Merge branch 'master' into upstream

435 files changed, 60151 insertions, 13211 deletions

diff --git a/drivers/ata/libata-eh.c b/drivers/ata/libata-eh.c
index b1b5104..3fa80f0 100644
--- a/drivers/ata/libata-eh.c
+++ b/drivers/ata/libata-eh.c
@@ -32,7 +32,6 @@
  *
  */
 
-#include <linux/config.h>
 #include <linux/kernel.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_host.h>
diff --git a/drivers/ata/sata_svw.c b/drivers/ata/sata_svw.c
index 2a7e349..d6d6658 100644
--- a/drivers/ata/sata_svw.c
+++ b/drivers/ata/sata_svw.c
@@ -268,7 +268,7 @@ static int k2_sata_proc_info(struct Scsi_Host *shost, char *page, char **start,
 	/* Match it to a port node */
 	index = (ap == ap->host->ports[0]) ? 0 : 1;
 	for (np = np->child; np != NULL; np = np->sibling) {
-		u32 *reg = (u32 *)get_property(np, "reg", NULL);
+		const u32 *reg = get_property(np, "reg", NULL);
 		if (!reg)
 			continue;
 		if (index == *reg)
diff --git a/drivers/atm/he.c b/drivers/atm/he.c
index dd96123..41e052f 100644
--- a/drivers/atm/he.c
+++ b/drivers/atm/he.c
@@ -1912,7 +1912,7 @@ he_service_rbrq(struct he_dev *he_dev, int group)
 				skb->tail = skb->data + skb->len;
 #ifdef USE_CHECKSUM_HW
 				if (vcc->vpi == 0 && vcc->vci >= ATM_NOT_RSV_VCI) {
-					skb->ip_summed = CHECKSUM_HW;
+					skb->ip_summed = CHECKSUM_COMPLETE;
 					skb->csum = TCP_CKSUM(skb->data,
 							he_vcc->pdu_len);
 				}
@@ -1928,7 +1928,9 @@ he_service_rbrq(struct he_dev *he_dev, int group)
 #ifdef notdef
 		ATM_SKB(skb)->vcc = vcc;
 #endif
+		spin_unlock(&he_dev->global_lock);
 		vcc->push(vcc, skb);
+		spin_lock(&he_dev->global_lock);
 
 		atomic_inc(&vcc->stats->rx);
 
diff --git a/drivers/base/hypervisor.c b/drivers/base/hypervisor.c
index 0c85e9d..7080b41 100644
--- a/drivers/base/hypervisor.c
+++ b/drivers/base/hypervisor.c
@@ -1,8 +1,9 @@
 /*
  * hypervisor.c - /sys/hypervisor subsystem.
  *
- * This file is released under the GPLv2
+ * Copyright (C) IBM Corp. 2006
  *
+ * This file is released under the GPLv2
  */
 
 #include <linux/kobject.h>
diff --git a/drivers/block/DAC960.c b/drivers/block/DAC960.c
index 4cd23c3..a360215 100644
--- a/drivers/block/DAC960.c
+++ b/drivers/block/DAC960.c
@@ -7115,7 +7115,7 @@ static struct pci_device_id DAC960_id_table[] = {
 	{
 		.vendor 	= PCI_VENDOR_ID_MYLEX,
 		.device		= PCI_DEVICE_ID_MYLEX_DAC960_GEM,
-		.subvendor	= PCI_ANY_ID,
+		.subvendor	= PCI_VENDOR_ID_MYLEX,
 		.subdevice	= PCI_ANY_ID,
 		.driver_data	= (unsigned long) &DAC960_GEM_privdata,
 	},
diff --git a/drivers/block/cciss_scsi.c b/drivers/block/cciss_scsi.c
index afdff32..05f79d7 100644
--- a/drivers/block/cciss_scsi.c
+++ b/drivers/block/cciss_scsi.c
@@ -251,10 +251,6 @@ scsi_cmd_stack_free(int ctlr)
 	stk->pool = NULL;
 }
 
-/* scsi_device_types comes from scsi.h */
-#define DEVICETYPE(n) (n<0 || n>MAX_SCSI_DEVICE_CODE) ? \
-	"Unknown" : scsi_device_types[n]
-
 #if 0
 static int xmargin=8;
 static int amargin=60;
@@ -389,7 +385,7 @@ cciss_scsi_add_entry(int ctlr, int hostno,
 	   time anyway (the scsi layer's inquiries will show that info) */
 	if (hostno != -1)
 		printk("cciss%d: %s device c%db%dt%dl%d added.\n", 
-			ctlr, DEVICETYPE(sd->devtype), hostno, 
+			ctlr, scsi_device_type(sd->devtype), hostno,
 			sd->bus, sd->target, sd->lun);
 	return 0;
 }
@@ -407,7 +403,7 @@ cciss_scsi_remove_entry(int ctlr, int hostno, int entry)
 		ccissscsi[ctlr].dev[i] = ccissscsi[ctlr].dev[i+1];
 	ccissscsi[ctlr].ndevices--;
 	printk("cciss%d: %s device c%db%dt%dl%d removed.\n",
-		ctlr, DEVICETYPE(sd.devtype), hostno, 
+		ctlr, scsi_device_type(sd.devtype), hostno,
 			sd.bus, sd.target, sd.lun);
 }
 
@@ -458,7 +454,7 @@ adjust_cciss_scsi_table(int ctlr, int hostno,
 		if (found == 0) { /* device no longer present. */ 
 			changes++;
 			/* printk("cciss%d: %s device c%db%dt%dl%d removed.\n",
-				ctlr, DEVICETYPE(csd->devtype), hostno, 
+				ctlr, scsi_device_type(csd->devtype), hostno,
 					csd->bus, csd->target, csd->lun); */
 			cciss_scsi_remove_entry(ctlr, hostno, i);
 			/* note, i not incremented */
@@ -468,7 +464,7 @@ adjust_cciss_scsi_table(int ctlr, int hostno,
 			printk("cciss%d: device c%db%dt%dl%d type changed "
 				"(device type now %s).\n",
 				ctlr, hostno, csd->bus, csd->target, csd->lun,
-					DEVICETYPE(csd->devtype));
+					scsi_device_type(csd->devtype));
 			csd->devtype = sd[j].devtype;
 			i++;	/* so just move along. */
 		} else 		/* device is same as it ever was, */
@@ -1098,7 +1094,7 @@ cciss_update_non_disk_devices(int cntl_num, int hostno)
 			if (ncurrent >= CCISS_MAX_SCSI_DEVS_PER_HBA) {
 				printk(KERN_INFO "cciss%d: %s ignored, "
 					"too many devices.\n", cntl_num,
-					DEVICETYPE(devtype));
+					scsi_device_type(devtype));
 				break;
 			}
 			memcpy(&currentsd[ncurrent].scsi3addr[0], 
diff --git a/drivers/block/cryptoloop.c b/drivers/block/cryptoloop.c
index 3d4261c..4053503 100644
--- a/drivers/block/cryptoloop.c
+++ b/drivers/block/cryptoloop.c
@@ -40,11 +40,13 @@ static int
 cryptoloop_init(struct loop_device *lo, const struct loop_info64 *info)
 {
 	int err = -EINVAL;
+	int cipher_len;
+	int mode_len;
 	char cms[LO_NAME_SIZE];			/* cipher-mode string */
 	char *cipher;
 	char *mode;
 	char *cmsp = cms;			/* c-m string pointer */
-	struct crypto_tfm *tfm = NULL;
+	struct crypto_blkcipher *tfm;
 
 	/* encryption breaks for non sector aligned offsets */
 
@@ -53,20 +55,39 @@ cryptoloop_init(struct loop_device *lo, const struct loop_info64 *info)
 
 	strncpy(cms, info->lo_crypt_name, LO_NAME_SIZE);
 	cms[LO_NAME_SIZE - 1] = 0;
-	cipher = strsep(&cmsp, "-");
-	mode = strsep(&cmsp, "-");
-
-	if (mode == NULL || strcmp(mode, "cbc") == 0)
-		tfm = crypto_alloc_tfm(cipher, CRYPTO_TFM_MODE_CBC |
-					       CRYPTO_TFM_REQ_MAY_SLEEP);
-	else if (strcmp(mode, "ecb") == 0)
-		tfm = crypto_alloc_tfm(cipher, CRYPTO_TFM_MODE_ECB |
-					       CRYPTO_TFM_REQ_MAY_SLEEP);
-	if (tfm == NULL)
+
+	cipher = cmsp;
+	cipher_len = strcspn(cmsp, "-");
+
+	mode = cmsp + cipher_len;
+	mode_len = 0;
+	if (*mode) {
+		mode++;
+		mode_len = strcspn(mode, "-");
+	}
+
+	if (!mode_len) {
+		mode = "cbc";
+		mode_len = 3;
+	}
+
+	if (cipher_len + mode_len + 3 > LO_NAME_SIZE)
 		return -EINVAL;
 
-	err = tfm->crt_u.cipher.cit_setkey(tfm, info->lo_encrypt_key,
-					   info->lo_encrypt_key_size);
+	memmove(cms, mode, mode_len);
+	cmsp = cms + mode_len;
+	*cmsp++ = '(';
+	memcpy(cmsp, info->lo_crypt_name, cipher_len);
+	cmsp += cipher_len;
+	*cmsp++ = ')';
+	*cmsp = 0;
+
+	tfm = crypto_alloc_blkcipher(cms, 0, CRYPTO_ALG_ASYNC);
+	if (IS_ERR(tfm))
+		return PTR_ERR(tfm);
+
+	err = crypto_blkcipher_setkey(tfm, info->lo_encrypt_key,
+				      info->lo_encrypt_key_size);
 	
 	if (err != 0)
 		goto out_free_tfm;
@@ -75,99 +96,49 @@ cryptoloop_init(struct loop_device *lo, const struct loop_info64 *info)
 	return 0;
 
  out_free_tfm:
-	crypto_free_tfm(tfm);
+	crypto_free_blkcipher(tfm);
 
  out:
 	return err;
 }
 
 
-typedef int (*encdec_ecb_t)(struct crypto_tfm *tfm,
+typedef int (*encdec_cbc_t)(struct blkcipher_desc *desc,
 			struct scatterlist *sg_out,
 			struct scatterlist *sg_in,
 			unsigned int nsg);
 
-
-static int
-cryptoloop_transfer_ecb(struct loop_device *lo, int cmd,
-			struct page *raw_page, unsigned raw_off,
-			struct page *loop_page, unsigned loop_off,
-			int size, sector_t IV)
-{
-	struct crypto_tfm *tfm = (struct crypto_tfm *) lo->key_data;
-	struct scatterlist sg_out = { NULL, };
-	struct scatterlist sg_in = { NULL, };
-
-	encdec_ecb_t encdecfunc;
-	struct page *in_page, *out_page;
-	unsigned in_offs, out_offs;
-
-	if (cmd == READ) {
-		in_page = raw_page;
-		in_offs = raw_off;
-		out_page = loop_page;
-		out_offs = loop_off;
-		encdecfunc = tfm->crt_u.cipher.cit_decrypt;
-	} else {
-		in_page = loop_page;
-		in_offs = loop_off;
-		out_page = raw_page;
-		out_offs = raw_off;
-		encdecfunc = tfm->crt_u.cipher.cit_encrypt;
-	}
-
-	while (size > 0) {
-		const int sz = min(size, LOOP_IV_SECTOR_SIZE);
-
-		sg_in.page = in_page;
-		sg_in.offset = in_offs;
-		sg_in.length = sz;
-
-		sg_out.page = out_page;
-		sg_out.offset = out_offs;
-		sg_out.length = sz;
-
-		encdecfunc(tfm, &sg_out, &sg_in, sz);
-
-		size -= sz;
-		in_offs += sz;
-		out_offs += sz;
-	}
-
-	return 0;
-}
-
-typedef int (*encdec_cbc_t)(struct crypto_tfm *tfm,
-			struct scatterlist *sg_out,
-			struct scatterlist *sg_in,
-			unsigned int nsg, u8 *iv);
-
 static int
-cryptoloop_transfer_cbc(struct loop_device *lo, int cmd,
-			struct page *raw_page, unsigned raw_off,
-			struct page *loop_page, unsigned loop_off,
-			int size, sector_t IV)
+cryptoloop_transfer(struct loop_device *lo, int cmd,
+		    struct page *raw_page, unsigned raw_off,
+		    struct page *loop_page, unsigned loop_off,
+		    int size, sector_t IV)
 {
-	struct crypto_tfm *tfm = (struct crypto_tfm *) lo->key_data;
+	struct crypto_blkcipher *tfm = lo->key_data;
+	struct blkcipher_desc desc = {
+		.tfm = tfm,
+		.flags = CRYPTO_TFM_REQ_MAY_SLEEP,
+	};
 	struct scatterlist sg_out = { NULL, };
 	struct scatterlist sg_in = { NULL, };
 
 	encdec_cbc_t encdecfunc;
 	struct page *in_page, *out_page;
 	unsigned in_offs, out_offs;
+	int err;
 
 	if (cmd == READ) {
 		in_page = raw_page;
 		in_offs = raw_off;
 		out_page = loop_page;
 		out_offs = loop_off;
-		encdecfunc = tfm->crt_u.cipher.cit_decrypt_iv;
+		encdecfunc = crypto_blkcipher_crt(tfm)->decrypt;
 	} else {
 		in_page = loop_page;
 		in_offs = loop_off;
 		out_page = raw_page;
 		out_offs = raw_off;
-		encdecfunc = tfm->crt_u.cipher.cit_encrypt_iv;
+		encdecfunc = crypto_blkcipher_crt(tfm)->encrypt;
 	}
 
 	while (size > 0) {
@@ -183,7 +154,10 @@ cryptoloop_transfer_cbc(struct loop_device *lo, int cmd,
 		sg_out.offset = out_offs;
 		sg_out.length = sz;
 
-		encdecfunc(tfm, &sg_out, &sg_in, sz, (u8 *)iv);
+		desc.info = iv;
+		err = encdecfunc(&desc, &sg_out, &sg_in, sz);
+		if (err)
+			return err;
 
 		IV++;
 		size -= sz;
@@ -195,32 +169,6 @@ cryptoloop_transfer_cbc(struct loop_device *lo, int cmd,
 }
 
 static int
-cryptoloop_transfer(struct loop_device *lo, int cmd,
-		    struct page *raw_page, unsigned raw_off,
-		    struct page *loop_page, unsigned loop_off,
-		    int size, sector_t IV)
-{
-	struct crypto_tfm *tfm = (struct crypto_tfm *) lo->key_data;
-	if(tfm->crt_cipher.cit_mode == CRYPTO_TFM_MODE_ECB)
-	{
-		lo->transfer = cryptoloop_transfer_ecb;
-		return cryptoloop_transfer_ecb(lo, cmd, raw_page, raw_off,
-					       loop_page, loop_off, size, IV);
-	}	
-	if(tfm->crt_cipher.cit_mode == CRYPTO_TFM_MODE_CBC)
-	{	
-		lo->transfer = cryptoloop_transfer_cbc;
-		return cryptoloop_transfer_cbc(lo, cmd, raw_page, raw_off,
-					       loop_page, loop_off, size, IV);
-	}
-	
-	/*  This is not supposed to happen */
-
-	printk( KERN_ERR "cryptoloop: unsupported cipher mode in cryptoloop_transfer!\n");
-	return -EINVAL;
-}
-
-static int
 cryptoloop_ioctl(struct loop_device *lo, int cmd, unsigned long arg)
 {
 	return -EINVAL;
@@ -229,9 +177,9 @@ cryptoloop_ioctl(struct loop_device *lo, int cmd, unsigned long arg)
 static int
 cryptoloop_release(struct loop_device *lo)
 {
-	struct crypto_tfm *tfm = (struct crypto_tfm *) lo->key_data;
+	struct crypto_blkcipher *tfm = lo->key_data;
 	if (tfm != NULL) {
-		crypto_free_tfm(tfm);
+		crypto_free_blkcipher(tfm);
 		lo->key_data = NULL;
 		return 0;
 	}
diff --git a/drivers/char/Kconfig b/drivers/char/Kconfig
index c40e487..52ea94b 100644
--- a/drivers/char/Kconfig
+++ b/drivers/char/Kconfig
@@ -495,6 +495,21 @@ config LEGACY_PTY_COUNT
 	  When not in use, each legacy PTY occupies 12 bytes on 32-bit
 	  architectures and 24 bytes on 64-bit architectures.
 
+config BRIQ_PANEL
+	tristate 'Total Impact briQ front panel driver'
+	depends on PPC_CHRP
+	---help---
+	  The briQ is a small footprint CHRP computer with a frontpanel VFD, a
+	  tristate led and two switches. It is the size of a CDROM drive.
+
+	  If you have such one and want anything showing on the VFD then you
+	  must answer Y here.
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called briq_panel.
+
+	  It's safe to say N here.
+
 config PRINTER
 	tristate "Parallel printer support"
 	depends on PARPORT
@@ -596,6 +611,13 @@ config HVC_CONSOLE
 	  console. This driver allows each pSeries partition to have a console
 	  which is accessed via the HMC.
 
+config HVC_ISERIES
+	bool "iSeries Hypervisor Virtual Console support"
+	depends on PPC_ISERIES && !VIOCONS
+	select HVC_DRIVER
+	help
+	  iSeries machines support a hypervisor virtual console.
+
 config HVC_RTAS
 	bool "IBM RTAS Console support"
 	depends on PPC_RTAS
diff --git a/drivers/char/Makefile b/drivers/char/Makefile
index 6e0f446..8c6dfc62 100644
--- a/drivers/char/Makefile
+++ b/drivers/char/Makefile
@@ -42,6 +42,7 @@ obj-$(CONFIG_AMIGA_BUILTIN_SERIAL) += amiserial.o
 obj-$(CONFIG_SX)		+= sx.o generic_serial.o
 obj-$(CONFIG_RIO)		+= rio/ generic_serial.o
 obj-$(CONFIG_HVC_CONSOLE)	+= hvc_vio.o hvsi.o
+obj-$(CONFIG_HVC_ISERIES)	+= hvc_iseries.o
 obj-$(CONFIG_HVC_RTAS)		+= hvc_rtas.o
 obj-$(CONFIG_HVC_DRIVER)	+= hvc_console.o
 obj-$(CONFIG_RAW_DRIVER)	+= raw.o
@@ -51,6 +52,7 @@ obj-$(CONFIG_VIOCONS)		+= viocons.o
 obj-$(CONFIG_VIOTAPE)		+= viotape.o
 obj-$(CONFIG_HVCS)		+= hvcs.o
 obj-$(CONFIG_SGI_MBCS)		+= mbcs.o
+obj-$(CONFIG_BRIQ_PANEL)	+= briq_panel.o
 
 obj-$(CONFIG_PRINTER)		+= lp.o
 obj-$(CONFIG_TIPAR)		+= tipar.o
diff --git a/drivers/char/agp/agp.h b/drivers/char/agp/agp.h
index 3c623b6..8b3317f 100644
--- a/drivers/char/agp/agp.h
+++ b/drivers/char/agp/agp.h
@@ -117,7 +117,7 @@ struct agp_bridge_driver {
 };
 
 struct agp_bridge_data {
-	struct agp_version *version;
+	const struct agp_version *version;
 	struct agp_bridge_driver *driver;
 	struct vm_operations_struct *vm_ops;
 	void *previous_size;
diff --git a/drivers/char/agp/backend.c b/drivers/char/agp/backend.c
index 509adc4..d59e037 100644
--- a/drivers/char/agp/backend.c
+++ b/drivers/char/agp/backend.c
@@ -44,7 +44,7 @@
  * past 0.99 at all due to some boolean logic error. */
 #define AGPGART_VERSION_MAJOR 0
 #define AGPGART_VERSION_MINOR 101
-static struct agp_version agp_current_version =
+static const struct agp_version agp_current_version =
 {
 	.major = AGPGART_VERSION_MAJOR,
 	.minor = AGPGART_VERSION_MINOR,
diff --git a/drivers/char/agp/efficeon-agp.c b/drivers/char/agp/efficeon-agp.c
index b788b0a..30f730f 100644
--- a/drivers/char/agp/efficeon-agp.c
+++ b/drivers/char/agp/efficeon-agp.c
@@ -337,13 +337,6 @@ static struct agp_bridge_driver efficeon_driver = {
 	.agp_destroy_page	= agp_generic_destroy_page,
 };
 
-
-static int agp_efficeon_resume(struct pci_dev *pdev)
-{
-	printk(KERN_DEBUG PFX "agp_efficeon_resume()\n");
-	return efficeon_configure();
-}
-
 static int __devinit agp_efficeon_probe(struct pci_dev *pdev,
 				     const struct pci_device_id *ent)
 {
@@ -414,11 +407,18 @@ static void __devexit agp_efficeon_remove(struct pci_dev *pdev)
 	agp_put_bridge(bridge);
 }
 
+#ifdef CONFIG_PM
 static int agp_efficeon_suspend(struct pci_dev *dev, pm_message_t state)
 {
 	return 0;
 }
 
+static int agp_efficeon_resume(struct pci_dev *pdev)
+{
+	printk(KERN_DEBUG PFX "agp_efficeon_resume()\n");
+	return efficeon_configure();
+}
+#endif
 
 static struct pci_device_id agp_efficeon_pci_table[] = {
 	{
@@ -439,8 +439,10 @@ static struct pci_driver agp_efficeon_pci_driver = {
 	.id_table	= agp_efficeon_pci_table,
 	.probe		= agp_efficeon_probe,
 	.remove		= agp_efficeon_remove,
+#ifdef CONFIG_PM
 	.suspend	= agp_efficeon_suspend,
 	.resume		= agp_efficeon_resume,
+#endif
 };
 
 static int __init agp_efficeon_init(void)
diff --git a/drivers/char/agp/frontend.c b/drivers/char/agp/frontend.c
index d9c5a91..0f2ed2a 100644
--- a/drivers/char/agp/frontend.c
+++ b/drivers/char/agp/frontend.c
@@ -151,35 +151,12 @@ static void agp_add_seg_to_client(struct agp_client *client,
 	client->segments = seg;
 }
 
-/* Originally taken from linux/mm/mmap.c from the array
- * protection_map.
- * The original really should be exported to modules, or
- * some routine which does the conversion for you
- */
-
-static const pgprot_t my_protect_map[16] =
-{
-	__P000, __P001, __P010, __P011, __P100, __P101, __P110, __P111,
-	__S000, __S001, __S010, __S011, __S100, __S101, __S110, __S111
-};
-
 static pgprot_t agp_convert_mmap_flags(int prot)
 {
-#define _trans(x,bit1,bit2) \
-((bit1==bit2)?(x&bit1):(x&bit1)?bit2:0)
-
 	unsigned long prot_bits;
-	pgprot_t temp;
-
-	prot_bits = _trans(prot, PROT_READ, VM_READ) |
-	    _trans(prot, PROT_WRITE, VM_WRITE) |
-	    _trans(prot, PROT_EXEC, VM_EXEC);
-
-	prot_bits |= VM_SHARED;
 
-	temp = my_protect_map[prot_bits & 0x0000000f];
-
-	return temp;
+	prot_bits = calc_vm_prot_bits(prot) | VM_SHARED;
+	return vm_get_page_prot(prot_bits);
 }
 
 static int agp_create_segment(struct agp_client *client, struct agp_region *region)
diff --git a/drivers/char/agp/generic.c b/drivers/char/agp/generic.c
index cc5ea34..0dcdb36 100644
--- a/drivers/char/agp/generic.c
+++ b/drivers/char/agp/generic.c
@@ -568,25 +568,34 @@ static void agp_v3_parse_one(u32 *requested_mode, u32 *bridge_agpstat, u32 *vga_
 		*bridge_agpstat &= ~(AGPSTAT3_4X | AGPSTAT3_RSVD);
 		goto done;
 
+	} else if (*requested_mode & AGPSTAT3_4X) {
+		*bridge_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
+		*bridge_agpstat |= AGPSTAT3_4X;
+		goto done;
+
 	} else {
 
 		/*
-		 * If we didn't specify AGPx8, we can only do x4.
-		 * If the hardware can't do x4, we're up shit creek, and never
-		 *  should have got this far.
+		 * If we didn't specify an AGP mode, we see if both
+		 * the graphics card, and the bridge can do x8, and use if so.
+		 * If not, we fall back to x4 mode.
 		 */
-		*bridge_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
-		if ((*bridge_agpstat & AGPSTAT3_4X) && (*vga_agpstat & AGPSTAT3_4X))
-			*bridge_agpstat |= AGPSTAT3_4X;
-		else {
-			printk(KERN_INFO PFX "Badness. Don't know which AGP mode to set. "
-							"[bridge_agpstat:%x vga_agpstat:%x fell back to:- bridge_agpstat:%x vga_agpstat:%x]\n",
-							origbridge, origvga, *bridge_agpstat, *vga_agpstat);
-			if (!(*bridge_agpstat & AGPSTAT3_4X))
-				printk(KERN_INFO PFX "Bridge couldn't do AGP x4.\n");
-			if (!(*vga_agpstat & AGPSTAT3_4X))
-				printk(KERN_INFO PFX "Graphic card couldn't do AGP x4.\n");
-			return;
+		if ((*bridge_agpstat & AGPSTAT3_8X) && (*vga_agpstat & AGPSTAT3_8X)) {
+			printk(KERN_INFO PFX "No AGP mode specified. Setting to highest mode supported by bridge & card (x8).\n");
+			*bridge_agpstat &= ~(AGPSTAT3_4X | AGPSTAT3_RSVD);
+			*vga_agpstat &= ~(AGPSTAT3_4X | AGPSTAT3_RSVD);
+		} else {
+			printk(KERN_INFO PFX "Fell back to AGPx4 mode because");
+			if (!(*bridge_agpstat & AGPSTAT3_8X)) {
+				printk("bridge couldn't do x8. bridge_agpstat:%x (orig=%x)\n", *bridge_agpstat, origbridge);
+				*bridge_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
+				*bridge_agpstat |= AGPSTAT3_4X;
+			}
+			if (!(*vga_agpstat & AGPSTAT3_8X)) {
+				printk("graphics card couldn't do x8. vga_agpstat:%x (orig=%x)\n", *vga_agpstat, origvga);
+				*vga_agpstat &= ~(AGPSTAT3_8X | AGPSTAT3_RSVD);
+				*vga_agpstat |= AGPSTAT3_4X;
+			}
 		}
 	}
 
diff --git a/drivers/char/agp/intel-agp.c b/drivers/char/agp/intel-agp.c
index 61ac380..d1ede7d 100644
--- a/drivers/char/agp/intel-agp.c
+++ b/drivers/char/agp/intel-agp.c
@@ -2,14 +2,6 @@
  * Intel AGPGART routines.
  */
 
-/*
- * Intel(R) 855GM/852GM and 865G support added by David Dawes
- * <dawes@tungstengraphics.com>.
- *
- * Intel(R) 915G/915GM support added by Alan Hourihane
- * <alanh@tungstengraphics.com>.
- */
-
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/init.h>
@@ -17,6 +9,21 @@
 #include <linux/agp_backend.h>
 #include "agp.h"
 
+#define PCI_DEVICE_ID_INTEL_82946GZ_HB      0x2970
+#define PCI_DEVICE_ID_INTEL_82946GZ_IG      0x2972
+#define PCI_DEVICE_ID_INTEL_82965G_1_HB     0x2980
+#define PCI_DEVICE_ID_INTEL_82965G_1_IG     0x2982
+#define PCI_DEVICE_ID_INTEL_82965Q_HB       0x2990
+#define PCI_DEVICE_ID_INTEL_82965Q_IG       0x2992
+#define PCI_DEVICE_ID_INTEL_82965G_HB       0x29A0
+#define PCI_DEVICE_ID_INTEL_82965G_IG       0x29A2
+
+#define IS_I965 (agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82946GZ_HB || \
+                 agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82965G_1_HB || \
+                 agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82965Q_HB || \
+                 agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82965G_HB)
+
+
 /* Intel 815 register */
 #define INTEL_815_APCONT	0x51
 #define INTEL_815_ATTBASE_MASK	~0x1FFFFFFF
@@ -40,6 +47,8 @@
 #define I915_GMCH_GMS_STOLEN_48M	(0x6 << 4)
 #define I915_GMCH_GMS_STOLEN_64M	(0x7 << 4)
 
+/* Intel 965G registers */
+#define I965_MSAC 0x62
 
 /* Intel 7505 registers */
 #define INTEL_I7505_APSIZE	0x74
@@ -354,6 +363,7 @@ static struct aper_size_info_fixed intel_i830_sizes[] =
 	/* The 64M mode still requires a 128k gatt */
 	{64, 16384, 5},
 	{256, 65536, 6},
+	{512, 131072, 7},
 };
 
 static struct _intel_i830_private {
@@ -377,7 +387,11 @@ static void intel_i830_init_gtt_entries(void)
 	/* We obtain the size of the GTT, which is also stored (for some
 	 * reason) at the top of stolen memory. Then we add 4KB to that
 	 * for the video BIOS popup, which is also stored in there. */
-	size = agp_bridge->driver->fetch_size() + 4;
+
+	if (IS_I965)
+		size = 512 + 4;
+	else
+		size = agp_bridge->driver->fetch_size() + 4;
 
 	if (agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82830_HB ||
 	    agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82845G_HB) {
@@ -423,7 +437,7 @@ static void intel_i830_init_gtt_entries(void)
 			if (agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82915G_HB ||
 			    agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82915GM_HB ||
 			    agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82945G_HB ||
-			    agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82945GM_HB)
+			    agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82945GM_HB || IS_I965 )
 				gtt_entries = MB(48) - KB(size);
 			else
 				gtt_entries = 0;
@@ -433,7 +447,7 @@ static void intel_i830_init_gtt_entries(void)
 			if (agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82915G_HB ||
 			    agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82915GM_HB ||
 			    agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82945G_HB ||
-			    agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82945GM_HB)
+			    agp_bridge->dev->device == PCI_DEVICE_ID_INTEL_82945GM_HB || IS_I965)
 				gtt_entries = MB(64) - KB(size);
 			else
 				gtt_entries = 0;
@@ -791,6 +805,77 @@ static int intel_i915_create_gatt_table(struct agp_bridge_data *bridge)
 
 	return 0;
 }
+static int intel_i965_fetch_size(void)
+{
+       struct aper_size_info_fixed *values;
+       u32 offset = 0;
+       u8 temp;
+
+#define I965_512MB_ADDRESS_MASK (3<<1)
+
+       values = A_SIZE_FIX(agp_bridge->driver->aperture_sizes);
+
+       pci_read_config_byte(intel_i830_private.i830_dev, I965_MSAC, &temp);
+       temp &= I965_512MB_ADDRESS_MASK;
+       switch (temp) {
+       case 0x00:
+               offset = 0; /* 128MB */
+               break;
+       case 0x06:
+               offset = 3; /* 512MB */
+               break;
+       default:
+       case 0x02:
+               offset = 2; /* 256MB */
+               break;
+       }
+
+       agp_bridge->previous_size = agp_bridge->current_size = (void *)(values + offset);
+
+       return values[offset].size;
+}
+
+/* The intel i965 automatically initializes the agp aperture during POST.
++ * Use the memory already set aside for in the GTT.
++ */
+static int intel_i965_create_gatt_table(struct agp_bridge_data *bridge)
+{
+       int page_order;
+       struct aper_size_info_fixed *size;
+       int num_entries;
+       u32 temp;
+
+       size = agp_bridge->current_size;
+       page_order = size->page_order;
+       num_entries = size->num_entries;
+       agp_bridge->gatt_table_real = NULL;
+
+       pci_read_config_dword(intel_i830_private.i830_dev, I915_MMADDR, &temp);
+
+       temp &= 0xfff00000;
+       intel_i830_private.gtt = ioremap((temp + (512 * 1024)) , 512 * 1024);
+
+       if (!intel_i830_private.gtt)
+               return -ENOMEM;
+
+
+       intel_i830_private.registers = ioremap(temp,128 * 4096);
+       if (!intel_i830_private.registers)
+               return -ENOMEM;
+
+       temp = readl(intel_i830_private.registers+I810_PGETBL_CTL) & 0xfffff000;
+       global_cache_flush();   /* FIXME: ? */
+
+       /* we have to call this as early as possible after the MMIO base address is known */
+       intel_i830_init_gtt_entries();
+
+       agp_bridge->gatt_table = NULL;
+
+       agp_bridge->gatt_bus_addr = temp;
+
+       return 0;
+}
+
 
 static int intel_fetch_size(void)
 {
@@ -1307,7 +1392,7 @@ static struct agp_bridge_driver intel_830_driver = {
 	.owner			= THIS_MODULE,
 	.aperture_sizes		= intel_i830_sizes,
 	.size_type		= FIXED_APER_SIZE,
-	.num_aperture_sizes	= 3,
+	.num_aperture_sizes	= 4,
 	.needs_scratch_page	= TRUE,
 	.configure		= intel_i830_configure,
 	.fetch_size		= intel_i830_fetch_size,
@@ -1469,7 +1554,7 @@ static struct agp_bridge_driver intel_915_driver = {
 	.owner			= THIS_MODULE,
 	.aperture_sizes		= intel_i830_sizes,
 	.size_type		= FIXED_APER_SIZE,
-	.num_aperture_sizes	= 3,
+	.num_aperture_sizes	= 4,
 	.needs_scratch_page	= TRUE,
 	.configure		= intel_i915_configure,
 	.fetch_size		= intel_i915_fetch_size,
@@ -1489,6 +1574,29 @@ static struct agp_bridge_driver intel_915_driver = {
 	.agp_destroy_page	= agp_generic_destroy_page,
 };
 
+static struct agp_bridge_driver intel_i965_driver = {
+       .owner                  = THIS_MODULE,
+       .aperture_sizes         = intel_i830_sizes,
+       .size_type              = FIXED_APER_SIZE,
+       .num_aperture_sizes     = 4,
+       .needs_scratch_page     = TRUE,
+       .configure              = intel_i915_configure,
+       .fetch_size             = intel_i965_fetch_size,
+       .cleanup                = intel_i915_cleanup,
+       .tlb_flush              = intel_i810_tlbflush,
+       .mask_memory            = intel_i810_mask_memory,
+       .masks                  = intel_i810_masks,
+       .agp_enable             = intel_i810_agp_enable,
+       .cache_flush            = global_cache_flush,
+       .create_gatt_table      = intel_i965_create_gatt_table,
+       .free_gatt_table        = intel_i830_free_gatt_table,
+       .insert_memory          = intel_i915_insert_entries,
+       .remove_memory          = intel_i915_remove_entries,
+       .alloc_by_type          = intel_i830_alloc_by_type,
+       .free_by_type           = intel_i810_free_by_type,
+       .agp_alloc_page         = agp_generic_alloc_page,
+       .agp_destroy_page       = agp_generic_destroy_page,
+};
 
 static struct agp_bridge_driver intel_7505_driver = {
 	.owner			= THIS_MODULE,
@@ -1684,6 +1792,35 @@ static int __devinit agp_intel_probe(struct pci_dev *pdev,
 			bridge->driver = &intel_845_driver;
 		name = "945GM";
 		break;
+	case PCI_DEVICE_ID_INTEL_82946GZ_HB:
+		if (find_i830(PCI_DEVICE_ID_INTEL_82946GZ_IG))
+			bridge->driver = &intel_i965_driver;
+		else
+			bridge->driver = &intel_845_driver;
+		name = "946GZ";
+		break;
+	case PCI_DEVICE_ID_INTEL_82965G_1_HB:
+		if (find_i830(PCI_DEVICE_ID_INTEL_82965G_1_IG))
+			bridge->driver = &intel_i965_driver;
+		else
+			bridge->driver = &intel_845_driver;
+		name = "965G";
+		break;
+	case PCI_DEVICE_ID_INTEL_82965Q_HB:
+		if (find_i830(PCI_DEVICE_ID_INTEL_82965Q_IG))
+			bridge->driver = &intel_i965_driver;
+		else
+			bridge->driver = &intel_845_driver;
+		name = "965Q";
+		break;
+	case PCI_DEVICE_ID_INTEL_82965G_HB:
+		if (find_i830(PCI_DEVICE_ID_INTEL_82965G_IG))
+			bridge->driver = &intel_i965_driver;
+		else
+			bridge->driver = &intel_845_driver;
+		name = "965G";
+		break;
+
 	case PCI_DEVICE_ID_INTEL_7505_0:
 		bridge->driver = &intel_7505_driver;
 		name = "E7505";
@@ -1766,6 +1903,7 @@ static void __devexit agp_intel_remove(struct pci_dev *pdev)
 	agp_put_bridge(bridge);
 }
 
+#ifdef CONFIG_PM
 static int agp_intel_resume(struct pci_dev *pdev)
 {
 	struct agp_bridge_data *bridge = pci_get_drvdata(pdev);
@@ -1786,9 +1924,12 @@ static int agp_intel_resume(struct pci_dev *pdev)
 		intel_i830_configure();
 	else if (bridge->driver == &intel_810_driver)
 		intel_i810_configure();
+	else if (bridge->driver == &intel_i965_driver)
+		intel_i915_configure();
 
 	return 0;
 }
+#endif
 
 static struct pci_device_id agp_intel_pci_table[] = {
 #define ID(x)						\
@@ -1825,6 +1966,10 @@ static struct pci_device_id agp_intel_pci_table[] = {
 	ID(PCI_DEVICE_ID_INTEL_82915GM_HB),
 	ID(PCI_DEVICE_ID_INTEL_82945G_HB),
 	ID(PCI_DEVICE_ID_INTEL_82945GM_HB),
+	ID(PCI_DEVICE_ID_INTEL_82946GZ_HB),
+	ID(PCI_DEVICE_ID_INTEL_82965G_1_HB),
+	ID(PCI_DEVICE_ID_INTEL_82965Q_HB),
+	ID(PCI_DEVICE_ID_INTEL_82965G_HB),
 	{ }
 };
 
@@ -1835,7 +1980,9 @@ static struct pci_driver agp_intel_pci_driver = {
 	.id_table	= agp_intel_pci_table,
 	.probe		= agp_intel_probe,
 	.remove		= __devexit_p(agp_intel_remove),
+#ifdef CONFIG_PM
 	.resume		= agp_intel_resume,
+#endif
 };
 
 static int __init agp_intel_init(void)
diff --git a/drivers/char/agp/uninorth-agp.c b/drivers/char/agp/uninorth-agp.c
index 1de1b12..91b71e7 100644
--- a/drivers/char/agp/uninorth-agp.c
+++ b/drivers/char/agp/uninorth-agp.c
@@ -601,8 +601,8 @@ static int __devinit agp_uninorth_probe(struct pci_dev *pdev,
 		uninorth_node = of_find_node_by_name(NULL, "u3");
 	}
 	if (uninorth_node) {
-		int *revprop = (int *)
-			get_property(uninorth_node, "device-rev", NULL);
+		const int *revprop = get_property(uninorth_node,
+				"device-rev", NULL);
 		if (revprop != NULL)
 			uninorth_rev = *revprop & 0x3f;
 		of_node_put(uninorth_node);
diff --git a/drivers/char/agp/via-agp.c b/drivers/char/agp/via-agp.c
index b8ec25d..c149ac9c 100644
--- a/drivers/char/agp/via-agp.c
+++ b/drivers/char/agp/via-agp.c
@@ -9,7 +9,7 @@
 #include <linux/agp_backend.h>
 #include "agp.h"
 
-static struct pci_device_id agp_via_pci_table[];
+static const struct pci_device_id agp_via_pci_table[];
 
 #define VIA_GARTCTRL	0x80
 #define VIA_APSIZE	0x84
@@ -485,7 +485,7 @@ static int agp_via_resume(struct pci_dev *pdev)
 #endif /* CONFIG_PM */
 
 /* must be the same order as name table above */
-static struct pci_device_id agp_via_pci_table[] = {
+static const struct pci_device_id agp_via_pci_table[] = {
 #define ID(x) \
 	{						\
 	.class		= (PCI_CLASS_BRIDGE_HOST << 8),	\
diff --git a/drivers/char/briq_panel.c b/drivers/char/briq_panel.c
new file mode 100644
index 0000000..b8c2225
--- /dev/null
+++ b/drivers/char/briq_panel.c
@@ -0,0 +1,271 @@
+/*
+ * Drivers for the Total Impact PPC based computer "BRIQ"
+ * by Dr. Karsten Jeppesen
+ *
+ */
+
+#include <linux/module.h>
+
+#include <linux/types.h>
+#include <linux/errno.h>
+#include <linux/sched.h>
+#include <linux/tty.h>
+#include <linux/timer.h>
+#include <linux/config.h>
+#include <linux/kernel.h>
+#include <linux/wait.h>
+#include <linux/string.h>
+#include <linux/slab.h>
+#include <linux/ioport.h>
+#include <linux/delay.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/mm.h>
+#include <linux/init.h>
+
+#include <asm/uaccess.h>
+#include <asm/io.h>
+#include <asm/prom.h>
+
+#define		BRIQ_PANEL_MINOR	156
+#define		BRIQ_PANEL_VFD_IOPORT	0x0390
+#define		BRIQ_PANEL_LED_IOPORT	0x0398
+#define		BRIQ_PANEL_VER		"1.1 (04/20/2002)"
+#define		BRIQ_PANEL_MSG0		"Loading Linux"
+
+static int		vfd_is_open;
+static unsigned char	vfd[40];
+static int		vfd_cursor;
+static unsigned char	ledpb, led;
+
+static void update_vfd(void)
+{
+	int	i;
+
+	/* cursor home */
+	outb(0x02, BRIQ_PANEL_VFD_IOPORT);
+	for (i=0; i<20; i++)
+		outb(vfd[i], BRIQ_PANEL_VFD_IOPORT + 1);
+
+	/* cursor to next line */
+	outb(0xc0, BRIQ_PANEL_VFD_IOPORT);
+	for (i=20; i<40; i++)
+		outb(vfd[i], BRIQ_PANEL_VFD_IOPORT + 1);
+
+}
+
+static void set_led(char state)
+{
+	if (state == 'R')
+		led = 0x01;
+	else if (state == 'G')
+		led = 0x02;
+	else if (state == 'Y')
+		led = 0x03;
+	else if (state == 'X')
+		led = 0x00;
+	outb(led, BRIQ_PANEL_LED_IOPORT);
+}
+
+static int briq_panel_open(struct inode *ino, struct file *filep)
+{
+	/* enforce single access */
+	if (vfd_is_open)
+		return -EBUSY;
+	vfd_is_open = 1;
+
+	return 0;
+}
+
+static int briq_panel_release(struct inode *ino, struct file *filep)
+{
+	if (!vfd_is_open)
+		return -ENODEV;
+
+	vfd_is_open = 0;
+
+	return 0;
+}
+
+static ssize_t briq_panel_read(struct file *file, char __user *buf, size_t count,
+			 loff_t *ppos)
+{
+	unsigned short c;
+	unsigned char cp;
+
+#if 0	/*  Can't seek (pread) on this device  */
+	if (ppos != &file->f_pos)
+		return -ESPIPE;
+#endif
+
+	if (!vfd_is_open)
+		return -ENODEV;
+
+	c = (inb(BRIQ_PANEL_LED_IOPORT) & 0x000c) | (ledpb & 0x0003);
+	set_led(' ');
+	/* upper button released */
+	if ((!(ledpb & 0x0004)) && (c & 0x0004)) {
+		cp = ' ';
+		ledpb = c;
+		if (copy_to_user(buf, &cp, 1))
+			return -EFAULT;
+		return 1;
+	}
+	/* lower button released */
+	else if ((!(ledpb & 0x0008)) && (c & 0x0008)) {
+		cp = '\r';
+		ledpb = c;
+		if (copy_to_user(buf, &cp, 1))
+			return -EFAULT;
+		return 1;
+	} else {
+		ledpb = c;
+		return 0;
+	}
+}
+
+static void scroll_vfd( void )
+{
+	int	i;
+
+	for (i=0; i<20; i++) {
+		vfd[i] = vfd[i+20];
+		vfd[i+20] = ' ';
+	}
+	vfd_cursor = 20;
+}
+
+static ssize_t briq_panel_write(struct file *file, const char __user *buf, size_t len,
+			  loff_t *ppos)
+{
+	size_t indx = len;
+	int i, esc = 0;
+
+#if 0	/*  Can't seek (pwrite) on this device  */
+	if (ppos != &file->f_pos)
+		return -ESPIPE;
+#endif
+
+	if (!vfd_is_open)
+		return -EBUSY;
+
+	for (;;) {
+		char c;
+		if (!indx)
+			break;
+		if (get_user(c, buf))
+			return -EFAULT;
+		if (esc) {
+			set_led(c);
+			esc = 0;
+		} else if (c == 27) {
+			esc = 1;
+		} else if (c == 12) {
+			/* do a form feed */
+			for (i=0; i<40; i++)
+				vfd[i] = ' ';
+			vfd_cursor = 0;
+		} else if (c == 10) {
+			if (vfd_cursor < 20)
+				vfd_cursor = 20;
+			else if (vfd_cursor < 40)
+				vfd_cursor = 40;
+			else if (vfd_cursor < 60)
+				vfd_cursor = 60;
+			if (vfd_cursor > 59)
+				scroll_vfd();
+		} else {
+			/* just a character */
+			if (vfd_cursor > 39)
+				scroll_vfd();
+			vfd[vfd_cursor++] = c;
+		}
+		indx--;
+		buf++;
+	}
+	update_vfd();
+
+	return len;
+}
+
+static struct file_operations briq_panel_fops = {
+	.owner		= THIS_MODULE,
+	.read		= briq_panel_read,
+	.write		= briq_panel_write,
+	.open		= briq_panel_open,
+	.release	= briq_panel_release,
+};
+
+static struct miscdevice briq_panel_miscdev = {
+	BRIQ_PANEL_MINOR,
+	"briq_panel",
+	&briq_panel_fops
+};
+
+static int __init briq_panel_init(void)
+{
+	struct device_node *root = find_path_device("/");
+	const char *machine;
+	int i;
+
+	machine = get_property(root, "model", NULL);
+	if (!machine || strncmp(machine, "TotalImpact,BRIQ-1", 18) != 0)
+		return -ENODEV;
+
+	printk(KERN_INFO
+		"briq_panel: v%s Dr. Karsten Jeppesen (kj@totalimpact.com)\n",
+		BRIQ_PANEL_VER);
+
+	if (!request_region(BRIQ_PANEL_VFD_IOPORT, 4, "BRIQ Front Panel"))
+		return -EBUSY;
+
+	if (!request_region(BRIQ_PANEL_LED_IOPORT, 2, "BRIQ Front Panel")) {
+		release_region(BRIQ_PANEL_VFD_IOPORT, 4);
+		return -EBUSY;
+	}
+	ledpb = inb(BRIQ_PANEL_LED_IOPORT) & 0x000c;
+
+	if (misc_register(&briq_panel_miscdev) < 0) {
+		release_region(BRIQ_PANEL_VFD_IOPORT, 4);
+		release_region(BRIQ_PANEL_LED_IOPORT, 2);
+		return -EBUSY;
+	}
+
+	outb(0x38, BRIQ_PANEL_VFD_IOPORT);	/* Function set */
+	outb(0x01, BRIQ_PANEL_VFD_IOPORT);	/* Clear display */
+	outb(0x0c, BRIQ_PANEL_VFD_IOPORT);	/* Display on */
+	outb(0x06, BRIQ_PANEL_VFD_IOPORT);	/* Entry normal */
+	for (i=0; i<40; i++)
+		vfd[i]=' ';
+#ifndef MODULE
+	vfd[0] = 'L';
+	vfd[1] = 'o';
+	vfd[2] = 'a';
+	vfd[3] = 'd';
+	vfd[4] = 'i';
+	vfd[5] = 'n';
+	vfd[6] = 'g';
+	vfd[7] = ' ';
+	vfd[8] = '.';
+	vfd[9] = '.';
+	vfd[10] = '.';
+#endif /* !MODULE */
+
+	update_vfd();
+
+	return 0;
+}
+
+static void __exit briq_panel_exit(void)
+{
+	misc_deregister(&briq_panel_miscdev);
+	release_region(BRIQ_PANEL_VFD_IOPORT, 4);
+	release_region(BRIQ_PANEL_LED_IOPORT, 2);
+}
+
+module_init(briq_panel_init);
+module_exit(briq_panel_exit);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Karsten Jeppesen <karsten@jeppesens.com>");
+MODULE_DESCRIPTION("Driver for the Total Impact briQ front panel");
diff --git a/drivers/char/hvc_console.c b/drivers/char/hvc_console.c
index 613d67f..a76d2c4 100644
--- a/drivers/char/hvc_console.c
+++ b/drivers/char/hvc_console.c
@@ -80,7 +80,8 @@ struct hvc_struct {
 	struct tty_struct *tty;
 	unsigned int count;
 	int do_wakeup;
-	char outbuf[N_OUTBUF] __ALIGNED__;
+	char *outbuf;
+	int outbuf_size;
 	int n_outbuf;
 	uint32_t vtermno;
 	struct hv_ops *ops;
@@ -319,10 +320,8 @@ static int hvc_open(struct tty_struct *tty, struct file * filp)
 	struct kobject *kobjp;
 
 	/* Auto increments kobject reference if found. */
-	if (!(hp = hvc_get_by_index(tty->index))) {
-		printk(KERN_WARNING "hvc_console: tty open failed, no vty associated with tty.\n");
+	if (!(hp = hvc_get_by_index(tty->index)))
 		return -ENODEV;
-	}
 
 	spin_lock_irqsave(&hp->lock, flags);
 	/* Check and then increment for fast path open. */
@@ -505,7 +504,7 @@ static int hvc_write(struct tty_struct *tty, const unsigned char *buf, int count
 	if (hp->n_outbuf > 0)
 		hvc_push(hp);
 
-	while (count > 0 && (rsize = N_OUTBUF - hp->n_outbuf) > 0) {
+	while (count > 0 && (rsize = hp->outbuf_size - hp->n_outbuf) > 0) {
 		if (rsize > count)
 			rsize = count;
 		memcpy(hp->outbuf + hp->n_outbuf, buf, rsize);
@@ -538,7 +537,7 @@ static int hvc_write_room(struct tty_struct *tty)
 	if (!hp)
 		return -1;
 
-	return N_OUTBUF - hp->n_outbuf;
+	return hp->outbuf_size - hp->n_outbuf;
 }
 
 static int hvc_chars_in_buffer(struct tty_struct *tty)
@@ -729,12 +728,13 @@ static struct kobj_type hvc_kobj_type = {
 };
 
 struct hvc_struct __devinit *hvc_alloc(uint32_t vtermno, int irq,
-					struct hv_ops *ops)
+					struct hv_ops *ops, int outbuf_size)
 {
 	struct hvc_struct *hp;
 	int i;
 
-	hp = kmalloc(sizeof(*hp), GFP_KERNEL);
+	hp = kmalloc(ALIGN(sizeof(*hp), sizeof(long)) + outbuf_size,
+			GFP_KERNEL);
 	if (!hp)
 		return ERR_PTR(-ENOMEM);
 
@@ -743,6 +743,8 @@ struct hvc_struct __devinit *hvc_alloc(uint32_t vtermno, int irq,
 	hp->vtermno = vtermno;
 	hp->irq = irq;
 	hp->ops = ops;
+	hp->outbuf_size = outbuf_size;
+	hp->outbuf = &((char *)hp)[ALIGN(sizeof(*hp), sizeof(long))];
 
 	kobject_init(&hp->kobj);
 	hp->kobj.ktype = &hvc_kobj_type;
diff --git a/drivers/char/hvc_console.h b/drivers/char/hvc_console.h
index 96b7401..8c59818 100644
--- a/drivers/char/hvc_console.h
+++ b/drivers/char/hvc_console.h
@@ -56,7 +56,7 @@ extern int hvc_instantiate(uint32_t vtermno, int index, struct hv_ops *ops);
 
 /* register a vterm for hvc tty operation (module_init or hotplug add) */
 extern struct hvc_struct * __devinit hvc_alloc(uint32_t vtermno, int irq,
-						 struct hv_ops *ops);
+				struct hv_ops *ops, int outbuf_size);
 /* remove a vterm from hvc tty operation (modele_exit or hotplug remove) */
 extern int __devexit hvc_remove(struct hvc_struct *hp);
 
diff --git a/drivers/char/hvc_iseries.c b/drivers/char/hvc_iseries.c
new file mode 100644
index 0000000..4747729
--- /dev/null
+++ b/drivers/char/hvc_iseries.c
@@ -0,0 +1,594 @@
+/*
+ * iSeries vio driver interface to hvc_console.c
+ *
+ * This code is based heavily on hvc_vio.c and viocons.c
+ *
+ * Copyright (C) 2006 Stephen Rothwell, IBM Corporation
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
+ */
+#include <stdarg.h>
+#include <linux/types.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/console.h>
+
+#include <asm/hvconsole.h>
+#include <asm/vio.h>
+#include <asm/prom.h>
+#include <asm/iseries/vio.h>
+#include <asm/iseries/hv_call.h>
+#include <asm/iseries/hv_lp_config.h>
+#include <asm/iseries/hv_lp_event.h>
+
+#include "hvc_console.h"
+
+#define VTTY_PORTS 10
+
+static DEFINE_SPINLOCK(consolelock);
+static DEFINE_SPINLOCK(consoleloglock);
+
+static const char hvc_driver_name[] = "hvc_console";
+
+#define IN_BUF_SIZE	200
+
+/*
+ * Our port information.
+ */
+static struct port_info {
+	HvLpIndex lp;
+	u64 seq;	/* sequence number of last HV send */
+	u64 ack;	/* last ack from HV */
+	struct hvc_struct *hp;
+	int in_start;
+	int in_end;
+	unsigned char in_buf[IN_BUF_SIZE];
+} port_info[VTTY_PORTS] = {
+	[ 0 ... VTTY_PORTS - 1 ] = {
+		.lp = HvLpIndexInvalid
+	}
+};
+
+#define viochar_is_console(pi)	((pi) == &port_info[0])
+
+static struct vio_device_id hvc_driver_table[] __devinitdata = {
+	{"serial", "IBM,iSeries-vty"},
+	{ "", "" }
+};
+MODULE_DEVICE_TABLE(vio, hvc_driver_table);
+
+static void hvlog(char *fmt, ...)
+{
+	int i;
+	unsigned long flags;
+	va_list args;
+	static char buf[256];
+
+	spin_lock_irqsave(&consoleloglock, flags);
+	va_start(args, fmt);
+	i = vscnprintf(buf, sizeof(buf) - 1, fmt, args);
+	va_end(args);
+	buf[i++] = '\r';
+	HvCall_writeLogBuffer(buf, i);
+	spin_unlock_irqrestore(&consoleloglock, flags);
+}
+
+/*
+ * Initialize the common fields in a charLpEvent
+ */
+static void init_data_event(struct viocharlpevent *viochar, HvLpIndex lp)
+{
+	struct HvLpEvent *hev = &viochar->event;
+
+	memset(viochar, 0, sizeof(struct viocharlpevent));
+
+	hev->flags = HV_LP_EVENT_VALID | HV_LP_EVENT_DEFERRED_ACK |
+		HV_LP_EVENT_INT;
+	hev->xType = HvLpEvent_Type_VirtualIo;
+	hev->xSubtype = viomajorsubtype_chario | viochardata;
+	hev->xSourceLp = HvLpConfig_getLpIndex();
+	hev->xTargetLp = lp;
+	hev->xSizeMinus1 = sizeof(struct viocharlpevent);
+	hev->xSourceInstanceId = viopath_sourceinst(lp);
+	hev->xTargetInstanceId = viopath_targetinst(lp);
+}
+
+static int get_chars(uint32_t vtermno, char *buf, int count)
+{
+	struct port_info *pi;
+	int n = 0;
+	unsigned long flags;
+
+	if (vtermno >= VTTY_PORTS)
+		return -EINVAL;
+	if (count == 0)
+		return 0;
+
+	pi = &port_info[vtermno];
+	spin_lock_irqsave(&consolelock, flags);
+
+	if (pi->in_end == 0)
+		goto done;
+
+	n = pi->in_end - pi->in_start;
+	if (n > count)
+		n = count;
+	memcpy(buf, &pi->in_buf[pi->in_start], n);
+	pi->in_start += n;
+	if (pi->in_start == pi->in_end) {
+		pi->in_start = 0;
+		pi->in_end = 0;
+	}
+done:
+	spin_unlock_irqrestore(&consolelock, flags);
+	return n;
+}
+
+static int put_chars(uint32_t vtermno, const char *buf, int count)
+{
+	struct viocharlpevent *viochar;
+	struct port_info *pi;
+	HvLpEvent_Rc hvrc;
+	unsigned long flags;
+	int sent = 0;
+
+	if (vtermno >= VTTY_PORTS)
+		return -EINVAL;
+
+	pi = &port_info[vtermno];
+
+	spin_lock_irqsave(&consolelock, flags);
+
+	if (viochar_is_console(pi) && !viopath_isactive(pi->lp)) {
+		spin_lock_irqsave(&consoleloglock, flags);
+		HvCall_writeLogBuffer(buf, count);
+		spin_unlock_irqrestore(&consoleloglock, flags);
+		sent = count;
+		goto done;
+	}
+
+	viochar = vio_get_event_buffer(viomajorsubtype_chario);
+	if (viochar == NULL) {
+		hvlog("\n\rviocons: Can't get viochar buffer.");
+		goto done;
+	}
+
+	while ((count > 0) && ((pi->seq - pi->ack) < VIOCHAR_WINDOW)) {
+		int len;
+
+		len = (count > VIOCHAR_MAX_DATA) ? VIOCHAR_MAX_DATA : count;
+
+		if (viochar_is_console(pi)) {
+			spin_lock_irqsave(&consoleloglock, flags);
+			HvCall_writeLogBuffer(buf, len);
+			spin_unlock_irqrestore(&consoleloglock, flags);
+		}
+
+		init_data_event(viochar, pi->lp);
+
+		viochar->len = len;
+		viochar->event.xCorrelationToken = pi->seq++;
+		viochar->event.xSizeMinus1 =
+			offsetof(struct viocharlpevent, data) + len;
+
+		memcpy(viochar->data, buf, len);
+
+		hvrc = HvCallEvent_signalLpEvent(&viochar->event);
+		if (hvrc)
+			hvlog("\n\rerror sending event! return code %d\n\r",
+				(int)hvrc);
+		sent += len;
+		count -= len;
+		buf += len;
+	}
+
+	vio_free_event_buffer(viomajorsubtype_chario, viochar);
+done:
+	spin_unlock_irqrestore(&consolelock, flags);
+	return sent;
+}
+
+static struct hv_ops hvc_get_put_ops = {
+	.get_chars = get_chars,
+	.put_chars = put_chars,
+};
+
+static int __devinit hvc_vio_probe(struct vio_dev *vdev,
+			const struct vio_device_id *id)
+{
+	struct hvc_struct *hp;
+	struct port_info *pi;
+
+	/* probed with invalid parameters. */
+	if (!vdev || !id)
+		return -EPERM;
+
+	if (vdev->unit_address >= VTTY_PORTS)
+		return -ENODEV;
+
+	pi = &port_info[vdev->unit_address];
+
+	hp = hvc_alloc(vdev->unit_address, vdev->irq, &hvc_get_put_ops,
+			VIOCHAR_MAX_DATA);
+	if (IS_ERR(hp))
+		return PTR_ERR(hp);
+	pi->hp = hp;
+	dev_set_drvdata(&vdev->dev, pi);
+
+	return 0;
+}
+
+static int __devexit hvc_vio_remove(struct vio_dev *vdev)
+{
+	struct port_info *pi = dev_get_drvdata(&vdev->dev);
+	struct hvc_struct *hp = pi->hp;
+
+	return hvc_remove(hp);
+}
+
+static struct vio_driver hvc_vio_driver = {
+	.id_table	= hvc_driver_table,
+	.probe		= hvc_vio_probe,
+	.remove		= hvc_vio_remove,
+	.driver		= {
+		.name	= hvc_driver_name,
+		.owner	= THIS_MODULE,
+	}
+};
+
+static void hvc_open_event(struct HvLpEvent *event)
+{
+	unsigned long flags;
+	struct viocharlpevent *cevent = (struct viocharlpevent *)event;
+	u8 port = cevent->virtual_device;
+	struct port_info *pi;
+	int reject = 0;
+
+	if (hvlpevent_is_ack(event)) {
+		if (port >= VTTY_PORTS)
+			return;
+
+		spin_lock_irqsave(&consolelock, flags);
+
+		pi = &port_info[port];
+		if (event->xRc == HvLpEvent_Rc_Good) {
+			pi->seq = pi->ack = 0;
+			/*
+			 * This line allows connections from the primary
+			 * partition but once one is connected from the
+			 * primary partition nothing short of a reboot
+			 * of linux will allow access from the hosting
+			 * partition again without a required iSeries fix.
+			 */
+			pi->lp = event->xTargetLp;
+		}
+
+		spin_unlock_irqrestore(&consolelock, flags);
+		if (event->xRc != HvLpEvent_Rc_Good)
+			printk(KERN_WARNING
+			       "hvc: handle_open_event: event->xRc == (%d).\n",
+			       event->xRc);
+
+		if (event->xCorrelationToken != 0) {
+			atomic_t *aptr= (atomic_t *)event->xCorrelationToken;
+			atomic_set(aptr, 1);
+		} else
+			printk(KERN_WARNING
+			       "hvc: weird...got open ack without atomic\n");
+		return;
+	}
+
+	/* This had better require an ack, otherwise complain */
+	if (!hvlpevent_need_ack(event)) {
+		printk(KERN_WARNING "hvc: viocharopen without ack bit!\n");
+		return;
+	}
+
+	spin_lock_irqsave(&consolelock, flags);
+
+	/* Make sure this is a good virtual tty */
+	if (port >= VTTY_PORTS) {
+		event->xRc = HvLpEvent_Rc_SubtypeError;
+		cevent->subtype_result_code = viorc_openRejected;
+		/*
+		 * Flag state here since we can't printk while holding
+		 * the consolelock spinlock.
+		 */
+		reject = 1;
+	} else {
+		pi = &port_info[port];
+		if ((pi->lp != HvLpIndexInvalid) &&
+				(pi->lp != event->xSourceLp)) {
+			/*
+			 * If this is tty is already connected to a different
+			 * partition, fail.
+			 */
+			event->xRc = HvLpEvent_Rc_SubtypeError;
+			cevent->subtype_result_code = viorc_openRejected;
+			reject = 2;
+		} else {
+			pi->lp = event->xSourceLp;
+			event->xRc = HvLpEvent_Rc_Good;
+			cevent->subtype_result_code = viorc_good;
+			pi->seq = pi->ack = 0;
+		}
+	}
+
+	spin_unlock_irqrestore(&consolelock, flags);
+
+	if (reject == 1)
+		printk(KERN_WARNING "hvc: open rejected: bad virtual tty.\n");
+	else if (reject == 2)
+		printk(KERN_WARNING "hvc: open rejected: console in exclusive "
+				"use by another partition.\n");
+
+	/* Return the acknowledgement */
+	HvCallEvent_ackLpEvent(event);
+}
+
+/*
+ * Handle a close charLpEvent.  This should ONLY be an Interrupt because the
+ * virtual console should never actually issue a close event to the hypervisor
+ * because the virtual console never goes away.  A close event coming from the
+ * hypervisor simply means that there are no client consoles connected to the
+ * virtual console.
+ */
+static void hvc_close_event(struct HvLpEvent *event)
+{
+	unsigned long flags;
+	struct viocharlpevent *cevent = (struct viocharlpevent *)event;
+	u8 port = cevent->virtual_device;
+
+	if (!hvlpevent_is_int(event)) {
+		printk(KERN_WARNING
+			"hvc: got unexpected close acknowlegement\n");
+		return;
+	}
+
+	if (port >= VTTY_PORTS) {
+		printk(KERN_WARNING
+			"hvc: close message from invalid virtual device.\n");
+		return;
+	}
+
+	/* For closes, just mark the console partition invalid */
+	spin_lock_irqsave(&consolelock, flags);
+
+	if (port_info[port].lp == event->xSourceLp)
+		port_info[port].lp = HvLpIndexInvalid;
+
+	spin_unlock_irqrestore(&consolelock, flags);
+}
+
+static void hvc_data_event(struct HvLpEvent *event)
+{
+	unsigned long flags;
+	struct viocharlpevent *cevent = (struct viocharlpevent *)event;
+	struct port_info *pi;
+	int n;
+	u8 port = cevent->virtual_device;
+
+	if (port >= VTTY_PORTS) {
+		printk(KERN_WARNING "hvc: data on invalid virtual device %d\n",
+				port);
+		return;
+	}
+	if (cevent->len == 0)
+		return;
+
+	/*
+	 * Change 05/01/2003 - Ryan Arnold: If a partition other than
+	 * the current exclusive partition tries to send us data
+	 * events then just drop them on the floor because we don't
+	 * want his stinking data.  He isn't authorized to receive
+	 * data because he wasn't the first one to get the console,
+	 * therefore he shouldn't be allowed to send data either.
+	 * This will work without an iSeries fix.
+	 */
+	pi = &port_info[port];
+	if (pi->lp != event->xSourceLp)
+		return;
+
+	spin_lock_irqsave(&consolelock, flags);
+
+	n = IN_BUF_SIZE - pi->in_end;
+	if (n > cevent->len)
+		n = cevent->len;
+	if (n > 0) {
+		memcpy(&pi->in_buf[pi->in_end], cevent->data, n);
+		pi->in_end += n;
+	}
+	spin_unlock_irqrestore(&consolelock, flags);
+	if (n == 0)
+		printk(KERN_WARNING "hvc: input buffer overflow\n");
+}
+
+static void hvc_ack_event(struct HvLpEvent *event)
+{
+	struct viocharlpevent *cevent = (struct viocharlpevent *)event;
+	unsigned long flags;
+	u8 port = cevent->virtual_device;
+
+	if (port >= VTTY_PORTS) {
+		printk(KERN_WARNING "hvc: data on invalid virtual device\n");
+		return;
+	}
+
+	spin_lock_irqsave(&consolelock, flags);
+	port_info[port].ack = event->xCorrelationToken;
+	spin_unlock_irqrestore(&consolelock, flags);
+}
+
+static void hvc_config_event(struct HvLpEvent *event)
+{
+	struct viocharlpevent *cevent = (struct viocharlpevent *)event;
+
+	if (cevent->data[0] == 0x01)
+		printk(KERN_INFO "hvc: window resized to %d: %d: %d: %d\n",
+		       cevent->data[1], cevent->data[2],
+		       cevent->data[3], cevent->data[4]);
+	else
+		printk(KERN_WARNING "hvc: unknown config event\n");
+}
+
+static void hvc_handle_event(struct HvLpEvent *event)
+{
+	int charminor;
+
+	if (event == NULL)
+		return;
+
+	charminor = event->xSubtype & VIOMINOR_SUBTYPE_MASK;
+	switch (charminor) {
+	case viocharopen:
+		hvc_open_event(event);
+		break;
+	case viocharclose:
+		hvc_close_event(event);
+		break;
+	case viochardata:
+		hvc_data_event(event);
+		break;
+	case viocharack:
+		hvc_ack_event(event);
+		break;
+	case viocharconfig:
+		hvc_config_event(event);
+		break;
+	default:
+		if (hvlpevent_is_int(event) && hvlpevent_need_ack(event)) {
+			event->xRc = HvLpEvent_Rc_InvalidSubtype;
+			HvCallEvent_ackLpEvent(event);
+		}
+	}
+}
+
+static int send_open(HvLpIndex remoteLp, void *sem)
+{
+	return HvCallEvent_signalLpEventFast(remoteLp,
+			HvLpEvent_Type_VirtualIo,
+			viomajorsubtype_chario | viocharopen,
+			HvLpEvent_AckInd_DoAck, HvLpEvent_AckType_ImmediateAck,
+			viopath_sourceinst(remoteLp),
+			viopath_targetinst(remoteLp),
+			(u64)(unsigned long)sem, VIOVERSION << 16,
+			0, 0, 0, 0);
+}
+
+static int hvc_vio_init(void)
+{
+	atomic_t wait_flag;
+	int rc;
+
+	/* +2 for fudge */
+	rc = viopath_open(HvLpConfig_getPrimaryLpIndex(),
+			viomajorsubtype_chario, VIOCHAR_WINDOW + 2);
+	if (rc)
+		printk(KERN_WARNING "hvc: error opening to primary %d\n", rc);
+
+	if (viopath_hostLp == HvLpIndexInvalid)
+		vio_set_hostlp();
+
+	/*
+	 * And if the primary is not the same as the hosting LP, open to the
+	 * hosting lp
+	 */
+	if ((viopath_hostLp != HvLpIndexInvalid) &&
+	    (viopath_hostLp != HvLpConfig_getPrimaryLpIndex())) {
+		printk(KERN_INFO "hvc: open path to hosting (%d)\n",
+				viopath_hostLp);
+		rc = viopath_open(viopath_hostLp, viomajorsubtype_chario,
+				VIOCHAR_WINDOW + 2);	/* +2 for fudge */
+		if (rc)
+			printk(KERN_WARNING
+				"error opening to partition %d: %d\n",
+				viopath_hostLp, rc);
+	}
+
+	if (vio_setHandler(viomajorsubtype_chario, hvc_handle_event) < 0)
+		printk(KERN_WARNING
+			"hvc: error seting handler for console events!\n");
+
+	/*
+	 * First, try to open the console to the hosting lp.
+	 * Wait on a semaphore for the response.
+	 */
+	atomic_set(&wait_flag, 0);
+	if ((viopath_isactive(viopath_hostLp)) &&
+	    (send_open(viopath_hostLp, &wait_flag) == 0)) {
+		printk(KERN_INFO "hvc: hosting partition %d\n", viopath_hostLp);
+		while (atomic_read(&wait_flag) == 0)
+			mb();
+		atomic_set(&wait_flag, 0);
+	}
+
+	/*
+	 * If we don't have an active console, try the primary
+	 */
+	if ((!viopath_isactive(port_info[0].lp)) &&
+	    (viopath_isactive(HvLpConfig_getPrimaryLpIndex())) &&
+	    (send_open(HvLpConfig_getPrimaryLpIndex(), &wait_flag) == 0)) {
+		printk(KERN_INFO "hvc: opening console to primary partition\n");
+		while (atomic_read(&wait_flag) == 0)
+			mb();
+	}
+
+	/* Register as a vio device to receive callbacks */
+	rc = vio_register_driver(&hvc_vio_driver);
+
+	return rc;
+}
+module_init(hvc_vio_init); /* after drivers/char/hvc_console.c */
+
+static void hvc_vio_exit(void)
+{
+	vio_unregister_driver(&hvc_vio_driver);
+}
+module_exit(hvc_vio_exit);
+
+/* the device tree order defines our numbering */
+static int hvc_find_vtys(void)
+{
+	struct device_node *vty;
+	int num_found = 0;
+
+	for (vty = of_find_node_by_name(NULL, "vty"); vty != NULL;
+			vty = of_find_node_by_name(vty, "vty")) {
+		uint32_t *vtermno;
+
+		/* We have statically defined space for only a certain number
+		 * of console adapters.
+		 */
+		if ((num_found >= MAX_NR_HVC_CONSOLES) ||
+				(num_found >= VTTY_PORTS))
+			break;
+
+		vtermno = (uint32_t *)get_property(vty, "reg", NULL);
+		if (!vtermno)
+			continue;
+
+		if (!device_is_compatible(vty, "IBM,iSeries-vty"))
+			continue;
+
+		if (num_found == 0)
+			add_preferred_console("hvc", 0, NULL);
+		hvc_instantiate(*vtermno, num_found, &hvc_get_put_ops);
+		++num_found;
+	}
+
+	return num_found;
+}
+console_initcall(hvc_find_vtys);
diff --git a/drivers/char/hvc_rtas.c b/drivers/char/hvc_rtas.c
index 57106e02..4b97eaf 100644
--- a/drivers/char/hvc_rtas.c
+++ b/drivers/char/hvc_rtas.c
@@ -94,7 +94,7 @@ static int hvc_rtas_init(void)
 
 	/* Allocate an hvc_struct for the console device we instantiated
 	 * earlier.  Save off hp so that we can return it on exit */
-	hp = hvc_alloc(hvc_rtas_cookie, NO_IRQ, &hvc_rtas_get_put_ops);
+	hp = hvc_alloc(hvc_rtas_cookie, NO_IRQ, &hvc_rtas_get_put_ops, 16);
 	if (IS_ERR(hp))
 		return PTR_ERR(hp);
 
diff --git a/drivers/char/hvc_vio.c b/drivers/char/hvc_vio.c
index 9add81ce..cc95941 100644
--- a/drivers/char/hvc_vio.c
+++ b/drivers/char/hvc_vio.c
@@ -90,7 +90,8 @@ static int __devinit hvc_vio_probe(struct vio_dev *vdev,
 	if (!vdev || !id)
 		return -EPERM;
 
-	hp = hvc_alloc(vdev->unit_address, vdev->irq, &hvc_get_put_ops);
+	hp = hvc_alloc(vdev->unit_address, vdev->irq, &hvc_get_put_ops,
+			MAX_VIO_PUT_CHARS);
 	if (IS_ERR(hp))
 		return PTR_ERR(hp);
 	dev_set_drvdata(&vdev->dev, hp);
@@ -140,7 +141,7 @@ static int hvc_find_vtys(void)
 
 	for (vty = of_find_node_by_name(NULL, "vty"); vty != NULL;
 			vty = of_find_node_by_name(vty, "vty")) {
-		uint32_t *vtermno;
+		const uint32_t *vtermno;
 
 		/* We have statically defined space for only a certain number
 		 * of console adapters.
@@ -148,7 +149,7 @@ static int hvc_find_vtys(void)
 		if (num_found >= MAX_NR_HVC_CONSOLES)
 			break;
 
-		vtermno = (uint32_t *)get_property(vty, "reg", NULL);
+		vtermno = get_property(vty, "reg", NULL);
 		if (!vtermno)
 			continue;
 
diff --git a/drivers/char/hvsi.c b/drivers/char/hvsi.c
index 017f755..a89a95f 100644
--- a/drivers/char/hvsi.c
+++ b/drivers/char/hvsi.c
@@ -1274,11 +1274,10 @@ static int __init hvsi_console_init(void)
 			vty != NULL;
 			vty = of_find_compatible_node(vty, "serial", "hvterm-protocol")) {
 		struct hvsi_struct *hp;
-		uint32_t *vtermno;
-		uint32_t *irq;
+		const uint32_t *vtermno, *irq;
 
-		vtermno = (uint32_t *)get_property(vty, "reg", NULL);
-		irq = (uint32_t *)get_property(vty, "interrupts", NULL);
+		vtermno = get_property(vty, "reg", NULL);
+		irq = get_property(vty, "interrupts", NULL);
 		if (!vtermno || !irq)
 			continue;
 
diff --git a/drivers/char/istallion.c b/drivers/char/istallion.c
index 84dfc42..8c09997 100644
--- a/drivers/char/istallion.c
+++ b/drivers/char/istallion.c
@@ -3488,7 +3488,7 @@ static int stli_initecp(stlibrd_t *brdp)
  */
 	EBRDENABLE(brdp);
 	sigsp = (cdkecpsig_t __iomem *) EBRDGETMEMPTR(brdp, CDK_SIGADDR);
-	memcpy(&sig, sigsp, sizeof(cdkecpsig_t));
+	memcpy_fromio(&sig, sigsp, sizeof(cdkecpsig_t));
 	EBRDDISABLE(brdp);
 
 	if (sig.magic != cpu_to_le32(ECP_MAGIC))
diff --git a/drivers/char/tpm/tpm_atmel.h b/drivers/char/tpm/tpm_atmel.h
index 2e68eeb..aefd683 100644
--- a/drivers/char/tpm/tpm_atmel.h
+++ b/drivers/char/tpm/tpm_atmel.h
@@ -37,7 +37,7 @@ static void __iomem * atmel_get_base_addr(unsigned long *base, int *region_size)
 {
 	struct device_node *dn;
 	unsigned long address, size;
-	unsigned int *reg;
+	const unsigned int *reg;
 	int reglen;
 	int naddrc;
 	int nsizec;
@@ -52,7 +52,7 @@ static void __iomem * atmel_get_base_addr(unsigned long *base, int *region_size)
 		return NULL;
 	}
 
-	reg = (unsigned int *) get_property(dn, "reg", &reglen);
+	reg = get_property(dn, "reg", &reglen);
 	naddrc = prom_n_addr_cells(dn);
 	nsizec = prom_n_size_cells(dn);
 
diff --git a/drivers/char/viocons.c b/drivers/char/viocons.c
index 766f786..f3efeaf 100644
--- a/drivers/char/viocons.c
+++ b/drivers/char/viocons.c
@@ -43,7 +43,6 @@
 #include <linux/sysrq.h>
 
 #include <asm/iseries/vio.h>
-
 #include <asm/iseries/hv_lp_event.h>
 #include <asm/iseries/hv_call_event.h>
 #include <asm/iseries/hv_lp_config.h>
@@ -67,35 +66,6 @@ static int vio_sysrq_pressed;
 extern int sysrq_enabled;
 #endif
 
-/*
- * The structure of the events that flow between us and OS/400.  You can't
- * mess with this unless the OS/400 side changes too
- */
-struct viocharlpevent {
-	struct HvLpEvent event;
-	u32 reserved;
-	u16 version;
-	u16 subtype_result_code;
-	u8 virtual_device;
-	u8 len;
-	u8 data[VIOCHAR_MAX_DATA];
-};
-
-#define VIOCHAR_WINDOW		10
-#define VIOCHAR_HIGHWATERMARK	3
-
-enum viocharsubtype {
-	viocharopen = 0x0001,
-	viocharclose = 0x0002,
-	viochardata = 0x0003,
-	viocharack = 0x0004,
-	viocharconfig = 0x0005
-};
-
-enum viochar_rc {
-	viochar_rc_ebusy = 1
-};
-
 #define VIOCHAR_NUM_BUF		16
 
 /*
@@ -1183,6 +1153,7 @@ static int __init viocons_init(void)
 		port_info[i].magic = VIOTTY_MAGIC;
 	}
 	HvCall_setLogBufferFormatAndCodepage(HvCall_LogBuffer_ASCII, 437);
+	add_preferred_console("viocons", 0, NULL);
 	register_console(&viocons_early);
 	return 0;
 }
diff --git a/drivers/char/viotape.c b/drivers/char/viotape.c
index b72b204..73c78bf 100644
--- a/drivers/char/viotape.c
+++ b/drivers/char/viotape.c
@@ -940,7 +940,6 @@ static void vioHandleTapeEvent(struct HvLpEvent *event)
 
 static int viotape_probe(struct vio_dev *vdev, const struct vio_device_id *id)
 {
-	char tapename[32];
 	int i = vdev->unit_address;
 	int j;
 
@@ -956,10 +955,9 @@ static int viotape_probe(struct vio_dev *vdev, const struct vio_device_id *id)
 			"iseries!vt%d", i);
 	class_device_create(tape_class, NULL, MKDEV(VIOTAPE_MAJOR, i | 0x80),
 			NULL, "iseries!nvt%d", i);
-	sprintf(tapename, "iseries/vt%d", i);
-	printk(VIOTAPE_KERN_INFO "tape %s is iSeries "
+	printk(VIOTAPE_KERN_INFO "tape iseries/vt%d is iSeries "
 			"resource %10.10s type %4.4s, model %3.3s\n",
-			tapename, viotape_unitinfo[i].rsrcname,
+			i, viotape_unitinfo[i].rsrcname,
 			viotape_unitinfo[i].type, viotape_unitinfo[i].model);
 	return 0;
 }
diff --git a/drivers/cpufreq/cpufreq.c b/drivers/cpufreq/cpufreq.c
index b3df613..d35a9f0 100644
--- a/drivers/cpufreq/cpufreq.c
+++ b/drivers/cpufreq/cpufreq.c
@@ -32,7 +32,7 @@
 #define dprintk(msg...) cpufreq_debug_printk(CPUFREQ_DEBUG_CORE, "cpufreq-core", msg)
 
 /**
- * The "cpufreq driver" - the arch- or hardware-dependend low
+ * The "cpufreq driver" - the arch- or hardware-dependent low
  * level driver of CPUFreq support, and its spinlock. This lock
  * also protects the cpufreq_cpu_data array.
  */
diff --git a/drivers/cpufreq/cpufreq_ondemand.c b/drivers/cpufreq/cpufreq_ondemand.c
index 52cf1f0..bf8aa45 100644
--- a/drivers/cpufreq/cpufreq_ondemand.c
+++ b/drivers/cpufreq/cpufreq_ondemand.c
@@ -55,6 +55,10 @@ struct cpu_dbs_info_s {
 	struct cpufreq_policy *cur_policy;
  	struct work_struct work;
 	unsigned int enable;
+	struct cpufreq_frequency_table *freq_table;
+	unsigned int freq_lo;
+	unsigned int freq_lo_jiffies;
+	unsigned int freq_hi_jiffies;
 };
 static DEFINE_PER_CPU(struct cpu_dbs_info_s, cpu_dbs_info);
 
@@ -72,15 +76,15 @@ static DEFINE_MUTEX(dbs_mutex);
 
 static struct workqueue_struct	*kondemand_wq;
 
-struct dbs_tuners {
+static struct dbs_tuners {
 	unsigned int sampling_rate;
 	unsigned int up_threshold;
 	unsigned int ignore_nice;
-};
-
-static struct dbs_tuners dbs_tuners_ins = {
+	unsigned int powersave_bias;
+} dbs_tuners_ins = {
 	.up_threshold = DEF_FREQUENCY_UP_THRESHOLD,
 	.ignore_nice = 0,
+	.powersave_bias = 0,
 };
 
 static inline cputime64_t get_cpu_idle_time(unsigned int cpu)
@@ -96,6 +100,70 @@ static inline cputime64_t get_cpu_idle_time(unsigned int cpu)
 	return retval;
 }
 
+/*
+ * Find right freq to be set now with powersave_bias on.
+ * Returns the freq_hi to be used right now and will set freq_hi_jiffies,
+ * freq_lo, and freq_lo_jiffies in percpu area for averaging freqs.
+ */
+static unsigned int powersave_bias_target(struct cpufreq_policy *policy,
+					  unsigned int freq_next,
+					  unsigned int relation)
+{
+	unsigned int freq_req, freq_reduc, freq_avg;
+	unsigned int freq_hi, freq_lo;
+	unsigned int index = 0;
+	unsigned int jiffies_total, jiffies_hi, jiffies_lo;
+	struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, policy->cpu);
+
+	if (!dbs_info->freq_table) {
+		dbs_info->freq_lo = 0;
+		dbs_info->freq_lo_jiffies = 0;
+		return freq_next;
+	}
+
+	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_next,
+			relation, &index);
+	freq_req = dbs_info->freq_table[index].frequency;
+	freq_reduc = freq_req * dbs_tuners_ins.powersave_bias / 1000;
+	freq_avg = freq_req - freq_reduc;
+
+	/* Find freq bounds for freq_avg in freq_table */
+	index = 0;
+	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
+			CPUFREQ_RELATION_H, &index);
+	freq_lo = dbs_info->freq_table[index].frequency;
+	index = 0;
+	cpufreq_frequency_table_target(policy, dbs_info->freq_table, freq_avg,
+			CPUFREQ_RELATION_L, &index);
+	freq_hi = dbs_info->freq_table[index].frequency;
+
+	/* Find out how long we have to be in hi and lo freqs */
+	if (freq_hi == freq_lo) {
+		dbs_info->freq_lo = 0;
+		dbs_info->freq_lo_jiffies = 0;
+		return freq_lo;
+	}
+	jiffies_total = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
+	jiffies_hi = (freq_avg - freq_lo) * jiffies_total;
+	jiffies_hi += ((freq_hi - freq_lo) / 2);
+	jiffies_hi /= (freq_hi - freq_lo);
+	jiffies_lo = jiffies_total - jiffies_hi;
+	dbs_info->freq_lo = freq_lo;
+	dbs_info->freq_lo_jiffies = jiffies_lo;
+	dbs_info->freq_hi_jiffies = jiffies_hi;
+	return freq_hi;
+}
+
+static void ondemand_powersave_bias_init(void)
+{
+	int i;
+	for_each_online_cpu(i) {
+		struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, i);
+		dbs_info->freq_table = cpufreq_frequency_get_table(i);
+		dbs_info->freq_lo = 0;
+	}
+}
+
 /************************** sysfs interface ************************/
 static ssize_t show_sampling_rate_max(struct cpufreq_policy *policy, char *buf)
 {
@@ -124,6 +192,7 @@ static ssize_t show_##file_name						\
 show_one(sampling_rate, sampling_rate);
 show_one(up_threshold, up_threshold);
 show_one(ignore_nice_load, ignore_nice);
+show_one(powersave_bias, powersave_bias);
 
 static ssize_t store_sampling_rate(struct cpufreq_policy *unused,
 		const char *buf, size_t count)
@@ -198,6 +267,27 @@ static ssize_t store_ignore_nice_load(struct cpufreq_policy *policy,
 	return count;
 }
 
+static ssize_t store_powersave_bias(struct cpufreq_policy *unused,
+		const char *buf, size_t count)
+{
+	unsigned int input;
+	int ret;
+	ret = sscanf(buf, "%u", &input);
+
+	if (ret != 1)
+		return -EINVAL;
+
+	if (input > 1000)
+		input = 1000;
+
+	mutex_lock(&dbs_mutex);
+	dbs_tuners_ins.powersave_bias = input;
+	ondemand_powersave_bias_init();
+	mutex_unlock(&dbs_mutex);
+
+	return count;
+}
+
 #define define_one_rw(_name) \
 static struct freq_attr _name = \
 __ATTR(_name, 0644, show_##_name, store_##_name)
@@ -205,6 +295,7 @@ __ATTR(_name, 0644, show_##_name, store_##_name)
 define_one_rw(sampling_rate);
 define_one_rw(up_threshold);
 define_one_rw(ignore_nice_load);
+define_one_rw(powersave_bias);
 
 static struct attribute * dbs_attributes[] = {
 	&sampling_rate_max.attr,
@@ -212,6 +303,7 @@ static struct attribute * dbs_attributes[] = {
 	&sampling_rate.attr,
 	&up_threshold.attr,
 	&ignore_nice_load.attr,
+	&powersave_bias.attr,
 	NULL
 };
 
@@ -234,6 +326,7 @@ static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
 	if (!this_dbs_info->enable)
 		return;
 
+	this_dbs_info->freq_lo = 0;
 	policy = this_dbs_info->cur_policy;
 	cur_jiffies = jiffies64_to_cputime64(get_jiffies_64());
 	total_ticks = (unsigned int) cputime64_sub(cur_jiffies,
@@ -274,11 +367,18 @@ static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
 	/* Check for frequency increase */
 	if (load > dbs_tuners_ins.up_threshold) {
 		/* if we are already at full speed then break out early */
-		if (policy->cur == policy->max)
-			return;
-
-		__cpufreq_driver_target(policy, policy->max,
-			CPUFREQ_RELATION_H);
+		if (!dbs_tuners_ins.powersave_bias) {
+			if (policy->cur == policy->max)
+				return;
+
+			__cpufreq_driver_target(policy, policy->max,
+				CPUFREQ_RELATION_H);
+		} else {
+			int freq = powersave_bias_target(policy, policy->max,
+					CPUFREQ_RELATION_H);
+			__cpufreq_driver_target(policy, freq,
+				CPUFREQ_RELATION_L);
+		}
 		return;
 	}
 
@@ -293,37 +393,64 @@ static void dbs_check_cpu(struct cpu_dbs_info_s *this_dbs_info)
 	 * policy. To be safe, we focus 10 points under the threshold.
 	 */
 	if (load < (dbs_tuners_ins.up_threshold - 10)) {
-		unsigned int freq_next;
-		freq_next = (policy->cur * load) /
+		unsigned int freq_next = (policy->cur * load) /
 			(dbs_tuners_ins.up_threshold - 10);
-
-		__cpufreq_driver_target(policy, freq_next, CPUFREQ_RELATION_L);
+		if (!dbs_tuners_ins.powersave_bias) {
+			__cpufreq_driver_target(policy, freq_next,
+					CPUFREQ_RELATION_L);
+		} else {
+			int freq = powersave_bias_target(policy, freq_next,
+					CPUFREQ_RELATION_L);
+			__cpufreq_driver_target(policy, freq,
+				CPUFREQ_RELATION_L);
+		}
 	}
 }
 
+/* Sampling types */
+enum {DBS_NORMAL_SAMPLE, DBS_SUB_SAMPLE};
+
 static void do_dbs_timer(void *data)
 {
 	unsigned int cpu = smp_processor_id();
 	struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, cpu);
+	/* We want all CPUs to do sampling nearly on same jiffy */
+	int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
+	delay -= jiffies % delay;
 
 	if (!dbs_info->enable)
 		return;
-
-	lock_cpu_hotplug();
-	dbs_check_cpu(dbs_info);
-	unlock_cpu_hotplug();
-	queue_delayed_work_on(cpu, kondemand_wq, &dbs_info->work,
-			usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
+	/* Common NORMAL_SAMPLE setup */
+	INIT_WORK(&dbs_info->work, do_dbs_timer, (void *)DBS_NORMAL_SAMPLE);
+	if (!dbs_tuners_ins.powersave_bias ||
+	    (unsigned long) data == DBS_NORMAL_SAMPLE) {
+		lock_cpu_hotplug();
+		dbs_check_cpu(dbs_info);
+		unlock_cpu_hotplug();
+		if (dbs_info->freq_lo) {
+			/* Setup timer for SUB_SAMPLE */
+			INIT_WORK(&dbs_info->work, do_dbs_timer,
+					(void *)DBS_SUB_SAMPLE);
+			delay = dbs_info->freq_hi_jiffies;
+		}
+	} else {
+		__cpufreq_driver_target(dbs_info->cur_policy,
+	                        	dbs_info->freq_lo,
+	                        	CPUFREQ_RELATION_H);
+	}
+	queue_delayed_work_on(cpu, kondemand_wq, &dbs_info->work, delay);
 }
 
 static inline void dbs_timer_init(unsigned int cpu)
 {
 	struct cpu_dbs_info_s *dbs_info = &per_cpu(cpu_dbs_info, cpu);
+	/* We want all CPUs to do sampling nearly on same jiffy */
+	int delay = usecs_to_jiffies(dbs_tuners_ins.sampling_rate);
+	delay -= jiffies % delay;
 
-	INIT_WORK(&dbs_info->work, do_dbs_timer, 0);
-	queue_delayed_work_on(cpu, kondemand_wq, &dbs_info->work,
-			usecs_to_jiffies(dbs_tuners_ins.sampling_rate));
-	return;
+	ondemand_powersave_bias_init();
+	INIT_WORK(&dbs_info->work, do_dbs_timer, NULL);
+	queue_delayed_work_on(cpu, kondemand_wq, &dbs_info->work, delay);
 }
 
 static inline void dbs_timer_exit(struct cpu_dbs_info_s *dbs_info)
diff --git a/drivers/cpufreq/cpufreq_stats.c b/drivers/cpufreq/cpufreq_stats.c
index 25eee53..c2ecc59 100644
--- a/drivers/cpufreq/cpufreq_stats.c
+++ b/drivers/cpufreq/cpufreq_stats.c
@@ -350,12 +350,10 @@ __init cpufreq_stats_init(void)
 	}
 
 	register_hotcpu_notifier(&cpufreq_stat_cpu_notifier);
-	lock_cpu_hotplug();
 	for_each_online_cpu(cpu) {
 		cpufreq_stat_cpu_callback(&cpufreq_stat_cpu_notifier, CPU_ONLINE,
 			(void *)(long)cpu);
 	}
-	unlock_cpu_hotplug();
 	return 0;
 }
 static void
diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig
index 4263935..adb5541 100644
--- a/drivers/crypto/Kconfig
+++ b/drivers/crypto/Kconfig
@@ -2,22 +2,53 @@ menu "Hardware crypto devices"
 
 config CRYPTO_DEV_PADLOCK
 	tristate "Support for VIA PadLock ACE"
-	depends on CRYPTO && X86_32
+	depends on X86_32
+	select CRYPTO_ALGAPI
+	default m
 	help
 	  Some VIA processors come with an integrated crypto engine
 	  (so called VIA PadLock ACE, Advanced Cryptography Engine)
-	  that provides instructions for very fast {en,de}cryption 
-	  with some algorithms.
+	  that provides instructions for very fast cryptographic
+	  operations with supported algorithms.
 	  
 	  The instructions are used only when the CPU supports them.
-	  Otherwise software encryption is used. If you are unsure,
-	  say Y.
+	  Otherwise software encryption is used.
+
+	  Selecting M for this option will compile a helper module
+	  padlock.ko that should autoload all below configured
+	  algorithms. Don't worry if your hardware does not support
+	  some or all of them. In such case padlock.ko will
+	  simply write a single line into the kernel log informing
+	  about its failure but everything will keep working fine.
+
+	  If you are unsure, say M. The compiled module will be
+	  called padlock.ko
 
 config CRYPTO_DEV_PADLOCK_AES
-	bool "Support for AES in VIA PadLock"
+	tristate "PadLock driver for AES algorithm"
 	depends on CRYPTO_DEV_PADLOCK
-	default y
+	select CRYPTO_BLKCIPHER
+	default m
 	help
 	  Use VIA PadLock for AES algorithm.
 
+	  Available in VIA C3 and newer CPUs.
+
+	  If unsure say M. The compiled module will be
+	  called padlock-aes.ko
+
+config CRYPTO_DEV_PADLOCK_SHA
+	tristate "PadLock driver for SHA1 and SHA256 algorithms"
+	depends on CRYPTO_DEV_PADLOCK
+	select CRYPTO_SHA1
+	select CRYPTO_SHA256
+	default m
+	help
+	  Use VIA PadLock for SHA1/SHA256 algorithms.
+
+	  Available in VIA C7 and newer processors.
+
+	  If unsure say M. The compiled module will be
+	  called padlock-sha.ko
+
 endmenu
diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile
index 45426ca..4c3d0ec 100644
--- a/drivers/crypto/Makefile
+++ b/drivers/crypto/Makefile
@@ -1,7 +1,3 @@
-
 obj-$(CONFIG_CRYPTO_DEV_PADLOCK) += padlock.o
-
-padlock-objs-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
-
-padlock-objs := padlock-generic.o $(padlock-objs-y)
-
+obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o
+obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o
diff --git a/drivers/crypto/padlock-aes.c b/drivers/crypto/padlock-aes.c
index b643d71..d4501dc 100644
--- a/drivers/crypto/padlock-aes.c
+++ b/drivers/crypto/padlock-aes.c
@@ -43,11 +43,11 @@
  * ---------------------------------------------------------------------------
  */
 
+#include <crypto/algapi.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/types.h>
 #include <linux/errno.h>
-#include <linux/crypto.h>
 #include <linux/interrupt.h>
 #include <linux/kernel.h>
 #include <asm/byteorder.h>
@@ -59,6 +59,17 @@
 #define AES_EXTENDED_KEY_SIZE	64	/* in uint32_t units */
 #define AES_EXTENDED_KEY_SIZE_B	(AES_EXTENDED_KEY_SIZE * sizeof(uint32_t))
 
+/* Control word. */
+struct cword {
+	unsigned int __attribute__ ((__packed__))
+		rounds:4,
+		algo:3,
+		keygen:1,
+		interm:1,
+		encdec:1,
+		ksize:2;
+} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
+
 /* Whenever making any changes to the following
  * structure *make sure* you keep E, d_data
  * and cword aligned on 16 Bytes boundaries!!! */
@@ -286,9 +297,9 @@ aes_hw_extkey_available(uint8_t key_len)
 	return 0;
 }
 
-static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
+static inline struct aes_ctx *aes_ctx_common(void *ctx)
 {
-	unsigned long addr = (unsigned long)crypto_tfm_ctx(tfm);
+	unsigned long addr = (unsigned long)ctx;
 	unsigned long align = PADLOCK_ALIGNMENT;
 
 	if (align <= crypto_tfm_ctx_alignment())
@@ -296,16 +307,27 @@ static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
 	return (struct aes_ctx *)ALIGN(addr, align);
 }
 
+static inline struct aes_ctx *aes_ctx(struct crypto_tfm *tfm)
+{
+	return aes_ctx_common(crypto_tfm_ctx(tfm));
+}
+
+static inline struct aes_ctx *blk_aes_ctx(struct crypto_blkcipher *tfm)
+{
+	return aes_ctx_common(crypto_blkcipher_ctx(tfm));
+}
+
 static int aes_set_key(struct crypto_tfm *tfm, const u8 *in_key,
-		       unsigned int key_len, u32 *flags)
+		       unsigned int key_len)
 {
 	struct aes_ctx *ctx = aes_ctx(tfm);
 	const __le32 *key = (const __le32 *)in_key;
+	u32 *flags = &tfm->crt_flags;
 	uint32_t i, t, u, v, w;
 	uint32_t P[AES_EXTENDED_KEY_SIZE];
 	uint32_t rounds;
 
-	if (key_len != 16 && key_len != 24 && key_len != 32) {
+	if (key_len % 8) {
 		*flags |= CRYPTO_TFM_RES_BAD_KEY_LEN;
 		return -EINVAL;
 	}
@@ -430,80 +452,212 @@ static void aes_decrypt(struct crypto_tfm *tfm, u8 *out, const u8 *in)
 	padlock_xcrypt_ecb(in, out, ctx->D, &ctx->cword.decrypt, 1);
 }
 
-static unsigned int aes_encrypt_ecb(const struct cipher_desc *desc, u8 *out,
-				    const u8 *in, unsigned int nbytes)
+static struct crypto_alg aes_alg = {
+	.cra_name		=	"aes",
+	.cra_driver_name	=	"aes-padlock",
+	.cra_priority		=	PADLOCK_CRA_PRIORITY,
+	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
+	.cra_blocksize		=	AES_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct aes_ctx),
+	.cra_alignmask		=	PADLOCK_ALIGNMENT - 1,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(aes_alg.cra_list),
+	.cra_u			=	{
+		.cipher = {
+			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
+			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
+			.cia_setkey	   	= 	aes_set_key,
+			.cia_encrypt	 	=	aes_encrypt,
+			.cia_decrypt	  	=	aes_decrypt,
+		}
+	}
+};
+
+static int ecb_aes_encrypt(struct blkcipher_desc *desc,
+			   struct scatterlist *dst, struct scatterlist *src,
+			   unsigned int nbytes)
 {
-	struct aes_ctx *ctx = aes_ctx(desc->tfm);
-	padlock_xcrypt_ecb(in, out, ctx->E, &ctx->cword.encrypt,
-			   nbytes / AES_BLOCK_SIZE);
-	return nbytes & ~(AES_BLOCK_SIZE - 1);
+	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while ((nbytes = walk.nbytes)) {
+		padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
+				   ctx->E, &ctx->cword.encrypt,
+				   nbytes / AES_BLOCK_SIZE);
+		nbytes &= AES_BLOCK_SIZE - 1;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
 }
 
-static unsigned int aes_decrypt_ecb(const struct cipher_desc *desc, u8 *out,
-				    const u8 *in, unsigned int nbytes)
+static int ecb_aes_decrypt(struct blkcipher_desc *desc,
+			   struct scatterlist *dst, struct scatterlist *src,
+			   unsigned int nbytes)
 {
-	struct aes_ctx *ctx = aes_ctx(desc->tfm);
-	padlock_xcrypt_ecb(in, out, ctx->D, &ctx->cword.decrypt,
-			   nbytes / AES_BLOCK_SIZE);
-	return nbytes & ~(AES_BLOCK_SIZE - 1);
+	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while ((nbytes = walk.nbytes)) {
+		padlock_xcrypt_ecb(walk.src.virt.addr, walk.dst.virt.addr,
+				   ctx->D, &ctx->cword.decrypt,
+				   nbytes / AES_BLOCK_SIZE);
+		nbytes &= AES_BLOCK_SIZE - 1;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
 }
 
-static unsigned int aes_encrypt_cbc(const struct cipher_desc *desc, u8 *out,
-				    const u8 *in, unsigned int nbytes)
-{
-	struct aes_ctx *ctx = aes_ctx(desc->tfm);
-	u8 *iv;
+static struct crypto_alg ecb_aes_alg = {
+	.cra_name		=	"ecb(aes)",
+	.cra_driver_name	=	"ecb-aes-padlock",
+	.cra_priority		=	PADLOCK_COMPOSITE_PRIORITY,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
+	.cra_blocksize		=	AES_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct aes_ctx),
+	.cra_alignmask		=	PADLOCK_ALIGNMENT - 1,
+	.cra_type		=	&crypto_blkcipher_type,
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(ecb_aes_alg.cra_list),
+	.cra_u			=	{
+		.blkcipher = {
+			.min_keysize		=	AES_MIN_KEY_SIZE,
+			.max_keysize		=	AES_MAX_KEY_SIZE,
+			.setkey	   		= 	aes_set_key,
+			.encrypt		=	ecb_aes_encrypt,
+			.decrypt		=	ecb_aes_decrypt,
+		}
+	}
+};
 
-	iv = padlock_xcrypt_cbc(in, out, ctx->E, desc->info,
-				&ctx->cword.encrypt, nbytes / AES_BLOCK_SIZE);
-	memcpy(desc->info, iv, AES_BLOCK_SIZE);
+static int cbc_aes_encrypt(struct blkcipher_desc *desc,
+			   struct scatterlist *dst, struct scatterlist *src,
+			   unsigned int nbytes)
+{
+	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while ((nbytes = walk.nbytes)) {
+		u8 *iv = padlock_xcrypt_cbc(walk.src.virt.addr,
+					    walk.dst.virt.addr, ctx->E,
+					    walk.iv, &ctx->cword.encrypt,
+					    nbytes / AES_BLOCK_SIZE);
+		memcpy(walk.iv, iv, AES_BLOCK_SIZE);
+		nbytes &= AES_BLOCK_SIZE - 1;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
 
-	return nbytes & ~(AES_BLOCK_SIZE - 1);
+	return err;
 }
 
-static unsigned int aes_decrypt_cbc(const struct cipher_desc *desc, u8 *out,
-				    const u8 *in, unsigned int nbytes)
+static int cbc_aes_decrypt(struct blkcipher_desc *desc,
+			   struct scatterlist *dst, struct scatterlist *src,
+			   unsigned int nbytes)
 {
-	struct aes_ctx *ctx = aes_ctx(desc->tfm);
-	padlock_xcrypt_cbc(in, out, ctx->D, desc->info, &ctx->cword.decrypt,
-			   nbytes / AES_BLOCK_SIZE);
-	return nbytes & ~(AES_BLOCK_SIZE - 1);
+	struct aes_ctx *ctx = blk_aes_ctx(desc->tfm);
+	struct blkcipher_walk walk;
+	int err;
+
+	blkcipher_walk_init(&walk, dst, src, nbytes);
+	err = blkcipher_walk_virt(desc, &walk);
+
+	while ((nbytes = walk.nbytes)) {
+		padlock_xcrypt_cbc(walk.src.virt.addr, walk.dst.virt.addr,
+				   ctx->D, walk.iv, &ctx->cword.decrypt,
+				   nbytes / AES_BLOCK_SIZE);
+		nbytes &= AES_BLOCK_SIZE - 1;
+		err = blkcipher_walk_done(desc, &walk, nbytes);
+	}
+
+	return err;
 }
 
-static struct crypto_alg aes_alg = {
-	.cra_name		=	"aes",
-	.cra_driver_name	=	"aes-padlock",
-	.cra_priority		=	300,
-	.cra_flags		=	CRYPTO_ALG_TYPE_CIPHER,
+static struct crypto_alg cbc_aes_alg = {
+	.cra_name		=	"cbc(aes)",
+	.cra_driver_name	=	"cbc-aes-padlock",
+	.cra_priority		=	PADLOCK_COMPOSITE_PRIORITY,
+	.cra_flags		=	CRYPTO_ALG_TYPE_BLKCIPHER,
 	.cra_blocksize		=	AES_BLOCK_SIZE,
 	.cra_ctxsize		=	sizeof(struct aes_ctx),
 	.cra_alignmask		=	PADLOCK_ALIGNMENT - 1,
+	.cra_type		=	&crypto_blkcipher_type,
 	.cra_module		=	THIS_MODULE,
-	.cra_list		=	LIST_HEAD_INIT(aes_alg.cra_list),
+	.cra_list		=	LIST_HEAD_INIT(cbc_aes_alg.cra_list),
 	.cra_u			=	{
-		.cipher = {
-			.cia_min_keysize	=	AES_MIN_KEY_SIZE,
-			.cia_max_keysize	=	AES_MAX_KEY_SIZE,
-			.cia_setkey	   	= 	aes_set_key,
-			.cia_encrypt	 	=	aes_encrypt,
-			.cia_decrypt	  	=	aes_decrypt,
-			.cia_encrypt_ecb 	=	aes_encrypt_ecb,
-			.cia_decrypt_ecb  	=	aes_decrypt_ecb,
-			.cia_encrypt_cbc 	=	aes_encrypt_cbc,
-			.cia_decrypt_cbc  	=	aes_decrypt_cbc,
+		.blkcipher = {
+			.min_keysize		=	AES_MIN_KEY_SIZE,
+			.max_keysize		=	AES_MAX_KEY_SIZE,
+			.ivsize			=	AES_BLOCK_SIZE,
+			.setkey	   		= 	aes_set_key,
+			.encrypt		=	cbc_aes_encrypt,
+			.decrypt		=	cbc_aes_decrypt,
 		}
 	}
 };
 
-int __init padlock_init_aes(void)
+static int __init padlock_init(void)
 {
-	printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
+	int ret;
+
+	if (!cpu_has_xcrypt) {
+		printk(KERN_ERR PFX "VIA PadLock not detected.\n");
+		return -ENODEV;
+	}
+
+	if (!cpu_has_xcrypt_enabled) {
+		printk(KERN_ERR PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
+		return -ENODEV;
+	}
 
 	gen_tabs();
-	return crypto_register_alg(&aes_alg);
+	if ((ret = crypto_register_alg(&aes_alg)))
+		goto aes_err;
+
+	if ((ret = crypto_register_alg(&ecb_aes_alg)))
+		goto ecb_aes_err;
+
+	if ((ret = crypto_register_alg(&cbc_aes_alg)))
+		goto cbc_aes_err;
+
+	printk(KERN_NOTICE PFX "Using VIA PadLock ACE for AES algorithm.\n");
+
+out:
+	return ret;
+
+cbc_aes_err:
+	crypto_unregister_alg(&ecb_aes_alg);
+ecb_aes_err:
+	crypto_unregister_alg(&aes_alg);
+aes_err:
+	printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
+	goto out;
 }
 
-void __exit padlock_fini_aes(void)
+static void __exit padlock_fini(void)
 {
+	crypto_unregister_alg(&cbc_aes_alg);
+	crypto_unregister_alg(&ecb_aes_alg);
 	crypto_unregister_alg(&aes_alg);
 }
+
+module_init(padlock_init);
+module_exit(padlock_fini);
+
+MODULE_DESCRIPTION("VIA PadLock AES algorithm support");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Michal Ludvig");
+
+MODULE_ALIAS("aes-padlock");
diff --git a/drivers/crypto/padlock-generic.c b/drivers/crypto/padlock-generic.c
deleted file mode 100644
index 18cf0e8..0000000
--- a/drivers/crypto/padlock-generic.c
+++ /dev/null
@@ -1,63 +0,0 @@
-/* 
- * Cryptographic API.
- *
- * Support for VIA PadLock hardware crypto engine.
- *
- * Copyright (c) 2004  Michal Ludvig <michal@logix.cz>
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2 of the License, or
- * (at your option) any later version.
- */
-
-#include <linux/module.h>
-#include <linux/init.h>
-#include <linux/types.h>
-#include <linux/errno.h>
-#include <linux/crypto.h>
-#include <asm/byteorder.h>
-#include "padlock.h"
-
-static int __init
-padlock_init(void)
-{
-	int ret = -ENOSYS;
-	
-	if (!cpu_has_xcrypt) {
-		printk(KERN_ERR PFX "VIA PadLock not detected.\n");
-		return -ENODEV;
-	}
-
-	if (!cpu_has_xcrypt_enabled) {
-		printk(KERN_ERR PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
-		return -ENODEV;
-	}
-
-#ifdef CONFIG_CRYPTO_DEV_PADLOCK_AES
-	if ((ret = padlock_init_aes())) {
-		printk(KERN_ERR PFX "VIA PadLock AES initialization failed.\n");
-		return ret;
-	}
-#endif
-
-	if (ret == -ENOSYS)
-		printk(KERN_ERR PFX "Hmm, VIA PadLock was compiled without any algorithm.\n");
-
-	return ret;
-}
-
-static void __exit
-padlock_fini(void)
-{
-#ifdef CONFIG_CRYPTO_DEV_PADLOCK_AES
-	padlock_fini_aes();
-#endif
-}
-
-module_init(padlock_init);
-module_exit(padlock_fini);
-
-MODULE_DESCRIPTION("VIA PadLock crypto engine support.");
-MODULE_LICENSE("Dual BSD/GPL");
-MODULE_AUTHOR("Michal Ludvig");
diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c
new file mode 100644
index 0000000..a781fd2
--- /dev/null
+++ b/drivers/crypto/padlock-sha.c
@@ -0,0 +1,318 @@
+/*
+ * Cryptographic API.
+ *
+ * Support for VIA PadLock hardware crypto engine.
+ *
+ * Copyright (c) 2006  Michal Ludvig <michal@logix.cz>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <crypto/algapi.h>
+#include <linux/err.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/cryptohash.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/scatterlist.h>
+#include "padlock.h"
+
+#define SHA1_DEFAULT_FALLBACK	"sha1-generic"
+#define SHA1_DIGEST_SIZE        20
+#define SHA1_HMAC_BLOCK_SIZE    64
+
+#define SHA256_DEFAULT_FALLBACK "sha256-generic"
+#define SHA256_DIGEST_SIZE      32
+#define SHA256_HMAC_BLOCK_SIZE  64
+
+struct padlock_sha_ctx {
+	char		*data;
+	size_t		used;
+	int		bypass;
+	void (*f_sha_padlock)(const char *in, char *out, int count);
+	struct hash_desc fallback;
+};
+
+static inline struct padlock_sha_ctx *ctx(struct crypto_tfm *tfm)
+{
+	return crypto_tfm_ctx(tfm);
+}
+
+/* We'll need aligned address on the stack */
+#define NEAREST_ALIGNED(ptr) \
+	((void *)ALIGN((size_t)(ptr), PADLOCK_ALIGNMENT))
+
+static struct crypto_alg sha1_alg, sha256_alg;
+
+static void padlock_sha_bypass(struct crypto_tfm *tfm)
+{
+	if (ctx(tfm)->bypass)
+		return;
+
+	crypto_hash_init(&ctx(tfm)->fallback);
+	if (ctx(tfm)->data && ctx(tfm)->used) {
+		struct scatterlist sg;
+
+		sg_set_buf(&sg, ctx(tfm)->data, ctx(tfm)->used);
+		crypto_hash_update(&ctx(tfm)->fallback, &sg, sg.length);
+	}
+
+	ctx(tfm)->used = 0;
+	ctx(tfm)->bypass = 1;
+}
+
+static void padlock_sha_init(struct crypto_tfm *tfm)
+{
+	ctx(tfm)->used = 0;
+	ctx(tfm)->bypass = 0;
+}
+
+static void padlock_sha_update(struct crypto_tfm *tfm,
+			const uint8_t *data, unsigned int length)
+{
+	/* Our buffer is always one page. */
+	if (unlikely(!ctx(tfm)->bypass &&
+		     (ctx(tfm)->used + length > PAGE_SIZE)))
+		padlock_sha_bypass(tfm);
+
+	if (unlikely(ctx(tfm)->bypass)) {
+		struct scatterlist sg;
+		sg_set_buf(&sg, (uint8_t *)data, length);
+		crypto_hash_update(&ctx(tfm)->fallback, &sg, length);
+		return;
+	}
+
+	memcpy(ctx(tfm)->data + ctx(tfm)->used, data, length);
+	ctx(tfm)->used += length;
+}
+
+static inline void padlock_output_block(uint32_t *src,
+		 	uint32_t *dst, size_t count)
+{
+	while (count--)
+		*dst++ = swab32(*src++);
+}
+
+static void padlock_do_sha1(const char *in, char *out, int count)
+{
+	/* We can't store directly to *out as it may be unaligned. */
+	/* BTW Don't reduce the buffer size below 128 Bytes!
+	 *     PadLock microcode needs it that big. */
+	char buf[128+16];
+	char *result = NEAREST_ALIGNED(buf);
+
+	((uint32_t *)result)[0] = 0x67452301;
+	((uint32_t *)result)[1] = 0xEFCDAB89;
+	((uint32_t *)result)[2] = 0x98BADCFE;
+	((uint32_t *)result)[3] = 0x10325476;
+	((uint32_t *)result)[4] = 0xC3D2E1F0;
+ 
+	asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */
+		      : "+S"(in), "+D"(result)
+		      : "c"(count), "a"(0));
+
+	padlock_output_block((uint32_t *)result, (uint32_t *)out, 5);
+}
+
+static void padlock_do_sha256(const char *in, char *out, int count)
+{
+	/* We can't store directly to *out as it may be unaligned. */
+	/* BTW Don't reduce the buffer size below 128 Bytes!
+	 *     PadLock microcode needs it that big. */
+	char buf[128+16];
+	char *result = NEAREST_ALIGNED(buf);
+
+	((uint32_t *)result)[0] = 0x6A09E667;
+	((uint32_t *)result)[1] = 0xBB67AE85;
+	((uint32_t *)result)[2] = 0x3C6EF372;
+	((uint32_t *)result)[3] = 0xA54FF53A;
+	((uint32_t *)result)[4] = 0x510E527F;
+	((uint32_t *)result)[5] = 0x9B05688C;
+	((uint32_t *)result)[6] = 0x1F83D9AB;
+	((uint32_t *)result)[7] = 0x5BE0CD19;
+
+	asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */
+		      : "+S"(in), "+D"(result)
+		      : "c"(count), "a"(0));
+
+	padlock_output_block((uint32_t *)result, (uint32_t *)out, 8);
+}
+
+static void padlock_sha_final(struct crypto_tfm *tfm, uint8_t *out)
+{
+	if (unlikely(ctx(tfm)->bypass)) {
+		crypto_hash_final(&ctx(tfm)->fallback, out);
+		ctx(tfm)->bypass = 0;
+		return;
+	}
+
+	/* Pass the input buffer to PadLock microcode... */
+	ctx(tfm)->f_sha_padlock(ctx(tfm)->data, out, ctx(tfm)->used);
+
+	ctx(tfm)->used = 0;
+}
+
+static int padlock_cra_init(struct crypto_tfm *tfm)
+{
+	const char *fallback_driver_name = tfm->__crt_alg->cra_name;
+	struct crypto_hash *fallback_tfm;
+
+	/* For now we'll allocate one page. This
+	 * could eventually be configurable one day. */
+	ctx(tfm)->data = (char *)__get_free_page(GFP_KERNEL);
+	if (!ctx(tfm)->data)
+		return -ENOMEM;
+
+	/* Allocate a fallback and abort if it failed. */
+	fallback_tfm = crypto_alloc_hash(fallback_driver_name, 0,
+					 CRYPTO_ALG_ASYNC |
+					 CRYPTO_ALG_NEED_FALLBACK);
+	if (IS_ERR(fallback_tfm)) {
+		printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n",
+		       fallback_driver_name);
+		free_page((unsigned long)(ctx(tfm)->data));
+		return PTR_ERR(fallback_tfm);
+	}
+
+	ctx(tfm)->fallback.tfm = fallback_tfm;
+	return 0;
+}
+
+static int padlock_sha1_cra_init(struct crypto_tfm *tfm)
+{
+	ctx(tfm)->f_sha_padlock = padlock_do_sha1;
+
+	return padlock_cra_init(tfm);
+}
+
+static int padlock_sha256_cra_init(struct crypto_tfm *tfm)
+{
+	ctx(tfm)->f_sha_padlock = padlock_do_sha256;
+
+	return padlock_cra_init(tfm);
+}
+
+static void padlock_cra_exit(struct crypto_tfm *tfm)
+{
+	if (ctx(tfm)->data) {
+		free_page((unsigned long)(ctx(tfm)->data));
+		ctx(tfm)->data = NULL;
+	}
+
+	crypto_free_hash(ctx(tfm)->fallback.tfm);
+	ctx(tfm)->fallback.tfm = NULL;
+}
+
+static struct crypto_alg sha1_alg = {
+	.cra_name		=	"sha1",
+	.cra_driver_name	=	"sha1-padlock",
+	.cra_priority		=	PADLOCK_CRA_PRIORITY,
+	.cra_flags		=	CRYPTO_ALG_TYPE_DIGEST |
+					CRYPTO_ALG_NEED_FALLBACK,
+	.cra_blocksize		=	SHA1_HMAC_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct padlock_sha_ctx),
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(sha1_alg.cra_list),
+	.cra_init		=	padlock_sha1_cra_init,
+	.cra_exit		=	padlock_cra_exit,
+	.cra_u			=	{
+		.digest = {
+			.dia_digestsize	=	SHA1_DIGEST_SIZE,
+			.dia_init   	= 	padlock_sha_init,
+			.dia_update 	=	padlock_sha_update,
+			.dia_final  	=	padlock_sha_final,
+		}
+	}
+};
+
+static struct crypto_alg sha256_alg = {
+	.cra_name		=	"sha256",
+	.cra_driver_name	=	"sha256-padlock",
+	.cra_priority		=	PADLOCK_CRA_PRIORITY,
+	.cra_flags		=	CRYPTO_ALG_TYPE_DIGEST |
+					CRYPTO_ALG_NEED_FALLBACK,
+	.cra_blocksize		=	SHA256_HMAC_BLOCK_SIZE,
+	.cra_ctxsize		=	sizeof(struct padlock_sha_ctx),
+	.cra_module		=	THIS_MODULE,
+	.cra_list		=	LIST_HEAD_INIT(sha256_alg.cra_list),
+	.cra_init		=	padlock_sha256_cra_init,
+	.cra_exit		=	padlock_cra_exit,
+	.cra_u			=	{
+		.digest = {
+			.dia_digestsize	=	SHA256_DIGEST_SIZE,
+			.dia_init   	= 	padlock_sha_init,
+			.dia_update 	=	padlock_sha_update,
+			.dia_final  	=	padlock_sha_final,
+		}
+	}
+};
+
+static void __init padlock_sha_check_fallbacks(void)
+{
+	if (!crypto_has_hash("sha1", 0, CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_NEED_FALLBACK))
+		printk(KERN_WARNING PFX
+		       "Couldn't load fallback module for sha1.\n");
+
+	if (!crypto_has_hash("sha256", 0, CRYPTO_ALG_ASYNC |
+					CRYPTO_ALG_NEED_FALLBACK))
+		printk(KERN_WARNING PFX
+		       "Couldn't load fallback module for sha256.\n");
+}
+
+static int __init padlock_init(void)
+{
+	int rc = -ENODEV;
+
+	if (!cpu_has_phe) {
+		printk(KERN_ERR PFX "VIA PadLock Hash Engine not detected.\n");
+		return -ENODEV;
+	}
+
+	if (!cpu_has_phe_enabled) {
+		printk(KERN_ERR PFX "VIA PadLock detected, but not enabled. Hmm, strange...\n");
+		return -ENODEV;
+	}
+
+	padlock_sha_check_fallbacks();
+
+	rc = crypto_register_alg(&sha1_alg);
+	if (rc)
+		goto out;
+
+	rc = crypto_register_alg(&sha256_alg);
+	if (rc)
+		goto out_unreg1;
+
+	printk(KERN_NOTICE PFX "Using VIA PadLock ACE for SHA1/SHA256 algorithms.\n");
+
+	return 0;
+
+out_unreg1:
+	crypto_unregister_alg(&sha1_alg);
+out:
+	printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n");
+	return rc;
+}
+
+static void __exit padlock_fini(void)
+{
+	crypto_unregister_alg(&sha1_alg);
+	crypto_unregister_alg(&sha256_alg);
+}
+
+module_init(padlock_init);
+module_exit(padlock_fini);
+
+MODULE_DESCRIPTION("VIA PadLock SHA1/SHA256 algorithms support.");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Michal Ludvig");
+
+MODULE_ALIAS("sha1-padlock");
+MODULE_ALIAS("sha256-padlock");
diff --git a/drivers/crypto/padlock.c b/drivers/crypto/padlock.c
new file mode 100644
index 0000000..d6d7dd5
--- /dev/null
+++ b/drivers/crypto/padlock.c
@@ -0,0 +1,58 @@
+/*
+ * Cryptographic API.
+ *
+ * Support for VIA PadLock hardware crypto engine.
+ *
+ * Copyright (c) 2006  Michal Ludvig <michal@logix.cz>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/crypto.h>
+#include <linux/cryptohash.h>
+#include <linux/interrupt.h>
+#include <linux/kernel.h>
+#include <linux/scatterlist.h>
+#include "padlock.h"
+
+static int __init padlock_init(void)
+{
+	int success = 0;
+
+	if (crypto_has_cipher("aes-padlock", 0, 0))
+		success++;
+
+	if (crypto_has_hash("sha1-padlock", 0, 0))
+		success++;
+
+	if (crypto_has_hash("sha256-padlock", 0, 0))
+		success++;
+
+	if (!success) {
+		printk(KERN_WARNING PFX "No VIA PadLock drivers have been loaded.\n");
+		return -ENODEV;
+	}
+
+	printk(KERN_NOTICE PFX "%d drivers are available.\n", success);
+
+	return 0;
+}
+
+static void __exit padlock_fini(void)
+{
+}
+
+module_init(padlock_init);
+module_exit(padlock_fini);
+
+MODULE_DESCRIPTION("Load all configured PadLock algorithms.");
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Michal Ludvig");
+
diff --git a/drivers/crypto/padlock.h b/drivers/crypto/padlock.h
index b78489b..b728e45 100644
--- a/drivers/crypto/padlock.h
+++ b/drivers/crypto/padlock.h
@@ -15,22 +15,9 @@
 
 #define PADLOCK_ALIGNMENT 16
 
-/* Control word. */
-struct cword {
-	unsigned int __attribute__ ((__packed__))
-		rounds:4,
-		algo:3,
-		keygen:1,
-		interm:1,
-		encdec:1,
-		ksize:2;
-} __attribute__ ((__aligned__(PADLOCK_ALIGNMENT)));
-
 #define PFX	"padlock: "
 
-#ifdef CONFIG_CRYPTO_DEV_PADLOCK_AES
-int padlock_init_aes(void);
-void padlock_fini_aes(void);
-#endif
+#define PADLOCK_CRA_PRIORITY	300
+#define PADLOCK_COMPOSITE_PRIORITY 400
 
 #endif	/* _CRYPTO_PADLOCK_H */
diff --git a/drivers/i2c/busses/i2c-powermac.c b/drivers/i2c/busses/i2c-powermac.c
index 53bb435..d658d91 100644
--- a/drivers/i2c/busses/i2c-powermac.c
+++ b/drivers/i2c/busses/i2c-powermac.c
@@ -207,7 +207,8 @@ static int i2c_powermac_probe(struct device *dev)
 	struct pmac_i2c_bus *bus = dev->platform_data;
 	struct device_node *parent = NULL;
 	struct i2c_adapter *adapter;
-	char name[32], *basename;
+	char name[32];
+	const char *basename;
 	int rc;
 
 	if (bus == NULL)
diff --git a/drivers/ide/ppc/pmac.c b/drivers/ide/ppc/pmac.c
index ebf961f..996c694 100644
--- a/drivers/ide/ppc/pmac.c
+++ b/drivers/ide/ppc/pmac.c
@@ -1154,7 +1154,7 @@ static int
 pmac_ide_setup_device(pmac_ide_hwif_t *pmif, ide_hwif_t *hwif)
 {
 	struct device_node *np = pmif->node;
-	int *bidp;
+	const int *bidp;
 
 	pmif->cable_80 = 0;
 	pmif->broken_dma = pmif->broken_dma_warn = 0;
@@ -1176,14 +1176,14 @@ pmac_ide_setup_device(pmac_ide_hwif_t *pmif, ide_hwif_t *hwif)
 		pmif->broken_dma = 1;
 	}
 
-	bidp = (int *)get_property(np, "AAPL,bus-id", NULL);
+	bidp = get_property(np, "AAPL,bus-id", NULL);
 	pmif->aapl_bus_id =  bidp ? *bidp : 0;
 
 	/* Get cable type from device-tree */
 	if (pmif->kind == controller_kl_ata4 || pmif->kind == controller_un_ata6
 	    || pmif->kind == controller_k2_ata6
 	    || pmif->kind == controller_sh_ata6) {
-		char* cable = get_property(np, "cable-type", NULL);
+		const char* cable = get_property(np, "cable-type", NULL);
 		if (cable && !strncmp(cable, "80-", 3))
 			pmif->cable_80 = 1;
 	}
@@ -1326,7 +1326,7 @@ pmac_ide_macio_attach(struct macio_dev *mdev, const struct of_device_id *match)
 	if (macio_irq_count(mdev) == 0) {
 		printk(KERN_WARNING "ide%d: no intrs for device %s, using 13\n",
 			i, mdev->ofdev.node->full_name);
-		irq = 13;
+		irq = irq_create_mapping(NULL, 13);
 	} else
 		irq = macio_irq(mdev, 0);
 
diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig
index 69a53d4..9edface 100644
--- a/drivers/infiniband/Kconfig
+++ b/drivers/infiniband/Kconfig
@@ -14,7 +14,7 @@ config INFINIBAND_USER_MAD
 	---help---
 	  Userspace InfiniBand Management Datagram (MAD) support.  This
 	  is the kernel side of the userspace MAD support, which allows
-	  userspace processes to send and receive MADs. You will also 
+	  userspace processes to send and receive MADs. You will also
 	  need libibumad from <http://www.openib.org>.
 
 config INFINIBAND_USER_ACCESS
@@ -36,6 +36,8 @@ config INFINIBAND_ADDR_TRANS
 
 source "drivers/infiniband/hw/mthca/Kconfig"
 source "drivers/infiniband/hw/ipath/Kconfig"
+source "drivers/infiniband/hw/ehca/Kconfig"
+source "drivers/infiniband/hw/amso1100/Kconfig"
 
 source "drivers/infiniband/ulp/ipoib/Kconfig"
 
diff --git a/drivers/infiniband/Makefile b/drivers/infiniband/Makefile
index c7ff58c..2b5d109 100644
--- a/drivers/infiniband/Makefile
+++ b/drivers/infiniband/Makefile
@@ -1,6 +1,8 @@
 obj-$(CONFIG_INFINIBAND)		+= core/
 obj-$(CONFIG_INFINIBAND_MTHCA)		+= hw/mthca/
-obj-$(CONFIG_IPATH_CORE)		+= hw/ipath/
+obj-$(CONFIG_INFINIBAND_IPATH)		+= hw/ipath/
+obj-$(CONFIG_INFINIBAND_EHCA)		+= hw/ehca/
+obj-$(CONFIG_INFINIBAND_AMSO1100)	+= hw/amso1100/
 obj-$(CONFIG_INFINIBAND_IPOIB)		+= ulp/ipoib/
 obj-$(CONFIG_INFINIBAND_SRP)		+= ulp/srp/
 obj-$(CONFIG_INFINIBAND_ISER)		+= ulp/iser/
diff --git a/drivers/infiniband/core/Makefile b/drivers/infiniband/core/Makefile
index 68e73ec..163d991 100644
--- a/drivers/infiniband/core/Makefile
+++ b/drivers/infiniband/core/Makefile
@@ -1,7 +1,7 @@
 infiniband-$(CONFIG_INFINIBAND_ADDR_TRANS)	:= ib_addr.o rdma_cm.o
 
 obj-$(CONFIG_INFINIBAND) +=		ib_core.o ib_mad.o ib_sa.o \
-					ib_cm.o $(infiniband-y)
+					ib_cm.o iw_cm.o $(infiniband-y)
 obj-$(CONFIG_INFINIBAND_USER_MAD) +=	ib_umad.o
 obj-$(CONFIG_INFINIBAND_USER_ACCESS) +=	ib_uverbs.o ib_ucm.o
 
@@ -14,6 +14,8 @@ ib_sa-y :=			sa_query.o
 
 ib_cm-y :=			cm.o
 
+iw_cm-y :=			iwcm.o
+
 rdma_cm-y :=			cma.o
 
 ib_addr-y :=			addr.o
diff --git a/drivers/infiniband/core/addr.c b/drivers/infiniband/core/addr.c
index 1205e80..9cbf09e 100644
--- a/drivers/infiniband/core/addr.c
+++ b/drivers/infiniband/core/addr.c
@@ -61,12 +61,15 @@ static LIST_HEAD(req_list);
 static DECLARE_WORK(work, process_req, NULL);
 static struct workqueue_struct *addr_wq;
 
-static int copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev,
-		     unsigned char *dst_dev_addr)
+int rdma_copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev,
+		     const unsigned char *dst_dev_addr)
 {
 	switch (dev->type) {
 	case ARPHRD_INFINIBAND:
-		dev_addr->dev_type = IB_NODE_CA;
+		dev_addr->dev_type = RDMA_NODE_IB_CA;
+		break;
+	case ARPHRD_ETHER:
+		dev_addr->dev_type = RDMA_NODE_RNIC;
 		break;
 	default:
 		return -EADDRNOTAVAIL;
@@ -78,6 +81,7 @@ static int copy_addr(struct rdma_dev_addr *dev_addr, struct net_device *dev,
 		memcpy(dev_addr->dst_dev_addr, dst_dev_addr, MAX_ADDR_LEN);
 	return 0;
 }
+EXPORT_SYMBOL(rdma_copy_addr);
 
 int rdma_translate_ip(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
 {
@@ -89,7 +93,7 @@ int rdma_translate_ip(struct sockaddr *addr, struct rdma_dev_addr *dev_addr)
 	if (!dev)
 		return -EADDRNOTAVAIL;
 
-	ret = copy_addr(dev_addr, dev, NULL);
+	ret = rdma_copy_addr(dev_addr, dev, NULL);
 	dev_put(dev);
 	return ret;
 }
@@ -161,7 +165,7 @@ static int addr_resolve_remote(struct sockaddr_in *src_in,
 
 	/* If the device does ARP internally, return 'done' */
 	if (rt->idev->dev->flags & IFF_NOARP) {
-		copy_addr(addr, rt->idev->dev, NULL);
+		rdma_copy_addr(addr, rt->idev->dev, NULL);
 		goto put;
 	}
 
@@ -181,7 +185,7 @@ static int addr_resolve_remote(struct sockaddr_in *src_in,
 		src_in->sin_addr.s_addr = rt->rt_src;
 	}
 
-	ret = copy_addr(addr, neigh->dev, neigh->ha);
+	ret = rdma_copy_addr(addr, neigh->dev, neigh->ha);
 release:
 	neigh_release(neigh);
 put:
@@ -245,7 +249,7 @@ static int addr_resolve_local(struct sockaddr_in *src_in,
 	if (ZERONET(src_ip)) {
 		src_in->sin_family = dst_in->sin_family;
 		src_in->sin_addr.s_addr = dst_ip;
-		ret = copy_addr(addr, dev, dev->dev_addr);
+		ret = rdma_copy_addr(addr, dev, dev->dev_addr);
 	} else if (LOOPBACK(src_ip)) {
 		ret = rdma_translate_ip((struct sockaddr *)dst_in, addr);
 		if (!ret)
@@ -327,10 +331,10 @@ void rdma_addr_cancel(struct rdma_dev_addr *addr)
 }
 EXPORT_SYMBOL(rdma_addr_cancel);
 
-static int netevent_callback(struct notifier_block *self, unsigned long event, 
+static int netevent_callback(struct notifier_block *self, unsigned long event,
 	void *ctx)
 {
-	if (event == NETEVENT_NEIGH_UPDATE) {  
+	if (event == NETEVENT_NEIGH_UPDATE) {
 		struct neighbour *neigh = ctx;
 
 		if (neigh->dev->type == ARPHRD_INFINIBAND &&
diff --git a/drivers/infiniband/core/cache.c b/drivers/infiniband/core/cache.c
index 75313ad..20e9f64 100644
--- a/drivers/infiniband/core/cache.c
+++ b/drivers/infiniband/core/cache.c
@@ -62,12 +62,13 @@ struct ib_update_work {
 
 static inline int start_port(struct ib_device *device)
 {
-	return device->node_type == IB_NODE_SWITCH ? 0 : 1;
+	return (device->node_type == RDMA_NODE_IB_SWITCH) ? 0 : 1;
 }
 
 static inline int end_port(struct ib_device *device)
 {
-	return device->node_type == IB_NODE_SWITCH ? 0 : device->phys_port_cnt;
+	return (device->node_type == RDMA_NODE_IB_SWITCH) ?
+		0 : device->phys_port_cnt;
 }
 
 int ib_get_cached_gid(struct ib_device *device,
diff --git a/drivers/infiniband/core/cm.c b/drivers/infiniband/core/cm.c
index 0de335b..f35fcc4 100644
--- a/drivers/infiniband/core/cm.c
+++ b/drivers/infiniband/core/cm.c
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, 2005 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 Intel Corporation.  All rights reserved.
  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
  * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
@@ -41,6 +41,7 @@
 #include <linux/idr.h>
 #include <linux/interrupt.h>
 #include <linux/pci.h>
+#include <linux/random.h>
 #include <linux/rbtree.h>
 #include <linux/spinlock.h>
 #include <linux/workqueue.h>
@@ -73,6 +74,7 @@ static struct ib_cm {
 	struct rb_root remote_id_table;
 	struct rb_root remote_sidr_table;
 	struct idr local_id_table;
+	__be32 random_id_operand;
 	struct workqueue_struct *wq;
 } cm;
 
@@ -177,7 +179,7 @@ static int cm_alloc_msg(struct cm_id_private *cm_id_priv,
 	if (IS_ERR(ah))
 		return PTR_ERR(ah);
 
-	m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn, 
+	m = ib_create_send_mad(mad_agent, cm_id_priv->id.remote_cm_qpn,
 			       cm_id_priv->av.pkey_index,
 			       0, IB_MGMT_MAD_HDR, IB_MGMT_MAD_DATA,
 			       GFP_ATOMIC);
@@ -299,15 +301,17 @@ static int cm_init_av_by_path(struct ib_sa_path_rec *path, struct cm_av *av)
 static int cm_alloc_id(struct cm_id_private *cm_id_priv)
 {
 	unsigned long flags;
-	int ret;
+	int ret, id;
 	static int next_id;
 
 	do {
 		spin_lock_irqsave(&cm.lock, flags);
-		ret = idr_get_new_above(&cm.local_id_table, cm_id_priv, next_id++,
-					(__force int *) &cm_id_priv->id.local_id);
+		ret = idr_get_new_above(&cm.local_id_table, cm_id_priv,
+					next_id++, &id);
 		spin_unlock_irqrestore(&cm.lock, flags);
 	} while( (ret == -EAGAIN) && idr_pre_get(&cm.local_id_table, GFP_KERNEL) );
+
+	cm_id_priv->id.local_id = (__force __be32) (id ^ cm.random_id_operand);
 	return ret;
 }
 
@@ -316,7 +320,8 @@ static void cm_free_id(__be32 local_id)
 	unsigned long flags;
 
 	spin_lock_irqsave(&cm.lock, flags);
-	idr_remove(&cm.local_id_table, (__force int) local_id);
+	idr_remove(&cm.local_id_table,
+		   (__force int) (local_id ^ cm.random_id_operand));
 	spin_unlock_irqrestore(&cm.lock, flags);
 }
 
@@ -324,7 +329,8 @@ static struct cm_id_private * cm_get_id(__be32 local_id, __be32 remote_id)
 {
 	struct cm_id_private *cm_id_priv;
 
-	cm_id_priv = idr_find(&cm.local_id_table, (__force int) local_id);
+	cm_id_priv = idr_find(&cm.local_id_table,
+			      (__force int) (local_id ^ cm.random_id_operand));
 	if (cm_id_priv) {
 		if (cm_id_priv->id.remote_id == remote_id)
 			atomic_inc(&cm_id_priv->refcount);
@@ -679,6 +685,8 @@ static void cm_enter_timewait(struct cm_id_private *cm_id_priv)
 {
 	int wait_time;
 
+	cm_cleanup_timewait(cm_id_priv->timewait_info);
+
 	/*
 	 * The cm_id could be destroyed by the user before we exit timewait.
 	 * To protect against this, we search for the cm_id after exiting
@@ -1354,7 +1362,7 @@ static int cm_req_handler(struct cm_work *work)
 							    id.local_id);
 	if (IS_ERR(cm_id_priv->timewait_info)) {
 		ret = PTR_ERR(cm_id_priv->timewait_info);
-		goto error1;
+		goto destroy;
 	}
 	cm_id_priv->timewait_info->work.remote_id = req_msg->local_comm_id;
 	cm_id_priv->timewait_info->remote_ca_guid = req_msg->local_ca_guid;
@@ -1363,7 +1371,8 @@ static int cm_req_handler(struct cm_work *work)
 	listen_cm_id_priv = cm_match_req(work, cm_id_priv);
 	if (!listen_cm_id_priv) {
 		ret = -EINVAL;
-		goto error2;
+		kfree(cm_id_priv->timewait_info);
+		goto destroy;
 	}
 
 	cm_id_priv->id.cm_handler = listen_cm_id_priv->id.cm_handler;
@@ -1373,12 +1382,22 @@ static int cm_req_handler(struct cm_work *work)
 
 	cm_format_paths_from_req(req_msg, &work->path[0], &work->path[1]);
 	ret = cm_init_av_by_path(&work->path[0], &cm_id_priv->av);
-	if (ret)
-		goto error3;
+	if (ret) {
+		ib_get_cached_gid(work->port->cm_dev->device,
+				  work->port->port_num, 0, &work->path[0].sgid);
+		ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_GID,
+			       &work->path[0].sgid, sizeof work->path[0].sgid,
+			       NULL, 0);
+		goto rejected;
+	}
 	if (req_msg->alt_local_lid) {
 		ret = cm_init_av_by_path(&work->path[1], &cm_id_priv->alt_av);
-		if (ret)
-			goto error3;
+		if (ret) {
+			ib_send_cm_rej(cm_id, IB_CM_REJ_INVALID_ALT_GID,
+				       &work->path[0].sgid,
+				       sizeof work->path[0].sgid, NULL, 0);
+			goto rejected;
+		}
 	}
 	cm_id_priv->tid = req_msg->hdr.tid;
 	cm_id_priv->timeout_ms = cm_convert_to_ms(
@@ -1400,12 +1419,11 @@ static int cm_req_handler(struct cm_work *work)
 	cm_deref_id(listen_cm_id_priv);
 	return 0;
 
-error3:	atomic_dec(&cm_id_priv->refcount);
+rejected:
+	atomic_dec(&cm_id_priv->refcount);
 	cm_deref_id(listen_cm_id_priv);
-	cm_cleanup_timewait(cm_id_priv->timewait_info);
-error2:	kfree(cm_id_priv->timewait_info);
-	cm_id_priv->timewait_info = NULL;
-error1:	ib_destroy_cm_id(&cm_id_priv->id);
+destroy:
+	ib_destroy_cm_id(cm_id);
 	return ret;
 }
 
@@ -2072,8 +2090,9 @@ static struct cm_id_private * cm_acquire_rejected_id(struct cm_rej_msg *rej_msg)
 			spin_unlock_irqrestore(&cm.lock, flags);
 			return NULL;
 		}
-		cm_id_priv = idr_find(&cm.local_id_table,
-				      (__force int) timewait_info->work.local_id);
+		cm_id_priv = idr_find(&cm.local_id_table, (__force int)
+				      (timewait_info->work.local_id ^
+				       cm.random_id_operand));
 		if (cm_id_priv) {
 			if (cm_id_priv->id.remote_id == remote_id)
 				atomic_inc(&cm_id_priv->refcount);
@@ -3125,7 +3144,8 @@ static int cm_init_qp_init_attr(struct cm_id_private *cm_id_priv,
 		qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
 					   IB_ACCESS_REMOTE_WRITE;
 		if (cm_id_priv->responder_resources)
-			qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ;
+			qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ |
+						    IB_ACCESS_REMOTE_ATOMIC;
 		qp_attr->pkey_index = cm_id_priv->av.pkey_index;
 		qp_attr->port_num = cm_id_priv->av.port->port_num;
 		ret = 0;
@@ -3262,6 +3282,9 @@ static void cm_add_one(struct ib_device *device)
 	int ret;
 	u8 i;
 
+	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+		return;
+
 	cm_dev = kmalloc(sizeof(*cm_dev) + sizeof(*port) *
 			 device->phys_port_cnt, GFP_KERNEL);
 	if (!cm_dev)
@@ -3349,6 +3372,7 @@ static int __init ib_cm_init(void)
 	cm.remote_qp_table = RB_ROOT;
 	cm.remote_sidr_table = RB_ROOT;
 	idr_init(&cm.local_id_table);
+	get_random_bytes(&cm.random_id_operand, sizeof cm.random_id_operand);
 	idr_pre_get(&cm.local_id_table, GFP_KERNEL);
 
 	cm.wq = create_workqueue("ib_cm");
diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 5d625a8..1178bd4 100644
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -35,6 +35,7 @@
 #include <linux/mutex.h>
 #include <linux/random.h>
 #include <linux/idr.h>
+#include <linux/inetdevice.h>
 
 #include <net/tcp.h>
 
@@ -43,6 +44,7 @@
 #include <rdma/ib_cache.h>
 #include <rdma/ib_cm.h>
 #include <rdma/ib_sa.h>
+#include <rdma/iw_cm.h>
 
 MODULE_AUTHOR("Sean Hefty");
 MODULE_DESCRIPTION("Generic RDMA CM Agent");
@@ -60,6 +62,7 @@ static struct ib_client cma_client = {
 	.remove = cma_remove_one
 };
 
+static struct ib_sa_client sa_client;
 static LIST_HEAD(dev_list);
 static LIST_HEAD(listen_any_list);
 static DEFINE_MUTEX(lock);
@@ -124,6 +127,7 @@ struct rdma_id_private {
 	int			query_id;
 	union {
 		struct ib_cm_id	*ib;
+		struct iw_cm_id	*iw;
 	} cm_id;
 
 	u32			seq_num;
@@ -259,15 +263,24 @@ static void cma_detach_from_dev(struct rdma_id_private *id_priv)
 	id_priv->cma_dev = NULL;
 }
 
-static int cma_acquire_ib_dev(struct rdma_id_private *id_priv)
+static int cma_acquire_dev(struct rdma_id_private *id_priv)
 {
+	enum rdma_node_type dev_type = id_priv->id.route.addr.dev_addr.dev_type;
 	struct cma_device *cma_dev;
 	union ib_gid gid;
 	int ret = -ENODEV;
 
-	ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid),
+	switch (rdma_node_get_transport(dev_type)) {
+	case RDMA_TRANSPORT_IB:
+		ib_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
+		break;
+	case RDMA_TRANSPORT_IWARP:
+		iw_addr_get_sgid(&id_priv->id.route.addr.dev_addr, &gid);
+		break;
+	default:
+		return -ENODEV;
+	}
 
-	mutex_lock(&lock);
 	list_for_each_entry(cma_dev, &dev_list, list) {
 		ret = ib_find_cached_gid(cma_dev->device, &gid,
 					 &id_priv->id.port_num, NULL);
@@ -276,20 +289,9 @@ static int cma_acquire_ib_dev(struct rdma_id_private *id_priv)
 			break;
 		}
 	}
-	mutex_unlock(&lock);
 	return ret;
 }
 
-static int cma_acquire_dev(struct rdma_id_private *id_priv)
-{
-	switch (id_priv->id.route.addr.dev_addr.dev_type) {
-	case IB_NODE_CA:
-		return cma_acquire_ib_dev(id_priv);
-	default:
-		return -ENODEV;
-	}
-}
-
 static void cma_deref_id(struct rdma_id_private *id_priv)
 {
 	if (atomic_dec_and_test(&id_priv->refcount))
@@ -347,6 +349,16 @@ static int cma_init_ib_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
 					  IB_QP_PKEY_INDEX | IB_QP_PORT);
 }
 
+static int cma_init_iw_qp(struct rdma_id_private *id_priv, struct ib_qp *qp)
+{
+	struct ib_qp_attr qp_attr;
+
+	qp_attr.qp_state = IB_QPS_INIT;
+	qp_attr.qp_access_flags = IB_ACCESS_LOCAL_WRITE;
+
+	return ib_modify_qp(qp, &qp_attr, IB_QP_STATE | IB_QP_ACCESS_FLAGS);
+}
+
 int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 		   struct ib_qp_init_attr *qp_init_attr)
 {
@@ -362,10 +374,13 @@ int rdma_create_qp(struct rdma_cm_id *id, struct ib_pd *pd,
 	if (IS_ERR(qp))
 		return PTR_ERR(qp);
 
-	switch (id->device->node_type) {
-	case IB_NODE_CA:
+	switch (rdma_node_get_transport(id->device->node_type)) {
+	case RDMA_TRANSPORT_IB:
 		ret = cma_init_ib_qp(id_priv, qp);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = cma_init_iw_qp(id_priv, qp);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -451,13 +466,17 @@ int rdma_init_qp_attr(struct rdma_cm_id *id, struct ib_qp_attr *qp_attr,
 	int ret;
 
 	id_priv = container_of(id, struct rdma_id_private, id);
-	switch (id_priv->id.device->node_type) {
-	case IB_NODE_CA:
+	switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
+	case RDMA_TRANSPORT_IB:
 		ret = ib_cm_init_qp_attr(id_priv->cm_id.ib, qp_attr,
 					 qp_attr_mask);
 		if (qp_attr->qp_state == IB_QPS_RTR)
 			qp_attr->rq_psn = id_priv->seq_num;
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = iw_cm_init_qp_attr(id_priv->cm_id.iw, qp_attr,
+					qp_attr_mask);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -590,8 +609,8 @@ static int cma_notify_user(struct rdma_id_private *id_priv,
 
 static void cma_cancel_route(struct rdma_id_private *id_priv)
 {
-	switch (id_priv->id.device->node_type) {
-	case IB_NODE_CA:
+	switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
+	case RDMA_TRANSPORT_IB:
 		if (id_priv->query)
 			ib_sa_cancel_query(id_priv->query_id, id_priv->query);
 		break;
@@ -611,11 +630,15 @@ static void cma_destroy_listen(struct rdma_id_private *id_priv)
 	cma_exch(id_priv, CMA_DESTROYING);
 
 	if (id_priv->cma_dev) {
-		switch (id_priv->id.device->node_type) {
-		case IB_NODE_CA:
-	 		if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
+		switch (rdma_node_get_transport(id_priv->id.device->node_type)) {
+		case RDMA_TRANSPORT_IB:
+			if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
 				ib_destroy_cm_id(id_priv->cm_id.ib);
 			break;
+		case RDMA_TRANSPORT_IWARP:
+			if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
+				iw_destroy_cm_id(id_priv->cm_id.iw);
+			break;
 		default:
 			break;
 		}
@@ -689,19 +712,25 @@ void rdma_destroy_id(struct rdma_cm_id *id)
 	state = cma_exch(id_priv, CMA_DESTROYING);
 	cma_cancel_operation(id_priv, state);
 
+	mutex_lock(&lock);
 	if (id_priv->cma_dev) {
-		switch (id->device->node_type) {
-		case IB_NODE_CA:
-	 		if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
+		mutex_unlock(&lock);
+		switch (rdma_node_get_transport(id->device->node_type)) {
+		case RDMA_TRANSPORT_IB:
+			if (id_priv->cm_id.ib && !IS_ERR(id_priv->cm_id.ib))
 				ib_destroy_cm_id(id_priv->cm_id.ib);
 			break;
+		case RDMA_TRANSPORT_IWARP:
+			if (id_priv->cm_id.iw && !IS_ERR(id_priv->cm_id.iw))
+				iw_destroy_cm_id(id_priv->cm_id.iw);
+			break;
 		default:
 			break;
 		}
-	  	mutex_lock(&lock);
+		mutex_lock(&lock);
 		cma_detach_from_dev(id_priv);
-		mutex_unlock(&lock);
 	}
+	mutex_unlock(&lock);
 
 	cma_release_port(id_priv);
 	cma_deref_id(id_priv);
@@ -869,7 +898,7 @@ static struct rdma_id_private *cma_new_id(struct rdma_cm_id *listen_id,
 	ib_addr_set_sgid(&rt->addr.dev_addr, &rt->path_rec[0].sgid);
 	ib_addr_set_dgid(&rt->addr.dev_addr, &rt->path_rec[0].dgid);
 	ib_addr_set_pkey(&rt->addr.dev_addr, be16_to_cpu(rt->path_rec[0].pkey));
-	rt->addr.dev_addr.dev_type = IB_NODE_CA;
+	rt->addr.dev_addr.dev_type = RDMA_NODE_IB_CA;
 
 	id_priv = container_of(id, struct rdma_id_private, id);
 	id_priv->state = CMA_CONNECT;
@@ -898,7 +927,9 @@ static int cma_req_handler(struct ib_cm_id *cm_id, struct ib_cm_event *ib_event)
 	}
 
 	atomic_inc(&conn_id->dev_remove);
-	ret = cma_acquire_ib_dev(conn_id);
+	mutex_lock(&lock);
+	ret = cma_acquire_dev(conn_id);
+	mutex_unlock(&lock);
 	if (ret) {
 		ret = -ENODEV;
 		cma_release_remove(conn_id);
@@ -982,6 +1013,130 @@ static void cma_set_compare_data(enum rdma_port_space ps, struct sockaddr *addr,
 	}
 }
 
+static int cma_iw_handler(struct iw_cm_id *iw_id, struct iw_cm_event *iw_event)
+{
+	struct rdma_id_private *id_priv = iw_id->context;
+	enum rdma_cm_event_type event = 0;
+	struct sockaddr_in *sin;
+	int ret = 0;
+
+	atomic_inc(&id_priv->dev_remove);
+
+	switch (iw_event->event) {
+	case IW_CM_EVENT_CLOSE:
+		event = RDMA_CM_EVENT_DISCONNECTED;
+		break;
+	case IW_CM_EVENT_CONNECT_REPLY:
+		sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
+		*sin = iw_event->local_addr;
+		sin = (struct sockaddr_in *) &id_priv->id.route.addr.dst_addr;
+		*sin = iw_event->remote_addr;
+		if (iw_event->status)
+			event = RDMA_CM_EVENT_REJECTED;
+		else
+			event = RDMA_CM_EVENT_ESTABLISHED;
+		break;
+	case IW_CM_EVENT_ESTABLISHED:
+		event = RDMA_CM_EVENT_ESTABLISHED;
+		break;
+	default:
+		BUG_ON(1);
+	}
+
+	ret = cma_notify_user(id_priv, event, iw_event->status,
+			      iw_event->private_data,
+			      iw_event->private_data_len);
+	if (ret) {
+		/* Destroy the CM ID by returning a non-zero value. */
+		id_priv->cm_id.iw = NULL;
+		cma_exch(id_priv, CMA_DESTROYING);
+		cma_release_remove(id_priv);
+		rdma_destroy_id(&id_priv->id);
+		return ret;
+	}
+
+	cma_release_remove(id_priv);
+	return ret;
+}
+
+static int iw_conn_req_handler(struct iw_cm_id *cm_id,
+			       struct iw_cm_event *iw_event)
+{
+	struct rdma_cm_id *new_cm_id;
+	struct rdma_id_private *listen_id, *conn_id;
+	struct sockaddr_in *sin;
+	struct net_device *dev = NULL;
+	int ret;
+
+	listen_id = cm_id->context;
+	atomic_inc(&listen_id->dev_remove);
+	if (!cma_comp(listen_id, CMA_LISTEN)) {
+		ret = -ECONNABORTED;
+		goto out;
+	}
+
+	/* Create a new RDMA id for the new IW CM ID */
+	new_cm_id = rdma_create_id(listen_id->id.event_handler,
+				   listen_id->id.context,
+				   RDMA_PS_TCP);
+	if (!new_cm_id) {
+		ret = -ENOMEM;
+		goto out;
+	}
+	conn_id = container_of(new_cm_id, struct rdma_id_private, id);
+	atomic_inc(&conn_id->dev_remove);
+	conn_id->state = CMA_CONNECT;
+
+	dev = ip_dev_find(iw_event->local_addr.sin_addr.s_addr);
+	if (!dev) {
+		ret = -EADDRNOTAVAIL;
+		cma_release_remove(conn_id);
+		rdma_destroy_id(new_cm_id);
+		goto out;
+	}
+	ret = rdma_copy_addr(&conn_id->id.route.addr.dev_addr, dev, NULL);
+	if (ret) {
+		cma_release_remove(conn_id);
+		rdma_destroy_id(new_cm_id);
+		goto out;
+	}
+
+	mutex_lock(&lock);
+	ret = cma_acquire_dev(conn_id);
+	mutex_unlock(&lock);
+	if (ret) {
+		cma_release_remove(conn_id);
+		rdma_destroy_id(new_cm_id);
+		goto out;
+	}
+
+	conn_id->cm_id.iw = cm_id;
+	cm_id->context = conn_id;
+	cm_id->cm_handler = cma_iw_handler;
+
+	sin = (struct sockaddr_in *) &new_cm_id->route.addr.src_addr;
+	*sin = iw_event->local_addr;
+	sin = (struct sockaddr_in *) &new_cm_id->route.addr.dst_addr;
+	*sin = iw_event->remote_addr;
+
+	ret = cma_notify_user(conn_id, RDMA_CM_EVENT_CONNECT_REQUEST, 0,
+			      iw_event->private_data,
+			      iw_event->private_data_len);
+	if (ret) {
+		/* User wants to destroy the CM ID */
+		conn_id->cm_id.iw = NULL;
+		cma_exch(conn_id, CMA_DESTROYING);
+		cma_release_remove(conn_id);
+		rdma_destroy_id(&conn_id->id);
+	}
+
+out:
+	if (dev)
+		dev_put(dev);
+	cma_release_remove(listen_id);
+	return ret;
+}
+
 static int cma_ib_listen(struct rdma_id_private *id_priv)
 {
 	struct ib_cm_compare_data compare_data;
@@ -1011,6 +1166,30 @@ static int cma_ib_listen(struct rdma_id_private *id_priv)
 	return ret;
 }
 
+static int cma_iw_listen(struct rdma_id_private *id_priv, int backlog)
+{
+	int ret;
+	struct sockaddr_in *sin;
+
+	id_priv->cm_id.iw = iw_create_cm_id(id_priv->id.device,
+					    iw_conn_req_handler,
+					    id_priv);
+	if (IS_ERR(id_priv->cm_id.iw))
+		return PTR_ERR(id_priv->cm_id.iw);
+
+	sin = (struct sockaddr_in *) &id_priv->id.route.addr.src_addr;
+	id_priv->cm_id.iw->local_addr = *sin;
+
+	ret = iw_cm_listen(id_priv->cm_id.iw, backlog);
+
+	if (ret) {
+		iw_destroy_cm_id(id_priv->cm_id.iw);
+		id_priv->cm_id.iw = NULL;
+	}
+
+	return ret;
+}
+
 static int cma_listen_handler(struct rdma_cm_id *id,
 			      struct rdma_cm_event *event)
 {
@@ -1087,12 +1266,17 @@ int rdma_listen(struct rdma_cm_id *id, int backlog)
 
 	id_priv->backlog = backlog;
 	if (id->device) {
-		switch (id->device->node_type) {
-		case IB_NODE_CA:
+		switch (rdma_node_get_transport(id->device->node_type)) {
+		case RDMA_TRANSPORT_IB:
 			ret = cma_ib_listen(id_priv);
 			if (ret)
 				goto err;
 			break;
+		case RDMA_TRANSPORT_IWARP:
+			ret = cma_iw_listen(id_priv, backlog);
+			if (ret)
+				goto err;
+			break;
 		default:
 			ret = -ENOSYS;
 			goto err;
@@ -1140,7 +1324,7 @@ static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
 	path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(addr));
 	path_rec.numb_path = 1;
 
-	id_priv->query_id = ib_sa_path_rec_get(id_priv->id.device,
+	id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
 				id_priv->id.port_num, &path_rec,
 				IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
 				IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH,
@@ -1231,6 +1415,23 @@ err:
 }
 EXPORT_SYMBOL(rdma_set_ib_paths);
 
+static int cma_resolve_iw_route(struct rdma_id_private *id_priv, int timeout_ms)
+{
+	struct cma_work *work;
+
+	work = kzalloc(sizeof *work, GFP_KERNEL);
+	if (!work)
+		return -ENOMEM;
+
+	work->id = id_priv;
+	INIT_WORK(&work->work, cma_work_handler, work);
+	work->old_state = CMA_ROUTE_QUERY;
+	work->new_state = CMA_ROUTE_RESOLVED;
+	work->event.event = RDMA_CM_EVENT_ROUTE_RESOLVED;
+	queue_work(cma_wq, &work->work);
+	return 0;
+}
+
 int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
 {
 	struct rdma_id_private *id_priv;
@@ -1241,10 +1442,13 @@ int rdma_resolve_route(struct rdma_cm_id *id, int timeout_ms)
 		return -EINVAL;
 
 	atomic_inc(&id_priv->refcount);
-	switch (id->device->node_type) {
-	case IB_NODE_CA:
+	switch (rdma_node_get_transport(id->device->node_type)) {
+	case RDMA_TRANSPORT_IB:
 		ret = cma_resolve_ib_route(id_priv, timeout_ms);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = cma_resolve_iw_route(id_priv, timeout_ms);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -1309,16 +1513,26 @@ static void addr_handler(int status, struct sockaddr *src_addr,
 	enum rdma_cm_event_type event;
 
 	atomic_inc(&id_priv->dev_remove);
-	if (!id_priv->cma_dev && !status)
+
+	/*
+	 * Grab mutex to block rdma_destroy_id() from removing the device while
+	 * we're trying to acquire it.
+	 */
+	mutex_lock(&lock);
+	if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED)) {
+		mutex_unlock(&lock);
+		goto out;
+	}
+
+	if (!status && !id_priv->cma_dev)
 		status = cma_acquire_dev(id_priv);
+	mutex_unlock(&lock);
 
 	if (status) {
-		if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_BOUND))
+		if (!cma_comp_exch(id_priv, CMA_ADDR_RESOLVED, CMA_ADDR_BOUND))
 			goto out;
 		event = RDMA_CM_EVENT_ADDR_ERROR;
 	} else {
-		if (!cma_comp_exch(id_priv, CMA_ADDR_QUERY, CMA_ADDR_RESOLVED))
-			goto out;
 		memcpy(&id_priv->id.route.addr.src_addr, src_addr,
 		       ip_addr_size(src_addr));
 		event = RDMA_CM_EVENT_ADDR_RESOLVED;
@@ -1492,7 +1706,7 @@ static int cma_use_port(struct idr *ps, struct rdma_id_private *id_priv)
 	hlist_for_each_entry(cur_id, node, &bind_list->owners, node) {
 		if (cma_any_addr(&cur_id->id.route.addr.src_addr))
 			return -EADDRNOTAVAIL;
-		
+
 		cur_sin = (struct sockaddr_in *) &cur_id->id.route.addr.src_addr;
 		if (sin->sin_addr.s_addr == cur_sin->sin_addr.s_addr)
 			return -EADDRINUSE;
@@ -1542,8 +1756,11 @@ int rdma_bind_addr(struct rdma_cm_id *id, struct sockaddr *addr)
 
 	if (!cma_any_addr(addr)) {
 		ret = rdma_translate_ip(addr, &id->route.addr.dev_addr);
-		if (!ret)
+		if (!ret) {
+			mutex_lock(&lock);
 			ret = cma_acquire_dev(id_priv);
+			mutex_unlock(&lock);
+		}
 		if (ret)
 			goto err;
 	}
@@ -1649,6 +1866,47 @@ out:
 	return ret;
 }
 
+static int cma_connect_iw(struct rdma_id_private *id_priv,
+			  struct rdma_conn_param *conn_param)
+{
+	struct iw_cm_id *cm_id;
+	struct sockaddr_in* sin;
+	int ret;
+	struct iw_cm_conn_param iw_param;
+
+	cm_id = iw_create_cm_id(id_priv->id.device, cma_iw_handler, id_priv);
+	if (IS_ERR(cm_id)) {
+		ret = PTR_ERR(cm_id);
+		goto out;
+	}
+
+	id_priv->cm_id.iw = cm_id;
+
+	sin = (struct sockaddr_in*) &id_priv->id.route.addr.src_addr;
+	cm_id->local_addr = *sin;
+
+	sin = (struct sockaddr_in*) &id_priv->id.route.addr.dst_addr;
+	cm_id->remote_addr = *sin;
+
+	ret = cma_modify_qp_rtr(&id_priv->id);
+	if (ret) {
+		iw_destroy_cm_id(cm_id);
+		return ret;
+	}
+
+	iw_param.ord = conn_param->initiator_depth;
+	iw_param.ird = conn_param->responder_resources;
+	iw_param.private_data = conn_param->private_data;
+	iw_param.private_data_len = conn_param->private_data_len;
+	if (id_priv->id.qp)
+		iw_param.qpn = id_priv->qp_num;
+	else
+		iw_param.qpn = conn_param->qp_num;
+	ret = iw_cm_connect(cm_id, &iw_param);
+out:
+	return ret;
+}
+
 int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
 {
 	struct rdma_id_private *id_priv;
@@ -1664,10 +1922,13 @@ int rdma_connect(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
 		id_priv->srq = conn_param->srq;
 	}
 
-	switch (id->device->node_type) {
-	case IB_NODE_CA:
+	switch (rdma_node_get_transport(id->device->node_type)) {
+	case RDMA_TRANSPORT_IB:
 		ret = cma_connect_ib(id_priv, conn_param);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = cma_connect_iw(id_priv, conn_param);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -1708,6 +1969,28 @@ static int cma_accept_ib(struct rdma_id_private *id_priv,
 	return ib_send_cm_rep(id_priv->cm_id.ib, &rep);
 }
 
+static int cma_accept_iw(struct rdma_id_private *id_priv,
+		  struct rdma_conn_param *conn_param)
+{
+	struct iw_cm_conn_param iw_param;
+	int ret;
+
+	ret = cma_modify_qp_rtr(&id_priv->id);
+	if (ret)
+		return ret;
+
+	iw_param.ord = conn_param->initiator_depth;
+	iw_param.ird = conn_param->responder_resources;
+	iw_param.private_data = conn_param->private_data;
+	iw_param.private_data_len = conn_param->private_data_len;
+	if (id_priv->id.qp) {
+		iw_param.qpn = id_priv->qp_num;
+	} else
+		iw_param.qpn = conn_param->qp_num;
+
+	return iw_cm_accept(id_priv->cm_id.iw, &iw_param);
+}
+
 int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
 {
 	struct rdma_id_private *id_priv;
@@ -1723,13 +2006,16 @@ int rdma_accept(struct rdma_cm_id *id, struct rdma_conn_param *conn_param)
 		id_priv->srq = conn_param->srq;
 	}
 
-	switch (id->device->node_type) {
-	case IB_NODE_CA:
+	switch (rdma_node_get_transport(id->device->node_type)) {
+	case RDMA_TRANSPORT_IB:
 		if (conn_param)
 			ret = cma_accept_ib(id_priv, conn_param);
 		else
 			ret = cma_rep_recv(id_priv);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = cma_accept_iw(id_priv, conn_param);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -1756,12 +2042,16 @@ int rdma_reject(struct rdma_cm_id *id, const void *private_data,
 	if (!cma_comp(id_priv, CMA_CONNECT))
 		return -EINVAL;
 
-	switch (id->device->node_type) {
-	case IB_NODE_CA:
+	switch (rdma_node_get_transport(id->device->node_type)) {
+	case RDMA_TRANSPORT_IB:
 		ret = ib_send_cm_rej(id_priv->cm_id.ib,
 				     IB_CM_REJ_CONSUMER_DEFINED, NULL, 0,
 				     private_data, private_data_len);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = iw_cm_reject(id_priv->cm_id.iw,
+				   private_data, private_data_len);
+		break;
 	default:
 		ret = -ENOSYS;
 		break;
@@ -1780,17 +2070,20 @@ int rdma_disconnect(struct rdma_cm_id *id)
 	    !cma_comp(id_priv, CMA_DISCONNECT))
 		return -EINVAL;
 
-	ret = cma_modify_qp_err(id);
-	if (ret)
-		goto out;
-
-	switch (id->device->node_type) {
-	case IB_NODE_CA:
+	switch (rdma_node_get_transport(id->device->node_type)) {
+	case RDMA_TRANSPORT_IB:
+		ret = cma_modify_qp_err(id);
+		if (ret)
+			goto out;
 		/* Initiate or respond to a disconnect. */
 		if (ib_send_cm_dreq(id_priv->cm_id.ib, NULL, 0))
 			ib_send_cm_drep(id_priv->cm_id.ib, NULL, 0);
 		break;
+	case RDMA_TRANSPORT_IWARP:
+		ret = iw_cm_disconnect(id_priv->cm_id.iw, 0);
+		break;
 	default:
+		ret = -EINVAL;
 		break;
 	}
 out:
@@ -1907,12 +2200,15 @@ static int cma_init(void)
 	if (!cma_wq)
 		return -ENOMEM;
 
+	ib_sa_register_client(&sa_client);
+
 	ret = ib_register_client(&cma_client);
 	if (ret)
 		goto err;
 	return 0;
 
 err:
+	ib_sa_unregister_client(&sa_client);
 	destroy_workqueue(cma_wq);
 	return ret;
 }
@@ -1920,6 +2216,7 @@ err:
 static void cma_cleanup(void)
 {
 	ib_unregister_client(&cma_client);
+	ib_sa_unregister_client(&sa_client);
 	destroy_workqueue(cma_wq);
 	idr_destroy(&sdp_ps);
 	idr_destroy(&tcp_ps);
diff --git a/drivers/infiniband/core/device.c b/drivers/infiniband/core/device.c
index b2f3cb9..63d2a39 100644
--- a/drivers/infiniband/core/device.c
+++ b/drivers/infiniband/core/device.c
@@ -385,7 +385,7 @@ void *ib_get_client_data(struct ib_device *device, struct ib_client *client)
 EXPORT_SYMBOL(ib_get_client_data);
 
 /**
- * ib_set_client_data - Get IB client context
+ * ib_set_client_data - Set IB client context
  * @device:Device to set context for
  * @client:Client to set context for
  * @data:Context to set
@@ -505,7 +505,7 @@ int ib_query_port(struct ib_device *device,
 		  u8 port_num,
 		  struct ib_port_attr *port_attr)
 {
-	if (device->node_type == IB_NODE_SWITCH) {
+	if (device->node_type == RDMA_NODE_IB_SWITCH) {
 		if (port_num)
 			return -EINVAL;
 	} else if (port_num < 1 || port_num > device->phys_port_cnt)
@@ -580,7 +580,7 @@ int ib_modify_port(struct ib_device *device,
 		   u8 port_num, int port_modify_mask,
 		   struct ib_port_modify *port_modify)
 {
-	if (device->node_type == IB_NODE_SWITCH) {
+	if (device->node_type == RDMA_NODE_IB_SWITCH) {
 		if (port_num)
 			return -EINVAL;
 	} else if (port_num < 1 || port_num > device->phys_port_cnt)
diff --git a/drivers/infiniband/core/iwcm.c b/drivers/infiniband/core/iwcm.c
new file mode 100644
index 0000000..c3fb304
--- /dev/null
+++ b/drivers/infiniband/core/iwcm.c
@@ -0,0 +1,1019 @@
+/*
+ * Copyright (c) 2004, 2005 Intel Corporation.  All rights reserved.
+ * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
+ * Copyright (c) 2004, 2005 Voltaire Corporation.  All rights reserved.
+ * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ * Copyright (c) 2005 Network Appliance, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+#include <linux/dma-mapping.h>
+#include <linux/err.h>
+#include <linux/idr.h>
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/rbtree.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
+#include <linux/completion.h>
+
+#include <rdma/iw_cm.h>
+#include <rdma/ib_addr.h>
+
+#include "iwcm.h"
+
+MODULE_AUTHOR("Tom Tucker");
+MODULE_DESCRIPTION("iWARP CM");
+MODULE_LICENSE("Dual BSD/GPL");
+
+static struct workqueue_struct *iwcm_wq;
+struct iwcm_work {
+	struct work_struct work;
+	struct iwcm_id_private *cm_id;
+	struct list_head list;
+	struct iw_cm_event event;
+	struct list_head free_list;
+};
+
+/*
+ * The following services provide a mechanism for pre-allocating iwcm_work
+ * elements.  The design pre-allocates them  based on the cm_id type:
+ *	LISTENING IDS: 	Get enough elements preallocated to handle the
+ *			listen backlog.
+ *	ACTIVE IDS:	4: CONNECT_REPLY, ESTABLISHED, DISCONNECT, CLOSE
+ *	PASSIVE IDS:	3: ESTABLISHED, DISCONNECT, CLOSE
+ *
+ * Allocating them in connect and listen avoids having to deal
+ * with allocation failures on the event upcall from the provider (which
+ * is called in the interrupt context).
+ *
+ * One exception is when creating the cm_id for incoming connection requests.
+ * There are two cases:
+ * 1) in the event upcall, cm_event_handler(), for a listening cm_id.  If
+ *    the backlog is exceeded, then no more connection request events will
+ *    be processed.  cm_event_handler() returns -ENOMEM in this case.  Its up
+ *    to the provider to reject the connectino request.
+ * 2) in the connection request workqueue handler, cm_conn_req_handler().
+ *    If work elements cannot be allocated for the new connect request cm_id,
+ *    then IWCM will call the provider reject method.  This is ok since
+ *    cm_conn_req_handler() runs in the workqueue thread context.
+ */
+
+static struct iwcm_work *get_work(struct iwcm_id_private *cm_id_priv)
+{
+	struct iwcm_work *work;
+
+	if (list_empty(&cm_id_priv->work_free_list))
+		return NULL;
+	work = list_entry(cm_id_priv->work_free_list.next, struct iwcm_work,
+			  free_list);
+	list_del_init(&work->free_list);
+	return work;
+}
+
+static void put_work(struct iwcm_work *work)
+{
+	list_add(&work->free_list, &work->cm_id->work_free_list);
+}
+
+static void dealloc_work_entries(struct iwcm_id_private *cm_id_priv)
+{
+	struct list_head *e, *tmp;
+
+	list_for_each_safe(e, tmp, &cm_id_priv->work_free_list)
+		kfree(list_entry(e, struct iwcm_work, free_list));
+}
+
+static int alloc_work_entries(struct iwcm_id_private *cm_id_priv, int count)
+{
+	struct iwcm_work *work;
+
+	BUG_ON(!list_empty(&cm_id_priv->work_free_list));
+	while (count--) {
+		work = kmalloc(sizeof(struct iwcm_work), GFP_KERNEL);
+		if (!work) {
+			dealloc_work_entries(cm_id_priv);
+			return -ENOMEM;
+		}
+		work->cm_id = cm_id_priv;
+		INIT_LIST_HEAD(&work->list);
+		put_work(work);
+	}
+	return 0;
+}
+
+/*
+ * Save private data from incoming connection requests in the
+ * cm_id_priv so the low level driver doesn't have to.  Adjust
+ * the event ptr to point to the local copy.
+ */
+static int copy_private_data(struct iwcm_id_private *cm_id_priv,
+		       struct iw_cm_event *event)
+{
+	void *p;
+
+	p = kmalloc(event->private_data_len, GFP_ATOMIC);
+	if (!p)
+		return -ENOMEM;
+	memcpy(p, event->private_data, event->private_data_len);
+	event->private_data = p;
+	return 0;
+}
+
+/*
+ * Release a reference on cm_id. If the last reference is being removed
+ * and iw_destroy_cm_id is waiting, wake up the waiting thread.
+ */
+static int iwcm_deref_id(struct iwcm_id_private *cm_id_priv)
+{
+	int ret = 0;
+
+	BUG_ON(atomic_read(&cm_id_priv->refcount)==0);
+	if (atomic_dec_and_test(&cm_id_priv->refcount)) {
+		BUG_ON(!list_empty(&cm_id_priv->work_list));
+		if (waitqueue_active(&cm_id_priv->destroy_comp.wait)) {
+			BUG_ON(cm_id_priv->state != IW_CM_STATE_DESTROYING);
+			BUG_ON(test_bit(IWCM_F_CALLBACK_DESTROY,
+					&cm_id_priv->flags));
+			ret = 1;
+		}
+		complete(&cm_id_priv->destroy_comp);
+	}
+
+	return ret;
+}
+
+static void add_ref(struct iw_cm_id *cm_id)
+{
+	struct iwcm_id_private *cm_id_priv;
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+	atomic_inc(&cm_id_priv->refcount);
+}
+
+static void rem_ref(struct iw_cm_id *cm_id)
+{
+	struct iwcm_id_private *cm_id_priv;
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+	iwcm_deref_id(cm_id_priv);
+}
+
+static int cm_event_handler(struct iw_cm_id *cm_id, struct iw_cm_event *event);
+
+struct iw_cm_id *iw_create_cm_id(struct ib_device *device,
+				 iw_cm_handler cm_handler,
+				 void *context)
+{
+	struct iwcm_id_private *cm_id_priv;
+
+	cm_id_priv = kzalloc(sizeof(*cm_id_priv), GFP_KERNEL);
+	if (!cm_id_priv)
+		return ERR_PTR(-ENOMEM);
+
+	cm_id_priv->state = IW_CM_STATE_IDLE;
+	cm_id_priv->id.device = device;
+	cm_id_priv->id.cm_handler = cm_handler;
+	cm_id_priv->id.context = context;
+	cm_id_priv->id.event_handler = cm_event_handler;
+	cm_id_priv->id.add_ref = add_ref;
+	cm_id_priv->id.rem_ref = rem_ref;
+	spin_lock_init(&cm_id_priv->lock);
+	atomic_set(&cm_id_priv->refcount, 1);
+	init_waitqueue_head(&cm_id_priv->connect_wait);
+	init_completion(&cm_id_priv->destroy_comp);
+	INIT_LIST_HEAD(&cm_id_priv->work_list);
+	INIT_LIST_HEAD(&cm_id_priv->work_free_list);
+
+	return &cm_id_priv->id;
+}
+EXPORT_SYMBOL(iw_create_cm_id);
+
+
+static int iwcm_modify_qp_err(struct ib_qp *qp)
+{
+	struct ib_qp_attr qp_attr;
+
+	if (!qp)
+		return -EINVAL;
+
+	qp_attr.qp_state = IB_QPS_ERR;
+	return ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
+}
+
+/*
+ * This is really the RDMAC CLOSING state. It is most similar to the
+ * IB SQD QP state.
+ */
+static int iwcm_modify_qp_sqd(struct ib_qp *qp)
+{
+	struct ib_qp_attr qp_attr;
+
+	BUG_ON(qp == NULL);
+	qp_attr.qp_state = IB_QPS_SQD;
+	return ib_modify_qp(qp, &qp_attr, IB_QP_STATE);
+}
+
+/*
+ * CM_ID <-- CLOSING
+ *
+ * Block if a passive or active connection is currenlty being processed. Then
+ * process the event as follows:
+ * - If we are ESTABLISHED, move to CLOSING and modify the QP state
+ *   based on the abrupt flag
+ * - If the connection is already in the CLOSING or IDLE state, the peer is
+ *   disconnecting concurrently with us and we've already seen the
+ *   DISCONNECT event -- ignore the request and return 0
+ * - Disconnect on a listening endpoint returns -EINVAL
+ */
+int iw_cm_disconnect(struct iw_cm_id *cm_id, int abrupt)
+{
+	struct iwcm_id_private *cm_id_priv;
+	unsigned long flags;
+	int ret = 0;
+	struct ib_qp *qp = NULL;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+	/* Wait if we're currently in a connect or accept downcall */
+	wait_event(cm_id_priv->connect_wait,
+		   !test_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags));
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	switch (cm_id_priv->state) {
+	case IW_CM_STATE_ESTABLISHED:
+		cm_id_priv->state = IW_CM_STATE_CLOSING;
+
+		/* QP could be <nul> for user-mode client */
+		if (cm_id_priv->qp)
+			qp = cm_id_priv->qp;
+		else
+			ret = -EINVAL;
+		break;
+	case IW_CM_STATE_LISTEN:
+		ret = -EINVAL;
+		break;
+	case IW_CM_STATE_CLOSING:
+		/* remote peer closed first */
+	case IW_CM_STATE_IDLE:
+		/* accept or connect returned !0 */
+		break;
+	case IW_CM_STATE_CONN_RECV:
+		/*
+		 * App called disconnect before/without calling accept after
+		 * connect_request event delivered.
+		 */
+		break;
+	case IW_CM_STATE_CONN_SENT:
+		/* Can only get here if wait above fails */
+	default:
+		BUG();
+	}
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+
+	if (qp) {
+		if (abrupt)
+			ret = iwcm_modify_qp_err(qp);
+		else
+			ret = iwcm_modify_qp_sqd(qp);
+
+		/*
+		 * If both sides are disconnecting the QP could
+		 * already be in ERR or SQD states
+		 */
+		ret = 0;
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(iw_cm_disconnect);
+
+/*
+ * CM_ID <-- DESTROYING
+ *
+ * Clean up all resources associated with the connection and release
+ * the initial reference taken by iw_create_cm_id.
+ */
+static void destroy_cm_id(struct iw_cm_id *cm_id)
+{
+	struct iwcm_id_private *cm_id_priv;
+	unsigned long flags;
+	int ret;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+	/*
+	 * Wait if we're currently in a connect or accept downcall. A
+	 * listening endpoint should never block here.
+	 */
+	wait_event(cm_id_priv->connect_wait,
+		   !test_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags));
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	switch (cm_id_priv->state) {
+	case IW_CM_STATE_LISTEN:
+		cm_id_priv->state = IW_CM_STATE_DESTROYING;
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		/* destroy the listening endpoint */
+		ret = cm_id->device->iwcm->destroy_listen(cm_id);
+		spin_lock_irqsave(&cm_id_priv->lock, flags);
+		break;
+	case IW_CM_STATE_ESTABLISHED:
+		cm_id_priv->state = IW_CM_STATE_DESTROYING;
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		/* Abrupt close of the connection */
+		(void)iwcm_modify_qp_err(cm_id_priv->qp);
+		spin_lock_irqsave(&cm_id_priv->lock, flags);
+		break;
+	case IW_CM_STATE_IDLE:
+	case IW_CM_STATE_CLOSING:
+		cm_id_priv->state = IW_CM_STATE_DESTROYING;
+		break;
+	case IW_CM_STATE_CONN_RECV:
+		/*
+		 * App called destroy before/without calling accept after
+		 * receiving connection request event notification.
+		 */
+		cm_id_priv->state = IW_CM_STATE_DESTROYING;
+		break;
+	case IW_CM_STATE_CONN_SENT:
+	case IW_CM_STATE_DESTROYING:
+	default:
+		BUG();
+		break;
+	}
+	if (cm_id_priv->qp) {
+		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
+		cm_id_priv->qp = NULL;
+	}
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+
+	(void)iwcm_deref_id(cm_id_priv);
+}
+
+/*
+ * This function is only called by the application thread and cannot
+ * be called by the event thread. The function will wait for all
+ * references to be released on the cm_id and then kfree the cm_id
+ * object.
+ */
+void iw_destroy_cm_id(struct iw_cm_id *cm_id)
+{
+	struct iwcm_id_private *cm_id_priv;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+	BUG_ON(test_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags));
+
+	destroy_cm_id(cm_id);
+
+	wait_for_completion(&cm_id_priv->destroy_comp);
+
+	dealloc_work_entries(cm_id_priv);
+
+	kfree(cm_id_priv);
+}
+EXPORT_SYMBOL(iw_destroy_cm_id);
+
+/*
+ * CM_ID <-- LISTEN
+ *
+ * Start listening for connect requests. Generates one CONNECT_REQUEST
+ * event for each inbound connect request.
+ */
+int iw_cm_listen(struct iw_cm_id *cm_id, int backlog)
+{
+	struct iwcm_id_private *cm_id_priv;
+	unsigned long flags;
+	int ret = 0;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+
+	ret = alloc_work_entries(cm_id_priv, backlog);
+	if (ret)
+		return ret;
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	switch (cm_id_priv->state) {
+	case IW_CM_STATE_IDLE:
+		cm_id_priv->state = IW_CM_STATE_LISTEN;
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		ret = cm_id->device->iwcm->create_listen(cm_id, backlog);
+		if (ret)
+			cm_id_priv->state = IW_CM_STATE_IDLE;
+		spin_lock_irqsave(&cm_id_priv->lock, flags);
+		break;
+	default:
+		ret = -EINVAL;
+	}
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+
+	return ret;
+}
+EXPORT_SYMBOL(iw_cm_listen);
+
+/*
+ * CM_ID <-- IDLE
+ *
+ * Rejects an inbound connection request. No events are generated.
+ */
+int iw_cm_reject(struct iw_cm_id *cm_id,
+		 const void *private_data,
+		 u8 private_data_len)
+{
+	struct iwcm_id_private *cm_id_priv;
+	unsigned long flags;
+	int ret;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+	set_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	if (cm_id_priv->state != IW_CM_STATE_CONN_RECV) {
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+		wake_up_all(&cm_id_priv->connect_wait);
+		return -EINVAL;
+	}
+	cm_id_priv->state = IW_CM_STATE_IDLE;
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+
+	ret = cm_id->device->iwcm->reject(cm_id, private_data,
+					  private_data_len);
+
+	clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+	wake_up_all(&cm_id_priv->connect_wait);
+
+	return ret;
+}
+EXPORT_SYMBOL(iw_cm_reject);
+
+/*
+ * CM_ID <-- ESTABLISHED
+ *
+ * Accepts an inbound connection request and generates an ESTABLISHED
+ * event. Callers of iw_cm_disconnect and iw_destroy_cm_id will block
+ * until the ESTABLISHED event is received from the provider.
+ */
+int iw_cm_accept(struct iw_cm_id *cm_id,
+		 struct iw_cm_conn_param *iw_param)
+{
+	struct iwcm_id_private *cm_id_priv;
+	struct ib_qp *qp;
+	unsigned long flags;
+	int ret;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+	set_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	if (cm_id_priv->state != IW_CM_STATE_CONN_RECV) {
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+		wake_up_all(&cm_id_priv->connect_wait);
+		return -EINVAL;
+	}
+	/* Get the ib_qp given the QPN */
+	qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn);
+	if (!qp) {
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		return -EINVAL;
+	}
+	cm_id->device->iwcm->add_ref(qp);
+	cm_id_priv->qp = qp;
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+
+	ret = cm_id->device->iwcm->accept(cm_id, iw_param);
+	if (ret) {
+		/* An error on accept precludes provider events */
+		BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_RECV);
+		cm_id_priv->state = IW_CM_STATE_IDLE;
+		spin_lock_irqsave(&cm_id_priv->lock, flags);
+		if (cm_id_priv->qp) {
+			cm_id->device->iwcm->rem_ref(qp);
+			cm_id_priv->qp = NULL;
+		}
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+		wake_up_all(&cm_id_priv->connect_wait);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(iw_cm_accept);
+
+/*
+ * Active Side: CM_ID <-- CONN_SENT
+ *
+ * If successful, results in the generation of a CONNECT_REPLY
+ * event. iw_cm_disconnect and iw_cm_destroy will block until the
+ * CONNECT_REPLY event is received from the provider.
+ */
+int iw_cm_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
+{
+	struct iwcm_id_private *cm_id_priv;
+	int ret = 0;
+	unsigned long flags;
+	struct ib_qp *qp;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+
+	ret = alloc_work_entries(cm_id_priv, 4);
+	if (ret)
+		return ret;
+
+	set_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+
+	if (cm_id_priv->state != IW_CM_STATE_IDLE) {
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+		wake_up_all(&cm_id_priv->connect_wait);
+		return -EINVAL;
+	}
+
+	/* Get the ib_qp given the QPN */
+	qp = cm_id->device->iwcm->get_qp(cm_id->device, iw_param->qpn);
+	if (!qp) {
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		return -EINVAL;
+	}
+	cm_id->device->iwcm->add_ref(qp);
+	cm_id_priv->qp = qp;
+	cm_id_priv->state = IW_CM_STATE_CONN_SENT;
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+
+	ret = cm_id->device->iwcm->connect(cm_id, iw_param);
+	if (ret) {
+		spin_lock_irqsave(&cm_id_priv->lock, flags);
+		if (cm_id_priv->qp) {
+			cm_id->device->iwcm->rem_ref(qp);
+			cm_id_priv->qp = NULL;
+		}
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_SENT);
+		cm_id_priv->state = IW_CM_STATE_IDLE;
+		clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+		wake_up_all(&cm_id_priv->connect_wait);
+	}
+
+	return ret;
+}
+EXPORT_SYMBOL(iw_cm_connect);
+
+/*
+ * Passive Side: new CM_ID <-- CONN_RECV
+ *
+ * Handles an inbound connect request. The function creates a new
+ * iw_cm_id to represent the new connection and inherits the client
+ * callback function and other attributes from the listening parent.
+ *
+ * The work item contains a pointer to the listen_cm_id and the event. The
+ * listen_cm_id contains the client cm_handler, context and
+ * device. These are copied when the device is cloned. The event
+ * contains the new four tuple.
+ *
+ * An error on the child should not affect the parent, so this
+ * function does not return a value.
+ */
+static void cm_conn_req_handler(struct iwcm_id_private *listen_id_priv,
+				struct iw_cm_event *iw_event)
+{
+	unsigned long flags;
+	struct iw_cm_id *cm_id;
+	struct iwcm_id_private *cm_id_priv;
+	int ret;
+
+	/*
+	 * The provider should never generate a connection request
+	 * event with a bad status.
+	 */
+	BUG_ON(iw_event->status);
+
+	/*
+	 * We could be destroying the listening id. If so, ignore this
+	 * upcall.
+	 */
+	spin_lock_irqsave(&listen_id_priv->lock, flags);
+	if (listen_id_priv->state != IW_CM_STATE_LISTEN) {
+		spin_unlock_irqrestore(&listen_id_priv->lock, flags);
+		return;
+	}
+	spin_unlock_irqrestore(&listen_id_priv->lock, flags);
+
+	cm_id = iw_create_cm_id(listen_id_priv->id.device,
+				listen_id_priv->id.cm_handler,
+				listen_id_priv->id.context);
+	/* If the cm_id could not be created, ignore the request */
+	if (IS_ERR(cm_id))
+		return;
+
+	cm_id->provider_data = iw_event->provider_data;
+	cm_id->local_addr = iw_event->local_addr;
+	cm_id->remote_addr = iw_event->remote_addr;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+	cm_id_priv->state = IW_CM_STATE_CONN_RECV;
+
+	ret = alloc_work_entries(cm_id_priv, 3);
+	if (ret) {
+		iw_cm_reject(cm_id, NULL, 0);
+		iw_destroy_cm_id(cm_id);
+		return;
+	}
+
+	/* Call the client CM handler */
+	ret = cm_id->cm_handler(cm_id, iw_event);
+	if (ret) {
+		set_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags);
+		destroy_cm_id(cm_id);
+		if (atomic_read(&cm_id_priv->refcount)==0)
+			kfree(cm_id);
+	}
+
+	if (iw_event->private_data_len)
+		kfree(iw_event->private_data);
+}
+
+/*
+ * Passive Side: CM_ID <-- ESTABLISHED
+ *
+ * The provider generated an ESTABLISHED event which means that
+ * the MPA negotion has completed successfully and we are now in MPA
+ * FPDU mode.
+ *
+ * This event can only be received in the CONN_RECV state. If the
+ * remote peer closed, the ESTABLISHED event would be received followed
+ * by the CLOSE event. If the app closes, it will block until we wake
+ * it up after processing this event.
+ */
+static int cm_conn_est_handler(struct iwcm_id_private *cm_id_priv,
+			       struct iw_cm_event *iw_event)
+{
+	unsigned long flags;
+	int ret = 0;
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+
+	/*
+	 * We clear the CONNECT_WAIT bit here to allow the callback
+	 * function to call iw_cm_disconnect. Calling iw_destroy_cm_id
+	 * from a callback handler is not allowed.
+	 */
+	clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+	BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_RECV);
+	cm_id_priv->state = IW_CM_STATE_ESTABLISHED;
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, iw_event);
+	wake_up_all(&cm_id_priv->connect_wait);
+
+	return ret;
+}
+
+/*
+ * Active Side: CM_ID <-- ESTABLISHED
+ *
+ * The app has called connect and is waiting for the established event to
+ * post it's requests to the server. This event will wake up anyone
+ * blocked in iw_cm_disconnect or iw_destroy_id.
+ */
+static int cm_conn_rep_handler(struct iwcm_id_private *cm_id_priv,
+			       struct iw_cm_event *iw_event)
+{
+	unsigned long flags;
+	int ret = 0;
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	/*
+	 * Clear the connect wait bit so a callback function calling
+	 * iw_cm_disconnect will not wait and deadlock this thread
+	 */
+	clear_bit(IWCM_F_CONNECT_WAIT, &cm_id_priv->flags);
+	BUG_ON(cm_id_priv->state != IW_CM_STATE_CONN_SENT);
+	if (iw_event->status == IW_CM_EVENT_STATUS_ACCEPTED) {
+		cm_id_priv->id.local_addr = iw_event->local_addr;
+		cm_id_priv->id.remote_addr = iw_event->remote_addr;
+		cm_id_priv->state = IW_CM_STATE_ESTABLISHED;
+	} else {
+		/* REJECTED or RESET */
+		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
+		cm_id_priv->qp = NULL;
+		cm_id_priv->state = IW_CM_STATE_IDLE;
+	}
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+	ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, iw_event);
+
+	if (iw_event->private_data_len)
+		kfree(iw_event->private_data);
+
+	/* Wake up waiters on connect complete */
+	wake_up_all(&cm_id_priv->connect_wait);
+
+	return ret;
+}
+
+/*
+ * CM_ID <-- CLOSING
+ *
+ * If in the ESTABLISHED state, move to CLOSING.
+ */
+static void cm_disconnect_handler(struct iwcm_id_private *cm_id_priv,
+				  struct iw_cm_event *iw_event)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	if (cm_id_priv->state == IW_CM_STATE_ESTABLISHED)
+		cm_id_priv->state = IW_CM_STATE_CLOSING;
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+}
+
+/*
+ * CM_ID <-- IDLE
+ *
+ * If in the ESTBLISHED or CLOSING states, the QP will have have been
+ * moved by the provider to the ERR state. Disassociate the CM_ID from
+ * the QP,  move to IDLE, and remove the 'connected' reference.
+ *
+ * If in some other state, the cm_id was destroyed asynchronously.
+ * This is the last reference that will result in waking up
+ * the app thread blocked in iw_destroy_cm_id.
+ */
+static int cm_close_handler(struct iwcm_id_private *cm_id_priv,
+				  struct iw_cm_event *iw_event)
+{
+	unsigned long flags;
+	int ret = 0;
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+
+	if (cm_id_priv->qp) {
+		cm_id_priv->id.device->iwcm->rem_ref(cm_id_priv->qp);
+		cm_id_priv->qp = NULL;
+	}
+	switch (cm_id_priv->state) {
+	case IW_CM_STATE_ESTABLISHED:
+	case IW_CM_STATE_CLOSING:
+		cm_id_priv->state = IW_CM_STATE_IDLE;
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+		ret = cm_id_priv->id.cm_handler(&cm_id_priv->id, iw_event);
+		spin_lock_irqsave(&cm_id_priv->lock, flags);
+		break;
+	case IW_CM_STATE_DESTROYING:
+		break;
+	default:
+		BUG();
+	}
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+
+	return ret;
+}
+
+static int process_event(struct iwcm_id_private *cm_id_priv,
+			 struct iw_cm_event *iw_event)
+{
+	int ret = 0;
+
+	switch (iw_event->event) {
+	case IW_CM_EVENT_CONNECT_REQUEST:
+		cm_conn_req_handler(cm_id_priv, iw_event);
+		break;
+	case IW_CM_EVENT_CONNECT_REPLY:
+		ret = cm_conn_rep_handler(cm_id_priv, iw_event);
+		break;
+	case IW_CM_EVENT_ESTABLISHED:
+		ret = cm_conn_est_handler(cm_id_priv, iw_event);
+		break;
+	case IW_CM_EVENT_DISCONNECT:
+		cm_disconnect_handler(cm_id_priv, iw_event);
+		break;
+	case IW_CM_EVENT_CLOSE:
+		ret = cm_close_handler(cm_id_priv, iw_event);
+		break;
+	default:
+		BUG();
+	}
+
+	return ret;
+}
+
+/*
+ * Process events on the work_list for the cm_id. If the callback
+ * function requests that the cm_id be deleted, a flag is set in the
+ * cm_id flags to indicate that when the last reference is
+ * removed, the cm_id is to be destroyed. This is necessary to
+ * distinguish between an object that will be destroyed by the app
+ * thread asleep on the destroy_comp list vs. an object destroyed
+ * here synchronously when the last reference is removed.
+ */
+static void cm_work_handler(void *arg)
+{
+	struct iwcm_work *work = arg, lwork;
+	struct iwcm_id_private *cm_id_priv = work->cm_id;
+	unsigned long flags;
+	int empty;
+	int ret = 0;
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	empty = list_empty(&cm_id_priv->work_list);
+	while (!empty) {
+		work = list_entry(cm_id_priv->work_list.next,
+				  struct iwcm_work, list);
+		list_del_init(&work->list);
+		empty = list_empty(&cm_id_priv->work_list);
+		lwork = *work;
+		put_work(work);
+		spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+
+		ret = process_event(cm_id_priv, &work->event);
+		if (ret) {
+			set_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags);
+			destroy_cm_id(&cm_id_priv->id);
+		}
+		BUG_ON(atomic_read(&cm_id_priv->refcount)==0);
+		if (iwcm_deref_id(cm_id_priv))
+			return;
+
+		if (atomic_read(&cm_id_priv->refcount)==0 &&
+		    test_bit(IWCM_F_CALLBACK_DESTROY, &cm_id_priv->flags)) {
+			dealloc_work_entries(cm_id_priv);
+			kfree(cm_id_priv);
+			return;
+		}
+		spin_lock_irqsave(&cm_id_priv->lock, flags);
+	}
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+}
+
+/*
+ * This function is called on interrupt context. Schedule events on
+ * the iwcm_wq thread to allow callback functions to downcall into
+ * the CM and/or block.  Events are queued to a per-CM_ID
+ * work_list. If this is the first event on the work_list, the work
+ * element is also queued on the iwcm_wq thread.
+ *
+ * Each event holds a reference on the cm_id. Until the last posted
+ * event has been delivered and processed, the cm_id cannot be
+ * deleted.
+ *
+ * Returns:
+ * 	      0	- the event was handled.
+ *	-ENOMEM	- the event was not handled due to lack of resources.
+ */
+static int cm_event_handler(struct iw_cm_id *cm_id,
+			     struct iw_cm_event *iw_event)
+{
+	struct iwcm_work *work;
+	struct iwcm_id_private *cm_id_priv;
+	unsigned long flags;
+	int ret = 0;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	work = get_work(cm_id_priv);
+	if (!work) {
+		ret = -ENOMEM;
+		goto out;
+	}
+
+	INIT_WORK(&work->work, cm_work_handler, work);
+	work->cm_id = cm_id_priv;
+	work->event = *iw_event;
+
+	if ((work->event.event == IW_CM_EVENT_CONNECT_REQUEST ||
+	     work->event.event == IW_CM_EVENT_CONNECT_REPLY) &&
+	    work->event.private_data_len) {
+		ret = copy_private_data(cm_id_priv, &work->event);
+		if (ret) {
+			put_work(work);
+			goto out;
+		}
+	}
+
+	atomic_inc(&cm_id_priv->refcount);
+	if (list_empty(&cm_id_priv->work_list)) {
+		list_add_tail(&work->list, &cm_id_priv->work_list);
+		queue_work(iwcm_wq, &work->work);
+	} else
+		list_add_tail(&work->list, &cm_id_priv->work_list);
+out:
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+	return ret;
+}
+
+static int iwcm_init_qp_init_attr(struct iwcm_id_private *cm_id_priv,
+				  struct ib_qp_attr *qp_attr,
+				  int *qp_attr_mask)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	switch (cm_id_priv->state) {
+	case IW_CM_STATE_IDLE:
+	case IW_CM_STATE_CONN_SENT:
+	case IW_CM_STATE_CONN_RECV:
+	case IW_CM_STATE_ESTABLISHED:
+		*qp_attr_mask = IB_QP_STATE | IB_QP_ACCESS_FLAGS;
+		qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
+					   IB_ACCESS_REMOTE_WRITE|
+					   IB_ACCESS_REMOTE_READ;
+		ret = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+	return ret;
+}
+
+static int iwcm_init_qp_rts_attr(struct iwcm_id_private *cm_id_priv,
+				  struct ib_qp_attr *qp_attr,
+				  int *qp_attr_mask)
+{
+	unsigned long flags;
+	int ret;
+
+	spin_lock_irqsave(&cm_id_priv->lock, flags);
+	switch (cm_id_priv->state) {
+	case IW_CM_STATE_IDLE:
+	case IW_CM_STATE_CONN_SENT:
+	case IW_CM_STATE_CONN_RECV:
+	case IW_CM_STATE_ESTABLISHED:
+		*qp_attr_mask = 0;
+		ret = 0;
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	spin_unlock_irqrestore(&cm_id_priv->lock, flags);
+	return ret;
+}
+
+int iw_cm_init_qp_attr(struct iw_cm_id *cm_id,
+		       struct ib_qp_attr *qp_attr,
+		       int *qp_attr_mask)
+{
+	struct iwcm_id_private *cm_id_priv;
+	int ret;
+
+	cm_id_priv = container_of(cm_id, struct iwcm_id_private, id);
+	switch (qp_attr->qp_state) {
+	case IB_QPS_INIT:
+	case IB_QPS_RTR:
+		ret = iwcm_init_qp_init_attr(cm_id_priv,
+					     qp_attr, qp_attr_mask);
+		break;
+	case IB_QPS_RTS:
+		ret = iwcm_init_qp_rts_attr(cm_id_priv,
+					    qp_attr, qp_attr_mask);
+		break;
+	default:
+		ret = -EINVAL;
+		break;
+	}
+	return ret;
+}
+EXPORT_SYMBOL(iw_cm_init_qp_attr);
+
+static int __init iw_cm_init(void)
+{
+	iwcm_wq = create_singlethread_workqueue("iw_cm_wq");
+	if (!iwcm_wq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void __exit iw_cm_cleanup(void)
+{
+	destroy_workqueue(iwcm_wq);
+}
+
+module_init(iw_cm_init);
+module_exit(iw_cm_cleanup);
diff --git a/drivers/infiniband/core/iwcm.h b/drivers/infiniband/core/iwcm.h
new file mode 100644
index 0000000..3f6cc82
--- /dev/null
+++ b/drivers/infiniband/core/iwcm.h
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2005 Network Appliance, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef IWCM_H
+#define IWCM_H
+
+enum iw_cm_state {
+	IW_CM_STATE_IDLE,             /* unbound, inactive */
+	IW_CM_STATE_LISTEN,           /* listen waiting for connect */
+	IW_CM_STATE_CONN_RECV,        /* inbound waiting for user accept */
+	IW_CM_STATE_CONN_SENT,        /* outbound waiting for peer accept */
+	IW_CM_STATE_ESTABLISHED,      /* established */
+	IW_CM_STATE_CLOSING,	      /* disconnect */
+	IW_CM_STATE_DESTROYING        /* object being deleted */
+};
+
+struct iwcm_id_private {
+	struct iw_cm_id	id;
+	enum iw_cm_state state;
+	unsigned long flags;
+	struct ib_qp *qp;
+	struct completion destroy_comp;
+	wait_queue_head_t connect_wait;
+	struct list_head work_list;
+	spinlock_t lock;
+	atomic_t refcount;
+	struct list_head work_free_list;
+};
+
+#define IWCM_F_CALLBACK_DESTROY   1
+#define IWCM_F_CONNECT_WAIT       2
+
+#endif /* IWCM_H */
diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index 1c3cfbb..082f03c 100644
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -1246,8 +1246,8 @@ static int find_vendor_oui(struct ib_mad_mgmt_vendor_class *vendor_class,
 	int i;
 
 	for (i = 0; i < MAX_MGMT_OUI; i++)
-                /* Is there matching OUI for this vendor class ? */
-                if (!memcmp(vendor_class->oui[i], oui, 3))
+		/* Is there matching OUI for this vendor class ? */
+		if (!memcmp(vendor_class->oui[i], oui, 3))
 			return i;
 
 	return -1;
@@ -2237,7 +2237,7 @@ static void cancel_mads(struct ib_mad_agent_private *mad_agent_priv)
 	list_for_each_entry_safe(mad_send_wr, temp_mad_send_wr,
 				 &mad_agent_priv->send_list, agent_list) {
 		if (mad_send_wr->status == IB_WC_SUCCESS) {
- 			mad_send_wr->status = IB_WC_WR_FLUSH_ERR;
+			mad_send_wr->status = IB_WC_WR_FLUSH_ERR;
 			mad_send_wr->refcount -= (mad_send_wr->timeout > 0);
 		}
 	}
@@ -2528,10 +2528,10 @@ static int ib_mad_post_receive_mads(struct ib_mad_qp_info *qp_info,
 			}
 		}
 		sg_list.addr = dma_map_single(qp_info->port_priv->
-					      	device->dma_device,
+					        device->dma_device,
 					      &mad_priv->grh,
 					      sizeof *mad_priv -
-					      	sizeof mad_priv->header,
+					        sizeof mad_priv->header,
 					      DMA_FROM_DEVICE);
 		pci_unmap_addr_set(&mad_priv->header, mapping, sg_list.addr);
 		recv_wr.wr_id = (unsigned long)&mad_priv->header.mad_list;
@@ -2606,7 +2606,7 @@ static int ib_mad_port_start(struct ib_mad_port_private *port_priv)
 	struct ib_qp *qp;
 
 	attr = kmalloc(sizeof *attr, GFP_KERNEL);
- 	if (!attr) {
+	if (!attr) {
 		printk(KERN_ERR PFX "Couldn't kmalloc ib_qp_attr\n");
 		return -ENOMEM;
 	}
@@ -2876,7 +2876,10 @@ static void ib_mad_init_device(struct ib_device *device)
 {
 	int start, end, i;
 
-	if (device->node_type == IB_NODE_SWITCH) {
+	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+		return;
+
+	if (device->node_type == RDMA_NODE_IB_SWITCH) {
 		start = 0;
 		end   = 0;
 	} else {
@@ -2923,7 +2926,7 @@ static void ib_mad_remove_device(struct ib_device *device)
 {
 	int i, num_ports, cur_port;
 
-	if (device->node_type == IB_NODE_SWITCH) {
+	if (device->node_type == RDMA_NODE_IB_SWITCH) {
 		num_ports = 1;
 		cur_port = 0;
 	} else {
diff --git a/drivers/infiniband/core/mad_priv.h b/drivers/infiniband/core/mad_priv.h
index d147f3b..d06b590 100644
--- a/drivers/infiniband/core/mad_priv.h
+++ b/drivers/infiniband/core/mad_priv.h
@@ -38,8 +38,8 @@
 #define __IB_MAD_PRIV_H__
 
 #include <linux/completion.h>
+#include <linux/err.h>
 #include <linux/pci.h>
-#include <linux/kthread.h>
 #include <linux/workqueue.h>
 #include <rdma/ib_mad.h>
 #include <rdma/ib_smi.h>
diff --git a/drivers/infiniband/core/mad_rmpp.c b/drivers/infiniband/core/mad_rmpp.c
index ebcd5b1..1ef79d0 100644
--- a/drivers/infiniband/core/mad_rmpp.c
+++ b/drivers/infiniband/core/mad_rmpp.c
@@ -33,8 +33,6 @@
  * $Id: mad_rmpp.c 1921 2005-03-02 22:58:44Z sean.hefty $
  */
 
-#include <linux/dma-mapping.h>
-
 #include "mad_priv.h"
 #include "mad_rmpp.h"
 
@@ -60,6 +58,7 @@ struct mad_rmpp_recv {
 	int last_ack;
 	int seg_num;
 	int newwin;
+	int repwin;
 
 	__be64 tid;
 	u32 src_qp;
@@ -170,6 +169,32 @@ static struct ib_mad_send_buf *alloc_response_msg(struct ib_mad_agent *agent,
 	return msg;
 }
 
+static void ack_ds_ack(struct ib_mad_agent_private *agent,
+		       struct ib_mad_recv_wc *recv_wc)
+{
+	struct ib_mad_send_buf *msg;
+	struct ib_rmpp_mad *rmpp_mad;
+	int ret;
+
+	msg = alloc_response_msg(&agent->agent, recv_wc);
+	if (IS_ERR(msg))
+		return;
+
+	rmpp_mad = msg->mad;
+	memcpy(rmpp_mad, recv_wc->recv_buf.mad, msg->hdr_len);
+
+	rmpp_mad->mad_hdr.method ^= IB_MGMT_METHOD_RESP;
+	ib_set_rmpp_flags(&rmpp_mad->rmpp_hdr, IB_MGMT_RMPP_FLAG_ACTIVE);
+	rmpp_mad->rmpp_hdr.seg_num = 0;
+	rmpp_mad->rmpp_hdr.paylen_newwin = cpu_to_be32(1);
+
+	ret = ib_post_send_mad(msg, NULL);
+	if (ret) {
+		ib_destroy_ah(msg->ah);
+		ib_free_send_mad(msg);
+	}
+}
+
 void ib_rmpp_send_handler(struct ib_mad_send_wc *mad_send_wc)
 {
 	struct ib_rmpp_mad *rmpp_mad = mad_send_wc->send_buf->mad;
@@ -271,6 +296,7 @@ create_rmpp_recv(struct ib_mad_agent_private *agent,
 	rmpp_recv->newwin = 1;
 	rmpp_recv->seg_num = 1;
 	rmpp_recv->last_ack = 0;
+	rmpp_recv->repwin = 1;
 
 	mad_hdr = &mad_recv_wc->recv_buf.mad->mad_hdr;
 	rmpp_recv->tid = mad_hdr->tid;
@@ -365,7 +391,7 @@ static inline int window_size(struct ib_mad_agent_private *agent)
 static struct ib_mad_recv_buf * find_seg_location(struct list_head *rmpp_list,
 						  int seg_num)
 {
-        struct ib_mad_recv_buf *seg_buf;
+	struct ib_mad_recv_buf *seg_buf;
 	int cur_seg_num;
 
 	list_for_each_entry_reverse(seg_buf, rmpp_list, list) {
@@ -591,6 +617,16 @@ static inline void adjust_last_ack(struct ib_mad_send_wr_private *wr,
 			break;
 }
 
+static void process_ds_ack(struct ib_mad_agent_private *agent,
+			   struct ib_mad_recv_wc *mad_recv_wc, int newwin)
+{
+	struct mad_rmpp_recv *rmpp_recv;
+
+	rmpp_recv = find_rmpp_recv(agent, mad_recv_wc);
+	if (rmpp_recv && rmpp_recv->state == RMPP_STATE_COMPLETE)
+		rmpp_recv->repwin = newwin;
+}
+
 static void process_rmpp_ack(struct ib_mad_agent_private *agent,
 			     struct ib_mad_recv_wc *mad_recv_wc)
 {
@@ -616,8 +652,18 @@ static void process_rmpp_ack(struct ib_mad_agent_private *agent,
 
 	spin_lock_irqsave(&agent->lock, flags);
 	mad_send_wr = ib_find_send_mad(agent, mad_recv_wc);
-	if (!mad_send_wr)
-		goto out;	/* Unmatched ACK */
+	if (!mad_send_wr) {
+		if (!seg_num)
+			process_ds_ack(agent, mad_recv_wc, newwin);
+		goto out;	/* Unmatched or DS RMPP ACK */
+	}
+
+	if ((mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count) &&
+	    (mad_send_wr->timeout)) {
+		spin_unlock_irqrestore(&agent->lock, flags);
+		ack_ds_ack(agent, mad_recv_wc);
+		return;		/* Repeated ACK for DS RMPP transaction */
+	}
 
 	if ((mad_send_wr->last_ack == mad_send_wr->send_buf.seg_count) ||
 	    (!mad_send_wr->timeout) || (mad_send_wr->status != IB_WC_SUCCESS))
@@ -656,6 +702,9 @@ static void process_rmpp_ack(struct ib_mad_agent_private *agent,
 		if (mad_send_wr->refcount == 1)
 			ib_reset_mad_timeout(mad_send_wr,
 					     mad_send_wr->send_buf.timeout_ms);
+		spin_unlock_irqrestore(&agent->lock, flags);
+		ack_ds_ack(agent, mad_recv_wc);
+		return;
 	} else if (mad_send_wr->refcount == 1 &&
 		   mad_send_wr->seg_num < mad_send_wr->newwin &&
 		   mad_send_wr->seg_num < mad_send_wr->send_buf.seg_count) {
@@ -772,6 +821,39 @@ out:
 	return NULL;
 }
 
+static int init_newwin(struct ib_mad_send_wr_private *mad_send_wr)
+{
+	struct ib_mad_agent_private *agent = mad_send_wr->mad_agent_priv;
+	struct ib_mad_hdr *mad_hdr = mad_send_wr->send_buf.mad;
+	struct mad_rmpp_recv *rmpp_recv;
+	struct ib_ah_attr ah_attr;
+	unsigned long flags;
+	int newwin = 1;
+
+	if (!(mad_hdr->method & IB_MGMT_METHOD_RESP))
+		goto out;
+
+	spin_lock_irqsave(&agent->lock, flags);
+	list_for_each_entry(rmpp_recv, &agent->rmpp_list, list) {
+		if (rmpp_recv->tid != mad_hdr->tid ||
+		    rmpp_recv->mgmt_class != mad_hdr->mgmt_class ||
+		    rmpp_recv->class_version != mad_hdr->class_version ||
+		    (rmpp_recv->method & IB_MGMT_METHOD_RESP))
+			continue;
+
+		if (ib_query_ah(mad_send_wr->send_buf.ah, &ah_attr))
+			continue;
+
+		if (rmpp_recv->slid == ah_attr.dlid) {
+			newwin = rmpp_recv->repwin;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&agent->lock, flags);
+out:
+	return newwin;
+}
+
 int ib_send_rmpp_mad(struct ib_mad_send_wr_private *mad_send_wr)
 {
 	struct ib_rmpp_mad *rmpp_mad;
@@ -787,7 +869,7 @@ int ib_send_rmpp_mad(struct ib_mad_send_wr_private *mad_send_wr)
 		return IB_RMPP_RESULT_INTERNAL;
 	}
 
-	mad_send_wr->newwin = 1;
+	mad_send_wr->newwin = init_newwin(mad_send_wr);
 
 	/* We need to wait for the final ACK even if there isn't a response */
 	mad_send_wr->refcount += (mad_send_wr->timeout == 0);
diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index d6b8422..1706d3c 100644
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Voltaire, Inc.  All rights reserved.
+ * Copyright (c) 2006 Intel Corporation.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -75,6 +76,7 @@ struct ib_sa_device {
 struct ib_sa_query {
 	void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *);
 	void (*release)(struct ib_sa_query *);
+	struct ib_sa_client    *client;
 	struct ib_sa_port      *port;
 	struct ib_mad_send_buf *mad_buf;
 	struct ib_sa_sm_ah     *sm_ah;
@@ -415,6 +417,31 @@ static void ib_sa_event(struct ib_event_handler *handler, struct ib_event *event
 	}
 }
 
+void ib_sa_register_client(struct ib_sa_client *client)
+{
+	atomic_set(&client->users, 1);
+	init_completion(&client->comp);
+}
+EXPORT_SYMBOL(ib_sa_register_client);
+
+static inline void ib_sa_client_get(struct ib_sa_client *client)
+{
+	atomic_inc(&client->users);
+}
+
+static inline void ib_sa_client_put(struct ib_sa_client *client)
+{
+	if (atomic_dec_and_test(&client->users))
+		complete(&client->comp);
+}
+
+void ib_sa_unregister_client(struct ib_sa_client *client)
+{
+	ib_sa_client_put(client);
+	wait_for_completion(&client->comp);
+}
+EXPORT_SYMBOL(ib_sa_unregister_client);
+
 /**
  * ib_sa_cancel_query - try to cancel an SA query
  * @id:ID of query to cancel
@@ -557,6 +584,7 @@ static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)
 
 /**
  * ib_sa_path_rec_get - Start a Path get query
+ * @client:SA client
  * @device:device to send query on
  * @port_num: port number to send query on
  * @rec:Path Record to send in query
@@ -579,7 +607,8 @@ static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)
  * error code.  Otherwise it is a query ID that can be used to cancel
  * the query.
  */
-int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
+int ib_sa_path_rec_get(struct ib_sa_client *client,
+		       struct ib_device *device, u8 port_num,
 		       struct ib_sa_path_rec *rec,
 		       ib_sa_comp_mask comp_mask,
 		       int timeout_ms, gfp_t gfp_mask,
@@ -614,8 +643,10 @@ int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
 		goto err1;
 	}
 
-	query->callback = callback;
-	query->context  = context;
+	ib_sa_client_get(client);
+	query->sa_query.client = client;
+	query->callback        = callback;
+	query->context         = context;
 
 	mad = query->sa_query.mad_buf->mad;
 	init_mad(mad, agent);
@@ -639,6 +670,7 @@ int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
 
 err2:
 	*sa_query = NULL;
+	ib_sa_client_put(query->sa_query.client);
 	ib_free_send_mad(query->sa_query.mad_buf);
 
 err1:
@@ -671,6 +703,7 @@ static void ib_sa_service_rec_release(struct ib_sa_query *sa_query)
 
 /**
  * ib_sa_service_rec_query - Start Service Record operation
+ * @client:SA client
  * @device:device to send request on
  * @port_num: port number to send request on
  * @method:SA method - should be get, set, or delete
@@ -695,7 +728,8 @@ static void ib_sa_service_rec_release(struct ib_sa_query *sa_query)
  * error code.  Otherwise it is a request ID that can be used to cancel
  * the query.
  */
-int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method,
+int ib_sa_service_rec_query(struct ib_sa_client *client,
+			    struct ib_device *device, u8 port_num, u8 method,
 			    struct ib_sa_service_rec *rec,
 			    ib_sa_comp_mask comp_mask,
 			    int timeout_ms, gfp_t gfp_mask,
@@ -735,8 +769,10 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method,
 		goto err1;
 	}
 
-	query->callback = callback;
-	query->context  = context;
+	ib_sa_client_get(client);
+	query->sa_query.client = client;
+	query->callback        = callback;
+	query->context         = context;
 
 	mad = query->sa_query.mad_buf->mad;
 	init_mad(mad, agent);
@@ -761,6 +797,7 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method,
 
 err2:
 	*sa_query = NULL;
+	ib_sa_client_put(query->sa_query.client);
 	ib_free_send_mad(query->sa_query.mad_buf);
 
 err1:
@@ -791,7 +828,8 @@ static void ib_sa_mcmember_rec_release(struct ib_sa_query *sa_query)
 	kfree(container_of(sa_query, struct ib_sa_mcmember_query, sa_query));
 }
 
-int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
+int ib_sa_mcmember_rec_query(struct ib_sa_client *client,
+			     struct ib_device *device, u8 port_num,
 			     u8 method,
 			     struct ib_sa_mcmember_rec *rec,
 			     ib_sa_comp_mask comp_mask,
@@ -827,8 +865,10 @@ int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
 		goto err1;
 	}
 
-	query->callback = callback;
-	query->context  = context;
+	ib_sa_client_get(client);
+	query->sa_query.client = client;
+	query->callback        = callback;
+	query->context         = context;
 
 	mad = query->sa_query.mad_buf->mad;
 	init_mad(mad, agent);
@@ -853,6 +893,7 @@ int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
 
 err2:
 	*sa_query = NULL;
+	ib_sa_client_put(query->sa_query.client);
 	ib_free_send_mad(query->sa_query.mad_buf);
 
 err1:
@@ -887,8 +928,9 @@ static void send_handler(struct ib_mad_agent *agent,
 	idr_remove(&query_idr, query->id);
 	spin_unlock_irqrestore(&idr_lock, flags);
 
-        ib_free_send_mad(mad_send_wc->send_buf);
+	ib_free_send_mad(mad_send_wc->send_buf);
 	kref_put(&query->sm_ah->ref, free_sm_ah);
+	ib_sa_client_put(query->client);
 	query->release(query);
 }
 
@@ -919,7 +961,10 @@ static void ib_sa_add_one(struct ib_device *device)
 	struct ib_sa_device *sa_dev;
 	int s, e, i;
 
-	if (device->node_type == IB_NODE_SWITCH)
+	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+		return;
+
+	if (device->node_type == RDMA_NODE_IB_SWITCH)
 		s = e = 0;
 	else {
 		s = 1;
diff --git a/drivers/infiniband/core/smi.c b/drivers/infiniband/core/smi.c
index 35852e7..54b81e1 100644
--- a/drivers/infiniband/core/smi.c
+++ b/drivers/infiniband/core/smi.c
@@ -64,7 +64,7 @@ int smi_handle_dr_smp_send(struct ib_smp *smp,
 
 		/* C14-9:2 */
 		if (hop_ptr && hop_ptr < hop_cnt) {
-			if (node_type != IB_NODE_SWITCH)
+			if (node_type != RDMA_NODE_IB_SWITCH)
 				return 0;
 
 			/* smp->return_path set when received */
@@ -77,7 +77,7 @@ int smi_handle_dr_smp_send(struct ib_smp *smp,
 		if (hop_ptr == hop_cnt) {
 			/* smp->return_path set when received */
 			smp->hop_ptr++;
-			return (node_type == IB_NODE_SWITCH ||
+			return (node_type == RDMA_NODE_IB_SWITCH ||
 				smp->dr_dlid == IB_LID_PERMISSIVE);
 		}
 
@@ -95,7 +95,7 @@ int smi_handle_dr_smp_send(struct ib_smp *smp,
 
 		/* C14-13:2 */
 		if (2 <= hop_ptr && hop_ptr <= hop_cnt) {
-			if (node_type != IB_NODE_SWITCH)
+			if (node_type != RDMA_NODE_IB_SWITCH)
 				return 0;
 
 			smp->hop_ptr--;
@@ -107,7 +107,7 @@ int smi_handle_dr_smp_send(struct ib_smp *smp,
 		if (hop_ptr == 1) {
 			smp->hop_ptr--;
 			/* C14-13:3 -- SMPs destined for SM shouldn't be here */
-			return (node_type == IB_NODE_SWITCH ||
+			return (node_type == RDMA_NODE_IB_SWITCH ||
 				smp->dr_slid == IB_LID_PERMISSIVE);
 		}
 
@@ -142,7 +142,7 @@ int smi_handle_dr_smp_recv(struct ib_smp *smp,
 
 		/* C14-9:2 -- intermediate hop */
 		if (hop_ptr && hop_ptr < hop_cnt) {
-			if (node_type != IB_NODE_SWITCH)
+			if (node_type != RDMA_NODE_IB_SWITCH)
 				return 0;
 
 			smp->return_path[hop_ptr] = port_num;
@@ -156,7 +156,7 @@ int smi_handle_dr_smp_recv(struct ib_smp *smp,
 				smp->return_path[hop_ptr] = port_num;
 			/* smp->hop_ptr updated when sending */
 
-			return (node_type == IB_NODE_SWITCH ||
+			return (node_type == RDMA_NODE_IB_SWITCH ||
 				smp->dr_dlid == IB_LID_PERMISSIVE);
 		}
 
@@ -175,7 +175,7 @@ int smi_handle_dr_smp_recv(struct ib_smp *smp,
 
 		/* C14-13:2 */
 		if (2 <= hop_ptr && hop_ptr <= hop_cnt) {
-			if (node_type != IB_NODE_SWITCH)
+			if (node_type != RDMA_NODE_IB_SWITCH)
 				return 0;
 
 			/* smp->hop_ptr updated when sending */
@@ -190,7 +190,7 @@ int smi_handle_dr_smp_recv(struct ib_smp *smp,
 				return 1;
 			}
 			/* smp->hop_ptr updated when sending */
-			return (node_type == IB_NODE_SWITCH);
+			return (node_type == RDMA_NODE_IB_SWITCH);
 		}
 
 		/* C14-13:4 -- hop_ptr = 0 -> give to SM */
diff --git a/drivers/infiniband/core/sysfs.c b/drivers/infiniband/core/sysfs.c
index 21f9282..709323c 100644
--- a/drivers/infiniband/core/sysfs.c
+++ b/drivers/infiniband/core/sysfs.c
@@ -68,7 +68,7 @@ struct port_table_attribute {
 	int			index;
 };
 
-static inline int ibdev_is_alive(const struct ib_device *dev) 
+static inline int ibdev_is_alive(const struct ib_device *dev)
 {
 	return dev->reg_state == IB_DEV_REGISTERED;
 }
@@ -589,10 +589,11 @@ static ssize_t show_node_type(struct class_device *cdev, char *buf)
 		return -ENODEV;
 
 	switch (dev->node_type) {
-	case IB_NODE_CA:     return sprintf(buf, "%d: CA\n", dev->node_type);
-	case IB_NODE_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
-	case IB_NODE_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
-	default:             return sprintf(buf, "%d: <unknown>\n", dev->node_type);
+	case RDMA_NODE_IB_CA:	  return sprintf(buf, "%d: CA\n", dev->node_type);
+	case RDMA_NODE_RNIC:	  return sprintf(buf, "%d: RNIC\n", dev->node_type);
+	case RDMA_NODE_IB_SWITCH: return sprintf(buf, "%d: switch\n", dev->node_type);
+	case RDMA_NODE_IB_ROUTER: return sprintf(buf, "%d: router\n", dev->node_type);
+	default:		  return sprintf(buf, "%d: <unknown>\n", dev->node_type);
 	}
 }
 
@@ -708,7 +709,7 @@ int ib_device_register_sysfs(struct ib_device *device)
 	if (ret)
 		goto err_put;
 
-	if (device->node_type == IB_NODE_SWITCH) {
+	if (device->node_type == RDMA_NODE_IB_SWITCH) {
 		ret = add_port(device, 0);
 		if (ret)
 			goto err_put;
diff --git a/drivers/infiniband/core/ucm.c b/drivers/infiniband/core/ucm.c
index c1c6fda..ad4f4d5 100644
--- a/drivers/infiniband/core/ucm.c
+++ b/drivers/infiniband/core/ucm.c
@@ -309,9 +309,9 @@ static int ib_ucm_event_process(struct ib_cm_event *evt,
 		info	      = evt->param.apr_rcvd.apr_info;
 		break;
 	case IB_CM_SIDR_REQ_RECEIVED:
-		uvt->resp.u.sidr_req_resp.pkey = 
+		uvt->resp.u.sidr_req_resp.pkey =
 					evt->param.sidr_req_rcvd.pkey;
-		uvt->resp.u.sidr_req_resp.port = 
+		uvt->resp.u.sidr_req_resp.port =
 					evt->param.sidr_req_rcvd.port;
 		uvt->data_len = IB_CM_SIDR_REQ_PRIVATE_DATA_SIZE;
 		break;
@@ -1237,7 +1237,7 @@ static struct class ucm_class = {
 static ssize_t show_ibdev(struct class_device *class_dev, char *buf)
 {
 	struct ib_ucm_device *dev;
-	
+
 	dev = container_of(class_dev, struct ib_ucm_device, class_dev);
 	return sprintf(buf, "%s\n", dev->ib_dev->name);
 }
@@ -1247,7 +1247,8 @@ static void ib_ucm_add_one(struct ib_device *device)
 {
 	struct ib_ucm_device *ucm_dev;
 
-	if (!device->alloc_ucontext)
+	if (!device->alloc_ucontext ||
+	    rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
 		return;
 
 	ucm_dev = kzalloc(sizeof *ucm_dev, GFP_KERNEL);
diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index 1273f88..807fbd6 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
- * Copyright (c) 2005 Voltaire, Inc. All rights reserved. 
+ * Copyright (c) 2005 Voltaire, Inc. All rights reserved.
  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
@@ -1032,7 +1032,10 @@ static void ib_umad_add_one(struct ib_device *device)
 	struct ib_umad_device *umad_dev;
 	int s, e, i;
 
-	if (device->node_type == IB_NODE_SWITCH)
+	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+		return;
+
+	if (device->node_type == RDMA_NODE_IB_SWITCH)
 		s = e = 0;
 	else {
 		s = 1;
diff --git a/drivers/infiniband/core/uverbs_cmd.c b/drivers/infiniband/core/uverbs_cmd.c
index 30923eb..b72c7f6 100644
--- a/drivers/infiniband/core/uverbs_cmd.c
+++ b/drivers/infiniband/core/uverbs_cmd.c
@@ -155,7 +155,7 @@ static struct ib_uobject *__idr_get_uobj(struct idr *idr, int id,
 }
 
 static struct ib_uobject *idr_read_uobj(struct idr *idr, int id,
-					struct ib_ucontext *context)
+					struct ib_ucontext *context, int nested)
 {
 	struct ib_uobject *uobj;
 
@@ -163,7 +163,10 @@ static struct ib_uobject *idr_read_uobj(struct idr *idr, int id,
 	if (!uobj)
 		return NULL;
 
-	down_read(&uobj->mutex);
+	if (nested)
+		down_read_nested(&uobj->mutex, SINGLE_DEPTH_NESTING);
+	else
+		down_read(&uobj->mutex);
 	if (!uobj->live) {
 		put_uobj_read(uobj);
 		return NULL;
@@ -190,17 +193,18 @@ static struct ib_uobject *idr_write_uobj(struct idr *idr, int id,
 	return uobj;
 }
 
-static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context)
+static void *idr_read_obj(struct idr *idr, int id, struct ib_ucontext *context,
+			  int nested)
 {
 	struct ib_uobject *uobj;
 
-	uobj = idr_read_uobj(idr, id, context);
+	uobj = idr_read_uobj(idr, id, context, nested);
 	return uobj ? uobj->object : NULL;
 }
 
 static struct ib_pd *idr_read_pd(int pd_handle, struct ib_ucontext *context)
 {
-	return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context);
+	return idr_read_obj(&ib_uverbs_pd_idr, pd_handle, context, 0);
 }
 
 static void put_pd_read(struct ib_pd *pd)
@@ -208,9 +212,9 @@ static void put_pd_read(struct ib_pd *pd)
 	put_uobj_read(pd->uobject);
 }
 
-static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context)
+static struct ib_cq *idr_read_cq(int cq_handle, struct ib_ucontext *context, int nested)
 {
-	return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context);
+	return idr_read_obj(&ib_uverbs_cq_idr, cq_handle, context, nested);
 }
 
 static void put_cq_read(struct ib_cq *cq)
@@ -220,7 +224,7 @@ static void put_cq_read(struct ib_cq *cq)
 
 static struct ib_ah *idr_read_ah(int ah_handle, struct ib_ucontext *context)
 {
-	return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context);
+	return idr_read_obj(&ib_uverbs_ah_idr, ah_handle, context, 0);
 }
 
 static void put_ah_read(struct ib_ah *ah)
@@ -230,7 +234,7 @@ static void put_ah_read(struct ib_ah *ah)
 
 static struct ib_qp *idr_read_qp(int qp_handle, struct ib_ucontext *context)
 {
-	return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context);
+	return idr_read_obj(&ib_uverbs_qp_idr, qp_handle, context, 0);
 }
 
 static void put_qp_read(struct ib_qp *qp)
@@ -240,7 +244,7 @@ static void put_qp_read(struct ib_qp *qp)
 
 static struct ib_srq *idr_read_srq(int srq_handle, struct ib_ucontext *context)
 {
-	return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context);
+	return idr_read_obj(&ib_uverbs_srq_idr, srq_handle, context, 0);
 }
 
 static void put_srq_read(struct ib_srq *srq)
@@ -837,7 +841,6 @@ ssize_t ib_uverbs_create_cq(struct ib_uverbs_file *file,
 err_copy:
 	idr_remove_uobj(&ib_uverbs_cq_idr, &obj->uobject);
 
-
 err_free:
 	ib_destroy_cq(cq);
 
@@ -867,7 +870,7 @@ ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
 		   (unsigned long) cmd.response + sizeof resp,
 		   in_len - sizeof cmd, out_len - sizeof resp);
 
-	cq = idr_read_cq(cmd.cq_handle, file->ucontext);
+	cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
 	if (!cq)
 		return -EINVAL;
 
@@ -875,11 +878,10 @@ ssize_t ib_uverbs_resize_cq(struct ib_uverbs_file *file,
 	if (ret)
 		goto out;
 
-	memset(&resp, 0, sizeof resp);
 	resp.cqe = cq->cqe;
 
 	if (copy_to_user((void __user *) (unsigned long) cmd.response,
-			 &resp, sizeof resp))
+			 &resp, sizeof resp.cqe))
 		ret = -EFAULT;
 
 out:
@@ -894,7 +896,6 @@ ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
 {
 	struct ib_uverbs_poll_cq       cmd;
 	struct ib_uverbs_poll_cq_resp *resp;
-	struct ib_uobject	      *uobj;
 	struct ib_cq                  *cq;
 	struct ib_wc                  *wc;
 	int                            ret = 0;
@@ -915,16 +916,15 @@ ssize_t ib_uverbs_poll_cq(struct ib_uverbs_file *file,
 		goto out_wc;
 	}
 
-	uobj = idr_read_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext);
-	if (!uobj) {
+	cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
+	if (!cq) {
 		ret = -EINVAL;
 		goto out;
 	}
-	cq = uobj->object;
 
 	resp->count = ib_poll_cq(cq, cmd.ne, wc);
 
-	put_uobj_read(uobj);
+	put_cq_read(cq);
 
 	for (i = 0; i < resp->count; i++) {
 		resp->wc[i].wr_id 	   = wc[i].wr_id;
@@ -959,21 +959,19 @@ ssize_t ib_uverbs_req_notify_cq(struct ib_uverbs_file *file,
 				int out_len)
 {
 	struct ib_uverbs_req_notify_cq cmd;
-	struct ib_uobject	      *uobj;
 	struct ib_cq                  *cq;
 
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
-	uobj = idr_read_uobj(&ib_uverbs_cq_idr, cmd.cq_handle, file->ucontext);
-	if (!uobj)
+	cq = idr_read_cq(cmd.cq_handle, file->ucontext, 0);
+	if (!cq)
 		return -EINVAL;
-	cq = uobj->object;
 
 	ib_req_notify_cq(cq, cmd.solicited_only ?
 			 IB_CQ_SOLICITED : IB_CQ_NEXT_COMP);
 
-	put_uobj_read(uobj);
+	put_cq_read(cq);
 
 	return in_len;
 }
@@ -1064,9 +1062,9 @@ ssize_t ib_uverbs_create_qp(struct ib_uverbs_file *file,
 
 	srq = cmd.is_srq ? idr_read_srq(cmd.srq_handle, file->ucontext) : NULL;
 	pd  = idr_read_pd(cmd.pd_handle, file->ucontext);
-	scq = idr_read_cq(cmd.send_cq_handle, file->ucontext);
+	scq = idr_read_cq(cmd.send_cq_handle, file->ucontext, 0);
 	rcq = cmd.recv_cq_handle == cmd.send_cq_handle ?
-		scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext);
+		scq : idr_read_cq(cmd.recv_cq_handle, file->ucontext, 1);
 
 	if (!pd || !scq || !rcq || (cmd.is_srq && !srq)) {
 		ret = -EINVAL;
@@ -1274,6 +1272,7 @@ ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
 			    int out_len)
 {
 	struct ib_uverbs_modify_qp cmd;
+	struct ib_udata            udata;
 	struct ib_qp              *qp;
 	struct ib_qp_attr         *attr;
 	int                        ret;
@@ -1281,6 +1280,9 @@ ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
+	INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
+		   out_len);
+
 	attr = kmalloc(sizeof *attr, GFP_KERNEL);
 	if (!attr)
 		return -ENOMEM;
@@ -1337,7 +1339,7 @@ ssize_t ib_uverbs_modify_qp(struct ib_uverbs_file *file,
 	attr->alt_ah_attr.ah_flags 	    = cmd.alt_dest.is_global ? IB_AH_GRH : 0;
 	attr->alt_ah_attr.port_num 	    = cmd.alt_dest.port_num;
 
-	ret = ib_modify_qp(qp, attr, cmd.attr_mask);
+	ret = qp->device->modify_qp(qp, attr, cmd.attr_mask, &udata);
 
 	put_qp_read(qp);
 
@@ -1674,7 +1676,6 @@ ssize_t ib_uverbs_post_recv(struct ib_uverbs_file *file,
 				break;
 		}
 
-
 	if (copy_to_user((void __user *) (unsigned long) cmd.response,
 			 &resp, sizeof resp))
 		ret = -EFAULT;
@@ -1724,7 +1725,6 @@ ssize_t ib_uverbs_post_srq_recv(struct ib_uverbs_file *file,
 				break;
 		}
 
-
 	if (copy_to_user((void __user *) (unsigned long) cmd.response,
 			 &resp, sizeof resp))
 		ret = -EFAULT;
@@ -2055,6 +2055,7 @@ ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file,
 			     int out_len)
 {
 	struct ib_uverbs_modify_srq cmd;
+	struct ib_udata             udata;
 	struct ib_srq              *srq;
 	struct ib_srq_attr          attr;
 	int                         ret;
@@ -2062,6 +2063,9 @@ ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file,
 	if (copy_from_user(&cmd, buf, sizeof cmd))
 		return -EFAULT;
 
+	INIT_UDATA(&udata, buf + sizeof cmd, NULL, in_len - sizeof cmd,
+		   out_len);
+
 	srq = idr_read_srq(cmd.srq_handle, file->ucontext);
 	if (!srq)
 		return -EINVAL;
@@ -2069,7 +2073,7 @@ ssize_t ib_uverbs_modify_srq(struct ib_uverbs_file *file,
 	attr.max_wr    = cmd.max_wr;
 	attr.srq_limit = cmd.srq_limit;
 
-	ret = ib_modify_srq(srq, &attr, cmd.attr_mask);
+	ret = srq->device->modify_srq(srq, &attr, cmd.attr_mask, &udata);
 
 	put_srq_read(srq);
 
diff --git a/drivers/infiniband/core/verbs.c b/drivers/infiniband/core/verbs.c
index 468999c..8b5dd36 100644
--- a/drivers/infiniband/core/verbs.c
+++ b/drivers/infiniband/core/verbs.c
@@ -79,6 +79,23 @@ enum ib_rate mult_to_ib_rate(int mult)
 }
 EXPORT_SYMBOL(mult_to_ib_rate);
 
+enum rdma_transport_type
+rdma_node_get_transport(enum rdma_node_type node_type)
+{
+	switch (node_type) {
+	case RDMA_NODE_IB_CA:
+	case RDMA_NODE_IB_SWITCH:
+	case RDMA_NODE_IB_ROUTER:
+		return RDMA_TRANSPORT_IB;
+	case RDMA_NODE_RNIC:
+		return RDMA_TRANSPORT_IWARP;
+	default:
+		BUG();
+		return 0;
+	}
+}
+EXPORT_SYMBOL(rdma_node_get_transport);
+
 /* Protection domains */
 
 struct ib_pd *ib_alloc_pd(struct ib_device *device)
@@ -231,7 +248,7 @@ int ib_modify_srq(struct ib_srq *srq,
 		  struct ib_srq_attr *srq_attr,
 		  enum ib_srq_attr_mask srq_attr_mask)
 {
-	return srq->device->modify_srq(srq, srq_attr, srq_attr_mask);
+	return srq->device->modify_srq(srq, srq_attr, srq_attr_mask, NULL);
 }
 EXPORT_SYMBOL(ib_modify_srq);
 
@@ -547,7 +564,7 @@ int ib_modify_qp(struct ib_qp *qp,
 		 struct ib_qp_attr *qp_attr,
 		 int qp_attr_mask)
 {
-	return qp->device->modify_qp(qp, qp_attr, qp_attr_mask);
+	return qp->device->modify_qp(qp, qp_attr, qp_attr_mask, NULL);
 }
 EXPORT_SYMBOL(ib_modify_qp);
 
diff --git a/drivers/infiniband/hw/amso1100/Kbuild b/drivers/infiniband/hw/amso1100/Kbuild
new file mode 100644
index 0000000..06964c4
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/Kbuild
@@ -0,0 +1,8 @@
+ifdef CONFIG_INFINIBAND_AMSO1100_DEBUG
+EXTRA_CFLAGS += -DDEBUG
+endif
+
+obj-$(CONFIG_INFINIBAND_AMSO1100) += iw_c2.o
+
+iw_c2-y := c2.o c2_provider.o c2_rnic.o c2_alloc.o c2_mq.o c2_ae.o c2_vq.o \
+	c2_intr.o c2_cq.o c2_qp.o c2_cm.o c2_mm.o c2_pd.o
diff --git a/drivers/infiniband/hw/amso1100/Kconfig b/drivers/infiniband/hw/amso1100/Kconfig
new file mode 100644
index 0000000..809cb14
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/Kconfig
@@ -0,0 +1,15 @@
+config INFINIBAND_AMSO1100
+	tristate "Ammasso 1100 HCA support"
+	depends on PCI && INET && INFINIBAND
+	---help---
+	  This is a low-level driver for the Ammasso 1100 host
+	  channel adapter (HCA).
+
+config INFINIBAND_AMSO1100_DEBUG
+	bool "Verbose debugging output"
+	depends on INFINIBAND_AMSO1100
+	default n
+	---help---
+	  This option causes the amso1100 driver to produce a bunch of
+	  debug messages.  Select this if you are developing the driver
+	  or trying to diagnose a problem.
diff --git a/drivers/infiniband/hw/amso1100/c2.c b/drivers/infiniband/hw/amso1100/c2.c
new file mode 100644
index 0000000..9e9120f
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2.c
@@ -0,0 +1,1255 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/delay.h>
+#include <linux/ethtool.h>
+#include <linux/mii.h>
+#include <linux/if_vlan.h>
+#include <linux/crc32.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/tcp.h>
+#include <linux/init.h>
+#include <linux/dma-mapping.h>
+
+#include <asm/io.h>
+#include <asm/irq.h>
+#include <asm/byteorder.h>
+
+#include <rdma/ib_smi.h>
+#include "c2.h"
+#include "c2_provider.h"
+
+MODULE_AUTHOR("Tom Tucker <tom@opengridcomputing.com>");
+MODULE_DESCRIPTION("Ammasso AMSO1100 Low-level iWARP Driver");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_VERSION(DRV_VERSION);
+
+static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK
+    | NETIF_MSG_IFUP | NETIF_MSG_IFDOWN;
+
+static int debug = -1;		/* defaults above */
+module_param(debug, int, 0);
+MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
+
+static int c2_up(struct net_device *netdev);
+static int c2_down(struct net_device *netdev);
+static int c2_xmit_frame(struct sk_buff *skb, struct net_device *netdev);
+static void c2_tx_interrupt(struct net_device *netdev);
+static void c2_rx_interrupt(struct net_device *netdev);
+static irqreturn_t c2_interrupt(int irq, void *dev_id, struct pt_regs *regs);
+static void c2_tx_timeout(struct net_device *netdev);
+static int c2_change_mtu(struct net_device *netdev, int new_mtu);
+static void c2_reset(struct c2_port *c2_port);
+static struct net_device_stats *c2_get_stats(struct net_device *netdev);
+
+static struct pci_device_id c2_pci_table[] = {
+	{ PCI_DEVICE(0x18b8, 0xb001) },
+	{ 0 }
+};
+
+MODULE_DEVICE_TABLE(pci, c2_pci_table);
+
+static void c2_print_macaddr(struct net_device *netdev)
+{
+	pr_debug("%s: MAC %02X:%02X:%02X:%02X:%02X:%02X, "
+		"IRQ %u\n", netdev->name,
+		netdev->dev_addr[0], netdev->dev_addr[1], netdev->dev_addr[2],
+		netdev->dev_addr[3], netdev->dev_addr[4], netdev->dev_addr[5],
+		netdev->irq);
+}
+
+static void c2_set_rxbufsize(struct c2_port *c2_port)
+{
+	struct net_device *netdev = c2_port->netdev;
+
+	if (netdev->mtu > RX_BUF_SIZE)
+		c2_port->rx_buf_size =
+		    netdev->mtu + ETH_HLEN + sizeof(struct c2_rxp_hdr) +
+		    NET_IP_ALIGN;
+	else
+		c2_port->rx_buf_size = sizeof(struct c2_rxp_hdr) + RX_BUF_SIZE;
+}
+
+/*
+ * Allocate TX ring elements and chain them together.
+ * One-to-one association of adapter descriptors with ring elements.
+ */
+static int c2_tx_ring_alloc(struct c2_ring *tx_ring, void *vaddr,
+			    dma_addr_t base, void __iomem * mmio_txp_ring)
+{
+	struct c2_tx_desc *tx_desc;
+	struct c2_txp_desc __iomem *txp_desc;
+	struct c2_element *elem;
+	int i;
+
+	tx_ring->start = kmalloc(sizeof(*elem) * tx_ring->count, GFP_KERNEL);
+	if (!tx_ring->start)
+		return -ENOMEM;
+
+	elem = tx_ring->start;
+	tx_desc = vaddr;
+	txp_desc = mmio_txp_ring;
+	for (i = 0; i < tx_ring->count; i++, elem++, tx_desc++, txp_desc++) {
+		tx_desc->len = 0;
+		tx_desc->status = 0;
+
+		/* Set TXP_HTXD_UNINIT */
+		__raw_writeq(cpu_to_be64(0x1122334455667788ULL),
+			     (void __iomem *) txp_desc + C2_TXP_ADDR);
+		__raw_writew(0, (void __iomem *) txp_desc + C2_TXP_LEN);
+		__raw_writew(cpu_to_be16(TXP_HTXD_UNINIT),
+			     (void __iomem *) txp_desc + C2_TXP_FLAGS);
+
+		elem->skb = NULL;
+		elem->ht_desc = tx_desc;
+		elem->hw_desc = txp_desc;
+
+		if (i == tx_ring->count - 1) {
+			elem->next = tx_ring->start;
+			tx_desc->next_offset = base;
+		} else {
+			elem->next = elem + 1;
+			tx_desc->next_offset =
+			    base + (i + 1) * sizeof(*tx_desc);
+		}
+	}
+
+	tx_ring->to_use = tx_ring->to_clean = tx_ring->start;
+
+	return 0;
+}
+
+/*
+ * Allocate RX ring elements and chain them together.
+ * One-to-one association of adapter descriptors with ring elements.
+ */
+static int c2_rx_ring_alloc(struct c2_ring *rx_ring, void *vaddr,
+			    dma_addr_t base, void __iomem * mmio_rxp_ring)
+{
+	struct c2_rx_desc *rx_desc;
+	struct c2_rxp_desc __iomem *rxp_desc;
+	struct c2_element *elem;
+	int i;
+
+	rx_ring->start = kmalloc(sizeof(*elem) * rx_ring->count, GFP_KERNEL);
+	if (!rx_ring->start)
+		return -ENOMEM;
+
+	elem = rx_ring->start;
+	rx_desc = vaddr;
+	rxp_desc = mmio_rxp_ring;
+	for (i = 0; i < rx_ring->count; i++, elem++, rx_desc++, rxp_desc++) {
+		rx_desc->len = 0;
+		rx_desc->status = 0;
+
+		/* Set RXP_HRXD_UNINIT */
+		__raw_writew(cpu_to_be16(RXP_HRXD_OK),
+		       (void __iomem *) rxp_desc + C2_RXP_STATUS);
+		__raw_writew(0, (void __iomem *) rxp_desc + C2_RXP_COUNT);
+		__raw_writew(0, (void __iomem *) rxp_desc + C2_RXP_LEN);
+		__raw_writeq(cpu_to_be64(0x99aabbccddeeffULL),
+			     (void __iomem *) rxp_desc + C2_RXP_ADDR);
+		__raw_writew(cpu_to_be16(RXP_HRXD_UNINIT),
+			     (void __iomem *) rxp_desc + C2_RXP_FLAGS);
+
+		elem->skb = NULL;
+		elem->ht_desc = rx_desc;
+		elem->hw_desc = rxp_desc;
+
+		if (i == rx_ring->count - 1) {
+			elem->next = rx_ring->start;
+			rx_desc->next_offset = base;
+		} else {
+			elem->next = elem + 1;
+			rx_desc->next_offset =
+			    base + (i + 1) * sizeof(*rx_desc);
+		}
+	}
+
+	rx_ring->to_use = rx_ring->to_clean = rx_ring->start;
+
+	return 0;
+}
+
+/* Setup buffer for receiving */
+static inline int c2_rx_alloc(struct c2_port *c2_port, struct c2_element *elem)
+{
+	struct c2_dev *c2dev = c2_port->c2dev;
+	struct c2_rx_desc *rx_desc = elem->ht_desc;
+	struct sk_buff *skb;
+	dma_addr_t mapaddr;
+	u32 maplen;
+	struct c2_rxp_hdr *rxp_hdr;
+
+	skb = dev_alloc_skb(c2_port->rx_buf_size);
+	if (unlikely(!skb)) {
+		pr_debug("%s: out of memory for receive\n",
+			c2_port->netdev->name);
+		return -ENOMEM;
+	}
+
+	/* Zero out the rxp hdr in the sk_buff */
+	memset(skb->data, 0, sizeof(*rxp_hdr));
+
+	skb->dev = c2_port->netdev;
+
+	maplen = c2_port->rx_buf_size;
+	mapaddr =
+	    pci_map_single(c2dev->pcidev, skb->data, maplen,
+			   PCI_DMA_FROMDEVICE);
+
+	/* Set the sk_buff RXP_header to RXP_HRXD_READY */
+	rxp_hdr = (struct c2_rxp_hdr *) skb->data;
+	rxp_hdr->flags = RXP_HRXD_READY;
+
+	__raw_writew(0, elem->hw_desc + C2_RXP_STATUS);
+	__raw_writew(cpu_to_be16((u16) maplen - sizeof(*rxp_hdr)),
+		     elem->hw_desc + C2_RXP_LEN);
+	__raw_writeq(cpu_to_be64(mapaddr), elem->hw_desc + C2_RXP_ADDR);
+	__raw_writew(cpu_to_be16(RXP_HRXD_READY), elem->hw_desc + C2_RXP_FLAGS);
+
+	elem->skb = skb;
+	elem->mapaddr = mapaddr;
+	elem->maplen = maplen;
+	rx_desc->len = maplen;
+
+	return 0;
+}
+
+/*
+ * Allocate buffers for the Rx ring
+ * For receive:  rx_ring.to_clean is next received frame
+ */
+static int c2_rx_fill(struct c2_port *c2_port)
+{
+	struct c2_ring *rx_ring = &c2_port->rx_ring;
+	struct c2_element *elem;
+	int ret = 0;
+
+	elem = rx_ring->start;
+	do {
+		if (c2_rx_alloc(c2_port, elem)) {
+			ret = 1;
+			break;
+		}
+	} while ((elem = elem->next) != rx_ring->start);
+
+	rx_ring->to_clean = rx_ring->start;
+	return ret;
+}
+
+/* Free all buffers in RX ring, assumes receiver stopped */
+static void c2_rx_clean(struct c2_port *c2_port)
+{
+	struct c2_dev *c2dev = c2_port->c2dev;
+	struct c2_ring *rx_ring = &c2_port->rx_ring;
+	struct c2_element *elem;
+	struct c2_rx_desc *rx_desc;
+
+	elem = rx_ring->start;
+	do {
+		rx_desc = elem->ht_desc;
+		rx_desc->len = 0;
+
+		__raw_writew(0, elem->hw_desc + C2_RXP_STATUS);
+		__raw_writew(0, elem->hw_desc + C2_RXP_COUNT);
+		__raw_writew(0, elem->hw_desc + C2_RXP_LEN);
+		__raw_writeq(cpu_to_be64(0x99aabbccddeeffULL),
+			     elem->hw_desc + C2_RXP_ADDR);
+		__raw_writew(cpu_to_be16(RXP_HRXD_UNINIT),
+			     elem->hw_desc + C2_RXP_FLAGS);
+
+		if (elem->skb) {
+			pci_unmap_single(c2dev->pcidev, elem->mapaddr,
+					 elem->maplen, PCI_DMA_FROMDEVICE);
+			dev_kfree_skb(elem->skb);
+			elem->skb = NULL;
+		}
+	} while ((elem = elem->next) != rx_ring->start);
+}
+
+static inline int c2_tx_free(struct c2_dev *c2dev, struct c2_element *elem)
+{
+	struct c2_tx_desc *tx_desc = elem->ht_desc;
+
+	tx_desc->len = 0;
+
+	pci_unmap_single(c2dev->pcidev, elem->mapaddr, elem->maplen,
+			 PCI_DMA_TODEVICE);
+
+	if (elem->skb) {
+		dev_kfree_skb_any(elem->skb);
+		elem->skb = NULL;
+	}
+
+	return 0;
+}
+
+/* Free all buffers in TX ring, assumes transmitter stopped */
+static void c2_tx_clean(struct c2_port *c2_port)
+{
+	struct c2_ring *tx_ring = &c2_port->tx_ring;
+	struct c2_element *elem;
+	struct c2_txp_desc txp_htxd;
+	int retry;
+	unsigned long flags;
+
+	spin_lock_irqsave(&c2_port->tx_lock, flags);
+
+	elem = tx_ring->start;
+
+	do {
+		retry = 0;
+		do {
+			txp_htxd.flags =
+			    readw(elem->hw_desc + C2_TXP_FLAGS);
+
+			if (txp_htxd.flags == TXP_HTXD_READY) {
+				retry = 1;
+				__raw_writew(0,
+					     elem->hw_desc + C2_TXP_LEN);
+				__raw_writeq(0,
+					     elem->hw_desc + C2_TXP_ADDR);
+				__raw_writew(cpu_to_be16(TXP_HTXD_DONE),
+					     elem->hw_desc + C2_TXP_FLAGS);
+				c2_port->netstats.tx_dropped++;
+				break;
+			} else {
+				__raw_writew(0,
+					     elem->hw_desc + C2_TXP_LEN);
+				__raw_writeq(cpu_to_be64(0x1122334455667788ULL),
+					     elem->hw_desc + C2_TXP_ADDR);
+				__raw_writew(cpu_to_be16(TXP_HTXD_UNINIT),
+					     elem->hw_desc + C2_TXP_FLAGS);
+			}
+
+			c2_tx_free(c2_port->c2dev, elem);
+
+		} while ((elem = elem->next) != tx_ring->start);
+	} while (retry);
+
+	c2_port->tx_avail = c2_port->tx_ring.count - 1;
+	c2_port->c2dev->cur_tx = tx_ring->to_use - tx_ring->start;
+
+	if (c2_port->tx_avail > MAX_SKB_FRAGS + 1)
+		netif_wake_queue(c2_port->netdev);
+
+	spin_unlock_irqrestore(&c2_port->tx_lock, flags);
+}
+
+/*
+ * Process transmit descriptors marked 'DONE' by the firmware,
+ * freeing up their unneeded sk_buffs.
+ */
+static void c2_tx_interrupt(struct net_device *netdev)
+{
+	struct c2_port *c2_port = netdev_priv(netdev);
+	struct c2_dev *c2dev = c2_port->c2dev;
+	struct c2_ring *tx_ring = &c2_port->tx_ring;
+	struct c2_element *elem;
+	struct c2_txp_desc txp_htxd;
+
+	spin_lock(&c2_port->tx_lock);
+
+	for (elem = tx_ring->to_clean; elem != tx_ring->to_use;
+	     elem = elem->next) {
+		txp_htxd.flags =
+		    be16_to_cpu(readw(elem->hw_desc + C2_TXP_FLAGS));
+
+		if (txp_htxd.flags != TXP_HTXD_DONE)
+			break;
+
+		if (netif_msg_tx_done(c2_port)) {
+			/* PCI reads are expensive in fast path */
+			txp_htxd.len =
+			    be16_to_cpu(readw(elem->hw_desc + C2_TXP_LEN));
+			pr_debug("%s: tx done slot %3Zu status 0x%x len "
+				"%5u bytes\n",
+				netdev->name, elem - tx_ring->start,
+				txp_htxd.flags, txp_htxd.len);
+		}
+
+		c2_tx_free(c2dev, elem);
+		++(c2_port->tx_avail);
+	}
+
+	tx_ring->to_clean = elem;
+
+	if (netif_queue_stopped(netdev)
+	    && c2_port->tx_avail > MAX_SKB_FRAGS + 1)
+		netif_wake_queue(netdev);
+
+	spin_unlock(&c2_port->tx_lock);
+}
+
+static void c2_rx_error(struct c2_port *c2_port, struct c2_element *elem)
+{
+	struct c2_rx_desc *rx_desc = elem->ht_desc;
+	struct c2_rxp_hdr *rxp_hdr = (struct c2_rxp_hdr *) elem->skb->data;
+
+	if (rxp_hdr->status != RXP_HRXD_OK ||
+	    rxp_hdr->len > (rx_desc->len - sizeof(*rxp_hdr))) {
+		pr_debug("BAD RXP_HRXD\n");
+		pr_debug("  rx_desc : %p\n", rx_desc);
+		pr_debug("    index : %Zu\n",
+			elem - c2_port->rx_ring.start);
+		pr_debug("    len   : %u\n", rx_desc->len);
+		pr_debug("  rxp_hdr : %p [PA %p]\n", rxp_hdr,
+			(void *) __pa((unsigned long) rxp_hdr));
+		pr_debug("    flags : 0x%x\n", rxp_hdr->flags);
+		pr_debug("    status: 0x%x\n", rxp_hdr->status);
+		pr_debug("    len   : %u\n", rxp_hdr->len);
+		pr_debug("    rsvd  : 0x%x\n", rxp_hdr->rsvd);
+	}
+
+	/* Setup the skb for reuse since we're dropping this pkt */
+	elem->skb->tail = elem->skb->data = elem->skb->head;
+
+	/* Zero out the rxp hdr in the sk_buff */
+	memset(elem->skb->data, 0, sizeof(*rxp_hdr));
+
+	/* Write the descriptor to the adapter's rx ring */
+	__raw_writew(0, elem->hw_desc + C2_RXP_STATUS);
+	__raw_writew(0, elem->hw_desc + C2_RXP_COUNT);
+	__raw_writew(cpu_to_be16((u16) elem->maplen - sizeof(*rxp_hdr)),
+		     elem->hw_desc + C2_RXP_LEN);
+	__raw_writeq(cpu_to_be64(elem->mapaddr), elem->hw_desc + C2_RXP_ADDR);
+	__raw_writew(cpu_to_be16(RXP_HRXD_READY), elem->hw_desc + C2_RXP_FLAGS);
+
+	pr_debug("packet dropped\n");
+	c2_port->netstats.rx_dropped++;
+}
+
+static void c2_rx_interrupt(struct net_device *netdev)
+{
+	struct c2_port *c2_port = netdev_priv(netdev);
+	struct c2_dev *c2dev = c2_port->c2dev;
+	struct c2_ring *rx_ring = &c2_port->rx_ring;
+	struct c2_element *elem;
+	struct c2_rx_desc *rx_desc;
+	struct c2_rxp_hdr *rxp_hdr;
+	struct sk_buff *skb;
+	dma_addr_t mapaddr;
+	u32 maplen, buflen;
+	unsigned long flags;
+
+	spin_lock_irqsave(&c2dev->lock, flags);
+
+	/* Begin where we left off */
+	rx_ring->to_clean = rx_ring->start + c2dev->cur_rx;
+
+	for (elem = rx_ring->to_clean; elem->next != rx_ring->to_clean;
+	     elem = elem->next) {
+		rx_desc = elem->ht_desc;
+		mapaddr = elem->mapaddr;
+		maplen = elem->maplen;
+		skb = elem->skb;
+		rxp_hdr = (struct c2_rxp_hdr *) skb->data;
+
+		if (rxp_hdr->flags != RXP_HRXD_DONE)
+			break;
+		buflen = rxp_hdr->len;
+
+		/* Sanity check the RXP header */
+		if (rxp_hdr->status != RXP_HRXD_OK ||
+		    buflen > (rx_desc->len - sizeof(*rxp_hdr))) {
+			c2_rx_error(c2_port, elem);
+			continue;
+		}
+
+		/*
+		 * Allocate and map a new skb for replenishing the host
+		 * RX desc
+		 */
+		if (c2_rx_alloc(c2_port, elem)) {
+			c2_rx_error(c2_port, elem);
+			continue;
+		}
+
+		/* Unmap the old skb */
+		pci_unmap_single(c2dev->pcidev, mapaddr, maplen,
+				 PCI_DMA_FROMDEVICE);
+
+		prefetch(skb->data);
+
+		/*
+		 * Skip past the leading 8 bytes comprising of the
+		 * "struct c2_rxp_hdr", prepended by the adapter
+		 * to the usual Ethernet header ("struct ethhdr"),
+		 * to the start of the raw Ethernet packet.
+		 *
+		 * Fix up the various fields in the sk_buff before
+		 * passing it up to netif_rx(). The transfer size
+		 * (in bytes) specified by the adapter len field of
+		 * the "struct rxp_hdr_t" does NOT include the
+		 * "sizeof(struct c2_rxp_hdr)".
+		 */
+		skb->data += sizeof(*rxp_hdr);
+		skb->tail = skb->data + buflen;
+		skb->len = buflen;
+		skb->dev = netdev;
+		skb->protocol = eth_type_trans(skb, netdev);
+
+		netif_rx(skb);
+
+		netdev->last_rx = jiffies;
+		c2_port->netstats.rx_packets++;
+		c2_port->netstats.rx_bytes += buflen;
+	}
+
+	/* Save where we left off */
+	rx_ring->to_clean = elem;
+	c2dev->cur_rx = elem - rx_ring->start;
+	C2_SET_CUR_RX(c2dev, c2dev->cur_rx);
+
+	spin_unlock_irqrestore(&c2dev->lock, flags);
+}
+
+/*
+ * Handle netisr0 TX & RX interrupts.
+ */
+static irqreturn_t c2_interrupt(int irq, void *dev_id, struct pt_regs *regs)
+{
+	unsigned int netisr0, dmaisr;
+	int handled = 0;
+	struct c2_dev *c2dev = (struct c2_dev *) dev_id;
+
+	/* Process CCILNET interrupts */
+	netisr0 = readl(c2dev->regs + C2_NISR0);
+	if (netisr0) {
+
+		/*
+		 * There is an issue with the firmware that always
+		 * provides the status of RX for both TX & RX
+		 * interrupts.  So process both queues here.
+		 */
+		c2_rx_interrupt(c2dev->netdev);
+		c2_tx_interrupt(c2dev->netdev);
+
+		/* Clear the interrupt */
+		writel(netisr0, c2dev->regs + C2_NISR0);
+		handled++;
+	}
+
+	/* Process RNIC interrupts */
+	dmaisr = readl(c2dev->regs + C2_DISR);
+	if (dmaisr) {
+		writel(dmaisr, c2dev->regs + C2_DISR);
+		c2_rnic_interrupt(c2dev);
+		handled++;
+	}
+
+	if (handled) {
+		return IRQ_HANDLED;
+	} else {
+		return IRQ_NONE;
+	}
+}
+
+static int c2_up(struct net_device *netdev)
+{
+	struct c2_port *c2_port = netdev_priv(netdev);
+	struct c2_dev *c2dev = c2_port->c2dev;
+	struct c2_element *elem;
+	struct c2_rxp_hdr *rxp_hdr;
+	struct in_device *in_dev;
+	size_t rx_size, tx_size;
+	int ret, i;
+	unsigned int netimr0;
+
+	if (netif_msg_ifup(c2_port))
+		pr_debug("%s: enabling interface\n", netdev->name);
+
+	/* Set the Rx buffer size based on MTU */
+	c2_set_rxbufsize(c2_port);
+
+	/* Allocate DMA'able memory for Tx/Rx host descriptor rings */
+	rx_size = c2_port->rx_ring.count * sizeof(struct c2_rx_desc);
+	tx_size = c2_port->tx_ring.count * sizeof(struct c2_tx_desc);
+
+	c2_port->mem_size = tx_size + rx_size;
+	c2_port->mem = pci_alloc_consistent(c2dev->pcidev, c2_port->mem_size,
+					    &c2_port->dma);
+	if (c2_port->mem == NULL) {
+		pr_debug("Unable to allocate memory for "
+			"host descriptor rings\n");
+		return -ENOMEM;
+	}
+
+	memset(c2_port->mem, 0, c2_port->mem_size);
+
+	/* Create the Rx host descriptor ring */
+	if ((ret =
+	     c2_rx_ring_alloc(&c2_port->rx_ring, c2_port->mem, c2_port->dma,
+			      c2dev->mmio_rxp_ring))) {
+		pr_debug("Unable to create RX ring\n");
+		goto bail0;
+	}
+
+	/* Allocate Rx buffers for the host descriptor ring */
+	if (c2_rx_fill(c2_port)) {
+		pr_debug("Unable to fill RX ring\n");
+		goto bail1;
+	}
+
+	/* Create the Tx host descriptor ring */
+	if ((ret = c2_tx_ring_alloc(&c2_port->tx_ring, c2_port->mem + rx_size,
+				    c2_port->dma + rx_size,
+				    c2dev->mmio_txp_ring))) {
+		pr_debug("Unable to create TX ring\n");
+		goto bail1;
+	}
+
+	/* Set the TX pointer to where we left off */
+	c2_port->tx_avail = c2_port->tx_ring.count - 1;
+	c2_port->tx_ring.to_use = c2_port->tx_ring.to_clean =
+	    c2_port->tx_ring.start + c2dev->cur_tx;
+
+	/* missing: Initialize MAC */
+
+	BUG_ON(c2_port->tx_ring.to_use != c2_port->tx_ring.to_clean);
+
+	/* Reset the adapter, ensures the driver is in sync with the RXP */
+	c2_reset(c2_port);
+
+	/* Reset the READY bit in the sk_buff RXP headers & adapter HRXDQ */
+	for (i = 0, elem = c2_port->rx_ring.start; i < c2_port->rx_ring.count;
+	     i++, elem++) {
+		rxp_hdr = (struct c2_rxp_hdr *) elem->skb->data;
+		rxp_hdr->flags = 0;
+		__raw_writew(cpu_to_be16(RXP_HRXD_READY),
+			     elem->hw_desc + C2_RXP_FLAGS);
+	}
+
+	/* Enable network packets */
+	netif_start_queue(netdev);
+
+	/* Enable IRQ */
+	writel(0, c2dev->regs + C2_IDIS);
+	netimr0 = readl(c2dev->regs + C2_NIMR0);
+	netimr0 &= ~(C2_PCI_HTX_INT | C2_PCI_HRX_INT);
+	writel(netimr0, c2dev->regs + C2_NIMR0);
+
+	/* Tell the stack to ignore arp requests for ipaddrs bound to
+	 * other interfaces.  This is needed to prevent the host stack
+	 * from responding to arp requests to the ipaddr bound on the
+	 * rdma interface.
+	 */
+	in_dev = in_dev_get(netdev);
+	in_dev->cnf.arp_ignore = 1;
+	in_dev_put(in_dev);
+
+	return 0;
+
+      bail1:
+	c2_rx_clean(c2_port);
+	kfree(c2_port->rx_ring.start);
+
+      bail0:
+	pci_free_consistent(c2dev->pcidev, c2_port->mem_size, c2_port->mem,
+			    c2_port->dma);
+
+	return ret;
+}
+
+static int c2_down(struct net_device *netdev)
+{
+	struct c2_port *c2_port = netdev_priv(netdev);
+	struct c2_dev *c2dev = c2_port->c2dev;
+
+	if (netif_msg_ifdown(c2_port))
+		pr_debug("%s: disabling interface\n",
+			netdev->name);
+
+	/* Wait for all the queued packets to get sent */
+	c2_tx_interrupt(netdev);
+
+	/* Disable network packets */
+	netif_stop_queue(netdev);
+
+	/* Disable IRQs by clearing the interrupt mask */
+	writel(1, c2dev->regs + C2_IDIS);
+	writel(0, c2dev->regs + C2_NIMR0);
+
+	/* missing: Stop transmitter */
+
+	/* missing: Stop receiver */
+
+	/* Reset the adapter, ensures the driver is in sync with the RXP */
+	c2_reset(c2_port);
+
+	/* missing: Turn off LEDs here */
+
+	/* Free all buffers in the host descriptor rings */
+	c2_tx_clean(c2_port);
+	c2_rx_clean(c2_port);
+
+	/* Free the host descriptor rings */
+	kfree(c2_port->rx_ring.start);
+	kfree(c2_port->tx_ring.start);
+	pci_free_consistent(c2dev->pcidev, c2_port->mem_size, c2_port->mem,
+			    c2_port->dma);
+
+	return 0;
+}
+
+static void c2_reset(struct c2_port *c2_port)
+{
+	struct c2_dev *c2dev = c2_port->c2dev;
+	unsigned int cur_rx = c2dev->cur_rx;
+
+	/* Tell the hardware to quiesce */
+	C2_SET_CUR_RX(c2dev, cur_rx | C2_PCI_HRX_QUI);
+
+	/*
+	 * The hardware will reset the C2_PCI_HRX_QUI bit once
+	 * the RXP is quiesced.  Wait 2 seconds for this.
+	 */
+	ssleep(2);
+
+	cur_rx = C2_GET_CUR_RX(c2dev);
+
+	if (cur_rx & C2_PCI_HRX_QUI)
+		pr_debug("c2_reset: failed to quiesce the hardware!\n");
+
+	cur_rx &= ~C2_PCI_HRX_QUI;
+
+	c2dev->cur_rx = cur_rx;
+
+	pr_debug("Current RX: %u\n", c2dev->cur_rx);
+}
+
+static int c2_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
+{
+	struct c2_port *c2_port = netdev_priv(netdev);
+	struct c2_dev *c2dev = c2_port->c2dev;
+	struct c2_ring *tx_ring = &c2_port->tx_ring;
+	struct c2_element *elem;
+	dma_addr_t mapaddr;
+	u32 maplen;
+	unsigned long flags;
+	unsigned int i;
+
+	spin_lock_irqsave(&c2_port->tx_lock, flags);
+
+	if (unlikely(c2_port->tx_avail < (skb_shinfo(skb)->nr_frags + 1))) {
+		netif_stop_queue(netdev);
+		spin_unlock_irqrestore(&c2_port->tx_lock, flags);
+
+		pr_debug("%s: Tx ring full when queue awake!\n",
+			netdev->name);
+		return NETDEV_TX_BUSY;
+	}
+
+	maplen = skb_headlen(skb);
+	mapaddr =
+	    pci_map_single(c2dev->pcidev, skb->data, maplen, PCI_DMA_TODEVICE);
+
+	elem = tx_ring->to_use;
+	elem->skb = skb;
+	elem->mapaddr = mapaddr;
+	elem->maplen = maplen;
+
+	/* Tell HW to xmit */
+	__raw_writeq(cpu_to_be64(mapaddr), elem->hw_desc + C2_TXP_ADDR);
+	__raw_writew(cpu_to_be16(maplen), elem->hw_desc + C2_TXP_LEN);
+	__raw_writew(cpu_to_be16(TXP_HTXD_READY), elem->hw_desc + C2_TXP_FLAGS);
+
+	c2_port->netstats.tx_packets++;
+	c2_port->netstats.tx_bytes += maplen;
+
+	/* Loop thru additional data fragments and queue them */
+	if (skb_shinfo(skb)->nr_frags) {
+		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
+			skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
+			maplen = frag->size;
+			mapaddr =
+			    pci_map_page(c2dev->pcidev, frag->page,
+					 frag->page_offset, maplen,
+					 PCI_DMA_TODEVICE);
+
+			elem = elem->next;
+			elem->skb = NULL;
+			elem->mapaddr = mapaddr;
+			elem->maplen = maplen;
+
+			/* Tell HW to xmit */
+			__raw_writeq(cpu_to_be64(mapaddr),
+				     elem->hw_desc + C2_TXP_ADDR);
+			__raw_writew(cpu_to_be16(maplen),
+				     elem->hw_desc + C2_TXP_LEN);
+			__raw_writew(cpu_to_be16(TXP_HTXD_READY),
+				     elem->hw_desc + C2_TXP_FLAGS);
+
+			c2_port->netstats.tx_packets++;
+			c2_port->netstats.tx_bytes += maplen;
+		}
+	}
+
+	tx_ring->to_use = elem->next;
+	c2_port->tx_avail -= (skb_shinfo(skb)->nr_frags + 1);
+
+	if (c2_port->tx_avail <= MAX_SKB_FRAGS + 1) {
+		netif_stop_queue(netdev);
+		if (netif_msg_tx_queued(c2_port))
+			pr_debug("%s: transmit queue full\n",
+				netdev->name);
+	}
+
+	spin_unlock_irqrestore(&c2_port->tx_lock, flags);
+
+	netdev->trans_start = jiffies;
+
+	return NETDEV_TX_OK;
+}
+
+static struct net_device_stats *c2_get_stats(struct net_device *netdev)
+{
+	struct c2_port *c2_port = netdev_priv(netdev);
+
+	return &c2_port->netstats;
+}
+
+static void c2_tx_timeout(struct net_device *netdev)
+{
+	struct c2_port *c2_port = netdev_priv(netdev);
+
+	if (netif_msg_timer(c2_port))
+		pr_debug("%s: tx timeout\n", netdev->name);
+
+	c2_tx_clean(c2_port);
+}
+
+static int c2_change_mtu(struct net_device *netdev, int new_mtu)
+{
+	int ret = 0;
+
+	if (new_mtu < ETH_ZLEN || new_mtu > ETH_JUMBO_MTU)
+		return -EINVAL;
+
+	netdev->mtu = new_mtu;
+
+	if (netif_running(netdev)) {
+		c2_down(netdev);
+
+		c2_up(netdev);
+	}
+
+	return ret;
+}
+
+/* Initialize network device */
+static struct net_device *c2_devinit(struct c2_dev *c2dev,
+				     void __iomem * mmio_addr)
+{
+	struct c2_port *c2_port = NULL;
+	struct net_device *netdev = alloc_etherdev(sizeof(*c2_port));
+
+	if (!netdev) {
+		pr_debug("c2_port etherdev alloc failed");
+		return NULL;
+	}
+
+	SET_MODULE_OWNER(netdev);
+	SET_NETDEV_DEV(netdev, &c2dev->pcidev->dev);
+
+	netdev->open = c2_up;
+	netdev->stop = c2_down;
+	netdev->hard_start_xmit = c2_xmit_frame;
+	netdev->get_stats = c2_get_stats;
+	netdev->tx_timeout = c2_tx_timeout;
+	netdev->change_mtu = c2_change_mtu;
+	netdev->watchdog_timeo = C2_TX_TIMEOUT;
+	netdev->irq = c2dev->pcidev->irq;
+
+	c2_port = netdev_priv(netdev);
+	c2_port->netdev = netdev;
+	c2_port->c2dev = c2dev;
+	c2_port->msg_enable = netif_msg_init(debug, default_msg);
+	c2_port->tx_ring.count = C2_NUM_TX_DESC;
+	c2_port->rx_ring.count = C2_NUM_RX_DESC;
+
+	spin_lock_init(&c2_port->tx_lock);
+
+	/* Copy our 48-bit ethernet hardware address */
+	memcpy_fromio(netdev->dev_addr, mmio_addr + C2_REGS_ENADDR, 6);
+
+	/* Validate the MAC address */
+	if (!is_valid_ether_addr(netdev->dev_addr)) {
+		pr_debug("Invalid MAC Address\n");
+		c2_print_macaddr(netdev);
+		free_netdev(netdev);
+		return NULL;
+	}
+
+	c2dev->netdev = netdev;
+
+	return netdev;
+}
+
+static int __devinit c2_probe(struct pci_dev *pcidev,
+			      const struct pci_device_id *ent)
+{
+	int ret = 0, i;
+	unsigned long reg0_start, reg0_flags, reg0_len;
+	unsigned long reg2_start, reg2_flags, reg2_len;
+	unsigned long reg4_start, reg4_flags, reg4_len;
+	unsigned kva_map_size;
+	struct net_device *netdev = NULL;
+	struct c2_dev *c2dev = NULL;
+	void __iomem *mmio_regs = NULL;
+
+	printk(KERN_INFO PFX "AMSO1100 Gigabit Ethernet driver v%s loaded\n",
+		DRV_VERSION);
+
+	/* Enable PCI device */
+	ret = pci_enable_device(pcidev);
+	if (ret) {
+		printk(KERN_ERR PFX "%s: Unable to enable PCI device\n",
+			pci_name(pcidev));
+		goto bail0;
+	}
+
+	reg0_start = pci_resource_start(pcidev, BAR_0);
+	reg0_len = pci_resource_len(pcidev, BAR_0);
+	reg0_flags = pci_resource_flags(pcidev, BAR_0);
+
+	reg2_start = pci_resource_start(pcidev, BAR_2);
+	reg2_len = pci_resource_len(pcidev, BAR_2);
+	reg2_flags = pci_resource_flags(pcidev, BAR_2);
+
+	reg4_start = pci_resource_start(pcidev, BAR_4);
+	reg4_len = pci_resource_len(pcidev, BAR_4);
+	reg4_flags = pci_resource_flags(pcidev, BAR_4);
+
+	pr_debug("BAR0 size = 0x%lX bytes\n", reg0_len);
+	pr_debug("BAR2 size = 0x%lX bytes\n", reg2_len);
+	pr_debug("BAR4 size = 0x%lX bytes\n", reg4_len);
+
+	/* Make sure PCI base addr are MMIO */
+	if (!(reg0_flags & IORESOURCE_MEM) ||
+	    !(reg2_flags & IORESOURCE_MEM) || !(reg4_flags & IORESOURCE_MEM)) {
+		printk(KERN_ERR PFX "PCI regions not an MMIO resource\n");
+		ret = -ENODEV;
+		goto bail1;
+	}
+
+	/* Check for weird/broken PCI region reporting */
+	if ((reg0_len < C2_REG0_SIZE) ||
+	    (reg2_len < C2_REG2_SIZE) || (reg4_len < C2_REG4_SIZE)) {
+		printk(KERN_ERR PFX "Invalid PCI region sizes\n");
+		ret = -ENODEV;
+		goto bail1;
+	}
+
+	/* Reserve PCI I/O and memory resources */
+	ret = pci_request_regions(pcidev, DRV_NAME);
+	if (ret) {
+		printk(KERN_ERR PFX "%s: Unable to request regions\n",
+			pci_name(pcidev));
+		goto bail1;
+	}
+
+	if ((sizeof(dma_addr_t) > 4)) {
+		ret = pci_set_dma_mask(pcidev, DMA_64BIT_MASK);
+		if (ret < 0) {
+			printk(KERN_ERR PFX "64b DMA configuration failed\n");
+			goto bail2;
+		}
+	} else {
+		ret = pci_set_dma_mask(pcidev, DMA_32BIT_MASK);
+		if (ret < 0) {
+			printk(KERN_ERR PFX "32b DMA configuration failed\n");
+			goto bail2;
+		}
+	}
+
+	/* Enables bus-mastering on the device */
+	pci_set_master(pcidev);
+
+	/* Remap the adapter PCI registers in BAR4 */
+	mmio_regs = ioremap_nocache(reg4_start + C2_PCI_REGS_OFFSET,
+				    sizeof(struct c2_adapter_pci_regs));
+	if (mmio_regs == 0UL) {
+		printk(KERN_ERR PFX
+			"Unable to remap adapter PCI registers in BAR4\n");
+		ret = -EIO;
+		goto bail2;
+	}
+
+	/* Validate PCI regs magic */
+	for (i = 0; i < sizeof(c2_magic); i++) {
+		if (c2_magic[i] != readb(mmio_regs + C2_REGS_MAGIC + i)) {
+			printk(KERN_ERR PFX "Downlevel Firmware boot loader "
+				"[%d/%Zd: got 0x%x, exp 0x%x]. Use the cc_flash "
+			       "utility to update your boot loader\n",
+				i + 1, sizeof(c2_magic),
+				readb(mmio_regs + C2_REGS_MAGIC + i),
+				c2_magic[i]);
+			printk(KERN_ERR PFX "Adapter not claimed\n");
+			iounmap(mmio_regs);
+			ret = -EIO;
+			goto bail2;
+		}
+	}
+
+	/* Validate the adapter version */
+	if (be32_to_cpu(readl(mmio_regs + C2_REGS_VERS)) != C2_VERSION) {
+		printk(KERN_ERR PFX "Version mismatch "
+			"[fw=%u, c2=%u], Adapter not claimed\n",
+			be32_to_cpu(readl(mmio_regs + C2_REGS_VERS)),
+			C2_VERSION);
+		ret = -EINVAL;
+		iounmap(mmio_regs);
+		goto bail2;
+	}
+
+	/* Validate the adapter IVN */
+	if (be32_to_cpu(readl(mmio_regs + C2_REGS_IVN)) != C2_IVN) {
+		printk(KERN_ERR PFX "Downlevel FIrmware level. You should be using "
+		       "the OpenIB device support kit. "
+		       "[fw=0x%x, c2=0x%x], Adapter not claimed\n",
+			be32_to_cpu(readl(mmio_regs + C2_REGS_IVN)),
+			C2_IVN);
+		ret = -EINVAL;
+		iounmap(mmio_regs);
+		goto bail2;
+	}
+
+	/* Allocate hardware structure */
+	c2dev = (struct c2_dev *) ib_alloc_device(sizeof(*c2dev));
+	if (!c2dev) {
+		printk(KERN_ERR PFX "%s: Unable to alloc hardware struct\n",
+			pci_name(pcidev));
+		ret = -ENOMEM;
+		iounmap(mmio_regs);
+		goto bail2;
+	}
+
+	memset(c2dev, 0, sizeof(*c2dev));
+	spin_lock_init(&c2dev->lock);
+	c2dev->pcidev = pcidev;
+	c2dev->cur_tx = 0;
+
+	/* Get the last RX index */
+	c2dev->cur_rx =
+	    (be32_to_cpu(readl(mmio_regs + C2_REGS_HRX_CUR)) -
+	     0xffffc000) / sizeof(struct c2_rxp_desc);
+
+	/* Request an interrupt line for the driver */
+	ret = request_irq(pcidev->irq, c2_interrupt, SA_SHIRQ, DRV_NAME, c2dev);
+	if (ret) {
+		printk(KERN_ERR PFX "%s: requested IRQ %u is busy\n",
+			pci_name(pcidev), pcidev->irq);
+		iounmap(mmio_regs);
+		goto bail3;
+	}
+
+	/* Set driver specific data */
+	pci_set_drvdata(pcidev, c2dev);
+
+	/* Initialize network device */
+	if ((netdev = c2_devinit(c2dev, mmio_regs)) == NULL) {
+		iounmap(mmio_regs);
+		goto bail4;
+	}
+
+	/* Save off the actual size prior to unmapping mmio_regs */
+	kva_map_size = be32_to_cpu(readl(mmio_regs + C2_REGS_PCI_WINSIZE));
+
+	/* Unmap the adapter PCI registers in BAR4 */
+	iounmap(mmio_regs);
+
+	/* Register network device */
+	ret = register_netdev(netdev);
+	if (ret) {
+		printk(KERN_ERR PFX "Unable to register netdev, ret = %d\n",
+			ret);
+		goto bail5;
+	}
+
+	/* Disable network packets */
+	netif_stop_queue(netdev);
+
+	/* Remap the adapter HRXDQ PA space to kernel VA space */
+	c2dev->mmio_rxp_ring = ioremap_nocache(reg4_start + C2_RXP_HRXDQ_OFFSET,
+					       C2_RXP_HRXDQ_SIZE);
+	if (c2dev->mmio_rxp_ring == 0UL) {
+		printk(KERN_ERR PFX "Unable to remap MMIO HRXDQ region\n");
+		ret = -EIO;
+		goto bail6;
+	}
+
+	/* Remap the adapter HTXDQ PA space to kernel VA space */
+	c2dev->mmio_txp_ring = ioremap_nocache(reg4_start + C2_TXP_HTXDQ_OFFSET,
+					       C2_TXP_HTXDQ_SIZE);
+	if (c2dev->mmio_txp_ring == 0UL) {
+		printk(KERN_ERR PFX "Unable to remap MMIO HTXDQ region\n");
+		ret = -EIO;
+		goto bail7;
+	}
+
+	/* Save off the current RX index in the last 4 bytes of the TXP Ring */
+	C2_SET_CUR_RX(c2dev, c2dev->cur_rx);
+
+	/* Remap the PCI registers in adapter BAR0 to kernel VA space */
+	c2dev->regs = ioremap_nocache(reg0_start, reg0_len);
+	if (c2dev->regs == 0UL) {
+		printk(KERN_ERR PFX "Unable to remap BAR0\n");
+		ret = -EIO;
+		goto bail8;
+	}
+
+	/* Remap the PCI registers in adapter BAR4 to kernel VA space */
+	c2dev->pa = reg4_start + C2_PCI_REGS_OFFSET;
+	c2dev->kva = ioremap_nocache(reg4_start + C2_PCI_REGS_OFFSET,
+				     kva_map_size);
+	if (c2dev->kva == 0UL) {
+		printk(KERN_ERR PFX "Unable to remap BAR4\n");
+		ret = -EIO;
+		goto bail9;
+	}
+
+	/* Print out the MAC address */
+	c2_print_macaddr(netdev);
+
+	ret = c2_rnic_init(c2dev);
+	if (ret) {
+		printk(KERN_ERR PFX "c2_rnic_init failed: %d\n", ret);
+		goto bail10;
+	}
+
+	c2_register_device(c2dev);
+
+	return 0;
+
+ bail10:
+	iounmap(c2dev->kva);
+
+ bail9:
+	iounmap(c2dev->regs);
+
+ bail8:
+	iounmap(c2dev->mmio_txp_ring);
+
+ bail7:
+	iounmap(c2dev->mmio_rxp_ring);
+
+ bail6:
+	unregister_netdev(netdev);
+
+ bail5:
+	free_netdev(netdev);
+
+ bail4:
+	free_irq(pcidev->irq, c2dev);
+
+ bail3:
+	ib_dealloc_device(&c2dev->ibdev);
+
+ bail2:
+	pci_release_regions(pcidev);
+
+ bail1:
+	pci_disable_device(pcidev);
+
+ bail0:
+	return ret;
+}
+
+static void __devexit c2_remove(struct pci_dev *pcidev)
+{
+	struct c2_dev *c2dev = pci_get_drvdata(pcidev);
+	struct net_device *netdev = c2dev->netdev;
+
+	/* Unregister with OpenIB */
+	c2_unregister_device(c2dev);
+
+	/* Clean up the RNIC resources */
+	c2_rnic_term(c2dev);
+
+	/* Remove network device from the kernel */
+	unregister_netdev(netdev);
+
+	/* Free network device */
+	free_netdev(netdev);
+
+	/* Free the interrupt line */
+	free_irq(pcidev->irq, c2dev);
+
+	/* missing: Turn LEDs off here */
+
+	/* Unmap adapter PA space */
+	iounmap(c2dev->kva);
+	iounmap(c2dev->regs);
+	iounmap(c2dev->mmio_txp_ring);
+	iounmap(c2dev->mmio_rxp_ring);
+
+	/* Free the hardware structure */
+	ib_dealloc_device(&c2dev->ibdev);
+
+	/* Release reserved PCI I/O and memory resources */
+	pci_release_regions(pcidev);
+
+	/* Disable PCI device */
+	pci_disable_device(pcidev);
+
+	/* Clear driver specific data */
+	pci_set_drvdata(pcidev, NULL);
+}
+
+static struct pci_driver c2_pci_driver = {
+	.name = DRV_NAME,
+	.id_table = c2_pci_table,
+	.probe = c2_probe,
+	.remove = __devexit_p(c2_remove),
+};
+
+static int __init c2_init_module(void)
+{
+	return pci_module_init(&c2_pci_driver);
+}
+
+static void __exit c2_exit_module(void)
+{
+	pci_unregister_driver(&c2_pci_driver);
+}
+
+module_init(c2_init_module);
+module_exit(c2_exit_module);
diff --git a/drivers/infiniband/hw/amso1100/c2.h b/drivers/infiniband/hw/amso1100/c2.h
new file mode 100644
index 0000000..1b17dcd
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2.h
@@ -0,0 +1,551 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef __C2_H
+#define __C2_H
+
+#include <linux/netdevice.h>
+#include <linux/spinlock.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/dma-mapping.h>
+#include <linux/idr.h>
+#include <asm/semaphore.h>
+
+#include "c2_provider.h"
+#include "c2_mq.h"
+#include "c2_status.h"
+
+#define DRV_NAME     "c2"
+#define DRV_VERSION  "1.1"
+#define PFX          DRV_NAME ": "
+
+#define BAR_0                0
+#define BAR_2                2
+#define BAR_4                4
+
+#define RX_BUF_SIZE         (1536 + 8)
+#define ETH_JUMBO_MTU        9000
+#define C2_MAGIC            "CEPHEUS"
+#define C2_VERSION           4
+#define C2_IVN              (18 & 0x7fffffff)
+
+#define C2_REG0_SIZE        (16 * 1024)
+#define C2_REG2_SIZE        (2 * 1024 * 1024)
+#define C2_REG4_SIZE        (256 * 1024 * 1024)
+#define C2_NUM_TX_DESC       341
+#define C2_NUM_RX_DESC       256
+#define C2_PCI_REGS_OFFSET  (0x10000)
+#define C2_RXP_HRXDQ_OFFSET (((C2_REG4_SIZE)/2))
+#define C2_RXP_HRXDQ_SIZE   (4096)
+#define C2_TXP_HTXDQ_OFFSET (((C2_REG4_SIZE)/2) + C2_RXP_HRXDQ_SIZE)
+#define C2_TXP_HTXDQ_SIZE   (4096)
+#define C2_TX_TIMEOUT	    (6*HZ)
+
+/* CEPHEUS */
+static const u8 c2_magic[] = {
+	0x43, 0x45, 0x50, 0x48, 0x45, 0x55, 0x53
+};
+
+enum adapter_pci_regs {
+	C2_REGS_MAGIC = 0x0000,
+	C2_REGS_VERS = 0x0008,
+	C2_REGS_IVN = 0x000C,
+	C2_REGS_PCI_WINSIZE = 0x0010,
+	C2_REGS_Q0_QSIZE = 0x0014,
+	C2_REGS_Q0_MSGSIZE = 0x0018,
+	C2_REGS_Q0_POOLSTART = 0x001C,
+	C2_REGS_Q0_SHARED = 0x0020,
+	C2_REGS_Q1_QSIZE = 0x0024,
+	C2_REGS_Q1_MSGSIZE = 0x0028,
+	C2_REGS_Q1_SHARED = 0x0030,
+	C2_REGS_Q2_QSIZE = 0x0034,
+	C2_REGS_Q2_MSGSIZE = 0x0038,
+	C2_REGS_Q2_SHARED = 0x0040,
+	C2_REGS_ENADDR = 0x004C,
+	C2_REGS_RDMA_ENADDR = 0x0054,
+	C2_REGS_HRX_CUR = 0x006C,
+};
+
+struct c2_adapter_pci_regs {
+	char reg_magic[8];
+	u32 version;
+	u32 ivn;
+	u32 pci_window_size;
+	u32 q0_q_size;
+	u32 q0_msg_size;
+	u32 q0_pool_start;
+	u32 q0_shared;
+	u32 q1_q_size;
+	u32 q1_msg_size;
+	u32 q1_pool_start;
+	u32 q1_shared;
+	u32 q2_q_size;
+	u32 q2_msg_size;
+	u32 q2_pool_start;
+	u32 q2_shared;
+	u32 log_start;
+	u32 log_size;
+	u8 host_enaddr[8];
+	u8 rdma_enaddr[8];
+	u32 crash_entry;
+	u32 crash_ready[2];
+	u32 fw_txd_cur;
+	u32 fw_hrxd_cur;
+	u32 fw_rxd_cur;
+};
+
+enum pci_regs {
+	C2_HISR = 0x0000,
+	C2_DISR = 0x0004,
+	C2_HIMR = 0x0008,
+	C2_DIMR = 0x000C,
+	C2_NISR0 = 0x0010,
+	C2_NISR1 = 0x0014,
+	C2_NIMR0 = 0x0018,
+	C2_NIMR1 = 0x001C,
+	C2_IDIS = 0x0020,
+};
+
+enum {
+	C2_PCI_HRX_INT = 1 << 8,
+	C2_PCI_HTX_INT = 1 << 17,
+	C2_PCI_HRX_QUI = 1 << 31,
+};
+
+/*
+ * Cepheus registers in BAR0.
+ */
+struct c2_pci_regs {
+	u32 hostisr;
+	u32 dmaisr;
+	u32 hostimr;
+	u32 dmaimr;
+	u32 netisr0;
+	u32 netisr1;
+	u32 netimr0;
+	u32 netimr1;
+	u32 int_disable;
+};
+
+/* TXP flags */
+enum c2_txp_flags {
+	TXP_HTXD_DONE = 0,
+	TXP_HTXD_READY = 1 << 0,
+	TXP_HTXD_UNINIT = 1 << 1,
+};
+
+/* RXP flags */
+enum c2_rxp_flags {
+	RXP_HRXD_UNINIT = 0,
+	RXP_HRXD_READY = 1 << 0,
+	RXP_HRXD_DONE = 1 << 1,
+};
+
+/* RXP status */
+enum c2_rxp_status {
+	RXP_HRXD_ZERO = 0,
+	RXP_HRXD_OK = 1 << 0,
+	RXP_HRXD_BUF_OV = 1 << 1,
+};
+
+/* TXP descriptor fields */
+enum txp_desc {
+	C2_TXP_FLAGS = 0x0000,
+	C2_TXP_LEN = 0x0002,
+	C2_TXP_ADDR = 0x0004,
+};
+
+/* RXP descriptor fields */
+enum rxp_desc {
+	C2_RXP_FLAGS = 0x0000,
+	C2_RXP_STATUS = 0x0002,
+	C2_RXP_COUNT = 0x0004,
+	C2_RXP_LEN = 0x0006,
+	C2_RXP_ADDR = 0x0008,
+};
+
+struct c2_txp_desc {
+	u16 flags;
+	u16 len;
+	u64 addr;
+} __attribute__ ((packed));
+
+struct c2_rxp_desc {
+	u16 flags;
+	u16 status;
+	u16 count;
+	u16 len;
+	u64 addr;
+} __attribute__ ((packed));
+
+struct c2_rxp_hdr {
+	u16 flags;
+	u16 status;
+	u16 len;
+	u16 rsvd;
+} __attribute__ ((packed));
+
+struct c2_tx_desc {
+	u32 len;
+	u32 status;
+	dma_addr_t next_offset;
+};
+
+struct c2_rx_desc {
+	u32 len;
+	u32 status;
+	dma_addr_t next_offset;
+};
+
+struct c2_alloc {
+	u32 last;
+	u32 max;
+	spinlock_t lock;
+	unsigned long *table;
+};
+
+struct c2_array {
+	struct {
+		void **page;
+		int used;
+	} *page_list;
+};
+
+/*
+ * The MQ shared pointer pool is organized as a linked list of
+ * chunks. Each chunk contains a linked list of free shared pointers
+ * that can be allocated to a given user mode client.
+ *
+ */
+struct sp_chunk {
+	struct sp_chunk *next;
+	dma_addr_t dma_addr;
+	DECLARE_PCI_UNMAP_ADDR(mapping);
+	u16 head;
+	u16 shared_ptr[0];
+};
+
+struct c2_pd_table {
+	u32 last;
+	u32 max;
+	spinlock_t lock;
+	unsigned long *table;
+};
+
+struct c2_qp_table {
+	struct idr idr;
+	spinlock_t lock;
+	int last;
+};
+
+struct c2_element {
+	struct c2_element *next;
+	void *ht_desc;		/* host     descriptor */
+	void __iomem *hw_desc;	/* hardware descriptor */
+	struct sk_buff *skb;
+	dma_addr_t mapaddr;
+	u32 maplen;
+};
+
+struct c2_ring {
+	struct c2_element *to_clean;
+	struct c2_element *to_use;
+	struct c2_element *start;
+	unsigned long count;
+};
+
+struct c2_dev {
+	struct ib_device ibdev;
+	void __iomem *regs;
+	void __iomem *mmio_txp_ring; /* remapped adapter memory for hw rings */
+	void __iomem *mmio_rxp_ring;
+	spinlock_t lock;
+	struct pci_dev *pcidev;
+	struct net_device *netdev;
+	struct net_device *pseudo_netdev;
+	unsigned int cur_tx;
+	unsigned int cur_rx;
+	u32 adapter_handle;
+	int device_cap_flags;
+	void __iomem *kva;	/* KVA device memory */
+	unsigned long pa;	/* PA device memory */
+	void **qptr_array;
+
+	kmem_cache_t *host_msg_cache;
+
+	struct list_head cca_link;		/* adapter list */
+	struct list_head eh_wakeup_list;	/* event wakeup list */
+	wait_queue_head_t req_vq_wo;
+
+	/* Cached RNIC properties */
+	struct ib_device_attr props;
+
+	struct c2_pd_table pd_table;
+	struct c2_qp_table qp_table;
+	int ports;		/* num of GigE ports */
+	int devnum;
+	spinlock_t vqlock;	/* sync vbs req MQ */
+
+	/* Verbs Queues */
+	struct c2_mq req_vq;	/* Verbs Request MQ */
+	struct c2_mq rep_vq;	/* Verbs Reply MQ */
+	struct c2_mq aeq;	/* Async Events MQ */
+
+	/* Kernel client MQs */
+	struct sp_chunk *kern_mqsp_pool;
+
+	/* Device updates these values when posting messages to a host
+	 * target queue */
+	u16 req_vq_shared;
+	u16 rep_vq_shared;
+	u16 aeq_shared;
+	u16 irq_claimed;
+
+	/*
+	 * Shared host target pages for user-accessible MQs.
+	 */
+	int hthead;		/* index of first free entry */
+	void *htpages;		/* kernel vaddr */
+	int htlen;		/* length of htpages memory */
+	void *htuva;		/* user mapped vaddr */
+	spinlock_t htlock;	/* serialize allocation */
+
+	u64 adapter_hint_uva;	/* access to the activity FIFO */
+
+	//	spinlock_t aeq_lock;
+	//	spinlock_t rnic_lock;
+
+	u16 *hint_count;
+	dma_addr_t hint_count_dma;
+	u16 hints_read;
+
+	int init;		/* TRUE if it's ready */
+	char ae_cache_name[16];
+	char vq_cache_name[16];
+};
+
+struct c2_port {
+	u32 msg_enable;
+	struct c2_dev *c2dev;
+	struct net_device *netdev;
+
+	spinlock_t tx_lock;
+	u32 tx_avail;
+	struct c2_ring tx_ring;
+	struct c2_ring rx_ring;
+
+	void *mem;		/* PCI memory for host rings */
+	dma_addr_t dma;
+	unsigned long mem_size;
+
+	u32 rx_buf_size;
+
+	struct net_device_stats netstats;
+};
+
+/*
+ * Activity FIFO registers in BAR0.
+ */
+#define PCI_BAR0_HOST_HINT	0x100
+#define PCI_BAR0_ADAPTER_HINT	0x2000
+
+/*
+ * Ammasso PCI vendor id and Cepheus PCI device id.
+ */
+#define CQ_ARMED 	0x01
+#define CQ_WAIT_FOR_DMA	0x80
+
+/*
+ * The format of a hint is as follows:
+ * Lower 16 bits are the count of hints for the queue.
+ * Next 15 bits are the qp_index
+ * Upper most bit depends on who reads it:
+ *    If read by producer, then it means Full (1) or Not-Full (0)
+ *    If read by consumer, then it means Empty (1) or Not-Empty (0)
+ */
+#define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
+#define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
+#define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
+
+
+/*
+ * The following defines the offset in SDRAM for the c2_adapter_pci_regs_t
+ * struct.
+ */
+#define C2_ADAPTER_PCI_REGS_OFFSET 0x10000
+
+#ifndef readq
+static inline u64 readq(const void __iomem * addr)
+{
+	u64 ret = readl(addr + 4);
+	ret <<= 32;
+	ret |= readl(addr);
+
+	return ret;
+}
+#endif
+
+#ifndef writeq
+static inline void __raw_writeq(u64 val, void __iomem * addr)
+{
+	__raw_writel((u32) (val), addr);
+	__raw_writel((u32) (val >> 32), (addr + 4));
+}
+#endif
+
+#define C2_SET_CUR_RX(c2dev, cur_rx) \
+	__raw_writel(cpu_to_be32(cur_rx), c2dev->mmio_txp_ring + 4092)
+
+#define C2_GET_CUR_RX(c2dev) \
+	be32_to_cpu(readl(c2dev->mmio_txp_ring + 4092))
+
+static inline struct c2_dev *to_c2dev(struct ib_device *ibdev)
+{
+	return container_of(ibdev, struct c2_dev, ibdev);
+}
+
+static inline int c2_errno(void *reply)
+{
+	switch (c2_wr_get_result(reply)) {
+	case C2_OK:
+		return 0;
+	case CCERR_NO_BUFS:
+	case CCERR_INSUFFICIENT_RESOURCES:
+	case CCERR_ZERO_RDMA_READ_RESOURCES:
+		return -ENOMEM;
+	case CCERR_MR_IN_USE:
+	case CCERR_QP_IN_USE:
+		return -EBUSY;
+	case CCERR_ADDR_IN_USE:
+		return -EADDRINUSE;
+	case CCERR_ADDR_NOT_AVAIL:
+		return -EADDRNOTAVAIL;
+	case CCERR_CONN_RESET:
+		return -ECONNRESET;
+	case CCERR_NOT_IMPLEMENTED:
+	case CCERR_INVALID_WQE:
+		return -ENOSYS;
+	case CCERR_QP_NOT_PRIVILEGED:
+		return -EPERM;
+	case CCERR_STACK_ERROR:
+		return -EPROTO;
+	case CCERR_ACCESS_VIOLATION:
+	case CCERR_BASE_AND_BOUNDS_VIOLATION:
+		return -EFAULT;
+	case CCERR_STAG_STATE_NOT_INVALID:
+	case CCERR_INVALID_ADDRESS:
+	case CCERR_INVALID_CQ:
+	case CCERR_INVALID_EP:
+	case CCERR_INVALID_MODIFIER:
+	case CCERR_INVALID_MTU:
+	case CCERR_INVALID_PD_ID:
+	case CCERR_INVALID_QP:
+	case CCERR_INVALID_RNIC:
+	case CCERR_INVALID_STAG:
+		return -EINVAL;
+	default:
+		return -EAGAIN;
+	}
+}
+
+/* Device */
+extern int c2_register_device(struct c2_dev *c2dev);
+extern void c2_unregister_device(struct c2_dev *c2dev);
+extern int c2_rnic_init(struct c2_dev *c2dev);
+extern void c2_rnic_term(struct c2_dev *c2dev);
+extern void c2_rnic_interrupt(struct c2_dev *c2dev);
+extern int c2_del_addr(struct c2_dev *c2dev, u32 inaddr, u32 inmask);
+extern int c2_add_addr(struct c2_dev *c2dev, u32 inaddr, u32 inmask);
+
+/* QPs */
+extern int c2_alloc_qp(struct c2_dev *c2dev, struct c2_pd *pd,
+		       struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp);
+extern void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp);
+extern struct ib_qp *c2_get_qp(struct ib_device *device, int qpn);
+extern int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
+			struct ib_qp_attr *attr, int attr_mask);
+extern int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
+				 int ord, int ird);
+extern int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
+			struct ib_send_wr **bad_wr);
+extern int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
+			   struct ib_recv_wr **bad_wr);
+extern void __devinit c2_init_qp_table(struct c2_dev *c2dev);
+extern void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev);
+extern void c2_set_qp_state(struct c2_qp *, int);
+extern struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn);
+
+/* PDs */
+extern int c2_pd_alloc(struct c2_dev *c2dev, int privileged, struct c2_pd *pd);
+extern void c2_pd_free(struct c2_dev *c2dev, struct c2_pd *pd);
+extern int __devinit c2_init_pd_table(struct c2_dev *c2dev);
+extern void __devexit c2_cleanup_pd_table(struct c2_dev *c2dev);
+
+/* CQs */
+extern int c2_init_cq(struct c2_dev *c2dev, int entries,
+		      struct c2_ucontext *ctx, struct c2_cq *cq);
+extern void c2_free_cq(struct c2_dev *c2dev, struct c2_cq *cq);
+extern void c2_cq_event(struct c2_dev *c2dev, u32 mq_index);
+extern void c2_cq_clean(struct c2_dev *c2dev, struct c2_qp *qp, u32 mq_index);
+extern int c2_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry);
+extern int c2_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify);
+
+/* CM */
+extern int c2_llp_connect(struct iw_cm_id *cm_id,
+			  struct iw_cm_conn_param *iw_param);
+extern int c2_llp_accept(struct iw_cm_id *cm_id,
+			 struct iw_cm_conn_param *iw_param);
+extern int c2_llp_reject(struct iw_cm_id *cm_id, const void *pdata,
+			 u8 pdata_len);
+extern int c2_llp_service_create(struct iw_cm_id *cm_id, int backlog);
+extern int c2_llp_service_destroy(struct iw_cm_id *cm_id);
+
+/* MM */
+extern int c2_nsmr_register_phys_kern(struct c2_dev *c2dev, u64 *addr_list,
+ 				      int page_size, int pbl_depth, u32 length,
+ 				      u32 off, u64 *va, enum c2_acf acf,
+				      struct c2_mr *mr);
+extern int c2_stag_dealloc(struct c2_dev *c2dev, u32 stag_index);
+
+/* AE */
+extern void c2_ae_event(struct c2_dev *c2dev, u32 mq_index);
+
+/* MQSP Allocator */
+extern int c2_init_mqsp_pool(struct c2_dev *c2dev, gfp_t gfp_mask,
+			     struct sp_chunk **root);
+extern void c2_free_mqsp_pool(struct c2_dev *c2dev, struct sp_chunk *root);
+extern u16 *c2_alloc_mqsp(struct c2_dev *c2dev, struct sp_chunk *head,
+			  dma_addr_t *dma_addr, gfp_t gfp_mask);
+extern void c2_free_mqsp(u16 * mqsp);
+#endif
diff --git a/drivers/infiniband/hw/amso1100/c2_ae.c b/drivers/infiniband/hw/amso1100/c2_ae.c
new file mode 100644
index 0000000..08f46c8
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_ae.c
@@ -0,0 +1,321 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "c2.h"
+#include <rdma/iw_cm.h>
+#include "c2_status.h"
+#include "c2_ae.h"
+
+static int c2_convert_cm_status(u32 c2_status)
+{
+	switch (c2_status) {
+	case C2_CONN_STATUS_SUCCESS:
+		return 0;
+	case C2_CONN_STATUS_REJECTED:
+		return -ENETRESET;
+	case C2_CONN_STATUS_REFUSED:
+		return -ECONNREFUSED;
+	case C2_CONN_STATUS_TIMEDOUT:
+		return -ETIMEDOUT;
+	case C2_CONN_STATUS_NETUNREACH:
+		return -ENETUNREACH;
+	case C2_CONN_STATUS_HOSTUNREACH:
+		return -EHOSTUNREACH;
+	case C2_CONN_STATUS_INVALID_RNIC:
+		return -EINVAL;
+	case C2_CONN_STATUS_INVALID_QP:
+		return -EINVAL;
+	case C2_CONN_STATUS_INVALID_QP_STATE:
+		return -EINVAL;
+	case C2_CONN_STATUS_ADDR_NOT_AVAIL:
+		return -EADDRNOTAVAIL;
+	default:
+		printk(KERN_ERR PFX
+		       "%s - Unable to convert CM status: %d\n",
+		       __FUNCTION__, c2_status);
+		return -EIO;
+	}
+}
+
+#ifdef DEBUG
+static const char* to_event_str(int event)
+{
+	static const char* event_str[] = {
+		"CCAE_REMOTE_SHUTDOWN",
+		"CCAE_ACTIVE_CONNECT_RESULTS",
+		"CCAE_CONNECTION_REQUEST",
+		"CCAE_LLP_CLOSE_COMPLETE",
+		"CCAE_TERMINATE_MESSAGE_RECEIVED",
+		"CCAE_LLP_CONNECTION_RESET",
+		"CCAE_LLP_CONNECTION_LOST",
+		"CCAE_LLP_SEGMENT_SIZE_INVALID",
+		"CCAE_LLP_INVALID_CRC",
+		"CCAE_LLP_BAD_FPDU",
+		"CCAE_INVALID_DDP_VERSION",
+		"CCAE_INVALID_RDMA_VERSION",
+		"CCAE_UNEXPECTED_OPCODE",
+		"CCAE_INVALID_DDP_QUEUE_NUMBER",
+		"CCAE_RDMA_READ_NOT_ENABLED",
+		"CCAE_RDMA_WRITE_NOT_ENABLED",
+		"CCAE_RDMA_READ_TOO_SMALL",
+		"CCAE_NO_L_BIT",
+		"CCAE_TAGGED_INVALID_STAG",
+		"CCAE_TAGGED_BASE_BOUNDS_VIOLATION",
+		"CCAE_TAGGED_ACCESS_RIGHTS_VIOLATION",
+		"CCAE_TAGGED_INVALID_PD",
+		"CCAE_WRAP_ERROR",
+		"CCAE_BAD_CLOSE",
+		"CCAE_BAD_LLP_CLOSE",
+		"CCAE_INVALID_MSN_RANGE",
+		"CCAE_INVALID_MSN_GAP",
+		"CCAE_IRRQ_OVERFLOW",
+		"CCAE_IRRQ_MSN_GAP",
+		"CCAE_IRRQ_MSN_RANGE",
+		"CCAE_IRRQ_INVALID_STAG",
+		"CCAE_IRRQ_BASE_BOUNDS_VIOLATION",
+		"CCAE_IRRQ_ACCESS_RIGHTS_VIOLATION",
+		"CCAE_IRRQ_INVALID_PD",
+		"CCAE_IRRQ_WRAP_ERROR",
+		"CCAE_CQ_SQ_COMPLETION_OVERFLOW",
+		"CCAE_CQ_RQ_COMPLETION_ERROR",
+		"CCAE_QP_SRQ_WQE_ERROR",
+		"CCAE_QP_LOCAL_CATASTROPHIC_ERROR",
+		"CCAE_CQ_OVERFLOW",
+		"CCAE_CQ_OPERATION_ERROR",
+		"CCAE_SRQ_LIMIT_REACHED",
+		"CCAE_QP_RQ_LIMIT_REACHED",
+		"CCAE_SRQ_CATASTROPHIC_ERROR",
+		"CCAE_RNIC_CATASTROPHIC_ERROR"
+	};
+
+	if (event < CCAE_REMOTE_SHUTDOWN ||
+	    event > CCAE_RNIC_CATASTROPHIC_ERROR)
+		return "<invalid event>";
+
+	event -= CCAE_REMOTE_SHUTDOWN;
+	return event_str[event];
+}
+
+static const char *to_qp_state_str(int state)
+{
+	switch (state) {
+	case C2_QP_STATE_IDLE:
+		return "C2_QP_STATE_IDLE";
+	case C2_QP_STATE_CONNECTING:
+		return "C2_QP_STATE_CONNECTING";
+	case C2_QP_STATE_RTS:
+		return "C2_QP_STATE_RTS";
+	case C2_QP_STATE_CLOSING:
+		return "C2_QP_STATE_CLOSING";
+	case C2_QP_STATE_TERMINATE:
+		return "C2_QP_STATE_TERMINATE";
+	case C2_QP_STATE_ERROR:
+		return "C2_QP_STATE_ERROR";
+	default:
+		return "<invalid QP state>";
+	};
+}
+#endif
+
+void c2_ae_event(struct c2_dev *c2dev, u32 mq_index)
+{
+	struct c2_mq *mq = c2dev->qptr_array[mq_index];
+	union c2wr *wr;
+	void *resource_user_context;
+	struct iw_cm_event cm_event;
+	struct ib_event ib_event;
+	enum c2_resource_indicator resource_indicator;
+	enum c2_event_id event_id;
+	unsigned long flags;
+	int status;
+
+	/*
+	 * retreive the message
+	 */
+	wr = c2_mq_consume(mq);
+	if (!wr)
+		return;
+
+	memset(&ib_event, 0, sizeof(ib_event));
+	memset(&cm_event, 0, sizeof(cm_event));
+
+	event_id = c2_wr_get_id(wr);
+	resource_indicator = be32_to_cpu(wr->ae.ae_generic.resource_type);
+	resource_user_context =
+	    (void *) (unsigned long) wr->ae.ae_generic.user_context;
+
+	status = cm_event.status = c2_convert_cm_status(c2_wr_get_result(wr));
+
+	pr_debug("event received c2_dev=%p, event_id=%d, "
+		"resource_indicator=%d, user_context=%p, status = %d\n",
+		c2dev, event_id, resource_indicator, resource_user_context,
+		status);
+
+	switch (resource_indicator) {
+	case C2_RES_IND_QP:{
+
+		struct c2_qp *qp = (struct c2_qp *)resource_user_context;
+		struct iw_cm_id *cm_id = qp->cm_id;
+		struct c2wr_ae_active_connect_results *res;
+
+		if (!cm_id) {
+			pr_debug("event received, but cm_id is <nul>, qp=%p!\n",
+				qp);
+			goto ignore_it;
+		}
+		pr_debug("%s: event = %s, user_context=%llx, "
+			"resource_type=%x, "
+			"resource=%x, qp_state=%s\n",
+			__FUNCTION__,
+			to_event_str(event_id),
+			be64_to_cpu(wr->ae.ae_generic.user_context),
+			be32_to_cpu(wr->ae.ae_generic.resource_type),
+			be32_to_cpu(wr->ae.ae_generic.resource),
+			to_qp_state_str(be32_to_cpu(wr->ae.ae_generic.qp_state)));
+
+		c2_set_qp_state(qp, be32_to_cpu(wr->ae.ae_generic.qp_state));
+
+		switch (event_id) {
+		case CCAE_ACTIVE_CONNECT_RESULTS:
+			res = &wr->ae.ae_active_connect_results;
+			cm_event.event = IW_CM_EVENT_CONNECT_REPLY;
+			cm_event.local_addr.sin_addr.s_addr = res->laddr;
+			cm_event.remote_addr.sin_addr.s_addr = res->raddr;
+			cm_event.local_addr.sin_port = res->lport;
+			cm_event.remote_addr.sin_port =	res->rport;
+			if (status == 0) {
+				cm_event.private_data_len =
+					be32_to_cpu(res->private_data_length);
+				cm_event.private_data = res->private_data;
+			} else {
+				spin_lock_irqsave(&qp->lock, flags);
+				if (qp->cm_id) {
+					qp->cm_id->rem_ref(qp->cm_id);
+					qp->cm_id = NULL;
+				}
+				spin_unlock_irqrestore(&qp->lock, flags);
+				cm_event.private_data_len = 0;
+				cm_event.private_data = NULL;
+			}
+			if (cm_id->event_handler)
+				cm_id->event_handler(cm_id, &cm_event);
+			break;
+		case CCAE_TERMINATE_MESSAGE_RECEIVED:
+		case CCAE_CQ_SQ_COMPLETION_OVERFLOW:
+			ib_event.device = &c2dev->ibdev;
+			ib_event.element.qp = &qp->ibqp;
+			ib_event.event = IB_EVENT_QP_REQ_ERR;
+
+			if (qp->ibqp.event_handler)
+				qp->ibqp.event_handler(&ib_event,
+						       qp->ibqp.
+						       qp_context);
+			break;
+		case CCAE_BAD_CLOSE:
+		case CCAE_LLP_CLOSE_COMPLETE:
+		case CCAE_LLP_CONNECTION_RESET:
+		case CCAE_LLP_CONNECTION_LOST:
+			BUG_ON(cm_id->event_handler==(void*)0x6b6b6b6b);
+
+			spin_lock_irqsave(&qp->lock, flags);
+			if (qp->cm_id) {
+				qp->cm_id->rem_ref(qp->cm_id);
+				qp->cm_id = NULL;
+			}
+			spin_unlock_irqrestore(&qp->lock, flags);
+			cm_event.event = IW_CM_EVENT_CLOSE;
+			cm_event.status = 0;
+			if (cm_id->event_handler)
+				cm_id->event_handler(cm_id, &cm_event);
+			break;
+		default:
+			BUG_ON(1);
+			pr_debug("%s:%d Unexpected event_id=%d on QP=%p, "
+				"CM_ID=%p\n",
+				__FUNCTION__, __LINE__,
+				event_id, qp, cm_id);
+			break;
+		}
+		break;
+	}
+
+	case C2_RES_IND_EP:{
+
+		struct c2wr_ae_connection_request *req =
+			&wr->ae.ae_connection_request;
+		struct iw_cm_id *cm_id =
+			(struct iw_cm_id *)resource_user_context;
+
+		pr_debug("C2_RES_IND_EP event_id=%d\n", event_id);
+		if (event_id != CCAE_CONNECTION_REQUEST) {
+			pr_debug("%s: Invalid event_id: %d\n",
+				__FUNCTION__, event_id);
+			break;
+		}
+		cm_event.event = IW_CM_EVENT_CONNECT_REQUEST;
+		cm_event.provider_data = (void*)(unsigned long)req->cr_handle;
+		cm_event.local_addr.sin_addr.s_addr = req->laddr;
+		cm_event.remote_addr.sin_addr.s_addr = req->raddr;
+		cm_event.local_addr.sin_port = req->lport;
+		cm_event.remote_addr.sin_port = req->rport;
+		cm_event.private_data_len =
+			be32_to_cpu(req->private_data_length);
+		cm_event.private_data = req->private_data;
+
+		if (cm_id->event_handler)
+			cm_id->event_handler(cm_id, &cm_event);
+		break;
+	}
+
+	case C2_RES_IND_CQ:{
+		struct c2_cq *cq =
+		    (struct c2_cq *) resource_user_context;
+
+		pr_debug("IB_EVENT_CQ_ERR\n");
+		ib_event.device = &c2dev->ibdev;
+		ib_event.element.cq = &cq->ibcq;
+		ib_event.event = IB_EVENT_CQ_ERR;
+
+		if (cq->ibcq.event_handler)
+			cq->ibcq.event_handler(&ib_event,
+					       cq->ibcq.cq_context);
+	}
+
+	default:
+		printk("Bad resource indicator = %d\n",
+		       resource_indicator);
+		break;
+	}
+
+ ignore_it:
+	c2_mq_free(mq);
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_ae.h b/drivers/infiniband/hw/amso1100/c2_ae.h
new file mode 100644
index 0000000..3a065c3
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_ae.h
@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef _C2_AE_H_
+#define _C2_AE_H_
+
+/*
+ * WARNING: If you change this file, also bump C2_IVN_BASE
+ * in common/include/clustercore/c2_ivn.h.
+ */
+
+/*
+ * Asynchronous Event Identifiers
+ *
+ * These start at 0x80 only so it's obvious from inspection that
+ * they are not work-request statuses.  This isn't critical.
+ *
+ * NOTE: these event id's must fit in eight bits.
+ */
+enum c2_event_id {
+	CCAE_REMOTE_SHUTDOWN = 0x80,
+	CCAE_ACTIVE_CONNECT_RESULTS,
+	CCAE_CONNECTION_REQUEST,
+	CCAE_LLP_CLOSE_COMPLETE,
+	CCAE_TERMINATE_MESSAGE_RECEIVED,
+	CCAE_LLP_CONNECTION_RESET,
+	CCAE_LLP_CONNECTION_LOST,
+	CCAE_LLP_SEGMENT_SIZE_INVALID,
+	CCAE_LLP_INVALID_CRC,
+	CCAE_LLP_BAD_FPDU,
+	CCAE_INVALID_DDP_VERSION,
+	CCAE_INVALID_RDMA_VERSION,
+	CCAE_UNEXPECTED_OPCODE,
+	CCAE_INVALID_DDP_QUEUE_NUMBER,
+	CCAE_RDMA_READ_NOT_ENABLED,
+	CCAE_RDMA_WRITE_NOT_ENABLED,
+	CCAE_RDMA_READ_TOO_SMALL,
+	CCAE_NO_L_BIT,
+	CCAE_TAGGED_INVALID_STAG,
+	CCAE_TAGGED_BASE_BOUNDS_VIOLATION,
+	CCAE_TAGGED_ACCESS_RIGHTS_VIOLATION,
+	CCAE_TAGGED_INVALID_PD,
+	CCAE_WRAP_ERROR,
+	CCAE_BAD_CLOSE,
+	CCAE_BAD_LLP_CLOSE,
+	CCAE_INVALID_MSN_RANGE,
+	CCAE_INVALID_MSN_GAP,
+	CCAE_IRRQ_OVERFLOW,
+	CCAE_IRRQ_MSN_GAP,
+	CCAE_IRRQ_MSN_RANGE,
+	CCAE_IRRQ_INVALID_STAG,
+	CCAE_IRRQ_BASE_BOUNDS_VIOLATION,
+	CCAE_IRRQ_ACCESS_RIGHTS_VIOLATION,
+	CCAE_IRRQ_INVALID_PD,
+	CCAE_IRRQ_WRAP_ERROR,
+	CCAE_CQ_SQ_COMPLETION_OVERFLOW,
+	CCAE_CQ_RQ_COMPLETION_ERROR,
+	CCAE_QP_SRQ_WQE_ERROR,
+	CCAE_QP_LOCAL_CATASTROPHIC_ERROR,
+	CCAE_CQ_OVERFLOW,
+	CCAE_CQ_OPERATION_ERROR,
+	CCAE_SRQ_LIMIT_REACHED,
+	CCAE_QP_RQ_LIMIT_REACHED,
+	CCAE_SRQ_CATASTROPHIC_ERROR,
+	CCAE_RNIC_CATASTROPHIC_ERROR
+/* WARNING If you add more id's, make sure their values fit in eight bits. */
+};
+
+/*
+ * Resource Indicators and Identifiers
+ */
+enum c2_resource_indicator {
+	C2_RES_IND_QP = 1,
+	C2_RES_IND_EP,
+	C2_RES_IND_CQ,
+	C2_RES_IND_SRQ,
+};
+
+#endif /* _C2_AE_H_ */
diff --git a/drivers/infiniband/hw/amso1100/c2_alloc.c b/drivers/infiniband/hw/amso1100/c2_alloc.c
new file mode 100644
index 0000000..1d25299
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_alloc.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/errno.h>
+#include <linux/slab.h>
+#include <linux/bitmap.h>
+
+#include "c2.h"
+
+static int c2_alloc_mqsp_chunk(struct c2_dev *c2dev, gfp_t gfp_mask,
+			       struct sp_chunk **head)
+{
+	int i;
+	struct sp_chunk *new_head;
+
+	new_head = (struct sp_chunk *) __get_free_page(gfp_mask);
+	if (new_head == NULL)
+		return -ENOMEM;
+
+	new_head->dma_addr = dma_map_single(c2dev->ibdev.dma_device, new_head,
+					    PAGE_SIZE, DMA_FROM_DEVICE);
+	pci_unmap_addr_set(new_head, mapping, new_head->dma_addr);
+
+	new_head->next = NULL;
+	new_head->head = 0;
+
+	/* build list where each index is the next free slot */
+	for (i = 0;
+	     i < (PAGE_SIZE - sizeof(struct sp_chunk) -
+		  sizeof(u16)) / sizeof(u16) - 1;
+	     i++) {
+		new_head->shared_ptr[i] = i + 1;
+	}
+	/* terminate list */
+	new_head->shared_ptr[i] = 0xFFFF;
+
+	*head = new_head;
+	return 0;
+}
+
+int c2_init_mqsp_pool(struct c2_dev *c2dev, gfp_t gfp_mask,
+		      struct sp_chunk **root)
+{
+	return c2_alloc_mqsp_chunk(c2dev, gfp_mask, root);
+}
+
+void c2_free_mqsp_pool(struct c2_dev *c2dev, struct sp_chunk *root)
+{
+	struct sp_chunk *next;
+
+	while (root) {
+		next = root->next;
+		dma_unmap_single(c2dev->ibdev.dma_device,
+				 pci_unmap_addr(root, mapping), PAGE_SIZE,
+			         DMA_FROM_DEVICE);
+		__free_page((struct page *) root);
+		root = next;
+	}
+}
+
+u16 *c2_alloc_mqsp(struct c2_dev *c2dev, struct sp_chunk *head,
+		   dma_addr_t *dma_addr, gfp_t gfp_mask)
+{
+	u16 mqsp;
+
+	while (head) {
+		mqsp = head->head;
+		if (mqsp != 0xFFFF) {
+			head->head = head->shared_ptr[mqsp];
+			break;
+		} else if (head->next == NULL) {
+			if (c2_alloc_mqsp_chunk(c2dev, gfp_mask, &head->next) ==
+			    0) {
+				head = head->next;
+				mqsp = head->head;
+				head->head = head->shared_ptr[mqsp];
+				break;
+			} else
+				return NULL;
+		} else
+			head = head->next;
+	}
+	if (head) {
+		*dma_addr = head->dma_addr +
+			    ((unsigned long) &(head->shared_ptr[mqsp]) -
+			     (unsigned long) head);
+		pr_debug("%s addr %p dma_addr %llx\n", __FUNCTION__,
+			 &(head->shared_ptr[mqsp]), (u64)*dma_addr);
+		return &(head->shared_ptr[mqsp]);
+	}
+	return NULL;
+}
+
+void c2_free_mqsp(u16 * mqsp)
+{
+	struct sp_chunk *head;
+	u16 idx;
+
+	/* The chunk containing this ptr begins at the page boundary */
+	head = (struct sp_chunk *) ((unsigned long) mqsp & PAGE_MASK);
+
+	/* Link head to new mqsp */
+	*mqsp = head->head;
+
+	/* Compute the shared_ptr index */
+	idx = ((unsigned long) mqsp & ~PAGE_MASK) >> 1;
+	idx -= (unsigned long) &(((struct sp_chunk *) 0)->shared_ptr[0]) >> 1;
+
+	/* Point this index at the head */
+	head->shared_ptr[idx] = head->head;
+
+	/* Point head at this index */
+	head->head = idx;
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_cm.c b/drivers/infiniband/hw/amso1100/c2_cm.c
new file mode 100644
index 0000000..485254e
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_cm.c
@@ -0,0 +1,452 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc.  All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+#include "c2.h"
+#include "c2_wr.h"
+#include "c2_vq.h"
+#include <rdma/iw_cm.h>
+
+int c2_llp_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
+{
+	struct c2_dev *c2dev = to_c2dev(cm_id->device);
+	struct ib_qp *ibqp;
+	struct c2_qp *qp;
+	struct c2wr_qp_connect_req *wr;	/* variable size needs a malloc. */
+	struct c2_vq_req *vq_req;
+	int err;
+
+	ibqp = c2_get_qp(cm_id->device, iw_param->qpn);
+	if (!ibqp)
+		return -EINVAL;
+	qp = to_c2qp(ibqp);
+
+	/* Associate QP <--> CM_ID */
+	cm_id->provider_data = qp;
+	cm_id->add_ref(cm_id);
+	qp->cm_id = cm_id;
+
+	/*
+	 * only support the max private_data length
+	 */
+	if (iw_param->private_data_len > C2_MAX_PRIVATE_DATA_SIZE) {
+		err = -EINVAL;
+		goto bail0;
+	}
+	/*
+	 * Set the rdma read limits
+	 */
+	err = c2_qp_set_read_limits(c2dev, qp, iw_param->ord, iw_param->ird);
+	if (err)
+		goto bail0;
+
+	/*
+	 * Create and send a WR_QP_CONNECT...
+	 */
+	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
+	if (!wr) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+
+	c2_wr_set_id(wr, CCWR_QP_CONNECT);
+	wr->hdr.context = 0;
+	wr->rnic_handle = c2dev->adapter_handle;
+	wr->qp_handle = qp->adapter_handle;
+
+	wr->remote_addr = cm_id->remote_addr.sin_addr.s_addr;
+	wr->remote_port = cm_id->remote_addr.sin_port;
+
+	/*
+	 * Move any private data from the callers's buf into
+	 * the WR.
+	 */
+	if (iw_param->private_data) {
+		wr->private_data_length =
+			cpu_to_be32(iw_param->private_data_len);
+		memcpy(&wr->private_data[0], iw_param->private_data,
+		       iw_param->private_data_len);
+	} else
+		wr->private_data_length = 0;
+
+	/*
+	 * Send WR to adapter.  NOTE: There is no synch reply from
+	 * the adapter.
+	 */
+	err = vq_send_wr(c2dev, (union c2wr *) wr);
+	vq_req_free(c2dev, vq_req);
+
+ bail1:
+	kfree(wr);
+ bail0:
+	if (err) {
+		/*
+		 * If we fail, release reference on QP and
+		 * disassociate QP from CM_ID
+		 */
+		cm_id->provider_data = NULL;
+		qp->cm_id = NULL;
+		cm_id->rem_ref(cm_id);
+	}
+	return err;
+}
+
+int c2_llp_service_create(struct iw_cm_id *cm_id, int backlog)
+{
+	struct c2_dev *c2dev;
+	struct c2wr_ep_listen_create_req wr;
+	struct c2wr_ep_listen_create_rep *reply;
+	struct c2_vq_req *vq_req;
+	int err;
+
+	c2dev = to_c2dev(cm_id->device);
+	if (c2dev == NULL)
+		return -EINVAL;
+
+	/*
+	 * Allocate verbs request.
+	 */
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req)
+		return -ENOMEM;
+
+	/*
+	 * Build the WR
+	 */
+	c2_wr_set_id(&wr, CCWR_EP_LISTEN_CREATE);
+	wr.hdr.context = (u64) (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.local_addr = cm_id->local_addr.sin_addr.s_addr;
+	wr.local_port = cm_id->local_addr.sin_port;
+	wr.backlog = cpu_to_be32(backlog);
+	wr.user_context = (u64) (unsigned long) cm_id;
+
+	/*
+	 * Reference the request struct.  Dereferenced in the int handler.
+	 */
+	vq_req_get(c2dev, vq_req);
+
+	/*
+	 * Send WR to adapter
+	 */
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail0;
+	}
+
+	/*
+	 * Wait for reply from adapter
+	 */
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail0;
+
+	/*
+	 * Process reply
+	 */
+	reply =
+	    (struct c2wr_ep_listen_create_rep *) (unsigned long) vq_req->reply_msg;
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+
+	if ((err = c2_errno(reply)) != 0)
+		goto bail1;
+
+	/*
+	 * Keep the adapter handle. Used in subsequent destroy
+	 */
+	cm_id->provider_data = (void*)(unsigned long) reply->ep_handle;
+
+	/*
+	 * free vq stuff
+	 */
+	vq_repbuf_free(c2dev, reply);
+	vq_req_free(c2dev, vq_req);
+
+	return 0;
+
+ bail1:
+	vq_repbuf_free(c2dev, reply);
+ bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+
+int c2_llp_service_destroy(struct iw_cm_id *cm_id)
+{
+
+	struct c2_dev *c2dev;
+	struct c2wr_ep_listen_destroy_req wr;
+	struct c2wr_ep_listen_destroy_rep *reply;
+	struct c2_vq_req *vq_req;
+	int err;
+
+	c2dev = to_c2dev(cm_id->device);
+	if (c2dev == NULL)
+		return -EINVAL;
+
+	/*
+	 * Allocate verbs request.
+	 */
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req)
+		return -ENOMEM;
+
+	/*
+	 * Build the WR
+	 */
+	c2_wr_set_id(&wr, CCWR_EP_LISTEN_DESTROY);
+	wr.hdr.context = (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.ep_handle = (u32)(unsigned long)cm_id->provider_data;
+
+	/*
+	 * reference the request struct.  dereferenced in the int handler.
+	 */
+	vq_req_get(c2dev, vq_req);
+
+	/*
+	 * Send WR to adapter
+	 */
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail0;
+	}
+
+	/*
+	 * Wait for reply from adapter
+	 */
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail0;
+
+	/*
+	 * Process reply
+	 */
+	reply=(struct c2wr_ep_listen_destroy_rep *)(unsigned long)vq_req->reply_msg;
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+	if ((err = c2_errno(reply)) != 0)
+		goto bail1;
+
+ bail1:
+	vq_repbuf_free(c2dev, reply);
+ bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+int c2_llp_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
+{
+	struct c2_dev *c2dev = to_c2dev(cm_id->device);
+	struct c2_qp *qp;
+	struct ib_qp *ibqp;
+	struct c2wr_cr_accept_req *wr;	/* variable length WR */
+	struct c2_vq_req *vq_req;
+	struct c2wr_cr_accept_rep *reply;	/* VQ Reply msg ptr. */
+	int err;
+
+	ibqp = c2_get_qp(cm_id->device, iw_param->qpn);
+	if (!ibqp)
+		return -EINVAL;
+	qp = to_c2qp(ibqp);
+
+	/* Set the RDMA read limits */
+	err = c2_qp_set_read_limits(c2dev, qp, iw_param->ord, iw_param->ird);
+	if (err)
+		goto bail0;
+
+	/* Allocate verbs request. */
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+	vq_req->qp = qp;
+	vq_req->cm_id = cm_id;
+	vq_req->event = IW_CM_EVENT_ESTABLISHED;
+
+	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
+	if (!wr) {
+		err = -ENOMEM;
+		goto bail2;
+	}
+
+	/* Build the WR */
+	c2_wr_set_id(wr, CCWR_CR_ACCEPT);
+	wr->hdr.context = (unsigned long) vq_req;
+	wr->rnic_handle = c2dev->adapter_handle;
+	wr->ep_handle = (u32) (unsigned long) cm_id->provider_data;
+	wr->qp_handle = qp->adapter_handle;
+
+	/* Replace the cr_handle with the QP after accept */
+	cm_id->provider_data = qp;
+	cm_id->add_ref(cm_id);
+	qp->cm_id = cm_id;
+
+	cm_id->provider_data = qp;
+
+	/* Validate private_data length */
+	if (iw_param->private_data_len > C2_MAX_PRIVATE_DATA_SIZE) {
+		err = -EINVAL;
+		goto bail2;
+	}
+
+	if (iw_param->private_data) {
+		wr->private_data_length = cpu_to_be32(iw_param->private_data_len);
+		memcpy(&wr->private_data[0],
+		       iw_param->private_data, iw_param->private_data_len);
+	} else
+		wr->private_data_length = 0;
+
+	/* Reference the request struct.  Dereferenced in the int handler. */
+	vq_req_get(c2dev, vq_req);
+
+	/* Send WR to adapter */
+	err = vq_send_wr(c2dev, (union c2wr *) wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail2;
+	}
+
+	/* Wait for reply from adapter */
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail2;
+
+	/* Check that reply is present */
+	reply = (struct c2wr_cr_accept_rep *) (unsigned long) vq_req->reply_msg;
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail2;
+	}
+
+	err = c2_errno(reply);
+	vq_repbuf_free(c2dev, reply);
+
+	if (!err)
+		c2_set_qp_state(qp, C2_QP_STATE_RTS);
+ bail2:
+	kfree(wr);
+ bail1:
+	vq_req_free(c2dev, vq_req);
+ bail0:
+	if (err) {
+		/*
+		 * If we fail, release reference on QP and
+		 * disassociate QP from CM_ID
+		 */
+		cm_id->provider_data = NULL;
+		qp->cm_id = NULL;
+		cm_id->rem_ref(cm_id);
+	}
+	return err;
+}
+
+int c2_llp_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
+{
+	struct c2_dev *c2dev;
+	struct c2wr_cr_reject_req wr;
+	struct c2_vq_req *vq_req;
+	struct c2wr_cr_reject_rep *reply;
+	int err;
+
+	c2dev = to_c2dev(cm_id->device);
+
+	/*
+	 * Allocate verbs request.
+	 */
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req)
+		return -ENOMEM;
+
+	/*
+	 * Build the WR
+	 */
+	c2_wr_set_id(&wr, CCWR_CR_REJECT);
+	wr.hdr.context = (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.ep_handle = (u32) (unsigned long) cm_id->provider_data;
+
+	/*
+	 * reference the request struct.  dereferenced in the int handler.
+	 */
+	vq_req_get(c2dev, vq_req);
+
+	/*
+	 * Send WR to adapter
+	 */
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail0;
+	}
+
+	/*
+	 * Wait for reply from adapter
+	 */
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail0;
+
+	/*
+	 * Process reply
+	 */
+	reply = (struct c2wr_cr_reject_rep *) (unsigned long)
+		vq_req->reply_msg;
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+	err = c2_errno(reply);
+	/*
+	 * free vq stuff
+	 */
+	vq_repbuf_free(c2dev, reply);
+
+ bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_cq.c b/drivers/infiniband/hw/amso1100/c2_cq.c
new file mode 100644
index 0000000..9d7bcc5
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_cq.c
@@ -0,0 +1,433 @@
+/*
+ * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2005 Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+#include "c2.h"
+#include "c2_vq.h"
+#include "c2_status.h"
+
+#define C2_CQ_MSG_SIZE ((sizeof(struct c2wr_ce) + 32-1) & ~(32-1))
+
+static struct c2_cq *c2_cq_get(struct c2_dev *c2dev, int cqn)
+{
+	struct c2_cq *cq;
+	unsigned long flags;
+
+	spin_lock_irqsave(&c2dev->lock, flags);
+	cq = c2dev->qptr_array[cqn];
+	if (!cq) {
+		spin_unlock_irqrestore(&c2dev->lock, flags);
+		return NULL;
+	}
+	atomic_inc(&cq->refcount);
+	spin_unlock_irqrestore(&c2dev->lock, flags);
+	return cq;
+}
+
+static void c2_cq_put(struct c2_cq *cq)
+{
+	if (atomic_dec_and_test(&cq->refcount))
+		wake_up(&cq->wait);
+}
+
+void c2_cq_event(struct c2_dev *c2dev, u32 mq_index)
+{
+	struct c2_cq *cq;
+
+	cq = c2_cq_get(c2dev, mq_index);
+	if (!cq) {
+		printk("discarding events on destroyed CQN=%d\n", mq_index);
+		return;
+	}
+
+	(*cq->ibcq.comp_handler) (&cq->ibcq, cq->ibcq.cq_context);
+	c2_cq_put(cq);
+}
+
+void c2_cq_clean(struct c2_dev *c2dev, struct c2_qp *qp, u32 mq_index)
+{
+	struct c2_cq *cq;
+	struct c2_mq *q;
+
+	cq = c2_cq_get(c2dev, mq_index);
+	if (!cq)
+		return;
+
+	spin_lock_irq(&cq->lock);
+	q = &cq->mq;
+	if (q && !c2_mq_empty(q)) {
+		u16 priv = q->priv;
+		struct c2wr_ce *msg;
+
+		while (priv != be16_to_cpu(*q->shared)) {
+			msg = (struct c2wr_ce *)
+				(q->msg_pool.host + priv * q->msg_size);
+			if (msg->qp_user_context == (u64) (unsigned long) qp) {
+				msg->qp_user_context = (u64) 0;
+			}
+			priv = (priv + 1) % q->q_size;
+		}
+	}
+	spin_unlock_irq(&cq->lock);
+	c2_cq_put(cq);
+}
+
+static inline enum ib_wc_status c2_cqe_status_to_openib(u8 status)
+{
+	switch (status) {
+	case C2_OK:
+		return IB_WC_SUCCESS;
+	case CCERR_FLUSHED:
+		return IB_WC_WR_FLUSH_ERR;
+	case CCERR_BASE_AND_BOUNDS_VIOLATION:
+		return IB_WC_LOC_PROT_ERR;
+	case CCERR_ACCESS_VIOLATION:
+		return IB_WC_LOC_ACCESS_ERR;
+	case CCERR_TOTAL_LENGTH_TOO_BIG:
+		return IB_WC_LOC_LEN_ERR;
+	case CCERR_INVALID_WINDOW:
+		return IB_WC_MW_BIND_ERR;
+	default:
+		return IB_WC_GENERAL_ERR;
+	}
+}
+
+
+static inline int c2_poll_one(struct c2_dev *c2dev,
+			      struct c2_cq *cq, struct ib_wc *entry)
+{
+	struct c2wr_ce *ce;
+	struct c2_qp *qp;
+	int is_recv = 0;
+
+	ce = (struct c2wr_ce *) c2_mq_consume(&cq->mq);
+	if (!ce) {
+		return -EAGAIN;
+	}
+
+	/*
+	 * if the qp returned is null then this qp has already
+	 * been freed and we are unable process the completion.
+	 * try pulling the next message
+	 */
+	while ((qp =
+		(struct c2_qp *) (unsigned long) ce->qp_user_context) == NULL) {
+		c2_mq_free(&cq->mq);
+		ce = (struct c2wr_ce *) c2_mq_consume(&cq->mq);
+		if (!ce)
+			return -EAGAIN;
+	}
+
+	entry->status = c2_cqe_status_to_openib(c2_wr_get_result(ce));
+	entry->wr_id = ce->hdr.context;
+	entry->qp_num = ce->handle;
+	entry->wc_flags = 0;
+	entry->slid = 0;
+	entry->sl = 0;
+	entry->src_qp = 0;
+	entry->dlid_path_bits = 0;
+	entry->pkey_index = 0;
+
+	switch (c2_wr_get_id(ce)) {
+	case C2_WR_TYPE_SEND:
+		entry->opcode = IB_WC_SEND;
+		break;
+	case C2_WR_TYPE_RDMA_WRITE:
+		entry->opcode = IB_WC_RDMA_WRITE;
+		break;
+	case C2_WR_TYPE_RDMA_READ:
+		entry->opcode = IB_WC_RDMA_READ;
+		break;
+	case C2_WR_TYPE_BIND_MW:
+		entry->opcode = IB_WC_BIND_MW;
+		break;
+	case C2_WR_TYPE_RECV:
+		entry->byte_len = be32_to_cpu(ce->bytes_rcvd);
+		entry->opcode = IB_WC_RECV;
+		is_recv = 1;
+		break;
+	default:
+		break;
+	}
+
+	/* consume the WQEs */
+	if (is_recv)
+		c2_mq_lconsume(&qp->rq_mq, 1);
+	else
+		c2_mq_lconsume(&qp->sq_mq,
+			       be32_to_cpu(c2_wr_get_wqe_count(ce)) + 1);
+
+	/* free the message */
+	c2_mq_free(&cq->mq);
+
+	return 0;
+}
+
+int c2_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
+{
+	struct c2_dev *c2dev = to_c2dev(ibcq->device);
+	struct c2_cq *cq = to_c2cq(ibcq);
+	unsigned long flags;
+	int npolled, err;
+
+	spin_lock_irqsave(&cq->lock, flags);
+
+	for (npolled = 0; npolled < num_entries; ++npolled) {
+
+		err = c2_poll_one(c2dev, cq, entry + npolled);
+		if (err)
+			break;
+	}
+
+	spin_unlock_irqrestore(&cq->lock, flags);
+
+	return npolled;
+}
+
+int c2_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
+{
+	struct c2_mq_shared __iomem *shared;
+	struct c2_cq *cq;
+
+	cq = to_c2cq(ibcq);
+	shared = cq->mq.peer;
+
+	if (notify == IB_CQ_NEXT_COMP)
+		writeb(C2_CQ_NOTIFICATION_TYPE_NEXT, &shared->notification_type);
+	else if (notify == IB_CQ_SOLICITED)
+		writeb(C2_CQ_NOTIFICATION_TYPE_NEXT_SE, &shared->notification_type);
+	else
+		return -EINVAL;
+
+	writeb(CQ_WAIT_FOR_DMA | CQ_ARMED, &shared->armed);
+
+	/*
+	 * Now read back shared->armed to make the PCI
+	 * write synchronous.  This is necessary for
+	 * correct cq notification semantics.
+	 */
+	readb(&shared->armed);
+
+	return 0;
+}
+
+static void c2_free_cq_buf(struct c2_dev *c2dev, struct c2_mq *mq)
+{
+
+	dma_unmap_single(c2dev->ibdev.dma_device, pci_unmap_addr(mq, mapping),
+			 mq->q_size * mq->msg_size, DMA_FROM_DEVICE);
+	free_pages((unsigned long) mq->msg_pool.host,
+		   get_order(mq->q_size * mq->msg_size));
+}
+
+static int c2_alloc_cq_buf(struct c2_dev *c2dev, struct c2_mq *mq, int q_size,
+			   int msg_size)
+{
+	unsigned long pool_start;
+
+	pool_start = __get_free_pages(GFP_KERNEL,
+				      get_order(q_size * msg_size));
+	if (!pool_start)
+		return -ENOMEM;
+
+	c2_mq_rep_init(mq,
+		       0,		/* index (currently unknown) */
+		       q_size,
+		       msg_size,
+		       (u8 *) pool_start,
+		       NULL,	/* peer (currently unknown) */
+		       C2_MQ_HOST_TARGET);
+
+	mq->host_dma = dma_map_single(c2dev->ibdev.dma_device,
+				      (void *)pool_start,
+				      q_size * msg_size, DMA_FROM_DEVICE);
+	pci_unmap_addr_set(mq, mapping, mq->host_dma);
+
+	return 0;
+}
+
+int c2_init_cq(struct c2_dev *c2dev, int entries,
+	       struct c2_ucontext *ctx, struct c2_cq *cq)
+{
+	struct c2wr_cq_create_req wr;
+	struct c2wr_cq_create_rep *reply;
+	unsigned long peer_pa;
+	struct c2_vq_req *vq_req;
+	int err;
+
+	might_sleep();
+
+	cq->ibcq.cqe = entries - 1;
+	cq->is_kernel = !ctx;
+
+	/* Allocate a shared pointer */
+	cq->mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
+				      &cq->mq.shared_dma, GFP_KERNEL);
+	if (!cq->mq.shared)
+		return -ENOMEM;
+
+	/* Allocate pages for the message pool */
+	err = c2_alloc_cq_buf(c2dev, &cq->mq, entries + 1, C2_CQ_MSG_SIZE);
+	if (err)
+		goto bail0;
+
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+
+	memset(&wr, 0, sizeof(wr));
+	c2_wr_set_id(&wr, CCWR_CQ_CREATE);
+	wr.hdr.context = (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.msg_size = cpu_to_be32(cq->mq.msg_size);
+	wr.depth = cpu_to_be32(cq->mq.q_size);
+	wr.shared_ht = cpu_to_be64(cq->mq.shared_dma);
+	wr.msg_pool = cpu_to_be64(cq->mq.host_dma);
+	wr.user_context = (u64) (unsigned long) (cq);
+
+	vq_req_get(c2dev, vq_req);
+
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail2;
+	}
+
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail2;
+
+	reply = (struct c2wr_cq_create_rep *) (unsigned long) (vq_req->reply_msg);
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail2;
+	}
+
+	if ((err = c2_errno(reply)) != 0)
+		goto bail3;
+
+	cq->adapter_handle = reply->cq_handle;
+	cq->mq.index = be32_to_cpu(reply->mq_index);
+
+	peer_pa = c2dev->pa + be32_to_cpu(reply->adapter_shared);
+	cq->mq.peer = ioremap_nocache(peer_pa, PAGE_SIZE);
+	if (!cq->mq.peer) {
+		err = -ENOMEM;
+		goto bail3;
+	}
+
+	vq_repbuf_free(c2dev, reply);
+	vq_req_free(c2dev, vq_req);
+
+	spin_lock_init(&cq->lock);
+	atomic_set(&cq->refcount, 1);
+	init_waitqueue_head(&cq->wait);
+
+	/*
+	 * Use the MQ index allocated by the adapter to
+	 * store the CQ in the qptr_array
+	 */
+	cq->cqn = cq->mq.index;
+	c2dev->qptr_array[cq->cqn] = cq;
+
+	return 0;
+
+      bail3:
+	vq_repbuf_free(c2dev, reply);
+      bail2:
+	vq_req_free(c2dev, vq_req);
+      bail1:
+	c2_free_cq_buf(c2dev, &cq->mq);
+      bail0:
+	c2_free_mqsp(cq->mq.shared);
+
+	return err;
+}
+
+void c2_free_cq(struct c2_dev *c2dev, struct c2_cq *cq)
+{
+	int err;
+	struct c2_vq_req *vq_req;
+	struct c2wr_cq_destroy_req wr;
+	struct c2wr_cq_destroy_rep *reply;
+
+	might_sleep();
+
+	/* Clear CQ from the qptr array */
+	spin_lock_irq(&c2dev->lock);
+	c2dev->qptr_array[cq->mq.index] = NULL;
+	atomic_dec(&cq->refcount);
+	spin_unlock_irq(&c2dev->lock);
+
+	wait_event(cq->wait, !atomic_read(&cq->refcount));
+
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req) {
+		goto bail0;
+	}
+
+	memset(&wr, 0, sizeof(wr));
+	c2_wr_set_id(&wr, CCWR_CQ_DESTROY);
+	wr.hdr.context = (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.cq_handle = cq->adapter_handle;
+
+	vq_req_get(c2dev, vq_req);
+
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail1;
+	}
+
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail1;
+
+	reply = (struct c2wr_cq_destroy_rep *) (unsigned long) (vq_req->reply_msg);
+
+	vq_repbuf_free(c2dev, reply);
+      bail1:
+	vq_req_free(c2dev, vq_req);
+      bail0:
+	if (cq->is_kernel) {
+		c2_free_cq_buf(c2dev, &cq->mq);
+	}
+
+	return;
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_intr.c b/drivers/infiniband/hw/amso1100/c2_intr.c
new file mode 100644
index 0000000..0d0bc33
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_intr.c
@@ -0,0 +1,209 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "c2.h"
+#include <rdma/iw_cm.h>
+#include "c2_vq.h"
+
+static void handle_mq(struct c2_dev *c2dev, u32 index);
+static void handle_vq(struct c2_dev *c2dev, u32 mq_index);
+
+/*
+ * Handle RNIC interrupts
+ */
+void c2_rnic_interrupt(struct c2_dev *c2dev)
+{
+	unsigned int mq_index;
+
+	while (c2dev->hints_read != be16_to_cpu(*c2dev->hint_count)) {
+		mq_index = readl(c2dev->regs + PCI_BAR0_HOST_HINT);
+		if (mq_index & 0x80000000) {
+			break;
+		}
+
+		c2dev->hints_read++;
+		handle_mq(c2dev, mq_index);
+	}
+
+}
+
+/*
+ * Top level MQ handler
+ */
+static void handle_mq(struct c2_dev *c2dev, u32 mq_index)
+{
+	if (c2dev->qptr_array[mq_index] == NULL) {
+		pr_debug(KERN_INFO "handle_mq: stray activity for mq_index=%d\n",
+			mq_index);
+		return;
+	}
+
+	switch (mq_index) {
+	case (0):
+		/*
+		 * An index of 0 in the activity queue
+		 * indicates the req vq now has messages
+		 * available...
+		 *
+		 * Wake up any waiters waiting on req VQ
+		 * message availability.
+		 */
+		wake_up(&c2dev->req_vq_wo);
+		break;
+	case (1):
+		handle_vq(c2dev, mq_index);
+		break;
+	case (2):
+		/* We have to purge the VQ in case there are pending
+		 * accept reply requests that would result in the
+		 * generation of an ESTABLISHED event. If we don't
+		 * generate these first, a CLOSE event could end up
+		 * being delivered before the ESTABLISHED event.
+		 */
+		handle_vq(c2dev, 1);
+
+		c2_ae_event(c2dev, mq_index);
+		break;
+	default:
+		/* There is no event synchronization between CQ events
+		 * and AE or CM events. In fact, CQE could be
+		 * delivered for all of the I/O up to and including the
+		 * FLUSH for a peer disconenct prior to the ESTABLISHED
+		 * event being delivered to the app. The reason for this
+		 * is that CM events are delivered on a thread, while AE
+		 * and CM events are delivered on interrupt context.
+		 */
+		c2_cq_event(c2dev, mq_index);
+		break;
+	}
+
+	return;
+}
+
+/*
+ * Handles verbs WR replies.
+ */
+static void handle_vq(struct c2_dev *c2dev, u32 mq_index)
+{
+	void *adapter_msg, *reply_msg;
+	struct c2wr_hdr *host_msg;
+	struct c2wr_hdr tmp;
+	struct c2_mq *reply_vq;
+	struct c2_vq_req *req;
+	struct iw_cm_event cm_event;
+	int err;
+
+	reply_vq = (struct c2_mq *) c2dev->qptr_array[mq_index];
+
+	/*
+	 * get next msg from mq_index into adapter_msg.
+	 * don't free it yet.
+	 */
+	adapter_msg = c2_mq_consume(reply_vq);
+	if (adapter_msg == NULL) {
+		return;
+	}
+
+	host_msg = vq_repbuf_alloc(c2dev);
+
+	/*
+	 * If we can't get a host buffer, then we'll still
+	 * wakeup the waiter, we just won't give him the msg.
+	 * It is assumed the waiter will deal with this...
+	 */
+	if (!host_msg) {
+		pr_debug("handle_vq: no repbufs!\n");
+
+		/*
+		 * just copy the WR header into a local variable.
+		 * this allows us to still demux on the context
+		 */
+		host_msg = &tmp;
+		memcpy(host_msg, adapter_msg, sizeof(tmp));
+		reply_msg = NULL;
+	} else {
+		memcpy(host_msg, adapter_msg, reply_vq->msg_size);
+		reply_msg = host_msg;
+	}
+
+	/*
+	 * consume the msg from the MQ
+	 */
+	c2_mq_free(reply_vq);
+
+	/*
+	 * wakeup the waiter.
+	 */
+	req = (struct c2_vq_req *) (unsigned long) host_msg->context;
+	if (req == NULL) {
+		/*
+		 * We should never get here, as the adapter should
+		 * never send us a reply that we're not expecting.
+		 */
+		vq_repbuf_free(c2dev, host_msg);
+		pr_debug("handle_vq: UNEXPECTEDLY got NULL req\n");
+		return;
+	}
+
+	err = c2_errno(reply_msg);
+	if (!err) switch (req->event) {
+	case IW_CM_EVENT_ESTABLISHED:
+		c2_set_qp_state(req->qp,
+				C2_QP_STATE_RTS);
+	case IW_CM_EVENT_CLOSE:
+
+		/*
+		 * Move the QP to RTS if this is
+		 * the established event
+		 */
+		cm_event.event = req->event;
+		cm_event.status = 0;
+		cm_event.local_addr = req->cm_id->local_addr;
+		cm_event.remote_addr = req->cm_id->remote_addr;
+		cm_event.private_data = NULL;
+		cm_event.private_data_len = 0;
+		req->cm_id->event_handler(req->cm_id, &cm_event);
+		break;
+	default:
+		break;
+	}
+
+	req->reply_msg = (u64) (unsigned long) (reply_msg);
+	atomic_set(&req->reply_ready, 1);
+	wake_up(&req->wait_object);
+
+	/*
+	 * If the request was cancelled, then this put will
+	 * free the vq_req memory...and reply_msg!!!
+	 */
+	vq_req_put(c2dev, req);
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_mm.c b/drivers/infiniband/hw/amso1100/c2_mm.c
new file mode 100644
index 0000000..1e4f464
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_mm.c
@@ -0,0 +1,375 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "c2.h"
+#include "c2_vq.h"
+
+#define PBL_VIRT 1
+#define PBL_PHYS 2
+
+/*
+ * Send all the PBL messages to convey the remainder of the PBL
+ * Wait for the adapter's reply on the last one.
+ * This is indicated by setting the MEM_PBL_COMPLETE in the flags.
+ *
+ * NOTE:  vq_req is _not_ freed by this function.  The VQ Host
+ *	  Reply buffer _is_ freed by this function.
+ */
+static int
+send_pbl_messages(struct c2_dev *c2dev, u32 stag_index,
+		  unsigned long va, u32 pbl_depth,
+		  struct c2_vq_req *vq_req, int pbl_type)
+{
+	u32 pbe_count;		/* amt that fits in a PBL msg */
+	u32 count;		/* amt in this PBL MSG. */
+	struct c2wr_nsmr_pbl_req *wr;	/* PBL WR ptr */
+	struct c2wr_nsmr_pbl_rep *reply;	/* reply ptr */
+ 	int err, pbl_virt, pbl_index, i;
+
+	switch (pbl_type) {
+	case PBL_VIRT:
+		pbl_virt = 1;
+		break;
+	case PBL_PHYS:
+		pbl_virt = 0;
+		break;
+	default:
+		return -EINVAL;
+		break;
+	}
+
+	pbe_count = (c2dev->req_vq.msg_size -
+		     sizeof(struct c2wr_nsmr_pbl_req)) / sizeof(u64);
+	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
+	if (!wr) {
+		return -ENOMEM;
+	}
+	c2_wr_set_id(wr, CCWR_NSMR_PBL);
+
+	/*
+	 * Only the last PBL message will generate a reply from the verbs,
+	 * so we set the context to 0 indicating there is no kernel verbs
+	 * handler blocked awaiting this reply.
+	 */
+	wr->hdr.context = 0;
+	wr->rnic_handle = c2dev->adapter_handle;
+	wr->stag_index = stag_index;	/* already swapped */
+	wr->flags = 0;
+	pbl_index = 0;
+	while (pbl_depth) {
+		count = min(pbe_count, pbl_depth);
+		wr->addrs_length = cpu_to_be32(count);
+
+		/*
+		 *  If this is the last message, then reference the
+		 *  vq request struct cuz we're gonna wait for a reply.
+		 *  also make this PBL msg as the last one.
+		 */
+		if (count == pbl_depth) {
+			/*
+			 * reference the request struct.  dereferenced in the
+			 * int handler.
+			 */
+			vq_req_get(c2dev, vq_req);
+			wr->flags = cpu_to_be32(MEM_PBL_COMPLETE);
+
+			/*
+			 * This is the last PBL message.
+			 * Set the context to our VQ Request Object so we can
+			 * wait for the reply.
+			 */
+			wr->hdr.context = (unsigned long) vq_req;
+		}
+
+		/*
+		 * If pbl_virt is set then va is a virtual address
+		 * that describes a virtually contiguous memory
+		 * allocation. The wr needs the start of each virtual page
+		 * to be converted to the corresponding physical address
+		 * of the page. If pbl_virt is not set then va is an array
+		 * of physical addresses and there is no conversion to do.
+		 * Just fill in the wr with what is in the array.
+		 */
+		for (i = 0; i < count; i++) {
+			if (pbl_virt) {
+				va += PAGE_SIZE;
+			} else {
+ 				wr->paddrs[i] =
+				    cpu_to_be64(((u64 *)va)[pbl_index + i]);
+			}
+		}
+
+		/*
+		 * Send WR to adapter
+		 */
+		err = vq_send_wr(c2dev, (union c2wr *) wr);
+		if (err) {
+			if (count <= pbe_count) {
+				vq_req_put(c2dev, vq_req);
+			}
+			goto bail0;
+		}
+		pbl_depth -= count;
+		pbl_index += count;
+	}
+
+	/*
+	 *  Now wait for the reply...
+	 */
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err) {
+		goto bail0;
+	}
+
+	/*
+	 * Process reply
+	 */
+	reply = (struct c2wr_nsmr_pbl_rep *) (unsigned long) vq_req->reply_msg;
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	err = c2_errno(reply);
+
+	vq_repbuf_free(c2dev, reply);
+      bail0:
+	kfree(wr);
+	return err;
+}
+
+#define C2_PBL_MAX_DEPTH 131072
+int
+c2_nsmr_register_phys_kern(struct c2_dev *c2dev, u64 *addr_list,
+ 			   int page_size, int pbl_depth, u32 length,
+ 			   u32 offset, u64 *va, enum c2_acf acf,
+			   struct c2_mr *mr)
+{
+	struct c2_vq_req *vq_req;
+	struct c2wr_nsmr_register_req *wr;
+	struct c2wr_nsmr_register_rep *reply;
+	u16 flags;
+	int i, pbe_count, count;
+	int err;
+
+	if (!va || !length || !addr_list || !pbl_depth)
+		return -EINTR;
+
+	/*
+	 * Verify PBL depth is within rnic max
+	 */
+	if (pbl_depth > C2_PBL_MAX_DEPTH) {
+		return -EINTR;
+	}
+
+	/*
+	 * allocate verbs request object
+	 */
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req)
+		return -ENOMEM;
+
+	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
+	if (!wr) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	/*
+	 * build the WR
+	 */
+	c2_wr_set_id(wr, CCWR_NSMR_REGISTER);
+	wr->hdr.context = (unsigned long) vq_req;
+	wr->rnic_handle = c2dev->adapter_handle;
+
+	flags = (acf | MEM_VA_BASED | MEM_REMOTE);
+
+	/*
+	 * compute how many pbes can fit in the message
+	 */
+	pbe_count = (c2dev->req_vq.msg_size -
+		     sizeof(struct c2wr_nsmr_register_req)) / sizeof(u64);
+
+	if (pbl_depth <= pbe_count) {
+		flags |= MEM_PBL_COMPLETE;
+	}
+	wr->flags = cpu_to_be16(flags);
+	wr->stag_key = 0;	//stag_key;
+	wr->va = cpu_to_be64(*va);
+	wr->pd_id = mr->pd->pd_id;
+	wr->pbe_size = cpu_to_be32(page_size);
+	wr->length = cpu_to_be32(length);
+	wr->pbl_depth = cpu_to_be32(pbl_depth);
+	wr->fbo = cpu_to_be32(offset);
+	count = min(pbl_depth, pbe_count);
+	wr->addrs_length = cpu_to_be32(count);
+
+	/*
+	 * fill out the PBL for this message
+	 */
+	for (i = 0; i < count; i++) {
+		wr->paddrs[i] = cpu_to_be64(addr_list[i]);
+	}
+
+	/*
+	 * regerence the request struct
+	 */
+	vq_req_get(c2dev, vq_req);
+
+	/*
+	 * send the WR to the adapter
+	 */
+	err = vq_send_wr(c2dev, (union c2wr *) wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail1;
+	}
+
+	/*
+	 * wait for reply from adapter
+	 */
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err) {
+		goto bail1;
+	}
+
+	/*
+	 * process reply
+	 */
+	reply =
+	    (struct c2wr_nsmr_register_rep *) (unsigned long) (vq_req->reply_msg);
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+	if ((err = c2_errno(reply))) {
+		goto bail2;
+	}
+	//*p_pb_entries = be32_to_cpu(reply->pbl_depth);
+	mr->ibmr.lkey = mr->ibmr.rkey = be32_to_cpu(reply->stag_index);
+	vq_repbuf_free(c2dev, reply);
+
+	/*
+	 * if there are still more PBEs we need to send them to
+	 * the adapter and wait for a reply on the final one.
+	 * reuse vq_req for this purpose.
+	 */
+	pbl_depth -= count;
+	if (pbl_depth) {
+
+		vq_req->reply_msg = (unsigned long) NULL;
+		atomic_set(&vq_req->reply_ready, 0);
+		err = send_pbl_messages(c2dev,
+					cpu_to_be32(mr->ibmr.lkey),
+					(unsigned long) &addr_list[i],
+					pbl_depth, vq_req, PBL_PHYS);
+		if (err) {
+			goto bail1;
+		}
+	}
+
+	vq_req_free(c2dev, vq_req);
+	kfree(wr);
+
+	return err;
+
+      bail2:
+	vq_repbuf_free(c2dev, reply);
+      bail1:
+	kfree(wr);
+      bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+int c2_stag_dealloc(struct c2_dev *c2dev, u32 stag_index)
+{
+	struct c2_vq_req *vq_req;	/* verbs request object */
+	struct c2wr_stag_dealloc_req wr;	/* work request */
+	struct c2wr_stag_dealloc_rep *reply;	/* WR reply  */
+	int err;
+
+
+	/*
+	 * allocate verbs request object
+	 */
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req) {
+		return -ENOMEM;
+	}
+
+	/*
+	 * Build the WR
+	 */
+	c2_wr_set_id(&wr, CCWR_STAG_DEALLOC);
+	wr.hdr.context = (u64) (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.stag_index = cpu_to_be32(stag_index);
+
+	/*
+	 * reference the request struct.  dereferenced in the int handler.
+	 */
+	vq_req_get(c2dev, vq_req);
+
+	/*
+	 * Send WR to adapter
+	 */
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail0;
+	}
+
+	/*
+	 * Wait for reply from adapter
+	 */
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err) {
+		goto bail0;
+	}
+
+	/*
+	 * Process reply
+	 */
+	reply = (struct c2wr_stag_dealloc_rep *) (unsigned long) vq_req->reply_msg;
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	err = c2_errno(reply);
+
+	vq_repbuf_free(c2dev, reply);
+      bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_mq.c b/drivers/infiniband/hw/amso1100/c2_mq.c
new file mode 100644
index 0000000..b88a755
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_mq.c
@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include "c2.h"
+#include "c2_mq.h"
+
+void *c2_mq_alloc(struct c2_mq *q)
+{
+	BUG_ON(q->magic != C2_MQ_MAGIC);
+	BUG_ON(q->type != C2_MQ_ADAPTER_TARGET);
+
+	if (c2_mq_full(q)) {
+		return NULL;
+	} else {
+#ifdef DEBUG
+		struct c2wr_hdr *m =
+		    (struct c2wr_hdr *) (q->msg_pool.host + q->priv * q->msg_size);
+#ifdef CCMSGMAGIC
+		BUG_ON(m->magic != be32_to_cpu(~CCWR_MAGIC));
+		m->magic = cpu_to_be32(CCWR_MAGIC);
+#endif
+		return m;
+#else
+		return q->msg_pool.host + q->priv * q->msg_size;
+#endif
+	}
+}
+
+void c2_mq_produce(struct c2_mq *q)
+{
+	BUG_ON(q->magic != C2_MQ_MAGIC);
+	BUG_ON(q->type != C2_MQ_ADAPTER_TARGET);
+
+	if (!c2_mq_full(q)) {
+		q->priv = (q->priv + 1) % q->q_size;
+		q->hint_count++;
+		/* Update peer's offset. */
+		__raw_writew(cpu_to_be16(q->priv), &q->peer->shared);
+	}
+}
+
+void *c2_mq_consume(struct c2_mq *q)
+{
+	BUG_ON(q->magic != C2_MQ_MAGIC);
+	BUG_ON(q->type != C2_MQ_HOST_TARGET);
+
+	if (c2_mq_empty(q)) {
+		return NULL;
+	} else {
+#ifdef DEBUG
+		struct c2wr_hdr *m = (struct c2wr_hdr *)
+		    (q->msg_pool.host + q->priv * q->msg_size);
+#ifdef CCMSGMAGIC
+		BUG_ON(m->magic != be32_to_cpu(CCWR_MAGIC));
+#endif
+		return m;
+#else
+		return q->msg_pool.host + q->priv * q->msg_size;
+#endif
+	}
+}
+
+void c2_mq_free(struct c2_mq *q)
+{
+	BUG_ON(q->magic != C2_MQ_MAGIC);
+	BUG_ON(q->type != C2_MQ_HOST_TARGET);
+
+	if (!c2_mq_empty(q)) {
+
+#ifdef CCMSGMAGIC
+		{
+			struct c2wr_hdr __iomem *m = (struct c2wr_hdr __iomem *)
+			    (q->msg_pool.adapter + q->priv * q->msg_size);
+			__raw_writel(cpu_to_be32(~CCWR_MAGIC), &m->magic);
+		}
+#endif
+		q->priv = (q->priv + 1) % q->q_size;
+		/* Update peer's offset. */
+		__raw_writew(cpu_to_be16(q->priv), &q->peer->shared);
+	}
+}
+
+
+void c2_mq_lconsume(struct c2_mq *q, u32 wqe_count)
+{
+	BUG_ON(q->magic != C2_MQ_MAGIC);
+	BUG_ON(q->type != C2_MQ_ADAPTER_TARGET);
+
+	while (wqe_count--) {
+		BUG_ON(c2_mq_empty(q));
+		*q->shared = cpu_to_be16((be16_to_cpu(*q->shared)+1) % q->q_size);
+	}
+}
+
+#if 0
+u32 c2_mq_count(struct c2_mq *q)
+{
+	s32 count;
+
+	if (q->type == C2_MQ_HOST_TARGET)
+		count = be16_to_cpu(*q->shared) - q->priv;
+	else
+		count = q->priv - be16_to_cpu(*q->shared);
+
+	if (count < 0)
+		count += q->q_size;
+
+	return (u32) count;
+}
+#endif  /*  0  */
+
+void c2_mq_req_init(struct c2_mq *q, u32 index, u32 q_size, u32 msg_size,
+		    u8 __iomem *pool_start, u16 __iomem *peer, u32 type)
+{
+	BUG_ON(!q->shared);
+
+	/* This code assumes the byte swapping has already been done! */
+	q->index = index;
+	q->q_size = q_size;
+	q->msg_size = msg_size;
+	q->msg_pool.adapter = pool_start;
+	q->peer = (struct c2_mq_shared __iomem *) peer;
+	q->magic = C2_MQ_MAGIC;
+	q->type = type;
+	q->priv = 0;
+	q->hint_count = 0;
+	return;
+}
+void c2_mq_rep_init(struct c2_mq *q, u32 index, u32 q_size, u32 msg_size,
+		    u8 *pool_start, u16 __iomem *peer, u32 type)
+{
+	BUG_ON(!q->shared);
+
+	/* This code assumes the byte swapping has already been done! */
+	q->index = index;
+	q->q_size = q_size;
+	q->msg_size = msg_size;
+	q->msg_pool.host = pool_start;
+	q->peer = (struct c2_mq_shared __iomem *) peer;
+	q->magic = C2_MQ_MAGIC;
+	q->type = type;
+	q->priv = 0;
+	q->hint_count = 0;
+	return;
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_mq.h b/drivers/infiniband/hw/amso1100/c2_mq.h
new file mode 100644
index 0000000..9185bbb
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_mq.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef _C2_MQ_H_
+#define _C2_MQ_H_
+#include <linux/kernel.h>
+#include <linux/dma-mapping.h>
+#include "c2_wr.h"
+
+enum c2_shared_regs {
+
+	C2_SHARED_ARMED = 0x10,
+	C2_SHARED_NOTIFY = 0x18,
+	C2_SHARED_SHARED = 0x40,
+};
+
+struct c2_mq_shared {
+	u16 unused1;
+	u8 armed;
+	u8 notification_type;
+	u32 unused2;
+	u16 shared;
+	/* Pad to 64 bytes. */
+	u8 pad[64 - sizeof(u16) - 2 * sizeof(u8) - sizeof(u32) - sizeof(u16)];
+};
+
+enum c2_mq_type {
+	C2_MQ_HOST_TARGET = 1,
+	C2_MQ_ADAPTER_TARGET = 2,
+};
+
+/*
+ * c2_mq_t is for kernel-mode MQs like the VQs Cand the AEQ.
+ * c2_user_mq_t (which is the same format) is for user-mode MQs...
+ */
+#define C2_MQ_MAGIC 0x4d512020	/* 'MQ  ' */
+struct c2_mq {
+	u32 magic;
+	union {
+		u8 *host;
+		u8 __iomem *adapter;
+	} msg_pool;
+	dma_addr_t host_dma;
+	DECLARE_PCI_UNMAP_ADDR(mapping);
+	u16 hint_count;
+	u16 priv;
+	struct c2_mq_shared __iomem *peer;
+	u16 *shared;
+	dma_addr_t shared_dma;
+	u32 q_size;
+	u32 msg_size;
+	u32 index;
+	enum c2_mq_type type;
+};
+
+static __inline__ int c2_mq_empty(struct c2_mq *q)
+{
+	return q->priv == be16_to_cpu(*q->shared);
+}
+
+static __inline__ int c2_mq_full(struct c2_mq *q)
+{
+	return q->priv == (be16_to_cpu(*q->shared) + q->q_size - 1) % q->q_size;
+}
+
+extern void c2_mq_lconsume(struct c2_mq *q, u32 wqe_count);
+extern void *c2_mq_alloc(struct c2_mq *q);
+extern void c2_mq_produce(struct c2_mq *q);
+extern void *c2_mq_consume(struct c2_mq *q);
+extern void c2_mq_free(struct c2_mq *q);
+extern void c2_mq_req_init(struct c2_mq *q, u32 index, u32 q_size, u32 msg_size,
+		       u8 __iomem *pool_start, u16 __iomem *peer, u32 type);
+extern void c2_mq_rep_init(struct c2_mq *q, u32 index, u32 q_size, u32 msg_size,
+			   u8 *pool_start, u16 __iomem *peer, u32 type);
+
+#endif				/* _C2_MQ_H_ */
diff --git a/drivers/infiniband/hw/amso1100/c2_pd.c b/drivers/infiniband/hw/amso1100/c2_pd.c
new file mode 100644
index 0000000..00c7099
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_pd.c
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/init.h>
+#include <linux/errno.h>
+
+#include "c2.h"
+#include "c2_provider.h"
+
+int c2_pd_alloc(struct c2_dev *c2dev, int privileged, struct c2_pd *pd)
+{
+	u32 obj;
+	int ret = 0;
+
+	spin_lock(&c2dev->pd_table.lock);
+	obj = find_next_zero_bit(c2dev->pd_table.table, c2dev->pd_table.max,
+				 c2dev->pd_table.last);
+	if (obj >= c2dev->pd_table.max)
+		obj = find_first_zero_bit(c2dev->pd_table.table,
+					  c2dev->pd_table.max);
+	if (obj < c2dev->pd_table.max) {
+		pd->pd_id = obj;
+		__set_bit(obj, c2dev->pd_table.table);
+		c2dev->pd_table.last = obj+1;
+		if (c2dev->pd_table.last >= c2dev->pd_table.max)
+			c2dev->pd_table.last = 0;
+	} else
+		ret = -ENOMEM;
+	spin_unlock(&c2dev->pd_table.lock);
+	return ret;
+}
+
+void c2_pd_free(struct c2_dev *c2dev, struct c2_pd *pd)
+{
+	spin_lock(&c2dev->pd_table.lock);
+	__clear_bit(pd->pd_id, c2dev->pd_table.table);
+	spin_unlock(&c2dev->pd_table.lock);
+}
+
+int __devinit c2_init_pd_table(struct c2_dev *c2dev)
+{
+
+	c2dev->pd_table.last = 0;
+	c2dev->pd_table.max = c2dev->props.max_pd;
+	spin_lock_init(&c2dev->pd_table.lock);
+	c2dev->pd_table.table = kmalloc(BITS_TO_LONGS(c2dev->props.max_pd) *
+					sizeof(long), GFP_KERNEL);
+	if (!c2dev->pd_table.table)
+		return -ENOMEM;
+	bitmap_zero(c2dev->pd_table.table, c2dev->props.max_pd);
+	return 0;
+}
+
+void __devexit c2_cleanup_pd_table(struct c2_dev *c2dev)
+{
+	kfree(c2dev->pd_table.table);
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_provider.c b/drivers/infiniband/hw/amso1100/c2_provider.c
new file mode 100644
index 0000000..dd6af55
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_provider.c
@@ -0,0 +1,870 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/inetdevice.h>
+#include <linux/delay.h>
+#include <linux/ethtool.h>
+#include <linux/mii.h>
+#include <linux/if_vlan.h>
+#include <linux/crc32.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/tcp.h>
+#include <linux/init.h>
+#include <linux/dma-mapping.h>
+#include <linux/if_arp.h>
+#include <linux/vmalloc.h>
+
+#include <asm/io.h>
+#include <asm/irq.h>
+#include <asm/byteorder.h>
+
+#include <rdma/ib_smi.h>
+#include <rdma/ib_user_verbs.h>
+#include "c2.h"
+#include "c2_provider.h"
+#include "c2_user.h"
+
+static int c2_query_device(struct ib_device *ibdev,
+			   struct ib_device_attr *props)
+{
+	struct c2_dev *c2dev = to_c2dev(ibdev);
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	*props = c2dev->props;
+	return 0;
+}
+
+static int c2_query_port(struct ib_device *ibdev,
+			 u8 port, struct ib_port_attr *props)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	props->max_mtu = IB_MTU_4096;
+	props->lid = 0;
+	props->lmc = 0;
+	props->sm_lid = 0;
+	props->sm_sl = 0;
+	props->state = IB_PORT_ACTIVE;
+	props->phys_state = 0;
+	props->port_cap_flags =
+	    IB_PORT_CM_SUP |
+	    IB_PORT_REINIT_SUP |
+	    IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP;
+	props->gid_tbl_len = 1;
+	props->pkey_tbl_len = 1;
+	props->qkey_viol_cntr = 0;
+	props->active_width = 1;
+	props->active_speed = 1;
+
+	return 0;
+}
+
+static int c2_modify_port(struct ib_device *ibdev,
+			  u8 port, int port_modify_mask,
+			  struct ib_port_modify *props)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return 0;
+}
+
+static int c2_query_pkey(struct ib_device *ibdev,
+			 u8 port, u16 index, u16 * pkey)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	*pkey = 0;
+	return 0;
+}
+
+static int c2_query_gid(struct ib_device *ibdev, u8 port,
+			int index, union ib_gid *gid)
+{
+	struct c2_dev *c2dev = to_c2dev(ibdev);
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	memset(&(gid->raw[0]), 0, sizeof(gid->raw));
+	memcpy(&(gid->raw[0]), c2dev->pseudo_netdev->dev_addr, 6);
+
+	return 0;
+}
+
+/* Allocate the user context data structure. This keeps track
+ * of all objects associated with a particular user-mode client.
+ */
+static struct ib_ucontext *c2_alloc_ucontext(struct ib_device *ibdev,
+					     struct ib_udata *udata)
+{
+	struct c2_ucontext *context;
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	context = kmalloc(sizeof(*context), GFP_KERNEL);
+	if (!context)
+		return ERR_PTR(-ENOMEM);
+
+	return &context->ibucontext;
+}
+
+static int c2_dealloc_ucontext(struct ib_ucontext *context)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	kfree(context);
+	return 0;
+}
+
+static int c2_mmap_uar(struct ib_ucontext *context, struct vm_area_struct *vma)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return -ENOSYS;
+}
+
+static struct ib_pd *c2_alloc_pd(struct ib_device *ibdev,
+				 struct ib_ucontext *context,
+				 struct ib_udata *udata)
+{
+	struct c2_pd *pd;
+	int err;
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	pd = kmalloc(sizeof(*pd), GFP_KERNEL);
+	if (!pd)
+		return ERR_PTR(-ENOMEM);
+
+	err = c2_pd_alloc(to_c2dev(ibdev), !context, pd);
+	if (err) {
+		kfree(pd);
+		return ERR_PTR(err);
+	}
+
+	if (context) {
+		if (ib_copy_to_udata(udata, &pd->pd_id, sizeof(__u32))) {
+			c2_pd_free(to_c2dev(ibdev), pd);
+			kfree(pd);
+			return ERR_PTR(-EFAULT);
+		}
+	}
+
+	return &pd->ibpd;
+}
+
+static int c2_dealloc_pd(struct ib_pd *pd)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	c2_pd_free(to_c2dev(pd->device), to_c2pd(pd));
+	kfree(pd);
+
+	return 0;
+}
+
+static struct ib_ah *c2_ah_create(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return ERR_PTR(-ENOSYS);
+}
+
+static int c2_ah_destroy(struct ib_ah *ah)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return -ENOSYS;
+}
+
+static void c2_add_ref(struct ib_qp *ibqp)
+{
+	struct c2_qp *qp;
+	BUG_ON(!ibqp);
+	qp = to_c2qp(ibqp);
+	atomic_inc(&qp->refcount);
+}
+
+static void c2_rem_ref(struct ib_qp *ibqp)
+{
+	struct c2_qp *qp;
+	BUG_ON(!ibqp);
+	qp = to_c2qp(ibqp);
+	if (atomic_dec_and_test(&qp->refcount))
+		wake_up(&qp->wait);
+}
+
+struct ib_qp *c2_get_qp(struct ib_device *device, int qpn)
+{
+	struct c2_dev* c2dev = to_c2dev(device);
+	struct c2_qp *qp;
+
+	qp = c2_find_qpn(c2dev, qpn);
+	pr_debug("%s Returning QP=%p for QPN=%d, device=%p, refcount=%d\n",
+		__FUNCTION__, qp, qpn, device,
+		(qp?atomic_read(&qp->refcount):0));
+
+	return (qp?&qp->ibqp:NULL);
+}
+
+static struct ib_qp *c2_create_qp(struct ib_pd *pd,
+				  struct ib_qp_init_attr *init_attr,
+				  struct ib_udata *udata)
+{
+	struct c2_qp *qp;
+	int err;
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	switch (init_attr->qp_type) {
+	case IB_QPT_RC:
+		qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+		if (!qp) {
+			pr_debug("%s: Unable to allocate QP\n", __FUNCTION__);
+			return ERR_PTR(-ENOMEM);
+		}
+		spin_lock_init(&qp->lock);
+		if (pd->uobject) {
+			/* userspace specific */
+		}
+
+		err = c2_alloc_qp(to_c2dev(pd->device),
+				  to_c2pd(pd), init_attr, qp);
+
+		if (err && pd->uobject) {
+			/* userspace specific */
+		}
+
+		break;
+	default:
+		pr_debug("%s: Invalid QP type: %d\n", __FUNCTION__,
+			init_attr->qp_type);
+		return ERR_PTR(-EINVAL);
+		break;
+	}
+
+	if (err) {
+		kfree(qp);
+		return ERR_PTR(err);
+	}
+
+	return &qp->ibqp;
+}
+
+static int c2_destroy_qp(struct ib_qp *ib_qp)
+{
+	struct c2_qp *qp = to_c2qp(ib_qp);
+
+	pr_debug("%s:%u qp=%p,qp->state=%d\n",
+		__FUNCTION__, __LINE__,ib_qp,qp->state);
+	c2_free_qp(to_c2dev(ib_qp->device), qp);
+	kfree(qp);
+	return 0;
+}
+
+static struct ib_cq *c2_create_cq(struct ib_device *ibdev, int entries,
+				  struct ib_ucontext *context,
+				  struct ib_udata *udata)
+{
+	struct c2_cq *cq;
+	int err;
+
+	cq = kmalloc(sizeof(*cq), GFP_KERNEL);
+	if (!cq) {
+		pr_debug("%s: Unable to allocate CQ\n", __FUNCTION__);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	err = c2_init_cq(to_c2dev(ibdev), entries, NULL, cq);
+	if (err) {
+		pr_debug("%s: error initializing CQ\n", __FUNCTION__);
+		kfree(cq);
+		return ERR_PTR(err);
+	}
+
+	return &cq->ibcq;
+}
+
+static int c2_destroy_cq(struct ib_cq *ib_cq)
+{
+	struct c2_cq *cq = to_c2cq(ib_cq);
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	c2_free_cq(to_c2dev(ib_cq->device), cq);
+	kfree(cq);
+
+	return 0;
+}
+
+static inline u32 c2_convert_access(int acc)
+{
+	return (acc & IB_ACCESS_REMOTE_WRITE ? C2_ACF_REMOTE_WRITE : 0) |
+	    (acc & IB_ACCESS_REMOTE_READ ? C2_ACF_REMOTE_READ : 0) |
+	    (acc & IB_ACCESS_LOCAL_WRITE ? C2_ACF_LOCAL_WRITE : 0) |
+	    C2_ACF_LOCAL_READ | C2_ACF_WINDOW_BIND;
+}
+
+static struct ib_mr *c2_reg_phys_mr(struct ib_pd *ib_pd,
+				    struct ib_phys_buf *buffer_list,
+				    int num_phys_buf, int acc, u64 * iova_start)
+{
+	struct c2_mr *mr;
+	u64 *page_list;
+	u32 total_len;
+	int err, i, j, k, page_shift, pbl_depth;
+
+	pbl_depth = 0;
+	total_len = 0;
+
+	page_shift = PAGE_SHIFT;
+	/*
+	 * If there is only 1 buffer we assume this could
+	 * be a map of all phy mem...use a 32k page_shift.
+	 */
+	if (num_phys_buf == 1)
+		page_shift += 3;
+
+	for (i = 0; i < num_phys_buf; i++) {
+
+		if (buffer_list[i].addr & ~PAGE_MASK) {
+			pr_debug("Unaligned Memory Buffer: 0x%x\n",
+				(unsigned int) buffer_list[i].addr);
+			return ERR_PTR(-EINVAL);
+		}
+
+		if (!buffer_list[i].size) {
+			pr_debug("Invalid Buffer Size\n");
+			return ERR_PTR(-EINVAL);
+		}
+
+		total_len += buffer_list[i].size;
+		pbl_depth += ALIGN(buffer_list[i].size,
+				   (1 << page_shift)) >> page_shift;
+	}
+
+	page_list = vmalloc(sizeof(u64) * pbl_depth);
+	if (!page_list) {
+		pr_debug("couldn't vmalloc page_list of size %zd\n",
+			(sizeof(u64) * pbl_depth));
+		return ERR_PTR(-ENOMEM);
+	}
+
+	for (i = 0, j = 0; i < num_phys_buf; i++) {
+
+		int naddrs;
+
+ 		naddrs = ALIGN(buffer_list[i].size,
+			       (1 << page_shift)) >> page_shift;
+		for (k = 0; k < naddrs; k++)
+			page_list[j++] = (buffer_list[i].addr +
+						     (k << page_shift));
+	}
+
+	mr = kmalloc(sizeof(*mr), GFP_KERNEL);
+	if (!mr)
+		return ERR_PTR(-ENOMEM);
+
+	mr->pd = to_c2pd(ib_pd);
+	pr_debug("%s - page shift %d, pbl_depth %d, total_len %u, "
+		"*iova_start %llx, first pa %llx, last pa %llx\n",
+		__FUNCTION__, page_shift, pbl_depth, total_len,
+		*iova_start, page_list[0], page_list[pbl_depth-1]);
+  	err = c2_nsmr_register_phys_kern(to_c2dev(ib_pd->device), page_list,
+ 					 (1 << page_shift), pbl_depth,
+					 total_len, 0, iova_start,
+					 c2_convert_access(acc), mr);
+	vfree(page_list);
+	if (err) {
+		kfree(mr);
+		return ERR_PTR(err);
+	}
+
+	return &mr->ibmr;
+}
+
+static struct ib_mr *c2_get_dma_mr(struct ib_pd *pd, int acc)
+{
+	struct ib_phys_buf bl;
+	u64 kva = 0;
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	/* AMSO1100 limit */
+	bl.size = 0xffffffff;
+	bl.addr = 0;
+	return c2_reg_phys_mr(pd, &bl, 1, acc, &kva);
+}
+
+static struct ib_mr *c2_reg_user_mr(struct ib_pd *pd, struct ib_umem *region,
+				    int acc, struct ib_udata *udata)
+{
+	u64 *pages;
+	u64 kva = 0;
+	int shift, n, len;
+	int i, j, k;
+	int err = 0;
+	struct ib_umem_chunk *chunk;
+	struct c2_pd *c2pd = to_c2pd(pd);
+	struct c2_mr *c2mr;
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	shift = ffs(region->page_size) - 1;
+
+	c2mr = kmalloc(sizeof(*c2mr), GFP_KERNEL);
+	if (!c2mr)
+		return ERR_PTR(-ENOMEM);
+	c2mr->pd = c2pd;
+
+	n = 0;
+	list_for_each_entry(chunk, &region->chunk_list, list)
+		n += chunk->nents;
+
+	pages = kmalloc(n * sizeof(u64), GFP_KERNEL);
+	if (!pages) {
+		err = -ENOMEM;
+		goto err;
+	}
+
+	i = 0;
+	list_for_each_entry(chunk, &region->chunk_list, list) {
+		for (j = 0; j < chunk->nmap; ++j) {
+			len = sg_dma_len(&chunk->page_list[j]) >> shift;
+			for (k = 0; k < len; ++k) {
+				pages[i++] =
+					sg_dma_address(&chunk->page_list[j]) +
+					(region->page_size * k);
+			}
+		}
+	}
+
+	kva = (u64)region->virt_base;
+  	err = c2_nsmr_register_phys_kern(to_c2dev(pd->device),
+					 pages,
+ 					 region->page_size,
+					 i,
+					 region->length,
+					 region->offset,
+					 &kva,
+					 c2_convert_access(acc),
+					 c2mr);
+	kfree(pages);
+	if (err) {
+		kfree(c2mr);
+		return ERR_PTR(err);
+	}
+	return &c2mr->ibmr;
+
+err:
+	kfree(c2mr);
+	return ERR_PTR(err);
+}
+
+static int c2_dereg_mr(struct ib_mr *ib_mr)
+{
+	struct c2_mr *mr = to_c2mr(ib_mr);
+	int err;
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	err = c2_stag_dealloc(to_c2dev(ib_mr->device), ib_mr->lkey);
+	if (err)
+		pr_debug("c2_stag_dealloc failed: %d\n", err);
+	else
+		kfree(mr);
+
+	return err;
+}
+
+static ssize_t show_rev(struct class_device *cdev, char *buf)
+{
+	struct c2_dev *dev = container_of(cdev, struct c2_dev, ibdev.class_dev);
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return sprintf(buf, "%x\n", dev->props.hw_ver);
+}
+
+static ssize_t show_fw_ver(struct class_device *cdev, char *buf)
+{
+	struct c2_dev *dev = container_of(cdev, struct c2_dev, ibdev.class_dev);
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return sprintf(buf, "%x.%x.%x\n",
+		       (int) (dev->props.fw_ver >> 32),
+		       (int) (dev->props.fw_ver >> 16) & 0xffff,
+		       (int) (dev->props.fw_ver & 0xffff));
+}
+
+static ssize_t show_hca(struct class_device *cdev, char *buf)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return sprintf(buf, "AMSO1100\n");
+}
+
+static ssize_t show_board(struct class_device *cdev, char *buf)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return sprintf(buf, "%.*s\n", 32, "AMSO1100 Board ID");
+}
+
+static CLASS_DEVICE_ATTR(hw_rev, S_IRUGO, show_rev, NULL);
+static CLASS_DEVICE_ATTR(fw_ver, S_IRUGO, show_fw_ver, NULL);
+static CLASS_DEVICE_ATTR(hca_type, S_IRUGO, show_hca, NULL);
+static CLASS_DEVICE_ATTR(board_id, S_IRUGO, show_board, NULL);
+
+static struct class_device_attribute *c2_class_attributes[] = {
+	&class_device_attr_hw_rev,
+	&class_device_attr_fw_ver,
+	&class_device_attr_hca_type,
+	&class_device_attr_board_id
+};
+
+static int c2_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
+			int attr_mask, struct ib_udata *udata)
+{
+	int err;
+
+	err =
+	    c2_qp_modify(to_c2dev(ibqp->device), to_c2qp(ibqp), attr,
+			 attr_mask);
+
+	return err;
+}
+
+static int c2_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return -ENOSYS;
+}
+
+static int c2_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return -ENOSYS;
+}
+
+static int c2_process_mad(struct ib_device *ibdev,
+			  int mad_flags,
+			  u8 port_num,
+			  struct ib_wc *in_wc,
+			  struct ib_grh *in_grh,
+			  struct ib_mad *in_mad, struct ib_mad *out_mad)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return -ENOSYS;
+}
+
+static int c2_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	/* Request a connection */
+	return c2_llp_connect(cm_id, iw_param);
+}
+
+static int c2_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *iw_param)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	/* Accept the new connection */
+	return c2_llp_accept(cm_id, iw_param);
+}
+
+static int c2_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
+{
+	int err;
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	err = c2_llp_reject(cm_id, pdata, pdata_len);
+	return err;
+}
+
+static int c2_service_create(struct iw_cm_id *cm_id, int backlog)
+{
+	int err;
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	err = c2_llp_service_create(cm_id, backlog);
+	pr_debug("%s:%u err=%d\n",
+		__FUNCTION__, __LINE__,
+		err);
+	return err;
+}
+
+static int c2_service_destroy(struct iw_cm_id *cm_id)
+{
+	int err;
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+
+	err = c2_llp_service_destroy(cm_id);
+
+	return err;
+}
+
+static int c2_pseudo_up(struct net_device *netdev)
+{
+	struct in_device *ind;
+	struct c2_dev *c2dev = netdev->priv;
+
+	ind = in_dev_get(netdev);
+	if (!ind)
+		return 0;
+
+	pr_debug("adding...\n");
+	for_ifa(ind) {
+#ifdef DEBUG
+		u8 *ip = (u8 *) & ifa->ifa_address;
+
+		pr_debug("%s: %d.%d.%d.%d\n",
+		       ifa->ifa_label, ip[0], ip[1], ip[2], ip[3]);
+#endif
+		c2_add_addr(c2dev, ifa->ifa_address, ifa->ifa_mask);
+	}
+	endfor_ifa(ind);
+	in_dev_put(ind);
+
+	return 0;
+}
+
+static int c2_pseudo_down(struct net_device *netdev)
+{
+	struct in_device *ind;
+	struct c2_dev *c2dev = netdev->priv;
+
+	ind = in_dev_get(netdev);
+	if (!ind)
+		return 0;
+
+	pr_debug("deleting...\n");
+	for_ifa(ind) {
+#ifdef DEBUG
+		u8 *ip = (u8 *) & ifa->ifa_address;
+
+		pr_debug("%s: %d.%d.%d.%d\n",
+		       ifa->ifa_label, ip[0], ip[1], ip[2], ip[3]);
+#endif
+		c2_del_addr(c2dev, ifa->ifa_address, ifa->ifa_mask);
+	}
+	endfor_ifa(ind);
+	in_dev_put(ind);
+
+	return 0;
+}
+
+static int c2_pseudo_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
+{
+	kfree_skb(skb);
+	return NETDEV_TX_OK;
+}
+
+static int c2_pseudo_change_mtu(struct net_device *netdev, int new_mtu)
+{
+	int ret = 0;
+
+	if (new_mtu < ETH_ZLEN || new_mtu > ETH_JUMBO_MTU)
+		return -EINVAL;
+
+	netdev->mtu = new_mtu;
+
+	/* TODO: Tell rnic about new rmda interface mtu */
+	return ret;
+}
+
+static void setup(struct net_device *netdev)
+{
+	SET_MODULE_OWNER(netdev);
+	netdev->open = c2_pseudo_up;
+	netdev->stop = c2_pseudo_down;
+	netdev->hard_start_xmit = c2_pseudo_xmit_frame;
+	netdev->get_stats = NULL;
+	netdev->tx_timeout = NULL;
+	netdev->set_mac_address = NULL;
+	netdev->change_mtu = c2_pseudo_change_mtu;
+	netdev->watchdog_timeo = 0;
+	netdev->type = ARPHRD_ETHER;
+	netdev->mtu = 1500;
+	netdev->hard_header_len = ETH_HLEN;
+	netdev->addr_len = ETH_ALEN;
+	netdev->tx_queue_len = 0;
+	netdev->flags |= IFF_NOARP;
+	return;
+}
+
+static struct net_device *c2_pseudo_netdev_init(struct c2_dev *c2dev)
+{
+	char name[IFNAMSIZ];
+	struct net_device *netdev;
+
+	/* change ethxxx to iwxxx */
+	strcpy(name, "iw");
+	strcat(name, &c2dev->netdev->name[3]);
+	netdev = alloc_netdev(sizeof(*netdev), name, setup);
+	if (!netdev) {
+		printk(KERN_ERR PFX "%s -  etherdev alloc failed",
+			__FUNCTION__);
+		return NULL;
+	}
+
+	netdev->priv = c2dev;
+
+	SET_NETDEV_DEV(netdev, &c2dev->pcidev->dev);
+
+	memcpy_fromio(netdev->dev_addr, c2dev->kva + C2_REGS_RDMA_ENADDR, 6);
+
+	/* Print out the MAC address */
+	pr_debug("%s: MAC %02X:%02X:%02X:%02X:%02X:%02X\n",
+		netdev->name,
+		netdev->dev_addr[0], netdev->dev_addr[1], netdev->dev_addr[2],
+		netdev->dev_addr[3], netdev->dev_addr[4], netdev->dev_addr[5]);
+
+#if 0
+	/* Disable network packets */
+	netif_stop_queue(netdev);
+#endif
+	return netdev;
+}
+
+int c2_register_device(struct c2_dev *dev)
+{
+	int ret;
+	int i;
+
+	/* Register pseudo network device */
+	dev->pseudo_netdev = c2_pseudo_netdev_init(dev);
+	if (dev->pseudo_netdev) {
+		ret = register_netdev(dev->pseudo_netdev);
+		if (ret) {
+			printk(KERN_ERR PFX
+				"Unable to register netdev, ret = %d\n", ret);
+			free_netdev(dev->pseudo_netdev);
+			return ret;
+		}
+	}
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	strlcpy(dev->ibdev.name, "amso%d", IB_DEVICE_NAME_MAX);
+	dev->ibdev.owner = THIS_MODULE;
+	dev->ibdev.uverbs_cmd_mask =
+	    (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) |
+	    (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) |
+	    (1ull << IB_USER_VERBS_CMD_QUERY_PORT) |
+	    (1ull << IB_USER_VERBS_CMD_ALLOC_PD) |
+	    (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) |
+	    (1ull << IB_USER_VERBS_CMD_REG_MR) |
+	    (1ull << IB_USER_VERBS_CMD_DEREG_MR) |
+	    (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) |
+	    (1ull << IB_USER_VERBS_CMD_CREATE_CQ) |
+	    (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) |
+	    (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) |
+	    (1ull << IB_USER_VERBS_CMD_CREATE_QP) |
+	    (1ull << IB_USER_VERBS_CMD_MODIFY_QP) |
+	    (1ull << IB_USER_VERBS_CMD_POLL_CQ) |
+	    (1ull << IB_USER_VERBS_CMD_DESTROY_QP) |
+	    (1ull << IB_USER_VERBS_CMD_POST_SEND) |
+	    (1ull << IB_USER_VERBS_CMD_POST_RECV);
+
+	dev->ibdev.node_type = RDMA_NODE_RNIC;
+	memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid));
+	memcpy(&dev->ibdev.node_guid, dev->pseudo_netdev->dev_addr, 6);
+	dev->ibdev.phys_port_cnt = 1;
+	dev->ibdev.dma_device = &dev->pcidev->dev;
+	dev->ibdev.class_dev.dev = &dev->pcidev->dev;
+	dev->ibdev.query_device = c2_query_device;
+	dev->ibdev.query_port = c2_query_port;
+	dev->ibdev.modify_port = c2_modify_port;
+	dev->ibdev.query_pkey = c2_query_pkey;
+	dev->ibdev.query_gid = c2_query_gid;
+	dev->ibdev.alloc_ucontext = c2_alloc_ucontext;
+	dev->ibdev.dealloc_ucontext = c2_dealloc_ucontext;
+	dev->ibdev.mmap = c2_mmap_uar;
+	dev->ibdev.alloc_pd = c2_alloc_pd;
+	dev->ibdev.dealloc_pd = c2_dealloc_pd;
+	dev->ibdev.create_ah = c2_ah_create;
+	dev->ibdev.destroy_ah = c2_ah_destroy;
+	dev->ibdev.create_qp = c2_create_qp;
+	dev->ibdev.modify_qp = c2_modify_qp;
+	dev->ibdev.destroy_qp = c2_destroy_qp;
+	dev->ibdev.create_cq = c2_create_cq;
+	dev->ibdev.destroy_cq = c2_destroy_cq;
+	dev->ibdev.poll_cq = c2_poll_cq;
+	dev->ibdev.get_dma_mr = c2_get_dma_mr;
+	dev->ibdev.reg_phys_mr = c2_reg_phys_mr;
+	dev->ibdev.reg_user_mr = c2_reg_user_mr;
+	dev->ibdev.dereg_mr = c2_dereg_mr;
+
+	dev->ibdev.alloc_fmr = NULL;
+	dev->ibdev.unmap_fmr = NULL;
+	dev->ibdev.dealloc_fmr = NULL;
+	dev->ibdev.map_phys_fmr = NULL;
+
+	dev->ibdev.attach_mcast = c2_multicast_attach;
+	dev->ibdev.detach_mcast = c2_multicast_detach;
+	dev->ibdev.process_mad = c2_process_mad;
+
+	dev->ibdev.req_notify_cq = c2_arm_cq;
+	dev->ibdev.post_send = c2_post_send;
+	dev->ibdev.post_recv = c2_post_receive;
+
+	dev->ibdev.iwcm = kmalloc(sizeof(*dev->ibdev.iwcm), GFP_KERNEL);
+	dev->ibdev.iwcm->add_ref = c2_add_ref;
+	dev->ibdev.iwcm->rem_ref = c2_rem_ref;
+	dev->ibdev.iwcm->get_qp = c2_get_qp;
+	dev->ibdev.iwcm->connect = c2_connect;
+	dev->ibdev.iwcm->accept = c2_accept;
+	dev->ibdev.iwcm->reject = c2_reject;
+	dev->ibdev.iwcm->create_listen = c2_service_create;
+	dev->ibdev.iwcm->destroy_listen = c2_service_destroy;
+
+	ret = ib_register_device(&dev->ibdev);
+	if (ret)
+		return ret;
+
+	for (i = 0; i < ARRAY_SIZE(c2_class_attributes); ++i) {
+		ret = class_device_create_file(&dev->ibdev.class_dev,
+					       c2_class_attributes[i]);
+		if (ret) {
+			unregister_netdev(dev->pseudo_netdev);
+			free_netdev(dev->pseudo_netdev);
+			ib_unregister_device(&dev->ibdev);
+			return ret;
+		}
+	}
+
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	return 0;
+}
+
+void c2_unregister_device(struct c2_dev *dev)
+{
+	pr_debug("%s:%u\n", __FUNCTION__, __LINE__);
+	unregister_netdev(dev->pseudo_netdev);
+	free_netdev(dev->pseudo_netdev);
+	ib_unregister_device(&dev->ibdev);
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_provider.h b/drivers/infiniband/hw/amso1100/c2_provider.h
new file mode 100644
index 0000000..fc90622
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_provider.h
@@ -0,0 +1,181 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef C2_PROVIDER_H
+#define C2_PROVIDER_H
+#include <linux/inetdevice.h>
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_pack.h>
+
+#include "c2_mq.h"
+#include <rdma/iw_cm.h>
+
+#define C2_MPT_FLAG_ATOMIC        (1 << 14)
+#define C2_MPT_FLAG_REMOTE_WRITE  (1 << 13)
+#define C2_MPT_FLAG_REMOTE_READ   (1 << 12)
+#define C2_MPT_FLAG_LOCAL_WRITE   (1 << 11)
+#define C2_MPT_FLAG_LOCAL_READ    (1 << 10)
+
+struct c2_buf_list {
+	void *buf;
+	 DECLARE_PCI_UNMAP_ADDR(mapping)
+};
+
+
+/* The user context keeps track of objects allocated for a
+ * particular user-mode client. */
+struct c2_ucontext {
+	struct ib_ucontext ibucontext;
+};
+
+struct c2_mtt;
+
+/* All objects associated with a PD are kept in the
+ * associated user context if present.
+ */
+struct c2_pd {
+	struct ib_pd ibpd;
+	u32 pd_id;
+};
+
+struct c2_mr {
+	struct ib_mr ibmr;
+	struct c2_pd *pd;
+};
+
+struct c2_av;
+
+enum c2_ah_type {
+	C2_AH_ON_HCA,
+	C2_AH_PCI_POOL,
+	C2_AH_KMALLOC
+};
+
+struct c2_ah {
+	struct ib_ah ibah;
+};
+
+struct c2_cq {
+	struct ib_cq ibcq;
+	spinlock_t lock;
+	atomic_t refcount;
+	int cqn;
+	int is_kernel;
+	wait_queue_head_t wait;
+
+	u32 adapter_handle;
+	struct c2_mq mq;
+};
+
+struct c2_wq {
+	spinlock_t lock;
+};
+struct iw_cm_id;
+struct c2_qp {
+	struct ib_qp ibqp;
+	struct iw_cm_id *cm_id;
+	spinlock_t lock;
+	atomic_t refcount;
+	wait_queue_head_t wait;
+	int qpn;
+
+	u32 adapter_handle;
+	u32 send_sgl_depth;
+	u32 recv_sgl_depth;
+	u32 rdma_write_sgl_depth;
+	u8 state;
+
+	struct c2_mq sq_mq;
+	struct c2_mq rq_mq;
+};
+
+struct c2_cr_query_attrs {
+	u32 local_addr;
+	u32 remote_addr;
+	u16 local_port;
+	u16 remote_port;
+};
+
+static inline struct c2_pd *to_c2pd(struct ib_pd *ibpd)
+{
+	return container_of(ibpd, struct c2_pd, ibpd);
+}
+
+static inline struct c2_ucontext *to_c2ucontext(struct ib_ucontext *ibucontext)
+{
+	return container_of(ibucontext, struct c2_ucontext, ibucontext);
+}
+
+static inline struct c2_mr *to_c2mr(struct ib_mr *ibmr)
+{
+	return container_of(ibmr, struct c2_mr, ibmr);
+}
+
+
+static inline struct c2_ah *to_c2ah(struct ib_ah *ibah)
+{
+	return container_of(ibah, struct c2_ah, ibah);
+}
+
+static inline struct c2_cq *to_c2cq(struct ib_cq *ibcq)
+{
+	return container_of(ibcq, struct c2_cq, ibcq);
+}
+
+static inline struct c2_qp *to_c2qp(struct ib_qp *ibqp)
+{
+	return container_of(ibqp, struct c2_qp, ibqp);
+}
+
+static inline int is_rnic_addr(struct net_device *netdev, u32 addr)
+{
+	struct in_device *ind;
+	int ret = 0;
+
+	ind = in_dev_get(netdev);
+	if (!ind)
+		return 0;
+
+	for_ifa(ind) {
+		if (ifa->ifa_address == addr) {
+			ret = 1;
+			break;
+		}
+	}
+	endfor_ifa(ind);
+	in_dev_put(ind);
+	return ret;
+}
+#endif				/* C2_PROVIDER_H */
diff --git a/drivers/infiniband/hw/amso1100/c2_qp.c b/drivers/infiniband/hw/amso1100/c2_qp.c
new file mode 100644
index 0000000..1226113
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_qp.c
@@ -0,0 +1,975 @@
+/*
+ * Copyright (c) 2004 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems. All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#include "c2.h"
+#include "c2_vq.h"
+#include "c2_status.h"
+
+#define C2_MAX_ORD_PER_QP 128
+#define C2_MAX_IRD_PER_QP 128
+
+#define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
+#define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
+#define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
+
+#define NO_SUPPORT -1
+static const u8 c2_opcode[] = {
+	[IB_WR_SEND] = C2_WR_TYPE_SEND,
+	[IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
+	[IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
+	[IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
+	[IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
+	[IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
+	[IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
+};
+
+static int to_c2_state(enum ib_qp_state ib_state)
+{
+	switch (ib_state) {
+	case IB_QPS_RESET:
+		return C2_QP_STATE_IDLE;
+	case IB_QPS_RTS:
+		return C2_QP_STATE_RTS;
+	case IB_QPS_SQD:
+		return C2_QP_STATE_CLOSING;
+	case IB_QPS_SQE:
+		return C2_QP_STATE_CLOSING;
+	case IB_QPS_ERR:
+		return C2_QP_STATE_ERROR;
+	default:
+		return -1;
+	}
+}
+
+static int to_ib_state(enum c2_qp_state c2_state)
+{
+	switch (c2_state) {
+	case C2_QP_STATE_IDLE:
+		return IB_QPS_RESET;
+	case C2_QP_STATE_CONNECTING:
+		return IB_QPS_RTR;
+	case C2_QP_STATE_RTS:
+		return IB_QPS_RTS;
+	case C2_QP_STATE_CLOSING:
+		return IB_QPS_SQD;
+	case C2_QP_STATE_ERROR:
+		return IB_QPS_ERR;
+	case C2_QP_STATE_TERMINATE:
+		return IB_QPS_SQE;
+	default:
+		return -1;
+	}
+}
+
+static const char *to_ib_state_str(int ib_state)
+{
+	static const char *state_str[] = {
+		"IB_QPS_RESET",
+		"IB_QPS_INIT",
+		"IB_QPS_RTR",
+		"IB_QPS_RTS",
+		"IB_QPS_SQD",
+		"IB_QPS_SQE",
+		"IB_QPS_ERR"
+	};
+	if (ib_state < IB_QPS_RESET ||
+	    ib_state > IB_QPS_ERR)
+		return "<invalid IB QP state>";
+
+	ib_state -= IB_QPS_RESET;
+	return state_str[ib_state];
+}
+
+void c2_set_qp_state(struct c2_qp *qp, int c2_state)
+{
+	int new_state = to_ib_state(c2_state);
+
+	pr_debug("%s: qp[%p] state modify %s --> %s\n",
+	       __FUNCTION__,
+		qp,
+		to_ib_state_str(qp->state),
+		to_ib_state_str(new_state));
+	qp->state = new_state;
+}
+
+#define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
+
+int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
+		 struct ib_qp_attr *attr, int attr_mask)
+{
+	struct c2wr_qp_modify_req wr;
+	struct c2wr_qp_modify_rep *reply;
+	struct c2_vq_req *vq_req;
+	unsigned long flags;
+	u8 next_state;
+	int err;
+
+	pr_debug("%s:%d qp=%p, %s --> %s\n",
+		__FUNCTION__, __LINE__,
+		qp,
+		to_ib_state_str(qp->state),
+		to_ib_state_str(attr->qp_state));
+
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req)
+		return -ENOMEM;
+
+	c2_wr_set_id(&wr, CCWR_QP_MODIFY);
+	wr.hdr.context = (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.qp_handle = qp->adapter_handle;
+	wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
+	wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
+	wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
+	wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
+
+	if (attr_mask & IB_QP_STATE) {
+		/* Ensure the state is valid */
+		if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR)
+			return -EINVAL;
+
+		wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
+
+		if (attr->qp_state == IB_QPS_ERR) {
+			spin_lock_irqsave(&qp->lock, flags);
+			if (qp->cm_id && qp->state == IB_QPS_RTS) {
+				pr_debug("Generating CLOSE event for QP-->ERR, "
+					"qp=%p, cm_id=%p\n",qp,qp->cm_id);
+				/* Generate an CLOSE event */
+				vq_req->cm_id = qp->cm_id;
+				vq_req->event = IW_CM_EVENT_CLOSE;
+			}
+			spin_unlock_irqrestore(&qp->lock, flags);
+		}
+		next_state =  attr->qp_state;
+
+	} else if (attr_mask & IB_QP_CUR_STATE) {
+
+		if (attr->cur_qp_state != IB_QPS_RTR &&
+		    attr->cur_qp_state != IB_QPS_RTS &&
+		    attr->cur_qp_state != IB_QPS_SQD &&
+		    attr->cur_qp_state != IB_QPS_SQE)
+			return -EINVAL;
+		else
+			wr.next_qp_state =
+			    cpu_to_be32(to_c2_state(attr->cur_qp_state));
+
+		next_state = attr->cur_qp_state;
+
+	} else {
+		err = 0;
+		goto bail0;
+	}
+
+	/* reference the request struct */
+	vq_req_get(c2dev, vq_req);
+
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail0;
+	}
+
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail0;
+
+	reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	err = c2_errno(reply);
+	if (!err)
+		qp->state = next_state;
+#ifdef DEBUG
+	else
+		pr_debug("%s: c2_errno=%d\n", __FUNCTION__, err);
+#endif
+	/*
+	 * If we're going to error and generating the event here, then
+	 * we need to remove the reference because there will be no
+	 * close event generated by the adapter
+	*/
+	spin_lock_irqsave(&qp->lock, flags);
+	if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
+		qp->cm_id->rem_ref(qp->cm_id);
+		qp->cm_id = NULL;
+	}
+	spin_unlock_irqrestore(&qp->lock, flags);
+
+	vq_repbuf_free(c2dev, reply);
+      bail0:
+	vq_req_free(c2dev, vq_req);
+
+	pr_debug("%s:%d qp=%p, cur_state=%s\n",
+		__FUNCTION__, __LINE__,
+		qp,
+		to_ib_state_str(qp->state));
+	return err;
+}
+
+int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
+			  int ord, int ird)
+{
+	struct c2wr_qp_modify_req wr;
+	struct c2wr_qp_modify_rep *reply;
+	struct c2_vq_req *vq_req;
+	int err;
+
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req)
+		return -ENOMEM;
+
+	c2_wr_set_id(&wr, CCWR_QP_MODIFY);
+	wr.hdr.context = (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.qp_handle = qp->adapter_handle;
+	wr.ord = cpu_to_be32(ord);
+	wr.ird = cpu_to_be32(ird);
+	wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
+	wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
+	wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
+
+	/* reference the request struct */
+	vq_req_get(c2dev, vq_req);
+
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail0;
+	}
+
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail0;
+
+	reply = (struct c2wr_qp_modify_rep *) (unsigned long)
+		vq_req->reply_msg;
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	err = c2_errno(reply);
+	vq_repbuf_free(c2dev, reply);
+      bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
+{
+	struct c2_vq_req *vq_req;
+	struct c2wr_qp_destroy_req wr;
+	struct c2wr_qp_destroy_rep *reply;
+	unsigned long flags;
+	int err;
+
+	/*
+	 * Allocate a verb request message
+	 */
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req) {
+		return -ENOMEM;
+	}
+
+	/*
+	 * Initialize the WR
+	 */
+	c2_wr_set_id(&wr, CCWR_QP_DESTROY);
+	wr.hdr.context = (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.qp_handle = qp->adapter_handle;
+
+	/*
+	 * reference the request struct.  dereferenced in the int handler.
+	 */
+	vq_req_get(c2dev, vq_req);
+
+	spin_lock_irqsave(&qp->lock, flags);
+	if (qp->cm_id && qp->state == IB_QPS_RTS) {
+		pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
+			"qp=%p, cm_id=%p\n",qp,qp->cm_id);
+		/* Generate an CLOSE event */
+		vq_req->qp = qp;
+		vq_req->cm_id = qp->cm_id;
+		vq_req->event = IW_CM_EVENT_CLOSE;
+	}
+	spin_unlock_irqrestore(&qp->lock, flags);
+
+	/*
+	 * Send WR to adapter
+	 */
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail0;
+	}
+
+	/*
+	 * Wait for reply from adapter
+	 */
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err) {
+		goto bail0;
+	}
+
+	/*
+	 * Process reply
+	 */
+	reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	spin_lock_irqsave(&qp->lock, flags);
+	if (qp->cm_id) {
+		qp->cm_id->rem_ref(qp->cm_id);
+		qp->cm_id = NULL;
+	}
+	spin_unlock_irqrestore(&qp->lock, flags);
+
+	vq_repbuf_free(c2dev, reply);
+      bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
+{
+	int ret;
+
+        do {
+		spin_lock_irq(&c2dev->qp_table.lock);
+		ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
+					c2dev->qp_table.last++, &qp->qpn);
+		spin_unlock_irq(&c2dev->qp_table.lock);
+        } while ((ret == -EAGAIN) &&
+	 	 idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
+	return ret;
+}
+
+static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
+{
+	spin_lock_irq(&c2dev->qp_table.lock);
+	idr_remove(&c2dev->qp_table.idr, qpn);
+	spin_unlock_irq(&c2dev->qp_table.lock);
+}
+
+struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
+{
+	unsigned long flags;
+	struct c2_qp *qp;
+
+	spin_lock_irqsave(&c2dev->qp_table.lock, flags);
+	qp = idr_find(&c2dev->qp_table.idr, qpn);
+	spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
+	return qp;
+}
+
+int c2_alloc_qp(struct c2_dev *c2dev,
+		struct c2_pd *pd,
+		struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
+{
+	struct c2wr_qp_create_req wr;
+	struct c2wr_qp_create_rep *reply;
+	struct c2_vq_req *vq_req;
+	struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
+	struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
+	unsigned long peer_pa;
+	u32 q_size, msg_size, mmap_size;
+	void __iomem *mmap;
+	int err;
+
+	err = c2_alloc_qpn(c2dev, qp);
+	if (err)
+		return err;
+	qp->ibqp.qp_num = qp->qpn;
+	qp->ibqp.qp_type = IB_QPT_RC;
+
+	/* Allocate the SQ and RQ shared pointers */
+	qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
+					 &qp->sq_mq.shared_dma, GFP_KERNEL);
+	if (!qp->sq_mq.shared) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
+					 &qp->rq_mq.shared_dma, GFP_KERNEL);
+	if (!qp->rq_mq.shared) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+
+	/* Allocate the verbs request */
+	vq_req = vq_req_alloc(c2dev);
+	if (vq_req == NULL) {
+		err = -ENOMEM;
+		goto bail2;
+	}
+
+	/* Initialize the work request */
+	memset(&wr, 0, sizeof(wr));
+	c2_wr_set_id(&wr, CCWR_QP_CREATE);
+	wr.hdr.context = (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+	wr.sq_cq_handle = send_cq->adapter_handle;
+	wr.rq_cq_handle = recv_cq->adapter_handle;
+	wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
+	wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
+	wr.srq_handle = 0;
+	wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
+			       QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
+	wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
+	wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
+	wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
+	wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
+	wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
+	wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
+	wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
+	wr.pd_id = pd->pd_id;
+	wr.user_context = (unsigned long) qp;
+
+	vq_req_get(c2dev, vq_req);
+
+	/* Send the WR to the adapter */
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail3;
+	}
+
+	/* Wait for the verb reply  */
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err) {
+		goto bail3;
+	}
+
+	/* Process the reply */
+	reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail3;
+	}
+
+	if ((err = c2_wr_get_result(reply)) != 0) {
+		goto bail4;
+	}
+
+	/* Fill in the kernel QP struct */
+	atomic_set(&qp->refcount, 1);
+	qp->adapter_handle = reply->qp_handle;
+	qp->state = IB_QPS_RESET;
+	qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
+	qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
+	qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
+
+	/* Initialize the SQ MQ */
+	q_size = be32_to_cpu(reply->sq_depth);
+	msg_size = be32_to_cpu(reply->sq_msg_size);
+	peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
+	mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
+	mmap = ioremap_nocache(peer_pa, mmap_size);
+	if (!mmap) {
+		err = -ENOMEM;
+		goto bail5;
+	}
+
+	c2_mq_req_init(&qp->sq_mq,
+		       be32_to_cpu(reply->sq_mq_index),
+		       q_size,
+		       msg_size,
+		       mmap + sizeof(struct c2_mq_shared),	/* pool start */
+		       mmap,				/* peer */
+		       C2_MQ_ADAPTER_TARGET);
+
+	/* Initialize the RQ mq */
+	q_size = be32_to_cpu(reply->rq_depth);
+	msg_size = be32_to_cpu(reply->rq_msg_size);
+	peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
+	mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
+	mmap = ioremap_nocache(peer_pa, mmap_size);
+	if (!mmap) {
+		err = -ENOMEM;
+		goto bail6;
+	}
+
+	c2_mq_req_init(&qp->rq_mq,
+		       be32_to_cpu(reply->rq_mq_index),
+		       q_size,
+		       msg_size,
+		       mmap + sizeof(struct c2_mq_shared),	/* pool start */
+		       mmap,				/* peer */
+		       C2_MQ_ADAPTER_TARGET);
+
+	vq_repbuf_free(c2dev, reply);
+	vq_req_free(c2dev, vq_req);
+
+	return 0;
+
+      bail6:
+	iounmap(qp->sq_mq.peer);
+      bail5:
+	destroy_qp(c2dev, qp);
+      bail4:
+	vq_repbuf_free(c2dev, reply);
+      bail3:
+	vq_req_free(c2dev, vq_req);
+      bail2:
+	c2_free_mqsp(qp->rq_mq.shared);
+      bail1:
+	c2_free_mqsp(qp->sq_mq.shared);
+      bail0:
+	c2_free_qpn(c2dev, qp->qpn);
+	return err;
+}
+
+void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
+{
+	struct c2_cq *send_cq;
+	struct c2_cq *recv_cq;
+
+	send_cq = to_c2cq(qp->ibqp.send_cq);
+	recv_cq = to_c2cq(qp->ibqp.recv_cq);
+
+	/*
+	 * Lock CQs here, so that CQ polling code can do QP lookup
+	 * without taking a lock.
+	 */
+	spin_lock_irq(&send_cq->lock);
+	if (send_cq != recv_cq)
+		spin_lock(&recv_cq->lock);
+
+	c2_free_qpn(c2dev, qp->qpn);
+
+	if (send_cq != recv_cq)
+		spin_unlock(&recv_cq->lock);
+	spin_unlock_irq(&send_cq->lock);
+
+	/*
+	 * Destory qp in the rnic...
+	 */
+	destroy_qp(c2dev, qp);
+
+	/*
+	 * Mark any unreaped CQEs as null and void.
+	 */
+	c2_cq_clean(c2dev, qp, send_cq->cqn);
+	if (send_cq != recv_cq)
+		c2_cq_clean(c2dev, qp, recv_cq->cqn);
+	/*
+	 * Unmap the MQs and return the shared pointers
+	 * to the message pool.
+	 */
+	iounmap(qp->sq_mq.peer);
+	iounmap(qp->rq_mq.peer);
+	c2_free_mqsp(qp->sq_mq.shared);
+	c2_free_mqsp(qp->rq_mq.shared);
+
+	atomic_dec(&qp->refcount);
+	wait_event(qp->wait, !atomic_read(&qp->refcount));
+}
+
+/*
+ * Function: move_sgl
+ *
+ * Description:
+ * Move an SGL from the user's work request struct into a CCIL Work Request
+ * message, swapping to WR byte order and ensure the total length doesn't
+ * overflow.
+ *
+ * IN:
+ * dst		- ptr to CCIL Work Request message SGL memory.
+ * src		- ptr to the consumers SGL memory.
+ *
+ * OUT: none
+ *
+ * Return:
+ * CCIL status codes.
+ */
+static int
+move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
+	 u8 * actual_count)
+{
+	u32 tot = 0;		/* running total */
+	u8 acount = 0;		/* running total non-0 len sge's */
+
+	while (count > 0) {
+		/*
+		 * If the addition of this SGE causes the
+		 * total SGL length to exceed 2^32-1, then
+		 * fail-n-bail.
+		 *
+		 * If the current total plus the next element length
+		 * wraps, then it will go negative and be less than the
+		 * current total...
+		 */
+		if ((tot + src->length) < tot) {
+			return -EINVAL;
+		}
+		/*
+		 * Bug: 1456 (as well as 1498 & 1643)
+		 * Skip over any sge's supplied with len=0
+		 */
+		if (src->length) {
+			tot += src->length;
+			dst->stag = cpu_to_be32(src->lkey);
+			dst->to = cpu_to_be64(src->addr);
+			dst->length = cpu_to_be32(src->length);
+			dst++;
+			acount++;
+		}
+		src++;
+		count--;
+	}
+
+	if (acount == 0) {
+		/*
+		 * Bug: 1476 (as well as 1498, 1456 and 1643)
+		 * Setup the SGL in the WR to make it easier for the RNIC.
+		 * This way, the FW doesn't have to deal with special cases.
+		 * Setting length=0 should be sufficient.
+		 */
+		dst->stag = 0;
+		dst->to = 0;
+		dst->length = 0;
+	}
+
+	*p_len = tot;
+	*actual_count = acount;
+	return 0;
+}
+
+/*
+ * Function: c2_activity (private function)
+ *
+ * Description:
+ * Post an mq index to the host->adapter activity fifo.
+ *
+ * IN:
+ * c2dev	- ptr to c2dev structure
+ * mq_index	- mq index to post
+ * shared	- value most recently written to shared
+ *
+ * OUT:
+ *
+ * Return:
+ * none
+ */
+static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
+{
+	/*
+	 * First read the register to see if the FIFO is full, and if so,
+	 * spin until it's not.  This isn't perfect -- there is no
+	 * synchronization among the clients of the register, but in
+	 * practice it prevents multiple CPU from hammering the bus
+	 * with PCI RETRY. Note that when this does happen, the card
+	 * cannot get on the bus and the card and system hang in a
+	 * deadlock -- thus the need for this code. [TOT]
+	 */
+	while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000) {
+		set_current_state(TASK_UNINTERRUPTIBLE);
+		schedule_timeout(0);
+	}
+
+	__raw_writel(C2_HINT_MAKE(mq_index, shared),
+		     c2dev->regs + PCI_BAR0_ADAPTER_HINT);
+}
+
+/*
+ * Function: qp_wr_post
+ *
+ * Description:
+ * This in-line function allocates a MQ msg, then moves the host-copy of
+ * the completed WR into msg.  Then it posts the message.
+ *
+ * IN:
+ * q		- ptr to user MQ.
+ * wr		- ptr to host-copy of the WR.
+ * qp		- ptr to user qp
+ * size		- Number of bytes to post.  Assumed to be divisible by 4.
+ *
+ * OUT: none
+ *
+ * Return:
+ * CCIL status codes.
+ */
+static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
+{
+	union c2wr *msg;
+
+	msg = c2_mq_alloc(q);
+	if (msg == NULL) {
+		return -EINVAL;
+	}
+#ifdef CCMSGMAGIC
+	((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
+#endif
+
+	/*
+	 * Since all header fields in the WR are the same as the
+	 * CQE, set the following so the adapter need not.
+	 */
+	c2_wr_set_result(wr, CCERR_PENDING);
+
+	/*
+	 * Copy the wr down to the adapter
+	 */
+	memcpy((void *) msg, (void *) wr, size);
+
+	c2_mq_produce(q);
+	return 0;
+}
+
+
+int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
+		 struct ib_send_wr **bad_wr)
+{
+	struct c2_dev *c2dev = to_c2dev(ibqp->device);
+	struct c2_qp *qp = to_c2qp(ibqp);
+	union c2wr wr;
+	int err = 0;
+
+	u32 flags;
+	u32 tot_len;
+	u8 actual_sge_count;
+	u32 msg_size;
+
+	if (qp->state > IB_QPS_RTS)
+		return -EINVAL;
+
+	while (ib_wr) {
+
+		flags = 0;
+		wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
+		if (ib_wr->send_flags & IB_SEND_SIGNALED) {
+			flags |= SQ_SIGNALED;
+		}
+
+		switch (ib_wr->opcode) {
+		case IB_WR_SEND:
+			if (ib_wr->send_flags & IB_SEND_SOLICITED) {
+				c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
+				msg_size = sizeof(struct c2wr_send_req);
+			} else {
+				c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
+				msg_size = sizeof(struct c2wr_send_req);
+			}
+
+			wr.sqwr.send.remote_stag = 0;
+			msg_size += sizeof(struct c2_data_addr) * ib_wr->num_sge;
+			if (ib_wr->num_sge > qp->send_sgl_depth) {
+				err = -EINVAL;
+				break;
+			}
+			if (ib_wr->send_flags & IB_SEND_FENCE) {
+				flags |= SQ_READ_FENCE;
+			}
+			err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
+				       ib_wr->sg_list,
+				       ib_wr->num_sge,
+				       &tot_len, &actual_sge_count);
+			wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
+			c2_wr_set_sge_count(&wr, actual_sge_count);
+			break;
+		case IB_WR_RDMA_WRITE:
+			c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
+			msg_size = sizeof(struct c2wr_rdma_write_req) +
+			    (sizeof(struct c2_data_addr) * ib_wr->num_sge);
+			if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
+				err = -EINVAL;
+				break;
+			}
+			if (ib_wr->send_flags & IB_SEND_FENCE) {
+				flags |= SQ_READ_FENCE;
+			}
+			wr.sqwr.rdma_write.remote_stag =
+			    cpu_to_be32(ib_wr->wr.rdma.rkey);
+			wr.sqwr.rdma_write.remote_to =
+			    cpu_to_be64(ib_wr->wr.rdma.remote_addr);
+			err = move_sgl((struct c2_data_addr *)
+				       & (wr.sqwr.rdma_write.data),
+				       ib_wr->sg_list,
+				       ib_wr->num_sge,
+				       &tot_len, &actual_sge_count);
+			wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
+			c2_wr_set_sge_count(&wr, actual_sge_count);
+			break;
+		case IB_WR_RDMA_READ:
+			c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
+			msg_size = sizeof(struct c2wr_rdma_read_req);
+
+			/* IWarp only suppots 1 sge for RDMA reads */
+			if (ib_wr->num_sge > 1) {
+				err = -EINVAL;
+				break;
+			}
+
+			/*
+			 * Move the local and remote stag/to/len into the WR.
+			 */
+			wr.sqwr.rdma_read.local_stag =
+			    cpu_to_be32(ib_wr->sg_list->lkey);
+			wr.sqwr.rdma_read.local_to =
+			    cpu_to_be64(ib_wr->sg_list->addr);
+			wr.sqwr.rdma_read.remote_stag =
+			    cpu_to_be32(ib_wr->wr.rdma.rkey);
+			wr.sqwr.rdma_read.remote_to =
+			    cpu_to_be64(ib_wr->wr.rdma.remote_addr);
+			wr.sqwr.rdma_read.length =
+			    cpu_to_be32(ib_wr->sg_list->length);
+			break;
+		default:
+			/* error */
+			msg_size = 0;
+			err = -EINVAL;
+			break;
+		}
+
+		/*
+		 * If we had an error on the last wr build, then
+		 * break out.  Possible errors include bogus WR
+		 * type, and a bogus SGL length...
+		 */
+		if (err) {
+			break;
+		}
+
+		/*
+		 * Store flags
+		 */
+		c2_wr_set_flags(&wr, flags);
+
+		/*
+		 * Post the puppy!
+		 */
+		err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
+		if (err) {
+			break;
+		}
+
+		/*
+		 * Enqueue mq index to activity FIFO.
+		 */
+		c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
+
+		ib_wr = ib_wr->next;
+	}
+
+	if (err)
+		*bad_wr = ib_wr;
+	return err;
+}
+
+int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
+		    struct ib_recv_wr **bad_wr)
+{
+	struct c2_dev *c2dev = to_c2dev(ibqp->device);
+	struct c2_qp *qp = to_c2qp(ibqp);
+	union c2wr wr;
+	int err = 0;
+
+	if (qp->state > IB_QPS_RTS)
+		return -EINVAL;
+
+	/*
+	 * Try and post each work request
+	 */
+	while (ib_wr) {
+		u32 tot_len;
+		u8 actual_sge_count;
+
+		if (ib_wr->num_sge > qp->recv_sgl_depth) {
+			err = -EINVAL;
+			break;
+		}
+
+		/*
+		 * Create local host-copy of the WR
+		 */
+		wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
+		c2_wr_set_id(&wr, CCWR_RECV);
+		c2_wr_set_flags(&wr, 0);
+
+		/* sge_count is limited to eight bits. */
+		BUG_ON(ib_wr->num_sge >= 256);
+		err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
+			       ib_wr->sg_list,
+			       ib_wr->num_sge, &tot_len, &actual_sge_count);
+		c2_wr_set_sge_count(&wr, actual_sge_count);
+
+		/*
+		 * If we had an error on the last wr build, then
+		 * break out.  Possible errors include bogus WR
+		 * type, and a bogus SGL length...
+		 */
+		if (err) {
+			break;
+		}
+
+		err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
+		if (err) {
+			break;
+		}
+
+		/*
+		 * Enqueue mq index to activity FIFO
+		 */
+		c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
+
+		ib_wr = ib_wr->next;
+	}
+
+	if (err)
+		*bad_wr = ib_wr;
+	return err;
+}
+
+void __devinit c2_init_qp_table(struct c2_dev *c2dev)
+{
+	spin_lock_init(&c2dev->qp_table.lock);
+	idr_init(&c2dev->qp_table.idr);
+}
+
+void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
+{
+	idr_destroy(&c2dev->qp_table.idr);
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_rnic.c b/drivers/infiniband/hw/amso1100/c2_rnic.c
new file mode 100644
index 0000000..f49a32b
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_rnic.c
@@ -0,0 +1,664 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/pci.h>
+#include <linux/netdevice.h>
+#include <linux/etherdevice.h>
+#include <linux/delay.h>
+#include <linux/ethtool.h>
+#include <linux/mii.h>
+#include <linux/if_vlan.h>
+#include <linux/crc32.h>
+#include <linux/in.h>
+#include <linux/ip.h>
+#include <linux/tcp.h>
+#include <linux/init.h>
+#include <linux/dma-mapping.h>
+#include <linux/mm.h>
+#include <linux/inet.h>
+#include <linux/vmalloc.h>
+
+#include <linux/route.h>
+
+#include <asm/io.h>
+#include <asm/irq.h>
+#include <asm/byteorder.h>
+#include <rdma/ib_smi.h>
+#include "c2.h"
+#include "c2_vq.h"
+
+/* Device capabilities */
+#define C2_MIN_PAGESIZE  1024
+
+#define C2_MAX_MRS       32768
+#define C2_MAX_QPS       16000
+#define C2_MAX_WQE_SZ    256
+#define C2_MAX_QP_WR     ((128*1024)/C2_MAX_WQE_SZ)
+#define C2_MAX_SGES      4
+#define C2_MAX_SGE_RD    1
+#define C2_MAX_CQS       32768
+#define C2_MAX_CQES      4096
+#define C2_MAX_PDS       16384
+
+/*
+ * Send the adapter INIT message to the amso1100
+ */
+static int c2_adapter_init(struct c2_dev *c2dev)
+{
+	struct c2wr_init_req wr;
+	int err;
+
+	memset(&wr, 0, sizeof(wr));
+	c2_wr_set_id(&wr, CCWR_INIT);
+	wr.hdr.context = 0;
+	wr.hint_count = cpu_to_be64(c2dev->hint_count_dma);
+	wr.q0_host_shared = cpu_to_be64(c2dev->req_vq.shared_dma);
+	wr.q1_host_shared = cpu_to_be64(c2dev->rep_vq.shared_dma);
+	wr.q1_host_msg_pool = cpu_to_be64(c2dev->rep_vq.host_dma);
+	wr.q2_host_shared = cpu_to_be64(c2dev->aeq.shared_dma);
+	wr.q2_host_msg_pool = cpu_to_be64(c2dev->aeq.host_dma);
+
+	/* Post the init message */
+	err = vq_send_wr(c2dev, (union c2wr *) & wr);
+
+	return err;
+}
+
+/*
+ * Send the adapter TERM message to the amso1100
+ */
+static void c2_adapter_term(struct c2_dev *c2dev)
+{
+	struct c2wr_init_req wr;
+
+	memset(&wr, 0, sizeof(wr));
+	c2_wr_set_id(&wr, CCWR_TERM);
+	wr.hdr.context = 0;
+
+	/* Post the init message */
+	vq_send_wr(c2dev, (union c2wr *) & wr);
+	c2dev->init = 0;
+
+	return;
+}
+
+/*
+ * Query the adapter
+ */
+static int c2_rnic_query(struct c2_dev *c2dev, struct ib_device_attr *props)
+{
+	struct c2_vq_req *vq_req;
+	struct c2wr_rnic_query_req wr;
+	struct c2wr_rnic_query_rep *reply;
+	int err;
+
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req)
+		return -ENOMEM;
+
+	c2_wr_set_id(&wr, CCWR_RNIC_QUERY);
+	wr.hdr.context = (unsigned long) vq_req;
+	wr.rnic_handle = c2dev->adapter_handle;
+
+	vq_req_get(c2dev, vq_req);
+
+	err = vq_send_wr(c2dev, (union c2wr *) &wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail1;
+	}
+
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail1;
+
+	reply =
+	    (struct c2wr_rnic_query_rep *) (unsigned long) (vq_req->reply_msg);
+	if (!reply)
+		err = -ENOMEM;
+
+	err = c2_errno(reply);
+	if (err)
+		goto bail2;
+
+	props->fw_ver =
+		((u64)be32_to_cpu(reply->fw_ver_major) << 32) |
+		((be32_to_cpu(reply->fw_ver_minor) && 0xFFFF) << 16) |
+		(be32_to_cpu(reply->fw_ver_patch) && 0xFFFF);
+	memcpy(&props->sys_image_guid, c2dev->netdev->dev_addr, 6);
+	props->max_mr_size         = 0xFFFFFFFF;
+	props->page_size_cap       = ~(C2_MIN_PAGESIZE-1);
+	props->vendor_id           = be32_to_cpu(reply->vendor_id);
+	props->vendor_part_id      = be32_to_cpu(reply->part_number);
+	props->hw_ver              = be32_to_cpu(reply->hw_version);
+	props->max_qp              = be32_to_cpu(reply->max_qps);
+	props->max_qp_wr           = be32_to_cpu(reply->max_qp_depth);
+	props->device_cap_flags    = c2dev->device_cap_flags;
+	props->max_sge             = C2_MAX_SGES;
+	props->max_sge_rd          = C2_MAX_SGE_RD;
+	props->max_cq              = be32_to_cpu(reply->max_cqs);
+	props->max_cqe             = be32_to_cpu(reply->max_cq_depth);
+	props->max_mr              = be32_to_cpu(reply->max_mrs);
+	props->max_pd              = be32_to_cpu(reply->max_pds);
+	props->max_qp_rd_atom      = be32_to_cpu(reply->max_qp_ird);
+	props->max_ee_rd_atom      = 0;
+	props->max_res_rd_atom     = be32_to_cpu(reply->max_global_ird);
+	props->max_qp_init_rd_atom = be32_to_cpu(reply->max_qp_ord);
+	props->max_ee_init_rd_atom = 0;
+	props->atomic_cap          = IB_ATOMIC_NONE;
+	props->max_ee              = 0;
+	props->max_rdd             = 0;
+	props->max_mw              = be32_to_cpu(reply->max_mws);
+	props->max_raw_ipv6_qp     = 0;
+	props->max_raw_ethy_qp     = 0;
+	props->max_mcast_grp       = 0;
+	props->max_mcast_qp_attach = 0;
+	props->max_total_mcast_qp_attach = 0;
+	props->max_ah              = 0;
+	props->max_fmr             = 0;
+	props->max_map_per_fmr     = 0;
+	props->max_srq             = 0;
+	props->max_srq_wr          = 0;
+	props->max_srq_sge         = 0;
+	props->max_pkeys           = 0;
+	props->local_ca_ack_delay  = 0;
+
+ bail2:
+	vq_repbuf_free(c2dev, reply);
+
+ bail1:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+/*
+ * Add an IP address to the RNIC interface
+ */
+int c2_add_addr(struct c2_dev *c2dev, u32 inaddr, u32 inmask)
+{
+	struct c2_vq_req *vq_req;
+	struct c2wr_rnic_setconfig_req *wr;
+	struct c2wr_rnic_setconfig_rep *reply;
+	struct c2_netaddr netaddr;
+	int err, len;
+
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req)
+		return -ENOMEM;
+
+	len = sizeof(struct c2_netaddr);
+	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
+	if (!wr) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
+	wr->hdr.context = (unsigned long) vq_req;
+	wr->rnic_handle = c2dev->adapter_handle;
+	wr->option = cpu_to_be32(C2_CFG_ADD_ADDR);
+
+	netaddr.ip_addr = inaddr;
+	netaddr.netmask = inmask;
+	netaddr.mtu = 0;
+
+	memcpy(wr->data, &netaddr, len);
+
+	vq_req_get(c2dev, vq_req);
+
+	err = vq_send_wr(c2dev, (union c2wr *) wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail1;
+	}
+
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail1;
+
+	reply =
+	    (struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+
+	err = c2_errno(reply);
+	vq_repbuf_free(c2dev, reply);
+
+      bail1:
+	kfree(wr);
+      bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+/*
+ * Delete an IP address from the RNIC interface
+ */
+int c2_del_addr(struct c2_dev *c2dev, u32 inaddr, u32 inmask)
+{
+	struct c2_vq_req *vq_req;
+	struct c2wr_rnic_setconfig_req *wr;
+	struct c2wr_rnic_setconfig_rep *reply;
+	struct c2_netaddr netaddr;
+	int err, len;
+
+	vq_req = vq_req_alloc(c2dev);
+	if (!vq_req)
+		return -ENOMEM;
+
+	len = sizeof(struct c2_netaddr);
+	wr = kmalloc(c2dev->req_vq.msg_size, GFP_KERNEL);
+	if (!wr) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	c2_wr_set_id(wr, CCWR_RNIC_SETCONFIG);
+	wr->hdr.context = (unsigned long) vq_req;
+	wr->rnic_handle = c2dev->adapter_handle;
+	wr->option = cpu_to_be32(C2_CFG_DEL_ADDR);
+
+	netaddr.ip_addr = inaddr;
+	netaddr.netmask = inmask;
+	netaddr.mtu = 0;
+
+	memcpy(wr->data, &netaddr, len);
+
+	vq_req_get(c2dev, vq_req);
+
+	err = vq_send_wr(c2dev, (union c2wr *) wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail1;
+	}
+
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err)
+		goto bail1;
+
+	reply =
+	    (struct c2wr_rnic_setconfig_rep *) (unsigned long) (vq_req->reply_msg);
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+
+	err = c2_errno(reply);
+	vq_repbuf_free(c2dev, reply);
+
+      bail1:
+	kfree(wr);
+      bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+/*
+ * Open a single RNIC instance to use with all
+ * low level openib calls
+ */
+static int c2_rnic_open(struct c2_dev *c2dev)
+{
+	struct c2_vq_req *vq_req;
+	union c2wr wr;
+	struct c2wr_rnic_open_rep *reply;
+	int err;
+
+	vq_req = vq_req_alloc(c2dev);
+	if (vq_req == NULL) {
+		return -ENOMEM;
+	}
+
+	memset(&wr, 0, sizeof(wr));
+	c2_wr_set_id(&wr, CCWR_RNIC_OPEN);
+	wr.rnic_open.req.hdr.context = (unsigned long) (vq_req);
+	wr.rnic_open.req.flags = cpu_to_be16(RNIC_PRIV_MODE);
+	wr.rnic_open.req.port_num = cpu_to_be16(0);
+	wr.rnic_open.req.user_context = (unsigned long) c2dev;
+
+	vq_req_get(c2dev, vq_req);
+
+	err = vq_send_wr(c2dev, &wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail0;
+	}
+
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err) {
+		goto bail0;
+	}
+
+	reply = (struct c2wr_rnic_open_rep *) (unsigned long) (vq_req->reply_msg);
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	if ((err = c2_errno(reply)) != 0) {
+		goto bail1;
+	}
+
+	c2dev->adapter_handle = reply->rnic_handle;
+
+      bail1:
+	vq_repbuf_free(c2dev, reply);
+      bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+/*
+ * Close the RNIC instance
+ */
+static int c2_rnic_close(struct c2_dev *c2dev)
+{
+	struct c2_vq_req *vq_req;
+	union c2wr wr;
+	struct c2wr_rnic_close_rep *reply;
+	int err;
+
+	vq_req = vq_req_alloc(c2dev);
+	if (vq_req == NULL) {
+		return -ENOMEM;
+	}
+
+	memset(&wr, 0, sizeof(wr));
+	c2_wr_set_id(&wr, CCWR_RNIC_CLOSE);
+	wr.rnic_close.req.hdr.context = (unsigned long) vq_req;
+	wr.rnic_close.req.rnic_handle = c2dev->adapter_handle;
+
+	vq_req_get(c2dev, vq_req);
+
+	err = vq_send_wr(c2dev, &wr);
+	if (err) {
+		vq_req_put(c2dev, vq_req);
+		goto bail0;
+	}
+
+	err = vq_wait_for_reply(c2dev, vq_req);
+	if (err) {
+		goto bail0;
+	}
+
+	reply = (struct c2wr_rnic_close_rep *) (unsigned long) (vq_req->reply_msg);
+	if (!reply) {
+		err = -ENOMEM;
+		goto bail0;
+	}
+
+	if ((err = c2_errno(reply)) != 0) {
+		goto bail1;
+	}
+
+	c2dev->adapter_handle = 0;
+
+      bail1:
+	vq_repbuf_free(c2dev, reply);
+      bail0:
+	vq_req_free(c2dev, vq_req);
+	return err;
+}
+
+/*
+ * Called by c2_probe to initialize the RNIC. This principally
+ * involves initalizing the various limits and resouce pools that
+ * comprise the RNIC instance.
+ */
+int c2_rnic_init(struct c2_dev *c2dev)
+{
+	int err;
+	u32 qsize, msgsize;
+	void *q1_pages;
+	void *q2_pages;
+	void __iomem *mmio_regs;
+
+	/* Device capabilities */
+	c2dev->device_cap_flags =
+	    (IB_DEVICE_RESIZE_MAX_WR |
+	     IB_DEVICE_CURR_QP_STATE_MOD |
+	     IB_DEVICE_SYS_IMAGE_GUID |
+	     IB_DEVICE_ZERO_STAG |
+	     IB_DEVICE_SEND_W_INV | IB_DEVICE_MEM_WINDOW);
+
+	/* Allocate the qptr_array */
+	c2dev->qptr_array = vmalloc(C2_MAX_CQS * sizeof(void *));
+	if (!c2dev->qptr_array) {
+		return -ENOMEM;
+	}
+
+	/* Inialize the qptr_array */
+	memset(c2dev->qptr_array, 0, C2_MAX_CQS * sizeof(void *));
+	c2dev->qptr_array[0] = (void *) &c2dev->req_vq;
+	c2dev->qptr_array[1] = (void *) &c2dev->rep_vq;
+	c2dev->qptr_array[2] = (void *) &c2dev->aeq;
+
+	/* Initialize data structures */
+	init_waitqueue_head(&c2dev->req_vq_wo);
+	spin_lock_init(&c2dev->vqlock);
+	spin_lock_init(&c2dev->lock);
+
+	/* Allocate MQ shared pointer pool for kernel clients. User
+	 * mode client pools are hung off the user context
+	 */
+	err = c2_init_mqsp_pool(c2dev, GFP_KERNEL, &c2dev->kern_mqsp_pool);
+	if (err) {
+		goto bail0;
+	}
+
+	/* Allocate shared pointers for Q0, Q1, and Q2 from
+	 * the shared pointer pool.
+	 */
+
+	c2dev->hint_count = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
+					     &c2dev->hint_count_dma,
+					     GFP_KERNEL);
+	c2dev->req_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
+					     &c2dev->req_vq.shared_dma,
+					     GFP_KERNEL);
+	c2dev->rep_vq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
+					     &c2dev->rep_vq.shared_dma,
+					     GFP_KERNEL);
+	c2dev->aeq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
+					  &c2dev->aeq.shared_dma, GFP_KERNEL);
+	if (!c2dev->hint_count || !c2dev->req_vq.shared ||
+	    !c2dev->rep_vq.shared || !c2dev->aeq.shared) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+
+	mmio_regs = c2dev->kva;
+	/* Initialize the Verbs Request Queue */
+	c2_mq_req_init(&c2dev->req_vq, 0,
+		       be32_to_cpu(readl(mmio_regs + C2_REGS_Q0_QSIZE)),
+		       be32_to_cpu(readl(mmio_regs + C2_REGS_Q0_MSGSIZE)),
+		       mmio_regs +
+		       be32_to_cpu(readl(mmio_regs + C2_REGS_Q0_POOLSTART)),
+		       mmio_regs +
+		       be32_to_cpu(readl(mmio_regs + C2_REGS_Q0_SHARED)),
+		       C2_MQ_ADAPTER_TARGET);
+
+	/* Initialize the Verbs Reply Queue */
+	qsize = be32_to_cpu(readl(mmio_regs + C2_REGS_Q1_QSIZE));
+	msgsize = be32_to_cpu(readl(mmio_regs + C2_REGS_Q1_MSGSIZE));
+	q1_pages = kmalloc(qsize * msgsize, GFP_KERNEL);
+	if (!q1_pages) {
+		err = -ENOMEM;
+		goto bail1;
+	}
+	c2dev->rep_vq.host_dma = dma_map_single(c2dev->ibdev.dma_device,
+					        (void *)q1_pages, qsize * msgsize,
+				      		DMA_FROM_DEVICE);
+	pci_unmap_addr_set(&c2dev->rep_vq, mapping, c2dev->rep_vq.host_dma);
+	pr_debug("%s rep_vq va %p dma %llx\n", __FUNCTION__, q1_pages,
+		 (u64)c2dev->rep_vq.host_dma);
+	c2_mq_rep_init(&c2dev->rep_vq,
+		   1,
+		   qsize,
+		   msgsize,
+		   q1_pages,
+		   mmio_regs +
+		   be32_to_cpu(readl(mmio_regs + C2_REGS_Q1_SHARED)),
+		   C2_MQ_HOST_TARGET);
+
+	/* Initialize the Asynchronus Event Queue */
+	qsize = be32_to_cpu(readl(mmio_regs + C2_REGS_Q2_QSIZE));
+	msgsize = be32_to_cpu(readl(mmio_regs + C2_REGS_Q2_MSGSIZE));
+	q2_pages = kmalloc(qsize * msgsize, GFP_KERNEL);
+	if (!q2_pages) {
+		err = -ENOMEM;
+		goto bail2;
+	}
+	c2dev->aeq.host_dma = dma_map_single(c2dev->ibdev.dma_device,
+					        (void *)q2_pages, qsize * msgsize,
+				      		DMA_FROM_DEVICE);
+	pci_unmap_addr_set(&c2dev->aeq, mapping, c2dev->aeq.host_dma);
+	pr_debug("%s aeq va %p dma %llx\n", __FUNCTION__, q1_pages,
+		 (u64)c2dev->rep_vq.host_dma);
+	c2_mq_rep_init(&c2dev->aeq,
+		       2,
+		       qsize,
+		       msgsize,
+		       q2_pages,
+		       mmio_regs +
+		       be32_to_cpu(readl(mmio_regs + C2_REGS_Q2_SHARED)),
+		       C2_MQ_HOST_TARGET);
+
+	/* Initialize the verbs request allocator */
+	err = vq_init(c2dev);
+	if (err)
+		goto bail3;
+
+	/* Enable interrupts on the adapter */
+	writel(0, c2dev->regs + C2_IDIS);
+
+	/* create the WR init message */
+	err = c2_adapter_init(c2dev);
+	if (err)
+		goto bail4;
+	c2dev->init++;
+
+	/* open an adapter instance */
+	err = c2_rnic_open(c2dev);
+	if (err)
+		goto bail4;
+
+	/* Initialize cached the adapter limits */
+	if (c2_rnic_query(c2dev, &c2dev->props))
+		goto bail5;
+
+	/* Initialize the PD pool */
+	err = c2_init_pd_table(c2dev);
+	if (err)
+		goto bail5;
+
+	/* Initialize the QP pool */
+	c2_init_qp_table(c2dev);
+	return 0;
+
+      bail5:
+	c2_rnic_close(c2dev);
+      bail4:
+	vq_term(c2dev);
+      bail3:
+	dma_unmap_single(c2dev->ibdev.dma_device,
+			 pci_unmap_addr(&c2dev->aeq, mapping),
+			 c2dev->aeq.q_size * c2dev->aeq.msg_size,
+		  	 DMA_FROM_DEVICE);
+	kfree(q2_pages);
+      bail2:
+	dma_unmap_single(c2dev->ibdev.dma_device,
+			 pci_unmap_addr(&c2dev->rep_vq, mapping),
+			 c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
+		  	 DMA_FROM_DEVICE);
+	kfree(q1_pages);
+      bail1:
+	c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);
+      bail0:
+	vfree(c2dev->qptr_array);
+
+	return err;
+}
+
+/*
+ * Called by c2_remove to cleanup the RNIC resources.
+ */
+void c2_rnic_term(struct c2_dev *c2dev)
+{
+
+	/* Close the open adapter instance */
+	c2_rnic_close(c2dev);
+
+	/* Send the TERM message to the adapter */
+	c2_adapter_term(c2dev);
+
+	/* Disable interrupts on the adapter */
+	writel(1, c2dev->regs + C2_IDIS);
+
+	/* Free the QP pool */
+	c2_cleanup_qp_table(c2dev);
+
+	/* Free the PD pool */
+	c2_cleanup_pd_table(c2dev);
+
+	/* Free the verbs request allocator */
+	vq_term(c2dev);
+
+	/* Unmap and free the asynchronus event queue */
+	dma_unmap_single(c2dev->ibdev.dma_device,
+			 pci_unmap_addr(&c2dev->aeq, mapping),
+			 c2dev->aeq.q_size * c2dev->aeq.msg_size,
+		  	 DMA_FROM_DEVICE);
+	kfree(c2dev->aeq.msg_pool.host);
+
+	/* Unmap and free the verbs reply queue */
+	dma_unmap_single(c2dev->ibdev.dma_device,
+			 pci_unmap_addr(&c2dev->rep_vq, mapping),
+			 c2dev->rep_vq.q_size * c2dev->rep_vq.msg_size,
+		  	 DMA_FROM_DEVICE);
+	kfree(c2dev->rep_vq.msg_pool.host);
+
+	/* Free the MQ shared pointer pool */
+	c2_free_mqsp_pool(c2dev, c2dev->kern_mqsp_pool);
+
+	/* Free the qptr_array */
+	vfree(c2dev->qptr_array);
+
+	return;
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_status.h b/drivers/infiniband/hw/amso1100/c2_status.h
new file mode 100644
index 0000000..6ee4aa9
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_status.h
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef	_C2_STATUS_H_
+#define _C2_STATUS_H_
+
+/*
+ * Verbs Status Codes
+ */
+enum c2_status {
+	C2_OK = 0,		/* This must be zero */
+	CCERR_INSUFFICIENT_RESOURCES = 1,
+	CCERR_INVALID_MODIFIER = 2,
+	CCERR_INVALID_MODE = 3,
+	CCERR_IN_USE = 4,
+	CCERR_INVALID_RNIC = 5,
+	CCERR_INTERRUPTED_OPERATION = 6,
+	CCERR_INVALID_EH = 7,
+	CCERR_INVALID_CQ = 8,
+	CCERR_CQ_EMPTY = 9,
+	CCERR_NOT_IMPLEMENTED = 10,
+	CCERR_CQ_DEPTH_TOO_SMALL = 11,
+	CCERR_PD_IN_USE = 12,
+	CCERR_INVALID_PD = 13,
+	CCERR_INVALID_SRQ = 14,
+	CCERR_INVALID_ADDRESS = 15,
+	CCERR_INVALID_NETMASK = 16,
+	CCERR_INVALID_QP = 17,
+	CCERR_INVALID_QP_STATE = 18,
+	CCERR_TOO_MANY_WRS_POSTED = 19,
+	CCERR_INVALID_WR_TYPE = 20,
+	CCERR_INVALID_SGL_LENGTH = 21,
+	CCERR_INVALID_SQ_DEPTH = 22,
+	CCERR_INVALID_RQ_DEPTH = 23,
+	CCERR_INVALID_ORD = 24,
+	CCERR_INVALID_IRD = 25,
+	CCERR_QP_ATTR_CANNOT_CHANGE = 26,
+	CCERR_INVALID_STAG = 27,
+	CCERR_QP_IN_USE = 28,
+	CCERR_OUTSTANDING_WRS = 29,
+	CCERR_STAG_IN_USE = 30,
+	CCERR_INVALID_STAG_INDEX = 31,
+	CCERR_INVALID_SGL_FORMAT = 32,
+	CCERR_ADAPTER_TIMEOUT = 33,
+	CCERR_INVALID_CQ_DEPTH = 34,
+	CCERR_INVALID_PRIVATE_DATA_LENGTH = 35,
+	CCERR_INVALID_EP = 36,
+	CCERR_MR_IN_USE = CCERR_STAG_IN_USE,
+	CCERR_FLUSHED = 38,
+	CCERR_INVALID_WQE = 39,
+	CCERR_LOCAL_QP_CATASTROPHIC_ERROR = 40,
+	CCERR_REMOTE_TERMINATION_ERROR = 41,
+	CCERR_BASE_AND_BOUNDS_VIOLATION = 42,
+	CCERR_ACCESS_VIOLATION = 43,
+	CCERR_INVALID_PD_ID = 44,
+	CCERR_WRAP_ERROR = 45,
+	CCERR_INV_STAG_ACCESS_ERROR = 46,
+	CCERR_ZERO_RDMA_READ_RESOURCES = 47,
+	CCERR_QP_NOT_PRIVILEGED = 48,
+	CCERR_STAG_STATE_NOT_INVALID = 49,
+	CCERR_INVALID_PAGE_SIZE = 50,
+	CCERR_INVALID_BUFFER_SIZE = 51,
+	CCERR_INVALID_PBE = 52,
+	CCERR_INVALID_FBO = 53,
+	CCERR_INVALID_LENGTH = 54,
+	CCERR_INVALID_ACCESS_RIGHTS = 55,
+	CCERR_PBL_TOO_BIG = 56,
+	CCERR_INVALID_VA = 57,
+	CCERR_INVALID_REGION = 58,
+	CCERR_INVALID_WINDOW = 59,
+	CCERR_TOTAL_LENGTH_TOO_BIG = 60,
+	CCERR_INVALID_QP_ID = 61,
+	CCERR_ADDR_IN_USE = 62,
+	CCERR_ADDR_NOT_AVAIL = 63,
+	CCERR_NET_DOWN = 64,
+	CCERR_NET_UNREACHABLE = 65,
+	CCERR_CONN_ABORTED = 66,
+	CCERR_CONN_RESET = 67,
+	CCERR_NO_BUFS = 68,
+	CCERR_CONN_TIMEDOUT = 69,
+	CCERR_CONN_REFUSED = 70,
+	CCERR_HOST_UNREACHABLE = 71,
+	CCERR_INVALID_SEND_SGL_DEPTH = 72,
+	CCERR_INVALID_RECV_SGL_DEPTH = 73,
+	CCERR_INVALID_RDMA_WRITE_SGL_DEPTH = 74,
+	CCERR_INSUFFICIENT_PRIVILEGES = 75,
+	CCERR_STACK_ERROR = 76,
+	CCERR_INVALID_VERSION = 77,
+	CCERR_INVALID_MTU = 78,
+	CCERR_INVALID_IMAGE = 79,
+	CCERR_PENDING = 98,	/* not an error; user internally by adapter */
+	CCERR_DEFER = 99,	/* not an error; used internally by adapter */
+	CCERR_FAILED_WRITE = 100,
+	CCERR_FAILED_ERASE = 101,
+	CCERR_FAILED_VERIFICATION = 102,
+	CCERR_NOT_FOUND = 103,
+
+};
+
+/*
+ * CCAE_ACTIVE_CONNECT_RESULTS status result codes.
+ */
+enum c2_connect_status {
+	C2_CONN_STATUS_SUCCESS = C2_OK,
+	C2_CONN_STATUS_NO_MEM = CCERR_INSUFFICIENT_RESOURCES,
+	C2_CONN_STATUS_TIMEDOUT = CCERR_CONN_TIMEDOUT,
+	C2_CONN_STATUS_REFUSED = CCERR_CONN_REFUSED,
+	C2_CONN_STATUS_NETUNREACH = CCERR_NET_UNREACHABLE,
+	C2_CONN_STATUS_HOSTUNREACH = CCERR_HOST_UNREACHABLE,
+	C2_CONN_STATUS_INVALID_RNIC = CCERR_INVALID_RNIC,
+	C2_CONN_STATUS_INVALID_QP = CCERR_INVALID_QP,
+	C2_CONN_STATUS_INVALID_QP_STATE = CCERR_INVALID_QP_STATE,
+	C2_CONN_STATUS_REJECTED = CCERR_CONN_RESET,
+	C2_CONN_STATUS_ADDR_NOT_AVAIL = CCERR_ADDR_NOT_AVAIL,
+};
+
+/*
+ * Flash programming status codes.
+ */
+enum c2_flash_status {
+	C2_FLASH_STATUS_SUCCESS = 0x0000,
+	C2_FLASH_STATUS_VERIFY_ERR = 0x0002,
+	C2_FLASH_STATUS_IMAGE_ERR = 0x0004,
+	C2_FLASH_STATUS_ECLBS = 0x0400,
+	C2_FLASH_STATUS_PSLBS = 0x0800,
+	C2_FLASH_STATUS_VPENS = 0x1000,
+};
+
+#endif				/* _C2_STATUS_H_ */
diff --git a/drivers/infiniband/hw/amso1100/c2_user.h b/drivers/infiniband/hw/amso1100/c2_user.h
new file mode 100644
index 0000000..7e9e7ad
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_user.h
@@ -0,0 +1,82 @@
+/*
+ * Copyright (c) 2005 Topspin Communications.  All rights reserved.
+ * Copyright (c) 2005 Cisco Systems.  All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ *
+ */
+
+#ifndef C2_USER_H
+#define C2_USER_H
+
+#include <linux/types.h>
+
+/*
+ * Make sure that all structs defined in this file remain laid out so
+ * that they pack the same way on 32-bit and 64-bit architectures (to
+ * avoid incompatibility between 32-bit userspace and 64-bit kernels).
+ * In particular do not use pointer types -- pass pointers in __u64
+ * instead.
+ */
+
+struct c2_alloc_ucontext_resp {
+	__u32 qp_tab_size;
+	__u32 uarc_size;
+};
+
+struct c2_alloc_pd_resp {
+	__u32 pdn;
+	__u32 reserved;
+};
+
+struct c2_create_cq {
+	__u32 lkey;
+	__u32 pdn;
+	__u64 arm_db_page;
+	__u64 set_db_page;
+	__u32 arm_db_index;
+	__u32 set_db_index;
+};
+
+struct c2_create_cq_resp {
+	__u32 cqn;
+	__u32 reserved;
+};
+
+struct c2_create_qp {
+	__u32 lkey;
+	__u32 reserved;
+	__u64 sq_db_page;
+	__u64 rq_db_page;
+	__u32 sq_db_index;
+	__u32 rq_db_index;
+};
+
+#endif				/* C2_USER_H */
diff --git a/drivers/infiniband/hw/amso1100/c2_vq.c b/drivers/infiniband/hw/amso1100/c2_vq.c
new file mode 100644
index 0000000..40caeb5
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_vq.c
@@ -0,0 +1,260 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#include <linux/slab.h>
+#include <linux/spinlock.h>
+
+#include "c2_vq.h"
+#include "c2_provider.h"
+
+/*
+ * Verbs Request Objects:
+ *
+ * VQ Request Objects are allocated by the kernel verbs handlers.
+ * They contain a wait object, a refcnt, an atomic bool indicating that the
+ * adapter has replied, and a copy of the verb reply work request.
+ * A pointer to the VQ Request Object is passed down in the context
+ * field of the work request message, and reflected back by the adapter
+ * in the verbs reply message.  The function handle_vq() in the interrupt
+ * path will use this pointer to:
+ * 	1) append a copy of the verbs reply message
+ * 	2) mark that the reply is ready
+ * 	3) wake up the kernel verbs handler blocked awaiting the reply.
+ *
+ *
+ * The kernel verbs handlers do a "get" to put a 2nd reference on the
+ * VQ Request object.  If the kernel verbs handler exits before the adapter
+ * can respond, this extra reference will keep the VQ Request object around
+ * until the adapter's reply can be processed.  The reason we need this is
+ * because a pointer to this object is stuffed into the context field of
+ * the verbs work request message, and reflected back in the reply message.
+ * It is used in the interrupt handler (handle_vq()) to wake up the appropriate
+ * kernel verb handler that is blocked awaiting the verb reply.
+ * So handle_vq() will do a "put" on the object when it's done accessing it.
+ * NOTE:  If we guarantee that the kernel verb handler will never bail before
+ *        getting the reply, then we don't need these refcnts.
+ *
+ *
+ * VQ Request objects are freed by the kernel verbs handlers only
+ * after the verb has been processed, or when the adapter fails and
+ * does not reply.
+ *
+ *
+ * Verbs Reply Buffers:
+ *
+ * VQ Reply bufs are local host memory copies of a
+ * outstanding Verb Request reply
+ * message.  The are always allocated by the kernel verbs handlers, and _may_ be
+ * freed by either the kernel verbs handler -or- the interrupt handler.  The
+ * kernel verbs handler _must_ free the repbuf, then free the vq request object
+ * in that order.
+ */
+
+int vq_init(struct c2_dev *c2dev)
+{
+	sprintf(c2dev->vq_cache_name, "c2-vq:dev%c",
+		(char) ('0' + c2dev->devnum));
+	c2dev->host_msg_cache =
+	    kmem_cache_create(c2dev->vq_cache_name, c2dev->rep_vq.msg_size, 0,
+			      SLAB_HWCACHE_ALIGN, NULL, NULL);
+	if (c2dev->host_msg_cache == NULL) {
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+void vq_term(struct c2_dev *c2dev)
+{
+	kmem_cache_destroy(c2dev->host_msg_cache);
+}
+
+/* vq_req_alloc - allocate a VQ Request Object and initialize it.
+ * The refcnt is set to 1.
+ */
+struct c2_vq_req *vq_req_alloc(struct c2_dev *c2dev)
+{
+	struct c2_vq_req *r;
+
+	r = kmalloc(sizeof(struct c2_vq_req), GFP_KERNEL);
+	if (r) {
+		init_waitqueue_head(&r->wait_object);
+		r->reply_msg = (u64) NULL;
+		r->event = 0;
+		r->cm_id = NULL;
+		r->qp = NULL;
+		atomic_set(&r->refcnt, 1);
+		atomic_set(&r->reply_ready, 0);
+	}
+	return r;
+}
+
+
+/* vq_req_free - free the VQ Request Object.  It is assumed the verbs handler
+ * has already free the VQ Reply Buffer if it existed.
+ */
+void vq_req_free(struct c2_dev *c2dev, struct c2_vq_req *r)
+{
+	r->reply_msg = (u64) NULL;
+	if (atomic_dec_and_test(&r->refcnt)) {
+		kfree(r);
+	}
+}
+
+/* vq_req_get - reference a VQ Request Object.  Done
+ * only in the kernel verbs handlers.
+ */
+void vq_req_get(struct c2_dev *c2dev, struct c2_vq_req *r)
+{
+	atomic_inc(&r->refcnt);
+}
+
+
+/* vq_req_put - dereference and potentially free a VQ Request Object.
+ *
+ * This is only called by handle_vq() on the
+ * interrupt when it is done processing
+ * a verb reply message.  If the associated
+ * kernel verbs handler has already bailed,
+ * then this put will actually free the VQ
+ * Request object _and_ the VQ Reply Buffer
+ * if it exists.
+ */
+void vq_req_put(struct c2_dev *c2dev, struct c2_vq_req *r)
+{
+	if (atomic_dec_and_test(&r->refcnt)) {
+		if (r->reply_msg != (u64) NULL)
+			vq_repbuf_free(c2dev,
+				       (void *) (unsigned long) r->reply_msg);
+		kfree(r);
+	}
+}
+
+
+/*
+ * vq_repbuf_alloc - allocate a VQ Reply Buffer.
+ */
+void *vq_repbuf_alloc(struct c2_dev *c2dev)
+{
+	return kmem_cache_alloc(c2dev->host_msg_cache, SLAB_ATOMIC);
+}
+
+/*
+ * vq_send_wr - post a verbs request message to the Verbs Request Queue.
+ * If a message is not available in the MQ, then block until one is available.
+ * NOTE: handle_mq() on the interrupt context will wake up threads blocked here.
+ * When the adapter drains the Verbs Request Queue,
+ * it inserts MQ index 0 in to the
+ * adapter->host activity fifo and interrupts the host.
+ */
+int vq_send_wr(struct c2_dev *c2dev, union c2wr *wr)
+{
+	void *msg;
+	wait_queue_t __wait;
+
+	/*
+	 * grab adapter vq lock
+	 */
+	spin_lock(&c2dev->vqlock);
+
+	/*
+	 * allocate msg
+	 */
+	msg = c2_mq_alloc(&c2dev->req_vq);
+
+	/*
+	 * If we cannot get a msg, then we'll wait
+	 * When a messages are available, the int handler will wake_up()
+	 * any waiters.
+	 */
+	while (msg == NULL) {
+		pr_debug("%s:%d no available msg in VQ, waiting...\n",
+		       __FUNCTION__, __LINE__);
+		init_waitqueue_entry(&__wait, current);
+		add_wait_queue(&c2dev->req_vq_wo, &__wait);
+		spin_unlock(&c2dev->vqlock);
+		for (;;) {
+			set_current_state(TASK_INTERRUPTIBLE);
+			if (!c2_mq_full(&c2dev->req_vq)) {
+				break;
+			}
+			if (!signal_pending(current)) {
+				schedule_timeout(1 * HZ);	/* 1 second... */
+				continue;
+			}
+			set_current_state(TASK_RUNNING);
+			remove_wait_queue(&c2dev->req_vq_wo, &__wait);
+			return -EINTR;
+		}
+		set_current_state(TASK_RUNNING);
+		remove_wait_queue(&c2dev->req_vq_wo, &__wait);
+		spin_lock(&c2dev->vqlock);
+		msg = c2_mq_alloc(&c2dev->req_vq);
+	}
+
+	/*
+	 * copy wr into adapter msg
+	 */
+	memcpy(msg, wr, c2dev->req_vq.msg_size);
+
+	/*
+	 * post msg
+	 */
+	c2_mq_produce(&c2dev->req_vq);
+
+	/*
+	 * release adapter vq lock
+	 */
+	spin_unlock(&c2dev->vqlock);
+	return 0;
+}
+
+
+/*
+ * vq_wait_for_reply - block until the adapter posts a Verb Reply Message.
+ */
+int vq_wait_for_reply(struct c2_dev *c2dev, struct c2_vq_req *req)
+{
+	if (!wait_event_timeout(req->wait_object,
+				atomic_read(&req->reply_ready),
+				60*HZ))
+		return -ETIMEDOUT;
+
+	return 0;
+}
+
+/*
+ * vq_repbuf_free - Free a Verbs Reply Buffer.
+ */
+void vq_repbuf_free(struct c2_dev *c2dev, void *reply)
+{
+	kmem_cache_free(c2dev->host_msg_cache, reply);
+}
diff --git a/drivers/infiniband/hw/amso1100/c2_vq.h b/drivers/infiniband/hw/amso1100/c2_vq.h
new file mode 100644
index 0000000..3380562
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_vq.h
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef _C2_VQ_H_
+#define _C2_VQ_H_
+#include <linux/sched.h>
+#include "c2.h"
+#include "c2_wr.h"
+#include "c2_provider.h"
+
+struct c2_vq_req {
+	u64 reply_msg;		/* ptr to reply msg */
+	wait_queue_head_t wait_object;	/* wait object for vq reqs */
+	atomic_t reply_ready;	/* set when reply is ready */
+	atomic_t refcnt;	/* used to cancel WRs... */
+	int event;
+	struct iw_cm_id *cm_id;
+	struct c2_qp *qp;
+};
+
+extern int vq_init(struct c2_dev *c2dev);
+extern void vq_term(struct c2_dev *c2dev);
+
+extern struct c2_vq_req *vq_req_alloc(struct c2_dev *c2dev);
+extern void vq_req_free(struct c2_dev *c2dev, struct c2_vq_req *req);
+extern void vq_req_get(struct c2_dev *c2dev, struct c2_vq_req *req);
+extern void vq_req_put(struct c2_dev *c2dev, struct c2_vq_req *req);
+extern int vq_send_wr(struct c2_dev *c2dev, union c2wr * wr);
+
+extern void *vq_repbuf_alloc(struct c2_dev *c2dev);
+extern void vq_repbuf_free(struct c2_dev *c2dev, void *reply);
+
+extern int vq_wait_for_reply(struct c2_dev *c2dev, struct c2_vq_req *req);
+#endif				/* _C2_VQ_H_ */
diff --git a/drivers/infiniband/hw/amso1100/c2_wr.h b/drivers/infiniband/hw/amso1100/c2_wr.h
new file mode 100644
index 0000000..3ec6c43
--- /dev/null
+++ b/drivers/infiniband/hw/amso1100/c2_wr.h
@@ -0,0 +1,1520 @@
+/*
+ * Copyright (c) 2005 Ammasso, Inc. All rights reserved.
+ * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+#ifndef _C2_WR_H_
+#define _C2_WR_H_
+
+#ifdef CCDEBUG
+#define CCWR_MAGIC		0xb07700b0
+#endif
+
+#define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
+
+/* Maximum allowed size in bytes of private_data exchange
+ * on connect.
+ */
+#define C2_MAX_PRIVATE_DATA_SIZE 200
+
+/*
+ * These types are shared among the adapter, host, and CCIL consumer.
+ */
+enum c2_cq_notification_type {
+	C2_CQ_NOTIFICATION_TYPE_NONE = 1,
+	C2_CQ_NOTIFICATION_TYPE_NEXT,
+	C2_CQ_NOTIFICATION_TYPE_NEXT_SE
+};
+
+enum c2_setconfig_cmd {
+	C2_CFG_ADD_ADDR = 1,
+	C2_CFG_DEL_ADDR = 2,
+	C2_CFG_ADD_ROUTE = 3,
+	C2_CFG_DEL_ROUTE = 4
+};
+
+enum c2_getconfig_cmd {
+	C2_GETCONFIG_ROUTES = 1,
+	C2_GETCONFIG_ADDRS
+};
+
+/*
+ *  CCIL Work Request Identifiers
+ */
+enum c2wr_ids {
+	CCWR_RNIC_OPEN = 1,
+	CCWR_RNIC_QUERY,
+	CCWR_RNIC_SETCONFIG,
+	CCWR_RNIC_GETCONFIG,
+	CCWR_RNIC_CLOSE,
+	CCWR_CQ_CREATE,
+	CCWR_CQ_QUERY,
+	CCWR_CQ_MODIFY,
+	CCWR_CQ_DESTROY,
+	CCWR_QP_CONNECT,
+	CCWR_PD_ALLOC,
+	CCWR_PD_DEALLOC,
+	CCWR_SRQ_CREATE,
+	CCWR_SRQ_QUERY,
+	CCWR_SRQ_MODIFY,
+	CCWR_SRQ_DESTROY,
+	CCWR_QP_CREATE,
+	CCWR_QP_QUERY,
+	CCWR_QP_MODIFY,
+	CCWR_QP_DESTROY,
+	CCWR_NSMR_STAG_ALLOC,
+	CCWR_NSMR_REGISTER,
+	CCWR_NSMR_PBL,
+	CCWR_STAG_DEALLOC,
+	CCWR_NSMR_REREGISTER,
+	CCWR_SMR_REGISTER,
+	CCWR_MR_QUERY,
+	CCWR_MW_ALLOC,
+	CCWR_MW_QUERY,
+	CCWR_EP_CREATE,
+	CCWR_EP_GETOPT,
+	CCWR_EP_SETOPT,
+	CCWR_EP_DESTROY,
+	CCWR_EP_BIND,
+	CCWR_EP_CONNECT,
+	CCWR_EP_LISTEN,
+	CCWR_EP_SHUTDOWN,
+	CCWR_EP_LISTEN_CREATE,
+	CCWR_EP_LISTEN_DESTROY,
+	CCWR_EP_QUERY,
+	CCWR_CR_ACCEPT,
+	CCWR_CR_REJECT,
+	CCWR_CONSOLE,
+	CCWR_TERM,
+	CCWR_FLASH_INIT,
+	CCWR_FLASH,
+	CCWR_BUF_ALLOC,
+	CCWR_BUF_FREE,
+	CCWR_FLASH_WRITE,
+	CCWR_INIT,		/* WARNING: Don't move this ever again! */
+
+
+
+	/* Add new IDs here */
+
+
+
+	/*
+	 * WARNING: CCWR_LAST must always be the last verbs id defined!
+	 *          All the preceding IDs are fixed, and must not change.
+	 *          You can add new IDs, but must not remove or reorder
+	 *          any IDs. If you do, YOU will ruin any hope of
+	 *          compatability between versions.
+	 */
+	CCWR_LAST,
+
+	/*
+	 * Start over at 1 so that arrays indexed by user wr id's
+	 * begin at 1.  This is OK since the verbs and user wr id's
+	 * are always used on disjoint sets of queues.
+	 */
+	/*
+	 * The order of the CCWR_SEND_XX verbs must
+	 * match the order of the RDMA_OPs
+	 */
+	CCWR_SEND = 1,
+	CCWR_SEND_INV,
+	CCWR_SEND_SE,
+	CCWR_SEND_SE_INV,
+	CCWR_RDMA_WRITE,
+	CCWR_RDMA_READ,
+	CCWR_RDMA_READ_INV,
+	CCWR_MW_BIND,
+	CCWR_NSMR_FASTREG,
+	CCWR_STAG_INVALIDATE,
+	CCWR_RECV,
+	CCWR_NOP,
+	CCWR_UNIMPL,
+/* WARNING: This must always be the last user wr id defined! */
+};
+#define RDMA_SEND_OPCODE_FROM_WR_ID(x)   (x+2)
+
+/*
+ * SQ/RQ Work Request Types
+ */
+enum c2_wr_type {
+	C2_WR_TYPE_SEND = CCWR_SEND,
+	C2_WR_TYPE_SEND_SE = CCWR_SEND_SE,
+	C2_WR_TYPE_SEND_INV = CCWR_SEND_INV,
+	C2_WR_TYPE_SEND_SE_INV = CCWR_SEND_SE_INV,
+	C2_WR_TYPE_RDMA_WRITE = CCWR_RDMA_WRITE,
+	C2_WR_TYPE_RDMA_READ = CCWR_RDMA_READ,
+	C2_WR_TYPE_RDMA_READ_INV_STAG = CCWR_RDMA_READ_INV,
+	C2_WR_TYPE_BIND_MW = CCWR_MW_BIND,
+	C2_WR_TYPE_FASTREG_NSMR = CCWR_NSMR_FASTREG,
+	C2_WR_TYPE_INV_STAG = CCWR_STAG_INVALIDATE,
+	C2_WR_TYPE_RECV = CCWR_RECV,
+	C2_WR_TYPE_NOP = CCWR_NOP,
+};
+
+struct c2_netaddr {
+	u32 ip_addr;
+	u32 netmask;
+	u32 mtu;
+};
+
+struct c2_route {
+	u32 ip_addr;		/* 0 indicates the default route */
+	u32 netmask;		/* netmask associated with dst */
+	u32 flags;
+	union {
+		u32 ipaddr;	/* address of the nexthop interface */
+		u8 enaddr[6];
+	} nexthop;
+};
+
+/*
+ * A Scatter Gather Entry.
+ */
+struct c2_data_addr {
+	u32 stag;
+	u32 length;
+	u64 to;
+};
+
+/*
+ * MR and MW flags used by the consumer, RI, and RNIC.
+ */
+enum c2_mm_flags {
+	MEM_REMOTE = 0x0001,	/* allow mw binds with remote access. */
+	MEM_VA_BASED = 0x0002,	/* Not Zero-based */
+	MEM_PBL_COMPLETE = 0x0004,	/* PBL array is complete in this msg */
+	MEM_LOCAL_READ = 0x0008,	/* allow local reads */
+	MEM_LOCAL_WRITE = 0x0010,	/* allow local writes */
+	MEM_REMOTE_READ = 0x0020,	/* allow remote reads */
+	MEM_REMOTE_WRITE = 0x0040,	/* allow remote writes */
+	MEM_WINDOW_BIND = 0x0080,	/* binds allowed */
+	MEM_SHARED = 0x0100,	/* set if MR is shared */
+	MEM_STAG_VALID = 0x0200	/* set if STAG is in valid state */
+};
+
+/*
+ * CCIL API ACF flags defined in terms of the low level mem flags.
+ * This minimizes translation needed in the user API
+ */
+enum c2_acf {
+	C2_ACF_LOCAL_READ = MEM_LOCAL_READ,
+	C2_ACF_LOCAL_WRITE = MEM_LOCAL_WRITE,
+	C2_ACF_REMOTE_READ = MEM_REMOTE_READ,
+	C2_ACF_REMOTE_WRITE = MEM_REMOTE_WRITE,
+	C2_ACF_WINDOW_BIND = MEM_WINDOW_BIND
+};
+
+/*
+ * Image types of objects written to flash
+ */
+#define C2_FLASH_IMG_BITFILE 1
+#define C2_FLASH_IMG_OPTION_ROM 2
+#define C2_FLASH_IMG_VPD 3
+
+/*
+ *  to fix bug 1815 we define the max size allowable of the
+ *  terminate message (per the IETF spec).Refer to the IETF
+ *  protocal specification, section 12.1.6, page 64)
+ *  The message is prefixed by 20 types of DDP info.
+ *
+ *  Then the message has 6 bytes for the terminate control
+ *  and DDP segment length info plus a DDP header (either
+ *  14 or 18 byts) plus 28 bytes for the RDMA header.
+ *  Thus the max size in:
+ *  20 + (6 + 18 + 28) = 72
+ */
+#define C2_MAX_TERMINATE_MESSAGE_SIZE (72)
+
+/*
+ * Build String Length.  It must be the same as C2_BUILD_STR_LEN in ccil_api.h
+ */
+#define WR_BUILD_STR_LEN 64
+
+/*
+ * WARNING:  All of these structs need to align any 64bit types on
+ * 64 bit boundaries!  64bit types include u64 and u64.
+ */
+
+/*
+ * Clustercore Work Request Header.  Be sensitive to field layout
+ * and alignment.
+ */
+struct c2wr_hdr {
+	/* wqe_count is part of the cqe.  It is put here so the
+	 * adapter can write to it while the wr is pending without
+	 * clobbering part of the wr.  This word need not be dma'd
+	 * from the host to adapter by libccil, but we copy it anyway
+	 * to make the memcpy to the adapter better aligned.
+	 */
+	u32 wqe_count;
+
+	/* Put these fields next so that later 32- and 64-bit
+	 * quantities are naturally aligned.
+	 */
+	u8 id;
+	u8 result;		/* adapter -> host */
+	u8 sge_count;		/* host -> adapter */
+	u8 flags;		/* host -> adapter */
+
+	u64 context;
+#ifdef CCMSGMAGIC
+	u32 magic;
+	u32 pad;
+#endif
+} __attribute__((packed));
+
+/*
+ *------------------------ RNIC ------------------------
+ */
+
+/*
+ * WR_RNIC_OPEN
+ */
+
+/*
+ * Flags for the RNIC WRs
+ */
+enum c2_rnic_flags {
+	RNIC_IRD_STATIC = 0x0001,
+	RNIC_ORD_STATIC = 0x0002,
+	RNIC_QP_STATIC = 0x0004,
+	RNIC_SRQ_SUPPORTED = 0x0008,
+	RNIC_PBL_BLOCK_MODE = 0x0010,
+	RNIC_SRQ_MODEL_ARRIVAL = 0x0020,
+	RNIC_CQ_OVF_DETECTED = 0x0040,
+	RNIC_PRIV_MODE = 0x0080
+};
+
+struct c2wr_rnic_open_req {
+	struct c2wr_hdr hdr;
+	u64 user_context;
+	u16 flags;		/* See enum c2_rnic_flags */
+	u16 port_num;
+} __attribute__((packed));
+
+struct c2wr_rnic_open_rep {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+} __attribute__((packed));
+
+union c2wr_rnic_open {
+	struct c2wr_rnic_open_req req;
+	struct c2wr_rnic_open_rep rep;
+} __attribute__((packed));
+
+struct c2wr_rnic_query_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+} __attribute__((packed));
+
+/*
+ * WR_RNIC_QUERY
+ */
+struct c2wr_rnic_query_rep {
+	struct c2wr_hdr hdr;
+	u64 user_context;
+	u32 vendor_id;
+	u32 part_number;
+	u32 hw_version;
+	u32 fw_ver_major;
+	u32 fw_ver_minor;
+	u32 fw_ver_patch;
+	char fw_ver_build_str[WR_BUILD_STR_LEN];
+	u32 max_qps;
+	u32 max_qp_depth;
+	u32 max_srq_depth;
+	u32 max_send_sgl_depth;
+	u32 max_rdma_sgl_depth;
+	u32 max_cqs;
+	u32 max_cq_depth;
+	u32 max_cq_event_handlers;
+	u32 max_mrs;
+	u32 max_pbl_depth;
+	u32 max_pds;
+	u32 max_global_ird;
+	u32 max_global_ord;
+	u32 max_qp_ird;
+	u32 max_qp_ord;
+	u32 flags;
+	u32 max_mws;
+	u32 pbe_range_low;
+	u32 pbe_range_high;
+	u32 max_srqs;
+	u32 page_size;
+} __attribute__((packed));
+
+union c2wr_rnic_query {
+	struct c2wr_rnic_query_req req;
+	struct c2wr_rnic_query_rep rep;
+} __attribute__((packed));
+
+/*
+ * WR_RNIC_GETCONFIG
+ */
+
+struct c2wr_rnic_getconfig_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 option;		/* see c2_getconfig_cmd_t */
+	u64 reply_buf;
+	u32 reply_buf_len;
+} __attribute__((packed)) ;
+
+struct c2wr_rnic_getconfig_rep {
+	struct c2wr_hdr hdr;
+	u32 option;		/* see c2_getconfig_cmd_t */
+	u32 count_len;		/* length of the number of addresses configured */
+} __attribute__((packed)) ;
+
+union c2wr_rnic_getconfig {
+	struct c2wr_rnic_getconfig_req req;
+	struct c2wr_rnic_getconfig_rep rep;
+} __attribute__((packed)) ;
+
+/*
+ * WR_RNIC_SETCONFIG
+ */
+struct c2wr_rnic_setconfig_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 option;		/* See c2_setconfig_cmd_t */
+	/* variable data and pad. See c2_netaddr and c2_route */
+	u8 data[0];
+} __attribute__((packed)) ;
+
+struct c2wr_rnic_setconfig_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_rnic_setconfig {
+	struct c2wr_rnic_setconfig_req req;
+	struct c2wr_rnic_setconfig_rep rep;
+} __attribute__((packed)) ;
+
+/*
+ * WR_RNIC_CLOSE
+ */
+struct c2wr_rnic_close_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+} __attribute__((packed)) ;
+
+struct c2wr_rnic_close_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_rnic_close {
+	struct c2wr_rnic_close_req req;
+	struct c2wr_rnic_close_rep rep;
+} __attribute__((packed)) ;
+
+/*
+ *------------------------ CQ ------------------------
+ */
+struct c2wr_cq_create_req {
+	struct c2wr_hdr hdr;
+	u64 shared_ht;
+	u64 user_context;
+	u64 msg_pool;
+	u32 rnic_handle;
+	u32 msg_size;
+	u32 depth;
+} __attribute__((packed)) ;
+
+struct c2wr_cq_create_rep {
+	struct c2wr_hdr hdr;
+	u32 mq_index;
+	u32 adapter_shared;
+	u32 cq_handle;
+} __attribute__((packed)) ;
+
+union c2wr_cq_create {
+	struct c2wr_cq_create_req req;
+	struct c2wr_cq_create_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_cq_modify_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 cq_handle;
+	u32 new_depth;
+	u64 new_msg_pool;
+} __attribute__((packed)) ;
+
+struct c2wr_cq_modify_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_cq_modify {
+	struct c2wr_cq_modify_req req;
+	struct c2wr_cq_modify_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_cq_destroy_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 cq_handle;
+} __attribute__((packed)) ;
+
+struct c2wr_cq_destroy_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_cq_destroy {
+	struct c2wr_cq_destroy_req req;
+	struct c2wr_cq_destroy_rep rep;
+} __attribute__((packed)) ;
+
+/*
+ *------------------------ PD ------------------------
+ */
+struct c2wr_pd_alloc_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 pd_id;
+} __attribute__((packed)) ;
+
+struct c2wr_pd_alloc_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_pd_alloc {
+	struct c2wr_pd_alloc_req req;
+	struct c2wr_pd_alloc_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_pd_dealloc_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 pd_id;
+} __attribute__((packed)) ;
+
+struct c2wr_pd_dealloc_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_pd_dealloc {
+	struct c2wr_pd_dealloc_req req;
+	struct c2wr_pd_dealloc_rep rep;
+} __attribute__((packed)) ;
+
+/*
+ *------------------------ SRQ ------------------------
+ */
+struct c2wr_srq_create_req {
+	struct c2wr_hdr hdr;
+	u64 shared_ht;
+	u64 user_context;
+	u32 rnic_handle;
+	u32 srq_depth;
+	u32 srq_limit;
+	u32 sgl_depth;
+	u32 pd_id;
+} __attribute__((packed)) ;
+
+struct c2wr_srq_create_rep {
+	struct c2wr_hdr hdr;
+	u32 srq_depth;
+	u32 sgl_depth;
+	u32 msg_size;
+	u32 mq_index;
+	u32 mq_start;
+	u32 srq_handle;
+} __attribute__((packed)) ;
+
+union c2wr_srq_create {
+	struct c2wr_srq_create_req req;
+	struct c2wr_srq_create_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_srq_destroy_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 srq_handle;
+} __attribute__((packed)) ;
+
+struct c2wr_srq_destroy_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_srq_destroy {
+	struct c2wr_srq_destroy_req req;
+	struct c2wr_srq_destroy_rep rep;
+} __attribute__((packed)) ;
+
+/*
+ *------------------------ QP ------------------------
+ */
+enum c2wr_qp_flags {
+	QP_RDMA_READ = 0x00000001,	/* RDMA read enabled? */
+	QP_RDMA_WRITE = 0x00000002,	/* RDMA write enabled? */
+	QP_MW_BIND = 0x00000004,	/* MWs enabled */
+	QP_ZERO_STAG = 0x00000008,	/* enabled? */
+	QP_REMOTE_TERMINATION = 0x00000010,	/* remote end terminated */
+	QP_RDMA_READ_RESPONSE = 0x00000020	/* Remote RDMA read  */
+	    /* enabled? */
+};
+
+struct c2wr_qp_create_req {
+	struct c2wr_hdr hdr;
+	u64 shared_sq_ht;
+	u64 shared_rq_ht;
+	u64 user_context;
+	u32 rnic_handle;
+	u32 sq_cq_handle;
+	u32 rq_cq_handle;
+	u32 sq_depth;
+	u32 rq_depth;
+	u32 srq_handle;
+	u32 srq_limit;
+	u32 flags;		/* see enum c2wr_qp_flags */
+	u32 send_sgl_depth;
+	u32 recv_sgl_depth;
+	u32 rdma_write_sgl_depth;
+	u32 ord;
+	u32 ird;
+	u32 pd_id;
+} __attribute__((packed)) ;
+
+struct c2wr_qp_create_rep {
+	struct c2wr_hdr hdr;
+	u32 sq_depth;
+	u32 rq_depth;
+	u32 send_sgl_depth;
+	u32 recv_sgl_depth;
+	u32 rdma_write_sgl_depth;
+	u32 ord;
+	u32 ird;
+	u32 sq_msg_size;
+	u32 sq_mq_index;
+	u32 sq_mq_start;
+	u32 rq_msg_size;
+	u32 rq_mq_index;
+	u32 rq_mq_start;
+	u32 qp_handle;
+} __attribute__((packed)) ;
+
+union c2wr_qp_create {
+	struct c2wr_qp_create_req req;
+	struct c2wr_qp_create_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_qp_query_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 qp_handle;
+} __attribute__((packed)) ;
+
+struct c2wr_qp_query_rep {
+	struct c2wr_hdr hdr;
+	u64 user_context;
+	u32 rnic_handle;
+	u32 sq_depth;
+	u32 rq_depth;
+	u32 send_sgl_depth;
+	u32 rdma_write_sgl_depth;
+	u32 recv_sgl_depth;
+	u32 ord;
+	u32 ird;
+	u16 qp_state;
+	u16 flags;		/* see c2wr_qp_flags_t */
+	u32 qp_id;
+	u32 local_addr;
+	u32 remote_addr;
+	u16 local_port;
+	u16 remote_port;
+	u32 terminate_msg_length;	/* 0 if not present */
+	u8 data[0];
+	/* Terminate Message in-line here. */
+} __attribute__((packed)) ;
+
+union c2wr_qp_query {
+	struct c2wr_qp_query_req req;
+	struct c2wr_qp_query_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_qp_modify_req {
+	struct c2wr_hdr hdr;
+	u64 stream_msg;
+	u32 stream_msg_length;
+	u32 rnic_handle;
+	u32 qp_handle;
+	u32 next_qp_state;
+	u32 ord;
+	u32 ird;
+	u32 sq_depth;
+	u32 rq_depth;
+	u32 llp_ep_handle;
+} __attribute__((packed)) ;
+
+struct c2wr_qp_modify_rep {
+	struct c2wr_hdr hdr;
+	u32 ord;
+	u32 ird;
+	u32 sq_depth;
+	u32 rq_depth;
+	u32 sq_msg_size;
+	u32 sq_mq_index;
+	u32 sq_mq_start;
+	u32 rq_msg_size;
+	u32 rq_mq_index;
+	u32 rq_mq_start;
+} __attribute__((packed)) ;
+
+union c2wr_qp_modify {
+	struct c2wr_qp_modify_req req;
+	struct c2wr_qp_modify_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_qp_destroy_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 qp_handle;
+} __attribute__((packed)) ;
+
+struct c2wr_qp_destroy_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_qp_destroy {
+	struct c2wr_qp_destroy_req req;
+	struct c2wr_qp_destroy_rep rep;
+} __attribute__((packed)) ;
+
+/*
+ * The CCWR_QP_CONNECT msg is posted on the verbs request queue.  It can
+ * only be posted when a QP is in IDLE state.  After the connect request is
+ * submitted to the LLP, the adapter moves the QP to CONNECT_PENDING state.
+ * No synchronous reply from adapter to this WR.  The results of
+ * connection are passed back in an async event CCAE_ACTIVE_CONNECT_RESULTS
+ * See c2wr_ae_active_connect_results_t
+ */
+struct c2wr_qp_connect_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 qp_handle;
+	u32 remote_addr;
+	u16 remote_port;
+	u16 pad;
+	u32 private_data_length;
+	u8 private_data[0];	/* Private data in-line. */
+} __attribute__((packed)) ;
+
+struct c2wr_qp_connect {
+	struct c2wr_qp_connect_req req;
+	/* no synchronous reply.         */
+} __attribute__((packed)) ;
+
+
+/*
+ *------------------------ MM ------------------------
+ */
+
+struct c2wr_nsmr_stag_alloc_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 pbl_depth;
+	u32 pd_id;
+	u32 flags;
+} __attribute__((packed)) ;
+
+struct c2wr_nsmr_stag_alloc_rep {
+	struct c2wr_hdr hdr;
+	u32 pbl_depth;
+	u32 stag_index;
+} __attribute__((packed)) ;
+
+union c2wr_nsmr_stag_alloc {
+	struct c2wr_nsmr_stag_alloc_req req;
+	struct c2wr_nsmr_stag_alloc_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_nsmr_register_req {
+	struct c2wr_hdr hdr;
+	u64 va;
+	u32 rnic_handle;
+	u16 flags;
+	u8 stag_key;
+	u8 pad;
+	u32 pd_id;
+	u32 pbl_depth;
+	u32 pbe_size;
+	u32 fbo;
+	u32 length;
+	u32 addrs_length;
+	/* array of paddrs (must be aligned on a 64bit boundary) */
+	u64 paddrs[0];
+} __attribute__((packed)) ;
+
+struct c2wr_nsmr_register_rep {
+	struct c2wr_hdr hdr;
+	u32 pbl_depth;
+	u32 stag_index;
+} __attribute__((packed)) ;
+
+union c2wr_nsmr_register {
+	struct c2wr_nsmr_register_req req;
+	struct c2wr_nsmr_register_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_nsmr_pbl_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 flags;
+	u32 stag_index;
+	u32 addrs_length;
+	/* array of paddrs (must be aligned on a 64bit boundary) */
+	u64 paddrs[0];
+} __attribute__((packed)) ;
+
+struct c2wr_nsmr_pbl_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_nsmr_pbl {
+	struct c2wr_nsmr_pbl_req req;
+	struct c2wr_nsmr_pbl_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_mr_query_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 stag_index;
+} __attribute__((packed)) ;
+
+struct c2wr_mr_query_rep {
+	struct c2wr_hdr hdr;
+	u8 stag_key;
+	u8 pad[3];
+	u32 pd_id;
+	u32 flags;
+	u32 pbl_depth;
+} __attribute__((packed)) ;
+
+union c2wr_mr_query {
+	struct c2wr_mr_query_req req;
+	struct c2wr_mr_query_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_mw_query_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 stag_index;
+} __attribute__((packed)) ;
+
+struct c2wr_mw_query_rep {
+	struct c2wr_hdr hdr;
+	u8 stag_key;
+	u8 pad[3];
+	u32 pd_id;
+	u32 flags;
+} __attribute__((packed)) ;
+
+union c2wr_mw_query {
+	struct c2wr_mw_query_req req;
+	struct c2wr_mw_query_rep rep;
+} __attribute__((packed)) ;
+
+
+struct c2wr_stag_dealloc_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 stag_index;
+} __attribute__((packed)) ;
+
+struct c2wr_stag_dealloc_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed)) ;
+
+union c2wr_stag_dealloc {
+	struct c2wr_stag_dealloc_req req;
+	struct c2wr_stag_dealloc_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_nsmr_reregister_req {
+	struct c2wr_hdr hdr;
+	u64 va;
+	u32 rnic_handle;
+	u16 flags;
+	u8 stag_key;
+	u8 pad;
+	u32 stag_index;
+	u32 pd_id;
+	u32 pbl_depth;
+	u32 pbe_size;
+	u32 fbo;
+	u32 length;
+	u32 addrs_length;
+	u32 pad1;
+	/* array of paddrs (must be aligned on a 64bit boundary) */
+	u64 paddrs[0];
+} __attribute__((packed)) ;
+
+struct c2wr_nsmr_reregister_rep {
+	struct c2wr_hdr hdr;
+	u32 pbl_depth;
+	u32 stag_index;
+} __attribute__((packed)) ;
+
+union c2wr_nsmr_reregister {
+	struct c2wr_nsmr_reregister_req req;
+	struct c2wr_nsmr_reregister_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_smr_register_req {
+	struct c2wr_hdr hdr;
+	u64 va;
+	u32 rnic_handle;
+	u16 flags;
+	u8 stag_key;
+	u8 pad;
+	u32 stag_index;
+	u32 pd_id;
+} __attribute__((packed)) ;
+
+struct c2wr_smr_register_rep {
+	struct c2wr_hdr hdr;
+	u32 stag_index;
+} __attribute__((packed)) ;
+
+union c2wr_smr_register {
+	struct c2wr_smr_register_req req;
+	struct c2wr_smr_register_rep rep;
+} __attribute__((packed)) ;
+
+struct c2wr_mw_alloc_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 pd_id;
+} __attribute__((packed)) ;
+
+struct c2wr_mw_alloc_rep {
+	struct c2wr_hdr hdr;
+	u32 stag_index;
+} __attribute__((packed)) ;
+
+union c2wr_mw_alloc {
+	struct c2wr_mw_alloc_req req;
+	struct c2wr_mw_alloc_rep rep;
+} __attribute__((packed)) ;
+
+/*
+ *------------------------ WRs -----------------------
+ */
+
+struct c2wr_user_hdr {
+	struct c2wr_hdr hdr;		/* Has status and WR Type */
+} __attribute__((packed)) ;
+
+enum c2_qp_state {
+	C2_QP_STATE_IDLE = 0x01,
+	C2_QP_STATE_CONNECTING = 0x02,
+	C2_QP_STATE_RTS = 0x04,
+	C2_QP_STATE_CLOSING = 0x08,
+	C2_QP_STATE_TERMINATE = 0x10,
+	C2_QP_STATE_ERROR = 0x20,
+};
+
+/* Completion queue entry. */
+struct c2wr_ce {
+	struct c2wr_hdr hdr;		/* Has status and WR Type */
+	u64 qp_user_context;	/* c2_user_qp_t * */
+	u32 qp_state;		/* Current QP State */
+	u32 handle;		/* QPID or EP Handle */
+	u32 bytes_rcvd;		/* valid for RECV WCs */
+	u32 stag;
+} __attribute__((packed)) ;
+
+
+/*
+ * Flags used for all post-sq WRs.  These must fit in the flags
+ * field of the struct c2wr_hdr (eight bits).
+ */
+enum {
+	SQ_SIGNALED = 0x01,
+	SQ_READ_FENCE = 0x02,
+	SQ_FENCE = 0x04,
+};
+
+/*
+ * Common fields for all post-sq WRs.  Namely the standard header and a
+ * secondary header with fields common to all post-sq WRs.
+ */
+struct c2_sq_hdr {
+	struct c2wr_user_hdr user_hdr;
+} __attribute__((packed));
+
+/*
+ * Same as above but for post-rq WRs.
+ */
+struct c2_rq_hdr {
+	struct c2wr_user_hdr user_hdr;
+} __attribute__((packed));
+
+/*
+ * use the same struct for all sends.
+ */
+struct c2wr_send_req {
+	struct c2_sq_hdr sq_hdr;
+	u32 sge_len;
+	u32 remote_stag;
+	u8 data[0];		/* SGE array */
+} __attribute__((packed));
+
+union c2wr_send {
+	struct c2wr_send_req req;
+	struct c2wr_ce rep;
+} __attribute__((packed));
+
+struct c2wr_rdma_write_req {
+	struct c2_sq_hdr sq_hdr;
+	u64 remote_to;
+	u32 remote_stag;
+	u32 sge_len;
+	u8 data[0];		/* SGE array */
+} __attribute__((packed));
+
+union c2wr_rdma_write {
+	struct c2wr_rdma_write_req req;
+	struct c2wr_ce rep;
+} __attribute__((packed));
+
+struct c2wr_rdma_read_req {
+	struct c2_sq_hdr sq_hdr;
+	u64 local_to;
+	u64 remote_to;
+	u32 local_stag;
+	u32 remote_stag;
+	u32 length;
+} __attribute__((packed));
+
+union c2wr_rdma_read {
+	struct c2wr_rdma_read_req req;
+	struct c2wr_ce rep;
+} __attribute__((packed));
+
+struct c2wr_mw_bind_req {
+	struct c2_sq_hdr sq_hdr;
+	u64 va;
+	u8 stag_key;
+	u8 pad[3];
+	u32 mw_stag_index;
+	u32 mr_stag_index;
+	u32 length;
+	u32 flags;
+} __attribute__((packed));
+
+union c2wr_mw_bind {
+	struct c2wr_mw_bind_req req;
+	struct c2wr_ce rep;
+} __attribute__((packed));
+
+struct c2wr_nsmr_fastreg_req {
+	struct c2_sq_hdr sq_hdr;
+	u64 va;
+	u8 stag_key;
+	u8 pad[3];
+	u32 stag_index;
+	u32 pbe_size;
+	u32 fbo;
+	u32 length;
+	u32 addrs_length;
+	/* array of paddrs (must be aligned on a 64bit boundary) */
+	u64 paddrs[0];
+} __attribute__((packed));
+
+union c2wr_nsmr_fastreg {
+	struct c2wr_nsmr_fastreg_req req;
+	struct c2wr_ce rep;
+} __attribute__((packed));
+
+struct c2wr_stag_invalidate_req {
+	struct c2_sq_hdr sq_hdr;
+	u8 stag_key;
+	u8 pad[3];
+	u32 stag_index;
+} __attribute__((packed));
+
+union c2wr_stag_invalidate {
+	struct c2wr_stag_invalidate_req req;
+	struct c2wr_ce rep;
+} __attribute__((packed));
+
+union c2wr_sqwr {
+	struct c2_sq_hdr sq_hdr;
+	struct c2wr_send_req send;
+	struct c2wr_send_req send_se;
+	struct c2wr_send_req send_inv;
+	struct c2wr_send_req send_se_inv;
+	struct c2wr_rdma_write_req rdma_write;
+	struct c2wr_rdma_read_req rdma_read;
+	struct c2wr_mw_bind_req mw_bind;
+	struct c2wr_nsmr_fastreg_req nsmr_fastreg;
+	struct c2wr_stag_invalidate_req stag_inv;
+} __attribute__((packed));
+
+
+/*
+ * RQ WRs
+ */
+struct c2wr_rqwr {
+	struct c2_rq_hdr rq_hdr;
+	u8 data[0];		/* array of SGEs */
+} __attribute__((packed));
+
+union c2wr_recv {
+	struct c2wr_rqwr req;
+	struct c2wr_ce rep;
+} __attribute__((packed));
+
+/*
+ * All AEs start with this header.  Most AEs only need to convey the
+ * information in the header.  Some, like LLP connection events, need
+ * more info.  The union typdef c2wr_ae_t has all the possible AEs.
+ *
+ * hdr.context is the user_context from the rnic_open WR.  NULL If this
+ * is not affiliated with an rnic
+ *
+ * hdr.id is the AE identifier (eg;  CCAE_REMOTE_SHUTDOWN,
+ * CCAE_LLP_CLOSE_COMPLETE)
+ *
+ * resource_type is one of:  C2_RES_IND_QP, C2_RES_IND_CQ, C2_RES_IND_SRQ
+ *
+ * user_context is the context passed down when the host created the resource.
+ */
+struct c2wr_ae_hdr {
+	struct c2wr_hdr hdr;
+	u64 user_context;	/* user context for this res. */
+	u32 resource_type;	/* see enum c2_resource_indicator */
+	u32 resource;		/* handle for resource */
+	u32 qp_state;		/* current QP State */
+} __attribute__((packed));
+
+/*
+ * After submitting the CCAE_ACTIVE_CONNECT_RESULTS message on the AEQ,
+ * the adapter moves the QP into RTS state
+ */
+struct c2wr_ae_active_connect_results {
+	struct c2wr_ae_hdr ae_hdr;
+	u32 laddr;
+	u32 raddr;
+	u16 lport;
+	u16 rport;
+	u32 private_data_length;
+	u8 private_data[0];	/* data is in-line in the msg. */
+} __attribute__((packed));
+
+/*
+ * When connections are established by the stack (and the private data
+ * MPA frame is received), the adapter will generate an event to the host.
+ * The details of the connection, any private data, and the new connection
+ * request handle is passed up via the CCAE_CONNECTION_REQUEST msg on the
+ * AE queue:
+ */
+struct c2wr_ae_connection_request {
+	struct c2wr_ae_hdr ae_hdr;
+	u32 cr_handle;		/* connreq handle (sock ptr) */
+	u32 laddr;
+	u32 raddr;
+	u16 lport;
+	u16 rport;
+	u32 private_data_length;
+	u8 private_data[0];	/* data is in-line in the msg. */
+} __attribute__((packed));
+
+union c2wr_ae {
+	struct c2wr_ae_hdr ae_generic;
+	struct c2wr_ae_active_connect_results ae_active_connect_results;
+	struct c2wr_ae_connection_request ae_connection_request;
+} __attribute__((packed));
+
+struct c2wr_init_req {
+	struct c2wr_hdr hdr;
+	u64 hint_count;
+	u64 q0_host_shared;
+	u64 q1_host_shared;
+	u64 q1_host_msg_pool;
+	u64 q2_host_shared;
+	u64 q2_host_msg_pool;
+} __attribute__((packed));
+
+struct c2wr_init_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed));
+
+union c2wr_init {
+	struct c2wr_init_req req;
+	struct c2wr_init_rep rep;
+} __attribute__((packed));
+
+/*
+ * For upgrading flash.
+ */
+
+struct c2wr_flash_init_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+} __attribute__((packed));
+
+struct c2wr_flash_init_rep {
+	struct c2wr_hdr hdr;
+	u32 adapter_flash_buf_offset;
+	u32 adapter_flash_len;
+} __attribute__((packed));
+
+union c2wr_flash_init {
+	struct c2wr_flash_init_req req;
+	struct c2wr_flash_init_rep rep;
+} __attribute__((packed));
+
+struct c2wr_flash_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 len;
+} __attribute__((packed));
+
+struct c2wr_flash_rep {
+	struct c2wr_hdr hdr;
+	u32 status;
+} __attribute__((packed));
+
+union c2wr_flash {
+	struct c2wr_flash_req req;
+	struct c2wr_flash_rep rep;
+} __attribute__((packed));
+
+struct c2wr_buf_alloc_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 size;
+} __attribute__((packed));
+
+struct c2wr_buf_alloc_rep {
+	struct c2wr_hdr hdr;
+	u32 offset;		/* 0 if mem not available */
+	u32 size;		/* 0 if mem not available */
+} __attribute__((packed));
+
+union c2wr_buf_alloc {
+	struct c2wr_buf_alloc_req req;
+	struct c2wr_buf_alloc_rep rep;
+} __attribute__((packed));
+
+struct c2wr_buf_free_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 offset;		/* Must match value from alloc */
+	u32 size;		/* Must match value from alloc */
+} __attribute__((packed));
+
+struct c2wr_buf_free_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed));
+
+union c2wr_buf_free {
+	struct c2wr_buf_free_req req;
+	struct c2wr_ce rep;
+} __attribute__((packed));
+
+struct c2wr_flash_write_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 offset;
+	u32 size;
+	u32 type;
+	u32 flags;
+} __attribute__((packed));
+
+struct c2wr_flash_write_rep {
+	struct c2wr_hdr hdr;
+	u32 status;
+} __attribute__((packed));
+
+union c2wr_flash_write {
+	struct c2wr_flash_write_req req;
+	struct c2wr_flash_write_rep rep;
+} __attribute__((packed));
+
+/*
+ * Messages for LLP connection setup.
+ */
+
+/*
+ * Listen Request.  This allocates a listening endpoint to allow passive
+ * connection setup.  Newly established LLP connections are passed up
+ * via an AE.  See c2wr_ae_connection_request_t
+ */
+struct c2wr_ep_listen_create_req {
+	struct c2wr_hdr hdr;
+	u64 user_context;	/* returned in AEs. */
+	u32 rnic_handle;
+	u32 local_addr;		/* local addr, or 0  */
+	u16 local_port;		/* 0 means "pick one" */
+	u16 pad;
+	u32 backlog;		/* tradional tcp listen bl */
+} __attribute__((packed));
+
+struct c2wr_ep_listen_create_rep {
+	struct c2wr_hdr hdr;
+	u32 ep_handle;		/* handle to new listening ep */
+	u16 local_port;		/* resulting port... */
+	u16 pad;
+} __attribute__((packed));
+
+union c2wr_ep_listen_create {
+	struct c2wr_ep_listen_create_req req;
+	struct c2wr_ep_listen_create_rep rep;
+} __attribute__((packed));
+
+struct c2wr_ep_listen_destroy_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 ep_handle;
+} __attribute__((packed));
+
+struct c2wr_ep_listen_destroy_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed));
+
+union c2wr_ep_listen_destroy {
+	struct c2wr_ep_listen_destroy_req req;
+	struct c2wr_ep_listen_destroy_rep rep;
+} __attribute__((packed));
+
+struct c2wr_ep_query_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 ep_handle;
+} __attribute__((packed));
+
+struct c2wr_ep_query_rep {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 local_addr;
+	u32 remote_addr;
+	u16 local_port;
+	u16 remote_port;
+} __attribute__((packed));
+
+union c2wr_ep_query {
+	struct c2wr_ep_query_req req;
+	struct c2wr_ep_query_rep rep;
+} __attribute__((packed));
+
+
+/*
+ * The host passes this down to indicate acceptance of a pending iWARP
+ * connection.  The cr_handle was obtained from the CONNECTION_REQUEST
+ * AE passed up by the adapter.  See c2wr_ae_connection_request_t.
+ */
+struct c2wr_cr_accept_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 qp_handle;		/* QP to bind to this LLP conn */
+	u32 ep_handle;		/* LLP  handle to accept */
+	u32 private_data_length;
+	u8 private_data[0];	/* data in-line in msg. */
+} __attribute__((packed));
+
+/*
+ * adapter sends reply when private data is successfully submitted to
+ * the LLP.
+ */
+struct c2wr_cr_accept_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed));
+
+union c2wr_cr_accept {
+	struct c2wr_cr_accept_req req;
+	struct c2wr_cr_accept_rep rep;
+} __attribute__((packed));
+
+/*
+ * The host sends this down if a given iWARP connection request was
+ * rejected by the consumer.  The cr_handle was obtained from a
+ * previous c2wr_ae_connection_request_t AE sent by the adapter.
+ */
+struct  c2wr_cr_reject_req {
+	struct c2wr_hdr hdr;
+	u32 rnic_handle;
+	u32 ep_handle;		/* LLP handle to reject */
+} __attribute__((packed));
+
+/*
+ * Dunno if this is needed, but we'll add it for now.  The adapter will
+ * send the reject_reply after the LLP endpoint has been destroyed.
+ */
+struct  c2wr_cr_reject_rep {
+	struct c2wr_hdr hdr;
+} __attribute__((packed));
+
+union c2wr_cr_reject {
+	struct c2wr_cr_reject_req req;
+	struct c2wr_cr_reject_rep rep;
+} __attribute__((packed));
+
+/*
+ * console command.  Used to implement a debug console over the verbs
+ * request and reply queues.
+ */
+
+/*
+ * Console request message.  It contains:
+ *	- message hdr with id = CCWR_CONSOLE
+ *	- the physaddr/len of host memory to be used for the reply.
+ *	- the command string.  eg:  "netstat -s" or "zoneinfo"
+ */
+struct c2wr_console_req {
+	struct c2wr_hdr hdr;		/* id = CCWR_CONSOLE */
+	u64 reply_buf;		/* pinned host buf for reply */
+	u32 reply_buf_len;	/* length of reply buffer */
+	u8 command[0];		/* NUL terminated ascii string */
+	/* containing the command req */
+} __attribute__((packed));
+
+/*
+ * flags used in the console reply.
+ */
+enum c2_console_flags {
+	CONS_REPLY_TRUNCATED = 0x00000001	/* reply was truncated */
+} __attribute__((packed));
+
+/*
+ * Console reply message.
+ * hdr.result contains the c2_status_t error if the reply was _not_ generated,
+ * or C2_OK if the reply was generated.
+ */
+struct c2wr_console_rep {
+	struct c2wr_hdr hdr;		/* id = CCWR_CONSOLE */
+	u32 flags;
+} __attribute__((packed));
+
+union c2wr_console {
+	struct c2wr_console_req req;
+	struct c2wr_console_rep rep;
+} __attribute__((packed));
+
+
+/*
+ * Giant union with all WRs.  Makes life easier...
+ */
+union c2wr {
+	struct c2wr_hdr hdr;
+	struct c2wr_user_hdr user_hdr;
+	union c2wr_rnic_open rnic_open;
+	union c2wr_rnic_query rnic_query;
+	union c2wr_rnic_getconfig rnic_getconfig;
+	union c2wr_rnic_setconfig rnic_setconfig;
+	union c2wr_rnic_close rnic_close;
+	union c2wr_cq_create cq_create;
+	union c2wr_cq_modify cq_modify;
+	union c2wr_cq_destroy cq_destroy;
+	union c2wr_pd_alloc pd_alloc;
+	union c2wr_pd_dealloc pd_dealloc;
+	union c2wr_srq_create srq_create;
+	union c2wr_srq_destroy srq_destroy;
+	union c2wr_qp_create qp_create;
+	union c2wr_qp_query qp_query;
+	union c2wr_qp_modify qp_modify;
+	union c2wr_qp_destroy qp_destroy;
+	struct c2wr_qp_connect qp_connect;
+	union c2wr_nsmr_stag_alloc nsmr_stag_alloc;
+	union c2wr_nsmr_register nsmr_register;
+	union c2wr_nsmr_pbl nsmr_pbl;
+	union c2wr_mr_query mr_query;
+	union c2wr_mw_query mw_query;
+	union c2wr_stag_dealloc stag_dealloc;
+	union c2wr_sqwr sqwr;
+	struct c2wr_rqwr rqwr;
+	struct c2wr_ce ce;
+	union c2wr_ae ae;
+	union c2wr_init init;
+	union c2wr_ep_listen_create ep_listen_create;
+	union c2wr_ep_listen_destroy ep_listen_destroy;
+	union c2wr_cr_accept cr_accept;
+	union c2wr_cr_reject cr_reject;
+	union c2wr_console console;
+	union c2wr_flash_init flash_init;
+	union c2wr_flash flash;
+	union c2wr_buf_alloc buf_alloc;
+	union c2wr_buf_free buf_free;
+	union c2wr_flash_write flash_write;
+} __attribute__((packed));
+
+
+/*
+ * Accessors for the wr fields that are packed together tightly to
+ * reduce the wr message size.  The wr arguments are void* so that
+ * either a struct c2wr*, a struct c2wr_hdr*, or a pointer to any of the types
+ * in the struct c2wr union can be passed in.
+ */
+static __inline__ u8 c2_wr_get_id(void *wr)
+{
+	return ((struct c2wr_hdr *) wr)->id;
+}
+static __inline__ void c2_wr_set_id(void *wr, u8 id)
+{
+	((struct c2wr_hdr *) wr)->id = id;
+}
+static __inline__ u8 c2_wr_get_result(void *wr)
+{
+	return ((struct c2wr_hdr *) wr)->result;
+}
+static __inline__ void c2_wr_set_result(void *wr, u8 result)
+{
+	((struct c2wr_hdr *) wr)->result = result;
+}
+static __inline__ u8 c2_wr_get_flags(void *wr)
+{
+	return ((struct c2wr_hdr *) wr)->flags;
+}
+static __inline__ void c2_wr_set_flags(void *wr, u8 flags)
+{
+	((struct c2wr_hdr *) wr)->flags = flags;
+}
+static __inline__ u8 c2_wr_get_sge_count(void *wr)
+{
+	return ((struct c2wr_hdr *) wr)->sge_count;
+}
+static __inline__ void c2_wr_set_sge_count(void *wr, u8 sge_count)
+{
+	((struct c2wr_hdr *) wr)->sge_count = sge_count;
+}
+static __inline__ u32 c2_wr_get_wqe_count(void *wr)
+{
+	return ((struct c2wr_hdr *) wr)->wqe_count;
+}
+static __inline__ void c2_wr_set_wqe_count(void *wr, u32 wqe_count)
+{
+	((struct c2wr_hdr *) wr)->wqe_count = wqe_count;
+}
+
+#endif				/* _C2_WR_H_ */
diff --git a/drivers/infiniband/hw/ehca/Kconfig b/drivers/infiniband/hw/ehca/Kconfig
new file mode 100644
index 0000000..922389b
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/Kconfig
@@ -0,0 +1,16 @@
+config INFINIBAND_EHCA
+	tristate "eHCA support"
+	depends on IBMEBUS && INFINIBAND
+	---help---
+	This driver supports the IBM pSeries eHCA InfiniBand adapter.
+
+	To compile the driver as a module, choose M here. The module
+	will be called ib_ehca.
+
+config INFINIBAND_EHCA_SCALING
+	bool "Scaling support (EXPERIMENTAL)"
+	depends on IBMEBUS && INFINIBAND_EHCA && HOTPLUG_CPU && EXPERIMENTAL
+	---help---
+	eHCA scaling support schedules the CQ callbacks to different CPUs.
+
+	To enable this feature choose Y here.
diff --git a/drivers/infiniband/hw/ehca/Makefile b/drivers/infiniband/hw/ehca/Makefile
new file mode 100644
index 0000000..74d284e
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/Makefile
@@ -0,0 +1,16 @@
+#  Authors: Heiko J Schick <schickhj@de.ibm.com>
+#           Christoph Raisch <raisch@de.ibm.com>
+#           Joachim Fenkes <fenkes@de.ibm.com>
+#
+#  Copyright (c) 2005 IBM Corporation
+#
+#  All rights reserved.
+#
+#  This source code is distributed under a dual license of GPL v2.0 and OpenIB BSD.
+
+obj-$(CONFIG_INFINIBAND_EHCA) += ib_ehca.o
+
+ib_ehca-objs  = ehca_main.o ehca_hca.o ehca_mcast.o ehca_pd.o ehca_av.o ehca_eq.o \
+		ehca_cq.o ehca_qp.o ehca_sqp.o ehca_mrmw.o ehca_reqs.o ehca_irq.o \
+		ehca_uverbs.o ipz_pt_fn.o hcp_if.o hcp_phyp.o
+
diff --git a/drivers/infiniband/hw/ehca/ehca_av.c b/drivers/infiniband/hw/ehca/ehca_av.c
new file mode 100644
index 0000000..3bac197
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_av.c
@@ -0,0 +1,271 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  adress vector functions
+ *
+ *  Authors: Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Khadija Souissi <souissik@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <asm/current.h>
+
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+static struct kmem_cache *av_cache;
+
+struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr)
+{
+	int ret;
+	struct ehca_av *av;
+	struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
+					      ib_device);
+
+	av = kmem_cache_alloc(av_cache, SLAB_KERNEL);
+	if (!av) {
+		ehca_err(pd->device, "Out of memory pd=%p ah_attr=%p",
+			 pd, ah_attr);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	av->av.sl = ah_attr->sl;
+	av->av.dlid = ah_attr->dlid;
+	av->av.slid_path_bits = ah_attr->src_path_bits;
+
+	if (ehca_static_rate < 0) {
+		int ah_mult = ib_rate_to_mult(ah_attr->static_rate);
+		int ehca_mult =
+			ib_rate_to_mult(shca->sport[ah_attr->port_num].rate );
+
+		if (ah_mult >= ehca_mult)
+			av->av.ipd = 0;
+		else
+			av->av.ipd = (ah_mult > 0) ?
+				((ehca_mult - 1) / ah_mult) : 0;
+	} else
+	        av->av.ipd = ehca_static_rate;
+
+	av->av.lnh = ah_attr->ah_flags;
+	av->av.grh.word_0 = EHCA_BMASK_SET(GRH_IPVERSION_MASK, 6);
+	av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_TCLASS_MASK,
+					    ah_attr->grh.traffic_class);
+	av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
+					    ah_attr->grh.flow_label);
+	av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
+					    ah_attr->grh.hop_limit);
+	av->av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1B);
+	/* set sgid in grh.word_1 */
+	if (ah_attr->ah_flags & IB_AH_GRH) {
+		int rc;
+		struct ib_port_attr port_attr;
+		union ib_gid gid;
+		memset(&port_attr, 0, sizeof(port_attr));
+		rc = ehca_query_port(pd->device, ah_attr->port_num,
+				     &port_attr);
+		if (rc) { /* invalid port number */
+			ret = -EINVAL;
+			ehca_err(pd->device, "Invalid port number "
+				 "ehca_query_port() returned %x "
+				 "pd=%p ah_attr=%p", rc, pd, ah_attr);
+			goto create_ah_exit1;
+		}
+		memset(&gid, 0, sizeof(gid));
+		rc = ehca_query_gid(pd->device,
+				    ah_attr->port_num,
+				    ah_attr->grh.sgid_index, &gid);
+		if (rc) {
+			ret = -EINVAL;
+			ehca_err(pd->device, "Failed to retrieve sgid "
+				 "ehca_query_gid() returned %x "
+				 "pd=%p ah_attr=%p", rc, pd, ah_attr);
+			goto create_ah_exit1;
+		}
+		memcpy(&av->av.grh.word_1, &gid, sizeof(gid));
+	}
+	/* for the time being we use a hard coded PMTU of 2048 Bytes */
+	av->av.pmtu = 4;
+
+	/* dgid comes in grh.word_3 */
+	memcpy(&av->av.grh.word_3, &ah_attr->grh.dgid,
+	       sizeof(ah_attr->grh.dgid));
+
+	return &av->ib_ah;
+
+create_ah_exit1:
+	kmem_cache_free(av_cache, av);
+
+	return ERR_PTR(ret);
+}
+
+int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
+{
+	struct ehca_av *av;
+	struct ehca_ud_av new_ehca_av;
+	struct ehca_pd *my_pd = container_of(ah->pd, struct ehca_pd, ib_pd);
+	u32 cur_pid = current->tgid;
+
+	if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
+	    my_pd->ownpid != cur_pid) {
+		ehca_err(ah->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		return -EINVAL;
+	}
+
+	memset(&new_ehca_av, 0, sizeof(new_ehca_av));
+	new_ehca_av.sl = ah_attr->sl;
+	new_ehca_av.dlid = ah_attr->dlid;
+	new_ehca_av.slid_path_bits = ah_attr->src_path_bits;
+	new_ehca_av.ipd = ah_attr->static_rate;
+	new_ehca_av.lnh = EHCA_BMASK_SET(GRH_FLAG_MASK,
+					 (ah_attr->ah_flags & IB_AH_GRH) > 0);
+	new_ehca_av.grh.word_0 = EHCA_BMASK_SET(GRH_TCLASS_MASK,
+						ah_attr->grh.traffic_class);
+	new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_FLOWLABEL_MASK,
+						 ah_attr->grh.flow_label);
+	new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_HOPLIMIT_MASK,
+						 ah_attr->grh.hop_limit);
+	new_ehca_av.grh.word_0 |= EHCA_BMASK_SET(GRH_NEXTHEADER_MASK, 0x1b);
+
+	/* set sgid in grh.word_1 */
+	if (ah_attr->ah_flags & IB_AH_GRH) {
+		int rc;
+		struct ib_port_attr port_attr;
+		union ib_gid gid;
+		memset(&port_attr, 0, sizeof(port_attr));
+		rc = ehca_query_port(ah->device, ah_attr->port_num,
+				     &port_attr);
+		if (rc) { /* invalid port number */
+			ehca_err(ah->device, "Invalid port number "
+				 "ehca_query_port() returned %x "
+				 "ah=%p ah_attr=%p port_num=%x",
+				 rc, ah, ah_attr, ah_attr->port_num);
+			return -EINVAL;
+		}
+		memset(&gid, 0, sizeof(gid));
+		rc = ehca_query_gid(ah->device,
+				    ah_attr->port_num,
+				    ah_attr->grh.sgid_index, &gid);
+		if (rc) {
+			ehca_err(ah->device, "Failed to retrieve sgid "
+				 "ehca_query_gid() returned %x "
+				 "ah=%p ah_attr=%p port_num=%x "
+				 "sgid_index=%x",
+				 rc, ah, ah_attr, ah_attr->port_num,
+				 ah_attr->grh.sgid_index);
+			return -EINVAL;
+		}
+		memcpy(&new_ehca_av.grh.word_1, &gid, sizeof(gid));
+	}
+
+	new_ehca_av.pmtu = 4; /* see also comment in create_ah() */
+
+	memcpy(&new_ehca_av.grh.word_3, &ah_attr->grh.dgid,
+	       sizeof(ah_attr->grh.dgid));
+
+	av = container_of(ah, struct ehca_av, ib_ah);
+	av->av = new_ehca_av;
+
+	return 0;
+}
+
+int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr)
+{
+	struct ehca_av *av = container_of(ah, struct ehca_av, ib_ah);
+	struct ehca_pd *my_pd = container_of(ah->pd, struct ehca_pd, ib_pd);
+	u32 cur_pid = current->tgid;
+
+	if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
+	    my_pd->ownpid != cur_pid) {
+		ehca_err(ah->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		return -EINVAL;
+	}
+
+	memcpy(&ah_attr->grh.dgid, &av->av.grh.word_3,
+	       sizeof(ah_attr->grh.dgid));
+	ah_attr->sl = av->av.sl;
+
+	ah_attr->dlid = av->av.dlid;
+
+	ah_attr->src_path_bits = av->av.slid_path_bits;
+	ah_attr->static_rate = av->av.ipd;
+	ah_attr->ah_flags = EHCA_BMASK_GET(GRH_FLAG_MASK, av->av.lnh);
+	ah_attr->grh.traffic_class = EHCA_BMASK_GET(GRH_TCLASS_MASK,
+						    av->av.grh.word_0);
+	ah_attr->grh.hop_limit = EHCA_BMASK_GET(GRH_HOPLIMIT_MASK,
+						av->av.grh.word_0);
+	ah_attr->grh.flow_label = EHCA_BMASK_GET(GRH_FLOWLABEL_MASK,
+						 av->av.grh.word_0);
+
+	return 0;
+}
+
+int ehca_destroy_ah(struct ib_ah *ah)
+{
+	struct ehca_pd *my_pd = container_of(ah->pd, struct ehca_pd, ib_pd);
+	u32 cur_pid = current->tgid;
+
+	if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
+	    my_pd->ownpid != cur_pid) {
+		ehca_err(ah->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		return -EINVAL;
+	}
+
+	kmem_cache_free(av_cache, container_of(ah, struct ehca_av, ib_ah));
+
+	return 0;
+}
+
+int ehca_init_av_cache(void)
+{
+	av_cache = kmem_cache_create("ehca_cache_av",
+				   sizeof(struct ehca_av), 0,
+				   SLAB_HWCACHE_ALIGN,
+				   NULL, NULL);
+	if (!av_cache)
+		return -ENOMEM;
+	return 0;
+}
+
+void ehca_cleanup_av_cache(void)
+{
+	if (av_cache)
+		kmem_cache_destroy(av_cache);
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_classes.h b/drivers/infiniband/hw/ehca/ehca_classes.h
new file mode 100644
index 0000000..1c72203
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_classes.h
@@ -0,0 +1,346 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Struct definition for eHCA internal structures
+ *
+ *  Authors: Heiko J Schick <schickhj@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_CLASSES_H__
+#define __EHCA_CLASSES_H__
+
+#include "ehca_classes.h"
+#include "ipz_pt_fn.h"
+
+struct ehca_module;
+struct ehca_qp;
+struct ehca_cq;
+struct ehca_eq;
+struct ehca_mr;
+struct ehca_mw;
+struct ehca_pd;
+struct ehca_av;
+
+#ifdef CONFIG_PPC64
+#include "ehca_classes_pSeries.h"
+#endif
+
+#include <rdma/ib_verbs.h>
+#include <rdma/ib_user_verbs.h>
+
+#include "ehca_irq.h"
+
+struct ehca_eq {
+	u32 length;
+	struct ipz_queue ipz_queue;
+	struct ipz_eq_handle ipz_eq_handle;
+	struct work_struct work;
+	struct h_galpas galpas;
+	int is_initialized;
+	struct ehca_pfeq pf;
+	spinlock_t spinlock;
+	struct tasklet_struct interrupt_task;
+	u32 ist;
+};
+
+struct ehca_sport {
+	struct ib_cq *ibcq_aqp1;
+	struct ib_qp *ibqp_aqp1;
+	enum ib_rate  rate;
+	enum ib_port_state port_state;
+};
+
+struct ehca_shca {
+	struct ib_device ib_device;
+	struct ibmebus_dev *ibmebus_dev;
+	u8 num_ports;
+	int hw_level;
+	struct list_head shca_list;
+	struct ipz_adapter_handle ipz_hca_handle;
+	struct ehca_sport sport[2];
+	struct ehca_eq eq;
+	struct ehca_eq neq;
+	struct ehca_mr *maxmr;
+	struct ehca_pd *pd;
+	struct h_galpas galpas;
+};
+
+struct ehca_pd {
+	struct ib_pd ib_pd;
+	struct ipz_pd fw_pd;
+	u32 ownpid;
+};
+
+struct ehca_qp {
+	struct ib_qp ib_qp;
+	u32 qp_type;
+	struct ipz_queue ipz_squeue;
+	struct ipz_queue ipz_rqueue;
+	struct h_galpas galpas;
+	u32 qkey;
+	u32 real_qp_num;
+	u32 token;
+	spinlock_t spinlock_s;
+	spinlock_t spinlock_r;
+	u32 sq_max_inline_data_size;
+	struct ipz_qp_handle ipz_qp_handle;
+	struct ehca_pfqp pf;
+	struct ib_qp_init_attr init_attr;
+	u64 uspace_squeue;
+	u64 uspace_rqueue;
+	u64 uspace_fwh;
+	struct ehca_cq *send_cq;
+	struct ehca_cq *recv_cq;
+	unsigned int sqerr_purgeflag;
+	struct hlist_node list_entries;
+};
+
+/* must be power of 2 */
+#define QP_HASHTAB_LEN 8
+
+struct ehca_cq {
+	struct ib_cq ib_cq;
+	struct ipz_queue ipz_queue;
+	struct h_galpas galpas;
+	spinlock_t spinlock;
+	u32 cq_number;
+	u32 token;
+	u32 nr_of_entries;
+	struct ipz_cq_handle ipz_cq_handle;
+	struct ehca_pfcq pf;
+	spinlock_t cb_lock;
+	u64 uspace_queue;
+	u64 uspace_fwh;
+	struct hlist_head qp_hashtab[QP_HASHTAB_LEN];
+	struct list_head entry;
+	u32 nr_callbacks;
+	spinlock_t task_lock;
+	u32 ownpid;
+};
+
+enum ehca_mr_flag {
+	EHCA_MR_FLAG_FMR = 0x80000000,	 /* FMR, created with ehca_alloc_fmr */
+	EHCA_MR_FLAG_MAXMR = 0x40000000, /* max-MR                           */
+};
+
+struct ehca_mr {
+	union {
+		struct ib_mr ib_mr;	/* must always be first in ehca_mr */
+		struct ib_fmr ib_fmr;	/* must always be first in ehca_mr */
+	} ib;
+	spinlock_t mrlock;
+
+	enum ehca_mr_flag flags;
+	u32 num_pages;		/* number of MR pages */
+	u32 num_4k;		/* number of 4k "page" portions to form MR */
+	int acl;		/* ACL (stored here for usage in reregister) */
+	u64 *start;		/* virtual start address (stored here for */
+	                        /* usage in reregister) */
+	u64 size;		/* size (stored here for usage in reregister) */
+	u32 fmr_page_size;	/* page size for FMR */
+	u32 fmr_max_pages;	/* max pages for FMR */
+	u32 fmr_max_maps;	/* max outstanding maps for FMR */
+	u32 fmr_map_cnt;	/* map counter for FMR */
+	/* fw specific data */
+	struct ipz_mrmw_handle ipz_mr_handle;	/* MR handle for h-calls */
+	struct h_galpas galpas;
+	/* data for userspace bridge */
+	u32 nr_of_pages;
+	void *pagearray;
+};
+
+struct ehca_mw {
+	struct ib_mw ib_mw;	/* gen2 mw, must always be first in ehca_mw */
+	spinlock_t mwlock;
+
+	u8 never_bound;		/* indication MW was never bound */
+	struct ipz_mrmw_handle ipz_mw_handle;	/* MW handle for h-calls */
+	struct h_galpas galpas;
+};
+
+enum ehca_mr_pgi_type {
+	EHCA_MR_PGI_PHYS   = 1,  /* type of ehca_reg_phys_mr,
+				  * ehca_rereg_phys_mr,
+				  * ehca_reg_internal_maxmr */
+	EHCA_MR_PGI_USER   = 2,  /* type of ehca_reg_user_mr */
+	EHCA_MR_PGI_FMR    = 3   /* type of ehca_map_phys_fmr */
+};
+
+struct ehca_mr_pginfo {
+	enum ehca_mr_pgi_type type;
+	u64 num_pages;
+	u64 page_cnt;
+	u64 num_4k;       /* number of 4k "page" portions */
+	u64 page_4k_cnt;  /* counter for 4k "page" portions */
+	u64 next_4k;      /* next 4k "page" portion in buffer/chunk/listelem */
+
+	/* type EHCA_MR_PGI_PHYS section */
+	int num_phys_buf;
+	struct ib_phys_buf *phys_buf_array;
+	u64 next_buf;
+
+	/* type EHCA_MR_PGI_USER section */
+	struct ib_umem *region;
+	struct ib_umem_chunk *next_chunk;
+	u64 next_nmap;
+
+	/* type EHCA_MR_PGI_FMR section */
+	u64 *page_list;
+	u64 next_listelem;
+	/* next_4k also used within EHCA_MR_PGI_FMR */
+};
+
+/* output parameters for MR/FMR hipz calls */
+struct ehca_mr_hipzout_parms {
+	struct ipz_mrmw_handle handle;
+	u32 lkey;
+	u32 rkey;
+	u64 len;
+	u64 vaddr;
+	u32 acl;
+};
+
+/* output parameters for MW hipz calls */
+struct ehca_mw_hipzout_parms {
+	struct ipz_mrmw_handle handle;
+	u32 rkey;
+};
+
+struct ehca_av {
+	struct ib_ah ib_ah;
+	struct ehca_ud_av av;
+};
+
+struct ehca_ucontext {
+	struct ib_ucontext ib_ucontext;
+};
+
+struct ehca_module *ehca_module_new(void);
+
+int ehca_module_delete(struct ehca_module *me);
+
+int ehca_eq_ctor(struct ehca_eq *eq);
+
+int ehca_eq_dtor(struct ehca_eq *eq);
+
+struct ehca_shca *ehca_shca_new(void);
+
+int ehca_shca_delete(struct ehca_shca *me);
+
+struct ehca_sport *ehca_sport_new(struct ehca_shca *anchor);
+
+int ehca_init_pd_cache(void);
+void ehca_cleanup_pd_cache(void);
+int ehca_init_cq_cache(void);
+void ehca_cleanup_cq_cache(void);
+int ehca_init_qp_cache(void);
+void ehca_cleanup_qp_cache(void);
+int ehca_init_av_cache(void);
+void ehca_cleanup_av_cache(void);
+int ehca_init_mrmw_cache(void);
+void ehca_cleanup_mrmw_cache(void);
+
+extern spinlock_t ehca_qp_idr_lock;
+extern spinlock_t ehca_cq_idr_lock;
+extern struct idr ehca_qp_idr;
+extern struct idr ehca_cq_idr;
+
+extern int ehca_static_rate;
+extern int ehca_port_act_time;
+extern int ehca_use_hp_mr;
+
+struct ipzu_queue_resp {
+	u64 queue;        /* points to first queue entry */
+	u32 qe_size;      /* queue entry size */
+	u32 act_nr_of_sg;
+	u32 queue_length; /* queue length allocated in bytes */
+	u32 pagesize;
+	u32 toggle_state;
+	u32 dummy; /* padding for 8 byte alignment */
+};
+
+struct ehca_create_cq_resp {
+	u32 cq_number;
+	u32 token;
+	struct ipzu_queue_resp ipz_queue;
+	struct h_galpas galpas;
+};
+
+struct ehca_create_qp_resp {
+	u32 qp_num;
+	u32 token;
+	u32 qp_type;
+	u32 qkey;
+	/* qp_num assigned by ehca: sqp0/1 may have got different numbers */
+	u32 real_qp_num;
+	u32 dummy; /* padding for 8 byte alignment */
+	struct ipzu_queue_resp ipz_squeue;
+	struct ipzu_queue_resp ipz_rqueue;
+	struct h_galpas galpas;
+};
+
+struct ehca_alloc_cq_parms {
+	u32 nr_cqe;
+	u32 act_nr_of_entries;
+	u32 act_pages;
+	struct ipz_eq_handle eq_handle;
+};
+
+struct ehca_alloc_qp_parms {
+	int servicetype;
+	int sigtype;
+	int daqp_ctrl;
+	int max_send_sge;
+	int max_recv_sge;
+	int ud_av_l_key_ctl;
+
+	u16 act_nr_send_wqes;
+	u16 act_nr_recv_wqes;
+	u8  act_nr_recv_sges;
+	u8  act_nr_send_sges;
+
+	u32 nr_rq_pages;
+	u32 nr_sq_pages;
+
+	struct ipz_eq_handle ipz_eq_handle;
+	struct ipz_pd pd;
+};
+
+int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp);
+int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int qp_num);
+struct ehca_qp* ehca_cq_get_qp(struct ehca_cq *cq, int qp_num);
+
+#endif
diff --git a/drivers/infiniband/hw/ehca/ehca_classes_pSeries.h b/drivers/infiniband/hw/ehca/ehca_classes_pSeries.h
new file mode 100644
index 0000000..5665f21
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_classes_pSeries.h
@@ -0,0 +1,236 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  pSeries interface definitions
+ *
+ *  Authors: Waleri Fomin <fomin@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_CLASSES_PSERIES_H__
+#define __EHCA_CLASSES_PSERIES_H__
+
+#include "hcp_phyp.h"
+#include "ipz_pt_fn.h"
+
+
+struct ehca_pfqp {
+	struct ipz_qpt sqpt;
+	struct ipz_qpt rqpt;
+};
+
+struct ehca_pfcq {
+	struct ipz_qpt qpt;
+	u32 cqnr;
+};
+
+struct ehca_pfeq {
+	struct ipz_qpt qpt;
+	struct h_galpa galpa;
+	u32 eqnr;
+};
+
+struct ipz_adapter_handle {
+	u64 handle;
+};
+
+struct ipz_cq_handle {
+	u64 handle;
+};
+
+struct ipz_eq_handle {
+	u64 handle;
+};
+
+struct ipz_qp_handle {
+	u64 handle;
+};
+struct ipz_mrmw_handle {
+	u64 handle;
+};
+
+struct ipz_pd {
+	u32 value;
+};
+
+struct hcp_modify_qp_control_block {
+	u32 qkey;                      /* 00 */
+	u32 rdd;                       /* reliable datagram domain */
+	u32 send_psn;                  /* 02 */
+	u32 receive_psn;               /* 03 */
+	u32 prim_phys_port;            /* 04 */
+	u32 alt_phys_port;             /* 05 */
+	u32 prim_p_key_idx;            /* 06 */
+	u32 alt_p_key_idx;             /* 07 */
+	u32 rdma_atomic_ctrl;          /* 08 */
+	u32 qp_state;                  /* 09 */
+	u32 reserved_10;               /* 10 */
+	u32 rdma_nr_atomic_resp_res;   /* 11 */
+	u32 path_migration_state;      /* 12 */
+	u32 rdma_atomic_outst_dest_qp; /* 13 */
+	u32 dest_qp_nr;                /* 14 */
+	u32 min_rnr_nak_timer_field;   /* 15 */
+	u32 service_level;             /* 16 */
+	u32 send_grh_flag;             /* 17 */
+	u32 retry_count;               /* 18 */
+	u32 timeout;                   /* 19 */
+	u32 path_mtu;                  /* 20 */
+	u32 max_static_rate;           /* 21 */
+	u32 dlid;                      /* 22 */
+	u32 rnr_retry_count;           /* 23 */
+	u32 source_path_bits;          /* 24 */
+	u32 traffic_class;             /* 25 */
+	u32 hop_limit;                 /* 26 */
+	u32 source_gid_idx;            /* 27 */
+	u32 flow_label;                /* 28 */
+	u32 reserved_29;               /* 29 */
+	union {                        /* 30 */
+		u64 dw[2];
+		u8 byte[16];
+	} dest_gid;
+	u32 service_level_al;          /* 34 */
+	u32 send_grh_flag_al;          /* 35 */
+	u32 retry_count_al;            /* 36 */
+	u32 timeout_al;                /* 37 */
+	u32 max_static_rate_al;        /* 38 */
+	u32 dlid_al;                   /* 39 */
+	u32 rnr_retry_count_al;        /* 40 */
+	u32 source_path_bits_al;       /* 41 */
+	u32 traffic_class_al;          /* 42 */
+	u32 hop_limit_al;              /* 43 */
+	u32 source_gid_idx_al;         /* 44 */
+	u32 flow_label_al;             /* 45 */
+	u32 reserved_46;               /* 46 */
+	u32 reserved_47;               /* 47 */
+	union {                        /* 48 */
+		u64 dw[2];
+		u8 byte[16];
+	} dest_gid_al;
+	u32 max_nr_outst_send_wr;      /* 52 */
+	u32 max_nr_outst_recv_wr;      /* 53 */
+	u32 disable_ete_credit_check;  /* 54 */
+	u32 qp_number;                 /* 55 */
+	u64 send_queue_handle;         /* 56 */
+	u64 recv_queue_handle;         /* 58 */
+	u32 actual_nr_sges_in_sq_wqe;  /* 60 */
+	u32 actual_nr_sges_in_rq_wqe;  /* 61 */
+	u32 qp_enable;                 /* 62 */
+	u32 curr_srq_limit;            /* 63 */
+	u64 qp_aff_asyn_ev_log_reg;    /* 64 */
+	u64 shared_rq_hndl;            /* 66 */
+	u64 trigg_doorbell_qp_hndl;    /* 68 */
+	u32 reserved_70_127[58];       /* 70 */
+};
+
+#define MQPCB_MASK_QKEY                         EHCA_BMASK_IBM(0,0)
+#define MQPCB_MASK_SEND_PSN                     EHCA_BMASK_IBM(2,2)
+#define MQPCB_MASK_RECEIVE_PSN                  EHCA_BMASK_IBM(3,3)
+#define MQPCB_MASK_PRIM_PHYS_PORT               EHCA_BMASK_IBM(4,4)
+#define MQPCB_PRIM_PHYS_PORT                    EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_ALT_PHYS_PORT                EHCA_BMASK_IBM(5,5)
+#define MQPCB_MASK_PRIM_P_KEY_IDX               EHCA_BMASK_IBM(6,6)
+#define MQPCB_PRIM_P_KEY_IDX                    EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_ALT_P_KEY_IDX                EHCA_BMASK_IBM(7,7)
+#define MQPCB_MASK_RDMA_ATOMIC_CTRL             EHCA_BMASK_IBM(8,8)
+#define MQPCB_MASK_QP_STATE                     EHCA_BMASK_IBM(9,9)
+#define MQPCB_QP_STATE                          EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES      EHCA_BMASK_IBM(11,11)
+#define MQPCB_MASK_PATH_MIGRATION_STATE         EHCA_BMASK_IBM(12,12)
+#define MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP    EHCA_BMASK_IBM(13,13)
+#define MQPCB_MASK_DEST_QP_NR                   EHCA_BMASK_IBM(14,14)
+#define MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD      EHCA_BMASK_IBM(15,15)
+#define MQPCB_MASK_SERVICE_LEVEL                EHCA_BMASK_IBM(16,16)
+#define MQPCB_MASK_SEND_GRH_FLAG                EHCA_BMASK_IBM(17,17)
+#define MQPCB_MASK_RETRY_COUNT                  EHCA_BMASK_IBM(18,18)
+#define MQPCB_MASK_TIMEOUT                      EHCA_BMASK_IBM(19,19)
+#define MQPCB_MASK_PATH_MTU                     EHCA_BMASK_IBM(20,20)
+#define MQPCB_PATH_MTU                          EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_MAX_STATIC_RATE              EHCA_BMASK_IBM(21,21)
+#define MQPCB_MAX_STATIC_RATE                   EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_DLID                         EHCA_BMASK_IBM(22,22)
+#define MQPCB_DLID                              EHCA_BMASK_IBM(16,31)
+#define MQPCB_MASK_RNR_RETRY_COUNT              EHCA_BMASK_IBM(23,23)
+#define MQPCB_RNR_RETRY_COUNT                   EHCA_BMASK_IBM(29,31)
+#define MQPCB_MASK_SOURCE_PATH_BITS             EHCA_BMASK_IBM(24,24)
+#define MQPCB_SOURCE_PATH_BITS                  EHCA_BMASK_IBM(25,31)
+#define MQPCB_MASK_TRAFFIC_CLASS                EHCA_BMASK_IBM(25,25)
+#define MQPCB_TRAFFIC_CLASS                     EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_HOP_LIMIT                    EHCA_BMASK_IBM(26,26)
+#define MQPCB_HOP_LIMIT                         EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_SOURCE_GID_IDX               EHCA_BMASK_IBM(27,27)
+#define MQPCB_SOURCE_GID_IDX                    EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_FLOW_LABEL                   EHCA_BMASK_IBM(28,28)
+#define MQPCB_FLOW_LABEL                        EHCA_BMASK_IBM(12,31)
+#define MQPCB_MASK_DEST_GID                     EHCA_BMASK_IBM(30,30)
+#define MQPCB_MASK_SERVICE_LEVEL_AL             EHCA_BMASK_IBM(31,31)
+#define MQPCB_SERVICE_LEVEL_AL                  EHCA_BMASK_IBM(28,31)
+#define MQPCB_MASK_SEND_GRH_FLAG_AL             EHCA_BMASK_IBM(32,32)
+#define MQPCB_SEND_GRH_FLAG_AL                  EHCA_BMASK_IBM(31,31)
+#define MQPCB_MASK_RETRY_COUNT_AL               EHCA_BMASK_IBM(33,33)
+#define MQPCB_RETRY_COUNT_AL                    EHCA_BMASK_IBM(29,31)
+#define MQPCB_MASK_TIMEOUT_AL                   EHCA_BMASK_IBM(34,34)
+#define MQPCB_TIMEOUT_AL                        EHCA_BMASK_IBM(27,31)
+#define MQPCB_MASK_MAX_STATIC_RATE_AL           EHCA_BMASK_IBM(35,35)
+#define MQPCB_MAX_STATIC_RATE_AL                EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_DLID_AL                      EHCA_BMASK_IBM(36,36)
+#define MQPCB_DLID_AL                           EHCA_BMASK_IBM(16,31)
+#define MQPCB_MASK_RNR_RETRY_COUNT_AL           EHCA_BMASK_IBM(37,37)
+#define MQPCB_RNR_RETRY_COUNT_AL                EHCA_BMASK_IBM(29,31)
+#define MQPCB_MASK_SOURCE_PATH_BITS_AL          EHCA_BMASK_IBM(38,38)
+#define MQPCB_SOURCE_PATH_BITS_AL               EHCA_BMASK_IBM(25,31)
+#define MQPCB_MASK_TRAFFIC_CLASS_AL             EHCA_BMASK_IBM(39,39)
+#define MQPCB_TRAFFIC_CLASS_AL                  EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_HOP_LIMIT_AL                 EHCA_BMASK_IBM(40,40)
+#define MQPCB_HOP_LIMIT_AL                      EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_SOURCE_GID_IDX_AL            EHCA_BMASK_IBM(41,41)
+#define MQPCB_SOURCE_GID_IDX_AL                 EHCA_BMASK_IBM(24,31)
+#define MQPCB_MASK_FLOW_LABEL_AL                EHCA_BMASK_IBM(42,42)
+#define MQPCB_FLOW_LABEL_AL                     EHCA_BMASK_IBM(12,31)
+#define MQPCB_MASK_DEST_GID_AL                  EHCA_BMASK_IBM(44,44)
+#define MQPCB_MASK_MAX_NR_OUTST_SEND_WR         EHCA_BMASK_IBM(45,45)
+#define MQPCB_MAX_NR_OUTST_SEND_WR              EHCA_BMASK_IBM(16,31)
+#define MQPCB_MASK_MAX_NR_OUTST_RECV_WR         EHCA_BMASK_IBM(46,46)
+#define MQPCB_MAX_NR_OUTST_RECV_WR              EHCA_BMASK_IBM(16,31)
+#define MQPCB_MASK_DISABLE_ETE_CREDIT_CHECK     EHCA_BMASK_IBM(47,47)
+#define MQPCB_DISABLE_ETE_CREDIT_CHECK          EHCA_BMASK_IBM(31,31)
+#define MQPCB_QP_NUMBER                         EHCA_BMASK_IBM(8,31)
+#define MQPCB_MASK_QP_ENABLE                    EHCA_BMASK_IBM(48,48)
+#define MQPCB_QP_ENABLE                         EHCA_BMASK_IBM(31,31)
+#define MQPCB_MASK_CURR_SQR_LIMIT               EHCA_BMASK_IBM(49,49)
+#define MQPCB_CURR_SQR_LIMIT                    EHCA_BMASK_IBM(15,31)
+#define MQPCB_MASK_QP_AFF_ASYN_EV_LOG_REG       EHCA_BMASK_IBM(50,50)
+#define MQPCB_MASK_SHARED_RQ_HNDL               EHCA_BMASK_IBM(51,51)
+
+#endif /* __EHCA_CLASSES_PSERIES_H__ */
diff --git a/drivers/infiniband/hw/ehca/ehca_cq.c b/drivers/infiniband/hw/ehca/ehca_cq.c
new file mode 100644
index 0000000..458fe19
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_cq.c
@@ -0,0 +1,427 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Completion queue handling
+ *
+ *  Authors: Waleri Fomin <fomin@de.ibm.com>
+ *           Khadija Souissi <souissi@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *           Heiko J Schick <schickhj@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/current.h>
+
+#include "ehca_iverbs.h"
+#include "ehca_classes.h"
+#include "ehca_irq.h"
+#include "hcp_if.h"
+
+static struct kmem_cache *cq_cache;
+
+int ehca_cq_assign_qp(struct ehca_cq *cq, struct ehca_qp *qp)
+{
+	unsigned int qp_num = qp->real_qp_num;
+	unsigned int key = qp_num & (QP_HASHTAB_LEN-1);
+	unsigned long spl_flags;
+
+	spin_lock_irqsave(&cq->spinlock, spl_flags);
+	hlist_add_head(&qp->list_entries, &cq->qp_hashtab[key]);
+	spin_unlock_irqrestore(&cq->spinlock, spl_flags);
+
+	ehca_dbg(cq->ib_cq.device, "cq_num=%x real_qp_num=%x",
+		 cq->cq_number, qp_num);
+
+	return 0;
+}
+
+int ehca_cq_unassign_qp(struct ehca_cq *cq, unsigned int real_qp_num)
+{
+	int ret = -EINVAL;
+	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
+	struct hlist_node *iter;
+	struct ehca_qp *qp;
+	unsigned long spl_flags;
+
+	spin_lock_irqsave(&cq->spinlock, spl_flags);
+	hlist_for_each(iter, &cq->qp_hashtab[key]) {
+		qp = hlist_entry(iter, struct ehca_qp, list_entries);
+		if (qp->real_qp_num == real_qp_num) {
+			hlist_del(iter);
+			ehca_dbg(cq->ib_cq.device,
+				 "removed qp from cq .cq_num=%x real_qp_num=%x",
+				 cq->cq_number, real_qp_num);
+			ret = 0;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&cq->spinlock, spl_flags);
+	if (ret)
+		ehca_err(cq->ib_cq.device,
+			 "qp not found cq_num=%x real_qp_num=%x",
+			 cq->cq_number, real_qp_num);
+
+	return ret;
+}
+
+struct ehca_qp* ehca_cq_get_qp(struct ehca_cq *cq, int real_qp_num)
+{
+	struct ehca_qp *ret = NULL;
+	unsigned int key = real_qp_num & (QP_HASHTAB_LEN-1);
+	struct hlist_node *iter;
+	struct ehca_qp *qp;
+	hlist_for_each(iter, &cq->qp_hashtab[key]) {
+		qp = hlist_entry(iter, struct ehca_qp, list_entries);
+		if (qp->real_qp_num == real_qp_num) {
+			ret = qp;
+			break;
+		}
+	}
+	return ret;
+}
+
+struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe,
+			     struct ib_ucontext *context,
+			     struct ib_udata *udata)
+{
+	static const u32 additional_cqe = 20;
+	struct ib_cq *cq;
+	struct ehca_cq *my_cq;
+	struct ehca_shca *shca =
+		container_of(device, struct ehca_shca, ib_device);
+	struct ipz_adapter_handle adapter_handle;
+	struct ehca_alloc_cq_parms param; /* h_call's out parameters */
+	struct h_galpa gal;
+	void *vpage;
+	u32 counter;
+	u64 rpage, cqx_fec, h_ret;
+	int ipz_rc, ret, i;
+	unsigned long flags;
+
+	if (cqe >= 0xFFFFFFFF - 64 - additional_cqe)
+		return ERR_PTR(-EINVAL);
+
+	my_cq = kmem_cache_alloc(cq_cache, SLAB_KERNEL);
+	if (!my_cq) {
+		ehca_err(device, "Out of memory for ehca_cq struct device=%p",
+			 device);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	memset(my_cq, 0, sizeof(struct ehca_cq));
+	memset(&param, 0, sizeof(struct ehca_alloc_cq_parms));
+
+	spin_lock_init(&my_cq->spinlock);
+	spin_lock_init(&my_cq->cb_lock);
+	spin_lock_init(&my_cq->task_lock);
+	my_cq->ownpid = current->tgid;
+
+	cq = &my_cq->ib_cq;
+
+	adapter_handle = shca->ipz_hca_handle;
+	param.eq_handle = shca->eq.ipz_eq_handle;
+
+	do {
+		if (!idr_pre_get(&ehca_cq_idr, GFP_KERNEL)) {
+			cq = ERR_PTR(-ENOMEM);
+			ehca_err(device, "Can't reserve idr nr. device=%p",
+				 device);
+			goto create_cq_exit1;
+		}
+
+		spin_lock_irqsave(&ehca_cq_idr_lock, flags);
+		ret = idr_get_new(&ehca_cq_idr, my_cq, &my_cq->token);
+		spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+	} while (ret == -EAGAIN);
+
+	if (ret) {
+		cq = ERR_PTR(-ENOMEM);
+		ehca_err(device, "Can't allocate new idr entry. device=%p",
+			 device);
+		goto create_cq_exit1;
+	}
+
+	/*
+	 * CQs maximum depth is 4GB-64, but we need additional 20 as buffer
+	 * for receiving errors CQEs.
+	 */
+	param.nr_cqe = cqe + additional_cqe;
+	h_ret = hipz_h_alloc_resource_cq(adapter_handle, my_cq, &param);
+
+	if (h_ret != H_SUCCESS) {
+		ehca_err(device, "hipz_h_alloc_resource_cq() failed "
+			 "h_ret=%lx device=%p", h_ret, device);
+		cq = ERR_PTR(ehca2ib_return_code(h_ret));
+		goto create_cq_exit2;
+	}
+
+	ipz_rc = ipz_queue_ctor(&my_cq->ipz_queue, param.act_pages,
+				EHCA_PAGESIZE, sizeof(struct ehca_cqe), 0);
+	if (!ipz_rc) {
+		ehca_err(device, "ipz_queue_ctor() failed ipz_rc=%x device=%p",
+			 ipz_rc, device);
+		cq = ERR_PTR(-EINVAL);
+		goto create_cq_exit3;
+	}
+
+	for (counter = 0; counter < param.act_pages; counter++) {
+		vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
+		if (!vpage) {
+			ehca_err(device, "ipz_qpageit_get_inc() "
+				 "returns NULL device=%p", device);
+			cq = ERR_PTR(-EAGAIN);
+			goto create_cq_exit4;
+		}
+		rpage = virt_to_abs(vpage);
+
+		h_ret = hipz_h_register_rpage_cq(adapter_handle,
+						 my_cq->ipz_cq_handle,
+						 &my_cq->pf,
+						 0,
+						 0,
+						 rpage,
+						 1,
+						 my_cq->galpas.
+						 kernel);
+
+		if (h_ret < H_SUCCESS) {
+			ehca_err(device, "hipz_h_register_rpage_cq() failed "
+				 "ehca_cq=%p cq_num=%x h_ret=%lx counter=%i "
+				 "act_pages=%i", my_cq, my_cq->cq_number,
+				 h_ret, counter, param.act_pages);
+			cq = ERR_PTR(-EINVAL);
+			goto create_cq_exit4;
+		}
+
+		if (counter == (param.act_pages - 1)) {
+			vpage = ipz_qpageit_get_inc(&my_cq->ipz_queue);
+			if ((h_ret != H_SUCCESS) || vpage) {
+				ehca_err(device, "Registration of pages not "
+					 "complete ehca_cq=%p cq_num=%x "
+					 "h_ret=%lx", my_cq, my_cq->cq_number,
+					 h_ret);
+				cq = ERR_PTR(-EAGAIN);
+				goto create_cq_exit4;
+			}
+		} else {
+			if (h_ret != H_PAGE_REGISTERED) {
+				ehca_err(device, "Registration of page failed "
+					 "ehca_cq=%p cq_num=%x h_ret=%lx"
+					 "counter=%i act_pages=%i",
+					 my_cq, my_cq->cq_number,
+					 h_ret, counter, param.act_pages);
+				cq = ERR_PTR(-ENOMEM);
+				goto create_cq_exit4;
+			}
+		}
+	}
+
+	ipz_qeit_reset(&my_cq->ipz_queue);
+
+	gal = my_cq->galpas.kernel;
+	cqx_fec = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_fec));
+	ehca_dbg(device, "ehca_cq=%p cq_num=%x CQX_FEC=%lx",
+		 my_cq, my_cq->cq_number, cqx_fec);
+
+	my_cq->ib_cq.cqe = my_cq->nr_of_entries =
+		param.act_nr_of_entries - additional_cqe;
+	my_cq->cq_number = (my_cq->ipz_cq_handle.handle) & 0xffff;
+
+	for (i = 0; i < QP_HASHTAB_LEN; i++)
+		INIT_HLIST_HEAD(&my_cq->qp_hashtab[i]);
+
+	if (context) {
+		struct ipz_queue *ipz_queue = &my_cq->ipz_queue;
+		struct ehca_create_cq_resp resp;
+		struct vm_area_struct *vma;
+		memset(&resp, 0, sizeof(resp));
+		resp.cq_number = my_cq->cq_number;
+		resp.token = my_cq->token;
+		resp.ipz_queue.qe_size = ipz_queue->qe_size;
+		resp.ipz_queue.act_nr_of_sg = ipz_queue->act_nr_of_sg;
+		resp.ipz_queue.queue_length = ipz_queue->queue_length;
+		resp.ipz_queue.pagesize = ipz_queue->pagesize;
+		resp.ipz_queue.toggle_state = ipz_queue->toggle_state;
+		ret = ehca_mmap_nopage(((u64)(my_cq->token) << 32) | 0x12000000,
+				       ipz_queue->queue_length,
+				       (void**)&resp.ipz_queue.queue,
+				       &vma);
+		if (ret) {
+			ehca_err(device, "Could not mmap queue pages");
+			cq = ERR_PTR(ret);
+			goto create_cq_exit4;
+		}
+		my_cq->uspace_queue = resp.ipz_queue.queue;
+		resp.galpas = my_cq->galpas;
+		ret = ehca_mmap_register(my_cq->galpas.user.fw_handle,
+					 (void**)&resp.galpas.kernel.fw_handle,
+					 &vma);
+		if (ret) {
+			ehca_err(device, "Could not mmap fw_handle");
+			cq = ERR_PTR(ret);
+			goto create_cq_exit5;
+		}
+		my_cq->uspace_fwh = (u64)resp.galpas.kernel.fw_handle;
+		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
+			ehca_err(device, "Copy to udata failed.");
+			goto create_cq_exit6;
+		}
+	}
+
+	return cq;
+
+create_cq_exit6:
+	ehca_munmap(my_cq->uspace_fwh, EHCA_PAGESIZE);
+
+create_cq_exit5:
+	ehca_munmap(my_cq->uspace_queue, my_cq->ipz_queue.queue_length);
+
+create_cq_exit4:
+	ipz_queue_dtor(&my_cq->ipz_queue);
+
+create_cq_exit3:
+	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
+	if (h_ret != H_SUCCESS)
+		ehca_err(device, "hipz_h_destroy_cq() failed ehca_cq=%p "
+			 "cq_num=%x h_ret=%lx", my_cq, my_cq->cq_number, h_ret);
+
+create_cq_exit2:
+	spin_lock_irqsave(&ehca_cq_idr_lock, flags);
+	idr_remove(&ehca_cq_idr, my_cq->token);
+	spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+create_cq_exit1:
+	kmem_cache_free(cq_cache, my_cq);
+
+	return cq;
+}
+
+int ehca_destroy_cq(struct ib_cq *cq)
+{
+	u64 h_ret;
+	int ret;
+	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+	int cq_num = my_cq->cq_number;
+	struct ib_device *device = cq->device;
+	struct ehca_shca *shca = container_of(device, struct ehca_shca,
+					      ib_device);
+	struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
+	u32 cur_pid = current->tgid;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ehca_cq_idr_lock, flags);
+	while (my_cq->nr_callbacks)
+		yield();
+
+	idr_remove(&ehca_cq_idr, my_cq->token);
+	spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+	if (my_cq->uspace_queue && my_cq->ownpid != cur_pid) {
+		ehca_err(device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_cq->ownpid);
+		return -EINVAL;
+	}
+
+	/* un-mmap if vma alloc */
+	if (my_cq->uspace_queue ) {
+		ret = ehca_munmap(my_cq->uspace_queue,
+				  my_cq->ipz_queue.queue_length);
+		if (ret)
+			ehca_err(device, "Could not munmap queue ehca_cq=%p "
+				 "cq_num=%x", my_cq, cq_num);
+		ret = ehca_munmap(my_cq->uspace_fwh, EHCA_PAGESIZE);
+		if (ret)
+			ehca_err(device, "Could not munmap fwh ehca_cq=%p "
+				 "cq_num=%x", my_cq, cq_num);
+	}
+
+	h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 0);
+	if (h_ret == H_R_STATE) {
+		/* cq in err: read err data and destroy it forcibly */
+		ehca_dbg(device, "ehca_cq=%p cq_num=%x ressource=%lx in err "
+			 "state. Try to delete it forcibly.",
+			 my_cq, cq_num, my_cq->ipz_cq_handle.handle);
+		ehca_error_data(shca, my_cq, my_cq->ipz_cq_handle.handle);
+		h_ret = hipz_h_destroy_cq(adapter_handle, my_cq, 1);
+		if (h_ret == H_SUCCESS)
+			ehca_dbg(device, "cq_num=%x deleted successfully.",
+				 cq_num);
+	}
+	if (h_ret != H_SUCCESS) {
+		ehca_err(device, "hipz_h_destroy_cq() failed h_ret=%lx "
+			 "ehca_cq=%p cq_num=%x", h_ret, my_cq, cq_num);
+		return ehca2ib_return_code(h_ret);
+	}
+	ipz_queue_dtor(&my_cq->ipz_queue);
+	kmem_cache_free(cq_cache, my_cq);
+
+	return 0;
+}
+
+int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata)
+{
+	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+	u32 cur_pid = current->tgid;
+
+	if (my_cq->uspace_queue && my_cq->ownpid != cur_pid) {
+		ehca_err(cq->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_cq->ownpid);
+		return -EINVAL;
+	}
+
+	/* TODO: proper resize needs to be done */
+	ehca_err(cq->device, "not implemented yet");
+
+	return -EFAULT;
+}
+
+int ehca_init_cq_cache(void)
+{
+	cq_cache = kmem_cache_create("ehca_cache_cq",
+				     sizeof(struct ehca_cq), 0,
+				     SLAB_HWCACHE_ALIGN,
+				     NULL, NULL);
+	if (!cq_cache)
+		return -ENOMEM;
+	return 0;
+}
+
+void ehca_cleanup_cq_cache(void)
+{
+	if (cq_cache)
+		kmem_cache_destroy(cq_cache);
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_eq.c b/drivers/infiniband/hw/ehca/ehca_eq.c
new file mode 100644
index 0000000..5281dec
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_eq.c
@@ -0,0 +1,185 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Event queue handling
+ *
+ *  Authors: Waleri Fomin <fomin@de.ibm.com>
+ *           Khadija Souissi <souissi@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *           Heiko J Schick <schickhj@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_classes.h"
+#include "ehca_irq.h"
+#include "ehca_iverbs.h"
+#include "ehca_qes.h"
+#include "hcp_if.h"
+#include "ipz_pt_fn.h"
+
+int ehca_create_eq(struct ehca_shca *shca,
+		   struct ehca_eq *eq,
+		   const enum ehca_eq_type type, const u32 length)
+{
+	u64 ret;
+	u32 nr_pages;
+	u32 i;
+	void *vpage;
+	struct ib_device *ib_dev = &shca->ib_device;
+
+	spin_lock_init(&eq->spinlock);
+	eq->is_initialized = 0;
+
+	if (type != EHCA_EQ && type != EHCA_NEQ) {
+		ehca_err(ib_dev, "Invalid EQ type %x. eq=%p", type, eq);
+		return -EINVAL;
+	}
+	if (!length) {
+		ehca_err(ib_dev, "EQ length must not be zero. eq=%p", eq);
+		return -EINVAL;
+	}
+
+	ret = hipz_h_alloc_resource_eq(shca->ipz_hca_handle,
+				       &eq->pf,
+				       type,
+				       length,
+				       &eq->ipz_eq_handle,
+				       &eq->length,
+				       &nr_pages, &eq->ist);
+
+	if (ret != H_SUCCESS) {
+		ehca_err(ib_dev, "Can't allocate EQ/NEQ. eq=%p", eq);
+		return -EINVAL;
+	}
+
+	ret = ipz_queue_ctor(&eq->ipz_queue, nr_pages,
+			     EHCA_PAGESIZE, sizeof(struct ehca_eqe), 0);
+	if (!ret) {
+		ehca_err(ib_dev, "Can't allocate EQ pages eq=%p", eq);
+		goto create_eq_exit1;
+	}
+
+	for (i = 0; i < nr_pages; i++) {
+		u64 rpage;
+
+		if (!(vpage = ipz_qpageit_get_inc(&eq->ipz_queue))) {
+			ret = H_RESOURCE;
+			goto create_eq_exit2;
+		}
+
+		rpage = virt_to_abs(vpage);
+		ret = hipz_h_register_rpage_eq(shca->ipz_hca_handle,
+					       eq->ipz_eq_handle,
+					       &eq->pf,
+					       0, 0, rpage, 1);
+
+		if (i == (nr_pages - 1)) {
+			/* last page */
+			vpage = ipz_qpageit_get_inc(&eq->ipz_queue);
+			if (ret != H_SUCCESS || vpage)
+				goto create_eq_exit2;
+		} else {
+			if (ret != H_PAGE_REGISTERED || !vpage)
+				goto create_eq_exit2;
+		}
+	}
+
+	ipz_qeit_reset(&eq->ipz_queue);
+
+	/* register interrupt handlers and initialize work queues */
+	if (type == EHCA_EQ) {
+		ret = ibmebus_request_irq(NULL, eq->ist, ehca_interrupt_eq,
+					  SA_INTERRUPT, "ehca_eq",
+					  (void *)shca);
+		if (ret < 0)
+			ehca_err(ib_dev, "Can't map interrupt handler.");
+
+		tasklet_init(&eq->interrupt_task, ehca_tasklet_eq, (long)shca);
+	} else if (type == EHCA_NEQ) {
+		ret = ibmebus_request_irq(NULL, eq->ist, ehca_interrupt_neq,
+					  SA_INTERRUPT, "ehca_neq",
+					  (void *)shca);
+		if (ret < 0)
+			ehca_err(ib_dev, "Can't map interrupt handler.");
+
+		tasklet_init(&eq->interrupt_task, ehca_tasklet_neq, (long)shca);
+	}
+
+	eq->is_initialized = 1;
+
+	return 0;
+
+create_eq_exit2:
+	ipz_queue_dtor(&eq->ipz_queue);
+
+create_eq_exit1:
+	hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
+
+	return -EINVAL;
+}
+
+void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq)
+{
+	unsigned long flags;
+	void *eqe;
+
+	spin_lock_irqsave(&eq->spinlock, flags);
+	eqe = ipz_eqit_eq_get_inc_valid(&eq->ipz_queue);
+	spin_unlock_irqrestore(&eq->spinlock, flags);
+
+	return eqe;
+}
+
+int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq)
+{
+	unsigned long flags;
+	u64 h_ret;
+
+	spin_lock_irqsave(&eq->spinlock, flags);
+	ibmebus_free_irq(NULL, eq->ist, (void *)shca);
+
+	h_ret = hipz_h_destroy_eq(shca->ipz_hca_handle, eq);
+
+	spin_unlock_irqrestore(&eq->spinlock, flags);
+
+	if (h_ret != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "Can't free EQ resources.");
+		return -EINVAL;
+	}
+	ipz_queue_dtor(&eq->ipz_queue);
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_hca.c b/drivers/infiniband/hw/ehca/ehca_hca.c
new file mode 100644
index 0000000..5eae6ac
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_hca.c
@@ -0,0 +1,241 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  HCA query functions
+ *
+ *  Authors: Heiko J Schick <schickhj@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_tools.h"
+#include "hcp_if.h"
+
+int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props)
+{
+	int ret = 0;
+	struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
+					      ib_device);
+	struct hipz_query_hca *rblock;
+
+	rblock = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (!rblock) {
+		ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+		return -ENOMEM;
+	}
+
+	if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "Can't query device properties");
+		ret = -EINVAL;
+		goto query_device1;
+	}
+
+	memset(props, 0, sizeof(struct ib_device_attr));
+	props->fw_ver          = rblock->hw_ver;
+	props->max_mr_size     = rblock->max_mr_size;
+	props->vendor_id       = rblock->vendor_id >> 8;
+	props->vendor_part_id  = rblock->vendor_part_id >> 16;
+	props->hw_ver          = rblock->hw_ver;
+	props->max_qp          = min_t(int, rblock->max_qp, INT_MAX);
+	props->max_qp_wr       = min_t(int, rblock->max_wqes_wq, INT_MAX);
+	props->max_sge         = min_t(int, rblock->max_sge, INT_MAX);
+	props->max_sge_rd      = min_t(int, rblock->max_sge_rd, INT_MAX);
+	props->max_cq          = min_t(int, rblock->max_cq, INT_MAX);
+	props->max_cqe         = min_t(int, rblock->max_cqe, INT_MAX);
+	props->max_mr          = min_t(int, rblock->max_mr, INT_MAX);
+	props->max_mw          = min_t(int, rblock->max_mw, INT_MAX);
+	props->max_pd          = min_t(int, rblock->max_pd, INT_MAX);
+	props->max_ah          = min_t(int, rblock->max_ah, INT_MAX);
+	props->max_fmr         = min_t(int, rblock->max_mr, INT_MAX);
+	props->max_srq         = 0;
+	props->max_srq_wr      = 0;
+	props->max_srq_sge     = 0;
+	props->max_pkeys       = 16;
+	props->local_ca_ack_delay
+		= rblock->local_ca_ack_delay;
+	props->max_raw_ipv6_qp
+		= min_t(int, rblock->max_raw_ipv6_qp, INT_MAX);
+	props->max_raw_ethy_qp
+		= min_t(int, rblock->max_raw_ethy_qp, INT_MAX);
+	props->max_mcast_grp
+		= min_t(int, rblock->max_mcast_grp, INT_MAX);
+	props->max_mcast_qp_attach
+		= min_t(int, rblock->max_mcast_qp_attach, INT_MAX);
+	props->max_total_mcast_qp_attach
+		= min_t(int, rblock->max_total_mcast_qp_attach, INT_MAX);
+
+query_device1:
+	kfree(rblock);
+
+	return ret;
+}
+
+int ehca_query_port(struct ib_device *ibdev,
+		    u8 port, struct ib_port_attr *props)
+{
+	int ret = 0;
+	struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
+					      ib_device);
+	struct hipz_query_port *rblock;
+
+	rblock = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (!rblock) {
+		ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+		return -ENOMEM;
+	}
+
+	if (hipz_h_query_port(shca->ipz_hca_handle, port, rblock) != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "Can't query port properties");
+		ret = -EINVAL;
+		goto query_port1;
+	}
+
+	memset(props, 0, sizeof(struct ib_port_attr));
+	props->state = rblock->state;
+
+	switch (rblock->max_mtu) {
+	case 0x1:
+		props->active_mtu = props->max_mtu = IB_MTU_256;
+		break;
+	case 0x2:
+		props->active_mtu = props->max_mtu = IB_MTU_512;
+		break;
+	case 0x3:
+		props->active_mtu = props->max_mtu = IB_MTU_1024;
+		break;
+	case 0x4:
+		props->active_mtu = props->max_mtu = IB_MTU_2048;
+		break;
+	case 0x5:
+		props->active_mtu = props->max_mtu = IB_MTU_4096;
+		break;
+	default:
+		ehca_err(&shca->ib_device, "Unknown MTU size: %x.",
+			 rblock->max_mtu);
+		break;
+	}
+
+	props->gid_tbl_len     = rblock->gid_tbl_len;
+	props->max_msg_sz      = rblock->max_msg_sz;
+	props->bad_pkey_cntr   = rblock->bad_pkey_cntr;
+	props->qkey_viol_cntr  = rblock->qkey_viol_cntr;
+	props->pkey_tbl_len    = rblock->pkey_tbl_len;
+	props->lid             = rblock->lid;
+	props->sm_lid          = rblock->sm_lid;
+	props->lmc             = rblock->lmc;
+	props->sm_sl           = rblock->sm_sl;
+	props->subnet_timeout  = rblock->subnet_timeout;
+	props->init_type_reply = rblock->init_type_reply;
+
+	props->active_width    = IB_WIDTH_12X;
+	props->active_speed    = 0x1;
+
+query_port1:
+	kfree(rblock);
+
+	return ret;
+}
+
+int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
+{
+	int ret = 0;
+	struct ehca_shca *shca = container_of(ibdev, struct ehca_shca, ib_device);
+	struct hipz_query_port *rblock;
+
+	if (index > 16) {
+		ehca_err(&shca->ib_device, "Invalid index: %x.", index);
+		return -EINVAL;
+	}
+
+	rblock = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (!rblock) {
+		ehca_err(&shca->ib_device,  "Can't allocate rblock memory.");
+		return -ENOMEM;
+	}
+
+	if (hipz_h_query_port(shca->ipz_hca_handle, port, rblock) != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "Can't query port properties");
+		ret = -EINVAL;
+		goto query_pkey1;
+	}
+
+	memcpy(pkey, &rblock->pkey_entries + index, sizeof(u16));
+
+query_pkey1:
+	kfree(rblock);
+
+	return ret;
+}
+
+int ehca_query_gid(struct ib_device *ibdev, u8 port,
+		   int index, union ib_gid *gid)
+{
+	int ret = 0;
+	struct ehca_shca *shca = container_of(ibdev, struct ehca_shca,
+					      ib_device);
+	struct hipz_query_port *rblock;
+
+	if (index > 255) {
+		ehca_err(&shca->ib_device, "Invalid index: %x.", index);
+		return -EINVAL;
+	}
+
+	rblock = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (!rblock) {
+		ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+		return -ENOMEM;
+	}
+
+	if (hipz_h_query_port(shca->ipz_hca_handle, port, rblock) != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "Can't query port properties");
+		ret = -EINVAL;
+		goto query_gid1;
+	}
+
+	memcpy(&gid->raw[0], &rblock->gid_prefix, sizeof(u64));
+	memcpy(&gid->raw[8], &rblock->guid_entries[index], sizeof(u64));
+
+query_gid1:
+	kfree(rblock);
+
+	return ret;
+}
+
+int ehca_modify_port(struct ib_device *ibdev,
+		     u8 port, int port_modify_mask,
+		     struct ib_port_modify *props)
+{
+	/* Not implemented yet */
+	return -EFAULT;
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_irq.c b/drivers/infiniband/hw/ehca/ehca_irq.c
new file mode 100644
index 0000000..2a65b5b
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_irq.c
@@ -0,0 +1,762 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Functions for EQs, NEQs and interrupts
+ *
+ *  Authors: Heiko J Schick <schickhj@de.ibm.com>
+ *           Khadija Souissi <souissi@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_classes.h"
+#include "ehca_irq.h"
+#include "ehca_iverbs.h"
+#include "ehca_tools.h"
+#include "hcp_if.h"
+#include "hipz_fns.h"
+
+#define EQE_COMPLETION_EVENT   EHCA_BMASK_IBM(1,1)
+#define EQE_CQ_QP_NUMBER       EHCA_BMASK_IBM(8,31)
+#define EQE_EE_IDENTIFIER      EHCA_BMASK_IBM(2,7)
+#define EQE_CQ_NUMBER          EHCA_BMASK_IBM(8,31)
+#define EQE_QP_NUMBER          EHCA_BMASK_IBM(8,31)
+#define EQE_QP_TOKEN           EHCA_BMASK_IBM(32,63)
+#define EQE_CQ_TOKEN           EHCA_BMASK_IBM(32,63)
+
+#define NEQE_COMPLETION_EVENT  EHCA_BMASK_IBM(1,1)
+#define NEQE_EVENT_CODE        EHCA_BMASK_IBM(2,7)
+#define NEQE_PORT_NUMBER       EHCA_BMASK_IBM(8,15)
+#define NEQE_PORT_AVAILABILITY EHCA_BMASK_IBM(16,16)
+
+#define ERROR_DATA_LENGTH      EHCA_BMASK_IBM(52,63)
+#define ERROR_DATA_TYPE        EHCA_BMASK_IBM(0,7)
+
+#ifdef CONFIG_INFINIBAND_EHCA_SCALING
+
+static void queue_comp_task(struct ehca_cq *__cq);
+
+static struct ehca_comp_pool* pool;
+static struct notifier_block comp_pool_callback_nb;
+
+#endif
+
+static inline void comp_event_callback(struct ehca_cq *cq)
+{
+	if (!cq->ib_cq.comp_handler)
+		return;
+
+	spin_lock(&cq->cb_lock);
+	cq->ib_cq.comp_handler(&cq->ib_cq, cq->ib_cq.cq_context);
+	spin_unlock(&cq->cb_lock);
+
+	return;
+}
+
+static void print_error_data(struct ehca_shca * shca, void* data,
+			     u64* rblock, int length)
+{
+	u64 type = EHCA_BMASK_GET(ERROR_DATA_TYPE, rblock[2]);
+	u64 resource = rblock[1];
+
+	switch (type) {
+	case 0x1: /* Queue Pair */
+	{
+		struct ehca_qp *qp = (struct ehca_qp*)data;
+
+		/* only print error data if AER is set */
+		if (rblock[6] == 0)
+			return;
+
+		ehca_err(&shca->ib_device,
+			 "QP 0x%x (resource=%lx) has errors.",
+			 qp->ib_qp.qp_num, resource);
+		break;
+	}
+	case 0x4: /* Completion Queue */
+	{
+		struct ehca_cq *cq = (struct ehca_cq*)data;
+
+		ehca_err(&shca->ib_device,
+			 "CQ 0x%x (resource=%lx) has errors.",
+			 cq->cq_number, resource);
+		break;
+	}
+	default:
+		ehca_err(&shca->ib_device,
+			 "Unknown errror type: %lx on %s.",
+			 type, shca->ib_device.name);
+		break;
+	}
+
+	ehca_err(&shca->ib_device, "Error data is available: %lx.", resource);
+	ehca_err(&shca->ib_device, "EHCA ----- error data begin "
+		 "---------------------------------------------------");
+	ehca_dmp(rblock, length, "resource=%lx", resource);
+	ehca_err(&shca->ib_device, "EHCA ----- error data end "
+		 "----------------------------------------------------");
+
+	return;
+}
+
+int ehca_error_data(struct ehca_shca *shca, void *data,
+		    u64 resource)
+{
+
+	unsigned long ret;
+	u64 *rblock;
+	unsigned long block_count;
+
+	rblock = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (!rblock) {
+		ehca_err(&shca->ib_device, "Cannot allocate rblock memory.");
+		ret = -ENOMEM;
+		goto error_data1;
+	}
+
+	ret = hipz_h_error_data(shca->ipz_hca_handle,
+				resource,
+				rblock,
+				&block_count);
+
+	if (ret == H_R_STATE) {
+		ehca_err(&shca->ib_device,
+			 "No error data is available: %lx.", resource);
+	}
+	else if (ret == H_SUCCESS) {
+		int length;
+
+		length = EHCA_BMASK_GET(ERROR_DATA_LENGTH, rblock[0]);
+
+		if (length > PAGE_SIZE)
+			length = PAGE_SIZE;
+
+		print_error_data(shca, data, rblock, length);
+	}
+	else {
+		ehca_err(&shca->ib_device,
+			 "Error data could not be fetched: %lx", resource);
+	}
+
+	kfree(rblock);
+
+error_data1:
+	return ret;
+
+}
+
+static void qp_event_callback(struct ehca_shca *shca,
+			      u64 eqe,
+			      enum ib_event_type event_type)
+{
+	struct ib_event event;
+	struct ehca_qp *qp;
+	unsigned long flags;
+	u32 token = EHCA_BMASK_GET(EQE_QP_TOKEN, eqe);
+
+	spin_lock_irqsave(&ehca_qp_idr_lock, flags);
+	qp = idr_find(&ehca_qp_idr, token);
+	spin_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+
+	if (!qp)
+		return;
+
+	ehca_error_data(shca, qp, qp->ipz_qp_handle.handle);
+
+	if (!qp->ib_qp.event_handler)
+		return;
+
+	event.device     = &shca->ib_device;
+	event.event      = event_type;
+	event.element.qp = &qp->ib_qp;
+
+	qp->ib_qp.event_handler(&event, qp->ib_qp.qp_context);
+
+	return;
+}
+
+static void cq_event_callback(struct ehca_shca *shca,
+					  u64 eqe)
+{
+	struct ehca_cq *cq;
+	unsigned long flags;
+	u32 token = EHCA_BMASK_GET(EQE_CQ_TOKEN, eqe);
+
+	spin_lock_irqsave(&ehca_cq_idr_lock, flags);
+	cq = idr_find(&ehca_cq_idr, token);
+	spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+	if (!cq)
+		return;
+
+	ehca_error_data(shca, cq, cq->ipz_cq_handle.handle);
+
+	return;
+}
+
+static void parse_identifier(struct ehca_shca *shca, u64 eqe)
+{
+	u8 identifier = EHCA_BMASK_GET(EQE_EE_IDENTIFIER, eqe);
+
+	switch (identifier) {
+	case 0x02: /* path migrated */
+		qp_event_callback(shca, eqe, IB_EVENT_PATH_MIG);
+		break;
+	case 0x03: /* communication established */
+		qp_event_callback(shca, eqe, IB_EVENT_COMM_EST);
+		break;
+	case 0x04: /* send queue drained */
+		qp_event_callback(shca, eqe, IB_EVENT_SQ_DRAINED);
+		break;
+	case 0x05: /* QP error */
+	case 0x06: /* QP error */
+		qp_event_callback(shca, eqe, IB_EVENT_QP_FATAL);
+		break;
+	case 0x07: /* CQ error */
+	case 0x08: /* CQ error */
+		cq_event_callback(shca, eqe);
+		break;
+	case 0x09: /* MRMWPTE error */
+		ehca_err(&shca->ib_device, "MRMWPTE error.");
+		break;
+	case 0x0A: /* port event */
+		ehca_err(&shca->ib_device, "Port event.");
+		break;
+	case 0x0B: /* MR access error */
+		ehca_err(&shca->ib_device, "MR access error.");
+		break;
+	case 0x0C: /* EQ error */
+		ehca_err(&shca->ib_device, "EQ error.");
+		break;
+	case 0x0D: /* P/Q_Key mismatch */
+		ehca_err(&shca->ib_device, "P/Q_Key mismatch.");
+		break;
+	case 0x10: /* sampling complete */
+		ehca_err(&shca->ib_device, "Sampling complete.");
+		break;
+	case 0x11: /* unaffiliated access error */
+		ehca_err(&shca->ib_device, "Unaffiliated access error.");
+		break;
+	case 0x12: /* path migrating error */
+		ehca_err(&shca->ib_device, "Path migration error.");
+		break;
+	case 0x13: /* interface trace stopped */
+		ehca_err(&shca->ib_device, "Interface trace stopped.");
+		break;
+	case 0x14: /* first error capture info available */
+	default:
+		ehca_err(&shca->ib_device, "Unknown identifier: %x on %s.",
+			 identifier, shca->ib_device.name);
+		break;
+	}
+
+	return;
+}
+
+static void parse_ec(struct ehca_shca *shca, u64 eqe)
+{
+	struct ib_event event;
+	u8 ec   = EHCA_BMASK_GET(NEQE_EVENT_CODE, eqe);
+	u8 port = EHCA_BMASK_GET(NEQE_PORT_NUMBER, eqe);
+
+	switch (ec) {
+	case 0x30: /* port availability change */
+		if (EHCA_BMASK_GET(NEQE_PORT_AVAILABILITY, eqe)) {
+			ehca_info(&shca->ib_device,
+				  "port %x is active.", port);
+			event.device = &shca->ib_device;
+			event.event = IB_EVENT_PORT_ACTIVE;
+			event.element.port_num = port;
+			shca->sport[port - 1].port_state = IB_PORT_ACTIVE;
+			ib_dispatch_event(&event);
+		} else {
+			ehca_info(&shca->ib_device,
+				  "port %x is inactive.", port);
+			event.device = &shca->ib_device;
+			event.event = IB_EVENT_PORT_ERR;
+			event.element.port_num = port;
+			shca->sport[port - 1].port_state = IB_PORT_DOWN;
+			ib_dispatch_event(&event);
+		}
+		break;
+	case 0x31:
+		/* port configuration change
+		 * disruptive change is caused by
+		 * LID, PKEY or SM change
+		 */
+		ehca_warn(&shca->ib_device,
+			  "disruptive port %x configuration change", port);
+
+		ehca_info(&shca->ib_device,
+			 "port %x is inactive.", port);
+		event.device = &shca->ib_device;
+		event.event = IB_EVENT_PORT_ERR;
+		event.element.port_num = port;
+		shca->sport[port - 1].port_state = IB_PORT_DOWN;
+		ib_dispatch_event(&event);
+
+		ehca_info(&shca->ib_device,
+			 "port %x is active.", port);
+		event.device = &shca->ib_device;
+		event.event = IB_EVENT_PORT_ACTIVE;
+		event.element.port_num = port;
+		shca->sport[port - 1].port_state = IB_PORT_ACTIVE;
+		ib_dispatch_event(&event);
+		break;
+	case 0x32: /* adapter malfunction */
+		ehca_err(&shca->ib_device, "Adapter malfunction.");
+		break;
+	case 0x33:  /* trace stopped */
+		ehca_err(&shca->ib_device, "Traced stopped.");
+		break;
+	default:
+		ehca_err(&shca->ib_device, "Unknown event code: %x on %s.",
+			 ec, shca->ib_device.name);
+		break;
+	}
+
+	return;
+}
+
+static inline void reset_eq_pending(struct ehca_cq *cq)
+{
+	u64 CQx_EP;
+	struct h_galpa gal = cq->galpas.kernel;
+
+	hipz_galpa_store_cq(gal, cqx_ep, 0x0);
+	CQx_EP = hipz_galpa_load(gal, CQTEMM_OFFSET(cqx_ep));
+
+	return;
+}
+
+irqreturn_t ehca_interrupt_neq(int irq, void *dev_id, struct pt_regs *regs)
+{
+	struct ehca_shca *shca = (struct ehca_shca*)dev_id;
+
+	tasklet_hi_schedule(&shca->neq.interrupt_task);
+
+	return IRQ_HANDLED;
+}
+
+void ehca_tasklet_neq(unsigned long data)
+{
+	struct ehca_shca *shca = (struct ehca_shca*)data;
+	struct ehca_eqe *eqe;
+	u64 ret;
+
+	eqe = (struct ehca_eqe *)ehca_poll_eq(shca, &shca->neq);
+
+	while (eqe) {
+		if (!EHCA_BMASK_GET(NEQE_COMPLETION_EVENT, eqe->entry))
+			parse_ec(shca, eqe->entry);
+
+		eqe = (struct ehca_eqe *)ehca_poll_eq(shca, &shca->neq);
+	}
+
+	ret = hipz_h_reset_event(shca->ipz_hca_handle,
+				 shca->neq.ipz_eq_handle, 0xFFFFFFFFFFFFFFFFL);
+
+	if (ret != H_SUCCESS)
+		ehca_err(&shca->ib_device, "Can't clear notification events.");
+
+	return;
+}
+
+irqreturn_t ehca_interrupt_eq(int irq, void *dev_id, struct pt_regs *regs)
+{
+	struct ehca_shca *shca = (struct ehca_shca*)dev_id;
+
+	tasklet_hi_schedule(&shca->eq.interrupt_task);
+
+	return IRQ_HANDLED;
+}
+
+void ehca_tasklet_eq(unsigned long data)
+{
+	struct ehca_shca *shca = (struct ehca_shca*)data;
+	struct ehca_eqe *eqe;
+	int int_state;
+	int query_cnt = 0;
+
+	do {
+		eqe = (struct ehca_eqe *)ehca_poll_eq(shca, &shca->eq);
+
+		if ((shca->hw_level >= 2) && eqe)
+			int_state = 1;
+		else
+			int_state = 0;
+
+		while ((int_state == 1) || eqe) {
+			while (eqe) {
+				u64 eqe_value = eqe->entry;
+
+				ehca_dbg(&shca->ib_device,
+					 "eqe_value=%lx", eqe_value);
+
+				/* TODO: better structure */
+				if (EHCA_BMASK_GET(EQE_COMPLETION_EVENT,
+						   eqe_value)) {
+					unsigned long flags;
+					u32 token;
+					struct ehca_cq *cq;
+
+					ehca_dbg(&shca->ib_device,
+						 "... completion event");
+					token =
+						EHCA_BMASK_GET(EQE_CQ_TOKEN,
+							       eqe_value);
+					spin_lock_irqsave(&ehca_cq_idr_lock,
+							  flags);
+					cq = idr_find(&ehca_cq_idr, token);
+
+					if (cq == NULL) {
+						spin_unlock(&ehca_cq_idr_lock);
+						break;
+					}
+
+					reset_eq_pending(cq);
+#ifdef CONFIG_INFINIBAND_EHCA_SCALING
+					queue_comp_task(cq);
+					spin_unlock_irqrestore(&ehca_cq_idr_lock,
+							       flags);
+#else
+					spin_unlock_irqrestore(&ehca_cq_idr_lock,
+							       flags);
+					comp_event_callback(cq);
+#endif
+				} else {
+					ehca_dbg(&shca->ib_device,
+						 "... non completion event");
+					parse_identifier(shca, eqe_value);
+				}
+				eqe =
+					(struct ehca_eqe *)ehca_poll_eq(shca,
+								    &shca->eq);
+			}
+
+			if (shca->hw_level >= 2) {
+				int_state =
+				    hipz_h_query_int_state(shca->ipz_hca_handle,
+							   shca->eq.ist);
+				query_cnt++;
+				iosync();
+				if (query_cnt >= 100) {
+					query_cnt = 0;
+					int_state = 0;
+				}
+			}
+			eqe = (struct ehca_eqe *)ehca_poll_eq(shca, &shca->eq);
+
+		}
+	} while (int_state != 0);
+
+	return;
+}
+
+#ifdef CONFIG_INFINIBAND_EHCA_SCALING
+
+static inline int find_next_online_cpu(struct ehca_comp_pool* pool)
+{
+	unsigned long flags_last_cpu;
+
+	if (ehca_debug_level)
+		ehca_dmp(&cpu_online_map, sizeof(cpumask_t), "");
+
+	spin_lock_irqsave(&pool->last_cpu_lock, flags_last_cpu);
+	pool->last_cpu = next_cpu(pool->last_cpu, cpu_online_map);
+	if (pool->last_cpu == NR_CPUS)
+		pool->last_cpu = first_cpu(cpu_online_map);
+	spin_unlock_irqrestore(&pool->last_cpu_lock, flags_last_cpu);
+
+	return pool->last_cpu;
+}
+
+static void __queue_comp_task(struct ehca_cq *__cq,
+			      struct ehca_cpu_comp_task *cct)
+{
+	unsigned long flags_cct;
+	unsigned long flags_cq;
+
+	spin_lock_irqsave(&cct->task_lock, flags_cct);
+	spin_lock_irqsave(&__cq->task_lock, flags_cq);
+
+	if (__cq->nr_callbacks == 0) {
+		__cq->nr_callbacks++;
+		list_add_tail(&__cq->entry, &cct->cq_list);
+		cct->cq_jobs++;
+		wake_up(&cct->wait_queue);
+	}
+	else
+		__cq->nr_callbacks++;
+
+	spin_unlock_irqrestore(&__cq->task_lock, flags_cq);
+	spin_unlock_irqrestore(&cct->task_lock, flags_cct);
+}
+
+static void queue_comp_task(struct ehca_cq *__cq)
+{
+	int cpu;
+	int cpu_id;
+	struct ehca_cpu_comp_task *cct;
+
+	cpu = get_cpu();
+	cpu_id = find_next_online_cpu(pool);
+
+	BUG_ON(!cpu_online(cpu_id));
+
+	cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
+
+	if (cct->cq_jobs > 0) {
+		cpu_id = find_next_online_cpu(pool);
+		cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu_id);
+	}
+
+	__queue_comp_task(__cq, cct);
+
+	put_cpu();
+
+	return;
+}
+
+static void run_comp_task(struct ehca_cpu_comp_task* cct)
+{
+	struct ehca_cq *cq;
+	unsigned long flags_cct;
+	unsigned long flags_cq;
+
+	spin_lock_irqsave(&cct->task_lock, flags_cct);
+
+	while (!list_empty(&cct->cq_list)) {
+		cq = list_entry(cct->cq_list.next, struct ehca_cq, entry);
+		spin_unlock_irqrestore(&cct->task_lock, flags_cct);
+		comp_event_callback(cq);
+		spin_lock_irqsave(&cct->task_lock, flags_cct);
+
+		spin_lock_irqsave(&cq->task_lock, flags_cq);
+		cq->nr_callbacks--;
+		if (cq->nr_callbacks == 0) {
+			list_del_init(cct->cq_list.next);
+			cct->cq_jobs--;
+		}
+		spin_unlock_irqrestore(&cq->task_lock, flags_cq);
+
+	}
+
+	spin_unlock_irqrestore(&cct->task_lock, flags_cct);
+
+	return;
+}
+
+static int comp_task(void *__cct)
+{
+	struct ehca_cpu_comp_task* cct = __cct;
+	DECLARE_WAITQUEUE(wait, current);
+
+	set_current_state(TASK_INTERRUPTIBLE);
+	while(!kthread_should_stop()) {
+		add_wait_queue(&cct->wait_queue, &wait);
+
+		if (list_empty(&cct->cq_list))
+			schedule();
+		else
+			__set_current_state(TASK_RUNNING);
+
+		remove_wait_queue(&cct->wait_queue, &wait);
+
+		if (!list_empty(&cct->cq_list))
+			run_comp_task(__cct);
+
+		set_current_state(TASK_INTERRUPTIBLE);
+	}
+	__set_current_state(TASK_RUNNING);
+
+	return 0;
+}
+
+static struct task_struct *create_comp_task(struct ehca_comp_pool *pool,
+					    int cpu)
+{
+	struct ehca_cpu_comp_task *cct;
+
+	cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+	spin_lock_init(&cct->task_lock);
+	INIT_LIST_HEAD(&cct->cq_list);
+	init_waitqueue_head(&cct->wait_queue);
+	cct->task = kthread_create(comp_task, cct, "ehca_comp/%d", cpu);
+
+	return cct->task;
+}
+
+static void destroy_comp_task(struct ehca_comp_pool *pool,
+			      int cpu)
+{
+	struct ehca_cpu_comp_task *cct;
+	struct task_struct *task;
+	unsigned long flags_cct;
+
+	cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+
+	spin_lock_irqsave(&cct->task_lock, flags_cct);
+
+	task = cct->task;
+	cct->task = NULL;
+	cct->cq_jobs = 0;
+
+	spin_unlock_irqrestore(&cct->task_lock, flags_cct);
+
+	if (task)
+		kthread_stop(task);
+
+	return;
+}
+
+static void take_over_work(struct ehca_comp_pool *pool,
+			   int cpu)
+{
+	struct ehca_cpu_comp_task *cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+	LIST_HEAD(list);
+	struct ehca_cq *cq;
+	unsigned long flags_cct;
+
+	spin_lock_irqsave(&cct->task_lock, flags_cct);
+
+	list_splice_init(&cct->cq_list, &list);
+
+	while(!list_empty(&list)) {
+	       cq = list_entry(cct->cq_list.next, struct ehca_cq, entry);
+
+	       list_del(&cq->entry);
+	       __queue_comp_task(cq, per_cpu_ptr(pool->cpu_comp_tasks,
+						 smp_processor_id()));
+	}
+
+	spin_unlock_irqrestore(&cct->task_lock, flags_cct);
+
+}
+
+static int comp_pool_callback(struct notifier_block *nfb,
+			      unsigned long action,
+			      void *hcpu)
+{
+	unsigned int cpu = (unsigned long)hcpu;
+	struct ehca_cpu_comp_task *cct;
+
+	switch (action) {
+	case CPU_UP_PREPARE:
+		ehca_gen_dbg("CPU: %x (CPU_PREPARE)", cpu);
+		if(!create_comp_task(pool, cpu)) {
+			ehca_gen_err("Can't create comp_task for cpu: %x", cpu);
+			return NOTIFY_BAD;
+		}
+		break;
+	case CPU_UP_CANCELED:
+		ehca_gen_dbg("CPU: %x (CPU_CANCELED)", cpu);
+		cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+		kthread_bind(cct->task, any_online_cpu(cpu_online_map));
+		destroy_comp_task(pool, cpu);
+		break;
+	case CPU_ONLINE:
+		ehca_gen_dbg("CPU: %x (CPU_ONLINE)", cpu);
+		cct = per_cpu_ptr(pool->cpu_comp_tasks, cpu);
+		kthread_bind(cct->task, cpu);
+		wake_up_process(cct->task);
+		break;
+	case CPU_DOWN_PREPARE:
+		ehca_gen_dbg("CPU: %x (CPU_DOWN_PREPARE)", cpu);
+		break;
+	case CPU_DOWN_FAILED:
+		ehca_gen_dbg("CPU: %x (CPU_DOWN_FAILED)", cpu);
+		break;
+	case CPU_DEAD:
+		ehca_gen_dbg("CPU: %x (CPU_DEAD)", cpu);
+		destroy_comp_task(pool, cpu);
+		take_over_work(pool, cpu);
+		break;
+	}
+
+	return NOTIFY_OK;
+}
+
+#endif
+
+int ehca_create_comp_pool(void)
+{
+#ifdef CONFIG_INFINIBAND_EHCA_SCALING
+	int cpu;
+	struct task_struct *task;
+
+	pool = kzalloc(sizeof(struct ehca_comp_pool), GFP_KERNEL);
+	if (pool == NULL)
+		return -ENOMEM;
+
+	spin_lock_init(&pool->last_cpu_lock);
+	pool->last_cpu = any_online_cpu(cpu_online_map);
+
+	pool->cpu_comp_tasks = alloc_percpu(struct ehca_cpu_comp_task);
+	if (pool->cpu_comp_tasks == NULL) {
+		kfree(pool);
+		return -EINVAL;
+	}
+
+	for_each_online_cpu(cpu) {
+		task = create_comp_task(pool, cpu);
+		if (task) {
+			kthread_bind(task, cpu);
+			wake_up_process(task);
+		}
+	}
+
+	comp_pool_callback_nb.notifier_call = comp_pool_callback;
+	comp_pool_callback_nb.priority =0;
+	register_cpu_notifier(&comp_pool_callback_nb);
+#endif
+
+	return 0;
+}
+
+void ehca_destroy_comp_pool(void)
+{
+#ifdef CONFIG_INFINIBAND_EHCA_SCALING
+	int i;
+
+	unregister_cpu_notifier(&comp_pool_callback_nb);
+
+	for (i = 0; i < NR_CPUS; i++) {
+		if (cpu_online(i))
+			destroy_comp_task(pool, i);
+	}
+#endif
+
+	return;
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_irq.h b/drivers/infiniband/hw/ehca/ehca_irq.h
new file mode 100644
index 0000000..85bf1fe
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_irq.h
@@ -0,0 +1,77 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Function definitions and structs for EQs, NEQs and interrupts
+ *
+ *  Authors: Heiko J Schick <schickhj@de.ibm.com>
+ *           Khadija Souissi <souissi@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_IRQ_H
+#define __EHCA_IRQ_H
+
+
+struct ehca_shca;
+
+#include <linux/interrupt.h>
+#include <linux/types.h>
+#include <asm/atomic.h>
+
+int ehca_error_data(struct ehca_shca *shca, void *data, u64 resource);
+
+irqreturn_t ehca_interrupt_neq(int irq, void *dev_id, struct pt_regs *regs);
+void ehca_tasklet_neq(unsigned long data);
+
+irqreturn_t ehca_interrupt_eq(int irq, void *dev_id, struct pt_regs *regs);
+void ehca_tasklet_eq(unsigned long data);
+
+struct ehca_cpu_comp_task {
+	wait_queue_head_t wait_queue;
+	struct list_head cq_list;
+	struct task_struct *task;
+	spinlock_t task_lock;
+	int cq_jobs;
+};
+
+struct ehca_comp_pool {
+	struct ehca_cpu_comp_task *cpu_comp_tasks;
+	int last_cpu;
+	spinlock_t last_cpu_lock;
+};
+
+int ehca_create_comp_pool(void);
+void ehca_destroy_comp_pool(void);
+
+#endif
diff --git a/drivers/infiniband/hw/ehca/ehca_iverbs.h b/drivers/infiniband/hw/ehca/ehca_iverbs.h
new file mode 100644
index 0000000..319c39d
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_iverbs.h
@@ -0,0 +1,182 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Function definitions for internal functions
+ *
+ *  Authors: Heiko J Schick <schickhj@de.ibm.com>
+ *           Dietmar Decker <ddecker@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __EHCA_IVERBS_H__
+#define __EHCA_IVERBS_H__
+
+#include "ehca_classes.h"
+
+int ehca_query_device(struct ib_device *ibdev, struct ib_device_attr *props);
+
+int ehca_query_port(struct ib_device *ibdev, u8 port,
+		    struct ib_port_attr *props);
+
+int ehca_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 * pkey);
+
+int ehca_query_gid(struct ib_device *ibdev, u8 port, int index,
+		   union ib_gid *gid);
+
+int ehca_modify_port(struct ib_device *ibdev, u8 port, int port_modify_mask,
+		     struct ib_port_modify *props);
+
+struct ib_pd *ehca_alloc_pd(struct ib_device *device,
+			    struct ib_ucontext *context,
+			    struct ib_udata *udata);
+
+int ehca_dealloc_pd(struct ib_pd *pd);
+
+struct ib_ah *ehca_create_ah(struct ib_pd *pd, struct ib_ah_attr *ah_attr);
+
+int ehca_modify_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+
+int ehca_query_ah(struct ib_ah *ah, struct ib_ah_attr *ah_attr);
+
+int ehca_destroy_ah(struct ib_ah *ah);
+
+struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags);
+
+struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
+			       struct ib_phys_buf *phys_buf_array,
+			       int num_phys_buf,
+			       int mr_access_flags, u64 *iova_start);
+
+struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd,
+			       struct ib_umem *region,
+			       int mr_access_flags, struct ib_udata *udata);
+
+int ehca_rereg_phys_mr(struct ib_mr *mr,
+		       int mr_rereg_mask,
+		       struct ib_pd *pd,
+		       struct ib_phys_buf *phys_buf_array,
+		       int num_phys_buf, int mr_access_flags, u64 *iova_start);
+
+int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr);
+
+int ehca_dereg_mr(struct ib_mr *mr);
+
+struct ib_mw *ehca_alloc_mw(struct ib_pd *pd);
+
+int ehca_bind_mw(struct ib_qp *qp, struct ib_mw *mw,
+		 struct ib_mw_bind *mw_bind);
+
+int ehca_dealloc_mw(struct ib_mw *mw);
+
+struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
+			      int mr_access_flags,
+			      struct ib_fmr_attr *fmr_attr);
+
+int ehca_map_phys_fmr(struct ib_fmr *fmr,
+		      u64 *page_list, int list_len, u64 iova);
+
+int ehca_unmap_fmr(struct list_head *fmr_list);
+
+int ehca_dealloc_fmr(struct ib_fmr *fmr);
+
+enum ehca_eq_type {
+	EHCA_EQ = 0, /* Event Queue              */
+	EHCA_NEQ     /* Notification Event Queue */
+};
+
+int ehca_create_eq(struct ehca_shca *shca, struct ehca_eq *eq,
+		   enum ehca_eq_type type, const u32 length);
+
+int ehca_destroy_eq(struct ehca_shca *shca, struct ehca_eq *eq);
+
+void *ehca_poll_eq(struct ehca_shca *shca, struct ehca_eq *eq);
+
+
+struct ib_cq *ehca_create_cq(struct ib_device *device, int cqe,
+			     struct ib_ucontext *context,
+			     struct ib_udata *udata);
+
+int ehca_destroy_cq(struct ib_cq *cq);
+
+int ehca_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata);
+
+int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc);
+
+int ehca_peek_cq(struct ib_cq *cq, int wc_cnt);
+
+int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify cq_notify);
+
+struct ib_qp *ehca_create_qp(struct ib_pd *pd,
+			     struct ib_qp_init_attr *init_attr,
+			     struct ib_udata *udata);
+
+int ehca_destroy_qp(struct ib_qp *qp);
+
+int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+		   struct ib_udata *udata);
+
+int ehca_query_qp(struct ib_qp *qp, struct ib_qp_attr *qp_attr,
+		  int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr);
+
+int ehca_post_send(struct ib_qp *qp, struct ib_send_wr *send_wr,
+		   struct ib_send_wr **bad_send_wr);
+
+int ehca_post_recv(struct ib_qp *qp, struct ib_recv_wr *recv_wr,
+		   struct ib_recv_wr **bad_recv_wr);
+
+u64 ehca_define_sqp(struct ehca_shca *shca, struct ehca_qp *ibqp,
+		    struct ib_qp_init_attr *qp_init_attr);
+
+int ehca_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
+
+int ehca_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid);
+
+struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
+					struct ib_udata *udata);
+
+int ehca_dealloc_ucontext(struct ib_ucontext *context);
+
+int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+
+void ehca_poll_eqs(unsigned long data);
+
+int ehca_mmap_nopage(u64 foffset,u64 length,void **mapped,
+		     struct vm_area_struct **vma);
+
+int ehca_mmap_register(u64 physical,void **mapped,
+		       struct vm_area_struct **vma);
+
+int ehca_munmap(unsigned long addr, size_t len);
+
+#endif
diff --git a/drivers/infiniband/hw/ehca/ehca_main.c b/drivers/infiniband/hw/ehca/ehca_main.c
new file mode 100644
index 0000000..2380994
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_main.c
@@ -0,0 +1,818 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  module start stop, hca detection
+ *
+ *  Authors: Heiko J Schick <schickhj@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Joachim Fenkes <fenkes@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_classes.h"
+#include "ehca_iverbs.h"
+#include "ehca_mrmw.h"
+#include "ehca_tools.h"
+#include "hcp_if.h"
+
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_AUTHOR("Christoph Raisch <raisch@de.ibm.com>");
+MODULE_DESCRIPTION("IBM eServer HCA InfiniBand Device Driver");
+MODULE_VERSION("SVNEHCA_0016");
+
+int ehca_open_aqp1     = 0;
+int ehca_debug_level   = 0;
+int ehca_hw_level      = 0;
+int ehca_nr_ports      = 2;
+int ehca_use_hp_mr     = 0;
+int ehca_port_act_time = 30;
+int ehca_poll_all_eqs  = 1;
+int ehca_static_rate   = -1;
+
+module_param_named(open_aqp1,     ehca_open_aqp1,     int, 0);
+module_param_named(debug_level,   ehca_debug_level,   int, 0);
+module_param_named(hw_level,      ehca_hw_level,      int, 0);
+module_param_named(nr_ports,      ehca_nr_ports,      int, 0);
+module_param_named(use_hp_mr,     ehca_use_hp_mr,     int, 0);
+module_param_named(port_act_time, ehca_port_act_time, int, 0);
+module_param_named(poll_all_eqs,  ehca_poll_all_eqs,  int, 0);
+module_param_named(static_rate,   ehca_static_rate,   int, 0);
+
+MODULE_PARM_DESC(open_aqp1,
+		 "AQP1 on startup (0: no (default), 1: yes)");
+MODULE_PARM_DESC(debug_level,
+		 "debug level"
+		 " (0: no debug traces (default), 1: with debug traces)");
+MODULE_PARM_DESC(hw_level,
+		 "hardware level"
+		 " (0: autosensing (default), 1: v. 0.20, 2: v. 0.21)");
+MODULE_PARM_DESC(nr_ports,
+		 "number of connected ports (default: 2)");
+MODULE_PARM_DESC(use_hp_mr,
+		 "high performance MRs (0: no (default), 1: yes)");
+MODULE_PARM_DESC(port_act_time,
+		 "time to wait for port activation (default: 30 sec)");
+MODULE_PARM_DESC(poll_all_eqs,
+		 "polls all event queues periodically"
+		 " (0: no, 1: yes (default))");
+MODULE_PARM_DESC(static_rate,
+		 "set permanent static rate (default: disabled)");
+
+spinlock_t ehca_qp_idr_lock;
+spinlock_t ehca_cq_idr_lock;
+DEFINE_IDR(ehca_qp_idr);
+DEFINE_IDR(ehca_cq_idr);
+
+static struct list_head shca_list; /* list of all registered ehcas */
+static spinlock_t shca_list_lock;
+
+static struct timer_list poll_eqs_timer;
+
+static int ehca_create_slab_caches(void)
+{
+	int ret;
+
+	ret = ehca_init_pd_cache();
+	if (ret) {
+		ehca_gen_err("Cannot create PD SLAB cache.");
+		return ret;
+	}
+
+	ret = ehca_init_cq_cache();
+	if (ret) {
+		ehca_gen_err("Cannot create CQ SLAB cache.");
+		goto create_slab_caches2;
+	}
+
+	ret = ehca_init_qp_cache();
+	if (ret) {
+		ehca_gen_err("Cannot create QP SLAB cache.");
+		goto create_slab_caches3;
+	}
+
+	ret = ehca_init_av_cache();
+	if (ret) {
+		ehca_gen_err("Cannot create AV SLAB cache.");
+		goto create_slab_caches4;
+	}
+
+	ret = ehca_init_mrmw_cache();
+	if (ret) {
+		ehca_gen_err("Cannot create MR&MW SLAB cache.");
+		goto create_slab_caches5;
+	}
+
+	return 0;
+
+create_slab_caches5:
+	ehca_cleanup_av_cache();
+
+create_slab_caches4:
+	ehca_cleanup_qp_cache();
+
+create_slab_caches3:
+	ehca_cleanup_cq_cache();
+
+create_slab_caches2:
+	ehca_cleanup_pd_cache();
+
+	return ret;
+}
+
+static void ehca_destroy_slab_caches(void)
+{
+	ehca_cleanup_mrmw_cache();
+	ehca_cleanup_av_cache();
+	ehca_cleanup_qp_cache();
+	ehca_cleanup_cq_cache();
+	ehca_cleanup_pd_cache();
+}
+
+#define EHCA_HCAAVER  EHCA_BMASK_IBM(32,39)
+#define EHCA_REVID    EHCA_BMASK_IBM(40,63)
+
+int ehca_sense_attributes(struct ehca_shca *shca)
+{
+	int ret = 0;
+	u64 h_ret;
+	struct hipz_query_hca *rblock;
+
+	rblock = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (!rblock) {
+		ehca_gen_err("Cannot allocate rblock memory.");
+		return -ENOMEM;
+	}
+
+	h_ret = hipz_h_query_hca(shca->ipz_hca_handle, rblock);
+	if (h_ret != H_SUCCESS) {
+		ehca_gen_err("Cannot query device properties. h_ret=%lx",
+			     h_ret);
+		ret = -EPERM;
+		goto num_ports1;
+	}
+
+	if (ehca_nr_ports == 1)
+		shca->num_ports = 1;
+	else
+		shca->num_ports = (u8)rblock->num_ports;
+
+	ehca_gen_dbg(" ... found %x ports", rblock->num_ports);
+
+	if (ehca_hw_level == 0) {
+		u32 hcaaver;
+		u32 revid;
+
+		hcaaver = EHCA_BMASK_GET(EHCA_HCAAVER, rblock->hw_ver);
+		revid   = EHCA_BMASK_GET(EHCA_REVID, rblock->hw_ver);
+
+		ehca_gen_dbg(" ... hardware version=%x:%x", hcaaver, revid);
+
+		if ((hcaaver == 1) && (revid == 0))
+			shca->hw_level = 0;
+		else if ((hcaaver == 1) && (revid == 1))
+			shca->hw_level = 1;
+		else if ((hcaaver == 1) && (revid == 2))
+			shca->hw_level = 2;
+	}
+	ehca_gen_dbg(" ... hardware level=%x", shca->hw_level);
+
+	shca->sport[0].rate = IB_RATE_30_GBPS;
+	shca->sport[1].rate = IB_RATE_30_GBPS;
+
+num_ports1:
+	kfree(rblock);
+	return ret;
+}
+
+static int init_node_guid(struct ehca_shca *shca)
+{
+	int ret = 0;
+	struct hipz_query_hca *rblock;
+
+	rblock = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (!rblock) {
+		ehca_err(&shca->ib_device, "Can't allocate rblock memory.");
+		return -ENOMEM;
+	}
+
+	if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "Can't query device properties");
+		ret = -EINVAL;
+		goto init_node_guid1;
+	}
+
+	memcpy(&shca->ib_device.node_guid, &rblock->node_guid, sizeof(u64));
+
+init_node_guid1:
+	kfree(rblock);
+	return ret;
+}
+
+int ehca_register_device(struct ehca_shca *shca)
+{
+	int ret;
+
+	ret = init_node_guid(shca);
+	if (ret)
+		return ret;
+
+	strlcpy(shca->ib_device.name, "ehca%d", IB_DEVICE_NAME_MAX);
+	shca->ib_device.owner               = THIS_MODULE;
+
+	shca->ib_device.uverbs_abi_ver	    = 5;
+	shca->ib_device.uverbs_cmd_mask	    =
+		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
+		(1ull << IB_USER_VERBS_CMD_QUERY_DEVICE)	|
+		(1ull << IB_USER_VERBS_CMD_QUERY_PORT)		|
+		(1ull << IB_USER_VERBS_CMD_ALLOC_PD)		|
+		(1ull << IB_USER_VERBS_CMD_DEALLOC_PD)		|
+		(1ull << IB_USER_VERBS_CMD_REG_MR)		|
+		(1ull << IB_USER_VERBS_CMD_DEREG_MR)		|
+		(1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL)	|
+		(1ull << IB_USER_VERBS_CMD_CREATE_CQ)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_CQ)		|
+		(1ull << IB_USER_VERBS_CMD_CREATE_QP)		|
+		(1ull << IB_USER_VERBS_CMD_MODIFY_QP)		|
+		(1ull << IB_USER_VERBS_CMD_QUERY_QP)		|
+		(1ull << IB_USER_VERBS_CMD_DESTROY_QP)		|
+		(1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)	|
+		(1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
+
+	shca->ib_device.node_type           = RDMA_NODE_IB_CA;
+	shca->ib_device.phys_port_cnt       = shca->num_ports;
+	shca->ib_device.dma_device          = &shca->ibmebus_dev->ofdev.dev;
+	shca->ib_device.query_device        = ehca_query_device;
+	shca->ib_device.query_port          = ehca_query_port;
+	shca->ib_device.query_gid           = ehca_query_gid;
+	shca->ib_device.query_pkey          = ehca_query_pkey;
+	/* shca->in_device.modify_device    = ehca_modify_device    */
+	shca->ib_device.modify_port         = ehca_modify_port;
+	shca->ib_device.alloc_ucontext      = ehca_alloc_ucontext;
+	shca->ib_device.dealloc_ucontext    = ehca_dealloc_ucontext;
+	shca->ib_device.alloc_pd            = ehca_alloc_pd;
+	shca->ib_device.dealloc_pd          = ehca_dealloc_pd;
+	shca->ib_device.create_ah	    = ehca_create_ah;
+	/* shca->ib_device.modify_ah	    = ehca_modify_ah;	    */
+	shca->ib_device.query_ah	    = ehca_query_ah;
+	shca->ib_device.destroy_ah	    = ehca_destroy_ah;
+	shca->ib_device.create_qp	    = ehca_create_qp;
+	shca->ib_device.modify_qp	    = ehca_modify_qp;
+	shca->ib_device.query_qp	    = ehca_query_qp;
+	shca->ib_device.destroy_qp	    = ehca_destroy_qp;
+	shca->ib_device.post_send	    = ehca_post_send;
+	shca->ib_device.post_recv	    = ehca_post_recv;
+	shca->ib_device.create_cq	    = ehca_create_cq;
+	shca->ib_device.destroy_cq	    = ehca_destroy_cq;
+	shca->ib_device.resize_cq	    = ehca_resize_cq;
+	shca->ib_device.poll_cq		    = ehca_poll_cq;
+	/* shca->ib_device.peek_cq	    = ehca_peek_cq;	    */
+	shca->ib_device.req_notify_cq	    = ehca_req_notify_cq;
+	/* shca->ib_device.req_ncomp_notif  = ehca_req_ncomp_notif; */
+	shca->ib_device.get_dma_mr	    = ehca_get_dma_mr;
+	shca->ib_device.reg_phys_mr	    = ehca_reg_phys_mr;
+	shca->ib_device.reg_user_mr	    = ehca_reg_user_mr;
+	shca->ib_device.query_mr	    = ehca_query_mr;
+	shca->ib_device.dereg_mr	    = ehca_dereg_mr;
+	shca->ib_device.rereg_phys_mr	    = ehca_rereg_phys_mr;
+	shca->ib_device.alloc_mw	    = ehca_alloc_mw;
+	shca->ib_device.bind_mw		    = ehca_bind_mw;
+	shca->ib_device.dealloc_mw	    = ehca_dealloc_mw;
+	shca->ib_device.alloc_fmr	    = ehca_alloc_fmr;
+	shca->ib_device.map_phys_fmr	    = ehca_map_phys_fmr;
+	shca->ib_device.unmap_fmr	    = ehca_unmap_fmr;
+	shca->ib_device.dealloc_fmr	    = ehca_dealloc_fmr;
+	shca->ib_device.attach_mcast	    = ehca_attach_mcast;
+	shca->ib_device.detach_mcast	    = ehca_detach_mcast;
+	/* shca->ib_device.process_mad	    = ehca_process_mad;	    */
+	shca->ib_device.mmap		    = ehca_mmap;
+
+	ret = ib_register_device(&shca->ib_device);
+	if (ret)
+		ehca_err(&shca->ib_device,
+			 "ib_register_device() failed ret=%x", ret);
+
+	return ret;
+}
+
+static int ehca_create_aqp1(struct ehca_shca *shca, u32 port)
+{
+	struct ehca_sport *sport = &shca->sport[port - 1];
+	struct ib_cq *ibcq;
+	struct ib_qp *ibqp;
+	struct ib_qp_init_attr qp_init_attr;
+	int ret;
+
+	if (sport->ibcq_aqp1) {
+		ehca_err(&shca->ib_device, "AQP1 CQ is already created.");
+		return -EPERM;
+	}
+
+	ibcq = ib_create_cq(&shca->ib_device, NULL, NULL, (void*)(-1), 10);
+	if (IS_ERR(ibcq)) {
+		ehca_err(&shca->ib_device, "Cannot create AQP1 CQ.");
+		return PTR_ERR(ibcq);
+	}
+	sport->ibcq_aqp1 = ibcq;
+
+	if (sport->ibqp_aqp1) {
+		ehca_err(&shca->ib_device, "AQP1 QP is already created.");
+		ret = -EPERM;
+		goto create_aqp1;
+	}
+
+	memset(&qp_init_attr, 0, sizeof(struct ib_qp_init_attr));
+	qp_init_attr.send_cq          = ibcq;
+	qp_init_attr.recv_cq          = ibcq;
+	qp_init_attr.sq_sig_type      = IB_SIGNAL_ALL_WR;
+	qp_init_attr.cap.max_send_wr  = 100;
+	qp_init_attr.cap.max_recv_wr  = 100;
+	qp_init_attr.cap.max_send_sge = 2;
+	qp_init_attr.cap.max_recv_sge = 1;
+	qp_init_attr.qp_type          = IB_QPT_GSI;
+	qp_init_attr.port_num         = port;
+	qp_init_attr.qp_context       = NULL;
+	qp_init_attr.event_handler    = NULL;
+	qp_init_attr.srq              = NULL;
+
+	ibqp = ib_create_qp(&shca->pd->ib_pd, &qp_init_attr);
+	if (IS_ERR(ibqp)) {
+		ehca_err(&shca->ib_device, "Cannot create AQP1 QP.");
+		ret = PTR_ERR(ibqp);
+		goto create_aqp1;
+	}
+	sport->ibqp_aqp1 = ibqp;
+
+	return 0;
+
+create_aqp1:
+	ib_destroy_cq(sport->ibcq_aqp1);
+	return ret;
+}
+
+static int ehca_destroy_aqp1(struct ehca_sport *sport)
+{
+	int ret;
+
+	ret = ib_destroy_qp(sport->ibqp_aqp1);
+	if (ret) {
+		ehca_gen_err("Cannot destroy AQP1 QP. ret=%x", ret);
+		return ret;
+	}
+
+	ret = ib_destroy_cq(sport->ibcq_aqp1);
+	if (ret)
+		ehca_gen_err("Cannot destroy AQP1 CQ. ret=%x", ret);
+
+	return ret;
+}
+
+static ssize_t ehca_show_debug_level(struct device_driver *ddp, char *buf)
+{
+	return  snprintf(buf, PAGE_SIZE, "%d\n",
+			 ehca_debug_level);
+}
+
+static ssize_t ehca_store_debug_level(struct device_driver *ddp,
+				      const char *buf, size_t count)
+{
+	int value = (*buf) - '0';
+	if (value >= 0 && value <= 9)
+		ehca_debug_level = value;
+	return 1;
+}
+
+DRIVER_ATTR(debug_level, S_IRUSR | S_IWUSR,
+	    ehca_show_debug_level, ehca_store_debug_level);
+
+void ehca_create_driver_sysfs(struct ibmebus_driver *drv)
+{
+	driver_create_file(&drv->driver, &driver_attr_debug_level);
+}
+
+void ehca_remove_driver_sysfs(struct ibmebus_driver *drv)
+{
+	driver_remove_file(&drv->driver, &driver_attr_debug_level);
+}
+
+#define EHCA_RESOURCE_ATTR(name)                                           \
+static ssize_t  ehca_show_##name(struct device *dev,                       \
+				 struct device_attribute *attr,            \
+				 char *buf)                                \
+{									   \
+	struct ehca_shca *shca;						   \
+	struct hipz_query_hca *rblock;				           \
+	int data;                                                          \
+									   \
+	shca = dev->driver_data;					   \
+									   \
+	rblock = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);			   \
+	if (!rblock) {						           \
+		dev_err(dev, "Can't allocate rblock memory.");		   \
+		return 0;						   \
+	}								   \
+									   \
+	if (hipz_h_query_hca(shca->ipz_hca_handle, rblock) != H_SUCCESS) { \
+		dev_err(dev, "Can't query device properties");	   	   \
+		kfree(rblock);					   	   \
+		return 0;					   	   \
+	}								   \
+									   \
+	data = rblock->name;                                               \
+	kfree(rblock);                                                     \
+									   \
+	if ((strcmp(#name, "num_ports") == 0) && (ehca_nr_ports == 1))	   \
+		return snprintf(buf, 256, "1\n");			   \
+	else								   \
+		return snprintf(buf, 256, "%d\n", data);		   \
+									   \
+}									   \
+static DEVICE_ATTR(name, S_IRUGO, ehca_show_##name, NULL);
+
+EHCA_RESOURCE_ATTR(num_ports);
+EHCA_RESOURCE_ATTR(hw_ver);
+EHCA_RESOURCE_ATTR(max_eq);
+EHCA_RESOURCE_ATTR(cur_eq);
+EHCA_RESOURCE_ATTR(max_cq);
+EHCA_RESOURCE_ATTR(cur_cq);
+EHCA_RESOURCE_ATTR(max_qp);
+EHCA_RESOURCE_ATTR(cur_qp);
+EHCA_RESOURCE_ATTR(max_mr);
+EHCA_RESOURCE_ATTR(cur_mr);
+EHCA_RESOURCE_ATTR(max_mw);
+EHCA_RESOURCE_ATTR(cur_mw);
+EHCA_RESOURCE_ATTR(max_pd);
+EHCA_RESOURCE_ATTR(max_ah);
+
+static ssize_t ehca_show_adapter_handle(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct ehca_shca *shca = dev->driver_data;
+
+	return sprintf(buf, "%lx\n", shca->ipz_hca_handle.handle);
+
+}
+static DEVICE_ATTR(adapter_handle, S_IRUGO, ehca_show_adapter_handle, NULL);
+
+
+void ehca_create_device_sysfs(struct ibmebus_dev *dev)
+{
+	device_create_file(&dev->ofdev.dev, &dev_attr_adapter_handle);
+	device_create_file(&dev->ofdev.dev, &dev_attr_num_ports);
+	device_create_file(&dev->ofdev.dev, &dev_attr_hw_ver);
+	device_create_file(&dev->ofdev.dev, &dev_attr_max_eq);
+	device_create_file(&dev->ofdev.dev, &dev_attr_cur_eq);
+	device_create_file(&dev->ofdev.dev, &dev_attr_max_cq);
+	device_create_file(&dev->ofdev.dev, &dev_attr_cur_cq);
+	device_create_file(&dev->ofdev.dev, &dev_attr_max_qp);
+	device_create_file(&dev->ofdev.dev, &dev_attr_cur_qp);
+	device_create_file(&dev->ofdev.dev, &dev_attr_max_mr);
+	device_create_file(&dev->ofdev.dev, &dev_attr_cur_mr);
+	device_create_file(&dev->ofdev.dev, &dev_attr_max_mw);
+	device_create_file(&dev->ofdev.dev, &dev_attr_cur_mw);
+	device_create_file(&dev->ofdev.dev, &dev_attr_max_pd);
+	device_create_file(&dev->ofdev.dev, &dev_attr_max_ah);
+}
+
+void ehca_remove_device_sysfs(struct ibmebus_dev *dev)
+{
+	device_remove_file(&dev->ofdev.dev, &dev_attr_adapter_handle);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_num_ports);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_hw_ver);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_max_eq);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_cur_eq);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_max_cq);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_cur_cq);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_max_qp);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_cur_qp);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_max_mr);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_cur_mr);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_max_mw);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_cur_mw);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_max_pd);
+	device_remove_file(&dev->ofdev.dev, &dev_attr_max_ah);
+}
+
+static int __devinit ehca_probe(struct ibmebus_dev *dev,
+				const struct of_device_id *id)
+{
+	struct ehca_shca *shca;
+	u64 *handle;
+	struct ib_pd *ibpd;
+	int ret;
+
+	handle = (u64 *)get_property(dev->ofdev.node, "ibm,hca-handle", NULL);
+	if (!handle) {
+		ehca_gen_err("Cannot get eHCA handle for adapter: %s.",
+			     dev->ofdev.node->full_name);
+		return -ENODEV;
+	}
+
+	if (!(*handle)) {
+		ehca_gen_err("Wrong eHCA handle for adapter: %s.",
+			     dev->ofdev.node->full_name);
+		return -ENODEV;
+	}
+
+	shca = (struct ehca_shca *)ib_alloc_device(sizeof(*shca));
+	if (!shca) {
+		ehca_gen_err("Cannot allocate shca memory.");
+		return -ENOMEM;
+	}
+
+	shca->ibmebus_dev = dev;
+	shca->ipz_hca_handle.handle = *handle;
+	dev->ofdev.dev.driver_data = shca;
+
+	ret = ehca_sense_attributes(shca);
+	if (ret < 0) {
+		ehca_gen_err("Cannot sense eHCA attributes.");
+		goto probe1;
+	}
+
+	ret = ehca_register_device(shca);
+	if (ret) {
+		ehca_gen_err("Cannot register Infiniband device");
+		goto probe1;
+	}
+
+	/* create event queues */
+	ret = ehca_create_eq(shca, &shca->eq, EHCA_EQ, 2048);
+	if (ret) {
+		ehca_err(&shca->ib_device, "Cannot create EQ.");
+		goto probe2;
+	}
+
+	ret = ehca_create_eq(shca, &shca->neq, EHCA_NEQ, 513);
+	if (ret) {
+		ehca_err(&shca->ib_device, "Cannot create NEQ.");
+		goto probe3;
+	}
+
+	/* create internal protection domain */
+	ibpd = ehca_alloc_pd(&shca->ib_device, (void*)(-1), NULL);
+	if (IS_ERR(ibpd)) {
+		ehca_err(&shca->ib_device, "Cannot create internal PD.");
+		ret = PTR_ERR(ibpd);
+		goto probe4;
+	}
+
+	shca->pd = container_of(ibpd, struct ehca_pd, ib_pd);
+	shca->pd->ib_pd.device = &shca->ib_device;
+
+	/* create internal max MR */
+	ret = ehca_reg_internal_maxmr(shca, shca->pd, &shca->maxmr);
+
+	if (ret) {
+		ehca_err(&shca->ib_device, "Cannot create internal MR ret=%x",
+			 ret);
+		goto probe5;
+	}
+
+	/* create AQP1 for port 1 */
+	if (ehca_open_aqp1 == 1) {
+		shca->sport[0].port_state = IB_PORT_DOWN;
+		ret = ehca_create_aqp1(shca, 1);
+		if (ret) {
+			ehca_err(&shca->ib_device,
+				 "Cannot create AQP1 for port 1.");
+			goto probe6;
+		}
+	}
+
+	/* create AQP1 for port 2 */
+	if ((ehca_open_aqp1 == 1) && (shca->num_ports == 2)) {
+		shca->sport[1].port_state = IB_PORT_DOWN;
+		ret = ehca_create_aqp1(shca, 2);
+		if (ret) {
+			ehca_err(&shca->ib_device,
+				 "Cannot create AQP1 for port 2.");
+			goto probe7;
+		}
+	}
+
+	ehca_create_device_sysfs(dev);
+
+	spin_lock(&shca_list_lock);
+	list_add(&shca->shca_list, &shca_list);
+	spin_unlock(&shca_list_lock);
+
+	return 0;
+
+probe7:
+	ret = ehca_destroy_aqp1(&shca->sport[0]);
+	if (ret)
+		ehca_err(&shca->ib_device,
+			 "Cannot destroy AQP1 for port 1. ret=%x", ret);
+
+probe6:
+	ret = ehca_dereg_internal_maxmr(shca);
+	if (ret)
+		ehca_err(&shca->ib_device,
+			 "Cannot destroy internal MR. ret=%x", ret);
+
+probe5:
+	ret = ehca_dealloc_pd(&shca->pd->ib_pd);
+	if (ret)
+		ehca_err(&shca->ib_device,
+			 "Cannot destroy internal PD. ret=%x", ret);
+
+probe4:
+	ret = ehca_destroy_eq(shca, &shca->neq);
+	if (ret)
+		ehca_err(&shca->ib_device,
+			 "Cannot destroy NEQ. ret=%x", ret);
+
+probe3:
+	ret = ehca_destroy_eq(shca, &shca->eq);
+	if (ret)
+		ehca_err(&shca->ib_device,
+			 "Cannot destroy EQ. ret=%x", ret);
+
+probe2:
+	ib_unregister_device(&shca->ib_device);
+
+probe1:
+	ib_dealloc_device(&shca->ib_device);
+
+	return -EINVAL;
+}
+
+static int __devexit ehca_remove(struct ibmebus_dev *dev)
+{
+	struct ehca_shca *shca = dev->ofdev.dev.driver_data;
+	int ret;
+
+	ehca_remove_device_sysfs(dev);
+
+	if (ehca_open_aqp1 == 1) {
+		int i;
+		for (i = 0; i < shca->num_ports; i++) {
+			ret = ehca_destroy_aqp1(&shca->sport[i]);
+			if (ret)
+				ehca_err(&shca->ib_device,
+					 "Cannot destroy AQP1 for port %x "
+					 "ret=%x", ret, i);
+		}
+	}
+
+	ib_unregister_device(&shca->ib_device);
+
+	ret = ehca_dereg_internal_maxmr(shca);
+	if (ret)
+		ehca_err(&shca->ib_device,
+			 "Cannot destroy internal MR. ret=%x", ret);
+
+	ret = ehca_dealloc_pd(&shca->pd->ib_pd);
+	if (ret)
+		ehca_err(&shca->ib_device,
+			 "Cannot destroy internal PD. ret=%x", ret);
+
+	ret = ehca_destroy_eq(shca, &shca->eq);
+	if (ret)
+		ehca_err(&shca->ib_device, "Cannot destroy EQ. ret=%x", ret);
+
+	ret = ehca_destroy_eq(shca, &shca->neq);
+	if (ret)
+		ehca_err(&shca->ib_device, "Canot destroy NEQ. ret=%x", ret);
+
+	ib_dealloc_device(&shca->ib_device);
+
+	spin_lock(&shca_list_lock);
+	list_del(&shca->shca_list);
+	spin_unlock(&shca_list_lock);
+
+	return ret;
+}
+
+static struct of_device_id ehca_device_table[] =
+{
+	{
+		.name       = "lhca",
+		.compatible = "IBM,lhca",
+	},
+	{},
+};
+
+static struct ibmebus_driver ehca_driver = {
+	.name     = "ehca",
+	.id_table = ehca_device_table,
+	.probe    = ehca_probe,
+	.remove   = ehca_remove,
+};
+
+void ehca_poll_eqs(unsigned long data)
+{
+	struct ehca_shca *shca;
+
+	spin_lock(&shca_list_lock);
+	list_for_each_entry(shca, &shca_list, shca_list) {
+		if (shca->eq.is_initialized)
+			ehca_tasklet_eq((unsigned long)(void*)shca);
+	}
+	mod_timer(&poll_eqs_timer, jiffies + HZ);
+	spin_unlock(&shca_list_lock);
+}
+
+int __init ehca_module_init(void)
+{
+	int ret;
+
+	printk(KERN_INFO "eHCA Infiniband Device Driver "
+	                 "(Rel.: SVNEHCA_0016)\n");
+	idr_init(&ehca_qp_idr);
+	idr_init(&ehca_cq_idr);
+	spin_lock_init(&ehca_qp_idr_lock);
+	spin_lock_init(&ehca_cq_idr_lock);
+
+	INIT_LIST_HEAD(&shca_list);
+	spin_lock_init(&shca_list_lock);
+
+	if ((ret = ehca_create_comp_pool())) {
+		ehca_gen_err("Cannot create comp pool.");
+		return ret;
+	}
+
+	if ((ret = ehca_create_slab_caches())) {
+		ehca_gen_err("Cannot create SLAB caches");
+		ret = -ENOMEM;
+		goto module_init1;
+	}
+
+	if ((ret = ibmebus_register_driver(&ehca_driver))) {
+		ehca_gen_err("Cannot register eHCA device driver");
+		ret = -EINVAL;
+		goto module_init2;
+	}
+
+	ehca_create_driver_sysfs(&ehca_driver);
+
+	if (ehca_poll_all_eqs != 1) {
+		ehca_gen_err("WARNING!!!");
+		ehca_gen_err("It is possible to lose interrupts.");
+	} else {
+		init_timer(&poll_eqs_timer);
+		poll_eqs_timer.function = ehca_poll_eqs;
+		poll_eqs_timer.expires = jiffies + HZ;
+		add_timer(&poll_eqs_timer);
+	}
+
+	return 0;
+
+module_init2:
+	ehca_destroy_slab_caches();
+
+module_init1:
+	ehca_destroy_comp_pool();
+	return ret;
+};
+
+void __exit ehca_module_exit(void)
+{
+	if (ehca_poll_all_eqs == 1)
+		del_timer_sync(&poll_eqs_timer);
+
+	ehca_remove_driver_sysfs(&ehca_driver);
+	ibmebus_unregister_driver(&ehca_driver);
+
+	ehca_destroy_slab_caches();
+
+	ehca_destroy_comp_pool();
+
+	idr_destroy(&ehca_cq_idr);
+	idr_destroy(&ehca_qp_idr);
+};
+
+module_init(ehca_module_init);
+module_exit(ehca_module_exit);
diff --git a/drivers/infiniband/hw/ehca/ehca_mcast.c b/drivers/infiniband/hw/ehca/ehca_mcast.c
new file mode 100644
index 0000000..32a8706
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_mcast.c
@@ -0,0 +1,131 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  mcast  functions
+ *
+ *  Authors: Khadija Souissi <souissik@de.ibm.com>
+ *           Waleri Fomin <fomin@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Heiko J Schick <schickhj@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <linux/module.h>
+#include <linux/err.h>
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+#define MAX_MC_LID 0xFFFE
+#define MIN_MC_LID 0xC000	/* Multicast limits */
+#define EHCA_VALID_MULTICAST_GID(gid)  ((gid)[0] == 0xFF)
+#define EHCA_VALID_MULTICAST_LID(lid) \
+	(((lid) >= MIN_MC_LID) && ((lid) <= MAX_MC_LID))
+
+int ehca_attach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+	struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+	struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
+					      ib_device);
+	union ib_gid my_gid;
+	u64 subnet_prefix, interface_id, h_ret;
+
+	if (ibqp->qp_type != IB_QPT_UD) {
+		ehca_err(ibqp->device, "invalid qp_type=%x", ibqp->qp_type);
+		return -EINVAL;
+	}
+
+	if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
+		ehca_err(ibqp->device, "invalid mulitcast gid");
+		return -EINVAL;
+	} else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
+		ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
+		return -EINVAL;
+	}
+
+	memcpy(&my_gid.raw, gid->raw, sizeof(union ib_gid));
+
+	subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
+	interface_id = be64_to_cpu(my_gid.global.interface_id);
+	h_ret = hipz_h_attach_mcqp(shca->ipz_hca_handle,
+				   my_qp->ipz_qp_handle,
+				   my_qp->galpas.kernel,
+				   lid, subnet_prefix, interface_id);
+	if (h_ret != H_SUCCESS)
+		ehca_err(ibqp->device,
+			 "ehca_qp=%p qp_num=%x hipz_h_attach_mcqp() failed "
+			 "h_ret=%lx", my_qp, ibqp->qp_num, h_ret);
+
+	return ehca2ib_return_code(h_ret);
+}
+
+int ehca_detach_mcast(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
+{
+	struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+	struct ehca_shca *shca = container_of(ibqp->pd->device,
+					      struct ehca_shca, ib_device);
+	union ib_gid my_gid;
+	u64 subnet_prefix, interface_id, h_ret;
+
+	if (ibqp->qp_type != IB_QPT_UD) {
+		ehca_err(ibqp->device, "invalid qp_type %x", ibqp->qp_type);
+		return -EINVAL;
+	}
+
+	if (!(EHCA_VALID_MULTICAST_GID(gid->raw))) {
+		ehca_err(ibqp->device, "invalid mulitcast gid");
+		return -EINVAL;
+	} else if ((lid < MIN_MC_LID) || (lid > MAX_MC_LID)) {
+		ehca_err(ibqp->device, "invalid mulitcast lid=%x", lid);
+		return -EINVAL;
+	}
+
+	memcpy(&my_gid.raw, gid->raw, sizeof(union ib_gid));
+
+	subnet_prefix = be64_to_cpu(my_gid.global.subnet_prefix);
+	interface_id = be64_to_cpu(my_gid.global.interface_id);
+	h_ret = hipz_h_detach_mcqp(shca->ipz_hca_handle,
+				   my_qp->ipz_qp_handle,
+				   my_qp->galpas.kernel,
+				   lid, subnet_prefix, interface_id);
+	if (h_ret != H_SUCCESS)
+		ehca_err(ibqp->device,
+			 "ehca_qp=%p qp_num=%x hipz_h_detach_mcqp() failed "
+			 "h_ret=%lx", my_qp, ibqp->qp_num, h_ret);
+
+	return ehca2ib_return_code(h_ret);
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_mrmw.c b/drivers/infiniband/hw/ehca/ehca_mrmw.c
new file mode 100644
index 0000000..5ca6544
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_mrmw.c
@@ -0,0 +1,2261 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  MR/MW functions
+ *
+ *  Authors: Dietmar Decker <ddecker@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/current.h>
+
+#include "ehca_iverbs.h"
+#include "ehca_mrmw.h"
+#include "hcp_if.h"
+#include "hipz_hw.h"
+
+static struct kmem_cache *mr_cache;
+static struct kmem_cache *mw_cache;
+
+static struct ehca_mr *ehca_mr_new(void)
+{
+	struct ehca_mr *me;
+
+	me = kmem_cache_alloc(mr_cache, SLAB_KERNEL);
+	if (me) {
+		memset(me, 0, sizeof(struct ehca_mr));
+		spin_lock_init(&me->mrlock);
+	} else
+		ehca_gen_err("alloc failed");
+
+	return me;
+}
+
+static void ehca_mr_delete(struct ehca_mr *me)
+{
+	kmem_cache_free(mr_cache, me);
+}
+
+static struct ehca_mw *ehca_mw_new(void)
+{
+	struct ehca_mw *me;
+
+	me = kmem_cache_alloc(mw_cache, SLAB_KERNEL);
+	if (me) {
+		memset(me, 0, sizeof(struct ehca_mw));
+		spin_lock_init(&me->mwlock);
+	} else
+		ehca_gen_err("alloc failed");
+
+	return me;
+}
+
+static void ehca_mw_delete(struct ehca_mw *me)
+{
+	kmem_cache_free(mw_cache, me);
+}
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mr *ehca_get_dma_mr(struct ib_pd *pd, int mr_access_flags)
+{
+	struct ib_mr *ib_mr;
+	int ret;
+	struct ehca_mr *e_maxmr;
+	struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+	struct ehca_shca *shca =
+		container_of(pd->device, struct ehca_shca, ib_device);
+
+	if (shca->maxmr) {
+		e_maxmr = ehca_mr_new();
+		if (!e_maxmr) {
+			ehca_err(&shca->ib_device, "out of memory");
+			ib_mr = ERR_PTR(-ENOMEM);
+			goto get_dma_mr_exit0;
+		}
+
+		ret = ehca_reg_maxmr(shca, e_maxmr, (u64*)KERNELBASE,
+				     mr_access_flags, e_pd,
+				     &e_maxmr->ib.ib_mr.lkey,
+				     &e_maxmr->ib.ib_mr.rkey);
+		if (ret) {
+			ib_mr = ERR_PTR(ret);
+			goto get_dma_mr_exit0;
+		}
+		ib_mr = &e_maxmr->ib.ib_mr;
+	} else {
+		ehca_err(&shca->ib_device, "no internal max-MR exist!");
+		ib_mr = ERR_PTR(-EINVAL);
+		goto get_dma_mr_exit0;
+	}
+
+get_dma_mr_exit0:
+	if (IS_ERR(ib_mr))
+		ehca_err(&shca->ib_device, "rc=%lx pd=%p mr_access_flags=%x ",
+			 PTR_ERR(ib_mr), pd, mr_access_flags);
+	return ib_mr;
+} /* end ehca_get_dma_mr() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mr *ehca_reg_phys_mr(struct ib_pd *pd,
+			       struct ib_phys_buf *phys_buf_array,
+			       int num_phys_buf,
+			       int mr_access_flags,
+			       u64 *iova_start)
+{
+	struct ib_mr *ib_mr;
+	int ret;
+	struct ehca_mr *e_mr;
+	struct ehca_shca *shca =
+		container_of(pd->device, struct ehca_shca, ib_device);
+	struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+
+	u64 size;
+	struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
+	u32 num_pages_mr;
+	u32 num_pages_4k; /* 4k portion "pages" */
+
+	if ((num_phys_buf <= 0) || !phys_buf_array) {
+		ehca_err(pd->device, "bad input values: num_phys_buf=%x "
+			 "phys_buf_array=%p", num_phys_buf, phys_buf_array);
+		ib_mr = ERR_PTR(-EINVAL);
+		goto reg_phys_mr_exit0;
+	}
+	if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+	     !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+	    ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+	     !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
+		/*
+		 * Remote Write Access requires Local Write Access
+		 * Remote Atomic Access requires Local Write Access
+		 */
+		ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+			 mr_access_flags);
+		ib_mr = ERR_PTR(-EINVAL);
+		goto reg_phys_mr_exit0;
+	}
+
+	/* check physical buffer list and calculate size */
+	ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array, num_phys_buf,
+					    iova_start, &size);
+	if (ret) {
+		ib_mr = ERR_PTR(ret);
+		goto reg_phys_mr_exit0;
+	}
+	if ((size == 0) ||
+	    (((u64)iova_start + size) < (u64)iova_start)) {
+		ehca_err(pd->device, "bad input values: size=%lx iova_start=%p",
+			 size, iova_start);
+		ib_mr = ERR_PTR(-EINVAL);
+		goto reg_phys_mr_exit0;
+	}
+
+	e_mr = ehca_mr_new();
+	if (!e_mr) {
+		ehca_err(pd->device, "out of memory");
+		ib_mr = ERR_PTR(-ENOMEM);
+		goto reg_phys_mr_exit0;
+	}
+
+	/* determine number of MR pages */
+	num_pages_mr = ((((u64)iova_start % PAGE_SIZE) + size +
+			 PAGE_SIZE - 1) / PAGE_SIZE);
+	num_pages_4k = ((((u64)iova_start % EHCA_PAGESIZE) + size +
+			 EHCA_PAGESIZE - 1) / EHCA_PAGESIZE);
+
+	/* register MR on HCA */
+	if (ehca_mr_is_maxmr(size, iova_start)) {
+		e_mr->flags |= EHCA_MR_FLAG_MAXMR;
+		ret = ehca_reg_maxmr(shca, e_mr, iova_start, mr_access_flags,
+				     e_pd, &e_mr->ib.ib_mr.lkey,
+				     &e_mr->ib.ib_mr.rkey);
+		if (ret) {
+			ib_mr = ERR_PTR(ret);
+			goto reg_phys_mr_exit1;
+		}
+	} else {
+		pginfo.type           = EHCA_MR_PGI_PHYS;
+		pginfo.num_pages      = num_pages_mr;
+		pginfo.num_4k         = num_pages_4k;
+		pginfo.num_phys_buf   = num_phys_buf;
+		pginfo.phys_buf_array = phys_buf_array;
+		pginfo.next_4k        = (((u64)iova_start & ~PAGE_MASK) /
+					 EHCA_PAGESIZE);
+
+		ret = ehca_reg_mr(shca, e_mr, iova_start, size, mr_access_flags,
+				  e_pd, &pginfo, &e_mr->ib.ib_mr.lkey,
+				  &e_mr->ib.ib_mr.rkey);
+		if (ret) {
+			ib_mr = ERR_PTR(ret);
+			goto reg_phys_mr_exit1;
+		}
+	}
+
+	/* successful registration of all pages */
+	return &e_mr->ib.ib_mr;
+
+reg_phys_mr_exit1:
+	ehca_mr_delete(e_mr);
+reg_phys_mr_exit0:
+	if (IS_ERR(ib_mr))
+		ehca_err(pd->device, "rc=%lx pd=%p phys_buf_array=%p "
+			 "num_phys_buf=%x mr_access_flags=%x iova_start=%p",
+			 PTR_ERR(ib_mr), pd, phys_buf_array,
+			 num_phys_buf, mr_access_flags, iova_start);
+	return ib_mr;
+} /* end ehca_reg_phys_mr() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mr *ehca_reg_user_mr(struct ib_pd *pd,
+			       struct ib_umem *region,
+			       int mr_access_flags,
+			       struct ib_udata *udata)
+{
+	struct ib_mr *ib_mr;
+	struct ehca_mr *e_mr;
+	struct ehca_shca *shca =
+		container_of(pd->device, struct ehca_shca, ib_device);
+	struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+	struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
+	int ret;
+	u32 num_pages_mr;
+	u32 num_pages_4k; /* 4k portion "pages" */
+
+	if (!pd) {
+		ehca_gen_err("bad pd=%p", pd);
+		return ERR_PTR(-EFAULT);
+	}
+	if (!region) {
+		ehca_err(pd->device, "bad input values: region=%p", region);
+		ib_mr = ERR_PTR(-EINVAL);
+		goto reg_user_mr_exit0;
+	}
+	if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+	     !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+	    ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+	     !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
+		/*
+		 * Remote Write Access requires Local Write Access
+		 * Remote Atomic Access requires Local Write Access
+		 */
+		ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+			 mr_access_flags);
+		ib_mr = ERR_PTR(-EINVAL);
+		goto reg_user_mr_exit0;
+	}
+	if (region->page_size != PAGE_SIZE) {
+		ehca_err(pd->device, "page size not supported, "
+			 "region->page_size=%x", region->page_size);
+		ib_mr = ERR_PTR(-EINVAL);
+		goto reg_user_mr_exit0;
+	}
+
+	if ((region->length == 0) ||
+	    ((region->virt_base + region->length) < region->virt_base)) {
+		ehca_err(pd->device, "bad input values: length=%lx "
+			 "virt_base=%lx", region->length, region->virt_base);
+		ib_mr = ERR_PTR(-EINVAL);
+		goto reg_user_mr_exit0;
+	}
+
+	e_mr = ehca_mr_new();
+	if (!e_mr) {
+		ehca_err(pd->device, "out of memory");
+		ib_mr = ERR_PTR(-ENOMEM);
+		goto reg_user_mr_exit0;
+	}
+
+	/* determine number of MR pages */
+	num_pages_mr = (((region->virt_base % PAGE_SIZE) + region->length +
+			 PAGE_SIZE - 1) / PAGE_SIZE);
+	num_pages_4k = (((region->virt_base % EHCA_PAGESIZE) + region->length +
+			 EHCA_PAGESIZE - 1) / EHCA_PAGESIZE);
+
+	/* register MR on HCA */
+	pginfo.type       = EHCA_MR_PGI_USER;
+	pginfo.num_pages  = num_pages_mr;
+	pginfo.num_4k     = num_pages_4k;
+	pginfo.region     = region;
+	pginfo.next_4k	  = region->offset / EHCA_PAGESIZE;
+	pginfo.next_chunk = list_prepare_entry(pginfo.next_chunk,
+					       (&region->chunk_list),
+					       list);
+
+	ret = ehca_reg_mr(shca, e_mr, (u64*)region->virt_base,
+			  region->length, mr_access_flags, e_pd, &pginfo,
+			  &e_mr->ib.ib_mr.lkey, &e_mr->ib.ib_mr.rkey);
+	if (ret) {
+		ib_mr = ERR_PTR(ret);
+		goto reg_user_mr_exit1;
+	}
+
+	/* successful registration of all pages */
+	return &e_mr->ib.ib_mr;
+
+reg_user_mr_exit1:
+	ehca_mr_delete(e_mr);
+reg_user_mr_exit0:
+	if (IS_ERR(ib_mr))
+		ehca_err(pd->device, "rc=%lx pd=%p region=%p mr_access_flags=%x"
+			 " udata=%p",
+			 PTR_ERR(ib_mr), pd, region, mr_access_flags, udata);
+	return ib_mr;
+} /* end ehca_reg_user_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_rereg_phys_mr(struct ib_mr *mr,
+		       int mr_rereg_mask,
+		       struct ib_pd *pd,
+		       struct ib_phys_buf *phys_buf_array,
+		       int num_phys_buf,
+		       int mr_access_flags,
+		       u64 *iova_start)
+{
+	int ret;
+
+	struct ehca_shca *shca =
+		container_of(mr->device, struct ehca_shca, ib_device);
+	struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
+	struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
+	u64 new_size;
+	u64 *new_start;
+	u32 new_acl;
+	struct ehca_pd *new_pd;
+	u32 tmp_lkey, tmp_rkey;
+	unsigned long sl_flags;
+	u32 num_pages_mr = 0;
+	u32 num_pages_4k = 0; /* 4k portion "pages" */
+	struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
+	u32 cur_pid = current->tgid;
+
+	if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
+	    (my_pd->ownpid != cur_pid)) {
+		ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		ret = -EINVAL;
+		goto rereg_phys_mr_exit0;
+	}
+
+	if (!(mr_rereg_mask & IB_MR_REREG_TRANS)) {
+		/* TODO not supported, because PHYP rereg hCall needs pages */
+		ehca_err(mr->device, "rereg without IB_MR_REREG_TRANS not "
+			 "supported yet, mr_rereg_mask=%x", mr_rereg_mask);
+		ret = -EINVAL;
+		goto rereg_phys_mr_exit0;
+	}
+
+	if (mr_rereg_mask & IB_MR_REREG_PD) {
+		if (!pd) {
+			ehca_err(mr->device, "rereg with bad pd, pd=%p "
+				 "mr_rereg_mask=%x", pd, mr_rereg_mask);
+			ret = -EINVAL;
+			goto rereg_phys_mr_exit0;
+		}
+	}
+
+	if ((mr_rereg_mask &
+	     ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) ||
+	    (mr_rereg_mask == 0)) {
+		ret = -EINVAL;
+		goto rereg_phys_mr_exit0;
+	}
+
+	/* check other parameters */
+	if (e_mr == shca->maxmr) {
+		/* should be impossible, however reject to be sure */
+		ehca_err(mr->device, "rereg internal max-MR impossible, mr=%p "
+			 "shca->maxmr=%p mr->lkey=%x",
+			 mr, shca->maxmr, mr->lkey);
+		ret = -EINVAL;
+		goto rereg_phys_mr_exit0;
+	}
+	if (mr_rereg_mask & IB_MR_REREG_TRANS) { /* transl., i.e. addr/size */
+		if (e_mr->flags & EHCA_MR_FLAG_FMR) {
+			ehca_err(mr->device, "not supported for FMR, mr=%p "
+				 "flags=%x", mr, e_mr->flags);
+			ret = -EINVAL;
+			goto rereg_phys_mr_exit0;
+		}
+		if (!phys_buf_array || num_phys_buf <= 0) {
+			ehca_err(mr->device, "bad input values: mr_rereg_mask=%x"
+				 " phys_buf_array=%p num_phys_buf=%x",
+				 mr_rereg_mask, phys_buf_array, num_phys_buf);
+			ret = -EINVAL;
+			goto rereg_phys_mr_exit0;
+		}
+	}
+	if ((mr_rereg_mask & IB_MR_REREG_ACCESS) &&	/* change ACL */
+	    (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+	      !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+	     ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+	      !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)))) {
+		/*
+		 * Remote Write Access requires Local Write Access
+		 * Remote Atomic Access requires Local Write Access
+		 */
+		ehca_err(mr->device, "bad input values: mr_rereg_mask=%x "
+			 "mr_access_flags=%x", mr_rereg_mask, mr_access_flags);
+		ret = -EINVAL;
+		goto rereg_phys_mr_exit0;
+	}
+
+	/* set requested values dependent on rereg request */
+	spin_lock_irqsave(&e_mr->mrlock, sl_flags);
+	new_start = e_mr->start;  /* new == old address */
+	new_size  = e_mr->size;	  /* new == old length */
+	new_acl   = e_mr->acl;	  /* new == old access control */
+	new_pd    = container_of(mr->pd,struct ehca_pd,ib_pd); /*new == old PD*/
+
+	if (mr_rereg_mask & IB_MR_REREG_TRANS) {
+		new_start = iova_start;	/* change address */
+		/* check physical buffer list and calculate size */
+		ret = ehca_mr_chk_buf_and_calc_size(phys_buf_array,
+						    num_phys_buf, iova_start,
+						    &new_size);
+		if (ret)
+			goto rereg_phys_mr_exit1;
+		if ((new_size == 0) ||
+		    (((u64)iova_start + new_size) < (u64)iova_start)) {
+			ehca_err(mr->device, "bad input values: new_size=%lx "
+				 "iova_start=%p", new_size, iova_start);
+			ret = -EINVAL;
+			goto rereg_phys_mr_exit1;
+		}
+		num_pages_mr = ((((u64)new_start % PAGE_SIZE) + new_size +
+				 PAGE_SIZE - 1) / PAGE_SIZE);
+		num_pages_4k = ((((u64)new_start % EHCA_PAGESIZE) + new_size +
+				 EHCA_PAGESIZE - 1) / EHCA_PAGESIZE);
+		pginfo.type           = EHCA_MR_PGI_PHYS;
+		pginfo.num_pages      = num_pages_mr;
+		pginfo.num_4k         = num_pages_4k;
+		pginfo.num_phys_buf   = num_phys_buf;
+		pginfo.phys_buf_array = phys_buf_array;
+		pginfo.next_4k        = (((u64)iova_start & ~PAGE_MASK) /
+					 EHCA_PAGESIZE);
+	}
+	if (mr_rereg_mask & IB_MR_REREG_ACCESS)
+		new_acl = mr_access_flags;
+	if (mr_rereg_mask & IB_MR_REREG_PD)
+		new_pd = container_of(pd, struct ehca_pd, ib_pd);
+
+	ret = ehca_rereg_mr(shca, e_mr, new_start, new_size, new_acl,
+			    new_pd, &pginfo, &tmp_lkey, &tmp_rkey);
+	if (ret)
+		goto rereg_phys_mr_exit1;
+
+	/* successful reregistration */
+	if (mr_rereg_mask & IB_MR_REREG_PD)
+		mr->pd = pd;
+	mr->lkey = tmp_lkey;
+	mr->rkey = tmp_rkey;
+
+rereg_phys_mr_exit1:
+	spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
+rereg_phys_mr_exit0:
+	if (ret)
+		ehca_err(mr->device, "ret=%x mr=%p mr_rereg_mask=%x pd=%p "
+			 "phys_buf_array=%p num_phys_buf=%x mr_access_flags=%x "
+			 "iova_start=%p",
+			 ret, mr, mr_rereg_mask, pd, phys_buf_array,
+			 num_phys_buf, mr_access_flags, iova_start);
+	return ret;
+} /* end ehca_rereg_phys_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_query_mr(struct ib_mr *mr, struct ib_mr_attr *mr_attr)
+{
+	int ret = 0;
+	u64 h_ret;
+	struct ehca_shca *shca =
+		container_of(mr->device, struct ehca_shca, ib_device);
+	struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
+	struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
+	u32 cur_pid = current->tgid;
+	unsigned long sl_flags;
+	struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
+
+	if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
+	    (my_pd->ownpid != cur_pid)) {
+		ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		ret = -EINVAL;
+		goto query_mr_exit0;
+	}
+
+	if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
+		ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
+			 "e_mr->flags=%x", mr, e_mr, e_mr->flags);
+		ret = -EINVAL;
+		goto query_mr_exit0;
+	}
+
+	memset(mr_attr, 0, sizeof(struct ib_mr_attr));
+	spin_lock_irqsave(&e_mr->mrlock, sl_flags);
+
+	h_ret = hipz_h_query_mr(shca->ipz_hca_handle, e_mr, &hipzout);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(mr->device, "hipz_mr_query failed, h_ret=%lx mr=%p "
+			 "hca_hndl=%lx mr_hndl=%lx lkey=%x",
+			 h_ret, mr, shca->ipz_hca_handle.handle,
+			 e_mr->ipz_mr_handle.handle, mr->lkey);
+		ret = ehca_mrmw_map_hrc_query_mr(h_ret);
+		goto query_mr_exit1;
+	}
+	mr_attr->pd               = mr->pd;
+	mr_attr->device_virt_addr = hipzout.vaddr;
+	mr_attr->size             = hipzout.len;
+	mr_attr->lkey             = hipzout.lkey;
+	mr_attr->rkey             = hipzout.rkey;
+	ehca_mrmw_reverse_map_acl(&hipzout.acl, &mr_attr->mr_access_flags);
+
+query_mr_exit1:
+	spin_unlock_irqrestore(&e_mr->mrlock, sl_flags);
+query_mr_exit0:
+	if (ret)
+		ehca_err(mr->device, "ret=%x mr=%p mr_attr=%p",
+			 ret, mr, mr_attr);
+	return ret;
+} /* end ehca_query_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dereg_mr(struct ib_mr *mr)
+{
+	int ret = 0;
+	u64 h_ret;
+	struct ehca_shca *shca =
+		container_of(mr->device, struct ehca_shca, ib_device);
+	struct ehca_mr *e_mr = container_of(mr, struct ehca_mr, ib.ib_mr);
+	struct ehca_pd *my_pd = container_of(mr->pd, struct ehca_pd, ib_pd);
+	u32 cur_pid = current->tgid;
+
+	if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
+	    (my_pd->ownpid != cur_pid)) {
+		ehca_err(mr->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		ret = -EINVAL;
+		goto dereg_mr_exit0;
+	}
+
+	if ((e_mr->flags & EHCA_MR_FLAG_FMR)) {
+		ehca_err(mr->device, "not supported for FMR, mr=%p e_mr=%p "
+			 "e_mr->flags=%x", mr, e_mr, e_mr->flags);
+		ret = -EINVAL;
+		goto dereg_mr_exit0;
+	} else if (e_mr == shca->maxmr) {
+		/* should be impossible, however reject to be sure */
+		ehca_err(mr->device, "dereg internal max-MR impossible, mr=%p "
+			 "shca->maxmr=%p mr->lkey=%x",
+			 mr, shca->maxmr, mr->lkey);
+		ret = -EINVAL;
+		goto dereg_mr_exit0;
+	}
+
+	/* TODO: BUSY: MR still has bound window(s) */
+	h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(mr->device, "hipz_free_mr failed, h_ret=%lx shca=%p "
+			 "e_mr=%p hca_hndl=%lx mr_hndl=%lx mr->lkey=%x",
+			 h_ret, shca, e_mr, shca->ipz_hca_handle.handle,
+			 e_mr->ipz_mr_handle.handle, mr->lkey);
+		ret = ehca_mrmw_map_hrc_free_mr(h_ret);
+		goto dereg_mr_exit0;
+	}
+
+	/* successful deregistration */
+	ehca_mr_delete(e_mr);
+
+dereg_mr_exit0:
+	if (ret)
+		ehca_err(mr->device, "ret=%x mr=%p", ret, mr);
+	return ret;
+} /* end ehca_dereg_mr() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_mw *ehca_alloc_mw(struct ib_pd *pd)
+{
+	struct ib_mw *ib_mw;
+	u64 h_ret;
+	struct ehca_mw *e_mw;
+	struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+	struct ehca_shca *shca =
+		container_of(pd->device, struct ehca_shca, ib_device);
+	struct ehca_mw_hipzout_parms hipzout = {{0},0};
+
+	e_mw = ehca_mw_new();
+	if (!e_mw) {
+		ib_mw = ERR_PTR(-ENOMEM);
+		goto alloc_mw_exit0;
+	}
+
+	h_ret = hipz_h_alloc_resource_mw(shca->ipz_hca_handle, e_mw,
+					 e_pd->fw_pd, &hipzout);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(pd->device, "hipz_mw_allocate failed, h_ret=%lx "
+			 "shca=%p hca_hndl=%lx mw=%p",
+			 h_ret, shca, shca->ipz_hca_handle.handle, e_mw);
+		ib_mw = ERR_PTR(ehca_mrmw_map_hrc_alloc(h_ret));
+		goto alloc_mw_exit1;
+	}
+	/* successful MW allocation */
+	e_mw->ipz_mw_handle = hipzout.handle;
+	e_mw->ib_mw.rkey    = hipzout.rkey;
+	return &e_mw->ib_mw;
+
+alloc_mw_exit1:
+	ehca_mw_delete(e_mw);
+alloc_mw_exit0:
+	if (IS_ERR(ib_mw))
+		ehca_err(pd->device, "rc=%lx pd=%p", PTR_ERR(ib_mw), pd);
+	return ib_mw;
+} /* end ehca_alloc_mw() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_bind_mw(struct ib_qp *qp,
+		 struct ib_mw *mw,
+		 struct ib_mw_bind *mw_bind)
+{
+	/* TODO: not supported up to now */
+	ehca_gen_err("bind MW currently not supported by HCAD");
+
+	return -EPERM;
+} /* end ehca_bind_mw() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dealloc_mw(struct ib_mw *mw)
+{
+	u64 h_ret;
+	struct ehca_shca *shca =
+		container_of(mw->device, struct ehca_shca, ib_device);
+	struct ehca_mw *e_mw = container_of(mw, struct ehca_mw, ib_mw);
+
+	h_ret = hipz_h_free_resource_mw(shca->ipz_hca_handle, e_mw);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(mw->device, "hipz_free_mw failed, h_ret=%lx shca=%p "
+			 "mw=%p rkey=%x hca_hndl=%lx mw_hndl=%lx",
+			 h_ret, shca, mw, mw->rkey, shca->ipz_hca_handle.handle,
+			 e_mw->ipz_mw_handle.handle);
+		return ehca_mrmw_map_hrc_free_mw(h_ret);
+	}
+	/* successful deallocation */
+	ehca_mw_delete(e_mw);
+	return 0;
+} /* end ehca_dealloc_mw() */
+
+/*----------------------------------------------------------------------*/
+
+struct ib_fmr *ehca_alloc_fmr(struct ib_pd *pd,
+			      int mr_access_flags,
+			      struct ib_fmr_attr *fmr_attr)
+{
+	struct ib_fmr *ib_fmr;
+	struct ehca_shca *shca =
+		container_of(pd->device, struct ehca_shca, ib_device);
+	struct ehca_pd *e_pd = container_of(pd, struct ehca_pd, ib_pd);
+	struct ehca_mr *e_fmr;
+	int ret;
+	u32 tmp_lkey, tmp_rkey;
+	struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
+
+	/* check other parameters */
+	if (((mr_access_flags & IB_ACCESS_REMOTE_WRITE) &&
+	     !(mr_access_flags & IB_ACCESS_LOCAL_WRITE)) ||
+	    ((mr_access_flags & IB_ACCESS_REMOTE_ATOMIC) &&
+	     !(mr_access_flags & IB_ACCESS_LOCAL_WRITE))) {
+		/*
+		 * Remote Write Access requires Local Write Access
+		 * Remote Atomic Access requires Local Write Access
+		 */
+		ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+			 mr_access_flags);
+		ib_fmr = ERR_PTR(-EINVAL);
+		goto alloc_fmr_exit0;
+	}
+	if (mr_access_flags & IB_ACCESS_MW_BIND) {
+		ehca_err(pd->device, "bad input values: mr_access_flags=%x",
+			 mr_access_flags);
+		ib_fmr = ERR_PTR(-EINVAL);
+		goto alloc_fmr_exit0;
+	}
+	if ((fmr_attr->max_pages == 0) || (fmr_attr->max_maps == 0)) {
+		ehca_err(pd->device, "bad input values: fmr_attr->max_pages=%x "
+			 "fmr_attr->max_maps=%x fmr_attr->page_shift=%x",
+			 fmr_attr->max_pages, fmr_attr->max_maps,
+			 fmr_attr->page_shift);
+		ib_fmr = ERR_PTR(-EINVAL);
+		goto alloc_fmr_exit0;
+	}
+	if (((1 << fmr_attr->page_shift) != EHCA_PAGESIZE) &&
+	    ((1 << fmr_attr->page_shift) != PAGE_SIZE)) {
+		ehca_err(pd->device, "unsupported fmr_attr->page_shift=%x",
+			 fmr_attr->page_shift);
+		ib_fmr = ERR_PTR(-EINVAL);
+		goto alloc_fmr_exit0;
+	}
+
+	e_fmr = ehca_mr_new();
+	if (!e_fmr) {
+		ib_fmr = ERR_PTR(-ENOMEM);
+		goto alloc_fmr_exit0;
+	}
+	e_fmr->flags |= EHCA_MR_FLAG_FMR;
+
+	/* register MR on HCA */
+	ret = ehca_reg_mr(shca, e_fmr, NULL,
+			  fmr_attr->max_pages * (1 << fmr_attr->page_shift),
+			  mr_access_flags, e_pd, &pginfo,
+			  &tmp_lkey, &tmp_rkey);
+	if (ret) {
+		ib_fmr = ERR_PTR(ret);
+		goto alloc_fmr_exit1;
+	}
+
+	/* successful */
+	e_fmr->fmr_page_size = 1 << fmr_attr->page_shift;
+	e_fmr->fmr_max_pages = fmr_attr->max_pages;
+	e_fmr->fmr_max_maps = fmr_attr->max_maps;
+	e_fmr->fmr_map_cnt = 0;
+	return &e_fmr->ib.ib_fmr;
+
+alloc_fmr_exit1:
+	ehca_mr_delete(e_fmr);
+alloc_fmr_exit0:
+	if (IS_ERR(ib_fmr))
+		ehca_err(pd->device, "rc=%lx pd=%p mr_access_flags=%x "
+			 "fmr_attr=%p", PTR_ERR(ib_fmr), pd,
+			 mr_access_flags, fmr_attr);
+	return ib_fmr;
+} /* end ehca_alloc_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_map_phys_fmr(struct ib_fmr *fmr,
+		      u64 *page_list,
+		      int list_len,
+		      u64 iova)
+{
+	int ret;
+	struct ehca_shca *shca =
+		container_of(fmr->device, struct ehca_shca, ib_device);
+	struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
+	struct ehca_pd *e_pd = container_of(fmr->pd, struct ehca_pd, ib_pd);
+	struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
+	u32 tmp_lkey, tmp_rkey;
+
+	if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
+		ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
+			 e_fmr, e_fmr->flags);
+		ret = -EINVAL;
+		goto map_phys_fmr_exit0;
+	}
+	ret = ehca_fmr_check_page_list(e_fmr, page_list, list_len);
+	if (ret)
+		goto map_phys_fmr_exit0;
+	if (iova % e_fmr->fmr_page_size) {
+		/* only whole-numbered pages */
+		ehca_err(fmr->device, "bad iova, iova=%lx fmr_page_size=%x",
+			 iova, e_fmr->fmr_page_size);
+		ret = -EINVAL;
+		goto map_phys_fmr_exit0;
+	}
+	if (e_fmr->fmr_map_cnt >= e_fmr->fmr_max_maps) {
+		/* HCAD does not limit the maps, however trace this anyway */
+		ehca_info(fmr->device, "map limit exceeded, fmr=%p "
+			  "e_fmr->fmr_map_cnt=%x e_fmr->fmr_max_maps=%x",
+			  fmr, e_fmr->fmr_map_cnt, e_fmr->fmr_max_maps);
+	}
+
+	pginfo.type      = EHCA_MR_PGI_FMR;
+	pginfo.num_pages = list_len;
+	pginfo.num_4k    = list_len * (e_fmr->fmr_page_size / EHCA_PAGESIZE);
+	pginfo.page_list = page_list;
+	pginfo.next_4k   = ((iova & (e_fmr->fmr_page_size-1)) /
+			    EHCA_PAGESIZE);
+
+	ret = ehca_rereg_mr(shca, e_fmr, (u64*)iova,
+			    list_len * e_fmr->fmr_page_size,
+			    e_fmr->acl, e_pd, &pginfo, &tmp_lkey, &tmp_rkey);
+	if (ret)
+		goto map_phys_fmr_exit0;
+
+	/* successful reregistration */
+	e_fmr->fmr_map_cnt++;
+	e_fmr->ib.ib_fmr.lkey = tmp_lkey;
+	e_fmr->ib.ib_fmr.rkey = tmp_rkey;
+	return 0;
+
+map_phys_fmr_exit0:
+	if (ret)
+		ehca_err(fmr->device, "ret=%x fmr=%p page_list=%p list_len=%x "
+			 "iova=%lx",
+			 ret, fmr, page_list, list_len, iova);
+	return ret;
+} /* end ehca_map_phys_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_unmap_fmr(struct list_head *fmr_list)
+{
+	int ret = 0;
+	struct ib_fmr *ib_fmr;
+	struct ehca_shca *shca = NULL;
+	struct ehca_shca *prev_shca;
+	struct ehca_mr *e_fmr;
+	u32 num_fmr = 0;
+	u32 unmap_fmr_cnt = 0;
+
+	/* check all FMR belong to same SHCA, and check internal flag */
+	list_for_each_entry(ib_fmr, fmr_list, list) {
+		prev_shca = shca;
+		if (!ib_fmr) {
+			ehca_gen_err("bad fmr=%p in list", ib_fmr);
+			ret = -EINVAL;
+			goto unmap_fmr_exit0;
+		}
+		shca = container_of(ib_fmr->device, struct ehca_shca,
+				    ib_device);
+		e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
+		if ((shca != prev_shca) && prev_shca) {
+			ehca_err(&shca->ib_device, "SHCA mismatch, shca=%p "
+				 "prev_shca=%p e_fmr=%p",
+				 shca, prev_shca, e_fmr);
+			ret = -EINVAL;
+			goto unmap_fmr_exit0;
+		}
+		if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
+			ehca_err(&shca->ib_device, "not a FMR, e_fmr=%p "
+				 "e_fmr->flags=%x", e_fmr, e_fmr->flags);
+			ret = -EINVAL;
+			goto unmap_fmr_exit0;
+		}
+		num_fmr++;
+	}
+
+	/* loop over all FMRs to unmap */
+	list_for_each_entry(ib_fmr, fmr_list, list) {
+		unmap_fmr_cnt++;
+		e_fmr = container_of(ib_fmr, struct ehca_mr, ib.ib_fmr);
+		shca = container_of(ib_fmr->device, struct ehca_shca,
+				    ib_device);
+		ret = ehca_unmap_one_fmr(shca, e_fmr);
+		if (ret) {
+			/* unmap failed, stop unmapping of rest of FMRs */
+			ehca_err(&shca->ib_device, "unmap of one FMR failed, "
+				 "stop rest, e_fmr=%p num_fmr=%x "
+				 "unmap_fmr_cnt=%x lkey=%x", e_fmr, num_fmr,
+				 unmap_fmr_cnt, e_fmr->ib.ib_fmr.lkey);
+			goto unmap_fmr_exit0;
+		}
+	}
+
+unmap_fmr_exit0:
+	if (ret)
+		ehca_gen_err("ret=%x fmr_list=%p num_fmr=%x unmap_fmr_cnt=%x",
+			     ret, fmr_list, num_fmr, unmap_fmr_cnt);
+	return ret;
+} /* end ehca_unmap_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dealloc_fmr(struct ib_fmr *fmr)
+{
+	int ret;
+	u64 h_ret;
+	struct ehca_shca *shca =
+		container_of(fmr->device, struct ehca_shca, ib_device);
+	struct ehca_mr *e_fmr = container_of(fmr, struct ehca_mr, ib.ib_fmr);
+
+	if (!(e_fmr->flags & EHCA_MR_FLAG_FMR)) {
+		ehca_err(fmr->device, "not a FMR, e_fmr=%p e_fmr->flags=%x",
+			 e_fmr, e_fmr->flags);
+		ret = -EINVAL;
+		goto free_fmr_exit0;
+	}
+
+	h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(fmr->device, "hipz_free_mr failed, h_ret=%lx e_fmr=%p "
+			 "hca_hndl=%lx fmr_hndl=%lx fmr->lkey=%x",
+			 h_ret, e_fmr, shca->ipz_hca_handle.handle,
+			 e_fmr->ipz_mr_handle.handle, fmr->lkey);
+		ret = ehca_mrmw_map_hrc_free_mr(h_ret);
+		goto free_fmr_exit0;
+	}
+	/* successful deregistration */
+	ehca_mr_delete(e_fmr);
+	return 0;
+
+free_fmr_exit0:
+	if (ret)
+		ehca_err(&shca->ib_device, "ret=%x fmr=%p", ret, fmr);
+	return ret;
+} /* end ehca_dealloc_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_mr(struct ehca_shca *shca,
+		struct ehca_mr *e_mr,
+		u64 *iova_start,
+		u64 size,
+		int acl,
+		struct ehca_pd *e_pd,
+		struct ehca_mr_pginfo *pginfo,
+		u32 *lkey, /*OUT*/
+		u32 *rkey) /*OUT*/
+{
+	int ret;
+	u64 h_ret;
+	u32 hipz_acl;
+	struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
+
+	ehca_mrmw_map_acl(acl, &hipz_acl);
+	ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
+	if (ehca_use_hp_mr == 1)
+	        hipz_acl |= 0x00000001;
+
+	h_ret = hipz_h_alloc_resource_mr(shca->ipz_hca_handle, e_mr,
+					 (u64)iova_start, size, hipz_acl,
+					 e_pd->fw_pd, &hipzout);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "hipz_alloc_mr failed, h_ret=%lx "
+			 "hca_hndl=%lx", h_ret, shca->ipz_hca_handle.handle);
+		ret = ehca_mrmw_map_hrc_alloc(h_ret);
+		goto ehca_reg_mr_exit0;
+	}
+
+	e_mr->ipz_mr_handle = hipzout.handle;
+
+	ret = ehca_reg_mr_rpages(shca, e_mr, pginfo);
+	if (ret)
+		goto ehca_reg_mr_exit1;
+
+	/* successful registration */
+	e_mr->num_pages = pginfo->num_pages;
+	e_mr->num_4k    = pginfo->num_4k;
+	e_mr->start     = iova_start;
+	e_mr->size      = size;
+	e_mr->acl       = acl;
+	*lkey = hipzout.lkey;
+	*rkey = hipzout.rkey;
+	return 0;
+
+ehca_reg_mr_exit1:
+	h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "h_ret=%lx shca=%p e_mr=%p "
+			 "iova_start=%p size=%lx acl=%x e_pd=%p lkey=%x "
+			 "pginfo=%p num_pages=%lx num_4k=%lx ret=%x",
+			 h_ret, shca, e_mr, iova_start, size, acl, e_pd,
+			 hipzout.lkey, pginfo, pginfo->num_pages,
+			 pginfo->num_4k, ret);
+		ehca_err(&shca->ib_device, "internal error in ehca_reg_mr, "
+			 "not recoverable");
+	}
+ehca_reg_mr_exit0:
+	if (ret)
+		ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
+			 "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
+			 "num_pages=%lx num_4k=%lx",
+			 ret, shca, e_mr, iova_start, size, acl, e_pd, pginfo,
+			 pginfo->num_pages, pginfo->num_4k);
+	return ret;
+} /* end ehca_reg_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_mr_rpages(struct ehca_shca *shca,
+		       struct ehca_mr *e_mr,
+		       struct ehca_mr_pginfo *pginfo)
+{
+	int ret = 0;
+	u64 h_ret;
+	u32 rnum;
+	u64 rpage;
+	u32 i;
+	u64 *kpage;
+
+	kpage = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (!kpage) {
+		ehca_err(&shca->ib_device, "kpage alloc failed");
+		ret = -ENOMEM;
+		goto ehca_reg_mr_rpages_exit0;
+	}
+
+	/* max 512 pages per shot */
+	for (i = 0; i < ((pginfo->num_4k + 512 - 1) / 512); i++) {
+
+		if (i == ((pginfo->num_4k + 512 - 1) / 512) - 1) {
+			rnum = pginfo->num_4k % 512; /* last shot */
+			if (rnum == 0)
+				rnum = 512;      /* last shot is full */
+		} else
+			rnum = 512;
+
+		if (rnum > 1) {
+			ret = ehca_set_pagebuf(e_mr, pginfo, rnum, kpage);
+			if (ret) {
+				ehca_err(&shca->ib_device, "ehca_set_pagebuf "
+					 "bad rc, ret=%x rnum=%x kpage=%p",
+					 ret, rnum, kpage);
+				ret = -EFAULT;
+				goto ehca_reg_mr_rpages_exit1;
+			}
+			rpage = virt_to_abs(kpage);
+			if (!rpage) {
+				ehca_err(&shca->ib_device, "kpage=%p i=%x",
+					 kpage, i);
+				ret = -EFAULT;
+				goto ehca_reg_mr_rpages_exit1;
+			}
+		} else {  /* rnum==1 */
+			ret = ehca_set_pagebuf_1(e_mr, pginfo, &rpage);
+			if (ret) {
+				ehca_err(&shca->ib_device, "ehca_set_pagebuf_1 "
+					 "bad rc, ret=%x i=%x", ret, i);
+				ret = -EFAULT;
+				goto ehca_reg_mr_rpages_exit1;
+			}
+		}
+
+		h_ret = hipz_h_register_rpage_mr(shca->ipz_hca_handle, e_mr,
+						 0, /* pagesize 4k */
+						 0, rpage, rnum);
+
+		if (i == ((pginfo->num_4k + 512 - 1) / 512) - 1) {
+			/*
+			 * check for 'registration complete'==H_SUCCESS
+			 * and for 'page registered'==H_PAGE_REGISTERED
+			 */
+			if (h_ret != H_SUCCESS) {
+				ehca_err(&shca->ib_device, "last "
+					 "hipz_reg_rpage_mr failed, h_ret=%lx "
+					 "e_mr=%p i=%x hca_hndl=%lx mr_hndl=%lx"
+					 " lkey=%x", h_ret, e_mr, i,
+					 shca->ipz_hca_handle.handle,
+					 e_mr->ipz_mr_handle.handle,
+					 e_mr->ib.ib_mr.lkey);
+				ret = ehca_mrmw_map_hrc_rrpg_last(h_ret);
+				break;
+			} else
+				ret = 0;
+		} else if (h_ret != H_PAGE_REGISTERED) {
+			ehca_err(&shca->ib_device, "hipz_reg_rpage_mr failed, "
+				 "h_ret=%lx e_mr=%p i=%x lkey=%x hca_hndl=%lx "
+				 "mr_hndl=%lx", h_ret, e_mr, i,
+				 e_mr->ib.ib_mr.lkey,
+				 shca->ipz_hca_handle.handle,
+				 e_mr->ipz_mr_handle.handle);
+			ret = ehca_mrmw_map_hrc_rrpg_notlast(h_ret);
+			break;
+		} else
+			ret = 0;
+	} /* end for(i) */
+
+
+ehca_reg_mr_rpages_exit1:
+	kfree(kpage);
+ehca_reg_mr_rpages_exit0:
+	if (ret)
+		ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p pginfo=%p "
+			 "num_pages=%lx num_4k=%lx", ret, shca, e_mr, pginfo,
+			 pginfo->num_pages, pginfo->num_4k);
+	return ret;
+} /* end ehca_reg_mr_rpages() */
+
+/*----------------------------------------------------------------------*/
+
+inline int ehca_rereg_mr_rereg1(struct ehca_shca *shca,
+				struct ehca_mr *e_mr,
+				u64 *iova_start,
+				u64 size,
+				u32 acl,
+				struct ehca_pd *e_pd,
+				struct ehca_mr_pginfo *pginfo,
+				u32 *lkey, /*OUT*/
+				u32 *rkey) /*OUT*/
+{
+	int ret;
+	u64 h_ret;
+	u32 hipz_acl;
+	u64 *kpage;
+	u64 rpage;
+	struct ehca_mr_pginfo pginfo_save;
+	struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
+
+	ehca_mrmw_map_acl(acl, &hipz_acl);
+	ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
+
+	kpage = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (!kpage) {
+		ehca_err(&shca->ib_device, "kpage alloc failed");
+		ret = -ENOMEM;
+		goto ehca_rereg_mr_rereg1_exit0;
+	}
+
+	pginfo_save = *pginfo;
+	ret = ehca_set_pagebuf(e_mr, pginfo, pginfo->num_4k, kpage);
+	if (ret) {
+		ehca_err(&shca->ib_device, "set pagebuf failed, e_mr=%p "
+			 "pginfo=%p type=%x num_pages=%lx num_4k=%lx kpage=%p",
+			 e_mr, pginfo, pginfo->type, pginfo->num_pages,
+			 pginfo->num_4k,kpage);
+		goto ehca_rereg_mr_rereg1_exit1;
+	}
+	rpage = virt_to_abs(kpage);
+	if (!rpage) {
+		ehca_err(&shca->ib_device, "kpage=%p", kpage);
+		ret = -EFAULT;
+		goto ehca_rereg_mr_rereg1_exit1;
+	}
+	h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_mr,
+				      (u64)iova_start, size, hipz_acl,
+				      e_pd->fw_pd, rpage, &hipzout);
+	if (h_ret != H_SUCCESS) {
+		/*
+		 * reregistration unsuccessful, try it again with the 3 hCalls,
+		 * e.g. this is required in case H_MR_CONDITION
+		 * (MW bound or MR is shared)
+		 */
+		ehca_warn(&shca->ib_device, "hipz_h_reregister_pmr failed "
+			  "(Rereg1), h_ret=%lx e_mr=%p", h_ret, e_mr);
+		*pginfo = pginfo_save;
+		ret = -EAGAIN;
+	} else if ((u64*)hipzout.vaddr != iova_start) {
+		ehca_err(&shca->ib_device, "PHYP changed iova_start in "
+			 "rereg_pmr, iova_start=%p iova_start_out=%lx e_mr=%p "
+			 "mr_handle=%lx lkey=%x lkey_out=%x", iova_start,
+			 hipzout.vaddr, e_mr, e_mr->ipz_mr_handle.handle,
+			 e_mr->ib.ib_mr.lkey, hipzout.lkey);
+		ret = -EFAULT;
+	} else {
+		/*
+		 * successful reregistration
+		 * note: start and start_out are identical for eServer HCAs
+		 */
+		e_mr->num_pages = pginfo->num_pages;
+		e_mr->num_4k    = pginfo->num_4k;
+		e_mr->start     = iova_start;
+		e_mr->size      = size;
+		e_mr->acl       = acl;
+		*lkey = hipzout.lkey;
+		*rkey = hipzout.rkey;
+	}
+
+ehca_rereg_mr_rereg1_exit1:
+	kfree(kpage);
+ehca_rereg_mr_rereg1_exit0:
+	if ( ret && (ret != -EAGAIN) )
+		ehca_err(&shca->ib_device, "ret=%x lkey=%x rkey=%x "
+			 "pginfo=%p num_pages=%lx num_4k=%lx",
+			 ret, *lkey, *rkey, pginfo, pginfo->num_pages,
+			 pginfo->num_4k);
+	return ret;
+} /* end ehca_rereg_mr_rereg1() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_rereg_mr(struct ehca_shca *shca,
+		  struct ehca_mr *e_mr,
+		  u64 *iova_start,
+		  u64 size,
+		  int acl,
+		  struct ehca_pd *e_pd,
+		  struct ehca_mr_pginfo *pginfo,
+		  u32 *lkey,
+		  u32 *rkey)
+{
+	int ret = 0;
+	u64 h_ret;
+	int rereg_1_hcall = 1; /* 1: use hipz_h_reregister_pmr directly */
+	int rereg_3_hcall = 0; /* 1: use 3 hipz calls for reregistration */
+
+	/* first determine reregistration hCall(s) */
+	if ((pginfo->num_4k > 512) || (e_mr->num_4k > 512) ||
+	    (pginfo->num_4k > e_mr->num_4k)) {
+		ehca_dbg(&shca->ib_device, "Rereg3 case, pginfo->num_4k=%lx "
+			 "e_mr->num_4k=%x", pginfo->num_4k, e_mr->num_4k);
+		rereg_1_hcall = 0;
+		rereg_3_hcall = 1;
+	}
+
+	if (e_mr->flags & EHCA_MR_FLAG_MAXMR) {	/* check for max-MR */
+		rereg_1_hcall = 0;
+		rereg_3_hcall = 1;
+		e_mr->flags &= ~EHCA_MR_FLAG_MAXMR;
+		ehca_err(&shca->ib_device, "Rereg MR for max-MR! e_mr=%p",
+			 e_mr);
+	}
+
+	if (rereg_1_hcall) {
+		ret = ehca_rereg_mr_rereg1(shca, e_mr, iova_start, size,
+					   acl, e_pd, pginfo, lkey, rkey);
+		if (ret) {
+			if (ret == -EAGAIN)
+				rereg_3_hcall = 1;
+			else
+				goto ehca_rereg_mr_exit0;
+		}
+	}
+
+	if (rereg_3_hcall) {
+		struct ehca_mr save_mr;
+
+		/* first deregister old MR */
+		h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_mr);
+		if (h_ret != H_SUCCESS) {
+			ehca_err(&shca->ib_device, "hipz_free_mr failed, "
+				 "h_ret=%lx e_mr=%p hca_hndl=%lx mr_hndl=%lx "
+				 "mr->lkey=%x",
+				 h_ret, e_mr, shca->ipz_hca_handle.handle,
+				 e_mr->ipz_mr_handle.handle,
+				 e_mr->ib.ib_mr.lkey);
+			ret = ehca_mrmw_map_hrc_free_mr(h_ret);
+			goto ehca_rereg_mr_exit0;
+		}
+		/* clean ehca_mr_t, without changing struct ib_mr and lock */
+		save_mr = *e_mr;
+		ehca_mr_deletenew(e_mr);
+
+		/* set some MR values */
+		e_mr->flags = save_mr.flags;
+		e_mr->fmr_page_size = save_mr.fmr_page_size;
+		e_mr->fmr_max_pages = save_mr.fmr_max_pages;
+		e_mr->fmr_max_maps = save_mr.fmr_max_maps;
+		e_mr->fmr_map_cnt = save_mr.fmr_map_cnt;
+
+		ret = ehca_reg_mr(shca, e_mr, iova_start, size, acl,
+				      e_pd, pginfo, lkey, rkey);
+		if (ret) {
+			u32 offset = (u64)(&e_mr->flags) - (u64)e_mr;
+			memcpy(&e_mr->flags, &(save_mr.flags),
+			       sizeof(struct ehca_mr) - offset);
+			goto ehca_rereg_mr_exit0;
+		}
+	}
+
+ehca_rereg_mr_exit0:
+	if (ret)
+		ehca_err(&shca->ib_device, "ret=%x shca=%p e_mr=%p "
+			 "iova_start=%p size=%lx acl=%x e_pd=%p pginfo=%p "
+			 "num_pages=%lx lkey=%x rkey=%x rereg_1_hcall=%x "
+			 "rereg_3_hcall=%x", ret, shca, e_mr, iova_start, size,
+			 acl, e_pd, pginfo, pginfo->num_pages, *lkey, *rkey,
+			 rereg_1_hcall, rereg_3_hcall);
+	return ret;
+} /* end ehca_rereg_mr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_unmap_one_fmr(struct ehca_shca *shca,
+		       struct ehca_mr *e_fmr)
+{
+	int ret = 0;
+	u64 h_ret;
+	int rereg_1_hcall = 1; /* 1: use hipz_mr_reregister directly */
+	int rereg_3_hcall = 0; /* 1: use 3 hipz calls for unmapping */
+	struct ehca_pd *e_pd =
+		container_of(e_fmr->ib.ib_fmr.pd, struct ehca_pd, ib_pd);
+	struct ehca_mr save_fmr;
+	u32 tmp_lkey, tmp_rkey;
+	struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
+	struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
+
+	/* first check if reregistration hCall can be used for unmap */
+	if (e_fmr->fmr_max_pages > 512) {
+		rereg_1_hcall = 0;
+		rereg_3_hcall = 1;
+	}
+
+	if (rereg_1_hcall) {
+		/*
+		 * note: after using rereg hcall with len=0,
+		 * rereg hcall must be used again for registering pages
+		 */
+		h_ret = hipz_h_reregister_pmr(shca->ipz_hca_handle, e_fmr, 0,
+					      0, 0, e_pd->fw_pd, 0, &hipzout);
+		if (h_ret != H_SUCCESS) {
+			/*
+			 * should not happen, because length checked above,
+			 * FMRs are not shared and no MW bound to FMRs
+			 */
+			ehca_err(&shca->ib_device, "hipz_reregister_pmr failed "
+				 "(Rereg1), h_ret=%lx e_fmr=%p hca_hndl=%lx "
+				 "mr_hndl=%lx lkey=%x lkey_out=%x",
+				 h_ret, e_fmr, shca->ipz_hca_handle.handle,
+				 e_fmr->ipz_mr_handle.handle,
+				 e_fmr->ib.ib_fmr.lkey, hipzout.lkey);
+			rereg_3_hcall = 1;
+		} else {
+			/* successful reregistration */
+			e_fmr->start = NULL;
+			e_fmr->size = 0;
+			tmp_lkey = hipzout.lkey;
+			tmp_rkey = hipzout.rkey;
+		}
+	}
+
+	if (rereg_3_hcall) {
+		struct ehca_mr save_mr;
+
+		/* first free old FMR */
+		h_ret = hipz_h_free_resource_mr(shca->ipz_hca_handle, e_fmr);
+		if (h_ret != H_SUCCESS) {
+			ehca_err(&shca->ib_device, "hipz_free_mr failed, "
+				 "h_ret=%lx e_fmr=%p hca_hndl=%lx mr_hndl=%lx "
+				 "lkey=%x",
+				 h_ret, e_fmr, shca->ipz_hca_handle.handle,
+				 e_fmr->ipz_mr_handle.handle,
+				 e_fmr->ib.ib_fmr.lkey);
+			ret = ehca_mrmw_map_hrc_free_mr(h_ret);
+			goto ehca_unmap_one_fmr_exit0;
+		}
+		/* clean ehca_mr_t, without changing lock */
+		save_fmr = *e_fmr;
+		ehca_mr_deletenew(e_fmr);
+
+		/* set some MR values */
+		e_fmr->flags = save_fmr.flags;
+		e_fmr->fmr_page_size = save_fmr.fmr_page_size;
+		e_fmr->fmr_max_pages = save_fmr.fmr_max_pages;
+		e_fmr->fmr_max_maps = save_fmr.fmr_max_maps;
+		e_fmr->fmr_map_cnt = save_fmr.fmr_map_cnt;
+		e_fmr->acl = save_fmr.acl;
+
+		pginfo.type      = EHCA_MR_PGI_FMR;
+		pginfo.num_pages = 0;
+		pginfo.num_4k    = 0;
+		ret = ehca_reg_mr(shca, e_fmr, NULL,
+				  (e_fmr->fmr_max_pages * e_fmr->fmr_page_size),
+				  e_fmr->acl, e_pd, &pginfo, &tmp_lkey,
+				  &tmp_rkey);
+		if (ret) {
+			u32 offset = (u64)(&e_fmr->flags) - (u64)e_fmr;
+			memcpy(&e_fmr->flags, &(save_mr.flags),
+			       sizeof(struct ehca_mr) - offset);
+			goto ehca_unmap_one_fmr_exit0;
+		}
+	}
+
+ehca_unmap_one_fmr_exit0:
+	if (ret)
+		ehca_err(&shca->ib_device, "ret=%x tmp_lkey=%x tmp_rkey=%x "
+			 "fmr_max_pages=%x rereg_1_hcall=%x rereg_3_hcall=%x",
+			 ret, tmp_lkey, tmp_rkey, e_fmr->fmr_max_pages,
+			 rereg_1_hcall, rereg_3_hcall);
+	return ret;
+} /* end ehca_unmap_one_fmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_smr(struct ehca_shca *shca,
+		 struct ehca_mr *e_origmr,
+		 struct ehca_mr *e_newmr,
+		 u64 *iova_start,
+		 int acl,
+		 struct ehca_pd *e_pd,
+		 u32 *lkey, /*OUT*/
+		 u32 *rkey) /*OUT*/
+{
+	int ret = 0;
+	u64 h_ret;
+	u32 hipz_acl;
+	struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
+
+	ehca_mrmw_map_acl(acl, &hipz_acl);
+	ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
+
+	h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
+				    (u64)iova_start, hipz_acl, e_pd->fw_pd,
+				    &hipzout);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
+			 "shca=%p e_origmr=%p e_newmr=%p iova_start=%p acl=%x "
+			 "e_pd=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
+			 h_ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd,
+			 shca->ipz_hca_handle.handle,
+			 e_origmr->ipz_mr_handle.handle,
+			 e_origmr->ib.ib_mr.lkey);
+		ret = ehca_mrmw_map_hrc_reg_smr(h_ret);
+		goto ehca_reg_smr_exit0;
+	}
+	/* successful registration */
+	e_newmr->num_pages     = e_origmr->num_pages;
+	e_newmr->num_4k        = e_origmr->num_4k;
+	e_newmr->start         = iova_start;
+	e_newmr->size          = e_origmr->size;
+	e_newmr->acl           = acl;
+	e_newmr->ipz_mr_handle = hipzout.handle;
+	*lkey = hipzout.lkey;
+	*rkey = hipzout.rkey;
+	return 0;
+
+ehca_reg_smr_exit0:
+	if (ret)
+		ehca_err(&shca->ib_device, "ret=%x shca=%p e_origmr=%p "
+			 "e_newmr=%p iova_start=%p acl=%x e_pd=%p",
+			 ret, shca, e_origmr, e_newmr, iova_start, acl, e_pd);
+	return ret;
+} /* end ehca_reg_smr() */
+
+/*----------------------------------------------------------------------*/
+
+/* register internal max-MR to internal SHCA */
+int ehca_reg_internal_maxmr(
+	struct ehca_shca *shca,
+	struct ehca_pd *e_pd,
+	struct ehca_mr **e_maxmr)  /*OUT*/
+{
+	int ret;
+	struct ehca_mr *e_mr;
+	u64 *iova_start;
+	u64 size_maxmr;
+	struct ehca_mr_pginfo pginfo={0,0,0,0,0,0,0,NULL,0,NULL,NULL,0,NULL,0};
+	struct ib_phys_buf ib_pbuf;
+	u32 num_pages_mr;
+	u32 num_pages_4k; /* 4k portion "pages" */
+
+	e_mr = ehca_mr_new();
+	if (!e_mr) {
+		ehca_err(&shca->ib_device, "out of memory");
+		ret = -ENOMEM;
+		goto ehca_reg_internal_maxmr_exit0;
+	}
+	e_mr->flags |= EHCA_MR_FLAG_MAXMR;
+
+	/* register internal max-MR on HCA */
+	size_maxmr = (u64)high_memory - PAGE_OFFSET;
+	iova_start = (u64*)KERNELBASE;
+	ib_pbuf.addr = 0;
+	ib_pbuf.size = size_maxmr;
+	num_pages_mr = ((((u64)iova_start % PAGE_SIZE) + size_maxmr +
+			 PAGE_SIZE - 1) / PAGE_SIZE);
+	num_pages_4k = ((((u64)iova_start % EHCA_PAGESIZE) + size_maxmr +
+			 EHCA_PAGESIZE - 1) / EHCA_PAGESIZE);
+
+	pginfo.type           = EHCA_MR_PGI_PHYS;
+	pginfo.num_pages      = num_pages_mr;
+	pginfo.num_4k         = num_pages_4k;
+	pginfo.num_phys_buf   = 1;
+	pginfo.phys_buf_array = &ib_pbuf;
+
+	ret = ehca_reg_mr(shca, e_mr, iova_start, size_maxmr, 0, e_pd,
+			  &pginfo, &e_mr->ib.ib_mr.lkey,
+			  &e_mr->ib.ib_mr.rkey);
+	if (ret) {
+		ehca_err(&shca->ib_device, "reg of internal max MR failed, "
+			 "e_mr=%p iova_start=%p size_maxmr=%lx num_pages_mr=%x "
+			 "num_pages_4k=%x", e_mr, iova_start, size_maxmr,
+			 num_pages_mr, num_pages_4k);
+		goto ehca_reg_internal_maxmr_exit1;
+	}
+
+	/* successful registration of all pages */
+	e_mr->ib.ib_mr.device = e_pd->ib_pd.device;
+	e_mr->ib.ib_mr.pd = &e_pd->ib_pd;
+	e_mr->ib.ib_mr.uobject = NULL;
+	atomic_inc(&(e_pd->ib_pd.usecnt));
+	atomic_set(&(e_mr->ib.ib_mr.usecnt), 0);
+	*e_maxmr = e_mr;
+	return 0;
+
+ehca_reg_internal_maxmr_exit1:
+	ehca_mr_delete(e_mr);
+ehca_reg_internal_maxmr_exit0:
+	if (ret)
+		ehca_err(&shca->ib_device, "ret=%x shca=%p e_pd=%p e_maxmr=%p",
+			 ret, shca, e_pd, e_maxmr);
+	return ret;
+} /* end ehca_reg_internal_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_reg_maxmr(struct ehca_shca *shca,
+		   struct ehca_mr *e_newmr,
+		   u64 *iova_start,
+		   int acl,
+		   struct ehca_pd *e_pd,
+		   u32 *lkey,
+		   u32 *rkey)
+{
+	u64 h_ret;
+	struct ehca_mr *e_origmr = shca->maxmr;
+	u32 hipz_acl;
+	struct ehca_mr_hipzout_parms hipzout = {{0},0,0,0,0,0};
+
+	ehca_mrmw_map_acl(acl, &hipz_acl);
+	ehca_mrmw_set_pgsize_hipz_acl(&hipz_acl);
+
+	h_ret = hipz_h_register_smr(shca->ipz_hca_handle, e_newmr, e_origmr,
+				    (u64)iova_start, hipz_acl, e_pd->fw_pd,
+				    &hipzout);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "hipz_reg_smr failed, h_ret=%lx "
+			 "e_origmr=%p hca_hndl=%lx mr_hndl=%lx lkey=%x",
+			 h_ret, e_origmr, shca->ipz_hca_handle.handle,
+			 e_origmr->ipz_mr_handle.handle,
+			 e_origmr->ib.ib_mr.lkey);
+		return ehca_mrmw_map_hrc_reg_smr(h_ret);
+	}
+	/* successful registration */
+	e_newmr->num_pages     = e_origmr->num_pages;
+	e_newmr->num_4k        = e_origmr->num_4k;
+	e_newmr->start         = iova_start;
+	e_newmr->size          = e_origmr->size;
+	e_newmr->acl           = acl;
+	e_newmr->ipz_mr_handle = hipzout.handle;
+	*lkey = hipzout.lkey;
+	*rkey = hipzout.rkey;
+	return 0;
+} /* end ehca_reg_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+int ehca_dereg_internal_maxmr(struct ehca_shca *shca)
+{
+	int ret;
+	struct ehca_mr *e_maxmr;
+	struct ib_pd *ib_pd;
+
+	if (!shca->maxmr) {
+		ehca_err(&shca->ib_device, "bad call, shca=%p", shca);
+		ret = -EINVAL;
+		goto ehca_dereg_internal_maxmr_exit0;
+	}
+
+	e_maxmr = shca->maxmr;
+	ib_pd = e_maxmr->ib.ib_mr.pd;
+	shca->maxmr = NULL; /* remove internal max-MR indication from SHCA */
+
+	ret = ehca_dereg_mr(&e_maxmr->ib.ib_mr);
+	if (ret) {
+		ehca_err(&shca->ib_device, "dereg internal max-MR failed, "
+			 "ret=%x e_maxmr=%p shca=%p lkey=%x",
+			 ret, e_maxmr, shca, e_maxmr->ib.ib_mr.lkey);
+		shca->maxmr = e_maxmr;
+		goto ehca_dereg_internal_maxmr_exit0;
+	}
+
+	atomic_dec(&ib_pd->usecnt);
+
+ehca_dereg_internal_maxmr_exit0:
+	if (ret)
+		ehca_err(&shca->ib_device, "ret=%x shca=%p shca->maxmr=%p",
+			 ret, shca, shca->maxmr);
+	return ret;
+} /* end ehca_dereg_internal_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * check physical buffer array of MR verbs for validness and
+ * calculates MR size
+ */
+int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
+				  int num_phys_buf,
+				  u64 *iova_start,
+				  u64 *size)
+{
+	struct ib_phys_buf *pbuf = phys_buf_array;
+	u64 size_count = 0;
+	u32 i;
+
+	if (num_phys_buf == 0) {
+		ehca_gen_err("bad phys buf array len, num_phys_buf=0");
+		return -EINVAL;
+	}
+	/* check first buffer */
+	if (((u64)iova_start & ~PAGE_MASK) != (pbuf->addr & ~PAGE_MASK)) {
+		ehca_gen_err("iova_start/addr mismatch, iova_start=%p "
+			     "pbuf->addr=%lx pbuf->size=%lx",
+			     iova_start, pbuf->addr, pbuf->size);
+		return -EINVAL;
+	}
+	if (((pbuf->addr + pbuf->size) % PAGE_SIZE) &&
+	    (num_phys_buf > 1)) {
+		ehca_gen_err("addr/size mismatch in 1st buf, pbuf->addr=%lx "
+			     "pbuf->size=%lx", pbuf->addr, pbuf->size);
+		return -EINVAL;
+	}
+
+	for (i = 0; i < num_phys_buf; i++) {
+		if ((i > 0) && (pbuf->addr % PAGE_SIZE)) {
+			ehca_gen_err("bad address, i=%x pbuf->addr=%lx "
+				     "pbuf->size=%lx",
+				     i, pbuf->addr, pbuf->size);
+			return -EINVAL;
+		}
+		if (((i > 0) &&	/* not 1st */
+		     (i < (num_phys_buf - 1)) &&	/* not last */
+		     (pbuf->size % PAGE_SIZE)) || (pbuf->size == 0)) {
+			ehca_gen_err("bad size, i=%x pbuf->size=%lx",
+				     i, pbuf->size);
+			return -EINVAL;
+		}
+		size_count += pbuf->size;
+		pbuf++;
+	}
+
+	*size = size_count;
+	return 0;
+} /* end ehca_mr_chk_buf_and_calc_size() */
+
+/*----------------------------------------------------------------------*/
+
+/* check page list of map FMR verb for validness */
+int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
+			     u64 *page_list,
+			     int list_len)
+{
+	u32 i;
+	u64 *page;
+
+	if ((list_len == 0) || (list_len > e_fmr->fmr_max_pages)) {
+		ehca_gen_err("bad list_len, list_len=%x "
+			     "e_fmr->fmr_max_pages=%x fmr=%p",
+			     list_len, e_fmr->fmr_max_pages, e_fmr);
+		return -EINVAL;
+	}
+
+	/* each page must be aligned */
+	page = page_list;
+	for (i = 0; i < list_len; i++) {
+		if (*page % e_fmr->fmr_page_size) {
+			ehca_gen_err("bad page, i=%x *page=%lx page=%p fmr=%p "
+				     "fmr_page_size=%x", i, *page, page, e_fmr,
+				     e_fmr->fmr_page_size);
+			return -EINVAL;
+		}
+		page++;
+	}
+
+	return 0;
+} /* end ehca_fmr_check_page_list() */
+
+/*----------------------------------------------------------------------*/
+
+/* setup page buffer from page info */
+int ehca_set_pagebuf(struct ehca_mr *e_mr,
+		     struct ehca_mr_pginfo *pginfo,
+		     u32 number,
+		     u64 *kpage)
+{
+	int ret = 0;
+	struct ib_umem_chunk *prev_chunk;
+	struct ib_umem_chunk *chunk;
+	struct ib_phys_buf *pbuf;
+	u64 *fmrlist;
+	u64 num4k, pgaddr, offs4k;
+	u32 i = 0;
+	u32 j = 0;
+
+	if (pginfo->type == EHCA_MR_PGI_PHYS) {
+		/* loop over desired phys_buf_array entries */
+		while (i < number) {
+			pbuf   = pginfo->phys_buf_array + pginfo->next_buf;
+			num4k  = ((pbuf->addr % EHCA_PAGESIZE) + pbuf->size +
+				  EHCA_PAGESIZE - 1) / EHCA_PAGESIZE;
+			offs4k = (pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE;
+			while (pginfo->next_4k < offs4k + num4k) {
+				/* sanity check */
+				if ((pginfo->page_cnt >= pginfo->num_pages) ||
+				    (pginfo->page_4k_cnt >= pginfo->num_4k)) {
+					ehca_gen_err("page_cnt >= num_pages, "
+						     "page_cnt=%lx "
+						     "num_pages=%lx "
+						     "page_4k_cnt=%lx "
+						     "num_4k=%lx i=%x",
+						     pginfo->page_cnt,
+						     pginfo->num_pages,
+						     pginfo->page_4k_cnt,
+						     pginfo->num_4k, i);
+					ret = -EFAULT;
+					goto ehca_set_pagebuf_exit0;
+				}
+				*kpage = phys_to_abs(
+					(pbuf->addr & EHCA_PAGEMASK)
+					+ (pginfo->next_4k * EHCA_PAGESIZE));
+				if ( !(*kpage) && pbuf->addr ) {
+					ehca_gen_err("pbuf->addr=%lx "
+						     "pbuf->size=%lx "
+						     "next_4k=%lx", pbuf->addr,
+						     pbuf->size,
+						     pginfo->next_4k);
+					ret = -EFAULT;
+					goto ehca_set_pagebuf_exit0;
+				}
+				(pginfo->page_4k_cnt)++;
+				(pginfo->next_4k)++;
+				if (pginfo->next_4k %
+				    (PAGE_SIZE / EHCA_PAGESIZE) == 0)
+					(pginfo->page_cnt)++;
+				kpage++;
+				i++;
+				if (i >= number) break;
+			}
+			if (pginfo->next_4k >= offs4k + num4k) {
+				(pginfo->next_buf)++;
+				pginfo->next_4k = 0;
+			}
+		}
+	} else if (pginfo->type == EHCA_MR_PGI_USER) {
+		/* loop over desired chunk entries */
+		chunk      = pginfo->next_chunk;
+		prev_chunk = pginfo->next_chunk;
+		list_for_each_entry_continue(chunk,
+					     (&(pginfo->region->chunk_list)),
+					     list) {
+			for (i = pginfo->next_nmap; i < chunk->nmap; ) {
+				pgaddr = ( page_to_pfn(chunk->page_list[i].page)
+					   << PAGE_SHIFT );
+				*kpage = phys_to_abs(pgaddr +
+						     (pginfo->next_4k *
+						      EHCA_PAGESIZE));
+				if ( !(*kpage) ) {
+					ehca_gen_err("pgaddr=%lx "
+						     "chunk->page_list[i]=%lx "
+						     "i=%x next_4k=%lx mr=%p",
+						     pgaddr,
+						     (u64)sg_dma_address(
+							     &chunk->
+							     page_list[i]),
+						     i, pginfo->next_4k, e_mr);
+					ret = -EFAULT;
+					goto ehca_set_pagebuf_exit0;
+				}
+				(pginfo->page_4k_cnt)++;
+				(pginfo->next_4k)++;
+				kpage++;
+				if (pginfo->next_4k %
+				    (PAGE_SIZE / EHCA_PAGESIZE) == 0) {
+					(pginfo->page_cnt)++;
+					(pginfo->next_nmap)++;
+					pginfo->next_4k = 0;
+					i++;
+				}
+				j++;
+				if (j >= number) break;
+			}
+			if ((pginfo->next_nmap >= chunk->nmap) &&
+			    (j >= number)) {
+				pginfo->next_nmap = 0;
+				prev_chunk = chunk;
+				break;
+			} else if (pginfo->next_nmap >= chunk->nmap) {
+				pginfo->next_nmap = 0;
+				prev_chunk = chunk;
+			} else if (j >= number)
+				break;
+			else
+				prev_chunk = chunk;
+		}
+		pginfo->next_chunk =
+			list_prepare_entry(prev_chunk,
+					   (&(pginfo->region->chunk_list)),
+					   list);
+	} else if (pginfo->type == EHCA_MR_PGI_FMR) {
+		/* loop over desired page_list entries */
+		fmrlist = pginfo->page_list + pginfo->next_listelem;
+		for (i = 0; i < number; i++) {
+			*kpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) +
+					     pginfo->next_4k * EHCA_PAGESIZE);
+			if ( !(*kpage) ) {
+				ehca_gen_err("*fmrlist=%lx fmrlist=%p "
+					     "next_listelem=%lx next_4k=%lx",
+					     *fmrlist, fmrlist,
+					     pginfo->next_listelem,
+					     pginfo->next_4k);
+				ret = -EFAULT;
+				goto ehca_set_pagebuf_exit0;
+			}
+			(pginfo->page_4k_cnt)++;
+			(pginfo->next_4k)++;
+			kpage++;
+			if (pginfo->next_4k %
+			    (e_mr->fmr_page_size / EHCA_PAGESIZE) == 0) {
+				(pginfo->page_cnt)++;
+				(pginfo->next_listelem)++;
+				fmrlist++;
+				pginfo->next_4k = 0;
+			}
+		}
+	} else {
+		ehca_gen_err("bad pginfo->type=%x", pginfo->type);
+		ret = -EFAULT;
+		goto ehca_set_pagebuf_exit0;
+	}
+
+ehca_set_pagebuf_exit0:
+	if (ret)
+		ehca_gen_err("ret=%x e_mr=%p pginfo=%p type=%x num_pages=%lx "
+			     "num_4k=%lx next_buf=%lx next_4k=%lx number=%x "
+			     "kpage=%p page_cnt=%lx page_4k_cnt=%lx i=%x "
+			     "next_listelem=%lx region=%p next_chunk=%p "
+			     "next_nmap=%lx", ret, e_mr, pginfo, pginfo->type,
+			     pginfo->num_pages, pginfo->num_4k,
+			     pginfo->next_buf, pginfo->next_4k, number, kpage,
+			     pginfo->page_cnt, pginfo->page_4k_cnt, i,
+			     pginfo->next_listelem, pginfo->region,
+			     pginfo->next_chunk, pginfo->next_nmap);
+	return ret;
+} /* end ehca_set_pagebuf() */
+
+/*----------------------------------------------------------------------*/
+
+/* setup 1 page from page info page buffer */
+int ehca_set_pagebuf_1(struct ehca_mr *e_mr,
+		       struct ehca_mr_pginfo *pginfo,
+		       u64 *rpage)
+{
+	int ret = 0;
+	struct ib_phys_buf *tmp_pbuf;
+	u64 *fmrlist;
+	struct ib_umem_chunk *chunk;
+	struct ib_umem_chunk *prev_chunk;
+	u64 pgaddr, num4k, offs4k;
+
+	if (pginfo->type == EHCA_MR_PGI_PHYS) {
+		/* sanity check */
+		if ((pginfo->page_cnt >= pginfo->num_pages) ||
+		    (pginfo->page_4k_cnt >= pginfo->num_4k)) {
+			ehca_gen_err("page_cnt >= num_pages, page_cnt=%lx "
+				     "num_pages=%lx page_4k_cnt=%lx num_4k=%lx",
+				     pginfo->page_cnt, pginfo->num_pages,
+				     pginfo->page_4k_cnt, pginfo->num_4k);
+			ret = -EFAULT;
+			goto ehca_set_pagebuf_1_exit0;
+		}
+		tmp_pbuf = pginfo->phys_buf_array + pginfo->next_buf;
+		num4k  = ((tmp_pbuf->addr % EHCA_PAGESIZE) + tmp_pbuf->size +
+			  EHCA_PAGESIZE - 1) / EHCA_PAGESIZE;
+		offs4k = (tmp_pbuf->addr & ~PAGE_MASK) / EHCA_PAGESIZE;
+		*rpage = phys_to_abs((tmp_pbuf->addr & EHCA_PAGEMASK) +
+				     (pginfo->next_4k * EHCA_PAGESIZE));
+		if ( !(*rpage) && tmp_pbuf->addr ) {
+			ehca_gen_err("tmp_pbuf->addr=%lx"
+				     " tmp_pbuf->size=%lx next_4k=%lx",
+				     tmp_pbuf->addr, tmp_pbuf->size,
+				     pginfo->next_4k);
+			ret = -EFAULT;
+			goto ehca_set_pagebuf_1_exit0;
+		}
+		(pginfo->page_4k_cnt)++;
+		(pginfo->next_4k)++;
+		if (pginfo->next_4k % (PAGE_SIZE / EHCA_PAGESIZE) == 0)
+			(pginfo->page_cnt)++;
+		if (pginfo->next_4k >= offs4k + num4k) {
+			(pginfo->next_buf)++;
+			pginfo->next_4k = 0;
+		}
+	} else if (pginfo->type == EHCA_MR_PGI_USER) {
+		chunk      = pginfo->next_chunk;
+		prev_chunk = pginfo->next_chunk;
+		list_for_each_entry_continue(chunk,
+					     (&(pginfo->region->chunk_list)),
+					     list) {
+			pgaddr = ( page_to_pfn(chunk->page_list[
+						       pginfo->next_nmap].page)
+				   << PAGE_SHIFT);
+			*rpage = phys_to_abs(pgaddr +
+					     (pginfo->next_4k * EHCA_PAGESIZE));
+			if ( !(*rpage) ) {
+				ehca_gen_err("pgaddr=%lx chunk->page_list[]=%lx"
+					     " next_nmap=%lx next_4k=%lx mr=%p",
+					     pgaddr, (u64)sg_dma_address(
+						     &chunk->page_list[
+							     pginfo->
+							     next_nmap]),
+					     pginfo->next_nmap, pginfo->next_4k,
+					     e_mr);
+				ret = -EFAULT;
+				goto ehca_set_pagebuf_1_exit0;
+			}
+			(pginfo->page_4k_cnt)++;
+			(pginfo->next_4k)++;
+			if (pginfo->next_4k %
+			    (PAGE_SIZE / EHCA_PAGESIZE) == 0) {
+				(pginfo->page_cnt)++;
+				(pginfo->next_nmap)++;
+				pginfo->next_4k = 0;
+			}
+			if (pginfo->next_nmap >= chunk->nmap) {
+				pginfo->next_nmap = 0;
+				prev_chunk = chunk;
+			}
+			break;
+		}
+		pginfo->next_chunk =
+			list_prepare_entry(prev_chunk,
+					   (&(pginfo->region->chunk_list)),
+					   list);
+	} else if (pginfo->type == EHCA_MR_PGI_FMR) {
+		fmrlist = pginfo->page_list + pginfo->next_listelem;
+		*rpage = phys_to_abs((*fmrlist & EHCA_PAGEMASK) +
+				     pginfo->next_4k * EHCA_PAGESIZE);
+		if ( !(*rpage) ) {
+			ehca_gen_err("*fmrlist=%lx fmrlist=%p "
+				     "next_listelem=%lx next_4k=%lx",
+				     *fmrlist, fmrlist, pginfo->next_listelem,
+				     pginfo->next_4k);
+			ret = -EFAULT;
+			goto ehca_set_pagebuf_1_exit0;
+		}
+		(pginfo->page_4k_cnt)++;
+		(pginfo->next_4k)++;
+		if (pginfo->next_4k %
+		    (e_mr->fmr_page_size / EHCA_PAGESIZE) == 0) {
+			(pginfo->page_cnt)++;
+			(pginfo->next_listelem)++;
+			pginfo->next_4k = 0;
+		}
+	} else {
+		ehca_gen_err("bad pginfo->type=%x", pginfo->type);
+		ret = -EFAULT;
+		goto ehca_set_pagebuf_1_exit0;
+	}
+
+ehca_set_pagebuf_1_exit0:
+	if (ret)
+		ehca_gen_err("ret=%x e_mr=%p pginfo=%p type=%x num_pages=%lx "
+			     "num_4k=%lx next_buf=%lx next_4k=%lx rpage=%p "
+			     "page_cnt=%lx page_4k_cnt=%lx next_listelem=%lx "
+			     "region=%p next_chunk=%p next_nmap=%lx", ret, e_mr,
+			     pginfo, pginfo->type, pginfo->num_pages,
+			     pginfo->num_4k, pginfo->next_buf, pginfo->next_4k,
+			     rpage, pginfo->page_cnt, pginfo->page_4k_cnt,
+			     pginfo->next_listelem, pginfo->region,
+			     pginfo->next_chunk, pginfo->next_nmap);
+	return ret;
+} /* end ehca_set_pagebuf_1() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * check MR if it is a max-MR, i.e. uses whole memory
+ * in case it's a max-MR 1 is returned, else 0
+ */
+int ehca_mr_is_maxmr(u64 size,
+		     u64 *iova_start)
+{
+	/* a MR is treated as max-MR only if it fits following: */
+	if ((size == ((u64)high_memory - PAGE_OFFSET)) &&
+	    (iova_start == (void*)KERNELBASE)) {
+		ehca_gen_dbg("this is a max-MR");
+		return 1;
+	} else
+		return 0;
+} /* end ehca_mr_is_maxmr() */
+
+/*----------------------------------------------------------------------*/
+
+/* map access control for MR/MW. This routine is used for MR and MW. */
+void ehca_mrmw_map_acl(int ib_acl,
+		       u32 *hipz_acl)
+{
+	*hipz_acl = 0;
+	if (ib_acl & IB_ACCESS_REMOTE_READ)
+		*hipz_acl |= HIPZ_ACCESSCTRL_R_READ;
+	if (ib_acl & IB_ACCESS_REMOTE_WRITE)
+		*hipz_acl |= HIPZ_ACCESSCTRL_R_WRITE;
+	if (ib_acl & IB_ACCESS_REMOTE_ATOMIC)
+		*hipz_acl |= HIPZ_ACCESSCTRL_R_ATOMIC;
+	if (ib_acl & IB_ACCESS_LOCAL_WRITE)
+		*hipz_acl |= HIPZ_ACCESSCTRL_L_WRITE;
+	if (ib_acl & IB_ACCESS_MW_BIND)
+		*hipz_acl |= HIPZ_ACCESSCTRL_MW_BIND;
+} /* end ehca_mrmw_map_acl() */
+
+/*----------------------------------------------------------------------*/
+
+/* sets page size in hipz access control for MR/MW. */
+void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl) /*INOUT*/
+{
+	return; /* HCA supports only 4k */
+} /* end ehca_mrmw_set_pgsize_hipz_acl() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * reverse map access control for MR/MW.
+ * This routine is used for MR and MW.
+ */
+void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
+			       int *ib_acl) /*OUT*/
+{
+	*ib_acl = 0;
+	if (*hipz_acl & HIPZ_ACCESSCTRL_R_READ)
+		*ib_acl |= IB_ACCESS_REMOTE_READ;
+	if (*hipz_acl & HIPZ_ACCESSCTRL_R_WRITE)
+		*ib_acl |= IB_ACCESS_REMOTE_WRITE;
+	if (*hipz_acl & HIPZ_ACCESSCTRL_R_ATOMIC)
+		*ib_acl |= IB_ACCESS_REMOTE_ATOMIC;
+	if (*hipz_acl & HIPZ_ACCESSCTRL_L_WRITE)
+		*ib_acl |= IB_ACCESS_LOCAL_WRITE;
+	if (*hipz_acl & HIPZ_ACCESSCTRL_MW_BIND)
+		*ib_acl |= IB_ACCESS_MW_BIND;
+} /* end ehca_mrmw_reverse_map_acl() */
+
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * map HIPZ rc to IB retcodes for MR/MW allocations
+ * Used for hipz_mr_reg_alloc and hipz_mw_alloc.
+ */
+int ehca_mrmw_map_hrc_alloc(const u64 hipz_rc)
+{
+	switch (hipz_rc) {
+	case H_SUCCESS:	             /* successful completion */
+		return 0;
+	case H_ADAPTER_PARM:         /* invalid adapter handle */
+	case H_RT_PARM:              /* invalid resource type */
+	case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
+	case H_MLENGTH_PARM:         /* invalid memory length */
+	case H_MEM_ACCESS_PARM:      /* invalid access controls */
+	case H_CONSTRAINED:          /* resource constraint */
+		return -EINVAL;
+	case H_BUSY:                 /* long busy */
+		return -EBUSY;
+	default:
+		return -EINVAL;
+	}
+} /* end ehca_mrmw_map_hrc_alloc() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * map HIPZ rc to IB retcodes for MR register rpage
+ * Used for hipz_h_register_rpage_mr at registering last page
+ */
+int ehca_mrmw_map_hrc_rrpg_last(const u64 hipz_rc)
+{
+	switch (hipz_rc) {
+	case H_SUCCESS:         /* registration complete */
+		return 0;
+	case H_PAGE_REGISTERED:	/* page registered */
+	case H_ADAPTER_PARM:    /* invalid adapter handle */
+	case H_RH_PARM:         /* invalid resource handle */
+/*	case H_QT_PARM:            invalid queue type */
+	case H_PARAMETER:       /*
+				 * invalid logical address,
+				 * or count zero or greater 512
+				 */
+	case H_TABLE_FULL:      /* page table full */
+	case H_HARDWARE:        /* HCA not operational */
+		return -EINVAL;
+	case H_BUSY:            /* long busy */
+		return -EBUSY;
+	default:
+		return -EINVAL;
+	}
+} /* end ehca_mrmw_map_hrc_rrpg_last() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * map HIPZ rc to IB retcodes for MR register rpage
+ * Used for hipz_h_register_rpage_mr at registering one page, but not last page
+ */
+int ehca_mrmw_map_hrc_rrpg_notlast(const u64 hipz_rc)
+{
+	switch (hipz_rc) {
+	case H_PAGE_REGISTERED:	/* page registered */
+		return 0;
+	case H_SUCCESS:         /* registration complete */
+	case H_ADAPTER_PARM:    /* invalid adapter handle */
+	case H_RH_PARM:         /* invalid resource handle */
+/*	case H_QT_PARM:            invalid queue type */
+	case H_PARAMETER:       /*
+				 * invalid logical address,
+				 * or count zero or greater 512
+				 */
+	case H_TABLE_FULL:      /* page table full */
+	case H_HARDWARE:        /* HCA not operational */
+		return -EINVAL;
+	case H_BUSY:            /* long busy */
+		return -EBUSY;
+	default:
+		return -EINVAL;
+	}
+} /* end ehca_mrmw_map_hrc_rrpg_notlast() */
+
+/*----------------------------------------------------------------------*/
+
+/* map HIPZ rc to IB retcodes for MR query. Used for hipz_mr_query. */
+int ehca_mrmw_map_hrc_query_mr(const u64 hipz_rc)
+{
+	switch (hipz_rc) {
+	case H_SUCCESS:	             /* successful completion */
+		return 0;
+	case H_ADAPTER_PARM:         /* invalid adapter handle */
+	case H_RH_PARM:              /* invalid resource handle */
+		return -EINVAL;
+	case H_BUSY:                 /* long busy */
+		return -EBUSY;
+	default:
+		return -EINVAL;
+	}
+} /* end ehca_mrmw_map_hrc_query_mr() */
+
+/*----------------------------------------------------------------------*/
+/*----------------------------------------------------------------------*/
+
+/*
+ * map HIPZ rc to IB retcodes for freeing MR resource
+ * Used for hipz_h_free_resource_mr
+ */
+int ehca_mrmw_map_hrc_free_mr(const u64 hipz_rc)
+{
+	switch (hipz_rc) {
+	case H_SUCCESS:      /* resource freed */
+		return 0;
+	case H_ADAPTER_PARM: /* invalid adapter handle */
+	case H_RH_PARM:      /* invalid resource handle */
+	case H_R_STATE:      /* invalid resource state */
+	case H_HARDWARE:     /* HCA not operational */
+		return -EINVAL;
+	case H_RESOURCE:     /* Resource in use */
+	case H_BUSY:         /* long busy */
+		return -EBUSY;
+	default:
+		return -EINVAL;
+	}
+} /* end ehca_mrmw_map_hrc_free_mr() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * map HIPZ rc to IB retcodes for freeing MW resource
+ * Used for hipz_h_free_resource_mw
+ */
+int ehca_mrmw_map_hrc_free_mw(const u64 hipz_rc)
+{
+	switch (hipz_rc) {
+	case H_SUCCESS:	     /* resource freed */
+		return 0;
+	case H_ADAPTER_PARM: /* invalid adapter handle */
+	case H_RH_PARM:      /* invalid resource handle */
+	case H_R_STATE:      /* invalid resource state */
+	case H_HARDWARE:     /* HCA not operational */
+		return -EINVAL;
+	case H_RESOURCE:     /* Resource in use */
+	case H_BUSY:         /* long busy */
+		return -EBUSY;
+	default:
+		return -EINVAL;
+	}
+} /* end ehca_mrmw_map_hrc_free_mw() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * map HIPZ rc to IB retcodes for SMR registrations
+ * Used for hipz_h_register_smr.
+ */
+int ehca_mrmw_map_hrc_reg_smr(const u64 hipz_rc)
+{
+	switch (hipz_rc) {
+	case H_SUCCESS:	             /* successful completion */
+		return 0;
+	case H_ADAPTER_PARM:         /* invalid adapter handle */
+	case H_RH_PARM:              /* invalid resource handle */
+	case H_MEM_PARM:             /* invalid MR virtual address */
+	case H_MEM_ACCESS_PARM:      /* invalid access controls */
+	case H_NOT_ENOUGH_RESOURCES: /* insufficient resources */
+		return -EINVAL;
+	case H_BUSY:                 /* long busy */
+		return -EBUSY;
+	default:
+		return -EINVAL;
+	}
+} /* end ehca_mrmw_map_hrc_reg_smr() */
+
+/*----------------------------------------------------------------------*/
+
+/*
+ * MR destructor and constructor
+ * used in Reregister MR verb, sets all fields in ehca_mr_t to 0,
+ * except struct ib_mr and spinlock
+ */
+void ehca_mr_deletenew(struct ehca_mr *mr)
+{
+	mr->flags         = 0;
+	mr->num_pages     = 0;
+	mr->num_4k        = 0;
+	mr->acl           = 0;
+	mr->start         = NULL;
+	mr->fmr_page_size = 0;
+	mr->fmr_max_pages = 0;
+	mr->fmr_max_maps  = 0;
+	mr->fmr_map_cnt   = 0;
+	memset(&mr->ipz_mr_handle, 0, sizeof(mr->ipz_mr_handle));
+	memset(&mr->galpas, 0, sizeof(mr->galpas));
+	mr->nr_of_pages   = 0;
+	mr->pagearray     = NULL;
+} /* end ehca_mr_deletenew() */
+
+int ehca_init_mrmw_cache(void)
+{
+	mr_cache = kmem_cache_create("ehca_cache_mr",
+				     sizeof(struct ehca_mr), 0,
+				     SLAB_HWCACHE_ALIGN,
+				     NULL, NULL);
+	if (!mr_cache)
+		return -ENOMEM;
+	mw_cache = kmem_cache_create("ehca_cache_mw",
+				     sizeof(struct ehca_mw), 0,
+				     SLAB_HWCACHE_ALIGN,
+				     NULL, NULL);
+	if (!mw_cache) {
+		kmem_cache_destroy(mr_cache);
+		mr_cache = NULL;
+		return -ENOMEM;
+	}
+	return 0;
+}
+
+void ehca_cleanup_mrmw_cache(void)
+{
+	if (mr_cache)
+		kmem_cache_destroy(mr_cache);
+	if (mw_cache)
+		kmem_cache_destroy(mw_cache);
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_mrmw.h b/drivers/infiniband/hw/ehca/ehca_mrmw.h
new file mode 100644
index 0000000..d936e40
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_mrmw.h
@@ -0,0 +1,140 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  MR/MW declarations and inline functions
+ *
+ *  Authors: Dietmar Decker <ddecker@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef _EHCA_MRMW_H_
+#define _EHCA_MRMW_H_
+
+int ehca_reg_mr(struct ehca_shca *shca,
+		struct ehca_mr *e_mr,
+		u64 *iova_start,
+		u64 size,
+		int acl,
+		struct ehca_pd *e_pd,
+		struct ehca_mr_pginfo *pginfo,
+		u32 *lkey,
+		u32 *rkey);
+
+int ehca_reg_mr_rpages(struct ehca_shca *shca,
+		       struct ehca_mr *e_mr,
+		       struct ehca_mr_pginfo *pginfo);
+
+int ehca_rereg_mr(struct ehca_shca *shca,
+		  struct ehca_mr *e_mr,
+		  u64 *iova_start,
+		  u64 size,
+		  int mr_access_flags,
+		  struct ehca_pd *e_pd,
+		  struct ehca_mr_pginfo *pginfo,
+		  u32 *lkey,
+		  u32 *rkey);
+
+int ehca_unmap_one_fmr(struct ehca_shca *shca,
+		       struct ehca_mr *e_fmr);
+
+int ehca_reg_smr(struct ehca_shca *shca,
+		 struct ehca_mr *e_origmr,
+		 struct ehca_mr *e_newmr,
+		 u64 *iova_start,
+		 int acl,
+		 struct ehca_pd *e_pd,
+		 u32 *lkey,
+		 u32 *rkey);
+
+int ehca_reg_internal_maxmr(struct ehca_shca *shca,
+			    struct ehca_pd *e_pd,
+			    struct ehca_mr **maxmr);
+
+int ehca_reg_maxmr(struct ehca_shca *shca,
+		   struct ehca_mr *e_newmr,
+		   u64 *iova_start,
+		   int acl,
+		   struct ehca_pd *e_pd,
+		   u32 *lkey,
+		   u32 *rkey);
+
+int ehca_dereg_internal_maxmr(struct ehca_shca *shca);
+
+int ehca_mr_chk_buf_and_calc_size(struct ib_phys_buf *phys_buf_array,
+				  int num_phys_buf,
+				  u64 *iova_start,
+				  u64 *size);
+
+int ehca_fmr_check_page_list(struct ehca_mr *e_fmr,
+			     u64 *page_list,
+			     int list_len);
+
+int ehca_set_pagebuf(struct ehca_mr *e_mr,
+		     struct ehca_mr_pginfo *pginfo,
+		     u32 number,
+		     u64 *kpage);
+
+int ehca_set_pagebuf_1(struct ehca_mr *e_mr,
+		       struct ehca_mr_pginfo *pginfo,
+		       u64 *rpage);
+
+int ehca_mr_is_maxmr(u64 size,
+		     u64 *iova_start);
+
+void ehca_mrmw_map_acl(int ib_acl,
+		       u32 *hipz_acl);
+
+void ehca_mrmw_set_pgsize_hipz_acl(u32 *hipz_acl);
+
+void ehca_mrmw_reverse_map_acl(const u32 *hipz_acl,
+			       int *ib_acl);
+
+int ehca_mrmw_map_hrc_alloc(const u64 hipz_rc);
+
+int ehca_mrmw_map_hrc_rrpg_last(const u64 hipz_rc);
+
+int ehca_mrmw_map_hrc_rrpg_notlast(const u64 hipz_rc);
+
+int ehca_mrmw_map_hrc_query_mr(const u64 hipz_rc);
+
+int ehca_mrmw_map_hrc_free_mr(const u64 hipz_rc);
+
+int ehca_mrmw_map_hrc_free_mw(const u64 hipz_rc);
+
+int ehca_mrmw_map_hrc_reg_smr(const u64 hipz_rc);
+
+void ehca_mr_deletenew(struct ehca_mr *mr);
+
+#endif  /*_EHCA_MRMW_H_*/
diff --git a/drivers/infiniband/hw/ehca/ehca_pd.c b/drivers/infiniband/hw/ehca/ehca_pd.c
new file mode 100644
index 0000000..2c3cdc6
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_pd.c
@@ -0,0 +1,114 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  PD functions
+ *
+ *  Authors: Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/current.h>
+
+#include "ehca_tools.h"
+#include "ehca_iverbs.h"
+
+static struct kmem_cache *pd_cache;
+
+struct ib_pd *ehca_alloc_pd(struct ib_device *device,
+			    struct ib_ucontext *context, struct ib_udata *udata)
+{
+	struct ehca_pd *pd;
+
+	pd = kmem_cache_alloc(pd_cache, SLAB_KERNEL);
+	if (!pd) {
+		ehca_err(device, "device=%p context=%p out of memory",
+			 device, context);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	memset(pd, 0, sizeof(struct ehca_pd));
+	pd->ownpid = current->tgid;
+
+	/*
+	 * Kernel PD: when device = -1, 0
+	 * User   PD: when context != -1
+	 */
+	if (!context) {
+		/*
+		 * Kernel PDs after init reuses always
+		 * the one created in ehca_shca_reopen()
+		 */
+		struct ehca_shca *shca = container_of(device, struct ehca_shca,
+						      ib_device);
+		pd->fw_pd.value = shca->pd->fw_pd.value;
+	} else
+		pd->fw_pd.value = (u64)pd;
+
+	return &pd->ib_pd;
+}
+
+int ehca_dealloc_pd(struct ib_pd *pd)
+{
+	u32 cur_pid = current->tgid;
+	struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
+
+	if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
+	    my_pd->ownpid != cur_pid) {
+		ehca_err(pd->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		return -EINVAL;
+	}
+
+	kmem_cache_free(pd_cache,
+			container_of(pd, struct ehca_pd, ib_pd));
+
+	return 0;
+}
+
+int ehca_init_pd_cache(void)
+{
+	pd_cache = kmem_cache_create("ehca_cache_pd",
+				     sizeof(struct ehca_pd), 0,
+				     SLAB_HWCACHE_ALIGN,
+				     NULL, NULL);
+	if (!pd_cache)
+		return -ENOMEM;
+	return 0;
+}
+
+void ehca_cleanup_pd_cache(void)
+{
+	if (pd_cache)
+		kmem_cache_destroy(pd_cache);
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_qes.h b/drivers/infiniband/hw/ehca/ehca_qes.h
new file mode 100644
index 0000000..8707d29
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_qes.h
@@ -0,0 +1,259 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Hardware request structures
+ *
+ *  Authors: Waleri Fomin <fomin@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef _EHCA_QES_H_
+#define _EHCA_QES_H_
+
+#include "ehca_tools.h"
+
+/* virtual scatter gather entry to specify remote adresses with length */
+struct ehca_vsgentry {
+	u64 vaddr;
+	u32 lkey;
+	u32 length;
+};
+
+#define GRH_FLAG_MASK        EHCA_BMASK_IBM(7,7)
+#define GRH_IPVERSION_MASK   EHCA_BMASK_IBM(0,3)
+#define GRH_TCLASS_MASK      EHCA_BMASK_IBM(4,12)
+#define GRH_FLOWLABEL_MASK   EHCA_BMASK_IBM(13,31)
+#define GRH_PAYLEN_MASK      EHCA_BMASK_IBM(32,47)
+#define GRH_NEXTHEADER_MASK  EHCA_BMASK_IBM(48,55)
+#define GRH_HOPLIMIT_MASK    EHCA_BMASK_IBM(56,63)
+
+/*
+ * Unreliable Datagram Address Vector Format
+ * see IBTA Vol1 chapter 8.3 Global Routing Header
+ */
+struct ehca_ud_av {
+	u8 sl;
+	u8 lnh;
+	u16 dlid;
+	u8 reserved1;
+	u8 reserved2;
+	u8 reserved3;
+	u8 slid_path_bits;
+	u8 reserved4;
+	u8 ipd;
+	u8 reserved5;
+	u8 pmtu;
+	u32 reserved6;
+	u64 reserved7;
+	union {
+		struct {
+			u64 word_0; /* always set to 6  */
+			/*should be 0x1B for IB transport */
+			u64 word_1;
+			u64 word_2;
+			u64 word_3;
+			u64 word_4;
+		} grh;
+		struct {
+			u32 wd_0;
+			u32 wd_1;
+			/* DWord_1 --> SGID */
+
+			u32 sgid_wd3;
+			u32 sgid_wd2;
+
+			u32 sgid_wd1;
+			u32 sgid_wd0;
+			/* DWord_3 --> DGID */
+
+			u32 dgid_wd3;
+			u32 dgid_wd2;
+
+			u32 dgid_wd1;
+			u32 dgid_wd0;
+		} grh_l;
+	};
+};
+
+/* maximum number of sg entries allowed in a WQE */
+#define MAX_WQE_SG_ENTRIES 252
+
+#define WQE_OPTYPE_SEND             0x80
+#define WQE_OPTYPE_RDMAREAD         0x40
+#define WQE_OPTYPE_RDMAWRITE        0x20
+#define WQE_OPTYPE_CMPSWAP          0x10
+#define WQE_OPTYPE_FETCHADD         0x08
+#define WQE_OPTYPE_BIND             0x04
+
+#define WQE_WRFLAG_REQ_SIGNAL_COM   0x80
+#define WQE_WRFLAG_FENCE            0x40
+#define WQE_WRFLAG_IMM_DATA_PRESENT 0x20
+#define WQE_WRFLAG_SOLIC_EVENT      0x10
+
+#define WQEF_CACHE_HINT             0x80
+#define WQEF_CACHE_HINT_RD_WR       0x40
+#define WQEF_TIMED_WQE              0x20
+#define WQEF_PURGE                  0x08
+#define WQEF_HIGH_NIBBLE            0xF0
+
+#define MW_BIND_ACCESSCTRL_R_WRITE   0x40
+#define MW_BIND_ACCESSCTRL_R_READ    0x20
+#define MW_BIND_ACCESSCTRL_R_ATOMIC  0x10
+
+struct ehca_wqe {
+	u64 work_request_id;
+	u8 optype;
+	u8 wr_flag;
+	u16 pkeyi;
+	u8 wqef;
+	u8 nr_of_data_seg;
+	u16 wqe_provided_slid;
+	u32 destination_qp_number;
+	u32 resync_psn_sqp;
+	u32 local_ee_context_qkey;
+	u32 immediate_data;
+	union {
+		struct {
+			u64 remote_virtual_adress;
+			u32 rkey;
+			u32 reserved;
+			u64 atomic_1st_op_dma_len;
+			u64 atomic_2nd_op;
+			struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
+
+		} nud;
+		struct {
+			u64 ehca_ud_av_ptr;
+			u64 reserved1;
+			u64 reserved2;
+			u64 reserved3;
+			struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
+		} ud_avp;
+		struct {
+			struct ehca_ud_av ud_av;
+			struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES -
+						     2];
+		} ud_av;
+		struct {
+			u64 reserved0;
+			u64 reserved1;
+			u64 reserved2;
+			u64 reserved3;
+			struct ehca_vsgentry sg_list[MAX_WQE_SG_ENTRIES];
+		} all_rcv;
+
+		struct {
+			u64 reserved;
+			u32 rkey;
+			u32 old_rkey;
+			u64 reserved1;
+			u64 reserved2;
+			u64 virtual_address;
+			u32 reserved3;
+			u32 length;
+			u32 reserved4;
+			u16 reserved5;
+			u8 reserved6;
+			u8 lr_ctl;
+			u32 lkey;
+			u32 reserved7;
+			u64 reserved8;
+			u64 reserved9;
+			u64 reserved10;
+			u64 reserved11;
+		} bind;
+		struct {
+			u64 reserved12;
+			u64 reserved13;
+			u32 size;
+			u32 start;
+		} inline_data;
+	} u;
+
+};
+
+#define WC_SEND_RECEIVE EHCA_BMASK_IBM(0,0)
+#define WC_IMM_DATA     EHCA_BMASK_IBM(1,1)
+#define WC_GRH_PRESENT  EHCA_BMASK_IBM(2,2)
+#define WC_SE_BIT       EHCA_BMASK_IBM(3,3)
+#define WC_STATUS_ERROR_BIT 0x80000000
+#define WC_STATUS_REMOTE_ERROR_FLAGS 0x0000F800
+#define WC_STATUS_PURGE_BIT 0x10
+
+struct ehca_cqe {
+	u64 work_request_id;
+	u8 optype;
+	u8 w_completion_flags;
+	u16 reserved1;
+	u32 nr_bytes_transferred;
+	u32 immediate_data;
+	u32 local_qp_number;
+	u8 freed_resource_count;
+	u8 service_level;
+	u16 wqe_count;
+	u32 qp_token;
+	u32 qkey_ee_token;
+	u32 remote_qp_number;
+	u16 dlid;
+	u16 rlid;
+	u16 reserved2;
+	u16 pkey_index;
+	u32 cqe_timestamp;
+	u32 wqe_timestamp;
+	u8 wqe_timestamp_valid;
+	u8 reserved3;
+	u8 reserved4;
+	u8 cqe_flags;
+	u32 status;
+};
+
+struct ehca_eqe {
+	u64 entry;
+};
+
+struct ehca_mrte {
+	u64 starting_va;
+	u64 length; /* length of memory region in bytes*/
+	u32 pd;
+	u8 key_instance;
+	u8 pagesize;
+	u8 mr_control;
+	u8 local_remote_access_ctrl;
+	u8 reserved[0x20 - 0x18];
+	u64 at_pointer[4];
+};
+#endif /*_EHCA_QES_H_*/
diff --git a/drivers/infiniband/hw/ehca/ehca_qp.c b/drivers/infiniband/hw/ehca/ehca_qp.c
new file mode 100644
index 0000000..4394123
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_qp.c
@@ -0,0 +1,1507 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  QP functions
+ *
+ *  Authors: Waleri Fomin <fomin@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *           Heiko J Schick <schickhj@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <asm/current.h>
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+#include "hipz_fns.h"
+
+static struct kmem_cache *qp_cache;
+
+/*
+ * attributes not supported by query qp
+ */
+#define QP_ATTR_QUERY_NOT_SUPPORTED (IB_QP_MAX_DEST_RD_ATOMIC | \
+				     IB_QP_MAX_QP_RD_ATOMIC   | \
+				     IB_QP_ACCESS_FLAGS       | \
+				     IB_QP_EN_SQD_ASYNC_NOTIFY)
+
+/*
+ * ehca (internal) qp state values
+ */
+enum ehca_qp_state {
+	EHCA_QPS_RESET = 1,
+	EHCA_QPS_INIT = 2,
+	EHCA_QPS_RTR = 3,
+	EHCA_QPS_RTS = 5,
+	EHCA_QPS_SQD = 6,
+	EHCA_QPS_SQE = 8,
+	EHCA_QPS_ERR = 128
+};
+
+/*
+ * qp state transitions as defined by IB Arch Rel 1.1 page 431
+ */
+enum ib_qp_statetrans {
+	IB_QPST_ANY2RESET,
+	IB_QPST_ANY2ERR,
+	IB_QPST_RESET2INIT,
+	IB_QPST_INIT2RTR,
+	IB_QPST_INIT2INIT,
+	IB_QPST_RTR2RTS,
+	IB_QPST_RTS2SQD,
+	IB_QPST_RTS2RTS,
+	IB_QPST_SQD2RTS,
+	IB_QPST_SQE2RTS,
+	IB_QPST_SQD2SQD,
+	IB_QPST_MAX	/* nr of transitions, this must be last!!! */
+};
+
+/*
+ * ib2ehca_qp_state maps IB to ehca qp_state
+ * returns ehca qp state corresponding to given ib qp state
+ */
+static inline enum ehca_qp_state ib2ehca_qp_state(enum ib_qp_state ib_qp_state)
+{
+	switch (ib_qp_state) {
+	case IB_QPS_RESET:
+		return EHCA_QPS_RESET;
+	case IB_QPS_INIT:
+		return EHCA_QPS_INIT;
+	case IB_QPS_RTR:
+		return EHCA_QPS_RTR;
+	case IB_QPS_RTS:
+		return EHCA_QPS_RTS;
+	case IB_QPS_SQD:
+		return EHCA_QPS_SQD;
+	case IB_QPS_SQE:
+		return EHCA_QPS_SQE;
+	case IB_QPS_ERR:
+		return EHCA_QPS_ERR;
+	default:
+		ehca_gen_err("invalid ib_qp_state=%x", ib_qp_state);
+		return -EINVAL;
+	}
+}
+
+/*
+ * ehca2ib_qp_state maps ehca to IB qp_state
+ * returns ib qp state corresponding to given ehca qp state
+ */
+static inline enum ib_qp_state ehca2ib_qp_state(enum ehca_qp_state
+						ehca_qp_state)
+{
+	switch (ehca_qp_state) {
+	case EHCA_QPS_RESET:
+		return IB_QPS_RESET;
+	case EHCA_QPS_INIT:
+		return IB_QPS_INIT;
+	case EHCA_QPS_RTR:
+		return IB_QPS_RTR;
+	case EHCA_QPS_RTS:
+		return IB_QPS_RTS;
+	case EHCA_QPS_SQD:
+		return IB_QPS_SQD;
+	case EHCA_QPS_SQE:
+		return IB_QPS_SQE;
+	case EHCA_QPS_ERR:
+		return IB_QPS_ERR;
+	default:
+		ehca_gen_err("invalid ehca_qp_state=%x", ehca_qp_state);
+		return -EINVAL;
+	}
+}
+
+/*
+ * ehca_qp_type used as index for req_attr and opt_attr of
+ * struct ehca_modqp_statetrans
+ */
+enum ehca_qp_type {
+	QPT_RC = 0,
+	QPT_UC = 1,
+	QPT_UD = 2,
+	QPT_SQP = 3,
+	QPT_MAX
+};
+
+/*
+ * ib2ehcaqptype maps Ib to ehca qp_type
+ * returns ehca qp type corresponding to ib qp type
+ */
+static inline enum ehca_qp_type ib2ehcaqptype(enum ib_qp_type ibqptype)
+{
+	switch (ibqptype) {
+	case IB_QPT_SMI:
+	case IB_QPT_GSI:
+		return QPT_SQP;
+	case IB_QPT_RC:
+		return QPT_RC;
+	case IB_QPT_UC:
+		return QPT_UC;
+	case IB_QPT_UD:
+		return QPT_UD;
+	default:
+		ehca_gen_err("Invalid ibqptype=%x", ibqptype);
+		return -EINVAL;
+	}
+}
+
+static inline enum ib_qp_statetrans get_modqp_statetrans(int ib_fromstate,
+							 int ib_tostate)
+{
+	int index = -EINVAL;
+	switch (ib_tostate) {
+	case IB_QPS_RESET:
+		index = IB_QPST_ANY2RESET;
+		break;
+	case IB_QPS_INIT:
+		switch (ib_fromstate) {
+		case IB_QPS_RESET:
+			index = IB_QPST_RESET2INIT;
+			break;
+		case IB_QPS_INIT:
+			index = IB_QPST_INIT2INIT;
+			break;
+		}
+		break;
+	case IB_QPS_RTR:
+		if (ib_fromstate == IB_QPS_INIT)
+			index = IB_QPST_INIT2RTR;
+		break;
+	case IB_QPS_RTS:
+		switch (ib_fromstate) {
+		case IB_QPS_RTR:
+			index = IB_QPST_RTR2RTS;
+			break;
+		case IB_QPS_RTS:
+			index = IB_QPST_RTS2RTS;
+			break;
+		case IB_QPS_SQD:
+			index = IB_QPST_SQD2RTS;
+			break;
+		case IB_QPS_SQE:
+			index = IB_QPST_SQE2RTS;
+			break;
+		}
+		break;
+	case IB_QPS_SQD:
+		if (ib_fromstate == IB_QPS_RTS)
+			index = IB_QPST_RTS2SQD;
+		break;
+	case IB_QPS_SQE:
+		break;
+	case IB_QPS_ERR:
+		index = IB_QPST_ANY2ERR;
+		break;
+	default:
+		break;
+	}
+	return index;
+}
+
+enum ehca_service_type {
+	ST_RC = 0,
+	ST_UC = 1,
+	ST_RD = 2,
+	ST_UD = 3
+};
+
+/*
+ * ibqptype2servicetype returns hcp service type corresponding to given
+ * ib qp type used by create_qp()
+ */
+static inline int ibqptype2servicetype(enum ib_qp_type ibqptype)
+{
+	switch (ibqptype) {
+	case IB_QPT_SMI:
+	case IB_QPT_GSI:
+		return ST_UD;
+	case IB_QPT_RC:
+		return ST_RC;
+	case IB_QPT_UC:
+		return ST_UC;
+	case IB_QPT_UD:
+		return ST_UD;
+	case IB_QPT_RAW_IPV6:
+		return -EINVAL;
+	case IB_QPT_RAW_ETY:
+		return -EINVAL;
+	default:
+		ehca_gen_err("Invalid ibqptype=%x", ibqptype);
+		return -EINVAL;
+	}
+}
+
+/*
+ * init_qp_queues initializes/constructs r/squeue and registers queue pages.
+ */
+static inline int init_qp_queues(struct ehca_shca *shca,
+				 struct ehca_qp *my_qp,
+				 int nr_sq_pages,
+				 int nr_rq_pages,
+				 int swqe_size,
+				 int rwqe_size,
+				 int nr_send_sges, int nr_receive_sges)
+{
+	int ret, cnt, ipz_rc;
+	void *vpage;
+	u64 rpage, h_ret;
+	struct ib_device *ib_dev = &shca->ib_device;
+	struct ipz_adapter_handle ipz_hca_handle = shca->ipz_hca_handle;
+
+	ipz_rc = ipz_queue_ctor(&my_qp->ipz_squeue,
+				nr_sq_pages,
+				EHCA_PAGESIZE, swqe_size, nr_send_sges);
+	if (!ipz_rc) {
+		ehca_err(ib_dev,"Cannot allocate page for squeue. ipz_rc=%x",
+			 ipz_rc);
+		return -EBUSY;
+	}
+
+	ipz_rc = ipz_queue_ctor(&my_qp->ipz_rqueue,
+				nr_rq_pages,
+				EHCA_PAGESIZE, rwqe_size, nr_receive_sges);
+	if (!ipz_rc) {
+		ehca_err(ib_dev, "Cannot allocate page for rqueue. ipz_rc=%x",
+			 ipz_rc);
+		ret = -EBUSY;
+		goto init_qp_queues0;
+	}
+	/* register SQ pages */
+	for (cnt = 0; cnt < nr_sq_pages; cnt++) {
+		vpage = ipz_qpageit_get_inc(&my_qp->ipz_squeue);
+		if (!vpage) {
+			ehca_err(ib_dev, "SQ ipz_qpageit_get_inc() "
+				 "failed p_vpage= %p", vpage);
+			ret = -EINVAL;
+			goto init_qp_queues1;
+		}
+		rpage = virt_to_abs(vpage);
+
+		h_ret = hipz_h_register_rpage_qp(ipz_hca_handle,
+						 my_qp->ipz_qp_handle,
+						 &my_qp->pf, 0, 0,
+						 rpage, 1,
+						 my_qp->galpas.kernel);
+		if (h_ret < H_SUCCESS) {
+			ehca_err(ib_dev, "SQ hipz_qp_register_rpage()"
+				 " failed rc=%lx", h_ret);
+			ret = ehca2ib_return_code(h_ret);
+			goto init_qp_queues1;
+		}
+	}
+
+	ipz_qeit_reset(&my_qp->ipz_squeue);
+
+	/* register RQ pages */
+	for (cnt = 0; cnt < nr_rq_pages; cnt++) {
+		vpage = ipz_qpageit_get_inc(&my_qp->ipz_rqueue);
+		if (!vpage) {
+			ehca_err(ib_dev, "RQ ipz_qpageit_get_inc() "
+				 "failed p_vpage = %p", vpage);
+			ret = -EINVAL;
+			goto init_qp_queues1;
+		}
+
+		rpage = virt_to_abs(vpage);
+
+		h_ret = hipz_h_register_rpage_qp(ipz_hca_handle,
+						 my_qp->ipz_qp_handle,
+						 &my_qp->pf, 0, 1,
+						 rpage, 1,my_qp->galpas.kernel);
+		if (h_ret < H_SUCCESS) {
+			ehca_err(ib_dev, "RQ hipz_qp_register_rpage() failed "
+				 "rc=%lx", h_ret);
+			ret = ehca2ib_return_code(h_ret);
+			goto init_qp_queues1;
+		}
+		if (cnt == (nr_rq_pages - 1)) {	/* last page! */
+			if (h_ret != H_SUCCESS) {
+				ehca_err(ib_dev, "RQ hipz_qp_register_rpage() "
+					 "h_ret= %lx ", h_ret);
+				ret = ehca2ib_return_code(h_ret);
+				goto init_qp_queues1;
+			}
+			vpage = ipz_qpageit_get_inc(&my_qp->ipz_rqueue);
+			if (vpage) {
+				ehca_err(ib_dev, "ipz_qpageit_get_inc() "
+					 "should not succeed vpage=%p", vpage);
+				ret = -EINVAL;
+				goto init_qp_queues1;
+			}
+		} else {
+			if (h_ret != H_PAGE_REGISTERED) {
+				ehca_err(ib_dev, "RQ hipz_qp_register_rpage() "
+					 "h_ret= %lx ", h_ret);
+				ret = ehca2ib_return_code(h_ret);
+				goto init_qp_queues1;
+			}
+		}
+	}
+
+	ipz_qeit_reset(&my_qp->ipz_rqueue);
+
+	return 0;
+
+init_qp_queues1:
+	ipz_queue_dtor(&my_qp->ipz_rqueue);
+init_qp_queues0:
+	ipz_queue_dtor(&my_qp->ipz_squeue);
+	return ret;
+}
+
+struct ib_qp *ehca_create_qp(struct ib_pd *pd,
+			     struct ib_qp_init_attr *init_attr,
+			     struct ib_udata *udata)
+{
+	static int da_rc_msg_size[]={ 128, 256, 512, 1024, 2048, 4096 };
+	static int da_ud_sq_msg_size[]={ 128, 384, 896, 1920, 3968 };
+	struct ehca_qp *my_qp;
+	struct ehca_pd *my_pd = container_of(pd, struct ehca_pd, ib_pd);
+	struct ehca_shca *shca = container_of(pd->device, struct ehca_shca,
+					      ib_device);
+	struct ib_ucontext *context = NULL;
+	u64 h_ret;
+	int max_send_sge, max_recv_sge, ret;
+
+	/* h_call's out parameters */
+	struct ehca_alloc_qp_parms parms;
+	u32 swqe_size = 0, rwqe_size = 0;
+	u8 daqp_completion, isdaqp;
+	unsigned long flags;
+
+	if (init_attr->sq_sig_type != IB_SIGNAL_REQ_WR &&
+		init_attr->sq_sig_type != IB_SIGNAL_ALL_WR) {
+		ehca_err(pd->device, "init_attr->sg_sig_type=%x not allowed",
+			 init_attr->sq_sig_type);
+		return ERR_PTR(-EINVAL);
+	}
+
+	/* save daqp completion bits */
+	daqp_completion = init_attr->qp_type & 0x60;
+	/* save daqp bit */
+	isdaqp = (init_attr->qp_type & 0x80) ? 1 : 0;
+	init_attr->qp_type = init_attr->qp_type & 0x1F;
+
+	if (init_attr->qp_type != IB_QPT_UD &&
+	    init_attr->qp_type != IB_QPT_SMI &&
+	    init_attr->qp_type != IB_QPT_GSI &&
+	    init_attr->qp_type != IB_QPT_UC &&
+	    init_attr->qp_type != IB_QPT_RC) {
+		ehca_err(pd->device, "wrong QP Type=%x", init_attr->qp_type);
+		return ERR_PTR(-EINVAL);
+	}
+	if ((init_attr->qp_type != IB_QPT_RC && init_attr->qp_type != IB_QPT_UD)
+	    && isdaqp) {
+		ehca_err(pd->device, "unsupported LL QP Type=%x",
+			 init_attr->qp_type);
+		return ERR_PTR(-EINVAL);
+	} else if (init_attr->qp_type == IB_QPT_RC && isdaqp &&
+		   (init_attr->cap.max_send_wr > 255 ||
+		    init_attr->cap.max_recv_wr > 255 )) {
+		       ehca_err(pd->device, "Invalid Number of max_sq_wr =%x "
+				"or max_rq_wr=%x for QP Type=%x",
+				init_attr->cap.max_send_wr,
+				init_attr->cap.max_recv_wr,init_attr->qp_type);
+		       return ERR_PTR(-EINVAL);
+	} else if (init_attr->qp_type == IB_QPT_UD && isdaqp &&
+		  init_attr->cap.max_send_wr > 255) {
+		ehca_err(pd->device,
+			 "Invalid Number of max_send_wr=%x for UD QP_TYPE=%x",
+			 init_attr->cap.max_send_wr, init_attr->qp_type);
+		return ERR_PTR(-EINVAL);
+	}
+
+	if (pd->uobject && udata)
+		context = pd->uobject->context;
+
+	my_qp = kmem_cache_alloc(qp_cache, SLAB_KERNEL);
+	if (!my_qp) {
+		ehca_err(pd->device, "pd=%p not enough memory to alloc qp", pd);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	memset(my_qp, 0, sizeof(struct ehca_qp));
+	memset (&parms, 0, sizeof(struct ehca_alloc_qp_parms));
+	spin_lock_init(&my_qp->spinlock_s);
+	spin_lock_init(&my_qp->spinlock_r);
+
+	my_qp->recv_cq =
+		container_of(init_attr->recv_cq, struct ehca_cq, ib_cq);
+	my_qp->send_cq =
+		container_of(init_attr->send_cq, struct ehca_cq, ib_cq);
+
+	my_qp->init_attr = *init_attr;
+
+	do {
+		if (!idr_pre_get(&ehca_qp_idr, GFP_KERNEL)) {
+			ret = -ENOMEM;
+			ehca_err(pd->device, "Can't reserve idr resources.");
+			goto create_qp_exit0;
+		}
+
+		spin_lock_irqsave(&ehca_qp_idr_lock, flags);
+		ret = idr_get_new(&ehca_qp_idr, my_qp, &my_qp->token);
+		spin_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+	} while (ret == -EAGAIN);
+
+	if (ret) {
+		ret = -ENOMEM;
+		ehca_err(pd->device, "Can't allocate new idr entry.");
+		goto create_qp_exit0;
+	}
+
+	parms.servicetype = ibqptype2servicetype(init_attr->qp_type);
+	if (parms.servicetype < 0) {
+		ret = -EINVAL;
+		ehca_err(pd->device, "Invalid qp_type=%x", init_attr->qp_type);
+		goto create_qp_exit0;
+	}
+
+	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
+		parms.sigtype = HCALL_SIGT_EVERY;
+	else
+		parms.sigtype = HCALL_SIGT_BY_WQE;
+
+	/* UD_AV CIRCUMVENTION */
+	max_send_sge = init_attr->cap.max_send_sge;
+	max_recv_sge = init_attr->cap.max_recv_sge;
+	if (IB_QPT_UD == init_attr->qp_type ||
+	    IB_QPT_GSI == init_attr->qp_type ||
+	    IB_QPT_SMI == init_attr->qp_type) {
+		max_send_sge += 2;
+		max_recv_sge += 2;
+	}
+
+	parms.ipz_eq_handle = shca->eq.ipz_eq_handle;
+	parms.daqp_ctrl = isdaqp | daqp_completion;
+	parms.pd = my_pd->fw_pd;
+	parms.max_recv_sge = max_recv_sge;
+	parms.max_send_sge = max_send_sge;
+
+	h_ret = hipz_h_alloc_resource_qp(shca->ipz_hca_handle, my_qp, &parms);
+
+	if (h_ret != H_SUCCESS) {
+		ehca_err(pd->device, "h_alloc_resource_qp() failed h_ret=%lx",
+			 h_ret);
+		ret = ehca2ib_return_code(h_ret);
+		goto create_qp_exit1;
+	}
+
+	switch (init_attr->qp_type) {
+	case IB_QPT_RC:
+	        if (isdaqp == 0) {
+			swqe_size = offsetof(struct ehca_wqe, u.nud.sg_list[
+					     (parms.act_nr_send_sges)]);
+			rwqe_size = offsetof(struct ehca_wqe, u.nud.sg_list[
+					     (parms.act_nr_recv_sges)]);
+		} else { /* for daqp we need to use msg size, not wqe size */
+		        swqe_size = da_rc_msg_size[max_send_sge];
+			rwqe_size = da_rc_msg_size[max_recv_sge];
+			parms.act_nr_send_sges = 1;
+			parms.act_nr_recv_sges = 1;
+		}
+		break;
+	case IB_QPT_UC:
+		swqe_size = offsetof(struct ehca_wqe,
+				     u.nud.sg_list[parms.act_nr_send_sges]);
+		rwqe_size = offsetof(struct ehca_wqe,
+				     u.nud.sg_list[parms.act_nr_recv_sges]);
+		break;
+
+	case IB_QPT_UD:
+	case IB_QPT_GSI:
+	case IB_QPT_SMI:
+		/* UD circumvention */
+		parms.act_nr_recv_sges -= 2;
+		parms.act_nr_send_sges -= 2;
+		if (isdaqp) {
+		        swqe_size = da_ud_sq_msg_size[max_send_sge];
+			rwqe_size = da_rc_msg_size[max_recv_sge];
+			parms.act_nr_send_sges = 1;
+			parms.act_nr_recv_sges = 1;
+		} else {
+			swqe_size = offsetof(struct ehca_wqe,
+					     u.ud_av.sg_list[parms.act_nr_send_sges]);
+			rwqe_size = offsetof(struct ehca_wqe,
+					     u.ud_av.sg_list[parms.act_nr_recv_sges]);
+		}
+
+		if (IB_QPT_GSI == init_attr->qp_type ||
+		    IB_QPT_SMI == init_attr->qp_type) {
+			parms.act_nr_send_wqes = init_attr->cap.max_send_wr;
+			parms.act_nr_recv_wqes = init_attr->cap.max_recv_wr;
+			parms.act_nr_send_sges = init_attr->cap.max_send_sge;
+			parms.act_nr_recv_sges = init_attr->cap.max_recv_sge;
+			my_qp->real_qp_num =
+				(init_attr->qp_type == IB_QPT_SMI) ? 0 : 1;
+		}
+
+		break;
+
+	default:
+		break;
+	}
+
+	/* initializes r/squeue and registers queue pages */
+	ret = init_qp_queues(shca, my_qp,
+			     parms.nr_sq_pages, parms.nr_rq_pages,
+			     swqe_size, rwqe_size,
+			     parms.act_nr_send_sges, parms.act_nr_recv_sges);
+	if (ret) {
+		ehca_err(pd->device,
+			 "Couldn't initialize r/squeue and pages ret=%x", ret);
+		goto create_qp_exit2;
+	}
+
+	my_qp->ib_qp.pd = &my_pd->ib_pd;
+	my_qp->ib_qp.device = my_pd->ib_pd.device;
+
+	my_qp->ib_qp.recv_cq = init_attr->recv_cq;
+	my_qp->ib_qp.send_cq = init_attr->send_cq;
+
+	my_qp->ib_qp.qp_num = my_qp->real_qp_num;
+	my_qp->ib_qp.qp_type = init_attr->qp_type;
+
+	my_qp->qp_type = init_attr->qp_type;
+	my_qp->ib_qp.srq = init_attr->srq;
+
+	my_qp->ib_qp.qp_context = init_attr->qp_context;
+	my_qp->ib_qp.event_handler = init_attr->event_handler;
+
+	init_attr->cap.max_inline_data = 0; /* not supported yet */
+	init_attr->cap.max_recv_sge = parms.act_nr_recv_sges;
+	init_attr->cap.max_recv_wr = parms.act_nr_recv_wqes;
+	init_attr->cap.max_send_sge = parms.act_nr_send_sges;
+	init_attr->cap.max_send_wr = parms.act_nr_send_wqes;
+
+	/* NOTE: define_apq0() not supported yet */
+	if (init_attr->qp_type == IB_QPT_GSI) {
+		h_ret = ehca_define_sqp(shca, my_qp, init_attr);
+		if (h_ret != H_SUCCESS) {
+			ehca_err(pd->device, "ehca_define_sqp() failed rc=%lx",
+				 h_ret);
+			ret = ehca2ib_return_code(h_ret);
+			goto create_qp_exit3;
+		}
+	}
+	if (init_attr->send_cq) {
+		struct ehca_cq *cq = container_of(init_attr->send_cq,
+						  struct ehca_cq, ib_cq);
+		ret = ehca_cq_assign_qp(cq, my_qp);
+		if (ret) {
+			ehca_err(pd->device, "Couldn't assign qp to send_cq ret=%x",
+				 ret);
+			goto create_qp_exit3;
+		}
+		my_qp->send_cq = cq;
+	}
+	/* copy queues, galpa data to user space */
+	if (context && udata) {
+		struct ipz_queue *ipz_rqueue = &my_qp->ipz_rqueue;
+		struct ipz_queue *ipz_squeue = &my_qp->ipz_squeue;
+		struct ehca_create_qp_resp resp;
+		struct vm_area_struct * vma;
+		memset(&resp, 0, sizeof(resp));
+
+		resp.qp_num = my_qp->real_qp_num;
+		resp.token = my_qp->token;
+		resp.qp_type = my_qp->qp_type;
+		resp.qkey = my_qp->qkey;
+		resp.real_qp_num = my_qp->real_qp_num;
+		/* rqueue properties */
+		resp.ipz_rqueue.qe_size = ipz_rqueue->qe_size;
+		resp.ipz_rqueue.act_nr_of_sg = ipz_rqueue->act_nr_of_sg;
+		resp.ipz_rqueue.queue_length = ipz_rqueue->queue_length;
+		resp.ipz_rqueue.pagesize = ipz_rqueue->pagesize;
+		resp.ipz_rqueue.toggle_state = ipz_rqueue->toggle_state;
+		ret = ehca_mmap_nopage(((u64)(my_qp->token) << 32) | 0x22000000,
+				       ipz_rqueue->queue_length,
+				       (void**)&resp.ipz_rqueue.queue,
+				       &vma);
+		if (ret) {
+			ehca_err(pd->device, "Could not mmap rqueue pages");
+			goto create_qp_exit3;
+		}
+		my_qp->uspace_rqueue = resp.ipz_rqueue.queue;
+		/* squeue properties */
+		resp.ipz_squeue.qe_size = ipz_squeue->qe_size;
+		resp.ipz_squeue.act_nr_of_sg = ipz_squeue->act_nr_of_sg;
+		resp.ipz_squeue.queue_length = ipz_squeue->queue_length;
+		resp.ipz_squeue.pagesize = ipz_squeue->pagesize;
+		resp.ipz_squeue.toggle_state = ipz_squeue->toggle_state;
+		ret = ehca_mmap_nopage(((u64)(my_qp->token) << 32) | 0x23000000,
+				       ipz_squeue->queue_length,
+				       (void**)&resp.ipz_squeue.queue,
+				       &vma);
+		if (ret) {
+			ehca_err(pd->device, "Could not mmap squeue pages");
+			goto create_qp_exit4;
+		}
+		my_qp->uspace_squeue = resp.ipz_squeue.queue;
+		/* fw_handle */
+		resp.galpas = my_qp->galpas;
+		ret = ehca_mmap_register(my_qp->galpas.user.fw_handle,
+					 (void**)&resp.galpas.kernel.fw_handle,
+					 &vma);
+		if (ret) {
+			ehca_err(pd->device, "Could not mmap fw_handle");
+			goto create_qp_exit5;
+		}
+		my_qp->uspace_fwh = (u64)resp.galpas.kernel.fw_handle;
+
+		if (ib_copy_to_udata(udata, &resp, sizeof resp)) {
+			ehca_err(pd->device, "Copy to udata failed");
+			ret = -EINVAL;
+			goto create_qp_exit6;
+		}
+	}
+
+	return &my_qp->ib_qp;
+
+create_qp_exit6:
+	ehca_munmap(my_qp->uspace_fwh, EHCA_PAGESIZE);
+
+create_qp_exit5:
+	ehca_munmap(my_qp->uspace_squeue, my_qp->ipz_squeue.queue_length);
+
+create_qp_exit4:
+	ehca_munmap(my_qp->uspace_rqueue, my_qp->ipz_rqueue.queue_length);
+
+create_qp_exit3:
+	ipz_queue_dtor(&my_qp->ipz_rqueue);
+	ipz_queue_dtor(&my_qp->ipz_squeue);
+
+create_qp_exit2:
+	hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
+
+create_qp_exit1:
+	spin_lock_irqsave(&ehca_qp_idr_lock, flags);
+	idr_remove(&ehca_qp_idr, my_qp->token);
+	spin_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+create_qp_exit0:
+	kmem_cache_free(qp_cache, my_qp);
+	return ERR_PTR(ret);
+}
+
+/*
+ * prepare_sqe_rts called by internal_modify_qp() at trans sqe -> rts
+ * set purge bit of bad wqe and subsequent wqes to avoid reentering sqe
+ * returns total number of bad wqes in bad_wqe_cnt
+ */
+static int prepare_sqe_rts(struct ehca_qp *my_qp, struct ehca_shca *shca,
+			   int *bad_wqe_cnt)
+{
+	u64 h_ret;
+	struct ipz_queue *squeue;
+	void *bad_send_wqe_p, *bad_send_wqe_v;
+	void *squeue_start_p, *squeue_end_p;
+	void *squeue_start_v, *squeue_end_v;
+	struct ehca_wqe *wqe;
+	int qp_num = my_qp->ib_qp.qp_num;
+
+	/* get send wqe pointer */
+	h_ret = hipz_h_disable_and_get_wqe(shca->ipz_hca_handle,
+					   my_qp->ipz_qp_handle, &my_qp->pf,
+					   &bad_send_wqe_p, NULL, 2);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(&shca->ib_device, "hipz_h_disable_and_get_wqe() failed"
+			 " ehca_qp=%p qp_num=%x h_ret=%lx",
+			 my_qp, qp_num, h_ret);
+		return ehca2ib_return_code(h_ret);
+	}
+	bad_send_wqe_p = (void*)((u64)bad_send_wqe_p & (~(1L<<63)));
+	ehca_dbg(&shca->ib_device, "qp_num=%x bad_send_wqe_p=%p",
+		 qp_num, bad_send_wqe_p);
+	/* convert wqe pointer to vadr */
+	bad_send_wqe_v = abs_to_virt((u64)bad_send_wqe_p);
+	if (ehca_debug_level)
+		ehca_dmp(bad_send_wqe_v, 32, "qp_num=%x bad_wqe", qp_num);
+	squeue = &my_qp->ipz_squeue;
+	squeue_start_p = (void*)virt_to_abs(ipz_qeit_calc(squeue, 0L));
+	squeue_end_p = squeue_start_p+squeue->queue_length;
+	squeue_start_v = abs_to_virt((u64)squeue_start_p);
+	squeue_end_v = abs_to_virt((u64)squeue_end_p);
+	ehca_dbg(&shca->ib_device, "qp_num=%x squeue_start_v=%p squeue_end_v=%p",
+		 qp_num, squeue_start_v, squeue_end_v);
+
+	/* loop sets wqe's purge bit */
+	wqe = (struct ehca_wqe*)bad_send_wqe_v;
+	*bad_wqe_cnt = 0;
+	while (wqe->optype != 0xff && wqe->wqef != 0xff) {
+		if (ehca_debug_level)
+			ehca_dmp(wqe, 32, "qp_num=%x wqe", qp_num);
+		wqe->nr_of_data_seg = 0; /* suppress data access */
+		wqe->wqef = WQEF_PURGE; /* WQE to be purged */
+		wqe = (struct ehca_wqe*)((u8*)wqe+squeue->qe_size);
+		*bad_wqe_cnt = (*bad_wqe_cnt)+1;
+		if ((void*)wqe >= squeue_end_v) {
+			wqe = squeue_start_v;
+		}
+	}
+	/*
+	 * bad wqe will be reprocessed and ignored when pol_cq() is called,
+	 *  i.e. nr of wqes with flush error status is one less
+	 */
+	ehca_dbg(&shca->ib_device, "qp_num=%x flusherr_wqe_cnt=%x",
+		 qp_num, (*bad_wqe_cnt)-1);
+	wqe->wqef = 0;
+
+	return 0;
+}
+
+/*
+ * internal_modify_qp with circumvention to handle aqp0 properly
+ * smi_reset2init indicates if this is an internal reset-to-init-call for
+ * smi. This flag must always be zero if called from ehca_modify_qp()!
+ * This internal func was intorduced to avoid recursion of ehca_modify_qp()!
+ */
+static int internal_modify_qp(struct ib_qp *ibqp,
+			      struct ib_qp_attr *attr,
+			      int attr_mask, int smi_reset2init)
+{
+	enum ib_qp_state qp_cur_state, qp_new_state;
+	int cnt, qp_attr_idx, ret = 0;
+	enum ib_qp_statetrans statetrans;
+	struct hcp_modify_qp_control_block *mqpcb;
+	struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+	struct ehca_shca *shca =
+		container_of(ibqp->pd->device, struct ehca_shca, ib_device);
+	u64 update_mask;
+	u64 h_ret;
+	int bad_wqe_cnt = 0;
+	int squeue_locked = 0;
+	unsigned long spl_flags = 0;
+
+	/* do query_qp to obtain current attr values */
+	mqpcb = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL);
+	if (mqpcb == NULL) {
+		ehca_err(ibqp->device, "Could not get zeroed page for mqpcb "
+			 "ehca_qp=%p qp_num=%x ", my_qp, ibqp->qp_num);
+		return -ENOMEM;
+	}
+
+	h_ret = hipz_h_query_qp(shca->ipz_hca_handle,
+				my_qp->ipz_qp_handle,
+				&my_qp->pf,
+				mqpcb, my_qp->galpas.kernel);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(ibqp->device, "hipz_h_query_qp() failed "
+			 "ehca_qp=%p qp_num=%x h_ret=%lx",
+			 my_qp, ibqp->qp_num, h_ret);
+		ret = ehca2ib_return_code(h_ret);
+		goto modify_qp_exit1;
+	}
+
+	qp_cur_state = ehca2ib_qp_state(mqpcb->qp_state);
+
+	if (qp_cur_state == -EINVAL) {	/* invalid qp state */
+		ret = -EINVAL;
+		ehca_err(ibqp->device, "Invalid current ehca_qp_state=%x "
+			 "ehca_qp=%p qp_num=%x",
+			 mqpcb->qp_state, my_qp, ibqp->qp_num);
+		goto modify_qp_exit1;
+	}
+	/*
+	 * circumvention to set aqp0 initial state to init
+	 * as expected by IB spec
+	 */
+	if (smi_reset2init == 0 &&
+	    ibqp->qp_type == IB_QPT_SMI &&
+	    qp_cur_state == IB_QPS_RESET &&
+	    (attr_mask & IB_QP_STATE) &&
+	    attr->qp_state == IB_QPS_INIT) { /* RESET -> INIT */
+		struct ib_qp_attr smiqp_attr = {
+			.qp_state = IB_QPS_INIT,
+			.port_num = my_qp->init_attr.port_num,
+			.pkey_index = 0,
+			.qkey = 0
+		};
+		int smiqp_attr_mask = IB_QP_STATE | IB_QP_PORT |
+			IB_QP_PKEY_INDEX | IB_QP_QKEY;
+		int smirc = internal_modify_qp(
+			ibqp, &smiqp_attr, smiqp_attr_mask, 1);
+		if (smirc) {
+			ehca_err(ibqp->device, "SMI RESET -> INIT failed. "
+				 "ehca_modify_qp() rc=%x", smirc);
+			ret = H_PARAMETER;
+			goto modify_qp_exit1;
+		}
+		qp_cur_state = IB_QPS_INIT;
+		ehca_dbg(ibqp->device, "SMI RESET -> INIT succeeded");
+	}
+	/* is transmitted current state  equal to "real" current state */
+	if ((attr_mask & IB_QP_CUR_STATE) &&
+	    qp_cur_state != attr->cur_qp_state) {
+		ret = -EINVAL;
+		ehca_err(ibqp->device,
+			 "Invalid IB_QP_CUR_STATE attr->curr_qp_state=%x <>"
+			 " actual cur_qp_state=%x. ehca_qp=%p qp_num=%x",
+			 attr->cur_qp_state, qp_cur_state, my_qp, ibqp->qp_num);
+		goto modify_qp_exit1;
+	}
+
+	ehca_dbg(ibqp->device,"ehca_qp=%p qp_num=%x current qp_state=%x "
+		 "new qp_state=%x attribute_mask=%x",
+		 my_qp, ibqp->qp_num, qp_cur_state, attr->qp_state, attr_mask);
+
+	qp_new_state = attr_mask & IB_QP_STATE ? attr->qp_state : qp_cur_state;
+	if (!smi_reset2init &&
+	    !ib_modify_qp_is_ok(qp_cur_state, qp_new_state, ibqp->qp_type,
+				attr_mask)) {
+		ret = -EINVAL;
+		ehca_err(ibqp->device,
+			 "Invalid qp transition new_state=%x cur_state=%x "
+			 "ehca_qp=%p qp_num=%x attr_mask=%x", qp_new_state,
+			 qp_cur_state, my_qp, ibqp->qp_num, attr_mask);
+		goto modify_qp_exit1;
+	}
+
+	if ((mqpcb->qp_state = ib2ehca_qp_state(qp_new_state)))
+		update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_STATE, 1);
+	else {
+		ret = -EINVAL;
+		ehca_err(ibqp->device, "Invalid new qp state=%x "
+			 "ehca_qp=%p qp_num=%x",
+			 qp_new_state, my_qp, ibqp->qp_num);
+		goto modify_qp_exit1;
+	}
+
+	/* retrieve state transition struct to get req and opt attrs */
+	statetrans = get_modqp_statetrans(qp_cur_state, qp_new_state);
+	if (statetrans < 0) {
+		ret = -EINVAL;
+		ehca_err(ibqp->device, "<INVALID STATE CHANGE> qp_cur_state=%x "
+			 "new_qp_state=%x State_xsition=%x ehca_qp=%p "
+			 "qp_num=%x", qp_cur_state, qp_new_state,
+			 statetrans, my_qp, ibqp->qp_num);
+		goto modify_qp_exit1;
+	}
+
+	qp_attr_idx = ib2ehcaqptype(ibqp->qp_type);
+
+	if (qp_attr_idx < 0) {
+		ret = qp_attr_idx;
+		ehca_err(ibqp->device,
+			 "Invalid QP type=%x ehca_qp=%p qp_num=%x",
+			 ibqp->qp_type, my_qp, ibqp->qp_num);
+		goto modify_qp_exit1;
+	}
+
+	ehca_dbg(ibqp->device,
+		 "ehca_qp=%p qp_num=%x <VALID STATE CHANGE> qp_state_xsit=%x",
+		 my_qp, ibqp->qp_num, statetrans);
+
+	/* sqe -> rts: set purge bit of bad wqe before actual trans */
+	if ((my_qp->qp_type == IB_QPT_UD ||
+	     my_qp->qp_type == IB_QPT_GSI ||
+	     my_qp->qp_type == IB_QPT_SMI) &&
+	    statetrans == IB_QPST_SQE2RTS) {
+		/* mark next free wqe if kernel */
+		if (my_qp->uspace_squeue == 0) {
+			struct ehca_wqe *wqe;
+			/* lock send queue */
+			spin_lock_irqsave(&my_qp->spinlock_s, spl_flags);
+			squeue_locked = 1;
+			/* mark next free wqe */
+			wqe = (struct ehca_wqe*)
+				ipz_qeit_get(&my_qp->ipz_squeue);
+			wqe->optype = wqe->wqef = 0xff;
+			ehca_dbg(ibqp->device, "qp_num=%x next_free_wqe=%p",
+				 ibqp->qp_num, wqe);
+		}
+		ret = prepare_sqe_rts(my_qp, shca, &bad_wqe_cnt);
+		if (ret) {
+			ehca_err(ibqp->device, "prepare_sqe_rts() failed "
+				 "ehca_qp=%p qp_num=%x ret=%x",
+				 my_qp, ibqp->qp_num, ret);
+			goto modify_qp_exit2;
+		}
+	}
+
+	/*
+	 * enable RDMA_Atomic_Control if reset->init und reliable con
+	 * this is necessary since gen2 does not provide that flag,
+	 * but pHyp requires it
+	 */
+	if (statetrans == IB_QPST_RESET2INIT &&
+	    (ibqp->qp_type == IB_QPT_RC || ibqp->qp_type == IB_QPT_UC)) {
+		mqpcb->rdma_atomic_ctrl = 3;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RDMA_ATOMIC_CTRL, 1);
+	}
+	/* circ. pHyp requires #RDMA/Atomic Resp Res for UC INIT -> RTR */
+	if (statetrans == IB_QPST_INIT2RTR &&
+	    (ibqp->qp_type == IB_QPT_UC) &&
+	    !(attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)) {
+		mqpcb->rdma_nr_atomic_resp_res = 1; /* default to 1 */
+		update_mask |=
+			EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
+	}
+
+	if (attr_mask & IB_QP_PKEY_INDEX) {
+		mqpcb->prim_p_key_idx = attr->pkey_index;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_P_KEY_IDX, 1);
+	}
+	if (attr_mask & IB_QP_PORT) {
+		if (attr->port_num < 1 || attr->port_num > shca->num_ports) {
+			ret = -EINVAL;
+			ehca_err(ibqp->device, "Invalid port=%x. "
+				 "ehca_qp=%p qp_num=%x num_ports=%x",
+				 attr->port_num, my_qp, ibqp->qp_num,
+				 shca->num_ports);
+			goto modify_qp_exit2;
+		}
+		mqpcb->prim_phys_port = attr->port_num;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PRIM_PHYS_PORT, 1);
+	}
+	if (attr_mask & IB_QP_QKEY) {
+		mqpcb->qkey = attr->qkey;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_QKEY, 1);
+	}
+	if (attr_mask & IB_QP_AV) {
+		int ah_mult = ib_rate_to_mult(attr->ah_attr.static_rate);
+		int ehca_mult = ib_rate_to_mult(shca->sport[my_qp->
+						init_attr.port_num].rate);
+
+		mqpcb->dlid = attr->ah_attr.dlid;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DLID, 1);
+		mqpcb->source_path_bits = attr->ah_attr.src_path_bits;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS, 1);
+		mqpcb->service_level = attr->ah_attr.sl;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL, 1);
+
+		if (ah_mult < ehca_mult)
+			mqpcb->max_static_rate = (ah_mult > 0) ?
+			((ehca_mult - 1) / ah_mult) : 0;
+		else
+			mqpcb->max_static_rate = 0;
+
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE, 1);
+
+		/*
+		 * only if GRH is TRUE we might consider SOURCE_GID_IDX
+		 * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
+		 */
+		if (attr->ah_attr.ah_flags == IB_AH_GRH) {
+			mqpcb->send_grh_flag = 1 << 31;
+			update_mask |=
+				EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG, 1);
+			mqpcb->source_gid_idx = attr->ah_attr.grh.sgid_index;
+			update_mask |=
+				EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX, 1);
+
+			for (cnt = 0; cnt < 16; cnt++)
+				mqpcb->dest_gid.byte[cnt] =
+					attr->ah_attr.grh.dgid.raw[cnt];
+
+			update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_GID, 1);
+			mqpcb->flow_label = attr->ah_attr.grh.flow_label;
+			update_mask |= EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL, 1);
+			mqpcb->hop_limit = attr->ah_attr.grh.hop_limit;
+			update_mask |= EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT, 1);
+			mqpcb->traffic_class = attr->ah_attr.grh.traffic_class;
+			update_mask |=
+				EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS, 1);
+		}
+	}
+
+	if (attr_mask & IB_QP_PATH_MTU) {
+		mqpcb->path_mtu = attr->path_mtu;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_PATH_MTU, 1);
+	}
+	if (attr_mask & IB_QP_TIMEOUT) {
+		mqpcb->timeout = attr->timeout;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_TIMEOUT, 1);
+	}
+	if (attr_mask & IB_QP_RETRY_CNT) {
+		mqpcb->retry_count = attr->retry_cnt;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RETRY_COUNT, 1);
+	}
+	if (attr_mask & IB_QP_RNR_RETRY) {
+		mqpcb->rnr_retry_count = attr->rnr_retry;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RNR_RETRY_COUNT, 1);
+	}
+	if (attr_mask & IB_QP_RQ_PSN) {
+		mqpcb->receive_psn = attr->rq_psn;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_RECEIVE_PSN, 1);
+	}
+	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
+		mqpcb->rdma_nr_atomic_resp_res = attr->max_dest_rd_atomic < 3 ?
+			attr->max_dest_rd_atomic : 2;
+		update_mask |=
+			EHCA_BMASK_SET(MQPCB_MASK_RDMA_NR_ATOMIC_RESP_RES, 1);
+	}
+	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
+		mqpcb->rdma_atomic_outst_dest_qp = attr->max_rd_atomic < 3 ?
+			attr->max_rd_atomic : 2;
+		update_mask |=
+			EHCA_BMASK_SET
+			(MQPCB_MASK_RDMA_ATOMIC_OUTST_DEST_QP, 1);
+	}
+	if (attr_mask & IB_QP_ALT_PATH) {
+		int ah_mult = ib_rate_to_mult(attr->alt_ah_attr.static_rate);
+		int ehca_mult = ib_rate_to_mult(
+			shca->sport[my_qp->init_attr.port_num].rate);
+
+		mqpcb->dlid_al = attr->alt_ah_attr.dlid;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DLID_AL, 1);
+		mqpcb->source_path_bits_al = attr->alt_ah_attr.src_path_bits;
+		update_mask |=
+			EHCA_BMASK_SET(MQPCB_MASK_SOURCE_PATH_BITS_AL, 1);
+		mqpcb->service_level_al = attr->alt_ah_attr.sl;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SERVICE_LEVEL_AL, 1);
+
+		if (ah_mult < ehca_mult)
+			mqpcb->max_static_rate = (ah_mult > 0) ?
+			((ehca_mult - 1) / ah_mult) : 0;
+		else
+			mqpcb->max_static_rate_al = 0;
+
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_MAX_STATIC_RATE_AL, 1);
+
+		/*
+		 * only if GRH is TRUE we might consider SOURCE_GID_IDX
+		 * and DEST_GID otherwise phype will return H_ATTR_PARM!!!
+		 */
+		if (attr->alt_ah_attr.ah_flags == IB_AH_GRH) {
+			mqpcb->send_grh_flag_al = 1 << 31;
+			update_mask |=
+				EHCA_BMASK_SET(MQPCB_MASK_SEND_GRH_FLAG_AL, 1);
+			mqpcb->source_gid_idx_al =
+				attr->alt_ah_attr.grh.sgid_index;
+			update_mask |=
+				EHCA_BMASK_SET(MQPCB_MASK_SOURCE_GID_IDX_AL, 1);
+
+			for (cnt = 0; cnt < 16; cnt++)
+				mqpcb->dest_gid_al.byte[cnt] =
+					attr->alt_ah_attr.grh.dgid.raw[cnt];
+
+			update_mask |=
+				EHCA_BMASK_SET(MQPCB_MASK_DEST_GID_AL, 1);
+			mqpcb->flow_label_al = attr->alt_ah_attr.grh.flow_label;
+			update_mask |=
+				EHCA_BMASK_SET(MQPCB_MASK_FLOW_LABEL_AL, 1);
+			mqpcb->hop_limit_al = attr->alt_ah_attr.grh.hop_limit;
+			update_mask |=
+				EHCA_BMASK_SET(MQPCB_MASK_HOP_LIMIT_AL, 1);
+			mqpcb->traffic_class_al =
+				attr->alt_ah_attr.grh.traffic_class;
+			update_mask |=
+				EHCA_BMASK_SET(MQPCB_MASK_TRAFFIC_CLASS_AL, 1);
+		}
+	}
+
+	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
+		mqpcb->min_rnr_nak_timer_field = attr->min_rnr_timer;
+		update_mask |=
+			EHCA_BMASK_SET(MQPCB_MASK_MIN_RNR_NAK_TIMER_FIELD, 1);
+	}
+
+	if (attr_mask & IB_QP_SQ_PSN) {
+		mqpcb->send_psn = attr->sq_psn;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_SEND_PSN, 1);
+	}
+
+	if (attr_mask & IB_QP_DEST_QPN) {
+		mqpcb->dest_qp_nr = attr->dest_qp_num;
+		update_mask |= EHCA_BMASK_SET(MQPCB_MASK_DEST_QP_NR, 1);
+	}
+
+	if (attr_mask & IB_QP_PATH_MIG_STATE) {
+		mqpcb->path_migration_state = attr->path_mig_state;
+		update_mask |=
+			EHCA_BMASK_SET(MQPCB_MASK_PATH_MIGRATION_STATE, 1);
+	}
+
+	if (attr_mask & IB_QP_CAP) {
+		mqpcb->max_nr_outst_send_wr = attr->cap.max_send_wr+1;
+		update_mask |=
+			EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_SEND_WR, 1);
+		mqpcb->max_nr_outst_recv_wr = attr->cap.max_recv_wr+1;
+		update_mask |=
+			EHCA_BMASK_SET(MQPCB_MASK_MAX_NR_OUTST_RECV_WR, 1);
+		/* no support for max_send/recv_sge yet */
+	}
+
+	if (ehca_debug_level)
+		ehca_dmp(mqpcb, 4*70, "qp_num=%x", ibqp->qp_num);
+
+	h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
+				 my_qp->ipz_qp_handle,
+				 &my_qp->pf,
+				 update_mask,
+				 mqpcb, my_qp->galpas.kernel);
+
+	if (h_ret != H_SUCCESS) {
+		ret = ehca2ib_return_code(h_ret);
+		ehca_err(ibqp->device, "hipz_h_modify_qp() failed rc=%lx "
+			 "ehca_qp=%p qp_num=%x",h_ret, my_qp, ibqp->qp_num);
+		goto modify_qp_exit2;
+	}
+
+	if ((my_qp->qp_type == IB_QPT_UD ||
+	     my_qp->qp_type == IB_QPT_GSI ||
+	     my_qp->qp_type == IB_QPT_SMI) &&
+	    statetrans == IB_QPST_SQE2RTS) {
+		/* doorbell to reprocessing wqes */
+		iosync(); /* serialize GAL register access */
+		hipz_update_sqa(my_qp, bad_wqe_cnt-1);
+		ehca_gen_dbg("doorbell for %x wqes", bad_wqe_cnt);
+	}
+
+	if (statetrans == IB_QPST_RESET2INIT ||
+	    statetrans == IB_QPST_INIT2INIT) {
+		mqpcb->qp_enable = 1;
+		mqpcb->qp_state = EHCA_QPS_INIT;
+		update_mask = 0;
+		update_mask = EHCA_BMASK_SET(MQPCB_MASK_QP_ENABLE, 1);
+
+		h_ret = hipz_h_modify_qp(shca->ipz_hca_handle,
+					 my_qp->ipz_qp_handle,
+					 &my_qp->pf,
+					 update_mask,
+					 mqpcb,
+					 my_qp->galpas.kernel);
+
+		if (h_ret != H_SUCCESS) {
+			ret = ehca2ib_return_code(h_ret);
+			ehca_err(ibqp->device, "ENABLE in context of "
+				 "RESET_2_INIT failed! Maybe you didn't get "
+				 "a LID h_ret=%lx ehca_qp=%p qp_num=%x",
+				 h_ret, my_qp, ibqp->qp_num);
+			goto modify_qp_exit2;
+		}
+	}
+
+	if (statetrans == IB_QPST_ANY2RESET) {
+		ipz_qeit_reset(&my_qp->ipz_rqueue);
+		ipz_qeit_reset(&my_qp->ipz_squeue);
+	}
+
+	if (attr_mask & IB_QP_QKEY)
+		my_qp->qkey = attr->qkey;
+
+modify_qp_exit2:
+	if (squeue_locked) { /* this means: sqe -> rts */
+		spin_unlock_irqrestore(&my_qp->spinlock_s, spl_flags);
+		my_qp->sqerr_purgeflag = 1;
+	}
+
+modify_qp_exit1:
+	kfree(mqpcb);
+
+	return ret;
+}
+
+int ehca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+		   struct ib_udata *udata)
+{
+	struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+	struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
+					     ib_pd);
+	u32 cur_pid = current->tgid;
+
+	if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
+	    my_pd->ownpid != cur_pid) {
+		ehca_err(ibqp->pd->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		return -EINVAL;
+	}
+
+	return internal_modify_qp(ibqp, attr, attr_mask, 0);
+}
+
+int ehca_query_qp(struct ib_qp *qp,
+		  struct ib_qp_attr *qp_attr,
+		  int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
+{
+	struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
+	struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
+					     ib_pd);
+	struct ehca_shca *shca = container_of(qp->device, struct ehca_shca,
+					      ib_device);
+	struct ipz_adapter_handle adapter_handle = shca->ipz_hca_handle;
+	struct hcp_modify_qp_control_block *qpcb;
+	u32 cur_pid = current->tgid;
+	int cnt, ret = 0;
+	u64 h_ret;
+
+	if (my_pd->ib_pd.uobject  && my_pd->ib_pd.uobject->context  &&
+	    my_pd->ownpid != cur_pid) {
+		ehca_err(qp->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		return -EINVAL;
+	}
+
+	if (qp_attr_mask & QP_ATTR_QUERY_NOT_SUPPORTED) {
+		ehca_err(qp->device,"Invalid attribute mask "
+			 "ehca_qp=%p qp_num=%x qp_attr_mask=%x ",
+			 my_qp, qp->qp_num, qp_attr_mask);
+		return -EINVAL;
+	}
+
+	qpcb = kzalloc(H_CB_ALIGNMENT, GFP_KERNEL );
+	if (!qpcb) {
+		ehca_err(qp->device,"Out of memory for qpcb "
+			 "ehca_qp=%p qp_num=%x", my_qp, qp->qp_num);
+		return -ENOMEM;
+	}
+
+	h_ret = hipz_h_query_qp(adapter_handle,
+				my_qp->ipz_qp_handle,
+				&my_qp->pf,
+				qpcb, my_qp->galpas.kernel);
+
+	if (h_ret != H_SUCCESS) {
+		ret = ehca2ib_return_code(h_ret);
+		ehca_err(qp->device,"hipz_h_query_qp() failed "
+			 "ehca_qp=%p qp_num=%x h_ret=%lx",
+			 my_qp, qp->qp_num, h_ret);
+		goto query_qp_exit1;
+	}
+
+	qp_attr->cur_qp_state = ehca2ib_qp_state(qpcb->qp_state);
+	qp_attr->qp_state = qp_attr->cur_qp_state;
+
+	if (qp_attr->cur_qp_state == -EINVAL) {
+		ret = -EINVAL;
+		ehca_err(qp->device,"Got invalid ehca_qp_state=%x "
+			 "ehca_qp=%p qp_num=%x",
+			 qpcb->qp_state, my_qp, qp->qp_num);
+		goto query_qp_exit1;
+	}
+
+	if (qp_attr->qp_state == IB_QPS_SQD)
+		qp_attr->sq_draining = 1;
+
+	qp_attr->qkey = qpcb->qkey;
+	qp_attr->path_mtu = qpcb->path_mtu;
+	qp_attr->path_mig_state = qpcb->path_migration_state;
+	qp_attr->rq_psn = qpcb->receive_psn;
+	qp_attr->sq_psn = qpcb->send_psn;
+	qp_attr->min_rnr_timer = qpcb->min_rnr_nak_timer_field;
+	qp_attr->cap.max_send_wr = qpcb->max_nr_outst_send_wr-1;
+	qp_attr->cap.max_recv_wr = qpcb->max_nr_outst_recv_wr-1;
+	/* UD_AV CIRCUMVENTION */
+	if (my_qp->qp_type == IB_QPT_UD) {
+		qp_attr->cap.max_send_sge =
+			qpcb->actual_nr_sges_in_sq_wqe - 2;
+		qp_attr->cap.max_recv_sge =
+			qpcb->actual_nr_sges_in_rq_wqe - 2;
+	} else {
+		qp_attr->cap.max_send_sge =
+			qpcb->actual_nr_sges_in_sq_wqe;
+		qp_attr->cap.max_recv_sge =
+			qpcb->actual_nr_sges_in_rq_wqe;
+	}
+
+	qp_attr->cap.max_inline_data = my_qp->sq_max_inline_data_size;
+	qp_attr->dest_qp_num = qpcb->dest_qp_nr;
+
+	qp_attr->pkey_index =
+		EHCA_BMASK_GET(MQPCB_PRIM_P_KEY_IDX, qpcb->prim_p_key_idx);
+
+	qp_attr->port_num =
+		EHCA_BMASK_GET(MQPCB_PRIM_PHYS_PORT, qpcb->prim_phys_port);
+
+	qp_attr->timeout = qpcb->timeout;
+	qp_attr->retry_cnt = qpcb->retry_count;
+	qp_attr->rnr_retry = qpcb->rnr_retry_count;
+
+	qp_attr->alt_pkey_index =
+		EHCA_BMASK_GET(MQPCB_PRIM_P_KEY_IDX, qpcb->alt_p_key_idx);
+
+	qp_attr->alt_port_num = qpcb->alt_phys_port;
+	qp_attr->alt_timeout = qpcb->timeout_al;
+
+	/* primary av */
+	qp_attr->ah_attr.sl = qpcb->service_level;
+
+	if (qpcb->send_grh_flag) {
+		qp_attr->ah_attr.ah_flags = IB_AH_GRH;
+	}
+
+	qp_attr->ah_attr.static_rate = qpcb->max_static_rate;
+	qp_attr->ah_attr.dlid = qpcb->dlid;
+	qp_attr->ah_attr.src_path_bits = qpcb->source_path_bits;
+	qp_attr->ah_attr.port_num = qp_attr->port_num;
+
+	/* primary GRH */
+	qp_attr->ah_attr.grh.traffic_class = qpcb->traffic_class;
+	qp_attr->ah_attr.grh.hop_limit = qpcb->hop_limit;
+	qp_attr->ah_attr.grh.sgid_index = qpcb->source_gid_idx;
+	qp_attr->ah_attr.grh.flow_label = qpcb->flow_label;
+
+	for (cnt = 0; cnt < 16; cnt++)
+		qp_attr->ah_attr.grh.dgid.raw[cnt] =
+			qpcb->dest_gid.byte[cnt];
+
+	/* alternate AV */
+	qp_attr->alt_ah_attr.sl = qpcb->service_level_al;
+	if (qpcb->send_grh_flag_al) {
+		qp_attr->alt_ah_attr.ah_flags = IB_AH_GRH;
+	}
+
+	qp_attr->alt_ah_attr.static_rate = qpcb->max_static_rate_al;
+	qp_attr->alt_ah_attr.dlid = qpcb->dlid_al;
+	qp_attr->alt_ah_attr.src_path_bits = qpcb->source_path_bits_al;
+
+	/* alternate GRH */
+	qp_attr->alt_ah_attr.grh.traffic_class = qpcb->traffic_class_al;
+	qp_attr->alt_ah_attr.grh.hop_limit = qpcb->hop_limit_al;
+	qp_attr->alt_ah_attr.grh.sgid_index = qpcb->source_gid_idx_al;
+	qp_attr->alt_ah_attr.grh.flow_label = qpcb->flow_label_al;
+
+	for (cnt = 0; cnt < 16; cnt++)
+		qp_attr->alt_ah_attr.grh.dgid.raw[cnt] =
+			qpcb->dest_gid_al.byte[cnt];
+
+	/* return init attributes given in ehca_create_qp */
+	if (qp_init_attr)
+		*qp_init_attr = my_qp->init_attr;
+
+	if (ehca_debug_level)
+		ehca_dmp(qpcb, 4*70, "qp_num=%x", qp->qp_num);
+
+query_qp_exit1:
+	kfree(qpcb);
+
+	return ret;
+}
+
+int ehca_destroy_qp(struct ib_qp *ibqp)
+{
+	struct ehca_qp *my_qp = container_of(ibqp, struct ehca_qp, ib_qp);
+	struct ehca_shca *shca = container_of(ibqp->device, struct ehca_shca,
+					      ib_device);
+	struct ehca_pd *my_pd = container_of(my_qp->ib_qp.pd, struct ehca_pd,
+					     ib_pd);
+	u32 cur_pid = current->tgid;
+	u32 qp_num = ibqp->qp_num;
+	int ret;
+	u64 h_ret;
+	u8 port_num;
+	enum ib_qp_type	qp_type;
+	unsigned long flags;
+
+	if (my_pd->ib_pd.uobject && my_pd->ib_pd.uobject->context &&
+	    my_pd->ownpid != cur_pid) {
+		ehca_err(ibqp->device, "Invalid caller pid=%x ownpid=%x",
+			 cur_pid, my_pd->ownpid);
+		return -EINVAL;
+	}
+
+	if (my_qp->send_cq) {
+		ret = ehca_cq_unassign_qp(my_qp->send_cq,
+					      my_qp->real_qp_num);
+		if (ret) {
+			ehca_err(ibqp->device, "Couldn't unassign qp from "
+				 "send_cq ret=%x qp_num=%x cq_num=%x", ret,
+				 my_qp->ib_qp.qp_num, my_qp->send_cq->cq_number);
+			return ret;
+		}
+	}
+
+	spin_lock_irqsave(&ehca_qp_idr_lock, flags);
+	idr_remove(&ehca_qp_idr, my_qp->token);
+	spin_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+	/* un-mmap if vma alloc */
+	if (my_qp->uspace_rqueue) {
+		ret = ehca_munmap(my_qp->uspace_rqueue,
+				  my_qp->ipz_rqueue.queue_length);
+		if (ret)
+			ehca_err(ibqp->device, "Could not munmap rqueue "
+				 "qp_num=%x", qp_num);
+		ret = ehca_munmap(my_qp->uspace_squeue,
+				  my_qp->ipz_squeue.queue_length);
+		if (ret)
+			ehca_err(ibqp->device, "Could not munmap squeue "
+				 "qp_num=%x", qp_num);
+		ret = ehca_munmap(my_qp->uspace_fwh, EHCA_PAGESIZE);
+		if (ret)
+			ehca_err(ibqp->device, "Could not munmap fwh qp_num=%x",
+				 qp_num);
+	}
+
+	h_ret = hipz_h_destroy_qp(shca->ipz_hca_handle, my_qp);
+	if (h_ret != H_SUCCESS) {
+		ehca_err(ibqp->device, "hipz_h_destroy_qp() failed rc=%lx "
+			 "ehca_qp=%p qp_num=%x", h_ret, my_qp, qp_num);
+		return ehca2ib_return_code(h_ret);
+	}
+
+	port_num = my_qp->init_attr.port_num;
+	qp_type  = my_qp->init_attr.qp_type;
+
+	/* no support for IB_QPT_SMI yet */
+	if (qp_type == IB_QPT_GSI) {
+		struct ib_event event;
+		ehca_info(ibqp->device, "device %s: port %x is inactive.",
+			  shca->ib_device.name, port_num);
+		event.device = &shca->ib_device;
+		event.event = IB_EVENT_PORT_ERR;
+		event.element.port_num = port_num;
+		shca->sport[port_num - 1].port_state = IB_PORT_DOWN;
+		ib_dispatch_event(&event);
+	}
+
+	ipz_queue_dtor(&my_qp->ipz_rqueue);
+	ipz_queue_dtor(&my_qp->ipz_squeue);
+	kmem_cache_free(qp_cache, my_qp);
+	return 0;
+}
+
+int ehca_init_qp_cache(void)
+{
+	qp_cache = kmem_cache_create("ehca_cache_qp",
+				     sizeof(struct ehca_qp), 0,
+				     SLAB_HWCACHE_ALIGN,
+				     NULL, NULL);
+	if (!qp_cache)
+		return -ENOMEM;
+	return 0;
+}
+
+void ehca_cleanup_qp_cache(void)
+{
+	if (qp_cache)
+		kmem_cache_destroy(qp_cache);
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_reqs.c b/drivers/infiniband/hw/ehca/ehca_reqs.c
new file mode 100644
index 0000000..b46bda1
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_reqs.c
@@ -0,0 +1,653 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  post_send/recv, poll_cq, req_notify
+ *
+ *  Authors: Waleri Fomin <fomin@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <asm-powerpc/system.h>
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+#include "hipz_fns.h"
+
+static inline int ehca_write_rwqe(struct ipz_queue *ipz_rqueue,
+				  struct ehca_wqe *wqe_p,
+				  struct ib_recv_wr *recv_wr)
+{
+	u8 cnt_ds;
+	if (unlikely((recv_wr->num_sge < 0) ||
+		     (recv_wr->num_sge > ipz_rqueue->act_nr_of_sg))) {
+		ehca_gen_err("Invalid number of WQE SGE. "
+			 "num_sqe=%x max_nr_of_sg=%x",
+			 recv_wr->num_sge, ipz_rqueue->act_nr_of_sg);
+		return -EINVAL; /* invalid SG list length */
+	}
+
+	/* clear wqe header until sglist */
+	memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
+
+	wqe_p->work_request_id = recv_wr->wr_id;
+	wqe_p->nr_of_data_seg = recv_wr->num_sge;
+
+	for (cnt_ds = 0; cnt_ds < recv_wr->num_sge; cnt_ds++) {
+		wqe_p->u.all_rcv.sg_list[cnt_ds].vaddr =
+			recv_wr->sg_list[cnt_ds].addr;
+		wqe_p->u.all_rcv.sg_list[cnt_ds].lkey =
+			recv_wr->sg_list[cnt_ds].lkey;
+		wqe_p->u.all_rcv.sg_list[cnt_ds].length =
+			recv_wr->sg_list[cnt_ds].length;
+	}
+
+	if (ehca_debug_level) {
+		ehca_gen_dbg("RECEIVE WQE written into ipz_rqueue=%p", ipz_rqueue);
+		ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "recv wqe");
+	}
+
+	return 0;
+}
+
+#if defined(DEBUG_GSI_SEND_WR)
+
+/* need ib_mad struct */
+#include <rdma/ib_mad.h>
+
+static void trace_send_wr_ud(const struct ib_send_wr *send_wr)
+{
+	int idx;
+	int j;
+	while (send_wr) {
+		struct ib_mad_hdr *mad_hdr = send_wr->wr.ud.mad_hdr;
+		struct ib_sge *sge = send_wr->sg_list;
+		ehca_gen_dbg("send_wr#%x wr_id=%lx num_sge=%x "
+			     "send_flags=%x opcode=%x",idx, send_wr->wr_id,
+			     send_wr->num_sge, send_wr->send_flags,
+			     send_wr->opcode);
+		if (mad_hdr) {
+			ehca_gen_dbg("send_wr#%x mad_hdr base_version=%x "
+				     "mgmt_class=%x class_version=%x method=%x "
+				     "status=%x class_specific=%x tid=%lx "
+				     "attr_id=%x resv=%x attr_mod=%x",
+				     idx, mad_hdr->base_version,
+				     mad_hdr->mgmt_class,
+				     mad_hdr->class_version, mad_hdr->method,
+				     mad_hdr->status, mad_hdr->class_specific,
+				     mad_hdr->tid, mad_hdr->attr_id,
+				     mad_hdr->resv,
+				     mad_hdr->attr_mod);
+		}
+		for (j = 0; j < send_wr->num_sge; j++) {
+			u8 *data = (u8 *) abs_to_virt(sge->addr);
+			ehca_gen_dbg("send_wr#%x sge#%x addr=%p length=%x "
+				     "lkey=%x",
+				     idx, j, data, sge->length, sge->lkey);
+			/* assume length is n*16 */
+			ehca_dmp(data, sge->length, "send_wr#%x sge#%x",
+				 idx, j);
+			sge++;
+		} /* eof for j */
+		idx++;
+		send_wr = send_wr->next;
+	} /* eof while send_wr */
+}
+
+#endif /* DEBUG_GSI_SEND_WR */
+
+static inline int ehca_write_swqe(struct ehca_qp *qp,
+				  struct ehca_wqe *wqe_p,
+				  const struct ib_send_wr *send_wr)
+{
+	u32 idx;
+	u64 dma_length;
+	struct ehca_av *my_av;
+	u32 remote_qkey = send_wr->wr.ud.remote_qkey;
+
+	if (unlikely((send_wr->num_sge < 0) ||
+		     (send_wr->num_sge > qp->ipz_squeue.act_nr_of_sg))) {
+		ehca_gen_err("Invalid number of WQE SGE. "
+			 "num_sqe=%x max_nr_of_sg=%x",
+			 send_wr->num_sge, qp->ipz_squeue.act_nr_of_sg);
+		return -EINVAL; /* invalid SG list length */
+	}
+
+	/* clear wqe header until sglist */
+	memset(wqe_p, 0, offsetof(struct ehca_wqe, u.ud_av.sg_list));
+
+	wqe_p->work_request_id = send_wr->wr_id;
+
+	switch (send_wr->opcode) {
+	case IB_WR_SEND:
+	case IB_WR_SEND_WITH_IMM:
+		wqe_p->optype = WQE_OPTYPE_SEND;
+		break;
+	case IB_WR_RDMA_WRITE:
+	case IB_WR_RDMA_WRITE_WITH_IMM:
+		wqe_p->optype = WQE_OPTYPE_RDMAWRITE;
+		break;
+	case IB_WR_RDMA_READ:
+		wqe_p->optype = WQE_OPTYPE_RDMAREAD;
+		break;
+	default:
+		ehca_gen_err("Invalid opcode=%x", send_wr->opcode);
+		return -EINVAL; /* invalid opcode */
+	}
+
+	wqe_p->wqef = (send_wr->opcode) & WQEF_HIGH_NIBBLE;
+
+	wqe_p->wr_flag = 0;
+
+	if (send_wr->send_flags & IB_SEND_SIGNALED)
+		wqe_p->wr_flag |= WQE_WRFLAG_REQ_SIGNAL_COM;
+
+	if (send_wr->opcode == IB_WR_SEND_WITH_IMM ||
+	    send_wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM) {
+		/* this might not work as long as HW does not support it */
+		wqe_p->immediate_data = be32_to_cpu(send_wr->imm_data);
+		wqe_p->wr_flag |= WQE_WRFLAG_IMM_DATA_PRESENT;
+	}
+
+	wqe_p->nr_of_data_seg = send_wr->num_sge;
+
+	switch (qp->qp_type) {
+	case IB_QPT_SMI:
+	case IB_QPT_GSI:
+		/* no break is intential here */
+	case IB_QPT_UD:
+		/* IB 1.2 spec C10-15 compliance */
+		if (send_wr->wr.ud.remote_qkey & 0x80000000)
+			remote_qkey = qp->qkey;
+
+		wqe_p->destination_qp_number = send_wr->wr.ud.remote_qpn << 8;
+		wqe_p->local_ee_context_qkey = remote_qkey;
+		if (!send_wr->wr.ud.ah) {
+			ehca_gen_err("wr.ud.ah is NULL. qp=%p", qp);
+			return -EINVAL;
+		}
+		my_av = container_of(send_wr->wr.ud.ah, struct ehca_av, ib_ah);
+		wqe_p->u.ud_av.ud_av = my_av->av;
+
+		/*
+		 * omitted check of IB_SEND_INLINE
+		 * since HW does not support it
+		 */
+		for (idx = 0; idx < send_wr->num_sge; idx++) {
+			wqe_p->u.ud_av.sg_list[idx].vaddr =
+				send_wr->sg_list[idx].addr;
+			wqe_p->u.ud_av.sg_list[idx].lkey =
+				send_wr->sg_list[idx].lkey;
+			wqe_p->u.ud_av.sg_list[idx].length =
+				send_wr->sg_list[idx].length;
+		} /* eof for idx */
+		if (qp->qp_type == IB_QPT_SMI ||
+		    qp->qp_type == IB_QPT_GSI)
+			wqe_p->u.ud_av.ud_av.pmtu = 1;
+		if (qp->qp_type == IB_QPT_GSI) {
+			wqe_p->pkeyi = send_wr->wr.ud.pkey_index;
+#ifdef DEBUG_GSI_SEND_WR
+			trace_send_wr_ud(send_wr);
+#endif /* DEBUG_GSI_SEND_WR */
+		}
+		break;
+
+	case IB_QPT_UC:
+		if (send_wr->send_flags & IB_SEND_FENCE)
+			wqe_p->wr_flag |= WQE_WRFLAG_FENCE;
+		/* no break is intentional here */
+	case IB_QPT_RC:
+		/* TODO: atomic not implemented */
+		wqe_p->u.nud.remote_virtual_adress =
+			send_wr->wr.rdma.remote_addr;
+		wqe_p->u.nud.rkey = send_wr->wr.rdma.rkey;
+
+		/*
+		 * omitted checking of IB_SEND_INLINE
+		 * since HW does not support it
+		 */
+		dma_length = 0;
+		for (idx = 0; idx < send_wr->num_sge; idx++) {
+			wqe_p->u.nud.sg_list[idx].vaddr =
+				send_wr->sg_list[idx].addr;
+			wqe_p->u.nud.sg_list[idx].lkey =
+				send_wr->sg_list[idx].lkey;
+			wqe_p->u.nud.sg_list[idx].length =
+				send_wr->sg_list[idx].length;
+			dma_length += send_wr->sg_list[idx].length;
+		} /* eof idx */
+		wqe_p->u.nud.atomic_1st_op_dma_len = dma_length;
+
+		break;
+
+	default:
+		ehca_gen_err("Invalid qptype=%x", qp->qp_type);
+		return -EINVAL;
+	}
+
+	if (ehca_debug_level) {
+		ehca_gen_dbg("SEND WQE written into queue qp=%p ", qp);
+		ehca_dmp( wqe_p, 16*(6 + wqe_p->nr_of_data_seg), "send wqe");
+	}
+	return 0;
+}
+
+/* map_ib_wc_status converts raw cqe_status to ib_wc_status */
+static inline void map_ib_wc_status(u32 cqe_status,
+				    enum ib_wc_status *wc_status)
+{
+	if (unlikely(cqe_status & WC_STATUS_ERROR_BIT)) {
+		switch (cqe_status & 0x3F) {
+		case 0x01:
+		case 0x21:
+			*wc_status = IB_WC_LOC_LEN_ERR;
+			break;
+		case 0x02:
+		case 0x22:
+			*wc_status = IB_WC_LOC_QP_OP_ERR;
+			break;
+		case 0x03:
+		case 0x23:
+			*wc_status = IB_WC_LOC_EEC_OP_ERR;
+			break;
+		case 0x04:
+		case 0x24:
+			*wc_status = IB_WC_LOC_PROT_ERR;
+			break;
+		case 0x05:
+		case 0x25:
+			*wc_status = IB_WC_WR_FLUSH_ERR;
+			break;
+		case 0x06:
+			*wc_status = IB_WC_MW_BIND_ERR;
+			break;
+		case 0x07: /* remote error - look into bits 20:24 */
+			switch ((cqe_status
+				 & WC_STATUS_REMOTE_ERROR_FLAGS) >> 11) {
+			case 0x0:
+				/*
+				 * PSN Sequence Error!
+				 * couldn't find a matching status!
+				 */
+				*wc_status = IB_WC_GENERAL_ERR;
+				break;
+			case 0x1:
+				*wc_status = IB_WC_REM_INV_REQ_ERR;
+				break;
+			case 0x2:
+				*wc_status = IB_WC_REM_ACCESS_ERR;
+				break;
+			case 0x3:
+				*wc_status = IB_WC_REM_OP_ERR;
+				break;
+			case 0x4:
+				*wc_status = IB_WC_REM_INV_RD_REQ_ERR;
+				break;
+			}
+			break;
+		case 0x08:
+			*wc_status = IB_WC_RETRY_EXC_ERR;
+			break;
+		case 0x09:
+			*wc_status = IB_WC_RNR_RETRY_EXC_ERR;
+			break;
+		case 0x0A:
+		case 0x2D:
+			*wc_status = IB_WC_REM_ABORT_ERR;
+			break;
+		case 0x0B:
+		case 0x2E:
+			*wc_status = IB_WC_INV_EECN_ERR;
+			break;
+		case 0x0C:
+		case 0x2F:
+			*wc_status = IB_WC_INV_EEC_STATE_ERR;
+			break;
+		case 0x0D:
+			*wc_status = IB_WC_BAD_RESP_ERR;
+			break;
+		case 0x10:
+			/* WQE purged */
+			*wc_status = IB_WC_WR_FLUSH_ERR;
+			break;
+		default:
+			*wc_status = IB_WC_FATAL_ERR;
+
+		}
+	} else
+		*wc_status = IB_WC_SUCCESS;
+}
+
+int ehca_post_send(struct ib_qp *qp,
+		   struct ib_send_wr *send_wr,
+		   struct ib_send_wr **bad_send_wr)
+{
+	struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
+	struct ib_send_wr *cur_send_wr;
+	struct ehca_wqe *wqe_p;
+	int wqe_cnt = 0;
+	int ret = 0;
+	unsigned long spl_flags;
+
+	/* LOCK the QUEUE */
+	spin_lock_irqsave(&my_qp->spinlock_s, spl_flags);
+
+	/* loop processes list of send reqs */
+	for (cur_send_wr = send_wr; cur_send_wr != NULL;
+	     cur_send_wr = cur_send_wr->next) {
+		u64 start_offset = my_qp->ipz_squeue.current_q_offset;
+		/* get pointer next to free WQE */
+		wqe_p = ipz_qeit_get_inc(&my_qp->ipz_squeue);
+		if (unlikely(!wqe_p)) {
+			/* too many posted work requests: queue overflow */
+			if (bad_send_wr)
+				*bad_send_wr = cur_send_wr;
+			if (wqe_cnt == 0) {
+				ret = -ENOMEM;
+				ehca_err(qp->device, "Too many posted WQEs "
+					 "qp_num=%x", qp->qp_num);
+			}
+			goto post_send_exit0;
+		}
+		/* write a SEND WQE into the QUEUE */
+		ret = ehca_write_swqe(my_qp, wqe_p, cur_send_wr);
+		/*
+		 * if something failed,
+		 * reset the free entry pointer to the start value
+		 */
+		if (unlikely(ret)) {
+			my_qp->ipz_squeue.current_q_offset = start_offset;
+			*bad_send_wr = cur_send_wr;
+			if (wqe_cnt == 0) {
+				ret = -EINVAL;
+				ehca_err(qp->device, "Could not write WQE "
+					 "qp_num=%x", qp->qp_num);
+			}
+			goto post_send_exit0;
+		}
+		wqe_cnt++;
+		ehca_dbg(qp->device, "ehca_qp=%p qp_num=%x wqe_cnt=%d",
+			 my_qp, qp->qp_num, wqe_cnt);
+	} /* eof for cur_send_wr */
+
+post_send_exit0:
+	/* UNLOCK the QUEUE */
+	spin_unlock_irqrestore(&my_qp->spinlock_s, spl_flags);
+	iosync(); /* serialize GAL register access */
+	hipz_update_sqa(my_qp, wqe_cnt);
+	return ret;
+}
+
+int ehca_post_recv(struct ib_qp *qp,
+		   struct ib_recv_wr *recv_wr,
+		   struct ib_recv_wr **bad_recv_wr)
+{
+	struct ehca_qp *my_qp = container_of(qp, struct ehca_qp, ib_qp);
+	struct ib_recv_wr *cur_recv_wr;
+	struct ehca_wqe *wqe_p;
+	int wqe_cnt = 0;
+	int ret = 0;
+	unsigned long spl_flags;
+
+	/* LOCK the QUEUE */
+	spin_lock_irqsave(&my_qp->spinlock_r, spl_flags);
+
+	/* loop processes list of send reqs */
+	for (cur_recv_wr = recv_wr; cur_recv_wr != NULL;
+	     cur_recv_wr = cur_recv_wr->next) {
+		u64 start_offset = my_qp->ipz_rqueue.current_q_offset;
+		/* get pointer next to free WQE */
+		wqe_p = ipz_qeit_get_inc(&my_qp->ipz_rqueue);
+		if (unlikely(!wqe_p)) {
+			/* too many posted work requests: queue overflow */
+			if (bad_recv_wr)
+				*bad_recv_wr = cur_recv_wr;
+			if (wqe_cnt == 0) {
+				ret = -ENOMEM;
+				ehca_err(qp->device, "Too many posted WQEs "
+					 "qp_num=%x", qp->qp_num);
+			}
+			goto post_recv_exit0;
+		}
+		/* write a RECV WQE into the QUEUE */
+		ret = ehca_write_rwqe(&my_qp->ipz_rqueue, wqe_p, cur_recv_wr);
+		/*
+		 * if something failed,
+		 * reset the free entry pointer to the start value
+		 */
+		if (unlikely(ret)) {
+			my_qp->ipz_rqueue.current_q_offset = start_offset;
+			*bad_recv_wr = cur_recv_wr;
+			if (wqe_cnt == 0) {
+				ret = -EINVAL;
+				ehca_err(qp->device, "Could not write WQE "
+					 "qp_num=%x", qp->qp_num);
+			}
+			goto post_recv_exit0;
+		}
+		wqe_cnt++;
+		ehca_gen_dbg("ehca_qp=%p qp_num=%x wqe_cnt=%d",
+		     my_qp, qp->qp_num, wqe_cnt);
+	} /* eof for cur_recv_wr */
+
+post_recv_exit0:
+	spin_unlock_irqrestore(&my_qp->spinlock_r, spl_flags);
+	iosync(); /* serialize GAL register access */
+	hipz_update_rqa(my_qp, wqe_cnt);
+	return ret;
+}
+
+/*
+ * ib_wc_opcode table converts ehca wc opcode to ib
+ * Since we use zero to indicate invalid opcode, the actual ib opcode must
+ * be decremented!!!
+ */
+static const u8 ib_wc_opcode[255] = {
+	[0x01] = IB_WC_RECV+1,
+	[0x02] = IB_WC_RECV_RDMA_WITH_IMM+1,
+	[0x04] = IB_WC_BIND_MW+1,
+	[0x08] = IB_WC_FETCH_ADD+1,
+	[0x10] = IB_WC_COMP_SWAP+1,
+	[0x20] = IB_WC_RDMA_WRITE+1,
+	[0x40] = IB_WC_RDMA_READ+1,
+	[0x80] = IB_WC_SEND+1
+};
+
+/* internal function to poll one entry of cq */
+static inline int ehca_poll_cq_one(struct ib_cq *cq, struct ib_wc *wc)
+{
+	int ret = 0;
+	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+	struct ehca_cqe *cqe;
+	int cqe_count = 0;
+
+poll_cq_one_read_cqe:
+	cqe = (struct ehca_cqe *)
+		ipz_qeit_get_inc_valid(&my_cq->ipz_queue);
+	if (!cqe) {
+		ret = -EAGAIN;
+		ehca_dbg(cq->device, "Completion queue is empty ehca_cq=%p "
+			 "cq_num=%x ret=%x", my_cq, my_cq->cq_number, ret);
+		goto  poll_cq_one_exit0;
+	}
+
+	/* prevents loads being reordered across this point */
+	rmb();
+
+	cqe_count++;
+	if (unlikely(cqe->status & WC_STATUS_PURGE_BIT)) {
+		struct ehca_qp *qp=ehca_cq_get_qp(my_cq, cqe->local_qp_number);
+		int purgeflag;
+		unsigned long spl_flags;
+		if (!qp) {
+			ehca_err(cq->device, "cq_num=%x qp_num=%x "
+				 "could not find qp -> ignore cqe",
+				 my_cq->cq_number, cqe->local_qp_number);
+			ehca_dmp(cqe, 64, "cq_num=%x qp_num=%x",
+				 my_cq->cq_number, cqe->local_qp_number);
+			/* ignore this purged cqe */
+			goto poll_cq_one_read_cqe;
+		}
+		spin_lock_irqsave(&qp->spinlock_s, spl_flags);
+		purgeflag = qp->sqerr_purgeflag;
+		spin_unlock_irqrestore(&qp->spinlock_s, spl_flags);
+
+		if (purgeflag) {
+			ehca_dbg(cq->device, "Got CQE with purged bit qp_num=%x "
+				 "src_qp=%x",
+				 cqe->local_qp_number, cqe->remote_qp_number);
+			if (ehca_debug_level)
+				ehca_dmp(cqe, 64, "qp_num=%x src_qp=%x",
+					 cqe->local_qp_number,
+					 cqe->remote_qp_number);
+			/*
+			 * ignore this to avoid double cqes of bad wqe
+			 * that caused sqe and turn off purge flag
+			 */
+			qp->sqerr_purgeflag = 0;
+			goto poll_cq_one_read_cqe;
+		}
+	}
+
+	/* tracing cqe */
+	if (ehca_debug_level) {
+		ehca_dbg(cq->device,
+			 "Received COMPLETION ehca_cq=%p cq_num=%x -----",
+			 my_cq, my_cq->cq_number);
+		ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
+			 my_cq, my_cq->cq_number);
+		ehca_dbg(cq->device,
+			 "ehca_cq=%p cq_num=%x -------------------------",
+			 my_cq, my_cq->cq_number);
+	}
+
+	/* we got a completion! */
+	wc->wr_id = cqe->work_request_id;
+
+	/* eval ib_wc_opcode */
+	wc->opcode = ib_wc_opcode[cqe->optype]-1;
+	if (unlikely(wc->opcode == -1)) {
+		ehca_err(cq->device, "Invalid cqe->OPType=%x cqe->status=%x "
+			 "ehca_cq=%p cq_num=%x",
+			 cqe->optype, cqe->status, my_cq, my_cq->cq_number);
+		/* dump cqe for other infos */
+		ehca_dmp(cqe, 64, "ehca_cq=%p cq_num=%x",
+			 my_cq, my_cq->cq_number);
+		/* update also queue adder to throw away this entry!!! */
+		goto poll_cq_one_exit0;
+	}
+	/* eval ib_wc_status */
+	if (unlikely(cqe->status & WC_STATUS_ERROR_BIT)) {
+		/* complete with errors */
+		map_ib_wc_status(cqe->status, &wc->status);
+		wc->vendor_err = wc->status;
+	} else
+		wc->status = IB_WC_SUCCESS;
+
+	wc->qp_num = cqe->local_qp_number;
+	wc->byte_len = cqe->nr_bytes_transferred;
+	wc->pkey_index = cqe->pkey_index;
+	wc->slid = cqe->rlid;
+	wc->dlid_path_bits = cqe->dlid;
+	wc->src_qp = cqe->remote_qp_number;
+	wc->wc_flags = cqe->w_completion_flags;
+	wc->imm_data = cpu_to_be32(cqe->immediate_data);
+	wc->sl = cqe->service_level;
+
+	if (wc->status != IB_WC_SUCCESS)
+		ehca_dbg(cq->device,
+			 "ehca_cq=%p cq_num=%x WARNING unsuccessful cqe "
+			 "OPType=%x status=%x qp_num=%x src_qp=%x wr_id=%lx "
+			 "cqe=%p", my_cq, my_cq->cq_number, cqe->optype,
+			 cqe->status, cqe->local_qp_number,
+			 cqe->remote_qp_number, cqe->work_request_id, cqe);
+
+poll_cq_one_exit0:
+	if (cqe_count > 0)
+		hipz_update_feca(my_cq, cqe_count);
+
+	return ret;
+}
+
+int ehca_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc)
+{
+	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+	int nr;
+	struct ib_wc *current_wc = wc;
+	int ret = 0;
+	unsigned long spl_flags;
+
+	if (num_entries < 1) {
+		ehca_err(cq->device, "Invalid num_entries=%d ehca_cq=%p "
+			 "cq_num=%x", num_entries, my_cq, my_cq->cq_number);
+		ret = -EINVAL;
+		goto poll_cq_exit0;
+	}
+
+	spin_lock_irqsave(&my_cq->spinlock, spl_flags);
+	for (nr = 0; nr < num_entries; nr++) {
+		ret = ehca_poll_cq_one(cq, current_wc);
+		if (ret)
+			break;
+		current_wc++;
+	} /* eof for nr */
+	spin_unlock_irqrestore(&my_cq->spinlock, spl_flags);
+	if (ret == -EAGAIN  || !ret)
+		ret = nr;
+
+poll_cq_exit0:
+	return ret;
+}
+
+int ehca_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify cq_notify)
+{
+	struct ehca_cq *my_cq = container_of(cq, struct ehca_cq, ib_cq);
+
+	switch (cq_notify) {
+	case IB_CQ_SOLICITED:
+		hipz_set_cqx_n0(my_cq, 1);
+		break;
+	case IB_CQ_NEXT_COMP:
+		hipz_set_cqx_n1(my_cq, 1);
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_sqp.c b/drivers/infiniband/hw/ehca/ehca_sqp.c
new file mode 100644
index 0000000..9f16e9c
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_sqp.c
@@ -0,0 +1,111 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  SQP functions
+ *
+ *  Authors: Khadija Souissi <souissi@de.ibm.com>
+ *           Heiko J Schick <schickhj@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#include <linux/module.h>
+#include <linux/err.h>
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+#include "ehca_iverbs.h"
+#include "hcp_if.h"
+
+
+/**
+ * ehca_define_sqp - Defines special queue pair 1 (GSI QP). When special queue
+ * pair is created successfully, the corresponding port gets active.
+ *
+ * Define Special Queue pair 0 (SMI QP) is still not supported.
+ *
+ * @qp_init_attr: Queue pair init attributes with port and queue pair type
+ */
+
+u64 ehca_define_sqp(struct ehca_shca *shca,
+		    struct ehca_qp *ehca_qp,
+		    struct ib_qp_init_attr *qp_init_attr)
+{
+	u32 pma_qp_nr, bma_qp_nr;
+	u64 ret;
+	u8 port = qp_init_attr->port_num;
+	int counter;
+
+	shca->sport[port - 1].port_state = IB_PORT_DOWN;
+
+	switch (qp_init_attr->qp_type) {
+	case IB_QPT_SMI:
+		/* function not supported yet */
+		break;
+	case IB_QPT_GSI:
+		ret = hipz_h_define_aqp1(shca->ipz_hca_handle,
+					 ehca_qp->ipz_qp_handle,
+					 ehca_qp->galpas.kernel,
+					 (u32) qp_init_attr->port_num,
+					 &pma_qp_nr, &bma_qp_nr);
+
+		if (ret != H_SUCCESS) {
+			ehca_err(&shca->ib_device,
+				 "Can't define AQP1 for port %x. rc=%lx",
+				 port, ret);
+			return ret;
+		}
+		break;
+	default:
+		ehca_err(&shca->ib_device, "invalid qp_type=%x",
+			 qp_init_attr->qp_type);
+		return H_PARAMETER;
+	}
+
+	for (counter = 0;
+	     shca->sport[port - 1].port_state != IB_PORT_ACTIVE &&
+		     counter < ehca_port_act_time;
+	     counter++) {
+		ehca_dbg(&shca->ib_device, "... wait until port %x is active",
+			 port);
+		msleep_interruptible(1000);
+	}
+
+	if (counter == ehca_port_act_time) {
+		ehca_err(&shca->ib_device, "Port %x is not active.", port);
+		return H_HARDWARE;
+	}
+
+	return H_SUCCESS;
+}
diff --git a/drivers/infiniband/hw/ehca/ehca_tools.h b/drivers/infiniband/hw/ehca/ehca_tools.h
new file mode 100644
index 0000000..9f56bb8
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_tools.h
@@ -0,0 +1,172 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  auxiliary functions
+ *
+ *  Authors: Christoph Raisch <raisch@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Khadija Souissi <souissik@de.ibm.com>
+ *           Waleri Fomin <fomin@de.ibm.com>
+ *           Heiko J Schick <schickhj@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+
+#ifndef EHCA_TOOLS_H
+#define EHCA_TOOLS_H
+
+#include <linux/kernel.h>
+#include <linux/spinlock.h>
+#include <linux/delay.h>
+#include <linux/idr.h>
+#include <linux/kthread.h>
+#include <linux/mm.h>
+#include <linux/mman.h>
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/vmalloc.h>
+#include <linux/version.h>
+#include <linux/notifier.h>
+#include <linux/cpu.h>
+#include <linux/device.h>
+
+#include <asm/abs_addr.h>
+#include <asm/ibmebus.h>
+#include <asm/io.h>
+#include <asm/pgtable.h>
+
+extern int ehca_debug_level;
+
+#define ehca_dbg(ib_dev, format, arg...) \
+	do { \
+		if (unlikely(ehca_debug_level)) \
+			dev_printk(KERN_DEBUG, (ib_dev)->dma_device, \
+				   "PU%04x EHCA_DBG:%s " format "\n", \
+				   get_paca()->paca_index, __FUNCTION__, \
+				   ## arg); \
+	} while (0)
+
+#define ehca_info(ib_dev, format, arg...) \
+	dev_info((ib_dev)->dma_device, "PU%04x EHCA_INFO:%s " format "\n", \
+		 get_paca()->paca_index, __FUNCTION__, ## arg)
+
+#define ehca_warn(ib_dev, format, arg...) \
+	dev_warn((ib_dev)->dma_device, "PU%04x EHCA_WARN:%s " format "\n", \
+		 get_paca()->paca_index, __FUNCTION__, ## arg)
+
+#define ehca_err(ib_dev, format, arg...) \
+	dev_err((ib_dev)->dma_device, "PU%04x EHCA_ERR:%s " format "\n", \
+		get_paca()->paca_index, __FUNCTION__, ## arg)
+
+/* use this one only if no ib_dev available */
+#define ehca_gen_dbg(format, arg...) \
+	do { \
+		if (unlikely(ehca_debug_level)) \
+			printk(KERN_DEBUG "PU%04x EHCA_DBG:%s " format "\n",\
+			       get_paca()->paca_index, __FUNCTION__, ## arg); \
+	} while (0)
+
+#define ehca_gen_warn(format, arg...) \
+	do { \
+		if (unlikely(ehca_debug_level)) \
+			printk(KERN_INFO "PU%04x EHCA_WARN:%s " format "\n",\
+			       get_paca()->paca_index, __FUNCTION__, ## arg); \
+	} while (0)
+
+#define ehca_gen_err(format, arg...) \
+	printk(KERN_ERR "PU%04x EHCA_ERR:%s " format "\n", \
+		get_paca()->paca_index, __FUNCTION__, ## arg)
+
+/**
+ * ehca_dmp - printk a memory block, whose length is n*8 bytes.
+ * Each line has the following layout:
+ * <format string> adr=X ofs=Y <8 bytes hex> <8 bytes hex>
+ */
+#define ehca_dmp(adr, len, format, args...) \
+	do {				       \
+		unsigned int x;			      \
+		unsigned int l = (unsigned int)(len); \
+		unsigned char *deb = (unsigned char*)(adr);	\
+		for (x = 0; x < l; x += 16) { \
+			printk("EHCA_DMP:%s" format \
+			       " adr=%p ofs=%04x %016lx %016lx\n", \
+			       __FUNCTION__, ##args, deb, x, \
+			       *((u64 *)&deb[0]), *((u64 *)&deb[8])); \
+			deb += 16; \
+		} \
+	} while (0)
+
+/* define a bitmask, little endian version */
+#define EHCA_BMASK(pos,length) (((pos)<<16)+(length))
+
+/* define a bitmask, the ibm way... */
+#define EHCA_BMASK_IBM(from,to) (((63-to)<<16)+((to)-(from)+1))
+
+/* internal function, don't use */
+#define EHCA_BMASK_SHIFTPOS(mask) (((mask)>>16)&0xffff)
+
+/* internal function, don't use */
+#define EHCA_BMASK_MASK(mask) (0xffffffffffffffffULL >> ((64-(mask))&0xffff))
+
+/**
+ * EHCA_BMASK_SET - return value shifted and masked by mask
+ * variable|=EHCA_BMASK_SET(MY_MASK,0x4711) ORs the bits in variable
+ * variable&=~EHCA_BMASK_SET(MY_MASK,-1) clears the bits from the mask
+ * in variable
+ */
+#define EHCA_BMASK_SET(mask,value) \
+	((EHCA_BMASK_MASK(mask) & ((u64)(value)))<<EHCA_BMASK_SHIFTPOS(mask))
+
+/**
+ * EHCA_BMASK_GET - extract a parameter from value by mask
+ */
+#define EHCA_BMASK_GET(mask,value) \
+	(EHCA_BMASK_MASK(mask)& (((u64)(value))>>EHCA_BMASK_SHIFTPOS(mask)))
+
+
+/* Converts ehca to ib return code */
+static inline int ehca2ib_return_code(u64 ehca_rc)
+{
+	switch (ehca_rc) {
+	case H_SUCCESS:
+		return 0;
+	case H_BUSY:
+		return -EBUSY;
+	case H_NO_MEM:
+		return -ENOMEM;
+	default:
+		return -EINVAL;
+	}
+}
+
+
+#endif /* EHCA_TOOLS_H */
diff --git a/drivers/infiniband/hw/ehca/ehca_uverbs.c b/drivers/infiniband/hw/ehca/ehca_uverbs.c
new file mode 100644
index 0000000..e08764e
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ehca_uverbs.c
@@ -0,0 +1,392 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  userspace support verbs
+ *
+ *  Authors: Christoph Raisch <raisch@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Heiko J Schick <schickhj@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/current.h>
+
+#include "ehca_classes.h"
+#include "ehca_iverbs.h"
+#include "ehca_mrmw.h"
+#include "ehca_tools.h"
+#include "hcp_if.h"
+
+struct ib_ucontext *ehca_alloc_ucontext(struct ib_device *device,
+					struct ib_udata *udata)
+{
+	struct ehca_ucontext *my_context;
+
+	my_context = kzalloc(sizeof *my_context, GFP_KERNEL);
+	if (!my_context) {
+		ehca_err(device, "Out of memory device=%p", device);
+		return ERR_PTR(-ENOMEM);
+	}
+
+	return &my_context->ib_ucontext;
+}
+
+int ehca_dealloc_ucontext(struct ib_ucontext *context)
+{
+	kfree(container_of(context, struct ehca_ucontext, ib_ucontext));
+	return 0;
+}
+
+struct page *ehca_nopage(struct vm_area_struct *vma,
+			 unsigned long address, int *type)
+{
+	struct page *mypage = NULL;
+	u64 fileoffset = vma->vm_pgoff << PAGE_SHIFT;
+	u32 idr_handle = fileoffset >> 32;
+	u32 q_type = (fileoffset >> 28) & 0xF;	  /* CQ, QP,...        */
+	u32 rsrc_type = (fileoffset >> 24) & 0xF; /* sq,rq,cmnd_window */
+	u32 cur_pid = current->tgid;
+	unsigned long flags;
+	struct ehca_cq *cq;
+	struct ehca_qp *qp;
+	struct ehca_pd *pd;
+	u64 offset;
+	void *vaddr;
+
+	switch (q_type) {
+	case 1: /* CQ */
+		spin_lock_irqsave(&ehca_cq_idr_lock, flags);
+		cq = idr_find(&ehca_cq_idr, idr_handle);
+		spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+		/* make sure this mmap really belongs to the authorized user */
+		if (!cq) {
+			ehca_gen_err("cq is NULL ret=NOPAGE_SIGBUS");
+			return NOPAGE_SIGBUS;
+		}
+
+		if (cq->ownpid != cur_pid) {
+			ehca_err(cq->ib_cq.device,
+				 "Invalid caller pid=%x ownpid=%x",
+				 cur_pid, cq->ownpid);
+			return NOPAGE_SIGBUS;
+		}
+
+		if (rsrc_type == 2) {
+			ehca_dbg(cq->ib_cq.device, "cq=%p cq queuearea", cq);
+			offset = address - vma->vm_start;
+			vaddr = ipz_qeit_calc(&cq->ipz_queue, offset);
+			ehca_dbg(cq->ib_cq.device, "offset=%lx vaddr=%p",
+				 offset, vaddr);
+			mypage = virt_to_page(vaddr);
+		}
+		break;
+
+	case 2: /* QP */
+		spin_lock_irqsave(&ehca_qp_idr_lock, flags);
+		qp = idr_find(&ehca_qp_idr, idr_handle);
+		spin_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+		/* make sure this mmap really belongs to the authorized user */
+		if (!qp) {
+			ehca_gen_err("qp is NULL ret=NOPAGE_SIGBUS");
+			return NOPAGE_SIGBUS;
+		}
+
+		pd = container_of(qp->ib_qp.pd, struct ehca_pd, ib_pd);
+		if (pd->ownpid != cur_pid) {
+			ehca_err(qp->ib_qp.device,
+				 "Invalid caller pid=%x ownpid=%x",
+				 cur_pid, pd->ownpid);
+			return NOPAGE_SIGBUS;
+		}
+
+		if (rsrc_type == 2) {	/* rqueue */
+			ehca_dbg(qp->ib_qp.device, "qp=%p qp rqueuearea", qp);
+			offset = address - vma->vm_start;
+			vaddr = ipz_qeit_calc(&qp->ipz_rqueue, offset);
+			ehca_dbg(qp->ib_qp.device, "offset=%lx vaddr=%p",
+				 offset, vaddr);
+			mypage = virt_to_page(vaddr);
+		} else if (rsrc_type == 3) {	/* squeue */
+			ehca_dbg(qp->ib_qp.device, "qp=%p qp squeuearea", qp);
+			offset = address - vma->vm_start;
+			vaddr = ipz_qeit_calc(&qp->ipz_squeue, offset);
+			ehca_dbg(qp->ib_qp.device, "offset=%lx vaddr=%p",
+				 offset, vaddr);
+			mypage = virt_to_page(vaddr);
+		}
+		break;
+
+	default:
+		ehca_gen_err("bad queue type %x", q_type);
+		return NOPAGE_SIGBUS;
+	}
+
+	if (!mypage) {
+		ehca_gen_err("Invalid page adr==NULL ret=NOPAGE_SIGBUS");
+		return NOPAGE_SIGBUS;
+	}
+	get_page(mypage);
+
+	return mypage;
+}
+
+static struct vm_operations_struct ehcau_vm_ops = {
+	.nopage = ehca_nopage,
+};
+
+int ehca_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
+{
+	u64 fileoffset = vma->vm_pgoff << PAGE_SHIFT;
+	u32 idr_handle = fileoffset >> 32;
+	u32 q_type = (fileoffset >> 28) & 0xF;	  /* CQ, QP,...        */
+	u32 rsrc_type = (fileoffset >> 24) & 0xF; /* sq,rq,cmnd_window */
+	u32 cur_pid = current->tgid;
+	u32 ret;
+	u64 vsize, physical;
+	unsigned long flags;
+	struct ehca_cq *cq;
+	struct ehca_qp *qp;
+	struct ehca_pd *pd;
+
+	switch (q_type) {
+	case  1: /* CQ */
+		spin_lock_irqsave(&ehca_cq_idr_lock, flags);
+		cq = idr_find(&ehca_cq_idr, idr_handle);
+		spin_unlock_irqrestore(&ehca_cq_idr_lock, flags);
+
+		/* make sure this mmap really belongs to the authorized user */
+		if (!cq)
+			return -EINVAL;
+
+		if (cq->ownpid != cur_pid) {
+			ehca_err(cq->ib_cq.device,
+				 "Invalid caller pid=%x ownpid=%x",
+				 cur_pid, cq->ownpid);
+			return -ENOMEM;
+		}
+
+		if (!cq->ib_cq.uobject || cq->ib_cq.uobject->context != context)
+			return -EINVAL;
+
+		switch (rsrc_type) {
+		case 1: /* galpa fw handle */
+			ehca_dbg(cq->ib_cq.device, "cq=%p cq triggerarea", cq);
+			vma->vm_flags |= VM_RESERVED;
+			vsize = vma->vm_end - vma->vm_start;
+			if (vsize != EHCA_PAGESIZE) {
+				ehca_err(cq->ib_cq.device, "invalid vsize=%lx",
+					 vma->vm_end - vma->vm_start);
+				return -EINVAL;
+			}
+
+			physical = cq->galpas.user.fw_handle;
+			vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+			vma->vm_flags |= VM_IO | VM_RESERVED;
+
+			ehca_dbg(cq->ib_cq.device,
+				 "vsize=%lx physical=%lx", vsize, physical);
+			ret = remap_pfn_range(vma, vma->vm_start,
+					      physical >> PAGE_SHIFT, vsize,
+					      vma->vm_page_prot);
+			if (ret) {
+				ehca_err(cq->ib_cq.device,
+					 "remap_pfn_range() failed ret=%x",
+					 ret);
+				return -ENOMEM;
+			}
+			break;
+
+		case 2: /* cq queue_addr */
+			ehca_dbg(cq->ib_cq.device, "cq=%p cq q_addr", cq);
+			vma->vm_flags |= VM_RESERVED;
+			vma->vm_ops = &ehcau_vm_ops;
+			break;
+
+		default:
+			ehca_err(cq->ib_cq.device, "bad resource type %x",
+				 rsrc_type);
+			return -EINVAL;
+		}
+		break;
+
+	case 2: /* QP */
+		spin_lock_irqsave(&ehca_qp_idr_lock, flags);
+		qp = idr_find(&ehca_qp_idr, idr_handle);
+		spin_unlock_irqrestore(&ehca_qp_idr_lock, flags);
+
+		/* make sure this mmap really belongs to the authorized user */
+		if (!qp)
+			return -EINVAL;
+
+		pd = container_of(qp->ib_qp.pd, struct ehca_pd, ib_pd);
+		if (pd->ownpid != cur_pid) {
+			ehca_err(qp->ib_qp.device,
+				 "Invalid caller pid=%x ownpid=%x",
+				 cur_pid, pd->ownpid);
+			return -ENOMEM;
+		}
+
+		if (!qp->ib_qp.uobject || qp->ib_qp.uobject->context != context)
+			return -EINVAL;
+
+		switch (rsrc_type) {
+		case 1: /* galpa fw handle */
+			ehca_dbg(qp->ib_qp.device, "qp=%p qp triggerarea", qp);
+			vma->vm_flags |= VM_RESERVED;
+			vsize = vma->vm_end - vma->vm_start;
+			if (vsize != EHCA_PAGESIZE) {
+				ehca_err(qp->ib_qp.device, "invalid vsize=%lx",
+					 vma->vm_end - vma->vm_start);
+				return -EINVAL;
+			}
+
+			physical = qp->galpas.user.fw_handle;
+			vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
+			vma->vm_flags |= VM_IO | VM_RESERVED;
+
+			ehca_dbg(qp->ib_qp.device, "vsize=%lx physical=%lx",
+				 vsize, physical);
+			ret = remap_pfn_range(vma, vma->vm_start,
+					      physical >> PAGE_SHIFT, vsize,
+					      vma->vm_page_prot);
+			if (ret) {
+				ehca_err(qp->ib_qp.device,
+					 "remap_pfn_range() failed ret=%x",
+					 ret);
+				return -ENOMEM;
+			}
+			break;
+
+		case 2: /* qp rqueue_addr */
+			ehca_dbg(qp->ib_qp.device, "qp=%p qp rqueue_addr", qp);
+			vma->vm_flags |= VM_RESERVED;
+			vma->vm_ops = &ehcau_vm_ops;
+			break;
+
+		case 3: /* qp squeue_addr */
+			ehca_dbg(qp->ib_qp.device, "qp=%p qp squeue_addr", qp);
+			vma->vm_flags |= VM_RESERVED;
+			vma->vm_ops = &ehcau_vm_ops;
+			break;
+
+		default:
+			ehca_err(qp->ib_qp.device, "bad resource type %x",
+				 rsrc_type);
+			return -EINVAL;
+		}
+		break;
+
+	default:
+		ehca_gen_err("bad queue type %x", q_type);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+int ehca_mmap_nopage(u64 foffset, u64 length, void **mapped,
+		     struct vm_area_struct **vma)
+{
+	down_write(&current->mm->mmap_sem);
+	*mapped = (void*)do_mmap(NULL,0, length, PROT_WRITE,
+				 MAP_SHARED | MAP_ANONYMOUS,
+				 foffset);
+	up_write(&current->mm->mmap_sem);
+	if (!(*mapped)) {
+		ehca_gen_err("couldn't mmap foffset=%lx length=%lx",
+			     foffset, length);
+		return -EINVAL;
+	}
+
+	*vma = find_vma(current->mm, (u64)*mapped);
+	if (!(*vma)) {
+		down_write(&current->mm->mmap_sem);
+		do_munmap(current->mm, 0, length);
+		up_write(&current->mm->mmap_sem);
+		ehca_gen_err("couldn't find vma queue=%p", *mapped);
+		return -EINVAL;
+	}
+	(*vma)->vm_flags |= VM_RESERVED;
+	(*vma)->vm_ops = &ehcau_vm_ops;
+
+	return 0;
+}
+
+int ehca_mmap_register(u64 physical, void **mapped,
+		       struct vm_area_struct **vma)
+{
+	int ret;
+	unsigned long vsize;
+	/* ehca hw supports only 4k page */
+	ret = ehca_mmap_nopage(0, EHCA_PAGESIZE, mapped, vma);
+	if (ret) {
+		ehca_gen_err("could'nt mmap physical=%lx", physical);
+		return ret;
+	}
+
+	(*vma)->vm_flags |= VM_RESERVED;
+	vsize = (*vma)->vm_end - (*vma)->vm_start;
+	if (vsize != EHCA_PAGESIZE) {
+		ehca_gen_err("invalid vsize=%lx",
+			     (*vma)->vm_end - (*vma)->vm_start);
+		return -EINVAL;
+	}
+
+	(*vma)->vm_page_prot = pgprot_noncached((*vma)->vm_page_prot);
+	(*vma)->vm_flags |= VM_IO | VM_RESERVED;
+
+	ret = remap_pfn_range((*vma), (*vma)->vm_start,
+			      physical >> PAGE_SHIFT, vsize,
+			      (*vma)->vm_page_prot);
+	if (ret) {
+		ehca_gen_err("remap_pfn_range() failed ret=%x", ret);
+		return -ENOMEM;
+	}
+
+	return 0;
+
+}
+
+int ehca_munmap(unsigned long addr, size_t len) {
+	int ret = 0;
+	struct mm_struct *mm = current->mm;
+	if (mm) {
+		down_write(&mm->mmap_sem);
+		ret = do_munmap(mm, addr, len);
+		up_write(&mm->mmap_sem);
+	}
+	return ret;
+}
diff --git a/drivers/infiniband/hw/ehca/hcp_if.c b/drivers/infiniband/hw/ehca/hcp_if.c
new file mode 100644
index 0000000..3fb46e6
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/hcp_if.c
@@ -0,0 +1,874 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Firmware Infiniband Interface code for POWER
+ *
+ *  Authors: Christoph Raisch <raisch@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Gerd Bayer <gerd.bayer@de.ibm.com>
+ *           Waleri Fomin <fomin@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <asm/hvcall.h>
+#include "ehca_tools.h"
+#include "hcp_if.h"
+#include "hcp_phyp.h"
+#include "hipz_fns.h"
+#include "ipz_pt_fn.h"
+
+#define H_ALL_RES_QP_ENHANCED_OPS       EHCA_BMASK_IBM(9, 11)
+#define H_ALL_RES_QP_PTE_PIN            EHCA_BMASK_IBM(12, 12)
+#define H_ALL_RES_QP_SERVICE_TYPE       EHCA_BMASK_IBM(13, 15)
+#define H_ALL_RES_QP_LL_RQ_CQE_POSTING  EHCA_BMASK_IBM(18, 18)
+#define H_ALL_RES_QP_LL_SQ_CQE_POSTING  EHCA_BMASK_IBM(19, 21)
+#define H_ALL_RES_QP_SIGNALING_TYPE     EHCA_BMASK_IBM(22, 23)
+#define H_ALL_RES_QP_UD_AV_LKEY_CTRL    EHCA_BMASK_IBM(31, 31)
+#define H_ALL_RES_QP_RESOURCE_TYPE      EHCA_BMASK_IBM(56, 63)
+
+#define H_ALL_RES_QP_MAX_OUTST_SEND_WR  EHCA_BMASK_IBM(0, 15)
+#define H_ALL_RES_QP_MAX_OUTST_RECV_WR  EHCA_BMASK_IBM(16, 31)
+#define H_ALL_RES_QP_MAX_SEND_SGE       EHCA_BMASK_IBM(32, 39)
+#define H_ALL_RES_QP_MAX_RECV_SGE       EHCA_BMASK_IBM(40, 47)
+
+#define H_ALL_RES_QP_ACT_OUTST_SEND_WR  EHCA_BMASK_IBM(16, 31)
+#define H_ALL_RES_QP_ACT_OUTST_RECV_WR  EHCA_BMASK_IBM(48, 63)
+#define H_ALL_RES_QP_ACT_SEND_SGE       EHCA_BMASK_IBM(8, 15)
+#define H_ALL_RES_QP_ACT_RECV_SGE       EHCA_BMASK_IBM(24, 31)
+
+#define H_ALL_RES_QP_SQUEUE_SIZE_PAGES  EHCA_BMASK_IBM(0, 31)
+#define H_ALL_RES_QP_RQUEUE_SIZE_PAGES  EHCA_BMASK_IBM(32, 63)
+
+/* direct access qp controls */
+#define DAQP_CTRL_ENABLE    0x01
+#define DAQP_CTRL_SEND_COMP 0x20
+#define DAQP_CTRL_RECV_COMP 0x40
+
+static u32 get_longbusy_msecs(int longbusy_rc)
+{
+	switch (longbusy_rc) {
+	case H_LONG_BUSY_ORDER_1_MSEC:
+		return 1;
+	case H_LONG_BUSY_ORDER_10_MSEC:
+		return 10;
+	case H_LONG_BUSY_ORDER_100_MSEC:
+		return 100;
+	case H_LONG_BUSY_ORDER_1_SEC:
+		return 1000;
+	case H_LONG_BUSY_ORDER_10_SEC:
+		return 10000;
+	case H_LONG_BUSY_ORDER_100_SEC:
+		return 100000;
+	default:
+		return 1;
+	}
+}
+
+static long ehca_plpar_hcall_norets(unsigned long opcode,
+				    unsigned long arg1,
+				    unsigned long arg2,
+				    unsigned long arg3,
+				    unsigned long arg4,
+				    unsigned long arg5,
+				    unsigned long arg6,
+				    unsigned long arg7)
+{
+	long ret;
+	int i, sleep_msecs;
+
+	ehca_gen_dbg("opcode=%lx arg1=%lx arg2=%lx arg3=%lx arg4=%lx "
+		     "arg5=%lx arg6=%lx arg7=%lx",
+		     opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7);
+
+	for (i = 0; i < 5; i++) {
+		ret = plpar_hcall_norets(opcode, arg1, arg2, arg3, arg4,
+					 arg5, arg6, arg7);
+
+		if (H_IS_LONG_BUSY(ret)) {
+			sleep_msecs = get_longbusy_msecs(ret);
+			msleep_interruptible(sleep_msecs);
+			continue;
+		}
+
+		if (ret < H_SUCCESS)
+			ehca_gen_err("opcode=%lx ret=%lx"
+				     " arg1=%lx arg2=%lx arg3=%lx arg4=%lx"
+				     " arg5=%lx arg6=%lx arg7=%lx ",
+				     opcode, ret,
+				     arg1, arg2, arg3, arg4, arg5,
+				     arg6, arg7);
+
+		ehca_gen_dbg("opcode=%lx ret=%lx", opcode, ret);
+		return ret;
+
+	}
+
+	return H_BUSY;
+}
+
+static long ehca_plpar_hcall9(unsigned long opcode,
+			      unsigned long *outs, /* array of 9 outputs */
+			      unsigned long arg1,
+			      unsigned long arg2,
+			      unsigned long arg3,
+			      unsigned long arg4,
+			      unsigned long arg5,
+			      unsigned long arg6,
+			      unsigned long arg7,
+			      unsigned long arg8,
+			      unsigned long arg9)
+{
+	long ret;
+	int i, sleep_msecs;
+
+	ehca_gen_dbg("opcode=%lx arg1=%lx arg2=%lx arg3=%lx arg4=%lx "
+		     "arg5=%lx arg6=%lx arg7=%lx arg8=%lx arg9=%lx",
+		     opcode, arg1, arg2, arg3, arg4, arg5, arg6, arg7,
+		     arg8, arg9);
+
+	for (i = 0; i < 5; i++) {
+		ret = plpar_hcall9(opcode, outs,
+				   arg1, arg2, arg3, arg4, arg5,
+				   arg6, arg7, arg8, arg9);
+
+		if (H_IS_LONG_BUSY(ret)) {
+			sleep_msecs = get_longbusy_msecs(ret);
+			msleep_interruptible(sleep_msecs);
+			continue;
+		}
+
+		if (ret < H_SUCCESS)
+			ehca_gen_err("opcode=%lx ret=%lx"
+				     " arg1=%lx arg2=%lx arg3=%lx arg4=%lx"
+				     " arg5=%lx arg6=%lx arg7=%lx arg8=%lx"
+				     " arg9=%lx"
+				     " out1=%lx out2=%lx out3=%lx out4=%lx"
+				     " out5=%lx out6=%lx out7=%lx out8=%lx"
+				     " out9=%lx",
+				     opcode, ret,
+				     arg1, arg2, arg3, arg4, arg5,
+				     arg6, arg7, arg8, arg9,
+				     outs[0], outs[1], outs[2], outs[3],
+				     outs[4], outs[5], outs[6], outs[7],
+				     outs[8]);
+
+		ehca_gen_dbg("opcode=%lx ret=%lx out1=%lx out2=%lx out3=%lx "
+			     "out4=%lx out5=%lx out6=%lx out7=%lx out8=%lx "
+			     "out9=%lx",
+			     opcode, ret, outs[0], outs[1], outs[2], outs[3],
+			     outs[4], outs[5], outs[6], outs[7], outs[8]);
+		return ret;
+
+	}
+
+	return H_BUSY;
+}
+u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
+			     struct ehca_pfeq *pfeq,
+			     const u32 neq_control,
+			     const u32 number_of_entries,
+			     struct ipz_eq_handle *eq_handle,
+			     u32 *act_nr_of_entries,
+			     u32 *act_pages,
+			     u32 *eq_ist)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+	u64 allocate_controls;
+
+	/* resource type */
+	allocate_controls = 3ULL;
+
+	/* ISN is associated */
+	if (neq_control != 1)
+		allocate_controls = (1ULL << (63 - 7)) | allocate_controls;
+	else /* notification event queue */
+		allocate_controls = (1ULL << 63) | allocate_controls;
+
+	ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+				adapter_handle.handle,  /* r4 */
+				allocate_controls,      /* r5 */
+				number_of_entries,      /* r6 */
+				0, 0, 0, 0, 0, 0);
+	eq_handle->handle = outs[0];
+	*act_nr_of_entries = (u32)outs[3];
+	*act_pages = (u32)outs[4];
+	*eq_ist = (u32)outs[5];
+
+	if (ret == H_NOT_ENOUGH_RESOURCES)
+		ehca_gen_err("Not enough resource - ret=%lx ", ret);
+
+	return ret;
+}
+
+u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
+		       struct ipz_eq_handle eq_handle,
+		       const u64 event_mask)
+{
+	return ehca_plpar_hcall_norets(H_RESET_EVENTS,
+				       adapter_handle.handle, /* r4 */
+				       eq_handle.handle,      /* r5 */
+				       event_mask,	      /* r6 */
+				       0, 0, 0, 0);
+}
+
+u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
+			     struct ehca_cq *cq,
+			     struct ehca_alloc_cq_parms *param)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+				adapter_handle.handle,   /* r4  */
+				2,	                 /* r5  */
+				param->eq_handle.handle, /* r6  */
+				cq->token,	         /* r7  */
+				param->nr_cqe,           /* r8  */
+				0, 0, 0, 0);
+	cq->ipz_cq_handle.handle = outs[0];
+	param->act_nr_of_entries = (u32)outs[3];
+	param->act_pages = (u32)outs[4];
+
+	if (ret == H_SUCCESS)
+		hcp_galpas_ctor(&cq->galpas, outs[5], outs[6]);
+
+	if (ret == H_NOT_ENOUGH_RESOURCES)
+		ehca_gen_err("Not enough resources. ret=%lx", ret);
+
+	return ret;
+}
+
+u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
+			     struct ehca_qp *qp,
+			     struct ehca_alloc_qp_parms *parms)
+{
+	u64 ret;
+	u64 allocate_controls;
+	u64 max_r10_reg;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+	u16 max_nr_receive_wqes = qp->init_attr.cap.max_recv_wr + 1;
+	u16 max_nr_send_wqes = qp->init_attr.cap.max_send_wr + 1;
+	int daqp_ctrl = parms->daqp_ctrl;
+
+	allocate_controls =
+		EHCA_BMASK_SET(H_ALL_RES_QP_ENHANCED_OPS,
+			       (daqp_ctrl & DAQP_CTRL_ENABLE) ? 1 : 0)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_PTE_PIN, 0)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_SERVICE_TYPE, parms->servicetype)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_SIGNALING_TYPE, parms->sigtype)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_LL_RQ_CQE_POSTING,
+				 (daqp_ctrl & DAQP_CTRL_RECV_COMP) ? 1 : 0)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_LL_SQ_CQE_POSTING,
+				 (daqp_ctrl & DAQP_CTRL_SEND_COMP) ? 1 : 0)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_UD_AV_LKEY_CTRL,
+				 parms->ud_av_l_key_ctl)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_RESOURCE_TYPE, 1);
+
+	max_r10_reg =
+		EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_SEND_WR,
+			       max_nr_send_wqes)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_MAX_OUTST_RECV_WR,
+				 max_nr_receive_wqes)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_MAX_SEND_SGE,
+				 parms->max_send_sge)
+		| EHCA_BMASK_SET(H_ALL_RES_QP_MAX_RECV_SGE,
+				 parms->max_recv_sge);
+
+	ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+				adapter_handle.handle,	           /* r4  */
+				allocate_controls,	           /* r5  */
+				qp->send_cq->ipz_cq_handle.handle,
+				qp->recv_cq->ipz_cq_handle.handle,
+				parms->ipz_eq_handle.handle,
+				((u64)qp->token << 32) | parms->pd.value,
+				max_r10_reg,	                   /* r10 */
+				parms->ud_av_l_key_ctl,            /* r11 */
+				0);
+	qp->ipz_qp_handle.handle = outs[0];
+	qp->real_qp_num = (u32)outs[1];
+	parms->act_nr_send_sges =
+		(u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_SEND_WR, outs[2]);
+	parms->act_nr_recv_wqes =
+		(u16)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_OUTST_RECV_WR, outs[2]);
+	parms->act_nr_send_sges =
+		(u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_SEND_SGE, outs[3]);
+	parms->act_nr_recv_sges =
+		(u8)EHCA_BMASK_GET(H_ALL_RES_QP_ACT_RECV_SGE, outs[3]);
+	parms->nr_sq_pages =
+		(u32)EHCA_BMASK_GET(H_ALL_RES_QP_SQUEUE_SIZE_PAGES, outs[4]);
+	parms->nr_rq_pages =
+		(u32)EHCA_BMASK_GET(H_ALL_RES_QP_RQUEUE_SIZE_PAGES, outs[4]);
+
+	if (ret == H_SUCCESS)
+		hcp_galpas_ctor(&qp->galpas, outs[6], outs[6]);
+
+	if (ret == H_NOT_ENOUGH_RESOURCES)
+		ehca_gen_err("Not enough resources. ret=%lx", ret);
+
+	return ret;
+}
+
+u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
+		      const u8 port_id,
+		      struct hipz_query_port *query_port_response_block)
+{
+	u64 ret;
+	u64 r_cb = virt_to_abs(query_port_response_block);
+
+	if (r_cb & (EHCA_PAGESIZE-1)) {
+		ehca_gen_err("response block not page aligned");
+		return H_PARAMETER;
+	}
+
+	ret = ehca_plpar_hcall_norets(H_QUERY_PORT,
+				      adapter_handle.handle, /* r4 */
+				      port_id,	             /* r5 */
+				      r_cb,	             /* r6 */
+				      0, 0, 0, 0);
+
+	if (ehca_debug_level)
+		ehca_dmp(query_port_response_block, 64, "response_block");
+
+	return ret;
+}
+
+u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
+		     struct hipz_query_hca *query_hca_rblock)
+{
+	u64 r_cb = virt_to_abs(query_hca_rblock);
+
+	if (r_cb & (EHCA_PAGESIZE-1)) {
+		ehca_gen_err("response_block=%p not page aligned",
+			     query_hca_rblock);
+		return H_PARAMETER;
+	}
+
+	return ehca_plpar_hcall_norets(H_QUERY_HCA,
+				       adapter_handle.handle, /* r4 */
+				       r_cb,                  /* r5 */
+				       0, 0, 0, 0, 0);
+}
+
+u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
+			  const u8 pagesize,
+			  const u8 queue_type,
+			  const u64 resource_handle,
+			  const u64 logical_address_of_page,
+			  u64 count)
+{
+	return ehca_plpar_hcall_norets(H_REGISTER_RPAGES,
+				       adapter_handle.handle,      /* r4  */
+				       queue_type | pagesize << 8, /* r5  */
+				       resource_handle,	           /* r6  */
+				       logical_address_of_page,    /* r7  */
+				       count,	                   /* r8  */
+				       0, 0);
+}
+
+u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
+			     const struct ipz_eq_handle eq_handle,
+			     struct ehca_pfeq *pfeq,
+			     const u8 pagesize,
+			     const u8 queue_type,
+			     const u64 logical_address_of_page,
+			     const u64 count)
+{
+	if (count != 1) {
+		ehca_gen_err("Ppage counter=%lx", count);
+		return H_PARAMETER;
+	}
+	return hipz_h_register_rpage(adapter_handle,
+				     pagesize,
+				     queue_type,
+				     eq_handle.handle,
+				     logical_address_of_page, count);
+}
+
+u64 hipz_h_query_int_state(const struct ipz_adapter_handle adapter_handle,
+			   u32 ist)
+{
+	u64 ret;
+	ret = ehca_plpar_hcall_norets(H_QUERY_INT_STATE,
+				      adapter_handle.handle, /* r4 */
+				      ist,                   /* r5 */
+				      0, 0, 0, 0, 0);
+
+	if (ret != H_SUCCESS && ret != H_BUSY)
+		ehca_gen_err("Could not query interrupt state.");
+
+	return ret;
+}
+
+u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
+			     const struct ipz_cq_handle cq_handle,
+			     struct ehca_pfcq *pfcq,
+			     const u8 pagesize,
+			     const u8 queue_type,
+			     const u64 logical_address_of_page,
+			     const u64 count,
+			     const struct h_galpa gal)
+{
+	if (count != 1) {
+		ehca_gen_err("Page counter=%lx", count);
+		return H_PARAMETER;
+	}
+
+	return hipz_h_register_rpage(adapter_handle, pagesize, queue_type,
+				     cq_handle.handle, logical_address_of_page,
+				     count);
+}
+
+u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
+			     const struct ipz_qp_handle qp_handle,
+			     struct ehca_pfqp *pfqp,
+			     const u8 pagesize,
+			     const u8 queue_type,
+			     const u64 logical_address_of_page,
+			     const u64 count,
+			     const struct h_galpa galpa)
+{
+	if (count != 1) {
+		ehca_gen_err("Page counter=%lx", count);
+		return H_PARAMETER;
+	}
+
+	return hipz_h_register_rpage(adapter_handle,pagesize,queue_type,
+				     qp_handle.handle,logical_address_of_page,
+				     count);
+}
+
+u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
+			       const struct ipz_qp_handle qp_handle,
+			       struct ehca_pfqp *pfqp,
+			       void **log_addr_next_sq_wqe2processed,
+			       void **log_addr_next_rq_wqe2processed,
+			       int dis_and_get_function_code)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
+				adapter_handle.handle,     /* r4 */
+				dis_and_get_function_code, /* r5 */
+				qp_handle.handle,	   /* r6 */
+				0, 0, 0, 0, 0, 0);
+	if (log_addr_next_sq_wqe2processed)
+		*log_addr_next_sq_wqe2processed = (void*)outs[0];
+	if (log_addr_next_rq_wqe2processed)
+		*log_addr_next_rq_wqe2processed = (void*)outs[1];
+
+	return ret;
+}
+
+u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
+		     const struct ipz_qp_handle qp_handle,
+		     struct ehca_pfqp *pfqp,
+		     const u64 update_mask,
+		     struct hcp_modify_qp_control_block *mqpcb,
+		     struct h_galpa gal)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+	ret = ehca_plpar_hcall9(H_MODIFY_QP, outs,
+				adapter_handle.handle, /* r4 */
+				qp_handle.handle,      /* r5 */
+				update_mask,	       /* r6 */
+				virt_to_abs(mqpcb),    /* r7 */
+				0, 0, 0, 0, 0);
+
+	if (ret == H_NOT_ENOUGH_RESOURCES)
+		ehca_gen_err("Insufficient resources ret=%lx", ret);
+
+	return ret;
+}
+
+u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
+		    const struct ipz_qp_handle qp_handle,
+		    struct ehca_pfqp *pfqp,
+		    struct hcp_modify_qp_control_block *qqpcb,
+		    struct h_galpa gal)
+{
+	return ehca_plpar_hcall_norets(H_QUERY_QP,
+				       adapter_handle.handle, /* r4 */
+				       qp_handle.handle,      /* r5 */
+				       virt_to_abs(qqpcb),    /* r6 */
+				       0, 0, 0, 0);
+}
+
+u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
+		      struct ehca_qp *qp)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = hcp_galpas_dtor(&qp->galpas);
+	if (ret) {
+		ehca_gen_err("Could not destruct qp->galpas");
+		return H_RESOURCE;
+	}
+	ret = ehca_plpar_hcall9(H_DISABLE_AND_GETC, outs,
+				adapter_handle.handle,     /* r4 */
+				/* function code */
+				1,	                   /* r5 */
+				qp->ipz_qp_handle.handle,  /* r6 */
+				0, 0, 0, 0, 0, 0);
+	if (ret == H_HARDWARE)
+		ehca_gen_err("HCA not operational. ret=%lx", ret);
+
+	ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+				      adapter_handle.handle,     /* r4 */
+				      qp->ipz_qp_handle.handle,  /* r5 */
+				      0, 0, 0, 0, 0);
+
+	if (ret == H_RESOURCE)
+		ehca_gen_err("Resource still in use. ret=%lx", ret);
+
+	return ret;
+}
+
+u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
+		       const struct ipz_qp_handle qp_handle,
+		       struct h_galpa gal,
+		       u32 port)
+{
+	return ehca_plpar_hcall_norets(H_DEFINE_AQP0,
+				       adapter_handle.handle, /* r4 */
+				       qp_handle.handle,      /* r5 */
+				       port,                  /* r6 */
+				       0, 0, 0, 0);
+}
+
+u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
+		       const struct ipz_qp_handle qp_handle,
+		       struct h_galpa gal,
+		       u32 port, u32 * pma_qp_nr,
+		       u32 * bma_qp_nr)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = ehca_plpar_hcall9(H_DEFINE_AQP1, outs,
+				adapter_handle.handle, /* r4 */
+				qp_handle.handle,      /* r5 */
+				port,	               /* r6 */
+				0, 0, 0, 0, 0, 0);
+	*pma_qp_nr = (u32)outs[0];
+	*bma_qp_nr = (u32)outs[1];
+
+	if (ret == H_ALIAS_EXIST)
+		ehca_gen_err("AQP1 already exists. ret=%lx", ret);
+
+	return ret;
+}
+
+u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
+		       const struct ipz_qp_handle qp_handle,
+		       struct h_galpa gal,
+		       u16 mcg_dlid,
+		       u64 subnet_prefix, u64 interface_id)
+{
+	u64 ret;
+
+	ret = ehca_plpar_hcall_norets(H_ATTACH_MCQP,
+				      adapter_handle.handle,  /* r4 */
+				      qp_handle.handle,       /* r5 */
+				      mcg_dlid,               /* r6 */
+				      interface_id,           /* r7 */
+				      subnet_prefix,          /* r8 */
+				      0, 0);
+
+	if (ret == H_NOT_ENOUGH_RESOURCES)
+		ehca_gen_err("Not enough resources. ret=%lx", ret);
+
+	return ret;
+}
+
+u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
+		       const struct ipz_qp_handle qp_handle,
+		       struct h_galpa gal,
+		       u16 mcg_dlid,
+		       u64 subnet_prefix, u64 interface_id)
+{
+	return ehca_plpar_hcall_norets(H_DETACH_MCQP,
+				       adapter_handle.handle, /* r4 */
+				       qp_handle.handle,      /* r5 */
+				       mcg_dlid,              /* r6 */
+				       interface_id,          /* r7 */
+				       subnet_prefix,         /* r8 */
+				       0, 0);
+}
+
+u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
+		      struct ehca_cq *cq,
+		      u8 force_flag)
+{
+	u64 ret;
+
+	ret = hcp_galpas_dtor(&cq->galpas);
+	if (ret) {
+		ehca_gen_err("Could not destruct cp->galpas");
+		return H_RESOURCE;
+	}
+
+	ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+				      adapter_handle.handle,     /* r4 */
+				      cq->ipz_cq_handle.handle,  /* r5 */
+				      force_flag != 0 ? 1L : 0L, /* r6 */
+				      0, 0, 0, 0);
+
+	if (ret == H_RESOURCE)
+		ehca_gen_err("H_FREE_RESOURCE failed ret=%lx ", ret);
+
+	return ret;
+}
+
+u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
+		      struct ehca_eq *eq)
+{
+	u64 ret;
+
+	ret = hcp_galpas_dtor(&eq->galpas);
+	if (ret) {
+		ehca_gen_err("Could not destruct eq->galpas");
+		return H_RESOURCE;
+	}
+
+	ret = ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+				      adapter_handle.handle,     /* r4 */
+				      eq->ipz_eq_handle.handle,  /* r5 */
+				      0, 0, 0, 0, 0);
+
+	if (ret == H_RESOURCE)
+		ehca_gen_err("Resource in use. ret=%lx ", ret);
+
+	return ret;
+}
+
+u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
+			     const struct ehca_mr *mr,
+			     const u64 vaddr,
+			     const u64 length,
+			     const u32 access_ctrl,
+			     const struct ipz_pd pd,
+			     struct ehca_mr_hipzout_parms *outparms)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+				adapter_handle.handle,            /* r4 */
+				5,                                /* r5 */
+				vaddr,                            /* r6 */
+				length,                           /* r7 */
+				(((u64)access_ctrl) << 32ULL),    /* r8 */
+				pd.value,                         /* r9 */
+				0, 0, 0);
+	outparms->handle.handle = outs[0];
+	outparms->lkey = (u32)outs[2];
+	outparms->rkey = (u32)outs[3];
+
+	return ret;
+}
+
+u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
+			     const struct ehca_mr *mr,
+			     const u8 pagesize,
+			     const u8 queue_type,
+			     const u64 logical_address_of_page,
+			     const u64 count)
+{
+	u64 ret;
+
+	if ((count > 1) && (logical_address_of_page & (EHCA_PAGESIZE-1))) {
+		ehca_gen_err("logical_address_of_page not on a 4k boundary "
+			     "adapter_handle=%lx mr=%p mr_handle=%lx "
+			     "pagesize=%x queue_type=%x "
+			     "logical_address_of_page=%lx count=%lx",
+			     adapter_handle.handle, mr,
+			     mr->ipz_mr_handle.handle, pagesize, queue_type,
+			     logical_address_of_page, count);
+		ret = H_PARAMETER;
+	} else
+		ret = hipz_h_register_rpage(adapter_handle, pagesize,
+					    queue_type,
+					    mr->ipz_mr_handle.handle,
+					    logical_address_of_page, count);
+	return ret;
+}
+
+u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
+		    const struct ehca_mr *mr,
+		    struct ehca_mr_hipzout_parms *outparms)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = ehca_plpar_hcall9(H_QUERY_MR, outs,
+				adapter_handle.handle,     /* r4 */
+				mr->ipz_mr_handle.handle,  /* r5 */
+				0, 0, 0, 0, 0, 0, 0);
+	outparms->len = outs[0];
+	outparms->vaddr = outs[1];
+	outparms->acl  = outs[4] >> 32;
+	outparms->lkey = (u32)(outs[5] >> 32);
+	outparms->rkey = (u32)(outs[5] & (0xffffffff));
+
+	return ret;
+}
+
+u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
+			    const struct ehca_mr *mr)
+{
+	return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+				       adapter_handle.handle,    /* r4 */
+				       mr->ipz_mr_handle.handle, /* r5 */
+				       0, 0, 0, 0, 0);
+}
+
+u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
+			  const struct ehca_mr *mr,
+			  const u64 vaddr_in,
+			  const u64 length,
+			  const u32 access_ctrl,
+			  const struct ipz_pd pd,
+			  const u64 mr_addr_cb,
+			  struct ehca_mr_hipzout_parms *outparms)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = ehca_plpar_hcall9(H_REREGISTER_PMR, outs,
+				adapter_handle.handle,    /* r4 */
+				mr->ipz_mr_handle.handle, /* r5 */
+				vaddr_in,	          /* r6 */
+				length,                   /* r7 */
+				/* r8 */
+				((((u64)access_ctrl) << 32ULL) | pd.value),
+				mr_addr_cb,               /* r9 */
+				0, 0, 0);
+	outparms->vaddr = outs[1];
+	outparms->lkey = (u32)outs[2];
+	outparms->rkey = (u32)outs[3];
+
+	return ret;
+}
+
+u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
+			const struct ehca_mr *mr,
+			const struct ehca_mr *orig_mr,
+			const u64 vaddr_in,
+			const u32 access_ctrl,
+			const struct ipz_pd pd,
+			struct ehca_mr_hipzout_parms *outparms)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = ehca_plpar_hcall9(H_REGISTER_SMR, outs,
+				adapter_handle.handle,            /* r4 */
+				orig_mr->ipz_mr_handle.handle,    /* r5 */
+				vaddr_in,                         /* r6 */
+				(((u64)access_ctrl) << 32ULL),    /* r7 */
+				pd.value,                         /* r8 */
+				0, 0, 0, 0);
+	outparms->handle.handle = outs[0];
+	outparms->lkey = (u32)outs[2];
+	outparms->rkey = (u32)outs[3];
+
+	return ret;
+}
+
+u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
+			     const struct ehca_mw *mw,
+			     const struct ipz_pd pd,
+			     struct ehca_mw_hipzout_parms *outparms)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = ehca_plpar_hcall9(H_ALLOC_RESOURCE, outs,
+				adapter_handle.handle,      /* r4 */
+				6,                          /* r5 */
+				pd.value,                   /* r6 */
+				0, 0, 0, 0, 0, 0);
+	outparms->handle.handle = outs[0];
+	outparms->rkey = (u32)outs[3];
+
+	return ret;
+}
+
+u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
+		    const struct ehca_mw *mw,
+		    struct ehca_mw_hipzout_parms *outparms)
+{
+	u64 ret;
+	u64 outs[PLPAR_HCALL9_BUFSIZE];
+
+	ret = ehca_plpar_hcall9(H_QUERY_MW, outs,
+				adapter_handle.handle,    /* r4 */
+				mw->ipz_mw_handle.handle, /* r5 */
+				0, 0, 0, 0, 0, 0, 0);
+	outparms->rkey = (u32)outs[3];
+
+	return ret;
+}
+
+u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
+			    const struct ehca_mw *mw)
+{
+	return ehca_plpar_hcall_norets(H_FREE_RESOURCE,
+				       adapter_handle.handle,    /* r4 */
+				       mw->ipz_mw_handle.handle, /* r5 */
+				       0, 0, 0, 0, 0);
+}
+
+u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
+		      const u64 ressource_handle,
+		      void *rblock,
+		      unsigned long *byte_count)
+{
+	u64 r_cb = virt_to_abs(rblock);
+
+	if (r_cb & (EHCA_PAGESIZE-1)) {
+		ehca_gen_err("rblock not page aligned.");
+		return H_PARAMETER;
+	}
+
+	return ehca_plpar_hcall_norets(H_ERROR_DATA,
+				       adapter_handle.handle,
+				       ressource_handle,
+				       r_cb,
+				       0, 0, 0, 0);
+}
diff --git a/drivers/infiniband/hw/ehca/hcp_if.h b/drivers/infiniband/hw/ehca/hcp_if.h
new file mode 100644
index 0000000..587ebd4
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/hcp_if.h
@@ -0,0 +1,261 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Firmware Infiniband Interface code for POWER
+ *
+ *  Authors: Christoph Raisch <raisch@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Gerd Bayer <gerd.bayer@de.ibm.com>
+ *           Waleri Fomin <fomin@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HCP_IF_H__
+#define __HCP_IF_H__
+
+#include "ehca_classes.h"
+#include "ehca_tools.h"
+#include "hipz_hw.h"
+
+/*
+ * hipz_h_alloc_resource_eq allocates EQ resources in HW and FW, initalize
+ * resources, create the empty EQPT (ring).
+ */
+u64 hipz_h_alloc_resource_eq(const struct ipz_adapter_handle adapter_handle,
+			     struct ehca_pfeq *pfeq,
+			     const u32 neq_control,
+			     const u32 number_of_entries,
+			     struct ipz_eq_handle *eq_handle,
+			     u32 * act_nr_of_entries,
+			     u32 * act_pages,
+			     u32 * eq_ist);
+
+u64 hipz_h_reset_event(const struct ipz_adapter_handle adapter_handle,
+		       struct ipz_eq_handle eq_handle,
+		       const u64 event_mask);
+/*
+ * hipz_h_allocate_resource_cq allocates CQ resources in HW and FW, initialize
+ * resources, create the empty CQPT (ring).
+ */
+u64 hipz_h_alloc_resource_cq(const struct ipz_adapter_handle adapter_handle,
+			     struct ehca_cq *cq,
+			     struct ehca_alloc_cq_parms *param);
+
+
+/*
+ * hipz_h_alloc_resource_qp allocates QP resources in HW and FW,
+ * initialize resources, create empty QPPTs (2 rings).
+ */
+u64 hipz_h_alloc_resource_qp(const struct ipz_adapter_handle adapter_handle,
+			     struct ehca_qp *qp,
+			     struct ehca_alloc_qp_parms *parms);
+
+u64 hipz_h_query_port(const struct ipz_adapter_handle adapter_handle,
+		      const u8 port_id,
+		      struct hipz_query_port *query_port_response_block);
+
+u64 hipz_h_query_hca(const struct ipz_adapter_handle adapter_handle,
+		     struct hipz_query_hca *query_hca_rblock);
+
+/*
+ * hipz_h_register_rpage internal function in hcp_if.h for all
+ * hcp_H_REGISTER_RPAGE calls.
+ */
+u64 hipz_h_register_rpage(const struct ipz_adapter_handle adapter_handle,
+			  const u8 pagesize,
+			  const u8 queue_type,
+			  const u64 resource_handle,
+			  const u64 logical_address_of_page,
+			  u64 count);
+
+u64 hipz_h_register_rpage_eq(const struct ipz_adapter_handle adapter_handle,
+			     const struct ipz_eq_handle eq_handle,
+			     struct ehca_pfeq *pfeq,
+			     const u8 pagesize,
+			     const u8 queue_type,
+			     const u64 logical_address_of_page,
+			     const u64 count);
+
+u64 hipz_h_query_int_state(const struct ipz_adapter_handle
+			   hcp_adapter_handle,
+			   u32 ist);
+
+u64 hipz_h_register_rpage_cq(const struct ipz_adapter_handle adapter_handle,
+			     const struct ipz_cq_handle cq_handle,
+			     struct ehca_pfcq *pfcq,
+			     const u8 pagesize,
+			     const u8 queue_type,
+			     const u64 logical_address_of_page,
+			     const u64 count,
+			     const struct h_galpa gal);
+
+u64 hipz_h_register_rpage_qp(const struct ipz_adapter_handle adapter_handle,
+			     const struct ipz_qp_handle qp_handle,
+			     struct ehca_pfqp *pfqp,
+			     const u8 pagesize,
+			     const u8 queue_type,
+			     const u64 logical_address_of_page,
+			     const u64 count,
+			     const struct h_galpa galpa);
+
+u64 hipz_h_disable_and_get_wqe(const struct ipz_adapter_handle adapter_handle,
+			       const struct ipz_qp_handle qp_handle,
+			       struct ehca_pfqp *pfqp,
+			       void **log_addr_next_sq_wqe_tb_processed,
+			       void **log_addr_next_rq_wqe_tb_processed,
+			       int dis_and_get_function_code);
+enum hcall_sigt {
+	HCALL_SIGT_NO_CQE = 0,
+	HCALL_SIGT_BY_WQE = 1,
+	HCALL_SIGT_EVERY = 2
+};
+
+u64 hipz_h_modify_qp(const struct ipz_adapter_handle adapter_handle,
+		     const struct ipz_qp_handle qp_handle,
+		     struct ehca_pfqp *pfqp,
+		     const u64 update_mask,
+		     struct hcp_modify_qp_control_block *mqpcb,
+		     struct h_galpa gal);
+
+u64 hipz_h_query_qp(const struct ipz_adapter_handle adapter_handle,
+		    const struct ipz_qp_handle qp_handle,
+		    struct ehca_pfqp *pfqp,
+		    struct hcp_modify_qp_control_block *qqpcb,
+		    struct h_galpa gal);
+
+u64 hipz_h_destroy_qp(const struct ipz_adapter_handle adapter_handle,
+		      struct ehca_qp *qp);
+
+u64 hipz_h_define_aqp0(const struct ipz_adapter_handle adapter_handle,
+		       const struct ipz_qp_handle qp_handle,
+		       struct h_galpa gal,
+		       u32 port);
+
+u64 hipz_h_define_aqp1(const struct ipz_adapter_handle adapter_handle,
+		       const struct ipz_qp_handle qp_handle,
+		       struct h_galpa gal,
+		       u32 port, u32 * pma_qp_nr,
+		       u32 * bma_qp_nr);
+
+u64 hipz_h_attach_mcqp(const struct ipz_adapter_handle adapter_handle,
+		       const struct ipz_qp_handle qp_handle,
+		       struct h_galpa gal,
+		       u16 mcg_dlid,
+		       u64 subnet_prefix, u64 interface_id);
+
+u64 hipz_h_detach_mcqp(const struct ipz_adapter_handle adapter_handle,
+		       const struct ipz_qp_handle qp_handle,
+		       struct h_galpa gal,
+		       u16 mcg_dlid,
+		       u64 subnet_prefix, u64 interface_id);
+
+u64 hipz_h_destroy_cq(const struct ipz_adapter_handle adapter_handle,
+		      struct ehca_cq *cq,
+		      u8 force_flag);
+
+u64 hipz_h_destroy_eq(const struct ipz_adapter_handle adapter_handle,
+		      struct ehca_eq *eq);
+
+/*
+ * hipz_h_alloc_resource_mr allocates MR resources in HW and FW, initialize
+ * resources.
+ */
+u64 hipz_h_alloc_resource_mr(const struct ipz_adapter_handle adapter_handle,
+			     const struct ehca_mr *mr,
+			     const u64 vaddr,
+			     const u64 length,
+			     const u32 access_ctrl,
+			     const struct ipz_pd pd,
+			     struct ehca_mr_hipzout_parms *outparms);
+
+/* hipz_h_register_rpage_mr registers MR resource pages in HW and FW */
+u64 hipz_h_register_rpage_mr(const struct ipz_adapter_handle adapter_handle,
+			     const struct ehca_mr *mr,
+			     const u8 pagesize,
+			     const u8 queue_type,
+			     const u64 logical_address_of_page,
+			     const u64 count);
+
+/* hipz_h_query_mr queries MR in HW and FW */
+u64 hipz_h_query_mr(const struct ipz_adapter_handle adapter_handle,
+		    const struct ehca_mr *mr,
+		    struct ehca_mr_hipzout_parms *outparms);
+
+/* hipz_h_free_resource_mr frees MR resources in HW and FW */
+u64 hipz_h_free_resource_mr(const struct ipz_adapter_handle adapter_handle,
+			    const struct ehca_mr *mr);
+
+/* hipz_h_reregister_pmr reregisters MR in HW and FW */
+u64 hipz_h_reregister_pmr(const struct ipz_adapter_handle adapter_handle,
+			  const struct ehca_mr *mr,
+			  const u64 vaddr_in,
+			  const u64 length,
+			  const u32 access_ctrl,
+			  const struct ipz_pd pd,
+			  const u64 mr_addr_cb,
+			  struct ehca_mr_hipzout_parms *outparms);
+
+/* hipz_h_register_smr register shared MR in HW and FW */
+u64 hipz_h_register_smr(const struct ipz_adapter_handle adapter_handle,
+			const struct ehca_mr *mr,
+			const struct ehca_mr *orig_mr,
+			const u64 vaddr_in,
+			const u32 access_ctrl,
+			const struct ipz_pd pd,
+			struct ehca_mr_hipzout_parms *outparms);
+
+/*
+ * hipz_h_alloc_resource_mw allocates MW resources in HW and FW, initialize
+ * resources.
+ */
+u64 hipz_h_alloc_resource_mw(const struct ipz_adapter_handle adapter_handle,
+			     const struct ehca_mw *mw,
+			     const struct ipz_pd pd,
+			     struct ehca_mw_hipzout_parms *outparms);
+
+/* hipz_h_query_mw queries MW in HW and FW */
+u64 hipz_h_query_mw(const struct ipz_adapter_handle adapter_handle,
+		    const struct ehca_mw *mw,
+		    struct ehca_mw_hipzout_parms *outparms);
+
+/* hipz_h_free_resource_mw frees MW resources in HW and FW */
+u64 hipz_h_free_resource_mw(const struct ipz_adapter_handle adapter_handle,
+			    const struct ehca_mw *mw);
+
+u64 hipz_h_error_data(const struct ipz_adapter_handle adapter_handle,
+		      const u64 ressource_handle,
+		      void *rblock,
+		      unsigned long *byte_count);
+
+#endif /* __HCP_IF_H__ */
diff --git a/drivers/infiniband/hw/ehca/hcp_phyp.c b/drivers/infiniband/hw/ehca/hcp_phyp.c
new file mode 100644
index 0000000..0b1a477
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/hcp_phyp.c
@@ -0,0 +1,80 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *   load store abstraction for ehca register access with tracing
+ *
+ *  Authors: Christoph Raisch <raisch@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_classes.h"
+#include "hipz_hw.h"
+
+int hcall_map_page(u64 physaddr, u64 *mapaddr)
+{
+	*mapaddr = (u64)(ioremap(physaddr, EHCA_PAGESIZE));
+	return 0;
+}
+
+int hcall_unmap_page(u64 mapaddr)
+{
+	iounmap((volatile void __iomem*)mapaddr);
+	return 0;
+}
+
+int hcp_galpas_ctor(struct h_galpas *galpas,
+		    u64 paddr_kernel, u64 paddr_user)
+{
+	int ret = hcall_map_page(paddr_kernel, &galpas->kernel.fw_handle);
+	if (ret)
+		return ret;
+
+	galpas->user.fw_handle = paddr_user;
+
+	return 0;
+}
+
+int hcp_galpas_dtor(struct h_galpas *galpas)
+{
+	if (galpas->kernel.fw_handle) {
+		int ret = hcall_unmap_page(galpas->kernel.fw_handle);
+		if (ret)
+			return ret;
+	}
+
+	galpas->user.fw_handle = galpas->kernel.fw_handle = 0;
+
+	return 0;
+}
diff --git a/drivers/infiniband/hw/ehca/hcp_phyp.h b/drivers/infiniband/hw/ehca/hcp_phyp.h
new file mode 100644
index 0000000..5305c2a
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/hcp_phyp.h
@@ -0,0 +1,90 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  Firmware calls
+ *
+ *  Authors: Christoph Raisch <raisch@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Waleri Fomin <fomin@de.ibm.com>
+ *           Gerd Bayer <gerd.bayer@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HCP_PHYP_H__
+#define __HCP_PHYP_H__
+
+
+/*
+ * eHCA page (mapped into memory)
+ * resource to access eHCA register pages in CPU address space
+*/
+struct h_galpa {
+	u64 fw_handle;
+	/* for pSeries this is a 64bit memory address where
+	   I/O memory is mapped into CPU address space (kv) */
+};
+
+/*
+ * resource to access eHCA address space registers, all types
+ */
+struct h_galpas {
+	u32 pid;		/*PID of userspace galpa checking */
+	struct h_galpa user;	/* user space accessible resource,
+				   set to 0 if unused */
+	struct h_galpa kernel;	/* kernel space accessible resource,
+				   set to 0 if unused */
+};
+
+static inline u64 hipz_galpa_load(struct h_galpa galpa, u32 offset)
+{
+	u64 addr = galpa.fw_handle + offset;
+	return *(volatile u64 __force *)addr;
+}
+
+static inline void hipz_galpa_store(struct h_galpa galpa, u32 offset, u64 value)
+{
+	u64 addr = galpa.fw_handle + offset;
+	*(volatile u64 __force *)addr = value;
+}
+
+int hcp_galpas_ctor(struct h_galpas *galpas,
+		    u64 paddr_kernel, u64 paddr_user);
+
+int hcp_galpas_dtor(struct h_galpas *galpas);
+
+int hcall_map_page(u64 physaddr, u64 * mapaddr);
+
+int hcall_unmap_page(u64 mapaddr);
+
+#endif
diff --git a/drivers/infiniband/hw/ehca/hipz_fns.h b/drivers/infiniband/hw/ehca/hipz_fns.h
new file mode 100644
index 0000000..9dac93d
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/hipz_fns.h
@@ -0,0 +1,68 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  HW abstraction register functions
+ *
+ *  Authors: Christoph Raisch <raisch@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIPZ_FNS_H__
+#define __HIPZ_FNS_H__
+
+#include "ehca_classes.h"
+#include "hipz_hw.h"
+
+#include "hipz_fns_core.h"
+
+#define hipz_galpa_store_eq(gal, offset, value) \
+	hipz_galpa_store(gal, EQTEMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_eq(gal, offset) \
+	hipz_galpa_load(gal, EQTEMM_OFFSET(offset))
+
+#define hipz_galpa_store_qped(gal, offset, value) \
+	hipz_galpa_store(gal, QPEDMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_qped(gal, offset) \
+	hipz_galpa_load(gal, QPEDMM_OFFSET(offset))
+
+#define hipz_galpa_store_mrmw(gal, offset, value) \
+	hipz_galpa_store(gal, MRMWMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_mrmw(gal, offset) \
+	hipz_galpa_load(gal, MRMWMM_OFFSET(offset))
+
+#endif
diff --git a/drivers/infiniband/hw/ehca/hipz_fns_core.h b/drivers/infiniband/hw/ehca/hipz_fns_core.h
new file mode 100644
index 0000000..20898a1
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/hipz_fns_core.h
@@ -0,0 +1,100 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  HW abstraction register functions
+ *
+ *  Authors: Christoph Raisch <raisch@de.ibm.com>
+ *           Heiko J Schick <schickhj@de.ibm.com>
+ *           Hoang-Nam Nguyen <hnguyen@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIPZ_FNS_CORE_H__
+#define __HIPZ_FNS_CORE_H__
+
+#include "hcp_phyp.h"
+#include "hipz_hw.h"
+
+#define hipz_galpa_store_cq(gal, offset, value) \
+	hipz_galpa_store(gal, CQTEMM_OFFSET(offset), value)
+
+#define hipz_galpa_load_cq(gal, offset) \
+	hipz_galpa_load(gal, CQTEMM_OFFSET(offset))
+
+#define hipz_galpa_store_qp(gal,offset, value) \
+	hipz_galpa_store(gal, QPTEMM_OFFSET(offset), value)
+#define hipz_galpa_load_qp(gal, offset) \
+	hipz_galpa_load(gal,QPTEMM_OFFSET(offset))
+
+static inline void hipz_update_sqa(struct ehca_qp *qp, u16 nr_wqes)
+{
+	/*  ringing doorbell :-) */
+	hipz_galpa_store_qp(qp->galpas.kernel, qpx_sqa,
+			    EHCA_BMASK_SET(QPX_SQADDER, nr_wqes));
+}
+
+static inline void hipz_update_rqa(struct ehca_qp *qp, u16 nr_wqes)
+{
+	/*  ringing doorbell :-) */
+	hipz_galpa_store_qp(qp->galpas.kernel, qpx_rqa,
+			    EHCA_BMASK_SET(QPX_RQADDER, nr_wqes));
+}
+
+static inline void hipz_update_feca(struct ehca_cq *cq, u32 nr_cqes)
+{
+	hipz_galpa_store_cq(cq->galpas.kernel, cqx_feca,
+			    EHCA_BMASK_SET(CQX_FECADDER, nr_cqes));
+}
+
+static inline void hipz_set_cqx_n0(struct ehca_cq *cq, u32 value)
+{
+	u64 cqx_n0_reg;
+
+	hipz_galpa_store_cq(cq->galpas.kernel, cqx_n0,
+			    EHCA_BMASK_SET(CQX_N0_GENERATE_SOLICITED_COMP_EVENT,
+					   value));
+	cqx_n0_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n0);
+}
+
+static inline void hipz_set_cqx_n1(struct ehca_cq *cq, u32 value)
+{
+	u64 cqx_n1_reg;
+
+	hipz_galpa_store_cq(cq->galpas.kernel, cqx_n1,
+			    EHCA_BMASK_SET(CQX_N1_GENERATE_COMP_EVENT, value));
+	cqx_n1_reg = hipz_galpa_load_cq(cq->galpas.kernel, cqx_n1);
+}
+
+#endif /* __HIPZ_FNC_CORE_H__ */
diff --git a/drivers/infiniband/hw/ehca/hipz_hw.h b/drivers/infiniband/hw/ehca/hipz_hw.h
new file mode 100644
index 0000000..3fc92b0
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/hipz_hw.h
@@ -0,0 +1,388 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  eHCA register definitions
+ *
+ *  Authors: Waleri Fomin <fomin@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __HIPZ_HW_H__
+#define __HIPZ_HW_H__
+
+#include "ehca_tools.h"
+
+/* QP Table Entry Memory Map */
+struct hipz_qptemm {
+	u64 qpx_hcr;
+	u64 qpx_c;
+	u64 qpx_herr;
+	u64 qpx_aer;
+/* 0x20*/
+	u64 qpx_sqa;
+	u64 qpx_sqc;
+	u64 qpx_rqa;
+	u64 qpx_rqc;
+/* 0x40*/
+	u64 qpx_st;
+	u64 qpx_pmstate;
+	u64 qpx_pmfa;
+	u64 qpx_pkey;
+/* 0x60*/
+	u64 qpx_pkeya;
+	u64 qpx_pkeyb;
+	u64 qpx_pkeyc;
+	u64 qpx_pkeyd;
+/* 0x80*/
+	u64 qpx_qkey;
+	u64 qpx_dqp;
+	u64 qpx_dlidp;
+	u64 qpx_portp;
+/* 0xa0*/
+	u64 qpx_slidp;
+	u64 qpx_slidpp;
+	u64 qpx_dlida;
+	u64 qpx_porta;
+/* 0xc0*/
+	u64 qpx_slida;
+	u64 qpx_slidpa;
+	u64 qpx_slvl;
+	u64 qpx_ipd;
+/* 0xe0*/
+	u64 qpx_mtu;
+	u64 qpx_lato;
+	u64 qpx_rlimit;
+	u64 qpx_rnrlimit;
+/* 0x100*/
+	u64 qpx_t;
+	u64 qpx_sqhp;
+	u64 qpx_sqptp;
+	u64 qpx_nspsn;
+/* 0x120*/
+	u64 qpx_nspsnhwm;
+	u64 reserved1;
+	u64 qpx_sdsi;
+	u64 qpx_sdsbc;
+/* 0x140*/
+	u64 qpx_sqwsize;
+	u64 qpx_sqwts;
+	u64 qpx_lsn;
+	u64 qpx_nssn;
+/* 0x160 */
+	u64 qpx_mor;
+	u64 qpx_cor;
+	u64 qpx_sqsize;
+	u64 qpx_erc;
+/* 0x180*/
+	u64 qpx_rnrrc;
+	u64 qpx_ernrwt;
+	u64 qpx_rnrresp;
+	u64 qpx_lmsna;
+/* 0x1a0 */
+	u64 qpx_sqhpc;
+	u64 qpx_sqcptp;
+	u64 qpx_sigt;
+	u64 qpx_wqecnt;
+/* 0x1c0*/
+	u64 qpx_rqhp;
+	u64 qpx_rqptp;
+	u64 qpx_rqsize;
+	u64 qpx_nrr;
+/* 0x1e0*/
+	u64 qpx_rdmac;
+	u64 qpx_nrpsn;
+	u64 qpx_lapsn;
+	u64 qpx_lcr;
+/* 0x200*/
+	u64 qpx_rwc;
+	u64 qpx_rwva;
+	u64 qpx_rdsi;
+	u64 qpx_rdsbc;
+/* 0x220*/
+	u64 qpx_rqwsize;
+	u64 qpx_crmsn;
+	u64 qpx_rdd;
+	u64 qpx_larpsn;
+/* 0x240*/
+	u64 qpx_pd;
+	u64 qpx_scqn;
+	u64 qpx_rcqn;
+	u64 qpx_aeqn;
+/* 0x260*/
+	u64 qpx_aaelog;
+	u64 qpx_ram;
+	u64 qpx_rdmaqe0;
+	u64 qpx_rdmaqe1;
+/* 0x280*/
+	u64 qpx_rdmaqe2;
+	u64 qpx_rdmaqe3;
+	u64 qpx_nrpsnhwm;
+/* 0x298*/
+	u64 reserved[(0x400 - 0x298) / 8];
+/* 0x400 extended data */
+	u64 reserved_ext[(0x500 - 0x400) / 8];
+/* 0x500 */
+	u64 reserved2[(0x1000 - 0x500) / 8];
+/* 0x1000      */
+};
+
+#define QPX_SQADDER EHCA_BMASK_IBM(48,63)
+#define QPX_RQADDER EHCA_BMASK_IBM(48,63)
+
+#define QPTEMM_OFFSET(x) offsetof(struct hipz_qptemm,x)
+
+/* MRMWPT Entry Memory Map */
+struct hipz_mrmwmm {
+	/* 0x00 */
+	u64 mrx_hcr;
+
+	u64 mrx_c;
+	u64 mrx_herr;
+	u64 mrx_aer;
+	/* 0x20 */
+	u64 mrx_pp;
+	u64 reserved1;
+	u64 reserved2;
+	u64 reserved3;
+	/* 0x40 */
+	u64 reserved4[(0x200 - 0x40) / 8];
+	/* 0x200 */
+	u64 mrx_ctl[64];
+
+};
+
+#define MRMWMM_OFFSET(x) offsetof(struct hipz_mrmwmm,x)
+
+struct hipz_qpedmm {
+	/* 0x00 */
+	u64 reserved0[(0x400) / 8];
+	/* 0x400 */
+	u64 qpedx_phh;
+	u64 qpedx_ppsgp;
+	/* 0x410 */
+	u64 qpedx_ppsgu;
+	u64 qpedx_ppdgp;
+	/* 0x420 */
+	u64 qpedx_ppdgu;
+	u64 qpedx_aph;
+	/* 0x430 */
+	u64 qpedx_apsgp;
+	u64 qpedx_apsgu;
+	/* 0x440 */
+	u64 qpedx_apdgp;
+	u64 qpedx_apdgu;
+	/* 0x450 */
+	u64 qpedx_apav;
+	u64 qpedx_apsav;
+	/* 0x460  */
+	u64 qpedx_hcr;
+	u64 reserved1[4];
+	/* 0x488 */
+	u64 qpedx_rrl0;
+	/* 0x490 */
+	u64 qpedx_rrrkey0;
+	u64 qpedx_rrva0;
+	/* 0x4a0 */
+	u64 reserved2;
+	u64 qpedx_rrl1;
+	/* 0x4b0 */
+	u64 qpedx_rrrkey1;
+	u64 qpedx_rrva1;
+	/* 0x4c0 */
+	u64 reserved3;
+	u64 qpedx_rrl2;
+	/* 0x4d0 */
+	u64 qpedx_rrrkey2;
+	u64 qpedx_rrva2;
+	/* 0x4e0 */
+	u64 reserved4;
+	u64 qpedx_rrl3;
+	/* 0x4f0 */
+	u64 qpedx_rrrkey3;
+	u64 qpedx_rrva3;
+};
+
+#define QPEDMM_OFFSET(x) offsetof(struct hipz_qpedmm,x)
+
+/* CQ Table Entry Memory Map */
+struct hipz_cqtemm {
+	u64 cqx_hcr;
+	u64 cqx_c;
+	u64 cqx_herr;
+	u64 cqx_aer;
+/* 0x20  */
+	u64 cqx_ptp;
+	u64 cqx_tp;
+	u64 cqx_fec;
+	u64 cqx_feca;
+/* 0x40  */
+	u64 cqx_ep;
+	u64 cqx_eq;
+/* 0x50  */
+	u64 reserved1;
+	u64 cqx_n0;
+/* 0x60  */
+	u64 cqx_n1;
+	u64 reserved2[(0x1000 - 0x60) / 8];
+/* 0x1000 */
+};
+
+#define CQX_FEC_CQE_CNT           EHCA_BMASK_IBM(32,63)
+#define CQX_FECADDER              EHCA_BMASK_IBM(32,63)
+#define CQX_N0_GENERATE_SOLICITED_COMP_EVENT EHCA_BMASK_IBM(0,0)
+#define CQX_N1_GENERATE_COMP_EVENT EHCA_BMASK_IBM(0,0)
+
+#define CQTEMM_OFFSET(x) offsetof(struct hipz_cqtemm,x)
+
+/* EQ Table Entry Memory Map */
+struct hipz_eqtemm {
+	u64 eqx_hcr;
+	u64 eqx_c;
+
+	u64 eqx_herr;
+	u64 eqx_aer;
+/* 0x20 */
+	u64 eqx_ptp;
+	u64 eqx_tp;
+	u64 eqx_ssba;
+	u64 eqx_psba;
+
+/* 0x40 */
+	u64 eqx_cec;
+	u64 eqx_meql;
+	u64 eqx_xisbi;
+	u64 eqx_xisc;
+/* 0x60 */
+	u64 eqx_it;
+
+};
+
+#define EQTEMM_OFFSET(x) offsetof(struct hipz_eqtemm,x)
+
+/* access control defines for MR/MW */
+#define HIPZ_ACCESSCTRL_L_WRITE  0x00800000
+#define HIPZ_ACCESSCTRL_R_WRITE  0x00400000
+#define HIPZ_ACCESSCTRL_R_READ   0x00200000
+#define HIPZ_ACCESSCTRL_R_ATOMIC 0x00100000
+#define HIPZ_ACCESSCTRL_MW_BIND  0x00080000
+
+/* query hca response block */
+struct hipz_query_hca {
+	u32 cur_reliable_dg;
+	u32 cur_qp;
+	u32 cur_cq;
+	u32 cur_eq;
+	u32 cur_mr;
+	u32 cur_mw;
+	u32 cur_ee_context;
+	u32 cur_mcast_grp;
+	u32 cur_qp_attached_mcast_grp;
+	u32 reserved1;
+	u32 cur_ipv6_qp;
+	u32 cur_eth_qp;
+	u32 cur_hp_mr;
+	u32 reserved2[3];
+	u32 max_rd_domain;
+	u32 max_qp;
+	u32 max_cq;
+	u32 max_eq;
+	u32 max_mr;
+	u32 max_hp_mr;
+	u32 max_mw;
+	u32 max_mrwpte;
+	u32 max_special_mrwpte;
+	u32 max_rd_ee_context;
+	u32 max_mcast_grp;
+	u32 max_total_mcast_qp_attach;
+	u32 max_mcast_qp_attach;
+	u32 max_raw_ipv6_qp;
+	u32 max_raw_ethy_qp;
+	u32 internal_clock_frequency;
+	u32 max_pd;
+	u32 max_ah;
+	u32 max_cqe;
+	u32 max_wqes_wq;
+	u32 max_partitions;
+	u32 max_rr_ee_context;
+	u32 max_rr_qp;
+	u32 max_rr_hca;
+	u32 max_act_wqs_ee_context;
+	u32 max_act_wqs_qp;
+	u32 max_sge;
+	u32 max_sge_rd;
+	u32 memory_page_size_supported;
+	u64 max_mr_size;
+	u32 local_ca_ack_delay;
+	u32 num_ports;
+	u32 vendor_id;
+	u32 vendor_part_id;
+	u32 hw_ver;
+	u64 node_guid;
+	u64 hca_cap_indicators;
+	u32 data_counter_register_size;
+	u32 max_shared_rq;
+	u32 max_isns_eq;
+	u32 max_neq;
+} __attribute__ ((packed));
+
+/* query port response block */
+struct hipz_query_port {
+	u32 state;
+	u32 bad_pkey_cntr;
+	u32 lmc;
+	u32 lid;
+	u32 subnet_timeout;
+	u32 qkey_viol_cntr;
+	u32 sm_sl;
+	u32 sm_lid;
+	u32 capability_mask;
+	u32 init_type_reply;
+	u32 pkey_tbl_len;
+	u32 gid_tbl_len;
+	u64 gid_prefix;
+	u32 port_nr;
+	u16 pkey_entries[16];
+	u8  reserved1[32];
+	u32 trent_size;
+	u32 trbuf_size;
+	u64 max_msg_sz;
+	u32 max_mtu;
+	u32 vl_cap;
+	u8  reserved2[1900];
+	u64 guid_entries[255];
+} __attribute__ ((packed));
+
+#endif
diff --git a/drivers/infiniband/hw/ehca/ipz_pt_fn.c b/drivers/infiniband/hw/ehca/ipz_pt_fn.c
new file mode 100644
index 0000000..e028ff1
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ipz_pt_fn.c
@@ -0,0 +1,149 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  internal queue handling
+ *
+ *  Authors: Waleri Fomin <fomin@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "ehca_tools.h"
+#include "ipz_pt_fn.h"
+
+void *ipz_qpageit_get_inc(struct ipz_queue *queue)
+{
+	void *ret = ipz_qeit_get(queue);
+	queue->current_q_offset += queue->pagesize;
+	if (queue->current_q_offset > queue->queue_length) {
+		queue->current_q_offset -= queue->pagesize;
+		ret = NULL;
+	}
+	if (((u64)ret) % EHCA_PAGESIZE) {
+		ehca_gen_err("ERROR!! not at PAGE-Boundary");
+		return NULL;
+	}
+	return ret;
+}
+
+void *ipz_qeit_eq_get_inc(struct ipz_queue *queue)
+{
+	void *ret = ipz_qeit_get(queue);
+	u64 last_entry_in_q = queue->queue_length - queue->qe_size;
+
+	queue->current_q_offset += queue->qe_size;
+	if (queue->current_q_offset > last_entry_in_q) {
+		queue->current_q_offset = 0;
+		queue->toggle_state = (~queue->toggle_state) & 1;
+	}
+
+	return ret;
+}
+
+int ipz_queue_ctor(struct ipz_queue *queue,
+		   const u32 nr_of_pages,
+		   const u32 pagesize, const u32 qe_size, const u32 nr_of_sg)
+{
+	int pages_per_kpage = PAGE_SIZE >> EHCA_PAGESHIFT;
+	int f;
+
+	if (pagesize > PAGE_SIZE) {
+		ehca_gen_err("FATAL ERROR: pagesize=%x is greater "
+			     "than kernel page size", pagesize);
+		return 0;
+	}
+	if (!pages_per_kpage) {
+		ehca_gen_err("FATAL ERROR: invalid kernel page size. "
+			     "pages_per_kpage=%x", pages_per_kpage);
+		return 0;
+	}
+	queue->queue_length = nr_of_pages * pagesize;
+	queue->queue_pages = vmalloc(nr_of_pages * sizeof(void *));
+	if (!queue->queue_pages) {
+		ehca_gen_err("ERROR!! didn't get the memory");
+		return 0;
+	}
+	memset(queue->queue_pages, 0, nr_of_pages * sizeof(void *));
+	/*
+	 * allocate pages for queue:
+	 * outer loop allocates whole kernel pages (page aligned) and
+	 * inner loop divides a kernel page into smaller hca queue pages
+	 */
+	f = 0;
+	while (f < nr_of_pages) {
+		u8 *kpage = (u8*)get_zeroed_page(GFP_KERNEL);
+		int k;
+		if (!kpage)
+			goto ipz_queue_ctor_exit0; /*NOMEM*/
+		for (k = 0; k < pages_per_kpage && f < nr_of_pages; k++) {
+			(queue->queue_pages)[f] = (struct ipz_page *)kpage;
+			kpage += EHCA_PAGESIZE;
+			f++;
+		}
+	}
+
+	queue->current_q_offset = 0;
+	queue->qe_size = qe_size;
+	queue->act_nr_of_sg = nr_of_sg;
+	queue->pagesize = pagesize;
+	queue->toggle_state = 1;
+	return 1;
+
+ ipz_queue_ctor_exit0:
+	ehca_gen_err("Couldn't get alloc pages queue=%p f=%x nr_of_pages=%x",
+		     queue, f, nr_of_pages);
+	for (f = 0; f < nr_of_pages; f += pages_per_kpage) {
+		if (!(queue->queue_pages)[f])
+			break;
+		free_page((unsigned long)(queue->queue_pages)[f]);
+	}
+	return 0;
+}
+
+int ipz_queue_dtor(struct ipz_queue *queue)
+{
+	int pages_per_kpage = PAGE_SIZE >> EHCA_PAGESHIFT;
+	int g;
+	int nr_pages;
+
+	if (!queue || !queue->queue_pages) {
+		ehca_gen_dbg("queue or queue_pages is NULL");
+		return 0;
+	}
+	nr_pages = queue->queue_length / queue->pagesize;
+	for (g = 0; g < nr_pages; g += pages_per_kpage)
+		free_page((unsigned long)(queue->queue_pages)[g]);
+	vfree(queue->queue_pages);
+
+	return 1;
+}
diff --git a/drivers/infiniband/hw/ehca/ipz_pt_fn.h b/drivers/infiniband/hw/ehca/ipz_pt_fn.h
new file mode 100644
index 0000000..2f13509
--- /dev/null
+++ b/drivers/infiniband/hw/ehca/ipz_pt_fn.h
@@ -0,0 +1,247 @@
+/*
+ *  IBM eServer eHCA Infiniband device driver for Linux on POWER
+ *
+ *  internal queue handling
+ *
+ *  Authors: Waleri Fomin <fomin@de.ibm.com>
+ *           Reinhard Ernst <rernst@de.ibm.com>
+ *           Christoph Raisch <raisch@de.ibm.com>
+ *
+ *  Copyright (c) 2005 IBM Corporation
+ *
+ *  All rights reserved.
+ *
+ *  This source code is distributed under a dual license of GPL v2.0 and OpenIB
+ *  BSD.
+ *
+ * OpenIB BSD License
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * Redistributions of source code must retain the above copyright notice, this
+ * list of conditions and the following disclaimer.
+ *
+ * Redistributions in binary form must reproduce the above copyright notice,
+ * this list of conditions and the following disclaimer in the documentation
+ * and/or other materials
+ * provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
+ * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#ifndef __IPZ_PT_FN_H__
+#define __IPZ_PT_FN_H__
+
+#define EHCA_PAGESHIFT   12
+#define EHCA_PAGESIZE   4096UL
+#define EHCA_PAGEMASK   (~(EHCA_PAGESIZE-1))
+#define EHCA_PT_ENTRIES 512UL
+
+#include "ehca_tools.h"
+#include "ehca_qes.h"
+
+/* struct generic ehca page */
+struct ipz_page {
+	u8 entries[EHCA_PAGESIZE];
+};
+
+/* struct generic queue in linux kernel virtual memory (kv) */
+struct ipz_queue {
+	u64 current_q_offset;	/* current queue entry */
+
+	struct ipz_page **queue_pages;	/* array of pages belonging to queue */
+	u32 qe_size;		/* queue entry size */
+	u32 act_nr_of_sg;
+	u32 queue_length;	/* queue length allocated in bytes */
+	u32 pagesize;
+	u32 toggle_state;	/* toggle flag - per page */
+	u32 dummy3;		/* 64 bit alignment */
+};
+
+/*
+ * return current Queue Entry for a certain q_offset
+ * returns address (kv) of Queue Entry
+ */
+static inline void *ipz_qeit_calc(struct ipz_queue *queue, u64 q_offset)
+{
+	struct ipz_page *current_page;
+	if (q_offset >= queue->queue_length)
+		return NULL;
+	current_page = (queue->queue_pages)[q_offset >> EHCA_PAGESHIFT];
+	return  &current_page->entries[q_offset & (EHCA_PAGESIZE - 1)];
+}
+
+/*
+ * return current Queue Entry
+ * returns address (kv) of Queue Entry
+ */
+static inline void *ipz_qeit_get(struct ipz_queue *queue)
+{
+	return ipz_qeit_calc(queue, queue->current_q_offset);
+}
+
+/*
+ * return current Queue Page , increment Queue Page iterator from
+ * page to page in struct ipz_queue, last increment will return 0! and
+ * NOT wrap
+ * returns address (kv) of Queue Page
+ * warning don't use in parallel with ipz_QE_get_inc()
+ */
+void *ipz_qpageit_get_inc(struct ipz_queue *queue);
+
+/*
+ * return current Queue Entry, increment Queue Entry iterator by one
+ * step in struct ipz_queue, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * warning don't use in parallel with ipz_qpageit_get_inc()
+ * warning unpredictable results may occur if steps>act_nr_of_queue_entries
+ */
+static inline void *ipz_qeit_get_inc(struct ipz_queue *queue)
+{
+	void *ret = ipz_qeit_get(queue);
+	queue->current_q_offset += queue->qe_size;
+	if (queue->current_q_offset >= queue->queue_length) {
+		queue->current_q_offset = 0;
+		/* toggle the valid flag */
+		queue->toggle_state = (~queue->toggle_state) & 1;
+	}
+
+	return ret;
+}
+
+/*
+ * return current Queue Entry, increment Queue Entry iterator by one
+ * step in struct ipz_queue, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * returns 0 and does not increment, if wrong valid state
+ * warning don't use in parallel with ipz_qpageit_get_inc()
+ * warning unpredictable results may occur if steps>act_nr_of_queue_entries
+ */
+static inline void *ipz_qeit_get_inc_valid(struct ipz_queue *queue)
+{
+	struct ehca_cqe *cqe = ipz_qeit_get(queue);
+	u32 cqe_flags = cqe->cqe_flags;
+
+	if ((cqe_flags >> 7) != (queue->toggle_state & 1))
+		return NULL;
+
+	ipz_qeit_get_inc(queue);
+	return cqe;
+}
+
+/*
+ * returns and resets Queue Entry iterator
+ * returns address (kv) of first Queue Entry
+ */
+static inline void *ipz_qeit_reset(struct ipz_queue *queue)
+{
+	queue->current_q_offset = 0;
+	return ipz_qeit_get(queue);
+}
+
+/* struct generic page table */
+struct ipz_pt {
+	u64 entries[EHCA_PT_ENTRIES];
+};
+
+/* struct page table for a queue, only to be used in pf */
+struct ipz_qpt {
+	/* queue page tables (kv), use u64 because we know the element length */
+	u64 *qpts;
+	u32 n_qpts;
+	u32 n_ptes;       /*  number of page table entries */
+	u64 *current_pte_addr;
+};
+
+/*
+ * constructor for a ipz_queue_t, placement new for ipz_queue_t,
+ * new for all dependent datastructors
+ * all QP Tables are the same
+ * flow:
+ *    allocate+pin queue
+ * see ipz_qpt_ctor()
+ * returns true if ok, false if out of memory
+ */
+int ipz_queue_ctor(struct ipz_queue *queue, const u32 nr_of_pages,
+		   const u32 pagesize, const u32 qe_size,
+		   const u32 nr_of_sg);
+
+/*
+ * destructor for a ipz_queue_t
+ *  -# free queue
+ *  see ipz_queue_ctor()
+ *  returns true if ok, false if queue was NULL-ptr of free failed
+ */
+int ipz_queue_dtor(struct ipz_queue *queue);
+
+/*
+ * constructor for a ipz_qpt_t,
+ * placement new for struct ipz_queue, new for all dependent datastructors
+ * all QP Tables are the same,
+ * flow:
+ * -# allocate+pin queue
+ * -# initialise ptcb
+ * -# allocate+pin PTs
+ * -# link PTs to a ring, according to HCA Arch, set bit62 id needed
+ * -# the ring must have room for exactly nr_of_PTEs
+ * see ipz_qpt_ctor()
+ */
+void ipz_qpt_ctor(struct ipz_qpt *qpt,
+		  const u32 nr_of_qes,
+		  const u32 pagesize,
+		  const u32 qe_size,
+		  const u8 lowbyte, const u8 toggle,
+		  u32 * act_nr_of_QEs, u32 * act_nr_of_pages);
+
+/*
+ * return current Queue Entry, increment Queue Entry iterator by one
+ * step in struct ipz_queue, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * warning don't use in parallel with ipz_qpageit_get_inc()
+ * warning unpredictable results may occur if steps>act_nr_of_queue_entries
+ * fix EQ page problems
+ */
+void *ipz_qeit_eq_get_inc(struct ipz_queue *queue);
+
+/*
+ * return current Event Queue Entry, increment Queue Entry iterator
+ * by one step in struct ipz_queue if valid, will wrap in ringbuffer
+ * returns address (kv) of Queue Entry BEFORE increment
+ * returns 0 and does not increment, if wrong valid state
+ * warning don't use in parallel with ipz_queue_QPageit_get_inc()
+ * warning unpredictable results may occur if steps>act_nr_of_queue_entries
+ */
+static inline void *ipz_eqit_eq_get_inc_valid(struct ipz_queue *queue)
+{
+	void *ret = ipz_qeit_get(queue);
+	u32 qe = *(u8 *) ret;
+	if ((qe >> 7) != (queue->toggle_state & 1))
+		return NULL;
+	ipz_qeit_eq_get_inc(queue); /* this is a good one */
+	return ret;
+}
+
+/* returns address (GX) of first queue entry */
+static inline u64 ipz_qpt_get_firstpage(struct ipz_qpt *qpt)
+{
+	return be64_to_cpu(qpt->qpts[0]);
+}
+
+/* returns address (kv) of first page of queue page table */
+static inline void *ipz_qpt_get_qpt(struct ipz_qpt *qpt)
+{
+	return qpt->qpts;
+}
+
+#endif				/* __IPZ_PT_FN_H__ */
diff --git a/drivers/infiniband/hw/ipath/Kconfig b/drivers/infiniband/hw/ipath/Kconfig
index 1db9489..574a678 100644
--- a/drivers/infiniband/hw/ipath/Kconfig
+++ b/drivers/infiniband/hw/ipath/Kconfig
@@ -1,16 +1,9 @@
-config IPATH_CORE
-	tristate "QLogic InfiniPath Driver"
-	depends on 64BIT && PCI_MSI && NET
-	---help---
-	This is a low-level driver for QLogic InfiniPath host channel
-	adapters (HCAs) based on the HT-400 and PE-800 chips.
-
 config INFINIBAND_IPATH
-	tristate "QLogic InfiniPath Verbs Driver"
-	depends on IPATH_CORE && INFINIBAND
+	tristate "QLogic InfiniPath Driver"
+	depends on PCI_MSI && 64BIT && INFINIBAND
 	---help---
-	This is a driver that provides InfiniBand verbs support for
-	QLogic InfiniPath host channel adapters (HCAs).  This
-	allows these devices to be used with both kernel upper level
-	protocols such as IP-over-InfiniBand as well as with userspace
-	applications (in conjunction with InfiniBand userspace access).
+	This is a driver for QLogic InfiniPath host channel adapters,
+	including InfiniBand verbs support.  This driver allows these
+	devices to be used with both kernel upper level protocols such
+	as IP-over-InfiniBand as well as with userspace applications
+	(in conjunction with InfiniBand userspace access).
diff --git a/drivers/infiniband/hw/ipath/Makefile b/drivers/infiniband/hw/ipath/Makefile
index b0bf728..5e29cb0 100644
--- a/drivers/infiniband/hw/ipath/Makefile
+++ b/drivers/infiniband/hw/ipath/Makefile
@@ -1,36 +1,35 @@
 EXTRA_CFLAGS += -DIPATH_IDSTR='"QLogic kernel.org driver"' \
 	-DIPATH_KERN_TYPE=0
 
-obj-$(CONFIG_IPATH_CORE) += ipath_core.o
 obj-$(CONFIG_INFINIBAND_IPATH) += ib_ipath.o
 
-ipath_core-y := \
+ib_ipath-y := \
+	ipath_cq.o \
 	ipath_diag.o \
 	ipath_driver.o \
 	ipath_eeprom.o \
 	ipath_file_ops.o \
 	ipath_fs.o \
-	ipath_ht400.o \
+	ipath_iba6110.o \
+	ipath_iba6120.o \
 	ipath_init_chip.o \
 	ipath_intr.o \
-	ipath_layer.o \
-	ipath_pe800.o \
-	ipath_stats.o \
-	ipath_sysfs.o \
-	ipath_user_pages.o
-
-ipath_core-$(CONFIG_X86_64) += ipath_wc_x86_64.o
-
-ib_ipath-y := \
-	ipath_cq.o \
 	ipath_keys.o \
+	ipath_layer.o \
 	ipath_mad.o \
+	ipath_mmap.o \
 	ipath_mr.o \
 	ipath_qp.o \
 	ipath_rc.o \
 	ipath_ruc.o \
 	ipath_srq.o \
+	ipath_stats.o \
+	ipath_sysfs.o \
 	ipath_uc.o \
 	ipath_ud.o \
-	ipath_verbs.o \
-	ipath_verbs_mcast.o
+	ipath_user_pages.o \
+	ipath_verbs_mcast.o \
+	ipath_verbs.o
+
+ib_ipath-$(CONFIG_X86_64) += ipath_wc_x86_64.o
+ib_ipath-$(CONFIG_PPC64) += ipath_wc_ppc64.o
diff --git a/drivers/infiniband/hw/ipath/ipath_common.h b/drivers/infiniband/hw/ipath/ipath_common.h
index 062bd39..f577905 100644
--- a/drivers/infiniband/hw/ipath/ipath_common.h
+++ b/drivers/infiniband/hw/ipath/ipath_common.h
@@ -106,9 +106,9 @@ struct infinipath_stats {
 	__u64 sps_ether_spkts;
 	/* number of "ethernet" packets received by driver */
 	__u64 sps_ether_rpkts;
-	/* number of SMA packets sent by driver */
+	/* number of SMA packets sent by driver. Obsolete. */
 	__u64 sps_sma_spkts;
-	/* number of SMA packets received by driver */
+	/* number of SMA packets received by driver. Obsolete. */
 	__u64 sps_sma_rpkts;
 	/* number of times all ports rcvhdrq was full and packet dropped */
 	__u64 sps_hdrqfull;
@@ -138,7 +138,7 @@ struct infinipath_stats {
 	__u64 sps_pageunlocks;
 	/*
 	 * Number of packets dropped in kernel other than errors (ether
-	 * packets if ipath not configured, sma/mad, etc.)
+	 * packets if ipath not configured, etc.)
 	 */
 	__u64 sps_krdrops;
 	/* pad for future growth */
@@ -153,8 +153,6 @@ struct infinipath_stats {
 #define IPATH_STATUS_DISABLED      0x2	/* hardware disabled */
 /* Device has been disabled via admin request */
 #define IPATH_STATUS_ADMIN_DISABLED    0x4
-#define IPATH_STATUS_OIB_SMA       0x8	/* ipath_mad kernel SMA running */
-#define IPATH_STATUS_SMA          0x10	/* user SMA running */
 /* Chip has been found and initted */
 #define IPATH_STATUS_CHIP_PRESENT 0x20
 /* IB link is at ACTIVE, usable for data traffic */
@@ -465,12 +463,11 @@ struct __ipath_sendpkt {
 	struct ipath_iovec sps_iov[4];
 };
 
-/* Passed into SMA special file's ->read and ->write methods. */
-struct ipath_sma_pkt
-{
-	__u32 unit;	/* unit on which to send packet */
-	__u64 data;	/* address of payload in userspace */
-	__u32 len;	/* length of payload */
+/* Passed into diag data special file's ->write method. */
+struct ipath_diag_pkt {
+	__u32 unit;
+	__u64 data;
+	__u32 len;
 };
 
 /*
diff --git a/drivers/infiniband/hw/ipath/ipath_cq.c b/drivers/infiniband/hw/ipath/ipath_cq.c
index 3efee34..049221b 100644
--- a/drivers/infiniband/hw/ipath/ipath_cq.c
+++ b/drivers/infiniband/hw/ipath/ipath_cq.c
@@ -42,20 +42,28 @@
  * @entry: work completion entry to add
  * @sig: true if @entry is a solicitated entry
  *
- * This may be called with one of the qp->s_lock or qp->r_rq.lock held.
+ * This may be called with qp->s_lock held.
  */
 void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int solicited)
 {
+	struct ipath_cq_wc *wc = cq->queue;
 	unsigned long flags;
+	u32 head;
 	u32 next;
 
 	spin_lock_irqsave(&cq->lock, flags);
 
-	if (cq->head == cq->ibcq.cqe)
+	/*
+	 * Note that the head pointer might be writable by user processes.
+	 * Take care to verify it is a sane value.
+	 */
+	head = wc->head;
+	if (head >= (unsigned) cq->ibcq.cqe) {
+		head = cq->ibcq.cqe;
 		next = 0;
-	else
-		next = cq->head + 1;
-	if (unlikely(next == cq->tail)) {
+	} else
+		next = head + 1;
+	if (unlikely(next == wc->tail)) {
 		spin_unlock_irqrestore(&cq->lock, flags);
 		if (cq->ibcq.event_handler) {
 			struct ib_event ev;
@@ -67,8 +75,8 @@ void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int solicited)
 		}
 		return;
 	}
-	cq->queue[cq->head] = *entry;
-	cq->head = next;
+	wc->queue[head] = *entry;
+	wc->head = next;
 
 	if (cq->notify == IB_CQ_NEXT_COMP ||
 	    (cq->notify == IB_CQ_SOLICITED && solicited)) {
@@ -101,19 +109,20 @@ void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int solicited)
 int ipath_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry)
 {
 	struct ipath_cq *cq = to_icq(ibcq);
+	struct ipath_cq_wc *wc = cq->queue;
 	unsigned long flags;
 	int npolled;
 
 	spin_lock_irqsave(&cq->lock, flags);
 
 	for (npolled = 0; npolled < num_entries; ++npolled, ++entry) {
-		if (cq->tail == cq->head)
+		if (wc->tail == wc->head)
 			break;
-		*entry = cq->queue[cq->tail];
-		if (cq->tail == cq->ibcq.cqe)
-			cq->tail = 0;
+		*entry = wc->queue[wc->tail];
+		if (wc->tail >= cq->ibcq.cqe)
+			wc->tail = 0;
 		else
-			cq->tail++;
+			wc->tail++;
 	}
 
 	spin_unlock_irqrestore(&cq->lock, flags);
@@ -160,38 +169,74 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries,
 {
 	struct ipath_ibdev *dev = to_idev(ibdev);
 	struct ipath_cq *cq;
-	struct ib_wc *wc;
+	struct ipath_cq_wc *wc;
 	struct ib_cq *ret;
 
-	if (entries > ib_ipath_max_cqes) {
+	if (entries < 1 || entries > ib_ipath_max_cqes) {
 		ret = ERR_PTR(-EINVAL);
-		goto bail;
+		goto done;
 	}
 
 	if (dev->n_cqs_allocated == ib_ipath_max_cqs) {
 		ret = ERR_PTR(-ENOMEM);
-		goto bail;
+		goto done;
 	}
 
-	/*
-	 * Need to use vmalloc() if we want to support large #s of
-	 * entries.
-	 */
+	/* Allocate the completion queue structure. */
 	cq = kmalloc(sizeof(*cq), GFP_KERNEL);
 	if (!cq) {
 		ret = ERR_PTR(-ENOMEM);
-		goto bail;
+		goto done;
 	}
 
 	/*
-	 * Need to use vmalloc() if we want to support large #s of entries.
+	 * Allocate the completion queue entries and head/tail pointers.
+	 * This is allocated separately so that it can be resized and
+	 * also mapped into user space.
+	 * We need to use vmalloc() in order to support mmap and large
+	 * numbers of entries.
 	 */
-	wc = vmalloc(sizeof(*wc) * (entries + 1));
+	wc = vmalloc_user(sizeof(*wc) + sizeof(struct ib_wc) * entries);
 	if (!wc) {
-		kfree(cq);
 		ret = ERR_PTR(-ENOMEM);
-		goto bail;
+		goto bail_cq;
 	}
+
+	/*
+	 * Return the address of the WC as the offset to mmap.
+	 * See ipath_mmap() for details.
+	 */
+	if (udata && udata->outlen >= sizeof(__u64)) {
+		struct ipath_mmap_info *ip;
+		__u64 offset = (__u64) wc;
+		int err;
+
+		err = ib_copy_to_udata(udata, &offset, sizeof(offset));
+		if (err) {
+			ret = ERR_PTR(err);
+			goto bail_wc;
+		}
+
+		/* Allocate info for ipath_mmap(). */
+		ip = kmalloc(sizeof(*ip), GFP_KERNEL);
+		if (!ip) {
+			ret = ERR_PTR(-ENOMEM);
+			goto bail_wc;
+		}
+		cq->ip = ip;
+		ip->context = context;
+		ip->obj = wc;
+		kref_init(&ip->ref);
+		ip->mmap_cnt = 0;
+		ip->size = PAGE_ALIGN(sizeof(*wc) +
+				      sizeof(struct ib_wc) * entries);
+		spin_lock_irq(&dev->pending_lock);
+		ip->next = dev->pending_mmaps;
+		dev->pending_mmaps = ip;
+		spin_unlock_irq(&dev->pending_lock);
+	} else
+		cq->ip = NULL;
+
 	/*
 	 * ib_create_cq() will initialize cq->ibcq except for cq->ibcq.cqe.
 	 * The number of entries should be >= the number requested or return
@@ -202,15 +247,22 @@ struct ib_cq *ipath_create_cq(struct ib_device *ibdev, int entries,
 	cq->triggered = 0;
 	spin_lock_init(&cq->lock);
 	tasklet_init(&cq->comptask, send_complete, (unsigned long)cq);
-	cq->head = 0;
-	cq->tail = 0;
+	wc->head = 0;
+	wc->tail = 0;
 	cq->queue = wc;
 
 	ret = &cq->ibcq;
 
 	dev->n_cqs_allocated++;
+	goto done;
 
-bail:
+bail_wc:
+	vfree(wc);
+
+bail_cq:
+	kfree(cq);
+
+done:
 	return ret;
 }
 
@@ -229,7 +281,10 @@ int ipath_destroy_cq(struct ib_cq *ibcq)
 
 	tasklet_kill(&cq->comptask);
 	dev->n_cqs_allocated--;
-	vfree(cq->queue);
+	if (cq->ip)
+		kref_put(&cq->ip->ref, ipath_release_mmap_info);
+	else
+		vfree(cq->queue);
 	kfree(cq);
 
 	return 0;
@@ -253,7 +308,7 @@ int ipath_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
 	spin_lock_irqsave(&cq->lock, flags);
 	/*
 	 * Don't change IB_CQ_NEXT_COMP to IB_CQ_SOLICITED but allow
-	 * any other transitions.
+	 * any other transitions (see C11-31 and C11-32 in ch. 11.4.2.2).
 	 */
 	if (cq->notify != IB_CQ_NEXT_COMP)
 		cq->notify = notify;
@@ -264,46 +319,86 @@ int ipath_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
 int ipath_resize_cq(struct ib_cq *ibcq, int cqe, struct ib_udata *udata)
 {
 	struct ipath_cq *cq = to_icq(ibcq);
-	struct ib_wc *wc, *old_wc;
-	u32 n;
+	struct ipath_cq_wc *old_wc = cq->queue;
+	struct ipath_cq_wc *wc;
+	u32 head, tail, n;
 	int ret;
 
+	if (cqe < 1 || cqe > ib_ipath_max_cqes) {
+		ret = -EINVAL;
+		goto bail;
+	}
+
 	/*
 	 * Need to use vmalloc() if we want to support large #s of entries.
 	 */
-	wc = vmalloc(sizeof(*wc) * (cqe + 1));
+	wc = vmalloc_user(sizeof(*wc) + sizeof(struct ib_wc) * cqe);
 	if (!wc) {
 		ret = -ENOMEM;
 		goto bail;
 	}
 
+	/*
+	 * Return the address of the WC as the offset to mmap.
+	 * See ipath_mmap() for details.
+	 */
+	if (udata && udata->outlen >= sizeof(__u64)) {
+		__u64 offset = (__u64) wc;
+
+		ret = ib_copy_to_udata(udata, &offset, sizeof(offset));
+		if (ret)
+			goto bail;
+	}
+
 	spin_lock_irq(&cq->lock);
-	if (cq->head < cq->tail)
-		n = cq->ibcq.cqe + 1 + cq->head - cq->tail;
+	/*
+	 * Make sure head and tail are sane since they
+	 * might be user writable.
+	 */
+	head = old_wc->head;
+	if (head > (u32) cq->ibcq.cqe)
+		head = (u32) cq->ibcq.cqe;
+	tail = old_wc->tail;
+	if (tail > (u32) cq->ibcq.cqe)
+		tail = (u32) cq->ibcq.cqe;
+	if (head < tail)
+		n = cq->ibcq.cqe + 1 + head - tail;
 	else
-		n = cq->head - cq->tail;
+		n = head - tail;
 	if (unlikely((u32)cqe < n)) {
 		spin_unlock_irq(&cq->lock);
 		vfree(wc);
 		ret = -EOVERFLOW;
 		goto bail;
 	}
-	for (n = 0; cq->tail != cq->head; n++) {
-		wc[n] = cq->queue[cq->tail];
-		if (cq->tail == cq->ibcq.cqe)
-			cq->tail = 0;
+	for (n = 0; tail != head; n++) {
+		wc->queue[n] = old_wc->queue[tail];
+		if (tail == (u32) cq->ibcq.cqe)
+			tail = 0;
 		else
-			cq->tail++;
+			tail++;
 	}
 	cq->ibcq.cqe = cqe;
-	cq->head = n;
-	cq->tail = 0;
-	old_wc = cq->queue;
+	wc->head = n;
+	wc->tail = 0;
 	cq->queue = wc;
 	spin_unlock_irq(&cq->lock);
 
 	vfree(old_wc);
 
+	if (cq->ip) {
+		struct ipath_ibdev *dev = to_idev(ibcq->device);
+		struct ipath_mmap_info *ip = cq->ip;
+
+		ip->obj = wc;
+		ip->size = PAGE_ALIGN(sizeof(*wc) +
+				      sizeof(struct ib_wc) * cqe);
+		spin_lock_irq(&dev->pending_lock);
+		ip->next = dev->pending_mmaps;
+		dev->pending_mmaps = ip;
+		spin_unlock_irq(&dev->pending_lock);
+	}
+
 	ret = 0;
 
 bail:
diff --git a/drivers/infiniband/hw/ipath/ipath_debug.h b/drivers/infiniband/hw/ipath/ipath_debug.h
index f415bed..df69f0d8 100644
--- a/drivers/infiniband/hw/ipath/ipath_debug.h
+++ b/drivers/infiniband/hw/ipath/ipath_debug.h
@@ -60,7 +60,6 @@
 #define __IPATH_USER_SEND   0x1000	/* use user mode send */
 #define __IPATH_KERNEL_SEND 0x2000	/* use kernel mode send */
 #define __IPATH_EPKTDBG     0x4000	/* print ethernet packet data */
-#define __IPATH_SMADBG      0x8000	/* sma packet debug */
 #define __IPATH_IPATHDBG    0x10000	/* Ethernet (IPATH) gen debug */
 #define __IPATH_IPATHWARN   0x20000	/* Ethernet (IPATH) warnings */
 #define __IPATH_IPATHERR    0x40000	/* Ethernet (IPATH) errors */
@@ -84,7 +83,6 @@
 /* print mmap/nopage stuff, not using VDBG any more */
 #define __IPATH_MMDBG     0x0
 #define __IPATH_EPKTDBG   0x0	/* print ethernet packet data */
-#define __IPATH_SMADBG    0x0   /* process startup (init)/exit messages */
 #define __IPATH_IPATHDBG  0x0	/* Ethernet (IPATH) table dump on */
 #define __IPATH_IPATHWARN 0x0	/* Ethernet (IPATH) warnings on   */
 #define __IPATH_IPATHERR  0x0	/* Ethernet (IPATH) errors on   */
diff --git a/drivers/infiniband/hw/ipath/ipath_diag.c b/drivers/infiniband/hw/ipath/ipath_diag.c
index 147dd89..29958b6 100644
--- a/drivers/infiniband/hw/ipath/ipath_diag.c
+++ b/drivers/infiniband/hw/ipath/ipath_diag.c
@@ -41,11 +41,12 @@
  * through the /sys/bus/pci resource mmap interface.
  */
 
+#include <linux/io.h>
 #include <linux/pci.h>
+#include <linux/vmalloc.h>
 #include <asm/uaccess.h>
 
 #include "ipath_kernel.h"
-#include "ipath_layer.h"
 #include "ipath_common.h"
 
 int ipath_diag_inuse;
@@ -274,6 +275,158 @@ bail:
 	return ret;
 }
 
+static ssize_t ipath_diagpkt_write(struct file *fp,
+				   const char __user *data,
+				   size_t count, loff_t *off);
+
+static struct file_operations diagpkt_file_ops = {
+	.owner = THIS_MODULE,
+	.write = ipath_diagpkt_write,
+};
+
+static struct cdev *diagpkt_cdev;
+static struct class_device *diagpkt_class_dev;
+
+int __init ipath_diagpkt_add(void)
+{
+	return ipath_cdev_init(IPATH_DIAGPKT_MINOR,
+			       "ipath_diagpkt", &diagpkt_file_ops,
+			       &diagpkt_cdev, &diagpkt_class_dev);
+}
+
+void __exit ipath_diagpkt_remove(void)
+{
+	ipath_cdev_cleanup(&diagpkt_cdev, &diagpkt_class_dev);
+}
+
+/**
+ * ipath_diagpkt_write - write an IB packet
+ * @fp: the diag data device file pointer
+ * @data: ipath_diag_pkt structure saying where to get the packet
+ * @count: size of data to write
+ * @off: unused by this code
+ */
+static ssize_t ipath_diagpkt_write(struct file *fp,
+				   const char __user *data,
+				   size_t count, loff_t *off)
+{
+	u32 __iomem *piobuf;
+	u32 plen, clen, pbufn;
+	struct ipath_diag_pkt dp;
+	u32 *tmpbuf = NULL;
+	struct ipath_devdata *dd;
+	ssize_t ret = 0;
+	u64 val;
+
+	if (count < sizeof(dp)) {
+		ret = -EINVAL;
+		goto bail;
+	}
+
+	if (copy_from_user(&dp, data, sizeof(dp))) {
+		ret = -EFAULT;
+		goto bail;
+	}
+
+	/* send count must be an exact number of dwords */
+	if (dp.len & 3) {
+		ret = -EINVAL;
+		goto bail;
+	}
+
+	clen = dp.len >> 2;
+
+	dd = ipath_lookup(dp.unit);
+	if (!dd || !(dd->ipath_flags & IPATH_PRESENT) ||
+	    !dd->ipath_kregbase) {
+		ipath_cdbg(VERBOSE, "illegal unit %u for diag data send\n",
+			   dp.unit);
+		ret = -ENODEV;
+		goto bail;
+	}
+
+	if (ipath_diag_inuse && !diag_set_link &&
+	    !(dd->ipath_flags & IPATH_LINKACTIVE)) {
+		diag_set_link = 1;
+		ipath_cdbg(VERBOSE, "Trying to set to set link active for "
+			   "diag pkt\n");
+		ipath_set_linkstate(dd, IPATH_IB_LINKARM);
+		ipath_set_linkstate(dd, IPATH_IB_LINKACTIVE);
+	}
+
+	if (!(dd->ipath_flags & IPATH_INITTED)) {
+		/* no hardware, freeze, etc. */
+		ipath_cdbg(VERBOSE, "unit %u not usable\n", dd->ipath_unit);
+		ret = -ENODEV;
+		goto bail;
+	}
+	val = dd->ipath_lastibcstat & IPATH_IBSTATE_MASK;
+	if (val != IPATH_IBSTATE_INIT && val != IPATH_IBSTATE_ARM &&
+	    val != IPATH_IBSTATE_ACTIVE) {
+		ipath_cdbg(VERBOSE, "unit %u not ready (state %llx)\n",
+			   dd->ipath_unit, (unsigned long long) val);
+		ret = -EINVAL;
+		goto bail;
+	}
+
+	/* need total length before first word written */
+	/* +1 word is for the qword padding */
+	plen = sizeof(u32) + dp.len;
+
+	if ((plen + 4) > dd->ipath_ibmaxlen) {
+		ipath_dbg("Pkt len 0x%x > ibmaxlen %x\n",
+			  plen - 4, dd->ipath_ibmaxlen);
+		ret = -EINVAL;
+		goto bail;	/* before writing pbc */
+	}
+	tmpbuf = vmalloc(plen);
+	if (!tmpbuf) {
+		dev_info(&dd->pcidev->dev, "Unable to allocate tmp buffer, "
+			 "failing\n");
+		ret = -ENOMEM;
+		goto bail;
+	}
+
+	if (copy_from_user(tmpbuf,
+			   (const void __user *) (unsigned long) dp.data,
+			   dp.len)) {
+		ret = -EFAULT;
+		goto bail;
+	}
+
+	piobuf = ipath_getpiobuf(dd, &pbufn);
+	if (!piobuf) {
+		ipath_cdbg(VERBOSE, "No PIO buffers avail unit for %u\n",
+			   dd->ipath_unit);
+		ret = -EBUSY;
+		goto bail;
+	}
+
+	plen >>= 2;		/* in dwords */
+
+	if (ipath_debug & __IPATH_PKTDBG)
+		ipath_cdbg(VERBOSE, "unit %u 0x%x+1w pio%d\n",
+			   dd->ipath_unit, plen - 1, pbufn);
+
+	/* we have to flush after the PBC for correctness on some cpus
+	 * or WC buffer can be written out of order */
+	writeq(plen, piobuf);
+	ipath_flush_wc();
+	/* copy all by the trigger word, then flush, so it's written
+	 * to chip before trigger word, then write trigger word, then
+	 * flush again, so packet is sent. */
+	__iowrite32_copy(piobuf + 2, tmpbuf, clen - 1);
+	ipath_flush_wc();
+	__raw_writel(tmpbuf[clen - 1], piobuf + clen + 1);
+	ipath_flush_wc();
+
+	ret = sizeof(dp);
+
+bail:
+	vfree(tmpbuf);
+	return ret;
+}
+
 static int ipath_diag_release(struct inode *in, struct file *fp)
 {
 	mutex_lock(&ipath_mutex);
diff --git a/drivers/infiniband/hw/ipath/ipath_driver.c b/drivers/infiniband/hw/ipath/ipath_driver.c
index f98518d..2108466 100644
--- a/drivers/infiniband/hw/ipath/ipath_driver.c
+++ b/drivers/infiniband/hw/ipath/ipath_driver.c
@@ -39,7 +39,7 @@
 #include <linux/vmalloc.h>
 
 #include "ipath_kernel.h"
-#include "ipath_layer.h"
+#include "ipath_verbs.h"
 #include "ipath_common.h"
 
 static void ipath_update_pio_bufs(struct ipath_devdata *);
@@ -51,8 +51,6 @@ const char *ipath_get_unit_name(int unit)
 	return iname;
 }
 
-EXPORT_SYMBOL_GPL(ipath_get_unit_name);
-
 #define DRIVER_LOAD_MSG "QLogic " IPATH_DRV_NAME " loaded: "
 #define PFX IPATH_DRV_NAME ": "
 
@@ -60,13 +58,13 @@ EXPORT_SYMBOL_GPL(ipath_get_unit_name);
  * The size has to be longer than this string, so we can append
  * board/chip information to it in the init code.
  */
-const char ipath_core_version[] = IPATH_IDSTR "\n";
+const char ib_ipath_version[] = IPATH_IDSTR "\n";
 
 static struct idr unit_table;
 DEFINE_SPINLOCK(ipath_devs_lock);
 LIST_HEAD(ipath_dev_list);
 
-wait_queue_head_t ipath_sma_state_wait;
+wait_queue_head_t ipath_state_wait;
 
 unsigned ipath_debug = __IPATH_INFO;
 
@@ -403,10 +401,10 @@ static int __devinit ipath_init_one(struct pci_dev *pdev,
 	/* setup the chip-specific functions, as early as possible. */
 	switch (ent->device) {
 	case PCI_DEVICE_ID_INFINIPATH_HT:
-		ipath_init_ht400_funcs(dd);
+		ipath_init_iba6110_funcs(dd);
 		break;
 	case PCI_DEVICE_ID_INFINIPATH_PE800:
-		ipath_init_pe800_funcs(dd);
+		ipath_init_iba6120_funcs(dd);
 		break;
 	default:
 		ipath_dev_err(dd, "Found unknown QLogic deviceid 0x%x, "
@@ -440,7 +438,13 @@ static int __devinit ipath_init_one(struct pci_dev *pdev,
 	}
 	dd->ipath_pcirev = rev;
 
+#if defined(__powerpc__)
+	/* There isn't a generic way to specify writethrough mappings */
+	dd->ipath_kregbase = __ioremap(addr, len,
+		(_PAGE_NO_CACHE|_PAGE_WRITETHRU));
+#else
 	dd->ipath_kregbase = ioremap_nocache(addr, len);
+#endif
 
 	if (!dd->ipath_kregbase) {
 		ipath_dbg("Unable to map io addr %llx to kvirt, failing\n",
@@ -503,7 +507,7 @@ static int __devinit ipath_init_one(struct pci_dev *pdev,
 	ipathfs_add_device(dd);
 	ipath_user_add(dd);
 	ipath_diag_add(dd);
-	ipath_layer_add(dd);
+	ipath_register_ib_device(dd);
 
 	goto bail;
 
@@ -532,7 +536,7 @@ static void __devexit ipath_remove_one(struct pci_dev *pdev)
 		return;
 
 	dd = pci_get_drvdata(pdev);
-	ipath_layer_remove(dd);
+	ipath_unregister_ib_device(dd->verbs_dev);
 	ipath_diag_remove(dd);
 	ipath_user_remove(dd);
 	ipathfs_remove_device(dd);
@@ -607,21 +611,23 @@ void ipath_disarm_piobufs(struct ipath_devdata *dd, unsigned first,
  *
  * wait up to msecs milliseconds for IB link state change to occur for
  * now, take the easy polling route.  Currently used only by
- * ipath_layer_set_linkstate.  Returns 0 if state reached, otherwise
+ * ipath_set_linkstate.  Returns 0 if state reached, otherwise
  * -ETIMEDOUT state can have multiple states set, for any of several
  * transitions.
  */
-int ipath_wait_linkstate(struct ipath_devdata *dd, u32 state, int msecs)
+static int ipath_wait_linkstate(struct ipath_devdata *dd, u32 state,
+				int msecs)
 {
-	dd->ipath_sma_state_wanted = state;
-	wait_event_interruptible_timeout(ipath_sma_state_wait,
+	dd->ipath_state_wanted = state;
+	wait_event_interruptible_timeout(ipath_state_wait,
 					 (dd->ipath_flags & state),
 					 msecs_to_jiffies(msecs));
-	dd->ipath_sma_state_wanted = 0;
+	dd->ipath_state_wanted = 0;
 
 	if (!(dd->ipath_flags & state)) {
 		u64 val;
-		ipath_cdbg(SMA, "Didn't reach linkstate %s within %u ms\n",
+		ipath_cdbg(VERBOSE, "Didn't reach linkstate %s within %u"
+			   " ms\n",
 			   /* test INIT ahead of DOWN, both can be set */
 			   (state & IPATH_LINKINIT) ? "INIT" :
 			   ((state & IPATH_LINKDOWN) ? "DOWN" :
@@ -807,58 +813,6 @@ bail:
 	return skb;
 }
 
-/**
- * ipath_rcv_layer - receive a packet for the layered (ethernet) driver
- * @dd: the infinipath device
- * @etail: the sk_buff number
- * @tlen: the total packet length
- * @hdr: the ethernet header
- *
- * Separate routine for better overall optimization
- */
-static void ipath_rcv_layer(struct ipath_devdata *dd, u32 etail,
-			    u32 tlen, struct ether_header *hdr)
-{
-	u32 elen;
-	u8 pad, *bthbytes;
-	struct sk_buff *skb, *nskb;
-
-	if (dd->ipath_port0_skbs &&
-			hdr->sub_opcode == IPATH_ITH4X_OPCODE_ENCAP) {
-		/*
-		 * Allocate a new sk_buff to replace the one we give
-		 * to the network stack.
-		 */
-		nskb = ipath_alloc_skb(dd, GFP_ATOMIC);
-		if (!nskb) {
-			/* count OK packets that we drop */
-			ipath_stats.sps_krdrops++;
-			return;
-		}
-
-		bthbytes = (u8 *) hdr->bth;
-		pad = (bthbytes[1] >> 4) & 3;
-		/* +CRC32 */
-		elen = tlen - (sizeof(*hdr) + pad + sizeof(u32));
-
-		skb = dd->ipath_port0_skbs[etail];
-		dd->ipath_port0_skbs[etail] = nskb;
-		skb_put(skb, elen);
-
-		dd->ipath_f_put_tid(dd, etail + (u64 __iomem *)
-				    ((char __iomem *) dd->ipath_kregbase
-				     + dd->ipath_rcvegrbase), 0,
-				    virt_to_phys(nskb->data));
-
-		__ipath_layer_rcv(dd, hdr, skb);
-
-		/* another ether packet received */
-		ipath_stats.sps_ether_rpkts++;
-	}
-	else if (hdr->sub_opcode == IPATH_ITH4X_OPCODE_LID_ARP)
-		__ipath_layer_rcv_lid(dd, hdr);
-}
-
 static void ipath_rcv_hdrerr(struct ipath_devdata *dd,
 			     u32 eflags,
 			     u32 l,
@@ -972,26 +926,17 @@ reloop:
 		if (unlikely(eflags))
 			ipath_rcv_hdrerr(dd, eflags, l, etail, rc);
 		else if (etype == RCVHQ_RCV_TYPE_NON_KD) {
-				int ret = __ipath_verbs_rcv(dd, rc + 1,
-							    ebuf, tlen);
-				if (ret == -ENODEV)
-					ipath_cdbg(VERBOSE,
-						   "received IB packet, "
-						   "not SMA (QP=%x)\n", qp);
-				if (dd->ipath_lli_counter)
-					dd->ipath_lli_counter--;
-
-		} else if (etype == RCVHQ_RCV_TYPE_EAGER) {
-			if (qp == IPATH_KD_QP &&
-			    bthbytes[0] == ipath_layer_rcv_opcode &&
-			    ebuf)
-				ipath_rcv_layer(dd, etail, tlen,
-						(struct ether_header *)hdr);
-			else
-				ipath_cdbg(PKT, "typ %x, opcode %x (eager, "
-					   "qp=%x), len %x; ignored\n",
-					   etype, bthbytes[0], qp, tlen);
+			ipath_ib_rcv(dd->verbs_dev, rc + 1, ebuf, tlen);
+			if (dd->ipath_lli_counter)
+				dd->ipath_lli_counter--;
+			ipath_cdbg(PKT, "typ %x, opcode %x (eager, "
+				   "qp=%x), len %x; ignored\n",
+				   etype, bthbytes[0], qp, tlen);
 		}
+		else if (etype == RCVHQ_RCV_TYPE_EAGER)
+			ipath_cdbg(PKT, "typ %x, opcode %x (eager, "
+				   "qp=%x), len %x; ignored\n",
+				   etype, bthbytes[0], qp, tlen);
 		else if (etype == RCVHQ_RCV_TYPE_EXPECTED)
 			ipath_dbg("Bug: Expected TID, opcode %x; ignored\n",
 				  be32_to_cpu(hdr->bth[0]) & 0xff);
@@ -1024,7 +969,8 @@ reloop:
 		 */
 		if (l == hdrqtail || (i && !(i&0xf))) {
 			u64 lval;
-			if (l == hdrqtail) /* PE-800 interrupt only on last */
+			if (l == hdrqtail)
+				/* request IBA6120 interrupt only on last */
 				lval = dd->ipath_rhdrhead_intr_off | l;
 			else
 				lval = l;
@@ -1038,7 +984,7 @@ reloop:
 	}
 
 	if (!dd->ipath_rhdrhead_intr_off && !reloop) {
-		/* HT-400 workaround; we can have a race clearing chip
+		/* IBA6110 workaround; we can have a race clearing chip
 		 * interrupt with another interrupt about to be delivered,
 		 * and can clear it before it is delivered on the GPIO
 		 * workaround.  By doing the extra check here for the
@@ -1211,7 +1157,7 @@ int ipath_setrcvhdrsize(struct ipath_devdata *dd, unsigned rhdrsize)
  *
  * do appropriate marking as busy, etc.
  * returns buffer number if one found (>=0), negative number is error.
- * Used by ipath_sma_send_pkt and ipath_layer_send
+ * Used by ipath_layer_send
  */
 u32 __iomem *ipath_getpiobuf(struct ipath_devdata *dd, u32 * pbufnum)
 {
@@ -1317,13 +1263,6 @@ rescan:
 		goto bail;
 	}
 
-	if (updated)
-		/*
-		 * ran out of bufs, now some (at least this one we just
-		 * got) are now available, so tell the layered driver.
-		 */
-		__ipath_layer_intr(dd, IPATH_LAYER_INT_SEND_CONTINUE);
-
 	/*
 	 * set next starting place.  Since it's just an optimization,
 	 * it doesn't matter who wins on this, so no locking
@@ -1500,7 +1439,7 @@ int ipath_waitfor_mdio_cmdready(struct ipath_devdata *dd)
 	return ret;
 }
 
-void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
+static void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
 {
 	static const char *what[4] = {
 		[0] = "DOWN",
@@ -1511,7 +1450,7 @@ void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
 	int linkcmd = (which >> INFINIPATH_IBCC_LINKCMD_SHIFT) &
 			INFINIPATH_IBCC_LINKCMD_MASK;
 
-	ipath_cdbg(SMA, "Trying to move unit %u to %s, current ltstate "
+	ipath_cdbg(VERBOSE, "Trying to move unit %u to %s, current ltstate "
 		   "is %s\n", dd->ipath_unit,
 		   what[linkcmd],
 		   ipath_ibcstatus_str[
@@ -1520,7 +1459,7 @@ void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
 			    INFINIPATH_IBCS_LINKTRAININGSTATE_SHIFT) &
 			   INFINIPATH_IBCS_LINKTRAININGSTATE_MASK]);
 	/* flush all queued sends when going to DOWN or INIT, to be sure that
-	 * they don't block SMA and other MAD packets */
+	 * they don't block MAD packets */
 	if (!linkcmd || linkcmd == INFINIPATH_IBCC_LINKCMD_INIT) {
 		ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
 				 INFINIPATH_S_ABORT);
@@ -1534,6 +1473,180 @@ void ipath_set_ib_lstate(struct ipath_devdata *dd, int which)
 			 dd->ipath_ibcctrl | which);
 }
 
+int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate)
+{
+	u32 lstate;
+	int ret;
+
+	switch (newstate) {
+	case IPATH_IB_LINKDOWN:
+		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_POLL <<
+				    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+		/* don't wait */
+		ret = 0;
+		goto bail;
+
+	case IPATH_IB_LINKDOWN_SLEEP:
+		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_SLEEP <<
+				    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+		/* don't wait */
+		ret = 0;
+		goto bail;
+
+	case IPATH_IB_LINKDOWN_DISABLE:
+		ipath_set_ib_lstate(dd,
+				    INFINIPATH_IBCC_LINKINITCMD_DISABLE <<
+				    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
+		/* don't wait */
+		ret = 0;
+		goto bail;
+
+	case IPATH_IB_LINKINIT:
+		if (dd->ipath_flags & IPATH_LINKINIT) {
+			ret = 0;
+			goto bail;
+		}
+		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_INIT <<
+				    INFINIPATH_IBCC_LINKCMD_SHIFT);
+		lstate = IPATH_LINKINIT;
+		break;
+
+	case IPATH_IB_LINKARM:
+		if (dd->ipath_flags & IPATH_LINKARMED) {
+			ret = 0;
+			goto bail;
+		}
+		if (!(dd->ipath_flags &
+		      (IPATH_LINKINIT | IPATH_LINKACTIVE))) {
+			ret = -EINVAL;
+			goto bail;
+		}
+		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED <<
+				    INFINIPATH_IBCC_LINKCMD_SHIFT);
+		/*
+		 * Since the port can transition to ACTIVE by receiving
+		 * a non VL 15 packet, wait for either state.
+		 */
+		lstate = IPATH_LINKARMED | IPATH_LINKACTIVE;
+		break;
+
+	case IPATH_IB_LINKACTIVE:
+		if (dd->ipath_flags & IPATH_LINKACTIVE) {
+			ret = 0;
+			goto bail;
+		}
+		if (!(dd->ipath_flags & IPATH_LINKARMED)) {
+			ret = -EINVAL;
+			goto bail;
+		}
+		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE <<
+				    INFINIPATH_IBCC_LINKCMD_SHIFT);
+		lstate = IPATH_LINKACTIVE;
+		break;
+
+	default:
+		ipath_dbg("Invalid linkstate 0x%x requested\n", newstate);
+		ret = -EINVAL;
+		goto bail;
+	}
+	ret = ipath_wait_linkstate(dd, lstate, 2000);
+
+bail:
+	return ret;
+}
+
+/**
+ * ipath_set_mtu - set the MTU
+ * @dd: the infinipath device
+ * @arg: the new MTU
+ *
+ * we can handle "any" incoming size, the issue here is whether we
+ * need to restrict our outgoing size.   For now, we don't do any
+ * sanity checking on this, and we don't deal with what happens to
+ * programs that are already running when the size changes.
+ * NOTE: changing the MTU will usually cause the IBC to go back to
+ * link initialize (IPATH_IBSTATE_INIT) state...
+ */
+int ipath_set_mtu(struct ipath_devdata *dd, u16 arg)
+{
+	u32 piosize;
+	int changed = 0;
+	int ret;
+
+	/*
+	 * mtu is IB data payload max.  It's the largest power of 2 less
+	 * than piosize (or even larger, since it only really controls the
+	 * largest we can receive; we can send the max of the mtu and
+	 * piosize).  We check that it's one of the valid IB sizes.
+	 */
+	if (arg != 256 && arg != 512 && arg != 1024 && arg != 2048 &&
+	    arg != 4096) {
+		ipath_dbg("Trying to set invalid mtu %u, failing\n", arg);
+		ret = -EINVAL;
+		goto bail;
+	}
+	if (dd->ipath_ibmtu == arg) {
+		ret = 0;        /* same as current */
+		goto bail;
+	}
+
+	piosize = dd->ipath_ibmaxlen;
+	dd->ipath_ibmtu = arg;
+
+	if (arg >= (piosize - IPATH_PIO_MAXIBHDR)) {
+		/* Only if it's not the initial value (or reset to it) */
+		if (piosize != dd->ipath_init_ibmaxlen) {
+			dd->ipath_ibmaxlen = piosize;
+			changed = 1;
+		}
+	} else if ((arg + IPATH_PIO_MAXIBHDR) != dd->ipath_ibmaxlen) {
+		piosize = arg + IPATH_PIO_MAXIBHDR;
+		ipath_cdbg(VERBOSE, "ibmaxlen was 0x%x, setting to 0x%x "
+			   "(mtu 0x%x)\n", dd->ipath_ibmaxlen, piosize,
+			   arg);
+		dd->ipath_ibmaxlen = piosize;
+		changed = 1;
+	}
+
+	if (changed) {
+		/*
+		 * set the IBC maxpktlength to the size of our pio
+		 * buffers in words
+		 */
+		u64 ibc = dd->ipath_ibcctrl;
+		ibc &= ~(INFINIPATH_IBCC_MAXPKTLEN_MASK <<
+			 INFINIPATH_IBCC_MAXPKTLEN_SHIFT);
+
+		piosize = piosize - 2 * sizeof(u32);    /* ignore pbc */
+		dd->ipath_ibmaxlen = piosize;
+		piosize /= sizeof(u32); /* in words */
+		/*
+		 * for ICRC, which we only send in diag test pkt mode, and
+		 * we don't need to worry about that for mtu
+		 */
+		piosize += 1;
+
+		ibc |= piosize << INFINIPATH_IBCC_MAXPKTLEN_SHIFT;
+		dd->ipath_ibcctrl = ibc;
+		ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
+				 dd->ipath_ibcctrl);
+		dd->ipath_f_tidtemplate(dd);
+	}
+
+	ret = 0;
+
+bail:
+	return ret;
+}
+
+int ipath_set_lid(struct ipath_devdata *dd, u32 arg, u8 lmc)
+{
+	dd->ipath_lid = arg;
+	dd->ipath_lmc = lmc;
+
+	return 0;
+}
+
 /**
  * ipath_read_kreg64_port - read a device's per-port 64-bit kernel register
  * @dd: the infinipath device
@@ -1637,13 +1750,6 @@ void ipath_shutdown_device(struct ipath_devdata *dd)
 	ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_DISABLE <<
 			    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
 
-	/*
-	 * we are shutting down, so tell the layered driver.  We don't do
-	 * this on just a link state change, much like ethernet, a cable
-	 * unplug, etc. doesn't change driver state
-	 */
-	ipath_layer_intr(dd, IPATH_LAYER_INT_IF_DOWN);
-
 	/* disable IBC */
 	dd->ipath_control &= ~INFINIPATH_C_LINKENABLE;
 	ipath_write_kreg(dd, dd->ipath_kregs->kr_control,
@@ -1743,7 +1849,7 @@ static int __init infinipath_init(void)
 {
 	int ret;
 
-	ipath_dbg(KERN_INFO DRIVER_LOAD_MSG "%s", ipath_core_version);
+	ipath_dbg(KERN_INFO DRIVER_LOAD_MSG "%s", ib_ipath_version);
 
 	/*
 	 * These must be called before the driver is registered with
@@ -1776,8 +1882,18 @@ static int __init infinipath_init(void)
 		goto bail_group;
 	}
 
+	ret = ipath_diagpkt_add();
+	if (ret < 0) {
+		printk(KERN_ERR IPATH_DRV_NAME ": Unable to create "
+		       "diag data device: error %d\n", -ret);
+		goto bail_ipathfs;
+	}
+
 	goto bail;
 
+bail_ipathfs:
+	ipath_exit_ipathfs();
+
 bail_group:
 	ipath_driver_remove_group(&ipath_driver.driver);
 
@@ -1888,6 +2004,8 @@ static void __exit infinipath_cleanup(void)
 	struct ipath_devdata *dd, *tmp;
 	unsigned long flags;
 
+	ipath_diagpkt_remove();
+
 	ipath_exit_ipathfs();
 
 	ipath_driver_remove_group(&ipath_driver.driver);
@@ -1998,5 +2116,22 @@ bail:
 	return ret;
 }
 
+int ipath_set_rx_pol_inv(struct ipath_devdata *dd, u8 new_pol_inv)
+{
+	u64 val;
+	if ( new_pol_inv > INFINIPATH_XGXS_RX_POL_MASK ) {
+		return -1;
+	}
+	if ( dd->ipath_rx_pol_inv != new_pol_inv ) {
+		dd->ipath_rx_pol_inv = new_pol_inv;
+		val = ipath_read_kreg64(dd, dd->ipath_kregs->kr_xgxsconfig);
+		val &= ~(INFINIPATH_XGXS_RX_POL_MASK <<
+			 INFINIPATH_XGXS_RX_POL_SHIFT);
+		val |= ((u64)dd->ipath_rx_pol_inv) <<
+			INFINIPATH_XGXS_RX_POL_SHIFT;
+		ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
+	}
+	return 0;
+}
 module_init(infinipath_init);
 module_exit(infinipath_cleanup);
diff --git a/drivers/infiniband/hw/ipath/ipath_file_ops.c b/drivers/infiniband/hw/ipath/ipath_file_ops.c
index bbaa70e..29930e2 100644
--- a/drivers/infiniband/hw/ipath/ipath_file_ops.c
+++ b/drivers/infiniband/hw/ipath/ipath_file_ops.c
@@ -39,7 +39,6 @@
 #include <asm/pgtable.h>
 
 #include "ipath_kernel.h"
-#include "ipath_layer.h"
 #include "ipath_common.h"
 
 static int ipath_open(struct inode *, struct file *);
@@ -985,15 +984,17 @@ static int mmap_piobufs(struct vm_area_struct *vma,
 	 * write combining behavior we want on the PIO buffers!
 	 */
 
-	if (vma->vm_flags & VM_READ) {
-		dev_info(&dd->pcidev->dev,
-			 "Can't map piobufs as readable (flags=%lx)\n",
-			 vma->vm_flags);
-		ret = -EPERM;
-		goto bail;
-	}
+#if defined(__powerpc__)
+	/* There isn't a generic way to specify writethrough mappings */
+	pgprot_val(vma->vm_page_prot) |= _PAGE_NO_CACHE;
+	pgprot_val(vma->vm_page_prot) |= _PAGE_WRITETHRU;
+	pgprot_val(vma->vm_page_prot) &= ~_PAGE_GUARDED;
+#endif
 
-	/* don't allow them to later change to readable with mprotect */
+	/*
+	 * don't allow them to later change to readable with mprotect (for when
+	 * not initially mapped readable, as is normally the case)
+	 */
 	vma->vm_flags &= ~VM_MAYREAD;
 	vma->vm_flags |= VM_DONTCOPY | VM_DONTEXPAND;
 
@@ -1109,7 +1110,7 @@ static int ipath_mmap(struct file *fp, struct vm_area_struct *vma)
 		ret = mmap_rcvegrbufs(vma, pd);
 	else if (pgaddr == (u64) pd->port_rcvhdrq_phys) {
 		/*
-		 * The rcvhdrq itself; readonly except on HT-400 (so have
+		 * The rcvhdrq itself; readonly except on HT (so have
 		 * to allow writable mapping), multiple pages, contiguous
 		 * from an i/o perspective.
 		 */
@@ -1149,6 +1150,7 @@ static unsigned int ipath_poll(struct file *fp,
 	struct ipath_portdata *pd;
 	u32 head, tail;
 	int bit;
+	unsigned pollflag = 0;
 	struct ipath_devdata *dd;
 
 	pd = port_fp(fp);
@@ -1185,9 +1187,12 @@ static unsigned int ipath_poll(struct file *fp,
 			clear_bit(IPATH_PORT_WAITING_RCV, &pd->port_flag);
 			pd->port_rcvwait_to++;
 		}
+		else
+			pollflag = POLLIN | POLLRDNORM;
 	}
 	else {
 		/* it's already happened; don't do wait_event overhead */
+		pollflag = POLLIN | POLLRDNORM;
 		pd->port_rcvnowait++;
 	}
 
@@ -1195,7 +1200,7 @@ static unsigned int ipath_poll(struct file *fp,
 	ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
 			 dd->ipath_rcvctrl);
 
-	return 0;
+	return pollflag;
 }
 
 static int try_alloc_port(struct ipath_devdata *dd, int port,
@@ -1297,14 +1302,14 @@ static int find_best_unit(struct file *fp)
 	 * This code is present to allow a knowledgeable person to
 	 * specify the layout of processes to processors before opening
 	 * this driver, and then we'll assign the process to the "closest"
-	 * HT-400 to that processor (we assume reasonable connectivity,
+	 * InfiniPath chip to that processor (we assume reasonable connectivity,
 	 * for now).  This code assumes that if affinity has been set
 	 * before this point, that at most one cpu is set; for now this
 	 * is reasonable.  I check for both cpus_empty() and cpus_full(),
 	 * in case some kernel variant sets none of the bits when no
 	 * affinity is set.  2.6.11 and 12 kernels have all present
 	 * cpus set.  Some day we'll have to fix it up further to handle
-	 * a cpu subset.  This algorithm fails for two HT-400's connected
+	 * a cpu subset.  This algorithm fails for two HT chips connected
 	 * in tunnel fashion.  Eventually this needs real topology
 	 * information.  There may be some issues with dual core numbering
 	 * as well.  This needs more work prior to release.
@@ -1815,7 +1820,7 @@ int ipath_user_add(struct ipath_devdata *dd)
 		if (ret < 0) {
 			ipath_dev_err(dd, "Could not create wildcard "
 				      "minor: error %d\n", -ret);
-			goto bail_sma;
+			goto bail_user;
 		}
 
 		atomic_set(&user_setup, 1);
@@ -1831,7 +1836,7 @@ int ipath_user_add(struct ipath_devdata *dd)
 
 	goto bail;
 
-bail_sma:
+bail_user:
 	user_cleanup();
 bail:
 	return ret;
diff --git a/drivers/infiniband/hw/ipath/ipath_fs.c b/drivers/infiniband/hw/ipath/ipath_fs.c
index 0936d8e..a5eb30a 100644
--- a/drivers/infiniband/hw/ipath/ipath_fs.c
+++ b/drivers/infiniband/hw/ipath/ipath_fs.c
@@ -191,8 +191,8 @@ static ssize_t atomic_port_info_read(struct file *file, char __user *buf,
 	portinfo[4] = (dd->ipath_lid << 16);
 
 	/*
-	 * Notimpl yet SMLID (should we store this in the driver, in case
-	 * SMA dies?)  CapabilityMask is 0, we don't support any of these
+	 * Notimpl yet SMLID.
+	 * CapabilityMask is 0, we don't support any of these
 	 * DiagCode is 0; we don't store any diag info for now Notimpl yet
 	 * M_KeyLeasePeriod (we don't support M_Key)
 	 */
diff --git a/drivers/infiniband/hw/ipath/ipath_ht400.c b/drivers/infiniband/hw/ipath/ipath_iba6110.c
index 3db015d..bf2455a 100644
--- a/drivers/infiniband/hw/ipath/ipath_ht400.c
+++ b/drivers/infiniband/hw/ipath/ipath_iba6110.c
@@ -33,7 +33,7 @@
 
 /*
  * This file contains all of the code that is specific to the InfiniPath
- * HT-400 chip.
+ * HT chip.
  */
 
 #include <linux/pci.h>
@@ -43,7 +43,7 @@
 #include "ipath_registers.h"
 
 /*
- * This lists the InfiniPath HT400 registers, in the actual chip layout.
+ * This lists the InfiniPath registers, in the actual chip layout.
  * This structure should never be directly accessed.
  *
  * The names are in InterCap form because they're taken straight from
@@ -461,8 +461,9 @@ static void ipath_ht_handle_hwerrors(struct ipath_devdata *dd, char *msg,
 			 * times.
 			 */
 			if (dd->ipath_flags & IPATH_INITTED) {
-				ipath_dev_err(dd, "Fatal Error (freeze "
-					      "mode), no longer usable\n");
+				ipath_dev_err(dd, "Fatal Hardware Error (freeze "
+					      "mode), no longer usable, SN %.16s\n",
+						  dd->ipath_serial);
 				isfatal = 1;
 			}
 			*dd->ipath_statusp &= ~IPATH_STATUS_IB_READY;
@@ -537,7 +538,7 @@ static void ipath_ht_handle_hwerrors(struct ipath_devdata *dd, char *msg,
 	if (hwerrs & INFINIPATH_HWE_HTCMISCERR7)
 		strlcat(msg, "[HT core Misc7]", msgl);
 	if (hwerrs & INFINIPATH_HWE_MEMBISTFAILED) {
-		strlcat(msg, "[Memory BIST test failed, HT-400 unusable]",
+		strlcat(msg, "[Memory BIST test failed, InfiniPath hardware unusable]",
 			msgl);
 		/* ignore from now on, so disable until driver reloaded */
 		dd->ipath_hwerrmask &= ~INFINIPATH_HWE_MEMBISTFAILED;
@@ -553,7 +554,7 @@ static void ipath_ht_handle_hwerrors(struct ipath_devdata *dd, char *msg,
 
 	if (hwerrs & _IPATH_PLL_FAIL) {
 		snprintf(bitsmsg, sizeof bitsmsg,
-			 "[PLL failed (%llx), HT-400 unusable]",
+			 "[PLL failed (%llx), InfiniPath hardware unusable]",
 			 (unsigned long long) (hwerrs & _IPATH_PLL_FAIL));
 		strlcat(msg, bitsmsg, msgl);
 		/* ignore from now on, so disable until driver reloaded */
@@ -610,18 +611,18 @@ static int ipath_ht_boardname(struct ipath_devdata *dd, char *name,
 		break;
 	case 5:
 		/*
-		 * HT-460 original production board; two production levels, with
+		 * original production board; two production levels, with
 		 * different serial number ranges.   See ipath_ht_early_init() for
 		 * case where we enable IPATH_GPIO_INTR for later serial # range.
 		 */
-		n = "InfiniPath_HT-460";
+		n = "InfiniPath_QHT7040";
 		break;
 	case 6:
 		n = "OEM_Board_3";
 		break;
 	case 7:
-		/* HT-460 small form factor production board */
-		n = "InfiniPath_HT-465";
+		/* small form factor production board */
+		n = "InfiniPath_QHT7140";
 		break;
 	case 8:
 		n = "LS/X-1";
@@ -633,7 +634,7 @@ static int ipath_ht_boardname(struct ipath_devdata *dd, char *name,
 		n = "OEM_Board_2";
 		break;
 	case 11:
-		n = "InfiniPath_HT-470";
+		n = "InfiniPath_HT-470"; /* obsoleted */
 		break;
 	case 12:
 		n = "OEM_Board_4";
@@ -641,7 +642,7 @@ static int ipath_ht_boardname(struct ipath_devdata *dd, char *name,
 	default:		/* don't know, just print the number */
 		ipath_dev_err(dd, "Don't yet know about board "
 			      "with ID %u\n", boardrev);
-		snprintf(name, namelen, "Unknown_InfiniPath_HT-4xx_%u",
+		snprintf(name, namelen, "Unknown_InfiniPath_QHT7xxx_%u",
 			 boardrev);
 		break;
 	}
@@ -650,11 +651,10 @@ static int ipath_ht_boardname(struct ipath_devdata *dd, char *name,
 
 	if (dd->ipath_majrev != 3 || (dd->ipath_minrev < 2 || dd->ipath_minrev > 3)) {
 		/*
-		 * This version of the driver only supports the HT-400
-		 * Rev 3.2
+		 * This version of the driver only supports Rev 3.2 and 3.3
 		 */
 		ipath_dev_err(dd,
-			      "Unsupported HT-400 revision %u.%u!\n",
+			      "Unsupported InfiniPath hardware revision %u.%u!\n",
 			      dd->ipath_majrev, dd->ipath_minrev);
 		ret = 1;
 		goto bail;
@@ -738,7 +738,7 @@ static void ipath_check_htlink(struct ipath_devdata *dd)
 
 static int ipath_setup_ht_reset(struct ipath_devdata *dd)
 {
-	ipath_dbg("No reset possible for HT-400\n");
+	ipath_dbg("No reset possible for this InfiniPath hardware\n");
 	return 0;
 }
 
@@ -925,7 +925,7 @@ static int set_int_handler(struct ipath_devdata *dd, struct pci_dev *pdev,
 
 	/*
 	 * kernels with CONFIG_PCI_MSI set the vector in the irq field of
-	 * struct pci_device, so we use that to program the HT-400 internal
+	 * struct pci_device, so we use that to program the internal
 	 * interrupt register (not config space) with that value. The BIOS
 	 * must still have done the basic MSI setup.
 	 */
@@ -1013,7 +1013,7 @@ bail:
  * @dd: the infinipath device
  *
  * Called during driver unload.
- * This is currently a nop for the HT-400, not for all chips
+ * This is currently a nop for the HT chip, not for all chips
  */
 static void ipath_setup_ht_cleanup(struct ipath_devdata *dd)
 {
@@ -1290,6 +1290,15 @@ static int ipath_ht_bringup_serdes(struct ipath_devdata *dd)
 		val &= ~INFINIPATH_XGXS_RESET;
 		change = 1;
 	}
+	if (((val >> INFINIPATH_XGXS_RX_POL_SHIFT) &
+	     INFINIPATH_XGXS_RX_POL_MASK) != dd->ipath_rx_pol_inv ) {
+		/* need to compensate for Tx inversion in partner */
+		val &= ~(INFINIPATH_XGXS_RX_POL_MASK <<
+		         INFINIPATH_XGXS_RX_POL_SHIFT);
+		val |= dd->ipath_rx_pol_inv <<
+			INFINIPATH_XGXS_RX_POL_SHIFT;
+		change = 1;
+	}
 	if (change)
 		ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
 
@@ -1470,7 +1479,7 @@ static int ipath_ht_early_init(struct ipath_devdata *dd)
 	dd->ipath_rcvhdrsize = IPATH_DFLT_RCVHDRSIZE;
 
 	/*
-	 * For HT-400, we allocate a somewhat overly large eager buffer,
+	 * For HT, we allocate a somewhat overly large eager buffer,
 	 * such that we can guarantee that we can receive the largest
 	 * packet that we can send out.  To truly support a 4KB MTU,
 	 * we need to bump this to a large value.  To date, other than
@@ -1531,7 +1540,7 @@ static int ipath_ht_early_init(struct ipath_devdata *dd)
 	if(dd->ipath_boardrev == 5 && dd->ipath_serial[0] == '1' &&
 		dd->ipath_serial[1] == '2' && dd->ipath_serial[2] == '8') {
 		/*
-		 * Later production HT-460 has same changes as HT-465, so
+		 * Later production QHT7040 has same changes as QHT7140, so
 		 * can use GPIO interrupts.  They have serial #'s starting
 		 * with 128, rather than 112.
 		 */
@@ -1560,13 +1569,13 @@ static int ipath_ht_get_base_info(struct ipath_portdata *pd, void *kbase)
 }
 
 /**
- * ipath_init_ht400_funcs - set up the chip-specific function pointers
+ * ipath_init_iba6110_funcs - set up the chip-specific function pointers
  * @dd: the infinipath device
  *
  * This is global, and is called directly at init to set up the
  * chip-specific function pointers for later use.
  */
-void ipath_init_ht400_funcs(struct ipath_devdata *dd)
+void ipath_init_iba6110_funcs(struct ipath_devdata *dd)
 {
 	dd->ipath_f_intrsetup = ipath_ht_intconfig;
 	dd->ipath_f_bus = ipath_setup_ht_config;
diff --git a/drivers/infiniband/hw/ipath/ipath_pe800.c b/drivers/infiniband/hw/ipath/ipath_iba6120.c
index b83f66d..d86516d 100644
--- a/drivers/infiniband/hw/ipath/ipath_pe800.c
+++ b/drivers/infiniband/hw/ipath/ipath_iba6120.c
@@ -32,7 +32,7 @@
  */
 /*
  * This file contains all of the code that is specific to the
- * InfiniPath PE-800 chip.
+ * InfiniPath PCIe chip.
  */
 
 #include <linux/interrupt.h>
@@ -45,9 +45,9 @@
 
 /*
  * This file contains all the chip-specific register information and
- * access functions for the QLogic InfiniPath PE800, the PCI-Express chip.
+ * access functions for the QLogic InfiniPath PCI-Express chip.
  *
- * This lists the InfiniPath PE800 registers, in the actual chip layout.
+ * This lists the InfiniPath registers, in the actual chip layout.
  * This structure should never be directly accessed.
  */
 struct _infinipath_do_not_use_kernel_regs {
@@ -213,7 +213,6 @@ static const struct ipath_kregs ipath_pe_kregs = {
 	.kr_rcvhdraddr = IPATH_KREG_OFFSET(RcvHdrAddr0),
 	.kr_rcvhdrtailaddr = IPATH_KREG_OFFSET(RcvHdrTailAddr0),
 
-	/* This group is pe-800-specific; and used only in this file */
 	/* The rcvpktled register controls one of the debug port signals, so
 	 * a packet activity LED can be connected to it. */
 	.kr_rcvpktledcnt = IPATH_KREG_OFFSET(RcvPktLEDCnt),
@@ -364,8 +363,9 @@ static void ipath_pe_handle_hwerrors(struct ipath_devdata *dd, char *msg,
 			 * and we get here multiple times
 			 */
 			if (dd->ipath_flags & IPATH_INITTED) {
-				ipath_dev_err(dd, "Fatal Error (freeze "
-					      "mode), no longer usable\n");
+				ipath_dev_err(dd, "Fatal Hardware Error (freeze "
+					      "mode), no longer usable, SN %.16s\n",
+						  dd->ipath_serial);
 				isfatal = 1;
 			}
 			/*
@@ -388,7 +388,7 @@ static void ipath_pe_handle_hwerrors(struct ipath_devdata *dd, char *msg,
 	*msg = '\0';
 
 	if (hwerrs & INFINIPATH_HWE_MEMBISTFAILED) {
-		strlcat(msg, "[Memory BIST test failed, PE-800 unusable]",
+		strlcat(msg, "[Memory BIST test failed, InfiniPath hardware unusable]",
 			msgl);
 		/* ignore from now on, so disable until driver reloaded */
 		*dd->ipath_statusp |= IPATH_STATUS_HWERROR;
@@ -433,7 +433,7 @@ static void ipath_pe_handle_hwerrors(struct ipath_devdata *dd, char *msg,
 
 	if (hwerrs & _IPATH_PLL_FAIL) {
 		snprintf(bitsmsg, sizeof bitsmsg,
-			 "[PLL failed (%llx), PE-800 unusable]",
+			 "[PLL failed (%llx), InfiniPath hardware unusable]",
 			 (unsigned long long) hwerrs & _IPATH_PLL_FAIL);
 		strlcat(msg, bitsmsg, msgl);
 		/* ignore from now on, so disable until driver reloaded */
@@ -511,22 +511,25 @@ static int ipath_pe_boardname(struct ipath_devdata *dd, char *name,
 		n = "InfiniPath_Emulation";
 		break;
 	case 1:
-		n = "InfiniPath_PE-800-Bringup";
+		n = "InfiniPath_QLE7140-Bringup";
 		break;
 	case 2:
-		n = "InfiniPath_PE-880";
+		n = "InfiniPath_QLE7140";
 		break;
 	case 3:
-		n = "InfiniPath_PE-850";
+		n = "InfiniPath_QMI7140";
 		break;
 	case 4:
-		n = "InfiniPath_PE-860";
+		n = "InfiniPath_QEM7140";
+		break;
+	case 5:
+		n = "InfiniPath_QMH7140";
 		break;
 	default:
 		ipath_dev_err(dd,
 			      "Don't yet know about board with ID %u\n",
 			      boardrev);
-		snprintf(name, namelen, "Unknown_InfiniPath_PE-8xx_%u",
+		snprintf(name, namelen, "Unknown_InfiniPath_PCIe_%u",
 			 boardrev);
 		break;
 	}
@@ -534,7 +537,7 @@ static int ipath_pe_boardname(struct ipath_devdata *dd, char *name,
 		snprintf(name, namelen, "%s", n);
 
 	if (dd->ipath_majrev != 4 || !dd->ipath_minrev || dd->ipath_minrev>2) {
-		ipath_dev_err(dd, "Unsupported PE-800 revision %u.%u!\n",
+		ipath_dev_err(dd, "Unsupported InfiniPath hardware revision %u.%u!\n",
 			      dd->ipath_majrev, dd->ipath_minrev);
 		ret = 1;
 	} else
@@ -651,6 +654,15 @@ static int ipath_pe_bringup_serdes(struct ipath_devdata *dd)
 		val &= ~INFINIPATH_XGXS_RESET;
 		change = 1;
 	}
+	if (((val >> INFINIPATH_XGXS_RX_POL_SHIFT) &
+	     INFINIPATH_XGXS_RX_POL_MASK) != dd->ipath_rx_pol_inv ) {
+		/* need to compensate for Tx inversion in partner */
+		val &= ~(INFINIPATH_XGXS_RX_POL_MASK <<
+		         INFINIPATH_XGXS_RX_POL_SHIFT);
+		val |= dd->ipath_rx_pol_inv <<
+			INFINIPATH_XGXS_RX_POL_SHIFT;
+		change = 1;
+	}
 	if (change)
 		ipath_write_kreg(dd, dd->ipath_kregs->kr_xgxsconfig, val);
 
@@ -705,7 +717,7 @@ static void ipath_pe_quiet_serdes(struct ipath_devdata *dd)
 	ipath_write_kreg(dd, dd->ipath_kregs->kr_serdesconfig0, val);
 }
 
-/* this is not yet needed on the PE800, so just return 0. */
+/* this is not yet needed on this chip, so just return 0. */
 static int ipath_pe_intconfig(struct ipath_devdata *dd)
 {
 	return 0;
@@ -759,8 +771,8 @@ static void ipath_setup_pe_setextled(struct ipath_devdata *dd, u64 lst,
  *
  * This is called during driver unload.
  * We do the pci_disable_msi here, not in generic code, because it
- * isn't used for the HT-400. If we do end up needing pci_enable_msi
- * at some point in the future for HT-400, we'll move the call back
+ * isn't used for the HT chips. If we do end up needing pci_enable_msi
+ * at some point in the future for HT, we'll move the call back
  * into the main init_one code.
  */
 static void ipath_setup_pe_cleanup(struct ipath_devdata *dd)
@@ -780,10 +792,10 @@ static void ipath_setup_pe_cleanup(struct ipath_devdata *dd)
  * late in 2.6.16).
  * All that can be done is to edit the kernel source to remove the quirk
  * check until that is fixed.
- * We do not need to call enable_msi() for our HyperTransport chip (HT-400),
- * even those it uses MSI, and we want to avoid the quirk warning, so
- * So we call enable_msi only for the PE-800.  If we do end up needing
- * pci_enable_msi at some point in the future for HT-400, we'll move the
+ * We do not need to call enable_msi() for our HyperTransport chip,
+ * even though it uses MSI, and we want to avoid the quirk warning, so
+ * So we call enable_msi only for PCIe.  If we do end up needing
+ * pci_enable_msi at some point in the future for HT, we'll move the
  * call back into the main init_one code.
  * We save the msi lo and hi values, so we can restore them after
  * chip reset (the kernel PCI infrastructure doesn't yet handle that
@@ -971,8 +983,7 @@ static int ipath_setup_pe_reset(struct ipath_devdata *dd)
 	int ret;
 
 	/* Use ERROR so it shows up in logs, etc. */
-	ipath_dev_err(dd, "Resetting PE-800 unit %u\n",
-		      dd->ipath_unit);
+	ipath_dev_err(dd, "Resetting InfiniPath unit %u\n", dd->ipath_unit);
 	/* keep chip from being accessed in a few places */
 	dd->ipath_flags &= ~(IPATH_INITTED|IPATH_PRESENT);
 	val = dd->ipath_control | INFINIPATH_C_RESET;
@@ -1078,7 +1089,7 @@ static void ipath_pe_put_tid(struct ipath_devdata *dd, u64 __iomem *tidptr,
  * @port: the port
  *
  * clear all TID entries for a port, expected and eager.
- * Used from ipath_close().  On PE800, TIDs are only 32 bits,
+ * Used from ipath_close().  On this chip, TIDs are only 32 bits,
  * not 64, but they are still on 64 bit boundaries, so tidbase
  * is declared as u64 * for the pointer math, even though we write 32 bits
  */
@@ -1148,9 +1159,9 @@ static int ipath_pe_early_init(struct ipath_devdata *dd)
 	dd->ipath_flags |= IPATH_4BYTE_TID;
 
 	/*
-	 * For openib, we need to be able to handle an IB header of 96 bytes
-	 * or 24 dwords.  HT-400 has arbitrary sized receive buffers, so we
-	 * made them the same size as the PIO buffers.  The PE-800 does not
+	 * For openfabrics, we need to be able to handle an IB header of
+	 * 24 dwords.  HT chip has arbitrary sized receive buffers, so we
+	 * made them the same size as the PIO buffers.  This chip does not
 	 * handle arbitrary size buffers, so we need the header large enough
 	 * to handle largest IB header, but still have room for a 2KB MTU
 	 * standard IB packet.
@@ -1158,11 +1169,10 @@ static int ipath_pe_early_init(struct ipath_devdata *dd)
 	dd->ipath_rcvhdrentsize = 24;
 	dd->ipath_rcvhdrsize = IPATH_DFLT_RCVHDRSIZE;
 
-	/* For HT-400, we allocate a somewhat overly large eager buffer,
-	 * such that we can guarantee that we can receive the largest packet
-	 * that we can send out.  To truly support a 4KB MTU, we need to
-	 * bump this to a larger value.  We'll do this when I get around to
-	 * testing 4KB sends on the PE-800, which I have not yet done.
+	/*
+	 * To truly support a 4KB MTU (for usermode), we need to
+	 * bump this to a larger value.  For now, we use them for
+	 * the kernel only.
 	 */
 	dd->ipath_rcvegrbufsize = 2048;
 	/*
@@ -1175,9 +1185,9 @@ static int ipath_pe_early_init(struct ipath_devdata *dd)
 	dd->ipath_init_ibmaxlen = dd->ipath_ibmaxlen;
 
 	/*
-	 * For PE-800, we can request a receive interrupt for 1 or
+	 * We can request a receive interrupt for 1 or
 	 * more packets from current offset.  For now, we set this
-	 * up for a single packet, to match the HT-400 behavior.
+	 * up for a single packet.
 	 */
 	dd->ipath_rhdrhead_intr_off = 1ULL<<32;
 
@@ -1216,13 +1226,13 @@ static int ipath_pe_get_base_info(struct ipath_portdata *pd, void *kbase)
 }
 
 /**
- * ipath_init_pe800_funcs - set up the chip-specific function pointers
+ * ipath_init_iba6120_funcs - set up the chip-specific function pointers
  * @dd: the infinipath device
  *
  * This is global, and is called directly at init to set up the
  * chip-specific function pointers for later use.
  */
-void ipath_init_pe800_funcs(struct ipath_devdata *dd)
+void ipath_init_iba6120_funcs(struct ipath_devdata *dd)
 {
 	dd->ipath_f_intrsetup = ipath_pe_intconfig;
 	dd->ipath_f_bus = ipath_setup_pe_config;
diff --git a/drivers/infiniband/hw/ipath/ipath_init_chip.c b/drivers/infiniband/hw/ipath/ipath_init_chip.c
index 414cdd1..44669dc 100644
--- a/drivers/infiniband/hw/ipath/ipath_init_chip.c
+++ b/drivers/infiniband/hw/ipath/ipath_init_chip.c
@@ -53,8 +53,8 @@ module_param_named(cfgports, ipath_cfgports, ushort, S_IRUGO);
 MODULE_PARM_DESC(cfgports, "Set max number of ports to use");
 
 /*
- * Number of buffers reserved for driver (layered drivers and SMA
- * send).  Reserved at end of buffer list.   Initialized based on
+ * Number of buffers reserved for driver (verbs and layered drivers.)
+ * Reserved at end of buffer list.   Initialized based on
  * number of PIO buffers if not set via module interface.
  * The problem with this is that it's global, but we'll use different
  * numbers for different chip types.  So the default value is not
@@ -80,7 +80,7 @@ MODULE_PARM_DESC(kpiobufs, "Set number of PIO buffers for driver");
  *
  * Allocate the eager TID buffers and program them into infinipath.
  * We use the network layer alloc_skb() allocator to allocate the
- * memory, and either use the buffers as is for things like SMA
+ * memory, and either use the buffers as is for things like verbs
  * packets, or pass the buffers up to the ipath layered driver and
  * thence the network layer, replacing them as we do so (see
  * ipath_rcv_layer()).
@@ -240,7 +240,11 @@ static int init_chip_first(struct ipath_devdata *dd,
 			  "only supports %u\n", ipath_cfgports,
 			  dd->ipath_portcnt);
 	}
-	dd->ipath_pd = kzalloc(sizeof(*dd->ipath_pd) * dd->ipath_cfgports,
+	/*
+	 * Allocate full portcnt array, rather than just cfgports, because
+	 * cleanup iterates across all possible ports.
+	 */
+	dd->ipath_pd = kzalloc(sizeof(*dd->ipath_pd) * dd->ipath_portcnt,
 			       GFP_KERNEL);
 
 	if (!dd->ipath_pd) {
@@ -446,9 +450,9 @@ static void enable_chip(struct ipath_devdata *dd,
 	u32 val;
 	int i;
 
-	if (!reinit) {
-		init_waitqueue_head(&ipath_sma_state_wait);
-	}
+	if (!reinit)
+		init_waitqueue_head(&ipath_state_wait);
+
 	ipath_write_kreg(dd, dd->ipath_kregs->kr_rcvctrl,
 			 dd->ipath_rcvctrl);
 
@@ -687,7 +691,7 @@ int ipath_init_chip(struct ipath_devdata *dd, int reinit)
 	dd->ipath_pioavregs = ALIGN(val, sizeof(u64) * BITS_PER_BYTE / 2)
 		/ (sizeof(u64) * BITS_PER_BYTE / 2);
 	if (ipath_kpiobufs == 0) {
-		/* not set by user, or set explictly to default  */
+		/* not set by user (this is default) */
 		if ((dd->ipath_piobcnt2k + dd->ipath_piobcnt4k) > 128)
 			kpiobufs = 32;
 		else
@@ -946,6 +950,7 @@ static int ipath_set_kpiobufs(const char *str, struct kernel_param *kp)
 			dd->ipath_piobcnt2k + dd->ipath_piobcnt4k - val;
 	}
 
+	ipath_kpiobufs = val;
 	ret = 0;
 bail:
 	spin_unlock_irqrestore(&ipath_devs_lock, flags);
diff --git a/drivers/infiniband/hw/ipath/ipath_intr.c b/drivers/infiniband/hw/ipath/ipath_intr.c
index 280e732..49bf7bb 100644
--- a/drivers/infiniband/hw/ipath/ipath_intr.c
+++ b/drivers/infiniband/hw/ipath/ipath_intr.c
@@ -34,7 +34,7 @@
 #include <linux/pci.h>
 
 #include "ipath_kernel.h"
-#include "ipath_layer.h"
+#include "ipath_verbs.h"
 #include "ipath_common.h"
 
 /* These are all rcv-related errors which we want to count for stats */
@@ -201,7 +201,7 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
 				  ib_linkstate(lstate));
 		}
 		else
-			ipath_cdbg(SMA, "Unit %u link state %s, last "
+			ipath_cdbg(VERBOSE, "Unit %u link state %s, last "
 				   "was %s\n", dd->ipath_unit,
 				   ib_linkstate(lstate),
 				   ib_linkstate((unsigned)
@@ -213,7 +213,7 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
 		if (lstate == IPATH_IBSTATE_INIT ||
 		    lstate == IPATH_IBSTATE_ARM ||
 		    lstate == IPATH_IBSTATE_ACTIVE)
-			ipath_cdbg(SMA, "Unit %u link state down"
+			ipath_cdbg(VERBOSE, "Unit %u link state down"
 				   " (state 0x%x), from %s\n",
 				   dd->ipath_unit,
 				   (u32)val & IPATH_IBSTATE_MASK,
@@ -269,7 +269,7 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
 			     INFINIPATH_IBCS_LINKSTATE_MASK)
 			    == INFINIPATH_IBCS_L_STATE_ACTIVE)
 				/* if from up to down be more vocal */
-				ipath_cdbg(SMA,
+				ipath_cdbg(VERBOSE,
 					   "Unit %u link now down (%s)\n",
 					   dd->ipath_unit,
 					   ipath_ibcstatus_str[ltstate]);
@@ -289,8 +289,6 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd,
 		*dd->ipath_statusp |=
 			IPATH_STATUS_IB_READY | IPATH_STATUS_IB_CONF;
 		dd->ipath_f_setextled(dd, lstate, ltstate);
-
-		__ipath_layer_intr(dd, IPATH_LAYER_INT_IF_UP);
 	} else if ((val & IPATH_IBSTATE_MASK) == IPATH_IBSTATE_INIT) {
 		/*
 		 * set INIT and DOWN.  Down is checked by most of the other
@@ -598,11 +596,11 @@ static int handle_errors(struct ipath_devdata *dd, ipath_err_t errs)
 
 	if (!noprint && *msg)
 		ipath_dev_err(dd, "%s error\n", msg);
-	if (dd->ipath_sma_state_wanted & dd->ipath_flags) {
-		ipath_cdbg(VERBOSE, "sma wanted state %x, iflags now %x, "
-			   "waking\n", dd->ipath_sma_state_wanted,
+	if (dd->ipath_state_wanted & dd->ipath_flags) {
+		ipath_cdbg(VERBOSE, "driver wanted state %x, iflags now %x, "
+			   "waking\n", dd->ipath_state_wanted,
 			   dd->ipath_flags);
-		wake_up_interruptible(&ipath_sma_state_wait);
+		wake_up_interruptible(&ipath_state_wait);
 	}
 
 	return chkerrpkts;
@@ -708,11 +706,7 @@ static void handle_layer_pioavail(struct ipath_devdata *dd)
 {
 	int ret;
 
-	ret = __ipath_layer_intr(dd, IPATH_LAYER_INT_SEND_CONTINUE);
-	if (ret > 0)
-		goto set;
-
-	ret = __ipath_verbs_piobufavail(dd);
+	ret = ipath_ib_piobufavail(dd->verbs_dev);
 	if (ret > 0)
 		goto set;
 
diff --git a/drivers/infiniband/hw/ipath/ipath_kernel.h b/drivers/infiniband/hw/ipath/ipath_kernel.h
index e9f374f..a8a5627 100644
--- a/drivers/infiniband/hw/ipath/ipath_kernel.h
+++ b/drivers/infiniband/hw/ipath/ipath_kernel.h
@@ -132,12 +132,6 @@ struct _ipath_layer {
 	void *l_arg;
 };
 
-/* Verbs layer interface */
-struct _verbs_layer {
-	void *l_arg;
-	struct timer_list l_timer;
-};
-
 struct ipath_devdata {
 	struct list_head ipath_list;
 
@@ -198,7 +192,8 @@ struct ipath_devdata {
 	void (*ipath_f_setextled)(struct ipath_devdata *, u64, u64);
 	/* fill out chip-specific fields */
 	int (*ipath_f_get_base_info)(struct ipath_portdata *, void *);
-	struct _verbs_layer verbs_layer;
+	struct ipath_ibdev *verbs_dev;
+	struct timer_list verbs_timer;
 	/* total dwords sent (summed from counter) */
 	u64 ipath_sword;
 	/* total dwords rcvd (summed from counter) */
@@ -241,7 +236,7 @@ struct ipath_devdata {
 	u64 ipath_tidtemplate;
 	/* value to write to free TIDs */
 	u64 ipath_tidinvalid;
-	/* PE-800 rcv interrupt setup */
+	/* IBA6120 rcv interrupt setup */
 	u64 ipath_rhdrhead_intr_off;
 
 	/* size of memory at ipath_kregbase */
@@ -250,8 +245,8 @@ struct ipath_devdata {
 	u32 ipath_pioavregs;
 	/* IPATH_POLL, etc. */
 	u32 ipath_flags;
-	/* ipath_flags sma is waiting for */
-	u32 ipath_sma_state_wanted;
+	/* ipath_flags driver is waiting for */
+	u32 ipath_state_wanted;
 	/* last buffer for user use, first buf for kernel use is this
 	 * index. */
 	u32 ipath_lastport_piobuf;
@@ -311,10 +306,6 @@ struct ipath_devdata {
 	u32 ipath_pcibar0;
 	/* so we can rewrite it after a chip reset */
 	u32 ipath_pcibar1;
-	/* sequential tries for SMA send and no bufs */
-	u32 ipath_nosma_bufs;
-	/* duration (seconds) ipath_nosma_bufs set */
-	u32 ipath_nosma_secs;
 
 	/* HT/PCI Vendor ID (here for NodeInfo) */
 	u16 ipath_vendorid;
@@ -512,6 +503,8 @@ struct ipath_devdata {
 	u8 ipath_pci_cacheline;
 	/* LID mask control */
 	u8 ipath_lmc;
+	/* Rx Polarity inversion (compensate for ~tx on partner) */
+	u8 ipath_rx_pol_inv;
 
 	/* local link integrity counter */
 	u32 ipath_lli_counter;
@@ -523,18 +516,6 @@ extern struct list_head ipath_dev_list;
 extern spinlock_t ipath_devs_lock;
 extern struct ipath_devdata *ipath_lookup(int unit);
 
-extern u16 ipath_layer_rcv_opcode;
-extern int __ipath_layer_intr(struct ipath_devdata *, u32);
-extern int ipath_layer_intr(struct ipath_devdata *, u32);
-extern int __ipath_layer_rcv(struct ipath_devdata *, void *,
-			     struct sk_buff *);
-extern int __ipath_layer_rcv_lid(struct ipath_devdata *, void *);
-extern int __ipath_verbs_piobufavail(struct ipath_devdata *);
-extern int __ipath_verbs_rcv(struct ipath_devdata *, void *, void *, u32);
-
-void ipath_layer_add(struct ipath_devdata *);
-void ipath_layer_remove(struct ipath_devdata *);
-
 int ipath_init_chip(struct ipath_devdata *, int);
 int ipath_enable_wc(struct ipath_devdata *dd);
 void ipath_disable_wc(struct ipath_devdata *dd);
@@ -549,9 +530,8 @@ void ipath_cdev_cleanup(struct cdev **cdevp,
 
 int ipath_diag_add(struct ipath_devdata *);
 void ipath_diag_remove(struct ipath_devdata *);
-void ipath_diag_bringup_link(struct ipath_devdata *);
 
-extern wait_queue_head_t ipath_sma_state_wait;
+extern wait_queue_head_t ipath_state_wait;
 
 int ipath_user_add(struct ipath_devdata *dd);
 void ipath_user_remove(struct ipath_devdata *dd);
@@ -582,12 +562,14 @@ void ipath_free_pddata(struct ipath_devdata *, struct ipath_portdata *);
 
 int ipath_parse_ushort(const char *str, unsigned short *valp);
 
-int ipath_wait_linkstate(struct ipath_devdata *, u32, int);
-void ipath_set_ib_lstate(struct ipath_devdata *, int);
 void ipath_kreceive(struct ipath_devdata *);
 int ipath_setrcvhdrsize(struct ipath_devdata *, unsigned);
 int ipath_reset_device(int);
 void ipath_get_faststats(unsigned long);
+int ipath_set_linkstate(struct ipath_devdata *, u8);
+int ipath_set_mtu(struct ipath_devdata *, u16);
+int ipath_set_lid(struct ipath_devdata *, u32, u8);
+int ipath_set_rx_pol_inv(struct ipath_devdata *dd, u8 new_pol_inv);
 
 /* for use in system calls, where we want to know device type, etc. */
 #define port_fp(fp) ((struct ipath_portdata *) (fp)->private_data)
@@ -642,10 +624,8 @@ void ipath_free_data(struct ipath_portdata *dd);
 int ipath_waitfor_mdio_cmdready(struct ipath_devdata *);
 int ipath_waitfor_complete(struct ipath_devdata *, ipath_kreg, u64, u64 *);
 u32 __iomem *ipath_getpiobuf(struct ipath_devdata *, u32 *);
-/* init PE-800-specific func */
-void ipath_init_pe800_funcs(struct ipath_devdata *);
-/* init HT-400-specific func */
-void ipath_init_ht400_funcs(struct ipath_devdata *);
+void ipath_init_iba6120_funcs(struct ipath_devdata *);
+void ipath_init_iba6110_funcs(struct ipath_devdata *);
 void ipath_get_eeprom_info(struct ipath_devdata *);
 u64 ipath_snap_cntr(struct ipath_devdata *, ipath_creg);
 
@@ -801,7 +781,7 @@ static inline u32 ipath_read_creg32(const struct ipath_devdata *dd,
 
 struct device_driver;
 
-extern const char ipath_core_version[];
+extern const char ib_ipath_version[];
 
 int ipath_driver_create_group(struct device_driver *);
 void ipath_driver_remove_group(struct device_driver *);
@@ -810,6 +790,9 @@ int ipath_device_create_group(struct device *, struct ipath_devdata *);
 void ipath_device_remove_group(struct device *, struct ipath_devdata *);
 int ipath_expose_reset(struct device *);
 
+int ipath_diagpkt_add(void);
+void ipath_diagpkt_remove(void);
+
 int ipath_init_ipathfs(void);
 void ipath_exit_ipathfs(void);
 int ipathfs_add_device(struct ipath_devdata *);
@@ -831,10 +814,10 @@ const char *ipath_get_unit_name(int unit);
 
 extern struct mutex ipath_mutex;
 
-#define IPATH_DRV_NAME		"ipath_core"
+#define IPATH_DRV_NAME		"ib_ipath"
 #define IPATH_MAJOR		233
 #define IPATH_USER_MINOR_BASE	0
-#define IPATH_SMA_MINOR		128
+#define IPATH_DIAGPKT_MINOR	127
 #define IPATH_DIAG_MINOR_BASE	129
 #define IPATH_NMINORS		255
 
diff --git a/drivers/infiniband/hw/ipath/ipath_keys.c b/drivers/infiniband/hw/ipath/ipath_keys.c
index a5ca279..ba1b932 100644
--- a/drivers/infiniband/hw/ipath/ipath_keys.c
+++ b/drivers/infiniband/hw/ipath/ipath_keys.c
@@ -34,6 +34,7 @@
 #include <asm/io.h>
 
 #include "ipath_verbs.h"
+#include "ipath_kernel.h"
 
 /**
  * ipath_alloc_lkey - allocate an lkey
@@ -60,7 +61,7 @@ int ipath_alloc_lkey(struct ipath_lkey_table *rkt, struct ipath_mregion *mr)
 		r = (r + 1) & (rkt->max - 1);
 		if (r == n) {
 			spin_unlock_irqrestore(&rkt->lock, flags);
-			_VERBS_INFO("LKEY table full\n");
+			ipath_dbg(KERN_INFO "LKEY table full\n");
 			ret = 0;
 			goto bail;
 		}
diff --git a/drivers/infiniband/hw/ipath/ipath_layer.c b/drivers/infiniband/hw/ipath/ipath_layer.c
index b28c6f8..e46aa4e 100644
--- a/drivers/infiniband/hw/ipath/ipath_layer.c
+++ b/drivers/infiniband/hw/ipath/ipath_layer.c
@@ -42,26 +42,20 @@
 
 #include "ipath_kernel.h"
 #include "ipath_layer.h"
+#include "ipath_verbs.h"
 #include "ipath_common.h"
 
 /* Acquire before ipath_devs_lock. */
 static DEFINE_MUTEX(ipath_layer_mutex);
 
-static int ipath_verbs_registered;
-
 u16 ipath_layer_rcv_opcode;
 
 static int (*layer_intr)(void *, u32);
 static int (*layer_rcv)(void *, void *, struct sk_buff *);
 static int (*layer_rcv_lid)(void *, void *);
-static int (*verbs_piobufavail)(void *);
-static void (*verbs_rcv)(void *, void *, void *, u32);
 
 static void *(*layer_add_one)(int, struct ipath_devdata *);
 static void (*layer_remove_one)(void *);
-static void *(*verbs_add_one)(int, struct ipath_devdata *);
-static void (*verbs_remove_one)(void *);
-static void (*verbs_timer_cb)(void *);
 
 int __ipath_layer_intr(struct ipath_devdata *dd, u32 arg)
 {
@@ -107,302 +101,16 @@ int __ipath_layer_rcv_lid(struct ipath_devdata *dd, void *hdr)
 	return ret;
 }
 
-int __ipath_verbs_piobufavail(struct ipath_devdata *dd)
-{
-	int ret = -ENODEV;
-
-	if (dd->verbs_layer.l_arg && verbs_piobufavail)
-		ret = verbs_piobufavail(dd->verbs_layer.l_arg);
-
-	return ret;
-}
-
-int __ipath_verbs_rcv(struct ipath_devdata *dd, void *rc, void *ebuf,
-		      u32 tlen)
-{
-	int ret = -ENODEV;
-
-	if (dd->verbs_layer.l_arg && verbs_rcv) {
-		verbs_rcv(dd->verbs_layer.l_arg, rc, ebuf, tlen);
-		ret = 0;
-	}
-
-	return ret;
-}
-
-int ipath_layer_set_linkstate(struct ipath_devdata *dd, u8 newstate)
+void ipath_layer_lid_changed(struct ipath_devdata *dd)
 {
-	u32 lstate;
-	int ret;
-
-	switch (newstate) {
-	case IPATH_IB_LINKDOWN:
-		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_POLL <<
-				    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
-		/* don't wait */
-		ret = 0;
-		goto bail;
-
-	case IPATH_IB_LINKDOWN_SLEEP:
-		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_SLEEP <<
-				    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
-		/* don't wait */
-		ret = 0;
-		goto bail;
-
-	case IPATH_IB_LINKDOWN_DISABLE:
-		ipath_set_ib_lstate(dd,
-				    INFINIPATH_IBCC_LINKINITCMD_DISABLE <<
-				    INFINIPATH_IBCC_LINKINITCMD_SHIFT);
-		/* don't wait */
-		ret = 0;
-		goto bail;
-
-	case IPATH_IB_LINKINIT:
-		if (dd->ipath_flags & IPATH_LINKINIT) {
-			ret = 0;
-			goto bail;
-		}
-		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_INIT <<
-				    INFINIPATH_IBCC_LINKCMD_SHIFT);
-		lstate = IPATH_LINKINIT;
-		break;
-
-	case IPATH_IB_LINKARM:
-		if (dd->ipath_flags & IPATH_LINKARMED) {
-			ret = 0;
-			goto bail;
-		}
-		if (!(dd->ipath_flags &
-		      (IPATH_LINKINIT | IPATH_LINKACTIVE))) {
-			ret = -EINVAL;
-			goto bail;
-		}
-		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED <<
-				    INFINIPATH_IBCC_LINKCMD_SHIFT);
-		/*
-		 * Since the port can transition to ACTIVE by receiving
-		 * a non VL 15 packet, wait for either state.
-		 */
-		lstate = IPATH_LINKARMED | IPATH_LINKACTIVE;
-		break;
-
-	case IPATH_IB_LINKACTIVE:
-		if (dd->ipath_flags & IPATH_LINKACTIVE) {
-			ret = 0;
-			goto bail;
-		}
-		if (!(dd->ipath_flags & IPATH_LINKARMED)) {
-			ret = -EINVAL;
-			goto bail;
-		}
-		ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE <<
-				    INFINIPATH_IBCC_LINKCMD_SHIFT);
-		lstate = IPATH_LINKACTIVE;
-		break;
-
-	default:
-		ipath_dbg("Invalid linkstate 0x%x requested\n", newstate);
-		ret = -EINVAL;
-		goto bail;
-	}
-	ret = ipath_wait_linkstate(dd, lstate, 2000);
-
-bail:
-	return ret;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_set_linkstate);
-
-/**
- * ipath_layer_set_mtu - set the MTU
- * @dd: the infinipath device
- * @arg: the new MTU
- *
- * we can handle "any" incoming size, the issue here is whether we
- * need to restrict our outgoing size.   For now, we don't do any
- * sanity checking on this, and we don't deal with what happens to
- * programs that are already running when the size changes.
- * NOTE: changing the MTU will usually cause the IBC to go back to
- * link initialize (IPATH_IBSTATE_INIT) state...
- */
-int ipath_layer_set_mtu(struct ipath_devdata *dd, u16 arg)
-{
-	u32 piosize;
-	int changed = 0;
-	int ret;
-
-	/*
-	 * mtu is IB data payload max.  It's the largest power of 2 less
-	 * than piosize (or even larger, since it only really controls the
-	 * largest we can receive; we can send the max of the mtu and
-	 * piosize).  We check that it's one of the valid IB sizes.
-	 */
-	if (arg != 256 && arg != 512 && arg != 1024 && arg != 2048 &&
-	    arg != 4096) {
-		ipath_dbg("Trying to set invalid mtu %u, failing\n", arg);
-		ret = -EINVAL;
-		goto bail;
-	}
-	if (dd->ipath_ibmtu == arg) {
-		ret = 0;	/* same as current */
-		goto bail;
-	}
-
-	piosize = dd->ipath_ibmaxlen;
-	dd->ipath_ibmtu = arg;
-
-	if (arg >= (piosize - IPATH_PIO_MAXIBHDR)) {
-		/* Only if it's not the initial value (or reset to it) */
-		if (piosize != dd->ipath_init_ibmaxlen) {
-			dd->ipath_ibmaxlen = piosize;
-			changed = 1;
-		}
-	} else if ((arg + IPATH_PIO_MAXIBHDR) != dd->ipath_ibmaxlen) {
-		piosize = arg + IPATH_PIO_MAXIBHDR;
-		ipath_cdbg(VERBOSE, "ibmaxlen was 0x%x, setting to 0x%x "
-			   "(mtu 0x%x)\n", dd->ipath_ibmaxlen, piosize,
-			   arg);
-		dd->ipath_ibmaxlen = piosize;
-		changed = 1;
-	}
-
-	if (changed) {
-		/*
-		 * set the IBC maxpktlength to the size of our pio
-		 * buffers in words
-		 */
-		u64 ibc = dd->ipath_ibcctrl;
-		ibc &= ~(INFINIPATH_IBCC_MAXPKTLEN_MASK <<
-			 INFINIPATH_IBCC_MAXPKTLEN_SHIFT);
-
-		piosize = piosize - 2 * sizeof(u32);	/* ignore pbc */
-		dd->ipath_ibmaxlen = piosize;
-		piosize /= sizeof(u32);	/* in words */
-		/*
-		 * for ICRC, which we only send in diag test pkt mode, and
-		 * we don't need to worry about that for mtu
-		 */
-		piosize += 1;
-
-		ibc |= piosize << INFINIPATH_IBCC_MAXPKTLEN_SHIFT;
-		dd->ipath_ibcctrl = ibc;
-		ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
-				 dd->ipath_ibcctrl);
-		dd->ipath_f_tidtemplate(dd);
-	}
-
-	ret = 0;
-
-bail:
-	return ret;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_set_mtu);
-
-int ipath_set_lid(struct ipath_devdata *dd, u32 arg, u8 lmc)
-{
-	dd->ipath_lid = arg;
-	dd->ipath_lmc = lmc;
-
 	mutex_lock(&ipath_layer_mutex);
 
 	if (dd->ipath_layer.l_arg && layer_intr)
 		layer_intr(dd->ipath_layer.l_arg, IPATH_LAYER_INT_LID);
 
 	mutex_unlock(&ipath_layer_mutex);
-
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_set_lid);
-
-int ipath_layer_set_guid(struct ipath_devdata *dd, __be64 guid)
-{
-	/* XXX - need to inform anyone who cares this just happened. */
-	dd->ipath_guid = guid;
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_set_guid);
-
-__be64 ipath_layer_get_guid(struct ipath_devdata *dd)
-{
-	return dd->ipath_guid;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_guid);
-
-u32 ipath_layer_get_nguid(struct ipath_devdata *dd)
-{
-	return dd->ipath_nguid;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_nguid);
-
-u32 ipath_layer_get_majrev(struct ipath_devdata *dd)
-{
-	return dd->ipath_majrev;
 }
 
-EXPORT_SYMBOL_GPL(ipath_layer_get_majrev);
-
-u32 ipath_layer_get_minrev(struct ipath_devdata *dd)
-{
-	return dd->ipath_minrev;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_minrev);
-
-u32 ipath_layer_get_pcirev(struct ipath_devdata *dd)
-{
-	return dd->ipath_pcirev;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_pcirev);
-
-u32 ipath_layer_get_flags(struct ipath_devdata *dd)
-{
-	return dd->ipath_flags;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_flags);
-
-struct device *ipath_layer_get_device(struct ipath_devdata *dd)
-{
-	return &dd->pcidev->dev;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_device);
-
-u16 ipath_layer_get_deviceid(struct ipath_devdata *dd)
-{
-	return dd->ipath_deviceid;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_deviceid);
-
-u32 ipath_layer_get_vendorid(struct ipath_devdata *dd)
-{
-	return dd->ipath_vendorid;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_vendorid);
-
-u64 ipath_layer_get_lastibcstat(struct ipath_devdata *dd)
-{
-	return dd->ipath_lastibcstat;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_lastibcstat);
-
-u32 ipath_layer_get_ibmtu(struct ipath_devdata *dd)
-{
-	return dd->ipath_ibmtu;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_ibmtu);
-
 void ipath_layer_add(struct ipath_devdata *dd)
 {
 	mutex_lock(&ipath_layer_mutex);
@@ -411,10 +119,6 @@ void ipath_layer_add(struct ipath_devdata *dd)
 		dd->ipath_layer.l_arg =
 			layer_add_one(dd->ipath_unit, dd);
 
-	if (verbs_add_one)
-		dd->verbs_layer.l_arg =
-			verbs_add_one(dd->ipath_unit, dd);
-
 	mutex_unlock(&ipath_layer_mutex);
 }
 
@@ -427,11 +131,6 @@ void ipath_layer_remove(struct ipath_devdata *dd)
 		dd->ipath_layer.l_arg = NULL;
 	}
 
-	if (dd->verbs_layer.l_arg && verbs_remove_one) {
-		verbs_remove_one(dd->verbs_layer.l_arg);
-		dd->verbs_layer.l_arg = NULL;
-	}
-
 	mutex_unlock(&ipath_layer_mutex);
 }
 
@@ -463,9 +162,6 @@ int ipath_layer_register(void *(*l_add)(int, struct ipath_devdata *),
 		if (dd->ipath_layer.l_arg)
 			continue;
 
-		if (!(*dd->ipath_statusp & IPATH_STATUS_SMA))
-			*dd->ipath_statusp |= IPATH_STATUS_OIB_SMA;
-
 		spin_unlock_irqrestore(&ipath_devs_lock, flags);
 		dd->ipath_layer.l_arg = l_add(dd->ipath_unit, dd);
 		spin_lock_irqsave(&ipath_devs_lock, flags);
@@ -509,107 +205,6 @@ void ipath_layer_unregister(void)
 
 EXPORT_SYMBOL_GPL(ipath_layer_unregister);
 
-static void __ipath_verbs_timer(unsigned long arg)
-{
-	struct ipath_devdata *dd = (struct ipath_devdata *) arg;
-
-	/*
-	 * If port 0 receive packet interrupts are not available, or
-	 * can be missed, poll the receive queue
-	 */
-	if (dd->ipath_flags & IPATH_POLL_RX_INTR)
-		ipath_kreceive(dd);
-
-	/* Handle verbs layer timeouts. */
-	if (dd->verbs_layer.l_arg && verbs_timer_cb)
-		verbs_timer_cb(dd->verbs_layer.l_arg);
-
-	mod_timer(&dd->verbs_layer.l_timer, jiffies + 1);
-}
-
-/**
- * ipath_verbs_register - verbs layer registration
- * @l_piobufavail: callback for when PIO buffers become available
- * @l_rcv: callback for receiving a packet
- * @l_timer_cb: timer callback
- * @ipath_devdata: device data structure is put here
- */
-int ipath_verbs_register(void *(*l_add)(int, struct ipath_devdata *),
-			 void (*l_remove)(void *arg),
-			 int (*l_piobufavail) (void *arg),
-			 void (*l_rcv) (void *arg, void *rhdr,
-					void *data, u32 tlen),
-			 void (*l_timer_cb) (void *arg))
-{
-	struct ipath_devdata *dd, *tmp;
-	unsigned long flags;
-
-	mutex_lock(&ipath_layer_mutex);
-
-	verbs_add_one = l_add;
-	verbs_remove_one = l_remove;
-	verbs_piobufavail = l_piobufavail;
-	verbs_rcv = l_rcv;
-	verbs_timer_cb = l_timer_cb;
-
-	spin_lock_irqsave(&ipath_devs_lock, flags);
-
-	list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) {
-		if (!(dd->ipath_flags & IPATH_INITTED))
-			continue;
-
-		if (dd->verbs_layer.l_arg)
-			continue;
-
-		spin_unlock_irqrestore(&ipath_devs_lock, flags);
-		dd->verbs_layer.l_arg = l_add(dd->ipath_unit, dd);
-		spin_lock_irqsave(&ipath_devs_lock, flags);
-	}
-
-	spin_unlock_irqrestore(&ipath_devs_lock, flags);
-	mutex_unlock(&ipath_layer_mutex);
-
-	ipath_verbs_registered = 1;
-
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_verbs_register);
-
-void ipath_verbs_unregister(void)
-{
-	struct ipath_devdata *dd, *tmp;
-	unsigned long flags;
-
-	mutex_lock(&ipath_layer_mutex);
-	spin_lock_irqsave(&ipath_devs_lock, flags);
-
-	list_for_each_entry_safe(dd, tmp, &ipath_dev_list, ipath_list) {
-		*dd->ipath_statusp &= ~IPATH_STATUS_OIB_SMA;
-
-		if (dd->verbs_layer.l_arg && verbs_remove_one) {
-			spin_unlock_irqrestore(&ipath_devs_lock, flags);
-			verbs_remove_one(dd->verbs_layer.l_arg);
-			spin_lock_irqsave(&ipath_devs_lock, flags);
-			dd->verbs_layer.l_arg = NULL;
-		}
-	}
-
-	spin_unlock_irqrestore(&ipath_devs_lock, flags);
-
-	verbs_add_one = NULL;
-	verbs_remove_one = NULL;
-	verbs_piobufavail = NULL;
-	verbs_rcv = NULL;
-	verbs_timer_cb = NULL;
-
-	ipath_verbs_registered = 0;
-
-	mutex_unlock(&ipath_layer_mutex);
-}
-
-EXPORT_SYMBOL_GPL(ipath_verbs_unregister);
-
 int ipath_layer_open(struct ipath_devdata *dd, u32 * pktmax)
 {
 	int ret;
@@ -698,390 +293,6 @@ u16 ipath_layer_get_bcast(struct ipath_devdata *dd)
 
 EXPORT_SYMBOL_GPL(ipath_layer_get_bcast);
 
-u32 ipath_layer_get_cr_errpkey(struct ipath_devdata *dd)
-{
-	return ipath_read_creg32(dd, dd->ipath_cregs->cr_errpkey);
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_cr_errpkey);
-
-static void update_sge(struct ipath_sge_state *ss, u32 length)
-{
-	struct ipath_sge *sge = &ss->sge;
-
-	sge->vaddr += length;
-	sge->length -= length;
-	sge->sge_length -= length;
-	if (sge->sge_length == 0) {
-		if (--ss->num_sge)
-			*sge = *ss->sg_list++;
-	} else if (sge->length == 0 && sge->mr != NULL) {
-		if (++sge->n >= IPATH_SEGSZ) {
-			if (++sge->m >= sge->mr->mapsz)
-				return;
-			sge->n = 0;
-		}
-		sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
-		sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
-	}
-}
-
-#ifdef __LITTLE_ENDIAN
-static inline u32 get_upper_bits(u32 data, u32 shift)
-{
-	return data >> shift;
-}
-
-static inline u32 set_upper_bits(u32 data, u32 shift)
-{
-	return data << shift;
-}
-
-static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off)
-{
-	data <<= ((sizeof(u32) - n) * BITS_PER_BYTE);
-	data >>= ((sizeof(u32) - n - off) * BITS_PER_BYTE);
-	return data;
-}
-#else
-static inline u32 get_upper_bits(u32 data, u32 shift)
-{
-	return data << shift;
-}
-
-static inline u32 set_upper_bits(u32 data, u32 shift)
-{
-	return data >> shift;
-}
-
-static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off)
-{
-	data >>= ((sizeof(u32) - n) * BITS_PER_BYTE);
-	data <<= ((sizeof(u32) - n - off) * BITS_PER_BYTE);
-	return data;
-}
-#endif
-
-static void copy_io(u32 __iomem *piobuf, struct ipath_sge_state *ss,
-		    u32 length)
-{
-	u32 extra = 0;
-	u32 data = 0;
-	u32 last;
-
-	while (1) {
-		u32 len = ss->sge.length;
-		u32 off;
-
-		BUG_ON(len == 0);
-		if (len > length)
-			len = length;
-		if (len > ss->sge.sge_length)
-			len = ss->sge.sge_length;
-		/* If the source address is not aligned, try to align it. */
-		off = (unsigned long)ss->sge.vaddr & (sizeof(u32) - 1);
-		if (off) {
-			u32 *addr = (u32 *)((unsigned long)ss->sge.vaddr &
-					    ~(sizeof(u32) - 1));
-			u32 v = get_upper_bits(*addr, off * BITS_PER_BYTE);
-			u32 y;
-
-			y = sizeof(u32) - off;
-			if (len > y)
-				len = y;
-			if (len + extra >= sizeof(u32)) {
-				data |= set_upper_bits(v, extra *
-						       BITS_PER_BYTE);
-				len = sizeof(u32) - extra;
-				if (len == length) {
-					last = data;
-					break;
-				}
-				__raw_writel(data, piobuf);
-				piobuf++;
-				extra = 0;
-				data = 0;
-			} else {
-				/* Clear unused upper bytes */
-				data |= clear_upper_bytes(v, len, extra);
-				if (len == length) {
-					last = data;
-					break;
-				}
-				extra += len;
-			}
-		} else if (extra) {
-			/* Source address is aligned. */
-			u32 *addr = (u32 *) ss->sge.vaddr;
-			int shift = extra * BITS_PER_BYTE;
-			int ushift = 32 - shift;
-			u32 l = len;
-
-			while (l >= sizeof(u32)) {
-				u32 v = *addr;
-
-				data |= set_upper_bits(v, shift);
-				__raw_writel(data, piobuf);
-				data = get_upper_bits(v, ushift);
-				piobuf++;
-				addr++;
-				l -= sizeof(u32);
-			}
-			/*
-			 * We still have 'extra' number of bytes leftover.
-			 */
-			if (l) {
-				u32 v = *addr;
-
-				if (l + extra >= sizeof(u32)) {
-					data |= set_upper_bits(v, shift);
-					len -= l + extra - sizeof(u32);
-					if (len == length) {
-						last = data;
-						break;
-					}
-					__raw_writel(data, piobuf);
-					piobuf++;
-					extra = 0;
-					data = 0;
-				} else {
-					/* Clear unused upper bytes */
-					data |= clear_upper_bytes(v, l,
-								  extra);
-					if (len == length) {
-						last = data;
-						break;
-					}
-					extra += l;
-				}
-			} else if (len == length) {
-				last = data;
-				break;
-			}
-		} else if (len == length) {
-			u32 w;
-
-			/*
-			 * Need to round up for the last dword in the
-			 * packet.
-			 */
-			w = (len + 3) >> 2;
-			__iowrite32_copy(piobuf, ss->sge.vaddr, w - 1);
-			piobuf += w - 1;
-			last = ((u32 *) ss->sge.vaddr)[w - 1];
-			break;
-		} else {
-			u32 w = len >> 2;
-
-			__iowrite32_copy(piobuf, ss->sge.vaddr, w);
-			piobuf += w;
-
-			extra = len & (sizeof(u32) - 1);
-			if (extra) {
-				u32 v = ((u32 *) ss->sge.vaddr)[w];
-
-				/* Clear unused upper bytes */
-				data = clear_upper_bytes(v, extra, 0);
-			}
-		}
-		update_sge(ss, len);
-		length -= len;
-	}
-	/* Update address before sending packet. */
-	update_sge(ss, length);
-	/* must flush early everything before trigger word */
-	ipath_flush_wc();
-	__raw_writel(last, piobuf);
-	/* be sure trigger word is written */
-	ipath_flush_wc();
-}
-
-/**
- * ipath_verbs_send - send a packet from the verbs layer
- * @dd: the infinipath device
- * @hdrwords: the number of words in the header
- * @hdr: the packet header
- * @len: the length of the packet in bytes
- * @ss: the SGE to send
- *
- * This is like ipath_sma_send_pkt() in that we need to be able to send
- * packets after the chip is initialized (MADs) but also like
- * ipath_layer_send_hdr() since its used by the verbs layer.
- */
-int ipath_verbs_send(struct ipath_devdata *dd, u32 hdrwords,
-		     u32 *hdr, u32 len, struct ipath_sge_state *ss)
-{
-	u32 __iomem *piobuf;
-	u32 plen;
-	int ret;
-
-	/* +1 is for the qword padding of pbc */
-	plen = hdrwords + ((len + 3) >> 2) + 1;
-	if (unlikely((plen << 2) > dd->ipath_ibmaxlen)) {
-		ipath_dbg("packet len 0x%x too long, failing\n", plen);
-		ret = -EINVAL;
-		goto bail;
-	}
-
-	/* Get a PIO buffer to use. */
-	piobuf = ipath_getpiobuf(dd, NULL);
-	if (unlikely(piobuf == NULL)) {
-		ret = -EBUSY;
-		goto bail;
-	}
-
-	/*
-	 * Write len to control qword, no flags.
-	 * We have to flush after the PBC for correctness on some cpus
-	 * or WC buffer can be written out of order.
-	 */
-	writeq(plen, piobuf);
-	ipath_flush_wc();
-	piobuf += 2;
-	if (len == 0) {
-		/*
-		 * If there is just the header portion, must flush before
-		 * writing last word of header for correctness, and after
-		 * the last header word (trigger word).
-		 */
-		__iowrite32_copy(piobuf, hdr, hdrwords - 1);
-		ipath_flush_wc();
-		__raw_writel(hdr[hdrwords - 1], piobuf + hdrwords - 1);
-		ipath_flush_wc();
-		ret = 0;
-		goto bail;
-	}
-
-	__iowrite32_copy(piobuf, hdr, hdrwords);
-	piobuf += hdrwords;
-
-	/* The common case is aligned and contained in one segment. */
-	if (likely(ss->num_sge == 1 && len <= ss->sge.length &&
-		   !((unsigned long)ss->sge.vaddr & (sizeof(u32) - 1)))) {
-		u32 w;
-		u32 *addr = (u32 *) ss->sge.vaddr;
-
-		/* Update address before sending packet. */
-		update_sge(ss, len);
-		/* Need to round up for the last dword in the packet. */
-		w = (len + 3) >> 2;
-		__iowrite32_copy(piobuf, addr, w - 1);
-		/* must flush early everything before trigger word */
-		ipath_flush_wc();
-		__raw_writel(addr[w - 1], piobuf + w - 1);
-		/* be sure trigger word is written */
-		ipath_flush_wc();
-		ret = 0;
-		goto bail;
-	}
-	copy_io(piobuf, ss, len);
-	ret = 0;
-
-bail:
-	return ret;
-}
-
-EXPORT_SYMBOL_GPL(ipath_verbs_send);
-
-int ipath_layer_snapshot_counters(struct ipath_devdata *dd, u64 *swords,
-				  u64 *rwords, u64 *spkts, u64 *rpkts,
-				  u64 *xmit_wait)
-{
-	int ret;
-
-	if (!(dd->ipath_flags & IPATH_INITTED)) {
-		/* no hardware, freeze, etc. */
-		ipath_dbg("unit %u not usable\n", dd->ipath_unit);
-		ret = -EINVAL;
-		goto bail;
-	}
-	*swords = ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordsendcnt);
-	*rwords = ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordrcvcnt);
-	*spkts = ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktsendcnt);
-	*rpkts = ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktrcvcnt);
-	*xmit_wait = ipath_snap_cntr(dd, dd->ipath_cregs->cr_sendstallcnt);
-
-	ret = 0;
-
-bail:
-	return ret;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_snapshot_counters);
-
-/**
- * ipath_layer_get_counters - get various chip counters
- * @dd: the infinipath device
- * @cntrs: counters are placed here
- *
- * Return the counters needed by recv_pma_get_portcounters().
- */
-int ipath_layer_get_counters(struct ipath_devdata *dd,
-			      struct ipath_layer_counters *cntrs)
-{
-	int ret;
-
-	if (!(dd->ipath_flags & IPATH_INITTED)) {
-		/* no hardware, freeze, etc. */
-		ipath_dbg("unit %u not usable\n", dd->ipath_unit);
-		ret = -EINVAL;
-		goto bail;
-	}
-	cntrs->symbol_error_counter =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_ibsymbolerrcnt);
-	cntrs->link_error_recovery_counter =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_iblinkerrrecovcnt);
-	/*
-	 * The link downed counter counts when the other side downs the
-	 * connection.  We add in the number of times we downed the link
-	 * due to local link integrity errors to compensate.
-	 */
-	cntrs->link_downed_counter =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_iblinkdowncnt);
-	cntrs->port_rcv_errors =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_rxdroppktcnt) +
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_rcvovflcnt) +
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_portovflcnt) +
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_err_rlencnt) +
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_invalidrlencnt) +
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_erricrccnt) +
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_errvcrccnt) +
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_errlpcrccnt) +
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_badformatcnt);
-	cntrs->port_rcv_remphys_errors =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_rcvebpcnt);
-	cntrs->port_xmit_discards =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_unsupvlcnt);
-	cntrs->port_xmit_data =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordsendcnt);
-	cntrs->port_rcv_data =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordrcvcnt);
-	cntrs->port_xmit_packets =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktsendcnt);
-	cntrs->port_rcv_packets =
-		ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktrcvcnt);
-	cntrs->local_link_integrity_errors = dd->ipath_lli_errors;
-	cntrs->excessive_buffer_overrun_errors = 0; /* XXX */
-
-	ret = 0;
-
-bail:
-	return ret;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_counters);
-
-int ipath_layer_want_buffer(struct ipath_devdata *dd)
-{
-	set_bit(IPATH_S_PIOINTBUFAVAIL, &dd->ipath_sendctrl);
-	ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
-			 dd->ipath_sendctrl);
-
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_want_buffer);
-
 int ipath_layer_send_hdr(struct ipath_devdata *dd, struct ether_header *hdr)
 {
 	int ret = 0;
@@ -1153,389 +364,3 @@ int ipath_layer_set_piointbufavail_int(struct ipath_devdata *dd)
 }
 
 EXPORT_SYMBOL_GPL(ipath_layer_set_piointbufavail_int);
-
-int ipath_layer_enable_timer(struct ipath_devdata *dd)
-{
-	/*
-	 * HT-400 has a design flaw where the chip and kernel idea
-	 * of the tail register don't always agree, and therefore we won't
-	 * get an interrupt on the next packet received.
-	 * If the board supports per packet receive interrupts, use it.
-	 * Otherwise, the timer function periodically checks for packets
-	 * to cover this case.
-	 * Either way, the timer is needed for verbs layer related
-	 * processing.
-	 */
-	if (dd->ipath_flags & IPATH_GPIO_INTR) {
-		ipath_write_kreg(dd, dd->ipath_kregs->kr_debugportselect,
-				 0x2074076542310ULL);
-		/* Enable GPIO bit 2 interrupt */
-		ipath_write_kreg(dd, dd->ipath_kregs->kr_gpio_mask,
-				 (u64) (1 << 2));
-	}
-
-	init_timer(&dd->verbs_layer.l_timer);
-	dd->verbs_layer.l_timer.function = __ipath_verbs_timer;
-	dd->verbs_layer.l_timer.data = (unsigned long)dd;
-	dd->verbs_layer.l_timer.expires = jiffies + 1;
-	add_timer(&dd->verbs_layer.l_timer);
-
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_enable_timer);
-
-int ipath_layer_disable_timer(struct ipath_devdata *dd)
-{
-	/* Disable GPIO bit 2 interrupt */
-	if (dd->ipath_flags & IPATH_GPIO_INTR)
-		ipath_write_kreg(dd, dd->ipath_kregs->kr_gpio_mask, 0);
-
-	del_timer_sync(&dd->verbs_layer.l_timer);
-
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_disable_timer);
-
-/**
- * ipath_layer_set_verbs_flags - set the verbs layer flags
- * @dd: the infinipath device
- * @flags: the flags to set
- */
-int ipath_layer_set_verbs_flags(struct ipath_devdata *dd, unsigned flags)
-{
-	struct ipath_devdata *ss;
-	unsigned long lflags;
-
-	spin_lock_irqsave(&ipath_devs_lock, lflags);
-
-	list_for_each_entry(ss, &ipath_dev_list, ipath_list) {
-		if (!(ss->ipath_flags & IPATH_INITTED))
-			continue;
-		if ((flags & IPATH_VERBS_KERNEL_SMA) &&
-		    !(*ss->ipath_statusp & IPATH_STATUS_SMA))
-			*ss->ipath_statusp |= IPATH_STATUS_OIB_SMA;
-		else
-			*ss->ipath_statusp &= ~IPATH_STATUS_OIB_SMA;
-	}
-
-	spin_unlock_irqrestore(&ipath_devs_lock, lflags);
-
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_set_verbs_flags);
-
-/**
- * ipath_layer_get_npkeys - return the size of the PKEY table for port 0
- * @dd: the infinipath device
- */
-unsigned ipath_layer_get_npkeys(struct ipath_devdata *dd)
-{
-	return ARRAY_SIZE(dd->ipath_pd[0]->port_pkeys);
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_npkeys);
-
-/**
- * ipath_layer_get_pkey - return the indexed PKEY from the port 0 PKEY table
- * @dd: the infinipath device
- * @index: the PKEY index
- */
-unsigned ipath_layer_get_pkey(struct ipath_devdata *dd, unsigned index)
-{
-	unsigned ret;
-
-	if (index >= ARRAY_SIZE(dd->ipath_pd[0]->port_pkeys))
-		ret = 0;
-	else
-		ret = dd->ipath_pd[0]->port_pkeys[index];
-
-	return ret;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_pkey);
-
-/**
- * ipath_layer_get_pkeys - return the PKEY table for port 0
- * @dd: the infinipath device
- * @pkeys: the pkey table is placed here
- */
-int ipath_layer_get_pkeys(struct ipath_devdata *dd, u16 * pkeys)
-{
-	struct ipath_portdata *pd = dd->ipath_pd[0];
-
-	memcpy(pkeys, pd->port_pkeys, sizeof(pd->port_pkeys));
-
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_pkeys);
-
-/**
- * rm_pkey - decrecment the reference count for the given PKEY
- * @dd: the infinipath device
- * @key: the PKEY index
- *
- * Return true if this was the last reference and the hardware table entry
- * needs to be changed.
- */
-static int rm_pkey(struct ipath_devdata *dd, u16 key)
-{
-	int i;
-	int ret;
-
-	for (i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
-		if (dd->ipath_pkeys[i] != key)
-			continue;
-		if (atomic_dec_and_test(&dd->ipath_pkeyrefs[i])) {
-			dd->ipath_pkeys[i] = 0;
-			ret = 1;
-			goto bail;
-		}
-		break;
-	}
-
-	ret = 0;
-
-bail:
-	return ret;
-}
-
-/**
- * add_pkey - add the given PKEY to the hardware table
- * @dd: the infinipath device
- * @key: the PKEY
- *
- * Return an error code if unable to add the entry, zero if no change,
- * or 1 if the hardware PKEY register needs to be updated.
- */
-static int add_pkey(struct ipath_devdata *dd, u16 key)
-{
-	int i;
-	u16 lkey = key & 0x7FFF;
-	int any = 0;
-	int ret;
-
-	if (lkey == 0x7FFF) {
-		ret = 0;
-		goto bail;
-	}
-
-	/* Look for an empty slot or a matching PKEY. */
-	for (i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
-		if (!dd->ipath_pkeys[i]) {
-			any++;
-			continue;
-		}
-		/* If it matches exactly, try to increment the ref count */
-		if (dd->ipath_pkeys[i] == key) {
-			if (atomic_inc_return(&dd->ipath_pkeyrefs[i]) > 1) {
-				ret = 0;
-				goto bail;
-			}
-			/* Lost the race. Look for an empty slot below. */
-			atomic_dec(&dd->ipath_pkeyrefs[i]);
-			any++;
-		}
-		/*
-		 * It makes no sense to have both the limited and unlimited
-		 * PKEY set at the same time since the unlimited one will
-		 * disable the limited one.
-		 */
-		if ((dd->ipath_pkeys[i] & 0x7FFF) == lkey) {
-			ret = -EEXIST;
-			goto bail;
-		}
-	}
-	if (!any) {
-		ret = -EBUSY;
-		goto bail;
-	}
-	for (i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
-		if (!dd->ipath_pkeys[i] &&
-		    atomic_inc_return(&dd->ipath_pkeyrefs[i]) == 1) {
-			/* for ipathstats, etc. */
-			ipath_stats.sps_pkeys[i] = lkey;
-			dd->ipath_pkeys[i] = key;
-			ret = 1;
-			goto bail;
-		}
-	}
-	ret = -EBUSY;
-
-bail:
-	return ret;
-}
-
-/**
- * ipath_layer_set_pkeys - set the PKEY table for port 0
- * @dd: the infinipath device
- * @pkeys: the PKEY table
- */
-int ipath_layer_set_pkeys(struct ipath_devdata *dd, u16 * pkeys)
-{
-	struct ipath_portdata *pd;
-	int i;
-	int changed = 0;
-
-	pd = dd->ipath_pd[0];
-
-	for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
-		u16 key = pkeys[i];
-		u16 okey = pd->port_pkeys[i];
-
-		if (key == okey)
-			continue;
-		/*
-		 * The value of this PKEY table entry is changing.
-		 * Remove the old entry in the hardware's array of PKEYs.
-		 */
-		if (okey & 0x7FFF)
-			changed |= rm_pkey(dd, okey);
-		if (key & 0x7FFF) {
-			int ret = add_pkey(dd, key);
-
-			if (ret < 0)
-				key = 0;
-			else
-				changed |= ret;
-		}
-		pd->port_pkeys[i] = key;
-	}
-	if (changed) {
-		u64 pkey;
-
-		pkey = (u64) dd->ipath_pkeys[0] |
-			((u64) dd->ipath_pkeys[1] << 16) |
-			((u64) dd->ipath_pkeys[2] << 32) |
-			((u64) dd->ipath_pkeys[3] << 48);
-		ipath_cdbg(VERBOSE, "p0 new pkey reg %llx\n",
-			   (unsigned long long) pkey);
-		ipath_write_kreg(dd, dd->ipath_kregs->kr_partitionkey,
-				 pkey);
-	}
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_set_pkeys);
-
-/**
- * ipath_layer_get_linkdowndefaultstate - get the default linkdown state
- * @dd: the infinipath device
- *
- * Returns zero if the default is POLL, 1 if the default is SLEEP.
- */
-int ipath_layer_get_linkdowndefaultstate(struct ipath_devdata *dd)
-{
-	return !!(dd->ipath_ibcctrl & INFINIPATH_IBCC_LINKDOWNDEFAULTSTATE);
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_linkdowndefaultstate);
-
-/**
- * ipath_layer_set_linkdowndefaultstate - set the default linkdown state
- * @dd: the infinipath device
- * @sleep: the new state
- *
- * Note that this will only take effect when the link state changes.
- */
-int ipath_layer_set_linkdowndefaultstate(struct ipath_devdata *dd,
-					 int sleep)
-{
-	if (sleep)
-		dd->ipath_ibcctrl |= INFINIPATH_IBCC_LINKDOWNDEFAULTSTATE;
-	else
-		dd->ipath_ibcctrl &= ~INFINIPATH_IBCC_LINKDOWNDEFAULTSTATE;
-	ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
-			 dd->ipath_ibcctrl);
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_set_linkdowndefaultstate);
-
-int ipath_layer_get_phyerrthreshold(struct ipath_devdata *dd)
-{
-	return (dd->ipath_ibcctrl >>
-		INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT) &
-		INFINIPATH_IBCC_PHYERRTHRESHOLD_MASK;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_phyerrthreshold);
-
-/**
- * ipath_layer_set_phyerrthreshold - set the physical error threshold
- * @dd: the infinipath device
- * @n: the new threshold
- *
- * Note that this will only take effect when the link state changes.
- */
-int ipath_layer_set_phyerrthreshold(struct ipath_devdata *dd, unsigned n)
-{
-	unsigned v;
-
-	v = (dd->ipath_ibcctrl >> INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT) &
-		INFINIPATH_IBCC_PHYERRTHRESHOLD_MASK;
-	if (v != n) {
-		dd->ipath_ibcctrl &=
-			~(INFINIPATH_IBCC_PHYERRTHRESHOLD_MASK <<
-			  INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT);
-		dd->ipath_ibcctrl |=
-			(u64) n << INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT;
-		ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
-				 dd->ipath_ibcctrl);
-	}
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_set_phyerrthreshold);
-
-int ipath_layer_get_overrunthreshold(struct ipath_devdata *dd)
-{
-	return (dd->ipath_ibcctrl >>
-		INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT) &
-		INFINIPATH_IBCC_OVERRUNTHRESHOLD_MASK;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_get_overrunthreshold);
-
-/**
- * ipath_layer_set_overrunthreshold - set the overrun threshold
- * @dd: the infinipath device
- * @n: the new threshold
- *
- * Note that this will only take effect when the link state changes.
- */
-int ipath_layer_set_overrunthreshold(struct ipath_devdata *dd, unsigned n)
-{
-	unsigned v;
-
-	v = (dd->ipath_ibcctrl >> INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT) &
-		INFINIPATH_IBCC_OVERRUNTHRESHOLD_MASK;
-	if (v != n) {
-		dd->ipath_ibcctrl &=
-			~(INFINIPATH_IBCC_OVERRUNTHRESHOLD_MASK <<
-			  INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT);
-		dd->ipath_ibcctrl |=
-			(u64) n << INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT;
-		ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
-				 dd->ipath_ibcctrl);
-	}
-	return 0;
-}
-
-EXPORT_SYMBOL_GPL(ipath_layer_set_overrunthreshold);
-
-int ipath_layer_get_boardname(struct ipath_devdata *dd, char *name,
-			      size_t namelen)
-{
-	return dd->ipath_f_get_boardname(dd, name, namelen);
-}
-EXPORT_SYMBOL_GPL(ipath_layer_get_boardname);
-
-u32 ipath_layer_get_rcvhdrentsize(struct ipath_devdata *dd)
-{
-	return dd->ipath_rcvhdrentsize;
-}
-EXPORT_SYMBOL_GPL(ipath_layer_get_rcvhdrentsize);
diff --git a/drivers/infiniband/hw/ipath/ipath_layer.h b/drivers/infiniband/hw/ipath/ipath_layer.h
index 7148509..3854a4e 100644
--- a/drivers/infiniband/hw/ipath/ipath_layer.h
+++ b/drivers/infiniband/hw/ipath/ipath_layer.h
@@ -40,73 +40,9 @@
  */
 
 struct sk_buff;
-struct ipath_sge_state;
 struct ipath_devdata;
 struct ether_header;
 
-struct ipath_layer_counters {
-	u64 symbol_error_counter;
-	u64 link_error_recovery_counter;
-	u64 link_downed_counter;
-	u64 port_rcv_errors;
-	u64 port_rcv_remphys_errors;
-	u64 port_xmit_discards;
-	u64 port_xmit_data;
-	u64 port_rcv_data;
-	u64 port_xmit_packets;
-	u64 port_rcv_packets;
-	u32 local_link_integrity_errors;
-	u32 excessive_buffer_overrun_errors;
-};
-
-/*
- * A segment is a linear region of low physical memory.
- * XXX Maybe we should use phys addr here and kmap()/kunmap().
- * Used by the verbs layer.
- */
-struct ipath_seg {
-	void *vaddr;
-	size_t length;
-};
-
-/* The number of ipath_segs that fit in a page. */
-#define IPATH_SEGSZ     (PAGE_SIZE / sizeof (struct ipath_seg))
-
-struct ipath_segarray {
-	struct ipath_seg segs[IPATH_SEGSZ];
-};
-
-struct ipath_mregion {
-	u64 user_base;		/* User's address for this region */
-	u64 iova;		/* IB start address of this region */
-	size_t length;
-	u32 lkey;
-	u32 offset;		/* offset (bytes) to start of region */
-	int access_flags;
-	u32 max_segs;		/* number of ipath_segs in all the arrays */
-	u32 mapsz;		/* size of the map array */
-	struct ipath_segarray *map[0];	/* the segments */
-};
-
-/*
- * These keep track of the copy progress within a memory region.
- * Used by the verbs layer.
- */
-struct ipath_sge {
-	struct ipath_mregion *mr;
-	void *vaddr;		/* current pointer into the segment */
-	u32 sge_length;		/* length of the SGE */
-	u32 length;		/* remaining length of the segment */
-	u16 m;			/* current index: mr->map[m] */
-	u16 n;			/* current index: mr->map[m]->segs[n] */
-};
-
-struct ipath_sge_state {
-	struct ipath_sge *sg_list;	/* next SGE to be used if any */
-	struct ipath_sge sge;	/* progress state for the current SGE */
-	u8 num_sge;
-};
-
 int ipath_layer_register(void *(*l_add)(int, struct ipath_devdata *),
 			 void (*l_remove)(void *),
 			 int (*l_intr)(void *, u32),
@@ -114,62 +50,14 @@ int ipath_layer_register(void *(*l_add)(int, struct ipath_devdata *),
 				      struct sk_buff *),
 			 u16 rcv_opcode,
 			 int (*l_rcv_lid)(void *, void *));
-int ipath_verbs_register(void *(*l_add)(int, struct ipath_devdata *),
-			 void (*l_remove)(void *arg),
-			 int (*l_piobufavail)(void *arg),
-			 void (*l_rcv)(void *arg, void *rhdr,
-				       void *data, u32 tlen),
-			 void (*l_timer_cb)(void *arg));
 void ipath_layer_unregister(void);
-void ipath_verbs_unregister(void);
 int ipath_layer_open(struct ipath_devdata *, u32 * pktmax);
 u16 ipath_layer_get_lid(struct ipath_devdata *dd);
 int ipath_layer_get_mac(struct ipath_devdata *dd, u8 *);
 u16 ipath_layer_get_bcast(struct ipath_devdata *dd);
-u32 ipath_layer_get_cr_errpkey(struct ipath_devdata *dd);
-int ipath_layer_set_linkstate(struct ipath_devdata *dd, u8 state);
-int ipath_layer_set_mtu(struct ipath_devdata *, u16);
-int ipath_set_lid(struct ipath_devdata *, u32, u8);
 int ipath_layer_send_hdr(struct ipath_devdata *dd,
 			 struct ether_header *hdr);
-int ipath_verbs_send(struct ipath_devdata *dd, u32 hdrwords,
-		     u32 * hdr, u32 len, struct ipath_sge_state *ss);
 int ipath_layer_set_piointbufavail_int(struct ipath_devdata *dd);
-int ipath_layer_get_boardname(struct ipath_devdata *dd, char *name,
-			      size_t namelen);
-int ipath_layer_snapshot_counters(struct ipath_devdata *dd, u64 *swords,
-				  u64 *rwords, u64 *spkts, u64 *rpkts,
-				  u64 *xmit_wait);
-int ipath_layer_get_counters(struct ipath_devdata *dd,
-			     struct ipath_layer_counters *cntrs);
-int ipath_layer_want_buffer(struct ipath_devdata *dd);
-int ipath_layer_set_guid(struct ipath_devdata *, __be64 guid);
-__be64 ipath_layer_get_guid(struct ipath_devdata *);
-u32 ipath_layer_get_nguid(struct ipath_devdata *);
-u32 ipath_layer_get_majrev(struct ipath_devdata *);
-u32 ipath_layer_get_minrev(struct ipath_devdata *);
-u32 ipath_layer_get_pcirev(struct ipath_devdata *);
-u32 ipath_layer_get_flags(struct ipath_devdata *dd);
-struct device *ipath_layer_get_device(struct ipath_devdata *dd);
-u16 ipath_layer_get_deviceid(struct ipath_devdata *dd);
-u32 ipath_layer_get_vendorid(struct ipath_devdata *);
-u64 ipath_layer_get_lastibcstat(struct ipath_devdata *dd);
-u32 ipath_layer_get_ibmtu(struct ipath_devdata *dd);
-int ipath_layer_enable_timer(struct ipath_devdata *dd);
-int ipath_layer_disable_timer(struct ipath_devdata *dd);
-int ipath_layer_set_verbs_flags(struct ipath_devdata *dd, unsigned flags);
-unsigned ipath_layer_get_npkeys(struct ipath_devdata *dd);
-unsigned ipath_layer_get_pkey(struct ipath_devdata *dd, unsigned index);
-int ipath_layer_get_pkeys(struct ipath_devdata *dd, u16 *pkeys);
-int ipath_layer_set_pkeys(struct ipath_devdata *dd, u16 *pkeys);
-int ipath_layer_get_linkdowndefaultstate(struct ipath_devdata *dd);
-int ipath_layer_set_linkdowndefaultstate(struct ipath_devdata *dd,
-					 int sleep);
-int ipath_layer_get_phyerrthreshold(struct ipath_devdata *dd);
-int ipath_layer_set_phyerrthreshold(struct ipath_devdata *dd, unsigned n);
-int ipath_layer_get_overrunthreshold(struct ipath_devdata *dd);
-int ipath_layer_set_overrunthreshold(struct ipath_devdata *dd, unsigned n);
-u32 ipath_layer_get_rcvhdrentsize(struct ipath_devdata *dd);
 
 /* ipath_ether interrupt values */
 #define IPATH_LAYER_INT_IF_UP 0x2
@@ -178,9 +66,6 @@ u32 ipath_layer_get_rcvhdrentsize(struct ipath_devdata *dd);
 #define IPATH_LAYER_INT_SEND_CONTINUE 0x10
 #define IPATH_LAYER_INT_BCAST 0x40
 
-/* _verbs_layer.l_flags */
-#define IPATH_VERBS_KERNEL_SMA 0x1
-
 extern unsigned ipath_debug; /* debugging bit mask */
 
 #endif				/* _IPATH_LAYER_H */
diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c
index d340234..72d1db8 100644
--- a/drivers/infiniband/hw/ipath/ipath_mad.c
+++ b/drivers/infiniband/hw/ipath/ipath_mad.c
@@ -101,15 +101,15 @@ static int recv_subn_get_nodeinfo(struct ib_smp *smp,
 	nip->num_ports = ibdev->phys_port_cnt;
 	/* This is already in network order */
 	nip->sys_guid = to_idev(ibdev)->sys_image_guid;
-	nip->node_guid = ipath_layer_get_guid(dd);
+	nip->node_guid = dd->ipath_guid;
 	nip->port_guid = nip->sys_guid;
-	nip->partition_cap = cpu_to_be16(ipath_layer_get_npkeys(dd));
-	nip->device_id = cpu_to_be16(ipath_layer_get_deviceid(dd));
-	majrev = ipath_layer_get_majrev(dd);
-	minrev = ipath_layer_get_minrev(dd);
+	nip->partition_cap = cpu_to_be16(ipath_get_npkeys(dd));
+	nip->device_id = cpu_to_be16(dd->ipath_deviceid);
+	majrev = dd->ipath_majrev;
+	minrev = dd->ipath_minrev;
 	nip->revision = cpu_to_be32((majrev << 16) | minrev);
 	nip->local_port_num = port;
-	vendor = ipath_layer_get_vendorid(dd);
+	vendor = dd->ipath_vendorid;
 	nip->vendor_id[0] = 0;
 	nip->vendor_id[1] = vendor >> 8;
 	nip->vendor_id[2] = vendor;
@@ -133,13 +133,89 @@ static int recv_subn_get_guidinfo(struct ib_smp *smp,
 	 */
 	if (startgx == 0)
 		/* The first is a copy of the read-only HW GUID. */
-		*p = ipath_layer_get_guid(to_idev(ibdev)->dd);
+		*p = to_idev(ibdev)->dd->ipath_guid;
 	else
 		smp->status |= IB_SMP_INVALID_FIELD;
 
 	return reply(smp);
 }
 
+
+static int get_overrunthreshold(struct ipath_devdata *dd)
+{
+	return (dd->ipath_ibcctrl >>
+		INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT) &
+		INFINIPATH_IBCC_OVERRUNTHRESHOLD_MASK;
+}
+
+/**
+ * set_overrunthreshold - set the overrun threshold
+ * @dd: the infinipath device
+ * @n: the new threshold
+ *
+ * Note that this will only take effect when the link state changes.
+ */
+static int set_overrunthreshold(struct ipath_devdata *dd, unsigned n)
+{
+	unsigned v;
+
+	v = (dd->ipath_ibcctrl >> INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT) &
+		INFINIPATH_IBCC_OVERRUNTHRESHOLD_MASK;
+	if (v != n) {
+		dd->ipath_ibcctrl &=
+			~(INFINIPATH_IBCC_OVERRUNTHRESHOLD_MASK <<
+			  INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT);
+		dd->ipath_ibcctrl |=
+			(u64) n << INFINIPATH_IBCC_OVERRUNTHRESHOLD_SHIFT;
+		ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
+				 dd->ipath_ibcctrl);
+	}
+	return 0;
+}
+
+static int get_phyerrthreshold(struct ipath_devdata *dd)
+{
+	return (dd->ipath_ibcctrl >>
+		INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT) &
+		INFINIPATH_IBCC_PHYERRTHRESHOLD_MASK;
+}
+
+/**
+ * set_phyerrthreshold - set the physical error threshold
+ * @dd: the infinipath device
+ * @n: the new threshold
+ *
+ * Note that this will only take effect when the link state changes.
+ */
+static int set_phyerrthreshold(struct ipath_devdata *dd, unsigned n)
+{
+	unsigned v;
+
+	v = (dd->ipath_ibcctrl >> INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT) &
+		INFINIPATH_IBCC_PHYERRTHRESHOLD_MASK;
+	if (v != n) {
+		dd->ipath_ibcctrl &=
+			~(INFINIPATH_IBCC_PHYERRTHRESHOLD_MASK <<
+			  INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT);
+		dd->ipath_ibcctrl |=
+			(u64) n << INFINIPATH_IBCC_PHYERRTHRESHOLD_SHIFT;
+		ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
+				 dd->ipath_ibcctrl);
+	}
+	return 0;
+}
+
+/**
+ * get_linkdowndefaultstate - get the default linkdown state
+ * @dd: the infinipath device
+ *
+ * Returns zero if the default is POLL, 1 if the default is SLEEP.
+ */
+static int get_linkdowndefaultstate(struct ipath_devdata *dd)
+{
+	return !!(dd->ipath_ibcctrl & INFINIPATH_IBCC_LINKDOWNDEFAULTSTATE);
+}
+
 static int recv_subn_get_portinfo(struct ib_smp *smp,
 				  struct ib_device *ibdev, u8 port)
 {
@@ -166,7 +242,7 @@ static int recv_subn_get_portinfo(struct ib_smp *smp,
 	    (dev->mkeyprot_resv_lmc >> 6) == 0)
 		pip->mkey = dev->mkey;
 	pip->gid_prefix = dev->gid_prefix;
-	lid = ipath_layer_get_lid(dev->dd);
+	lid = dev->dd->ipath_lid;
 	pip->lid = lid ? cpu_to_be16(lid) : IB_LID_PERMISSIVE;
 	pip->sm_lid = cpu_to_be16(dev->sm_lid);
 	pip->cap_mask = cpu_to_be32(dev->port_cap_flags);
@@ -177,14 +253,14 @@ static int recv_subn_get_portinfo(struct ib_smp *smp,
 	pip->link_width_supported = 3;	/* 1x or 4x */
 	pip->link_width_active = 2;	/* 4x */
 	pip->linkspeed_portstate = 0x10;	/* 2.5Gbps */
-	ibcstat = ipath_layer_get_lastibcstat(dev->dd);
+	ibcstat = dev->dd->ipath_lastibcstat;
 	pip->linkspeed_portstate |= ((ibcstat >> 4) & 0x3) + 1;
 	pip->portphysstate_linkdown =
 		(ipath_cvt_physportstate[ibcstat & 0xf] << 4) |
-		(ipath_layer_get_linkdowndefaultstate(dev->dd) ? 1 : 2);
+		(get_linkdowndefaultstate(dev->dd) ? 1 : 2);
 	pip->mkeyprot_resv_lmc = dev->mkeyprot_resv_lmc;
 	pip->linkspeedactive_enabled = 0x11;	/* 2.5Gbps, 2.5Gbps */
-	switch (ipath_layer_get_ibmtu(dev->dd)) {
+	switch (dev->dd->ipath_ibmtu) {
 	case 4096:
 		mtu = IB_MTU_4096;
 		break;
@@ -217,7 +293,7 @@ static int recv_subn_get_portinfo(struct ib_smp *smp,
 	pip->mkey_violations = cpu_to_be16(dev->mkey_violations);
 	/* P_KeyViolations are counted by hardware. */
 	pip->pkey_violations =
-		cpu_to_be16((ipath_layer_get_cr_errpkey(dev->dd) -
+		cpu_to_be16((ipath_get_cr_errpkey(dev->dd) -
 			     dev->z_pkey_violations) & 0xFFFF);
 	pip->qkey_violations = cpu_to_be16(dev->qkey_violations);
 	/* Only the hardware GUID is supported for now */
@@ -226,8 +302,8 @@ static int recv_subn_get_portinfo(struct ib_smp *smp,
 	/* 32.768 usec. response time (guessing) */
 	pip->resv_resptimevalue = 3;
 	pip->localphyerrors_overrunerrors =
-		(ipath_layer_get_phyerrthreshold(dev->dd) << 4) |
-		ipath_layer_get_overrunthreshold(dev->dd);
+		(get_phyerrthreshold(dev->dd) << 4) |
+		get_overrunthreshold(dev->dd);
 	/* pip->max_credit_hint; */
 	/* pip->link_roundtrip_latency[3]; */
 
@@ -237,6 +313,20 @@ bail:
 	return ret;
 }
 
+/**
+ * get_pkeys - return the PKEY table for port 0
+ * @dd: the infinipath device
+ * @pkeys: the pkey table is placed here
+ */
+static int get_pkeys(struct ipath_devdata *dd, u16 * pkeys)
+{
+	struct ipath_portdata *pd = dd->ipath_pd[0];
+
+	memcpy(pkeys, pd->port_pkeys, sizeof(pd->port_pkeys));
+
+	return 0;
+}
+
 static int recv_subn_get_pkeytable(struct ib_smp *smp,
 				   struct ib_device *ibdev)
 {
@@ -249,9 +339,9 @@ static int recv_subn_get_pkeytable(struct ib_smp *smp,
 	memset(smp->data, 0, sizeof(smp->data));
 	if (startpx == 0) {
 		struct ipath_ibdev *dev = to_idev(ibdev);
-		unsigned i, n = ipath_layer_get_npkeys(dev->dd);
+		unsigned i, n = ipath_get_npkeys(dev->dd);
 
-		ipath_layer_get_pkeys(dev->dd, p);
+		get_pkeys(dev->dd, p);
 
 		for (i = 0; i < n; i++)
 			q[i] = cpu_to_be16(p[i]);
@@ -269,6 +359,24 @@ static int recv_subn_set_guidinfo(struct ib_smp *smp,
 }
 
 /**
+ * set_linkdowndefaultstate - set the default linkdown state
+ * @dd: the infinipath device
+ * @sleep: the new state
+ *
+ * Note that this will only take effect when the link state changes.
+ */
+static int set_linkdowndefaultstate(struct ipath_devdata *dd, int sleep)
+{
+	if (sleep)
+		dd->ipath_ibcctrl |= INFINIPATH_IBCC_LINKDOWNDEFAULTSTATE;
+	else
+		dd->ipath_ibcctrl &= ~INFINIPATH_IBCC_LINKDOWNDEFAULTSTATE;
+	ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl,
+			 dd->ipath_ibcctrl);
+	return 0;
+}
+
+/**
  * recv_subn_set_portinfo - set port information
  * @smp: the incoming SM packet
  * @ibdev: the infiniband device
@@ -290,7 +398,7 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 	u8 state;
 	u16 lstate;
 	u32 mtu;
-	int ret;
+	int ret, ore;
 
 	if (be32_to_cpu(smp->attr_mod) > ibdev->phys_port_cnt)
 		goto err;
@@ -304,7 +412,7 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 	dev->mkey_lease_period = be16_to_cpu(pip->mkey_lease_period);
 
 	lid = be16_to_cpu(pip->lid);
-	if (lid != ipath_layer_get_lid(dev->dd)) {
+	if (lid != dev->dd->ipath_lid) {
 		/* Must be a valid unicast LID address. */
 		if (lid == 0 || lid >= IPATH_MULTICAST_LID_BASE)
 			goto err;
@@ -342,11 +450,11 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 	case 0: /* NOP */
 		break;
 	case 1: /* SLEEP */
-		if (ipath_layer_set_linkdowndefaultstate(dev->dd, 1))
+		if (set_linkdowndefaultstate(dev->dd, 1))
 			goto err;
 		break;
 	case 2: /* POLL */
-		if (ipath_layer_set_linkdowndefaultstate(dev->dd, 0))
+		if (set_linkdowndefaultstate(dev->dd, 0))
 			goto err;
 		break;
 	default:
@@ -376,7 +484,7 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 		/* XXX We have already partially updated our state! */
 		goto err;
 	}
-	ipath_layer_set_mtu(dev->dd, mtu);
+	ipath_set_mtu(dev->dd, mtu);
 
 	dev->sm_sl = pip->neighbormtu_mastersmsl & 0xF;
 
@@ -392,20 +500,16 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 	 * later.
 	 */
 	if (pip->pkey_violations == 0)
-		dev->z_pkey_violations =
-			ipath_layer_get_cr_errpkey(dev->dd);
+		dev->z_pkey_violations = ipath_get_cr_errpkey(dev->dd);
 
 	if (pip->qkey_violations == 0)
 		dev->qkey_violations = 0;
 
-	if (ipath_layer_set_phyerrthreshold(
-		    dev->dd,
-		    (pip->localphyerrors_overrunerrors >> 4) & 0xF))
+	ore = pip->localphyerrors_overrunerrors;
+	if (set_phyerrthreshold(dev->dd, (ore >> 4) & 0xF))
 		goto err;
 
-	if (ipath_layer_set_overrunthreshold(
-		    dev->dd,
-		    (pip->localphyerrors_overrunerrors & 0xF)))
+	if (set_overrunthreshold(dev->dd, (ore & 0xF)))
 		goto err;
 
 	dev->subnet_timeout = pip->clientrereg_resv_subnetto & 0x1F;
@@ -423,7 +527,7 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 	 * is down or is being set to down.
 	 */
 	state = pip->linkspeed_portstate & 0xF;
-	flags = ipath_layer_get_flags(dev->dd);
+	flags = dev->dd->ipath_flags;
 	lstate = (pip->portphysstate_linkdown >> 4) & 0xF;
 	if (lstate && !(state == IB_PORT_DOWN || state == IB_PORT_NOP))
 		goto err;
@@ -439,7 +543,7 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 		/* FALLTHROUGH */
 	case IB_PORT_DOWN:
 		if (lstate == 0)
-			if (ipath_layer_get_linkdowndefaultstate(dev->dd))
+			if (get_linkdowndefaultstate(dev->dd))
 				lstate = IPATH_IB_LINKDOWN_SLEEP;
 			else
 				lstate = IPATH_IB_LINKDOWN;
@@ -451,7 +555,7 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 			lstate = IPATH_IB_LINKDOWN_DISABLE;
 		else
 			goto err;
-		ipath_layer_set_linkstate(dev->dd, lstate);
+		ipath_set_linkstate(dev->dd, lstate);
 		if (flags & IPATH_LINKACTIVE) {
 			event.event = IB_EVENT_PORT_ERR;
 			ib_dispatch_event(&event);
@@ -460,7 +564,7 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 	case IB_PORT_ARMED:
 		if (!(flags & (IPATH_LINKINIT | IPATH_LINKACTIVE)))
 			break;
-		ipath_layer_set_linkstate(dev->dd, IPATH_IB_LINKARM);
+		ipath_set_linkstate(dev->dd, IPATH_IB_LINKARM);
 		if (flags & IPATH_LINKACTIVE) {
 			event.event = IB_EVENT_PORT_ERR;
 			ib_dispatch_event(&event);
@@ -469,7 +573,7 @@ static int recv_subn_set_portinfo(struct ib_smp *smp,
 	case IB_PORT_ACTIVE:
 		if (!(flags & IPATH_LINKARMED))
 			break;
-		ipath_layer_set_linkstate(dev->dd, IPATH_IB_LINKACTIVE);
+		ipath_set_linkstate(dev->dd, IPATH_IB_LINKACTIVE);
 		event.event = IB_EVENT_PORT_ACTIVE;
 		ib_dispatch_event(&event);
 		break;
@@ -493,6 +597,152 @@ done:
 	return ret;
 }
 
+/**
+ * rm_pkey - decrecment the reference count for the given PKEY
+ * @dd: the infinipath device
+ * @key: the PKEY index
+ *
+ * Return true if this was the last reference and the hardware table entry
+ * needs to be changed.
+ */
+static int rm_pkey(struct ipath_devdata *dd, u16 key)
+{
+	int i;
+	int ret;
+
+	for (i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
+		if (dd->ipath_pkeys[i] != key)
+			continue;
+		if (atomic_dec_and_test(&dd->ipath_pkeyrefs[i])) {
+			dd->ipath_pkeys[i] = 0;
+			ret = 1;
+			goto bail;
+		}
+		break;
+	}
+
+	ret = 0;
+
+bail:
+	return ret;
+}
+
+/**
+ * add_pkey - add the given PKEY to the hardware table
+ * @dd: the infinipath device
+ * @key: the PKEY
+ *
+ * Return an error code if unable to add the entry, zero if no change,
+ * or 1 if the hardware PKEY register needs to be updated.
+ */
+static int add_pkey(struct ipath_devdata *dd, u16 key)
+{
+	int i;
+	u16 lkey = key & 0x7FFF;
+	int any = 0;
+	int ret;
+
+	if (lkey == 0x7FFF) {
+		ret = 0;
+		goto bail;
+	}
+
+	/* Look for an empty slot or a matching PKEY. */
+	for (i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
+		if (!dd->ipath_pkeys[i]) {
+			any++;
+			continue;
+		}
+		/* If it matches exactly, try to increment the ref count */
+		if (dd->ipath_pkeys[i] == key) {
+			if (atomic_inc_return(&dd->ipath_pkeyrefs[i]) > 1) {
+				ret = 0;
+				goto bail;
+			}
+			/* Lost the race. Look for an empty slot below. */
+			atomic_dec(&dd->ipath_pkeyrefs[i]);
+			any++;
+		}
+		/*
+		 * It makes no sense to have both the limited and unlimited
+		 * PKEY set at the same time since the unlimited one will
+		 * disable the limited one.
+		 */
+		if ((dd->ipath_pkeys[i] & 0x7FFF) == lkey) {
+			ret = -EEXIST;
+			goto bail;
+		}
+	}
+	if (!any) {
+		ret = -EBUSY;
+		goto bail;
+	}
+	for (i = 0; i < ARRAY_SIZE(dd->ipath_pkeys); i++) {
+		if (!dd->ipath_pkeys[i] &&
+		    atomic_inc_return(&dd->ipath_pkeyrefs[i]) == 1) {
+			/* for ipathstats, etc. */
+			ipath_stats.sps_pkeys[i] = lkey;
+			dd->ipath_pkeys[i] = key;
+			ret = 1;
+			goto bail;
+		}
+	}
+	ret = -EBUSY;
+
+bail:
+	return ret;
+}
+
+/**
+ * set_pkeys - set the PKEY table for port 0
+ * @dd: the infinipath device
+ * @pkeys: the PKEY table
+ */
+static int set_pkeys(struct ipath_devdata *dd, u16 *pkeys)
+{
+	struct ipath_portdata *pd;
+	int i;
+	int changed = 0;
+
+	pd = dd->ipath_pd[0];
+
+	for (i = 0; i < ARRAY_SIZE(pd->port_pkeys); i++) {
+		u16 key = pkeys[i];
+		u16 okey = pd->port_pkeys[i];
+
+		if (key == okey)
+			continue;
+		/*
+		 * The value of this PKEY table entry is changing.
+		 * Remove the old entry in the hardware's array of PKEYs.
+		 */
+		if (okey & 0x7FFF)
+			changed |= rm_pkey(dd, okey);
+		if (key & 0x7FFF) {
+			int ret = add_pkey(dd, key);
+
+			if (ret < 0)
+				key = 0;
+			else
+				changed |= ret;
+		}
+		pd->port_pkeys[i] = key;
+	}
+	if (changed) {
+		u64 pkey;
+
+		pkey = (u64) dd->ipath_pkeys[0] |
+			((u64) dd->ipath_pkeys[1] << 16) |
+			((u64) dd->ipath_pkeys[2] << 32) |
+			((u64) dd->ipath_pkeys[3] << 48);
+		ipath_cdbg(VERBOSE, "p0 new pkey reg %llx\n",
+			   (unsigned long long) pkey);
+		ipath_write_kreg(dd, dd->ipath_kregs->kr_partitionkey,
+				 pkey);
+	}
+	return 0;
+}
+
 static int recv_subn_set_pkeytable(struct ib_smp *smp,
 				   struct ib_device *ibdev)
 {
@@ -500,13 +750,12 @@ static int recv_subn_set_pkeytable(struct ib_smp *smp,
 	__be16 *p = (__be16 *) smp->data;
 	u16 *q = (u16 *) smp->data;
 	struct ipath_ibdev *dev = to_idev(ibdev);
-	unsigned i, n = ipath_layer_get_npkeys(dev->dd);
+	unsigned i, n = ipath_get_npkeys(dev->dd);
 
 	for (i = 0; i < n; i++)
 		q[i] = be16_to_cpu(p[i]);
 
-	if (startpx != 0 ||
-	    ipath_layer_set_pkeys(dev->dd, q) != 0)
+	if (startpx != 0 || set_pkeys(dev->dd, q) != 0)
 		smp->status |= IB_SMP_INVALID_FIELD;
 
 	return recv_subn_get_pkeytable(smp, ibdev);
@@ -844,10 +1093,10 @@ static int recv_pma_get_portcounters(struct ib_perf *pmp,
 	struct ib_pma_portcounters *p = (struct ib_pma_portcounters *)
 		pmp->data;
 	struct ipath_ibdev *dev = to_idev(ibdev);
-	struct ipath_layer_counters cntrs;
+	struct ipath_verbs_counters cntrs;
 	u8 port_select = p->port_select;
 
-	ipath_layer_get_counters(dev->dd, &cntrs);
+	ipath_get_counters(dev->dd, &cntrs);
 
 	/* Adjust counters for any resets done. */
 	cntrs.symbol_error_counter -= dev->z_symbol_error_counter;
@@ -944,8 +1193,8 @@ static int recv_pma_get_portcounters_ext(struct ib_perf *pmp,
 	u64 swords, rwords, spkts, rpkts, xwait;
 	u8 port_select = p->port_select;
 
-	ipath_layer_snapshot_counters(dev->dd, &swords, &rwords, &spkts,
-				      &rpkts, &xwait);
+	ipath_snapshot_counters(dev->dd, &swords, &rwords, &spkts,
+				&rpkts, &xwait);
 
 	/* Adjust counters for any resets done. */
 	swords -= dev->z_port_xmit_data;
@@ -978,13 +1227,13 @@ static int recv_pma_set_portcounters(struct ib_perf *pmp,
 	struct ib_pma_portcounters *p = (struct ib_pma_portcounters *)
 		pmp->data;
 	struct ipath_ibdev *dev = to_idev(ibdev);
-	struct ipath_layer_counters cntrs;
+	struct ipath_verbs_counters cntrs;
 
 	/*
 	 * Since the HW doesn't support clearing counters, we save the
 	 * current count and subtract it from future responses.
 	 */
-	ipath_layer_get_counters(dev->dd, &cntrs);
+	ipath_get_counters(dev->dd, &cntrs);
 
 	if (p->counter_select & IB_PMA_SEL_SYMBOL_ERROR)
 		dev->z_symbol_error_counter = cntrs.symbol_error_counter;
@@ -1041,8 +1290,8 @@ static int recv_pma_set_portcounters_ext(struct ib_perf *pmp,
 	struct ipath_ibdev *dev = to_idev(ibdev);
 	u64 swords, rwords, spkts, rpkts, xwait;
 
-	ipath_layer_snapshot_counters(dev->dd, &swords, &rwords, &spkts,
-				      &rpkts, &xwait);
+	ipath_snapshot_counters(dev->dd, &swords, &rwords, &spkts,
+				&rpkts, &xwait);
 
 	if (p->counter_select & IB_PMA_SELX_PORT_XMIT_DATA)
 		dev->z_port_xmit_data = swords;
diff --git a/drivers/infiniband/hw/ipath/ipath_mmap.c b/drivers/infiniband/hw/ipath/ipath_mmap.c
new file mode 100644
index 0000000..11b7378
--- /dev/null
+++ b/drivers/infiniband/hw/ipath/ipath_mmap.c
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2006 QLogic, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include <linux/config.h>
+#include <linux/module.h>
+#include <linux/vmalloc.h>
+#include <linux/mm.h>
+#include <linux/errno.h>
+#include <asm/pgtable.h>
+
+#include "ipath_verbs.h"
+
+/**
+ * ipath_release_mmap_info - free mmap info structure
+ * @ref: a pointer to the kref within struct ipath_mmap_info
+ */
+void ipath_release_mmap_info(struct kref *ref)
+{
+	struct ipath_mmap_info *ip =
+		container_of(ref, struct ipath_mmap_info, ref);
+
+	vfree(ip->obj);
+	kfree(ip);
+}
+
+/*
+ * open and close keep track of how many times the CQ is mapped,
+ * to avoid releasing it.
+ */
+static void ipath_vma_open(struct vm_area_struct *vma)
+{
+	struct ipath_mmap_info *ip = vma->vm_private_data;
+
+	kref_get(&ip->ref);
+	ip->mmap_cnt++;
+}
+
+static void ipath_vma_close(struct vm_area_struct *vma)
+{
+	struct ipath_mmap_info *ip = vma->vm_private_data;
+
+	ip->mmap_cnt--;
+	kref_put(&ip->ref, ipath_release_mmap_info);
+}
+
+static struct vm_operations_struct ipath_vm_ops = {
+	.open =     ipath_vma_open,
+	.close =    ipath_vma_close,
+};
+
+/**
+ * ipath_mmap - create a new mmap region
+ * @context: the IB user context of the process making the mmap() call
+ * @vma: the VMA to be initialized
+ * Return zero if the mmap is OK. Otherwise, return an errno.
+ */
+int ipath_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
+{
+	struct ipath_ibdev *dev = to_idev(context->device);
+	unsigned long offset = vma->vm_pgoff << PAGE_SHIFT;
+	unsigned long size = vma->vm_end - vma->vm_start;
+	struct ipath_mmap_info *ip, **pp;
+	int ret = -EINVAL;
+
+	/*
+	 * Search the device's list of objects waiting for a mmap call.
+	 * Normally, this list is very short since a call to create a
+	 * CQ, QP, or SRQ is soon followed by a call to mmap().
+	 */
+	spin_lock_irq(&dev->pending_lock);
+	for (pp = &dev->pending_mmaps; (ip = *pp); pp = &ip->next) {
+		/* Only the creator is allowed to mmap the object */
+		if (context != ip->context || (void *) offset != ip->obj)
+			continue;
+		/* Don't allow a mmap larger than the object. */
+		if (size > ip->size)
+			break;
+
+		*pp = ip->next;
+		spin_unlock_irq(&dev->pending_lock);
+
+		ret = remap_vmalloc_range(vma, ip->obj, 0);
+		if (ret)
+			goto done;
+		vma->vm_ops = &ipath_vm_ops;
+		vma->vm_private_data = ip;
+		ipath_vma_open(vma);
+		goto done;
+	}
+	spin_unlock_irq(&dev->pending_lock);
+done:
+	return ret;
+}
diff --git a/drivers/infiniband/hw/ipath/ipath_mr.c b/drivers/infiniband/hw/ipath/ipath_mr.c
index 4ac31a5..b36f6fb 100644
--- a/drivers/infiniband/hw/ipath/ipath_mr.c
+++ b/drivers/infiniband/hw/ipath/ipath_mr.c
@@ -36,6 +36,18 @@
 
 #include "ipath_verbs.h"
 
+/* Fast memory region */
+struct ipath_fmr {
+	struct ib_fmr ibfmr;
+	u8 page_shift;
+	struct ipath_mregion mr;        /* must be last */
+};
+
+static inline struct ipath_fmr *to_ifmr(struct ib_fmr *ibfmr)
+{
+	return container_of(ibfmr, struct ipath_fmr, ibfmr);
+}
+
 /**
  * ipath_get_dma_mr - get a DMA memory region
  * @pd: protection domain for this memory region
diff --git a/drivers/infiniband/hw/ipath/ipath_qp.c b/drivers/infiniband/hw/ipath/ipath_qp.c
index 83e557b..224b0f4 100644
--- a/drivers/infiniband/hw/ipath/ipath_qp.c
+++ b/drivers/infiniband/hw/ipath/ipath_qp.c
@@ -35,7 +35,7 @@
 #include <linux/vmalloc.h>
 
 #include "ipath_verbs.h"
-#include "ipath_common.h"
+#include "ipath_kernel.h"
 
 #define BITS_PER_PAGE		(PAGE_SIZE*BITS_PER_BYTE)
 #define BITS_PER_PAGE_MASK	(BITS_PER_PAGE-1)
@@ -44,19 +44,6 @@
 #define find_next_offset(map, off) find_next_zero_bit((map)->page, \
 						      BITS_PER_PAGE, off)
 
-#define TRANS_INVALID	0
-#define TRANS_ANY2RST	1
-#define TRANS_RST2INIT	2
-#define TRANS_INIT2INIT	3
-#define TRANS_INIT2RTR	4
-#define TRANS_RTR2RTS	5
-#define TRANS_RTS2RTS	6
-#define TRANS_SQERR2RTS	7
-#define TRANS_ANY2ERR	8
-#define TRANS_RTS2SQD	9  /* XXX Wait for expected ACKs & signal event */
-#define TRANS_SQD2SQD	10 /* error if not drained & parameter change */
-#define TRANS_SQD2RTS	11 /* error if not drained */
-
 /*
  * Convert the AETH credit code into the number of credits.
  */
@@ -287,7 +274,7 @@ void ipath_free_all_qps(struct ipath_qp_table *qpt)
 				free_qpn(qpt, qp->ibqp.qp_num);
 			if (!atomic_dec_and_test(&qp->refcount) ||
 			    !ipath_destroy_qp(&qp->ibqp))
-				_VERBS_INFO("QP memory leak!\n");
+				ipath_dbg(KERN_INFO "QP memory leak!\n");
 			qp = nqp;
 		}
 	}
@@ -355,8 +342,10 @@ static void ipath_reset_qp(struct ipath_qp *qp)
 	qp->s_last = 0;
 	qp->s_ssn = 1;
 	qp->s_lsn = 0;
-	qp->r_rq.head = 0;
-	qp->r_rq.tail = 0;
+	if (qp->r_rq.wq) {
+		qp->r_rq.wq->head = 0;
+		qp->r_rq.wq->tail = 0;
+	}
 	qp->r_reuse_sge = 0;
 }
 
@@ -373,8 +362,8 @@ void ipath_error_qp(struct ipath_qp *qp)
 	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
 	struct ib_wc wc;
 
-	_VERBS_INFO("QP%d/%d in error state\n",
-		    qp->ibqp.qp_num, qp->remote_qpn);
+	ipath_dbg(KERN_INFO "QP%d/%d in error state\n",
+		  qp->ibqp.qp_num, qp->remote_qpn);
 
 	spin_lock(&dev->pending_lock);
 	/* XXX What if its already removed by the timeout code? */
@@ -410,15 +399,32 @@ void ipath_error_qp(struct ipath_qp *qp)
 	qp->s_hdrwords = 0;
 	qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
 
-	wc.opcode = IB_WC_RECV;
-	spin_lock(&qp->r_rq.lock);
-	while (qp->r_rq.tail != qp->r_rq.head) {
-		wc.wr_id = get_rwqe_ptr(&qp->r_rq, qp->r_rq.tail)->wr_id;
-		if (++qp->r_rq.tail >= qp->r_rq.size)
-			qp->r_rq.tail = 0;
-		ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
+	if (qp->r_rq.wq) {
+		struct ipath_rwq *wq;
+		u32 head;
+		u32 tail;
+
+		spin_lock(&qp->r_rq.lock);
+
+		/* sanity check pointers before trusting them */
+		wq = qp->r_rq.wq;
+		head = wq->head;
+		if (head >= qp->r_rq.size)
+			head = 0;
+		tail = wq->tail;
+		if (tail >= qp->r_rq.size)
+			tail = 0;
+		wc.opcode = IB_WC_RECV;
+		while (tail != head) {
+			wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;
+			if (++tail >= qp->r_rq.size)
+				tail = 0;
+			ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
+		}
+		wq->tail = tail;
+
+		spin_unlock(&qp->r_rq.lock);
 	}
-	spin_unlock(&qp->r_rq.lock);
 }
 
 /**
@@ -426,11 +432,12 @@ void ipath_error_qp(struct ipath_qp *qp)
  * @ibqp: the queue pair who's attributes we're modifying
  * @attr: the new attributes
  * @attr_mask: the mask of attributes to modify
+ * @udata: user data for ipathverbs.so
  *
  * Returns 0 on success, otherwise returns an errno.
  */
 int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
-		    int attr_mask)
+		    int attr_mask, struct ib_udata *udata)
 {
 	struct ipath_ibdev *dev = to_idev(ibqp->device);
 	struct ipath_qp *qp = to_iqp(ibqp);
@@ -448,19 +455,46 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 				attr_mask))
 		goto inval;
 
-	if (attr_mask & IB_QP_AV)
+	if (attr_mask & IB_QP_AV) {
 		if (attr->ah_attr.dlid == 0 ||
 		    attr->ah_attr.dlid >= IPATH_MULTICAST_LID_BASE)
 			goto inval;
 
+		if ((attr->ah_attr.ah_flags & IB_AH_GRH) &&
+		    (attr->ah_attr.grh.sgid_index > 1))
+			goto inval;
+	}
+
 	if (attr_mask & IB_QP_PKEY_INDEX)
-		if (attr->pkey_index >= ipath_layer_get_npkeys(dev->dd))
+		if (attr->pkey_index >= ipath_get_npkeys(dev->dd))
 			goto inval;
 
 	if (attr_mask & IB_QP_MIN_RNR_TIMER)
 		if (attr->min_rnr_timer > 31)
 			goto inval;
 
+	if (attr_mask & IB_QP_PORT)
+		if (attr->port_num == 0 ||
+		    attr->port_num > ibqp->device->phys_port_cnt)
+			goto inval;
+
+	if (attr_mask & IB_QP_PATH_MTU)
+		if (attr->path_mtu > IB_MTU_4096)
+			goto inval;
+
+	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
+		if (attr->max_dest_rd_atomic > 1)
+			goto inval;
+
+	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
+		if (attr->max_rd_atomic > 1)
+			goto inval;
+
+	if (attr_mask & IB_QP_PATH_MIG_STATE)
+		if (attr->path_mig_state != IB_MIG_MIGRATED &&
+		    attr->path_mig_state != IB_MIG_REARM)
+			goto inval;
+
 	switch (new_state) {
 	case IB_QPS_RESET:
 		ipath_reset_qp(qp);
@@ -511,6 +545,9 @@ int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	if (attr_mask & IB_QP_MIN_RNR_TIMER)
 		qp->r_min_rnr_timer = attr->min_rnr_timer;
 
+	if (attr_mask & IB_QP_TIMEOUT)
+		qp->timeout = attr->timeout;
+
 	if (attr_mask & IB_QP_QKEY)
 		qp->qkey = attr->qkey;
 
@@ -543,7 +580,7 @@ int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	attr->dest_qp_num = qp->remote_qpn;
 	attr->qp_access_flags = qp->qp_access_flags;
 	attr->cap.max_send_wr = qp->s_size - 1;
-	attr->cap.max_recv_wr = qp->r_rq.size - 1;
+	attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1;
 	attr->cap.max_send_sge = qp->s_max_sge;
 	attr->cap.max_recv_sge = qp->r_rq.max_sge;
 	attr->cap.max_inline_data = 0;
@@ -557,7 +594,7 @@ int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	attr->max_dest_rd_atomic = 1;
 	attr->min_rnr_timer = qp->r_min_rnr_timer;
 	attr->port_num = 1;
-	attr->timeout = 0;
+	attr->timeout = qp->timeout;
 	attr->retry_cnt = qp->s_retry_cnt;
 	attr->rnr_retry = qp->s_rnr_retry;
 	attr->alt_port_num = 0;
@@ -569,9 +606,10 @@ int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 	init_attr->recv_cq = qp->ibqp.recv_cq;
 	init_attr->srq = qp->ibqp.srq;
 	init_attr->cap = attr->cap;
-	init_attr->sq_sig_type =
-		(qp->s_flags & (1 << IPATH_S_SIGNAL_REQ_WR))
-		? IB_SIGNAL_REQ_WR : 0;
+	if (qp->s_flags & (1 << IPATH_S_SIGNAL_REQ_WR))
+		init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
+	else
+		init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
 	init_attr->qp_type = qp->ibqp.qp_type;
 	init_attr->port_num = 1;
 	return 0;
@@ -596,13 +634,23 @@ __be32 ipath_compute_aeth(struct ipath_qp *qp)
 	} else {
 		u32 min, max, x;
 		u32 credits;
-
+		struct ipath_rwq *wq = qp->r_rq.wq;
+		u32 head;
+		u32 tail;
+
+		/* sanity check pointers before trusting them */
+		head = wq->head;
+		if (head >= qp->r_rq.size)
+			head = 0;
+		tail = wq->tail;
+		if (tail >= qp->r_rq.size)
+			tail = 0;
 		/*
 		 * Compute the number of credits available (RWQEs).
 		 * XXX Not holding the r_rq.lock here so there is a small
 		 * chance that the pair of reads are not atomic.
 		 */
-		credits = qp->r_rq.head - qp->r_rq.tail;
+		credits = head - tail;
 		if ((int)credits < 0)
 			credits += qp->r_rq.size;
 		/*
@@ -679,27 +727,37 @@ struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
 	case IB_QPT_UD:
 	case IB_QPT_SMI:
 	case IB_QPT_GSI:
-		qp = kmalloc(sizeof(*qp), GFP_KERNEL);
+		sz = sizeof(*qp);
+		if (init_attr->srq) {
+			struct ipath_srq *srq = to_isrq(init_attr->srq);
+
+			sz += sizeof(*qp->r_sg_list) *
+				srq->rq.max_sge;
+		} else
+			sz += sizeof(*qp->r_sg_list) *
+				init_attr->cap.max_recv_sge;
+		qp = kmalloc(sz, GFP_KERNEL);
 		if (!qp) {
-			vfree(swq);
 			ret = ERR_PTR(-ENOMEM);
-			goto bail;
+			goto bail_swq;
 		}
 		if (init_attr->srq) {
+			sz = 0;
 			qp->r_rq.size = 0;
 			qp->r_rq.max_sge = 0;
 			qp->r_rq.wq = NULL;
+			init_attr->cap.max_recv_wr = 0;
+			init_attr->cap.max_recv_sge = 0;
 		} else {
 			qp->r_rq.size = init_attr->cap.max_recv_wr + 1;
 			qp->r_rq.max_sge = init_attr->cap.max_recv_sge;
-			sz = (sizeof(struct ipath_sge) * qp->r_rq.max_sge) +
+			sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) +
 				sizeof(struct ipath_rwqe);
-			qp->r_rq.wq = vmalloc(qp->r_rq.size * sz);
+			qp->r_rq.wq = vmalloc_user(sizeof(struct ipath_rwq) +
+					      qp->r_rq.size * sz);
 			if (!qp->r_rq.wq) {
-				kfree(qp);
-				vfree(swq);
 				ret = ERR_PTR(-ENOMEM);
-				goto bail;
+				goto bail_qp;
 			}
 		}
 
@@ -719,24 +777,19 @@ struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
 		qp->s_wq = swq;
 		qp->s_size = init_attr->cap.max_send_wr + 1;
 		qp->s_max_sge = init_attr->cap.max_send_sge;
-		qp->s_flags = init_attr->sq_sig_type == IB_SIGNAL_REQ_WR ?
-			1 << IPATH_S_SIGNAL_REQ_WR : 0;
+		if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR)
+			qp->s_flags = 1 << IPATH_S_SIGNAL_REQ_WR;
+		else
+			qp->s_flags = 0;
 		dev = to_idev(ibpd->device);
 		err = ipath_alloc_qpn(&dev->qp_table, qp,
 				      init_attr->qp_type);
 		if (err) {
-			vfree(swq);
-			vfree(qp->r_rq.wq);
-			kfree(qp);
 			ret = ERR_PTR(err);
-			goto bail;
+			goto bail_rwq;
 		}
+		qp->ip = NULL;
 		ipath_reset_qp(qp);
-
-		/* Tell the core driver that the kernel SMA is present. */
-		if (init_attr->qp_type == IB_QPT_SMI)
-			ipath_layer_set_verbs_flags(dev->dd,
-						    IPATH_VERBS_KERNEL_SMA);
 		break;
 
 	default:
@@ -747,8 +800,63 @@ struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
 
 	init_attr->cap.max_inline_data = 0;
 
+	/*
+	 * Return the address of the RWQ as the offset to mmap.
+	 * See ipath_mmap() for details.
+	 */
+	if (udata && udata->outlen >= sizeof(__u64)) {
+		struct ipath_mmap_info *ip;
+		__u64 offset = (__u64) qp->r_rq.wq;
+		int err;
+
+		err = ib_copy_to_udata(udata, &offset, sizeof(offset));
+		if (err) {
+			ret = ERR_PTR(err);
+			goto bail_rwq;
+		}
+
+		if (qp->r_rq.wq) {
+			/* Allocate info for ipath_mmap(). */
+			ip = kmalloc(sizeof(*ip), GFP_KERNEL);
+			if (!ip) {
+				ret = ERR_PTR(-ENOMEM);
+				goto bail_rwq;
+			}
+			qp->ip = ip;
+			ip->context = ibpd->uobject->context;
+			ip->obj = qp->r_rq.wq;
+			kref_init(&ip->ref);
+			ip->mmap_cnt = 0;
+			ip->size = PAGE_ALIGN(sizeof(struct ipath_rwq) +
+					      qp->r_rq.size * sz);
+			spin_lock_irq(&dev->pending_lock);
+			ip->next = dev->pending_mmaps;
+			dev->pending_mmaps = ip;
+			spin_unlock_irq(&dev->pending_lock);
+		}
+	}
+
+	spin_lock(&dev->n_qps_lock);
+	if (dev->n_qps_allocated == ib_ipath_max_qps) {
+		spin_unlock(&dev->n_qps_lock);
+		ret = ERR_PTR(-ENOMEM);
+		goto bail_ip;
+	}
+
+	dev->n_qps_allocated++;
+	spin_unlock(&dev->n_qps_lock);
+
 	ret = &qp->ibqp;
+	goto bail;
 
+bail_ip:
+	kfree(qp->ip);
+bail_rwq:
+	vfree(qp->r_rq.wq);
+bail_qp:
+	kfree(qp);
+bail_swq:
+	vfree(swq);
 bail:
 	return ret;
 }
@@ -768,15 +876,12 @@ int ipath_destroy_qp(struct ib_qp *ibqp)
 	struct ipath_ibdev *dev = to_idev(ibqp->device);
 	unsigned long flags;
 
-	/* Tell the core driver that the kernel SMA is gone. */
-	if (qp->ibqp.qp_type == IB_QPT_SMI)
-		ipath_layer_set_verbs_flags(dev->dd, 0);
-
-	spin_lock_irqsave(&qp->r_rq.lock, flags);
-	spin_lock(&qp->s_lock);
+	spin_lock_irqsave(&qp->s_lock, flags);
 	qp->state = IB_QPS_ERR;
-	spin_unlock(&qp->s_lock);
-	spin_unlock_irqrestore(&qp->r_rq.lock, flags);
+	spin_unlock_irqrestore(&qp->s_lock, flags);
+	spin_lock(&dev->n_qps_lock);
+	dev->n_qps_allocated--;
+	spin_unlock(&dev->n_qps_lock);
 
 	/* Stop the sending tasklet. */
 	tasklet_kill(&qp->s_task);
@@ -797,8 +902,11 @@ int ipath_destroy_qp(struct ib_qp *ibqp)
 	if (atomic_read(&qp->refcount) != 0)
 		ipath_free_qp(&dev->qp_table, qp);
 
+	if (qp->ip)
+		kref_put(&qp->ip->ref, ipath_release_mmap_info);
+	else
+		vfree(qp->r_rq.wq);
 	vfree(qp->s_wq);
-	vfree(qp->r_rq.wq);
 	kfree(qp);
 	return 0;
 }
@@ -850,8 +958,8 @@ void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc)
 	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
 	struct ipath_swqe *wqe = get_swqe_ptr(qp, qp->s_last);
 
-	_VERBS_INFO("Send queue error on QP%d/%d: err: %d\n",
-		    qp->ibqp.qp_num, qp->remote_qpn, wc->status);
+	ipath_dbg(KERN_INFO "Send queue error on QP%d/%d: err: %d\n",
+		  qp->ibqp.qp_num, qp->remote_qpn, wc->status);
 
 	spin_lock(&dev->pending_lock);
 	/* XXX What if its already removed by the timeout code? */
diff --git a/drivers/infiniband/hw/ipath/ipath_rc.c b/drivers/infiniband/hw/ipath/ipath_rc.c
index 774d161..a086540 100644
--- a/drivers/infiniband/hw/ipath/ipath_rc.c
+++ b/drivers/infiniband/hw/ipath/ipath_rc.c
@@ -32,7 +32,7 @@
  */
 
 #include "ipath_verbs.h"
-#include "ipath_common.h"
+#include "ipath_kernel.h"
 
 /* cut down ridiculously long IB macro names */
 #define OP(x) IB_OPCODE_RC_##x
@@ -540,7 +540,7 @@ static void send_rc_ack(struct ipath_qp *qp)
 		lrh0 = IPATH_LRH_GRH;
 	}
 	/* read pkey_index w/o lock (its atomic) */
-	bth0 = ipath_layer_get_pkey(dev->dd, qp->s_pkey_index);
+	bth0 = ipath_get_pkey(dev->dd, qp->s_pkey_index);
 	if (qp->r_nak_state)
 		ohdr->u.aeth = cpu_to_be32((qp->r_msn & IPATH_MSN_MASK) |
 					    (qp->r_nak_state <<
@@ -557,7 +557,7 @@ static void send_rc_ack(struct ipath_qp *qp)
 	hdr.lrh[0] = cpu_to_be16(lrh0);
 	hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
 	hdr.lrh[2] = cpu_to_be16(hwords + SIZE_OF_CRC);
-	hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->dd));
+	hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid);
 	ohdr->bth[0] = cpu_to_be32(bth0);
 	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
 	ohdr->bth[2] = cpu_to_be32(qp->r_ack_psn & IPATH_PSN_MASK);
@@ -1323,8 +1323,7 @@ void ipath_rc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
 		 * the eager header buffer size to 56 bytes so the last 4
 		 * bytes of the BTH header (PSN) is in the data buffer.
 		 */
-		header_in_data =
-			ipath_layer_get_rcvhdrentsize(dev->dd) == 16;
+		header_in_data = dev->dd->ipath_rcvhdrentsize == 16;
 		if (header_in_data) {
 			psn = be32_to_cpu(((__be32 *) data)[0]);
 			data += sizeof(__be32);
diff --git a/drivers/infiniband/hw/ipath/ipath_registers.h b/drivers/infiniband/hw/ipath/ipath_registers.h
index 89df8f5..6e23b3d 100644
--- a/drivers/infiniband/hw/ipath/ipath_registers.h
+++ b/drivers/infiniband/hw/ipath/ipath_registers.h
@@ -36,8 +36,7 @@
 
 /*
  * This file should only be included by kernel source, and by the diags.  It
- * defines the registers, and their contents, for the InfiniPath HT-400
- * chip.
+ * defines the registers, and their contents, for InfiniPath chips.
  */
 
 /*
@@ -283,10 +282,12 @@
 #define INFINIPATH_XGXS_RESET          0x7ULL
 #define INFINIPATH_XGXS_MDIOADDR_MASK  0xfULL
 #define INFINIPATH_XGXS_MDIOADDR_SHIFT 4
+#define INFINIPATH_XGXS_RX_POL_SHIFT 19
+#define INFINIPATH_XGXS_RX_POL_MASK 0xfULL
 
 #define INFINIPATH_RT_ADDR_MASK 0xFFFFFFFFFFULL	/* 40 bits valid */
 
-/* TID entries (memory), HT400-only */
+/* TID entries (memory), HT-only */
 #define INFINIPATH_RT_VALID 0x8000000000000000ULL
 #define INFINIPATH_RT_ADDR_SHIFT 0
 #define INFINIPATH_RT_BUFSIZE_MASK 0x3FFF
diff --git a/drivers/infiniband/hw/ipath/ipath_ruc.c b/drivers/infiniband/hw/ipath/ipath_ruc.c
index 772bc59f..5c1da2d 100644
--- a/drivers/infiniband/hw/ipath/ipath_ruc.c
+++ b/drivers/infiniband/hw/ipath/ipath_ruc.c
@@ -32,7 +32,7 @@
  */
 
 #include "ipath_verbs.h"
-#include "ipath_common.h"
+#include "ipath_kernel.h"
 
 /*
  * Convert the AETH RNR timeout code into the number of milliseconds.
@@ -106,6 +106,54 @@ void ipath_insert_rnr_queue(struct ipath_qp *qp)
 	spin_unlock_irqrestore(&dev->pending_lock, flags);
 }
 
+static int init_sge(struct ipath_qp *qp, struct ipath_rwqe *wqe)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	int user = to_ipd(qp->ibqp.pd)->user;
+	int i, j, ret;
+	struct ib_wc wc;
+
+	qp->r_len = 0;
+	for (i = j = 0; i < wqe->num_sge; i++) {
+		if (wqe->sg_list[i].length == 0)
+			continue;
+		/* Check LKEY */
+		if ((user && wqe->sg_list[i].lkey == 0) ||
+		    !ipath_lkey_ok(&dev->lk_table,
+				   &qp->r_sg_list[j], &wqe->sg_list[i],
+				   IB_ACCESS_LOCAL_WRITE))
+			goto bad_lkey;
+		qp->r_len += wqe->sg_list[i].length;
+		j++;
+	}
+	qp->r_sge.sge = qp->r_sg_list[0];
+	qp->r_sge.sg_list = qp->r_sg_list + 1;
+	qp->r_sge.num_sge = j;
+	ret = 1;
+	goto bail;
+
+bad_lkey:
+	wc.wr_id = wqe->wr_id;
+	wc.status = IB_WC_LOC_PROT_ERR;
+	wc.opcode = IB_WC_RECV;
+	wc.vendor_err = 0;
+	wc.byte_len = 0;
+	wc.imm_data = 0;
+	wc.qp_num = qp->ibqp.qp_num;
+	wc.src_qp = 0;
+	wc.wc_flags = 0;
+	wc.pkey_index = 0;
+	wc.slid = 0;
+	wc.sl = 0;
+	wc.dlid_path_bits = 0;
+	wc.port_num = 0;
+	/* Signal solicited completion event. */
+	ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
+	ret = 0;
+bail:
+	return ret;
+}
+
 /**
  * ipath_get_rwqe - copy the next RWQE into the QP's RWQE
  * @qp: the QP
@@ -119,71 +167,71 @@ int ipath_get_rwqe(struct ipath_qp *qp, int wr_id_only)
 {
 	unsigned long flags;
 	struct ipath_rq *rq;
+	struct ipath_rwq *wq;
 	struct ipath_srq *srq;
 	struct ipath_rwqe *wqe;
-	int ret = 1;
+	void (*handler)(struct ib_event *, void *);
+	u32 tail;
+	int ret;
 
-	if (!qp->ibqp.srq) {
+	if (qp->ibqp.srq) {
+		srq = to_isrq(qp->ibqp.srq);
+		handler = srq->ibsrq.event_handler;
+		rq = &srq->rq;
+	} else {
+		srq = NULL;
+		handler = NULL;
 		rq = &qp->r_rq;
-		spin_lock_irqsave(&rq->lock, flags);
-
-		if (unlikely(rq->tail == rq->head)) {
-			ret = 0;
-			goto done;
-		}
-		wqe = get_rwqe_ptr(rq, rq->tail);
-		qp->r_wr_id = wqe->wr_id;
-		if (!wr_id_only) {
-			qp->r_sge.sge = wqe->sg_list[0];
-			qp->r_sge.sg_list = wqe->sg_list + 1;
-			qp->r_sge.num_sge = wqe->num_sge;
-			qp->r_len = wqe->length;
-		}
-		if (++rq->tail >= rq->size)
-			rq->tail = 0;
-		goto done;
 	}
 
-	srq = to_isrq(qp->ibqp.srq);
-	rq = &srq->rq;
 	spin_lock_irqsave(&rq->lock, flags);
-
-	if (unlikely(rq->tail == rq->head)) {
-		ret = 0;
-		goto done;
-	}
-	wqe = get_rwqe_ptr(rq, rq->tail);
+	wq = rq->wq;
+	tail = wq->tail;
+	/* Validate tail before using it since it is user writable. */
+	if (tail >= rq->size)
+		tail = 0;
+	do {
+		if (unlikely(tail == wq->head)) {
+			spin_unlock_irqrestore(&rq->lock, flags);
+			ret = 0;
+			goto bail;
+		}
+		wqe = get_rwqe_ptr(rq, tail);
+		if (++tail >= rq->size)
+			tail = 0;
+	} while (!wr_id_only && !init_sge(qp, wqe));
 	qp->r_wr_id = wqe->wr_id;
-	if (!wr_id_only) {
-		qp->r_sge.sge = wqe->sg_list[0];
-		qp->r_sge.sg_list = wqe->sg_list + 1;
-		qp->r_sge.num_sge = wqe->num_sge;
-		qp->r_len = wqe->length;
-	}
-	if (++rq->tail >= rq->size)
-		rq->tail = 0;
-	if (srq->ibsrq.event_handler) {
-		struct ib_event ev;
+	wq->tail = tail;
+
+	ret = 1;
+	if (handler) {
 		u32 n;
 
-		if (rq->head < rq->tail)
-			n = rq->size + rq->head - rq->tail;
+		/*
+		 * validate head pointer value and compute
+		 * the number of remaining WQEs.
+		 */
+		n = wq->head;
+		if (n >= rq->size)
+			n = 0;
+		if (n < tail)
+			n += rq->size - tail;
 		else
-			n = rq->head - rq->tail;
+			n -= tail;
 		if (n < srq->limit) {
+			struct ib_event ev;
+
 			srq->limit = 0;
 			spin_unlock_irqrestore(&rq->lock, flags);
 			ev.device = qp->ibqp.device;
 			ev.element.srq = qp->ibqp.srq;
 			ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
-			srq->ibsrq.event_handler(&ev,
-						 srq->ibsrq.srq_context);
+			handler(&ev, srq->ibsrq.srq_context);
 			goto bail;
 		}
 	}
-
-done:
 	spin_unlock_irqrestore(&rq->lock, flags);
+
 bail:
 	return ret;
 }
@@ -422,6 +470,15 @@ done:
 		wake_up(&qp->wait);
 }
 
+static int want_buffer(struct ipath_devdata *dd)
+{
+	set_bit(IPATH_S_PIOINTBUFAVAIL, &dd->ipath_sendctrl);
+	ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
+			 dd->ipath_sendctrl);
+
+	return 0;
+}
+
 /**
  * ipath_no_bufs_available - tell the layer driver we need buffers
  * @qp: the QP that caused the problem
@@ -438,7 +495,7 @@ void ipath_no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev)
 		list_add_tail(&qp->piowait, &dev->piowait);
 	spin_unlock_irqrestore(&dev->pending_lock, flags);
 	/*
-	 * Note that as soon as ipath_layer_want_buffer() is called and
+	 * Note that as soon as want_buffer() is called and
 	 * possibly before it returns, ipath_ib_piobufavail()
 	 * could be called.  If we are still in the tasklet function,
 	 * tasklet_hi_schedule() will not call us until the next time
@@ -448,7 +505,7 @@ void ipath_no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev)
 	 */
 	clear_bit(IPATH_S_BUSY, &qp->s_flags);
 	tasklet_unlock(&qp->s_task);
-	ipath_layer_want_buffer(dev->dd);
+	want_buffer(dev->dd);
 	dev->n_piowait++;
 }
 
@@ -563,7 +620,7 @@ u32 ipath_make_grh(struct ipath_ibdev *dev, struct ib_grh *hdr,
 	hdr->hop_limit = grh->hop_limit;
 	/* The SGID is 32-bit aligned. */
 	hdr->sgid.global.subnet_prefix = dev->gid_prefix;
-	hdr->sgid.global.interface_id = ipath_layer_get_guid(dev->dd);
+	hdr->sgid.global.interface_id = dev->dd->ipath_guid;
 	hdr->dgid = grh->dgid;
 
 	/* GRH header size in 32-bit words. */
@@ -595,8 +652,7 @@ void ipath_do_ruc_send(unsigned long data)
 	if (test_and_set_bit(IPATH_S_BUSY, &qp->s_flags))
 		goto bail;
 
-	if (unlikely(qp->remote_ah_attr.dlid ==
-		     ipath_layer_get_lid(dev->dd))) {
+	if (unlikely(qp->remote_ah_attr.dlid == dev->dd->ipath_lid)) {
 		ipath_ruc_loopback(qp);
 		goto clear;
 	}
@@ -663,8 +719,8 @@ again:
 	qp->s_hdr.lrh[1] = cpu_to_be16(qp->remote_ah_attr.dlid);
 	qp->s_hdr.lrh[2] = cpu_to_be16(qp->s_hdrwords + nwords +
 				       SIZE_OF_CRC);
-	qp->s_hdr.lrh[3] = cpu_to_be16(ipath_layer_get_lid(dev->dd));
-	bth0 |= ipath_layer_get_pkey(dev->dd, qp->s_pkey_index);
+	qp->s_hdr.lrh[3] = cpu_to_be16(dev->dd->ipath_lid);
+	bth0 |= ipath_get_pkey(dev->dd, qp->s_pkey_index);
 	bth0 |= extra_bytes << 20;
 	ohdr->bth[0] = cpu_to_be32(bth0);
 	ohdr->bth[1] = cpu_to_be32(qp->remote_qpn);
diff --git a/drivers/infiniband/hw/ipath/ipath_srq.c b/drivers/infiniband/hw/ipath/ipath_srq.c
index f760434..941e866 100644
--- a/drivers/infiniband/hw/ipath/ipath_srq.c
+++ b/drivers/infiniband/hw/ipath/ipath_srq.c
@@ -48,66 +48,39 @@ int ipath_post_srq_receive(struct ib_srq *ibsrq, struct ib_recv_wr *wr,
 			   struct ib_recv_wr **bad_wr)
 {
 	struct ipath_srq *srq = to_isrq(ibsrq);
-	struct ipath_ibdev *dev = to_idev(ibsrq->device);
+	struct ipath_rwq *wq;
 	unsigned long flags;
 	int ret;
 
 	for (; wr; wr = wr->next) {
 		struct ipath_rwqe *wqe;
 		u32 next;
-		int i, j;
+		int i;
 
-		if (wr->num_sge > srq->rq.max_sge) {
+		if ((unsigned) wr->num_sge > srq->rq.max_sge) {
 			*bad_wr = wr;
 			ret = -ENOMEM;
 			goto bail;
 		}
 
 		spin_lock_irqsave(&srq->rq.lock, flags);
-		next = srq->rq.head + 1;
+		wq = srq->rq.wq;
+		next = wq->head + 1;
 		if (next >= srq->rq.size)
 			next = 0;
-		if (next == srq->rq.tail) {
+		if (next == wq->tail) {
 			spin_unlock_irqrestore(&srq->rq.lock, flags);
 			*bad_wr = wr;
 			ret = -ENOMEM;
 			goto bail;
 		}
 
-		wqe = get_rwqe_ptr(&srq->rq, srq->rq.head);
+		wqe = get_rwqe_ptr(&srq->rq, wq->head);
 		wqe->wr_id = wr->wr_id;
-		wqe->sg_list[0].mr = NULL;
-		wqe->sg_list[0].vaddr = NULL;
-		wqe->sg_list[0].length = 0;
-		wqe->sg_list[0].sge_length = 0;
-		wqe->length = 0;
-		for (i = 0, j = 0; i < wr->num_sge; i++) {
-			/* Check LKEY */
-			if (to_ipd(srq->ibsrq.pd)->user &&
-			    wr->sg_list[i].lkey == 0) {
-				spin_unlock_irqrestore(&srq->rq.lock,
-						       flags);
-				*bad_wr = wr;
-				ret = -EINVAL;
-				goto bail;
-			}
-			if (wr->sg_list[i].length == 0)
-				continue;
-			if (!ipath_lkey_ok(&dev->lk_table,
-					   &wqe->sg_list[j],
-					   &wr->sg_list[i],
-					   IB_ACCESS_LOCAL_WRITE)) {
-				spin_unlock_irqrestore(&srq->rq.lock,
-						       flags);
-				*bad_wr = wr;
-				ret = -EINVAL;
-				goto bail;
-			}
-			wqe->length += wr->sg_list[i].length;
-			j++;
-		}
-		wqe->num_sge = j;
-		srq->rq.head = next;
+		wqe->num_sge = wr->num_sge;
+		for (i = 0; i < wr->num_sge; i++)
+			wqe->sg_list[i] = wr->sg_list[i];
+		wq->head = next;
 		spin_unlock_irqrestore(&srq->rq.lock, flags);
 	}
 	ret = 0;
@@ -133,53 +106,95 @@ struct ib_srq *ipath_create_srq(struct ib_pd *ibpd,
 
 	if (dev->n_srqs_allocated == ib_ipath_max_srqs) {
 		ret = ERR_PTR(-ENOMEM);
-		goto bail;
+		goto done;
 	}
 
 	if (srq_init_attr->attr.max_wr == 0) {
 		ret = ERR_PTR(-EINVAL);
-		goto bail;
+		goto done;
 	}
 
 	if ((srq_init_attr->attr.max_sge > ib_ipath_max_srq_sges) ||
 	    (srq_init_attr->attr.max_wr > ib_ipath_max_srq_wrs)) {
 		ret = ERR_PTR(-EINVAL);
-		goto bail;
+		goto done;
 	}
 
 	srq = kmalloc(sizeof(*srq), GFP_KERNEL);
 	if (!srq) {
 		ret = ERR_PTR(-ENOMEM);
-		goto bail;
+		goto done;
 	}
 
 	/*
 	 * Need to use vmalloc() if we want to support large #s of entries.
 	 */
 	srq->rq.size = srq_init_attr->attr.max_wr + 1;
-	sz = sizeof(struct ipath_sge) * srq_init_attr->attr.max_sge +
+	srq->rq.max_sge = srq_init_attr->attr.max_sge;
+	sz = sizeof(struct ib_sge) * srq->rq.max_sge +
 		sizeof(struct ipath_rwqe);
-	srq->rq.wq = vmalloc(srq->rq.size * sz);
+	srq->rq.wq = vmalloc_user(sizeof(struct ipath_rwq) + srq->rq.size * sz);
 	if (!srq->rq.wq) {
-		kfree(srq);
 		ret = ERR_PTR(-ENOMEM);
-		goto bail;
+		goto bail_srq;
 	}
 
 	/*
+	 * Return the address of the RWQ as the offset to mmap.
+	 * See ipath_mmap() for details.
+	 */
+	if (udata && udata->outlen >= sizeof(__u64)) {
+		struct ipath_mmap_info *ip;
+		__u64 offset = (__u64) srq->rq.wq;
+		int err;
+
+		err = ib_copy_to_udata(udata, &offset, sizeof(offset));
+		if (err) {
+			ret = ERR_PTR(err);
+			goto bail_wq;
+		}
+
+		/* Allocate info for ipath_mmap(). */
+		ip = kmalloc(sizeof(*ip), GFP_KERNEL);
+		if (!ip) {
+			ret = ERR_PTR(-ENOMEM);
+			goto bail_wq;
+		}
+		srq->ip = ip;
+		ip->context = ibpd->uobject->context;
+		ip->obj = srq->rq.wq;
+		kref_init(&ip->ref);
+		ip->mmap_cnt = 0;
+		ip->size = PAGE_ALIGN(sizeof(struct ipath_rwq) +
+				      srq->rq.size * sz);
+		spin_lock_irq(&dev->pending_lock);
+		ip->next = dev->pending_mmaps;
+		dev->pending_mmaps = ip;
+		spin_unlock_irq(&dev->pending_lock);
+	} else
+		srq->ip = NULL;
+
+	/*
 	 * ib_create_srq() will initialize srq->ibsrq.
 	 */
 	spin_lock_init(&srq->rq.lock);
-	srq->rq.head = 0;
-	srq->rq.tail = 0;
+	srq->rq.wq->head = 0;
+	srq->rq.wq->tail = 0;
 	srq->rq.max_sge = srq_init_attr->attr.max_sge;
 	srq->limit = srq_init_attr->attr.srq_limit;
 
+	dev->n_srqs_allocated++;
+
 	ret = &srq->ibsrq;
+	goto done;
 
-	dev->n_srqs_allocated++;
+bail_wq:
+	vfree(srq->rq.wq);
 
-bail:
+bail_srq:
+	kfree(srq);
+
+done:
 	return ret;
 }
 
@@ -188,83 +203,130 @@ bail:
  * @ibsrq: the SRQ to modify
  * @attr: the new attributes of the SRQ
  * @attr_mask: indicates which attributes to modify
+ * @udata: user data for ipathverbs.so
  */
 int ipath_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
-		     enum ib_srq_attr_mask attr_mask)
+		     enum ib_srq_attr_mask attr_mask,
+		     struct ib_udata *udata)
 {
 	struct ipath_srq *srq = to_isrq(ibsrq);
-	unsigned long flags;
-	int ret;
+	int ret = 0;
 
-	if (attr_mask & IB_SRQ_MAX_WR)
-		if ((attr->max_wr > ib_ipath_max_srq_wrs) ||
-		    (attr->max_sge > srq->rq.max_sge)) {
-			ret = -EINVAL;
-			goto bail;
-		}
+	if (attr_mask & IB_SRQ_MAX_WR) {
+		struct ipath_rwq *owq;
+		struct ipath_rwq *wq;
+		struct ipath_rwqe *p;
+		u32 sz, size, n, head, tail;
 
-	if (attr_mask & IB_SRQ_LIMIT)
-		if (attr->srq_limit >= srq->rq.size) {
+		/* Check that the requested sizes are below the limits. */
+		if ((attr->max_wr > ib_ipath_max_srq_wrs) ||
+		    ((attr_mask & IB_SRQ_LIMIT) ?
+		     attr->srq_limit : srq->limit) > attr->max_wr) {
 			ret = -EINVAL;
 			goto bail;
 		}
 
-	if (attr_mask & IB_SRQ_MAX_WR) {
-		struct ipath_rwqe *wq, *p;
-		u32 sz, size, n;
-
 		sz = sizeof(struct ipath_rwqe) +
-			attr->max_sge * sizeof(struct ipath_sge);
+			srq->rq.max_sge * sizeof(struct ib_sge);
 		size = attr->max_wr + 1;
-		wq = vmalloc(size * sz);
+		wq = vmalloc_user(sizeof(struct ipath_rwq) + size * sz);
 		if (!wq) {
 			ret = -ENOMEM;
 			goto bail;
 		}
 
-		spin_lock_irqsave(&srq->rq.lock, flags);
-		if (srq->rq.head < srq->rq.tail)
-			n = srq->rq.size + srq->rq.head - srq->rq.tail;
+		/*
+		 * Return the address of the RWQ as the offset to mmap.
+		 * See ipath_mmap() for details.
+		 */
+		if (udata && udata->inlen >= sizeof(__u64)) {
+			__u64 offset_addr;
+			__u64 offset = (__u64) wq;
+
+			ret = ib_copy_from_udata(&offset_addr, udata,
+						 sizeof(offset_addr));
+			if (ret) {
+				vfree(wq);
+				goto bail;
+			}
+			udata->outbuf = (void __user *) offset_addr;
+			ret = ib_copy_to_udata(udata, &offset,
+					       sizeof(offset));
+			if (ret) {
+				vfree(wq);
+				goto bail;
+			}
+		}
+
+		spin_lock_irq(&srq->rq.lock);
+		/*
+		 * validate head pointer value and compute
+		 * the number of remaining WQEs.
+		 */
+		owq = srq->rq.wq;
+		head = owq->head;
+		if (head >= srq->rq.size)
+			head = 0;
+		tail = owq->tail;
+		if (tail >= srq->rq.size)
+			tail = 0;
+		n = head;
+		if (n < tail)
+			n += srq->rq.size - tail;
 		else
-			n = srq->rq.head - srq->rq.tail;
-		if (size <= n || size <= srq->limit) {
-			spin_unlock_irqrestore(&srq->rq.lock, flags);
+			n -= tail;
+		if (size <= n) {
+			spin_unlock_irq(&srq->rq.lock);
 			vfree(wq);
 			ret = -EINVAL;
 			goto bail;
 		}
 		n = 0;
-		p = wq;
-		while (srq->rq.tail != srq->rq.head) {
+		p = wq->wq;
+		while (tail != head) {
 			struct ipath_rwqe *wqe;
 			int i;
 
-			wqe = get_rwqe_ptr(&srq->rq, srq->rq.tail);
+			wqe = get_rwqe_ptr(&srq->rq, tail);
 			p->wr_id = wqe->wr_id;
-			p->length = wqe->length;
 			p->num_sge = wqe->num_sge;
 			for (i = 0; i < wqe->num_sge; i++)
 				p->sg_list[i] = wqe->sg_list[i];
 			n++;
 			p = (struct ipath_rwqe *)((char *) p + sz);
-			if (++srq->rq.tail >= srq->rq.size)
-				srq->rq.tail = 0;
+			if (++tail >= srq->rq.size)
+				tail = 0;
 		}
-		vfree(srq->rq.wq);
 		srq->rq.wq = wq;
 		srq->rq.size = size;
-		srq->rq.head = n;
-		srq->rq.tail = 0;
-		srq->rq.max_sge = attr->max_sge;
-		spin_unlock_irqrestore(&srq->rq.lock, flags);
-	}
-
-	if (attr_mask & IB_SRQ_LIMIT) {
-		spin_lock_irqsave(&srq->rq.lock, flags);
-		srq->limit = attr->srq_limit;
-		spin_unlock_irqrestore(&srq->rq.lock, flags);
+		wq->head = n;
+		wq->tail = 0;
+		if (attr_mask & IB_SRQ_LIMIT)
+			srq->limit = attr->srq_limit;
+		spin_unlock_irq(&srq->rq.lock);
+
+		vfree(owq);
+
+		if (srq->ip) {
+			struct ipath_mmap_info *ip = srq->ip;
+			struct ipath_ibdev *dev = to_idev(srq->ibsrq.device);
+
+			ip->obj = wq;
+			ip->size = PAGE_ALIGN(sizeof(struct ipath_rwq) +
+					      size * sz);
+			spin_lock_irq(&dev->pending_lock);
+			ip->next = dev->pending_mmaps;
+			dev->pending_mmaps = ip;
+			spin_unlock_irq(&dev->pending_lock);
+		}
+	} else if (attr_mask & IB_SRQ_LIMIT) {
+		spin_lock_irq(&srq->rq.lock);
+		if (attr->srq_limit >= srq->rq.size)
+			ret = -EINVAL;
+		else
+			srq->limit = attr->srq_limit;
+		spin_unlock_irq(&srq->rq.lock);
 	}
-	ret = 0;
 
 bail:
 	return ret;
diff --git a/drivers/infiniband/hw/ipath/ipath_stats.c b/drivers/infiniband/hw/ipath/ipath_stats.c
index 70351b7..30a8259 100644
--- a/drivers/infiniband/hw/ipath/ipath_stats.c
+++ b/drivers/infiniband/hw/ipath/ipath_stats.c
@@ -271,33 +271,6 @@ void ipath_get_faststats(unsigned long opaque)
 		}
 	}
 
-	if (dd->ipath_nosma_bufs) {
-		dd->ipath_nosma_secs += 5;
-		if (dd->ipath_nosma_secs >= 30) {
-			ipath_cdbg(SMA, "No SMA bufs avail %u seconds; "
-				   "cancelling pending sends\n",
-				   dd->ipath_nosma_secs);
-			/*
-			 * issue an abort as well, in case we have a packet
-			 * stuck in launch fifo.  This could corrupt an
-			 * outgoing user packet in the worst case,
-			 * but this is a pretty catastrophic, anyway.
-			 */
-			ipath_write_kreg(dd, dd->ipath_kregs->kr_sendctrl,
-					 INFINIPATH_S_ABORT);
-			ipath_disarm_piobufs(dd, dd->ipath_lastport_piobuf,
-					     dd->ipath_piobcnt2k +
-					     dd->ipath_piobcnt4k -
-					     dd->ipath_lastport_piobuf);
-			/* start again, if necessary */
-			dd->ipath_nosma_secs = 0;
-		} else
-			ipath_cdbg(SMA, "No SMA bufs avail %u tries, "
-				   "after %u seconds\n",
-				   dd->ipath_nosma_bufs,
-				   dd->ipath_nosma_secs);
-	}
-
 done:
 	mod_timer(&dd->ipath_stats_timer, jiffies + HZ * 5);
 }
diff --git a/drivers/infiniband/hw/ipath/ipath_sysfs.c b/drivers/infiniband/hw/ipath/ipath_sysfs.c
index b98821d..e299148 100644
--- a/drivers/infiniband/hw/ipath/ipath_sysfs.c
+++ b/drivers/infiniband/hw/ipath/ipath_sysfs.c
@@ -35,7 +35,6 @@
 #include <linux/pci.h>
 
 #include "ipath_kernel.h"
-#include "ipath_layer.h"
 #include "ipath_common.h"
 
 /**
@@ -76,7 +75,7 @@ bail:
 static ssize_t show_version(struct device_driver *dev, char *buf)
 {
 	/* The string printed here is already newline-terminated. */
-	return scnprintf(buf, PAGE_SIZE, "%s", ipath_core_version);
+	return scnprintf(buf, PAGE_SIZE, "%s", ib_ipath_version);
 }
 
 static ssize_t show_num_units(struct device_driver *dev, char *buf)
@@ -108,8 +107,8 @@ static const char *ipath_status_str[] = {
 	"Initted",
 	"Disabled",
 	"Admin_Disabled",
-	"OIB_SMA",
-	"SMA",
+	"", /* This used to be the old "OIB_SMA" status. */
+	"", /* This used to be the old "SMA" status. */
 	"Present",
 	"IB_link_up",
 	"IB_configured",
@@ -227,7 +226,6 @@ static ssize_t store_mlid(struct device *dev,
 	unit = dd->ipath_unit;
 
 	dd->ipath_mlid = mlid;
-	ipath_layer_intr(dd, IPATH_LAYER_INT_BCAST);
 
 	goto bail;
 invalid:
@@ -467,7 +465,7 @@ static ssize_t store_link_state(struct device *dev,
 	if (ret < 0)
 		goto invalid;
 
-	r = ipath_layer_set_linkstate(dd, state);
+	r = ipath_set_linkstate(dd, state);
 	if (r < 0) {
 		ret = r;
 		goto bail;
@@ -502,7 +500,7 @@ static ssize_t store_mtu(struct device *dev,
 	if (ret < 0)
 		goto invalid;
 
-	r = ipath_layer_set_mtu(dd, mtu);
+	r = ipath_set_mtu(dd, mtu);
 	if (r < 0)
 		ret = r;
 
@@ -563,6 +561,33 @@ bail:
 	return ret;
 }
 
+static ssize_t store_rx_pol_inv(struct device *dev,
+			  struct device_attribute *attr,
+			  const char *buf,
+			  size_t count)
+{
+	struct ipath_devdata *dd = dev_get_drvdata(dev);
+	int ret, r;
+	u16 val;
+
+	ret = ipath_parse_ushort(buf, &val);
+	if (ret < 0)
+		goto invalid;
+
+	r = ipath_set_rx_pol_inv(dd, val);
+	if (r < 0) {
+		ret = r;
+		goto bail;
+	}
+
+	goto bail;
+invalid:
+	ipath_dev_err(dd, "attempt to set invalid Rx Polarity invert\n");
+bail:
+	return ret;
+}
+
+
 static DRIVER_ATTR(num_units, S_IRUGO, show_num_units, NULL);
 static DRIVER_ATTR(version, S_IRUGO, show_version, NULL);
 
@@ -589,6 +614,7 @@ static DEVICE_ATTR(status, S_IRUGO, show_status, NULL);
 static DEVICE_ATTR(status_str, S_IRUGO, show_status_str, NULL);
 static DEVICE_ATTR(boardversion, S_IRUGO, show_boardversion, NULL);
 static DEVICE_ATTR(unit, S_IRUGO, show_unit, NULL);
+static DEVICE_ATTR(rx_pol_inv, S_IWUSR, NULL, store_rx_pol_inv);
 
 static struct attribute *dev_attributes[] = {
 	&dev_attr_guid.attr,
@@ -603,6 +629,7 @@ static struct attribute *dev_attributes[] = {
 	&dev_attr_boardversion.attr,
 	&dev_attr_unit.attr,
 	&dev_attr_enabled.attr,
+	&dev_attr_rx_pol_inv.attr,
 	NULL
 };
 
diff --git a/drivers/infiniband/hw/ipath/ipath_uc.c b/drivers/infiniband/hw/ipath/ipath_uc.c
index c33abea..0fd3cde 100644
--- a/drivers/infiniband/hw/ipath/ipath_uc.c
+++ b/drivers/infiniband/hw/ipath/ipath_uc.c
@@ -32,7 +32,7 @@
  */
 
 #include "ipath_verbs.h"
-#include "ipath_common.h"
+#include "ipath_kernel.h"
 
 /* cut down ridiculously long IB macro names */
 #define OP(x) IB_OPCODE_UC_##x
@@ -261,8 +261,7 @@ void ipath_uc_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
 		 * size to 56 bytes so the last 4 bytes of
 		 * the BTH header (PSN) is in the data buffer.
 		 */
-		header_in_data =
-			ipath_layer_get_rcvhdrentsize(dev->dd) == 16;
+		header_in_data = dev->dd->ipath_rcvhdrentsize == 16;
 		if (header_in_data) {
 			psn = be32_to_cpu(((__be32 *) data)[0]);
 			data += sizeof(__be32);
diff --git a/drivers/infiniband/hw/ipath/ipath_ud.c b/drivers/infiniband/hw/ipath/ipath_ud.c
index 3466129..6991d1d 100644
--- a/drivers/infiniband/hw/ipath/ipath_ud.c
+++ b/drivers/infiniband/hw/ipath/ipath_ud.c
@@ -34,7 +34,54 @@
 #include <rdma/ib_smi.h>
 
 #include "ipath_verbs.h"
-#include "ipath_common.h"
+#include "ipath_kernel.h"
+
+static int init_sge(struct ipath_qp *qp, struct ipath_rwqe *wqe,
+		    u32 *lengthp, struct ipath_sge_state *ss)
+{
+	struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
+	int user = to_ipd(qp->ibqp.pd)->user;
+	int i, j, ret;
+	struct ib_wc wc;
+
+	*lengthp = 0;
+	for (i = j = 0; i < wqe->num_sge; i++) {
+		if (wqe->sg_list[i].length == 0)
+			continue;
+		/* Check LKEY */
+		if ((user && wqe->sg_list[i].lkey == 0) ||
+		    !ipath_lkey_ok(&dev->lk_table,
+				   j ? &ss->sg_list[j - 1] : &ss->sge,
+				   &wqe->sg_list[i], IB_ACCESS_LOCAL_WRITE))
+			goto bad_lkey;
+		*lengthp += wqe->sg_list[i].length;
+		j++;
+	}
+	ss->num_sge = j;
+	ret = 1;
+	goto bail;
+
+bad_lkey:
+	wc.wr_id = wqe->wr_id;
+	wc.status = IB_WC_LOC_PROT_ERR;
+	wc.opcode = IB_WC_RECV;
+	wc.vendor_err = 0;
+	wc.byte_len = 0;
+	wc.imm_data = 0;
+	wc.qp_num = qp->ibqp.qp_num;
+	wc.src_qp = 0;
+	wc.wc_flags = 0;
+	wc.pkey_index = 0;
+	wc.slid = 0;
+	wc.sl = 0;
+	wc.dlid_path_bits = 0;
+	wc.port_num = 0;
+	/* Signal solicited completion event. */
+	ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
+	ret = 0;
+bail:
+	return ret;
+}
 
 /**
  * ipath_ud_loopback - handle send on loopback QPs
@@ -46,6 +93,8 @@
  *
  * This is called from ipath_post_ud_send() to forward a WQE addressed
  * to the same HCA.
+ * Note that the receive interrupt handler may be calling ipath_ud_rcv()
+ * while this is being called.
  */
 static void ipath_ud_loopback(struct ipath_qp *sqp,
 			      struct ipath_sge_state *ss,
@@ -60,7 +109,11 @@ static void ipath_ud_loopback(struct ipath_qp *sqp,
 	struct ipath_srq *srq;
 	struct ipath_sge_state rsge;
 	struct ipath_sge *sge;
+	struct ipath_rwq *wq;
 	struct ipath_rwqe *wqe;
+	void (*handler)(struct ib_event *, void *);
+	u32 tail;
+	u32 rlen;
 
 	qp = ipath_lookup_qpn(&dev->qp_table, wr->wr.ud.remote_qpn);
 	if (!qp)
@@ -94,6 +147,13 @@ static void ipath_ud_loopback(struct ipath_qp *sqp,
 		wc->imm_data = 0;
 	}
 
+	if (wr->num_sge > 1) {
+		rsge.sg_list = kmalloc((wr->num_sge - 1) *
+					sizeof(struct ipath_sge),
+				       GFP_ATOMIC);
+	} else
+		rsge.sg_list = NULL;
+
 	/*
 	 * Get the next work request entry to find where to put the data.
 	 * Note that it is safe to drop the lock after changing rq->tail
@@ -101,37 +161,52 @@ static void ipath_ud_loopback(struct ipath_qp *sqp,
 	 */
 	if (qp->ibqp.srq) {
 		srq = to_isrq(qp->ibqp.srq);
+		handler = srq->ibsrq.event_handler;
 		rq = &srq->rq;
 	} else {
 		srq = NULL;
+		handler = NULL;
 		rq = &qp->r_rq;
 	}
+
 	spin_lock_irqsave(&rq->lock, flags);
-	if (rq->tail == rq->head) {
-		spin_unlock_irqrestore(&rq->lock, flags);
-		dev->n_pkt_drops++;
-		goto done;
+	wq = rq->wq;
+	tail = wq->tail;
+	while (1) {
+		if (unlikely(tail == wq->head)) {
+			spin_unlock_irqrestore(&rq->lock, flags);
+			dev->n_pkt_drops++;
+			goto bail_sge;
+		}
+		wqe = get_rwqe_ptr(rq, tail);
+		if (++tail >= rq->size)
+			tail = 0;
+		if (init_sge(qp, wqe, &rlen, &rsge))
+			break;
+		wq->tail = tail;
 	}
 	/* Silently drop packets which are too big. */
-	wqe = get_rwqe_ptr(rq, rq->tail);
-	if (wc->byte_len > wqe->length) {
+	if (wc->byte_len > rlen) {
 		spin_unlock_irqrestore(&rq->lock, flags);
 		dev->n_pkt_drops++;
-		goto done;
+		goto bail_sge;
 	}
+	wq->tail = tail;
 	wc->wr_id = wqe->wr_id;
-	rsge.sge = wqe->sg_list[0];
-	rsge.sg_list = wqe->sg_list + 1;
-	rsge.num_sge = wqe->num_sge;
-	if (++rq->tail >= rq->size)
-		rq->tail = 0;
-	if (srq && srq->ibsrq.event_handler) {
+	if (handler) {
 		u32 n;
 
-		if (rq->head < rq->tail)
-			n = rq->size + rq->head - rq->tail;
+		/*
+		 * validate head pointer value and compute
+		 * the number of remaining WQEs.
+		 */
+		n = wq->head;
+		if (n >= rq->size)
+			n = 0;
+		if (n < tail)
+			n += rq->size - tail;
 		else
-			n = rq->head - rq->tail;
+			n -= tail;
 		if (n < srq->limit) {
 			struct ib_event ev;
 
@@ -140,12 +215,12 @@ static void ipath_ud_loopback(struct ipath_qp *sqp,
 			ev.device = qp->ibqp.device;
 			ev.element.srq = qp->ibqp.srq;
 			ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
-			srq->ibsrq.event_handler(&ev,
-						 srq->ibsrq.srq_context);
+			handler(&ev, srq->ibsrq.srq_context);
 		} else
 			spin_unlock_irqrestore(&rq->lock, flags);
 	} else
 		spin_unlock_irqrestore(&rq->lock, flags);
+
 	ah_attr = &to_iah(wr->wr.ud.ah)->attr;
 	if (ah_attr->ah_flags & IB_AH_GRH) {
 		ipath_copy_sge(&rsge, &ah_attr->grh, sizeof(struct ib_grh));
@@ -186,7 +261,7 @@ static void ipath_ud_loopback(struct ipath_qp *sqp,
 	wc->src_qp = sqp->ibqp.qp_num;
 	/* XXX do we know which pkey matched? Only needed for GSI. */
 	wc->pkey_index = 0;
-	wc->slid = ipath_layer_get_lid(dev->dd) |
+	wc->slid = dev->dd->ipath_lid |
 		(ah_attr->src_path_bits &
 		 ((1 << (dev->mkeyprot_resv_lmc & 7)) - 1));
 	wc->sl = ah_attr->sl;
@@ -196,6 +271,8 @@ static void ipath_ud_loopback(struct ipath_qp *sqp,
 	ipath_cq_enter(to_icq(qp->ibqp.recv_cq), wc,
 		       wr->send_flags & IB_SEND_SOLICITED);
 
+bail_sge:
+	kfree(rsge.sg_list);
 done:
 	if (atomic_dec_and_test(&qp->refcount))
 		wake_up(&qp->wait);
@@ -276,7 +353,7 @@ int ipath_post_ud_send(struct ipath_qp *qp, struct ib_send_wr *wr)
 		ss.num_sge++;
 	}
 	/* Check for invalid packet size. */
-	if (len > ipath_layer_get_ibmtu(dev->dd)) {
+	if (len > dev->dd->ipath_ibmtu) {
 		ret = -EINVAL;
 		goto bail;
 	}
@@ -298,7 +375,7 @@ int ipath_post_ud_send(struct ipath_qp *qp, struct ib_send_wr *wr)
 		dev->n_unicast_xmit++;
 		lid = ah_attr->dlid &
 			~((1 << (dev->mkeyprot_resv_lmc & 7)) - 1);
-		if (unlikely(lid == ipath_layer_get_lid(dev->dd))) {
+		if (unlikely(lid == dev->dd->ipath_lid)) {
 			/*
 			 * Pass in an uninitialized ib_wc to save stack
 			 * space.
@@ -327,7 +404,7 @@ int ipath_post_ud_send(struct ipath_qp *qp, struct ib_send_wr *wr)
 		qp->s_hdr.u.l.grh.sgid.global.subnet_prefix =
 			dev->gid_prefix;
 		qp->s_hdr.u.l.grh.sgid.global.interface_id =
-			ipath_layer_get_guid(dev->dd);
+			dev->dd->ipath_guid;
 		qp->s_hdr.u.l.grh.dgid = ah_attr->grh.dgid;
 		/*
 		 * Don't worry about sending to locally attached multicast
@@ -357,7 +434,7 @@ int ipath_post_ud_send(struct ipath_qp *qp, struct ib_send_wr *wr)
 	qp->s_hdr.lrh[0] = cpu_to_be16(lrh0);
 	qp->s_hdr.lrh[1] = cpu_to_be16(ah_attr->dlid);	/* DEST LID */
 	qp->s_hdr.lrh[2] = cpu_to_be16(hwords + nwords + SIZE_OF_CRC);
-	lid = ipath_layer_get_lid(dev->dd);
+	lid = dev->dd->ipath_lid;
 	if (lid) {
 		lid |= ah_attr->src_path_bits &
 			((1 << (dev->mkeyprot_resv_lmc & 7)) - 1);
@@ -368,7 +445,7 @@ int ipath_post_ud_send(struct ipath_qp *qp, struct ib_send_wr *wr)
 		bth0 |= 1 << 23;
 	bth0 |= extra_bytes << 20;
 	bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? IPATH_DEFAULT_P_KEY :
-		ipath_layer_get_pkey(dev->dd, qp->s_pkey_index);
+		ipath_get_pkey(dev->dd, qp->s_pkey_index);
 	ohdr->bth[0] = cpu_to_be32(bth0);
 	/*
 	 * Use the multicast QP if the destination LID is a multicast LID.
@@ -433,13 +510,9 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
 	int opcode;
 	u32 hdrsize;
 	u32 pad;
-	unsigned long flags;
 	struct ib_wc wc;
 	u32 qkey;
 	u32 src_qp;
-	struct ipath_rq *rq;
-	struct ipath_srq *srq;
-	struct ipath_rwqe *wqe;
 	u16 dlid;
 	int header_in_data;
 
@@ -458,8 +531,7 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
 		 * the eager header buffer size to 56 bytes so the last 12
 		 * bytes of the IB header is in the data buffer.
 		 */
-		header_in_data =
-			ipath_layer_get_rcvhdrentsize(dev->dd) == 16;
+		header_in_data = dev->dd->ipath_rcvhdrentsize == 16;
 		if (header_in_data) {
 			qkey = be32_to_cpu(((__be32 *) data)[1]);
 			src_qp = be32_to_cpu(((__be32 *) data)[2]);
@@ -547,19 +619,10 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
 
 	/*
 	 * Get the next work request entry to find where to put the data.
-	 * Note that it is safe to drop the lock after changing rq->tail
-	 * since ipath_post_receive() won't fill the empty slot.
 	 */
-	if (qp->ibqp.srq) {
-		srq = to_isrq(qp->ibqp.srq);
-		rq = &srq->rq;
-	} else {
-		srq = NULL;
-		rq = &qp->r_rq;
-	}
-	spin_lock_irqsave(&rq->lock, flags);
-	if (rq->tail == rq->head) {
-		spin_unlock_irqrestore(&rq->lock, flags);
+	if (qp->r_reuse_sge)
+		qp->r_reuse_sge = 0;
+	else if (!ipath_get_rwqe(qp, 0)) {
 		/*
 		 * Count VL15 packets dropped due to no receive buffer.
 		 * Otherwise, count them as buffer overruns since usually,
@@ -573,39 +636,11 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
 		goto bail;
 	}
 	/* Silently drop packets which are too big. */
-	wqe = get_rwqe_ptr(rq, rq->tail);
-	if (wc.byte_len > wqe->length) {
-		spin_unlock_irqrestore(&rq->lock, flags);
+	if (wc.byte_len > qp->r_len) {
+		qp->r_reuse_sge = 1;
 		dev->n_pkt_drops++;
 		goto bail;
 	}
-	wc.wr_id = wqe->wr_id;
-	qp->r_sge.sge = wqe->sg_list[0];
-	qp->r_sge.sg_list = wqe->sg_list + 1;
-	qp->r_sge.num_sge = wqe->num_sge;
-	if (++rq->tail >= rq->size)
-		rq->tail = 0;
-	if (srq && srq->ibsrq.event_handler) {
-		u32 n;
-
-		if (rq->head < rq->tail)
-			n = rq->size + rq->head - rq->tail;
-		else
-			n = rq->head - rq->tail;
-		if (n < srq->limit) {
-			struct ib_event ev;
-
-			srq->limit = 0;
-			spin_unlock_irqrestore(&rq->lock, flags);
-			ev.device = qp->ibqp.device;
-			ev.element.srq = qp->ibqp.srq;
-			ev.event = IB_EVENT_SRQ_LIMIT_REACHED;
-			srq->ibsrq.event_handler(&ev,
-						 srq->ibsrq.srq_context);
-		} else
-			spin_unlock_irqrestore(&rq->lock, flags);
-	} else
-		spin_unlock_irqrestore(&rq->lock, flags);
 	if (has_grh) {
 		ipath_copy_sge(&qp->r_sge, &hdr->u.l.grh,
 			       sizeof(struct ib_grh));
@@ -614,6 +649,7 @@ void ipath_ud_rcv(struct ipath_ibdev *dev, struct ipath_ib_header *hdr,
 		ipath_skip_sge(&qp->r_sge, sizeof(struct ib_grh));
 	ipath_copy_sge(&qp->r_sge, data,
 		       wc.byte_len - sizeof(struct ib_grh));
+	wc.wr_id = qp->r_wr_id;
 	wc.status = IB_WC_SUCCESS;
 	wc.opcode = IB_WC_RECV;
 	wc.vendor_err = 0;
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.c b/drivers/infiniband/hw/ipath/ipath_verbs.c
index d70a9b6..b8381c5 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.c
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.c
@@ -33,15 +33,13 @@
 
 #include <rdma/ib_mad.h>
 #include <rdma/ib_user_verbs.h>
+#include <linux/io.h>
 #include <linux/utsname.h>
 
 #include "ipath_kernel.h"
 #include "ipath_verbs.h"
 #include "ipath_common.h"
 
-/* Not static, because we don't want the compiler removing it */
-const char ipath_verbs_version[] = "ipath_verbs " IPATH_IDSTR;
-
 static unsigned int ib_ipath_qp_table_size = 251;
 module_param_named(qp_table_size, ib_ipath_qp_table_size, uint, S_IRUGO);
 MODULE_PARM_DESC(qp_table_size, "QP table size");
@@ -52,10 +50,6 @@ module_param_named(lkey_table_size, ib_ipath_lkey_table_size, uint,
 MODULE_PARM_DESC(lkey_table_size,
 		 "LKEY table size in bits (2^n, 1 <= n <= 23)");
 
-unsigned int ib_ipath_debug;	/* debug mask */
-module_param_named(debug, ib_ipath_debug, uint, S_IWUSR | S_IRUGO);
-MODULE_PARM_DESC(debug, "Verbs debug mask");
-
 static unsigned int ib_ipath_max_pds = 0xFFFF;
 module_param_named(max_pds, ib_ipath_max_pds, uint, S_IWUSR | S_IRUGO);
 MODULE_PARM_DESC(max_pds,
@@ -79,6 +73,10 @@ module_param_named(max_qp_wrs, ib_ipath_max_qp_wrs, uint,
 		   S_IWUSR | S_IRUGO);
 MODULE_PARM_DESC(max_qp_wrs, "Maximum number of QP WRs to support");
 
+unsigned int ib_ipath_max_qps = 16384;
+module_param_named(max_qps, ib_ipath_max_qps, uint, S_IWUSR | S_IRUGO);
+MODULE_PARM_DESC(max_qps, "Maximum number of QPs to support");
+
 unsigned int ib_ipath_max_sges = 0x60;
 module_param_named(max_sges, ib_ipath_max_sges, uint, S_IWUSR | S_IRUGO);
 MODULE_PARM_DESC(max_sges, "Maximum number of SGEs to support");
@@ -109,9 +107,9 @@ module_param_named(max_srq_wrs, ib_ipath_max_srq_wrs,
 		   uint, S_IWUSR | S_IRUGO);
 MODULE_PARM_DESC(max_srq_wrs, "Maximum number of SRQ WRs support");
 
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("QLogic <support@pathscale.com>");
-MODULE_DESCRIPTION("QLogic InfiniPath driver");
+static unsigned int ib_ipath_disable_sma;
+module_param_named(disable_sma, ib_ipath_disable_sma, uint, S_IWUSR | S_IRUGO);
+MODULE_PARM_DESC(ib_ipath_disable_sma, "Disable the SMA");
 
 const int ib_ipath_state_ops[IB_QPS_ERR + 1] = {
 	[IB_QPS_RESET] = 0,
@@ -125,6 +123,16 @@ const int ib_ipath_state_ops[IB_QPS_ERR + 1] = {
 	[IB_QPS_ERR] = 0,
 };
 
+struct ipath_ucontext {
+	struct ib_ucontext ibucontext;
+};
+
+static inline struct ipath_ucontext *to_iucontext(struct ib_ucontext
+						  *ibucontext)
+{
+	return container_of(ibucontext, struct ipath_ucontext, ibucontext);
+}
+
 /*
  * Translate ib_wr_opcode into ib_wc_opcode.
  */
@@ -277,11 +285,12 @@ static int ipath_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 			      struct ib_recv_wr **bad_wr)
 {
 	struct ipath_qp *qp = to_iqp(ibqp);
+	struct ipath_rwq *wq = qp->r_rq.wq;
 	unsigned long flags;
 	int ret;
 
 	/* Check that state is OK to post receive. */
-	if (!(ib_ipath_state_ops[qp->state] & IPATH_POST_RECV_OK)) {
+	if (!(ib_ipath_state_ops[qp->state] & IPATH_POST_RECV_OK) || !wq) {
 		*bad_wr = wr;
 		ret = -EINVAL;
 		goto bail;
@@ -290,59 +299,31 @@ static int ipath_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
 	for (; wr; wr = wr->next) {
 		struct ipath_rwqe *wqe;
 		u32 next;
-		int i, j;
+		int i;
 
-		if (wr->num_sge > qp->r_rq.max_sge) {
+		if ((unsigned) wr->num_sge > qp->r_rq.max_sge) {
 			*bad_wr = wr;
 			ret = -ENOMEM;
 			goto bail;
 		}
 
 		spin_lock_irqsave(&qp->r_rq.lock, flags);
-		next = qp->r_rq.head + 1;
+		next = wq->head + 1;
 		if (next >= qp->r_rq.size)
 			next = 0;
-		if (next == qp->r_rq.tail) {
+		if (next == wq->tail) {
 			spin_unlock_irqrestore(&qp->r_rq.lock, flags);
 			*bad_wr = wr;
 			ret = -ENOMEM;
 			goto bail;
 		}
 
-		wqe = get_rwqe_ptr(&qp->r_rq, qp->r_rq.head);
+		wqe = get_rwqe_ptr(&qp->r_rq, wq->head);
 		wqe->wr_id = wr->wr_id;
-		wqe->sg_list[0].mr = NULL;
-		wqe->sg_list[0].vaddr = NULL;
-		wqe->sg_list[0].length = 0;
-		wqe->sg_list[0].sge_length = 0;
-		wqe->length = 0;
-		for (i = 0, j = 0; i < wr->num_sge; i++) {
-			/* Check LKEY */
-			if (to_ipd(qp->ibqp.pd)->user &&
-			    wr->sg_list[i].lkey == 0) {
-				spin_unlock_irqrestore(&qp->r_rq.lock,
-						       flags);
-				*bad_wr = wr;
-				ret = -EINVAL;
-				goto bail;
-			}
-			if (wr->sg_list[i].length == 0)
-				continue;
-			if (!ipath_lkey_ok(
-				    &to_idev(qp->ibqp.device)->lk_table,
-				    &wqe->sg_list[j], &wr->sg_list[i],
-				    IB_ACCESS_LOCAL_WRITE)) {
-				spin_unlock_irqrestore(&qp->r_rq.lock,
-						       flags);
-				*bad_wr = wr;
-				ret = -EINVAL;
-				goto bail;
-			}
-			wqe->length += wr->sg_list[i].length;
-			j++;
-		}
-		wqe->num_sge = j;
-		qp->r_rq.head = next;
+		wqe->num_sge = wr->num_sge;
+		for (i = 0; i < wr->num_sge; i++)
+			wqe->sg_list[i] = wr->sg_list[i];
+		wq->head = next;
 		spin_unlock_irqrestore(&qp->r_rq.lock, flags);
 	}
 	ret = 0;
@@ -377,6 +358,9 @@ static void ipath_qp_rcv(struct ipath_ibdev *dev,
 	switch (qp->ibqp.qp_type) {
 	case IB_QPT_SMI:
 	case IB_QPT_GSI:
+		if (ib_ipath_disable_sma)
+			break;
+		/* FALLTHROUGH */
 	case IB_QPT_UD:
 		ipath_ud_rcv(dev, hdr, has_grh, data, tlen, qp);
 		break;
@@ -395,7 +379,7 @@ static void ipath_qp_rcv(struct ipath_ibdev *dev,
 }
 
 /**
- * ipath_ib_rcv - process and incoming packet
+ * ipath_ib_rcv - process an incoming packet
  * @arg: the device pointer
  * @rhdr: the header of the packet
  * @data: the packet data
@@ -404,9 +388,9 @@ static void ipath_qp_rcv(struct ipath_ibdev *dev,
  * This is called from ipath_kreceive() to process an incoming packet at
  * interrupt level. Tlen is the length of the header + data + CRC in bytes.
  */
-static void ipath_ib_rcv(void *arg, void *rhdr, void *data, u32 tlen)
+void ipath_ib_rcv(struct ipath_ibdev *dev, void *rhdr, void *data,
+		  u32 tlen)
 {
-	struct ipath_ibdev *dev = (struct ipath_ibdev *) arg;
 	struct ipath_ib_header *hdr = rhdr;
 	struct ipath_other_headers *ohdr;
 	struct ipath_qp *qp;
@@ -427,7 +411,7 @@ static void ipath_ib_rcv(void *arg, void *rhdr, void *data, u32 tlen)
 	lid = be16_to_cpu(hdr->lrh[1]);
 	if (lid < IPATH_MULTICAST_LID_BASE) {
 		lid &= ~((1 << (dev->mkeyprot_resv_lmc & 7)) - 1);
-		if (unlikely(lid != ipath_layer_get_lid(dev->dd))) {
+		if (unlikely(lid != dev->dd->ipath_lid)) {
 			dev->rcv_errors++;
 			goto bail;
 		}
@@ -495,9 +479,8 @@ bail:;
  * This is called from ipath_do_rcv_timer() at interrupt level to check for
  * QPs which need retransmits and to collect performance numbers.
  */
-static void ipath_ib_timer(void *arg)
+void ipath_ib_timer(struct ipath_ibdev *dev)
 {
-	struct ipath_ibdev *dev = (struct ipath_ibdev *) arg;
 	struct ipath_qp *resend = NULL;
 	struct list_head *last;
 	struct ipath_qp *qp;
@@ -539,19 +522,19 @@ static void ipath_ib_timer(void *arg)
 	if (dev->pma_sample_status == IB_PMA_SAMPLE_STATUS_STARTED &&
 	    --dev->pma_sample_start == 0) {
 		dev->pma_sample_status = IB_PMA_SAMPLE_STATUS_RUNNING;
-		ipath_layer_snapshot_counters(dev->dd, &dev->ipath_sword,
-					      &dev->ipath_rword,
-					      &dev->ipath_spkts,
-					      &dev->ipath_rpkts,
-					      &dev->ipath_xmit_wait);
+		ipath_snapshot_counters(dev->dd, &dev->ipath_sword,
+					&dev->ipath_rword,
+					&dev->ipath_spkts,
+					&dev->ipath_rpkts,
+					&dev->ipath_xmit_wait);
 	}
 	if (dev->pma_sample_status == IB_PMA_SAMPLE_STATUS_RUNNING) {
 		if (dev->pma_sample_interval == 0) {
 			u64 ta, tb, tc, td, te;
 
 			dev->pma_sample_status = IB_PMA_SAMPLE_STATUS_DONE;
-			ipath_layer_snapshot_counters(dev->dd, &ta, &tb,
-						      &tc, &td, &te);
+			ipath_snapshot_counters(dev->dd, &ta, &tb,
+						&tc, &td, &te);
 
 			dev->ipath_sword = ta - dev->ipath_sword;
 			dev->ipath_rword = tb - dev->ipath_rword;
@@ -581,6 +564,362 @@ static void ipath_ib_timer(void *arg)
 	}
 }
 
+static void update_sge(struct ipath_sge_state *ss, u32 length)
+{
+	struct ipath_sge *sge = &ss->sge;
+
+	sge->vaddr += length;
+	sge->length -= length;
+	sge->sge_length -= length;
+	if (sge->sge_length == 0) {
+		if (--ss->num_sge)
+			*sge = *ss->sg_list++;
+	} else if (sge->length == 0 && sge->mr != NULL) {
+		if (++sge->n >= IPATH_SEGSZ) {
+			if (++sge->m >= sge->mr->mapsz)
+				return;
+			sge->n = 0;
+		}
+		sge->vaddr = sge->mr->map[sge->m]->segs[sge->n].vaddr;
+		sge->length = sge->mr->map[sge->m]->segs[sge->n].length;
+	}
+}
+
+#ifdef __LITTLE_ENDIAN
+static inline u32 get_upper_bits(u32 data, u32 shift)
+{
+	return data >> shift;
+}
+
+static inline u32 set_upper_bits(u32 data, u32 shift)
+{
+	return data << shift;
+}
+
+static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off)
+{
+	data <<= ((sizeof(u32) - n) * BITS_PER_BYTE);
+	data >>= ((sizeof(u32) - n - off) * BITS_PER_BYTE);
+	return data;
+}
+#else
+static inline u32 get_upper_bits(u32 data, u32 shift)
+{
+	return data << shift;
+}
+
+static inline u32 set_upper_bits(u32 data, u32 shift)
+{
+	return data >> shift;
+}
+
+static inline u32 clear_upper_bytes(u32 data, u32 n, u32 off)
+{
+	data >>= ((sizeof(u32) - n) * BITS_PER_BYTE);
+	data <<= ((sizeof(u32) - n - off) * BITS_PER_BYTE);
+	return data;
+}
+#endif
+
+static void copy_io(u32 __iomem *piobuf, struct ipath_sge_state *ss,
+		    u32 length)
+{
+	u32 extra = 0;
+	u32 data = 0;
+	u32 last;
+
+	while (1) {
+		u32 len = ss->sge.length;
+		u32 off;
+
+		BUG_ON(len == 0);
+		if (len > length)
+			len = length;
+		if (len > ss->sge.sge_length)
+			len = ss->sge.sge_length;
+		/* If the source address is not aligned, try to align it. */
+		off = (unsigned long)ss->sge.vaddr & (sizeof(u32) - 1);
+		if (off) {
+			u32 *addr = (u32 *)((unsigned long)ss->sge.vaddr &
+					    ~(sizeof(u32) - 1));
+			u32 v = get_upper_bits(*addr, off * BITS_PER_BYTE);
+			u32 y;
+
+			y = sizeof(u32) - off;
+			if (len > y)
+				len = y;
+			if (len + extra >= sizeof(u32)) {
+				data |= set_upper_bits(v, extra *
+						       BITS_PER_BYTE);
+				len = sizeof(u32) - extra;
+				if (len == length) {
+					last = data;
+					break;
+				}
+				__raw_writel(data, piobuf);
+				piobuf++;
+				extra = 0;
+				data = 0;
+			} else {
+				/* Clear unused upper bytes */
+				data |= clear_upper_bytes(v, len, extra);
+				if (len == length) {
+					last = data;
+					break;
+				}
+				extra += len;
+			}
+		} else if (extra) {
+			/* Source address is aligned. */
+			u32 *addr = (u32 *) ss->sge.vaddr;
+			int shift = extra * BITS_PER_BYTE;
+			int ushift = 32 - shift;
+			u32 l = len;
+
+			while (l >= sizeof(u32)) {
+				u32 v = *addr;
+
+				data |= set_upper_bits(v, shift);
+				__raw_writel(data, piobuf);
+				data = get_upper_bits(v, ushift);
+				piobuf++;
+				addr++;
+				l -= sizeof(u32);
+			}
+			/*
+			 * We still have 'extra' number of bytes leftover.
+			 */
+			if (l) {
+				u32 v = *addr;
+
+				if (l + extra >= sizeof(u32)) {
+					data |= set_upper_bits(v, shift);
+					len -= l + extra - sizeof(u32);
+					if (len == length) {
+						last = data;
+						break;
+					}
+					__raw_writel(data, piobuf);
+					piobuf++;
+					extra = 0;
+					data = 0;
+				} else {
+					/* Clear unused upper bytes */
+					data |= clear_upper_bytes(v, l,
+								  extra);
+					if (len == length) {
+						last = data;
+						break;
+					}
+					extra += l;
+				}
+			} else if (len == length) {
+				last = data;
+				break;
+			}
+		} else if (len == length) {
+			u32 w;
+
+			/*
+			 * Need to round up for the last dword in the
+			 * packet.
+			 */
+			w = (len + 3) >> 2;
+			__iowrite32_copy(piobuf, ss->sge.vaddr, w - 1);
+			piobuf += w - 1;
+			last = ((u32 *) ss->sge.vaddr)[w - 1];
+			break;
+		} else {
+			u32 w = len >> 2;
+
+			__iowrite32_copy(piobuf, ss->sge.vaddr, w);
+			piobuf += w;
+
+			extra = len & (sizeof(u32) - 1);
+			if (extra) {
+				u32 v = ((u32 *) ss->sge.vaddr)[w];
+
+				/* Clear unused upper bytes */
+				data = clear_upper_bytes(v, extra, 0);
+			}
+		}
+		update_sge(ss, len);
+		length -= len;
+	}
+	/* Update address before sending packet. */
+	update_sge(ss, length);
+	/* must flush early everything before trigger word */
+	ipath_flush_wc();
+	__raw_writel(last, piobuf);
+	/* be sure trigger word is written */
+	ipath_flush_wc();
+}
+
+/**
+ * ipath_verbs_send - send a packet
+ * @dd: the infinipath device
+ * @hdrwords: the number of words in the header
+ * @hdr: the packet header
+ * @len: the length of the packet in bytes
+ * @ss: the SGE to send
+ */
+int ipath_verbs_send(struct ipath_devdata *dd, u32 hdrwords,
+		     u32 *hdr, u32 len, struct ipath_sge_state *ss)
+{
+	u32 __iomem *piobuf;
+	u32 plen;
+	int ret;
+
+	/* +1 is for the qword padding of pbc */
+	plen = hdrwords + ((len + 3) >> 2) + 1;
+	if (unlikely((plen << 2) > dd->ipath_ibmaxlen)) {
+		ipath_dbg("packet len 0x%x too long, failing\n", plen);
+		ret = -EINVAL;
+		goto bail;
+	}
+
+	/* Get a PIO buffer to use. */
+	piobuf = ipath_getpiobuf(dd, NULL);
+	if (unlikely(piobuf == NULL)) {
+		ret = -EBUSY;
+		goto bail;
+	}
+
+	/*
+	 * Write len to control qword, no flags.
+	 * We have to flush after the PBC for correctness on some cpus
+	 * or WC buffer can be written out of order.
+	 */
+	writeq(plen, piobuf);
+	ipath_flush_wc();
+	piobuf += 2;
+	if (len == 0) {
+		/*
+		 * If there is just the header portion, must flush before
+		 * writing last word of header for correctness, and after
+		 * the last header word (trigger word).
+		 */
+		__iowrite32_copy(piobuf, hdr, hdrwords - 1);
+		ipath_flush_wc();
+		__raw_writel(hdr[hdrwords - 1], piobuf + hdrwords - 1);
+		ipath_flush_wc();
+		ret = 0;
+		goto bail;
+	}
+
+	__iowrite32_copy(piobuf, hdr, hdrwords);
+	piobuf += hdrwords;
+
+	/* The common case is aligned and contained in one segment. */
+	if (likely(ss->num_sge == 1 && len <= ss->sge.length &&
+		   !((unsigned long)ss->sge.vaddr & (sizeof(u32) - 1)))) {
+		u32 w;
+		u32 *addr = (u32 *) ss->sge.vaddr;
+
+		/* Update address before sending packet. */
+		update_sge(ss, len);
+		/* Need to round up for the last dword in the packet. */
+		w = (len + 3) >> 2;
+		__iowrite32_copy(piobuf, addr, w - 1);
+		/* must flush early everything before trigger word */
+		ipath_flush_wc();
+		__raw_writel(addr[w - 1], piobuf + w - 1);
+		/* be sure trigger word is written */
+		ipath_flush_wc();
+		ret = 0;
+		goto bail;
+	}
+	copy_io(piobuf, ss, len);
+	ret = 0;
+
+bail:
+	return ret;
+}
+
+int ipath_snapshot_counters(struct ipath_devdata *dd, u64 *swords,
+			    u64 *rwords, u64 *spkts, u64 *rpkts,
+			    u64 *xmit_wait)
+{
+	int ret;
+
+	if (!(dd->ipath_flags & IPATH_INITTED)) {
+		/* no hardware, freeze, etc. */
+		ipath_dbg("unit %u not usable\n", dd->ipath_unit);
+		ret = -EINVAL;
+		goto bail;
+	}
+	*swords = ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordsendcnt);
+	*rwords = ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordrcvcnt);
+	*spkts = ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktsendcnt);
+	*rpkts = ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktrcvcnt);
+	*xmit_wait = ipath_snap_cntr(dd, dd->ipath_cregs->cr_sendstallcnt);
+
+	ret = 0;
+
+bail:
+	return ret;
+}
+
+/**
+ * ipath_get_counters - get various chip counters
+ * @dd: the infinipath device
+ * @cntrs: counters are placed here
+ *
+ * Return the counters needed by recv_pma_get_portcounters().
+ */
+int ipath_get_counters(struct ipath_devdata *dd,
+		       struct ipath_verbs_counters *cntrs)
+{
+	int ret;
+
+	if (!(dd->ipath_flags & IPATH_INITTED)) {
+		/* no hardware, freeze, etc. */
+		ipath_dbg("unit %u not usable\n", dd->ipath_unit);
+		ret = -EINVAL;
+		goto bail;
+	}
+	cntrs->symbol_error_counter =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_ibsymbolerrcnt);
+	cntrs->link_error_recovery_counter =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_iblinkerrrecovcnt);
+	/*
+	 * The link downed counter counts when the other side downs the
+	 * connection.  We add in the number of times we downed the link
+	 * due to local link integrity errors to compensate.
+	 */
+	cntrs->link_downed_counter =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_iblinkdowncnt);
+	cntrs->port_rcv_errors =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_rxdroppktcnt) +
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_rcvovflcnt) +
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_portovflcnt) +
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_err_rlencnt) +
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_invalidrlencnt) +
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_erricrccnt) +
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_errvcrccnt) +
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_errlpcrccnt) +
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_badformatcnt);
+	cntrs->port_rcv_remphys_errors =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_rcvebpcnt);
+	cntrs->port_xmit_discards =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_unsupvlcnt);
+	cntrs->port_xmit_data =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordsendcnt);
+	cntrs->port_rcv_data =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_wordrcvcnt);
+	cntrs->port_xmit_packets =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktsendcnt);
+	cntrs->port_rcv_packets =
+		ipath_snap_cntr(dd, dd->ipath_cregs->cr_pktrcvcnt);
+	cntrs->local_link_integrity_errors = dd->ipath_lli_errors;
+	cntrs->excessive_buffer_overrun_errors = 0; /* XXX */
+
+	ret = 0;
+
+bail:
+	return ret;
+}
+
 /**
  * ipath_ib_piobufavail - callback when a PIO buffer is available
  * @arg: the device pointer
@@ -591,9 +930,8 @@ static void ipath_ib_timer(void *arg)
  * QPs waiting for buffers (for now, just do a tasklet_hi_schedule and
  * return zero).
  */
-static int ipath_ib_piobufavail(void *arg)
+int ipath_ib_piobufavail(struct ipath_ibdev *dev)
 {
-	struct ipath_ibdev *dev = (struct ipath_ibdev *) arg;
 	struct ipath_qp *qp;
 	unsigned long flags;
 
@@ -624,14 +962,14 @@ static int ipath_query_device(struct ib_device *ibdev,
 		IB_DEVICE_BAD_QKEY_CNTR | IB_DEVICE_SHUTDOWN_PORT |
 		IB_DEVICE_SYS_IMAGE_GUID;
 	props->page_size_cap = PAGE_SIZE;
-	props->vendor_id = ipath_layer_get_vendorid(dev->dd);
-	props->vendor_part_id = ipath_layer_get_deviceid(dev->dd);
-	props->hw_ver = ipath_layer_get_pcirev(dev->dd);
+	props->vendor_id = dev->dd->ipath_vendorid;
+	props->vendor_part_id = dev->dd->ipath_deviceid;
+	props->hw_ver = dev->dd->ipath_pcirev;
 
 	props->sys_image_guid = dev->sys_image_guid;
 
 	props->max_mr_size = ~0ull;
-	props->max_qp = dev->qp_table.max;
+	props->max_qp = ib_ipath_max_qps;
 	props->max_qp_wr = ib_ipath_max_qp_wrs;
 	props->max_sge = ib_ipath_max_sges;
 	props->max_cq = ib_ipath_max_cqs;
@@ -647,7 +985,7 @@ static int ipath_query_device(struct ib_device *ibdev,
 	props->max_srq_sge = ib_ipath_max_srq_sges;
 	/* props->local_ca_ack_delay */
 	props->atomic_cap = IB_ATOMIC_HCA;
-	props->max_pkeys = ipath_layer_get_npkeys(dev->dd);
+	props->max_pkeys = ipath_get_npkeys(dev->dd);
 	props->max_mcast_grp = ib_ipath_max_mcast_grps;
 	props->max_mcast_qp_attach = ib_ipath_max_mcast_qp_attached;
 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
@@ -672,12 +1010,17 @@ const u8 ipath_cvt_physportstate[16] = {
 	[INFINIPATH_IBCS_LT_STATE_RECOVERIDLE] = 6,
 };
 
+u32 ipath_get_cr_errpkey(struct ipath_devdata *dd)
+{
+	return ipath_read_creg32(dd, dd->ipath_cregs->cr_errpkey);
+}
+
 static int ipath_query_port(struct ib_device *ibdev,
 			    u8 port, struct ib_port_attr *props)
 {
 	struct ipath_ibdev *dev = to_idev(ibdev);
 	enum ib_mtu mtu;
-	u16 lid = ipath_layer_get_lid(dev->dd);
+	u16 lid = dev->dd->ipath_lid;
 	u64 ibcstat;
 
 	memset(props, 0, sizeof(*props));
@@ -685,16 +1028,16 @@ static int ipath_query_port(struct ib_device *ibdev,
 	props->lmc = dev->mkeyprot_resv_lmc & 7;
 	props->sm_lid = dev->sm_lid;
 	props->sm_sl = dev->sm_sl;
-	ibcstat = ipath_layer_get_lastibcstat(dev->dd);
+	ibcstat = dev->dd->ipath_lastibcstat;
 	props->state = ((ibcstat >> 4) & 0x3) + 1;
 	/* See phys_state_show() */
 	props->phys_state = ipath_cvt_physportstate[
-		ipath_layer_get_lastibcstat(dev->dd) & 0xf];
+		dev->dd->ipath_lastibcstat & 0xf];
 	props->port_cap_flags = dev->port_cap_flags;
 	props->gid_tbl_len = 1;
 	props->max_msg_sz = 0x80000000;
-	props->pkey_tbl_len = ipath_layer_get_npkeys(dev->dd);
-	props->bad_pkey_cntr = ipath_layer_get_cr_errpkey(dev->dd) -
+	props->pkey_tbl_len = ipath_get_npkeys(dev->dd);
+	props->bad_pkey_cntr = ipath_get_cr_errpkey(dev->dd) -
 		dev->z_pkey_violations;
 	props->qkey_viol_cntr = dev->qkey_violations;
 	props->active_width = IB_WIDTH_4X;
@@ -704,7 +1047,7 @@ static int ipath_query_port(struct ib_device *ibdev,
 	props->init_type_reply = 0;
 
 	props->max_mtu = IB_MTU_4096;
-	switch (ipath_layer_get_ibmtu(dev->dd)) {
+	switch (dev->dd->ipath_ibmtu) {
 	case 4096:
 		mtu = IB_MTU_4096;
 		break;
@@ -763,7 +1106,7 @@ static int ipath_modify_port(struct ib_device *ibdev,
 	dev->port_cap_flags |= props->set_port_cap_mask;
 	dev->port_cap_flags &= ~props->clr_port_cap_mask;
 	if (port_modify_mask & IB_PORT_SHUTDOWN)
-		ipath_layer_set_linkstate(dev->dd, IPATH_IB_LINKDOWN);
+		ipath_set_linkstate(dev->dd, IPATH_IB_LINKDOWN);
 	if (port_modify_mask & IB_PORT_RESET_QKEY_CNTR)
 		dev->qkey_violations = 0;
 	return 0;
@@ -780,7 +1123,7 @@ static int ipath_query_gid(struct ib_device *ibdev, u8 port,
 		goto bail;
 	}
 	gid->global.subnet_prefix = dev->gid_prefix;
-	gid->global.interface_id = ipath_layer_get_guid(dev->dd);
+	gid->global.interface_id = dev->dd->ipath_guid;
 
 	ret = 0;
 
@@ -803,18 +1146,22 @@ static struct ib_pd *ipath_alloc_pd(struct ib_device *ibdev,
 	 * we allow allocations of more than we report for this value.
 	 */
 
-	if (dev->n_pds_allocated == ib_ipath_max_pds) {
+	pd = kmalloc(sizeof *pd, GFP_KERNEL);
+	if (!pd) {
 		ret = ERR_PTR(-ENOMEM);
 		goto bail;
 	}
 
-	pd = kmalloc(sizeof *pd, GFP_KERNEL);
-	if (!pd) {
+	spin_lock(&dev->n_pds_lock);
+	if (dev->n_pds_allocated == ib_ipath_max_pds) {
+		spin_unlock(&dev->n_pds_lock);
+		kfree(pd);
 		ret = ERR_PTR(-ENOMEM);
 		goto bail;
 	}
 
 	dev->n_pds_allocated++;
+	spin_unlock(&dev->n_pds_lock);
 
 	/* ib_alloc_pd() will initialize pd->ibpd. */
 	pd->user = udata != NULL;
@@ -830,7 +1177,9 @@ static int ipath_dealloc_pd(struct ib_pd *ibpd)
 	struct ipath_pd *pd = to_ipd(ibpd);
 	struct ipath_ibdev *dev = to_idev(ibpd->device);
 
+	spin_lock(&dev->n_pds_lock);
 	dev->n_pds_allocated--;
+	spin_unlock(&dev->n_pds_lock);
 
 	kfree(pd);
 
@@ -851,11 +1200,6 @@ static struct ib_ah *ipath_create_ah(struct ib_pd *pd,
 	struct ib_ah *ret;
 	struct ipath_ibdev *dev = to_idev(pd->device);
 
-	if (dev->n_ahs_allocated == ib_ipath_max_ahs) {
-		ret = ERR_PTR(-ENOMEM);
-		goto bail;
-	}
-
 	/* A multicast address requires a GRH (see ch. 8.4.1). */
 	if (ah_attr->dlid >= IPATH_MULTICAST_LID_BASE &&
 	    ah_attr->dlid != IPATH_PERMISSIVE_LID &&
@@ -881,7 +1225,16 @@ static struct ib_ah *ipath_create_ah(struct ib_pd *pd,
 		goto bail;
 	}
 
+	spin_lock(&dev->n_ahs_lock);
+	if (dev->n_ahs_allocated == ib_ipath_max_ahs) {
+		spin_unlock(&dev->n_ahs_lock);
+		kfree(ah);
+		ret = ERR_PTR(-ENOMEM);
+		goto bail;
+	}
+
 	dev->n_ahs_allocated++;
+	spin_unlock(&dev->n_ahs_lock);
 
 	/* ib_create_ah() will initialize ah->ibah. */
 	ah->attr = *ah_attr;
@@ -903,7 +1256,9 @@ static int ipath_destroy_ah(struct ib_ah *ibah)
 	struct ipath_ibdev *dev = to_idev(ibah->device);
 	struct ipath_ah *ah = to_iah(ibah);
 
+	spin_lock(&dev->n_ahs_lock);
 	dev->n_ahs_allocated--;
+	spin_unlock(&dev->n_ahs_lock);
 
 	kfree(ah);
 
@@ -919,25 +1274,50 @@ static int ipath_query_ah(struct ib_ah *ibah, struct ib_ah_attr *ah_attr)
 	return 0;
 }
 
+/**
+ * ipath_get_npkeys - return the size of the PKEY table for port 0
+ * @dd: the infinipath device
+ */
+unsigned ipath_get_npkeys(struct ipath_devdata *dd)
+{
+	return ARRAY_SIZE(dd->ipath_pd[0]->port_pkeys);
+}
+
+/**
+ * ipath_get_pkey - return the indexed PKEY from the port 0 PKEY table
+ * @dd: the infinipath device
+ * @index: the PKEY index
+ */
+unsigned ipath_get_pkey(struct ipath_devdata *dd, unsigned index)
+{
+	unsigned ret;
+
+	if (index >= ARRAY_SIZE(dd->ipath_pd[0]->port_pkeys))
+		ret = 0;
+	else
+		ret = dd->ipath_pd[0]->port_pkeys[index];
+
+	return ret;
+}
+
 static int ipath_query_pkey(struct ib_device *ibdev, u8 port, u16 index,
 			    u16 *pkey)
 {
 	struct ipath_ibdev *dev = to_idev(ibdev);
 	int ret;
 
-	if (index >= ipath_layer_get_npkeys(dev->dd)) {
+	if (index >= ipath_get_npkeys(dev->dd)) {
 		ret = -EINVAL;
 		goto bail;
 	}
 
-	*pkey = ipath_layer_get_pkey(dev->dd, index);
+	*pkey = ipath_get_pkey(dev->dd, index);
 	ret = 0;
 
 bail:
 	return ret;
 }
 
-
 /**
  * ipath_alloc_ucontext - allocate a ucontest
  * @ibdev: the infiniband device
@@ -970,26 +1350,91 @@ static int ipath_dealloc_ucontext(struct ib_ucontext *context)
 
 static int ipath_verbs_register_sysfs(struct ib_device *dev);
 
+static void __verbs_timer(unsigned long arg)
+{
+	struct ipath_devdata *dd = (struct ipath_devdata *) arg;
+
+	/*
+	 * If port 0 receive packet interrupts are not available, or
+	 * can be missed, poll the receive queue
+	 */
+	if (dd->ipath_flags & IPATH_POLL_RX_INTR)
+		ipath_kreceive(dd);
+
+	/* Handle verbs layer timeouts. */
+	ipath_ib_timer(dd->verbs_dev);
+
+	mod_timer(&dd->verbs_timer, jiffies + 1);
+}
+
+static int enable_timer(struct ipath_devdata *dd)
+{
+	/*
+	 * Early chips had a design flaw where the chip and kernel idea
+	 * of the tail register don't always agree, and therefore we won't
+	 * get an interrupt on the next packet received.
+	 * If the board supports per packet receive interrupts, use it.
+	 * Otherwise, the timer function periodically checks for packets
+	 * to cover this case.
+	 * Either way, the timer is needed for verbs layer related
+	 * processing.
+	 */
+	if (dd->ipath_flags & IPATH_GPIO_INTR) {
+		ipath_write_kreg(dd, dd->ipath_kregs->kr_debugportselect,
+				 0x2074076542310ULL);
+		/* Enable GPIO bit 2 interrupt */
+		ipath_write_kreg(dd, dd->ipath_kregs->kr_gpio_mask,
+				 (u64) (1 << 2));
+	}
+
+	init_timer(&dd->verbs_timer);
+	dd->verbs_timer.function = __verbs_timer;
+	dd->verbs_timer.data = (unsigned long)dd;
+	dd->verbs_timer.expires = jiffies + 1;
+	add_timer(&dd->verbs_timer);
+
+	return 0;
+}
+
+static int disable_timer(struct ipath_devdata *dd)
+{
+	/* Disable GPIO bit 2 interrupt */
+	if (dd->ipath_flags & IPATH_GPIO_INTR)
+		ipath_write_kreg(dd, dd->ipath_kregs->kr_gpio_mask, 0);
+
+	del_timer_sync(&dd->verbs_timer);
+
+	return 0;
+}
+
 /**
  * ipath_register_ib_device - register our device with the infiniband core
- * @unit: the device number to register
  * @dd: the device data structure
  * Return the allocated ipath_ibdev pointer or NULL on error.
  */
-static void *ipath_register_ib_device(int unit, struct ipath_devdata *dd)
+int ipath_register_ib_device(struct ipath_devdata *dd)
 {
-	struct ipath_layer_counters cntrs;
+	struct ipath_verbs_counters cntrs;
 	struct ipath_ibdev *idev;
 	struct ib_device *dev;
 	int ret;
 
 	idev = (struct ipath_ibdev *)ib_alloc_device(sizeof *idev);
-	if (idev == NULL)
+	if (idev == NULL) {
+		ret = -ENOMEM;
 		goto bail;
+	}
 
 	dev = &idev->ibdev;
 
 	/* Only need to initialize non-zero fields. */
+	spin_lock_init(&idev->n_pds_lock);
+	spin_lock_init(&idev->n_ahs_lock);
+	spin_lock_init(&idev->n_cqs_lock);
+	spin_lock_init(&idev->n_qps_lock);
+	spin_lock_init(&idev->n_srqs_lock);
+	spin_lock_init(&idev->n_mcast_grps_lock);
+
 	spin_lock_init(&idev->qp_table.lock);
 	spin_lock_init(&idev->lk_table.lock);
 	idev->sm_lid = __constant_be16_to_cpu(IB_LID_PERMISSIVE);
@@ -1030,7 +1475,7 @@ static void *ipath_register_ib_device(int unit, struct ipath_devdata *dd)
 	idev->link_width_enabled = 3;	/* 1x or 4x */
 
 	/* Snapshot current HW counters to "clear" them. */
-	ipath_layer_get_counters(dd, &cntrs);
+	ipath_get_counters(dd, &cntrs);
 	idev->z_symbol_error_counter = cntrs.symbol_error_counter;
 	idev->z_link_error_recovery_counter =
 		cntrs.link_error_recovery_counter;
@@ -1054,14 +1499,14 @@ static void *ipath_register_ib_device(int unit, struct ipath_devdata *dd)
 	 * device types in the system, we can't be sure this is unique.
 	 */
 	if (!sys_image_guid)
-		sys_image_guid = ipath_layer_get_guid(dd);
+		sys_image_guid = dd->ipath_guid;
 	idev->sys_image_guid = sys_image_guid;
-	idev->ib_unit = unit;
+	idev->ib_unit = dd->ipath_unit;
 	idev->dd = dd;
 
 	strlcpy(dev->name, "ipath%d", IB_DEVICE_NAME_MAX);
 	dev->owner = THIS_MODULE;
-	dev->node_guid = ipath_layer_get_guid(dd);
+	dev->node_guid = dd->ipath_guid;
 	dev->uverbs_abi_ver = IPATH_UVERBS_ABI_VERSION;
 	dev->uverbs_cmd_mask =
 		(1ull << IB_USER_VERBS_CMD_GET_CONTEXT)		|
@@ -1093,9 +1538,9 @@ static void *ipath_register_ib_device(int unit, struct ipath_devdata *dd)
 		(1ull << IB_USER_VERBS_CMD_QUERY_SRQ)		|
 		(1ull << IB_USER_VERBS_CMD_DESTROY_SRQ)		|
 		(1ull << IB_USER_VERBS_CMD_POST_SRQ_RECV);
-	dev->node_type = IB_NODE_CA;
+	dev->node_type = RDMA_NODE_IB_CA;
 	dev->phys_port_cnt = 1;
-	dev->dma_device = ipath_layer_get_device(dd);
+	dev->dma_device = &dd->pcidev->dev;
 	dev->class_dev.dev = dev->dma_device;
 	dev->query_device = ipath_query_device;
 	dev->modify_device = ipath_modify_device;
@@ -1137,9 +1582,10 @@ static void *ipath_register_ib_device(int unit, struct ipath_devdata *dd)
 	dev->attach_mcast = ipath_multicast_attach;
 	dev->detach_mcast = ipath_multicast_detach;
 	dev->process_mad = ipath_process_mad;
+	dev->mmap = ipath_mmap;
 
 	snprintf(dev->node_desc, sizeof(dev->node_desc),
-		 IPATH_IDSTR " %s kernel_SMA", system_utsname.nodename);
+		 IPATH_IDSTR " %s", system_utsname.nodename);
 
 	ret = ib_register_device(dev);
 	if (ret)
@@ -1148,7 +1594,7 @@ static void *ipath_register_ib_device(int unit, struct ipath_devdata *dd)
 	if (ipath_verbs_register_sysfs(dev))
 		goto err_class;
 
-	ipath_layer_enable_timer(dd);
+	enable_timer(dd);
 
 	goto bail;
 
@@ -1160,37 +1606,32 @@ err_lk:
 	kfree(idev->qp_table.table);
 err_qp:
 	ib_dealloc_device(dev);
-	_VERBS_ERROR("ib_ipath%d cannot register verbs (%d)!\n",
-		     unit, -ret);
+	ipath_dev_err(dd, "cannot register verbs: %d!\n", -ret);
 	idev = NULL;
 
 bail:
-	return idev;
+	dd->verbs_dev = idev;
+	return ret;
 }
 
-static void ipath_unregister_ib_device(void *arg)
+void ipath_unregister_ib_device(struct ipath_ibdev *dev)
 {
-	struct ipath_ibdev *dev = (struct ipath_ibdev *) arg;
 	struct ib_device *ibdev = &dev->ibdev;
 
-	ipath_layer_disable_timer(dev->dd);
+	disable_timer(dev->dd);
 
 	ib_unregister_device(ibdev);
 
 	if (!list_empty(&dev->pending[0]) ||
 	    !list_empty(&dev->pending[1]) ||
 	    !list_empty(&dev->pending[2]))
-		_VERBS_ERROR("ipath%d pending list not empty!\n",
-			     dev->ib_unit);
+		ipath_dev_err(dev->dd, "pending list not empty!\n");
 	if (!list_empty(&dev->piowait))
-		_VERBS_ERROR("ipath%d piowait list not empty!\n",
-			     dev->ib_unit);
+		ipath_dev_err(dev->dd, "piowait list not empty!\n");
 	if (!list_empty(&dev->rnrwait))
-		_VERBS_ERROR("ipath%d rnrwait list not empty!\n",
-			     dev->ib_unit);
+		ipath_dev_err(dev->dd, "rnrwait list not empty!\n");
 	if (!ipath_mcast_tree_empty())
-		_VERBS_ERROR("ipath%d multicast table memory leak!\n",
-			     dev->ib_unit);
+		ipath_dev_err(dev->dd, "multicast table memory leak!\n");
 	/*
 	 * Note that ipath_unregister_ib_device() can be called before all
 	 * the QPs are destroyed!
@@ -1201,25 +1642,12 @@ static void ipath_unregister_ib_device(void *arg)
 	ib_dealloc_device(ibdev);
 }
 
-static int __init ipath_verbs_init(void)
-{
-	return ipath_verbs_register(ipath_register_ib_device,
-				    ipath_unregister_ib_device,
-				    ipath_ib_piobufavail, ipath_ib_rcv,
-				    ipath_ib_timer);
-}
-
-static void __exit ipath_verbs_cleanup(void)
-{
-	ipath_verbs_unregister();
-}
-
 static ssize_t show_rev(struct class_device *cdev, char *buf)
 {
 	struct ipath_ibdev *dev =
 		container_of(cdev, struct ipath_ibdev, ibdev.class_dev);
 
-	return sprintf(buf, "%x\n", ipath_layer_get_pcirev(dev->dd));
+	return sprintf(buf, "%x\n", dev->dd->ipath_pcirev);
 }
 
 static ssize_t show_hca(struct class_device *cdev, char *buf)
@@ -1228,7 +1656,7 @@ static ssize_t show_hca(struct class_device *cdev, char *buf)
 		container_of(cdev, struct ipath_ibdev, ibdev.class_dev);
 	int ret;
 
-	ret = ipath_layer_get_boardname(dev->dd, buf, 128);
+	ret = dev->dd->ipath_f_get_boardname(dev->dd, buf, 128);
 	if (ret < 0)
 		goto bail;
 	strcat(buf, "\n");
@@ -1305,6 +1733,3 @@ static int ipath_verbs_register_sysfs(struct ib_device *dev)
 bail:
 	return ret;
 }
-
-module_init(ipath_verbs_init);
-module_exit(ipath_verbs_cleanup);
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs.h b/drivers/infiniband/hw/ipath/ipath_verbs.h
index 2df6847..09bbb3f 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs.h
+++ b/drivers/infiniband/hw/ipath/ipath_verbs.h
@@ -38,10 +38,10 @@
 #include <linux/spinlock.h>
 #include <linux/kernel.h>
 #include <linux/interrupt.h>
+#include <linux/kref.h>
 #include <rdma/ib_pack.h>
 
 #include "ipath_layer.h"
-#include "verbs_debug.h"
 
 #define QPN_MAX                 (1 << 24)
 #define QPNMAP_ENTRIES          (QPN_MAX / PAGE_SIZE / BITS_PER_BYTE)
@@ -50,7 +50,7 @@
  * Increment this value if any changes that break userspace ABI
  * compatibility are made.
  */
-#define IPATH_UVERBS_ABI_VERSION       1
+#define IPATH_UVERBS_ABI_VERSION       2
 
 /*
  * Define an ib_cq_notify value that is not valid so we know when CQ
@@ -152,19 +152,6 @@ struct ipath_mcast {
 	int n_attached;
 };
 
-/* Memory region */
-struct ipath_mr {
-	struct ib_mr ibmr;
-	struct ipath_mregion mr;	/* must be last */
-};
-
-/* Fast memory region */
-struct ipath_fmr {
-	struct ib_fmr ibfmr;
-	u8 page_shift;
-	struct ipath_mregion mr;	/* must be last */
-};
-
 /* Protection domain */
 struct ipath_pd {
 	struct ib_pd ibpd;
@@ -178,58 +165,89 @@ struct ipath_ah {
 };
 
 /*
- * Quick description of our CQ/QP locking scheme:
- *
- * We have one global lock that protects dev->cq/qp_table.  Each
- * struct ipath_cq/qp also has its own lock.  An individual qp lock
- * may be taken inside of an individual cq lock.  Both cqs attached to
- * a qp may be locked, with the send cq locked first.  No other
- * nesting should be done.
- *
- * Each struct ipath_cq/qp also has an atomic_t ref count.  The
- * pointer from the cq/qp_table to the struct counts as one reference.
- * This reference also is good for access through the consumer API, so
- * modifying the CQ/QP etc doesn't need to take another reference.
- * Access because of a completion being polled does need a reference.
- *
- * Finally, each struct ipath_cq/qp has a wait_queue_head_t for the
- * destroy function to sleep on.
- *
- * This means that access from the consumer API requires nothing but
- * taking the struct's lock.
- *
- * Access because of a completion event should go as follows:
- * - lock cq/qp_table and look up struct
- * - increment ref count in struct
- * - drop cq/qp_table lock
- * - lock struct, do your thing, and unlock struct
- * - decrement ref count; if zero, wake up waiters
- *
- * To destroy a CQ/QP, we can do the following:
- * - lock cq/qp_table, remove pointer, unlock cq/qp_table lock
- * - decrement ref count
- * - wait_event until ref count is zero
- *
- * It is the consumer's responsibilty to make sure that no QP
- * operations (WQE posting or state modification) are pending when the
- * QP is destroyed.  Also, the consumer must make sure that calls to
- * qp_modify are serialized.
- *
- * Possible optimizations (wait for profile data to see if/where we
- * have locks bouncing between CPUs):
- * - split cq/qp table lock into n separate (cache-aligned) locks,
- *   indexed (say) by the page in the table
+ * This structure is used by ipath_mmap() to validate an offset
+ * when an mmap() request is made.  The vm_area_struct then uses
+ * this as its vm_private_data.
+ */
+struct ipath_mmap_info {
+	struct ipath_mmap_info *next;
+	struct ib_ucontext *context;
+	void *obj;
+	struct kref ref;
+	unsigned size;
+	unsigned mmap_cnt;
+};
+
+/*
+ * This structure is used to contain the head pointer, tail pointer,
+ * and completion queue entries as a single memory allocation so
+ * it can be mmap'ed into user space.
  */
+struct ipath_cq_wc {
+	u32 head;		/* index of next entry to fill */
+	u32 tail;		/* index of next ib_poll_cq() entry */
+	struct ib_wc queue[1];	/* this is actually size ibcq.cqe + 1 */
+};
 
+/*
+ * The completion queue structure.
+ */
 struct ipath_cq {
 	struct ib_cq ibcq;
 	struct tasklet_struct comptask;
 	spinlock_t lock;
 	u8 notify;
 	u8 triggered;
-	u32 head;		/* new records added to the head */
-	u32 tail;		/* poll_cq() reads from here. */
-	struct ib_wc *queue;	/* this is actually ibcq.cqe + 1 */
+	struct ipath_cq_wc *queue;
+	struct ipath_mmap_info *ip;
+};
+
+/*
+ * A segment is a linear region of low physical memory.
+ * XXX Maybe we should use phys addr here and kmap()/kunmap().
+ * Used by the verbs layer.
+ */
+struct ipath_seg {
+	void *vaddr;
+	size_t length;
+};
+
+/* The number of ipath_segs that fit in a page. */
+#define IPATH_SEGSZ     (PAGE_SIZE / sizeof (struct ipath_seg))
+
+struct ipath_segarray {
+	struct ipath_seg segs[IPATH_SEGSZ];
+};
+
+struct ipath_mregion {
+	u64 user_base;		/* User's address for this region */
+	u64 iova;		/* IB start address of this region */
+	size_t length;
+	u32 lkey;
+	u32 offset;		/* offset (bytes) to start of region */
+	int access_flags;
+	u32 max_segs;		/* number of ipath_segs in all the arrays */
+	u32 mapsz;		/* size of the map array */
+	struct ipath_segarray *map[0];	/* the segments */
+};
+
+/*
+ * These keep track of the copy progress within a memory region.
+ * Used by the verbs layer.
+ */
+struct ipath_sge {
+	struct ipath_mregion *mr;
+	void *vaddr;		/* current pointer into the segment */
+	u32 sge_length;		/* length of the SGE */
+	u32 length;		/* remaining length of the segment */
+	u16 m;			/* current index: mr->map[m] */
+	u16 n;			/* current index: mr->map[m]->segs[n] */
+};
+
+/* Memory region */
+struct ipath_mr {
+	struct ib_mr ibmr;
+	struct ipath_mregion mr;	/* must be last */
 };
 
 /*
@@ -248,32 +266,50 @@ struct ipath_swqe {
 
 /*
  * Receive work request queue entry.
- * The size of the sg_list is determined when the QP is created and stored
- * in qp->r_max_sge.
+ * The size of the sg_list is determined when the QP (or SRQ) is created
+ * and stored in qp->r_rq.max_sge (or srq->rq.max_sge).
  */
 struct ipath_rwqe {
 	u64 wr_id;
-	u32 length;		/* total length of data in sg_list */
 	u8 num_sge;
-	struct ipath_sge sg_list[0];
+	struct ib_sge sg_list[0];
 };
 
-struct ipath_rq {
-	spinlock_t lock;
+/*
+ * This structure is used to contain the head pointer, tail pointer,
+ * and receive work queue entries as a single memory allocation so
+ * it can be mmap'ed into user space.
+ * Note that the wq array elements are variable size so you can't
+ * just index into the array to get the N'th element;
+ * use get_rwqe_ptr() instead.
+ */
+struct ipath_rwq {
 	u32 head;		/* new work requests posted to the head */
 	u32 tail;		/* receives pull requests from here. */
+	struct ipath_rwqe wq[0];
+};
+
+struct ipath_rq {
+	struct ipath_rwq *wq;
+	spinlock_t lock;
 	u32 size;		/* size of RWQE array */
 	u8 max_sge;
-	struct ipath_rwqe *wq;	/* RWQE array */
 };
 
 struct ipath_srq {
 	struct ib_srq ibsrq;
 	struct ipath_rq rq;
+	struct ipath_mmap_info *ip;
 	/* send signal when number of RWQEs < limit */
 	u32 limit;
 };
 
+struct ipath_sge_state {
+	struct ipath_sge *sg_list;      /* next SGE to be used if any */
+	struct ipath_sge sge;   /* progress state for the current SGE */
+	u8 num_sge;
+};
+
 /*
  * Variables prefixed with s_ are for the requester (sender).
  * Variables prefixed with r_ are for the responder (receiver).
@@ -293,6 +329,7 @@ struct ipath_qp {
 	atomic_t refcount;
 	wait_queue_head_t wait;
 	struct tasklet_struct s_task;
+	struct ipath_mmap_info *ip;
 	struct ipath_sge_state *s_cur_sge;
 	struct ipath_sge_state s_sge;	/* current send request data */
 	/* current RDMA read send data */
@@ -334,6 +371,7 @@ struct ipath_qp {
 	u8 s_retry;		/* requester retry counter */
 	u8 s_rnr_retry;		/* requester RNR retry counter */
 	u8 s_pkey_index;	/* PKEY index to use */
+	u8 timeout;		/* Timeout for this QP */
 	enum ib_mtu path_mtu;
 	u32 remote_qpn;
 	u32 qkey;		/* QKEY for this QP (for UD or RD) */
@@ -345,7 +383,8 @@ struct ipath_qp {
 	u32 s_ssn;		/* SSN of tail entry */
 	u32 s_lsn;		/* limit sequence number (credit) */
 	struct ipath_swqe *s_wq;	/* send work queue */
-	struct ipath_rq r_rq;	/* receive work queue */
+	struct ipath_rq r_rq;		/* receive work queue */
+	struct ipath_sge r_sg_list[0];	/* verified SGEs */
 };
 
 /*
@@ -369,15 +408,15 @@ static inline struct ipath_swqe *get_swqe_ptr(struct ipath_qp *qp,
 
 /*
  * Since struct ipath_rwqe is not a fixed size, we can't simply index into
- * struct ipath_rq.wq.  This function does the array index computation.
+ * struct ipath_rwq.wq.  This function does the array index computation.
  */
 static inline struct ipath_rwqe *get_rwqe_ptr(struct ipath_rq *rq,
 					      unsigned n)
 {
 	return (struct ipath_rwqe *)
-		((char *) rq->wq +
+		((char *) rq->wq->wq +
 		 (sizeof(struct ipath_rwqe) +
-		  rq->max_sge * sizeof(struct ipath_sge)) * n);
+		  rq->max_sge * sizeof(struct ib_sge)) * n);
 }
 
 /*
@@ -417,6 +456,7 @@ struct ipath_ibdev {
 	struct ib_device ibdev;
 	struct list_head dev_list;
 	struct ipath_devdata *dd;
+	struct ipath_mmap_info *pending_mmaps;
 	int ib_unit;		/* This is the device number */
 	u16 sm_lid;		/* in host order */
 	u8 sm_sl;
@@ -435,11 +475,20 @@ struct ipath_ibdev {
 	__be64 sys_image_guid;	/* in network order */
 	__be64 gid_prefix;	/* in network order */
 	__be64 mkey;
+
 	u32 n_pds_allocated;	/* number of PDs allocated for device */
+	spinlock_t n_pds_lock;
 	u32 n_ahs_allocated;	/* number of AHs allocated for device */
+	spinlock_t n_ahs_lock;
 	u32 n_cqs_allocated;	/* number of CQs allocated for device */
+	spinlock_t n_cqs_lock;
+	u32 n_qps_allocated;	/* number of QPs allocated for device */
+	spinlock_t n_qps_lock;
 	u32 n_srqs_allocated;	/* number of SRQs allocated for device */
+	spinlock_t n_srqs_lock;
 	u32 n_mcast_grps_allocated; /* number of mcast groups allocated */
+	spinlock_t n_mcast_grps_lock;
+
 	u64 ipath_sword;	/* total dwords sent (sample result) */
 	u64 ipath_rword;	/* total dwords received (sample result) */
 	u64 ipath_spkts;	/* total packets sent (sample result) */
@@ -494,8 +543,19 @@ struct ipath_ibdev {
 	struct ipath_opcode_stats opstats[128];
 };
 
-struct ipath_ucontext {
-	struct ib_ucontext ibucontext;
+struct ipath_verbs_counters {
+	u64 symbol_error_counter;
+	u64 link_error_recovery_counter;
+	u64 link_downed_counter;
+	u64 port_rcv_errors;
+	u64 port_rcv_remphys_errors;
+	u64 port_xmit_discards;
+	u64 port_xmit_data;
+	u64 port_rcv_data;
+	u64 port_xmit_packets;
+	u64 port_rcv_packets;
+	u32 local_link_integrity_errors;
+	u32 excessive_buffer_overrun_errors;
 };
 
 static inline struct ipath_mr *to_imr(struct ib_mr *ibmr)
@@ -503,11 +563,6 @@ static inline struct ipath_mr *to_imr(struct ib_mr *ibmr)
 	return container_of(ibmr, struct ipath_mr, ibmr);
 }
 
-static inline struct ipath_fmr *to_ifmr(struct ib_fmr *ibfmr)
-{
-	return container_of(ibfmr, struct ipath_fmr, ibfmr);
-}
-
 static inline struct ipath_pd *to_ipd(struct ib_pd *ibpd)
 {
 	return container_of(ibpd, struct ipath_pd, ibpd);
@@ -545,12 +600,6 @@ int ipath_process_mad(struct ib_device *ibdev,
 		      struct ib_grh *in_grh,
 		      struct ib_mad *in_mad, struct ib_mad *out_mad);
 
-static inline struct ipath_ucontext *to_iucontext(struct ib_ucontext
-						  *ibucontext)
-{
-	return container_of(ibucontext, struct ipath_ucontext, ibucontext);
-}
-
 /*
  * Compare the lower 24 bits of the two values.
  * Returns an integer <, ==, or > than zero.
@@ -562,6 +611,13 @@ static inline int ipath_cmp24(u32 a, u32 b)
 
 struct ipath_mcast *ipath_mcast_find(union ib_gid *mgid);
 
+int ipath_snapshot_counters(struct ipath_devdata *dd, u64 *swords,
+			    u64 *rwords, u64 *spkts, u64 *rpkts,
+			    u64 *xmit_wait);
+
+int ipath_get_counters(struct ipath_devdata *dd,
+		       struct ipath_verbs_counters *cntrs);
+
 int ipath_multicast_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid);
 
 int ipath_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid);
@@ -579,7 +635,7 @@ struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
 int ipath_destroy_qp(struct ib_qp *ibqp);
 
 int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
-		    int attr_mask);
+		    int attr_mask, struct ib_udata *udata);
 
 int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
 		   int attr_mask, struct ib_qp_init_attr *init_attr);
@@ -592,6 +648,9 @@ void ipath_sqerror_qp(struct ipath_qp *qp, struct ib_wc *wc);
 
 void ipath_get_credit(struct ipath_qp *qp, u32 aeth);
 
+int ipath_verbs_send(struct ipath_devdata *dd, u32 hdrwords,
+		     u32 *hdr, u32 len, struct ipath_sge_state *ss);
+
 void ipath_cq_enter(struct ipath_cq *cq, struct ib_wc *entry, int sig);
 
 int ipath_rkey_ok(struct ipath_ibdev *dev, struct ipath_sge_state *ss,
@@ -638,7 +697,8 @@ struct ib_srq *ipath_create_srq(struct ib_pd *ibpd,
 				struct ib_udata *udata);
 
 int ipath_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
-		     enum ib_srq_attr_mask attr_mask);
+		     enum ib_srq_attr_mask attr_mask,
+		     struct ib_udata *udata);
 
 int ipath_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr);
 
@@ -680,6 +740,10 @@ int ipath_unmap_fmr(struct list_head *fmr_list);
 
 int ipath_dealloc_fmr(struct ib_fmr *ibfmr);
 
+void ipath_release_mmap_info(struct kref *ref);
+
+int ipath_mmap(struct ib_ucontext *context, struct vm_area_struct *vma);
+
 void ipath_no_bufs_available(struct ipath_qp *qp, struct ipath_ibdev *dev);
 
 void ipath_insert_rnr_queue(struct ipath_qp *qp);
@@ -700,6 +764,22 @@ int ipath_make_rc_req(struct ipath_qp *qp, struct ipath_other_headers *ohdr,
 int ipath_make_uc_req(struct ipath_qp *qp, struct ipath_other_headers *ohdr,
 		      u32 pmtu, u32 *bth0p, u32 *bth2p);
 
+int ipath_register_ib_device(struct ipath_devdata *);
+
+void ipath_unregister_ib_device(struct ipath_ibdev *);
+
+void ipath_ib_rcv(struct ipath_ibdev *, void *, void *, u32);
+
+int ipath_ib_piobufavail(struct ipath_ibdev *);
+
+void ipath_ib_timer(struct ipath_ibdev *);
+
+unsigned ipath_get_npkeys(struct ipath_devdata *);
+
+u32 ipath_get_cr_errpkey(struct ipath_devdata *);
+
+unsigned ipath_get_pkey(struct ipath_devdata *, unsigned);
+
 extern const enum ib_wc_opcode ib_ipath_wc_opcode[];
 
 extern const u8 ipath_cvt_physportstate[];
@@ -714,6 +794,8 @@ extern unsigned int ib_ipath_max_cqs;
 
 extern unsigned int ib_ipath_max_qp_wrs;
 
+extern unsigned int ib_ipath_max_qps;
+
 extern unsigned int ib_ipath_max_sges;
 
 extern unsigned int ib_ipath_max_mcast_grps;
diff --git a/drivers/infiniband/hw/ipath/ipath_verbs_mcast.c b/drivers/infiniband/hw/ipath/ipath_verbs_mcast.c
index ee0e1d9..085e28b 100644
--- a/drivers/infiniband/hw/ipath/ipath_verbs_mcast.c
+++ b/drivers/infiniband/hw/ipath/ipath_verbs_mcast.c
@@ -207,12 +207,17 @@ static int ipath_mcast_add(struct ipath_ibdev *dev,
 		goto bail;
 	}
 
+	spin_lock(&dev->n_mcast_grps_lock);
 	if (dev->n_mcast_grps_allocated == ib_ipath_max_mcast_grps) {
+		spin_unlock(&dev->n_mcast_grps_lock);
 		ret = ENOMEM;
 		goto bail;
 	}
 
 	dev->n_mcast_grps_allocated++;
+	spin_unlock(&dev->n_mcast_grps_lock);
+
+	mcast->n_attached++;
 
 	list_add_tail_rcu(&mqp->list, &mcast->qp_list);
 
@@ -343,7 +348,9 @@ int ipath_multicast_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
 		atomic_dec(&mcast->refcount);
 		wait_event(mcast->wait, !atomic_read(&mcast->refcount));
 		ipath_mcast_free(mcast);
+		spin_lock(&dev->n_mcast_grps_lock);
 		dev->n_mcast_grps_allocated--;
+		spin_unlock(&dev->n_mcast_grps_lock);
 	}
 
 	ret = 0;
diff --git a/drivers/infiniband/hw/ipath/ipath_wc_ppc64.c b/drivers/infiniband/hw/ipath/ipath_wc_ppc64.c
new file mode 100644
index 0000000..036fde6
--- /dev/null
+++ b/drivers/infiniband/hw/ipath/ipath_wc_ppc64.c
@@ -0,0 +1,52 @@
+/*
+ * Copyright (c) 2006 QLogic, Inc. All rights reserved.
+ *
+ * This software is available to you under a choice of one of two
+ * licenses.  You may choose to be licensed under the terms of the GNU
+ * General Public License (GPL) Version 2, available from the file
+ * COPYING in the main directory of this source tree, or the
+ * OpenIB.org BSD license below:
+ *
+ *     Redistribution and use in source and binary forms, with or
+ *     without modification, are permitted provided that the following
+ *     conditions are met:
+ *
+ *      - Redistributions of source code must retain the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer.
+ *
+ *      - Redistributions in binary form must reproduce the above
+ *        copyright notice, this list of conditions and the following
+ *        disclaimer in the documentation and/or other materials
+ *        provided with the distribution.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+/*
+ * This file is conditionally built on PowerPC only.  Otherwise weak symbol
+ * versions of the functions exported from here are used.
+ */
+
+#include "ipath_kernel.h"
+
+/**
+ * ipath_unordered_wc - indicate whether write combining is ordered
+ *
+ * PowerPC systems (at least those in the 970 processor family)
+ * write partially filled store buffers in address order, but will write
+ * completely filled store buffers in "random" order, and therefore must
+ * have serialization for correctness with current InfiniPath chips.
+ *
+ */
+int ipath_unordered_wc(void)
+{
+	return 1;
+}
diff --git a/drivers/infiniband/hw/ipath/verbs_debug.h b/drivers/infiniband/hw/ipath/verbs_debug.h
deleted file mode 100644
index 6186676..0000000
--- a/drivers/infiniband/hw/ipath/verbs_debug.h
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright (c) 2006 QLogic, Inc. All rights reserved.
- * Copyright (c) 2003, 2004, 2005, 2006 PathScale, Inc. All rights reserved.
- *
- * This software is available to you under a choice of one of two
- * licenses.  You may choose to be licensed under the terms of the GNU
- * General Public License (GPL) Version 2, available from the file
- * COPYING in the main directory of this source tree, or the
- * OpenIB.org BSD license below:
- *
- *     Redistribution and use in source and binary forms, with or
- *     without modification, are permitted provided that the following
- *     conditions are met:
- *
- *      - Redistributions of source code must retain the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer.
- *
- *      - Redistributions in binary form must reproduce the above
- *        copyright notice, this list of conditions and the following
- *        disclaimer in the documentation and/or other materials
- *        provided with the distribution.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
- * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
- * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
- * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
- * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- * SOFTWARE.
- */
-
-#ifndef _VERBS_DEBUG_H
-#define _VERBS_DEBUG_H
-
-/*
- * This file contains tracing code for the ib_ipath kernel module.
- */
-#ifndef _VERBS_DEBUGGING	/* tracing enabled or not */
-#define _VERBS_DEBUGGING 1
-#endif
-
-extern unsigned ib_ipath_debug;
-
-#define _VERBS_ERROR(fmt,...) \
-	do { \
-		printk(KERN_ERR "%s: " fmt, "ib_ipath", ##__VA_ARGS__); \
-	} while(0)
-
-#define _VERBS_UNIT_ERROR(unit,fmt,...) \
-	do { \
-		printk(KERN_ERR "%s: " fmt, "ib_ipath", ##__VA_ARGS__); \
-	} while(0)
-
-#if _VERBS_DEBUGGING
-
-/*
- * Mask values for debugging.  The scheme allows us to compile out any
- * of the debug tracing stuff, and if compiled in, to enable or
- * disable dynamically.
- * This can be set at modprobe time also:
- *      modprobe ib_path ib_ipath_debug=3
- */
-
-#define __VERBS_INFO        0x1	/* generic low verbosity stuff */
-#define __VERBS_DBG         0x2	/* generic debug */
-#define __VERBS_VDBG        0x4	/* verbose debug */
-#define __VERBS_SMADBG      0x8000	/* sma packet debug */
-
-#define _VERBS_INFO(fmt,...) \
-	do { \
-		if (unlikely(ib_ipath_debug&__VERBS_INFO)) \
-			printk(KERN_INFO "%s: " fmt,"ib_ipath", \
-			       ##__VA_ARGS__); \
-	} while(0)
-
-#define _VERBS_DBG(fmt,...) \
-	do { \
-		if (unlikely(ib_ipath_debug&__VERBS_DBG)) \
-			printk(KERN_DEBUG "%s: " fmt, __func__, \
-			       ##__VA_ARGS__); \
-	} while(0)
-
-#define _VERBS_VDBG(fmt,...) \
-	do { \
-		if (unlikely(ib_ipath_debug&__VERBS_VDBG)) \
-			printk(KERN_DEBUG "%s: " fmt, __func__, \
-			       ##__VA_ARGS__); \
-	} while(0)
-
-#define _VERBS_SMADBG(fmt,...) \
-	do { \
-		if (unlikely(ib_ipath_debug&__VERBS_SMADBG)) \
-			printk(KERN_DEBUG "%s: " fmt, __func__, \
-			       ##__VA_ARGS__); \
-	} while(0)
-
-#else /* ! _VERBS_DEBUGGING */
-
-#define _VERBS_INFO(fmt,...)
-#define _VERBS_DBG(fmt,...)
-#define _VERBS_VDBG(fmt,...)
-#define _VERBS_SMADBG(fmt,...)
-
-#endif /* _VERBS_DEBUGGING */
-
-#endif /* _VERBS_DEBUG_H */
diff --git a/drivers/infiniband/hw/mthca/mthca_av.c b/drivers/infiniband/hw/mthca/mthca_av.c
index e215041..6959945 100644
--- a/drivers/infiniband/hw/mthca/mthca_av.c
+++ b/drivers/infiniband/hw/mthca/mthca_av.c
@@ -90,7 +90,7 @@ static enum ib_rate tavor_rate_to_ib(u8 mthca_rate, u8 port_rate)
 	case MTHCA_RATE_TAVOR_1X:     return IB_RATE_2_5_GBPS;
 	case MTHCA_RATE_TAVOR_1X_DDR: return IB_RATE_5_GBPS;
 	case MTHCA_RATE_TAVOR_4X:     return IB_RATE_10_GBPS;
-	default:		      return port_rate;
+	default:		      return mult_to_ib_rate(port_rate);
 	}
 }
 
diff --git a/drivers/infiniband/hw/mthca/mthca_catas.c b/drivers/infiniband/hw/mthca/mthca_catas.c
index c3bec74..cd044ea 100644
--- a/drivers/infiniband/hw/mthca/mthca_catas.c
+++ b/drivers/infiniband/hw/mthca/mthca_catas.c
@@ -34,6 +34,7 @@
 
 #include <linux/jiffies.h>
 #include <linux/timer.h>
+#include <linux/workqueue.h>
 
 #include "mthca_dev.h"
 
@@ -48,9 +49,41 @@ enum {
 
 static DEFINE_SPINLOCK(catas_lock);
 
+static LIST_HEAD(catas_list);
+static struct workqueue_struct *catas_wq;
+static struct work_struct catas_work;
+
+static int catas_reset_disable;
+module_param_named(catas_reset_disable, catas_reset_disable, int, 0644);
+MODULE_PARM_DESC(catas_reset_disable, "disable reset on catastrophic event if nonzero");
+
+static void catas_reset(void *work_ptr)
+{
+	struct mthca_dev *dev, *tmpdev;
+	LIST_HEAD(tlist);
+	int ret;
+
+	mutex_lock(&mthca_device_mutex);
+
+	spin_lock_irq(&catas_lock);
+	list_splice_init(&catas_list, &tlist);
+	spin_unlock_irq(&catas_lock);
+
+	list_for_each_entry_safe(dev, tmpdev, &tlist, catas_err.list) {
+		ret = __mthca_restart_one(dev->pdev);
+		if (ret)
+			mthca_err(dev, "Reset failed (%d)\n", ret);
+		else
+			mthca_dbg(dev, "Reset succeeded\n");
+	}
+
+	mutex_unlock(&mthca_device_mutex);
+}
+
 static void handle_catas(struct mthca_dev *dev)
 {
 	struct ib_event event;
+	unsigned long flags;
 	const char *type;
 	int i;
 
@@ -82,6 +115,14 @@ static void handle_catas(struct mthca_dev *dev)
 	for (i = 0; i < dev->catas_err.size; ++i)
 		mthca_err(dev, "  buf[%02x]: %08x\n",
 			  i, swab32(readl(dev->catas_err.map + i)));
+
+	if (catas_reset_disable)
+		return;
+
+	spin_lock_irqsave(&catas_lock, flags);
+	list_add(&dev->catas_err.list, &catas_list);
+	queue_work(catas_wq, &catas_work);
+	spin_unlock_irqrestore(&catas_lock, flags);
 }
 
 static void poll_catas(unsigned long dev_ptr)
@@ -135,6 +176,7 @@ void mthca_start_catas_poll(struct mthca_dev *dev)
 	dev->catas_err.timer.data     = (unsigned long) dev;
 	dev->catas_err.timer.function = poll_catas;
 	dev->catas_err.timer.expires  = jiffies + MTHCA_CATAS_POLL_INTERVAL;
+	INIT_LIST_HEAD(&dev->catas_err.list);
 	add_timer(&dev->catas_err.timer);
 }
 
@@ -153,4 +195,24 @@ void mthca_stop_catas_poll(struct mthca_dev *dev)
 				    dev->catas_err.addr),
 				   dev->catas_err.size * 4);
 	}
+
+	spin_lock_irq(&catas_lock);
+	list_del(&dev->catas_err.list);
+	spin_unlock_irq(&catas_lock);
+}
+
+int __init mthca_catas_init(void)
+{
+	INIT_WORK(&catas_work, catas_reset, NULL);
+
+	catas_wq = create_singlethread_workqueue("mthca_catas");
+	if (!catas_wq)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void mthca_catas_cleanup(void)
+{
+	destroy_workqueue(catas_wq);
 }
diff --git a/drivers/infiniband/hw/mthca/mthca_cmd.c b/drivers/infiniband/hw/mthca/mthca_cmd.c
index deabc14..99a94d7 100644
--- a/drivers/infiniband/hw/mthca/mthca_cmd.c
+++ b/drivers/infiniband/hw/mthca/mthca_cmd.c
@@ -34,7 +34,7 @@
  * $Id: mthca_cmd.c 1349 2004-12-16 21:09:43Z roland $
  */
 
-#include <linux/sched.h>
+#include <linux/completion.h>
 #include <linux/pci.h>
 #include <linux/errno.h>
 #include <asm/io.h>
diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 3e27a08..e393681 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -544,11 +544,11 @@ static inline int mthca_poll_one(struct mthca_dev *dev,
 		wq = &(*cur_qp)->rq;
 		wqe = be32_to_cpu(cqe->wqe);
 		wqe_index = wqe >> wq->wqe_shift;
-               /*
-		* WQE addr == base - 1 might be reported in receive completion
-		* with error instead of (rq size - 1) by Sinai FW 1.0.800 and
-		* Arbel FW 5.1.400.  This bug should be fixed in later FW revs.
-		*/
+		/*
+		 * WQE addr == base - 1 might be reported in receive completion
+		 * with error instead of (rq size - 1) by Sinai FW 1.0.800 and
+		 * Arbel FW 5.1.400.  This bug should be fixed in later FW revs.
+		 */
 		if (unlikely(wqe_index < 0))
 			wqe_index = wq->max - 1;
 		entry->wr_id = (*cur_qp)->wrid[wqe_index];
diff --git a/drivers/infiniband/hw/mthca/mthca_dev.h b/drivers/infiniband/hw/mthca/mthca_dev.h
index f8160b8..fe5cecf 100644
--- a/drivers/infiniband/hw/mthca/mthca_dev.h
+++ b/drivers/infiniband/hw/mthca/mthca_dev.h
@@ -45,6 +45,7 @@
 #include <linux/dma-mapping.h>
 #include <linux/timer.h>
 #include <linux/mutex.h>
+#include <linux/list.h>
 
 #include <asm/semaphore.h>
 
@@ -283,8 +284,11 @@ struct mthca_catas_err {
 	unsigned long		stop;
 	u32			size;
 	struct timer_list	timer;
+	struct list_head	list;
 };
 
+extern struct mutex mthca_device_mutex;
+
 struct mthca_dev {
 	struct ib_device  ib_dev;
 	struct pci_dev   *pdev;
@@ -450,6 +454,9 @@ void mthca_unregister_device(struct mthca_dev *dev);
 
 void mthca_start_catas_poll(struct mthca_dev *dev);
 void mthca_stop_catas_poll(struct mthca_dev *dev);
+int __mthca_restart_one(struct pci_dev *pdev);
+int mthca_catas_init(void);
+void mthca_catas_cleanup(void);
 
 int mthca_uar_alloc(struct mthca_dev *dev, struct mthca_uar *uar);
 void mthca_uar_free(struct mthca_dev *dev, struct mthca_uar *uar);
@@ -506,7 +513,7 @@ int mthca_alloc_srq(struct mthca_dev *dev, struct mthca_pd *pd,
 		    struct ib_srq_attr *attr, struct mthca_srq *srq);
 void mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq);
 int mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
-		     enum ib_srq_attr_mask attr_mask);
+		     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);
 int mthca_query_srq(struct ib_srq *srq, struct ib_srq_attr *srq_attr);
 int mthca_max_srq_sge(struct mthca_dev *dev);
 void mthca_srq_event(struct mthca_dev *dev, u32 srqn,
@@ -521,7 +528,8 @@ void mthca_qp_event(struct mthca_dev *dev, u32 qpn,
 		    enum ib_event_type event_type);
 int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
 		   struct ib_qp_init_attr *qp_init_attr);
-int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask);
+int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+		    struct ib_udata *udata);
 int mthca_tavor_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
 			  struct ib_send_wr **bad_wr);
 int mthca_tavor_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *wr,
diff --git a/drivers/infiniband/hw/mthca/mthca_mad.c b/drivers/infiniband/hw/mthca/mthca_mad.c
index d9bc030..45e106f 100644
--- a/drivers/infiniband/hw/mthca/mthca_mad.c
+++ b/drivers/infiniband/hw/mthca/mthca_mad.c
@@ -119,7 +119,7 @@ static void smp_snoop(struct ib_device *ibdev,
 
 			mthca_update_rate(to_mdev(ibdev), port_num);
 			update_sm_ah(to_mdev(ibdev), port_num,
-				     be16_to_cpu(pinfo->lid),
+				     be16_to_cpu(pinfo->sm_lid),
 				     pinfo->neighbormtu_mastersmsl & 0xf);
 
 			event.device           = ibdev;
diff --git a/drivers/infiniband/hw/mthca/mthca_main.c b/drivers/infiniband/hw/mthca/mthca_main.c
index 7b82c19..47ea021 100644
--- a/drivers/infiniband/hw/mthca/mthca_main.c
+++ b/drivers/infiniband/hw/mthca/mthca_main.c
@@ -80,6 +80,8 @@ static int tune_pci = 0;
 module_param(tune_pci, int, 0444);
 MODULE_PARM_DESC(tune_pci, "increase PCI burst from the default set by BIOS if nonzero");
 
+struct mutex mthca_device_mutex;
+
 static const char mthca_version[] __devinitdata =
 	DRV_NAME ": Mellanox InfiniBand HCA driver v"
 	DRV_VERSION " (" DRV_RELDATE ")\n";
@@ -978,28 +980,15 @@ static struct {
 					MTHCA_FLAG_SINAI_OPT }
 };
 
-static int __devinit mthca_init_one(struct pci_dev *pdev,
-				    const struct pci_device_id *id)
+static int __mthca_init_one(struct pci_dev *pdev, int hca_type)
 {
-	static int mthca_version_printed = 0;
 	int ddr_hidden = 0;
 	int err;
 	struct mthca_dev *mdev;
 
-	if (!mthca_version_printed) {
-		printk(KERN_INFO "%s", mthca_version);
-		++mthca_version_printed;
-	}
-
 	printk(KERN_INFO PFX "Initializing %s\n",
 	       pci_name(pdev));
 
-	if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) {
-		printk(KERN_ERR PFX "%s has invalid driver data %lx\n",
-		       pci_name(pdev), id->driver_data);
-		return -ENODEV;
-	}
-
 	err = pci_enable_device(pdev);
 	if (err) {
 		dev_err(&pdev->dev, "Cannot enable PCI device, "
@@ -1065,7 +1054,7 @@ static int __devinit mthca_init_one(struct pci_dev *pdev,
 
 	mdev->pdev = pdev;
 
-	mdev->mthca_flags = mthca_hca_table[id->driver_data].flags;
+	mdev->mthca_flags = mthca_hca_table[hca_type].flags;
 	if (ddr_hidden)
 		mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN;
 
@@ -1099,13 +1088,13 @@ static int __devinit mthca_init_one(struct pci_dev *pdev,
 	if (err)
 		goto err_cmd;
 
-	if (mdev->fw_ver < mthca_hca_table[id->driver_data].latest_fw) {
+	if (mdev->fw_ver < mthca_hca_table[hca_type].latest_fw) {
 		mthca_warn(mdev, "HCA FW version %d.%d.%d is old (%d.%d.%d is current).\n",
 			   (int) (mdev->fw_ver >> 32), (int) (mdev->fw_ver >> 16) & 0xffff,
 			   (int) (mdev->fw_ver & 0xffff),
-			   (int) (mthca_hca_table[id->driver_data].latest_fw >> 32),
-			   (int) (mthca_hca_table[id->driver_data].latest_fw >> 16) & 0xffff,
-			   (int) (mthca_hca_table[id->driver_data].latest_fw & 0xffff));
+			   (int) (mthca_hca_table[hca_type].latest_fw >> 32),
+			   (int) (mthca_hca_table[hca_type].latest_fw >> 16) & 0xffff,
+			   (int) (mthca_hca_table[hca_type].latest_fw & 0xffff));
 		mthca_warn(mdev, "If you have problems, try updating your HCA FW.\n");
 	}
 
@@ -1122,6 +1111,7 @@ static int __devinit mthca_init_one(struct pci_dev *pdev,
 		goto err_unregister;
 
 	pci_set_drvdata(pdev, mdev);
+	mdev->hca_type = hca_type;
 
 	return 0;
 
@@ -1166,7 +1156,7 @@ err_disable_pdev:
 	return err;
 }
 
-static void __devexit mthca_remove_one(struct pci_dev *pdev)
+static void __mthca_remove_one(struct pci_dev *pdev)
 {
 	struct mthca_dev *mdev = pci_get_drvdata(pdev);
 	u8 status;
@@ -1211,6 +1201,51 @@ static void __devexit mthca_remove_one(struct pci_dev *pdev)
 	}
 }
 
+int __mthca_restart_one(struct pci_dev *pdev)
+{
+	struct mthca_dev *mdev;
+
+	mdev = pci_get_drvdata(pdev);
+	if (!mdev)
+		return -ENODEV;
+	__mthca_remove_one(pdev);
+	return __mthca_init_one(pdev, mdev->hca_type);
+}
+
+static int __devinit mthca_init_one(struct pci_dev *pdev,
+			     const struct pci_device_id *id)
+{
+	static int mthca_version_printed = 0;
+	int ret;
+
+	mutex_lock(&mthca_device_mutex);
+
+	if (!mthca_version_printed) {
+		printk(KERN_INFO "%s", mthca_version);
+		++mthca_version_printed;
+	}
+
+	if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) {
+		printk(KERN_ERR PFX "%s has invalid driver data %lx\n",
+		       pci_name(pdev), id->driver_data);
+		mutex_unlock(&mthca_device_mutex);
+		return -ENODEV;
+	}
+
+	ret = __mthca_init_one(pdev, id->driver_data);
+
+	mutex_unlock(&mthca_device_mutex);
+
+	return ret;
+}
+
+static void __devexit mthca_remove_one(struct pci_dev *pdev)
+{
+	mutex_lock(&mthca_device_mutex);
+	__mthca_remove_one(pdev);
+	mutex_unlock(&mthca_device_mutex);
+}
+
 static struct pci_device_id mthca_pci_table[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR),
 	  .driver_data = TAVOR },
@@ -1248,13 +1283,24 @@ static int __init mthca_init(void)
 {
 	int ret;
 
+	mutex_init(&mthca_device_mutex);
+	ret = mthca_catas_init();
+	if (ret)
+		return ret;
+
 	ret = pci_register_driver(&mthca_driver);
-	return ret < 0 ? ret : 0;
+	if (ret < 0) {
+		mthca_catas_cleanup();
+		return ret;
+	}
+
+	return 0;
 }
 
 static void __exit mthca_cleanup(void)
 {
 	pci_unregister_driver(&mthca_driver);
+	mthca_catas_cleanup();
 }
 
 module_init(mthca_init);
diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c
index 265b1d1..981fe2e 100644
--- a/drivers/infiniband/hw/mthca/mthca_provider.c
+++ b/drivers/infiniband/hw/mthca/mthca_provider.c
@@ -1288,7 +1288,7 @@ int mthca_register_device(struct mthca_dev *dev)
 		(1ull << IB_USER_VERBS_CMD_DESTROY_QP)		|
 		(1ull << IB_USER_VERBS_CMD_ATTACH_MCAST)	|
 		(1ull << IB_USER_VERBS_CMD_DETACH_MCAST);
-	dev->ib_dev.node_type            = IB_NODE_CA;
+	dev->ib_dev.node_type            = RDMA_NODE_IB_CA;
 	dev->ib_dev.phys_port_cnt        = dev->limits.num_ports;
 	dev->ib_dev.dma_device           = &dev->pdev->dev;
 	dev->ib_dev.class_dev.dev        = &dev->pdev->dev;
diff --git a/drivers/infiniband/hw/mthca/mthca_qp.c b/drivers/infiniband/hw/mthca/mthca_qp.c
index 2e8f6f3..5e5c58b 100644
--- a/drivers/infiniband/hw/mthca/mthca_qp.c
+++ b/drivers/infiniband/hw/mthca/mthca_qp.c
@@ -408,7 +408,7 @@ static void to_ib_ah_attr(struct mthca_dev *dev, struct ib_ah_attr *ib_ah_attr,
 	ib_ah_attr->sl       	  = be32_to_cpu(path->sl_tclass_flowlabel) >> 28;
 	ib_ah_attr->src_path_bits = path->g_mylmc & 0x7f;
 	ib_ah_attr->static_rate   = mthca_rate_to_ib(dev,
-						     path->static_rate & 0x7,
+						     path->static_rate & 0xf,
 						     ib_ah_attr->port_num);
 	ib_ah_attr->ah_flags      = (path->g_mylmc & (1 << 7)) ? IB_AH_GRH : 0;
 	if (ib_ah_attr->ah_flags) {
@@ -472,10 +472,14 @@ int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_m
 	if (qp->transport == RC || qp->transport == UC) {
 		to_ib_ah_attr(dev, &qp_attr->ah_attr, &context->pri_path);
 		to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context->alt_path);
+		qp_attr->alt_pkey_index =
+			be32_to_cpu(context->alt_path.port_pkey) & 0x7f;
+		qp_attr->alt_port_num 	= qp_attr->alt_ah_attr.port_num;
 	}
 
-	qp_attr->pkey_index     = be32_to_cpu(context->pri_path.port_pkey) & 0x7f;
-	qp_attr->alt_pkey_index = be32_to_cpu(context->alt_path.port_pkey) & 0x7f;
+	qp_attr->pkey_index = be32_to_cpu(context->pri_path.port_pkey) & 0x7f;
+	qp_attr->port_num   =
+		(be32_to_cpu(context->pri_path.port_pkey) >> 24) & 0x3;
 
 	/* qp_attr->en_sqd_async_notify is only applicable in modify qp */
 	qp_attr->sq_draining = mthca_state == MTHCA_QP_STATE_DRAINING;
@@ -486,11 +490,9 @@ int mthca_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_m
 		1 << ((be32_to_cpu(context->params2) >> 21) & 0x7);
 	qp_attr->min_rnr_timer 	    =
 		(be32_to_cpu(context->rnr_nextrecvpsn) >> 24) & 0x1f;
-	qp_attr->port_num 	    = qp_attr->ah_attr.port_num;
 	qp_attr->timeout 	    = context->pri_path.ackto >> 3;
 	qp_attr->retry_cnt 	    = (be32_to_cpu(context->params1) >> 16) & 0x7;
 	qp_attr->rnr_retry 	    = context->pri_path.rnr_retry >> 5;
-	qp_attr->alt_port_num 	    = qp_attr->alt_ah_attr.port_num;
 	qp_attr->alt_timeout 	    = context->alt_path.ackto >> 3;
 	qp_init_attr->cap 	    = qp_attr->cap;
 
@@ -527,7 +529,8 @@ static int mthca_path_set(struct mthca_dev *dev, struct ib_ah_attr *ah,
 	return 0;
 }
 
-int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
+int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask,
+		    struct ib_udata *udata)
 {
 	struct mthca_dev *dev = to_mdev(ibqp->device);
 	struct mthca_qp *qp = to_mqp(ibqp);
@@ -842,11 +845,10 @@ int mthca_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask)
 	 * entries and reinitialize the QP.
 	 */
 	if (new_state == IB_QPS_RESET && !qp->ibqp.uobject) {
-		mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq), qp->qpn,
+		mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq), qp->qpn,
 			       qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
 		if (qp->ibqp.send_cq != qp->ibqp.recv_cq)
-			mthca_cq_clean(dev, to_mcq(qp->ibqp.recv_cq), qp->qpn,
-				       qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
+			mthca_cq_clean(dev, to_mcq(qp->ibqp.send_cq), qp->qpn, NULL);
 
 		mthca_wq_reset(&qp->sq);
 		qp->sq.last = get_send_wqe(qp, qp->sq.max - 1);
diff --git a/drivers/infiniband/hw/mthca/mthca_srq.c b/drivers/infiniband/hw/mthca/mthca_srq.c
index b60a9d7..0f316c8 100644
--- a/drivers/infiniband/hw/mthca/mthca_srq.c
+++ b/drivers/infiniband/hw/mthca/mthca_srq.c
@@ -358,7 +358,7 @@ void mthca_free_srq(struct mthca_dev *dev, struct mthca_srq *srq)
 }
 
 int mthca_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
-		     enum ib_srq_attr_mask attr_mask)
+		     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
 {
 	struct mthca_dev *dev = to_mdev(ibsrq->device);
 	struct mthca_srq *srq = to_msrq(ibsrq);
diff --git a/drivers/infiniband/hw/mthca/mthca_uar.c b/drivers/infiniband/hw/mthca/mthca_uar.c
index 8e92198..8b72848 100644
--- a/drivers/infiniband/hw/mthca/mthca_uar.c
+++ b/drivers/infiniband/hw/mthca/mthca_uar.c
@@ -60,7 +60,7 @@ int mthca_init_uar_table(struct mthca_dev *dev)
 	ret = mthca_alloc_init(&dev->uar_table.alloc,
 			       dev->limits.num_uars,
 			       dev->limits.num_uars - 1,
-			       dev->limits.reserved_uars);
+			       dev->limits.reserved_uars + 1);
 	if (ret)
 		return ret;
 
diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index 474aa21..0b8a79d 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -336,6 +336,8 @@ static inline void ipoib_unregister_debugfs(void) { }
 extern int ipoib_sendq_size;
 extern int ipoib_recvq_size;
 
+extern struct ib_sa_client ipoib_sa_client;
+
 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
 extern int ipoib_debug_level;
 
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_ib.c b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
index 5033666..f426a69 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_ib.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_ib.c
@@ -169,117 +169,129 @@ static int ipoib_ib_post_receives(struct net_device *dev)
 	return 0;
 }
 
-static void ipoib_ib_handle_wc(struct net_device *dev,
-			       struct ib_wc *wc)
+static void ipoib_ib_handle_rx_wc(struct net_device *dev, struct ib_wc *wc)
 {
 	struct ipoib_dev_priv *priv = netdev_priv(dev);
-	unsigned int wr_id = wc->wr_id;
+	unsigned int wr_id = wc->wr_id & ~IPOIB_OP_RECV;
+	struct sk_buff *skb;
+	dma_addr_t addr;
 
-	ipoib_dbg_data(priv, "called: id %d, op %d, status: %d\n",
+	ipoib_dbg_data(priv, "recv completion: id %d, op %d, status: %d\n",
 		       wr_id, wc->opcode, wc->status);
 
-	if (wr_id & IPOIB_OP_RECV) {
-		wr_id &= ~IPOIB_OP_RECV;
-
-		if (wr_id < ipoib_recvq_size) {
-			struct sk_buff *skb  = priv->rx_ring[wr_id].skb;
-			dma_addr_t      addr = priv->rx_ring[wr_id].mapping;
-
-			if (unlikely(wc->status != IB_WC_SUCCESS)) {
-				if (wc->status != IB_WC_WR_FLUSH_ERR)
-					ipoib_warn(priv, "failed recv event "
-						   "(status=%d, wrid=%d vend_err %x)\n",
-						   wc->status, wr_id, wc->vendor_err);
-				dma_unmap_single(priv->ca->dma_device, addr,
-						 IPOIB_BUF_SIZE, DMA_FROM_DEVICE);
-				dev_kfree_skb_any(skb);
-				priv->rx_ring[wr_id].skb = NULL;
-				return;
-			}
+	if (unlikely(wr_id >= ipoib_recvq_size)) {
+		ipoib_warn(priv, "recv completion event with wrid %d (> %d)\n",
+			   wr_id, ipoib_recvq_size);
+		return;
+	}
 
-			/*
-			 * If we can't allocate a new RX buffer, dump
-			 * this packet and reuse the old buffer.
-			 */
-			if (unlikely(ipoib_alloc_rx_skb(dev, wr_id))) {
-				++priv->stats.rx_dropped;
-				goto repost;
-			}
+	skb  = priv->rx_ring[wr_id].skb;
+	addr = priv->rx_ring[wr_id].mapping;
 
-			ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n",
-				       wc->byte_len, wc->slid);
+	if (unlikely(wc->status != IB_WC_SUCCESS)) {
+		if (wc->status != IB_WC_WR_FLUSH_ERR)
+			ipoib_warn(priv, "failed recv event "
+				   "(status=%d, wrid=%d vend_err %x)\n",
+				   wc->status, wr_id, wc->vendor_err);
+		dma_unmap_single(priv->ca->dma_device, addr,
+				 IPOIB_BUF_SIZE, DMA_FROM_DEVICE);
+		dev_kfree_skb_any(skb);
+		priv->rx_ring[wr_id].skb = NULL;
+		return;
+	}
 
-			dma_unmap_single(priv->ca->dma_device, addr,
-					 IPOIB_BUF_SIZE, DMA_FROM_DEVICE);
+	/*
+	 * If we can't allocate a new RX buffer, dump
+	 * this packet and reuse the old buffer.
+	 */
+	if (unlikely(ipoib_alloc_rx_skb(dev, wr_id))) {
+		++priv->stats.rx_dropped;
+		goto repost;
+	}
 
-			skb_put(skb, wc->byte_len);
-			skb_pull(skb, IB_GRH_BYTES);
+	ipoib_dbg_data(priv, "received %d bytes, SLID 0x%04x\n",
+		       wc->byte_len, wc->slid);
 
-			if (wc->slid != priv->local_lid ||
-			    wc->src_qp != priv->qp->qp_num) {
-				skb->protocol = ((struct ipoib_header *) skb->data)->proto;
-				skb->mac.raw = skb->data;
-				skb_pull(skb, IPOIB_ENCAP_LEN);
+	dma_unmap_single(priv->ca->dma_device, addr,
+			 IPOIB_BUF_SIZE, DMA_FROM_DEVICE);
 
-				dev->last_rx = jiffies;
-				++priv->stats.rx_packets;
-				priv->stats.rx_bytes += skb->len;
+	skb_put(skb, wc->byte_len);
+	skb_pull(skb, IB_GRH_BYTES);
 
-				skb->dev = dev;
-				/* XXX get correct PACKET_ type here */
-				skb->pkt_type = PACKET_HOST;
-				netif_rx_ni(skb);
-			} else {
-				ipoib_dbg_data(priv, "dropping loopback packet\n");
-				dev_kfree_skb_any(skb);
-			}
+	if (wc->slid != priv->local_lid ||
+	    wc->src_qp != priv->qp->qp_num) {
+		skb->protocol = ((struct ipoib_header *) skb->data)->proto;
+		skb->mac.raw = skb->data;
+		skb_pull(skb, IPOIB_ENCAP_LEN);
 
-		repost:
-			if (unlikely(ipoib_ib_post_receive(dev, wr_id)))
-				ipoib_warn(priv, "ipoib_ib_post_receive failed "
-					   "for buf %d\n", wr_id);
-		} else
-			ipoib_warn(priv, "completion event with wrid %d\n",
-				   wr_id);
+		dev->last_rx = jiffies;
+		++priv->stats.rx_packets;
+		priv->stats.rx_bytes += skb->len;
 
+		skb->dev = dev;
+		/* XXX get correct PACKET_ type here */
+		skb->pkt_type = PACKET_HOST;
+		netif_rx_ni(skb);
 	} else {
-		struct ipoib_tx_buf *tx_req;
-		unsigned long flags;
+		ipoib_dbg_data(priv, "dropping loopback packet\n");
+		dev_kfree_skb_any(skb);
+	}
 
-		if (wr_id >= ipoib_sendq_size) {
-			ipoib_warn(priv, "completion event with wrid %d (> %d)\n",
-				   wr_id, ipoib_sendq_size);
-			return;
-		}
+repost:
+	if (unlikely(ipoib_ib_post_receive(dev, wr_id)))
+		ipoib_warn(priv, "ipoib_ib_post_receive failed "
+			   "for buf %d\n", wr_id);
+}
 
-		ipoib_dbg_data(priv, "send complete, wrid %d\n", wr_id);
+static void ipoib_ib_handle_tx_wc(struct net_device *dev, struct ib_wc *wc)
+{
+	struct ipoib_dev_priv *priv = netdev_priv(dev);
+	unsigned int wr_id = wc->wr_id;
+	struct ipoib_tx_buf *tx_req;
+	unsigned long flags;
 
-		tx_req = &priv->tx_ring[wr_id];
+	ipoib_dbg_data(priv, "send completion: id %d, op %d, status: %d\n",
+		       wr_id, wc->opcode, wc->status);
 
-		dma_unmap_single(priv->ca->dma_device,
-				 pci_unmap_addr(tx_req, mapping),
-				 tx_req->skb->len,
-				 DMA_TO_DEVICE);
+	if (unlikely(wr_id >= ipoib_sendq_size)) {
+		ipoib_warn(priv, "send completion event with wrid %d (> %d)\n",
+			   wr_id, ipoib_sendq_size);
+		return;
+	}
 
-		++priv->stats.tx_packets;
-		priv->stats.tx_bytes += tx_req->skb->len;
+	tx_req = &priv->tx_ring[wr_id];
 
-		dev_kfree_skb_any(tx_req->skb);
+	dma_unmap_single(priv->ca->dma_device,
+			 pci_unmap_addr(tx_req, mapping),
+			 tx_req->skb->len,
+			 DMA_TO_DEVICE);
 
-		spin_lock_irqsave(&priv->tx_lock, flags);
-		++priv->tx_tail;
-		if (netif_queue_stopped(dev) &&
-		    test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags) &&
-		    priv->tx_head - priv->tx_tail <= ipoib_sendq_size >> 1)
-			netif_wake_queue(dev);
-		spin_unlock_irqrestore(&priv->tx_lock, flags);
+	++priv->stats.tx_packets;
+	priv->stats.tx_bytes += tx_req->skb->len;
 
-		if (wc->status != IB_WC_SUCCESS &&
-		    wc->status != IB_WC_WR_FLUSH_ERR)
-			ipoib_warn(priv, "failed send event "
-				   "(status=%d, wrid=%d vend_err %x)\n",
-				   wc->status, wr_id, wc->vendor_err);
-	}
+	dev_kfree_skb_any(tx_req->skb);
+
+	spin_lock_irqsave(&priv->tx_lock, flags);
+	++priv->tx_tail;
+	if (netif_queue_stopped(dev) &&
+	    test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags) &&
+	    priv->tx_head - priv->tx_tail <= ipoib_sendq_size >> 1)
+		netif_wake_queue(dev);
+	spin_unlock_irqrestore(&priv->tx_lock, flags);
+
+	if (wc->status != IB_WC_SUCCESS &&
+	    wc->status != IB_WC_WR_FLUSH_ERR)
+		ipoib_warn(priv, "failed send event "
+			   "(status=%d, wrid=%d vend_err %x)\n",
+			   wc->status, wr_id, wc->vendor_err);
+}
+
+static void ipoib_ib_handle_wc(struct net_device *dev, struct ib_wc *wc)
+{
+	if (wc->wr_id & IPOIB_OP_RECV)
+		ipoib_ib_handle_rx_wc(dev, wc);
+	else
+		ipoib_ib_handle_tx_wc(dev, wc);
 }
 
 void ipoib_ib_completion(struct ib_cq *cq, void *dev_ptr)
@@ -320,7 +332,7 @@ void ipoib_send(struct net_device *dev, struct sk_buff *skb,
 	struct ipoib_tx_buf *tx_req;
 	dma_addr_t addr;
 
-	if (skb->len > dev->mtu + INFINIBAND_ALEN) {
+	if (unlikely(skb->len > dev->mtu + INFINIBAND_ALEN)) {
 		ipoib_warn(priv, "packet len %d (> %d) too long to send, dropping\n",
 			   skb->len, dev->mtu + INFINIBAND_ALEN);
 		++priv->stats.tx_dropped;
@@ -619,8 +631,10 @@ void ipoib_ib_dev_flush(void *_dev)
 	 * The device could have been brought down between the start and when
 	 * we get here, don't bring it back up if it's not configured up
 	 */
-	if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags))
+	if (test_bit(IPOIB_FLAG_ADMIN_UP, &priv->flags)) {
 		ipoib_ib_dev_up(dev);
+		ipoib_mcast_restart_task(dev);
+	}
 
 	mutex_lock(&priv->vlan_mutex);
 
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index cf71d2a..1eaf00e 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -40,7 +40,6 @@
 
 #include <linux/init.h>
 #include <linux/slab.h>
-#include <linux/vmalloc.h>
 #include <linux/kernel.h>
 
 #include <linux/if_arp.h>	/* For ARPHRD_xxx */
@@ -82,6 +81,8 @@ static const u8 ipv4_bcast_addr[] = {
 
 struct workqueue_struct *ipoib_workqueue;
 
+struct ib_sa_client ipoib_sa_client;
+
 static void ipoib_add_one(struct ib_device *device);
 static void ipoib_remove_one(struct ib_device *device);
 
@@ -336,7 +337,8 @@ void ipoib_flush_paths(struct net_device *dev)
 	struct ipoib_path *path, *tp;
 	LIST_HEAD(remove_list);
 
-	spin_lock_irq(&priv->lock);
+	spin_lock_irq(&priv->tx_lock);
+	spin_lock(&priv->lock);
 
 	list_splice(&priv->path_list, &remove_list);
 	INIT_LIST_HEAD(&priv->path_list);
@@ -347,12 +349,15 @@ void ipoib_flush_paths(struct net_device *dev)
 	list_for_each_entry_safe(path, tp, &remove_list, list) {
 		if (path->query)
 			ib_sa_cancel_query(path->query_id, path->query);
-		spin_unlock_irq(&priv->lock);
+		spin_unlock(&priv->lock);
+		spin_unlock_irq(&priv->tx_lock);
 		wait_for_completion(&path->done);
 		path_free(dev, path);
-		spin_lock_irq(&priv->lock);
+		spin_lock_irq(&priv->tx_lock);
+		spin_lock(&priv->lock);
 	}
-	spin_unlock_irq(&priv->lock);
+	spin_unlock(&priv->lock);
+	spin_unlock_irq(&priv->tx_lock);
 }
 
 static void path_rec_completion(int status,
@@ -459,7 +464,7 @@ static int path_rec_start(struct net_device *dev,
 	init_completion(&path->done);
 
 	path->query_id =
-		ib_sa_path_rec_get(priv->ca, priv->port,
+		ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,
 				   &path->pathrec,
 				   IB_SA_PATH_REC_DGID		|
 				   IB_SA_PATH_REC_SGID		|
@@ -615,7 +620,7 @@ static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	struct ipoib_neigh *neigh;
 	unsigned long flags;
 
-	if (!spin_trylock_irqsave(&priv->tx_lock, flags))
+	if (unlikely(!spin_trylock_irqsave(&priv->tx_lock, flags)))
 		return NETDEV_TX_LOCKED;
 
 	/*
@@ -628,7 +633,7 @@ static int ipoib_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		return NETDEV_TX_BUSY;
 	}
 
-	if (skb->dst && skb->dst->neighbour) {
+	if (likely(skb->dst && skb->dst->neighbour)) {
 		if (unlikely(!*to_ipoib_neigh(skb->dst->neighbour))) {
 			ipoib_path_lookup(skb, dev);
 			goto out;
@@ -1107,13 +1112,16 @@ static void ipoib_add_one(struct ib_device *device)
 	struct ipoib_dev_priv *priv;
 	int s, e, p;
 
+	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+		return;
+
 	dev_list = kmalloc(sizeof *dev_list, GFP_KERNEL);
 	if (!dev_list)
 		return;
 
 	INIT_LIST_HEAD(dev_list);
 
-	if (device->node_type == IB_NODE_SWITCH) {
+	if (device->node_type == RDMA_NODE_IB_SWITCH) {
 		s = 0;
 		e = 0;
 	} else {
@@ -1137,6 +1145,9 @@ static void ipoib_remove_one(struct ib_device *device)
 	struct ipoib_dev_priv *priv, *tmp;
 	struct list_head *dev_list;
 
+	if (rdma_node_get_transport(device->node_type) != RDMA_TRANSPORT_IB)
+		return;
+
 	dev_list = ib_get_client_data(device, &ipoib_client);
 
 	list_for_each_entry_safe(priv, tmp, dev_list, list) {
@@ -1181,13 +1192,16 @@ static int __init ipoib_init_module(void)
 		goto err_fs;
 	}
 
+	ib_sa_register_client(&ipoib_sa_client);
+
 	ret = ib_register_client(&ipoib_client);
 	if (ret)
-		goto err_wq;
+		goto err_sa;
 
 	return 0;
 
-err_wq:
+err_sa:
+	ib_sa_unregister_client(&ipoib_sa_client);
 	destroy_workqueue(ipoib_workqueue);
 
 err_fs:
@@ -1199,6 +1213,7 @@ err_fs:
 static void __exit ipoib_cleanup_module(void)
 {
 	ib_unregister_client(&ipoib_client);
+	ib_sa_unregister_client(&ipoib_sa_client);
 	ipoib_unregister_debugfs();
 	destroy_workqueue(ipoib_workqueue);
 }
diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index ec356ce..3faa182 100644
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -361,7 +361,7 @@ static int ipoib_mcast_sendonly_join(struct ipoib_mcast *mcast)
 
 	init_completion(&mcast->done);
 
-	ret = ib_sa_mcmember_rec_set(priv->ca, priv->port, &rec,
+	ret = ib_sa_mcmember_rec_set(&ipoib_sa_client, priv->ca, priv->port, &rec,
 				     IB_SA_MCMEMBER_REC_MGID		|
 				     IB_SA_MCMEMBER_REC_PORT_GID	|
 				     IB_SA_MCMEMBER_REC_PKEY		|
@@ -472,22 +472,32 @@ static void ipoib_mcast_join(struct net_device *dev, struct ipoib_mcast *mcast,
 
 	if (create) {
 		comp_mask |=
-			IB_SA_MCMEMBER_REC_QKEY		|
-			IB_SA_MCMEMBER_REC_SL		|
-			IB_SA_MCMEMBER_REC_FLOW_LABEL	|
-			IB_SA_MCMEMBER_REC_TRAFFIC_CLASS;
+			IB_SA_MCMEMBER_REC_QKEY			|
+			IB_SA_MCMEMBER_REC_MTU_SELECTOR		|
+			IB_SA_MCMEMBER_REC_MTU			|
+			IB_SA_MCMEMBER_REC_TRAFFIC_CLASS	|
+			IB_SA_MCMEMBER_REC_RATE_SELECTOR	|
+			IB_SA_MCMEMBER_REC_RATE			|
+			IB_SA_MCMEMBER_REC_SL			|
+			IB_SA_MCMEMBER_REC_FLOW_LABEL		|
+			IB_SA_MCMEMBER_REC_HOP_LIMIT;
 
 		rec.qkey	  = priv->broadcast->mcmember.qkey;
+		rec.mtu_selector  = IB_SA_EQ;
+		rec.mtu		  = priv->broadcast->mcmember.mtu;
+		rec.traffic_class = priv->broadcast->mcmember.traffic_class;
+		rec.rate_selector = IB_SA_EQ;
+		rec.rate	  = priv->broadcast->mcmember.rate;
 		rec.sl		  = priv->broadcast->mcmember.sl;
 		rec.flow_label	  = priv->broadcast->mcmember.flow_label;
-		rec.traffic_class = priv->broadcast->mcmember.traffic_class;
+		rec.hop_limit	  = priv->broadcast->mcmember.hop_limit;
 	}
 
 	init_completion(&mcast->done);
 
-	ret = ib_sa_mcmember_rec_set(priv->ca, priv->port, &rec, comp_mask,
-				     mcast->backoff * 1000, GFP_ATOMIC,
-				     ipoib_mcast_join_complete,
+	ret = ib_sa_mcmember_rec_set(&ipoib_sa_client, priv->ca, priv->port,
+				     &rec, comp_mask, mcast->backoff * 1000,
+				     GFP_ATOMIC, ipoib_mcast_join_complete,
 				     mcast, &mcast->query);
 
 	if (ret < 0) {
@@ -528,7 +538,7 @@ void ipoib_mcast_join_task(void *dev_ptr)
 			priv->local_rate = attr.active_speed *
 				ib_width_enum_to_int(attr.active_width);
 		} else
-			ipoib_warn(priv, "ib_query_port failed\n");
+		ipoib_warn(priv, "ib_query_port failed\n");
 	}
 
 	if (!priv->broadcast) {
@@ -681,7 +691,7 @@ static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
 	 * Just make one shot at leaving and don't wait for a reply;
 	 * if we fail, too bad.
 	 */
-	ret = ib_sa_mcmember_rec_delete(priv->ca, priv->port, &rec,
+	ret = ib_sa_mcmember_rec_delete(&ipoib_sa_client, priv->ca, priv->port, &rec,
 					IB_SA_MCMEMBER_REC_MGID		|
 					IB_SA_MCMEMBER_REC_PORT_GID	|
 					IB_SA_MCMEMBER_REC_PKEY		|
@@ -795,7 +805,7 @@ void ipoib_mcast_dev_flush(struct net_device *dev)
 	}
 
 	if (priv->broadcast) {
- 		rb_erase(&priv->broadcast->rb_node, &priv->multicast_tree);
+		rb_erase(&priv->broadcast->rb_node, &priv->multicast_tree);
 		list_add_tail(&priv->broadcast->list, &remove_list);
 		priv->broadcast = NULL;
 	}
diff --git a/drivers/infiniband/ulp/iser/Kconfig b/drivers/infiniband/ulp/iser/Kconfig
index fead87d..365a1b5 100644
--- a/drivers/infiniband/ulp/iser/Kconfig
+++ b/drivers/infiniband/ulp/iser/Kconfig
@@ -1,6 +1,6 @@
 config INFINIBAND_ISER
 	tristate "ISCSI RDMA Protocol"
-	depends on INFINIBAND && SCSI
+	depends on INFINIBAND && SCSI && INET
 	select SCSI_ISCSI_ATTRS
 	---help---
 	  Support for the ISCSI RDMA Protocol over InfiniBand.  This
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.c b/drivers/infiniband/ulp/iser/iscsi_iser.c
index 1437d7e..2a14fe2 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.c
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.c
@@ -141,18 +141,11 @@ iscsi_iser_cmd_init(struct iscsi_cmd_task *ctask)
 
 	if (sc->sc_data_direction == DMA_TO_DEVICE) {
 		BUG_ON(ctask->total_length == 0);
-		/* bytes to be sent via RDMA operations */
-		iser_ctask->rdma_data_count = ctask->total_length -
-					 ctask->imm_count -
-					 ctask->unsol_count;
 
-		debug_scsi("cmd [itt %x total %d imm %d unsol_data %d "
-			   "rdma_data %d]\n",
+		debug_scsi("cmd [itt %x total %d imm %d unsol_data %d\n",
 			   ctask->itt, ctask->total_length, ctask->imm_count,
-			   ctask->unsol_count, iser_ctask->rdma_data_count);
-	} else
-		/* bytes to be sent via RDMA operations */
-		iser_ctask->rdma_data_count = ctask->total_length;
+			   ctask->unsol_count);
+	}
 
 	iser_ctask_rdma_init(iser_ctask);
 }
@@ -196,13 +189,10 @@ iscsi_iser_ctask_xmit_unsol_data(struct iscsi_conn *conn,
 {
 	struct iscsi_data  hdr;
 	int error = 0;
-	struct iscsi_iser_cmd_task *iser_ctask = ctask->dd_data;
 
 	/* Send data-out PDUs while there's still unsolicited data to send */
 	while (ctask->unsol_count > 0) {
-		iscsi_prep_unsolicit_data_pdu(ctask, &hdr,
-					      iser_ctask->rdma_data_count);
-
+		iscsi_prep_unsolicit_data_pdu(ctask, &hdr);
 		debug_scsi("Sending data-out: itt 0x%x, data count %d\n",
 			   hdr.itt, ctask->data_count);
 
@@ -555,6 +545,7 @@ static struct scsi_host_template iscsi_iser_sht = {
 	.queuecommand           = iscsi_queuecommand,
 	.can_queue		= ISCSI_XMIT_CMDS_MAX - 1,
 	.sg_tablesize           = ISCSI_ISER_SG_TABLESIZE,
+	.max_sectors		= 1024,
 	.cmd_per_lun            = ISCSI_MAX_CMD_PER_LUN,
 	.eh_abort_handler       = iscsi_eh_abort,
 	.eh_host_reset_handler	= iscsi_eh_host_reset,
diff --git a/drivers/infiniband/ulp/iser/iscsi_iser.h b/drivers/infiniband/ulp/iser/iscsi_iser.h
index 3350ba6..2cf9ae0 100644
--- a/drivers/infiniband/ulp/iser/iscsi_iser.h
+++ b/drivers/infiniband/ulp/iser/iscsi_iser.h
@@ -82,8 +82,12 @@
 		       __func__ , ## arg);		\
 	} while (0)
 
+#define SHIFT_4K	12
+#define SIZE_4K	(1UL << SHIFT_4K)
+#define MASK_4K	(~(SIZE_4K-1))
+
 					/* support upto 512KB in one RDMA */
-#define ISCSI_ISER_SG_TABLESIZE         (0x80000 >> PAGE_SHIFT)
+#define ISCSI_ISER_SG_TABLESIZE         (0x80000 >> SHIFT_4K)
 #define ISCSI_ISER_MAX_LUN		256
 #define ISCSI_ISER_MAX_CMD_LEN		16
 
@@ -171,6 +175,7 @@ struct iser_mem_reg {
 	u64  va;
 	u64  len;
 	void *mem_h;
+	int  is_fmr;
 };
 
 struct iser_regd_buf {
@@ -257,7 +262,6 @@ struct iscsi_iser_conn {
 struct iscsi_iser_cmd_task {
 	struct iser_desc             desc;
 	struct iscsi_iser_conn	     *iser_conn;
-	int			     rdma_data_count;/* RDMA bytes           */
 	enum iser_task_status 	     status;
 	int                          command_sent;  /* set if command  sent  */
 	int                          dir[ISER_DIRS_NUM];      /* set if dir use*/
diff --git a/drivers/infiniband/ulp/iser/iser_memory.c b/drivers/infiniband/ulp/iser/iser_memory.c
index 31950a5..d0b03f4 100644
--- a/drivers/infiniband/ulp/iser/iser_memory.c
+++ b/drivers/infiniband/ulp/iser/iser_memory.c
@@ -42,6 +42,7 @@
 #include "iscsi_iser.h"
 
 #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */
+
 /**
  * Decrements the reference count for the
  * registered buffer & releases it
@@ -55,7 +56,7 @@ int iser_regd_buff_release(struct iser_regd_buf *regd_buf)
 	if ((atomic_read(&regd_buf->ref_count) == 0) ||
 	    atomic_dec_and_test(&regd_buf->ref_count)) {
 		/* if we used the dma mr, unreg is just NOP */
-		if (regd_buf->reg.rkey != 0)
+		if (regd_buf->reg.is_fmr)
 			iser_unreg_mem(&regd_buf->reg);
 
 		if (regd_buf->dma_addr) {
@@ -90,9 +91,9 @@ void iser_reg_single(struct iser_device *device,
 	BUG_ON(dma_mapping_error(dma_addr));
 
 	regd_buf->reg.lkey = device->mr->lkey;
-	regd_buf->reg.rkey = 0; /* indicate there's no need to unreg */
 	regd_buf->reg.len  = regd_buf->data_size;
 	regd_buf->reg.va   = dma_addr;
+	regd_buf->reg.is_fmr = 0;
 
 	regd_buf->dma_addr  = dma_addr;
 	regd_buf->direction = direction;
@@ -239,7 +240,7 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data,
 	int i;
 
 	/* compute the offset of first element */
-	page_vec->offset = (u64) sg[0].offset;
+	page_vec->offset = (u64) sg[0].offset & ~MASK_4K;
 
 	for (i = 0; i < data->dma_nents; i++) {
 		total_sz += sg_dma_len(&sg[i]);
@@ -247,21 +248,30 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data,
 		first_addr = sg_dma_address(&sg[i]);
 		last_addr  = first_addr + sg_dma_len(&sg[i]);
 
-		start_aligned = !(first_addr & ~PAGE_MASK);
-		end_aligned   = !(last_addr  & ~PAGE_MASK);
+		start_aligned = !(first_addr & ~MASK_4K);
+		end_aligned   = !(last_addr  & ~MASK_4K);
 
 		/* continue to collect page fragments till aligned or SG ends */
 		while (!end_aligned && (i + 1 < data->dma_nents)) {
 			i++;
 			total_sz += sg_dma_len(&sg[i]);
 			last_addr = sg_dma_address(&sg[i]) + sg_dma_len(&sg[i]);
-			end_aligned = !(last_addr  & ~PAGE_MASK);
+			end_aligned = !(last_addr  & ~MASK_4K);
 		}
 
-		first_addr = first_addr & PAGE_MASK;
-
-		for (page = first_addr; page < last_addr; page += PAGE_SIZE)
-			page_vec->pages[cur_page++] = page;
+		/* handle the 1st page in the 1st DMA element */
+		if (cur_page == 0) {
+			page = first_addr & MASK_4K;
+			page_vec->pages[cur_page] = page;
+			cur_page++;
+			page += SIZE_4K;
+		} else
+			page = first_addr;
+
+		for (; page < last_addr; page += SIZE_4K) {
+			page_vec->pages[cur_page] = page;
+			cur_page++;
+		}
 
 	}
 	page_vec->data_size = total_sz;
@@ -269,8 +279,7 @@ static int iser_sg_to_page_vec(struct iser_data_buf *data,
 	return cur_page;
 }
 
-#define MASK_4K			((1UL << 12) - 1) /* 0xFFF */
-#define IS_4K_ALIGNED(addr)	((((unsigned long)addr) & MASK_4K) == 0)
+#define IS_4K_ALIGNED(addr)	((((unsigned long)addr) & ~MASK_4K) == 0)
 
 /**
  * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned
@@ -320,9 +329,9 @@ static void iser_data_buf_dump(struct iser_data_buf *data)
 	struct scatterlist *sg = (struct scatterlist *)data->buf;
 	int i;
 
-	for (i = 0; i < data->size; i++)
+	for (i = 0; i < data->dma_nents; i++)
 		iser_err("sg[%d] dma_addr:0x%lX page:0x%p "
-			 "off:%d sz:%d dma_len:%d\n",
+			 "off:0x%x sz:0x%x dma_len:0x%x\n",
 			 i, (unsigned long)sg_dma_address(&sg[i]),
 			 sg[i].page, sg[i].offset,
 			 sg[i].length,sg_dma_len(&sg[i]));
@@ -352,7 +361,7 @@ static void iser_page_vec_build(struct iser_data_buf *data,
 
 	page_vec->length = page_vec_len;
 
-	if (page_vec_len * PAGE_SIZE < page_vec->data_size) {
+	if (page_vec_len * SIZE_4K < page_vec->data_size) {
 		iser_err("page_vec too short to hold this SG\n");
 		iser_data_buf_dump(data);
 		iser_dump_page_vec(page_vec);
@@ -370,15 +379,18 @@ int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *iser_ctask,
 		      enum   iser_data_dir        cmd_dir)
 {
 	struct iser_conn     *ib_conn = iser_ctask->iser_conn->ib_conn;
+	struct iser_device   *device = ib_conn->device;
 	struct iser_data_buf *mem = &iser_ctask->data[cmd_dir];
 	struct iser_regd_buf *regd_buf;
 	int aligned_len;
 	int err;
+	int i;
+	struct scatterlist *sg;
 
 	regd_buf = &iser_ctask->rdma_regd[cmd_dir];
 
 	aligned_len = iser_data_buf_aligned_len(mem);
-	if (aligned_len != mem->size) {
+	if (aligned_len != mem->dma_nents) {
 		iser_err("rdma alignment violation %d/%d aligned\n",
 			 aligned_len, mem->size);
 		iser_data_buf_dump(mem);
@@ -389,10 +401,38 @@ int iser_reg_rdma_mem(struct iscsi_iser_cmd_task *iser_ctask,
 		mem = &iser_ctask->data_copy[cmd_dir];
 	}
 
-	iser_page_vec_build(mem, ib_conn->page_vec);
-	err = iser_reg_page_vec(ib_conn, ib_conn->page_vec, &regd_buf->reg);
-	if (err)
-		return err;
+	/* if there a single dma entry, FMR is not needed */
+	if (mem->dma_nents == 1) {
+		sg = (struct scatterlist *)mem->buf;
+
+		regd_buf->reg.lkey = device->mr->lkey;
+		regd_buf->reg.rkey = device->mr->rkey;
+		regd_buf->reg.len  = sg_dma_len(&sg[0]);
+		regd_buf->reg.va   = sg_dma_address(&sg[0]);
+		regd_buf->reg.is_fmr = 0;
+
+		iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X  "
+			 "va: 0x%08lX sz: %ld]\n",
+			 (unsigned int)regd_buf->reg.lkey,
+			 (unsigned int)regd_buf->reg.rkey,
+			 (unsigned long)regd_buf->reg.va,
+			 (unsigned long)regd_buf->reg.len);
+	} else { /* use FMR for multiple dma entries */
+		iser_page_vec_build(mem, ib_conn->page_vec);
+		err = iser_reg_page_vec(ib_conn, ib_conn->page_vec, &regd_buf->reg);
+		if (err) {
+			iser_data_buf_dump(mem);
+			iser_err("mem->dma_nents = %d (dlength = 0x%x)\n", mem->dma_nents,
+				 ntoh24(iser_ctask->desc.iscsi_header.dlength));
+			iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n",
+				 ib_conn->page_vec->data_size, ib_conn->page_vec->length,
+				 ib_conn->page_vec->offset);
+			for (i=0 ; i<ib_conn->page_vec->length ; i++)
+				iser_err("page_vec[%d] = 0x%llx\n", i,
+					 (unsigned long long) ib_conn->page_vec->pages[i]);
+			return err;
+		}
+	}
 
 	/* take a reference on this regd buf such that it will not be released *
 	 * (eg in send dto completion) before we get the scsi response         */
diff --git a/drivers/infiniband/ulp/iser/iser_verbs.c b/drivers/infiniband/ulp/iser/iser_verbs.c
index 72febf1..ecdca7f 100644
--- a/drivers/infiniband/ulp/iser/iser_verbs.c
+++ b/drivers/infiniband/ulp/iser/iser_verbs.c
@@ -88,8 +88,9 @@ static int iser_create_device_ib_res(struct iser_device *device)
 		     iser_cq_tasklet_fn,
 		     (unsigned long)device);
 
-	device->mr = ib_get_dma_mr(device->pd,
-				   IB_ACCESS_LOCAL_WRITE);
+	device->mr = ib_get_dma_mr(device->pd, IB_ACCESS_LOCAL_WRITE |
+				   IB_ACCESS_REMOTE_WRITE |
+				   IB_ACCESS_REMOTE_READ);
 	if (IS_ERR(device->mr))
 		goto dma_mr_err;
 
@@ -150,7 +151,7 @@ static int iser_create_ib_conn_res(struct iser_conn *ib_conn)
 	}
 	ib_conn->page_vec->pages = (u64 *) (ib_conn->page_vec + 1);
 
-	params.page_shift        = PAGE_SHIFT;
+	params.page_shift        = SHIFT_4K;
 	/* when the first/last SG element are not start/end *
 	 * page aligned, the map whould be of N+1 pages     */
 	params.max_pages_per_fmr = ISCSI_ISER_SG_TABLESIZE + 1;
@@ -604,8 +605,9 @@ int iser_reg_page_vec(struct iser_conn     *ib_conn,
 
 	mem_reg->lkey  = mem->fmr->lkey;
 	mem_reg->rkey  = mem->fmr->rkey;
-	mem_reg->len   = page_vec->length * PAGE_SIZE;
+	mem_reg->len   = page_vec->length * SIZE_4K;
 	mem_reg->va    = io_addr;
+	mem_reg->is_fmr = 1;
 	mem_reg->mem_h = (void *)mem;
 
 	mem_reg->va   += page_vec->offset;
diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index fd8344c..44b9e5b 100644
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -96,6 +96,8 @@ static struct ib_client srp_client = {
 	.remove = srp_remove_one
 };
 
+static struct ib_sa_client srp_sa_client;
+
 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 {
 	return (struct srp_target_port *) host->hostdata;
@@ -267,7 +269,8 @@ static int srp_lookup_path(struct srp_target_port *target)
 
 	init_completion(&target->done);
 
-	target->path_query_id = ib_sa_path_rec_get(target->srp_host->dev->dev,
+	target->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
+						   target->srp_host->dev->dev,
 						   target->srp_host->port,
 						   &target->path,
 						   IB_SA_PATH_REC_DGID		|
@@ -330,7 +333,7 @@ static int srp_send_req(struct srp_target_port *target)
 	req->priv.req_buf_fmt 	= cpu_to_be16(SRP_BUF_FORMAT_DIRECT |
 					      SRP_BUF_FORMAT_INDIRECT);
 	/*
-	 * In the published SRP specification (draft rev. 16a), the 
+	 * In the published SRP specification (draft rev. 16a), the
 	 * port identifier format is 8 bytes of ID extension followed
 	 * by 8 bytes of GUID.  Older drafts put the two halves in the
 	 * opposite order, so that the GUID comes first.
@@ -1449,12 +1452,28 @@ static ssize_t show_zero_req_lim(struct class_device *cdev, char *buf)
 	return sprintf(buf, "%d\n", target->zero_req_lim);
 }
 
-static CLASS_DEVICE_ATTR(id_ext,	S_IRUGO, show_id_ext,		NULL);
-static CLASS_DEVICE_ATTR(ioc_guid,	S_IRUGO, show_ioc_guid,		NULL);
-static CLASS_DEVICE_ATTR(service_id,	S_IRUGO, show_service_id,	NULL);
-static CLASS_DEVICE_ATTR(pkey,		S_IRUGO, show_pkey,		NULL);
-static CLASS_DEVICE_ATTR(dgid,		S_IRUGO, show_dgid,		NULL);
-static CLASS_DEVICE_ATTR(zero_req_lim,	S_IRUGO, show_zero_req_lim,	NULL);
+static ssize_t show_local_ib_port(struct class_device *cdev, char *buf)
+{
+	struct srp_target_port *target = host_to_target(class_to_shost(cdev));
+
+	return sprintf(buf, "%d\n", target->srp_host->port);
+}
+
+static ssize_t show_local_ib_device(struct class_device *cdev, char *buf)
+{
+	struct srp_target_port *target = host_to_target(class_to_shost(cdev));
+
+	return sprintf(buf, "%s\n", target->srp_host->dev->dev->name);
+}
+
+static CLASS_DEVICE_ATTR(id_ext,	  S_IRUGO, show_id_ext,		 NULL);
+static CLASS_DEVICE_ATTR(ioc_guid,	  S_IRUGO, show_ioc_guid,	 NULL);
+static CLASS_DEVICE_ATTR(service_id,	  S_IRUGO, show_service_id,	 NULL);
+static CLASS_DEVICE_ATTR(pkey,		  S_IRUGO, show_pkey,		 NULL);
+static CLASS_DEVICE_ATTR(dgid,		  S_IRUGO, show_dgid,		 NULL);
+static CLASS_DEVICE_ATTR(zero_req_lim,	  S_IRUGO, show_zero_req_lim,	 NULL);
+static CLASS_DEVICE_ATTR(local_ib_port,   S_IRUGO, show_local_ib_port,	 NULL);
+static CLASS_DEVICE_ATTR(local_ib_device, S_IRUGO, show_local_ib_device, NULL);
 
 static struct class_device_attribute *srp_host_attrs[] = {
 	&class_device_attr_id_ext,
@@ -1463,6 +1482,8 @@ static struct class_device_attribute *srp_host_attrs[] = {
 	&class_device_attr_pkey,
 	&class_device_attr_dgid,
 	&class_device_attr_zero_req_lim,
+	&class_device_attr_local_ib_port,
+	&class_device_attr_local_ib_device,
 	NULL
 };
 
@@ -1881,7 +1902,7 @@ static void srp_add_one(struct ib_device *device)
 	if (IS_ERR(srp_dev->fmr_pool))
 		srp_dev->fmr_pool = NULL;
 
-	if (device->node_type == IB_NODE_SWITCH) {
+	if (device->node_type == RDMA_NODE_IB_SWITCH) {
 		s = 0;
 		e = 0;
 	} else {
@@ -1980,9 +2001,12 @@ static int __init srp_init_module(void)
 		return ret;
 	}
 
+	ib_sa_register_client(&srp_sa_client);
+
 	ret = ib_register_client(&srp_client);
 	if (ret) {
 		printk(KERN_ERR PFX "couldn't register IB client\n");
+		ib_sa_unregister_client(&srp_sa_client);
 		class_unregister(&srp_class);
 		return ret;
 	}
@@ -1993,6 +2017,7 @@ static int __init srp_init_module(void)
 static void __exit srp_cleanup_module(void)
 {
 	ib_unregister_client(&srp_client);
+	ib_sa_unregister_client(&srp_sa_client);
 	class_unregister(&srp_class);
 }
 
diff --git a/drivers/macintosh/adbhid.c b/drivers/macintosh/adbhid.c
index c69d23b..efd51e0 100644
--- a/drivers/macintosh/adbhid.c
+++ b/drivers/macintosh/adbhid.c
@@ -45,8 +45,8 @@
 #include <linux/pmu.h>
 
 #include <asm/machdep.h>
-#include <asm/backlight.h>
 #ifdef CONFIG_PPC_PMAC
+#include <asm/backlight.h>
 #include <asm/pmac_feature.h>
 #endif
 
diff --git a/drivers/macintosh/macio_asic.c b/drivers/macintosh/macio_asic.c
index 82657bc..d562160 100644
--- a/drivers/macintosh/macio_asic.c
+++ b/drivers/macintosh/macio_asic.c
@@ -139,7 +139,9 @@ static int macio_uevent(struct device *dev, char **envp, int num_envp,
 {
 	struct macio_dev * macio_dev;
 	struct of_device * of;
-	char *scratch, *compat, *compat2;
+	char *scratch;
+	const char *compat, *compat2;
+
 	int i = 0;
 	int length, cplen, cplen2, seen = 0;
 
@@ -173,7 +175,7 @@ static int macio_uevent(struct device *dev, char **envp, int num_envp,
          * it's not really legal to split it out with commas. We split it
          * up using a number of environment variables instead. */
 
-	compat = (char *) get_property(of->node, "compatible", &cplen);
+	compat = get_property(of->node, "compatible", &cplen);
 	compat2 = compat;
 	cplen2= cplen;
 	while (compat && cplen > 0) {
@@ -454,7 +456,7 @@ static struct macio_dev * macio_add_one_device(struct macio_chip *chip,
 					       struct resource *parent_res)
 {
 	struct macio_dev *dev;
-	u32 *reg;
+	const u32 *reg;
 	
 	if (np == NULL)
 		return NULL;
@@ -489,7 +491,7 @@ static struct macio_dev * macio_add_one_device(struct macio_chip *chip,
 #endif
 			MAX_NODE_NAME_SIZE, np->name);
 	} else {
-		reg = (u32 *)get_property(np, "reg", NULL);
+		reg = get_property(np, "reg", NULL);
 		sprintf(dev->ofdev.dev.bus_id, "%1d.%08x:%.*s",
 			chip->lbus.index,
 			reg ? *reg : 0, MAX_NODE_NAME_SIZE, np->name);
diff --git a/drivers/macintosh/macio_sysfs.c b/drivers/macintosh/macio_sysfs.c
index cae24a1..8566bdf 100644
--- a/drivers/macintosh/macio_sysfs.c
+++ b/drivers/macintosh/macio_sysfs.c
@@ -16,12 +16,12 @@ static ssize_t
 compatible_show (struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct of_device *of;
-	char *compat;
+	const char *compat;
 	int cplen;
 	int length = 0;
 
 	of = &to_macio_device (dev)->ofdev;
-	compat = (char *) get_property(of->node, "compatible", &cplen);
+	compat = get_property(of->node, "compatible", &cplen);
 	if (!compat) {
 		*buf = '\0';
 		return 0;
@@ -42,12 +42,12 @@ static ssize_t modalias_show (struct device *dev, struct device_attribute *attr,
 			      char *buf)
 {
 	struct of_device *of;
-	char *compat;
+	const char *compat;
 	int cplen;
 	int length;
 
 	of = &to_macio_device (dev)->ofdev;
-	compat = (char *) get_property (of->node, "compatible", &cplen);
+	compat = get_property(of->node, "compatible", &cplen);
 	if (!compat) compat = "", cplen = 1;
 	length = sprintf (buf, "of:N%sT%s", of->node->name, of->node->type);
 	buf += length;
diff --git a/drivers/macintosh/smu.c b/drivers/macintosh/smu.c
index 00ef468..090e40f 100644
--- a/drivers/macintosh/smu.c
+++ b/drivers/macintosh/smu.c
@@ -454,7 +454,7 @@ EXPORT_SYMBOL(smu_present);
 int __init smu_init (void)
 {
 	struct device_node *np;
-	u32 *data;
+	const u32 *data;
 
         np = of_find_node_by_type(NULL, "smu");
         if (np == NULL)
@@ -490,7 +490,7 @@ int __init smu_init (void)
 		printk(KERN_ERR "SMU: Can't find doorbell GPIO !\n");
 		goto fail;
 	}
-	data = (u32 *)get_property(smu->db_node, "reg", NULL);
+	data = get_property(smu->db_node, "reg", NULL);
 	if (data == NULL) {
 		of_node_put(smu->db_node);
 		smu->db_node = NULL;
@@ -511,7 +511,7 @@ int __init smu_init (void)
 		smu->msg_node = of_find_node_by_name(NULL, "smu-interrupt");
 		if (smu->msg_node == NULL)
 			break;
-		data = (u32 *)get_property(smu->msg_node, "reg", NULL);
+		data = get_property(smu->msg_node, "reg", NULL);
 		if (data == NULL) {
 			of_node_put(smu->msg_node);
 			smu->msg_node = NULL;
@@ -982,11 +982,11 @@ static struct smu_sdbp_header *smu_create_sdb_partition(int id)
 /* Note: Only allowed to return error code in pointers (using ERR_PTR)
  * when interruptible is 1
  */
-struct smu_sdbp_header *__smu_get_sdb_partition(int id, unsigned int *size,
-						int interruptible)
+const struct smu_sdbp_header *__smu_get_sdb_partition(int id,
+		unsigned int *size, int interruptible)
 {
 	char pname[32];
-	struct smu_sdbp_header *part;
+	const struct smu_sdbp_header *part;
 
 	if (!smu)
 		return NULL;
@@ -1003,8 +1003,7 @@ struct smu_sdbp_header *__smu_get_sdb_partition(int id, unsigned int *size,
 	} else
 		mutex_lock(&smu_part_access);
 
-	part = (struct smu_sdbp_header *)get_property(smu->of_node,
-						      pname, size);
+	part = get_property(smu->of_node, pname, size);
 	if (part == NULL) {
 		DPRINTK("trying to extract from SMU ...\n");
 		part = smu_create_sdb_partition(id);
@@ -1015,7 +1014,7 @@ struct smu_sdbp_header *__smu_get_sdb_partition(int id, unsigned int *size,
 	return part;
 }
 
-struct smu_sdbp_header *smu_get_sdb_partition(int id, unsigned int *size)
+const struct smu_sdbp_header *smu_get_sdb_partition(int id, unsigned int *size)
 {
 	return __smu_get_sdb_partition(id, size, 0);
 }
@@ -1094,7 +1093,7 @@ static ssize_t smu_write(struct file *file, const char __user *buf,
 		pp->mode = smu_file_events;
 		return 0;
 	} else if (hdr.cmdtype == SMU_CMDTYPE_GET_PARTITION) {
-		struct smu_sdbp_header *part;
+		const struct smu_sdbp_header *part;
 		part = __smu_get_sdb_partition(hdr.cmd, NULL, 1);
 		if (part == NULL)
 			return -EINVAL;
diff --git a/drivers/macintosh/therm_adt746x.c b/drivers/macintosh/therm_adt746x.c
index 7f86478..a0f30d0 100644
--- a/drivers/macintosh/therm_adt746x.c
+++ b/drivers/macintosh/therm_adt746x.c
@@ -47,7 +47,7 @@ static u8 FAN_SPD_SET[2] = {0x30, 0x31};
 
 static u8 default_limits_local[3] = {70, 50, 70};    /* local, sensor1, sensor2 */
 static u8 default_limits_chip[3] = {80, 65, 80};    /* local, sensor1, sensor2 */
-static char *sensor_location[3] = {NULL, NULL, NULL};
+static const char *sensor_location[3] = {NULL, NULL, NULL};
 
 static int limit_adjust = 0;
 static int fan_speed = -1;
@@ -553,7 +553,7 @@ static int __init
 thermostat_init(void)
 {
 	struct device_node* np;
-	u32 *prop;
+	const u32 *prop;
 	int i = 0, offset = 0;
 	
 	np = of_find_node_by_name(NULL, "fan");
@@ -566,13 +566,13 @@ thermostat_init(void)
 	else
 		return -ENODEV;
 
-	prop = (u32 *)get_property(np, "hwsensor-params-version", NULL);
+	prop = get_property(np, "hwsensor-params-version", NULL);
 	printk(KERN_INFO "adt746x: version %d (%ssupported)\n", *prop,
 			 (*prop == 1)?"":"un");
 	if (*prop != 1)
 		return -ENODEV;
 
-	prop = (u32 *)get_property(np, "reg", NULL);
+	prop = get_property(np, "reg", NULL);
 	if (!prop)
 		return -ENODEV;
 
diff --git a/drivers/macintosh/therm_pm72.c b/drivers/macintosh/therm_pm72.c
index 20bf672..d00c0c3 100644
--- a/drivers/macintosh/therm_pm72.c
+++ b/drivers/macintosh/therm_pm72.c
@@ -660,7 +660,7 @@ static int read_eeprom(int cpu, struct mpu_data *out)
 {
 	struct device_node *np;
 	char nodename[64];
-	u8 *data;
+	const u8 *data;
 	int len;
 
 	/* prom.c routine for finding a node by path is a bit brain dead
@@ -673,7 +673,7 @@ static int read_eeprom(int cpu, struct mpu_data *out)
 		printk(KERN_ERR "therm_pm72: Failed to retrieve cpuid node from device-tree\n");
 		return -ENODEV;
 	}
-	data = (u8 *)get_property(np, "cpuid", &len);
+	data = get_property(np, "cpuid", &len);
 	if (data == NULL) {
 		printk(KERN_ERR "therm_pm72: Failed to retrieve cpuid property from device-tree\n");
 		of_node_put(np);
@@ -1336,7 +1336,7 @@ static int init_backside_state(struct backside_pid_state *state)
 	 */
 	u3 = of_find_node_by_path("/u3@0,f8000000");
 	if (u3 != NULL) {
-		u32 *vers = (u32 *)get_property(u3, "device-rev", NULL);
+		const u32 *vers = get_property(u3, "device-rev", NULL);
 		if (vers)
 			if (((*vers) & 0x3f) < 0x34)
 				u3h = 0;
@@ -2111,8 +2111,8 @@ static void fcu_lookup_fans(struct device_node *fcu_node)
 
 	while ((np = of_get_next_child(fcu_node, np)) != NULL) {
 		int type = -1;
-		char *loc;
-		u32 *reg;
+		const char *loc;
+		const u32 *reg;
 
 		DBG(" control: %s, type: %s\n", np->name, np->type);
 
@@ -2128,8 +2128,8 @@ static void fcu_lookup_fans(struct device_node *fcu_node)
 			continue;
 
 		/* Lookup for a matching location */
-		loc = (char *)get_property(np, "location", NULL);
-		reg = (u32 *)get_property(np, "reg", NULL);
+		loc = get_property(np, "location", NULL);
+		reg = get_property(np, "reg", NULL);
 		if (loc == NULL || reg == NULL)
 			continue;
 		DBG(" matching location: %s, reg: 0x%08x\n", loc, *reg);
diff --git a/drivers/macintosh/therm_windtunnel.c b/drivers/macintosh/therm_windtunnel.c
index c7d1c29..738faab 100644
--- a/drivers/macintosh/therm_windtunnel.c
+++ b/drivers/macintosh/therm_windtunnel.c
@@ -484,14 +484,14 @@ struct apple_thermal_info {
 static int __init
 g4fan_init( void )
 {
-	struct apple_thermal_info *info;
+	const struct apple_thermal_info *info;
 	struct device_node *np;
 
 	init_MUTEX( &x.lock );
 
 	if( !(np=of_find_node_by_name(NULL, "power-mgt")) )
 		return -ENODEV;
-	info = (struct apple_thermal_info*)get_property(np, "thermal-info", NULL);
+	info = get_property(np, "thermal-info", NULL);
 	of_node_put(np);
 
 	if( !info || !machine_is_compatible("PowerMac3,6") )
diff --git a/drivers/macintosh/via-cuda.c b/drivers/macintosh/via-cuda.c
index 69d5452..7512d1c 100644
--- a/drivers/macintosh/via-cuda.c
+++ b/drivers/macintosh/via-cuda.c
@@ -123,7 +123,7 @@ int __init find_via_cuda(void)
 {
     struct adb_request req;
     phys_addr_t taddr;
-    u32 *reg;
+    const u32 *reg;
     int err;
 
     if (vias != 0)
@@ -132,7 +132,7 @@ int __init find_via_cuda(void)
     if (vias == 0)
 	return 0;
 
-    reg = (u32 *)get_property(vias, "reg", NULL);
+    reg = get_property(vias, "reg", NULL);
     if (reg == NULL) {
 	    printk(KERN_ERR "via-cuda: No \"reg\" property !\n");
 	    goto fail;
diff --git a/drivers/macintosh/via-pmu-led.c b/drivers/macintosh/via-pmu-led.c
index 5189d54..179af10 100644
--- a/drivers/macintosh/via-pmu-led.c
+++ b/drivers/macintosh/via-pmu-led.c
@@ -120,7 +120,7 @@ static int __init via_pmu_led_init(void)
 	dt = of_find_node_by_path("/");
 	if (dt == NULL)
 		return -ENODEV;
-	model = (const char *)get_property(dt, "model", NULL);
+	model = get_property(dt, "model", NULL);
 	if (model == NULL)
 		return -ENODEV;
 	if (strncmp(model, "PowerBook", strlen("PowerBook")) != 0 &&
diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c
index 14610a6..dda0398 100644
--- a/drivers/macintosh/via-pmu.c
+++ b/drivers/macintosh/via-pmu.c
@@ -280,7 +280,7 @@ static char *pbook_type[] = {
 int __init find_via_pmu(void)
 {
 	u64 taddr;
-	u32 *reg;
+	const u32 *reg;
 
 	if (via != 0)
 		return 1;
@@ -288,7 +288,7 @@ int __init find_via_pmu(void)
 	if (vias == NULL)
 		return 0;
 
-	reg = (u32 *)get_property(vias, "reg", NULL);
+	reg = get_property(vias, "reg", NULL);
 	if (reg == NULL) {
 		printk(KERN_ERR "via-pmu: No \"reg\" property !\n");
 		goto fail;
@@ -330,7 +330,7 @@ int __init find_via_pmu(void)
 		
 		gpiop = of_find_node_by_name(NULL, "gpio");
 		if (gpiop) {
-			reg = (u32 *)get_property(gpiop, "reg", NULL);
+			reg = get_property(gpiop, "reg", NULL);
 			if (reg)
 				gaddr = of_translate_address(gpiop, reg);
 			if (gaddr != OF_BAD_ADDR)
@@ -479,9 +479,9 @@ static int __init via_pmu_dev_init(void)
 		pmu_batteries[1].flags |= PMU_BATT_TYPE_SMART;
 	} else {
 		struct device_node* prim = find_devices("power-mgt");
-		u32 *prim_info = NULL;
+		const u32 *prim_info = NULL;
 		if (prim)
-			prim_info = (u32 *)get_property(prim, "prim-info", NULL);
+			prim_info = get_property(prim, "prim-info", NULL);
 		if (prim_info) {
 			/* Other stuffs here yet unknown */
 			pmu_battery_count = (prim_info[6] >> 16) & 0xff;
diff --git a/drivers/macintosh/windfarm_pm81.c b/drivers/macintosh/windfarm_pm81.c
index f1df6ef..2ff546e 100644
--- a/drivers/macintosh/windfarm_pm81.c
+++ b/drivers/macintosh/windfarm_pm81.c
@@ -396,7 +396,7 @@ static void wf_smu_sys_fans_tick(struct wf_smu_sys_fans_state *st)
 static void wf_smu_create_cpu_fans(void)
 {
 	struct wf_cpu_pid_param pid_param;
-	struct smu_sdbp_header *hdr;
+	const struct smu_sdbp_header *hdr;
 	struct smu_sdbp_cpupiddata *piddata;
 	struct smu_sdbp_fvt *fvt;
 	s32 tmax, tdelta, maxpow, powadj;
@@ -702,7 +702,7 @@ static struct notifier_block wf_smu_events = {
 
 static int wf_init_pm(void)
 {
-	struct smu_sdbp_header *hdr;
+	const struct smu_sdbp_header *hdr;
 
 	hdr = smu_get_sdb_partition(SMU_SDB_SENSORTREE_ID, NULL);
 	if (hdr != 0) {
diff --git a/drivers/macintosh/windfarm_pm91.c b/drivers/macintosh/windfarm_pm91.c
index 0d6372e..59e9ffe 100644
--- a/drivers/macintosh/windfarm_pm91.c
+++ b/drivers/macintosh/windfarm_pm91.c
@@ -144,7 +144,7 @@ static struct wf_smu_slots_fans_state *wf_smu_slots_fans;
 static void wf_smu_create_cpu_fans(void)
 {
 	struct wf_cpu_pid_param pid_param;
-	struct smu_sdbp_header *hdr;
+	const struct smu_sdbp_header *hdr;
 	struct smu_sdbp_cpupiddata *piddata;
 	struct smu_sdbp_fvt *fvt;
 	s32 tmax, tdelta, maxpow, powadj;
diff --git a/drivers/macintosh/windfarm_smu_controls.c b/drivers/macintosh/windfarm_smu_controls.c
index a9e88ed..bff1f37 100644
--- a/drivers/macintosh/windfarm_smu_controls.c
+++ b/drivers/macintosh/windfarm_smu_controls.c
@@ -159,14 +159,15 @@ static struct smu_fan_control *smu_fan_create(struct device_node *node,
 					      int pwm_fan)
 {
 	struct smu_fan_control *fct;
-	s32 *v; u32 *reg;
-	char *l;
+	const s32 *v;
+	const u32 *reg;
+	const char *l;
 
 	fct = kmalloc(sizeof(struct smu_fan_control), GFP_KERNEL);
 	if (fct == NULL)
 		return NULL;
 	fct->ctrl.ops = &smu_fan_ops;
-	l = (char *)get_property(node, "location", NULL);
+	l = get_property(node, "location", NULL);
 	if (l == NULL)
 		goto fail;
 
@@ -223,17 +224,17 @@ static struct smu_fan_control *smu_fan_create(struct device_node *node,
 		goto fail;
 
 	/* Get min & max values*/
-	v = (s32 *)get_property(node, "min-value", NULL);
+	v = get_property(node, "min-value", NULL);
 	if (v == NULL)
 		goto fail;
 	fct->min = *v;
-	v = (s32 *)get_property(node, "max-value", NULL);
+	v = get_property(node, "max-value", NULL);
 	if (v == NULL)
 		goto fail;
 	fct->max = *v;
 
 	/* Get "reg" value */
-	reg = (u32 *)get_property(node, "reg", NULL);
+	reg = get_property(node, "reg", NULL);
 	if (reg == NULL)
 		goto fail;
 	fct->reg = *reg;
diff --git a/drivers/macintosh/windfarm_smu_sat.c b/drivers/macintosh/windfarm_smu_sat.c
index e295a07..aceb61d 100644
--- a/drivers/macintosh/windfarm_smu_sat.c
+++ b/drivers/macintosh/windfarm_smu_sat.c
@@ -233,15 +233,15 @@ static void wf_sat_create(struct i2c_adapter *adapter, struct device_node *dev)
 {
 	struct wf_sat *sat;
 	struct wf_sat_sensor *sens;
-	u32 *reg;
-	char *loc, *type;
+	const u32 *reg;
+	const char *loc, *type;
 	u8 addr, chip, core;
 	struct device_node *child;
 	int shift, cpu, index;
 	char *name;
 	int vsens[2], isens[2];
 
-	reg = (u32 *) get_property(dev, "reg", NULL);
+	reg = get_property(dev, "reg", NULL);
 	if (reg == NULL)
 		return;
 	addr = *reg;
@@ -268,7 +268,7 @@ static void wf_sat_create(struct i2c_adapter *adapter, struct device_node *dev)
 	isens[0] = isens[1] = -1;
 	child = NULL;
 	while ((child = of_get_next_child(dev, child)) != NULL) {
-		reg = (u32 *) get_property(child, "reg", NULL);
+		reg = get_property(child, "reg", NULL);
 		type = get_property(child, "device_type", NULL);
 		loc = get_property(child, "location", NULL);
 		if (reg == NULL || loc == NULL)
diff --git a/drivers/macintosh/windfarm_smu_sensors.c b/drivers/macintosh/windfarm_smu_sensors.c
index bed25dc..defe992 100644
--- a/drivers/macintosh/windfarm_smu_sensors.c
+++ b/drivers/macintosh/windfarm_smu_sensors.c
@@ -198,14 +198,14 @@ static struct wf_sensor_ops smu_slotspow_ops = {
 static struct smu_ad_sensor *smu_ads_create(struct device_node *node)
 {
 	struct smu_ad_sensor *ads;
-	char *c, *l;
-	u32 *v;
+	const char *c, *l;
+	const u32 *v;
 
 	ads = kmalloc(sizeof(struct smu_ad_sensor), GFP_KERNEL);
 	if (ads == NULL)
 		return NULL;
-	c = (char *)get_property(node, "device_type", NULL);
-	l = (char *)get_property(node, "location", NULL);
+	c = get_property(node, "device_type", NULL);
+	l = get_property(node, "location", NULL);
 	if (c == NULL || l == NULL)
 		goto fail;
 
@@ -255,7 +255,7 @@ static struct smu_ad_sensor *smu_ads_create(struct device_node *node)
 	} else
 		goto fail;
 
-	v = (u32 *)get_property(node, "reg", NULL);
+	v = get_property(node, "reg", NULL);
 	if (v == NULL)
 		goto fail;
 	ads->reg = *v;
@@ -382,7 +382,7 @@ smu_cpu_power_create(struct wf_sensor *volts, struct wf_sensor *amps)
 
 static void smu_fetch_param_partitions(void)
 {
-	struct smu_sdbp_header *hdr;
+	const struct smu_sdbp_header *hdr;
 
 	/* Get CPU voltage/current/power calibration data */
 	hdr = smu_get_sdb_partition(SMU_SDB_CPUVCP_ID, NULL);
diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c
index 6022ed1..bdbd349 100644
--- a/drivers/md/dm-crypt.c
+++ b/drivers/md/dm-crypt.c
@@ -5,6 +5,7 @@
  * This file is released under the GPL.
  */
 
+#include <linux/err.h>
 #include <linux/module.h>
 #include <linux/init.h>
 #include <linux/kernel.h>
@@ -78,11 +79,13 @@ struct crypt_config {
 	 */
 	struct crypt_iv_operations *iv_gen_ops;
 	char *iv_mode;
-	void *iv_gen_private;
+	struct crypto_cipher *iv_gen_private;
 	sector_t iv_offset;
 	unsigned int iv_size;
 
-	struct crypto_tfm *tfm;
+	char cipher[CRYPTO_MAX_ALG_NAME];
+	char chainmode[CRYPTO_MAX_ALG_NAME];
+	struct crypto_blkcipher *tfm;
 	unsigned int key_size;
 	u8 key[0];
 };
@@ -96,12 +99,12 @@ static kmem_cache_t *_crypt_io_pool;
 /*
  * Different IV generation algorithms:
  *
- * plain: the initial vector is the 32-bit low-endian version of the sector
+ * plain: the initial vector is the 32-bit little-endian version of the sector
  *        number, padded with zeros if neccessary.
  *
- * ess_iv: "encrypted sector|salt initial vector", the sector number is
- *         encrypted with the bulk cipher using a salt as key. The salt
- *         should be derived from the bulk cipher's key via hashing.
+ * essiv: "encrypted sector|salt initial vector", the sector number is
+ *        encrypted with the bulk cipher using a salt as key. The salt
+ *        should be derived from the bulk cipher's key via hashing.
  *
  * plumb: unimplemented, see:
  * http://article.gmane.org/gmane.linux.kernel.device-mapper.dm-crypt/454
@@ -118,11 +121,13 @@ static int crypt_iv_plain_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
 static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
 	                      const char *opts)
 {
-	struct crypto_tfm *essiv_tfm;
-	struct crypto_tfm *hash_tfm;
+	struct crypto_cipher *essiv_tfm;
+	struct crypto_hash *hash_tfm;
+	struct hash_desc desc;
 	struct scatterlist sg;
 	unsigned int saltsize;
 	u8 *salt;
+	int err;
 
 	if (opts == NULL) {
 		ti->error = "Digest algorithm missing for ESSIV mode";
@@ -130,76 +135,70 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti,
 	}
 
 	/* Hash the cipher key with the given hash algorithm */
-	hash_tfm = crypto_alloc_tfm(opts, CRYPTO_TFM_REQ_MAY_SLEEP);
-	if (hash_tfm == NULL) {
+	hash_tfm = crypto_alloc_hash(opts, 0, CRYPTO_ALG_ASYNC);
+	if (IS_ERR(hash_tfm)) {
 		ti->error = "Error initializing ESSIV hash";
-		return -EINVAL;
+		return PTR_ERR(hash_tfm);
 	}
 
-	if (crypto_tfm_alg_type(hash_tfm) != CRYPTO_ALG_TYPE_DIGEST) {
-		ti->error = "Expected digest algorithm for ESSIV hash";
-		crypto_free_tfm(hash_tfm);
-		return -EINVAL;
-	}
-
-	saltsize = crypto_tfm_alg_digestsize(hash_tfm);
+	saltsize = crypto_hash_digestsize(hash_tfm);
 	salt = kmalloc(saltsize, GFP_KERNEL);
 	if (salt == NULL) {
 		ti->error = "Error kmallocing salt storage in ESSIV";
-		crypto_free_tfm(hash_tfm);
+		crypto_free_hash(hash_tfm);
 		return -ENOMEM;
 	}
 
 	sg_set_buf(&sg, cc->key, cc->key_size);
-	crypto_digest_digest(hash_tfm, &sg, 1, salt);
-	crypto_free_tfm(hash_tfm);
+	desc.tfm = hash_tfm;
+	desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP;
+	err = crypto_hash_digest(&desc, &sg, cc->key_size, salt);
+	crypto_free_hash(hash_tfm);
+
+	if (err) {
+		ti->error = "Error calculating hash in ESSIV";
+		return err;
+	}
 
 	/* Setup the essiv_tfm with the given salt */
-	essiv_tfm = crypto_alloc_tfm(crypto_tfm_alg_name(cc->tfm),
-	                             CRYPTO_TFM_MODE_ECB |
-	                             CRYPTO_TFM_REQ_MAY_SLEEP);
-	if (essiv_tfm == NULL) {
+	essiv_tfm = crypto_alloc_cipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
+	if (IS_ERR(essiv_tfm)) {
 		ti->error = "Error allocating crypto tfm for ESSIV";
 		kfree(salt);
-		return -EINVAL;
+		return PTR_ERR(essiv_tfm);
 	}
-	if (crypto_tfm_alg_blocksize(essiv_tfm)
-	    != crypto_tfm_alg_ivsize(cc->tfm)) {
+	if (crypto_cipher_blocksize(essiv_tfm) !=
+	    crypto_blkcipher_ivsize(cc->tfm)) {
 		ti->error = "Block size of ESSIV cipher does "
 			        "not match IV size of block cipher";
-		crypto_free_tfm(essiv_tfm);
+		crypto_free_cipher(essiv_tfm);
 		kfree(salt);
 		return -EINVAL;
 	}
-	if (crypto_cipher_setkey(essiv_tfm, salt, saltsize) < 0) {
+	err = crypto_cipher_setkey(essiv_tfm, salt, saltsize);
+	if (err) {
 		ti->error = "Failed to set key for ESSIV cipher";
-		crypto_free_tfm(essiv_tfm);
+		crypto_free_cipher(essiv_tfm);
 		kfree(salt);
-		return -EINVAL;
+		return err;
 	}
 	kfree(salt);
 
-	cc->iv_gen_private = (void *)essiv_tfm;
+	cc->iv_gen_private = essiv_tfm;
 	return 0;
 }
 
 static void crypt_iv_essiv_dtr(struct crypt_config *cc)
 {
-	crypto_free_tfm((struct crypto_tfm *)cc->iv_gen_private);
+	crypto_free_cipher(cc->iv_gen_private);
 	cc->iv_gen_private = NULL;
 }
 
 static int crypt_iv_essiv_gen(struct crypt_config *cc, u8 *iv, sector_t sector)
 {
-	struct scatterlist sg;
-
 	memset(iv, 0, cc->iv_size);
 	*(u64 *)iv = cpu_to_le64(sector);
-
-	sg_set_buf(&sg, iv, cc->iv_size);
-	crypto_cipher_encrypt((struct crypto_tfm *)cc->iv_gen_private,
-	                      &sg, &sg, cc->iv_size);
-
+	crypto_cipher_encrypt_one(cc->iv_gen_private, iv, iv);
 	return 0;
 }
 
@@ -220,6 +219,11 @@ crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out,
                           int write, sector_t sector)
 {
 	u8 iv[cc->iv_size];
+	struct blkcipher_desc desc = {
+		.tfm = cc->tfm,
+		.info = iv,
+		.flags = CRYPTO_TFM_REQ_MAY_SLEEP,
+	};
 	int r;
 
 	if (cc->iv_gen_ops) {
@@ -228,14 +232,14 @@ crypt_convert_scatterlist(struct crypt_config *cc, struct scatterlist *out,
 			return r;
 
 		if (write)
-			r = crypto_cipher_encrypt_iv(cc->tfm, out, in, length, iv);
+			r = crypto_blkcipher_encrypt_iv(&desc, out, in, length);
 		else
-			r = crypto_cipher_decrypt_iv(cc->tfm, out, in, length, iv);
+			r = crypto_blkcipher_decrypt_iv(&desc, out, in, length);
 	} else {
 		if (write)
-			r = crypto_cipher_encrypt(cc->tfm, out, in, length);
+			r = crypto_blkcipher_encrypt(&desc, out, in, length);
 		else
-			r = crypto_cipher_decrypt(cc->tfm, out, in, length);
+			r = crypto_blkcipher_decrypt(&desc, out, in, length);
 	}
 
 	return r;
@@ -510,13 +514,12 @@ static void crypt_encode_key(char *hex, u8 *key, unsigned int size)
 static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 {
 	struct crypt_config *cc;
-	struct crypto_tfm *tfm;
+	struct crypto_blkcipher *tfm;
 	char *tmp;
 	char *cipher;
 	char *chainmode;
 	char *ivmode;
 	char *ivopts;
-	unsigned int crypto_flags;
 	unsigned int key_size;
 	unsigned long long tmpll;
 
@@ -556,31 +559,25 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		ivmode = "plain";
 	}
 
-	/* Choose crypto_flags according to chainmode */
-	if (strcmp(chainmode, "cbc") == 0)
-		crypto_flags = CRYPTO_TFM_MODE_CBC;
-	else if (strcmp(chainmode, "ecb") == 0)
-		crypto_flags = CRYPTO_TFM_MODE_ECB;
-	else {
-		ti->error = "Unknown chaining mode";
+	if (strcmp(chainmode, "ecb") && !ivmode) {
+		ti->error = "This chaining mode requires an IV mechanism";
 		goto bad1;
 	}
 
-	if (crypto_flags != CRYPTO_TFM_MODE_ECB && !ivmode) {
-		ti->error = "This chaining mode requires an IV mechanism";
+	if (snprintf(cc->cipher, CRYPTO_MAX_ALG_NAME, "%s(%s)", chainmode, 
+		     cipher) >= CRYPTO_MAX_ALG_NAME) {
+		ti->error = "Chain mode + cipher name is too long";
 		goto bad1;
 	}
 
-	tfm = crypto_alloc_tfm(cipher, crypto_flags | CRYPTO_TFM_REQ_MAY_SLEEP);
-	if (!tfm) {
+	tfm = crypto_alloc_blkcipher(cc->cipher, 0, CRYPTO_ALG_ASYNC);
+	if (IS_ERR(tfm)) {
 		ti->error = "Error allocating crypto tfm";
 		goto bad1;
 	}
-	if (crypto_tfm_alg_type(tfm) != CRYPTO_ALG_TYPE_CIPHER) {
-		ti->error = "Expected cipher algorithm";
-		goto bad2;
-	}
 
+	strcpy(cc->cipher, cipher);
+	strcpy(cc->chainmode, chainmode);
 	cc->tfm = tfm;
 
 	/*
@@ -603,12 +600,12 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 	    cc->iv_gen_ops->ctr(cc, ti, ivopts) < 0)
 		goto bad2;
 
-	if (tfm->crt_cipher.cit_decrypt_iv && tfm->crt_cipher.cit_encrypt_iv)
+	cc->iv_size = crypto_blkcipher_ivsize(tfm);
+	if (cc->iv_size)
 		/* at least a 64 bit sector number should fit in our buffer */
-		cc->iv_size = max(crypto_tfm_alg_ivsize(tfm),
+		cc->iv_size = max(cc->iv_size,
 		                  (unsigned int)(sizeof(u64) / sizeof(u8)));
 	else {
-		cc->iv_size = 0;
 		if (cc->iv_gen_ops) {
 			DMWARN("Selected cipher does not support IVs");
 			if (cc->iv_gen_ops->dtr)
@@ -629,7 +626,7 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 		goto bad4;
 	}
 
-	if (tfm->crt_cipher.cit_setkey(tfm, cc->key, key_size) < 0) {
+	if (crypto_blkcipher_setkey(tfm, cc->key, key_size) < 0) {
 		ti->error = "Error setting key";
 		goto bad5;
 	}
@@ -675,7 +672,7 @@ bad3:
 	if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
 		cc->iv_gen_ops->dtr(cc);
 bad2:
-	crypto_free_tfm(tfm);
+	crypto_free_blkcipher(tfm);
 bad1:
 	/* Must zero key material before freeing */
 	memset(cc, 0, sizeof(*cc) + cc->key_size * sizeof(u8));
@@ -693,7 +690,7 @@ static void crypt_dtr(struct dm_target *ti)
 	kfree(cc->iv_mode);
 	if (cc->iv_gen_ops && cc->iv_gen_ops->dtr)
 		cc->iv_gen_ops->dtr(cc);
-	crypto_free_tfm(cc->tfm);
+	crypto_free_blkcipher(cc->tfm);
 	dm_put_device(ti, cc->dev);
 
 	/* Must zero key material before freeing */
@@ -858,18 +855,9 @@ static int crypt_status(struct dm_target *ti, status_type_t type,
 		break;
 
 	case STATUSTYPE_TABLE:
-		cipher = crypto_tfm_alg_name(cc->tfm);
+		cipher = crypto_blkcipher_name(cc->tfm);
 
-		switch(cc->tfm->crt_cipher.cit_mode) {
-		case CRYPTO_TFM_MODE_CBC:
-			chainmode = "cbc";
-			break;
-		case CRYPTO_TFM_MODE_ECB:
-			chainmode = "ecb";
-			break;
-		default:
-			BUG();
-		}
+		chainmode = cc->chainmode;
 
 		if (cc->iv_mode)
 			DMEMIT("%s-%s-%s ", cipher, chainmode, cc->iv_mode);
diff --git a/drivers/message/fusion/mptfc.c b/drivers/message/fusion/mptfc.c
index 85696f3..e57bb03 100644
--- a/drivers/message/fusion/mptfc.c
+++ b/drivers/message/fusion/mptfc.c
@@ -162,7 +162,13 @@ static struct fc_function_template mptfc_transport_functions = {
 	.show_starget_port_id = 1,
 	.set_rport_dev_loss_tmo = mptfc_set_rport_loss_tmo,
 	.show_rport_dev_loss_tmo = 1,
-
+	.show_host_supported_speeds = 1,
+	.show_host_maxframe_size = 1,
+	.show_host_speed = 1,
+	.show_host_fabric_name = 1,
+	.show_host_port_type = 1,
+	.show_host_port_state = 1,
+	.show_host_symbolic_name = 1,
 };
 
 static void
@@ -839,33 +845,95 @@ mptfc_SetFcPortPage1_defaults(MPT_ADAPTER *ioc)
 static void
 mptfc_init_host_attr(MPT_ADAPTER *ioc,int portnum)
 {
-	unsigned class = 0, cos = 0;
+	unsigned	class = 0;
+	unsigned	cos = 0;
+	unsigned	speed;
+	unsigned	port_type;
+	unsigned	port_state;
+	FCPortPage0_t	*pp0;
+	struct Scsi_Host *sh;
+	char		*sn;
 
 	/* don't know what to do as only one scsi (fc) host was allocated */
 	if (portnum != 0)
 		return;
 
-	class = ioc->fc_port_page0[portnum].SupportedServiceClass;
+	pp0 = &ioc->fc_port_page0[portnum];
+	sh = ioc->sh;
+
+	sn = fc_host_symbolic_name(sh);
+	snprintf(sn, FC_SYMBOLIC_NAME_SIZE, "%s %s%08xh",
+	    ioc->prod_name,
+	    MPT_FW_REV_MAGIC_ID_STRING,
+	    ioc->facts.FWVersion.Word);
+
+	fc_host_tgtid_bind_type(sh) = FC_TGTID_BIND_BY_WWPN;
+
+	fc_host_maxframe_size(sh) = pp0->MaxFrameSize;
+
+	fc_host_node_name(sh) =
+	    	(u64)pp0->WWNN.High << 32 | (u64)pp0->WWNN.Low;
+
+	fc_host_port_name(sh) =
+	    	(u64)pp0->WWPN.High << 32 | (u64)pp0->WWPN.Low;
+
+	fc_host_port_id(sh) = pp0->PortIdentifier;
+
+	class = pp0->SupportedServiceClass;
 	if (class & MPI_FCPORTPAGE0_SUPPORT_CLASS_1)
 		cos |= FC_COS_CLASS1;
 	if (class & MPI_FCPORTPAGE0_SUPPORT_CLASS_2)
 		cos |= FC_COS_CLASS2;
 	if (class & MPI_FCPORTPAGE0_SUPPORT_CLASS_3)
 		cos |= FC_COS_CLASS3;
+	fc_host_supported_classes(sh) = cos;
+
+	if (pp0->CurrentSpeed == MPI_FCPORTPAGE0_CURRENT_SPEED_1GBIT)
+		speed = FC_PORTSPEED_1GBIT;
+	else if (pp0->CurrentSpeed == MPI_FCPORTPAGE0_CURRENT_SPEED_2GBIT)
+		speed = FC_PORTSPEED_2GBIT;
+	else if (pp0->CurrentSpeed == MPI_FCPORTPAGE0_CURRENT_SPEED_4GBIT)
+		speed = FC_PORTSPEED_4GBIT;
+	else if (pp0->CurrentSpeed == MPI_FCPORTPAGE0_CURRENT_SPEED_10GBIT)
+		speed = FC_PORTSPEED_10GBIT;
+	else
+		speed = FC_PORTSPEED_UNKNOWN;
+	fc_host_speed(sh) = speed;
+
+	speed = 0;
+	if (pp0->SupportedSpeeds & MPI_FCPORTPAGE0_SUPPORT_1GBIT_SPEED)
+		speed |= FC_PORTSPEED_1GBIT;
+	if (pp0->SupportedSpeeds & MPI_FCPORTPAGE0_SUPPORT_2GBIT_SPEED)
+		speed |= FC_PORTSPEED_2GBIT;
+	if (pp0->SupportedSpeeds & MPI_FCPORTPAGE0_SUPPORT_4GBIT_SPEED)
+		speed |= FC_PORTSPEED_4GBIT;
+	if (pp0->SupportedSpeeds & MPI_FCPORTPAGE0_SUPPORT_10GBIT_SPEED)
+		speed |= FC_PORTSPEED_10GBIT;
+	fc_host_supported_speeds(sh) = speed;
+
+	port_state = FC_PORTSTATE_UNKNOWN;
+	if (pp0->PortState == MPI_FCPORTPAGE0_PORTSTATE_ONLINE)
+		port_state = FC_PORTSTATE_ONLINE;
+	else if (pp0->PortState == MPI_FCPORTPAGE0_PORTSTATE_OFFLINE)
+		port_state = FC_PORTSTATE_LINKDOWN;
+	fc_host_port_state(sh) = port_state;
+
+	port_type = FC_PORTTYPE_UNKNOWN;
+	if (pp0->Flags & MPI_FCPORTPAGE0_FLAGS_ATTACH_POINT_TO_POINT)
+		port_type = FC_PORTTYPE_PTP;
+	else if (pp0->Flags & MPI_FCPORTPAGE0_FLAGS_ATTACH_PRIVATE_LOOP)
+		port_type = FC_PORTTYPE_LPORT;
+	else if (pp0->Flags & MPI_FCPORTPAGE0_FLAGS_ATTACH_PUBLIC_LOOP)
+		port_type = FC_PORTTYPE_NLPORT;
+	else if (pp0->Flags & MPI_FCPORTPAGE0_FLAGS_ATTACH_FABRIC_DIRECT)
+		port_type = FC_PORTTYPE_NPORT;
+	fc_host_port_type(sh) = port_type;
+
+	fc_host_fabric_name(sh) =
+	    (pp0->Flags & MPI_FCPORTPAGE0_FLAGS_FABRIC_WWN_VALID) ?
+		(u64) pp0->FabricWWNN.High << 32 | (u64) pp0->FabricWWPN.Low :
+		(u64)pp0->WWNN.High << 32 | (u64)pp0->WWNN.Low;
 
-	fc_host_node_name(ioc->sh) =
-	    	(u64)ioc->fc_port_page0[portnum].WWNN.High << 32
-		    | (u64)ioc->fc_port_page0[portnum].WWNN.Low;
-
-	fc_host_port_name(ioc->sh) =
-	    	(u64)ioc->fc_port_page0[portnum].WWPN.High << 32
-		    | (u64)ioc->fc_port_page0[portnum].WWPN.Low;
-
-	fc_host_port_id(ioc->sh) = ioc->fc_port_page0[portnum].PortIdentifier;
-
-	fc_host_supported_classes(ioc->sh) = cos;
-
-	fc_host_tgtid_bind_type(ioc->sh) = FC_TGTID_BIND_BY_WWPN;
 }
 
 static void
diff --git a/drivers/message/fusion/mptsas.c b/drivers/message/fusion/mptsas.c
index f66f220..b752a47 100644
--- a/drivers/message/fusion/mptsas.c
+++ b/drivers/message/fusion/mptsas.c
@@ -852,6 +852,10 @@ static int mptsas_get_linkerrors(struct sas_phy *phy)
 	dma_addr_t dma_handle;
 	int error;
 
+	/* FIXME: only have link errors on local phys */
+	if (!scsi_is_sas_phy_local(phy))
+		return -EINVAL;
+
 	hdr.PageVersion = MPI_SASPHY1_PAGEVERSION;
 	hdr.ExtPageLength = 0;
 	hdr.PageNumber = 1 /* page number 1*/;
@@ -924,6 +928,10 @@ static int mptsas_phy_reset(struct sas_phy *phy, int hard_reset)
 	unsigned long timeleft;
 	int error = -ERESTARTSYS;
 
+	/* FIXME: fusion doesn't allow non-local phy reset */
+	if (!scsi_is_sas_phy_local(phy))
+		return -EINVAL;
+
 	/* not implemented for expanders */
 	if (phy->identify.target_port_protocols & SAS_PROTOCOL_SMP)
 		return -ENXIO;
@@ -1570,9 +1578,6 @@ static int mptsas_probe_one_phy(struct device *dev,
 
 	if (!phy_info->phy) {
 
-		if (local)
-			phy->local_attached = 1;
-
 		error = sas_phy_add(phy);
 		if (error) {
 			sas_phy_free(phy);
@@ -1642,14 +1647,18 @@ static int mptsas_probe_one_phy(struct device *dev,
 
 			for (i = 0; i < port_info->num_phys; i++)
 				if (port_info->phy_info[i].identify.sas_address ==
-				    identify.sas_address)
+				    identify.sas_address) {
+					sas_port_mark_backlink(port);
 					goto out;
+				}
 
 		} else if (scsi_is_sas_rphy(parent)) {
 			struct sas_rphy *parent_rphy = dev_to_rphy(parent);
 			if (identify.sas_address ==
-			    parent_rphy->identify.sas_address)
+			    parent_rphy->identify.sas_address) {
+				sas_port_mark_backlink(port);
 				goto out;
+			}
 		}
 
 		switch (identify.device_type) {
diff --git a/drivers/mtd/Kconfig b/drivers/mtd/Kconfig
index 1344ad7..a03e862 100644
--- a/drivers/mtd/Kconfig
+++ b/drivers/mtd/Kconfig
@@ -101,7 +101,7 @@ config MTD_REDBOOT_PARTS_READONLY
 
 config MTD_CMDLINE_PARTS
 	bool "Command line partition table parsing"
-	depends on MTD_PARTITIONS = "y"
+	depends on MTD_PARTITIONS = "y" && MTD = "y"
 	---help---
 	  Allow generic configuration of the MTD partition tables via the kernel
 	  command line. Multiple flash resources are supported for hardware where
@@ -263,6 +263,14 @@ config RFD_FTL
 
 		http://www.gensw.com/pages/prod/bios/rfd.htm
 
+config SSFDC
+	tristate "NAND SSFDC (SmartMedia) read only translation layer"
+	depends on MTD
+	default n
+	help
+	  This enables read only access to SmartMedia formatted NAND
+	  flash. You can mount it with FAT file system.
+
 source "drivers/mtd/chips/Kconfig"
 
 source "drivers/mtd/maps/Kconfig"
diff --git a/drivers/mtd/Makefile b/drivers/mtd/Makefile
index fc93744..1e36b9a 100644
--- a/drivers/mtd/Makefile
+++ b/drivers/mtd/Makefile
@@ -21,6 +21,7 @@ obj-$(CONFIG_FTL)		+= ftl.o mtd_blkdevs.o
 obj-$(CONFIG_NFTL)		+= nftl.o mtd_blkdevs.o
 obj-$(CONFIG_INFTL)		+= inftl.o mtd_blkdevs.o
 obj-$(CONFIG_RFD_FTL)		+= rfd_ftl.o mtd_blkdevs.o
+obj-$(CONFIG_SSFDC)		+= ssfdc.o mtd_blkdevs.o
 
 nftl-objs		:= nftlcore.o nftlmount.o
 inftl-objs		:= inftlcore.o inftlmount.o
diff --git a/drivers/mtd/chips/cfi_cmdset_0001.c b/drivers/mtd/chips/cfi_cmdset_0001.c
index 39edb82..7ea49a0 100644
--- a/drivers/mtd/chips/cfi_cmdset_0001.c
+++ b/drivers/mtd/chips/cfi_cmdset_0001.c
@@ -908,7 +908,7 @@ static void __xipram xip_enable(struct map_info *map, struct flchip *chip,
 
 static int __xipram xip_wait_for_operation(
 		struct map_info *map, struct flchip *chip,
-		unsigned long adr, int *chip_op_time )
+		unsigned long adr, unsigned int chip_op_time )
 {
 	struct cfi_private *cfi = map->fldrv_priv;
 	struct cfi_pri_intelext *cfip = cfi->cmdset_priv;
@@ -917,7 +917,7 @@ static int __xipram xip_wait_for_operation(
 	flstate_t oldstate, newstate;
 
        	start = xip_currtime();
-	usec = *chip_op_time * 8;
+	usec = chip_op_time * 8;
 	if (usec == 0)
 		usec = 500000;
 	done = 0;
@@ -1027,8 +1027,8 @@ static int __xipram xip_wait_for_operation(
 #define XIP_INVAL_CACHED_RANGE(map, from, size)  \
 	INVALIDATE_CACHED_RANGE(map, from, size)
 
-#define INVAL_CACHE_AND_WAIT(map, chip, cmd_adr, inval_adr, inval_len, p_usec) \
-	xip_wait_for_operation(map, chip, cmd_adr, p_usec)
+#define INVAL_CACHE_AND_WAIT(map, chip, cmd_adr, inval_adr, inval_len, usec) \
+	xip_wait_for_operation(map, chip, cmd_adr, usec)
 
 #else
 
@@ -1040,64 +1040,64 @@ static int __xipram xip_wait_for_operation(
 static int inval_cache_and_wait_for_operation(
 		struct map_info *map, struct flchip *chip,
 		unsigned long cmd_adr, unsigned long inval_adr, int inval_len,
-		int *chip_op_time )
+		unsigned int chip_op_time)
 {
 	struct cfi_private *cfi = map->fldrv_priv;
 	map_word status, status_OK = CMD(0x80);
-	int z, chip_state = chip->state;
-	unsigned long timeo;
+	int chip_state = chip->state;
+	unsigned int timeo, sleep_time;
 
 	spin_unlock(chip->mutex);
 	if (inval_len)
 		INVALIDATE_CACHED_RANGE(map, inval_adr, inval_len);
-	if (*chip_op_time)
-		cfi_udelay(*chip_op_time);
 	spin_lock(chip->mutex);
 
-	timeo = *chip_op_time * 8 * HZ / 1000000;
-	if (timeo < HZ/2)
-		timeo = HZ/2;
-	timeo += jiffies;
+	/* set our timeout to 8 times the expected delay */
+	timeo = chip_op_time * 8;
+	if (!timeo)
+		timeo = 500000;
+	sleep_time = chip_op_time / 2;
 
-	z = 0;
 	for (;;) {
-		if (chip->state != chip_state) {
-			/* Someone's suspended the operation: sleep */
-			DECLARE_WAITQUEUE(wait, current);
-
-			set_current_state(TASK_UNINTERRUPTIBLE);
-			add_wait_queue(&chip->wq, &wait);
-			spin_unlock(chip->mutex);
-			schedule();
-			remove_wait_queue(&chip->wq, &wait);
-			timeo = jiffies + (HZ / 2); /* FIXME */
-			spin_lock(chip->mutex);
-			continue;
-		}
-
 		status = map_read(map, cmd_adr);
 		if (map_word_andequal(map, status, status_OK, status_OK))
 			break;
 
-		/* OK Still waiting */
-		if (time_after(jiffies, timeo)) {
+		if (!timeo) {
 			map_write(map, CMD(0x70), cmd_adr);
 			chip->state = FL_STATUS;
 			return -ETIME;
 		}
 
-		/* Latency issues. Drop the lock, wait a while and retry */
-		z++;
+		/* OK Still waiting. Drop the lock, wait a while and retry. */
 		spin_unlock(chip->mutex);
-		cfi_udelay(1);
+		if (sleep_time >= 1000000/HZ) {
+			/*
+			 * Half of the normal delay still remaining
+			 * can be performed with a sleeping delay instead
+			 * of busy waiting.
+			 */
+			msleep(sleep_time/1000);
+			timeo -= sleep_time;
+			sleep_time = 1000000/HZ;
+		} else {
+			udelay(1);
+			cond_resched();
+			timeo--;
+		}
 		spin_lock(chip->mutex);
-	}
 
-	if (!z) {
-		if (!--(*chip_op_time))
-			*chip_op_time = 1;
-	} else if (z > 1)
-		++(*chip_op_time);
+		if (chip->state != chip_state) {
+			/* Someone's suspended the operation: sleep */
+			DECLARE_WAITQUEUE(wait, current);
+			set_current_state(TASK_UNINTERRUPTIBLE);
+			add_wait_queue(&chip->wq, &wait);
+			spin_unlock(chip->mutex);
+			schedule();
+			remove_wait_queue(&chip->wq, &wait);
+			spin_lock(chip->mutex);
+		}
+	}
 
 	/* Done and happy. */
  	chip->state = FL_STATUS;
@@ -1107,8 +1107,7 @@ static int inval_cache_and_wait_for_operation(
 #endif
 
 #define WAIT_TIMEOUT(map, chip, adr, udelay) \
-	({ int __udelay = (udelay); \
-	   INVAL_CACHE_AND_WAIT(map, chip, adr, 0, 0, &__udelay); })
+	INVAL_CACHE_AND_WAIT(map, chip, adr, 0, 0, udelay);
 
 
 static int do_point_onechip (struct map_info *map, struct flchip *chip, loff_t adr, size_t len)
@@ -1332,7 +1331,7 @@ static int __xipram do_write_oneword(struct map_info *map, struct flchip *chip,
 
 	ret = INVAL_CACHE_AND_WAIT(map, chip, adr,
 				   adr, map_bankwidth(map),
-				   &chip->word_write_time);
+				   chip->word_write_time);
 	if (ret) {
 		xip_enable(map, chip, adr);
 		printk(KERN_ERR "%s: word write error (status timeout)\n", map->name);
@@ -1569,7 +1568,7 @@ static int __xipram do_write_buffer(struct map_info *map, struct flchip *chip,
 
 	ret = INVAL_CACHE_AND_WAIT(map, chip, cmd_adr,
 				   adr, len,
-				   &chip->buffer_write_time);
+				   chip->buffer_write_time);
 	if (ret) {
 		map_write(map, CMD(0x70), cmd_adr);
 		chip->state = FL_STATUS;
@@ -1704,7 +1703,7 @@ static int __xipram do_erase_oneblock(struct map_info *map, struct flchip *chip,
 
 	ret = INVAL_CACHE_AND_WAIT(map, chip, adr,
 				   adr, len,
-				   &chip->erase_time);
+				   chip->erase_time);
 	if (ret) {
 		map_write(map, CMD(0x70), adr);
 		chip->state = FL_STATUS;
diff --git a/drivers/mtd/chips/cfi_cmdset_0002.c b/drivers/mtd/chips/cfi_cmdset_0002.c
index 9885726..702ae4c 100644
--- a/drivers/mtd/chips/cfi_cmdset_0002.c
+++ b/drivers/mtd/chips/cfi_cmdset_0002.c
@@ -45,9 +45,11 @@
 #define MAX_WORD_RETRIES 3
 
 #define MANUFACTURER_AMD	0x0001
+#define MANUFACTURER_ATMEL	0x001F
 #define MANUFACTURER_SST	0x00BF
 #define SST49LF004B	        0x0060
 #define SST49LF008A		0x005a
+#define AT49BV6416		0x00d6
 
 static int cfi_amdstd_read (struct mtd_info *, loff_t, size_t, size_t *, u_char *);
 static int cfi_amdstd_write_words(struct mtd_info *, loff_t, size_t, size_t *, const u_char *);
@@ -68,6 +70,9 @@ static int get_chip(struct map_info *map, struct flchip *chip, unsigned long adr
 static void put_chip(struct map_info *map, struct flchip *chip, unsigned long adr);
 #include "fwh_lock.h"
 
+static int cfi_atmel_lock(struct mtd_info *mtd, loff_t ofs, size_t len);
+static int cfi_atmel_unlock(struct mtd_info *mtd, loff_t ofs, size_t len);
+
 static struct mtd_chip_driver cfi_amdstd_chipdrv = {
 	.probe		= NULL, /* Not usable directly */
 	.destroy	= cfi_amdstd_destroy,
@@ -161,6 +166,26 @@ static void fixup_use_write_buffers(struct mtd_info *mtd, void *param)
 	}
 }
 
+/* Atmel chips don't use the same PRI format as AMD chips */
+static void fixup_convert_atmel_pri(struct mtd_info *mtd, void *param)
+{
+	struct map_info *map = mtd->priv;
+	struct cfi_private *cfi = map->fldrv_priv;
+	struct cfi_pri_amdstd *extp = cfi->cmdset_priv;
+	struct cfi_pri_atmel atmel_pri;
+
+	memcpy(&atmel_pri, extp, sizeof(atmel_pri));
+	memset((char *)extp + 5, 0, sizeof(*extp) - 5);
+
+	if (atmel_pri.Features & 0x02)
+		extp->EraseSuspend = 2;
+
+	if (atmel_pri.BottomBoot)
+		extp->TopBottom = 2;
+	else
+		extp->TopBottom = 3;
+}
+
 static void fixup_use_secsi(struct mtd_info *mtd, void *param)
 {
 	/* Setup for chips with a secsi area */
@@ -179,6 +204,17 @@ static void fixup_use_erase_chip(struct mtd_info *mtd, void *param)
 
 }
 
+/*
+ * Some Atmel chips (e.g. the AT49BV6416) power-up with all sectors
+ * locked by default.
+ */
+static void fixup_use_atmel_lock(struct mtd_info *mtd, void *param)
+{
+	mtd->lock = cfi_atmel_lock;
+	mtd->unlock = cfi_atmel_unlock;
+	mtd->flags |= MTD_STUPID_LOCK;
+}
+
 static struct cfi_fixup cfi_fixup_table[] = {
 #ifdef AMD_BOOTLOC_BUG
 	{ CFI_MFR_AMD, CFI_ID_ANY, fixup_amd_bootblock, NULL },
@@ -192,6 +228,7 @@ static struct cfi_fixup cfi_fixup_table[] = {
 #if !FORCE_WORD_WRITE
 	{ CFI_MFR_ANY, CFI_ID_ANY, fixup_use_write_buffers, NULL, },
 #endif
+	{ CFI_MFR_ATMEL, CFI_ID_ANY, fixup_convert_atmel_pri, NULL },
 	{ 0, 0, NULL, NULL }
 };
 static struct cfi_fixup jedec_fixup_table[] = {
@@ -207,6 +244,7 @@ static struct cfi_fixup fixup_table[] = {
 	 * we know that is the case.
 	 */
 	{ CFI_MFR_ANY, CFI_ID_ANY, fixup_use_erase_chip, NULL },
+	{ CFI_MFR_ATMEL, AT49BV6416, fixup_use_atmel_lock, NULL },
 	{ 0, 0, NULL, NULL }
 };
 
@@ -1607,6 +1645,80 @@ static int cfi_amdstd_erase_chip(struct mtd_info *mtd, struct erase_info *instr)
 	return 0;
 }
 
+static int do_atmel_lock(struct map_info *map, struct flchip *chip,
+			 unsigned long adr, int len, void *thunk)
+{
+	struct cfi_private *cfi = map->fldrv_priv;
+	int ret;
+
+	spin_lock(chip->mutex);
+	ret = get_chip(map, chip, adr + chip->start, FL_LOCKING);
+	if (ret)
+		goto out_unlock;
+	chip->state = FL_LOCKING;
+
+	DEBUG(MTD_DEBUG_LEVEL3, "MTD %s(): LOCK 0x%08lx len %d\n",
+	      __func__, adr, len);
+
+	cfi_send_gen_cmd(0xAA, cfi->addr_unlock1, chip->start, map, cfi,
+			 cfi->device_type, NULL);
+	cfi_send_gen_cmd(0x55, cfi->addr_unlock2, chip->start, map, cfi,
+			 cfi->device_type, NULL);
+	cfi_send_gen_cmd(0x80, cfi->addr_unlock1, chip->start, map, cfi,
+			 cfi->device_type, NULL);
+	cfi_send_gen_cmd(0xAA, cfi->addr_unlock1, chip->start, map, cfi,
+			 cfi->device_type, NULL);
+	cfi_send_gen_cmd(0x55, cfi->addr_unlock2, chip->start, map, cfi,
+			 cfi->device_type, NULL);
+	map_write(map, CMD(0x40), chip->start + adr);
+
+	chip->state = FL_READY;
+	put_chip(map, chip, adr + chip->start);
+	ret = 0;
+
+out_unlock:
+	spin_unlock(chip->mutex);
+	return ret;
+}
+
+static int do_atmel_unlock(struct map_info *map, struct flchip *chip,
+			   unsigned long adr, int len, void *thunk)
+{
+	struct cfi_private *cfi = map->fldrv_priv;
+	int ret;
+
+	spin_lock(chip->mutex);
+	ret = get_chip(map, chip, adr + chip->start, FL_UNLOCKING);
+	if (ret)
+		goto out_unlock;
+	chip->state = FL_UNLOCKING;
+
+	DEBUG(MTD_DEBUG_LEVEL3, "MTD %s(): LOCK 0x%08lx len %d\n",
+	      __func__, adr, len);
+
+	cfi_send_gen_cmd(0xAA, cfi->addr_unlock1, chip->start, map, cfi,
+			 cfi->device_type, NULL);
+	map_write(map, CMD(0x70), adr);
+
+	chip->state = FL_READY;
+	put_chip(map, chip, adr + chip->start);
+	ret = 0;
+
+out_unlock:
+	spin_unlock(chip->mutex);
+	return ret;
+}
+
+static int cfi_atmel_lock(struct mtd_info *mtd, loff_t ofs, size_t len)
+{
+	return cfi_varsize_frob(mtd, do_atmel_lock, ofs, len, NULL);
+}
+
+static int cfi_atmel_unlock(struct mtd_info *mtd, loff_t ofs, size_t len)
+{
+	return cfi_varsize_frob(mtd, do_atmel_unlock, ofs, len, NULL);
+}
+
 
 static void cfi_amdstd_sync (struct mtd_info *mtd)
 {
diff --git a/drivers/mtd/chips/jedec_probe.c b/drivers/mtd/chips/jedec_probe.c
index 8f39d0a..1154dac 100644
--- a/drivers/mtd/chips/jedec_probe.c
+++ b/drivers/mtd/chips/jedec_probe.c
@@ -111,6 +111,7 @@
 #define MX29LV040C	0x004F
 #define MX29LV160T	0x22C4
 #define MX29LV160B	0x2249
+#define MX29F040	0x00A4
 #define MX29F016	0x00AD
 #define MX29F002T	0x00B0
 #define MX29F004T	0x0045
@@ -1172,6 +1173,19 @@ static const struct amd_flash_info jedec_table[] = {
 		}
 	}, {
 		.mfr_id		= MANUFACTURER_MACRONIX,
+		.dev_id		= MX29F040,
+		.name		= "Macronix MX29F040",
+		.uaddr		= {
+			[0] = MTD_UADDR_0x0555_0x02AA /* x8 */
+		},
+		.DevSize	= SIZE_512KiB,
+		.CmdSet		= P_ID_AMD_STD,
+		.NumEraseRegions= 1,
+		.regions	= {
+			ERASEINFO(0x10000,8),
+		}
+        }, {
+		.mfr_id		= MANUFACTURER_MACRONIX,
 		.dev_id		= MX29F016,
 		.name		= "Macronix MX29F016",
 		.uaddr		= {
diff --git a/drivers/mtd/devices/block2mtd.c b/drivers/mtd/devices/block2mtd.c
index ede3561..401c6a2 100644
--- a/drivers/mtd/devices/block2mtd.c
+++ b/drivers/mtd/devices/block2mtd.c
@@ -18,6 +18,7 @@
 #include <linux/mtd/mtd.h>
 #include <linux/buffer_head.h>
 #include <linux/mutex.h>
+#include <linux/mount.h>
 
 #define VERSION "$Revision: 1.30 $"
 
@@ -236,6 +237,8 @@ static int _block2mtd_write(struct block2mtd_dev *dev, const u_char *buf,
 	}
 	return 0;
 }
+
+
 static int block2mtd_write(struct mtd_info *mtd, loff_t to, size_t len,
 		size_t *retlen, const u_char *buf)
 {
@@ -299,6 +302,19 @@ static struct block2mtd_dev *add_device(char *devname, int erase_size)
 
 	/* Get a handle on the device */
 	bdev = open_bdev_excl(devname, O_RDWR, NULL);
+#ifndef MODULE
+	if (IS_ERR(bdev)) {
+
+		/* We might not have rootfs mounted at this point. Try
+		   to resolve the device name by other means. */
+
+		dev_t dev = name_to_dev_t(devname);
+		if (dev != 0) {
+			bdev = open_by_devnum(dev, FMODE_WRITE | FMODE_READ);
+		}
+	}
+#endif
+
 	if (IS_ERR(bdev)) {
 		ERROR("error: cannot open device %s", devname);
 		goto devinit_err;
@@ -393,26 +409,6 @@ static int parse_num(size_t *num, const char *token)
 }
 
 
-static int parse_name(char **pname, const char *token, size_t limit)
-{
-	size_t len;
-	char *name;
-
-	len = strlen(token) + 1;
-	if (len > limit)
-		return -ENOSPC;
-
-	name = kmalloc(len, GFP_KERNEL);
-	if (!name)
-		return -ENOMEM;
-
-	strcpy(name, token);
-
-	*pname = name;
-	return 0;
-}
-
-
 static inline void kill_final_newline(char *str)
 {
 	char *newline = strrchr(str, '\n');
@@ -426,9 +422,15 @@ static inline void kill_final_newline(char *str)
 	return 0;				\
 } while (0)
 
-static int block2mtd_setup(const char *val, struct kernel_param *kp)
+#ifndef MODULE
+static int block2mtd_init_called = 0;
+static __initdata char block2mtd_paramline[80 + 12]; /* 80 for device, 12 for erase size */
+#endif
+
+
+static int block2mtd_setup2(const char *val)
 {
-	char buf[80+12]; /* 80 for device, 12 for erase size */
+	char buf[80 + 12]; /* 80 for device, 12 for erase size */
 	char *str = buf;
 	char *token[2];
 	char *name;
@@ -450,13 +452,9 @@ static int block2mtd_setup(const char *val, struct kernel_param *kp)
 	if (!token[0])
 		parse_err("no argument");
 
-	ret = parse_name(&name, token[0], 80);
-	if (ret == -ENOMEM)
-		parse_err("out of memory");
-	if (ret == -ENOSPC)
-		parse_err("name too long");
-	if (ret)
-		return 0;
+	name = token[0];
+	if (strlen(name) + 1 > 80)
+		parse_err("device name too long");
 
 	if (token[1]) {
 		ret = parse_num(&erase_size, token[1]);
@@ -472,13 +470,48 @@ static int block2mtd_setup(const char *val, struct kernel_param *kp)
 }
 
 
+static int block2mtd_setup(const char *val, struct kernel_param *kp)
+{
+#ifdef MODULE
+	return block2mtd_setup2(val);
+#else
+	/* If more parameters are later passed in via
+	   /sys/module/block2mtd/parameters/block2mtd
+	   and block2mtd_init() has already been called,
+	   we can parse the argument now. */
+
+	if (block2mtd_init_called)
+		return block2mtd_setup2(val);
+
+	/* During early boot stage, we only save the parameters
+	   here. We must parse them later: if the param passed
+	   from kernel boot command line, block2mtd_setup() is
+	   called so early that it is not possible to resolve
+	   the device (even kmalloc() fails). Deter that work to
+	   block2mtd_setup2(). */
+
+	strlcpy(block2mtd_paramline, val, sizeof(block2mtd_paramline));
+
+	return 0;
+#endif
+}
+
+
 module_param_call(block2mtd, block2mtd_setup, NULL, NULL, 0200);
 MODULE_PARM_DESC(block2mtd, "Device to use. \"block2mtd=<dev>[,<erasesize>]\"");
 
 static int __init block2mtd_init(void)
 {
+	int ret = 0;
 	INFO("version " VERSION);
-	return 0;
+
+#ifndef MODULE
+	if (strlen(block2mtd_paramline))
+		ret = block2mtd_setup2(block2mtd_paramline);
+	block2mtd_init_called = 1;
+#endif
+
+	return ret;
 }
 
 
diff --git a/drivers/mtd/devices/m25p80.c b/drivers/mtd/devices/m25p80.c
index a846614..ef4a731 100644
--- a/drivers/mtd/devices/m25p80.c
+++ b/drivers/mtd/devices/m25p80.c
@@ -406,13 +406,13 @@ struct flash_info {
 
 static struct flash_info __devinitdata m25p_data [] = {
 	/* REVISIT: fill in JEDEC ids, for parts that have them */
-	{ "m25p05", 0x05, 0x0000, 32 * 1024, 2 },
-	{ "m25p10", 0x10, 0x0000, 32 * 1024, 4 },
-	{ "m25p20", 0x11, 0x0000, 64 * 1024, 4 },
-	{ "m25p40", 0x12, 0x0000, 64 * 1024, 8 },
+	{ "m25p05", 0x05, 0x2010, 32 * 1024, 2 },
+	{ "m25p10", 0x10, 0x2011, 32 * 1024, 4 },
+	{ "m25p20", 0x11, 0x2012, 64 * 1024, 4 },
+	{ "m25p40", 0x12, 0x2013, 64 * 1024, 8 },
 	{ "m25p80", 0x13, 0x0000, 64 * 1024, 16 },
-	{ "m25p16", 0x14, 0x0000, 64 * 1024, 32 },
-	{ "m25p32", 0x15, 0x0000, 64 * 1024, 64 },
+	{ "m25p16", 0x14, 0x2015, 64 * 1024, 32 },
+	{ "m25p32", 0x15, 0x2016, 64 * 1024, 64 },
 	{ "m25p64", 0x16, 0x2017, 64 * 1024, 128 },
 };
 
diff --git a/drivers/mtd/devices/pmc551.c b/drivers/mtd/devices/pmc551.c
index 6f9bbf6..354e165 100644
--- a/drivers/mtd/devices/pmc551.c
+++ b/drivers/mtd/devices/pmc551.c
@@ -4,82 +4,82 @@
  * PMC551 PCI Mezzanine Ram Device
  *
  * Author:
- *       Mark Ferrell <mferrell@mvista.com>
- *       Copyright 1999,2000 Nortel Networks
+ *	Mark Ferrell <mferrell@mvista.com>
+ *	Copyright 1999,2000 Nortel Networks
  *
  * License:
- *	 As part of this driver was derived from the slram.c driver it
- *	 falls under the same license, which is GNU General Public
- *	 License v2
+ *	As part of this driver was derived from the slram.c driver it
+ *	falls under the same license, which is GNU General Public
+ *	License v2
  *
  * Description:
- *	 This driver is intended to support the PMC551 PCI Ram device
- *	 from Ramix Inc.  The PMC551 is a PMC Mezzanine module for
- *	 cPCI embedded systems.  The device contains a single SROM
- *	 that initially programs the V370PDC chipset onboard the
- *	 device, and various banks of DRAM/SDRAM onboard.  This driver
- *	 implements this PCI Ram device as an MTD (Memory Technology
- *	 Device) so that it can be used to hold a file system, or for
- *	 added swap space in embedded systems.  Since the memory on
- *	 this board isn't as fast as main memory we do not try to hook
- *	 it into main memory as that would simply reduce performance
- *	 on the system.  Using it as a block device allows us to use
- *	 it as high speed swap or for a high speed disk device of some
- *	 sort.  Which becomes very useful on diskless systems in the
- *	 embedded market I might add.
+ *	This driver is intended to support the PMC551 PCI Ram device
+ *	from Ramix Inc.  The PMC551 is a PMC Mezzanine module for
+ *	cPCI embedded systems.  The device contains a single SROM
+ *	that initially programs the V370PDC chipset onboard the
+ *	device, and various banks of DRAM/SDRAM onboard.  This driver
+ *	implements this PCI Ram device as an MTD (Memory Technology
+ *	Device) so that it can be used to hold a file system, or for
+ *	added swap space in embedded systems.  Since the memory on
+ *	this board isn't as fast as main memory we do not try to hook
+ *	it into main memory as that would simply reduce performance
+ *	on the system.  Using it as a block device allows us to use
+ *	it as high speed swap or for a high speed disk device of some
+ *	sort.  Which becomes very useful on diskless systems in the
+ *	embedded market I might add.
  *
  * Notes:
- *	 Due to what I assume is more buggy SROM, the 64M PMC551 I
- *	 have available claims that all 4 of it's DRAM banks have 64M
- *	 of ram configured (making a grand total of 256M onboard).
- *	 This is slightly annoying since the BAR0 size reflects the
- *	 aperture size, not the dram size, and the V370PDC supplies no
- *	 other method for memory size discovery.  This problem is
- *	 mostly only relevant when compiled as a module, as the
- *	 unloading of the module with an aperture size smaller then
- *	 the ram will cause the driver to detect the onboard memory
- *	 size to be equal to the aperture size when the module is
- *	 reloaded.  Soooo, to help, the module supports an msize
- *	 option to allow the specification of the onboard memory, and
- *	 an asize option, to allow the specification of the aperture
- *	 size.  The aperture must be equal to or less then the memory
- *	 size, the driver will correct this if you screw it up.  This
- *	 problem is not relevant for compiled in drivers as compiled
- *	 in drivers only init once.
+ *	Due to what I assume is more buggy SROM, the 64M PMC551 I
+ *	have available claims that all 4 of it's DRAM banks have 64M
+ *	of ram configured (making a grand total of 256M onboard).
+ *	This is slightly annoying since the BAR0 size reflects the
+ *	aperture size, not the dram size, and the V370PDC supplies no
+ *	other method for memory size discovery.  This problem is
+ *	mostly only relevant when compiled as a module, as the
+ *	unloading of the module with an aperture size smaller then
+ *	the ram will cause the driver to detect the onboard memory
+ *	size to be equal to the aperture size when the module is
+ *	reloaded.  Soooo, to help, the module supports an msize
+ *	option to allow the specification of the onboard memory, and
+ *	an asize option, to allow the specification of the aperture
+ *	size.  The aperture must be equal to or less then the memory
+ *	size, the driver will correct this if you screw it up.  This
+ *	problem is not relevant for compiled in drivers as compiled
+ *	in drivers only init once.
  *
  * Credits:
- *       Saeed Karamooz <saeed@ramix.com> of Ramix INC. for the
- *       initial example code of how to initialize this device and for
- *       help with questions I had concerning operation of the device.
+ *	Saeed Karamooz <saeed@ramix.com> of Ramix INC. for the
+ *	initial example code of how to initialize this device and for
+ *	help with questions I had concerning operation of the device.
  *
- *       Most of the MTD code for this driver was originally written
- *       for the slram.o module in the MTD drivers package which
- *       allows the mapping of system memory into an MTD device.
- *       Since the PMC551 memory module is accessed in the same
- *       fashion as system memory, the slram.c code became a very nice
- *       fit to the needs of this driver.  All we added was PCI
- *       detection/initialization to the driver and automatically figure
- *       out the size via the PCI detection.o, later changes by Corey
- *       Minyard set up the card to utilize a 1M sliding apature.
+ *	Most of the MTD code for this driver was originally written
+ *	for the slram.o module in the MTD drivers package which
+ *	allows the mapping of system memory into an MTD device.
+ *	Since the PMC551 memory module is accessed in the same
+ *	fashion as system memory, the slram.c code became a very nice
+ *	fit to the needs of this driver.  All we added was PCI
+ *	detection/initialization to the driver and automatically figure
+ *	out the size via the PCI detection.o, later changes by Corey
+ *	Minyard set up the card to utilize a 1M sliding apature.
  *
- *	 Corey Minyard <minyard@nortelnetworks.com>
- *       * Modified driver to utilize a sliding aperture instead of
- *         mapping all memory into kernel space which turned out to
- *         be very wasteful.
- *       * Located a bug in the SROM's initialization sequence that
- *         made the memory unusable, added a fix to code to touch up
- *         the DRAM some.
+ *	Corey Minyard <minyard@nortelnetworks.com>
+ *	* Modified driver to utilize a sliding aperture instead of
+ *	 mapping all memory into kernel space which turned out to
+ *	 be very wasteful.
+ *	* Located a bug in the SROM's initialization sequence that
+ *	 made the memory unusable, added a fix to code to touch up
+ *	 the DRAM some.
  *
  * Bugs/FIXME's:
- *       * MUST fix the init function to not spin on a register
- *       waiting for it to set .. this does not safely handle busted
- *       devices that never reset the register correctly which will
- *       cause the system to hang w/ a reboot being the only chance at
- *       recover. [sort of fixed, could be better]
- *       * Add I2C handling of the SROM so we can read the SROM's information
- *       about the aperture size.  This should always accurately reflect the
- *       onboard memory size.
- *       * Comb the init routine.  It's still a bit cludgy on a few things.
+ *	* MUST fix the init function to not spin on a register
+ *	waiting for it to set .. this does not safely handle busted
+ *	devices that never reset the register correctly which will
+ *	cause the system to hang w/ a reboot being the only chance at
+ *	recover. [sort of fixed, could be better]
+ *	* Add I2C handling of the SROM so we can read the SROM's information
+ *	about the aperture size.  This should always accurately reflect the
+ *	onboard memory size.
+ *	* Comb the init routine.  It's still a bit cludgy on a few things.
  */
 
 #include <linux/kernel.h>
@@ -99,84 +99,83 @@
 #include <asm/system.h>
 #include <linux/pci.h>
 
-#ifndef CONFIG_PCI
-#error Enable PCI in your kernel config
-#endif
-
 #include <linux/mtd/mtd.h>
 #include <linux/mtd/pmc551.h>
 #include <linux/mtd/compatmac.h>
 
 static struct mtd_info *pmc551list;
 
-static int pmc551_erase (struct mtd_info *mtd, struct erase_info *instr)
+static int pmc551_erase(struct mtd_info *mtd, struct erase_info *instr)
 {
-        struct mypriv *priv = mtd->priv;
-        u32 soff_hi, soff_lo; /* start address offset hi/lo */
-        u32 eoff_hi, eoff_lo; /* end address offset hi/lo */
-        unsigned long end;
+	struct mypriv *priv = mtd->priv;
+	u32 soff_hi, soff_lo;	/* start address offset hi/lo */
+	u32 eoff_hi, eoff_lo;	/* end address offset hi/lo */
+	unsigned long end;
 	u_char *ptr;
 	size_t retlen;
 
 #ifdef CONFIG_MTD_PMC551_DEBUG
-	printk(KERN_DEBUG "pmc551_erase(pos:%ld, len:%ld)\n", (long)instr->addr, (long)instr->len);
+	printk(KERN_DEBUG "pmc551_erase(pos:%ld, len:%ld)\n", (long)instr->addr,
+		(long)instr->len);
 #endif
 
-        end = instr->addr + instr->len - 1;
+	end = instr->addr + instr->len - 1;
 
-        /* Is it past the end? */
-        if ( end > mtd->size ) {
+	/* Is it past the end? */
+	if (end > mtd->size) {
 #ifdef CONFIG_MTD_PMC551_DEBUG
-	printk(KERN_DEBUG "pmc551_erase() out of bounds (%ld > %ld)\n", (long)end, (long)mtd->size);
+		printk(KERN_DEBUG "pmc551_erase() out of bounds (%ld > %ld)\n",
+			(long)end, (long)mtd->size);
 #endif
-                return -EINVAL;
-        }
-
-        eoff_hi = end & ~(priv->asize - 1);
-        soff_hi = instr->addr & ~(priv->asize - 1);
-        eoff_lo = end & (priv->asize - 1);
-        soff_lo = instr->addr & (priv->asize - 1);
-
-	pmc551_point (mtd, instr->addr, instr->len, &retlen, &ptr);
-
-        if ( soff_hi == eoff_hi || mtd->size == priv->asize) {
-                /* The whole thing fits within one access, so just one shot
-                   will do it. */
-                memset(ptr, 0xff, instr->len);
-        } else {
-                /* We have to do multiple writes to get all the data
-                   written. */
-                while (soff_hi != eoff_hi) {
+		return -EINVAL;
+	}
+
+	eoff_hi = end & ~(priv->asize - 1);
+	soff_hi = instr->addr & ~(priv->asize - 1);
+	eoff_lo = end & (priv->asize - 1);
+	soff_lo = instr->addr & (priv->asize - 1);
+
+	pmc551_point(mtd, instr->addr, instr->len, &retlen, &ptr);
+
+	if (soff_hi == eoff_hi || mtd->size == priv->asize) {
+		/* The whole thing fits within one access, so just one shot
+		   will do it. */
+		memset(ptr, 0xff, instr->len);
+	} else {
+		/* We have to do multiple writes to get all the data
+		   written. */
+		while (soff_hi != eoff_hi) {
 #ifdef CONFIG_MTD_PMC551_DEBUG
-			printk( KERN_DEBUG "pmc551_erase() soff_hi: %ld, eoff_hi: %ld\n", (long)soff_hi, (long)eoff_hi);
+			printk(KERN_DEBUG "pmc551_erase() soff_hi: %ld, "
+				"eoff_hi: %ld\n", (long)soff_hi, (long)eoff_hi);
 #endif
-                        memset(ptr, 0xff, priv->asize);
-                        if (soff_hi + priv->asize >= mtd->size) {
-                                goto out;
-                        }
-                        soff_hi += priv->asize;
-			pmc551_point (mtd,(priv->base_map0|soff_hi),
-				      priv->asize, &retlen, &ptr);
-                }
-                memset (ptr, 0xff, eoff_lo);
-        }
-
-out:
+			memset(ptr, 0xff, priv->asize);
+			if (soff_hi + priv->asize >= mtd->size) {
+				goto out;
+			}
+			soff_hi += priv->asize;
+			pmc551_point(mtd, (priv->base_map0 | soff_hi),
+				     priv->asize, &retlen, &ptr);
+		}
+		memset(ptr, 0xff, eoff_lo);
+	}
+
+      out:
 	instr->state = MTD_ERASE_DONE;
 #ifdef CONFIG_MTD_PMC551_DEBUG
 	printk(KERN_DEBUG "pmc551_erase() done\n");
 #endif
 
-        mtd_erase_callback(instr);
-        return 0;
+	mtd_erase_callback(instr);
+	return 0;
 }
 
-
-static int pmc551_point (struct mtd_info *mtd, loff_t from, size_t len, size_t *retlen, u_char **mtdbuf)
+static int pmc551_point(struct mtd_info *mtd, loff_t from, size_t len,
+			size_t * retlen, u_char ** mtdbuf)
 {
-        struct mypriv *priv = mtd->priv;
-        u32 soff_hi;
-        u32 soff_lo;
+	struct mypriv *priv = mtd->priv;
+	u32 soff_hi;
+	u32 soff_lo;
 
 #ifdef CONFIG_MTD_PMC551_DEBUG
 	printk(KERN_DEBUG "pmc551_point(%ld, %ld)\n", (long)from, (long)len);
@@ -184,18 +183,19 @@ static int pmc551_point (struct mtd_info *mtd, loff_t from, size_t len, size_t *
 
 	if (from + len > mtd->size) {
 #ifdef CONFIG_MTD_PMC551_DEBUG
-		printk(KERN_DEBUG "pmc551_point() out of bounds (%ld > %ld)\n", (long)from+len, (long)mtd->size);
+		printk(KERN_DEBUG "pmc551_point() out of bounds (%ld > %ld)\n",
+			(long)from + len, (long)mtd->size);
 #endif
 		return -EINVAL;
 	}
 
-        soff_hi = from & ~(priv->asize - 1);
-        soff_lo = from & (priv->asize - 1);
+	soff_hi = from & ~(priv->asize - 1);
+	soff_lo = from & (priv->asize - 1);
 
 	/* Cheap hack optimization */
-	if( priv->curr_map0 != from ) {
-        	pci_write_config_dword ( priv->dev, PMC551_PCI_MEM_MAP0,
-                                 	(priv->base_map0 | soff_hi) );
+	if (priv->curr_map0 != from) {
+		pci_write_config_dword(priv->dev, PMC551_PCI_MEM_MAP0,
+					(priv->base_map0 | soff_hi));
 		priv->curr_map0 = soff_hi;
 	}
 
@@ -204,137 +204,144 @@ static int pmc551_point (struct mtd_info *mtd, loff_t from, size_t len, size_t *
 	return 0;
 }
 
-
-static void pmc551_unpoint (struct mtd_info *mtd, u_char *addr, loff_t from, size_t len)
+static void pmc551_unpoint(struct mtd_info *mtd, u_char * addr, loff_t from,
+			   size_t len)
 {
 #ifdef CONFIG_MTD_PMC551_DEBUG
 	printk(KERN_DEBUG "pmc551_unpoint()\n");
 #endif
 }
 
-
-static int pmc551_read (struct mtd_info *mtd, loff_t from, size_t len, size_t *retlen, u_char *buf)
+static int pmc551_read(struct mtd_info *mtd, loff_t from, size_t len,
+			size_t * retlen, u_char * buf)
 {
-        struct mypriv *priv = mtd->priv;
-        u32 soff_hi, soff_lo; /* start address offset hi/lo */
-        u32 eoff_hi, eoff_lo; /* end address offset hi/lo */
-        unsigned long end;
+	struct mypriv *priv = mtd->priv;
+	u32 soff_hi, soff_lo;	/* start address offset hi/lo */
+	u32 eoff_hi, eoff_lo;	/* end address offset hi/lo */
+	unsigned long end;
 	u_char *ptr;
-        u_char *copyto = buf;
+	u_char *copyto = buf;
 
 #ifdef CONFIG_MTD_PMC551_DEBUG
-	printk(KERN_DEBUG "pmc551_read(pos:%ld, len:%ld) asize: %ld\n", (long)from, (long)len, (long)priv->asize);
+	printk(KERN_DEBUG "pmc551_read(pos:%ld, len:%ld) asize: %ld\n",
+		(long)from, (long)len, (long)priv->asize);
 #endif
 
-        end = from + len - 1;
+	end = from + len - 1;
 
-        /* Is it past the end? */
-        if (end > mtd->size) {
+	/* Is it past the end? */
+	if (end > mtd->size) {
 #ifdef CONFIG_MTD_PMC551_DEBUG
-	printk(KERN_DEBUG "pmc551_read() out of bounds (%ld > %ld)\n", (long) end, (long)mtd->size);
+		printk(KERN_DEBUG "pmc551_read() out of bounds (%ld > %ld)\n",
+			(long)end, (long)mtd->size);
 #endif
-                return -EINVAL;
-        }
-
-        soff_hi = from & ~(priv->asize - 1);
-        eoff_hi = end & ~(priv->asize - 1);
-        soff_lo = from & (priv->asize - 1);
-        eoff_lo = end & (priv->asize - 1);
-
-	pmc551_point (mtd, from, len, retlen, &ptr);
-
-        if (soff_hi == eoff_hi) {
-                /* The whole thing fits within one access, so just one shot
-                   will do it. */
-                memcpy(copyto, ptr, len);
-                copyto += len;
-        } else {
-                /* We have to do multiple writes to get all the data
-                   written. */
-                while (soff_hi != eoff_hi) {
+		return -EINVAL;
+	}
+
+	soff_hi = from & ~(priv->asize - 1);
+	eoff_hi = end & ~(priv->asize - 1);
+	soff_lo = from & (priv->asize - 1);
+	eoff_lo = end & (priv->asize - 1);
+
+	pmc551_point(mtd, from, len, retlen, &ptr);
+
+	if (soff_hi == eoff_hi) {
+		/* The whole thing fits within one access, so just one shot
+		   will do it. */
+		memcpy(copyto, ptr, len);
+		copyto += len;
+	} else {
+		/* We have to do multiple writes to get all the data
+		   written. */
+		while (soff_hi != eoff_hi) {
 #ifdef CONFIG_MTD_PMC551_DEBUG
-			printk( KERN_DEBUG "pmc551_read() soff_hi: %ld, eoff_hi: %ld\n", (long)soff_hi, (long)eoff_hi);
+			printk(KERN_DEBUG "pmc551_read() soff_hi: %ld, "
+				"eoff_hi: %ld\n", (long)soff_hi, (long)eoff_hi);
 #endif
-                        memcpy(copyto, ptr, priv->asize);
-                        copyto += priv->asize;
-                        if (soff_hi + priv->asize >= mtd->size) {
-                                goto out;
-                        }
-                        soff_hi += priv->asize;
-			pmc551_point (mtd, soff_hi, priv->asize, retlen, &ptr);
-                }
-                memcpy(copyto, ptr, eoff_lo);
-                copyto += eoff_lo;
-        }
-
-out:
+			memcpy(copyto, ptr, priv->asize);
+			copyto += priv->asize;
+			if (soff_hi + priv->asize >= mtd->size) {
+				goto out;
+			}
+			soff_hi += priv->asize;
+			pmc551_point(mtd, soff_hi, priv->asize, retlen, &ptr);
+		}
+		memcpy(copyto, ptr, eoff_lo);
+		copyto += eoff_lo;
+	}
+
+      out:
 #ifdef CONFIG_MTD_PMC551_DEBUG
 	printk(KERN_DEBUG "pmc551_read() done\n");
 #endif
-        *retlen = copyto - buf;
-        return 0;
+	*retlen = copyto - buf;
+	return 0;
 }
 
-static int pmc551_write (struct mtd_info *mtd, loff_t to, size_t len, size_t *retlen, const u_char *buf)
+static int pmc551_write(struct mtd_info *mtd, loff_t to, size_t len,
+			size_t * retlen, const u_char * buf)
 {
-        struct mypriv *priv = mtd->priv;
-        u32 soff_hi, soff_lo; /* start address offset hi/lo */
-        u32 eoff_hi, eoff_lo; /* end address offset hi/lo */
-        unsigned long end;
+	struct mypriv *priv = mtd->priv;
+	u32 soff_hi, soff_lo;	/* start address offset hi/lo */
+	u32 eoff_hi, eoff_lo;	/* end address offset hi/lo */
+	unsigned long end;
 	u_char *ptr;
-        const u_char *copyfrom = buf;
-
+	const u_char *copyfrom = buf;
 
 #ifdef CONFIG_MTD_PMC551_DEBUG
-	printk(KERN_DEBUG "pmc551_write(pos:%ld, len:%ld) asize:%ld\n", (long)to, (long)len, (long)priv->asize);
+	printk(KERN_DEBUG "pmc551_write(pos:%ld, len:%ld) asize:%ld\n",
+		(long)to, (long)len, (long)priv->asize);
 #endif
 
-        end = to + len - 1;
-        /* Is it past the end?  or did the u32 wrap? */
-        if (end > mtd->size ) {
+	end = to + len - 1;
+	/* Is it past the end?  or did the u32 wrap? */
+	if (end > mtd->size) {
 #ifdef CONFIG_MTD_PMC551_DEBUG
-	printk(KERN_DEBUG "pmc551_write() out of bounds (end: %ld, size: %ld, to: %ld)\n", (long) end, (long)mtd->size, (long)to);
+		printk(KERN_DEBUG "pmc551_write() out of bounds (end: %ld, "
+			"size: %ld, to: %ld)\n", (long)end, (long)mtd->size,
+			(long)to);
 #endif
-                return -EINVAL;
-        }
-
-        soff_hi = to & ~(priv->asize - 1);
-        eoff_hi = end & ~(priv->asize - 1);
-        soff_lo = to & (priv->asize - 1);
-        eoff_lo = end & (priv->asize - 1);
-
-	pmc551_point (mtd, to, len, retlen, &ptr);
-
-        if (soff_hi == eoff_hi) {
-                /* The whole thing fits within one access, so just one shot
-                   will do it. */
-                memcpy(ptr, copyfrom, len);
-                copyfrom += len;
-        } else {
-                /* We have to do multiple writes to get all the data
-                   written. */
-                while (soff_hi != eoff_hi) {
+		return -EINVAL;
+	}
+
+	soff_hi = to & ~(priv->asize - 1);
+	eoff_hi = end & ~(priv->asize - 1);
+	soff_lo = to & (priv->asize - 1);
+	eoff_lo = end & (priv->asize - 1);
+
+	pmc551_point(mtd, to, len, retlen, &ptr);
+
+	if (soff_hi == eoff_hi) {
+		/* The whole thing fits within one access, so just one shot
+		   will do it. */
+		memcpy(ptr, copyfrom, len);
+		copyfrom += len;
+	} else {
+		/* We have to do multiple writes to get all the data
+		   written. */
+		while (soff_hi != eoff_hi) {
 #ifdef CONFIG_MTD_PMC551_DEBUG
-			printk( KERN_DEBUG "pmc551_write() soff_hi: %ld, eoff_hi: %ld\n", (long)soff_hi, (long)eoff_hi);
+			printk(KERN_DEBUG "pmc551_write() soff_hi: %ld, "
+				"eoff_hi: %ld\n", (long)soff_hi, (long)eoff_hi);
 #endif
-                	memcpy(ptr, copyfrom, priv->asize);
-                	copyfrom += priv->asize;
-                        if (soff_hi >= mtd->size) {
-                                goto out;
-                        }
-                        soff_hi += priv->asize;
-			pmc551_point (mtd, soff_hi, priv->asize, retlen, &ptr);
-                }
-                memcpy(ptr, copyfrom, eoff_lo);
-                copyfrom += eoff_lo;
-        }
-
-out:
+			memcpy(ptr, copyfrom, priv->asize);
+			copyfrom += priv->asize;
+			if (soff_hi >= mtd->size) {
+				goto out;
+			}
+			soff_hi += priv->asize;
+			pmc551_point(mtd, soff_hi, priv->asize, retlen, &ptr);
+		}
+		memcpy(ptr, copyfrom, eoff_lo);
+		copyfrom += eoff_lo;
+	}
+
+      out:
 #ifdef CONFIG_MTD_PMC551_DEBUG
 	printk(KERN_DEBUG "pmc551_write() done\n");
 #endif
-        *retlen = copyfrom - buf;
-        return 0;
+	*retlen = copyfrom - buf;
+	return 0;
 }
 
 /*
@@ -349,58 +356,58 @@ out:
  * mechanism
  * returns the size of the memory region found.
  */
-static u32 fixup_pmc551 (struct pci_dev *dev)
+static u32 fixup_pmc551(struct pci_dev *dev)
 {
 #ifdef CONFIG_MTD_PMC551_BUGFIX
-        u32 dram_data;
+	u32 dram_data;
 #endif
-        u32 size, dcmd, cfg, dtmp;
-        u16 cmd, tmp, i;
+	u32 size, dcmd, cfg, dtmp;
+	u16 cmd, tmp, i;
 	u8 bcmd, counter;
 
-        /* Sanity Check */
-        if(!dev) {
-                return -ENODEV;
-        }
+	/* Sanity Check */
+	if (!dev) {
+		return -ENODEV;
+	}
 
 	/*
 	 * Attempt to reset the card
 	 * FIXME: Stop Spinning registers
 	 */
-	counter=0;
+	counter = 0;
 	/* unlock registers */
-	pci_write_config_byte(dev, PMC551_SYS_CTRL_REG, 0xA5 );
+	pci_write_config_byte(dev, PMC551_SYS_CTRL_REG, 0xA5);
 	/* read in old data */
-	pci_read_config_byte(dev, PMC551_SYS_CTRL_REG, &bcmd );
+	pci_read_config_byte(dev, PMC551_SYS_CTRL_REG, &bcmd);
 	/* bang the reset line up and down for a few */
-	for(i=0;i<10;i++) {
-		counter=0;
+	for (i = 0; i < 10; i++) {
+		counter = 0;
 		bcmd &= ~0x80;
-		while(counter++ < 100) {
+		while (counter++ < 100) {
 			pci_write_config_byte(dev, PMC551_SYS_CTRL_REG, bcmd);
 		}
-		counter=0;
+		counter = 0;
 		bcmd |= 0x80;
-		while(counter++ < 100) {
+		while (counter++ < 100) {
 			pci_write_config_byte(dev, PMC551_SYS_CTRL_REG, bcmd);
 		}
 	}
-	bcmd |= (0x40|0x20);
+	bcmd |= (0x40 | 0x20);
 	pci_write_config_byte(dev, PMC551_SYS_CTRL_REG, bcmd);
 
-        /*
+	/*
 	 * Take care and turn off the memory on the device while we
 	 * tweak the configurations
 	 */
-        pci_read_config_word(dev, PCI_COMMAND, &cmd);
-        tmp = cmd & ~(PCI_COMMAND_IO|PCI_COMMAND_MEMORY);
-        pci_write_config_word(dev, PCI_COMMAND, tmp);
+	pci_read_config_word(dev, PCI_COMMAND, &cmd);
+	tmp = cmd & ~(PCI_COMMAND_IO | PCI_COMMAND_MEMORY);
+	pci_write_config_word(dev, PCI_COMMAND, tmp);
 
 	/*
 	 * Disable existing aperture before probing memory size
 	 */
 	pci_read_config_dword(dev, PMC551_PCI_MEM_MAP0, &dcmd);
-        dtmp=(dcmd|PMC551_PCI_MEM_MAP_ENABLE|PMC551_PCI_MEM_MAP_REG_EN);
+	dtmp = (dcmd | PMC551_PCI_MEM_MAP_ENABLE | PMC551_PCI_MEM_MAP_REG_EN);
 	pci_write_config_dword(dev, PMC551_PCI_MEM_MAP0, dtmp);
 	/*
 	 * Grab old BAR0 config so that we can figure out memory size
@@ -411,220 +418,230 @@ static u32 fixup_pmc551 (struct pci_dev *dev)
 	 * then write all 1's to the memory space, read back the result into
 	 * "size", and then write back all the old config.
 	 */
-	pci_read_config_dword( dev, PCI_BASE_ADDRESS_0, &cfg );
+	pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &cfg);
 #ifndef CONFIG_MTD_PMC551_BUGFIX
-	pci_write_config_dword( dev, PCI_BASE_ADDRESS_0, ~0 );
-	pci_read_config_dword( dev, PCI_BASE_ADDRESS_0, &size );
-	size = (size&PCI_BASE_ADDRESS_MEM_MASK);
-	size &= ~(size-1);
-	pci_write_config_dword( dev, PCI_BASE_ADDRESS_0, cfg );
+	pci_write_config_dword(dev, PCI_BASE_ADDRESS_0, ~0);
+	pci_read_config_dword(dev, PCI_BASE_ADDRESS_0, &size);
+	size = (size & PCI_BASE_ADDRESS_MEM_MASK);
+	size &= ~(size - 1);
+	pci_write_config_dword(dev, PCI_BASE_ADDRESS_0, cfg);
 #else
-        /*
-         * Get the size of the memory by reading all the DRAM size values
-         * and adding them up.
-         *
-         * KLUDGE ALERT: the boards we are using have invalid column and
-         * row mux values.  We fix them here, but this will break other
-         * memory configurations.
-         */
-        pci_read_config_dword(dev, PMC551_DRAM_BLK0, &dram_data);
-        size = PMC551_DRAM_BLK_GET_SIZE(dram_data);
-        dram_data = PMC551_DRAM_BLK_SET_COL_MUX(dram_data, 0x5);
-        dram_data = PMC551_DRAM_BLK_SET_ROW_MUX(dram_data, 0x9);
-        pci_write_config_dword(dev, PMC551_DRAM_BLK0, dram_data);
-
-        pci_read_config_dword(dev, PMC551_DRAM_BLK1, &dram_data);
-        size += PMC551_DRAM_BLK_GET_SIZE(dram_data);
-        dram_data = PMC551_DRAM_BLK_SET_COL_MUX(dram_data, 0x5);
-        dram_data = PMC551_DRAM_BLK_SET_ROW_MUX(dram_data, 0x9);
-        pci_write_config_dword(dev, PMC551_DRAM_BLK1, dram_data);
-
-        pci_read_config_dword(dev, PMC551_DRAM_BLK2, &dram_data);
-        size += PMC551_DRAM_BLK_GET_SIZE(dram_data);
-        dram_data = PMC551_DRAM_BLK_SET_COL_MUX(dram_data, 0x5);
-        dram_data = PMC551_DRAM_BLK_SET_ROW_MUX(dram_data, 0x9);
-        pci_write_config_dword(dev, PMC551_DRAM_BLK2, dram_data);
-
-        pci_read_config_dword(dev, PMC551_DRAM_BLK3, &dram_data);
-        size += PMC551_DRAM_BLK_GET_SIZE(dram_data);
-        dram_data = PMC551_DRAM_BLK_SET_COL_MUX(dram_data, 0x5);
-        dram_data = PMC551_DRAM_BLK_SET_ROW_MUX(dram_data, 0x9);
-        pci_write_config_dword(dev, PMC551_DRAM_BLK3, dram_data);
-
-        /*
-         * Oops .. something went wrong
-         */
-        if( (size &= PCI_BASE_ADDRESS_MEM_MASK) == 0) {
-                return -ENODEV;
-        }
-#endif /* CONFIG_MTD_PMC551_BUGFIX */
-
-	if ((cfg&PCI_BASE_ADDRESS_SPACE) != PCI_BASE_ADDRESS_SPACE_MEMORY) {
-                return -ENODEV;
+	/*
+	 * Get the size of the memory by reading all the DRAM size values
+	 * and adding them up.
+	 *
+	 * KLUDGE ALERT: the boards we are using have invalid column and
+	 * row mux values.  We fix them here, but this will break other
+	 * memory configurations.
+	 */
+	pci_read_config_dword(dev, PMC551_DRAM_BLK0, &dram_data);
+	size = PMC551_DRAM_BLK_GET_SIZE(dram_data);
+	dram_data = PMC551_DRAM_BLK_SET_COL_MUX(dram_data, 0x5);
+	dram_data = PMC551_DRAM_BLK_SET_ROW_MUX(dram_data, 0x9);
+	pci_write_config_dword(dev, PMC551_DRAM_BLK0, dram_data);
+
+	pci_read_config_dword(dev, PMC551_DRAM_BLK1, &dram_data);
+	size += PMC551_DRAM_BLK_GET_SIZE(dram_data);
+	dram_data = PMC551_DRAM_BLK_SET_COL_MUX(dram_data, 0x5);
+	dram_data = PMC551_DRAM_BLK_SET_ROW_MUX(dram_data, 0x9);
+	pci_write_config_dword(dev, PMC551_DRAM_BLK1, dram_data);
+
+	pci_read_config_dword(dev, PMC551_DRAM_BLK2, &dram_data);
+	size += PMC551_DRAM_BLK_GET_SIZE(dram_data);
+	dram_data = PMC551_DRAM_BLK_SET_COL_MUX(dram_data, 0x5);
+	dram_data = PMC551_DRAM_BLK_SET_ROW_MUX(dram_data, 0x9);
+	pci_write_config_dword(dev, PMC551_DRAM_BLK2, dram_data);
+
+	pci_read_config_dword(dev, PMC551_DRAM_BLK3, &dram_data);
+	size += PMC551_DRAM_BLK_GET_SIZE(dram_data);
+	dram_data = PMC551_DRAM_BLK_SET_COL_MUX(dram_data, 0x5);
+	dram_data = PMC551_DRAM_BLK_SET_ROW_MUX(dram_data, 0x9);
+	pci_write_config_dword(dev, PMC551_DRAM_BLK3, dram_data);
+
+	/*
+	 * Oops .. something went wrong
+	 */
+	if ((size &= PCI_BASE_ADDRESS_MEM_MASK) == 0) {
+		return -ENODEV;
+	}
+#endif				/* CONFIG_MTD_PMC551_BUGFIX */
+
+	if ((cfg & PCI_BASE_ADDRESS_SPACE) != PCI_BASE_ADDRESS_SPACE_MEMORY) {
+		return -ENODEV;
 	}
 
-        /*
-         * Precharge Dram
-         */
-        pci_write_config_word( dev, PMC551_SDRAM_MA, 0x0400 );
-        pci_write_config_word( dev, PMC551_SDRAM_CMD, 0x00bf );
-
-        /*
-         * Wait until command has gone through
-         * FIXME: register spinning issue
-         */
-        do {	pci_read_config_word( dev, PMC551_SDRAM_CMD, &cmd );
-		if(counter++ > 100)break;
-        } while ( (PCI_COMMAND_IO) & cmd );
-
-        /*
+	/*
+	 * Precharge Dram
+	 */
+	pci_write_config_word(dev, PMC551_SDRAM_MA, 0x0400);
+	pci_write_config_word(dev, PMC551_SDRAM_CMD, 0x00bf);
+
+	/*
+	 * Wait until command has gone through
+	 * FIXME: register spinning issue
+	 */
+	do {
+		pci_read_config_word(dev, PMC551_SDRAM_CMD, &cmd);
+		if (counter++ > 100)
+			break;
+	} while ((PCI_COMMAND_IO) & cmd);
+
+	/*
 	 * Turn on auto refresh
 	 * The loop is taken directly from Ramix's example code.  I assume that
 	 * this must be held high for some duration of time, but I can find no
 	 * documentation refrencing the reasons why.
-         */
-        for ( i = 1; i<=8 ; i++) {
-                pci_write_config_word (dev, PMC551_SDRAM_CMD, 0x0df);
-
-                /*
-                 * Make certain command has gone through
-                 * FIXME: register spinning issue
-                 */
-		counter=0;
-                do {	pci_read_config_word(dev, PMC551_SDRAM_CMD, &cmd);
-			if(counter++ > 100)break;
-                } while ( (PCI_COMMAND_IO) & cmd );
-        }
-
-        pci_write_config_word ( dev, PMC551_SDRAM_MA, 0x0020);
-        pci_write_config_word ( dev, PMC551_SDRAM_CMD, 0x0ff);
-
-        /*
-         * Wait until command completes
-         * FIXME: register spinning issue
-         */
-	counter=0;
-        do {	pci_read_config_word ( dev, PMC551_SDRAM_CMD, &cmd);
-		if(counter++ > 100)break;
-        } while ( (PCI_COMMAND_IO) & cmd );
-
-        pci_read_config_dword ( dev, PMC551_DRAM_CFG, &dcmd);
-        dcmd |= 0x02000000;
-        pci_write_config_dword ( dev, PMC551_DRAM_CFG, dcmd);
-
-        /*
-         * Check to make certain fast back-to-back, if not
-         * then set it so
-         */
-        pci_read_config_word( dev, PCI_STATUS, &cmd);
-        if((cmd&PCI_COMMAND_FAST_BACK) == 0) {
-                cmd |= PCI_COMMAND_FAST_BACK;
-                pci_write_config_word( dev, PCI_STATUS, cmd);
-        }
-
-        /*
-         * Check to make certain the DEVSEL is set correctly, this device
-         * has a tendancy to assert DEVSEL and TRDY when a write is performed
-         * to the memory when memory is read-only
-         */
-        if((cmd&PCI_STATUS_DEVSEL_MASK) != 0x0) {
-                cmd &= ~PCI_STATUS_DEVSEL_MASK;
-                pci_write_config_word( dev, PCI_STATUS, cmd );
-        }
-        /*
-         * Set to be prefetchable and put everything back based on old cfg.
+	 */
+	for (i = 1; i <= 8; i++) {
+		pci_write_config_word(dev, PMC551_SDRAM_CMD, 0x0df);
+
+		/*
+		 * Make certain command has gone through
+		 * FIXME: register spinning issue
+		 */
+		counter = 0;
+		do {
+			pci_read_config_word(dev, PMC551_SDRAM_CMD, &cmd);
+			if (counter++ > 100)
+				break;
+		} while ((PCI_COMMAND_IO) & cmd);
+	}
+
+	pci_write_config_word(dev, PMC551_SDRAM_MA, 0x0020);
+	pci_write_config_word(dev, PMC551_SDRAM_CMD, 0x0ff);
+
+	/*
+	 * Wait until command completes
+	 * FIXME: register spinning issue
+	 */
+	counter = 0;
+	do {
+		pci_read_config_word(dev, PMC551_SDRAM_CMD, &cmd);
+		if (counter++ > 100)
+			break;
+	} while ((PCI_COMMAND_IO) & cmd);
+
+	pci_read_config_dword(dev, PMC551_DRAM_CFG, &dcmd);
+	dcmd |= 0x02000000;
+	pci_write_config_dword(dev, PMC551_DRAM_CFG, dcmd);
+
+	/*
+	 * Check to make certain fast back-to-back, if not
+	 * then set it so
+	 */
+	pci_read_config_word(dev, PCI_STATUS, &cmd);
+	if ((cmd & PCI_COMMAND_FAST_BACK) == 0) {
+		cmd |= PCI_COMMAND_FAST_BACK;
+		pci_write_config_word(dev, PCI_STATUS, cmd);
+	}
+
+	/*
+	 * Check to make certain the DEVSEL is set correctly, this device
+	 * has a tendancy to assert DEVSEL and TRDY when a write is performed
+	 * to the memory when memory is read-only
+	 */
+	if ((cmd & PCI_STATUS_DEVSEL_MASK) != 0x0) {
+		cmd &= ~PCI_STATUS_DEVSEL_MASK;
+		pci_write_config_word(dev, PCI_STATUS, cmd);
+	}
+	/*
+	 * Set to be prefetchable and put everything back based on old cfg.
 	 * it's possible that the reset of the V370PDC nuked the original
 	 * setup
-         */
+	 */
+	/*
+	   cfg |= PCI_BASE_ADDRESS_MEM_PREFETCH;
+	   pci_write_config_dword( dev, PCI_BASE_ADDRESS_0, cfg );
+	 */
+
 	/*
-        cfg |= PCI_BASE_ADDRESS_MEM_PREFETCH;
-	pci_write_config_dword( dev, PCI_BASE_ADDRESS_0, cfg );
-	*/
-
-        /*
-         * Turn PCI memory and I/O bus access back on
-         */
-        pci_write_config_word( dev, PCI_COMMAND,
-                               PCI_COMMAND_MEMORY | PCI_COMMAND_IO );
+	 * Turn PCI memory and I/O bus access back on
+	 */
+	pci_write_config_word(dev, PCI_COMMAND,
+			      PCI_COMMAND_MEMORY | PCI_COMMAND_IO);
 #ifdef CONFIG_MTD_PMC551_DEBUG
-        /*
-         * Some screen fun
-         */
-        printk(KERN_DEBUG "pmc551: %d%c (0x%x) of %sprefetchable memory at 0x%llx\n",
-	       (size<1024)?size:(size<1048576)?size>>10:size>>20,
-               (size<1024)?'B':(size<1048576)?'K':'M',
-	       size, ((dcmd&(0x1<<3)) == 0)?"non-":"",
-               (unsigned long long)((dev->resource[0].start)&PCI_BASE_ADDRESS_MEM_MASK));
-
-        /*
-         * Check to see the state of the memory
-         */
-        pci_read_config_dword( dev, PMC551_DRAM_BLK0, &dcmd );
-        printk(KERN_DEBUG "pmc551: DRAM_BLK0 Flags: %s,%s\n"
-			  "pmc551: DRAM_BLK0 Size: %d at %d\n"
-			  "pmc551: DRAM_BLK0 Row MUX: %d, Col MUX: %d\n",
-               (((0x1<<1)&dcmd) == 0)?"RW":"RO",
-               (((0x1<<0)&dcmd) == 0)?"Off":"On",
-	       PMC551_DRAM_BLK_GET_SIZE(dcmd),
-	       ((dcmd>>20)&0x7FF), ((dcmd>>13)&0x7), ((dcmd>>9)&0xF) );
-
-        pci_read_config_dword( dev, PMC551_DRAM_BLK1, &dcmd );
-        printk(KERN_DEBUG "pmc551: DRAM_BLK1 Flags: %s,%s\n"
-			  "pmc551: DRAM_BLK1 Size: %d at %d\n"
-			  "pmc551: DRAM_BLK1 Row MUX: %d, Col MUX: %d\n",
-               (((0x1<<1)&dcmd) == 0)?"RW":"RO",
-               (((0x1<<0)&dcmd) == 0)?"Off":"On",
-	       PMC551_DRAM_BLK_GET_SIZE(dcmd),
-	       ((dcmd>>20)&0x7FF), ((dcmd>>13)&0x7), ((dcmd>>9)&0xF) );
-
-        pci_read_config_dword( dev, PMC551_DRAM_BLK2, &dcmd );
-        printk(KERN_DEBUG "pmc551: DRAM_BLK2 Flags: %s,%s\n"
-			  "pmc551: DRAM_BLK2 Size: %d at %d\n"
-			  "pmc551: DRAM_BLK2 Row MUX: %d, Col MUX: %d\n",
-               (((0x1<<1)&dcmd) == 0)?"RW":"RO",
-               (((0x1<<0)&dcmd) == 0)?"Off":"On",
-	       PMC551_DRAM_BLK_GET_SIZE(dcmd),
-	       ((dcmd>>20)&0x7FF), ((dcmd>>13)&0x7), ((dcmd>>9)&0xF) );
-
-        pci_read_config_dword( dev, PMC551_DRAM_BLK3, &dcmd );
-        printk(KERN_DEBUG "pmc551: DRAM_BLK3 Flags: %s,%s\n"
-			  "pmc551: DRAM_BLK3 Size: %d at %d\n"
-			  "pmc551: DRAM_BLK3 Row MUX: %d, Col MUX: %d\n",
-               (((0x1<<1)&dcmd) == 0)?"RW":"RO",
-               (((0x1<<0)&dcmd) == 0)?"Off":"On",
-	       PMC551_DRAM_BLK_GET_SIZE(dcmd),
-	       ((dcmd>>20)&0x7FF), ((dcmd>>13)&0x7), ((dcmd>>9)&0xF) );
-
-        pci_read_config_word( dev, PCI_COMMAND, &cmd );
-        printk( KERN_DEBUG "pmc551: Memory Access %s\n",
-                (((0x1<<1)&cmd) == 0)?"off":"on" );
-        printk( KERN_DEBUG "pmc551: I/O Access %s\n",
-                (((0x1<<0)&cmd) == 0)?"off":"on" );
-
-        pci_read_config_word( dev, PCI_STATUS, &cmd );
-        printk( KERN_DEBUG "pmc551: Devsel %s\n",
-                ((PCI_STATUS_DEVSEL_MASK&cmd)==0x000)?"Fast":
-                ((PCI_STATUS_DEVSEL_MASK&cmd)==0x200)?"Medium":
-                ((PCI_STATUS_DEVSEL_MASK&cmd)==0x400)?"Slow":"Invalid" );
-
-        printk( KERN_DEBUG "pmc551: %sFast Back-to-Back\n",
-                ((PCI_COMMAND_FAST_BACK&cmd) == 0)?"Not ":"" );
-
-	pci_read_config_byte(dev, PMC551_SYS_CTRL_REG, &bcmd );
-	printk( KERN_DEBUG "pmc551: EEPROM is under %s control\n"
-			   "pmc551: System Control Register is %slocked to PCI access\n"
-			   "pmc551: System Control Register is %slocked to EEPROM access\n",
-		(bcmd&0x1)?"software":"hardware",
-		(bcmd&0x20)?"":"un", (bcmd&0x40)?"":"un");
+	/*
+	 * Some screen fun
+	 */
+	printk(KERN_DEBUG "pmc551: %d%c (0x%x) of %sprefetchable memory at "
+		"0x%llx\n", (size < 1024) ? size : (size < 1048576) ?
+		size >> 10 : size >> 20,
+		(size < 1024) ? 'B' : (size < 1048576) ? 'K' : 'M', size,
+		((dcmd & (0x1 << 3)) == 0) ? "non-" : "",
+		(unsigned long long)pci_resource_start(dev, 0));
+
+	/*
+	 * Check to see the state of the memory
+	 */
+	pci_read_config_dword(dev, PMC551_DRAM_BLK0, &dcmd);
+	printk(KERN_DEBUG "pmc551: DRAM_BLK0 Flags: %s,%s\n"
+		"pmc551: DRAM_BLK0 Size: %d at %d\n"
+		"pmc551: DRAM_BLK0 Row MUX: %d, Col MUX: %d\n",
+		(((0x1 << 1) & dcmd) == 0) ? "RW" : "RO",
+		(((0x1 << 0) & dcmd) == 0) ? "Off" : "On",
+		PMC551_DRAM_BLK_GET_SIZE(dcmd),
+		((dcmd >> 20) & 0x7FF), ((dcmd >> 13) & 0x7),
+		((dcmd >> 9) & 0xF));
+
+	pci_read_config_dword(dev, PMC551_DRAM_BLK1, &dcmd);
+	printk(KERN_DEBUG "pmc551: DRAM_BLK1 Flags: %s,%s\n"
+		"pmc551: DRAM_BLK1 Size: %d at %d\n"
+		"pmc551: DRAM_BLK1 Row MUX: %d, Col MUX: %d\n",
+		(((0x1 << 1) & dcmd) == 0) ? "RW" : "RO",
+		(((0x1 << 0) & dcmd) == 0) ? "Off" : "On",
+		PMC551_DRAM_BLK_GET_SIZE(dcmd),
+		((dcmd >> 20) & 0x7FF), ((dcmd >> 13) & 0x7),
+		((dcmd >> 9) & 0xF));
+
+	pci_read_config_dword(dev, PMC551_DRAM_BLK2, &dcmd);
+	printk(KERN_DEBUG "pmc551: DRAM_BLK2 Flags: %s,%s\n"
+		"pmc551: DRAM_BLK2 Size: %d at %d\n"
+		"pmc551: DRAM_BLK2 Row MUX: %d, Col MUX: %d\n",
+		(((0x1 << 1) & dcmd) == 0) ? "RW" : "RO",
+		(((0x1 << 0) & dcmd) == 0) ? "Off" : "On",
+		PMC551_DRAM_BLK_GET_SIZE(dcmd),
+		((dcmd >> 20) & 0x7FF), ((dcmd >> 13) & 0x7),
+		((dcmd >> 9) & 0xF));
+
+	pci_read_config_dword(dev, PMC551_DRAM_BLK3, &dcmd);
+	printk(KERN_DEBUG "pmc551: DRAM_BLK3 Flags: %s,%s\n"
+		"pmc551: DRAM_BLK3 Size: %d at %d\n"
+		"pmc551: DRAM_BLK3 Row MUX: %d, Col MUX: %d\n",
+		(((0x1 << 1) & dcmd) == 0) ? "RW" : "RO",
+		(((0x1 << 0) & dcmd) == 0) ? "Off" : "On",
+		PMC551_DRAM_BLK_GET_SIZE(dcmd),
+		((dcmd >> 20) & 0x7FF), ((dcmd >> 13) & 0x7),
+		((dcmd >> 9) & 0xF));
+
+	pci_read_config_word(dev, PCI_COMMAND, &cmd);
+	printk(KERN_DEBUG "pmc551: Memory Access %s\n",
+		(((0x1 << 1) & cmd) == 0) ? "off" : "on");
+	printk(KERN_DEBUG "pmc551: I/O Access %s\n",
+		(((0x1 << 0) & cmd) == 0) ? "off" : "on");
+
+	pci_read_config_word(dev, PCI_STATUS, &cmd);
+	printk(KERN_DEBUG "pmc551: Devsel %s\n",
+		((PCI_STATUS_DEVSEL_MASK & cmd) == 0x000) ? "Fast" :
+		((PCI_STATUS_DEVSEL_MASK & cmd) == 0x200) ? "Medium" :
+		((PCI_STATUS_DEVSEL_MASK & cmd) == 0x400) ? "Slow" : "Invalid");
+
+	printk(KERN_DEBUG "pmc551: %sFast Back-to-Back\n",
+		((PCI_COMMAND_FAST_BACK & cmd) == 0) ? "Not " : "");
+
+	pci_read_config_byte(dev, PMC551_SYS_CTRL_REG, &bcmd);
+	printk(KERN_DEBUG "pmc551: EEPROM is under %s control\n"
+		"pmc551: System Control Register is %slocked to PCI access\n"
+		"pmc551: System Control Register is %slocked to EEPROM access\n",
+		(bcmd & 0x1) ? "software" : "hardware",
+		(bcmd & 0x20) ? "" : "un", (bcmd & 0x40) ? "" : "un");
 #endif
-        return size;
+	return size;
 }
 
 /*
  * Kernel version specific module stuffages
  */
 
-
 MODULE_LICENSE("GPL");
 MODULE_AUTHOR("Mark Ferrell <mferrell@mvista.com>");
 MODULE_DESCRIPTION(PMC551_VERSION);
@@ -632,11 +649,11 @@ MODULE_DESCRIPTION(PMC551_VERSION);
 /*
  * Stuff these outside the ifdef so as to not bust compiled in driver support
  */
-static int msize=0;
+static int msize = 0;
 #if defined(CONFIG_MTD_PMC551_APERTURE_SIZE)
-static int asize=CONFIG_MTD_PMC551_APERTURE_SIZE
+static int asize = CONFIG_MTD_PMC551_APERTURE_SIZE
 #else
-static int asize=0;
+static int asize = 0;
 #endif
 
 module_param(msize, int, 0);
@@ -649,164 +666,174 @@ MODULE_PARM_DESC(asize, "aperture size, must be <= memsize [1-1024]");
  */
 static int __init init_pmc551(void)
 {
-        struct pci_dev *PCI_Device = NULL;
-        struct mypriv *priv;
-        int count, found=0;
-        struct mtd_info *mtd;
-        u32 length = 0;
-
-	if(msize) {
-		msize = (1 << (ffs(msize) - 1))<<20;
-		if (msize > (1<<30)) {
-			printk(KERN_NOTICE "pmc551: Invalid memory size [%d]\n", msize);
+	struct pci_dev *PCI_Device = NULL;
+	struct mypriv *priv;
+	int count, found = 0;
+	struct mtd_info *mtd;
+	u32 length = 0;
+
+	if (msize) {
+		msize = (1 << (ffs(msize) - 1)) << 20;
+		if (msize > (1 << 30)) {
+			printk(KERN_NOTICE "pmc551: Invalid memory size [%d]\n",
+				msize);
 			return -EINVAL;
 		}
 	}
 
-	if(asize) {
-		asize = (1 << (ffs(asize) - 1))<<20;
-		if (asize > (1<<30) ) {
-			printk(KERN_NOTICE "pmc551: Invalid aperture size [%d]\n", asize);
+	if (asize) {
+		asize = (1 << (ffs(asize) - 1)) << 20;
+		if (asize > (1 << 30)) {
+			printk(KERN_NOTICE "pmc551: Invalid aperture size "
+				"[%d]\n", asize);
 			return -EINVAL;
 		}
 	}
 
-        printk(KERN_INFO PMC551_VERSION);
-
-        /*
-         * PCU-bus chipset probe.
-         */
-        for( count = 0; count < MAX_MTD_DEVICES; count++ ) {
-
-                if ((PCI_Device = pci_find_device(PCI_VENDOR_ID_V3_SEMI,
-                                                  PCI_DEVICE_ID_V3_SEMI_V370PDC,
-						  PCI_Device ) ) == NULL) {
-                        break;
-                }
-
-                printk(KERN_NOTICE "pmc551: Found PCI V370PDC at 0x%llx\n",
-				    (unsigned long long)PCI_Device->resource[0].start);
-
-                /*
-                 * The PMC551 device acts VERY weird if you don't init it
-                 * first.  i.e. it will not correctly report devsel.  If for
-                 * some reason the sdram is in a wrote-protected state the
-                 * device will DEVSEL when it is written to causing problems
-                 * with the oldproc.c driver in
-                 * some kernels (2.2.*)
-                 */
-                if((length = fixup_pmc551(PCI_Device)) <= 0) {
-                        printk(KERN_NOTICE "pmc551: Cannot init SDRAM\n");
-                        break;
-                }
+	printk(KERN_INFO PMC551_VERSION);
+
+	/*
+	 * PCU-bus chipset probe.
+	 */
+	for (count = 0; count < MAX_MTD_DEVICES; count++) {
+
+		if ((PCI_Device = pci_get_device(PCI_VENDOR_ID_V3_SEMI,
+						  PCI_DEVICE_ID_V3_SEMI_V370PDC,
+						  PCI_Device)) == NULL) {
+			break;
+		}
+
+		printk(KERN_NOTICE "pmc551: Found PCI V370PDC at 0x%llx\n",
+			(unsigned long long)pci_resource_start(PCI_Device, 0));
+
+		/*
+		 * The PMC551 device acts VERY weird if you don't init it
+		 * first.  i.e. it will not correctly report devsel.  If for
+		 * some reason the sdram is in a wrote-protected state the
+		 * device will DEVSEL when it is written to causing problems
+		 * with the oldproc.c driver in
+		 * some kernels (2.2.*)
+		 */
+		if ((length = fixup_pmc551(PCI_Device)) <= 0) {
+			printk(KERN_NOTICE "pmc551: Cannot init SDRAM\n");
+			break;
+		}
 
 		/*
 		 * This is needed until the driver is capable of reading the
 		 * onboard I2C SROM to discover the "real" memory size.
 		 */
-		if(msize) {
+		if (msize) {
 			length = msize;
-			printk(KERN_NOTICE "pmc551: Using specified memory size 0x%x\n", length);
+			printk(KERN_NOTICE "pmc551: Using specified memory "
+				"size 0x%x\n", length);
 		} else {
 			msize = length;
 		}
 
-                mtd = kmalloc(sizeof(struct mtd_info), GFP_KERNEL);
-                if (!mtd) {
-                        printk(KERN_NOTICE "pmc551: Cannot allocate new MTD device.\n");
-                        break;
-                }
-
-                memset(mtd, 0, sizeof(struct mtd_info));
-
-                priv = kmalloc (sizeof(struct mypriv), GFP_KERNEL);
-                if (!priv) {
-                        printk(KERN_NOTICE "pmc551: Cannot allocate new MTD device.\n");
-                        kfree(mtd);
-                        break;
-                }
-                memset(priv, 0, sizeof(*priv));
-                mtd->priv = priv;
-                priv->dev = PCI_Device;
-
-		if(asize > length) {
-			printk(KERN_NOTICE "pmc551: reducing aperture size to fit %dM\n",length>>20);
+		mtd = kzalloc(sizeof(struct mtd_info), GFP_KERNEL);
+		if (!mtd) {
+			printk(KERN_NOTICE "pmc551: Cannot allocate new MTD "
+				"device.\n");
+			break;
+		}
+
+		priv = kzalloc(sizeof(struct mypriv), GFP_KERNEL);
+		if (!priv) {
+			printk(KERN_NOTICE "pmc551: Cannot allocate new MTD "
+				"device.\n");
+			kfree(mtd);
+			break;
+		}
+		mtd->priv = priv;
+		priv->dev = PCI_Device;
+
+		if (asize > length) {
+			printk(KERN_NOTICE "pmc551: reducing aperture size to "
+				"fit %dM\n", length >> 20);
 			priv->asize = asize = length;
 		} else if (asize == 0 || asize == length) {
-			printk(KERN_NOTICE "pmc551: Using existing aperture size %dM\n", length>>20);
+			printk(KERN_NOTICE "pmc551: Using existing aperture "
+				"size %dM\n", length >> 20);
 			priv->asize = asize = length;
 		} else {
-			printk(KERN_NOTICE "pmc551: Using specified aperture size %dM\n", asize>>20);
+			printk(KERN_NOTICE "pmc551: Using specified aperture "
+				"size %dM\n", asize >> 20);
 			priv->asize = asize;
 		}
-                priv->start = ioremap(((PCI_Device->resource[0].start)
-                                       & PCI_BASE_ADDRESS_MEM_MASK),
-                                      priv->asize);
+		priv->start = pci_iomap(PCI_Device, 0, priv->asize);
 
 		if (!priv->start) {
 			printk(KERN_NOTICE "pmc551: Unable to map IO space\n");
-                        kfree(mtd->priv);
-                        kfree(mtd);
+			kfree(mtd->priv);
+			kfree(mtd);
 			break;
 		}
-
 #ifdef CONFIG_MTD_PMC551_DEBUG
-		printk( KERN_DEBUG "pmc551: setting aperture to %d\n",
-			ffs(priv->asize>>20)-1);
+		printk(KERN_DEBUG "pmc551: setting aperture to %d\n",
+			ffs(priv->asize >> 20) - 1);
 #endif
 
-                priv->base_map0 = ( PMC551_PCI_MEM_MAP_REG_EN
-				  | PMC551_PCI_MEM_MAP_ENABLE
-				  | (ffs(priv->asize>>20)-1)<<4 );
-                priv->curr_map0 = priv->base_map0;
-                pci_write_config_dword ( priv->dev, PMC551_PCI_MEM_MAP0,
-                                         priv->curr_map0 );
+		priv->base_map0 = (PMC551_PCI_MEM_MAP_REG_EN
+				   | PMC551_PCI_MEM_MAP_ENABLE
+				   | (ffs(priv->asize >> 20) - 1) << 4);
+		priv->curr_map0 = priv->base_map0;
+		pci_write_config_dword(priv->dev, PMC551_PCI_MEM_MAP0,
+					priv->curr_map0);
 
 #ifdef CONFIG_MTD_PMC551_DEBUG
-		printk( KERN_DEBUG "pmc551: aperture set to %d\n",
-			(priv->base_map0 & 0xF0)>>4 );
+		printk(KERN_DEBUG "pmc551: aperture set to %d\n",
+			(priv->base_map0 & 0xF0) >> 4);
 #endif
 
-                mtd->size 	= msize;
-                mtd->flags 	= MTD_CAP_RAM;
-                mtd->erase 	= pmc551_erase;
-                mtd->read 	= pmc551_read;
-                mtd->write 	= pmc551_write;
-                mtd->point 	= pmc551_point;
-                mtd->unpoint 	= pmc551_unpoint;
-                mtd->type 	= MTD_RAM;
-                mtd->name 	= "PMC551 RAM board";
-                mtd->erasesize 	= 0x10000;
-                mtd->writesize  = 1;
-                mtd->owner = THIS_MODULE;
-
-                if (add_mtd_device(mtd)) {
-                        printk(KERN_NOTICE "pmc551: Failed to register new device\n");
-			iounmap(priv->start);
-                        kfree(mtd->priv);
-                        kfree(mtd);
-                        break;
-                }
-                printk(KERN_NOTICE "Registered pmc551 memory device.\n");
-                printk(KERN_NOTICE "Mapped %dM of memory from 0x%p to 0x%p\n",
-                       priv->asize>>20,
-                       priv->start,
-                       priv->start + priv->asize);
-                printk(KERN_NOTICE "Total memory is %d%c\n",
-	       		(length<1024)?length:
-				(length<1048576)?length>>10:length>>20,
-               		(length<1024)?'B':(length<1048576)?'K':'M');
+		mtd->size = msize;
+		mtd->flags = MTD_CAP_RAM;
+		mtd->erase = pmc551_erase;
+		mtd->read = pmc551_read;
+		mtd->write = pmc551_write;
+		mtd->point = pmc551_point;
+		mtd->unpoint = pmc551_unpoint;
+		mtd->type = MTD_RAM;
+		mtd->name = "PMC551 RAM board";
+		mtd->erasesize = 0x10000;
+		mtd->writesize = 1;
+		mtd->owner = THIS_MODULE;
+
+		if (add_mtd_device(mtd)) {
+			printk(KERN_NOTICE "pmc551: Failed to register new "
+				"device\n");
+			pci_iounmap(PCI_Device, priv->start);
+			kfree(mtd->priv);
+			kfree(mtd);
+			break;
+		}
+
+		/* Keep a reference as the add_mtd_device worked */
+		pci_dev_get(PCI_Device);
+
+		printk(KERN_NOTICE "Registered pmc551 memory device.\n");
+		printk(KERN_NOTICE "Mapped %dM of memory from 0x%p to 0x%p\n",
+			priv->asize >> 20,
+			priv->start, priv->start + priv->asize);
+		printk(KERN_NOTICE "Total memory is %d%c\n",
+			(length < 1024) ? length :
+			(length < 1048576) ? length >> 10 : length >> 20,
+			(length < 1024) ? 'B' : (length < 1048576) ? 'K' : 'M');
 		priv->nextpmc551 = pmc551list;
 		pmc551list = mtd;
 		found++;
-        }
+	}
+
+	/* Exited early, reference left over */
+	if (PCI_Device)
+		pci_dev_put(PCI_Device);
 
-        if( !pmc551list ) {
-                printk(KERN_NOTICE "pmc551: not detected\n");
-                return -ENODEV;
-        } else {
+	if (!pmc551list) {
+		printk(KERN_NOTICE "pmc551: not detected\n");
+		return -ENODEV;
+	} else {
 		printk(KERN_NOTICE "pmc551: %d pmc551 devices loaded\n", found);
-                return 0;
+		return 0;
 	}
 }
 
@@ -815,23 +842,24 @@ static int __init init_pmc551(void)
  */
 static void __exit cleanup_pmc551(void)
 {
-        int found=0;
-        struct mtd_info *mtd;
+	int found = 0;
+	struct mtd_info *mtd;
 	struct mypriv *priv;
 
-	while((mtd=pmc551list)) {
+	while ((mtd = pmc551list)) {
 		priv = mtd->priv;
 		pmc551list = priv->nextpmc551;
 
-		if(priv->start) {
-			printk (KERN_DEBUG "pmc551: unmapping %dM starting at 0x%p\n",
-				priv->asize>>20, priv->start);
-			iounmap (priv->start);
+		if (priv->start) {
+			printk(KERN_DEBUG "pmc551: unmapping %dM starting at "
+				"0x%p\n", priv->asize >> 20, priv->start);
+			pci_iounmap(priv->dev, priv->start);
 		}
+		pci_dev_put(priv->dev);
 
-		kfree (mtd->priv);
-		del_mtd_device (mtd);
-		kfree (mtd);
+		kfree(mtd->priv);
+		del_mtd_device(mtd);
+		kfree(mtd);
 		found++;
 	}
 
diff --git a/drivers/mtd/maps/Kconfig b/drivers/mtd/maps/Kconfig
index 83d0b2a..24747bd 100644
--- a/drivers/mtd/maps/Kconfig
+++ b/drivers/mtd/maps/Kconfig
@@ -13,13 +13,13 @@ config MTD_COMPLEX_MAPPINGS
 
 config MTD_PHYSMAP
 	tristate "CFI Flash device in physical memory map"
-	depends on MTD_CFI
+	depends on MTD_CFI || MTD_JEDECPROBE || MTD_ROM
 	help
-	  This provides a 'mapping' driver which allows the CFI probe and
-	  command set driver code to communicate with flash chips which
-	  are mapped physically into the CPU's memory. You will need to
-	  configure the physical address and size of the flash chips on
-	  your particular board as well as the bus width, either statically
+	  This provides a 'mapping' driver which allows the NOR Flash and
+	  ROM driver code to communicate with chips which are mapped
+	  physically into the CPU's memory. You will need to configure
+	  the physical address and size of the flash chips on your
+	  particular board as well as the bus width, either statically
 	  with config options or at run-time.
 
 config MTD_PHYSMAP_START
@@ -447,14 +447,6 @@ config MTD_DC21285
 	  21285 bridge used with Intel's StrongARM processors. More info at
 	  <http://www.intel.com/design/bridge/docs/21285_documentation.htm>.
 
-config MTD_IQ80310
-	tristate "CFI Flash device mapped on the XScale IQ80310 board"
-	depends on MTD_CFI && ARCH_IQ80310
-	help
-	  This enables access routines for the flash chips on the Intel XScale
-	  IQ80310 evaluation board. If you have one of these boards and would
-	  like to use the flash chips on it, say 'Y'.
-
 config MTD_IXP4XX
 	tristate "CFI Flash device mapped on Intel IXP4xx based systems"
 	depends on MTD_CFI && MTD_COMPLEX_MAPPINGS && ARCH_IXP4XX
diff --git a/drivers/mtd/maps/Makefile b/drivers/mtd/maps/Makefile
index ab71f17..191c192 100644
--- a/drivers/mtd/maps/Makefile
+++ b/drivers/mtd/maps/Makefile
@@ -15,7 +15,6 @@ obj-$(CONFIG_MTD_CFI_FLAGADM)	+= cfi_flagadm.o
 obj-$(CONFIG_MTD_CSTM_MIPS_IXX)	+= cstm_mips_ixx.o
 obj-$(CONFIG_MTD_DC21285)	+= dc21285.o
 obj-$(CONFIG_MTD_DILNETPC)	+= dilnetpc.o
-obj-$(CONFIG_MTD_IQ80310)	+= iq80310.o
 obj-$(CONFIG_MTD_L440GX)	+= l440gx.o
 obj-$(CONFIG_MTD_AMD76XROM)	+= amd76xrom.o
 obj-$(CONFIG_MTD_ICHXROM)	+= ichxrom.o
diff --git a/drivers/mtd/maps/amd76xrom.c b/drivers/mtd/maps/amd76xrom.c
index 447955b..797caff 100644
--- a/drivers/mtd/maps/amd76xrom.c
+++ b/drivers/mtd/maps/amd76xrom.c
@@ -57,6 +57,7 @@ static void amd76xrom_cleanup(struct amd76xrom_window *window)
 		/* Disable writes through the rom window */
 		pci_read_config_byte(window->pdev, 0x40, &byte);
 		pci_write_config_byte(window->pdev, 0x40, byte & ~1);
+		pci_dev_put(window->pdev);
 	}
 
 	/* Free all of the mtd devices */
@@ -91,7 +92,7 @@ static int __devinit amd76xrom_init_one (struct pci_dev *pdev,
 	struct amd76xrom_map_info *map = NULL;
 	unsigned long map_top;
 
-	/* Remember the pci dev I find the window in */
+	/* Remember the pci dev I find the window in - already have a ref */
 	window->pdev = pdev;
 
 	/* Assume the rom window is properly setup, and find it's size */
@@ -302,7 +303,7 @@ static int __init init_amd76xrom(void)
 	struct pci_device_id *id;
 	pdev = NULL;
 	for(id = amd76xrom_pci_tbl; id->vendor; id++) {
-		pdev = pci_find_device(id->vendor, id->device, NULL);
+		pdev = pci_get_device(id->vendor, id->device, NULL);
 		if (pdev) {
 			break;
 		}
diff --git a/drivers/mtd/maps/arctic-mtd.c b/drivers/mtd/maps/arctic-mtd.c
index d95ae58..642d96b 100644
--- a/drivers/mtd/maps/arctic-mtd.c
+++ b/drivers/mtd/maps/arctic-mtd.c
@@ -96,6 +96,8 @@ static struct mtd_partition arctic_partitions[PARTITIONS] = {
 static int __init
 init_arctic_mtd(void)
 {
+	int err = 0;
+
 	printk("%s: 0x%08x at 0x%08x\n", NAME, SIZE, PADDR);
 
 	arctic_mtd_map.virt = ioremap(PADDR, SIZE);
@@ -109,12 +111,20 @@ init_arctic_mtd(void)
 	printk("%s: probing %d-bit flash bus\n", NAME, BUSWIDTH * 8);
 	arctic_mtd = do_map_probe("cfi_probe", &arctic_mtd_map);
 
-	if (!arctic_mtd)
+	if (!arctic_mtd) {
+		iounmap((void *) arctic_mtd_map.virt);
 		return -ENXIO;
+	}
 
 	arctic_mtd->owner = THIS_MODULE;
 
-	return add_mtd_partitions(arctic_mtd, arctic_partitions, PARTITIONS);
+	err = add_mtd_partitions(arctic_mtd, arctic_partitions, PARTITIONS);
+	if (err) {
+		printk("%s: add_mtd_partitions failed\n", NAME);
+		iounmap((void *) arctic_mtd_map.virt);
+	}
+
+	return err;
 }
 
 static void __exit
diff --git a/drivers/mtd/maps/beech-mtd.c b/drivers/mtd/maps/beech-mtd.c
index 5df7361..a64b1a5 100644
--- a/drivers/mtd/maps/beech-mtd.c
+++ b/drivers/mtd/maps/beech-mtd.c
@@ -72,6 +72,8 @@ static struct mtd_partition beech_partitions[2] = {
 static int __init
 init_beech_mtd(void)
 {
+	int err = 0;
+
 	printk("%s: 0x%08x at 0x%08x\n", NAME, SIZE, PADDR);
 
 	beech_mtd_map.virt = ioremap(PADDR, SIZE);
@@ -86,12 +88,20 @@ init_beech_mtd(void)
 	printk("%s: probing %d-bit flash bus\n", NAME, BUSWIDTH * 8);
 	beech_mtd = do_map_probe("cfi_probe", &beech_mtd_map);
 
-	if (!beech_mtd)
+	if (!beech_mtd) {
+		iounmap((void *) beech_mtd_map.virt);
 		return -ENXIO;
+	}
 
 	beech_mtd->owner = THIS_MODULE;
 
-	return add_mtd_partitions(beech_mtd, beech_partitions, 2);
+	err = add_mtd_partitions(beech_mtd, beech_partitions, 2);
+	if (err) {
+		printk("%s: add_mtd_partitions failed\n", NAME);
+		iounmap((void *) beech_mtd_map.virt);
+	}
+
+	return err;
 }
 
 static void __exit
diff --git a/drivers/mtd/maps/cstm_mips_ixx.c b/drivers/mtd/maps/cstm_mips_ixx.c
index aa56def..d6bef10 100644
--- a/drivers/mtd/maps/cstm_mips_ixx.c
+++ b/drivers/mtd/maps/cstm_mips_ixx.c
@@ -171,7 +171,14 @@ int __init init_cstm_mips_ixx(void)
 		cstm_mips_ixx_map[i].phys = cstm_mips_ixx_board_desc[i].window_addr;
 		cstm_mips_ixx_map[i].virt = ioremap(cstm_mips_ixx_board_desc[i].window_addr, cstm_mips_ixx_board_desc[i].window_size);
 		if (!cstm_mips_ixx_map[i].virt) {
+			int j = 0;
 			printk(KERN_WARNING "Failed to ioremap\n");
+			for (j = 0; j < i; j++) {
+				if (cstm_mips_ixx_map[j].virt) {
+					iounmap((void *)cstm_mips_ixx_map[j].virt);
+					cstm_mips_ixx_map[j].virt = 0;
+				}
+			}
 			return -EIO;
 	        }
 		cstm_mips_ixx_map[i].name = cstm_mips_ixx_board_desc[i].name;
@@ -204,8 +211,15 @@ int __init init_cstm_mips_ixx(void)
 	                cstm_mips_ixx_map[i].map_priv_2 = (unsigned long)mymtd;
 		        add_mtd_partitions(mymtd, parts, cstm_mips_ixx_board_desc[i].num_partitions);
 		}
-		else
-	           return -ENXIO;
+		else {
+			for (i = 0; i < PHYSMAP_NUMBER; i++) {
+				if (cstm_mips_ixx_map[i].virt) {
+					iounmap((void *)cstm_mips_ixx_map[i].virt);
+					cstm_mips_ixx_map[i].virt = 0;
+				}
+			}
+			return -ENXIO;
+		}
 	}
 	return 0;
 }
diff --git a/drivers/mtd/maps/ebony.c b/drivers/mtd/maps/ebony.c
index 641e1dd..1488bb9 100644
--- a/drivers/mtd/maps/ebony.c
+++ b/drivers/mtd/maps/ebony.c
@@ -108,6 +108,7 @@ int __init init_ebony(void)
 					ARRAY_SIZE(ebony_small_partitions));
 	} else {
 		printk("map probe failed for flash\n");
+		iounmap(ebony_small_map.virt);
 		return -ENXIO;
 	}
 
@@ -117,6 +118,7 @@ int __init init_ebony(void)
 
 	if (!ebony_large_map.virt) {
 		printk("Failed to ioremap flash\n");
+		iounmap(ebony_small_map.virt);
 		return -EIO;
 	}
 
@@ -129,6 +131,8 @@ int __init init_ebony(void)
 					ARRAY_SIZE(ebony_large_partitions));
 	} else {
 		printk("map probe failed for flash\n");
+		iounmap(ebony_small_map.virt);
+		iounmap(ebony_large_map.virt);
 		return -ENXIO;
 	}
 
diff --git a/drivers/mtd/maps/fortunet.c b/drivers/mtd/maps/fortunet.c
index c6bf4e1..7c50c27 100644
--- a/drivers/mtd/maps/fortunet.c
+++ b/drivers/mtd/maps/fortunet.c
@@ -218,8 +218,11 @@ int __init init_fortunet(void)
 				map_regions[ix].map_info.size);
 			if(!map_regions[ix].map_info.virt)
 			{
+				int j = 0;
 				printk(MTD_FORTUNET_PK "%s flash failed to ioremap!\n",
 					map_regions[ix].map_info.name);
+				for (j = 0 ; j < ix; j++)
+					iounmap(map_regions[j].map_info.virt);
 				return -ENXIO;
 			}
 			simple_map_init(&map_regions[ix].map_info);
diff --git a/drivers/mtd/maps/ichxrom.c b/drivers/mtd/maps/ichxrom.c
index db4b570..2bb3e636 100644
--- a/drivers/mtd/maps/ichxrom.c
+++ b/drivers/mtd/maps/ichxrom.c
@@ -61,6 +61,7 @@ static void ichxrom_cleanup(struct ichxrom_window *window)
 	/* Disable writes through the rom window */
 	pci_read_config_word(window->pdev, BIOS_CNTL, &word);
 	pci_write_config_word(window->pdev, BIOS_CNTL, word & ~1);
+	pci_dev_put(window->pdev);
 
 	/* Free all of the mtd devices */
 	list_for_each_entry_safe(map, scratch, &window->maps, list) {
@@ -355,7 +356,7 @@ static int __init init_ichxrom(void)
 
 	pdev = NULL;
 	for (id = ichxrom_pci_tbl; id->vendor; id++) {
-		pdev = pci_find_device(id->vendor, id->device, NULL);
+		pdev = pci_get_device(id->vendor, id->device, NULL);
 		if (pdev) {
 			break;
 		}
diff --git a/drivers/mtd/maps/iq80310.c b/drivers/mtd/maps/iq80310.c
deleted file mode 100644
index 62d9e87..0000000
--- a/drivers/mtd/maps/iq80310.c
+++ /dev/null
@@ -1,118 +0,0 @@
-/*
- * $Id: iq80310.c,v 1.21 2005/11/07 11:14:27 gleixner Exp $
- *
- * Mapping for the Intel XScale IQ80310 evaluation board
- *
- * Author:	Nicolas Pitre
- * Copyright:	(C) 2001 MontaVista Software Inc.
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License version 2 as
- * published by the Free Software Foundation.
- */
-
-#include <linux/module.h>
-#include <linux/types.h>
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/slab.h>
-#include <asm/io.h>
-#include <linux/mtd/mtd.h>
-#include <linux/mtd/map.h>
-#include <linux/mtd/partitions.h>
-
-
-#define WINDOW_ADDR 	0
-#define WINDOW_SIZE 	8*1024*1024
-#define BUSWIDTH 	1
-
-static struct mtd_info *mymtd;
-
-static struct map_info iq80310_map = {
-	.name = "IQ80310 flash",
-	.size = WINDOW_SIZE,
-	.bankwidth = BUSWIDTH,
-	.phys = WINDOW_ADDR
-};
-
-static struct mtd_partition iq80310_partitions[4] = {
-	{
-		.name =		"Firmware",
-		.size =		0x00080000,
-		.offset =	0,
-		.mask_flags =	MTD_WRITEABLE  /* force read-only */
-	},{
-		.name =		"Kernel",
-		.size =		0x000a0000,
-		.offset =	0x00080000,
-	},{
-		.name =		"Filesystem",
-		.size =		0x00600000,
-		.offset =	0x00120000
-	},{
-		.name =		"RedBoot",
-		.size =		0x000e0000,
-		.offset =	0x00720000,
-		.mask_flags =	MTD_WRITEABLE
-	}
-};
-
-static struct mtd_info *mymtd;
-static struct mtd_partition *parsed_parts;
-static const char *probes[] = { "RedBoot", "cmdlinepart", NULL };
-
-static int __init init_iq80310(void)
-{
-	struct mtd_partition *parts;
-	int nb_parts = 0;
-	int parsed_nr_parts = 0;
-	int ret;
-
-	iq80310_map.virt = ioremap(WINDOW_ADDR, WINDOW_SIZE);
-	if (!iq80310_map.virt) {
-		printk("Failed to ioremap\n");
-		return -EIO;
-	}
-	simple_map_init(&iq80310_map);
-
-	mymtd = do_map_probe("cfi_probe", &iq80310_map);
-	if (!mymtd) {
-		iounmap((void *)iq80310_map.virt);
-		return -ENXIO;
-	}
-	mymtd->owner = THIS_MODULE;
-
-	ret = parse_mtd_partitions(mymtd, probes, &parsed_parts, 0);
-
-	if (ret > 0)
-		parsed_nr_parts = ret;
-
-	if (parsed_nr_parts > 0) {
-		parts = parsed_parts;
-		nb_parts = parsed_nr_parts;
-	} else {
-		parts = iq80310_partitions;
-		nb_parts = ARRAY_SIZE(iq80310_partitions);
-	}
-	add_mtd_partitions(mymtd, parts, nb_parts);
-	return 0;
-}
-
-static void __exit cleanup_iq80310(void)
-{
-	if (mymtd) {
-		del_mtd_partitions(mymtd);
-		map_destroy(mymtd);
-		kfree(parsed_parts);
-	}
-	if (iq80310_map.virt)
-		iounmap((void *)iq80310_map.virt);
-}
-
-module_init(init_iq80310);
-module_exit(cleanup_iq80310);
-
-
-MODULE_LICENSE("GPL");
-MODULE_AUTHOR("Nicolas Pitre <nico@cam.org>");
-MODULE_DESCRIPTION("MTD map driver for Intel XScale IQ80310 evaluation board");
diff --git a/drivers/mtd/maps/ixp4xx.c b/drivers/mtd/maps/ixp4xx.c
index 986c586..7a828e3 100644
--- a/drivers/mtd/maps/ixp4xx.c
+++ b/drivers/mtd/maps/ixp4xx.c
@@ -253,7 +253,7 @@ static int ixp4xx_flash_probe(struct platform_device *dev)
 	/* Use the fast version */
 	info->map.write = ixp4xx_write16,
 
-	err = parse_mtd_partitions(info->mtd, probes, &info->partitions, 0);
+	err = parse_mtd_partitions(info->mtd, probes, &info->partitions, dev->resource->start);
 	if (err > 0) {
 		err = add_mtd_partitions(info->mtd, info->partitions, err);
 		if(err)
diff --git a/drivers/mtd/maps/l440gx.c b/drivers/mtd/maps/l440gx.c
index 6b784ef..67620ad 100644
--- a/drivers/mtd/maps/l440gx.c
+++ b/drivers/mtd/maps/l440gx.c
@@ -61,14 +61,17 @@ static int __init init_l440gx(void)
 	struct resource *pm_iobase;
 	__u16 word;
 
-	dev = pci_find_device(PCI_VENDOR_ID_INTEL,
+	dev = pci_get_device(PCI_VENDOR_ID_INTEL,
 		PCI_DEVICE_ID_INTEL_82371AB_0, NULL);
 
-	pm_dev = pci_find_device(PCI_VENDOR_ID_INTEL,
+	pm_dev = pci_get_device(PCI_VENDOR_ID_INTEL,
 		PCI_DEVICE_ID_INTEL_82371AB_3, NULL);
 
+	pci_dev_put(dev);
+
 	if (!dev || !pm_dev) {
 		printk(KERN_NOTICE "L440GX flash mapping: failed to find PIIX4 ISA bridge, cannot continue\n");
+		pci_dev_put(pm_dev);
 		return -ENODEV;
 	}
 
@@ -76,6 +79,7 @@ static int __init init_l440gx(void)
 
 	if (!l440gx_map.virt) {
 		printk(KERN_WARNING "Failed to ioremap L440GX flash region\n");
+		pci_dev_put(pm_dev);
 		return -ENOMEM;
 	}
 	simple_map_init(&l440gx_map);
@@ -99,8 +103,12 @@ static int __init init_l440gx(void)
 		pm_iobase->start += iobase & ~1;
 		pm_iobase->end += iobase & ~1;
 
+		pci_dev_put(pm_dev);
+
 		/* Allocate the resource region */
 		if (pci_assign_resource(pm_dev, PIIXE_IOBASE_RESOURCE) != 0) {
+			pci_dev_put(dev);
+			pci_dev_put(pm_dev);
 			printk(KERN_WARNING "Could not allocate pm iobase resource\n");
 			iounmap(l440gx_map.virt);
 			return -ENXIO;
diff --git a/drivers/mtd/maps/lasat.c b/drivers/mtd/maps/lasat.c
index 1c13d2d..e343763 100644
--- a/drivers/mtd/maps/lasat.c
+++ b/drivers/mtd/maps/lasat.c
@@ -79,6 +79,7 @@ static int __init init_lasat(void)
 		return 0;
 	}
 
+	iounmap(lasat_map.virt);
 	return -ENXIO;
 }
 
@@ -89,6 +90,7 @@ static void __exit cleanup_lasat(void)
 		map_destroy(lasat_mtd);
 	}
 	if (lasat_map.virt) {
+		iounmap(lasat_map.virt);
 		lasat_map.virt = 0;
 	}
 }
diff --git a/drivers/mtd/maps/nettel.c b/drivers/mtd/maps/nettel.c
index 0994b5b..198e840 100644
--- a/drivers/mtd/maps/nettel.c
+++ b/drivers/mtd/maps/nettel.c
@@ -277,6 +277,7 @@ int __init nettel_init(void)
 	nettel_amd_map.virt = ioremap_nocache(amdaddr, maxsize);
 	if (!nettel_amd_map.virt) {
 		printk("SNAPGEAR: failed to ioremap() BOOTCS\n");
+		iounmap(nettel_mmcrp);
 		return(-EIO);
 	}
 	simple_map_init(&nettel_amd_map);
@@ -337,7 +338,8 @@ int __init nettel_init(void)
 		nettel_amd_map.virt = NULL;
 #else
 		/* Only AMD flash supported */
-		return(-ENXIO);
+		rc = -ENXIO;
+		goto out_unmap2;
 #endif
 	}
 
@@ -361,14 +363,15 @@ int __init nettel_init(void)
 	nettel_intel_map.virt = ioremap_nocache(intel0addr, maxsize);
 	if (!nettel_intel_map.virt) {
 		printk("SNAPGEAR: failed to ioremap() ROMCS1\n");
-		return(-EIO);
+		rc = -EIO;
+		goto out_unmap2;
 	}
 	simple_map_init(&nettel_intel_map);
 
 	intel_mtd = do_map_probe("cfi_probe", &nettel_intel_map);
 	if (!intel_mtd) {
-		iounmap(nettel_intel_map.virt);
-		return(-ENXIO);
+		rc = -ENXIO;
+		goto out_unmap1;
 	}
 
 	/* Set PAR to the detected size */
@@ -394,13 +397,14 @@ int __init nettel_init(void)
 	nettel_intel_map.virt = ioremap_nocache(intel0addr, maxsize);
 	if (!nettel_intel_map.virt) {
 		printk("SNAPGEAR: failed to ioremap() ROMCS1/2\n");
-		return(-EIO);
+		rc = -EIO;
+		goto out_unmap2;
 	}
 
 	intel_mtd = do_map_probe("cfi_probe", &nettel_intel_map);
 	if (! intel_mtd) {
-		iounmap((void *) nettel_intel_map.virt);
-		return(-ENXIO);
+		rc = -ENXIO;
+		goto out_unmap1;
 	}
 
 	intel1size = intel_mtd->size - intel0size;
@@ -456,6 +460,18 @@ int __init nettel_init(void)
 #endif
 
 	return(rc);
+
+#ifdef CONFIG_MTD_CFI_INTELEXT
+out_unmap1:
+	iounmap((void *) nettel_intel_map.virt);
+#endif
+
+out_unmap2:
+	iounmap(nettel_mmcrp);
+	iounmap(nettel_amd_map.virt);
+
+	return(rc);
+		
 }
 
 /****************************************************************************/
@@ -469,6 +485,10 @@ void __exit nettel_cleanup(void)
 		del_mtd_partitions(amd_mtd);
 		map_destroy(amd_mtd);
 	}
+	if (nettel_mmcrp) {
+		iounmap(nettel_mmcrp);
+		nettel_mmcrp = NULL;
+	}
 	if (nettel_amd_map.virt) {
 		iounmap(nettel_amd_map.virt);
 		nettel_amd_map.virt = NULL;
diff --git a/drivers/mtd/maps/ocotea.c b/drivers/mtd/maps/ocotea.c
index 2f07602..5522eac 100644
--- a/drivers/mtd/maps/ocotea.c
+++ b/drivers/mtd/maps/ocotea.c
@@ -97,6 +97,7 @@ int __init init_ocotea(void)
 					ARRAY_SIZE(ocotea_small_partitions));
 	} else {
 		printk("map probe failed for flash\n");
+		iounmap(ocotea_small_map.virt);
 		return -ENXIO;
 	}
 
@@ -106,6 +107,7 @@ int __init init_ocotea(void)
 
 	if (!ocotea_large_map.virt) {
 		printk("Failed to ioremap flash\n");
+		iounmap(ocotea_small_map.virt);
 		return -EIO;
 	}
 
@@ -118,6 +120,8 @@ int __init init_ocotea(void)
 					ARRAY_SIZE(ocotea_large_partitions));
 	} else {
 		printk("map probe failed for flash\n");
+		iounmap(ocotea_small_map.virt);
+		iounmap(ocotea_large_map.virt);
 		return -ENXIO;
 	}
 
diff --git a/drivers/mtd/maps/pcmciamtd.c b/drivers/mtd/maps/pcmciamtd.c
index c861134..995347b 100644
--- a/drivers/mtd/maps/pcmciamtd.c
+++ b/drivers/mtd/maps/pcmciamtd.c
@@ -602,6 +602,10 @@ static int pcmciamtd_config(struct pcmcia_device *link)
 	ret = pcmcia_request_configuration(link, &link->conf);
 	if(ret != CS_SUCCESS) {
 		cs_error(link, RequestConfiguration, ret);
+		if (dev->win_base) {
+			iounmap(dev->win_base);
+			dev->win_base = NULL;
+		}
 		return -ENODEV;
 	}
 
diff --git a/drivers/mtd/maps/physmap.c b/drivers/mtd/maps/physmap.c
index 7799a25..bc7cc71 100644
--- a/drivers/mtd/maps/physmap.c
+++ b/drivers/mtd/maps/physmap.c
@@ -158,9 +158,42 @@ err_out:
 	return err;
 }
 
+#ifdef CONFIG_PM
+static int physmap_flash_suspend(struct platform_device *dev, pm_message_t state)
+{
+	struct physmap_flash_info *info = platform_get_drvdata(dev);
+	int ret = 0;
+
+	if (info)
+		ret = info->mtd->suspend(info->mtd);
+
+	return ret;
+}
+
+static int physmap_flash_resume(struct platform_device *dev)
+{
+	struct physmap_flash_info *info = platform_get_drvdata(dev);
+	if (info)
+		info->mtd->resume(info->mtd);
+	return 0;
+}
+
+static void physmap_flash_shutdown(struct platform_device *dev)
+{
+	struct physmap_flash_info *info = platform_get_drvdata(dev);
+	if (info && info->mtd->suspend(info->mtd) == 0)
+		info->mtd->resume(info->mtd);
+}
+#endif
+
 static struct platform_driver physmap_flash_driver = {
 	.probe		= physmap_flash_probe,
 	.remove		= physmap_flash_remove,
+#ifdef CONFIG_PM
+	.suspend	= physmap_flash_suspend,
+	.resume		= physmap_flash_resume,
+	.shutdown	= physmap_flash_shutdown,
+#endif
 	.driver		= {
 		.name	= "physmap-flash",
 	},
diff --git a/drivers/mtd/maps/redwood.c b/drivers/mtd/maps/redwood.c
index ec8fdae..2257d2b 100644
--- a/drivers/mtd/maps/redwood.c
+++ b/drivers/mtd/maps/redwood.c
@@ -126,6 +126,8 @@ static struct mtd_info *redwood_mtd;
 
 int __init init_redwood_flash(void)
 {
+	int err = 0;
+
 	printk(KERN_NOTICE "redwood: flash mapping: %x at %x\n",
 			WINDOW_SIZE, WINDOW_ADDR);
 
@@ -141,11 +143,18 @@ int __init init_redwood_flash(void)
 
 	if (redwood_mtd) {
 		redwood_mtd->owner = THIS_MODULE;
-		return add_mtd_partitions(redwood_mtd,
+		err = add_mtd_partitions(redwood_mtd,
 				redwood_flash_partitions,
 				NUM_REDWOOD_FLASH_PARTITIONS);
+		if (err) {
+			printk("init_redwood_flash: add_mtd_partitions failed\n");
+			iounmap(redwood_flash_map.virt);
+		}
+		return err;
+
 	}
 
+	iounmap(redwood_flash_map.virt);
 	return -ENXIO;
 }
 
diff --git a/drivers/mtd/maps/sbc8240.c b/drivers/mtd/maps/sbc8240.c
index 7d0fcf8..b8c1331 100644
--- a/drivers/mtd/maps/sbc8240.c
+++ b/drivers/mtd/maps/sbc8240.c
@@ -156,7 +156,7 @@ int __init init_sbc8240_mtd (void)
 	};
 
 	int devicesfound = 0;
-	int i;
+	int i,j;
 
 	for (i = 0; i < NUM_FLASH_BANKS; i++) {
 		printk (KERN_NOTICE MSG_PREFIX
@@ -166,6 +166,10 @@ int __init init_sbc8240_mtd (void)
 			(unsigned long) ioremap (pt[i].addr, pt[i].size);
 		if (!sbc8240_map[i].map_priv_1) {
 			printk (MSG_PREFIX "failed to ioremap\n");
+			for (j = 0; j < i; j++) {
+				iounmap((void *) sbc8240_map[j].map_priv_1);
+				sbc8240_map[j].map_priv_1 = 0;
+			}
 			return -EIO;
 		}
 		simple_map_init(&sbc8240_mtd[i]);
@@ -175,6 +179,11 @@ int __init init_sbc8240_mtd (void)
 		if (sbc8240_mtd[i]) {
 			sbc8240_mtd[i]->module = THIS_MODULE;
 			devicesfound++;
+		} else {
+			if (sbc8240_map[i].map_priv_1) {
+				iounmap((void *) sbc8240_map[i].map_priv_1);
+				sbc8240_map[i].map_priv_1 = 0;
+			}
 		}
 	}
 
diff --git a/drivers/mtd/maps/scx200_docflash.c b/drivers/mtd/maps/scx200_docflash.c
index 7391fd5..5e2bce2 100644
--- a/drivers/mtd/maps/scx200_docflash.c
+++ b/drivers/mtd/maps/scx200_docflash.c
@@ -87,19 +87,23 @@ static int __init init_scx200_docflash(void)
 
 	printk(KERN_DEBUG NAME ": NatSemi SCx200 DOCCS Flash Driver\n");
 
-	if ((bridge = pci_find_device(PCI_VENDOR_ID_NS,
+	if ((bridge = pci_get_device(PCI_VENDOR_ID_NS,
 				      PCI_DEVICE_ID_NS_SCx200_BRIDGE,
 				      NULL)) == NULL)
 		return -ENODEV;
 
 	/* check that we have found the configuration block */
-	if (!scx200_cb_present())
+	if (!scx200_cb_present()) {
+		pci_dev_put(bridge);
 		return -ENODEV;
+	}
 
 	if (probe) {
 		/* Try to use the present flash mapping if any */
 		pci_read_config_dword(bridge, SCx200_DOCCS_BASE, &base);
 		pci_read_config_dword(bridge, SCx200_DOCCS_CTRL, &ctrl);
+		pci_dev_put(bridge);
+
 		pmr = inl(scx200_cb_base + SCx200_PMR);
 
 		if (base == 0
@@ -127,6 +131,7 @@ static int __init init_scx200_docflash(void)
 			return -ENOMEM;
 		}
 	} else {
+		pci_dev_put(bridge);
 		for (u = size; u > 1; u >>= 1)
 			;
 		if (u != 1) {
diff --git a/drivers/mtd/maps/walnut.c b/drivers/mtd/maps/walnut.c
index ec80eec..ca93212 100644
--- a/drivers/mtd/maps/walnut.c
+++ b/drivers/mtd/maps/walnut.c
@@ -68,6 +68,7 @@ int __init init_walnut(void)
 
 	if (WALNUT_FLASH_ONBD_N(fpga_brds1)) {
 		printk("The on-board flash is disabled (U79 sw 5)!");
+		iounmap(fpga_status_adr);
 		return -EIO;
 	}
 	if (WALNUT_FLASH_SRAM_SEL(fpga_brds1))
@@ -81,6 +82,7 @@ int __init init_walnut(void)
 
 	if (!walnut_map.virt) {
 		printk("Failed to ioremap flash.\n");
+		iounmap(fpga_status_adr);
 		return -EIO;
 	}
 
@@ -93,9 +95,11 @@ int __init init_walnut(void)
 					ARRAY_SIZE(walnut_partitions));
 	} else {
 		printk("map probe failed for flash\n");
+		iounmap(fpga_status_adr);
 		return -ENXIO;
 	}
 
+	iounmap(fpga_status_adr);
 	return 0;
 }
 
diff --git a/drivers/mtd/mtdchar.c b/drivers/mtd/mtdchar.c
index fb8b4f7..5b6acfc 100644
--- a/drivers/mtd/mtdchar.c
+++ b/drivers/mtd/mtdchar.c
@@ -62,15 +62,12 @@ static loff_t mtd_lseek (struct file *file, loff_t offset, int orig)
 	struct mtd_info *mtd = mfi->mtd;
 
 	switch (orig) {
-	case 0:
-		/* SEEK_SET */
+	case SEEK_SET:
 		break;
-	case 1:
-		/* SEEK_CUR */
+	case SEEK_CUR:
 		offset += file->f_pos;
 		break;
-	case 2:
-		/* SEEK_END */
+	case SEEK_END:
 		offset += mtd->size;
 		break;
 	default:
diff --git a/drivers/mtd/mtdcore.c b/drivers/mtd/mtdcore.c
index 168d3ba..c4d26de 100644
--- a/drivers/mtd/mtdcore.c
+++ b/drivers/mtd/mtdcore.c
@@ -57,6 +57,16 @@ int add_mtd_device(struct mtd_info *mtd)
 			mtd->index = i;
 			mtd->usecount = 0;
 
+			/* Some chips always power up locked. Unlock them now */
+			if ((mtd->flags & MTD_WRITEABLE)
+			    && (mtd->flags & MTD_STUPID_LOCK) && mtd->unlock) {
+				if (mtd->unlock(mtd, 0, mtd->size))
+					printk(KERN_WARNING
+					       "%s: unlock failed, "
+					       "writes may not work\n",
+					       mtd->name);
+			}
+
 			DEBUG(0, "mtd: Giving out device %d to %s\n",i, mtd->name);
 			/* No need to get a refcount on the module containing
 			   the notifier, since we hold the mtd_table_mutex */
diff --git a/drivers/mtd/nand/Kconfig b/drivers/mtd/nand/Kconfig
index 3db77ee..c99302e 100644
--- a/drivers/mtd/nand/Kconfig
+++ b/drivers/mtd/nand/Kconfig
@@ -11,7 +11,7 @@ config MTD_NAND
 	help
 	  This enables support for accessing all type of NAND flash
 	  devices. For further information see
-	  <http://www.linux-mtd.infradead.org/tech/nand.html>.
+	  <http://www.linux-mtd.infradead.org/doc/nand.html>.
 
 config MTD_NAND_VERIFY_WRITE
 	bool "Verify NAND page writes"
diff --git a/drivers/mtd/nand/au1550nd.c b/drivers/mtd/nand/au1550nd.c
index 3122833..09e421a 100644
--- a/drivers/mtd/nand/au1550nd.c
+++ b/drivers/mtd/nand/au1550nd.c
@@ -21,18 +21,7 @@
 #include <linux/version.h>
 #include <asm/io.h>
 
-/* fixme: this is ugly */
-#if LINUX_VERSION_CODE > KERNEL_VERSION(2, 6, 0)
 #include <asm/mach-au1x00/au1xxx.h>
-#else
-#include <asm/au1000.h>
-#ifdef CONFIG_MIPS_PB1550
-#include <asm/pb1550.h>
-#endif
-#ifdef CONFIG_MIPS_DB1550
-#include <asm/db1x00.h>
-#endif
-#endif
 
 /*
  * MTD structure for NAND controller
diff --git a/drivers/mtd/nand/edb7312.c b/drivers/mtd/nand/edb7312.c
index 516c0e5..12017f3 100644
--- a/drivers/mtd/nand/edb7312.c
+++ b/drivers/mtd/nand/edb7312.c
@@ -198,6 +198,9 @@ static void __exit ep7312_cleanup(void)
 	/* Release resources, unregister device */
 	nand_release(ap7312_mtd);
 
+	/* Release io resource */
+	iounmap((void *)this->IO_ADDR_R);
+
 	/* Free the MTD device structure */
 	kfree(ep7312_mtd);
 }
diff --git a/drivers/mtd/nand/nand_base.c b/drivers/mtd/nand/nand_base.c
index 0a54d00..975b2ef 100644
--- a/drivers/mtd/nand/nand_base.c
+++ b/drivers/mtd/nand/nand_base.c
@@ -2224,7 +2224,7 @@ static struct nand_flash_dev *nand_get_flash_type(struct mtd_info *mtd,
 	}
 
 	/* Try to identify manufacturer */
-	for (maf_idx = 0; nand_manuf_ids[maf_idx].id != 0x0; maf_id++) {
+	for (maf_idx = 0; nand_manuf_ids[maf_idx].id != 0x0; maf_idx++) {
 		if (nand_manuf_ids[maf_idx].id == *maf_id)
 			break;
 	}
diff --git a/drivers/mtd/nand/ndfc.c b/drivers/mtd/nand/ndfc.c
index e5bd88f..039c759 100644
--- a/drivers/mtd/nand/ndfc.c
+++ b/drivers/mtd/nand/ndfc.c
@@ -168,7 +168,7 @@ static void ndfc_chip_init(struct ndfc_nand_mtd *mtd)
 	chip->ecc.mode = NAND_ECC_HW;
 	chip->ecc.size = 256;
 	chip->ecc.bytes = 3;
-	chip->ecclayout = mtd->pl_chip->ecclayout;
+	chip->ecclayout = chip->ecc.layout = mtd->pl_chip->ecclayout;
 	mtd->mtd.priv = chip;
 	mtd->mtd.owner = THIS_MODULE;
 }
diff --git a/drivers/mtd/nand/ppchameleonevb.c b/drivers/mtd/nand/ppchameleonevb.c
index 22fa65c..eb7d4d4 100644
--- a/drivers/mtd/nand/ppchameleonevb.c
+++ b/drivers/mtd/nand/ppchameleonevb.c
@@ -276,6 +276,7 @@ static int __init ppchameleonevb_init(void)
 	/* Scan to find existence of the device (it could not be mounted) */
 	if (nand_scan(ppchameleon_mtd, 1)) {
 		iounmap((void *)ppchameleon_fio_base);
+		ppchameleon_fio_base = NULL;
 		kfree(ppchameleon_mtd);
 		goto nand_evb_init;
 	}
@@ -314,6 +315,8 @@ static int __init ppchameleonevb_init(void)
 	ppchameleonevb_mtd = kmalloc(sizeof(struct mtd_info) + sizeof(struct nand_chip), GFP_KERNEL);
 	if (!ppchameleonevb_mtd) {
 		printk("Unable to allocate PPChameleonEVB NAND MTD device structure.\n");
+		if (ppchameleon_fio_base)
+			iounmap(ppchameleon_fio_base);
 		return -ENOMEM;
 	}
 
@@ -322,6 +325,8 @@ static int __init ppchameleonevb_init(void)
 	if (!ppchameleonevb_fio_base) {
 		printk("ioremap PPChameleonEVB NAND flash failed\n");
 		kfree(ppchameleonevb_mtd);
+		if (ppchameleon_fio_base)
+			iounmap(ppchameleon_fio_base);
 		return -EIO;
 	}
 
@@ -378,6 +383,8 @@ static int __init ppchameleonevb_init(void)
 	if (nand_scan(ppchameleonevb_mtd, 1)) {
 		iounmap((void *)ppchameleonevb_fio_base);
 		kfree(ppchameleonevb_mtd);
+		if (ppchameleon_fio_base)
+			iounmap(ppchameleon_fio_base);
 		return -ENXIO;
 	}
 #ifdef CONFIG_MTD_PARTITIONS
diff --git a/drivers/mtd/nand/sharpsl.c b/drivers/mtd/nand/sharpsl.c
index fbeedc3..51c7288 100644
--- a/drivers/mtd/nand/sharpsl.c
+++ b/drivers/mtd/nand/sharpsl.c
@@ -78,7 +78,7 @@ static struct mtd_partition sharpsl_nand_default_partition_info[] = {
 /*
  *	hardware specific access to control-lines
  *	ctrl:
- *	NAND_CNE: bit 0 -> bit 0 & 4
+ *	NAND_CNE: bit 0 -> ! bit 0 & 4
  *	NAND_CLE: bit 1 -> bit 1
  *	NAND_ALE: bit 2 -> bit 2
  *
@@ -92,7 +92,10 @@ static void sharpsl_nand_hwcontrol(struct mtd_info *mtd, int cmd,
 		unsigned char bits = ctrl & 0x07;
 
 		bits |= (ctrl & 0x01) << 4;
-		writeb((readb(FLASHCTL) & 0x17) | bits, FLASHCTL);
+
+		bits ^= 0x11;
+
+		writeb((readb(FLASHCTL) & ~0x17) | bits, FLASHCTL);
 	}
 
 	if (cmd != NAND_CMD_NONE)
diff --git a/drivers/mtd/ssfdc.c b/drivers/mtd/ssfdc.c
new file mode 100644
index 0000000..79d3bb6
--- /dev/null
+++ b/drivers/mtd/ssfdc.c
@@ -0,0 +1,474 @@
+/*
+ * Linux driver for SSFDC Flash Translation Layer (Read only)
+ * (c) 2005 Eptar srl
+ * Author: Claudio Lanconelli <lanconelli.claudio@eptar.com>
+ *
+ * Based on NTFL and MTDBLOCK_RO drivers
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ */
+
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/slab.h>
+#include <linux/hdreg.h>
+#include <linux/mtd/mtd.h>
+#include <linux/mtd/nand.h>
+#include <linux/mtd/blktrans.h>
+
+struct ssfdcr_record {
+	struct mtd_blktrans_dev mbd;
+	int usecount;
+	unsigned char heads;
+	unsigned char sectors;
+	unsigned short cylinders;
+	int cis_block;			/* block n. containing CIS/IDI */
+	int erase_size;			/* phys_block_size */
+	unsigned short *logic_block_map; /* all zones (max 8192 phys blocks on
+					    the 128MiB) */
+	int map_len;			/* n. phys_blocks on the card */
+};
+
+#define SSFDCR_MAJOR		257
+#define SSFDCR_PARTN_BITS	3
+
+#define SECTOR_SIZE		512
+#define SECTOR_SHIFT		9
+#define OOB_SIZE		16
+
+#define MAX_LOGIC_BLK_PER_ZONE	1000
+#define MAX_PHYS_BLK_PER_ZONE	1024
+
+#define KiB(x)	( (x) * 1024L )
+#define MiB(x)	( KiB(x) * 1024L )
+
+/** CHS Table
+		1MiB	2MiB	4MiB	8MiB	16MiB	32MiB	64MiB	128MiB
+NCylinder	125	125	250	250	500	500	500	500
+NHead		4	4	4	4	4	8	8	16
+NSector		4	8	8	16	16	16	32	32
+SumSector	2,000	4,000	8,000	16,000	32,000	64,000	128,000	256,000
+SectorSize	512	512	512	512	512	512	512	512
+**/
+
+typedef struct {
+	unsigned long size;
+	unsigned short cyl;
+	unsigned char head;
+	unsigned char sec;
+} chs_entry_t;
+
+/* Must be ordered by size */
+static const chs_entry_t chs_table[] = {
+	{ MiB(  1), 125,  4,  4 },
+	{ MiB(  2), 125,  4,  8 },
+	{ MiB(  4), 250,  4,  8 },
+	{ MiB(  8), 250,  4, 16 },
+	{ MiB( 16), 500,  4, 16 },
+	{ MiB( 32), 500,  8, 16 },
+	{ MiB( 64), 500,  8, 32 },
+	{ MiB(128), 500, 16, 32 },
+	{ 0 },
+};
+
+static int get_chs(unsigned long size, unsigned short *cyl, unsigned char *head,
+			unsigned char *sec)
+{
+	int k;
+	int found = 0;
+
+	k = 0;
+	while (chs_table[k].size > 0 && size > chs_table[k].size)
+		k++;
+
+	if (chs_table[k].size > 0) {
+		if (cyl)
+			*cyl = chs_table[k].cyl;
+		if (head)
+			*head = chs_table[k].head;
+		if (sec)
+			*sec = chs_table[k].sec;
+		found = 1;
+	}
+
+	return found;
+}
+
+/* These bytes are the signature for the CIS/IDI sector */
+static const uint8_t cis_numbers[] = {
+	0x01, 0x03, 0xD9, 0x01, 0xFF, 0x18, 0x02, 0xDF, 0x01, 0x20
+};
+
+/* Read and check for a valid CIS sector */
+static int get_valid_cis_sector(struct mtd_info *mtd)
+{
+	int ret, k, cis_sector;
+	size_t retlen;
+	loff_t offset;
+	uint8_t *sect_buf;
+
+	cis_sector = -1;
+
+	sect_buf = kmalloc(SECTOR_SIZE, GFP_KERNEL);
+	if (!sect_buf)
+		goto out;
+
+	/*
+	 * Look for CIS/IDI sector on the first GOOD block (give up after 4 bad
+	 * blocks). If the first good block doesn't contain CIS number the flash
+	 * is not SSFDC formatted
+	 */
+	for (k = 0, offset = 0; k < 4; k++, offset += mtd->erasesize) {
+		if (!mtd->block_isbad(mtd, offset)) {
+			ret = mtd->read(mtd, offset, SECTOR_SIZE, &retlen,
+				sect_buf);
+
+			/* CIS pattern match on the sector buffer */
+			if (ret < 0 || retlen != SECTOR_SIZE) {
+				printk(KERN_WARNING
+					"SSFDC_RO:can't read CIS/IDI sector\n");
+			} else if (!memcmp(sect_buf, cis_numbers,
+					sizeof(cis_numbers))) {
+				/* Found */
+				cis_sector = (int)(offset >> SECTOR_SHIFT);
+			} else {
+				DEBUG(MTD_DEBUG_LEVEL1,
+					"SSFDC_RO: CIS/IDI sector not found"
+					" on %s (mtd%d)\n", mtd->name,
+					mtd->index);
+			}
+			break;
+		}
+	}
+
+	kfree(sect_buf);
+ out:
+	return cis_sector;
+}
+
+/* Read physical sector (wrapper to MTD_READ) */
+static int read_physical_sector(struct mtd_info *mtd, uint8_t *sect_buf,
+				int sect_no)
+{
+	int ret;
+	size_t retlen;
+	loff_t offset = (loff_t)sect_no << SECTOR_SHIFT;
+
+	ret = mtd->read(mtd, offset, SECTOR_SIZE, &retlen, sect_buf);
+	if (ret < 0 || retlen != SECTOR_SIZE)
+		return -1;
+
+	return 0;
+}
+
+/* Read redundancy area (wrapper to MTD_READ_OOB */
+static int read_raw_oob(struct mtd_info *mtd, loff_t offs, uint8_t *buf)
+{
+	struct mtd_oob_ops ops;
+	int ret;
+
+	ops.mode = MTD_OOB_RAW;
+	ops.ooboffs = 0;
+	ops.ooblen = mtd->oobsize;
+	ops.len = OOB_SIZE;
+	ops.oobbuf = buf;
+	ops.datbuf = NULL;
+
+	ret = mtd->read_oob(mtd, offs, &ops);
+	if (ret < 0 || ops.retlen != OOB_SIZE)
+		return -1;
+
+	return 0;
+}
+
+/* Parity calculator on a word of n bit size */
+static int get_parity(int number, int size)
+{
+ 	int k;
+	int parity;
+
+	parity = 1;
+	for (k = 0; k < size; k++) {
+		parity += (number >> k);
+		parity &= 1;
+	}
+	return parity;
+}
+
+/* Read and validate the logical block address field stored in the OOB */
+static int get_logical_address(uint8_t *oob_buf)
+{
+	int block_address, parity;
+	int offset[2] = {6, 11}; /* offset of the 2 address fields within OOB */
+	int j;
+	int ok = 0;
+
+	/*
+	 * Look for the first valid logical address
+	 * Valid address has fixed pattern on most significant bits and
+	 * parity check
+	 */
+	for (j = 0; j < ARRAY_SIZE(offset); j++) {
+		block_address = ((int)oob_buf[offset[j]] << 8) |
+			oob_buf[offset[j]+1];
+
+		/* Check for the signature bits in the address field (MSBits) */
+		if ((block_address & ~0x7FF) == 0x1000) {
+			parity = block_address & 0x01;
+			block_address &= 0x7FF;
+			block_address >>= 1;
+
+			if (get_parity(block_address, 10) != parity) {
+				DEBUG(MTD_DEBUG_LEVEL0,
+					"SSFDC_RO: logical address field%d"
+					"parity error(0x%04X)\n", j+1,
+					block_address);
+			} else {
+				ok = 1;
+				break;
+			}
+		}
+	}
+
+	if (!ok)
+		block_address = -2;
+
+	DEBUG(MTD_DEBUG_LEVEL3, "SSFDC_RO: get_logical_address() %d\n",
+		block_address);
+
+	return block_address;
+}
+
+/* Build the logic block map */
+static int build_logical_block_map(struct ssfdcr_record *ssfdc)
+{
+	unsigned long offset;
+	uint8_t oob_buf[OOB_SIZE];
+	int ret, block_address, phys_block;
+	struct mtd_info *mtd = ssfdc->mbd.mtd;
+
+	DEBUG(MTD_DEBUG_LEVEL1, "SSFDC_RO: build_block_map() nblks=%d (%luK)\n",
+	      ssfdc->map_len,
+	      (unsigned long)ssfdc->map_len * ssfdc->erase_size / 1024);
+
+	/* Scan every physical block, skip CIS block */
+	for (phys_block = ssfdc->cis_block + 1; phys_block < ssfdc->map_len;
+			phys_block++) {
+		offset = (unsigned long)phys_block * ssfdc->erase_size;
+		if (mtd->block_isbad(mtd, offset))
+			continue;	/* skip bad blocks */
+
+		ret = read_raw_oob(mtd, offset, oob_buf);
+		if (ret < 0) {
+			DEBUG(MTD_DEBUG_LEVEL0,
+				"SSFDC_RO: mtd read_oob() failed at %lu\n",
+				offset);
+			return -1;
+		}
+		block_address = get_logical_address(oob_buf);
+
+		/* Skip invalid addresses */
+		if (block_address >= 0 &&
+				block_address < MAX_LOGIC_BLK_PER_ZONE) {
+			int zone_index;
+
+			zone_index = phys_block / MAX_PHYS_BLK_PER_ZONE;
+			block_address += zone_index * MAX_LOGIC_BLK_PER_ZONE;
+			ssfdc->logic_block_map[block_address] =
+				(unsigned short)phys_block;
+
+			DEBUG(MTD_DEBUG_LEVEL2,
+				"SSFDC_RO: build_block_map() phys_block=%d,"
+				"logic_block_addr=%d, zone=%d\n",
+				phys_block, block_address, zone_index);
+		}
+	}
+	return 0;
+}
+
+static void ssfdcr_add_mtd(struct mtd_blktrans_ops *tr, struct mtd_info *mtd)
+{
+	struct ssfdcr_record *ssfdc;
+	int cis_sector;
+
+	/* Check for small page NAND flash */
+	if (mtd->type != MTD_NANDFLASH || mtd->oobsize != OOB_SIZE)
+		return;
+
+	/* Check for SSDFC format by reading CIS/IDI sector */
+	cis_sector = get_valid_cis_sector(mtd);
+	if (cis_sector == -1)
+		return;
+
+	ssfdc = kzalloc(sizeof(struct ssfdcr_record), GFP_KERNEL);
+	if (!ssfdc) {
+		printk(KERN_WARNING
+			"SSFDC_RO: out of memory for data structures\n");
+		return;
+	}
+
+	ssfdc->mbd.mtd = mtd;
+	ssfdc->mbd.devnum = -1;
+	ssfdc->mbd.blksize = SECTOR_SIZE;
+	ssfdc->mbd.tr = tr;
+	ssfdc->mbd.readonly = 1;
+
+	ssfdc->cis_block = cis_sector / (mtd->erasesize >> SECTOR_SHIFT);
+	ssfdc->erase_size = mtd->erasesize;
+	ssfdc->map_len = mtd->size / mtd->erasesize;
+
+	DEBUG(MTD_DEBUG_LEVEL1,
+		"SSFDC_RO: cis_block=%d,erase_size=%d,map_len=%d,n_zones=%d\n",
+		ssfdc->cis_block, ssfdc->erase_size, ssfdc->map_len,
+		(ssfdc->map_len + MAX_PHYS_BLK_PER_ZONE - 1) /
+		MAX_PHYS_BLK_PER_ZONE);
+
+	/* Set geometry */
+	ssfdc->heads = 16;
+	ssfdc->sectors = 32;
+	get_chs(mtd->size, NULL, &ssfdc->heads, &ssfdc->sectors);
+	ssfdc->cylinders = (unsigned short)((mtd->size >> SECTOR_SHIFT) /
+			((long)ssfdc->sectors * (long)ssfdc->heads));
+
+	DEBUG(MTD_DEBUG_LEVEL1, "SSFDC_RO: using C:%d H:%d S:%d == %ld sects\n",
+		ssfdc->cylinders, ssfdc->heads , ssfdc->sectors,
+		(long)ssfdc->cylinders * (long)ssfdc->heads *
+		(long)ssfdc->sectors);
+
+	ssfdc->mbd.size = (long)ssfdc->heads * (long)ssfdc->cylinders *
+				(long)ssfdc->sectors;
+
+	/* Allocate logical block map */
+	ssfdc->logic_block_map = kmalloc(sizeof(ssfdc->logic_block_map[0]) *
+					 ssfdc->map_len, GFP_KERNEL);
+	if (!ssfdc->logic_block_map) {
+		printk(KERN_WARNING
+			"SSFDC_RO: out of memory for data structures\n");
+		goto out_err;
+	}
+	memset(ssfdc->logic_block_map, 0xff, sizeof(ssfdc->logic_block_map[0]) *
+		ssfdc->map_len);
+
+	/* Build logical block map */
+	if (build_logical_block_map(ssfdc) < 0)
+		goto out_err;
+
+	/* Register device + partitions */
+	if (add_mtd_blktrans_dev(&ssfdc->mbd))
+		goto out_err;
+
+	printk(KERN_INFO "SSFDC_RO: Found ssfdc%c on mtd%d (%s)\n",
+		ssfdc->mbd.devnum + 'a', mtd->index, mtd->name);
+	return;
+
+out_err:
+	kfree(ssfdc->logic_block_map);
+        kfree(ssfdc);
+}
+
+static void ssfdcr_remove_dev(struct mtd_blktrans_dev *dev)
+{
+	struct ssfdcr_record *ssfdc = (struct ssfdcr_record *)dev;
+
+	DEBUG(MTD_DEBUG_LEVEL1, "SSFDC_RO: remove_dev (i=%d)\n", dev->devnum);
+
+	del_mtd_blktrans_dev(dev);
+	kfree(ssfdc->logic_block_map);
+	kfree(ssfdc);
+}
+
+static int ssfdcr_readsect(struct mtd_blktrans_dev *dev,
+				unsigned long logic_sect_no, char *buf)
+{
+	struct ssfdcr_record *ssfdc = (struct ssfdcr_record *)dev;
+	int sectors_per_block, offset, block_address;
+
+	sectors_per_block = ssfdc->erase_size >> SECTOR_SHIFT;
+	offset = (int)(logic_sect_no % sectors_per_block);
+	block_address = (int)(logic_sect_no / sectors_per_block);
+
+	DEBUG(MTD_DEBUG_LEVEL3,
+		"SSFDC_RO: ssfdcr_readsect(%lu) sec_per_blk=%d, ofst=%d,"
+		" block_addr=%d\n", logic_sect_no, sectors_per_block, offset,
+		block_address);
+
+	if (block_address >= ssfdc->map_len)
+		BUG();
+
+	block_address = ssfdc->logic_block_map[block_address];
+
+	DEBUG(MTD_DEBUG_LEVEL3,
+		"SSFDC_RO: ssfdcr_readsect() phys_block_addr=%d\n",
+		block_address);
+
+	if (block_address < 0xffff) {
+		unsigned long sect_no;
+
+		sect_no = (unsigned long)block_address * sectors_per_block +
+				offset;
+
+		DEBUG(MTD_DEBUG_LEVEL3,
+			"SSFDC_RO: ssfdcr_readsect() phys_sect_no=%lu\n",
+			sect_no);
+
+		if (read_physical_sector(ssfdc->mbd.mtd, buf, sect_no) < 0)
+			return -EIO;
+	} else {
+		memset(buf, 0xff, SECTOR_SIZE);
+	}
+
+	return 0;
+}
+
+static int ssfdcr_getgeo(struct mtd_blktrans_dev *dev,  struct hd_geometry *geo)
+{
+	struct ssfdcr_record *ssfdc = (struct ssfdcr_record *)dev;
+
+	DEBUG(MTD_DEBUG_LEVEL1, "SSFDC_RO: ssfdcr_getgeo() C=%d, H=%d, S=%d\n",
+			ssfdc->cylinders, ssfdc->heads, ssfdc->sectors);
+
+	geo->heads = ssfdc->heads;
+	geo->sectors = ssfdc->sectors;
+	geo->cylinders = ssfdc->cylinders;
+
+	return 0;
+}
+
+/****************************************************************************
+ *
+ * Module stuff
+ *
+ ****************************************************************************/
+
+static struct mtd_blktrans_ops ssfdcr_tr = {
+	.name		= "ssfdc",
+	.major		= SSFDCR_MAJOR,
+	.part_bits	= SSFDCR_PARTN_BITS,
+	.getgeo		= ssfdcr_getgeo,
+	.readsect	= ssfdcr_readsect,
+	.add_mtd	= ssfdcr_add_mtd,
+	.remove_dev	= ssfdcr_remove_dev,
+	.owner		= THIS_MODULE,
+};
+
+static int __init init_ssfdcr(void)
+{
+	printk(KERN_INFO "SSFDC read-only Flash Translation layer\n");
+
+	return register_mtd_blktrans(&ssfdcr_tr);
+}
+
+static void __exit cleanup_ssfdcr(void)
+{
+	deregister_mtd_blktrans(&ssfdcr_tr);
+}
+
+module_init(init_ssfdcr);
+module_exit(cleanup_ssfdcr);
+
+MODULE_LICENSE("GPL");
+MODULE_AUTHOR("Claudio Lanconelli <lanconelli.claudio@eptar.com>");
+MODULE_DESCRIPTION("Flash Translation Layer for read-only SSFDC SmartMedia card");
diff --git a/drivers/net/3c59x.c b/drivers/net/3c59x.c
index 80e8ca0..29dede2 100644
--- a/drivers/net/3c59x.c
+++ b/drivers/net/3c59x.c
@@ -2077,7 +2077,7 @@ boomerang_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	vp->tx_ring[entry].next = 0;
 #if DO_ZEROCOPY
-	if (skb->ip_summed != CHECKSUM_HW)
+	if (skb->ip_summed != CHECKSUM_PARTIAL)
 			vp->tx_ring[entry].status = cpu_to_le32(skb->len | TxIntrUploaded);
 	else
 			vp->tx_ring[entry].status = cpu_to_le32(skb->len | TxIntrUploaded | AddTCPChksum | AddUDPChksum);
diff --git a/drivers/net/8139cp.c b/drivers/net/8139cp.c
index 1428bb7..a48b211 100644
--- a/drivers/net/8139cp.c
+++ b/drivers/net/8139cp.c
@@ -813,7 +813,7 @@ static int cp_start_xmit (struct sk_buff *skb, struct net_device *dev)
 
 		if (mss)
 			flags |= LargeSend | ((mss & MSSMask) << MSSShift);
-		else if (skb->ip_summed == CHECKSUM_HW) {
+		else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 			const struct iphdr *ip = skb->nh.iph;
 			if (ip->protocol == IPPROTO_TCP)
 				flags |= IPCS | TCPCS;
@@ -867,7 +867,7 @@ static int cp_start_xmit (struct sk_buff *skb, struct net_device *dev)
 			if (mss)
 				ctrl |= LargeSend |
 					((mss & MSSMask) << MSSShift);
-			else if (skb->ip_summed == CHECKSUM_HW) {
+			else if (skb->ip_summed == CHECKSUM_PARTIAL) {
 				if (ip->protocol == IPPROTO_TCP)
 					ctrl |= IPCS | TCPCS;
 				else if (ip->protocol == IPPROTO_UDP)
@@ -898,7 +898,7 @@ static int cp_start_xmit (struct sk_buff *skb, struct net_device *dev)
 		txd->addr = cpu_to_le64(first_mapping);
 		wmb();
 
-		if (skb->ip_summed == CHECKSUM_HW) {
+		if (skb->ip_summed == CHECKSUM_PARTIAL) {
 			if (ip->protocol == IPPROTO_TCP)
 				txd->opts1 = cpu_to_le32(first_eor | first_len |
 							 FirstFrag | DescOwn |
diff --git a/drivers/net/acenic.c b/drivers/net/acenic.c
index 1c01e9b..8265486 100644
--- a/drivers/net/acenic.c
+++ b/drivers/net/acenic.c
@@ -2040,7 +2040,7 @@ static void ace_rx_int(struct net_device *dev, u32 rxretprd, u32 rxretcsm)
 		 */
 		if (bd_flags & BD_FLG_TCP_UDP_SUM) {
 			skb->csum = htons(csum);
-			skb->ip_summed = CHECKSUM_HW;
+			skb->ip_summed = CHECKSUM_COMPLETE;
 		} else {
 			skb->ip_summed = CHECKSUM_NONE;
 		}
@@ -2511,7 +2511,7 @@ restart:
 
 		mapping = ace_map_tx_skb(ap, skb, skb, idx);
 		flagsize = (skb->len << 16) | (BD_FLG_END);
-		if (skb->ip_summed == CHECKSUM_HW)
+		if (skb->ip_summed == CHECKSUM_PARTIAL)
 			flagsize |= BD_FLG_TCP_UDP_SUM;
 #if ACENIC_DO_VLAN
 		if (vlan_tx_tag_present(skb)) {
@@ -2534,7 +2534,7 @@ restart:
 
 		mapping = ace_map_tx_skb(ap, skb, NULL, idx);
 		flagsize = (skb_headlen(skb) << 16);
-		if (skb->ip_summed == CHECKSUM_HW)
+		if (skb->ip_summed == CHECKSUM_PARTIAL)
 			flagsize |= BD_FLG_TCP_UDP_SUM;
 #if ACENIC_DO_VLAN
 		if (vlan_tx_tag_present(skb)) {
@@ -2560,7 +2560,7 @@ restart:
 					       PCI_DMA_TODEVICE);
 
 			flagsize = (frag->size << 16);
-			if (skb->ip_summed == CHECKSUM_HW)
+			if (skb->ip_summed == CHECKSUM_PARTIAL)
 				flagsize |= BD_FLG_TCP_UDP_SUM;
 			idx = (idx + 1) % ACE_TX_RING_ENTRIES(ap);
 
diff --git a/drivers/net/arcnet/com20020-pci.c b/drivers/net/arcnet/com20020-pci.c
index 979a33d..96d8a69 100644
--- a/drivers/net/arcnet/com20020-pci.c
+++ b/drivers/net/arcnet/com20020-pci.c
@@ -161,6 +161,7 @@ static struct pci_device_id com20020pci_id_table[] = {
 	{ 0x1571, 0xa204, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ARC_CAN_10MBIT },
 	{ 0x1571, 0xa205, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ARC_CAN_10MBIT },
 	{ 0x1571, 0xa206, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ARC_CAN_10MBIT },
+	{ 0x10B5, 0x9030, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ARC_CAN_10MBIT },
 	{ 0x10B5, 0x9050, PCI_ANY_ID, PCI_ANY_ID, 0, 0, ARC_CAN_10MBIT },
 	{0,}
 };
diff --git a/drivers/net/bmac.c b/drivers/net/bmac.c
index 6fad83f..7116096 100644
--- a/drivers/net/bmac.c
+++ b/drivers/net/bmac.c
@@ -1264,7 +1264,8 @@ static int __devinit bmac_probe(struct macio_dev *mdev, const struct of_device_i
 {
 	int j, rev, ret;
 	struct bmac_data *bp;
-	unsigned char *addr;
+	const unsigned char *prop_addr;
+	unsigned char addr[6];
 	struct net_device *dev;
 	int is_bmac_plus = ((int)match->data) != 0;
 
@@ -1272,14 +1273,16 @@ static int __devinit bmac_probe(struct macio_dev *mdev, const struct of_device_i
 		printk(KERN_ERR "BMAC: can't use, need 3 addrs and 3 intrs\n");
 		return -ENODEV;
 	}
-	addr = get_property(macio_get_of_node(mdev), "mac-address", NULL);
-	if (addr == NULL) {
-		addr = get_property(macio_get_of_node(mdev), "local-mac-address", NULL);
-		if (addr == NULL) {
+	prop_addr = get_property(macio_get_of_node(mdev), "mac-address", NULL);
+	if (prop_addr == NULL) {
+		prop_addr = get_property(macio_get_of_node(mdev),
+				"local-mac-address", NULL);
+		if (prop_addr == NULL) {
 			printk(KERN_ERR "BMAC: Can't get mac-address\n");
 			return -ENODEV;
 		}
 	}
+	memcpy(addr, prop_addr, sizeof(addr));
 
 	dev = alloc_etherdev(PRIV_BYTES);
 	if (!dev) {
diff --git a/drivers/net/bnx2.c b/drivers/net/bnx2.c
index 652eb05..7857b463 100644
--- a/drivers/net/bnx2.c
+++ b/drivers/net/bnx2.c
@@ -4423,7 +4423,7 @@ bnx2_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	ring_prod = TX_RING_IDX(prod);
 
 	vlan_tag_flags = 0;
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		vlan_tag_flags |= TX_BD_FLAGS_TCP_UDP_CKSUM;
 	}
 
diff --git a/drivers/net/cassini.c b/drivers/net/cassini.c
index a31544c..558fdb8 100644
--- a/drivers/net/cassini.c
+++ b/drivers/net/cassini.c
@@ -2167,7 +2167,7 @@ end_copy_pkt:
 			cas_page_unmap(addr);
 	}
 	skb->csum = ntohs(i ^ 0xffff);
-	skb->ip_summed = CHECKSUM_HW;
+	skb->ip_summed = CHECKSUM_COMPLETE;
 	skb->protocol = eth_type_trans(skb, cp->dev);
 	return len;
 }
@@ -2821,7 +2821,7 @@ static inline int cas_xmit_tx_ringN(struct cas *cp, int ring,
 	}
 
 	ctrl = 0;
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		u64 csum_start_off, csum_stuff_off;
 
 		csum_start_off = (u64) (skb->h.raw - skb->data);
diff --git a/drivers/net/chelsio/sge.c b/drivers/net/chelsio/sge.c
index 61b3754..ddd0bdb 100644
--- a/drivers/net/chelsio/sge.c
+++ b/drivers/net/chelsio/sge.c
@@ -1470,9 +1470,9 @@ int t1_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		}
 
 		if (!(adapter->flags & UDP_CSUM_CAPABLE) &&
-		    skb->ip_summed == CHECKSUM_HW &&
+		    skb->ip_summed == CHECKSUM_PARTIAL &&
 		    skb->nh.iph->protocol == IPPROTO_UDP)
-			if (unlikely(skb_checksum_help(skb, 0))) {
+			if (unlikely(skb_checksum_help(skb))) {
 				dev_kfree_skb_any(skb);
 				return NETDEV_TX_OK;
 			}
@@ -1495,11 +1495,11 @@ int t1_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		cpl = (struct cpl_tx_pkt *)__skb_push(skb, sizeof(*cpl));
 		cpl->opcode = CPL_TX_PKT;
 		cpl->ip_csum_dis = 1;    /* SW calculates IP csum */
-		cpl->l4_csum_dis = skb->ip_summed == CHECKSUM_HW ? 0 : 1;
+		cpl->l4_csum_dis = skb->ip_summed == CHECKSUM_PARTIAL ? 0 : 1;
 		/* the length field isn't used so don't bother setting it */
 
-		st->tx_cso += (skb->ip_summed == CHECKSUM_HW);
-		sge->stats.tx_do_cksum += (skb->ip_summed == CHECKSUM_HW);
+		st->tx_cso += (skb->ip_summed == CHECKSUM_PARTIAL);
+		sge->stats.tx_do_cksum += (skb->ip_summed == CHECKSUM_PARTIAL);
 		sge->stats.tx_reg_pkts++;
 	}
 	cpl->iff = dev->if_port;
diff --git a/drivers/net/dl2k.c b/drivers/net/dl2k.c
index 402961e..b74e676 100644
--- a/drivers/net/dl2k.c
+++ b/drivers/net/dl2k.c
@@ -611,7 +611,7 @@ start_xmit (struct sk_buff *skb, struct net_device *dev)
 	txdesc = &np->tx_ring[entry];
 
 #if 0
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		txdesc->status |=
 		    cpu_to_le64 (TCPChecksumEnable | UDPChecksumEnable |
 				 IPChecksumEnable);
diff --git a/drivers/net/e1000/e1000_main.c b/drivers/net/e1000/e1000_main.c
index 98ef9f8..2ab9f96 100644
--- a/drivers/net/e1000/e1000_main.c
+++ b/drivers/net/e1000/e1000_main.c
@@ -2600,7 +2600,7 @@ e1000_tx_csum(struct e1000_adapter *adapter, struct e1000_tx_ring *tx_ring,
 	unsigned int i;
 	uint8_t css;
 
-	if (likely(skb->ip_summed == CHECKSUM_HW)) {
+	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
 		css = skb->h.raw - skb->data;
 
 		i = tx_ring->next_to_use;
@@ -2927,11 +2927,11 @@ e1000_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
 	}
 
 	/* reserve a descriptor for the offload context */
-	if ((mss) || (skb->ip_summed == CHECKSUM_HW))
+	if ((mss) || (skb->ip_summed == CHECKSUM_PARTIAL))
 		count++;
 	count++;
 #else
-	if (skb->ip_summed == CHECKSUM_HW)
+	if (skb->ip_summed == CHECKSUM_PARTIAL)
 		count++;
 #endif
 
@@ -3608,7 +3608,7 @@ e1000_rx_checksum(struct e1000_adapter *adapter,
 		 */
 		csum = ntohl(csum ^ 0xFFFF);
 		skb->csum = csum;
-		skb->ip_summed = CHECKSUM_HW;
+		skb->ip_summed = CHECKSUM_COMPLETE;
 	}
 	adapter->hw_csum_good++;
 }
diff --git a/drivers/net/forcedeth.c b/drivers/net/forcedeth.c
index 11b8f1b..32cacf1 100644
--- a/drivers/net/forcedeth.c
+++ b/drivers/net/forcedeth.c
@@ -1503,7 +1503,8 @@ static int nv_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		tx_flags_extra = NV_TX2_TSO | (skb_shinfo(skb)->gso_size << NV_TX2_TSO_SHIFT);
 	else
 #endif
-	tx_flags_extra = (skb->ip_summed == CHECKSUM_HW ? (NV_TX2_CHECKSUM_L3|NV_TX2_CHECKSUM_L4) : 0);
+	tx_flags_extra = skb->ip_summed == CHECKSUM_PARTIAL ?
+			 NV_TX2_CHECKSUM_L3 | NV_TX2_CHECKSUM_L4 : 0;
 
 	/* vlan tag */
 	if (np->vlangrp && vlan_tx_tag_present(skb)) {
diff --git a/drivers/net/gianfar.c b/drivers/net/gianfar.c
index ebbbd6c..ba96091 100644
--- a/drivers/net/gianfar.c
+++ b/drivers/net/gianfar.c
@@ -947,7 +947,7 @@ static int gfar_start_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	/* Set up checksumming */
 	if (likely((dev->features & NETIF_F_IP_CSUM)
-			&& (CHECKSUM_HW == skb->ip_summed))) {
+			&& (CHECKSUM_PARTIAL == skb->ip_summed))) {
 		fcb = gfar_add_fcb(skb, txbdp);
 		status |= TXBD_TOE;
 		gfar_tx_checksum(skb, fcb);
diff --git a/drivers/net/hamachi.c b/drivers/net/hamachi.c
index 409c6aa..763373a 100644
--- a/drivers/net/hamachi.c
+++ b/drivers/net/hamachi.c
@@ -1648,7 +1648,7 @@ static int hamachi_rx(struct net_device *dev)
 						* could do the pseudo myself and return
 						* CHECKSUM_UNNECESSARY
 						*/
-						skb->ip_summed = CHECKSUM_HW;
+						skb->ip_summed = CHECKSUM_COMPLETE;
 					}
 				}	
 			}
diff --git a/drivers/net/ibm_emac/ibm_emac_core.c b/drivers/net/ibm_emac/ibm_emac_core.c
index 82468e2..57e214d 100644
--- a/drivers/net/ibm_emac/ibm_emac_core.c
+++ b/drivers/net/ibm_emac/ibm_emac_core.c
@@ -1036,7 +1036,7 @@ static inline u16 emac_tx_csum(struct ocp_enet_private *dev,
 			       struct sk_buff *skb)
 {
 #if defined(CONFIG_IBM_EMAC_TAH)
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		++dev->stats.tx_packets_csum;
 		return EMAC_TX_CTRL_TAH_CSUM;
 	}
diff --git a/drivers/net/ibmveth.c b/drivers/net/ibmveth.c
index 0464e78..e56eac8 100644
--- a/drivers/net/ibmveth.c
+++ b/drivers/net/ibmveth.c
@@ -702,7 +702,8 @@ static int ibmveth_start_xmit(struct sk_buff *skb, struct net_device *netdev)
 					     desc[3].desc,
 					     desc[4].desc,
 					     desc[5].desc,
-					     correlator);
+					     correlator,
+					     &correlator);
 	} while ((lpar_rc == H_BUSY) && (retry_count--));
 
 	if(lpar_rc != H_SUCCESS && lpar_rc != H_DROPPED) {
diff --git a/drivers/net/ibmveth.h b/drivers/net/ibmveth.h
index 8385bf8..f5b25bf 100644
--- a/drivers/net/ibmveth.h
+++ b/drivers/net/ibmveth.h
@@ -41,16 +41,6 @@
 #define IbmVethMcastRemoveFilter     0x2UL
 #define IbmVethMcastClearFilterTable 0x3UL
 
-/* hcall numbers */
-#define H_VIO_SIGNAL             0x104
-#define H_REGISTER_LOGICAL_LAN   0x114
-#define H_FREE_LOGICAL_LAN       0x118
-#define H_ADD_LOGICAL_LAN_BUFFER 0x11C
-#define H_SEND_LOGICAL_LAN       0x120
-#define H_MULTICAST_CTRL         0x130
-#define H_CHANGE_LOGICAL_LAN_MAC 0x14C
-#define H_FREE_LOGICAL_LAN_BUFFER 0x1D4
-
 /* hcall macros */
 #define h_register_logical_lan(ua, buflst, rxq, fltlst, mac) \
   plpar_hcall_norets(H_REGISTER_LOGICAL_LAN, ua, buflst, rxq, fltlst, mac)
@@ -61,8 +51,21 @@
 #define h_add_logical_lan_buffer(ua, buf) \
   plpar_hcall_norets(H_ADD_LOGICAL_LAN_BUFFER, ua, buf)
 
-#define h_send_logical_lan(ua, buf1, buf2, buf3, buf4, buf5, buf6, correlator) \
-  plpar_hcall_8arg_2ret(H_SEND_LOGICAL_LAN, ua, buf1, buf2, buf3, buf4, buf5, buf6, correlator, &correlator)
+static inline long h_send_logical_lan(unsigned long unit_address,
+		unsigned long desc1, unsigned long desc2, unsigned long desc3,
+		unsigned long desc4, unsigned long desc5, unsigned long desc6,
+		unsigned long corellator_in, unsigned long *corellator_out)
+{
+	long rc;
+	unsigned long retbuf[PLPAR_HCALL9_BUFSIZE];
+
+	rc = plpar_hcall9(H_SEND_LOGICAL_LAN, retbuf, unit_address, desc1,
+			desc2, desc3, desc4, desc5, desc6, corellator_in);
+
+	*corellator_out = retbuf[0];
+
+	return rc;
+}
 
 #define h_multicast_ctrl(ua, cmd, mac) \
   plpar_hcall_norets(H_MULTICAST_CTRL, ua, cmd, mac)
diff --git a/drivers/net/ioc3-eth.c b/drivers/net/ioc3-eth.c
index 68d8af7..65f897dd 100644
--- a/drivers/net/ioc3-eth.c
+++ b/drivers/net/ioc3-eth.c
@@ -1387,7 +1387,7 @@ static int ioc3_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	 * MAC header which should not be summed and the TCP/UDP pseudo headers
 	 * manually.
 	 */
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		int proto = ntohs(skb->nh.iph->protocol);
 		unsigned int csoff;
 		struct iphdr *ih = skb->nh.iph;
diff --git a/drivers/net/irda/ali-ircc.c b/drivers/net/irda/ali-ircc.c
index e3c8cd5..68d4c41 100644
--- a/drivers/net/irda/ali-ircc.c
+++ b/drivers/net/irda/ali-ircc.c
@@ -249,7 +249,7 @@ static void __exit ali_ircc_cleanup(void)
 
 	IRDA_DEBUG(2, "%s(), ---------------- Start ----------------\n", __FUNCTION__);	
 
-	for (i=0; i < 4; i++) {
+	for (i=0; i < ARRAY_SIZE(dev_self); i++) {
 		if (dev_self[i])
 			ali_ircc_close(dev_self[i]);
 	}
@@ -273,6 +273,12 @@ static int ali_ircc_open(int i, chipio_t *info)
 	int err;
 			
 	IRDA_DEBUG(2, "%s(), ---------------- Start ----------------\n", __FUNCTION__);	
+
+	if (i >= ARRAY_SIZE(dev_self)) {
+		IRDA_ERROR("%s(), maximum number of supported chips reached!\n",
+			   __FUNCTION__);
+		return -ENOMEM;
+	}
 	
 	/* Set FIR FIFO and DMA Threshold */
 	if ((ali_ircc_setup(info)) == -1)
diff --git a/drivers/net/irda/irport.c b/drivers/net/irda/irport.c
index 44efd49..ba4f3eb 100644
--- a/drivers/net/irda/irport.c
+++ b/drivers/net/irda/irport.c
@@ -1090,7 +1090,7 @@ static int __init irport_init(void)
 {
  	int i;
 
- 	for (i=0; (io[i] < 2000) && (i < 4); i++) {
+ 	for (i=0; (io[i] < 2000) && (i < ARRAY_SIZE(dev_self)); i++) {
  		if (irport_open(i, io[i], irq[i]) != NULL)
  			return 0;
  	}
@@ -1112,7 +1112,7 @@ static void __exit irport_cleanup(void)
 
         IRDA_DEBUG( 4, "%s()\n", __FUNCTION__);
 
-	for (i=0; i < 4; i++) {
+	for (i=0; i < ARRAY_SIZE(dev_self); i++) {
  		if (dev_self[i])
  			irport_close(dev_self[i]);
  	}
diff --git a/drivers/net/irda/via-ircc.c b/drivers/net/irda/via-ircc.c
index 8bafb45..79b85f3 100644
--- a/drivers/net/irda/via-ircc.c
+++ b/drivers/net/irda/via-ircc.c
@@ -279,7 +279,7 @@ static void via_ircc_clean(void)
 
 	IRDA_DEBUG(3, "%s()\n", __FUNCTION__);
 
-	for (i=0; i < 4; i++) {
+	for (i=0; i < ARRAY_SIZE(dev_self); i++) {
 		if (dev_self[i])
 			via_ircc_close(dev_self[i]);
 	}
@@ -327,6 +327,9 @@ static __devinit int via_ircc_open(int i, chipio_t * info, unsigned int id)
 
 	IRDA_DEBUG(3, "%s()\n", __FUNCTION__);
 
+	if (i >= ARRAY_SIZE(dev_self))
+		return -ENOMEM;
+
 	/* Allocate new instance of the driver */
 	dev = alloc_irdadev(sizeof(struct via_ircc_cb));
 	if (dev == NULL) 
diff --git a/drivers/net/irda/w83977af_ir.c b/drivers/net/irda/w83977af_ir.c
index 0ea65c4..8421597 100644
--- a/drivers/net/irda/w83977af_ir.c
+++ b/drivers/net/irda/w83977af_ir.c
@@ -117,7 +117,7 @@ static int __init w83977af_init(void)
 
 	IRDA_DEBUG(0, "%s()\n", __FUNCTION__ );
 
-	for (i=0; (io[i] < 2000) && (i < 4); i++) { 
+	for (i=0; (io[i] < 2000) && (i < ARRAY_SIZE(dev_self)); i++) {
 		if (w83977af_open(i, io[i], irq[i], dma[i]) == 0)
 			return 0;
 	}
@@ -136,7 +136,7 @@ static void __exit w83977af_cleanup(void)
 
         IRDA_DEBUG(4, "%s()\n", __FUNCTION__ );
 
-	for (i=0; i < 4; i++) {
+	for (i=0; i < ARRAY_SIZE(dev_self); i++) {
 		if (dev_self[i])
 			w83977af_close(dev_self[i]);
 	}
diff --git a/drivers/net/ixgb/ixgb_main.c b/drivers/net/ixgb/ixgb_main.c
index 7bbd447..9405b44 100644
--- a/drivers/net/ixgb/ixgb_main.c
+++ b/drivers/net/ixgb/ixgb_main.c
@@ -1232,7 +1232,7 @@ ixgb_tx_csum(struct ixgb_adapter *adapter, struct sk_buff *skb)
 	unsigned int i;
 	uint8_t css, cso;
 
-	if(likely(skb->ip_summed == CHECKSUM_HW)) {
+	if(likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
 		css = skb->h.raw - skb->data;
 		cso = (skb->h.raw + skb->csum) - skb->data;
 
diff --git a/drivers/net/lp486e.c b/drivers/net/lp486e.c
index b783a69..393aba9 100644
--- a/drivers/net/lp486e.c
+++ b/drivers/net/lp486e.c
@@ -442,16 +442,16 @@ init_rx_bufs(struct net_device *dev, int num) {
 		if (rbd) {
 			rbd->pad = 0;
 			rbd->count = 0;
-			rbd->skb = dev_alloc_skb(RX_SKB_SIZE);
+			rbd->skb = dev_alloc_skb(RX_SKBSIZE);
 			if (!rbd->skb) {
 				printk("dev_alloc_skb failed");
 			}
 			rbd->next = rfd->rbd;
 			if (i) {
 				rfd->rbd->prev = rbd;
-				rbd->size = RX_SKB_SIZE;
+				rbd->size = RX_SKBSIZE;
 			} else {
-				rbd->size = (RX_SKB_SIZE | RBD_EL);
+				rbd->size = (RX_SKBSIZE | RBD_EL);
 				lp->rbd_tail = rbd;
 			}
 
diff --git a/drivers/net/mace.c b/drivers/net/mace.c
index 29e4b5a..5d80e0e 100644
--- a/drivers/net/mace.c
+++ b/drivers/net/mace.c
@@ -113,7 +113,7 @@ static int __devinit mace_probe(struct macio_dev *mdev, const struct of_device_i
 	struct device_node *mace = macio_get_of_node(mdev);
 	struct net_device *dev;
 	struct mace_data *mp;
-	unsigned char *addr;
+	const unsigned char *addr;
 	int j, rev, rc = -EBUSY;
 
 	if (macio_resource_count(mdev) != 3 || macio_irq_count(mdev) != 3) {
diff --git a/drivers/net/mv643xx_eth.c b/drivers/net/mv643xx_eth.c
index eeab1df..d4dcc85 100644
--- a/drivers/net/mv643xx_eth.c
+++ b/drivers/net/mv643xx_eth.c
@@ -385,7 +385,7 @@ static int mv643xx_eth_receive_queue(struct net_device *dev, int budget)
 	struct pkt_info pkt_info;
 
 	while (budget-- > 0 && eth_port_receive(mp, &pkt_info) == ETH_OK) {
-		dma_unmap_single(NULL, pkt_info.buf_ptr, RX_SKB_SIZE,
+		dma_unmap_single(NULL, pkt_info.buf_ptr, ETH_RX_SKB_SIZE,
 							DMA_FROM_DEVICE);
 		mp->rx_desc_count--;
 		received_packets++;
@@ -1147,7 +1147,7 @@ static void eth_tx_submit_descs_for_skb(struct mv643xx_private *mp,
 	desc->byte_cnt = length;
 	desc->buf_ptr = dma_map_single(NULL, skb->data, length, DMA_TO_DEVICE);
 
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		BUG_ON(skb->protocol != ETH_P_IP);
 
 		cmd_sts |= ETH_GEN_TCP_UDP_CHECKSUM |
diff --git a/drivers/net/myri10ge/myri10ge.c b/drivers/net/myri10ge/myri10ge.c
index 9bdd43a..9f16681 100644
--- a/drivers/net/myri10ge/myri10ge.c
+++ b/drivers/net/myri10ge/myri10ge.c
@@ -930,7 +930,7 @@ static inline void myri10ge_vlan_ip_csum(struct sk_buff *skb, u16 hw_csum)
 	    (vh->h_vlan_encapsulated_proto == htons(ETH_P_IP) ||
 	     vh->h_vlan_encapsulated_proto == htons(ETH_P_IPV6))) {
 		skb->csum = hw_csum;
-		skb->ip_summed = CHECKSUM_HW;
+		skb->ip_summed = CHECKSUM_COMPLETE;
 	}
 }
 
@@ -973,7 +973,7 @@ myri10ge_rx_done(struct myri10ge_priv *mgp, struct myri10ge_rx_buf *rx,
 		if ((skb->protocol == ntohs(ETH_P_IP)) ||
 		    (skb->protocol == ntohs(ETH_P_IPV6))) {
 			skb->csum = ntohs((u16) csum);
-			skb->ip_summed = CHECKSUM_HW;
+			skb->ip_summed = CHECKSUM_COMPLETE;
 		} else
 			myri10ge_vlan_ip_csum(skb, ntohs((u16) csum));
 	}
@@ -1897,13 +1897,13 @@ again:
 	pseudo_hdr_offset = 0;
 	odd_flag = 0;
 	flags = (MXGEFW_FLAGS_NO_TSO | MXGEFW_FLAGS_FIRST);
-	if (likely(skb->ip_summed == CHECKSUM_HW)) {
+	if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
 		cksum_offset = (skb->h.raw - skb->data);
 		pseudo_hdr_offset = (skb->h.raw + skb->csum) - skb->data;
 		/* If the headers are excessively large, then we must
 		 * fall back to a software checksum */
 		if (unlikely(cksum_offset > 255 || pseudo_hdr_offset > 127)) {
-			if (skb_checksum_help(skb, 0))
+			if (skb_checksum_help(skb))
 				goto drop;
 			cksum_offset = 0;
 			pseudo_hdr_offset = 0;
diff --git a/drivers/net/ns83820.c b/drivers/net/ns83820.c
index 0e76859..5143f5d 100644
--- a/drivers/net/ns83820.c
+++ b/drivers/net/ns83820.c
@@ -1153,7 +1153,7 @@ again:
 	if (!nr_frags)
 		frag = NULL;
 	extsts = 0;
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		extsts |= EXTSTS_IPPKT;
 		if (IPPROTO_TCP == skb->nh.iph->protocol)
 			extsts |= EXTSTS_TCPPKT;
diff --git a/drivers/net/ppp_mppe.c b/drivers/net/ppp_mppe.c
index 51ff9a9..f3655fd 100644
--- a/drivers/net/ppp_mppe.c
+++ b/drivers/net/ppp_mppe.c
@@ -43,6 +43,7 @@
  *                    deprecated in 2.6
  */
 
+#include <linux/err.h>
 #include <linux/module.h>
 #include <linux/kernel.h>
 #include <linux/version.h>
@@ -64,12 +65,13 @@ MODULE_LICENSE("Dual BSD/GPL");
 MODULE_ALIAS("ppp-compress-" __stringify(CI_MPPE));
 MODULE_VERSION("1.0.2");
 
-static void
+static unsigned int
 setup_sg(struct scatterlist *sg, const void *address, unsigned int length)
 {
 	sg[0].page = virt_to_page(address);
 	sg[0].offset = offset_in_page(address);
 	sg[0].length = length;
+	return length;
 }
 
 #define SHA1_PAD_SIZE 40
@@ -95,8 +97,8 @@ static inline void sha_pad_init(struct sha_pad *shapad)
  * State for an MPPE (de)compressor.
  */
 struct ppp_mppe_state {
-	struct crypto_tfm *arc4;
-	struct crypto_tfm *sha1;
+	struct crypto_blkcipher *arc4;
+	struct crypto_hash *sha1;
 	unsigned char *sha1_digest;
 	unsigned char master_key[MPPE_MAX_KEY_LEN];
 	unsigned char session_key[MPPE_MAX_KEY_LEN];
@@ -136,14 +138,21 @@ struct ppp_mppe_state {
  */
 static void get_new_key_from_sha(struct ppp_mppe_state * state, unsigned char *InterimKey)
 {
+	struct hash_desc desc;
 	struct scatterlist sg[4];
+	unsigned int nbytes;
 
-	setup_sg(&sg[0], state->master_key, state->keylen);
-	setup_sg(&sg[1], sha_pad->sha_pad1, sizeof(sha_pad->sha_pad1));
-	setup_sg(&sg[2], state->session_key, state->keylen);
-	setup_sg(&sg[3], sha_pad->sha_pad2, sizeof(sha_pad->sha_pad2));
+	nbytes = setup_sg(&sg[0], state->master_key, state->keylen);
+	nbytes += setup_sg(&sg[1], sha_pad->sha_pad1,
+			   sizeof(sha_pad->sha_pad1));
+	nbytes += setup_sg(&sg[2], state->session_key, state->keylen);
+	nbytes += setup_sg(&sg[3], sha_pad->sha_pad2,
+			   sizeof(sha_pad->sha_pad2));
 
-	crypto_digest_digest (state->sha1, sg, 4, state->sha1_digest);
+	desc.tfm = state->sha1;
+	desc.flags = 0;
+
+	crypto_hash_digest(&desc, sg, nbytes, state->sha1_digest);
 
 	memcpy(InterimKey, state->sha1_digest, state->keylen);
 }
@@ -156,14 +165,15 @@ static void mppe_rekey(struct ppp_mppe_state * state, int initial_key)
 {
 	unsigned char InterimKey[MPPE_MAX_KEY_LEN];
 	struct scatterlist sg_in[1], sg_out[1];
+	struct blkcipher_desc desc = { .tfm = state->arc4 };
 
 	get_new_key_from_sha(state, InterimKey);
 	if (!initial_key) {
-		crypto_cipher_setkey(state->arc4, InterimKey, state->keylen);
+		crypto_blkcipher_setkey(state->arc4, InterimKey, state->keylen);
 		setup_sg(sg_in, InterimKey, state->keylen);
 		setup_sg(sg_out, state->session_key, state->keylen);
-		if (crypto_cipher_encrypt(state->arc4, sg_out, sg_in,
-				      state->keylen) != 0) {
+		if (crypto_blkcipher_encrypt(&desc, sg_out, sg_in,
+					     state->keylen) != 0) {
     		    printk(KERN_WARNING "mppe_rekey: cipher_encrypt failed\n");
 		}
 	} else {
@@ -175,7 +185,7 @@ static void mppe_rekey(struct ppp_mppe_state * state, int initial_key)
 		state->session_key[1] = 0x26;
 		state->session_key[2] = 0x9e;
 	}
-	crypto_cipher_setkey(state->arc4, state->session_key, state->keylen);
+	crypto_blkcipher_setkey(state->arc4, state->session_key, state->keylen);
 }
 
 /*
@@ -196,15 +206,19 @@ static void *mppe_alloc(unsigned char *options, int optlen)
 
 	memset(state, 0, sizeof(*state));
 
-	state->arc4 = crypto_alloc_tfm("arc4", 0);
-	if (!state->arc4)
+	state->arc4 = crypto_alloc_blkcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC);
+	if (IS_ERR(state->arc4)) {
+		state->arc4 = NULL;
 		goto out_free;
+	}
 
-	state->sha1 = crypto_alloc_tfm("sha1", 0);
-	if (!state->sha1)
+	state->sha1 = crypto_alloc_hash("sha1", 0, CRYPTO_ALG_ASYNC);
+	if (IS_ERR(state->sha1)) {
+		state->sha1 = NULL;
 		goto out_free;
+	}
 
-	digestsize = crypto_tfm_alg_digestsize(state->sha1);
+	digestsize = crypto_hash_digestsize(state->sha1);
 	if (digestsize < MPPE_MAX_KEY_LEN)
 		goto out_free;
 
@@ -229,9 +243,9 @@ static void *mppe_alloc(unsigned char *options, int optlen)
 	    if (state->sha1_digest)
 		kfree(state->sha1_digest);
 	    if (state->sha1)
-		crypto_free_tfm(state->sha1);
+		crypto_free_hash(state->sha1);
 	    if (state->arc4)
-		crypto_free_tfm(state->arc4);
+		crypto_free_blkcipher(state->arc4);
 	    kfree(state);
 	out:
 	return NULL;
@@ -247,9 +261,9 @@ static void mppe_free(void *arg)
 	    if (state->sha1_digest)
 		kfree(state->sha1_digest);
 	    if (state->sha1)
-		crypto_free_tfm(state->sha1);
+		crypto_free_hash(state->sha1);
 	    if (state->arc4)
-		crypto_free_tfm(state->arc4);
+		crypto_free_blkcipher(state->arc4);
 	    kfree(state);
 	}
 }
@@ -356,6 +370,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf,
 	      int isize, int osize)
 {
 	struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg;
+	struct blkcipher_desc desc = { .tfm = state->arc4 };
 	int proto;
 	struct scatterlist sg_in[1], sg_out[1];
 
@@ -413,7 +428,7 @@ mppe_compress(void *arg, unsigned char *ibuf, unsigned char *obuf,
 	/* Encrypt packet */
 	setup_sg(sg_in, ibuf, isize);
 	setup_sg(sg_out, obuf, osize);
-	if (crypto_cipher_encrypt(state->arc4, sg_out, sg_in, isize) != 0) {
+	if (crypto_blkcipher_encrypt(&desc, sg_out, sg_in, isize) != 0) {
 		printk(KERN_DEBUG "crypto_cypher_encrypt failed\n");
 		return -1;
 	}
@@ -462,6 +477,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf,
 		int osize)
 {
 	struct ppp_mppe_state *state = (struct ppp_mppe_state *) arg;
+	struct blkcipher_desc desc = { .tfm = state->arc4 };
 	unsigned ccount;
 	int flushed = MPPE_BITS(ibuf) & MPPE_BIT_FLUSHED;
 	int sanity = 0;
@@ -599,7 +615,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf,
 	 */
 	setup_sg(sg_in, ibuf, 1);
 	setup_sg(sg_out, obuf, 1);
-	if (crypto_cipher_decrypt(state->arc4, sg_out, sg_in, 1) != 0) {
+	if (crypto_blkcipher_decrypt(&desc, sg_out, sg_in, 1) != 0) {
 		printk(KERN_DEBUG "crypto_cypher_decrypt failed\n");
 		return DECOMP_ERROR;
 	}
@@ -619,7 +635,7 @@ mppe_decompress(void *arg, unsigned char *ibuf, int isize, unsigned char *obuf,
 	/* And finally, decrypt the rest of the packet. */
 	setup_sg(sg_in, ibuf + 1, isize - 1);
 	setup_sg(sg_out, obuf + 1, osize - 1);
-	if (crypto_cipher_decrypt(state->arc4, sg_out, sg_in, isize - 1) != 0) {
+	if (crypto_blkcipher_decrypt(&desc, sg_out, sg_in, isize - 1)) {
 		printk(KERN_DEBUG "crypto_cypher_decrypt failed\n");
 		return DECOMP_ERROR;
 	}
@@ -694,8 +710,8 @@ static struct compressor ppp_mppe = {
 static int __init ppp_mppe_init(void)
 {
 	int answer;
-	if (!(crypto_alg_available("arc4", 0) &&
-	      crypto_alg_available("sha1", 0)))
+	if (!(crypto_has_blkcipher("ecb(arc4)", 0, CRYPTO_ALG_ASYNC) &&
+	      crypto_has_hash("sha1", 0, CRYPTO_ALG_ASYNC)))
 		return -ENODEV;
 
 	sha_pad = kmalloc(sizeof(struct sha_pad), GFP_KERNEL);
diff --git a/drivers/net/r8169.c b/drivers/net/r8169.c
index 4c2f575..d9b960a 100644
--- a/drivers/net/r8169.c
+++ b/drivers/net/r8169.c
@@ -2169,7 +2169,7 @@ static inline u32 rtl8169_tso_csum(struct sk_buff *skb, struct net_device *dev)
 		if (mss)
 			return LargeSend | ((mss & MSSMask) << MSSShift);
 	}
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		const struct iphdr *ip = skb->nh.iph;
 
 		if (ip->protocol == IPPROTO_TCP)
diff --git a/drivers/net/s2io.c b/drivers/net/s2io.c
index e72e0e0..5b3713f 100644
--- a/drivers/net/s2io.c
+++ b/drivers/net/s2io.c
@@ -3893,7 +3893,7 @@ static int s2io_xmit(struct sk_buff *skb, struct net_device *dev)
 		txdp->Control_1 |= TXD_TCP_LSO_MSS(s2io_tcp_mss(skb));
 	}
 #endif
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		txdp->Control_2 |=
 		    (TXD_TX_CKO_IPV4_EN | TXD_TX_CKO_TCP_EN |
 		     TXD_TX_CKO_UDP_EN);
diff --git a/drivers/net/sk98lin/skge.c b/drivers/net/sk98lin/skge.c
index ee62845..eb3b351 100644
--- a/drivers/net/sk98lin/skge.c
+++ b/drivers/net/sk98lin/skge.c
@@ -1559,7 +1559,7 @@ struct sk_buff	*pMessage)	/* pointer to send-message              */
 	pTxd->VDataHigh = (SK_U32) (PhysAddr >> 32);
 	pTxd->pMBuf     = pMessage;
 
-	if (pMessage->ip_summed == CHECKSUM_HW) {
+	if (pMessage->ip_summed == CHECKSUM_PARTIAL) {
 		u16 hdrlen = pMessage->h.raw - pMessage->data;
 		u16 offset = hdrlen + pMessage->csum;
 
@@ -1678,7 +1678,7 @@ struct sk_buff	*pMessage)	/* pointer to send-message              */
 	/* 
 	** Does the HW need to evaluate checksum for TCP or UDP packets? 
 	*/
-	if (pMessage->ip_summed == CHECKSUM_HW) {
+	if (pMessage->ip_summed == CHECKSUM_PARTIAL) {
 		u16 hdrlen = pMessage->h.raw - pMessage->data;
 		u16 offset = hdrlen + pMessage->csum;
 
@@ -2158,7 +2158,7 @@ rx_start:
 
 #ifdef USE_SK_RX_CHECKSUM
 		pMsg->csum = pRxd->TcpSums & 0xffff;
-		pMsg->ip_summed = CHECKSUM_HW;
+		pMsg->ip_summed = CHECKSUM_COMPLETE;
 #else
 		pMsg->ip_summed = CHECKSUM_NONE;
 #endif
diff --git a/drivers/net/skge.c b/drivers/net/skge.c
index ad878df..b3d6fa3 100644
--- a/drivers/net/skge.c
+++ b/drivers/net/skge.c
@@ -2338,7 +2338,7 @@ static int skge_xmit_frame(struct sk_buff *skb, struct net_device *dev)
 	td->dma_lo = map;
 	td->dma_hi = map >> 32;
 
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		int offset = skb->h.raw - skb->data;
 
 		/* This seems backwards, but it is what the sk98lin
@@ -2642,7 +2642,7 @@ static inline struct sk_buff *skge_rx_get(struct skge_port *skge,
 	skb->dev = skge->netdev;
 	if (skge->rx_csum) {
 		skb->csum = csum;
-		skb->ip_summed = CHECKSUM_HW;
+		skb->ip_summed = CHECKSUM_COMPLETE;
 	}
 
 	skb->protocol = eth_type_trans(skb, skge->netdev);
diff --git a/drivers/net/sky2.c b/drivers/net/sky2.c
index 933e87f..8e92566 100644
--- a/drivers/net/sky2.c
+++ b/drivers/net/sky2.c
@@ -1163,7 +1163,7 @@ static unsigned tx_le_req(const struct sk_buff *skb)
 	if (skb_is_gso(skb))
 		++count;
 
-	if (skb->ip_summed == CHECKSUM_HW)
+	if (skb->ip_summed == CHECKSUM_PARTIAL)
 		++count;
 
 	return count;
@@ -1272,7 +1272,7 @@ static int sky2_xmit_frame(struct sk_buff *skb, struct net_device *dev)
 #endif
 
 	/* Handle TCP checksum offload */
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		u16 hdr = skb->h.raw - skb->data;
 		u16 offset = hdr + skb->csum;
 
@@ -2000,7 +2000,7 @@ static int sky2_status_intr(struct sky2_hw *hw, int to_do)
 #endif
 		case OP_RXCHKS:
 			skb = sky2->rx_ring[sky2->rx_next].skb;
-			skb->ip_summed = CHECKSUM_HW;
+			skb->ip_summed = CHECKSUM_COMPLETE;
 			skb->csum = le16_to_cpu(status);
 			break;
 
diff --git a/drivers/net/spider_net.c b/drivers/net/spider_net.c
index 8890721..d64e718 100644
--- a/drivers/net/spider_net.c
+++ b/drivers/net/spider_net.c
@@ -1697,10 +1697,10 @@ spider_net_setup_phy(struct spider_net_card *card)
  */
 static int
 spider_net_download_firmware(struct spider_net_card *card,
-			     u8 *firmware_ptr)
+			     const void *firmware_ptr)
 {
 	int sequencer, i;
-	u32 *fw_ptr = (u32 *)firmware_ptr;
+	const u32 *fw_ptr = firmware_ptr;
 
 	/* stop sequencers */
 	spider_net_write_reg(card, SPIDER_NET_GSINIT,
@@ -1757,7 +1757,7 @@ spider_net_init_firmware(struct spider_net_card *card)
 {
 	struct firmware *firmware = NULL;
 	struct device_node *dn;
-	u8 *fw_prop = NULL;
+	const u8 *fw_prop = NULL;
 	int err = -ENOENT;
 	int fw_size;
 
@@ -1783,7 +1783,7 @@ try_host_fw:
 	if (!dn)
 		goto out_err;
 
-	fw_prop = (u8 *)get_property(dn, "firmware", &fw_size);
+	fw_prop = get_property(dn, "firmware", &fw_size);
 	if (!fw_prop)
 		goto out_err;
 
@@ -1986,7 +1986,7 @@ spider_net_setup_netdev(struct spider_net_card *card)
 	struct net_device *netdev = card->netdev;
 	struct device_node *dn;
 	struct sockaddr addr;
-	u8 *mac;
+	const u8 *mac;
 
 	SET_MODULE_OWNER(netdev);
 	SET_NETDEV_DEV(netdev, &card->pdev->dev);
@@ -2019,7 +2019,7 @@ spider_net_setup_netdev(struct spider_net_card *card)
 	if (!dn)
 		return -EIO;
 
-	mac = (u8 *)get_property(dn, "local-mac-address", NULL);
+	mac = get_property(dn, "local-mac-address", NULL);
 	if (!mac)
 		return -EIO;
 	memcpy(addr.sa_data, mac, ETH_ALEN);
diff --git a/drivers/net/starfire.c b/drivers/net/starfire.c
index c0a62b0..2607aa5 100644
--- a/drivers/net/starfire.c
+++ b/drivers/net/starfire.c
@@ -1230,7 +1230,7 @@ static int start_tx(struct sk_buff *skb, struct net_device *dev)
 	}
 
 #if defined(ZEROCOPY) && defined(HAS_BROKEN_FIRMWARE)
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		if (skb_padto(skb, (skb->len + PADDING_MASK) & ~PADDING_MASK))
 			return NETDEV_TX_OK;
 	}
@@ -1252,7 +1252,7 @@ static int start_tx(struct sk_buff *skb, struct net_device *dev)
 				status |= TxDescIntr;
 				np->reap_tx = 0;
 			}
-			if (skb->ip_summed == CHECKSUM_HW) {
+			if (skb->ip_summed == CHECKSUM_PARTIAL) {
 				status |= TxCalTCP;
 				np->stats.tx_compressed++;
 			}
@@ -1499,7 +1499,7 @@ static int __netdev_rx(struct net_device *dev, int *quota)
 		 * Until then, the printk stays. :-) -Ion
 		 */
 		else if (le16_to_cpu(desc->status2) & 0x0040) {
-			skb->ip_summed = CHECKSUM_HW;
+			skb->ip_summed = CHECKSUM_COMPLETE;
 			skb->csum = le16_to_cpu(desc->csum);
 			printk(KERN_DEBUG "%s: checksum_hw, status2 = %#x\n", dev->name, le16_to_cpu(desc->status2));
 		}
diff --git a/drivers/net/sungem.c b/drivers/net/sungem.c
index b70bbd7..b388651 100644
--- a/drivers/net/sungem.c
+++ b/drivers/net/sungem.c
@@ -855,7 +855,7 @@ static int gem_rx(struct gem *gp, int work_to_do)
 		}
 
 		skb->csum = ntohs((status & RXDCTRL_TCPCSUM) ^ 0xffff);
-		skb->ip_summed = CHECKSUM_HW;
+		skb->ip_summed = CHECKSUM_COMPLETE;
 		skb->protocol = eth_type_trans(skb, gp->dev);
 
 		netif_receive_skb(skb);
@@ -1026,7 +1026,7 @@ static int gem_start_xmit(struct sk_buff *skb, struct net_device *dev)
 	unsigned long flags;
 
 	ctrl = 0;
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		u64 csum_start_off, csum_stuff_off;
 
 		csum_start_off = (u64) (skb->h.raw - skb->data);
@@ -2896,7 +2896,7 @@ static int __devinit gem_get_device_address(struct gem *gp)
 	if (use_idprom)
 		memcpy(dev->dev_addr, idprom->id_ethaddr, 6);
 #elif defined(CONFIG_PPC_PMAC)
-	unsigned char *addr;
+	const unsigned char *addr;
 
 	addr = get_property(gp->of_node, "local-mac-address", NULL);
 	if (addr == NULL) {
diff --git a/drivers/net/sunhme.c b/drivers/net/sunhme.c
index c6f5bc3..17981da 100644
--- a/drivers/net/sunhme.c
+++ b/drivers/net/sunhme.c
@@ -1207,7 +1207,7 @@ static void happy_meal_transceiver_check(struct happy_meal *hp, void __iomem *tr
  * flags, thus:
  *
  * 	skb->csum = rxd->rx_flags & 0xffff;
- * 	skb->ip_summed = CHECKSUM_HW;
+ * 	skb->ip_summed = CHECKSUM_COMPLETE;
  *
  * before sending off the skb to the protocols, and we are good as gold.
  */
@@ -2074,7 +2074,7 @@ static void happy_meal_rx(struct happy_meal *hp, struct net_device *dev)
 
 		/* This card is _fucking_ hot... */
 		skb->csum = ntohs(csum ^ 0xffff);
-		skb->ip_summed = CHECKSUM_HW;
+		skb->ip_summed = CHECKSUM_COMPLETE;
 
 		RXD(("len=%d csum=%4x]", len, csum));
 		skb->protocol = eth_type_trans(skb, dev);
@@ -2268,7 +2268,7 @@ static int happy_meal_start_xmit(struct sk_buff *skb, struct net_device *dev)
  	u32 tx_flags;
 
 	tx_flags = TXFLAG_OWN;
-	if (skb->ip_summed == CHECKSUM_HW) {
+	if (skb->ip_summed == CHECKSUM_PARTIAL) {
 		u32 csum_start_off, csum_stuff_off;
 
 		csum_start_off = (u32) (skb->h.raw - skb->data);
diff --git a/drivers/net/tg3.c b/drivers/net/tg3.c
index eafabb2..fb70261 100644
--- a/drivers/net/tg3.c
+++ b/drivers/net/tg3.c
@@ -149,122 +149,67 @@ module_param(tg3_debug, int, 0);
 MODULE_PARM_DESC(tg3_debug, "Tigon3 bitmapped debugging message enable value");
 
 static struct pci_device_id tg3_pci_tbl[] = {
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3,
-	  PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0UL },
-	{ 0, }
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5700)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5701)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702FE)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705_2)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705M_2)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702X)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703X)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5702A3)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5703A3)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5782)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5788)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5789)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5901_2)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5704S_2)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5705F)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5720)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5721)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5750M)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751M)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5751F)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5752M)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753M)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5753F)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5754M)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5755M)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5786)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5787M)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5714S)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5715S)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5780S)},
+	{PCI_DEVICE(PCI_VENDOR_ID_BROADCOM, PCI_DEVICE_ID_TIGON3_5781)},
+	{PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9DXX)},
+	{PCI_DEVICE(PCI_VENDOR_ID_SYSKONNECT, PCI_DEVICE_ID_SYSKONNECT_9MXX)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1000)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1001)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC1003)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ALTIMA, PCI_DEVICE_ID_ALTIMA_AC9100)},
+	{PCI_DEVICE(PCI_VENDOR_ID_APPLE, PCI_DEVICE_ID_APPLE_TIGON3)},
+	{}
 };
 
 MODULE_DEVICE_TABLE(pci, tg3_pci_tbl);
 
-static struct {
+static const struct {
 	const char string[ETH_GSTRING_LEN];
 } ethtool_stats_keys[TG3_NUM_STATS] = {
 	{ "rx_octets" },
@@ -345,7 +290,7 @@ static struct {
 	{ "nic_tx_threshold_hit" }
 };
 
-static struct {
+static const struct {
 	const char string[ETH_GSTRING_LEN];
 } ethtool_test_keys[TG3_NUM_TEST] = {
 	{ "nvram test     (online) " },
@@ -3851,11 +3796,11 @@ static int tg3_start_xmit(struct sk_buff *skb, struct net_device *dev)
 		skb->h.th->check = 0;
 
 	}
-	else if (skb->ip_summed == CHECKSUM_HW)
+	else if (skb->ip_summed == CHECKSUM_PARTIAL)
 		base_flags |= TXD_FLAG_TCPUDP_CSUM;
 #else
 	mss = 0;
-	if (skb->ip_summed == CHECKSUM_HW)
+	if (skb->ip_summed == CHECKSUM_PARTIAL)
 		base_flags |= TXD_FLAG_TCPUDP_CSUM;
 #endif
 #if TG3_VLAN_TAG_USED
@@ -3981,7 +3926,7 @@ static int tg3_start_xmit_dma_bug(struct sk_buff *skb, struct net_device *dev)
 
 	entry = tp->tx_prod;
 	base_flags = 0;
-	if (skb->ip_summed == CHECKSUM_HW)
+	if (skb->ip_summed == CHECKSUM_PARTIAL)
 		base_flags |= TXD_FLAG_TCPUDP_CSUM;
 #if TG3_TSO_SUPPORT != 0
 	mss = 0;
@@ -4969,7 +4914,7 @@ static int tg3_halt(struct tg3 *tp, int kind, int silent)
 #define TG3_FW_BSS_ADDR		0x08000a70
 #define TG3_FW_BSS_LEN		0x10
 
-static u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
+static const u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
 	0x00000000, 0x10000003, 0x00000000, 0x0000000d, 0x0000000d, 0x3c1d0800,
 	0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100000, 0x0e000018, 0x00000000,
 	0x0000000d, 0x3c1d0800, 0x37bd3ffc, 0x03a0f021, 0x3c100800, 0x26100034,
@@ -5063,7 +5008,7 @@ static u32 tg3FwText[(TG3_FW_TEXT_LEN / sizeof(u32)) + 1] = {
 	0x27bd0008, 0x03e00008, 0x00000000, 0x00000000, 0x00000000
 };
 
-static u32 tg3FwRodata[(TG3_FW_RODATA_LEN / sizeof(u32)) + 1] = {
+static const u32 tg3FwRodata[(TG3_FW_RODATA_LEN / sizeof(u32)) + 1] = {
 	0x35373031, 0x726c7341, 0x00000000, 0x00000000, 0x53774576, 0x656e7430,
 	0x00000000, 0x726c7045, 0x76656e74, 0x31000000, 0x556e6b6e, 0x45766e74,
 	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
@@ -5128,13 +5073,13 @@ static int tg3_halt_cpu(struct tg3 *tp, u32 offset)
 struct fw_info {
 	unsigned int text_base;
 	unsigned int text_len;
-	u32 *text_data;
+	const u32 *text_data;
 	unsigned int rodata_base;
 	unsigned int rodata_len;
-	u32 *rodata_data;
+	const u32 *rodata_data;
 	unsigned int data_base;
 	unsigned int data_len;
-	u32 *data_data;
+	const u32 *data_data;
 };
 
 /* tp->lock is held. */
@@ -5266,7 +5211,7 @@ static int tg3_load_5701_a0_firmware_fix(struct tg3 *tp)
 #define TG3_TSO_FW_BSS_ADDR		0x08001b80
 #define TG3_TSO_FW_BSS_LEN		0x894
 
-static u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
+static const u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
 	0x0e000003, 0x00000000, 0x08001b24, 0x00000000, 0x10000003, 0x00000000,
 	0x0000000d, 0x0000000d, 0x3c1d0800, 0x37bd4000, 0x03a0f021, 0x3c100800,
 	0x26100000, 0x0e000010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
@@ -5553,7 +5498,7 @@ static u32 tg3TsoFwText[(TG3_TSO_FW_TEXT_LEN / 4) + 1] = {
 	0xac470014, 0xac4a0018, 0x03e00008, 0xac4b001c, 0x00000000, 0x00000000,
 };
 
-static u32 tg3TsoFwRodata[] = {
+static const u32 tg3TsoFwRodata[] = {
 	0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
 	0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x496e0000, 0x73746b6f,
 	0x66662a2a, 0x00000000, 0x53774576, 0x656e7430, 0x00000000, 0x00000000,
@@ -5561,7 +5506,7 @@ static u32 tg3TsoFwRodata[] = {
 	0x00000000,
 };
 
-static u32 tg3TsoFwData[] = {
+static const u32 tg3TsoFwData[] = {
 	0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x362e3000, 0x00000000,
 	0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000, 0x00000000,
 	0x00000000,
@@ -5583,7 +5528,7 @@ static u32 tg3TsoFwData[] = {
 #define TG3_TSO5_FW_BSS_ADDR		0x00010f50
 #define TG3_TSO5_FW_BSS_LEN		0x88
 
-static u32 tg3Tso5FwText[(TG3_TSO5_FW_TEXT_LEN / 4) + 1] = {
+static const u32 tg3Tso5FwText[(TG3_TSO5_FW_TEXT_LEN / 4) + 1] = {
 	0x0c004003, 0x00000000, 0x00010f04, 0x00000000, 0x10000003, 0x00000000,
 	0x0000000d, 0x0000000d, 0x3c1d0001, 0x37bde000, 0x03a0f021, 0x3c100001,
 	0x26100000, 0x0c004010, 0x00000000, 0x0000000d, 0x27bdffe0, 0x3c04fefe,
@@ -5742,14 +5687,14 @@ static u32 tg3Tso5FwText[(TG3_TSO5_FW_TEXT_LEN / 4) + 1] = {
 	0x00000000, 0x00000000, 0x00000000,
 };
 
-static u32 tg3Tso5FwRodata[(TG3_TSO5_FW_RODATA_LEN / 4) + 1] = {
+static const u32 tg3Tso5FwRodata[(TG3_TSO5_FW_RODATA_LEN / 4) + 1] = {
 	0x4d61696e, 0x43707542, 0x00000000, 0x4d61696e, 0x43707541, 0x00000000,
 	0x00000000, 0x00000000, 0x73746b6f, 0x66666c64, 0x00000000, 0x00000000,
 	0x73746b6f, 0x66666c64, 0x00000000, 0x00000000, 0x66617461, 0x6c457272,
 	0x00000000, 0x00000000, 0x00000000,
 };
 
-static u32 tg3Tso5FwData[(TG3_TSO5_FW_DATA_LEN / 4) + 1] = {
+static const u32 tg3Tso5FwData[(TG3_TSO5_FW_DATA_LEN / 4) + 1] = {
 	0x00000000, 0x73746b6f, 0x66666c64, 0x5f76312e, 0x322e3000, 0x00000000,
 	0x00000000, 0x00000000, 0x00000000,
 };
diff --git a/drivers/net/typhoon.c b/drivers/net/typhoon.c
index 4103c37..c6e601d 100644
--- a/drivers/net/typhoon.c
+++ b/drivers/net/typhoon.c
@@ -830,7 +830,7 @@ typhoon_start_tx(struct sk_buff *skb, struct net_device *dev)
 	first_txd->addrHi = (u64)((unsigned long) skb) >> 32;
 	first_txd->processFlags = 0;
 
-	if(skb->ip_summed == CHECKSUM_HW) {
+	if(skb->ip_summed == CHECKSUM_PARTIAL) {
 		/* The 3XP will figure out if this is UDP/TCP */
 		first_txd->processFlags |= TYPHOON_TX_PF_TCP_CHKSUM;
 		first_txd->processFlags |= TYPHOON_TX_PF_UDP_CHKSUM;
diff --git a/drivers/net/via-rhine.c b/drivers/net/via-rhine.c
index ae97108..6654715 100644
--- a/drivers/net/via-rhine.c
+++ b/drivers/net/via-rhine.c
@@ -1230,7 +1230,7 @@ static int rhine_start_tx(struct sk_buff *skb, struct net_device *dev)
 	rp->tx_skbuff[entry] = skb;
 
 	if ((rp->quirks & rqRhineI) &&
-	    (((unsigned long)skb->data & 3) || skb_shinfo(skb)->nr_frags != 0 || skb->ip_summed == CHECKSUM_HW)) {
+	    (((unsigned long)skb->data & 3) || skb_shinfo(skb)->nr_frags != 0 || skb->ip_summed == CHECKSUM_PARTIAL)) {
 		/* Must use alignment buffer. */
 		if (skb->len > PKT_BUF_SZ) {
 			/* packet too long, drop it */
diff --git a/drivers/net/via-velocity.c b/drivers/net/via-velocity.c
index aa9cd92..f1e0c74 100644
--- a/drivers/net/via-velocity.c
+++ b/drivers/net/via-velocity.c
@@ -2002,7 +2002,7 @@ static int velocity_xmit(struct sk_buff *skb, struct net_device *dev)
 	 *	Handle hardware checksum
 	 */
 	if ((vptr->flags & VELOCITY_FLAGS_TX_CSUM)
-				 && (skb->ip_summed == CHECKSUM_HW)) {
+				 && (skb->ip_summed == CHECKSUM_PARTIAL)) {
 		struct iphdr *ip = skb->nh.iph;
 		if (ip->protocol == IPPROTO_TCP)
 			td_ptr->tdesc1.TCR |= TCR0_TCPCK;
diff --git a/drivers/net/wireless/airo.c b/drivers/net/wireless/airo.c
index a4dd139..170c500 100644
--- a/drivers/net/wireless/airo.c
+++ b/drivers/net/wireless/airo.c
@@ -19,6 +19,7 @@
 
 ======================================================================*/
 
+#include <linux/err.h>
 #include <linux/init.h>
 
 #include <linux/kernel.h>
@@ -1203,7 +1204,7 @@ struct airo_info {
 	struct iw_spy_data	spy_data;
 	struct iw_public_data	wireless_data;
 	/* MIC stuff */
-	struct crypto_tfm	*tfm;
+	struct crypto_cipher	*tfm;
 	mic_module		mod[2];
 	mic_statistics		micstats;
 	HostRxDesc rxfids[MPI_MAX_FIDS]; // rx/tx/config MPI350 descriptors
@@ -1271,7 +1272,8 @@ static int flashrestart(struct airo_info *ai,struct net_device *dev);
 
 static int RxSeqValid (struct airo_info *ai,miccntx *context,int mcast,u32 micSeq);
 static void MoveWindow(miccntx *context, u32 micSeq);
-static void emmh32_setseed(emmh32_context *context, u8 *pkey, int keylen, struct crypto_tfm *);
+static void emmh32_setseed(emmh32_context *context, u8 *pkey, int keylen,
+			   struct crypto_cipher *tfm);
 static void emmh32_init(emmh32_context *context);
 static void emmh32_update(emmh32_context *context, u8 *pOctets, int len);
 static void emmh32_final(emmh32_context *context, u8 digest[4]);
@@ -1339,10 +1341,11 @@ static int micsetup(struct airo_info *ai) {
 	int i;
 
 	if (ai->tfm == NULL)
-	        ai->tfm = crypto_alloc_tfm("aes", CRYPTO_TFM_REQ_MAY_SLEEP);
+	        ai->tfm = crypto_alloc_cipher("aes", 0, CRYPTO_ALG_ASYNC);
 
-        if (ai->tfm == NULL) {
+        if (IS_ERR(ai->tfm)) {
                 airo_print_err(ai->dev->name, "failed to load transform for AES");
+                ai->tfm = NULL;
                 return ERROR;
         }
 
@@ -1608,7 +1611,8 @@ static void MoveWindow(miccntx *context, u32 micSeq)
 static unsigned char aes_counter[16];
 
 /* expand the key to fill the MMH coefficient array */
-static void emmh32_setseed(emmh32_context *context, u8 *pkey, int keylen, struct crypto_tfm *tfm)
+static void emmh32_setseed(emmh32_context *context, u8 *pkey, int keylen,
+			   struct crypto_cipher *tfm)
 {
   /* take the keying material, expand if necessary, truncate at 16-bytes */
   /* run through AES counter mode to generate context->coeff[] */
@@ -1616,7 +1620,6 @@ static void emmh32_setseed(emmh32_context *context, u8 *pkey, int keylen, struct
 	int i,j;
 	u32 counter;
 	u8 *cipher, plain[16];
-	struct scatterlist sg[1];
 
 	crypto_cipher_setkey(tfm, pkey, 16);
 	counter = 0;
@@ -1627,9 +1630,8 @@ static void emmh32_setseed(emmh32_context *context, u8 *pkey, int keylen, struct
 		aes_counter[12] = (u8)(counter >> 24);
 		counter++;
 		memcpy (plain, aes_counter, 16);
-		sg_set_buf(sg, plain, 16);
-		crypto_cipher_encrypt(tfm, sg, sg, 16);
-		cipher = kmap(sg->page) + sg->offset;
+		crypto_cipher_encrypt_one(tfm, plain, plain);
+		cipher = plain;
 		for (j=0; (j<16) && (i< (sizeof(context->coeff)/sizeof(context->coeff[0]))); ) {
 			context->coeff[i++] = ntohl(*(u32 *)&cipher[j]);
 			j += 4;
@@ -2432,7 +2434,7 @@ void stop_airo_card( struct net_device *dev, int freeres )
 				ai->shared, ai->shared_dma);
 		}
         }
-	crypto_free_tfm(ai->tfm);
+	crypto_free_cipher(ai->tfm);
 	del_airo_dev( dev );
 	free_netdev( dev );
 }
diff --git a/drivers/pci/hotplug/rpaphp_core.c b/drivers/pci/hotplug/rpaphp_core.c
index 076bd6d..7288a3e 100644
--- a/drivers/pci/hotplug/rpaphp_core.c
+++ b/drivers/pci/hotplug/rpaphp_core.c
@@ -176,16 +176,16 @@ static int get_max_bus_speed(struct hotplug_slot *hotplug_slot, enum pci_bus_spe
 	return 0;
 }
 
-static int get_children_props(struct device_node *dn, int **drc_indexes,
-		int **drc_names, int **drc_types, int **drc_power_domains)
+static int get_children_props(struct device_node *dn, const int **drc_indexes,
+		const int **drc_names, const int **drc_types,
+		const int **drc_power_domains)
 {
-	int *indexes, *names;
-	int *types, *domains;
+	const int *indexes, *names, *types, *domains;
 
-	indexes = (int *) get_property(dn, "ibm,drc-indexes", NULL);
-	names = (int *) get_property(dn, "ibm,drc-names", NULL);
-	types = (int *) get_property(dn, "ibm,drc-types", NULL);
-	domains = (int *) get_property(dn, "ibm,drc-power-domains", NULL);
+	indexes = get_property(dn, "ibm,drc-indexes", NULL);
+	names = get_property(dn, "ibm,drc-names", NULL);
+	types = get_property(dn, "ibm,drc-types", NULL);
+	domains = get_property(dn, "ibm,drc-power-domains", NULL);
 
 	if (!indexes || !names || !types || !domains) {
 		/* Slot does not have dynamically-removable children */
@@ -212,13 +212,13 @@ static int get_children_props(struct device_node *dn, int **drc_indexes,
 int rpaphp_get_drc_props(struct device_node *dn, int *drc_index,
 		char **drc_name, char **drc_type, int *drc_power_domain)
 {
-	int *indexes, *names;
-	int *types, *domains;
-	unsigned int *my_index;
+	const int *indexes, *names;
+	const int *types, *domains;
+	const unsigned int *my_index;
 	char *name_tmp, *type_tmp;
 	int i, rc;
 
-	my_index = (int *) get_property(dn, "ibm,my-drc-index", NULL);
+	my_index = get_property(dn, "ibm,my-drc-index", NULL);
 	if (!my_index) {
 		/* Node isn't DLPAR/hotplug capable */
 		return -EINVAL;
@@ -265,10 +265,10 @@ static int is_php_type(char *drc_type)
 	return 1;
 }
 
-static int is_php_dn(struct device_node *dn, int **indexes, int **names,
-		int **types, int **power_domains)
+static int is_php_dn(struct device_node *dn, const int **indexes,
+		const int **names, const int **types, const int **power_domains)
 {
-	int *drc_types;
+	const int *drc_types;
 	int rc;
 
 	rc = get_children_props(dn, indexes, names, &drc_types, power_domains);
@@ -296,7 +296,7 @@ int rpaphp_add_slot(struct device_node *dn)
 	struct slot *slot;
 	int retval = 0;
 	int i;
-	int *indexes, *names, *types, *power_domains;
+	const int *indexes, *names, *types, *power_domains;
 	char *name, *type;
 
 	dbg("Entry %s: dn->full_name=%s\n", __FUNCTION__, dn->full_name);
diff --git a/drivers/s390/Kconfig b/drivers/s390/Kconfig
index 4d36208..ae89b9b8 100644
--- a/drivers/s390/Kconfig
+++ b/drivers/s390/Kconfig
@@ -213,17 +213,35 @@ config MONREADER
 	help
 	  Character device driver for reading z/VM monitor service records
 
+config MONWRITER
+	tristate "API for writing z/VM monitor service records"
+	default "m"
+	help
+	  Character device driver for writing z/VM monitor service records
+
 endmenu
 
 menu "Cryptographic devices"
 
-config Z90CRYPT
+config ZCRYPT
 	tristate "Support for PCI-attached cryptographic adapters"
-        default "m"
-        help
+	select ZCRYPT_MONOLITHIC if ZCRYPT="y"
+	default "m"
+	help
 	  Select this option if you want to use a PCI-attached cryptographic
-	  adapter like the PCI Cryptographic Accelerator (PCICA) or the PCI
-	  Cryptographic Coprocessor (PCICC).  This option is also available
-	  as a module called z90crypt.ko.
+	  adapter like:
+	  + PCI Cryptographic Accelerator (PCICA)
+	  + PCI Cryptographic Coprocessor (PCICC)
+	  + PCI-X Cryptographic Coprocessor (PCIXCC)
+	  + Crypto Express2 Coprocessor (CEX2C)
+	  + Crypto Express2 Accelerator (CEX2A)
+
+config ZCRYPT_MONOLITHIC
+	bool "Monolithic zcrypt module"
+	depends on ZCRYPT="m"
+	help
+	  Select this option if you want to have a single module z90crypt.ko
+	  that contains all parts of the crypto device driver (ap bus,
+	  request router and all the card drivers).
 
 endmenu
diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c
index 25c1ef6..d0647d1 100644
--- a/drivers/s390/block/dasd.c
+++ b/drivers/s390/block/dasd.c
@@ -184,7 +184,7 @@ dasd_state_known_to_basic(struct dasd_device * device)
 	device->debug_area = debug_register(device->cdev->dev.bus_id, 1, 2,
 					    8 * sizeof (long));
 	debug_register_view(device->debug_area, &debug_sprintf_view);
-	debug_set_level(device->debug_area, DBF_EMERG);
+	debug_set_level(device->debug_area, DBF_WARNING);
 	DBF_DEV_EVENT(DBF_EMERG, device, "%s", "debug area created");
 
 	device->state = DASD_STATE_BASIC;
@@ -893,7 +893,7 @@ dasd_handle_killed_request(struct ccw_device *cdev, unsigned long intparm)
 
 	device = (struct dasd_device *) cqr->device;
 	if (device == NULL ||
-	    device != dasd_device_from_cdev(cdev) ||
+	    device != dasd_device_from_cdev_locked(cdev) ||
 	    strncmp(device->discipline->ebcname, (char *) &cqr->magic, 4)) {
 		MESSAGE(KERN_DEBUG, "invalid device in request: bus_id %s",
 			cdev->dev.bus_id);
@@ -970,7 +970,7 @@ dasd_int_handler(struct ccw_device *cdev, unsigned long intparm,
 	/* first of all check for state change pending interrupt */
 	mask = DEV_STAT_ATTENTION | DEV_STAT_DEV_END | DEV_STAT_UNIT_EXCEP;
 	if ((irb->scsw.dstat & mask) == mask) {
-		device = dasd_device_from_cdev(cdev);
+		device = dasd_device_from_cdev_locked(cdev);
 		if (!IS_ERR(device)) {
 			dasd_handle_state_change_pending(device);
 			dasd_put_device(device);
@@ -2169,7 +2169,7 @@ dasd_init(void)
 		goto failed;
 	}
 	debug_register_view(dasd_debug_area, &debug_sprintf_view);
-	debug_set_level(dasd_debug_area, DBF_EMERG);
+	debug_set_level(dasd_debug_area, DBF_WARNING);
 
 	DBF_EVENT(DBF_EMERG, "%s", "debug area created");
 
diff --git a/drivers/s390/block/dasd_devmap.c b/drivers/s390/block/dasd_devmap.c
index 9af02c7..91cf971 100644
--- a/drivers/s390/block/dasd_devmap.c
+++ b/drivers/s390/block/dasd_devmap.c
@@ -258,8 +258,12 @@ dasd_parse_keyword( char *parsestring ) {
                 return residual_str;
         }
 	if (strncmp("nopav", parsestring, length) == 0) {
-		dasd_nopav = 1;
-		MESSAGE(KERN_INFO, "%s", "disable PAV mode");
+		if (MACHINE_IS_VM)
+			MESSAGE(KERN_INFO, "%s", "'nopav' not supported on VM");
+		else {
+			dasd_nopav = 1;
+			MESSAGE(KERN_INFO, "%s", "disable PAV mode");
+		}
 		return residual_str;
 	}
 	if (strncmp("fixedbuffers", parsestring, length) == 0) {
@@ -523,17 +527,17 @@ dasd_create_device(struct ccw_device *cdev)
 {
 	struct dasd_devmap *devmap;
 	struct dasd_device *device;
+	unsigned long flags;
 	int rc;
 
 	devmap = dasd_devmap_from_cdev(cdev);
 	if (IS_ERR(devmap))
 		return (void *) devmap;
-	cdev->dev.driver_data = devmap;
 
 	device = dasd_alloc_device();
 	if (IS_ERR(device))
 		return device;
-	atomic_set(&device->ref_count, 2);
+	atomic_set(&device->ref_count, 3);
 
 	spin_lock(&dasd_devmap_lock);
 	if (!devmap->device) {
@@ -552,6 +556,11 @@ dasd_create_device(struct ccw_device *cdev)
 		dasd_free_device(device);
 		return ERR_PTR(rc);
 	}
+
+	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
+	cdev->dev.driver_data = device;
+	spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
+
 	return device;
 }
 
@@ -569,6 +578,7 @@ dasd_delete_device(struct dasd_device *device)
 {
 	struct ccw_device *cdev;
 	struct dasd_devmap *devmap;
+	unsigned long flags;
 
 	/* First remove device pointer from devmap. */
 	devmap = dasd_find_busid(device->cdev->dev.bus_id);
@@ -582,9 +592,16 @@ dasd_delete_device(struct dasd_device *device)
 	devmap->device = NULL;
 	spin_unlock(&dasd_devmap_lock);
 
-	/* Drop ref_count by 2, one for the devmap reference and
-	 * one for the passed reference. */
-	atomic_sub(2, &device->ref_count);
+	/* Disconnect dasd_device structure from ccw_device structure. */
+	spin_lock_irqsave(get_ccwdev_lock(device->cdev), flags);
+	device->cdev->dev.driver_data = NULL;
+	spin_unlock_irqrestore(get_ccwdev_lock(device->cdev), flags);
+
+	/*
+	 * Drop ref_count by 3, one for the devmap reference, one for
+	 * the cdev reference and one for the passed reference.
+	 */
+	atomic_sub(3, &device->ref_count);
 
 	/* Wait for reference counter to drop to zero. */
 	wait_event(dasd_delete_wq, atomic_read(&device->ref_count) == 0);
@@ -593,9 +610,6 @@ dasd_delete_device(struct dasd_device *device)
 	cdev = device->cdev;
 	device->cdev = NULL;
 
-	/* Disconnect dasd_devmap structure from ccw_device structure. */
-	cdev->dev.driver_data = NULL;
-
 	/* Put ccw_device structure. */
 	put_device(&cdev->dev);
 
@@ -615,21 +629,32 @@ dasd_put_device_wake(struct dasd_device *device)
 
 /*
  * Return dasd_device structure associated with cdev.
+ * This function needs to be called with the ccw device
+ * lock held. It can be used from interrupt context.
+ */
+struct dasd_device *
+dasd_device_from_cdev_locked(struct ccw_device *cdev)
+{
+	struct dasd_device *device = cdev->dev.driver_data;
+
+	if (!device)
+		return ERR_PTR(-ENODEV);
+	dasd_get_device(device);
+	return device;
+}
+
+/*
+ * Return dasd_device structure associated with cdev.
  */
 struct dasd_device *
 dasd_device_from_cdev(struct ccw_device *cdev)
 {
-	struct dasd_devmap *devmap;
 	struct dasd_device *device;
+	unsigned long flags;
 
-	device = ERR_PTR(-ENODEV);
-	spin_lock(&dasd_devmap_lock);
-	devmap = cdev->dev.driver_data;
-	if (devmap && devmap->device) {
-		device = devmap->device;
-		dasd_get_device(device);
-	}
-	spin_unlock(&dasd_devmap_lock);
+	spin_lock_irqsave(get_ccwdev_lock(cdev), flags);
+	device = dasd_device_from_cdev_locked(cdev);
+	spin_unlock_irqrestore(get_ccwdev_lock(cdev), flags);
 	return device;
 }
 
@@ -730,16 +755,17 @@ static ssize_t
 dasd_discipline_show(struct device *dev, struct device_attribute *attr,
 		     char *buf)
 {
-	struct dasd_devmap *devmap;
-	char *dname;
+	struct dasd_device *device;
+	ssize_t len;
 
-	spin_lock(&dasd_devmap_lock);
-	dname = "none";
-	devmap = dev->driver_data;
-	if (devmap && devmap->device && devmap->device->discipline)
-		dname = devmap->device->discipline->name;
-	spin_unlock(&dasd_devmap_lock);
-	return snprintf(buf, PAGE_SIZE, "%s\n", dname);
+	device = dasd_device_from_cdev(to_ccwdev(dev));
+	if (!IS_ERR(device) && device->discipline) {
+		len = snprintf(buf, PAGE_SIZE, "%s\n",
+			       device->discipline->name);
+		dasd_put_device(device);
+	} else
+		len = snprintf(buf, PAGE_SIZE, "none\n");
+	return len;
 }
 
 static DEVICE_ATTR(discipline, 0444, dasd_discipline_show, NULL);
diff --git a/drivers/s390/block/dasd_eer.c b/drivers/s390/block/dasd_eer.c
index da65f1b..e0bf30e 100644
--- a/drivers/s390/block/dasd_eer.c
+++ b/drivers/s390/block/dasd_eer.c
@@ -678,7 +678,7 @@ int __init dasd_eer_init(void)
 	return 0;
 }
 
-void __exit dasd_eer_exit(void)
+void dasd_eer_exit(void)
 {
 	WARN_ON(misc_deregister(&dasd_eer_dev) != 0);
 }
diff --git a/drivers/s390/block/dasd_int.h b/drivers/s390/block/dasd_int.h
index 3ccf06d..9f52004 100644
--- a/drivers/s390/block/dasd_int.h
+++ b/drivers/s390/block/dasd_int.h
@@ -534,6 +534,7 @@ int dasd_add_sysfs_files(struct ccw_device *);
 void dasd_remove_sysfs_files(struct ccw_device *);
 
 struct dasd_device *dasd_device_from_cdev(struct ccw_device *);
+struct dasd_device *dasd_device_from_cdev_locked(struct ccw_device *);
 struct dasd_device *dasd_device_from_devindex(int);
 
 int dasd_parse(void);
diff --git a/drivers/s390/block/xpram.c b/drivers/s390/block/xpram.c
index ca7d51f..cab2c73 100644
--- a/drivers/s390/block/xpram.c
+++ b/drivers/s390/block/xpram.c
@@ -453,7 +453,7 @@ static int __init xpram_init(void)
 		PRINT_WARN("No expanded memory available\n");
 		return -ENODEV;
 	}
-	xpram_pages = xpram_highest_page_index();
+	xpram_pages = xpram_highest_page_index() + 1;
 	PRINT_INFO("  %u pages expanded memory found (%lu KB).\n",
 		   xpram_pages, (unsigned long) xpram_pages*4);
 	rc = xpram_setup_sizes(xpram_pages);
diff --git a/drivers/s390/char/Makefile b/drivers/s390/char/Makefile
index 0c0162ff..c3e97b4 100644
--- a/drivers/s390/char/Makefile
+++ b/drivers/s390/char/Makefile
@@ -28,3 +28,4 @@ obj-$(CONFIG_S390_TAPE) += tape.o tape_class.o
 obj-$(CONFIG_S390_TAPE_34XX) += tape_34xx.o
 obj-$(CONFIG_S390_TAPE_3590) += tape_3590.o
 obj-$(CONFIG_MONREADER) += monreader.o
+obj-$(CONFIG_MONWRITER) += monwriter.o
diff --git a/drivers/s390/char/monwriter.c b/drivers/s390/char/monwriter.c
new file mode 100644
index 0000000..1e3939a
--- /dev/null
+++ b/drivers/s390/char/monwriter.c
@@ -0,0 +1,292 @@
+/*
+ * drivers/s390/char/monwriter.c
+ *
+ * Character device driver for writing z/VM *MONITOR service records.
+ *
+ * Copyright (C) IBM Corp. 2006
+ *
+ * Author(s): Melissa Howland <Melissa.Howland@us.ibm.com>
+ */
+
+#include <linux/module.h>
+#include <linux/moduleparam.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/kernel.h>
+#include <linux/miscdevice.h>
+#include <linux/ctype.h>
+#include <linux/poll.h>
+#include <asm/uaccess.h>
+#include <asm/ebcdic.h>
+#include <asm/io.h>
+#include <asm/appldata.h>
+#include <asm/monwriter.h>
+
+#define MONWRITE_MAX_DATALEN	4024
+
+static int mon_max_bufs = 255;
+
+struct mon_buf {
+	struct list_head list;
+	struct monwrite_hdr hdr;
+	int diag_done;
+	char *data;
+};
+
+struct mon_private {
+	struct list_head list;
+	struct monwrite_hdr hdr;
+	size_t hdr_to_read;
+	size_t data_to_read;
+	struct mon_buf *current_buf;
+	int mon_buf_count;
+};
+
+/*
+ * helper functions
+ */
+
+static int monwrite_diag(struct monwrite_hdr *myhdr, char *buffer, int fcn)
+{
+	struct appldata_product_id id;
+	int rc;
+
+	strcpy(id.prod_nr, "LNXAPPL");
+	id.prod_fn = myhdr->applid;
+	id.record_nr = myhdr->record_num;
+	id.version_nr = myhdr->version;
+	id.release_nr = myhdr->release;
+	id.mod_lvl = myhdr->mod_level;
+	rc = appldata_asm(&id, fcn, (void *) buffer, myhdr->datalen);
+	if (rc <= 0)
+		return rc;
+	if (rc == 5)
+		return -EPERM;
+	printk("DIAG X'DC' error with return code: %i\n", rc);
+	return -EINVAL;
+}
+
+static inline struct mon_buf *monwrite_find_hdr(struct mon_private *monpriv,
+						struct monwrite_hdr *monhdr)
+{
+	struct mon_buf *entry, *next;
+
+	list_for_each_entry_safe(entry, next, &monpriv->list, list)
+		if (entry->hdr.applid == monhdr->applid &&
+		    entry->hdr.record_num == monhdr->record_num &&
+		    entry->hdr.version == monhdr->version &&
+		    entry->hdr.release == monhdr->release &&
+		    entry->hdr.mod_level == monhdr->mod_level)
+			return entry;
+	return NULL;
+}
+
+static int monwrite_new_hdr(struct mon_private *monpriv)
+{
+	struct monwrite_hdr *monhdr = &monpriv->hdr;
+	struct mon_buf *monbuf;
+	int rc;
+
+	if (monhdr->datalen > MONWRITE_MAX_DATALEN ||
+	    monhdr->mon_function > MONWRITE_START_CONFIG ||
+	    monhdr->hdrlen != sizeof(struct monwrite_hdr))
+		return -EINVAL;
+	monbuf = monwrite_find_hdr(monpriv, monhdr);
+	if (monbuf) {
+		if (monhdr->mon_function == MONWRITE_STOP_INTERVAL) {
+			monhdr->datalen = monbuf->hdr.datalen;
+			rc = monwrite_diag(monhdr, monbuf->data,
+					   APPLDATA_STOP_REC);
+			list_del(&monbuf->list);
+			monpriv->mon_buf_count--;
+			kfree(monbuf->data);
+			kfree(monbuf);
+			monbuf = NULL;
+		}
+	} else {
+		if (monpriv->mon_buf_count >= mon_max_bufs)
+			return -ENOSPC;
+		monbuf = kzalloc(sizeof(struct mon_buf), GFP_KERNEL);
+		if (!monbuf)
+			return -ENOMEM;
+		monbuf->data = kzalloc(monbuf->hdr.datalen,
+				       GFP_KERNEL | GFP_DMA);
+		if (!monbuf->data) {
+			kfree(monbuf);
+			return -ENOMEM;
+		}
+		monbuf->hdr = *monhdr;
+		list_add_tail(&monbuf->list, &monpriv->list);
+		monpriv->mon_buf_count++;
+	}
+	monpriv->current_buf = monbuf;
+	return 0;
+}
+
+static int monwrite_new_data(struct mon_private *monpriv)
+{
+	struct monwrite_hdr *monhdr = &monpriv->hdr;
+	struct mon_buf *monbuf = monpriv->current_buf;
+	int rc = 0;
+
+	switch (monhdr->mon_function) {
+	case MONWRITE_START_INTERVAL:
+		if (!monbuf->diag_done) {
+			rc = monwrite_diag(monhdr, monbuf->data,
+					   APPLDATA_START_INTERVAL_REC);
+			monbuf->diag_done = 1;
+		}
+		break;
+	case MONWRITE_START_CONFIG:
+		if (!monbuf->diag_done) {
+			rc = monwrite_diag(monhdr, monbuf->data,
+					   APPLDATA_START_CONFIG_REC);
+			monbuf->diag_done = 1;
+		}
+		break;
+	case MONWRITE_GEN_EVENT:
+		rc = monwrite_diag(monhdr, monbuf->data,
+				   APPLDATA_GEN_EVENT_REC);
+		list_del(&monpriv->current_buf->list);
+		kfree(monpriv->current_buf->data);
+		kfree(monpriv->current_buf);
+		monpriv->current_buf = NULL;
+		break;
+	default:
+		/* monhdr->mon_function is checked in monwrite_new_hdr */
+		BUG();
+	}
+	return rc;
+}
+
+/*
+ * file operations
+ */
+
+static int monwrite_open(struct inode *inode, struct file *filp)
+{
+	struct mon_private *monpriv;
+
+	monpriv = kzalloc(sizeof(struct mon_private), GFP_KERNEL);
+	if (!monpriv)
+		return -ENOMEM;
+	INIT_LIST_HEAD(&monpriv->list);
+	monpriv->hdr_to_read = sizeof(monpriv->hdr);
+	filp->private_data = monpriv;
+	return nonseekable_open(inode, filp);
+}
+
+static int monwrite_close(struct inode *inode, struct file *filp)
+{
+	struct mon_private *monpriv = filp->private_data;
+	struct mon_buf *entry, *next;
+
+	list_for_each_entry_safe(entry, next, &monpriv->list, list) {
+		if (entry->hdr.mon_function != MONWRITE_GEN_EVENT)
+			monwrite_diag(&entry->hdr, entry->data,
+				      APPLDATA_STOP_REC);
+		monpriv->mon_buf_count--;
+		list_del(&entry->list);
+		kfree(entry->data);
+		kfree(entry);
+	}
+	kfree(monpriv);
+	return 0;
+}
+
+static ssize_t monwrite_write(struct file *filp, const char __user *data,
+			      size_t count, loff_t *ppos)
+{
+	struct mon_private *monpriv = filp->private_data;
+	size_t len, written;
+	void *to;
+	int rc;
+
+	for (written = 0; written < count; ) {
+		if (monpriv->hdr_to_read) {
+			len = min(count - written, monpriv->hdr_to_read);
+			to = (char *) &monpriv->hdr +
+				sizeof(monpriv->hdr) - monpriv->hdr_to_read;
+			if (copy_from_user(to, data + written, len)) {
+				rc = -EFAULT;
+				goto out_error;
+			}
+			monpriv->hdr_to_read -= len;
+			written += len;
+			if (monpriv->hdr_to_read > 0)
+				continue;
+			rc = monwrite_new_hdr(monpriv);
+			if (rc)
+				goto out_error;
+			monpriv->data_to_read = monpriv->current_buf ?
+				monpriv->current_buf->hdr.datalen : 0;
+		}
+
+		if (monpriv->data_to_read) {
+			len = min(count - written, monpriv->data_to_read);
+			to = monpriv->current_buf->data +
+				monpriv->hdr.datalen - monpriv->data_to_read;
+			if (copy_from_user(to, data + written, len)) {
+				rc = -EFAULT;
+				goto out_error;
+			}
+			monpriv->data_to_read -= len;
+			written += len;
+			if (monpriv->data_to_read > 0)
+				continue;
+			rc = monwrite_new_data(monpriv);
+			if (rc)
+				goto out_error;
+		}
+		monpriv->hdr_to_read = sizeof(monpriv->hdr);
+	}
+	return written;
+
+out_error:
+	monpriv->data_to_read = 0;
+	monpriv->hdr_to_read = sizeof(struct monwrite_hdr);
+	return rc;
+}
+
+static struct file_operations monwrite_fops = {
+	.owner	 = THIS_MODULE,
+	.open	 = &monwrite_open,
+	.release = &monwrite_close,
+	.write	 = &monwrite_write,
+};
+
+static struct miscdevice mon_dev = {
+	.name	= "monwriter",
+	.fops	= &monwrite_fops,
+	.minor	= MISC_DYNAMIC_MINOR,
+};
+
+/*
+ * module init/exit
+ */
+
+static int __init mon_init(void)
+{
+	if (MACHINE_IS_VM)
+		return misc_register(&mon_dev);
+	else
+		return -ENODEV;
+}
+
+static void __exit mon_exit(void)
+{
+	WARN_ON(misc_deregister(&mon_dev) != 0);
+}
+
+module_init(mon_init);
+module_exit(mon_exit);
+
+module_param_named(max_bufs, mon_max_bufs, int, 0644);
+MODULE_PARM_DESC(max_bufs, "Maximum number of sample monitor data buffers"
+		 "that can be active at one time");
+
+MODULE_AUTHOR("Melissa Howland <Melissa.Howland@us.ibm.com>");
+MODULE_DESCRIPTION("Character device driver for writing z/VM "
+		   "APPLDATA monitor records.");
+MODULE_LICENSE("GPL");
diff --git a/drivers/s390/char/vmcp.c b/drivers/s390/char/vmcp.c
index 19762f3..1678b6c 100644
--- a/drivers/s390/char/vmcp.c
+++ b/drivers/s390/char/vmcp.c
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2004,2005 IBM Corporation
- * Interface implementation for communication with the v/VM control program
+ * Interface implementation for communication with the z/VM control program
  * Author(s): Christian Borntraeger <cborntra@de.ibm.com>
  *
  *
diff --git a/drivers/s390/char/vmcp.h b/drivers/s390/char/vmcp.h
index 87389e7..8a5975f 100644
--- a/drivers/s390/char/vmcp.h
+++ b/drivers/s390/char/vmcp.h
@@ -1,6 +1,6 @@
 /*
  * Copyright (C) 2004, 2005 IBM Corporation
- * Interface implementation for communication with the v/VM control program
+ * Interface implementation for communication with the z/VM control program
  * Version 1.0
  * Author(s): Christian Borntraeger <cborntra@de.ibm.com>
  *
diff --git a/drivers/s390/cio/chsc.c b/drivers/s390/cio/chsc.c
index c28444a..3bb4e47 100644
--- a/drivers/s390/cio/chsc.c
+++ b/drivers/s390/cio/chsc.c
@@ -256,7 +256,7 @@ s390_subchannel_remove_chpid(struct device *dev, void *data)
 	/* trigger path verification. */
 	if (sch->driver && sch->driver->verify)
 		sch->driver->verify(&sch->dev);
-	else if (sch->vpm == mask)
+	else if (sch->lpm == mask)
 		goto out_unreg;
 out_unlock:
 	spin_unlock_irq(&sch->lock);
@@ -378,6 +378,7 @@ __s390_process_res_acc(struct subchannel_id schid, void *data)
 
 	if (chp_mask == 0) {
 		spin_unlock_irq(&sch->lock);
+		put_device(&sch->dev);
 		return 0;
 	}
 	old_lpm = sch->lpm;
@@ -392,7 +393,7 @@ __s390_process_res_acc(struct subchannel_id schid, void *data)
 
 	spin_unlock_irq(&sch->lock);
 	put_device(&sch->dev);
-	return (res_data->fla_mask == 0xffff) ? -ENODEV : 0;
+	return 0;
 }
 
 
diff --git a/drivers/s390/cio/cio.c b/drivers/s390/cio/cio.c
index 89320c1..2e2882d 100644
--- a/drivers/s390/cio/cio.c
+++ b/drivers/s390/cio/cio.c
@@ -16,11 +16,10 @@
 #include <linux/device.h>
 #include <linux/kernel_stat.h>
 #include <linux/interrupt.h>
-
 #include <asm/cio.h>
 #include <asm/delay.h>
 #include <asm/irq.h>
-
+#include <asm/setup.h>
 #include "airq.h"
 #include "cio.h"
 #include "css.h"
@@ -192,7 +191,7 @@ cio_start_key (struct subchannel *sch,	/* subchannel structure */
 	sch->orb.pfch = sch->options.prefetch == 0;
 	sch->orb.spnd = sch->options.suspend;
 	sch->orb.ssic = sch->options.suspend && sch->options.inter;
-	sch->orb.lpm = (lpm != 0) ? (lpm & sch->opm) : sch->lpm;
+	sch->orb.lpm = (lpm != 0) ? lpm : sch->lpm;
 #ifdef CONFIG_64BIT
 	/*
 	 * for 64 bit we always support 64 bit IDAWs with 4k page size only
@@ -570,10 +569,7 @@ cio_validate_subchannel (struct subchannel *sch, struct subchannel_id schid)
 	sch->opm = 0xff;
 	if (!cio_is_console(sch->schid))
 		chsc_validate_chpids(sch);
-	sch->lpm = sch->schib.pmcw.pim &
-		sch->schib.pmcw.pam &
-		sch->schib.pmcw.pom &
-		sch->opm;
+	sch->lpm = sch->schib.pmcw.pam & sch->opm;
 
 	CIO_DEBUG(KERN_INFO, 0,
 		  "Detected device %04x on subchannel 0.%x.%04X"
@@ -841,14 +837,26 @@ __clear_subchannel_easy(struct subchannel_id schid)
 	return -EBUSY;
 }
 
-extern void do_reipl(unsigned long devno);
-static int
-__shutdown_subchannel_easy(struct subchannel_id schid, void *data)
+struct sch_match_id {
+	struct subchannel_id schid;
+	struct ccw_dev_id devid;
+	int rc;
+};
+
+static int __shutdown_subchannel_easy_and_match(struct subchannel_id schid,
+	void *data)
 {
 	struct schib schib;
+	struct sch_match_id *match_id = data;
 
 	if (stsch_err(schid, &schib))
 		return -ENXIO;
+	if (match_id && schib.pmcw.dnv &&
+		(schib.pmcw.dev == match_id->devid.devno) &&
+		(schid.ssid == match_id->devid.ssid)) {
+		match_id->schid = schid;
+		match_id->rc = 0;
+	}
 	if (!schib.pmcw.ena)
 		return 0;
 	switch(__disable_subchannel_easy(schid, &schib)) {
@@ -864,18 +872,71 @@ __shutdown_subchannel_easy(struct subchannel_id schid, void *data)
 	return 0;
 }
 
-void
-clear_all_subchannels(void)
+static int clear_all_subchannels_and_match(struct ccw_dev_id *devid,
+	struct subchannel_id *schid)
 {
+	struct sch_match_id match_id;
+
+	match_id.devid = *devid;
+	match_id.rc = -ENODEV;
 	local_irq_disable();
-	for_each_subchannel(__shutdown_subchannel_easy, NULL);
+	for_each_subchannel(__shutdown_subchannel_easy_and_match, &match_id);
+	if (match_id.rc == 0)
+		*schid = match_id.schid;
+	return match_id.rc;
 }
 
+
+void clear_all_subchannels(void)
+{
+	local_irq_disable();
+	for_each_subchannel(__shutdown_subchannel_easy_and_match, NULL);
+}
+
+extern void do_reipl_asm(__u32 schid);
+
 /* Make sure all subchannels are quiet before we re-ipl an lpar. */
-void
-reipl(unsigned long devno)
+void reipl_ccw_dev(struct ccw_dev_id *devid)
 {
-	clear_all_subchannels();
+	struct subchannel_id schid;
+
+	if (clear_all_subchannels_and_match(devid, &schid))
+		panic("IPL Device not found\n");
 	cio_reset_channel_paths();
-	do_reipl(devno);
+	do_reipl_asm(*((__u32*)&schid));
+}
+
+extern struct schib ipl_schib;
+
+/*
+ * ipl_save_parameters gets called very early. It is not allowed to access
+ * anything in the bss section at all. The bss section is not cleared yet,
+ * but may contain some ipl parameters written by the firmware.
+ * These parameters (if present) are copied to 0x2000.
+ * To avoid corruption of the ipl parameters, all variables used by this
+ * function must reside on the stack or in the data section.
+ */
+void ipl_save_parameters(void)
+{
+	struct subchannel_id schid;
+	unsigned int *ipl_ptr;
+	void *src, *dst;
+
+	schid = *(struct subchannel_id *)__LC_SUBCHANNEL_ID;
+	if (!schid.one)
+		return;
+	if (stsch(schid, &ipl_schib))
+		return;
+	if (!ipl_schib.pmcw.dnv)
+		return;
+	ipl_devno = ipl_schib.pmcw.dev;
+	ipl_flags |= IPL_DEVNO_VALID;
+	if (!ipl_schib.pmcw.qf)
+		return;
+	ipl_flags |= IPL_PARMBLOCK_VALID;
+	ipl_ptr = (unsigned int *)__LC_IPL_PARMBLOCK_PTR;
+	src = (void *)(unsigned long)*ipl_ptr;
+	dst = (void *)IPL_PARMBLOCK_ORIGIN;
+	memmove(dst, src, PAGE_SIZE);
+	*ipl_ptr = IPL_PARMBLOCK_ORIGIN;
 }
diff --git a/drivers/s390/cio/css.c b/drivers/s390/cio/css.c
index 13eeea3..7086a74 100644
--- a/drivers/s390/cio/css.c
+++ b/drivers/s390/cio/css.c
@@ -182,136 +182,141 @@ get_subchannel_by_schid(struct subchannel_id schid)
 	return dev ? to_subchannel(dev) : NULL;
 }
 
-
-static inline int
-css_get_subchannel_status(struct subchannel *sch, struct subchannel_id schid)
+static inline int css_get_subchannel_status(struct subchannel *sch)
 {
 	struct schib schib;
-	int cc;
 
-	cc = stsch(schid, &schib);
-	if (cc)
-		return CIO_GONE;
-	if (!schib.pmcw.dnv)
+	if (stsch(sch->schid, &schib) || !schib.pmcw.dnv)
 		return CIO_GONE;
-	if (sch && sch->schib.pmcw.dnv &&
-	    (schib.pmcw.dev != sch->schib.pmcw.dev))
+	if (sch->schib.pmcw.dnv && (schib.pmcw.dev != sch->schib.pmcw.dev))
 		return CIO_REVALIDATE;
-	if (sch && !sch->lpm)
+	if (!sch->lpm)
 		return CIO_NO_PATH;
 	return CIO_OPER;
 }
-	
-static int
-css_evaluate_subchannel(struct subchannel_id schid, int slow)
+
+static int css_evaluate_known_subchannel(struct subchannel *sch, int slow)
 {
 	int event, ret, disc;
-	struct subchannel *sch;
 	unsigned long flags;
+	enum { NONE, UNREGISTER, UNREGISTER_PROBE, REPROBE } action;
 
-	sch = get_subchannel_by_schid(schid);
-	disc = sch ? device_is_disconnected(sch) : 0;
+	spin_lock_irqsave(&sch->lock, flags);
+	disc = device_is_disconnected(sch);
 	if (disc && slow) {
-		if (sch)
-			put_device(&sch->dev);
-		return 0; /* Already processed. */
+		/* Disconnected devices are evaluated directly only.*/
+		spin_unlock_irqrestore(&sch->lock, flags);
+		return 0;
 	}
-	/*
-	 * We've got a machine check, so running I/O won't get an interrupt.
-	 * Kill any pending timers.
-	 */
-	if (sch)
-		device_kill_pending_timer(sch);
+	/* No interrupt after machine check - kill pending timers. */
+	device_kill_pending_timer(sch);
 	if (!disc && !slow) {
-		if (sch)
-			put_device(&sch->dev);
-		return -EAGAIN; /* Will be done on the slow path. */
+		/* Non-disconnected devices are evaluated on the slow path. */
+		spin_unlock_irqrestore(&sch->lock, flags);
+		return -EAGAIN;
 	}
-	event = css_get_subchannel_status(sch, schid);
+	event = css_get_subchannel_status(sch);
 	CIO_MSG_EVENT(4, "Evaluating schid 0.%x.%04x, event %d, %s, %s path.\n",
-		      schid.ssid, schid.sch_no, event,
-		      sch?(disc?"disconnected":"normal"):"unknown",
-		      slow?"slow":"fast");
+		      sch->schid.ssid, sch->schid.sch_no, event,
+		      disc ? "disconnected" : "normal",
+		      slow ? "slow" : "fast");
+	/* Analyze subchannel status. */
+	action = NONE;
 	switch (event) {
 	case CIO_NO_PATH:
-	case CIO_GONE:
-		if (!sch) {
-			/* Never used this subchannel. Ignore. */
-			ret = 0;
+		if (disc) {
+			/* Check if paths have become available. */
+			action = REPROBE;
 			break;
 		}
-		if (disc && (event == CIO_NO_PATH)) {
-			/*
-			 * Uargh, hack again. Because we don't get a machine
-			 * check on configure on, our path bookkeeping can
-			 * be out of date here (it's fine while we only do
-			 * logical varying or get chsc machine checks). We
-			 * need to force reprobing or we might miss devices
-			 * coming operational again. It won't do harm in real
-			 * no path situations.
-			 */
-			spin_lock_irqsave(&sch->lock, flags);
-			device_trigger_reprobe(sch);
+		/* fall through */
+	case CIO_GONE:
+		/* Prevent unwanted effects when opening lock. */
+		cio_disable_subchannel(sch);
+		device_set_disconnected(sch);
+		/* Ask driver what to do with device. */
+		action = UNREGISTER;
+		if (sch->driver && sch->driver->notify) {
 			spin_unlock_irqrestore(&sch->lock, flags);
-			ret = 0;
-			break;
-		}
-		if (sch->driver && sch->driver->notify &&
-		    sch->driver->notify(&sch->dev, event)) {
-			cio_disable_subchannel(sch);
-			device_set_disconnected(sch);
-			ret = 0;
-			break;
+			ret = sch->driver->notify(&sch->dev, event);
+			spin_lock_irqsave(&sch->lock, flags);
+			if (ret)
+				action = NONE;
 		}
-		/*
-		 * Unregister subchannel.
-		 * The device will be killed automatically.
-		 */
-		cio_disable_subchannel(sch);
-		css_sch_device_unregister(sch);
-		/* Reset intparm to zeroes. */
-		sch->schib.pmcw.intparm = 0;
-		cio_modify(sch);
-		put_device(&sch->dev);
-		ret = 0;
 		break;
 	case CIO_REVALIDATE:
-		/* 
-		 * Revalidation machine check. Sick.
-		 * We don't notify the driver since we have to throw the device
-		 * away in any case.
-		 */
-		if (!disc) {
-			css_sch_device_unregister(sch);
-			/* Reset intparm to zeroes. */
-			sch->schib.pmcw.intparm = 0;
-			cio_modify(sch);
-			put_device(&sch->dev);
-			ret = css_probe_device(schid);
-		} else {
-			/*
-			 * We can't immediately deregister the disconnected
-			 * device since it might block.
-			 */
-			spin_lock_irqsave(&sch->lock, flags);
-			device_trigger_reprobe(sch);
-			spin_unlock_irqrestore(&sch->lock, flags);
-			ret = 0;
-		}
+		/* Device will be removed, so no notify necessary. */
+		if (disc)
+			/* Reprobe because immediate unregister might block. */
+			action = REPROBE;
+		else
+			action = UNREGISTER_PROBE;
 		break;
 	case CIO_OPER:
-		if (disc) {
-			spin_lock_irqsave(&sch->lock, flags);
+		if (disc)
 			/* Get device operational again. */
-			device_trigger_reprobe(sch);
-			spin_unlock_irqrestore(&sch->lock, flags);
-		}
-		ret = sch ? 0 : css_probe_device(schid);
+			action = REPROBE;
+		break;
+	}
+	/* Perform action. */
+	ret = 0;
+	switch (action) {
+	case UNREGISTER:
+	case UNREGISTER_PROBE:
+		/* Unregister device (will use subchannel lock). */
+		spin_unlock_irqrestore(&sch->lock, flags);
+		css_sch_device_unregister(sch);
+		spin_lock_irqsave(&sch->lock, flags);
+
+		/* Reset intparm to zeroes. */
+		sch->schib.pmcw.intparm = 0;
+		cio_modify(sch);
+
+		/* Probe if necessary. */
+		if (action == UNREGISTER_PROBE)
+			ret = css_probe_device(sch->schid);
+		break;
+	case REPROBE:
+		device_trigger_reprobe(sch);
 		break;
 	default:
-		BUG();
-		ret = 0;
+		break;
+	}
+	spin_unlock_irqrestore(&sch->lock, flags);
+
+	return ret;
+}
+
+static int css_evaluate_new_subchannel(struct subchannel_id schid, int slow)
+{
+	struct schib schib;
+
+	if (!slow) {
+		/* Will be done on the slow path. */
+		return -EAGAIN;
 	}
+	if (stsch(schid, &schib) || !schib.pmcw.dnv) {
+		/* Unusable - ignore. */
+		return 0;
+	}
+	CIO_MSG_EVENT(4, "Evaluating schid 0.%x.%04x, event %d, unknown, "
+			 "slow path.\n", schid.ssid, schid.sch_no, CIO_OPER);
+
+	return css_probe_device(schid);
+}
+
+static int css_evaluate_subchannel(struct subchannel_id schid, int slow)
+{
+	struct subchannel *sch;
+	int ret;
+
+	sch = get_subchannel_by_schid(schid);
+	if (sch) {
+		ret = css_evaluate_known_subchannel(sch, slow);
+		put_device(&sch->dev);
+	} else
+		ret = css_evaluate_new_subchannel(schid, slow);
+
 	return ret;
 }
 
diff --git a/drivers/s390/cio/device.c b/drivers/s390/cio/device.c
index 646da56..6889456 100644
--- a/drivers/s390/cio/device.c
+++ b/drivers/s390/cio/device.c
@@ -52,53 +52,81 @@ ccw_bus_match (struct device * dev, struct device_driver * drv)
 	return 1;
 }
 
-/*
- * Hotplugging interface for ccw devices.
- * Heavily modeled on pci and usb hotplug.
- */
-static int
-ccw_uevent (struct device *dev, char **envp, int num_envp,
-	     char *buffer, int buffer_size)
+/* Store modalias string delimited by prefix/suffix string into buffer with
+ * specified size. Return length of resulting string (excluding trailing '\0')
+ * even if string doesn't fit buffer (snprintf semantics). */
+static int snprint_alias(char *buf, size_t size, const char *prefix,
+			 struct ccw_device_id *id, const char *suffix)
 {
-	struct ccw_device *cdev = to_ccwdev(dev);
-	int i = 0;
-	int length = 0;
+	int len;
 
-	if (!cdev)
-		return -ENODEV;
+	len = snprintf(buf, size, "%sccw:t%04Xm%02X", prefix, id->cu_type,
+		       id->cu_model);
+	if (len > size)
+		return len;
+	buf += len;
+	size -= len;
 
-	/* what we want to pass to /sbin/hotplug */
+	if (id->dev_type != 0)
+		len += snprintf(buf, size, "dt%04Xdm%02X%s", id->dev_type,
+				id->dev_model, suffix);
+	else
+		len += snprintf(buf, size, "dtdm%s", suffix);
 
-	envp[i++] = buffer;
-	length += scnprintf(buffer, buffer_size - length, "CU_TYPE=%04X",
-			   cdev->id.cu_type);
-	if ((buffer_size - length <= 0) || (i >= num_envp))
-		return -ENOMEM;
-	++length;
-	buffer += length;
+	return len;
+}
 
+/* Set up environment variables for ccw device uevent. Return 0 on success,
+ * non-zero otherwise. */
+static int ccw_uevent(struct device *dev, char **envp, int num_envp,
+		      char *buffer, int buffer_size)
+{
+	struct ccw_device *cdev = to_ccwdev(dev);
+	struct ccw_device_id *id = &(cdev->id);
+	int i = 0;
+	int len;
+
+	/* CU_TYPE= */
+	len = snprintf(buffer, buffer_size, "CU_TYPE=%04X", id->cu_type) + 1;
+	if (len > buffer_size || i >= num_envp)
+		return -ENOMEM;
 	envp[i++] = buffer;
-	length += scnprintf(buffer, buffer_size - length, "CU_MODEL=%02X",
-			   cdev->id.cu_model);
-	if ((buffer_size - length <= 0) || (i >= num_envp))
+	buffer += len;
+	buffer_size -= len;
+
+	/* CU_MODEL= */
+	len = snprintf(buffer, buffer_size, "CU_MODEL=%02X", id->cu_model) + 1;
+	if (len > buffer_size || i >= num_envp)
 		return -ENOMEM;
-	++length;
-	buffer += length;
+	envp[i++] = buffer;
+	buffer += len;
+	buffer_size -= len;
 
 	/* The next two can be zero, that's ok for us */
-	envp[i++] = buffer;
-	length += scnprintf(buffer, buffer_size - length, "DEV_TYPE=%04X",
-			   cdev->id.dev_type);
-	if ((buffer_size - length <= 0) || (i >= num_envp))
+	/* DEV_TYPE= */
+	len = snprintf(buffer, buffer_size, "DEV_TYPE=%04X", id->dev_type) + 1;
+	if (len > buffer_size || i >= num_envp)
 		return -ENOMEM;
-	++length;
-	buffer += length;
+	envp[i++] = buffer;
+	buffer += len;
+	buffer_size -= len;
 
+	/* DEV_MODEL= */
+	len = snprintf(buffer, buffer_size, "DEV_MODEL=%02X",
+			(unsigned char) id->dev_model) + 1;
+	if (len > buffer_size || i >= num_envp)
+		return -ENOMEM;
 	envp[i++] = buffer;
-	length += scnprintf(buffer, buffer_size - length, "DEV_MODEL=%02X",
-			   cdev->id.dev_model);
-	if ((buffer_size - length <= 0) || (i >= num_envp))
+	buffer += len;
+	buffer_size -= len;
+
+	/* MODALIAS=  */
+	len = snprint_alias(buffer, buffer_size, "MODALIAS=", id, "") + 1;
+	if (len > buffer_size || i >= num_envp)
 		return -ENOMEM;
+	envp[i++] = buffer;
+	buffer += len;
+	buffer_size -= len;
 
 	envp[i] = NULL;
 
@@ -251,16 +279,11 @@ modalias_show (struct device *dev, struct device_attribute *attr, char *buf)
 {
 	struct ccw_device *cdev = to_ccwdev(dev);
 	struct ccw_device_id *id = &(cdev->id);
-	int ret;
+	int len;
 
-	ret = sprintf(buf, "ccw:t%04Xm%02X",
-			id->cu_type, id->cu_model);
-	if (id->dev_type != 0)
-		ret += sprintf(buf + ret, "dt%04Xdm%02X\n",
-				id->dev_type, id->dev_model);
-	else
-		ret += sprintf(buf + ret, "dtdm\n");
-	return ret;
+	len = snprint_alias(buf, PAGE_SIZE, "", id, "\n") + 1;
+
+	return len > PAGE_SIZE ? PAGE_SIZE : len;
 }
 
 static ssize_t
diff --git a/drivers/s390/cio/device_fsm.c b/drivers/s390/cio/device_fsm.c
index 35e162b..dace46f 100644
--- a/drivers/s390/cio/device_fsm.c
+++ b/drivers/s390/cio/device_fsm.c
@@ -232,10 +232,7 @@ ccw_device_recog_done(struct ccw_device *cdev, int state)
 	 */
 	old_lpm = sch->lpm;
 	stsch(sch->schid, &sch->schib);
-	sch->lpm = sch->schib.pmcw.pim &
-		sch->schib.pmcw.pam &
-		sch->schib.pmcw.pom &
-		sch->opm;
+	sch->lpm = sch->schib.pmcw.pam & sch->opm;
 	/* Check since device may again have become not operational. */
 	if (!sch->schib.pmcw.dnv)
 		state = DEV_STATE_NOT_OPER;
@@ -267,6 +264,7 @@ ccw_device_recog_done(struct ccw_device *cdev, int state)
 			notify = 1;
 		}
 		/* fill out sense information */
+		memset(&cdev->id, 0, sizeof(cdev->id));
 		cdev->id.cu_type   = cdev->private->senseid.cu_type;
 		cdev->id.cu_model  = cdev->private->senseid.cu_model;
 		cdev->id.dev_type  = cdev->private->senseid.dev_type;
@@ -454,8 +452,8 @@ ccw_device_sense_pgid_done(struct ccw_device *cdev, int err)
 		return;
 	}
 	/* Start Path Group verification. */
-	sch->vpm = 0;	/* Start with no path groups set. */
 	cdev->private->state = DEV_STATE_VERIFY;
+	cdev->private->flags.doverify = 0;
 	ccw_device_verify_start(cdev);
 }
 
@@ -555,7 +553,19 @@ ccw_device_nopath_notify(void *data)
 void
 ccw_device_verify_done(struct ccw_device *cdev, int err)
 {
-	cdev->private->flags.doverify = 0;
+	struct subchannel *sch;
+
+	sch = to_subchannel(cdev->dev.parent);
+	/* Update schib - pom may have changed. */
+	stsch(sch->schid, &sch->schib);
+	/* Update lpm with verified path mask. */
+	sch->lpm = sch->vpm;
+	/* Repeat path verification? */
+	if (cdev->private->flags.doverify) {
+		cdev->private->flags.doverify = 0;
+		ccw_device_verify_start(cdev);
+		return;
+	}
 	switch (err) {
 	case -EOPNOTSUPP: /* path grouping not supported, just set online. */
 		cdev->private->options.pgroup = 0;
@@ -613,6 +623,7 @@ ccw_device_online(struct ccw_device *cdev)
 	if (!cdev->private->options.pgroup) {
 		/* Start initial path verification. */
 		cdev->private->state = DEV_STATE_VERIFY;
+		cdev->private->flags.doverify = 0;
 		ccw_device_verify_start(cdev);
 		return 0;
 	}
@@ -659,7 +670,6 @@ ccw_device_offline(struct ccw_device *cdev)
 	/* Are we doing path grouping? */
 	if (!cdev->private->options.pgroup) {
 		/* No, set state offline immediately. */
-		sch->vpm = 0;
 		ccw_device_done(cdev, DEV_STATE_OFFLINE);
 		return 0;
 	}
@@ -780,6 +790,7 @@ ccw_device_online_verify(struct ccw_device *cdev, enum dev_event dev_event)
 	}
 	/* Device is idle, we can do the path verification. */
 	cdev->private->state = DEV_STATE_VERIFY;
+	cdev->private->flags.doverify = 0;
 	ccw_device_verify_start(cdev);
 }
 
@@ -1042,9 +1053,9 @@ ccw_device_wait4io_timeout(struct ccw_device *cdev, enum dev_event dev_event)
 }
 
 static void
-ccw_device_wait4io_verify(struct ccw_device *cdev, enum dev_event dev_event)
+ccw_device_delay_verify(struct ccw_device *cdev, enum dev_event dev_event)
 {
-	/* When the I/O has terminated, we have to start verification. */
+	/* Start verification after current task finished. */
 	cdev->private->flags.doverify = 1;
 }
 
@@ -1110,10 +1121,7 @@ device_trigger_reprobe(struct subchannel *sch)
 	 * The pim, pam, pom values may not be accurate, but they are the best
 	 * we have before performing device selection :/
 	 */
-	sch->lpm = sch->schib.pmcw.pim &
-		sch->schib.pmcw.pam &
-		sch->schib.pmcw.pom &
-		sch->opm;
+	sch->lpm = sch->schib.pmcw.pam & sch->opm;
 	/* Re-set some bits in the pmcw that were lost. */
 	sch->schib.pmcw.isc = 3;
 	sch->schib.pmcw.csense = 1;
@@ -1237,7 +1245,7 @@ fsm_func_t *dev_jumptable[NR_DEV_STATES][NR_DEV_EVENTS] = {
 		[DEV_EVENT_NOTOPER]	= ccw_device_online_notoper,
 		[DEV_EVENT_INTERRUPT]	= ccw_device_verify_irq,
 		[DEV_EVENT_TIMEOUT]	= ccw_device_onoff_timeout,
-		[DEV_EVENT_VERIFY]	= ccw_device_nop,
+		[DEV_EVENT_VERIFY]	= ccw_device_delay_verify,
 	},
 	[DEV_STATE_ONLINE] = {
 		[DEV_EVENT_NOTOPER]	= ccw_device_online_notoper,
@@ -1280,7 +1288,7 @@ fsm_func_t *dev_jumptable[NR_DEV_STATES][NR_DEV_EVENTS] = {
 		[DEV_EVENT_NOTOPER]	= ccw_device_online_notoper,
 		[DEV_EVENT_INTERRUPT]	= ccw_device_wait4io_irq,
 		[DEV_EVENT_TIMEOUT]	= ccw_device_wait4io_timeout,
-		[DEV_EVENT_VERIFY]	= ccw_device_wait4io_verify,
+		[DEV_EVENT_VERIFY]	= ccw_device_delay_verify,
 	},
 	[DEV_STATE_QUIESCE] = {
 		[DEV_EVENT_NOTOPER]	= ccw_device_quiesce_done,
@@ -1293,7 +1301,7 @@ fsm_func_t *dev_jumptable[NR_DEV_STATES][NR_DEV_EVENTS] = {
 		[DEV_EVENT_NOTOPER]	= ccw_device_nop,
 		[DEV_EVENT_INTERRUPT]	= ccw_device_start_id,
 		[DEV_EVENT_TIMEOUT]	= ccw_device_bug,
-		[DEV_EVENT_VERIFY]	= ccw_device_nop,
+		[DEV_EVENT_VERIFY]	= ccw_device_start_id,
 	},
 	[DEV_STATE_DISCONNECTED_SENSE_ID] = {
 		[DEV_EVENT_NOTOPER]	= ccw_device_recog_notoper,
diff --git a/drivers/s390/cio/device_ops.c b/drivers/s390/cio/device_ops.c
index 9e3de0bd..93a897e 100644
--- a/drivers/s390/cio/device_ops.c
+++ b/drivers/s390/cio/device_ops.c
@@ -96,6 +96,12 @@ ccw_device_start_key(struct ccw_device *cdev, struct ccw1 *cpa,
 	ret = cio_set_options (sch, flags);
 	if (ret)
 		return ret;
+	/* Adjust requested path mask to excluded varied off paths. */
+	if (lpm) {
+		lpm &= sch->opm;
+		if (lpm == 0)
+			return -EACCES;
+	}
 	ret = cio_start_key (sch, cpa, lpm, key);
 	if (ret == 0)
 		cdev->private->intparm = intparm;
@@ -250,7 +256,7 @@ ccw_device_get_path_mask(struct ccw_device *cdev)
 	if (!sch)
 		return 0;
 	else
-		return sch->vpm;
+		return sch->lpm;
 }
 
 static void
@@ -304,7 +310,7 @@ __ccw_device_retry_loop(struct ccw_device *cdev, struct ccw1 *ccw, long magic, _
 	sch = to_subchannel(cdev->dev.parent);
 	do {
 		ret = cio_start (sch, ccw, lpm);
-		if ((ret == -EBUSY) || (ret == -EACCES)) {
+		if (ret == -EBUSY) {
 			/* Try again later. */
 			spin_unlock_irq(&sch->lock);
 			msleep(10);
@@ -433,6 +439,13 @@ read_conf_data_lpm (struct ccw_device *cdev, void **buffer, int *length, __u8 lp
 	if (!ciw || ciw->cmd == 0)
 		return -EOPNOTSUPP;
 
+	/* Adjust requested path mask to excluded varied off paths. */
+	if (lpm) {
+		lpm &= sch->opm;
+		if (lpm == 0)
+			return -EACCES;
+	}
+
 	rcd_ccw = kzalloc(sizeof(struct ccw1), GFP_KERNEL | GFP_DMA);
 	if (!rcd_ccw)
 		return -ENOMEM;
diff --git a/drivers/s390/cio/device_pgid.c b/drivers/s390/cio/device_pgid.c
index 1693a10..8ca2d07 100644
--- a/drivers/s390/cio/device_pgid.c
+++ b/drivers/s390/cio/device_pgid.c
@@ -245,18 +245,17 @@ __ccw_device_do_pgid(struct ccw_device *cdev, __u8 func)
 	memset(&cdev->private->irb, 0, sizeof(struct irb));
 
 	/* Try multiple times. */
-	ret = -ENODEV;
+	ret = -EACCES;
 	if (cdev->private->iretry > 0) {
 		cdev->private->iretry--;
 		ret = cio_start (sch, cdev->private->iccws,
 				 cdev->private->imask);
-		/* ret is 0, -EBUSY, -EACCES or -ENODEV */
-		if ((ret != -EACCES) && (ret != -ENODEV))
+		/* We expect an interrupt in case of success or busy
+		 * indication. */
+		if ((ret == 0) || (ret == -EBUSY))
 			return ret;
 	}
-	/* PGID command failed on this path. Switch it off. */
-	sch->lpm &= ~cdev->private->imask;
-	sch->vpm &= ~cdev->private->imask;
+	/* PGID command failed on this path. */
 	CIO_MSG_EVENT(2, "SPID - Device %04x on Subchannel "
 		      "0.%x.%04x, lpm %02X, became 'not operational'\n",
 		      cdev->private->devno, sch->schid.ssid,
@@ -286,18 +285,17 @@ static int __ccw_device_do_nop(struct ccw_device *cdev)
 	memset(&cdev->private->irb, 0, sizeof(struct irb));
 
 	/* Try multiple times. */
-	ret = -ENODEV;
+	ret = -EACCES;
 	if (cdev->private->iretry > 0) {
 		cdev->private->iretry--;
 		ret = cio_start (sch, cdev->private->iccws,
 				 cdev->private->imask);
-		/* ret is 0, -EBUSY, -EACCES or -ENODEV */
-		if ((ret != -EACCES) && (ret != -ENODEV))
+		/* We expect an interrupt in case of success or busy
+		 * indication. */
+		if ((ret == 0) || (ret == -EBUSY))
 			return ret;
 	}
-	/* nop command failed on this path. Switch it off. */
-	sch->lpm &= ~cdev->private->imask;
-	sch->vpm &= ~cdev->private->imask;
+	/* nop command failed on this path. */
 	CIO_MSG_EVENT(2, "NOP - Device %04x on Subchannel "
 		      "0.%x.%04x, lpm %02X, became 'not operational'\n",
 		      cdev->private->devno, sch->schid.ssid,
@@ -372,27 +370,32 @@ static void
 __ccw_device_verify_start(struct ccw_device *cdev)
 {
 	struct subchannel *sch;
-	__u8 imask, func;
+	__u8 func;
 	int ret;
 
 	sch = to_subchannel(cdev->dev.parent);
-	while (sch->vpm != sch->lpm) {
-		/* Find first unequal bit in vpm vs. lpm */
-		for (imask = 0x80; imask != 0; imask >>= 1)
-			if ((sch->vpm & imask) != (sch->lpm & imask))
-				break;
-		cdev->private->imask = imask;
+	/* Repeat for all paths. */
+	for (; cdev->private->imask; cdev->private->imask >>= 1,
+				     cdev->private->iretry = 5) {
+		if ((cdev->private->imask & sch->schib.pmcw.pam) == 0)
+			/* Path not available, try next. */
+			continue;
 		if (cdev->private->options.pgroup) {
-			func = (sch->vpm & imask) ?
-				SPID_FUNC_RESIGN : SPID_FUNC_ESTABLISH;
+			if (sch->opm & cdev->private->imask)
+				func = SPID_FUNC_ESTABLISH;
+			else
+				func = SPID_FUNC_RESIGN;
 			ret = __ccw_device_do_pgid(cdev, func);
 		} else
 			ret = __ccw_device_do_nop(cdev);
+		/* We expect an interrupt in case of success or busy
+		 * indication. */
 		if (ret == 0 || ret == -EBUSY)
 			return;
-		cdev->private->iretry = 5;
+		/* Permanent path failure, try next. */
 	}
-	ccw_device_verify_done(cdev, (sch->lpm != 0) ? 0 : -ENODEV);
+	/* Done with all paths. */
+	ccw_device_verify_done(cdev, (sch->vpm != 0) ? 0 : -ENODEV);
 }
 		
 /*
@@ -421,14 +424,14 @@ ccw_device_verify_irq(struct ccw_device *cdev, enum dev_event dev_event)
 	else
 		ret = __ccw_device_check_nop(cdev);
 	memset(&cdev->private->irb, 0, sizeof(struct irb));
+
 	switch (ret) {
 	/* 0, -ETIME, -EAGAIN, -EOPNOTSUPP or -EACCES */
 	case 0:
-		/* Establish or Resign Path Group done. Update vpm. */
-		if ((sch->lpm & cdev->private->imask) != 0)
-			sch->vpm |= cdev->private->imask;
-		else
-			sch->vpm &= ~cdev->private->imask;
+		/* Path verification ccw finished successfully, update lpm. */
+		sch->vpm |= sch->opm & cdev->private->imask;
+		/* Go on with next path. */
+		cdev->private->imask >>= 1;
 		cdev->private->iretry = 5;
 		__ccw_device_verify_start(cdev);
 		break;
@@ -441,6 +444,10 @@ ccw_device_verify_irq(struct ccw_device *cdev, enum dev_event dev_event)
 			cdev->private->options.pgroup = 0;
 		else
 			cdev->private->flags.pgid_single = 1;
+		/* Retry */
+		sch->vpm = 0;
+		cdev->private->imask = 0x80;
+		cdev->private->iretry = 5;
 		/* fall through. */
 	case -EAGAIN:		/* Try again. */
 		__ccw_device_verify_start(cdev);
@@ -449,8 +456,7 @@ ccw_device_verify_irq(struct ccw_device *cdev, enum dev_event dev_event)
 		ccw_device_verify_done(cdev, -ETIME);
 		break;
 	case -EACCES:		/* channel is not operational. */
-		sch->lpm &= ~cdev->private->imask;
-		sch->vpm &= ~cdev->private->imask;
+		cdev->private->imask >>= 1;
 		cdev->private->iretry = 5;
 		__ccw_device_verify_start(cdev);
 		break;
@@ -463,19 +469,17 @@ ccw_device_verify_start(struct ccw_device *cdev)
 	struct subchannel *sch = to_subchannel(cdev->dev.parent);
 
 	cdev->private->flags.pgid_single = 0;
+	cdev->private->imask = 0x80;
 	cdev->private->iretry = 5;
-	/*
-	 * Update sch->lpm with current values to catch paths becoming
-	 * available again.
-	 */
+
+	/* Start with empty vpm. */
+	sch->vpm = 0;
+
+	/* Get current pam. */
 	if (stsch(sch->schid, &sch->schib)) {
 		ccw_device_verify_done(cdev, -ENODEV);
 		return;
 	}
-	sch->lpm = sch->schib.pmcw.pim &
-		sch->schib.pmcw.pam &
-		sch->schib.pmcw.pom &
-		sch->opm;
 	__ccw_device_verify_start(cdev);
 }
 
@@ -524,7 +528,6 @@ ccw_device_disband_irq(struct ccw_device *cdev, enum dev_event dev_event)
 	switch (ret) {
 	/* 0, -ETIME, -EAGAIN, -EOPNOTSUPP or -EACCES */
 	case 0:			/* disband successful. */
-		sch->vpm = 0;
 		ccw_device_disband_done(cdev, ret);
 		break;
 	case -EOPNOTSUPP:
diff --git a/drivers/s390/cio/qdio.c b/drivers/s390/cio/qdio.c
index 7c93a87..cde822d 100644
--- a/drivers/s390/cio/qdio.c
+++ b/drivers/s390/cio/qdio.c
@@ -115,7 +115,7 @@ qdio_min(int a,int b)
 static inline __u64 
 qdio_get_micros(void)
 {
-        return (get_clock() >> 10); /* time>>12 is microseconds */
+	return (get_clock() >> 12); /* time>>12 is microseconds */
 }
 
 /* 
@@ -1129,7 +1129,7 @@ out:
 
 #ifdef QDIO_USE_PROCESSING_STATE
 	if (last_position>=0)
-		set_slsb(q, &last_position, SLSB_P_INPUT_NOT_INIT, &count);
+		set_slsb(q, &last_position, SLSB_P_INPUT_PROCESSING, &count);
 #endif /* QDIO_USE_PROCESSING_STATE */
 
 	QDIO_DBF_HEX4(0,trace,&q->first_to_check,sizeof(int));
diff --git a/drivers/s390/cio/qdio.h b/drivers/s390/cio/qdio.h
index ceb3ab3..1245693 100644
--- a/drivers/s390/cio/qdio.h
+++ b/drivers/s390/cio/qdio.h
@@ -191,49 +191,49 @@ enum qdio_irq_states {
 #if QDIO_VERBOSE_LEVEL>8
 #define QDIO_PRINT_STUPID(x...) printk( KERN_DEBUG QDIO_PRINTK_HEADER x)
 #else
-#define QDIO_PRINT_STUPID(x...)
+#define QDIO_PRINT_STUPID(x...) do { } while (0)
 #endif
 
 #if QDIO_VERBOSE_LEVEL>7
 #define QDIO_PRINT_ALL(x...) printk( QDIO_PRINTK_HEADER x)
 #else
-#define QDIO_PRINT_ALL(x...)
+#define QDIO_PRINT_ALL(x...) do { } while (0)
 #endif
 
 #if QDIO_VERBOSE_LEVEL>6
 #define QDIO_PRINT_INFO(x...) printk( QDIO_PRINTK_HEADER x)
 #else
-#define QDIO_PRINT_INFO(x...)
+#define QDIO_PRINT_INFO(x...) do { } while (0)
 #endif
 
 #if QDIO_VERBOSE_LEVEL>5
 #define QDIO_PRINT_WARN(x...) printk( QDIO_PRINTK_HEADER x)
 #else
-#define QDIO_PRINT_WARN(x...)
+#define QDIO_PRINT_WARN(x...) do { } while (0)
 #endif
 
 #if QDIO_VERBOSE_LEVEL>4
 #define QDIO_PRINT_ERR(x...) printk( QDIO_PRINTK_HEADER x)
 #else
-#define QDIO_PRINT_ERR(x...)
+#define QDIO_PRINT_ERR(x...) do { } while (0)
 #endif
 
 #if QDIO_VERBOSE_LEVEL>3
 #define QDIO_PRINT_CRIT(x...) printk( QDIO_PRINTK_HEADER x)
 #else
-#define QDIO_PRINT_CRIT(x...)
+#define QDIO_PRINT_CRIT(x...) do { } while (0)
 #endif
 
 #if QDIO_VERBOSE_LEVEL>2
 #define QDIO_PRINT_ALERT(x...) printk( QDIO_PRINTK_HEADER x)
 #else
-#define QDIO_PRINT_ALERT(x...)
+#define QDIO_PRINT_ALERT(x...) do { } while (0)
 #endif
 
 #if QDIO_VERBOSE_LEVEL>1
 #define QDIO_PRINT_EMERG(x...) printk( QDIO_PRINTK_HEADER x)
 #else
-#define QDIO_PRINT_EMERG(x...)
+#define QDIO_PRINT_EMERG(x...) do { } while (0)
 #endif
 
 #define HEXDUMP16(importance,header,ptr) \
diff --git a/drivers/s390/crypto/Makefile b/drivers/s390/crypto/Makefile
index 15edebb..f0a12d2 100644
--- a/drivers/s390/crypto/Makefile
+++ b/drivers/s390/crypto/Makefile
@@ -2,5 +2,16 @@
 # S/390 crypto devices
 #
 
-z90crypt-objs := z90main.o z90hardware.o
-obj-$(CONFIG_Z90CRYPT) += z90crypt.o
+ifdef CONFIG_ZCRYPT_MONOLITHIC
+
+z90crypt-objs := zcrypt_mono.o ap_bus.o zcrypt_api.o \
+		zcrypt_pcica.o zcrypt_pcicc.o zcrypt_pcixcc.o zcrypt_cex2a.o
+obj-$(CONFIG_ZCRYPT) += z90crypt.o
+
+else
+
+ap-objs := ap_bus.o
+obj-$(CONFIG_ZCRYPT) += ap.o zcrypt_api.o zcrypt_pcicc.o zcrypt_pcixcc.o
+obj-$(CONFIG_ZCRYPT) += zcrypt_pcica.o zcrypt_cex2a.o
+
+endif
diff --git a/drivers/s390/crypto/ap_bus.c b/drivers/s390/crypto/ap_bus.c
new file mode 100644
index 0000000..6ed0985
--- /dev/null
+++ b/drivers/s390/crypto/ap_bus.c
@@ -0,0 +1,1221 @@
+/*
+ * linux/drivers/s390/crypto/ap_bus.c
+ *
+ * Copyright (C) 2006 IBM Corporation
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *	      Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *	      Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *
+ * Adjunct processor bus.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/delay.h>
+#include <linux/err.h>
+#include <linux/interrupt.h>
+#include <linux/workqueue.h>
+#include <linux/notifier.h>
+#include <linux/kthread.h>
+#include <linux/mutex.h>
+#include <asm/s390_rdev.h>
+
+#include "ap_bus.h"
+
+/* Some prototypes. */
+static void ap_scan_bus(void *);
+static void ap_poll_all(unsigned long);
+static void ap_poll_timeout(unsigned long);
+static int ap_poll_thread_start(void);
+static void ap_poll_thread_stop(void);
+
+/**
+ * Module description.
+ */
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("Adjunct Processor Bus driver, "
+		   "Copyright 2006 IBM Corporation");
+MODULE_LICENSE("GPL");
+
+/**
+ * Module parameter
+ */
+int ap_domain_index = -1;	/* Adjunct Processor Domain Index */
+module_param_named(domain, ap_domain_index, int, 0000);
+MODULE_PARM_DESC(domain, "domain index for ap devices");
+EXPORT_SYMBOL(ap_domain_index);
+
+static int ap_thread_flag = 1;
+module_param_named(poll_thread, ap_thread_flag, int, 0000);
+MODULE_PARM_DESC(poll_thread, "Turn on/off poll thread, default is 1 (on).");
+
+static struct device *ap_root_device = NULL;
+
+/**
+ * Workqueue & timer for bus rescan.
+ */
+static struct workqueue_struct *ap_work_queue;
+static struct timer_list ap_config_timer;
+static int ap_config_time = AP_CONFIG_TIME;
+static DECLARE_WORK(ap_config_work, ap_scan_bus, NULL);
+
+/**
+ * Tasklet & timer for AP request polling.
+ */
+static struct timer_list ap_poll_timer = TIMER_INITIALIZER(ap_poll_timeout,0,0);
+static DECLARE_TASKLET(ap_tasklet, ap_poll_all, 0);
+static atomic_t ap_poll_requests = ATOMIC_INIT(0);
+static DECLARE_WAIT_QUEUE_HEAD(ap_poll_wait);
+static struct task_struct *ap_poll_kthread = NULL;
+static DEFINE_MUTEX(ap_poll_thread_mutex);
+
+/**
+ * Test if ap instructions are available.
+ *
+ * Returns 0 if the ap instructions are installed.
+ */
+static inline int ap_instructions_available(void)
+{
+	register unsigned long reg0 asm ("0") = AP_MKQID(0,0);
+	register unsigned long reg1 asm ("1") = -ENODEV;
+	register unsigned long reg2 asm ("2") = 0UL;
+
+	asm volatile(
+		"   .long 0xb2af0000\n"		/* PQAP(TAPQ) */
+		"0: la    %1,0\n"
+		"1:\n"
+		EX_TABLE(0b, 1b)
+		: "+d" (reg0), "+d" (reg1), "+d" (reg2) : : "cc" );
+	return reg1;
+}
+
+/**
+ * Test adjunct processor queue.
+ * @qid: the ap queue number
+ * @queue_depth: pointer to queue depth value
+ * @device_type: pointer to device type value
+ *
+ * Returns ap queue status structure.
+ */
+static inline struct ap_queue_status
+ap_test_queue(ap_qid_t qid, int *queue_depth, int *device_type)
+{
+	register unsigned long reg0 asm ("0") = qid;
+	register struct ap_queue_status reg1 asm ("1");
+	register unsigned long reg2 asm ("2") = 0UL;
+
+	asm volatile(".long 0xb2af0000"		/* PQAP(TAPQ) */
+		     : "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
+	*device_type = (int) (reg2 >> 24);
+	*queue_depth = (int) (reg2 & 0xff);
+	return reg1;
+}
+
+/**
+ * Reset adjunct processor queue.
+ * @qid: the ap queue number
+ *
+ * Returns ap queue status structure.
+ */
+static inline struct ap_queue_status ap_reset_queue(ap_qid_t qid)
+{
+	register unsigned long reg0 asm ("0") = qid | 0x01000000UL;
+	register struct ap_queue_status reg1 asm ("1");
+	register unsigned long reg2 asm ("2") = 0UL;
+
+	asm volatile(
+		".long 0xb2af0000"		/* PQAP(RAPQ) */
+		: "+d" (reg0), "=d" (reg1), "+d" (reg2) : : "cc");
+	return reg1;
+}
+
+/**
+ * Send message to adjunct processor queue.
+ * @qid: the ap queue number
+ * @psmid: the program supplied message identifier
+ * @msg: the message text
+ * @length: the message length
+ *
+ * Returns ap queue status structure.
+ *
+ * Condition code 1 on NQAP can't happen because the L bit is 1.
+ *
+ * Condition code 2 on NQAP also means the send is incomplete,
+ * because a segment boundary was reached. The NQAP is repeated.
+ */
+static inline struct ap_queue_status
+__ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
+{
+	typedef struct { char _[length]; } msgblock;
+	register unsigned long reg0 asm ("0") = qid | 0x40000000UL;
+	register struct ap_queue_status reg1 asm ("1");
+	register unsigned long reg2 asm ("2") = (unsigned long) msg;
+	register unsigned long reg3 asm ("3") = (unsigned long) length;
+	register unsigned long reg4 asm ("4") = (unsigned int) (psmid >> 32);
+	register unsigned long reg5 asm ("5") = (unsigned int) psmid;
+
+	asm volatile (
+		"0: .long 0xb2ad0042\n"		/* DQAP */
+		"   brc   2,0b"
+		: "+d" (reg0), "=d" (reg1), "+d" (reg2), "+d" (reg3)
+		: "d" (reg4), "d" (reg5), "m" (*(msgblock *) msg)
+		: "cc" );
+	return reg1;
+}
+
+int ap_send(ap_qid_t qid, unsigned long long psmid, void *msg, size_t length)
+{
+	struct ap_queue_status status;
+
+	status = __ap_send(qid, psmid, msg, length);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		return 0;
+	case AP_RESPONSE_Q_FULL:
+		return -EBUSY;
+	default:	/* Device is gone. */
+		return -ENODEV;
+	}
+}
+EXPORT_SYMBOL(ap_send);
+
+/*
+ * Receive message from adjunct processor queue.
+ * @qid: the ap queue number
+ * @psmid: pointer to program supplied message identifier
+ * @msg: the message text
+ * @length: the message length
+ *
+ * Returns ap queue status structure.
+ *
+ * Condition code 1 on DQAP means the receive has taken place
+ * but only partially.	The response is incomplete, hence the
+ * DQAP is repeated.
+ *
+ * Condition code 2 on DQAP also means the receive is incomplete,
+ * this time because a segment boundary was reached. Again, the
+ * DQAP is repeated.
+ *
+ * Note that gpr2 is used by the DQAP instruction to keep track of
+ * any 'residual' length, in case the instruction gets interrupted.
+ * Hence it gets zeroed before the instruction.
+ */
+static inline struct ap_queue_status
+__ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
+{
+	typedef struct { char _[length]; } msgblock;
+	register unsigned long reg0 asm("0") = qid | 0x80000000UL;
+	register struct ap_queue_status reg1 asm ("1");
+	register unsigned long reg2 asm("2") = 0UL;
+	register unsigned long reg4 asm("4") = (unsigned long) msg;
+	register unsigned long reg5 asm("5") = (unsigned long) length;
+	register unsigned long reg6 asm("6") = 0UL;
+	register unsigned long reg7 asm("7") = 0UL;
+
+
+	asm volatile(
+		"0: .long 0xb2ae0064\n"
+		"   brc   6,0b\n"
+		: "+d" (reg0), "=d" (reg1), "+d" (reg2),
+		"+d" (reg4), "+d" (reg5), "+d" (reg6), "+d" (reg7),
+		"=m" (*(msgblock *) msg) : : "cc" );
+	*psmid = (((unsigned long long) reg6) << 32) + reg7;
+	return reg1;
+}
+
+int ap_recv(ap_qid_t qid, unsigned long long *psmid, void *msg, size_t length)
+{
+	struct ap_queue_status status;
+
+	status = __ap_recv(qid, psmid, msg, length);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		return 0;
+	case AP_RESPONSE_NO_PENDING_REPLY:
+		if (status.queue_empty)
+			return -ENOENT;
+		return -EBUSY;
+	default:
+		return -ENODEV;
+	}
+}
+EXPORT_SYMBOL(ap_recv);
+
+/**
+ * Check if an AP queue is available. The test is repeated for
+ * AP_MAX_RESET times.
+ * @qid: the ap queue number
+ * @queue_depth: pointer to queue depth value
+ * @device_type: pointer to device type value
+ */
+static int ap_query_queue(ap_qid_t qid, int *queue_depth, int *device_type)
+{
+	struct ap_queue_status status;
+	int t_depth, t_device_type, rc, i;
+
+	rc = -EBUSY;
+	for (i = 0; i < AP_MAX_RESET; i++) {
+		status = ap_test_queue(qid, &t_depth, &t_device_type);
+		switch (status.response_code) {
+		case AP_RESPONSE_NORMAL:
+			*queue_depth = t_depth + 1;
+			*device_type = t_device_type;
+			rc = 0;
+			break;
+		case AP_RESPONSE_Q_NOT_AVAIL:
+			rc = -ENODEV;
+			break;
+		case AP_RESPONSE_RESET_IN_PROGRESS:
+			break;
+		case AP_RESPONSE_DECONFIGURED:
+			rc = -ENODEV;
+			break;
+		case AP_RESPONSE_CHECKSTOPPED:
+			rc = -ENODEV;
+			break;
+		case AP_RESPONSE_BUSY:
+			break;
+		default:
+			BUG();
+		}
+		if (rc != -EBUSY)
+			break;
+		if (i < AP_MAX_RESET - 1)
+			udelay(5);
+	}
+	return rc;
+}
+
+/**
+ * Reset an AP queue and wait for it to become available again.
+ * @qid: the ap queue number
+ */
+static int ap_init_queue(ap_qid_t qid)
+{
+	struct ap_queue_status status;
+	int rc, dummy, i;
+
+	rc = -ENODEV;
+	status = ap_reset_queue(qid);
+	for (i = 0; i < AP_MAX_RESET; i++) {
+		switch (status.response_code) {
+		case AP_RESPONSE_NORMAL:
+			if (status.queue_empty)
+				rc = 0;
+			break;
+		case AP_RESPONSE_Q_NOT_AVAIL:
+		case AP_RESPONSE_DECONFIGURED:
+		case AP_RESPONSE_CHECKSTOPPED:
+			i = AP_MAX_RESET;	/* return with -ENODEV */
+			break;
+		case AP_RESPONSE_RESET_IN_PROGRESS:
+		case AP_RESPONSE_BUSY:
+		default:
+			break;
+		}
+		if (rc != -ENODEV)
+			break;
+		if (i < AP_MAX_RESET - 1) {
+			udelay(5);
+			status = ap_test_queue(qid, &dummy, &dummy);
+		}
+	}
+	return rc;
+}
+
+/**
+ * AP device related attributes.
+ */
+static ssize_t ap_hwtype_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->device_type);
+}
+static DEVICE_ATTR(hwtype, 0444, ap_hwtype_show, NULL);
+
+static ssize_t ap_depth_show(struct device *dev, struct device_attribute *attr,
+			     char *buf)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	return snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->queue_depth);
+}
+static DEVICE_ATTR(depth, 0444, ap_depth_show, NULL);
+
+static ssize_t ap_request_count_show(struct device *dev,
+				     struct device_attribute *attr,
+				     char *buf)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	int rc;
+
+	spin_lock_bh(&ap_dev->lock);
+	rc = snprintf(buf, PAGE_SIZE, "%d\n", ap_dev->total_request_count);
+	spin_unlock_bh(&ap_dev->lock);
+	return rc;
+}
+
+static DEVICE_ATTR(request_count, 0444, ap_request_count_show, NULL);
+
+static ssize_t ap_modalias_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	return sprintf(buf, "ap:t%02X", to_ap_dev(dev)->device_type);
+}
+
+static DEVICE_ATTR(modalias, 0444, ap_modalias_show, NULL);
+
+static struct attribute *ap_dev_attrs[] = {
+	&dev_attr_hwtype.attr,
+	&dev_attr_depth.attr,
+	&dev_attr_request_count.attr,
+	&dev_attr_modalias.attr,
+	NULL
+};
+static struct attribute_group ap_dev_attr_group = {
+	.attrs = ap_dev_attrs
+};
+
+/**
+ * AP bus driver registration/unregistration.
+ */
+static int ap_bus_match(struct device *dev, struct device_driver *drv)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	struct ap_driver *ap_drv = to_ap_drv(drv);
+	struct ap_device_id *id;
+
+	/**
+	 * Compare device type of the device with the list of
+	 * supported types of the device_driver.
+	 */
+	for (id = ap_drv->ids; id->match_flags; id++) {
+		if ((id->match_flags & AP_DEVICE_ID_MATCH_DEVICE_TYPE) &&
+		    (id->dev_type != ap_dev->device_type))
+			continue;
+		return 1;
+	}
+	return 0;
+}
+
+/**
+ * uevent function for AP devices. It sets up a single environment
+ * variable DEV_TYPE which contains the hardware device type.
+ */
+static int ap_uevent (struct device *dev, char **envp, int num_envp,
+		       char *buffer, int buffer_size)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	int length;
+
+	if (!ap_dev)
+		return -ENODEV;
+
+	/* Set up DEV_TYPE environment variable. */
+	envp[0] = buffer;
+	length = scnprintf(buffer, buffer_size, "DEV_TYPE=%04X",
+			   ap_dev->device_type);
+	if (buffer_size - length <= 0)
+		return -ENOMEM;
+	envp[1] = 0;
+	return 0;
+}
+
+static struct bus_type ap_bus_type = {
+	.name = "ap",
+	.match = &ap_bus_match,
+	.uevent = &ap_uevent,
+};
+
+static int ap_device_probe(struct device *dev)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	struct ap_driver *ap_drv = to_ap_drv(dev->driver);
+	int rc;
+
+	ap_dev->drv = ap_drv;
+	rc = ap_drv->probe ? ap_drv->probe(ap_dev) : -ENODEV;
+	if (rc)
+		ap_dev->unregistered = 1;
+	return rc;
+}
+
+/**
+ * Flush all requests from the request/pending queue of an AP device.
+ * @ap_dev: pointer to the AP device.
+ */
+static inline void __ap_flush_queue(struct ap_device *ap_dev)
+{
+	struct ap_message *ap_msg, *next;
+
+	list_for_each_entry_safe(ap_msg, next, &ap_dev->pendingq, list) {
+		list_del_init(&ap_msg->list);
+		ap_dev->pendingq_count--;
+		ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
+	}
+	list_for_each_entry_safe(ap_msg, next, &ap_dev->requestq, list) {
+		list_del_init(&ap_msg->list);
+		ap_dev->requestq_count--;
+		ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
+	}
+}
+
+void ap_flush_queue(struct ap_device *ap_dev)
+{
+	spin_lock_bh(&ap_dev->lock);
+	__ap_flush_queue(ap_dev);
+	spin_unlock_bh(&ap_dev->lock);
+}
+EXPORT_SYMBOL(ap_flush_queue);
+
+static int ap_device_remove(struct device *dev)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	struct ap_driver *ap_drv = ap_dev->drv;
+
+	spin_lock_bh(&ap_dev->lock);
+	__ap_flush_queue(ap_dev);
+	/**
+	 * set ->unregistered to 1 while holding the lock. This prevents
+	 * new messages to be put on the queue from now on.
+	 */
+	ap_dev->unregistered = 1;
+	spin_unlock_bh(&ap_dev->lock);
+	if (ap_drv->remove)
+		ap_drv->remove(ap_dev);
+	return 0;
+}
+
+int ap_driver_register(struct ap_driver *ap_drv, struct module *owner,
+		       char *name)
+{
+	struct device_driver *drv = &ap_drv->driver;
+
+	drv->bus = &ap_bus_type;
+	drv->probe = ap_device_probe;
+	drv->remove = ap_device_remove;
+	drv->owner = owner;
+	drv->name = name;
+	return driver_register(drv);
+}
+EXPORT_SYMBOL(ap_driver_register);
+
+void ap_driver_unregister(struct ap_driver *ap_drv)
+{
+	driver_unregister(&ap_drv->driver);
+}
+EXPORT_SYMBOL(ap_driver_unregister);
+
+/**
+ * AP bus attributes.
+ */
+static ssize_t ap_domain_show(struct bus_type *bus, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n", ap_domain_index);
+}
+
+static BUS_ATTR(ap_domain, 0444, ap_domain_show, NULL);
+
+static ssize_t ap_config_time_show(struct bus_type *bus, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n", ap_config_time);
+}
+
+static ssize_t ap_config_time_store(struct bus_type *bus,
+				    const char *buf, size_t count)
+{
+	int time;
+
+	if (sscanf(buf, "%d\n", &time) != 1 || time < 5 || time > 120)
+		return -EINVAL;
+	ap_config_time = time;
+	if (!timer_pending(&ap_config_timer) ||
+	    !mod_timer(&ap_config_timer, jiffies + ap_config_time * HZ)) {
+		ap_config_timer.expires = jiffies + ap_config_time * HZ;
+		add_timer(&ap_config_timer);
+	}
+	return count;
+}
+
+static BUS_ATTR(config_time, 0644, ap_config_time_show, ap_config_time_store);
+
+static ssize_t ap_poll_thread_show(struct bus_type *bus, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%d\n", ap_poll_kthread ? 1 : 0);
+}
+
+static ssize_t ap_poll_thread_store(struct bus_type *bus,
+				    const char *buf, size_t count)
+{
+	int flag, rc;
+
+	if (sscanf(buf, "%d\n", &flag) != 1)
+		return -EINVAL;
+	if (flag) {
+		rc = ap_poll_thread_start();
+		if (rc)
+			return rc;
+	}
+	else
+		ap_poll_thread_stop();
+	return count;
+}
+
+static BUS_ATTR(poll_thread, 0644, ap_poll_thread_show, ap_poll_thread_store);
+
+static struct bus_attribute *const ap_bus_attrs[] = {
+	&bus_attr_ap_domain,
+	&bus_attr_config_time,
+	&bus_attr_poll_thread,
+	NULL
+};
+
+/**
+ * Pick one of the 16 ap domains.
+ */
+static inline int ap_select_domain(void)
+{
+	int queue_depth, device_type, count, max_count, best_domain;
+	int rc, i, j;
+
+	/**
+	 * We want to use a single domain. Either the one specified with
+	 * the "domain=" parameter or the domain with the maximum number
+	 * of devices.
+	 */
+	if (ap_domain_index >= 0 && ap_domain_index < AP_DOMAINS)
+		/* Domain has already been selected. */
+		return 0;
+	best_domain = -1;
+	max_count = 0;
+	for (i = 0; i < AP_DOMAINS; i++) {
+		count = 0;
+		for (j = 0; j < AP_DEVICES; j++) {
+			ap_qid_t qid = AP_MKQID(j, i);
+			rc = ap_query_queue(qid, &queue_depth, &device_type);
+			if (rc)
+				continue;
+			count++;
+		}
+		if (count > max_count) {
+			max_count = count;
+			best_domain = i;
+		}
+	}
+	if (best_domain >= 0){
+		ap_domain_index = best_domain;
+		return 0;
+	}
+	return -ENODEV;
+}
+
+/**
+ * Find the device type if query queue returned a device type of 0.
+ * @ap_dev: pointer to the AP device.
+ */
+static int ap_probe_device_type(struct ap_device *ap_dev)
+{
+	static unsigned char msg[] = {
+		0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x01,0x00,0x43,0x43,0x41,0x2d,0x41,0x50,
+		0x50,0x4c,0x20,0x20,0x20,0x01,0x01,0x01,
+		0x00,0x00,0x00,0x00,0x50,0x4b,0x00,0x00,
+		0x00,0x00,0x01,0x1c,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x05,0xb8,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x70,0x00,0x41,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x54,0x32,0x01,0x00,0xa0,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0xb8,0x05,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,
+		0x49,0x43,0x53,0x46,0x20,0x20,0x20,0x20,
+		0x50,0x4b,0x0a,0x00,0x50,0x4b,0x43,0x53,
+		0x2d,0x31,0x2e,0x32,0x37,0x00,0x11,0x22,
+		0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
+		0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,
+		0x99,0x00,0x11,0x22,0x33,0x44,0x55,0x66,
+		0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x44,
+		0x55,0x66,0x77,0x88,0x99,0x00,0x11,0x22,
+		0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
+		0x11,0x22,0x33,0x5d,0x00,0x5b,0x00,0x77,
+		0x88,0x1e,0x00,0x00,0x57,0x00,0x00,0x00,
+		0x00,0x04,0x00,0x00,0x4f,0x00,0x00,0x00,
+		0x03,0x02,0x00,0x00,0x40,0x01,0x00,0x01,
+		0xce,0x02,0x68,0x2d,0x5f,0xa9,0xde,0x0c,
+		0xf6,0xd2,0x7b,0x58,0x4b,0xf9,0x28,0x68,
+		0x3d,0xb4,0xf4,0xef,0x78,0xd5,0xbe,0x66,
+		0x63,0x42,0xef,0xf8,0xfd,0xa4,0xf8,0xb0,
+		0x8e,0x29,0xc2,0xc9,0x2e,0xd8,0x45,0xb8,
+		0x53,0x8c,0x6f,0x4e,0x72,0x8f,0x6c,0x04,
+		0x9c,0x88,0xfc,0x1e,0xc5,0x83,0x55,0x57,
+		0xf7,0xdd,0xfd,0x4f,0x11,0x36,0x95,0x5d,
+	};
+	struct ap_queue_status status;
+	unsigned long long psmid;
+	char *reply;
+	int rc, i;
+
+	reply = (void *) get_zeroed_page(GFP_KERNEL);
+	if (!reply) {
+		rc = -ENOMEM;
+		goto out;
+	}
+
+	status = __ap_send(ap_dev->qid, 0x0102030405060708ULL,
+			   msg, sizeof(msg));
+	if (status.response_code != AP_RESPONSE_NORMAL) {
+		rc = -ENODEV;
+		goto out_free;
+	}
+
+	/* Wait for the test message to complete. */
+	for (i = 0; i < 6; i++) {
+		mdelay(300);
+		status = __ap_recv(ap_dev->qid, &psmid, reply, 4096);
+		if (status.response_code == AP_RESPONSE_NORMAL &&
+		    psmid == 0x0102030405060708ULL)
+			break;
+	}
+	if (i < 6) {
+		/* Got an answer. */
+		if (reply[0] == 0x00 && reply[1] == 0x86)
+			ap_dev->device_type = AP_DEVICE_TYPE_PCICC;
+		else
+			ap_dev->device_type = AP_DEVICE_TYPE_PCICA;
+		rc = 0;
+	} else
+		rc = -ENODEV;
+
+out_free:
+	free_page((unsigned long) reply);
+out:
+	return rc;
+}
+
+/**
+ * Scan the ap bus for new devices.
+ */
+static int __ap_scan_bus(struct device *dev, void *data)
+{
+	return to_ap_dev(dev)->qid == (ap_qid_t)(unsigned long) data;
+}
+
+static void ap_device_release(struct device *dev)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+
+	kfree(ap_dev);
+}
+
+static void ap_scan_bus(void *data)
+{
+	struct ap_device *ap_dev;
+	struct device *dev;
+	ap_qid_t qid;
+	int queue_depth, device_type;
+	int rc, i;
+
+	if (ap_select_domain() != 0)
+		return;
+	for (i = 0; i < AP_DEVICES; i++) {
+		qid = AP_MKQID(i, ap_domain_index);
+		dev = bus_find_device(&ap_bus_type, NULL,
+				      (void *)(unsigned long)qid,
+				      __ap_scan_bus);
+		if (dev) {
+			put_device(dev);
+			continue;
+		}
+		rc = ap_query_queue(qid, &queue_depth, &device_type);
+		if (rc)
+			continue;
+		rc = ap_init_queue(qid);
+		if (rc)
+			continue;
+		ap_dev = kzalloc(sizeof(*ap_dev), GFP_KERNEL);
+		if (!ap_dev)
+			break;
+		ap_dev->qid = qid;
+		ap_dev->queue_depth = queue_depth;
+		spin_lock_init(&ap_dev->lock);
+		INIT_LIST_HEAD(&ap_dev->pendingq);
+		INIT_LIST_HEAD(&ap_dev->requestq);
+		if (device_type == 0)
+			ap_probe_device_type(ap_dev);
+		else
+			ap_dev->device_type = device_type;
+
+		ap_dev->device.bus = &ap_bus_type;
+		ap_dev->device.parent = ap_root_device;
+		snprintf(ap_dev->device.bus_id, BUS_ID_SIZE, "card%02x",
+			 AP_QID_DEVICE(ap_dev->qid));
+		ap_dev->device.release = ap_device_release;
+		rc = device_register(&ap_dev->device);
+		if (rc) {
+			kfree(ap_dev);
+			continue;
+		}
+		/* Add device attributes. */
+		rc = sysfs_create_group(&ap_dev->device.kobj,
+					&ap_dev_attr_group);
+		if (rc)
+			device_unregister(&ap_dev->device);
+	}
+}
+
+static void
+ap_config_timeout(unsigned long ptr)
+{
+	queue_work(ap_work_queue, &ap_config_work);
+	ap_config_timer.expires = jiffies + ap_config_time * HZ;
+	add_timer(&ap_config_timer);
+}
+
+/**
+ * Set up the timer to run the poll tasklet
+ */
+static inline void ap_schedule_poll_timer(void)
+{
+	if (timer_pending(&ap_poll_timer))
+		return;
+	mod_timer(&ap_poll_timer, jiffies + AP_POLL_TIME);
+}
+
+/**
+ * Receive pending reply messages from an AP device.
+ * @ap_dev: pointer to the AP device
+ * @flags: pointer to control flags, bit 2^0 is set if another poll is
+ *	   required, bit 2^1 is set if the poll timer needs to get armed
+ * Returns 0 if the device is still present, -ENODEV if not.
+ */
+static inline int ap_poll_read(struct ap_device *ap_dev, unsigned long *flags)
+{
+	struct ap_queue_status status;
+	struct ap_message *ap_msg;
+
+	if (ap_dev->queue_count <= 0)
+		return 0;
+	status = __ap_recv(ap_dev->qid, &ap_dev->reply->psmid,
+			   ap_dev->reply->message, ap_dev->reply->length);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		atomic_dec(&ap_poll_requests);
+		ap_dev->queue_count--;
+		list_for_each_entry(ap_msg, &ap_dev->pendingq, list) {
+			if (ap_msg->psmid != ap_dev->reply->psmid)
+				continue;
+			list_del_init(&ap_msg->list);
+			ap_dev->pendingq_count--;
+			ap_dev->drv->receive(ap_dev, ap_msg, ap_dev->reply);
+			break;
+		}
+		if (ap_dev->queue_count > 0)
+			*flags |= 1;
+		break;
+	case AP_RESPONSE_NO_PENDING_REPLY:
+		if (status.queue_empty) {
+			/* The card shouldn't forget requests but who knows. */
+			ap_dev->queue_count = 0;
+			list_splice_init(&ap_dev->pendingq, &ap_dev->requestq);
+			ap_dev->requestq_count += ap_dev->pendingq_count;
+			ap_dev->pendingq_count = 0;
+		} else
+			*flags |= 2;
+		break;
+	default:
+		return -ENODEV;
+	}
+	return 0;
+}
+
+/**
+ * Send messages from the request queue to an AP device.
+ * @ap_dev: pointer to the AP device
+ * @flags: pointer to control flags, bit 2^0 is set if another poll is
+ *	   required, bit 2^1 is set if the poll timer needs to get armed
+ * Returns 0 if the device is still present, -ENODEV if not.
+ */
+static inline int ap_poll_write(struct ap_device *ap_dev, unsigned long *flags)
+{
+	struct ap_queue_status status;
+	struct ap_message *ap_msg;
+
+	if (ap_dev->requestq_count <= 0 ||
+	    ap_dev->queue_count >= ap_dev->queue_depth)
+		return 0;
+	/* Start the next request on the queue. */
+	ap_msg = list_entry(ap_dev->requestq.next, struct ap_message, list);
+	status = __ap_send(ap_dev->qid, ap_msg->psmid,
+			   ap_msg->message, ap_msg->length);
+	switch (status.response_code) {
+	case AP_RESPONSE_NORMAL:
+		atomic_inc(&ap_poll_requests);
+		ap_dev->queue_count++;
+		list_move_tail(&ap_msg->list, &ap_dev->pendingq);
+		ap_dev->requestq_count--;
+		ap_dev->pendingq_count++;
+		if (ap_dev->queue_count < ap_dev->queue_depth &&
+		    ap_dev->requestq_count > 0)
+			*flags |= 1;
+		*flags |= 2;
+		break;
+	case AP_RESPONSE_Q_FULL:
+		*flags |= 2;
+		break;
+	case AP_RESPONSE_MESSAGE_TOO_BIG:
+		return -EINVAL;
+	default:
+		return -ENODEV;
+	}
+	return 0;
+}
+
+/**
+ * Poll AP device for pending replies and send new messages. If either
+ * ap_poll_read or ap_poll_write returns -ENODEV unregister the device.
+ * @ap_dev: pointer to the bus device
+ * @flags: pointer to control flags, bit 2^0 is set if another poll is
+ *	   required, bit 2^1 is set if the poll timer needs to get armed
+ * Returns 0.
+ */
+static inline int ap_poll_queue(struct ap_device *ap_dev, unsigned long *flags)
+{
+	int rc;
+
+	rc = ap_poll_read(ap_dev, flags);
+	if (rc)
+		return rc;
+	return ap_poll_write(ap_dev, flags);
+}
+
+/**
+ * Queue a message to a device.
+ * @ap_dev: pointer to the AP device
+ * @ap_msg: the message to be queued
+ */
+static int __ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
+{
+	struct ap_queue_status status;
+
+	if (list_empty(&ap_dev->requestq) &&
+	    ap_dev->queue_count < ap_dev->queue_depth) {
+		status = __ap_send(ap_dev->qid, ap_msg->psmid,
+				   ap_msg->message, ap_msg->length);
+		switch (status.response_code) {
+		case AP_RESPONSE_NORMAL:
+			list_add_tail(&ap_msg->list, &ap_dev->pendingq);
+			atomic_inc(&ap_poll_requests);
+			ap_dev->pendingq_count++;
+			ap_dev->queue_count++;
+			ap_dev->total_request_count++;
+			break;
+		case AP_RESPONSE_Q_FULL:
+			list_add_tail(&ap_msg->list, &ap_dev->requestq);
+			ap_dev->requestq_count++;
+			ap_dev->total_request_count++;
+			return -EBUSY;
+		case AP_RESPONSE_MESSAGE_TOO_BIG:
+			ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-EINVAL));
+			return -EINVAL;
+		default:	/* Device is gone. */
+			ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
+			return -ENODEV;
+		}
+	} else {
+		list_add_tail(&ap_msg->list, &ap_dev->requestq);
+		ap_dev->requestq_count++;
+		ap_dev->total_request_count++;
+		return -EBUSY;
+	}
+	ap_schedule_poll_timer();
+	return 0;
+}
+
+void ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
+{
+	unsigned long flags;
+	int rc;
+
+	spin_lock_bh(&ap_dev->lock);
+	if (!ap_dev->unregistered) {
+		/* Make room on the queue by polling for finished requests. */
+		rc = ap_poll_queue(ap_dev, &flags);
+		if (!rc)
+			rc = __ap_queue_message(ap_dev, ap_msg);
+		if (!rc)
+			wake_up(&ap_poll_wait);
+	} else {
+		ap_dev->drv->receive(ap_dev, ap_msg, ERR_PTR(-ENODEV));
+		rc = 0;
+	}
+	spin_unlock_bh(&ap_dev->lock);
+	if (rc == -ENODEV)
+		device_unregister(&ap_dev->device);
+}
+EXPORT_SYMBOL(ap_queue_message);
+
+/**
+ * Cancel a crypto request. This is done by removing the request
+ * from the devive pendingq or requestq queue. Note that the
+ * request stays on the AP queue. When it finishes the message
+ * reply will be discarded because the psmid can't be found.
+ * @ap_dev: AP device that has the message queued
+ * @ap_msg: the message that is to be removed
+ */
+void ap_cancel_message(struct ap_device *ap_dev, struct ap_message *ap_msg)
+{
+	struct ap_message *tmp;
+
+	spin_lock_bh(&ap_dev->lock);
+	if (!list_empty(&ap_msg->list)) {
+		list_for_each_entry(tmp, &ap_dev->pendingq, list)
+			if (tmp->psmid == ap_msg->psmid) {
+				ap_dev->pendingq_count--;
+				goto found;
+			}
+		ap_dev->requestq_count--;
+	found:
+		list_del_init(&ap_msg->list);
+	}
+	spin_unlock_bh(&ap_dev->lock);
+}
+EXPORT_SYMBOL(ap_cancel_message);
+
+/**
+ * AP receive polling for finished AP requests
+ */
+static void ap_poll_timeout(unsigned long unused)
+{
+	tasklet_schedule(&ap_tasklet);
+}
+
+/**
+ * Poll all AP devices on the bus in a round robin fashion. Continue
+ * polling until bit 2^0 of the control flags is not set. If bit 2^1
+ * of the control flags has been set arm the poll timer.
+ */
+static int __ap_poll_all(struct device *dev, void *data)
+{
+	struct ap_device *ap_dev = to_ap_dev(dev);
+	int rc;
+
+	spin_lock(&ap_dev->lock);
+	if (!ap_dev->unregistered) {
+		rc = ap_poll_queue(to_ap_dev(dev), (unsigned long *) data);
+	} else
+		rc = 0;
+	spin_unlock(&ap_dev->lock);
+	if (rc)
+		device_unregister(&ap_dev->device);
+	return 0;
+}
+
+static void ap_poll_all(unsigned long dummy)
+{
+	unsigned long flags;
+
+	do {
+		flags = 0;
+		bus_for_each_dev(&ap_bus_type, NULL, &flags, __ap_poll_all);
+	} while (flags & 1);
+	if (flags & 2)
+		ap_schedule_poll_timer();
+}
+
+/**
+ * AP bus poll thread. The purpose of this thread is to poll for
+ * finished requests in a loop if there is a "free" cpu - that is
+ * a cpu that doesn't have anything better to do. The polling stops
+ * as soon as there is another task or if all messages have been
+ * delivered.
+ */
+static int ap_poll_thread(void *data)
+{
+	DECLARE_WAITQUEUE(wait, current);
+	unsigned long flags;
+	int requests;
+
+	set_user_nice(current, -20);
+	while (1) {
+		if (need_resched()) {
+			schedule();
+			continue;
+		}
+		add_wait_queue(&ap_poll_wait, &wait);
+		set_current_state(TASK_INTERRUPTIBLE);
+		if (kthread_should_stop())
+			break;
+		requests = atomic_read(&ap_poll_requests);
+		if (requests <= 0)
+			schedule();
+		set_current_state(TASK_RUNNING);
+		remove_wait_queue(&ap_poll_wait, &wait);
+
+		local_bh_disable();
+		flags = 0;
+		bus_for_each_dev(&ap_bus_type, NULL, &flags, __ap_poll_all);
+		local_bh_enable();
+	}
+	set_current_state(TASK_RUNNING);
+	remove_wait_queue(&ap_poll_wait, &wait);
+	return 0;
+}
+
+static int ap_poll_thread_start(void)
+{
+	int rc;
+
+	mutex_lock(&ap_poll_thread_mutex);
+	if (!ap_poll_kthread) {
+		ap_poll_kthread = kthread_run(ap_poll_thread, NULL, "appoll");
+		rc = IS_ERR(ap_poll_kthread) ? PTR_ERR(ap_poll_kthread) : 0;
+		if (rc)
+			ap_poll_kthread = NULL;
+	}
+	else
+		rc = 0;
+	mutex_unlock(&ap_poll_thread_mutex);
+	return rc;
+}
+
+static void ap_poll_thread_stop(void)
+{
+	mutex_lock(&ap_poll_thread_mutex);
+	if (ap_poll_kthread) {
+		kthread_stop(ap_poll_kthread);
+		ap_poll_kthread = NULL;
+	}
+	mutex_unlock(&ap_poll_thread_mutex);
+}
+
+/**
+ * The module initialization code.
+ */
+int __init ap_module_init(void)
+{
+	int rc, i;
+
+	if (ap_domain_index < -1 || ap_domain_index >= AP_DOMAINS) {
+		printk(KERN_WARNING "Invalid param: domain = %d. "
+		       " Not loading.\n", ap_domain_index);
+		return -EINVAL;
+	}
+	if (ap_instructions_available() != 0) {
+		printk(KERN_WARNING "AP instructions not installed.\n");
+		return -ENODEV;
+	}
+
+	/* Create /sys/bus/ap. */
+	rc = bus_register(&ap_bus_type);
+	if (rc)
+		goto out;
+	for (i = 0; ap_bus_attrs[i]; i++) {
+		rc = bus_create_file(&ap_bus_type, ap_bus_attrs[i]);
+		if (rc)
+			goto out_bus;
+	}
+
+	/* Create /sys/devices/ap. */
+	ap_root_device = s390_root_dev_register("ap");
+	rc = IS_ERR(ap_root_device) ? PTR_ERR(ap_root_device) : 0;
+	if (rc)
+		goto out_bus;
+
+	ap_work_queue = create_singlethread_workqueue("kapwork");
+	if (!ap_work_queue) {
+		rc = -ENOMEM;
+		goto out_root;
+	}
+
+	if (ap_select_domain() == 0)
+		ap_scan_bus(NULL);
+
+	/* Setup the ap bus rescan timer. */
+	init_timer(&ap_config_timer);
+	ap_config_timer.function = ap_config_timeout;
+	ap_config_timer.data = 0;
+	ap_config_timer.expires = jiffies + ap_config_time * HZ;
+	add_timer(&ap_config_timer);
+
+	/* Start the low priority AP bus poll thread. */
+	if (ap_thread_flag) {
+		rc = ap_poll_thread_start();
+		if (rc)
+			goto out_work;
+	}
+
+	return 0;
+
+out_work:
+	del_timer_sync(&ap_config_timer);
+	del_timer_sync(&ap_poll_timer);
+	destroy_workqueue(ap_work_queue);
+out_root:
+	s390_root_dev_unregister(ap_root_device);
+out_bus:
+	while (i--)
+		bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
+	bus_unregister(&ap_bus_type);
+out:
+	return rc;
+}
+
+static int __ap_match_all(struct device *dev, void *data)
+{
+	return 1;
+}
+
+/**
+ * The module termination code
+ */
+void ap_module_exit(void)
+{
+	int i;
+	struct device *dev;
+
+	ap_poll_thread_stop();
+	del_timer_sync(&ap_config_timer);
+	del_timer_sync(&ap_poll_timer);
+	destroy_workqueue(ap_work_queue);
+	s390_root_dev_unregister(ap_root_device);
+	while ((dev = bus_find_device(&ap_bus_type, NULL, NULL,
+		    __ap_match_all)))
+	{
+		device_unregister(dev);
+		put_device(dev);
+	}
+	for (i = 0; ap_bus_attrs[i]; i++)
+		bus_remove_file(&ap_bus_type, ap_bus_attrs[i]);
+	bus_unregister(&ap_bus_type);
+}
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+module_init(ap_module_init);
+module_exit(ap_module_exit);
+#endif
diff --git a/drivers/s390/crypto/ap_bus.h b/drivers/s390/crypto/ap_bus.h
new file mode 100644
index 0000000..83b69c0
--- /dev/null
+++ b/drivers/s390/crypto/ap_bus.h
@@ -0,0 +1,158 @@
+/*
+ * linux/drivers/s390/crypto/ap_bus.h
+ *
+ * Copyright (C) 2006 IBM Corporation
+ * Author(s): Cornelia Huck <cornelia.huck@de.ibm.com>
+ *	      Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *	      Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *
+ * Adjunct processor bus header file.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _AP_BUS_H_
+#define _AP_BUS_H_
+
+#include <linux/device.h>
+#include <linux/mod_devicetable.h>
+#include <linux/types.h>
+
+#define AP_DEVICES 64		/* Number of AP devices. */
+#define AP_DOMAINS 16		/* Number of AP domains. */
+#define AP_MAX_RESET 90		/* Maximum number of resets. */
+#define AP_CONFIG_TIME 30	/* Time in seconds between AP bus rescans. */
+#define AP_POLL_TIME 1		/* Time in ticks between receive polls. */
+
+extern int ap_domain_index;
+
+/**
+ * The ap_qid_t identifier of an ap queue. It contains a
+ * 6 bit device index and a 4 bit queue index (domain).
+ */
+typedef unsigned int ap_qid_t;
+
+#define AP_MKQID(_device,_queue) (((_device) & 63) << 8 | ((_queue) & 15))
+#define AP_QID_DEVICE(_qid) (((_qid) >> 8) & 63)
+#define AP_QID_QUEUE(_qid) ((_qid) & 15)
+
+/**
+ * The ap queue status word is returned by all three AP functions
+ * (PQAP, NQAP and DQAP).  There's a set of flags in the first
+ * byte, followed by a 1 byte response code.
+ */
+struct ap_queue_status {
+	unsigned int queue_empty	: 1;
+	unsigned int replies_waiting	: 1;
+	unsigned int queue_full		: 1;
+	unsigned int pad1		: 5;
+	unsigned int response_code	: 8;
+	unsigned int pad2		: 16;
+};
+
+#define AP_RESPONSE_NORMAL		0x00
+#define AP_RESPONSE_Q_NOT_AVAIL		0x01
+#define AP_RESPONSE_RESET_IN_PROGRESS	0x02
+#define AP_RESPONSE_DECONFIGURED	0x03
+#define AP_RESPONSE_CHECKSTOPPED	0x04
+#define AP_RESPONSE_BUSY		0x05
+#define AP_RESPONSE_Q_FULL		0x10
+#define AP_RESPONSE_NO_PENDING_REPLY	0x10
+#define AP_RESPONSE_INDEX_TOO_BIG	0x11
+#define AP_RESPONSE_NO_FIRST_PART	0x13
+#define AP_RESPONSE_MESSAGE_TOO_BIG	0x15
+
+/**
+ * Known device types
+ */
+#define AP_DEVICE_TYPE_PCICC	3
+#define AP_DEVICE_TYPE_PCICA	4
+#define AP_DEVICE_TYPE_PCIXCC	5
+#define AP_DEVICE_TYPE_CEX2A	6
+#define AP_DEVICE_TYPE_CEX2C	7
+
+struct ap_device;
+struct ap_message;
+
+struct ap_driver {
+	struct device_driver driver;
+	struct ap_device_id *ids;
+
+	int (*probe)(struct ap_device *);
+	void (*remove)(struct ap_device *);
+	/* receive is called from tasklet context */
+	void (*receive)(struct ap_device *, struct ap_message *,
+			struct ap_message *);
+};
+
+#define to_ap_drv(x) container_of((x), struct ap_driver, driver)
+
+int ap_driver_register(struct ap_driver *, struct module *, char *);
+void ap_driver_unregister(struct ap_driver *);
+
+struct ap_device {
+	struct device device;
+	struct ap_driver *drv;		/* Pointer to AP device driver. */
+	spinlock_t lock;		/* Per device lock. */
+
+	ap_qid_t qid;			/* AP queue id. */
+	int queue_depth;		/* AP queue depth.*/
+	int device_type;		/* AP device type. */
+	int unregistered;		/* marks AP device as unregistered */
+
+	int queue_count;		/* # messages currently on AP queue. */
+
+	struct list_head pendingq;	/* List of message sent to AP queue. */
+	int pendingq_count;		/* # requests on pendingq list. */
+	struct list_head requestq;	/* List of message yet to be sent. */
+	int requestq_count;		/* # requests on requestq list. */
+	int total_request_count;	/* # requests ever for this AP device. */
+
+	struct ap_message *reply;	/* Per device reply message. */
+
+	void *private;			/* ap driver private pointer. */
+};
+
+#define to_ap_dev(x) container_of((x), struct ap_device, device)
+
+struct ap_message {
+	struct list_head list;		/* Request queueing. */
+	unsigned long long psmid;	/* Message id. */
+	void *message;			/* Pointer to message buffer. */
+	size_t length;			/* Message length. */
+
+	void *private;			/* ap driver private pointer. */
+};
+
+#define AP_DEVICE(dt)					\
+	.dev_type=(dt),					\
+	.match_flags=AP_DEVICE_ID_MATCH_DEVICE_TYPE,
+
+/**
+ * Note: don't use ap_send/ap_recv after using ap_queue_message
+ * for the first time. Otherwise the ap message queue will get
+ * confused.
+ */
+int ap_send(ap_qid_t, unsigned long long, void *, size_t);
+int ap_recv(ap_qid_t, unsigned long long *, void *, size_t);
+
+void ap_queue_message(struct ap_device *ap_dev, struct ap_message *ap_msg);
+void ap_cancel_message(struct ap_device *ap_dev, struct ap_message *ap_msg);
+void ap_flush_queue(struct ap_device *ap_dev);
+
+int ap_module_init(void);
+void ap_module_exit(void);
+
+#endif /* _AP_BUS_H_ */
diff --git a/drivers/s390/crypto/z90common.h b/drivers/s390/crypto/z90common.h
deleted file mode 100644
index dbbcda3..0000000
--- a/drivers/s390/crypto/z90common.h
+++ /dev/null
@@ -1,166 +0,0 @@
-/*
- *  linux/drivers/s390/crypto/z90common.h
- *
- *  z90crypt 1.3.3
- *
- *  Copyright (C)  2001, 2005 IBM Corporation
- *  Author(s): Robert Burroughs (burrough@us.ibm.com)
- *             Eric Rossman (edrossma@us.ibm.com)
- *
- *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#ifndef _Z90COMMON_H_
-#define _Z90COMMON_H_
-
-
-#define RESPBUFFSIZE 256
-#define PCI_FUNC_KEY_DECRYPT 0x5044
-#define PCI_FUNC_KEY_ENCRYPT 0x504B
-extern int ext_bitlens;
-
-enum devstat {
-	DEV_GONE,
-	DEV_ONLINE,
-	DEV_QUEUE_FULL,
-	DEV_EMPTY,
-	DEV_NO_WORK,
-	DEV_BAD_MESSAGE,
-	DEV_TSQ_EXCEPTION,
-	DEV_RSQ_EXCEPTION,
-	DEV_SEN_EXCEPTION,
-	DEV_REC_EXCEPTION
-};
-
-enum hdstat {
-	HD_NOT_THERE,
-	HD_BUSY,
-	HD_DECONFIGURED,
-	HD_CHECKSTOPPED,
-	HD_ONLINE,
-	HD_TSQ_EXCEPTION
-};
-
-#define Z90C_NO_DEVICES		1
-#define Z90C_AMBIGUOUS_DOMAIN	2
-#define Z90C_INCORRECT_DOMAIN	3
-#define ENOTINIT		4
-
-#define SEN_BUSY	 7
-#define SEN_USER_ERROR	 8
-#define SEN_QUEUE_FULL	11
-#define SEN_NOT_AVAIL	16
-#define SEN_PAD_ERROR	17
-#define SEN_RETRY	18
-#define SEN_RELEASED	24
-
-#define REC_EMPTY	 4
-#define REC_BUSY	 6
-#define REC_OPERAND_INV	 8
-#define REC_OPERAND_SIZE 9
-#define REC_EVEN_MOD	10
-#define REC_NO_WORK	11
-#define REC_HARDWAR_ERR	12
-#define REC_NO_RESPONSE	13
-#define REC_RETRY_DEV	14
-#define REC_USER_GONE	15
-#define REC_BAD_MESSAGE	16
-#define REC_INVALID_PAD	17
-#define REC_USE_PCICA	18
-
-#define WRONG_DEVICE_TYPE 20
-
-#define REC_FATAL_ERROR 32
-#define SEN_FATAL_ERROR 33
-#define TSQ_FATAL_ERROR 34
-#define RSQ_FATAL_ERROR 35
-
-#define Z90CRYPT_NUM_TYPES	6
-#define PCICA		0
-#define PCICC		1
-#define PCIXCC_MCL2	2
-#define PCIXCC_MCL3	3
-#define CEX2C		4
-#define CEX2A		5
-#define NILDEV		-1
-#define ANYDEV		-1
-#define PCIXCC_UNK	-2
-
-enum hdevice_type {
-	PCICC_HW  = 3,
-	PCICA_HW  = 4,
-	PCIXCC_HW = 5,
-	CEX2A_HW  = 6,
-	CEX2C_HW  = 7
-};
-
-struct CPRBX {
-	unsigned short cprb_len;
-	unsigned char  cprb_ver_id;
-	unsigned char  pad_000[3];
-	unsigned char  func_id[2];
-	unsigned char  cprb_flags[4];
-	unsigned int   req_parml;
-	unsigned int   req_datal;
-	unsigned int   rpl_msgbl;
-	unsigned int   rpld_parml;
-	unsigned int   rpl_datal;
-	unsigned int   rpld_datal;
-	unsigned int   req_extbl;
-	unsigned char  pad_001[4];
-	unsigned int   rpld_extbl;
-	unsigned char  req_parmb[16];
-	unsigned char  req_datab[16];
-	unsigned char  rpl_parmb[16];
-	unsigned char  rpl_datab[16];
-	unsigned char  req_extb[16];
-	unsigned char  rpl_extb[16];
-	unsigned short ccp_rtcode;
-	unsigned short ccp_rscode;
-	unsigned int   mac_data_len;
-	unsigned char  logon_id[8];
-	unsigned char  mac_value[8];
-	unsigned char  mac_content_flgs;
-	unsigned char  pad_002;
-	unsigned short domain;
-	unsigned char  pad_003[12];
-	unsigned char  pad_004[36];
-};
-
-#ifndef DEV_NAME
-#define DEV_NAME	"z90crypt"
-#endif
-#define PRINTK(fmt, args...) \
-	printk(KERN_DEBUG DEV_NAME ": %s -> " fmt, __FUNCTION__ , ## args)
-#define PRINTKN(fmt, args...) \
-	printk(KERN_DEBUG DEV_NAME ": " fmt, ## args)
-#define PRINTKW(fmt, args...) \
-	printk(KERN_WARNING DEV_NAME ": %s -> " fmt, __FUNCTION__ , ## args)
-#define PRINTKC(fmt, args...) \
-	printk(KERN_CRIT DEV_NAME ": %s -> " fmt, __FUNCTION__ , ## args)
-
-#ifdef Z90CRYPT_DEBUG
-#define PDEBUG(fmt, args...) \
-	printk(KERN_DEBUG DEV_NAME ": %s -> " fmt, __FUNCTION__ , ## args)
-#else
-#define PDEBUG(fmt, args...) do {} while (0)
-#endif
-
-#define UMIN(a,b) ((a) < (b) ? (a) : (b))
-#define IS_EVEN(x) ((x) == (2 * ((x) / 2)))
-
-#endif
diff --git a/drivers/s390/crypto/z90crypt.h b/drivers/s390/crypto/z90crypt.h
deleted file mode 100644
index 0ca1d12..0000000
--- a/drivers/s390/crypto/z90crypt.h
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- *  linux/drivers/s390/crypto/z90crypt.h
- *
- *  z90crypt 1.3.3 (kernel-private header)
- *
- *  Copyright (C)  2001, 2005 IBM Corporation
- *  Author(s): Robert Burroughs (burrough@us.ibm.com)
- *             Eric Rossman (edrossma@us.ibm.com)
- *
- *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#ifndef _Z90CRYPT_H_
-#define _Z90CRYPT_H_
-
-#include <asm/z90crypt.h>
-
-/**
- * local errno definitions
- */
-#define ENOBUFF	  129	// filp->private_data->...>work_elem_p->buffer is NULL
-#define EWORKPEND 130	// user issues ioctl while another pending
-#define ERELEASED 131	// user released while ioctl pending
-#define EQUIESCE  132	// z90crypt quiescing (no more work allowed)
-#define ETIMEOUT  133	// request timed out
-#define EUNKNOWN  134	// some unrecognized error occured (retry may succeed)
-#define EGETBUFF  135	// Error getting buffer or hardware lacks capability
-			// (retry in software)
-
-/**
- * DEPRECATED STRUCTURES
- */
-
-/**
- * This structure is DEPRECATED and the corresponding ioctl() has been
- * replaced with individual ioctl()s for each piece of data!
- * This structure will NOT survive past version 1.3.1, so switch to the
- * new ioctl()s.
- */
-#define MASK_LENGTH 64 // mask length
-struct ica_z90_status {
-	int totalcount;
-	int leedslitecount; // PCICA
-	int leeds2count;    // PCICC
-	// int PCIXCCCount; is not in struct for backward compatibility
-	int requestqWaitCount;
-	int pendingqWaitCount;
-	int totalOpenCount;
-	int cryptoDomain;
-	// status: 0=not there, 1=PCICA, 2=PCICC, 3=PCIXCC_MCL2, 4=PCIXCC_MCL3,
-	//         5=CEX2C
-	unsigned char status[MASK_LENGTH];
-	// qdepth: # work elements waiting for each device
-	unsigned char qdepth[MASK_LENGTH];
-};
-
-#endif /* _Z90CRYPT_H_ */
diff --git a/drivers/s390/crypto/z90hardware.c b/drivers/s390/crypto/z90hardware.c
deleted file mode 100644
index be60795..0000000
--- a/drivers/s390/crypto/z90hardware.c
+++ /dev/null
@@ -1,2531 +0,0 @@
-/*
- *  linux/drivers/s390/crypto/z90hardware.c
- *
- *  z90crypt 1.3.3
- *
- *  Copyright (C)  2001, 2005 IBM Corporation
- *  Author(s): Robert Burroughs (burrough@us.ibm.com)
- *             Eric Rossman (edrossma@us.ibm.com)
- *
- *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#include <asm/uaccess.h>
-#include <linux/compiler.h>
-#include <linux/delay.h>
-#include <linux/init.h>
-#include <linux/module.h>
-#include "z90crypt.h"
-#include "z90common.h"
-
-struct cca_token_hdr {
-	unsigned char  token_identifier;
-	unsigned char  version;
-	unsigned short token_length;
-	unsigned char  reserved[4];
-};
-
-#define CCA_TKN_HDR_ID_EXT 0x1E
-
-struct cca_private_ext_ME_sec {
-	unsigned char  section_identifier;
-	unsigned char  version;
-	unsigned short section_length;
-	unsigned char  private_key_hash[20];
-	unsigned char  reserved1[4];
-	unsigned char  key_format;
-	unsigned char  reserved2;
-	unsigned char  key_name_hash[20];
-	unsigned char  key_use_flags[4];
-	unsigned char  reserved3[6];
-	unsigned char  reserved4[24];
-	unsigned char  confounder[24];
-	unsigned char  exponent[128];
-	unsigned char  modulus[128];
-};
-
-#define CCA_PVT_USAGE_ALL 0x80
-
-struct cca_public_sec {
-	unsigned char  section_identifier;
-	unsigned char  version;
-	unsigned short section_length;
-	unsigned char  reserved[2];
-	unsigned short exponent_len;
-	unsigned short modulus_bit_len;
-	unsigned short modulus_byte_len;
-	unsigned char  exponent[3];
-};
-
-struct cca_private_ext_ME {
-	struct cca_token_hdr	      pvtMEHdr;
-	struct cca_private_ext_ME_sec pvtMESec;
-	struct cca_public_sec	      pubMESec;
-};
-
-struct cca_public_key {
-	struct cca_token_hdr  pubHdr;
-	struct cca_public_sec pubSec;
-};
-
-struct cca_pvt_ext_CRT_sec {
-	unsigned char  section_identifier;
-	unsigned char  version;
-	unsigned short section_length;
-	unsigned char  private_key_hash[20];
-	unsigned char  reserved1[4];
-	unsigned char  key_format;
-	unsigned char  reserved2;
-	unsigned char  key_name_hash[20];
-	unsigned char  key_use_flags[4];
-	unsigned short p_len;
-	unsigned short q_len;
-	unsigned short dp_len;
-	unsigned short dq_len;
-	unsigned short u_len;
-	unsigned short mod_len;
-	unsigned char  reserved3[4];
-	unsigned short pad_len;
-	unsigned char  reserved4[52];
-	unsigned char  confounder[8];
-};
-
-#define CCA_PVT_EXT_CRT_SEC_ID_PVT 0x08
-#define CCA_PVT_EXT_CRT_SEC_FMT_CL 0x40
-
-struct cca_private_ext_CRT {
-	struct cca_token_hdr	   pvtCrtHdr;
-	struct cca_pvt_ext_CRT_sec pvtCrtSec;
-	struct cca_public_sec	   pubCrtSec;
-};
-
-struct ap_status_word {
-	unsigned char q_stat_flags;
-	unsigned char response_code;
-	unsigned char reserved[2];
-};
-
-#define AP_Q_STATUS_EMPTY		0x80
-#define AP_Q_STATUS_REPLIES_WAITING	0x40
-#define AP_Q_STATUS_ARRAY_FULL		0x20
-
-#define AP_RESPONSE_NORMAL		0x00
-#define AP_RESPONSE_Q_NOT_AVAIL		0x01
-#define AP_RESPONSE_RESET_IN_PROGRESS	0x02
-#define AP_RESPONSE_DECONFIGURED	0x03
-#define AP_RESPONSE_CHECKSTOPPED	0x04
-#define AP_RESPONSE_BUSY		0x05
-#define AP_RESPONSE_Q_FULL		0x10
-#define AP_RESPONSE_NO_PENDING_REPLY	0x10
-#define AP_RESPONSE_INDEX_TOO_BIG	0x11
-#define AP_RESPONSE_NO_FIRST_PART	0x13
-#define AP_RESPONSE_MESSAGE_TOO_BIG	0x15
-
-#define AP_MAX_CDX_BITL		4
-#define AP_RQID_RESERVED_BITL	4
-#define SKIP_BITL		(AP_MAX_CDX_BITL + AP_RQID_RESERVED_BITL)
-
-struct type4_hdr {
-	unsigned char  reserved1;
-	unsigned char  msg_type_code;
-	unsigned short msg_len;
-	unsigned char  request_code;
-	unsigned char  msg_fmt;
-	unsigned short reserved2;
-};
-
-#define TYPE4_TYPE_CODE 0x04
-#define TYPE4_REQU_CODE 0x40
-
-#define TYPE4_SME_LEN 0x0188
-#define TYPE4_LME_LEN 0x0308
-#define TYPE4_SCR_LEN 0x01E0
-#define TYPE4_LCR_LEN 0x03A0
-
-#define TYPE4_SME_FMT 0x00
-#define TYPE4_LME_FMT 0x10
-#define TYPE4_SCR_FMT 0x40
-#define TYPE4_LCR_FMT 0x50
-
-struct type4_sme {
-	struct type4_hdr header;
-	unsigned char	 message[128];
-	unsigned char	 exponent[128];
-	unsigned char	 modulus[128];
-};
-
-struct type4_lme {
-	struct type4_hdr header;
-	unsigned char	 message[256];
-	unsigned char	 exponent[256];
-	unsigned char	 modulus[256];
-};
-
-struct type4_scr {
-	struct type4_hdr header;
-	unsigned char	 message[128];
-	unsigned char	 dp[72];
-	unsigned char	 dq[64];
-	unsigned char	 p[72];
-	unsigned char	 q[64];
-	unsigned char	 u[72];
-};
-
-struct type4_lcr {
-	struct type4_hdr header;
-	unsigned char	 message[256];
-	unsigned char	 dp[136];
-	unsigned char	 dq[128];
-	unsigned char	 p[136];
-	unsigned char	 q[128];
-	unsigned char	 u[136];
-};
-
-union type4_msg {
-	struct type4_sme sme;
-	struct type4_lme lme;
-	struct type4_scr scr;
-	struct type4_lcr lcr;
-};
-
-struct type84_hdr {
-	unsigned char  reserved1;
-	unsigned char  code;
-	unsigned short len;
-	unsigned char  reserved2[4];
-};
-
-#define TYPE84_RSP_CODE 0x84
-
-struct type6_hdr {
-	unsigned char reserved1;
-	unsigned char type;
-	unsigned char reserved2[2];
-	unsigned char right[4];
-	unsigned char reserved3[2];
-	unsigned char reserved4[2];
-	unsigned char apfs[4];
-	unsigned int  offset1;
-	unsigned int  offset2;
-	unsigned int  offset3;
-	unsigned int  offset4;
-	unsigned char agent_id[16];
-	unsigned char rqid[2];
-	unsigned char reserved5[2];
-	unsigned char function_code[2];
-	unsigned char reserved6[2];
-	unsigned int  ToCardLen1;
-	unsigned int  ToCardLen2;
-	unsigned int  ToCardLen3;
-	unsigned int  ToCardLen4;
-	unsigned int  FromCardLen1;
-	unsigned int  FromCardLen2;
-	unsigned int  FromCardLen3;
-	unsigned int  FromCardLen4;
-};
-
-struct CPRB {
-	unsigned char cprb_len[2];
-	unsigned char cprb_ver_id;
-	unsigned char pad_000;
-	unsigned char srpi_rtcode[4];
-	unsigned char srpi_verb;
-	unsigned char flags;
-	unsigned char func_id[2];
-	unsigned char checkpoint_flag;
-	unsigned char resv2;
-	unsigned char req_parml[2];
-	unsigned char req_parmp[4];
-	unsigned char req_datal[4];
-	unsigned char req_datap[4];
-	unsigned char rpl_parml[2];
-	unsigned char pad_001[2];
-	unsigned char rpl_parmp[4];
-	unsigned char rpl_datal[4];
-	unsigned char rpl_datap[4];
-	unsigned char ccp_rscode[2];
-	unsigned char ccp_rtcode[2];
-	unsigned char repd_parml[2];
-	unsigned char mac_data_len[2];
-	unsigned char repd_datal[4];
-	unsigned char req_pc[2];
-	unsigned char res_origin[8];
-	unsigned char mac_value[8];
-	unsigned char logon_id[8];
-	unsigned char usage_domain[2];
-	unsigned char resv3[18];
-	unsigned char svr_namel[2];
-	unsigned char svr_name[8];
-};
-
-struct type6_msg {
-	struct type6_hdr header;
-	struct CPRB	 CPRB;
-};
-
-struct type86_hdr {
-	unsigned char reserved1;
-	unsigned char type;
-	unsigned char format;
-	unsigned char reserved2;
-	unsigned char reply_code;
-	unsigned char reserved3[3];
-};
-
-#define TYPE86_RSP_CODE 0x86
-#define TYPE86_FMT2	0x02
-
-struct type86_fmt2_msg {
-	struct type86_hdr header;
-	unsigned char	  reserved[4];
-	unsigned char	  apfs[4];
-	unsigned int	  count1;
-	unsigned int	  offset1;
-	unsigned int	  count2;
-	unsigned int	  offset2;
-	unsigned int	  count3;
-	unsigned int	  offset3;
-	unsigned int	  count4;
-	unsigned int	  offset4;
-};
-
-static struct type6_hdr static_type6_hdr = {
-	0x00,
-	0x06,
-	{0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	0x00000058,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	{0x01,0x00,0x43,0x43,0x41,0x2D,0x41,0x50,
-	 0x50,0x4C,0x20,0x20,0x20,0x01,0x01,0x01},
-	{0x00,0x00},
-	{0x00,0x00},
-	{0x50,0x44},
-	{0x00,0x00},
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000
-};
-
-static struct type6_hdr static_type6_hdrX = {
-	0x00,
-	0x06,
-	{0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	0x00000058,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	{0x43,0x41,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00},
-	{0x50,0x44},
-	{0x00,0x00},
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000
-};
-
-static struct CPRB static_cprb = {
-	{0x70,0x00},
-	0x41,
-	0x00,
-	{0x00,0x00,0x00,0x00},
-	0x00,
-	0x00,
-	{0x54,0x32},
-	0x01,
-	0x00,
-	{0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00,0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00},
-	{0x08,0x00},
-	{0x49,0x43,0x53,0x46,0x20,0x20,0x20,0x20}
-};
-
-struct function_and_rules_block {
-	unsigned char function_code[2];
-	unsigned char ulen[2];
-	unsigned char only_rule[8];
-};
-
-static struct function_and_rules_block static_pkd_function_and_rules = {
-	{0x50,0x44},
-	{0x0A,0x00},
-	{'P','K','C','S','-','1','.','2'}
-};
-
-static struct function_and_rules_block static_pke_function_and_rules = {
-	{0x50,0x4B},
-	{0x0A,0x00},
-	{'P','K','C','S','-','1','.','2'}
-};
-
-struct T6_keyBlock_hdr {
-	unsigned char blen[2];
-	unsigned char ulen[2];
-	unsigned char flags[2];
-};
-
-static struct T6_keyBlock_hdr static_T6_keyBlock_hdr = {
-	{0x89,0x01},
-	{0x87,0x01},
-	{0x00}
-};
-
-static struct CPRBX static_cprbx = {
-	0x00DC,
-	0x02,
-	{0x00,0x00,0x00},
-	{0x54,0x32},
-	{0x00,0x00,0x00,0x00},
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	0x00000000,
-	{0x00,0x00,0x00,0x00},
-	0x00000000,
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	0x0000,
-	0x0000,
-	0x00000000,
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	0x00,
-	0x00,
-	0x0000,
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-	{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-	 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}
-};
-
-static struct function_and_rules_block static_pkd_function_and_rulesX_MCL2 = {
-	{0x50,0x44},
-	{0x00,0x0A},
-	{'P','K','C','S','-','1','.','2'}
-};
-
-static struct function_and_rules_block static_pke_function_and_rulesX_MCL2 = {
-	{0x50,0x4B},
-	{0x00,0x0A},
-	{'Z','E','R','O','-','P','A','D'}
-};
-
-static struct function_and_rules_block static_pkd_function_and_rulesX = {
-	{0x50,0x44},
-	{0x00,0x0A},
-	{'Z','E','R','O','-','P','A','D'}
-};
-
-static struct function_and_rules_block static_pke_function_and_rulesX = {
-	{0x50,0x4B},
-	{0x00,0x0A},
-	{'M','R','P',' ',' ',' ',' ',' '}
-};
-
-static unsigned char static_PKE_function_code[2] = {0x50, 0x4B};
-
-struct T6_keyBlock_hdrX {
-	unsigned short blen;
-	unsigned short ulen;
-	unsigned char flags[2];
-};
-
-static unsigned char static_pad[256] = {
-0x1B,0x7B,0x5D,0xB5,0x75,0x01,0x3D,0xFD,0x8D,0xD1,0xC7,0x03,0x2D,0x09,0x23,0x57,
-0x89,0x49,0xB9,0x3F,0xBB,0x99,0x41,0x5B,0x75,0x21,0x7B,0x9D,0x3B,0x6B,0x51,0x39,
-0xBB,0x0D,0x35,0xB9,0x89,0x0F,0x93,0xA5,0x0B,0x47,0xF1,0xD3,0xBB,0xCB,0xF1,0x9D,
-0x23,0x73,0x71,0xFF,0xF3,0xF5,0x45,0xFB,0x61,0x29,0x23,0xFD,0xF1,0x29,0x3F,0x7F,
-0x17,0xB7,0x1B,0xA9,0x19,0xBD,0x57,0xA9,0xD7,0x95,0xA3,0xCB,0xED,0x1D,0xDB,0x45,
-0x7D,0x11,0xD1,0x51,0x1B,0xED,0x71,0xE9,0xB1,0xD1,0xAB,0xAB,0x21,0x2B,0x1B,0x9F,
-0x3B,0x9F,0xF7,0xF7,0xBD,0x63,0xEB,0xAD,0xDF,0xB3,0x6F,0x5B,0xDB,0x8D,0xA9,0x5D,
-0xE3,0x7D,0x77,0x49,0x47,0xF5,0xA7,0xFD,0xAB,0x2F,0x27,0x35,0x77,0xD3,0x49,0xC9,
-0x09,0xEB,0xB1,0xF9,0xBF,0x4B,0xCB,0x2B,0xEB,0xEB,0x05,0xFF,0x7D,0xC7,0x91,0x8B,
-0x09,0x83,0xB9,0xB9,0x69,0x33,0x39,0x6B,0x79,0x75,0x19,0xBF,0xBB,0x07,0x1D,0xBD,
-0x29,0xBF,0x39,0x95,0x93,0x1D,0x35,0xC7,0xC9,0x4D,0xE5,0x97,0x0B,0x43,0x9B,0xF1,
-0x16,0x93,0x03,0x1F,0xA5,0xFB,0xDB,0xF3,0x27,0x4F,0x27,0x61,0x05,0x1F,0xB9,0x23,
-0x2F,0xC3,0x81,0xA9,0x23,0x71,0x55,0x55,0xEB,0xED,0x41,0xE5,0xF3,0x11,0xF1,0x43,
-0x69,0x03,0xBD,0x0B,0x37,0x0F,0x51,0x8F,0x0B,0xB5,0x89,0x5B,0x67,0xA9,0xD9,0x4F,
-0x01,0xF9,0x21,0x77,0x37,0x73,0x79,0xC5,0x7F,0x51,0xC1,0xCF,0x97,0xA1,0x75,0xAD,
-0x35,0x9D,0xD3,0xD3,0xA7,0x9D,0x5D,0x41,0x6F,0x65,0x1B,0xCF,0xA9,0x87,0x91,0x09
-};
-
-static struct cca_private_ext_ME static_pvt_me_key = {
-	{
-		0x1E,
-		0x00,
-		0x0183,
-		{0x00,0x00,0x00,0x00}
-	},
-
-	{
-		0x02,
-		0x00,
-		0x016C,
-		{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00},
-		{0x00,0x00,0x00,0x00},
-		0x00,
-		0x00,
-		{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00},
-		{0x80,0x00,0x00,0x00},
-		{0x00,0x00,0x00,0x00,0x00,0x00},
-		{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-		{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-		{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00},
-		{0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-		 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}
-	},
-
-	{
-		0x04,
-		0x00,
-		0x000F,
-		{0x00,0x00},
-		0x0003,
-		0x0000,
-		0x0000,
-		{0x01,0x00,0x01}
-	}
-};
-
-static struct cca_public_key static_public_key = {
-	{
-		0x1E,
-		0x00,
-		0x0000,
-		{0x00,0x00,0x00,0x00}
-	},
-
-	{
-		0x04,
-		0x00,
-		0x0000,
-		{0x00,0x00},
-		0x0000,
-		0x0000,
-		0x0000,
-		{0x01,0x00,0x01}
-	}
-};
-
-#define FIXED_TYPE6_ME_LEN 0x0000025F
-
-#define FIXED_TYPE6_ME_EN_LEN 0x000000F0
-
-#define FIXED_TYPE6_ME_LENX 0x000002CB
-
-#define FIXED_TYPE6_ME_EN_LENX 0x0000015C
-
-static struct cca_public_sec static_cca_pub_sec = {
-	0x04,
-	0x00,
-	0x000f,
-	{0x00,0x00},
-	0x0003,
-	0x0000,
-	0x0000,
-	{0x01,0x00,0x01}
-};
-
-#define FIXED_TYPE6_CR_LEN 0x00000177
-
-#define FIXED_TYPE6_CR_LENX 0x000001E3
-
-#define MAX_RESPONSE_SIZE 0x00000710
-
-#define MAX_RESPONSEX_SIZE 0x0000077C
-
-#define RESPONSE_CPRB_SIZE  0x000006B8
-#define RESPONSE_CPRBX_SIZE 0x00000724
-
-struct type50_hdr {
-	u8    reserved1;
-	u8    msg_type_code;
-	u16   msg_len;
-	u8    reserved2;
-	u8    ignored;
-	u16   reserved3;
-};
-
-#define TYPE50_TYPE_CODE 0x50
-
-#define TYPE50_MEB1_LEN (sizeof(struct type50_meb1_msg))
-#define TYPE50_MEB2_LEN (sizeof(struct type50_meb2_msg))
-#define TYPE50_CRB1_LEN (sizeof(struct type50_crb1_msg))
-#define TYPE50_CRB2_LEN (sizeof(struct type50_crb2_msg))
-
-#define TYPE50_MEB1_FMT 0x0001
-#define TYPE50_MEB2_FMT 0x0002
-#define TYPE50_CRB1_FMT 0x0011
-#define TYPE50_CRB2_FMT 0x0012
-
-struct type50_meb1_msg {
-	struct type50_hdr	header;
-	u16			keyblock_type;
-	u8			reserved[6];
-	u8			exponent[128];
-	u8			modulus[128];
-	u8			message[128];
-};
-
-struct type50_meb2_msg {
-	struct type50_hdr	header;
-	u16			keyblock_type;
-	u8			reserved[6];
-	u8			exponent[256];
-	u8			modulus[256];
-	u8			message[256];
-};
-
-struct type50_crb1_msg {
-	struct type50_hdr	header;
-	u16			keyblock_type;
-	u8			reserved[6];
-	u8			p[64];
-	u8			q[64];
-	u8			dp[64];
-	u8			dq[64];
-	u8			u[64];
-	u8			message[128];
-};
-
-struct type50_crb2_msg {
-	struct type50_hdr	header;
-	u16			keyblock_type;
-	u8			reserved[6];
-	u8			p[128];
-	u8			q[128];
-	u8			dp[128];
-	u8			dq[128];
-	u8			u[128];
-	u8			message[256];
-};
-
-union type50_msg {
-	struct type50_meb1_msg meb1;
-	struct type50_meb2_msg meb2;
-	struct type50_crb1_msg crb1;
-	struct type50_crb2_msg crb2;
-};
-
-struct type80_hdr {
-	u8	reserved1;
-	u8	type;
-	u16	len;
-	u8	code;
-	u8	reserved2[3];
-	u8	reserved3[8];
-};
-
-#define TYPE80_RSP_CODE 0x80
-
-struct error_hdr {
-	unsigned char reserved1;
-	unsigned char type;
-	unsigned char reserved2[2];
-	unsigned char reply_code;
-	unsigned char reserved3[3];
-};
-
-#define TYPE82_RSP_CODE 0x82
-#define TYPE88_RSP_CODE 0x88
-
-#define REP82_ERROR_MACHINE_FAILURE  0x10
-#define REP82_ERROR_PREEMPT_FAILURE  0x12
-#define REP82_ERROR_CHECKPT_FAILURE  0x14
-#define REP82_ERROR_MESSAGE_TYPE     0x20
-#define REP82_ERROR_INVALID_COMM_CD  0x21
-#define REP82_ERROR_INVALID_MSG_LEN  0x23
-#define REP82_ERROR_RESERVD_FIELD    0x24
-#define REP82_ERROR_FORMAT_FIELD     0x29
-#define REP82_ERROR_INVALID_COMMAND  0x30
-#define REP82_ERROR_MALFORMED_MSG    0x40
-#define REP82_ERROR_RESERVED_FIELDO  0x50
-#define REP82_ERROR_WORD_ALIGNMENT   0x60
-#define REP82_ERROR_MESSAGE_LENGTH   0x80
-#define REP82_ERROR_OPERAND_INVALID  0x82
-#define REP82_ERROR_OPERAND_SIZE     0x84
-#define REP82_ERROR_EVEN_MOD_IN_OPND 0x85
-#define REP82_ERROR_RESERVED_FIELD   0x88
-#define REP82_ERROR_TRANSPORT_FAIL   0x90
-#define REP82_ERROR_PACKET_TRUNCATED 0xA0
-#define REP82_ERROR_ZERO_BUFFER_LEN  0xB0
-
-#define REP88_ERROR_MODULE_FAILURE   0x10
-#define REP88_ERROR_MODULE_TIMEOUT   0x11
-#define REP88_ERROR_MODULE_NOTINIT   0x13
-#define REP88_ERROR_MODULE_NOTAVAIL  0x14
-#define REP88_ERROR_MODULE_DISABLED  0x15
-#define REP88_ERROR_MODULE_IN_DIAGN  0x17
-#define REP88_ERROR_FASTPATH_DISABLD 0x19
-#define REP88_ERROR_MESSAGE_TYPE     0x20
-#define REP88_ERROR_MESSAGE_MALFORMD 0x22
-#define REP88_ERROR_MESSAGE_LENGTH   0x23
-#define REP88_ERROR_RESERVED_FIELD   0x24
-#define REP88_ERROR_KEY_TYPE         0x34
-#define REP88_ERROR_INVALID_KEY      0x82
-#define REP88_ERROR_OPERAND          0x84
-#define REP88_ERROR_OPERAND_EVEN_MOD 0x85
-
-#define CALLER_HEADER 12
-
-static inline int
-testq(int q_nr, int *q_depth, int *dev_type, struct ap_status_word *stat)
-{
-	int ccode;
-
-	asm volatile
-#ifdef CONFIG_64BIT
-	("	llgfr	0,%4		\n"
-	 "	slgr	1,1		\n"
-	 "	lgr	2,1		\n"
-	 "0:	.long	0xb2af0000	\n"
-	 "1:	ipm	%0		\n"
-	 "	srl	%0,28		\n"
-	 "	iihh	%0,0		\n"
-	 "	iihl	%0,0		\n"
-	 "	lgr	%1,1		\n"
-	 "	lgr	%3,2		\n"
-	 "	srl	%3,24		\n"
-	 "	sll	2,24		\n"
-	 "	srl	2,24		\n"
-	 "	lgr	%2,2		\n"
-	 "2:				\n"
-	 ".section .fixup,\"ax\"	\n"
-	 "3:				\n"
-	 "	lhi	%0,%h5		\n"
-	 "	jg	2b		\n"
-	 ".previous			\n"
-	 ".section __ex_table,\"a\"	\n"
-	 "	.align	8		\n"
-	 "	.quad	0b,3b		\n"
-	 "	.quad	1b,3b		\n"
-	 ".previous"
-	 :"=d" (ccode),"=d" (*stat),"=d" (*q_depth), "=d" (*dev_type)
-	 :"d" (q_nr), "K" (DEV_TSQ_EXCEPTION)
-	 :"cc","0","1","2","memory");
-#else
-	("	lr	0,%4		\n"
-	 "	slr	1,1		\n"
-	 "	lr	2,1		\n"
-	 "0:	.long	0xb2af0000	\n"
-	 "1:	ipm	%0		\n"
-	 "	srl	%0,28		\n"
-	 "	lr	%1,1		\n"
-	 "	lr	%3,2		\n"
-	 "	srl	%3,24		\n"
-	 "	sll	2,24		\n"
-	 "	srl	2,24		\n"
-	 "	lr	%2,2		\n"
-	 "2:				\n"
-	 ".section .fixup,\"ax\"	\n"
-	 "3:				\n"
-	 "	lhi	%0,%h5		\n"
-	 "	bras	1,4f		\n"
-	 "	.long	2b		\n"
-	 "4:				\n"
-	 "	l	1,0(1)		\n"
-	 "	br	1		\n"
-	 ".previous			\n"
-	 ".section __ex_table,\"a\"	\n"
-	 "	.align	4		\n"
-	 "	.long	0b,3b		\n"
-	 "	.long	1b,3b		\n"
-	 ".previous"
-	 :"=d" (ccode),"=d" (*stat),"=d" (*q_depth), "=d" (*dev_type)
-	 :"d" (q_nr), "K" (DEV_TSQ_EXCEPTION)
-	 :"cc","0","1","2","memory");
-#endif
-	return ccode;
-}
-
-static inline int
-resetq(int q_nr, struct ap_status_word *stat_p)
-{
-	int ccode;
-
-	asm volatile
-#ifdef CONFIG_64BIT
-	("	llgfr	0,%2		\n"
-	 "	lghi	1,1		\n"
-	 "	sll	1,24		\n"
-	 "	or	0,1		\n"
-	 "	slgr	1,1		\n"
-	 "	lgr	2,1		\n"
-	 "0:	.long	0xb2af0000	\n"
-	 "1:	ipm	%0		\n"
-	 "	srl	%0,28		\n"
-	 "	iihh	%0,0		\n"
-	 "	iihl	%0,0		\n"
-	 "	lgr	%1,1		\n"
-	 "2:				\n"
-	 ".section .fixup,\"ax\"	\n"
-	 "3:				\n"
-	 "	lhi	%0,%h3		\n"
-	 "	jg	2b		\n"
-	 ".previous			\n"
-	 ".section __ex_table,\"a\"	\n"
-	 "	.align	8		\n"
-	 "	.quad	0b,3b		\n"
-	 "	.quad	1b,3b		\n"
-	 ".previous"
-	 :"=d" (ccode),"=d" (*stat_p)
-	 :"d" (q_nr), "K" (DEV_RSQ_EXCEPTION)
-	 :"cc","0","1","2","memory");
-#else
-	("	lr	0,%2		\n"
-	 "	lhi	1,1		\n"
-	 "	sll	1,24		\n"
-	 "	or	0,1		\n"
-	 "	slr	1,1		\n"
-	 "	lr	2,1		\n"
-	 "0:	.long	0xb2af0000	\n"
-	 "1:	ipm	%0		\n"
-	 "	srl	%0,28		\n"
-	 "	lr	%1,1		\n"
-	 "2:				\n"
-	 ".section .fixup,\"ax\"	\n"
-	 "3:				\n"
-	 "	lhi	%0,%h3		\n"
-	 "	bras	1,4f		\n"
-	 "	.long	2b		\n"
-	 "4:				\n"
-	 "	l	1,0(1)		\n"
-	 "	br	1		\n"
-	 ".previous			\n"
-	 ".section __ex_table,\"a\"	\n"
-	 "	.align	4		\n"
-	 "	.long	0b,3b		\n"
-	 "	.long	1b,3b		\n"
-	 ".previous"
-	 :"=d" (ccode),"=d" (*stat_p)
-	 :"d" (q_nr), "K" (DEV_RSQ_EXCEPTION)
-	 :"cc","0","1","2","memory");
-#endif
-	return ccode;
-}
-
-static inline int
-sen(int msg_len, unsigned char *msg_ext, struct ap_status_word *stat)
-{
-	int ccode;
-
-	asm volatile
-#ifdef CONFIG_64BIT
-	("	lgr	6,%3		\n"
-	 "	llgfr	7,%2		\n"
-	 "	llgt	0,0(6)		\n"
-	 "	lghi	1,64		\n"
-	 "	sll	1,24		\n"
-	 "	or	0,1		\n"
-	 "	la	6,4(6)		\n"
-	 "	llgt	2,0(6)		\n"
-	 "	llgt	3,4(6)		\n"
-	 "	la	6,8(6)		\n"
-	 "	slr	1,1		\n"
-	 "0:	.long	0xb2ad0026	\n"
-	 "1:	brc	2,0b		\n"
-	 "	ipm	%0		\n"
-	 "	srl	%0,28		\n"
-	 "	iihh	%0,0		\n"
-	 "	iihl	%0,0		\n"
-	 "	lgr	%1,1		\n"
-	 "2:				\n"
-	 ".section .fixup,\"ax\"	\n"
-	 "3:				\n"
-	 "	lhi	%0,%h4		\n"
-	 "	jg	2b		\n"
-	 ".previous			\n"
-	 ".section __ex_table,\"a\"	\n"
-	 "	.align	8		\n"
-	 "	.quad	0b,3b		\n"
-	 "	.quad	1b,3b		\n"
-	 ".previous"
-	 :"=d" (ccode),"=d" (*stat)
-	 :"d" (msg_len),"a" (msg_ext), "K" (DEV_SEN_EXCEPTION)
-	 :"cc","0","1","2","3","6","7","memory");
-#else
-	("	lr	6,%3		\n"
-	 "	lr	7,%2		\n"
-	 "	l	0,0(6)		\n"
-	 "	lhi	1,64		\n"
-	 "	sll	1,24		\n"
-	 "	or	0,1		\n"
-	 "	la	6,4(6)		\n"
-	 "	l	2,0(6)		\n"
-	 "	l	3,4(6)		\n"
-	 "	la	6,8(6)		\n"
-	 "	slr	1,1		\n"
-	 "0:	.long	0xb2ad0026	\n"
-	 "1:	brc	2,0b		\n"
-	 "	ipm	%0		\n"
-	 "	srl	%0,28		\n"
-	 "	lr	%1,1		\n"
-	 "2:				\n"
-	 ".section .fixup,\"ax\"	\n"
-	 "3:				\n"
-	 "	lhi	%0,%h4		\n"
-	 "	bras	1,4f		\n"
-	 "	.long	2b		\n"
-	 "4:				\n"
-	 "	l	1,0(1)		\n"
-	 "	br	1		\n"
-	 ".previous			\n"
-	 ".section __ex_table,\"a\"	\n"
-	 "	.align	4		\n"
-	 "	.long	0b,3b		\n"
-	 "	.long	1b,3b		\n"
-	 ".previous"
-	 :"=d" (ccode),"=d" (*stat)
-	 :"d" (msg_len),"a" (msg_ext), "K" (DEV_SEN_EXCEPTION)
-	 :"cc","0","1","2","3","6","7","memory");
-#endif
-	return ccode;
-}
-
-static inline int
-rec(int q_nr, int buff_l, unsigned char *rsp, unsigned char *id,
-    struct ap_status_word *st)
-{
-	int ccode;
-
-	asm volatile
-#ifdef CONFIG_64BIT
-	("	llgfr	0,%2		\n"
-	 "	lgr	3,%4		\n"
-	 "	lgr	6,%3		\n"
-	 "	llgfr	7,%5		\n"
-	 "	lghi	1,128		\n"
-	 "	sll	1,24		\n"
-	 "	or	0,1		\n"
-	 "	slgr	1,1		\n"
-	 "	lgr	2,1		\n"
-	 "	lgr	4,1		\n"
-	 "	lgr	5,1		\n"
-	 "0:	.long	0xb2ae0046	\n"
-	 "1:	brc	2,0b		\n"
-	 "	brc	4,0b		\n"
-	 "	ipm	%0		\n"
-	 "	srl	%0,28		\n"
-	 "	iihh	%0,0		\n"
-	 "	iihl	%0,0		\n"
-	 "	lgr	%1,1		\n"
-	 "	st	4,0(3)		\n"
-	 "	st	5,4(3)		\n"
-	 "2:				\n"
-	 ".section .fixup,\"ax\"	\n"
-	 "3:				\n"
-	 "	lhi   %0,%h6		\n"
-	 "	jg    2b		\n"
-	 ".previous			\n"
-	 ".section __ex_table,\"a\"	\n"
-	 "   .align	8		\n"
-	 "   .quad	0b,3b		\n"
-	 "   .quad	1b,3b		\n"
-	 ".previous"
-	 :"=d"(ccode),"=d"(*st)
-	 :"d" (q_nr), "d" (rsp), "d" (id), "d" (buff_l), "K" (DEV_REC_EXCEPTION)
-	 :"cc","0","1","2","3","4","5","6","7","memory");
-#else
-	("	lr	0,%2		\n"
-	 "	lr	3,%4		\n"
-	 "	lr	6,%3		\n"
-	 "	lr	7,%5		\n"
-	 "	lhi	1,128		\n"
-	 "	sll	1,24		\n"
-	 "	or	0,1		\n"
-	 "	slr	1,1		\n"
-	 "	lr	2,1		\n"
-	 "	lr	4,1		\n"
-	 "	lr	5,1		\n"
-	 "0:	.long	0xb2ae0046	\n"
-	 "1:	brc	2,0b		\n"
-	 "	brc	4,0b		\n"
-	 "	ipm	%0		\n"
-	 "	srl	%0,28		\n"
-	 "	lr	%1,1		\n"
-	 "	st	4,0(3)		\n"
-	 "	st	5,4(3)		\n"
-	 "2:				\n"
-	 ".section .fixup,\"ax\"	\n"
-	 "3:				\n"
-	 "	lhi   %0,%h6		\n"
-	 "	bras  1,4f		\n"
-	 "	.long 2b		\n"
-	 "4:				\n"
-	 "	l     1,0(1)		\n"
-	 "	br    1			\n"
-	 ".previous			\n"
-	 ".section __ex_table,\"a\"	\n"
-	 "   .align	4		\n"
-	 "   .long	0b,3b		\n"
-	 "   .long	1b,3b		\n"
-	 ".previous"
-	 :"=d"(ccode),"=d"(*st)
-	 :"d" (q_nr), "d" (rsp), "d" (id), "d" (buff_l), "K" (DEV_REC_EXCEPTION)
-	 :"cc","0","1","2","3","4","5","6","7","memory");
-#endif
-	return ccode;
-}
-
-static inline void
-itoLe2(int *i_p, unsigned char *lechars)
-{
-	*lechars       = *((unsigned char *) i_p + sizeof(int) - 1);
-	*(lechars + 1) = *((unsigned char *) i_p + sizeof(int) - 2);
-}
-
-static inline void
-le2toI(unsigned char *lechars, int *i_p)
-{
-	unsigned char *ic_p;
-	*i_p = 0;
-	ic_p = (unsigned char *) i_p;
-	*(ic_p + 2) = *(lechars + 1);
-	*(ic_p + 3) = *(lechars);
-}
-
-static inline int
-is_empty(unsigned char *ptr, int len)
-{
-	return !memcmp(ptr, (unsigned char *) &static_pvt_me_key+60, len);
-}
-
-enum hdstat
-query_online(int deviceNr, int cdx, int resetNr, int *q_depth, int *dev_type)
-{
-	int q_nr, i, t_depth, t_dev_type;
-	enum devstat ccode;
-	struct ap_status_word stat_word;
-	enum hdstat stat;
-	int break_out;
-
-	q_nr = (deviceNr << SKIP_BITL) + cdx;
-	stat = HD_BUSY;
-	ccode = testq(q_nr, &t_depth, &t_dev_type, &stat_word);
-	PDEBUG("ccode %d response_code %02X\n", ccode, stat_word.response_code);
-	break_out = 0;
-	for (i = 0; i < resetNr; i++) {
-		if (ccode > 3) {
-			PRINTKC("Exception testing device %d\n", i);
-			return HD_TSQ_EXCEPTION;
-		}
-		switch (ccode) {
-		case 0:
-			PDEBUG("t_dev_type %d\n", t_dev_type);
-			break_out = 1;
-			stat = HD_ONLINE;
-			*q_depth = t_depth + 1;
-			switch (t_dev_type) {
-			case PCICA_HW:
-				*dev_type = PCICA;
-				break;
-			case PCICC_HW:
-				*dev_type = PCICC;
-				break;
-			case PCIXCC_HW:
-				*dev_type = PCIXCC_UNK;
-				break;
-			case CEX2C_HW:
-				*dev_type = CEX2C;
-				break;
-			case CEX2A_HW:
-				*dev_type = CEX2A;
-				break;
-			default:
-				*dev_type = NILDEV;
-				break;
-			}
-			PDEBUG("available device %d: Q depth = %d, dev "
-			       "type = %d, stat = %02X%02X%02X%02X\n",
-			       deviceNr, *q_depth, *dev_type,
-			       stat_word.q_stat_flags,
-			       stat_word.response_code,
-			       stat_word.reserved[0],
-			       stat_word.reserved[1]);
-			break;
-		case 3:
-			switch (stat_word.response_code) {
-			case AP_RESPONSE_NORMAL:
-				stat = HD_ONLINE;
-				break_out = 1;
-				*q_depth = t_depth + 1;
-				*dev_type = t_dev_type;
-				PDEBUG("cc3, available device "
-				       "%d: Q depth = %d, dev "
-				       "type = %d, stat = "
-				       "%02X%02X%02X%02X\n",
-				       deviceNr, *q_depth,
-				       *dev_type,
-				       stat_word.q_stat_flags,
-				       stat_word.response_code,
-				       stat_word.reserved[0],
-				       stat_word.reserved[1]);
-				break;
-			case AP_RESPONSE_Q_NOT_AVAIL:
-				stat = HD_NOT_THERE;
-				break_out = 1;
-				break;
-			case AP_RESPONSE_RESET_IN_PROGRESS:
-				PDEBUG("device %d in reset\n",
-				       deviceNr);
-				break;
-			case AP_RESPONSE_DECONFIGURED:
-				stat = HD_DECONFIGURED;
-				break_out = 1;
-				break;
-			case AP_RESPONSE_CHECKSTOPPED:
-				stat = HD_CHECKSTOPPED;
-				break_out = 1;
-				break;
-			case AP_RESPONSE_BUSY:
-				PDEBUG("device %d busy\n",
-				       deviceNr);
-				break;
-			default:
-				break;
-			}
-			break;
-		default:
-			stat = HD_NOT_THERE;
-			break_out = 1;
-			break;
-		}
-		if (break_out)
-			break;
-
-		udelay(5);
-
-		ccode = testq(q_nr, &t_depth, &t_dev_type, &stat_word);
-	}
-	return stat;
-}
-
-enum devstat
-reset_device(int deviceNr, int cdx, int resetNr)
-{
-	int q_nr, ccode = 0, dummy_qdepth, dummy_devType, i;
-	struct ap_status_word stat_word;
-	enum devstat stat;
-	int break_out;
-
-	q_nr = (deviceNr << SKIP_BITL) + cdx;
-	stat = DEV_GONE;
-	ccode = resetq(q_nr, &stat_word);
-	if (ccode > 3)
-		return DEV_RSQ_EXCEPTION;
-
-	break_out = 0;
-	for (i = 0; i < resetNr; i++) {
-		switch (ccode) {
-		case 0:
-			stat = DEV_ONLINE;
-			if (stat_word.q_stat_flags & AP_Q_STATUS_EMPTY)
-				break_out = 1;
-			break;
-		case 3:
-			switch (stat_word.response_code) {
-			case AP_RESPONSE_NORMAL:
-				stat = DEV_ONLINE;
-				if (stat_word.q_stat_flags & AP_Q_STATUS_EMPTY)
-					break_out = 1;
-				break;
-			case AP_RESPONSE_Q_NOT_AVAIL:
-			case AP_RESPONSE_DECONFIGURED:
-			case AP_RESPONSE_CHECKSTOPPED:
-				stat = DEV_GONE;
-				break_out = 1;
-				break;
-			case AP_RESPONSE_RESET_IN_PROGRESS:
-			case AP_RESPONSE_BUSY:
-			default:
-				break;
-			}
-			break;
-		default:
-			stat = DEV_GONE;
-			break_out = 1;
-			break;
-		}
-		if (break_out == 1)
-			break;
-		udelay(5);
-
-		ccode = testq(q_nr, &dummy_qdepth, &dummy_devType, &stat_word);
-		if (ccode > 3) {
-			stat = DEV_TSQ_EXCEPTION;
-			break;
-		}
-	}
-	PDEBUG("Number of testq's needed for reset: %d\n", i);
-
-	if (i >= resetNr) {
-	  stat = DEV_GONE;
-	}
-
-	return stat;
-}
-
-#ifdef DEBUG_HYDRA_MSGS
-static inline void
-print_buffer(unsigned char *buffer, int bufflen)
-{
-	int i;
-	for (i = 0; i < bufflen; i += 16) {
-		PRINTK("%04X: %02X%02X%02X%02X %02X%02X%02X%02X "
-		       "%02X%02X%02X%02X %02X%02X%02X%02X\n", i,
-		       buffer[i+0], buffer[i+1], buffer[i+2], buffer[i+3],
-		       buffer[i+4], buffer[i+5], buffer[i+6], buffer[i+7],
-		       buffer[i+8], buffer[i+9], buffer[i+10], buffer[i+11],
-		       buffer[i+12], buffer[i+13], buffer[i+14], buffer[i+15]);
-	}
-}
-#endif
-
-enum devstat
-send_to_AP(int dev_nr, int cdx, int msg_len, unsigned char *msg_ext)
-{
-	struct ap_status_word stat_word;
-	enum devstat stat;
-	int ccode;
-	u32 *q_nr_p = (u32 *)msg_ext;
-
-	*q_nr_p = (dev_nr << SKIP_BITL) + cdx;
-	PDEBUG("msg_len passed to sen: %d\n", msg_len);
-	PDEBUG("q number passed to sen: %02x%02x%02x%02x\n",
-	       msg_ext[0], msg_ext[1], msg_ext[2], msg_ext[3]);
-	stat = DEV_GONE;
-
-#ifdef DEBUG_HYDRA_MSGS
-	PRINTK("Request header: %02X%02X%02X%02X %02X%02X%02X%02X "
-	       "%02X%02X%02X%02X\n",
-	       msg_ext[0], msg_ext[1], msg_ext[2], msg_ext[3],
-	       msg_ext[4], msg_ext[5], msg_ext[6], msg_ext[7],
-	       msg_ext[8], msg_ext[9], msg_ext[10], msg_ext[11]);
-	print_buffer(msg_ext+CALLER_HEADER, msg_len);
-#endif
-
-	ccode = sen(msg_len, msg_ext, &stat_word);
-	if (ccode > 3)
-		return DEV_SEN_EXCEPTION;
-
-	PDEBUG("nq cc: %u, st: %02x%02x%02x%02x\n",
-	       ccode, stat_word.q_stat_flags, stat_word.response_code,
-	       stat_word.reserved[0], stat_word.reserved[1]);
-	switch (ccode) {
-	case 0:
-		stat = DEV_ONLINE;
-		break;
-	case 1:
-		stat = DEV_GONE;
-		break;
-	case 3:
-		switch (stat_word.response_code) {
-		case AP_RESPONSE_NORMAL:
-			stat = DEV_ONLINE;
-			break;
-		case AP_RESPONSE_Q_FULL:
-			stat = DEV_QUEUE_FULL;
-			break;
-		default:
-			stat = DEV_GONE;
-			break;
-		}
-		break;
-	default:
-		stat = DEV_GONE;
-		break;
-	}
-
-	return stat;
-}
-
-enum devstat
-receive_from_AP(int dev_nr, int cdx, int resplen, unsigned char *resp,
-		unsigned char *psmid)
-{
-	int ccode;
-	struct ap_status_word stat_word;
-	enum devstat stat;
-
-	memset(resp, 0x00, 8);
-
-	ccode = rec((dev_nr << SKIP_BITL) + cdx, resplen, resp, psmid,
-		    &stat_word);
-	if (ccode > 3)
-		return DEV_REC_EXCEPTION;
-
-	PDEBUG("dq cc: %u, st: %02x%02x%02x%02x\n",
-	       ccode, stat_word.q_stat_flags, stat_word.response_code,
-	       stat_word.reserved[0], stat_word.reserved[1]);
-
-	stat = DEV_GONE;
-	switch (ccode) {
-	case 0:
-		stat = DEV_ONLINE;
-#ifdef DEBUG_HYDRA_MSGS
-		print_buffer(resp, resplen);
-#endif
-		break;
-	case 3:
-		switch (stat_word.response_code) {
-		case AP_RESPONSE_NORMAL:
-			stat = DEV_ONLINE;
-			break;
-		case AP_RESPONSE_NO_PENDING_REPLY:
-			if (stat_word.q_stat_flags & AP_Q_STATUS_EMPTY)
-				stat = DEV_EMPTY;
-			else
-				stat = DEV_NO_WORK;
-			break;
-		case AP_RESPONSE_INDEX_TOO_BIG:
-		case AP_RESPONSE_NO_FIRST_PART:
-		case AP_RESPONSE_MESSAGE_TOO_BIG:
-			stat = DEV_BAD_MESSAGE;
-			break;
-		default:
-			break;
-		}
-		break;
-	default:
-		break;
-	}
-
-	return stat;
-}
-
-static inline int
-pad_msg(unsigned char *buffer, int  totalLength, int msgLength)
-{
-	int pad_len;
-
-	for (pad_len = 0; pad_len < (totalLength - msgLength); pad_len++)
-		if (buffer[pad_len] != 0x00)
-			break;
-	pad_len -= 3;
-	if (pad_len < 8)
-		return SEN_PAD_ERROR;
-
-	buffer[0] = 0x00;
-	buffer[1] = 0x02;
-
-	memcpy(buffer+2, static_pad, pad_len);
-
-	buffer[pad_len + 2] = 0x00;
-
-	return 0;
-}
-
-static inline int
-is_common_public_key(unsigned char *key, int len)
-{
-	int i;
-
-	for (i = 0; i < len; i++)
-		if (key[i])
-			break;
-	key += i;
-	len -= i;
-	if (((len == 1) && (key[0] == 3)) ||
-	    ((len == 3) && (key[0] == 1) && (key[1] == 0) && (key[2] == 1)))
-		return 1;
-
-	return 0;
-}
-
-static int
-ICAMEX_msg_to_type4MEX_msg(struct ica_rsa_modexpo *icaMex_p, int *z90cMsg_l_p,
-			   union type4_msg *z90cMsg_p)
-{
-	int mod_len, msg_size, mod_tgt_len, exp_tgt_len, inp_tgt_len;
-	unsigned char *mod_tgt, *exp_tgt, *inp_tgt;
-	union type4_msg *tmp_type4_msg;
-
-	mod_len = icaMex_p->inputdatalength;
-
-	msg_size = ((mod_len <= 128) ? TYPE4_SME_LEN : TYPE4_LME_LEN) +
-		    CALLER_HEADER;
-
-	memset(z90cMsg_p, 0, msg_size);
-
-	tmp_type4_msg = (union type4_msg *)
-		((unsigned char *) z90cMsg_p + CALLER_HEADER);
-
-	tmp_type4_msg->sme.header.msg_type_code = TYPE4_TYPE_CODE;
-	tmp_type4_msg->sme.header.request_code = TYPE4_REQU_CODE;
-
-	if (mod_len <= 128) {
-		tmp_type4_msg->sme.header.msg_fmt = TYPE4_SME_FMT;
-		tmp_type4_msg->sme.header.msg_len = TYPE4_SME_LEN;
-		mod_tgt = tmp_type4_msg->sme.modulus;
-		mod_tgt_len = sizeof(tmp_type4_msg->sme.modulus);
-		exp_tgt = tmp_type4_msg->sme.exponent;
-		exp_tgt_len = sizeof(tmp_type4_msg->sme.exponent);
-		inp_tgt = tmp_type4_msg->sme.message;
-		inp_tgt_len = sizeof(tmp_type4_msg->sme.message);
-	} else {
-		tmp_type4_msg->lme.header.msg_fmt = TYPE4_LME_FMT;
-		tmp_type4_msg->lme.header.msg_len = TYPE4_LME_LEN;
-		mod_tgt = tmp_type4_msg->lme.modulus;
-		mod_tgt_len = sizeof(tmp_type4_msg->lme.modulus);
-		exp_tgt = tmp_type4_msg->lme.exponent;
-		exp_tgt_len = sizeof(tmp_type4_msg->lme.exponent);
-		inp_tgt = tmp_type4_msg->lme.message;
-		inp_tgt_len = sizeof(tmp_type4_msg->lme.message);
-	}
-
-	mod_tgt += (mod_tgt_len - mod_len);
-	if (copy_from_user(mod_tgt, icaMex_p->n_modulus, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(mod_tgt, mod_len))
-		return SEN_USER_ERROR;
-	exp_tgt += (exp_tgt_len - mod_len);
-	if (copy_from_user(exp_tgt, icaMex_p->b_key, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(exp_tgt, mod_len))
-		return SEN_USER_ERROR;
-	inp_tgt += (inp_tgt_len - mod_len);
-	if (copy_from_user(inp_tgt, icaMex_p->inputdata, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(inp_tgt, mod_len))
-		return SEN_USER_ERROR;
-
-	*z90cMsg_l_p = msg_size - CALLER_HEADER;
-
-	return 0;
-}
-
-static int
-ICACRT_msg_to_type4CRT_msg(struct ica_rsa_modexpo_crt *icaMsg_p,
-			   int *z90cMsg_l_p, union type4_msg *z90cMsg_p)
-{
-	int mod_len, short_len, long_len, tmp_size, p_tgt_len, q_tgt_len,
-	    dp_tgt_len, dq_tgt_len, u_tgt_len, inp_tgt_len;
-	unsigned char *p_tgt, *q_tgt, *dp_tgt, *dq_tgt, *u_tgt, *inp_tgt;
-	union type4_msg *tmp_type4_msg;
-
-	mod_len = icaMsg_p->inputdatalength;
-	short_len = mod_len / 2;
-	long_len = mod_len / 2 + 8;
-
-	tmp_size = ((mod_len <= 128) ? TYPE4_SCR_LEN : TYPE4_LCR_LEN) +
-		    CALLER_HEADER;
-
-	memset(z90cMsg_p, 0, tmp_size);
-
-	tmp_type4_msg = (union type4_msg *)
-		((unsigned char *) z90cMsg_p + CALLER_HEADER);
-
-	tmp_type4_msg->scr.header.msg_type_code = TYPE4_TYPE_CODE;
-	tmp_type4_msg->scr.header.request_code = TYPE4_REQU_CODE;
-	if (mod_len <= 128) {
-		tmp_type4_msg->scr.header.msg_fmt = TYPE4_SCR_FMT;
-		tmp_type4_msg->scr.header.msg_len = TYPE4_SCR_LEN;
-		p_tgt = tmp_type4_msg->scr.p;
-		p_tgt_len = sizeof(tmp_type4_msg->scr.p);
-		q_tgt = tmp_type4_msg->scr.q;
-		q_tgt_len = sizeof(tmp_type4_msg->scr.q);
-		dp_tgt = tmp_type4_msg->scr.dp;
-		dp_tgt_len = sizeof(tmp_type4_msg->scr.dp);
-		dq_tgt = tmp_type4_msg->scr.dq;
-		dq_tgt_len = sizeof(tmp_type4_msg->scr.dq);
-		u_tgt = tmp_type4_msg->scr.u;
-		u_tgt_len = sizeof(tmp_type4_msg->scr.u);
-		inp_tgt = tmp_type4_msg->scr.message;
-		inp_tgt_len = sizeof(tmp_type4_msg->scr.message);
-	} else {
-		tmp_type4_msg->lcr.header.msg_fmt = TYPE4_LCR_FMT;
-		tmp_type4_msg->lcr.header.msg_len = TYPE4_LCR_LEN;
-		p_tgt = tmp_type4_msg->lcr.p;
-		p_tgt_len = sizeof(tmp_type4_msg->lcr.p);
-		q_tgt = tmp_type4_msg->lcr.q;
-		q_tgt_len = sizeof(tmp_type4_msg->lcr.q);
-		dp_tgt = tmp_type4_msg->lcr.dp;
-		dp_tgt_len = sizeof(tmp_type4_msg->lcr.dp);
-		dq_tgt = tmp_type4_msg->lcr.dq;
-		dq_tgt_len = sizeof(tmp_type4_msg->lcr.dq);
-		u_tgt = tmp_type4_msg->lcr.u;
-		u_tgt_len = sizeof(tmp_type4_msg->lcr.u);
-		inp_tgt = tmp_type4_msg->lcr.message;
-		inp_tgt_len = sizeof(tmp_type4_msg->lcr.message);
-	}
-
-	p_tgt += (p_tgt_len - long_len);
-	if (copy_from_user(p_tgt, icaMsg_p->np_prime, long_len))
-		return SEN_RELEASED;
-	if (is_empty(p_tgt, long_len))
-		return SEN_USER_ERROR;
-	q_tgt += (q_tgt_len - short_len);
-	if (copy_from_user(q_tgt, icaMsg_p->nq_prime, short_len))
-		return SEN_RELEASED;
-	if (is_empty(q_tgt, short_len))
-		return SEN_USER_ERROR;
-	dp_tgt += (dp_tgt_len - long_len);
-	if (copy_from_user(dp_tgt, icaMsg_p->bp_key, long_len))
-		return SEN_RELEASED;
-	if (is_empty(dp_tgt, long_len))
-		return SEN_USER_ERROR;
-	dq_tgt += (dq_tgt_len - short_len);
-	if (copy_from_user(dq_tgt, icaMsg_p->bq_key, short_len))
-		return SEN_RELEASED;
-	if (is_empty(dq_tgt, short_len))
-		return SEN_USER_ERROR;
-	u_tgt += (u_tgt_len - long_len);
-	if (copy_from_user(u_tgt, icaMsg_p->u_mult_inv, long_len))
-		return SEN_RELEASED;
-	if (is_empty(u_tgt, long_len))
-		return SEN_USER_ERROR;
-	inp_tgt += (inp_tgt_len - mod_len);
-	if (copy_from_user(inp_tgt, icaMsg_p->inputdata, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(inp_tgt, mod_len))
-		return SEN_USER_ERROR;
-
-	*z90cMsg_l_p = tmp_size - CALLER_HEADER;
-
-	return 0;
-}
-
-static int
-ICAMEX_msg_to_type6MEX_de_msg(struct ica_rsa_modexpo *icaMsg_p, int cdx,
-			      int *z90cMsg_l_p, struct type6_msg *z90cMsg_p)
-{
-	int mod_len, vud_len, tmp_size, total_CPRB_len, parmBlock_l;
-	unsigned char *temp;
-	struct type6_hdr *tp6Hdr_p;
-	struct CPRB *cprb_p;
-	struct cca_private_ext_ME *key_p;
-	static int deprecated_msg_count = 0;
-
-	mod_len = icaMsg_p->inputdatalength;
-	tmp_size = FIXED_TYPE6_ME_LEN + mod_len;
-	total_CPRB_len = tmp_size - sizeof(struct type6_hdr);
-	parmBlock_l = total_CPRB_len - sizeof(struct CPRB);
-	tmp_size = 4*((tmp_size + 3)/4) + CALLER_HEADER;
-
-	memset(z90cMsg_p, 0, tmp_size);
-
-	temp = (unsigned char *)z90cMsg_p + CALLER_HEADER;
-	memcpy(temp, &static_type6_hdr, sizeof(struct type6_hdr));
-	tp6Hdr_p = (struct type6_hdr *)temp;
-	tp6Hdr_p->ToCardLen1 = 4*((total_CPRB_len+3)/4);
-	tp6Hdr_p->FromCardLen1 = RESPONSE_CPRB_SIZE;
-
-	temp += sizeof(struct type6_hdr);
-	memcpy(temp, &static_cprb, sizeof(struct CPRB));
-	cprb_p = (struct CPRB *) temp;
-	cprb_p->usage_domain[0]= (unsigned char)cdx;
-	itoLe2(&parmBlock_l, cprb_p->req_parml);
-	itoLe2((int *)&(tp6Hdr_p->FromCardLen1), cprb_p->rpl_parml);
-
-	temp += sizeof(struct CPRB);
-	memcpy(temp, &static_pkd_function_and_rules,
-	       sizeof(struct function_and_rules_block));
-
-	temp += sizeof(struct function_and_rules_block);
-	vud_len = 2 + icaMsg_p->inputdatalength;
-	itoLe2(&vud_len, temp);
-
-	temp += 2;
-	if (copy_from_user(temp, icaMsg_p->inputdata, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(temp, mod_len))
-		return SEN_USER_ERROR;
-
-	temp += mod_len;
-	memcpy(temp, &static_T6_keyBlock_hdr, sizeof(struct T6_keyBlock_hdr));
-
-	temp += sizeof(struct T6_keyBlock_hdr);
-	memcpy(temp, &static_pvt_me_key, sizeof(struct cca_private_ext_ME));
-	key_p = (struct cca_private_ext_ME *)temp;
-	temp = key_p->pvtMESec.exponent + sizeof(key_p->pvtMESec.exponent)
-	       - mod_len;
-	if (copy_from_user(temp, icaMsg_p->b_key, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(temp, mod_len))
-		return SEN_USER_ERROR;
-
-	if (is_common_public_key(temp, mod_len)) {
-		if (deprecated_msg_count < 20) {
-			PRINTK("Common public key used for modex decrypt\n");
-			deprecated_msg_count++;
-			if (deprecated_msg_count == 20)
-				PRINTK("No longer issuing messages about common"
-				       " public key for modex decrypt.\n");
-		}
-		return SEN_NOT_AVAIL;
-	}
-
-	temp = key_p->pvtMESec.modulus + sizeof(key_p->pvtMESec.modulus)
-	       - mod_len;
-	if (copy_from_user(temp, icaMsg_p->n_modulus, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(temp, mod_len))
-		return SEN_USER_ERROR;
-
-	key_p->pubMESec.modulus_bit_len = 8 * mod_len;
-
-	*z90cMsg_l_p = tmp_size - CALLER_HEADER;
-
-	return 0;
-}
-
-static int
-ICAMEX_msg_to_type6MEX_en_msg(struct ica_rsa_modexpo *icaMsg_p, int cdx,
-			      int *z90cMsg_l_p, struct type6_msg *z90cMsg_p)
-{
-	int mod_len, vud_len, exp_len, key_len;
-	int pad_len, tmp_size, total_CPRB_len, parmBlock_l, i;
-	unsigned char *temp_exp, *exp_p, *temp;
-	struct type6_hdr *tp6Hdr_p;
-	struct CPRB *cprb_p;
-	struct cca_public_key *key_p;
-	struct T6_keyBlock_hdr *keyb_p;
-
-	temp_exp = kmalloc(256, GFP_KERNEL);
-	if (!temp_exp)
-		return EGETBUFF;
-	mod_len = icaMsg_p->inputdatalength;
-	if (copy_from_user(temp_exp, icaMsg_p->b_key, mod_len)) {
-		kfree(temp_exp);
-		return SEN_RELEASED;
-	}
-	if (is_empty(temp_exp, mod_len)) {
-		kfree(temp_exp);
-		return SEN_USER_ERROR;
-	}
-
-	exp_p = temp_exp;
-	for (i = 0; i < mod_len; i++)
-		if (exp_p[i])
-			break;
-	if (i >= mod_len) {
-		kfree(temp_exp);
-		return SEN_USER_ERROR;
-	}
-
-	exp_len = mod_len - i;
-	exp_p += i;
-
-	PDEBUG("exp_len after computation: %08x\n", exp_len);
-	tmp_size = FIXED_TYPE6_ME_EN_LEN + 2 * mod_len + exp_len;
-	total_CPRB_len = tmp_size - sizeof(struct type6_hdr);
-	parmBlock_l = total_CPRB_len - sizeof(struct CPRB);
-	tmp_size = 4*((tmp_size + 3)/4) + CALLER_HEADER;
-
-	vud_len = 2 + mod_len;
-	memset(z90cMsg_p, 0, tmp_size);
-
-	temp = (unsigned char *)z90cMsg_p + CALLER_HEADER;
-	memcpy(temp, &static_type6_hdr, sizeof(struct type6_hdr));
-	tp6Hdr_p = (struct type6_hdr *)temp;
-	tp6Hdr_p->ToCardLen1 = 4*((total_CPRB_len+3)/4);
-	tp6Hdr_p->FromCardLen1 = RESPONSE_CPRB_SIZE;
-	memcpy(tp6Hdr_p->function_code, static_PKE_function_code,
-	       sizeof(static_PKE_function_code));
-	temp += sizeof(struct type6_hdr);
-	memcpy(temp, &static_cprb, sizeof(struct CPRB));
-	cprb_p = (struct CPRB *) temp;
-	cprb_p->usage_domain[0]= (unsigned char)cdx;
-	itoLe2((int *)&(tp6Hdr_p->FromCardLen1), cprb_p->rpl_parml);
-	temp += sizeof(struct CPRB);
-	memcpy(temp, &static_pke_function_and_rules,
-		 sizeof(struct function_and_rules_block));
-	temp += sizeof(struct function_and_rules_block);
-	temp += 2;
-	if (copy_from_user(temp, icaMsg_p->inputdata, mod_len)) {
-		kfree(temp_exp);
-		return SEN_RELEASED;
-	}
-	if (is_empty(temp, mod_len)) {
-		kfree(temp_exp);
-		return SEN_USER_ERROR;
-	}
-	if ((temp[0] != 0x00) || (temp[1] != 0x02)) {
-		kfree(temp_exp);
-		return SEN_NOT_AVAIL;
-	}
-	for (i = 2; i < mod_len; i++)
-		if (temp[i] == 0x00)
-			break;
-	if ((i < 9) || (i > (mod_len - 2))) {
-		kfree(temp_exp);
-		return SEN_NOT_AVAIL;
-	}
-	pad_len = i + 1;
-	vud_len = mod_len - pad_len;
-	memmove(temp, temp+pad_len, vud_len);
-	temp -= 2;
-	vud_len += 2;
-	itoLe2(&vud_len, temp);
-	temp += (vud_len);
-	keyb_p = (struct T6_keyBlock_hdr *)temp;
-	temp += sizeof(struct T6_keyBlock_hdr);
-	memcpy(temp, &static_public_key, sizeof(static_public_key));
-	key_p = (struct cca_public_key *)temp;
-	temp = key_p->pubSec.exponent;
-	memcpy(temp, exp_p, exp_len);
-	kfree(temp_exp);
-	temp += exp_len;
-	if (copy_from_user(temp, icaMsg_p->n_modulus, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(temp, mod_len))
-		return SEN_USER_ERROR;
-	key_p->pubSec.modulus_bit_len = 8 * mod_len;
-	key_p->pubSec.modulus_byte_len = mod_len;
-	key_p->pubSec.exponent_len = exp_len;
-	key_p->pubSec.section_length = CALLER_HEADER + mod_len + exp_len;
-	key_len = key_p->pubSec.section_length + sizeof(struct cca_token_hdr);
-	key_p->pubHdr.token_length = key_len;
-	key_len += 4;
-	itoLe2(&key_len, keyb_p->ulen);
-	key_len += 2;
-	itoLe2(&key_len, keyb_p->blen);
-	parmBlock_l -= pad_len;
-	itoLe2(&parmBlock_l, cprb_p->req_parml);
-	*z90cMsg_l_p = tmp_size - CALLER_HEADER;
-
-	return 0;
-}
-
-static int
-ICACRT_msg_to_type6CRT_msg(struct ica_rsa_modexpo_crt *icaMsg_p, int cdx,
-			   int *z90cMsg_l_p, struct type6_msg *z90cMsg_p)
-{
-	int mod_len, vud_len, tmp_size, total_CPRB_len, parmBlock_l, short_len;
-	int long_len, pad_len, keyPartsLen, tmp_l;
-	unsigned char *tgt_p, *temp;
-	struct type6_hdr *tp6Hdr_p;
-	struct CPRB *cprb_p;
-	struct cca_token_hdr *keyHdr_p;
-	struct cca_pvt_ext_CRT_sec *pvtSec_p;
-	struct cca_public_sec *pubSec_p;
-
-	mod_len = icaMsg_p->inputdatalength;
-	short_len = mod_len / 2;
-	long_len = 8 + short_len;
-	keyPartsLen = 3 * long_len + 2 * short_len;
-	pad_len = (8 - (keyPartsLen % 8)) % 8;
-	keyPartsLen += pad_len + mod_len;
-	tmp_size = FIXED_TYPE6_CR_LEN + keyPartsLen + mod_len;
-	total_CPRB_len = tmp_size -  sizeof(struct type6_hdr);
-	parmBlock_l = total_CPRB_len - sizeof(struct CPRB);
-	vud_len = 2 + mod_len;
-	tmp_size = 4*((tmp_size + 3)/4) + CALLER_HEADER;
-
-	memset(z90cMsg_p, 0, tmp_size);
-	tgt_p = (unsigned char *)z90cMsg_p + CALLER_HEADER;
-	memcpy(tgt_p, &static_type6_hdr, sizeof(struct type6_hdr));
-	tp6Hdr_p = (struct type6_hdr *)tgt_p;
-	tp6Hdr_p->ToCardLen1 = 4*((total_CPRB_len+3)/4);
-	tp6Hdr_p->FromCardLen1 = RESPONSE_CPRB_SIZE;
-	tgt_p += sizeof(struct type6_hdr);
-	cprb_p = (struct CPRB *) tgt_p;
-	memcpy(tgt_p, &static_cprb, sizeof(struct CPRB));
-	cprb_p->usage_domain[0]= *((unsigned char *)(&(cdx))+3);
-	itoLe2(&parmBlock_l, cprb_p->req_parml);
-	memcpy(cprb_p->rpl_parml, cprb_p->req_parml,
-	       sizeof(cprb_p->req_parml));
-	tgt_p += sizeof(struct CPRB);
-	memcpy(tgt_p, &static_pkd_function_and_rules,
-	       sizeof(struct function_and_rules_block));
-	tgt_p += sizeof(struct function_and_rules_block);
-	itoLe2(&vud_len, tgt_p);
-	tgt_p += 2;
-	if (copy_from_user(tgt_p, icaMsg_p->inputdata, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, mod_len))
-		return SEN_USER_ERROR;
-	tgt_p += mod_len;
-	tmp_l = sizeof(struct T6_keyBlock_hdr) + sizeof(struct cca_token_hdr) +
-		sizeof(struct cca_pvt_ext_CRT_sec) + 0x0F + keyPartsLen;
-	itoLe2(&tmp_l, tgt_p);
-	temp = tgt_p + 2;
-	tmp_l -= 2;
-	itoLe2(&tmp_l, temp);
-	tgt_p += sizeof(struct T6_keyBlock_hdr);
-	keyHdr_p = (struct cca_token_hdr *)tgt_p;
-	keyHdr_p->token_identifier = CCA_TKN_HDR_ID_EXT;
-	tmp_l -= 4;
-	keyHdr_p->token_length = tmp_l;
-	tgt_p += sizeof(struct cca_token_hdr);
-	pvtSec_p = (struct cca_pvt_ext_CRT_sec *)tgt_p;
-	pvtSec_p->section_identifier = CCA_PVT_EXT_CRT_SEC_ID_PVT;
-	pvtSec_p->section_length =
-		sizeof(struct cca_pvt_ext_CRT_sec) + keyPartsLen;
-	pvtSec_p->key_format = CCA_PVT_EXT_CRT_SEC_FMT_CL;
-	pvtSec_p->key_use_flags[0] = CCA_PVT_USAGE_ALL;
-	pvtSec_p->p_len = long_len;
-	pvtSec_p->q_len = short_len;
-	pvtSec_p->dp_len = long_len;
-	pvtSec_p->dq_len = short_len;
-	pvtSec_p->u_len = long_len;
-	pvtSec_p->mod_len = mod_len;
-	pvtSec_p->pad_len = pad_len;
-	tgt_p += sizeof(struct cca_pvt_ext_CRT_sec);
-	if (copy_from_user(tgt_p, icaMsg_p->np_prime, long_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, long_len))
-		return SEN_USER_ERROR;
-	tgt_p += long_len;
-	if (copy_from_user(tgt_p, icaMsg_p->nq_prime, short_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, short_len))
-		return SEN_USER_ERROR;
-	tgt_p += short_len;
-	if (copy_from_user(tgt_p, icaMsg_p->bp_key, long_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, long_len))
-		return SEN_USER_ERROR;
-	tgt_p += long_len;
-	if (copy_from_user(tgt_p, icaMsg_p->bq_key, short_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, short_len))
-		return SEN_USER_ERROR;
-	tgt_p += short_len;
-	if (copy_from_user(tgt_p, icaMsg_p->u_mult_inv, long_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, long_len))
-		return SEN_USER_ERROR;
-	tgt_p += long_len;
-	tgt_p += pad_len;
-	memset(tgt_p, 0xFF, mod_len);
-	tgt_p += mod_len;
-	memcpy(tgt_p, &static_cca_pub_sec, sizeof(struct cca_public_sec));
-	pubSec_p = (struct cca_public_sec *) tgt_p;
-	pubSec_p->modulus_bit_len = 8 * mod_len;
-	*z90cMsg_l_p = tmp_size - CALLER_HEADER;
-
-	return 0;
-}
-
-static int
-ICAMEX_msg_to_type6MEX_msgX(struct ica_rsa_modexpo *icaMsg_p, int cdx,
-			    int *z90cMsg_l_p, struct type6_msg *z90cMsg_p,
-			    int dev_type)
-{
-	int mod_len, exp_len, vud_len, tmp_size, total_CPRB_len, parmBlock_l;
-	int key_len, i;
-	unsigned char *temp_exp, *tgt_p, *temp, *exp_p;
-	struct type6_hdr *tp6Hdr_p;
-	struct CPRBX *cprbx_p;
-	struct cca_public_key *key_p;
-	struct T6_keyBlock_hdrX *keyb_p;
-
-	temp_exp = kmalloc(256, GFP_KERNEL);
-	if (!temp_exp)
-		return EGETBUFF;
-	mod_len = icaMsg_p->inputdatalength;
-	if (copy_from_user(temp_exp, icaMsg_p->b_key, mod_len)) {
-		kfree(temp_exp);
-		return SEN_RELEASED;
-	}
-	if (is_empty(temp_exp, mod_len)) {
-		kfree(temp_exp);
-		return SEN_USER_ERROR;
-	}
-	exp_p = temp_exp;
-	for (i = 0; i < mod_len; i++)
-		if (exp_p[i])
-			break;
-	if (i >= mod_len) {
-		kfree(temp_exp);
-		return SEN_USER_ERROR;
-	}
-	exp_len = mod_len - i;
-	exp_p += i;
-	PDEBUG("exp_len after computation: %08x\n", exp_len);
-	tmp_size = FIXED_TYPE6_ME_EN_LENX + 2 * mod_len + exp_len;
-	total_CPRB_len = tmp_size - sizeof(struct type6_hdr);
-	parmBlock_l = total_CPRB_len - sizeof(struct CPRBX);
-	tmp_size = tmp_size + CALLER_HEADER;
-	vud_len = 2 + mod_len;
-	memset(z90cMsg_p, 0, tmp_size);
-	tgt_p = (unsigned char *)z90cMsg_p + CALLER_HEADER;
-	memcpy(tgt_p, &static_type6_hdrX, sizeof(struct type6_hdr));
-	tp6Hdr_p = (struct type6_hdr *)tgt_p;
-	tp6Hdr_p->ToCardLen1 = total_CPRB_len;
-	tp6Hdr_p->FromCardLen1 = RESPONSE_CPRBX_SIZE;
-	memcpy(tp6Hdr_p->function_code, static_PKE_function_code,
-	       sizeof(static_PKE_function_code));
-	tgt_p += sizeof(struct type6_hdr);
-	memcpy(tgt_p, &static_cprbx, sizeof(struct CPRBX));
-	cprbx_p = (struct CPRBX *) tgt_p;
-	cprbx_p->domain = (unsigned short)cdx;
-	cprbx_p->rpl_msgbl = RESPONSE_CPRBX_SIZE;
-	tgt_p += sizeof(struct CPRBX);
-	if (dev_type == PCIXCC_MCL2)
-		memcpy(tgt_p, &static_pke_function_and_rulesX_MCL2,
-		       sizeof(struct function_and_rules_block));
-	else
-		memcpy(tgt_p, &static_pke_function_and_rulesX,
-		       sizeof(struct function_and_rules_block));
-	tgt_p += sizeof(struct function_and_rules_block);
-
-	tgt_p += 2;
-	if (copy_from_user(tgt_p, icaMsg_p->inputdata, mod_len)) {
-		kfree(temp_exp);
-		return SEN_RELEASED;
-	}
-	if (is_empty(tgt_p, mod_len)) {
-		kfree(temp_exp);
-		return SEN_USER_ERROR;
-	}
-	tgt_p -= 2;
-	*((short *)tgt_p) = (short) vud_len;
-	tgt_p += vud_len;
-	keyb_p = (struct T6_keyBlock_hdrX *)tgt_p;
-	tgt_p += sizeof(struct T6_keyBlock_hdrX);
-	memcpy(tgt_p, &static_public_key, sizeof(static_public_key));
-	key_p = (struct cca_public_key *)tgt_p;
-	temp = key_p->pubSec.exponent;
-	memcpy(temp, exp_p, exp_len);
-	kfree(temp_exp);
-	temp += exp_len;
-	if (copy_from_user(temp, icaMsg_p->n_modulus, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(temp, mod_len))
-		return SEN_USER_ERROR;
-	key_p->pubSec.modulus_bit_len = 8 * mod_len;
-	key_p->pubSec.modulus_byte_len = mod_len;
-	key_p->pubSec.exponent_len = exp_len;
-	key_p->pubSec.section_length = CALLER_HEADER + mod_len + exp_len;
-	key_len = key_p->pubSec.section_length + sizeof(struct cca_token_hdr);
-	key_p->pubHdr.token_length = key_len;
-	key_len += 4;
-	keyb_p->ulen = (unsigned short)key_len;
-	key_len += 2;
-	keyb_p->blen = (unsigned short)key_len;
-	cprbx_p->req_parml = parmBlock_l;
-	*z90cMsg_l_p = tmp_size - CALLER_HEADER;
-
-	return 0;
-}
-
-static int
-ICACRT_msg_to_type6CRT_msgX(struct ica_rsa_modexpo_crt *icaMsg_p, int cdx,
-			    int *z90cMsg_l_p, struct type6_msg *z90cMsg_p,
-			    int dev_type)
-{
-	int mod_len, vud_len, tmp_size, total_CPRB_len, parmBlock_l, short_len;
-	int long_len, pad_len, keyPartsLen, tmp_l;
-	unsigned char *tgt_p, *temp;
-	struct type6_hdr *tp6Hdr_p;
-	struct CPRBX *cprbx_p;
-	struct cca_token_hdr *keyHdr_p;
-	struct cca_pvt_ext_CRT_sec *pvtSec_p;
-	struct cca_public_sec *pubSec_p;
-
-	mod_len = icaMsg_p->inputdatalength;
-	short_len = mod_len / 2;
-	long_len = 8 + short_len;
-	keyPartsLen = 3 * long_len + 2 * short_len;
-	pad_len = (8 - (keyPartsLen % 8)) % 8;
-	keyPartsLen += pad_len + mod_len;
-	tmp_size = FIXED_TYPE6_CR_LENX + keyPartsLen + mod_len;
-	total_CPRB_len = tmp_size -  sizeof(struct type6_hdr);
-	parmBlock_l = total_CPRB_len - sizeof(struct CPRBX);
-	vud_len = 2 + mod_len;
-	tmp_size = tmp_size + CALLER_HEADER;
-	memset(z90cMsg_p, 0, tmp_size);
-	tgt_p = (unsigned char *)z90cMsg_p + CALLER_HEADER;
-	memcpy(tgt_p, &static_type6_hdrX, sizeof(struct type6_hdr));
-	tp6Hdr_p = (struct type6_hdr *)tgt_p;
-	tp6Hdr_p->ToCardLen1 = total_CPRB_len;
-	tp6Hdr_p->FromCardLen1 = RESPONSE_CPRBX_SIZE;
-	tgt_p += sizeof(struct type6_hdr);
-	cprbx_p = (struct CPRBX *) tgt_p;
-	memcpy(tgt_p, &static_cprbx, sizeof(struct CPRBX));
-	cprbx_p->domain = (unsigned short)cdx;
-	cprbx_p->req_parml = parmBlock_l;
-	cprbx_p->rpl_msgbl = parmBlock_l;
-	tgt_p += sizeof(struct CPRBX);
-	if (dev_type == PCIXCC_MCL2)
-		memcpy(tgt_p, &static_pkd_function_and_rulesX_MCL2,
-		       sizeof(struct function_and_rules_block));
-	else
-		memcpy(tgt_p, &static_pkd_function_and_rulesX,
-		       sizeof(struct function_and_rules_block));
-	tgt_p += sizeof(struct function_and_rules_block);
-	*((short *)tgt_p) = (short) vud_len;
-	tgt_p += 2;
-	if (copy_from_user(tgt_p, icaMsg_p->inputdata, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, mod_len))
-		return SEN_USER_ERROR;
-	tgt_p += mod_len;
-	tmp_l = sizeof(struct T6_keyBlock_hdr) + sizeof(struct cca_token_hdr) +
-		sizeof(struct cca_pvt_ext_CRT_sec) + 0x0F + keyPartsLen;
-	*((short *)tgt_p) = (short) tmp_l;
-	temp = tgt_p + 2;
-	tmp_l -= 2;
-	*((short *)temp) = (short) tmp_l;
-	tgt_p += sizeof(struct T6_keyBlock_hdr);
-	keyHdr_p = (struct cca_token_hdr *)tgt_p;
-	keyHdr_p->token_identifier = CCA_TKN_HDR_ID_EXT;
-	tmp_l -= 4;
-	keyHdr_p->token_length = tmp_l;
-	tgt_p += sizeof(struct cca_token_hdr);
-	pvtSec_p = (struct cca_pvt_ext_CRT_sec *)tgt_p;
-	pvtSec_p->section_identifier = CCA_PVT_EXT_CRT_SEC_ID_PVT;
-	pvtSec_p->section_length =
-		sizeof(struct cca_pvt_ext_CRT_sec) + keyPartsLen;
-	pvtSec_p->key_format = CCA_PVT_EXT_CRT_SEC_FMT_CL;
-	pvtSec_p->key_use_flags[0] = CCA_PVT_USAGE_ALL;
-	pvtSec_p->p_len = long_len;
-	pvtSec_p->q_len = short_len;
-	pvtSec_p->dp_len = long_len;
-	pvtSec_p->dq_len = short_len;
-	pvtSec_p->u_len = long_len;
-	pvtSec_p->mod_len = mod_len;
-	pvtSec_p->pad_len = pad_len;
-	tgt_p += sizeof(struct cca_pvt_ext_CRT_sec);
-	if (copy_from_user(tgt_p, icaMsg_p->np_prime, long_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, long_len))
-		return SEN_USER_ERROR;
-	tgt_p += long_len;
-	if (copy_from_user(tgt_p, icaMsg_p->nq_prime, short_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, short_len))
-		return SEN_USER_ERROR;
-	tgt_p += short_len;
-	if (copy_from_user(tgt_p, icaMsg_p->bp_key, long_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, long_len))
-		return SEN_USER_ERROR;
-	tgt_p += long_len;
-	if (copy_from_user(tgt_p, icaMsg_p->bq_key, short_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, short_len))
-		return SEN_USER_ERROR;
-	tgt_p += short_len;
-	if (copy_from_user(tgt_p, icaMsg_p->u_mult_inv, long_len))
-		return SEN_RELEASED;
-	if (is_empty(tgt_p, long_len))
-		return SEN_USER_ERROR;
-	tgt_p += long_len;
-	tgt_p += pad_len;
-	memset(tgt_p, 0xFF, mod_len);
-	tgt_p += mod_len;
-	memcpy(tgt_p, &static_cca_pub_sec, sizeof(struct cca_public_sec));
-	pubSec_p = (struct cca_public_sec *) tgt_p;
-	pubSec_p->modulus_bit_len = 8 * mod_len;
-	*z90cMsg_l_p = tmp_size - CALLER_HEADER;
-
-	return 0;
-}
-
-static int
-ICAMEX_msg_to_type50MEX_msg(struct ica_rsa_modexpo *icaMex_p, int *z90cMsg_l_p,
-			    union type50_msg *z90cMsg_p)
-{
-	int mod_len, msg_size, mod_tgt_len, exp_tgt_len, inp_tgt_len;
-	unsigned char *mod_tgt, *exp_tgt, *inp_tgt;
-	union type50_msg *tmp_type50_msg;
-
-	mod_len = icaMex_p->inputdatalength;
-
-	msg_size = ((mod_len <= 128) ? TYPE50_MEB1_LEN : TYPE50_MEB2_LEN) +
-		    CALLER_HEADER;
-
-	memset(z90cMsg_p, 0, msg_size);
-
-	tmp_type50_msg = (union type50_msg *)
-		((unsigned char *) z90cMsg_p + CALLER_HEADER);
-
-	tmp_type50_msg->meb1.header.msg_type_code = TYPE50_TYPE_CODE;
-
-	if (mod_len <= 128) {
-		tmp_type50_msg->meb1.header.msg_len = TYPE50_MEB1_LEN;
-		tmp_type50_msg->meb1.keyblock_type = TYPE50_MEB1_FMT;
-		mod_tgt = tmp_type50_msg->meb1.modulus;
-		mod_tgt_len = sizeof(tmp_type50_msg->meb1.modulus);
-		exp_tgt = tmp_type50_msg->meb1.exponent;
-		exp_tgt_len = sizeof(tmp_type50_msg->meb1.exponent);
-		inp_tgt = tmp_type50_msg->meb1.message;
-		inp_tgt_len = sizeof(tmp_type50_msg->meb1.message);
-	} else {
-		tmp_type50_msg->meb2.header.msg_len = TYPE50_MEB2_LEN;
-		tmp_type50_msg->meb2.keyblock_type = TYPE50_MEB2_FMT;
-		mod_tgt = tmp_type50_msg->meb2.modulus;
-		mod_tgt_len = sizeof(tmp_type50_msg->meb2.modulus);
-		exp_tgt = tmp_type50_msg->meb2.exponent;
-		exp_tgt_len = sizeof(tmp_type50_msg->meb2.exponent);
-		inp_tgt = tmp_type50_msg->meb2.message;
-		inp_tgt_len = sizeof(tmp_type50_msg->meb2.message);
-	}
-
-	mod_tgt += (mod_tgt_len - mod_len);
-	if (copy_from_user(mod_tgt, icaMex_p->n_modulus, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(mod_tgt, mod_len))
-		return SEN_USER_ERROR;
-	exp_tgt += (exp_tgt_len - mod_len);
-	if (copy_from_user(exp_tgt, icaMex_p->b_key, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(exp_tgt, mod_len))
-		return SEN_USER_ERROR;
-	inp_tgt += (inp_tgt_len - mod_len);
-	if (copy_from_user(inp_tgt, icaMex_p->inputdata, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(inp_tgt, mod_len))
-		return SEN_USER_ERROR;
-
-	*z90cMsg_l_p = msg_size - CALLER_HEADER;
-
-	return 0;
-}
-
-static int
-ICACRT_msg_to_type50CRT_msg(struct ica_rsa_modexpo_crt *icaMsg_p,
-			    int *z90cMsg_l_p, union type50_msg *z90cMsg_p)
-{
-	int mod_len, short_len, long_len, tmp_size, p_tgt_len, q_tgt_len,
-	    dp_tgt_len, dq_tgt_len, u_tgt_len, inp_tgt_len, long_offset;
-	unsigned char *p_tgt, *q_tgt, *dp_tgt, *dq_tgt, *u_tgt, *inp_tgt,
-		      temp[8];
-	union type50_msg *tmp_type50_msg;
-
-	mod_len = icaMsg_p->inputdatalength;
-	short_len = mod_len / 2;
-	long_len = mod_len / 2 + 8;
-	long_offset = 0;
-
-	if (long_len > 128) {
-		memset(temp, 0x00, sizeof(temp));
-		if (copy_from_user(temp, icaMsg_p->np_prime, long_len-128))
-			return SEN_RELEASED;
-		if (!is_empty(temp, 8))
-			return SEN_NOT_AVAIL;
-		if (copy_from_user(temp, icaMsg_p->bp_key, long_len-128))
-			return SEN_RELEASED;
-		if (!is_empty(temp, 8))
-			return SEN_NOT_AVAIL;
-		if (copy_from_user(temp, icaMsg_p->u_mult_inv, long_len-128))
-			return SEN_RELEASED;
-		if (!is_empty(temp, 8))
-			return SEN_NOT_AVAIL;
-		long_offset = long_len - 128;
-		long_len = 128;
-	}
-
-	tmp_size = ((long_len <= 64) ? TYPE50_CRB1_LEN : TYPE50_CRB2_LEN) +
-		    CALLER_HEADER;
-
-	memset(z90cMsg_p, 0, tmp_size);
-
-	tmp_type50_msg = (union type50_msg *)
-		((unsigned char *) z90cMsg_p + CALLER_HEADER);
-
-	tmp_type50_msg->crb1.header.msg_type_code = TYPE50_TYPE_CODE;
-	if (long_len <= 64) {
-		tmp_type50_msg->crb1.header.msg_len = TYPE50_CRB1_LEN;
-		tmp_type50_msg->crb1.keyblock_type = TYPE50_CRB1_FMT;
-		p_tgt = tmp_type50_msg->crb1.p;
-		p_tgt_len = sizeof(tmp_type50_msg->crb1.p);
-		q_tgt = tmp_type50_msg->crb1.q;
-		q_tgt_len = sizeof(tmp_type50_msg->crb1.q);
-		dp_tgt = tmp_type50_msg->crb1.dp;
-		dp_tgt_len = sizeof(tmp_type50_msg->crb1.dp);
-		dq_tgt = tmp_type50_msg->crb1.dq;
-		dq_tgt_len = sizeof(tmp_type50_msg->crb1.dq);
-		u_tgt = tmp_type50_msg->crb1.u;
-		u_tgt_len = sizeof(tmp_type50_msg->crb1.u);
-		inp_tgt = tmp_type50_msg->crb1.message;
-		inp_tgt_len = sizeof(tmp_type50_msg->crb1.message);
-	} else {
-		tmp_type50_msg->crb2.header.msg_len = TYPE50_CRB2_LEN;
-		tmp_type50_msg->crb2.keyblock_type = TYPE50_CRB2_FMT;
-		p_tgt = tmp_type50_msg->crb2.p;
-		p_tgt_len = sizeof(tmp_type50_msg->crb2.p);
-		q_tgt = tmp_type50_msg->crb2.q;
-		q_tgt_len = sizeof(tmp_type50_msg->crb2.q);
-		dp_tgt = tmp_type50_msg->crb2.dp;
-		dp_tgt_len = sizeof(tmp_type50_msg->crb2.dp);
-		dq_tgt = tmp_type50_msg->crb2.dq;
-		dq_tgt_len = sizeof(tmp_type50_msg->crb2.dq);
-		u_tgt = tmp_type50_msg->crb2.u;
-		u_tgt_len = sizeof(tmp_type50_msg->crb2.u);
-		inp_tgt = tmp_type50_msg->crb2.message;
-		inp_tgt_len = sizeof(tmp_type50_msg->crb2.message);
-	}
-
-	p_tgt += (p_tgt_len - long_len);
-	if (copy_from_user(p_tgt, icaMsg_p->np_prime + long_offset, long_len))
-		return SEN_RELEASED;
-	if (is_empty(p_tgt, long_len))
-		return SEN_USER_ERROR;
-	q_tgt += (q_tgt_len - short_len);
-	if (copy_from_user(q_tgt, icaMsg_p->nq_prime, short_len))
-		return SEN_RELEASED;
-	if (is_empty(q_tgt, short_len))
-		return SEN_USER_ERROR;
-	dp_tgt += (dp_tgt_len - long_len);
-	if (copy_from_user(dp_tgt, icaMsg_p->bp_key + long_offset, long_len))
-		return SEN_RELEASED;
-	if (is_empty(dp_tgt, long_len))
-		return SEN_USER_ERROR;
-	dq_tgt += (dq_tgt_len - short_len);
-	if (copy_from_user(dq_tgt, icaMsg_p->bq_key, short_len))
-		return SEN_RELEASED;
-	if (is_empty(dq_tgt, short_len))
-		return SEN_USER_ERROR;
-	u_tgt += (u_tgt_len - long_len);
-	if (copy_from_user(u_tgt, icaMsg_p->u_mult_inv + long_offset, long_len))
-		return SEN_RELEASED;
-	if (is_empty(u_tgt, long_len))
-		return SEN_USER_ERROR;
-	inp_tgt += (inp_tgt_len - mod_len);
-	if (copy_from_user(inp_tgt, icaMsg_p->inputdata, mod_len))
-		return SEN_RELEASED;
-	if (is_empty(inp_tgt, mod_len))
-		return SEN_USER_ERROR;
-
-	*z90cMsg_l_p = tmp_size - CALLER_HEADER;
-
-	return 0;
-}
-
-int
-convert_request(unsigned char *buffer, int func, unsigned short function,
-		int cdx, int dev_type, int *msg_l_p, unsigned char *msg_p)
-{
-	if (dev_type == PCICA) {
-		if (func == ICARSACRT)
-			return ICACRT_msg_to_type4CRT_msg(
-				(struct ica_rsa_modexpo_crt *) buffer,
-				msg_l_p, (union type4_msg *) msg_p);
-		else
-			return ICAMEX_msg_to_type4MEX_msg(
-				(struct ica_rsa_modexpo *) buffer,
-				msg_l_p, (union type4_msg *) msg_p);
-	}
-	if (dev_type == PCICC) {
-		if (func == ICARSACRT)
-			return ICACRT_msg_to_type6CRT_msg(
-				(struct ica_rsa_modexpo_crt *) buffer,
-				cdx, msg_l_p, (struct type6_msg *)msg_p);
-		if (function == PCI_FUNC_KEY_ENCRYPT)
-			return ICAMEX_msg_to_type6MEX_en_msg(
-				(struct ica_rsa_modexpo *) buffer,
-				cdx, msg_l_p, (struct type6_msg *) msg_p);
-		else
-			return ICAMEX_msg_to_type6MEX_de_msg(
-				(struct ica_rsa_modexpo *) buffer,
-				cdx, msg_l_p, (struct type6_msg *) msg_p);
-	}
-	if ((dev_type == PCIXCC_MCL2) ||
-	    (dev_type == PCIXCC_MCL3) ||
-	    (dev_type == CEX2C)) {
-		if (func == ICARSACRT)
-			return ICACRT_msg_to_type6CRT_msgX(
-				(struct ica_rsa_modexpo_crt *) buffer,
-				cdx, msg_l_p, (struct type6_msg *) msg_p,
-				dev_type);
-		else
-			return ICAMEX_msg_to_type6MEX_msgX(
-				(struct ica_rsa_modexpo *) buffer,
-				cdx, msg_l_p, (struct type6_msg *) msg_p,
-				dev_type);
-	}
-	if (dev_type == CEX2A) {
-		if (func == ICARSACRT)
-			return ICACRT_msg_to_type50CRT_msg(
-				(struct ica_rsa_modexpo_crt *) buffer,
-				msg_l_p, (union type50_msg *) msg_p);
-		else
-			return ICAMEX_msg_to_type50MEX_msg(
-				(struct ica_rsa_modexpo *) buffer,
-				msg_l_p, (union type50_msg *) msg_p);
-	}
-
-	return 0;
-}
-
-int ext_bitlens_msg_count = 0;
-static inline void
-unset_ext_bitlens(void)
-{
-	if (!ext_bitlens_msg_count) {
-		PRINTK("Unable to use coprocessors for extended bitlengths. "
-		       "Using PCICAs/CEX2As (if present) for extended "
-		       "bitlengths. This is not an error.\n");
-		ext_bitlens_msg_count++;
-	}
-	ext_bitlens = 0;
-}
-
-int
-convert_response(unsigned char *response, unsigned char *buffer,
-		 int *respbufflen_p, unsigned char *resp_buff)
-{
-	struct ica_rsa_modexpo *icaMsg_p = (struct ica_rsa_modexpo *) buffer;
-	struct error_hdr *errh_p = (struct error_hdr *) response;
-	struct type80_hdr *t80h_p = (struct type80_hdr *) response;
-	struct type84_hdr *t84h_p = (struct type84_hdr *) response;
-	struct type86_fmt2_msg *t86m_p =  (struct type86_fmt2_msg *) response;
-	int reply_code, service_rc, service_rs, src_l;
-	unsigned char *src_p, *tgt_p;
-	struct CPRB *cprb_p;
-	struct CPRBX *cprbx_p;
-
-	src_p = 0;
-	reply_code = 0;
-	service_rc = 0;
-	service_rs = 0;
-	src_l = 0;
-	switch (errh_p->type) {
-	case TYPE82_RSP_CODE:
-	case TYPE88_RSP_CODE:
-		reply_code = errh_p->reply_code;
-		src_p = (unsigned char *)errh_p;
-		PRINTK("Hardware error: Type %02X Message Header: "
-		       "%02x%02x%02x%02x%02x%02x%02x%02x\n",
-		       errh_p->type,
-		       src_p[0], src_p[1], src_p[2], src_p[3],
-		       src_p[4], src_p[5], src_p[6], src_p[7]);
-		break;
-	case TYPE80_RSP_CODE:
-		src_l = icaMsg_p->outputdatalength;
-		src_p = response + (int)t80h_p->len - src_l;
-		break;
-	case TYPE84_RSP_CODE:
-		src_l = icaMsg_p->outputdatalength;
-		src_p = response + (int)t84h_p->len - src_l;
-		break;
-	case TYPE86_RSP_CODE:
-		reply_code = t86m_p->header.reply_code;
-		if (reply_code != 0)
-			break;
-		cprb_p = (struct CPRB *)
-			(response + sizeof(struct type86_fmt2_msg));
-		cprbx_p = (struct CPRBX *) cprb_p;
-		if (cprb_p->cprb_ver_id != 0x02) {
-			le2toI(cprb_p->ccp_rtcode, &service_rc);
-			if (service_rc != 0) {
-				le2toI(cprb_p->ccp_rscode, &service_rs);
-				if ((service_rc == 8) && (service_rs == 66))
-					PDEBUG("Bad block format on PCICC\n");
-				else if ((service_rc == 8) && (service_rs == 65))
-					PDEBUG("Probably an even modulus on "
-					       "PCICC\n");
-				else if ((service_rc == 8) && (service_rs == 770)) {
-					PDEBUG("Invalid key length on PCICC\n");
-					unset_ext_bitlens();
-					return REC_USE_PCICA;
-				}
-				else if ((service_rc == 8) && (service_rs == 783)) {
-					PDEBUG("Extended bitlengths not enabled"
-					       "on PCICC\n");
-					unset_ext_bitlens();
-					return REC_USE_PCICA;
-				}
-				else
-					PRINTK("service rc/rs (PCICC): %d/%d\n",
-					       service_rc, service_rs);
-				return REC_OPERAND_INV;
-			}
-			src_p = (unsigned char *)cprb_p + sizeof(struct CPRB);
-			src_p += 4;
-			le2toI(src_p, &src_l);
-			src_l -= 2;
-			src_p += 2;
-		} else {
-			service_rc = (int)cprbx_p->ccp_rtcode;
-			if (service_rc != 0) {
-				service_rs = (int) cprbx_p->ccp_rscode;
-				if ((service_rc == 8) && (service_rs == 66))
-					PDEBUG("Bad block format on PCIXCC\n");
-				else if ((service_rc == 8) && (service_rs == 65))
-					PDEBUG("Probably an even modulus on "
-					       "PCIXCC\n");
-				else if ((service_rc == 8) && (service_rs == 770)) {
-					PDEBUG("Invalid key length on PCIXCC\n");
-					unset_ext_bitlens();
-					return REC_USE_PCICA;
-				}
-				else if ((service_rc == 8) && (service_rs == 783)) {
-					PDEBUG("Extended bitlengths not enabled"
-					       "on PCIXCC\n");
-					unset_ext_bitlens();
-					return REC_USE_PCICA;
-				}
-				else
-					PRINTK("service rc/rs (PCIXCC): %d/%d\n",
-					       service_rc, service_rs);
-				return REC_OPERAND_INV;
-			}
-			src_p = (unsigned char *)
-				cprbx_p + sizeof(struct CPRBX);
-			src_p += 4;
-			src_l = (int)(*((short *) src_p));
-			src_l -= 2;
-			src_p += 2;
-		}
-		break;
-	default:
-		src_p = (unsigned char *)errh_p;
-		PRINTK("Unrecognized Message Header: "
-		       "%02x%02x%02x%02x%02x%02x%02x%02x\n",
-		       src_p[0], src_p[1], src_p[2], src_p[3],
-		       src_p[4], src_p[5], src_p[6], src_p[7]);
-		return REC_BAD_MESSAGE;
-	}
-
-	if (reply_code)
-		switch (reply_code) {
-		case REP82_ERROR_MACHINE_FAILURE:
-			if (errh_p->type == TYPE82_RSP_CODE)
-				PRINTKW("Machine check failure\n");
-			else
-				PRINTKW("Module failure\n");
-			return REC_HARDWAR_ERR;
-		case REP82_ERROR_OPERAND_INVALID:
-			return REC_OPERAND_INV;
-		case REP88_ERROR_MESSAGE_MALFORMD:
-			PRINTKW("Message malformed\n");
-			return REC_OPERAND_INV;
-		case REP82_ERROR_OPERAND_SIZE:
-			return REC_OPERAND_SIZE;
-		case REP82_ERROR_EVEN_MOD_IN_OPND:
-			return REC_EVEN_MOD;
-		case REP82_ERROR_MESSAGE_TYPE:
-			return WRONG_DEVICE_TYPE;
-		case REP82_ERROR_TRANSPORT_FAIL:
-			PRINTKW("Transport failed (APFS = %02X%02X%02X%02X)\n",
-				t86m_p->apfs[0], t86m_p->apfs[1],
-				t86m_p->apfs[2], t86m_p->apfs[3]);
-			return REC_HARDWAR_ERR;
-		default:
-			PRINTKW("reply code = %d\n", reply_code);
-			return REC_HARDWAR_ERR;
-		}
-
-	if (service_rc != 0)
-		return REC_OPERAND_INV;
-
-	if ((src_l > icaMsg_p->outputdatalength) ||
-	    (src_l > RESPBUFFSIZE) ||
-	    (src_l <= 0))
-		return REC_OPERAND_SIZE;
-
-	PDEBUG("Length returned = %d\n", src_l);
-	tgt_p = resp_buff + icaMsg_p->outputdatalength - src_l;
-	memcpy(tgt_p, src_p, src_l);
-	if ((errh_p->type == TYPE86_RSP_CODE) && (resp_buff < tgt_p)) {
-		memset(resp_buff, 0, icaMsg_p->outputdatalength - src_l);
-		if (pad_msg(resp_buff, icaMsg_p->outputdatalength, src_l))
-			return REC_INVALID_PAD;
-	}
-	*respbufflen_p = icaMsg_p->outputdatalength;
-	if (*respbufflen_p == 0)
-		PRINTK("Zero *respbufflen_p\n");
-
-	return 0;
-}
-
diff --git a/drivers/s390/crypto/z90main.c b/drivers/s390/crypto/z90main.c
deleted file mode 100644
index b2f20ab..0000000
--- a/drivers/s390/crypto/z90main.c
+++ /dev/null
@@ -1,3379 +0,0 @@
-/*
- *  linux/drivers/s390/crypto/z90main.c
- *
- *  z90crypt 1.3.3
- *
- *  Copyright (C)  2001, 2005 IBM Corporation
- *  Author(s): Robert Burroughs (burrough@us.ibm.com)
- *             Eric Rossman (edrossma@us.ibm.com)
- *
- *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
- *
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation; either version 2, or (at your option)
- * any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program; if not, write to the Free Software
- * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
- */
-
-#include <asm/uaccess.h>       // copy_(from|to)_user
-#include <linux/compat.h>
-#include <linux/compiler.h>
-#include <linux/delay.h>       // mdelay
-#include <linux/init.h>
-#include <linux/interrupt.h>   // for tasklets
-#include <linux/miscdevice.h>
-#include <linux/module.h>
-#include <linux/moduleparam.h>
-#include <linux/proc_fs.h>
-#include <linux/syscalls.h>
-#include "z90crypt.h"
-#include "z90common.h"
-
-/**
- * Defaults that may be modified.
- */
-
-/**
- * You can specify a different minor at compile time.
- */
-#ifndef Z90CRYPT_MINOR
-#define Z90CRYPT_MINOR	MISC_DYNAMIC_MINOR
-#endif
-
-/**
- * You can specify a different domain at compile time or on the insmod
- * command line.
- */
-#ifndef DOMAIN_INDEX
-#define DOMAIN_INDEX	-1
-#endif
-
-/**
- * This is the name under which the device is registered in /proc/modules.
- */
-#define REG_NAME	"z90crypt"
-
-/**
- * Cleanup should run every CLEANUPTIME seconds and should clean up requests
- * older than CLEANUPTIME seconds in the past.
- */
-#ifndef CLEANUPTIME
-#define CLEANUPTIME 15
-#endif
-
-/**
- * Config should run every CONFIGTIME seconds
- */
-#ifndef CONFIGTIME
-#define CONFIGTIME 30
-#endif
-
-/**
- * The first execution of the config task should take place
- * immediately after initialization
- */
-#ifndef INITIAL_CONFIGTIME
-#define INITIAL_CONFIGTIME 1
-#endif
-
-/**
- * Reader should run every READERTIME milliseconds
- * With the 100Hz patch for s390, z90crypt can lock the system solid while
- * under heavy load. We'll try to avoid that.
- */
-#ifndef READERTIME
-#if HZ > 1000
-#define READERTIME 2
-#else
-#define READERTIME 10
-#endif
-#endif
-
-/**
- * turn long device array index into device pointer
- */
-#define LONG2DEVPTR(ndx) (z90crypt.device_p[(ndx)])
-
-/**
- * turn short device array index into long device array index
- */
-#define SHRT2LONG(ndx) (z90crypt.overall_device_x.device_index[(ndx)])
-
-/**
- * turn short device array index into device pointer
- */
-#define SHRT2DEVPTR(ndx) LONG2DEVPTR(SHRT2LONG(ndx))
-
-/**
- * Status for a work-element
- */
-#define STAT_DEFAULT	0x00 // request has not been processed
-
-#define STAT_ROUTED	0x80 // bit 7: requests get routed to specific device
-			     //	       else, device is determined each write
-#define STAT_FAILED	0x40 // bit 6: this bit is set if the request failed
-			     //	       before being sent to the hardware.
-#define STAT_WRITTEN	0x30 // bits 5-4: work to be done, not sent to device
-//			0x20 // UNUSED state
-#define STAT_READPEND	0x10 // bits 5-4: work done, we're returning data now
-#define STAT_NOWORK	0x00 // bits off: no work on any queue
-#define STAT_RDWRMASK	0x30 // mask for bits 5-4
-
-/**
- * Macros to check the status RDWRMASK
- */
-#define CHK_RDWRMASK(statbyte) ((statbyte) & STAT_RDWRMASK)
-#define SET_RDWRMASK(statbyte, newval) \
-	{(statbyte) &= ~STAT_RDWRMASK; (statbyte) |= newval;}
-
-/**
- * Audit Trail.	 Progress of a Work element
- * audit[0]: Unless noted otherwise, these bits are all set by the process
- */
-#define FP_COPYFROM	0x80 // Caller's buffer has been copied to work element
-#define FP_BUFFREQ	0x40 // Low Level buffer requested
-#define FP_BUFFGOT	0x20 // Low Level buffer obtained
-#define FP_SENT		0x10 // Work element sent to a crypto device
-			     // (may be set by process or by reader task)
-#define FP_PENDING	0x08 // Work element placed on pending queue
-			     // (may be set by process or by reader task)
-#define FP_REQUEST	0x04 // Work element placed on request queue
-#define FP_ASLEEP	0x02 // Work element about to sleep
-#define FP_AWAKE	0x01 // Work element has been awakened
-
-/**
- * audit[1]: These bits are set by the reader task and/or the cleanup task
- */
-#define FP_NOTPENDING	  0x80 // Work element removed from pending queue
-#define FP_AWAKENING	  0x40 // Caller about to be awakened
-#define FP_TIMEDOUT	  0x20 // Caller timed out
-#define FP_RESPSIZESET	  0x10 // Response size copied to work element
-#define FP_RESPADDRCOPIED 0x08 // Response address copied to work element
-#define FP_RESPBUFFCOPIED 0x04 // Response buffer copied to work element
-#define FP_REMREQUEST	  0x02 // Work element removed from request queue
-#define FP_SIGNALED	  0x01 // Work element was awakened by a signal
-
-/**
- * audit[2]: unused
- */
-
-/**
- * state of the file handle in private_data.status
- */
-#define STAT_OPEN 0
-#define STAT_CLOSED 1
-
-/**
- * PID() expands to the process ID of the current process
- */
-#define PID() (current->pid)
-
-/**
- * Selected Constants.	The number of APs and the number of devices
- */
-#ifndef Z90CRYPT_NUM_APS
-#define Z90CRYPT_NUM_APS 64
-#endif
-#ifndef Z90CRYPT_NUM_DEVS
-#define Z90CRYPT_NUM_DEVS Z90CRYPT_NUM_APS
-#endif
-
-/**
- * Buffer size for receiving responses. The maximum Response Size
- * is actually the maximum request size, since in an error condition
- * the request itself may be returned unchanged.
- */
-#define MAX_RESPONSE_SIZE 0x0000077C
-
-/**
- * A count and status-byte mask
- */
-struct status {
-	int	      st_count;		    // # of enabled devices
-	int	      disabled_count;	    // # of disabled devices
-	int	      user_disabled_count;  // # of devices disabled via proc fs
-	unsigned char st_mask[Z90CRYPT_NUM_APS]; // current status mask
-};
-
-/**
- * The array of device indexes is a mechanism for fast indexing into
- * a long (and sparse) array.  For instance, if APs 3, 9 and 47 are
- * installed, z90CDeviceIndex[0] is 3, z90CDeviceIndex[1] is 9, and
- * z90CDeviceIndex[2] is 47.
- */
-struct device_x {
-	int device_index[Z90CRYPT_NUM_DEVS];
-};
-
-/**
- * All devices are arranged in a single array: 64 APs
- */
-struct device {
-	int		 dev_type;	    // PCICA, PCICC, PCIXCC_MCL2,
-					    // PCIXCC_MCL3, CEX2C, CEX2A
-	enum devstat	 dev_stat;	    // current device status
-	int		 dev_self_x;	    // Index in array
-	int		 disabled;	    // Set when device is in error
-	int		 user_disabled;	    // Set when device is disabled by user
-	int		 dev_q_depth;	    // q depth
-	unsigned char *	 dev_resp_p;	    // Response buffer address
-	int		 dev_resp_l;	    // Response Buffer length
-	int		 dev_caller_count;  // Number of callers
-	int		 dev_total_req_cnt; // # requests for device since load
-	struct list_head dev_caller_list;   // List of callers
-};
-
-/**
- * There's a struct status and a struct device_x for each device type.
- */
-struct hdware_block {
-	struct status	hdware_mask;
-	struct status	type_mask[Z90CRYPT_NUM_TYPES];
-	struct device_x type_x_addr[Z90CRYPT_NUM_TYPES];
-	unsigned char	device_type_array[Z90CRYPT_NUM_APS];
-};
-
-/**
- * z90crypt is the topmost data structure in the hierarchy.
- */
-struct z90crypt {
-	int		     max_count;		// Nr of possible crypto devices
-	struct status	     mask;
-	int		     q_depth_array[Z90CRYPT_NUM_DEVS];
-	int		     dev_type_array[Z90CRYPT_NUM_DEVS];
-	struct device_x	     overall_device_x;	// array device indexes
-	struct device *	     device_p[Z90CRYPT_NUM_DEVS];
-	int		     terminating;
-	int		     domain_established;// TRUE:  domain has been found
-	int		     cdx;		// Crypto Domain Index
-	int		     len;		// Length of this data structure
-	struct hdware_block *hdware_info;
-};
-
-/**
- * An array of these structures is pointed to from dev_caller
- * The length of the array depends on the device type. For APs,
- * there are 8.
- *
- * The caller buffer is allocated to the user at OPEN. At WRITE,
- * it contains the request; at READ, the response. The function
- * send_to_crypto_device converts the request to device-dependent
- * form and use the caller's OPEN-allocated buffer for the response.
- *
- * For the contents of caller_dev_dep_req and caller_dev_dep_req_p
- * because that points to it, see the discussion in z90hardware.c.
- * Search for "extended request message block".
- */
-struct caller {
-	int		 caller_buf_l;		 // length of original request
-	unsigned char *	 caller_buf_p;		 // Original request on WRITE
-	int		 caller_dev_dep_req_l;	 // len device dependent request
-	unsigned char *	 caller_dev_dep_req_p;	 // Device dependent form
-	unsigned char	 caller_id[8];		 // caller-supplied message id
-	struct list_head caller_liste;
-	unsigned char	 caller_dev_dep_req[MAX_RESPONSE_SIZE];
-};
-
-/**
- * Function prototypes from z90hardware.c
- */
-enum hdstat query_online(int deviceNr, int cdx, int resetNr, int *q_depth,
-			 int *dev_type);
-enum devstat reset_device(int deviceNr, int cdx, int resetNr);
-enum devstat send_to_AP(int dev_nr, int cdx, int msg_len, unsigned char *msg_ext);
-enum devstat receive_from_AP(int dev_nr, int cdx, int resplen,
-			     unsigned char *resp, unsigned char *psmid);
-int convert_request(unsigned char *buffer, int func, unsigned short function,
-		    int cdx, int dev_type, int *msg_l_p, unsigned char *msg_p);
-int convert_response(unsigned char *response, unsigned char *buffer,
-		     int *respbufflen_p, unsigned char *resp_buff);
-
-/**
- * Low level function prototypes
- */
-static int create_z90crypt(int *cdx_p);
-static int refresh_z90crypt(int *cdx_p);
-static int find_crypto_devices(struct status *deviceMask);
-static int create_crypto_device(int index);
-static int destroy_crypto_device(int index);
-static void destroy_z90crypt(void);
-static int refresh_index_array(struct status *status_str,
-			       struct device_x *index_array);
-static int probe_device_type(struct device *devPtr);
-static int probe_PCIXCC_type(struct device *devPtr);
-
-/**
- * proc fs definitions
- */
-static struct proc_dir_entry *z90crypt_entry;
-
-/**
- * data structures
- */
-
-/**
- * work_element.opener points back to this structure
- */
-struct priv_data {
-	pid_t	opener_pid;
-	unsigned char	status;		// 0: open  1: closed
-};
-
-/**
- * A work element is allocated for each request
- */
-struct work_element {
-	struct priv_data *priv_data;
-	pid_t		  pid;
-	int		  devindex;	  // index of device processing this w_e
-					  // (If request did not specify device,
-					  // -1 until placed onto a queue)
-	int		  devtype;
-	struct list_head  liste;	  // used for requestq and pendingq
-	char		  buffer[128];	  // local copy of user request
-	int		  buff_size;	  // size of the buffer for the request
-	char		  resp_buff[RESPBUFFSIZE];
-	int		  resp_buff_size;
-	char __user *	  resp_addr;	  // address of response in user space
-	unsigned int	  funccode;	  // function code of request
-	wait_queue_head_t waitq;
-	unsigned long	  requestsent;	  // time at which the request was sent
-	atomic_t	  alarmrung;	  // wake-up signal
-	unsigned char	  caller_id[8];	  // pid + counter, for this w_e
-	unsigned char	  status[1];	  // bits to mark status of the request
-	unsigned char	  audit[3];	  // record of work element's progress
-	unsigned char *	  requestptr;	  // address of request buffer
-	int		  retcode;	  // return code of request
-};
-
-/**
- * High level function prototypes
- */
-static int z90crypt_open(struct inode *, struct file *);
-static int z90crypt_release(struct inode *, struct file *);
-static ssize_t z90crypt_read(struct file *, char __user *, size_t, loff_t *);
-static ssize_t z90crypt_write(struct file *, const char __user *,
-							size_t, loff_t *);
-static long z90crypt_unlocked_ioctl(struct file *, unsigned int, unsigned long);
-static long z90crypt_compat_ioctl(struct file *, unsigned int, unsigned long);
-
-static void z90crypt_reader_task(unsigned long);
-static void z90crypt_schedule_reader_task(unsigned long);
-static void z90crypt_config_task(unsigned long);
-static void z90crypt_cleanup_task(unsigned long);
-
-static int z90crypt_status(char *, char **, off_t, int, int *, void *);
-static int z90crypt_status_write(struct file *, const char __user *,
-				 unsigned long, void *);
-
-/**
- * Storage allocated at initialization and used throughout the life of
- * this insmod
- */
-static int domain = DOMAIN_INDEX;
-static struct z90crypt z90crypt;
-static int quiesce_z90crypt;
-static spinlock_t queuespinlock;
-static struct list_head request_list;
-static int requestq_count;
-static struct list_head pending_list;
-static int pendingq_count;
-
-static struct tasklet_struct reader_tasklet;
-static struct timer_list reader_timer;
-static struct timer_list config_timer;
-static struct timer_list cleanup_timer;
-static atomic_t total_open;
-static atomic_t z90crypt_step;
-
-static struct file_operations z90crypt_fops = {
-	.owner		= THIS_MODULE,
-	.read		= z90crypt_read,
-	.write		= z90crypt_write,
-	.unlocked_ioctl	= z90crypt_unlocked_ioctl,
-#ifdef CONFIG_COMPAT
-	.compat_ioctl	= z90crypt_compat_ioctl,
-#endif
-	.open		= z90crypt_open,
-	.release	= z90crypt_release
-};
-
-static struct miscdevice z90crypt_misc_device = {
-	.minor	    = Z90CRYPT_MINOR,
-	.name	    = DEV_NAME,
-	.fops	    = &z90crypt_fops,
-};
-
-/**
- * Documentation values.
- */
-MODULE_AUTHOR("zSeries Linux Crypto Team: Robert H. Burroughs, Eric D. Rossman"
-	      "and Jochen Roehrig");
-MODULE_DESCRIPTION("zSeries Linux Cryptographic Coprocessor device driver, "
-		   "Copyright 2001, 2005 IBM Corporation");
-MODULE_LICENSE("GPL");
-module_param(domain, int, 0);
-MODULE_PARM_DESC(domain, "domain index for device");
-
-#ifdef CONFIG_COMPAT
-/**
- * ioctl32 conversion routines
- */
-struct ica_rsa_modexpo_32 { // For 32-bit callers
-	compat_uptr_t	inputdata;
-	unsigned int	inputdatalength;
-	compat_uptr_t	outputdata;
-	unsigned int	outputdatalength;
-	compat_uptr_t	b_key;
-	compat_uptr_t	n_modulus;
-};
-
-static long
-trans_modexpo32(struct file *filp, unsigned int cmd, unsigned long arg)
-{
-	struct ica_rsa_modexpo_32 __user *mex32u = compat_ptr(arg);
-	struct ica_rsa_modexpo_32  mex32k;
-	struct ica_rsa_modexpo __user *mex64;
-	long ret = 0;
-	unsigned int i;
-
-	if (!access_ok(VERIFY_WRITE, mex32u, sizeof(struct ica_rsa_modexpo_32)))
-		return -EFAULT;
-	mex64 = compat_alloc_user_space(sizeof(struct ica_rsa_modexpo));
-	if (!access_ok(VERIFY_WRITE, mex64, sizeof(struct ica_rsa_modexpo)))
-		return -EFAULT;
-	if (copy_from_user(&mex32k, mex32u, sizeof(struct ica_rsa_modexpo_32)))
-		return -EFAULT;
-	if (__put_user(compat_ptr(mex32k.inputdata), &mex64->inputdata)   ||
-	    __put_user(mex32k.inputdatalength, &mex64->inputdatalength)   ||
-	    __put_user(compat_ptr(mex32k.outputdata), &mex64->outputdata) ||
-	    __put_user(mex32k.outputdatalength, &mex64->outputdatalength) ||
-	    __put_user(compat_ptr(mex32k.b_key), &mex64->b_key)           ||
-	    __put_user(compat_ptr(mex32k.n_modulus), &mex64->n_modulus))
-		return -EFAULT;
-	ret = z90crypt_unlocked_ioctl(filp, cmd, (unsigned long)mex64);
-	if (!ret)
-		if (__get_user(i, &mex64->outputdatalength) ||
-		    __put_user(i, &mex32u->outputdatalength))
-			ret = -EFAULT;
-	return ret;
-}
-
-struct ica_rsa_modexpo_crt_32 { // For 32-bit callers
-	compat_uptr_t	inputdata;
-	unsigned int	inputdatalength;
-	compat_uptr_t	outputdata;
-	unsigned int	outputdatalength;
-	compat_uptr_t	bp_key;
-	compat_uptr_t	bq_key;
-	compat_uptr_t	np_prime;
-	compat_uptr_t	nq_prime;
-	compat_uptr_t	u_mult_inv;
-};
-
-static long
-trans_modexpo_crt32(struct file *filp, unsigned int cmd, unsigned long arg)
-{
-	struct ica_rsa_modexpo_crt_32 __user *crt32u = compat_ptr(arg);
-	struct ica_rsa_modexpo_crt_32  crt32k;
-	struct ica_rsa_modexpo_crt __user *crt64;
-	long ret = 0;
-	unsigned int i;
-
-	if (!access_ok(VERIFY_WRITE, crt32u,
-		       sizeof(struct ica_rsa_modexpo_crt_32)))
-		return -EFAULT;
-	crt64 = compat_alloc_user_space(sizeof(struct ica_rsa_modexpo_crt));
-	if (!access_ok(VERIFY_WRITE, crt64, sizeof(struct ica_rsa_modexpo_crt)))
-		return -EFAULT;
-	if (copy_from_user(&crt32k, crt32u,
-			   sizeof(struct ica_rsa_modexpo_crt_32)))
-		return -EFAULT;
-	if (__put_user(compat_ptr(crt32k.inputdata), &crt64->inputdata)   ||
-	    __put_user(crt32k.inputdatalength, &crt64->inputdatalength)   ||
-	    __put_user(compat_ptr(crt32k.outputdata), &crt64->outputdata) ||
-	    __put_user(crt32k.outputdatalength, &crt64->outputdatalength) ||
-	    __put_user(compat_ptr(crt32k.bp_key), &crt64->bp_key)         ||
-	    __put_user(compat_ptr(crt32k.bq_key), &crt64->bq_key)         ||
-	    __put_user(compat_ptr(crt32k.np_prime), &crt64->np_prime)     ||
-	    __put_user(compat_ptr(crt32k.nq_prime), &crt64->nq_prime)     ||
-	    __put_user(compat_ptr(crt32k.u_mult_inv), &crt64->u_mult_inv))
-		return -EFAULT;
-	ret = z90crypt_unlocked_ioctl(filp, cmd, (unsigned long)crt64);
-	if (!ret)
-		if (__get_user(i, &crt64->outputdatalength) ||
-		    __put_user(i, &crt32u->outputdatalength))
-			ret = -EFAULT;
-	return ret;
-}
-
-static long
-z90crypt_compat_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
-{
-	switch (cmd) {
-	case ICAZ90STATUS:
-	case Z90QUIESCE:
-	case Z90STAT_TOTALCOUNT:
-	case Z90STAT_PCICACOUNT:
-	case Z90STAT_PCICCCOUNT:
-	case Z90STAT_PCIXCCCOUNT:
-	case Z90STAT_PCIXCCMCL2COUNT:
-	case Z90STAT_PCIXCCMCL3COUNT:
-	case Z90STAT_CEX2CCOUNT:
-	case Z90STAT_REQUESTQ_COUNT:
-	case Z90STAT_PENDINGQ_COUNT:
-	case Z90STAT_TOTALOPEN_COUNT:
-	case Z90STAT_DOMAIN_INDEX:
-	case Z90STAT_STATUS_MASK:
-	case Z90STAT_QDEPTH_MASK:
-	case Z90STAT_PERDEV_REQCNT:
-		return z90crypt_unlocked_ioctl(filp, cmd, arg);
-	case ICARSAMODEXPO:
-		return trans_modexpo32(filp, cmd, arg);
-	case ICARSACRT:
-		return trans_modexpo_crt32(filp, cmd, arg);
-	default:
-		return -ENOIOCTLCMD;
-  	}
-}
-#endif
-
-/**
- * The module initialization code.
- */
-static int __init
-z90crypt_init_module(void)
-{
-	int result, nresult;
-	struct proc_dir_entry *entry;
-
-	PDEBUG("PID %d\n", PID());
-
-	if ((domain < -1) || (domain > 15)) {
-		PRINTKW("Invalid param: domain = %d.  Not loading.\n", domain);
-		return -EINVAL;
-	}
-
-	/* Register as misc device with given minor (or get a dynamic one). */
-	result = misc_register(&z90crypt_misc_device);
-	if (result < 0) {
-		PRINTKW(KERN_ERR "misc_register (minor %d) failed with %d\n",
-			z90crypt_misc_device.minor, result);
-		return result;
-	}
-
-	PDEBUG("Registered " DEV_NAME " with result %d\n", result);
-
-	result = create_z90crypt(&domain);
-	if (result != 0) {
-		PRINTKW("create_z90crypt (domain index %d) failed with %d.\n",
-			domain, result);
-		result = -ENOMEM;
-		goto init_module_cleanup;
-	}
-
-	if (result == 0) {
-		PRINTKN("Version %d.%d.%d loaded, built on %s %s\n",
-			z90crypt_VERSION, z90crypt_RELEASE, z90crypt_VARIANT,
-			__DATE__, __TIME__);
-		PDEBUG("create_z90crypt (domain index %d) successful.\n",
-		       domain);
-	} else
-		PRINTK("No devices at startup\n");
-
-	/* Initialize globals. */
-	spin_lock_init(&queuespinlock);
-
-	INIT_LIST_HEAD(&pending_list);
-	pendingq_count = 0;
-
-	INIT_LIST_HEAD(&request_list);
-	requestq_count = 0;
-
-	quiesce_z90crypt = 0;
-
-	atomic_set(&total_open, 0);
-	atomic_set(&z90crypt_step, 0);
-
-	/* Set up the cleanup task. */
-	init_timer(&cleanup_timer);
-	cleanup_timer.function = z90crypt_cleanup_task;
-	cleanup_timer.data = 0;
-	cleanup_timer.expires = jiffies + (CLEANUPTIME * HZ);
-	add_timer(&cleanup_timer);
-
-	/* Set up the proc file system */
-	entry = create_proc_entry("driver/z90crypt", 0644, 0);
-	if (entry) {
-		entry->nlink = 1;
-		entry->data = 0;
-		entry->read_proc = z90crypt_status;
-		entry->write_proc = z90crypt_status_write;
-	}
-	else
-		PRINTK("Couldn't create z90crypt proc entry\n");
-	z90crypt_entry = entry;
-
-	/* Set up the configuration task. */
-	init_timer(&config_timer);
-	config_timer.function = z90crypt_config_task;
-	config_timer.data = 0;
-	config_timer.expires = jiffies + (INITIAL_CONFIGTIME * HZ);
-	add_timer(&config_timer);
-
-	/* Set up the reader task */
-	tasklet_init(&reader_tasklet, z90crypt_reader_task, 0);
-	init_timer(&reader_timer);
-	reader_timer.function = z90crypt_schedule_reader_task;
-	reader_timer.data = 0;
-	reader_timer.expires = jiffies + (READERTIME * HZ / 1000);
-	add_timer(&reader_timer);
-
-	return 0; // success
-
-init_module_cleanup:
-	if ((nresult = misc_deregister(&z90crypt_misc_device)))
-		PRINTK("misc_deregister failed with %d.\n", nresult);
-	else
-		PDEBUG("misc_deregister successful.\n");
-
-	return result; // failure
-}
-
-/**
- * The module termination code
- */
-static void __exit
-z90crypt_cleanup_module(void)
-{
-	int nresult;
-
-	PDEBUG("PID %d\n", PID());
-
-	remove_proc_entry("driver/z90crypt", 0);
-
-	if ((nresult = misc_deregister(&z90crypt_misc_device)))
-		PRINTK("misc_deregister failed with %d.\n", nresult);
-	else
-		PDEBUG("misc_deregister successful.\n");
-
-	/* Remove the tasks */
-	tasklet_kill(&reader_tasklet);
-	del_timer(&reader_timer);
-	del_timer(&config_timer);
-	del_timer(&cleanup_timer);
-
-	destroy_z90crypt();
-
-	PRINTKN("Unloaded.\n");
-}
-
-/**
- * Functions running under a process id
- *
- * The I/O functions:
- *     z90crypt_open
- *     z90crypt_release
- *     z90crypt_read
- *     z90crypt_write
- *     z90crypt_unlocked_ioctl
- *     z90crypt_status
- *     z90crypt_status_write
- *	 disable_card
- *	 enable_card
- *
- * Helper functions:
- *     z90crypt_rsa
- *	 z90crypt_prepare
- *	 z90crypt_send
- *	 z90crypt_process_results
- *
- */
-static int
-z90crypt_open(struct inode *inode, struct file *filp)
-{
-	struct priv_data *private_data_p;
-
-	if (quiesce_z90crypt)
-		return -EQUIESCE;
-
-	private_data_p = kzalloc(sizeof(struct priv_data), GFP_KERNEL);
-	if (!private_data_p) {
-		PRINTK("Memory allocate failed\n");
-		return -ENOMEM;
-	}
-
-	private_data_p->status = STAT_OPEN;
-	private_data_p->opener_pid = PID();
-	filp->private_data = private_data_p;
-	atomic_inc(&total_open);
-
-	return 0;
-}
-
-static int
-z90crypt_release(struct inode *inode, struct file *filp)
-{
-	struct priv_data *private_data_p = filp->private_data;
-
-	PDEBUG("PID %d (filp %p)\n", PID(), filp);
-
-	private_data_p->status = STAT_CLOSED;
-	memset(private_data_p, 0, sizeof(struct priv_data));
-	kfree(private_data_p);
-	atomic_dec(&total_open);
-
-	return 0;
-}
-
-/*
- * there are two read functions, of which compile options will choose one
- * without USE_GET_RANDOM_BYTES
- *   => read() always returns -EPERM;
- * otherwise
- *   => read() uses get_random_bytes() kernel function
- */
-#ifndef USE_GET_RANDOM_BYTES
-/**
- * z90crypt_read will not be supported beyond z90crypt 1.3.1
- */
-static ssize_t
-z90crypt_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos)
-{
-	PDEBUG("filp %p (PID %d)\n", filp, PID());
-	return -EPERM;
-}
-#else // we want to use get_random_bytes
-/**
- * read() just returns a string of random bytes.  Since we have no way
- * to generate these cryptographically, we just execute get_random_bytes
- * for the length specified.
- */
-#include <linux/random.h>
-static ssize_t
-z90crypt_read(struct file *filp, char __user *buf, size_t count, loff_t *f_pos)
-{
-	unsigned char *temp_buff;
-
-	PDEBUG("filp %p (PID %d)\n", filp, PID());
-
-	if (quiesce_z90crypt)
-		return -EQUIESCE;
-	if (count < 0) {
-		PRINTK("Requested random byte count negative: %ld\n", count);
-		return -EINVAL;
-	}
-	if (count > RESPBUFFSIZE) {
-		PDEBUG("count[%d] > RESPBUFFSIZE", count);
-		return -EINVAL;
-	}
-	if (count == 0)
-		return 0;
-	temp_buff = kmalloc(RESPBUFFSIZE, GFP_KERNEL);
-	if (!temp_buff) {
-		PRINTK("Memory allocate failed\n");
-		return -ENOMEM;
-	}
-	get_random_bytes(temp_buff, count);
-
-	if (copy_to_user(buf, temp_buff, count) != 0) {
-		kfree(temp_buff);
-		return -EFAULT;
-	}
-	kfree(temp_buff);
-	return count;
-}
-#endif
-
-/**
- * Write is is not allowed
- */
-static ssize_t
-z90crypt_write(struct file *filp, const char __user *buf, size_t count, loff_t *f_pos)
-{
-	PDEBUG("filp %p (PID %d)\n", filp, PID());
-	return -EPERM;
-}
-
-/**
- * New status functions
- */
-static inline int
-get_status_totalcount(void)
-{
-	return z90crypt.hdware_info->hdware_mask.st_count;
-}
-
-static inline int
-get_status_PCICAcount(void)
-{
-	return z90crypt.hdware_info->type_mask[PCICA].st_count;
-}
-
-static inline int
-get_status_PCICCcount(void)
-{
-	return z90crypt.hdware_info->type_mask[PCICC].st_count;
-}
-
-static inline int
-get_status_PCIXCCcount(void)
-{
-	return z90crypt.hdware_info->type_mask[PCIXCC_MCL2].st_count +
-	       z90crypt.hdware_info->type_mask[PCIXCC_MCL3].st_count;
-}
-
-static inline int
-get_status_PCIXCCMCL2count(void)
-{
-	return z90crypt.hdware_info->type_mask[PCIXCC_MCL2].st_count;
-}
-
-static inline int
-get_status_PCIXCCMCL3count(void)
-{
-	return z90crypt.hdware_info->type_mask[PCIXCC_MCL3].st_count;
-}
-
-static inline int
-get_status_CEX2Ccount(void)
-{
-	return z90crypt.hdware_info->type_mask[CEX2C].st_count;
-}
-
-static inline int
-get_status_CEX2Acount(void)
-{
-	return z90crypt.hdware_info->type_mask[CEX2A].st_count;
-}
-
-static inline int
-get_status_requestq_count(void)
-{
-	return requestq_count;
-}
-
-static inline int
-get_status_pendingq_count(void)
-{
-	return pendingq_count;
-}
-
-static inline int
-get_status_totalopen_count(void)
-{
-	return atomic_read(&total_open);
-}
-
-static inline int
-get_status_domain_index(void)
-{
-	return z90crypt.cdx;
-}
-
-static inline unsigned char *
-get_status_status_mask(unsigned char status[Z90CRYPT_NUM_APS])
-{
-	int i, ix;
-
-	memcpy(status, z90crypt.hdware_info->device_type_array,
-	       Z90CRYPT_NUM_APS);
-
-	for (i = 0; i < get_status_totalcount(); i++) {
-		ix = SHRT2LONG(i);
-		if (LONG2DEVPTR(ix)->user_disabled)
-			status[ix] = 0x0d;
-	}
-
-	return status;
-}
-
-static inline unsigned char *
-get_status_qdepth_mask(unsigned char qdepth[Z90CRYPT_NUM_APS])
-{
-	int i, ix;
-
-	memset(qdepth, 0, Z90CRYPT_NUM_APS);
-
-	for (i = 0; i < get_status_totalcount(); i++) {
-		ix = SHRT2LONG(i);
-		qdepth[ix] = LONG2DEVPTR(ix)->dev_caller_count;
-	}
-
-	return qdepth;
-}
-
-static inline unsigned int *
-get_status_perdevice_reqcnt(unsigned int reqcnt[Z90CRYPT_NUM_APS])
-{
-	int i, ix;
-
-	memset(reqcnt, 0, Z90CRYPT_NUM_APS * sizeof(int));
-
-	for (i = 0; i < get_status_totalcount(); i++) {
-		ix = SHRT2LONG(i);
-		reqcnt[ix] = LONG2DEVPTR(ix)->dev_total_req_cnt;
-	}
-
-	return reqcnt;
-}
-
-static inline void
-init_work_element(struct work_element *we_p,
-		  struct priv_data *priv_data, pid_t pid)
-{
-	int step;
-
-	we_p->requestptr = (unsigned char *)we_p + sizeof(struct work_element);
-	/* Come up with a unique id for this caller. */
-	step = atomic_inc_return(&z90crypt_step);
-	memcpy(we_p->caller_id+0, (void *) &pid, sizeof(pid));
-	memcpy(we_p->caller_id+4, (void *) &step, sizeof(step));
-	we_p->pid = pid;
-	we_p->priv_data = priv_data;
-	we_p->status[0] = STAT_DEFAULT;
-	we_p->audit[0] = 0x00;
-	we_p->audit[1] = 0x00;
-	we_p->audit[2] = 0x00;
-	we_p->resp_buff_size = 0;
-	we_p->retcode = 0;
-	we_p->devindex = -1;
-	we_p->devtype = -1;
-	atomic_set(&we_p->alarmrung, 0);
-	init_waitqueue_head(&we_p->waitq);
-	INIT_LIST_HEAD(&(we_p->liste));
-}
-
-static inline int
-allocate_work_element(struct work_element **we_pp,
-		      struct priv_data *priv_data_p, pid_t pid)
-{
-	struct work_element *we_p;
-
-	we_p = (struct work_element *) get_zeroed_page(GFP_KERNEL);
-	if (!we_p)
-		return -ENOMEM;
-	init_work_element(we_p, priv_data_p, pid);
-	*we_pp = we_p;
-	return 0;
-}
-
-static inline void
-remove_device(struct device *device_p)
-{
-	if (!device_p || (device_p->disabled != 0))
-		return;
-	device_p->disabled = 1;
-	z90crypt.hdware_info->type_mask[device_p->dev_type].disabled_count++;
-	z90crypt.hdware_info->hdware_mask.disabled_count++;
-}
-
-/**
- * Bitlength limits for each card
- *
- * There are new MCLs which allow more bitlengths. See the table for details.
- * The MCL must be applied and the newer bitlengths enabled for these to work.
- *
- * Card Type    Old limit    New limit
- * PCICA          ??-2048     same (the lower limit is less than 128 bit...)
- * PCICC         512-1024     512-2048
- * PCIXCC_MCL2   512-2048     ----- (applying any GA LIC will make an MCL3 card)
- * PCIXCC_MCL3   -----        128-2048
- * CEX2C         512-2048     128-2048
- * CEX2A          ??-2048     same (the lower limit is less than 128 bit...)
- *
- * ext_bitlens (extended bitlengths) is a global, since you should not apply an
- * MCL to just one card in a machine. We assume, at first, that all cards have
- * these capabilities.
- */
-int ext_bitlens = 1; // This is global
-#define PCIXCC_MIN_MOD_SIZE	 16	//  128 bits
-#define OLD_PCIXCC_MIN_MOD_SIZE	 64	//  512 bits
-#define PCICC_MIN_MOD_SIZE	 64	//  512 bits
-#define OLD_PCICC_MAX_MOD_SIZE	128	// 1024 bits
-#define MAX_MOD_SIZE		256	// 2048 bits
-
-static inline int
-select_device_type(int *dev_type_p, int bytelength)
-{
-	static int count = 0;
-	int PCICA_avail, PCIXCC_MCL3_avail, CEX2C_avail, CEX2A_avail,
-	    index_to_use;
-	struct status *stat;
-	if ((*dev_type_p != PCICC) && (*dev_type_p != PCICA) &&
-	    (*dev_type_p != PCIXCC_MCL2) && (*dev_type_p != PCIXCC_MCL3) &&
-	    (*dev_type_p != CEX2C) && (*dev_type_p != CEX2A) &&
-	    (*dev_type_p != ANYDEV))
-		return -1;
-	if (*dev_type_p != ANYDEV) {
-		stat = &z90crypt.hdware_info->type_mask[*dev_type_p];
-		if (stat->st_count >
-		    (stat->disabled_count + stat->user_disabled_count))
-			return 0;
-		return -1;
-	}
-
-	/**
-	 * Assumption: PCICA, PCIXCC_MCL3, CEX2C, and CEX2A are all similar in
-	 * speed.
-	 *
-	 * PCICA and CEX2A do NOT co-exist, so it would be either one or the
-	 * other present.
-	 */
-	stat = &z90crypt.hdware_info->type_mask[PCICA];
-	PCICA_avail = stat->st_count -
-			(stat->disabled_count + stat->user_disabled_count);
-	stat = &z90crypt.hdware_info->type_mask[PCIXCC_MCL3];
-	PCIXCC_MCL3_avail = stat->st_count -
-			(stat->disabled_count + stat->user_disabled_count);
-	stat = &z90crypt.hdware_info->type_mask[CEX2C];
-	CEX2C_avail = stat->st_count -
-			(stat->disabled_count + stat->user_disabled_count);
-	stat = &z90crypt.hdware_info->type_mask[CEX2A];
-	CEX2A_avail = stat->st_count -
-			(stat->disabled_count + stat->user_disabled_count);
-	if (PCICA_avail || PCIXCC_MCL3_avail || CEX2C_avail || CEX2A_avail) {
-		/**
-		 * bitlength is a factor, PCICA or CEX2A are the most capable,
-		 * even with the new MCL for PCIXCC.
-		 */
-		if ((bytelength < PCIXCC_MIN_MOD_SIZE) ||
-		    (!ext_bitlens && (bytelength < OLD_PCIXCC_MIN_MOD_SIZE))) {
-			if (PCICA_avail) {
-				*dev_type_p = PCICA;
-				return 0;
-			}
-			if (CEX2A_avail) {
-				*dev_type_p = CEX2A;
-				return 0;
-			}
-			return -1;
-		}
-
-		index_to_use = count % (PCICA_avail + PCIXCC_MCL3_avail +
-					CEX2C_avail + CEX2A_avail);
-		if (index_to_use < PCICA_avail)
-			*dev_type_p = PCICA;
-		else if (index_to_use < (PCICA_avail + PCIXCC_MCL3_avail))
-			*dev_type_p = PCIXCC_MCL3;
-		else if (index_to_use < (PCICA_avail + PCIXCC_MCL3_avail +
-					 CEX2C_avail))
-			*dev_type_p = CEX2C;
-		else
-			*dev_type_p = CEX2A;
-		count++;
-		return 0;
-	}
-
-	/* Less than OLD_PCIXCC_MIN_MOD_SIZE cannot go to a PCIXCC_MCL2 */
-	if (bytelength < OLD_PCIXCC_MIN_MOD_SIZE)
-		return -1;
-	stat = &z90crypt.hdware_info->type_mask[PCIXCC_MCL2];
-	if (stat->st_count >
-	    (stat->disabled_count + stat->user_disabled_count)) {
-		*dev_type_p = PCIXCC_MCL2;
-		return 0;
-	}
-
-	/**
-	 * Less than PCICC_MIN_MOD_SIZE or more than OLD_PCICC_MAX_MOD_SIZE
-	 * (if we don't have the MCL applied and the newer bitlengths enabled)
-	 * cannot go to a PCICC
-	 */
-	if ((bytelength < PCICC_MIN_MOD_SIZE) ||
-	    (!ext_bitlens && (bytelength > OLD_PCICC_MAX_MOD_SIZE))) {
-		return -1;
-	}
-	stat = &z90crypt.hdware_info->type_mask[PCICC];
-	if (stat->st_count >
-	    (stat->disabled_count + stat->user_disabled_count)) {
-		*dev_type_p = PCICC;
-		return 0;
-	}
-
-	return -1;
-}
-
-/**
- * Try the selected number, then the selected type (can be ANYDEV)
- */
-static inline int
-select_device(int *dev_type_p, int *device_nr_p, int bytelength)
-{
-	int i, indx, devTp, low_count, low_indx;
-	struct device_x *index_p;
-	struct device *dev_ptr;
-
-	PDEBUG("device type = %d, index = %d\n", *dev_type_p, *device_nr_p);
-	if ((*device_nr_p >= 0) && (*device_nr_p < Z90CRYPT_NUM_DEVS)) {
-		PDEBUG("trying index = %d\n", *device_nr_p);
-		dev_ptr = z90crypt.device_p[*device_nr_p];
-
-		if (dev_ptr &&
-		    (dev_ptr->dev_stat != DEV_GONE) &&
-		    (dev_ptr->disabled == 0) &&
-		    (dev_ptr->user_disabled == 0)) {
-			PDEBUG("selected by number, index = %d\n",
-			       *device_nr_p);
-			*dev_type_p = dev_ptr->dev_type;
-			return *device_nr_p;
-		}
-	}
-	*device_nr_p = -1;
-	PDEBUG("trying type = %d\n", *dev_type_p);
-	devTp = *dev_type_p;
-	if (select_device_type(&devTp, bytelength) == -1) {
-		PDEBUG("failed to select by type\n");
-		return -1;
-	}
-	PDEBUG("selected type = %d\n", devTp);
-	index_p = &z90crypt.hdware_info->type_x_addr[devTp];
-	low_count = 0x0000FFFF;
-	low_indx = -1;
-	for (i = 0; i < z90crypt.hdware_info->type_mask[devTp].st_count; i++) {
-		indx = index_p->device_index[i];
-		dev_ptr = z90crypt.device_p[indx];
-		if (dev_ptr &&
-		    (dev_ptr->dev_stat != DEV_GONE) &&
-		    (dev_ptr->disabled == 0) &&
-		    (dev_ptr->user_disabled == 0) &&
-		    (devTp == dev_ptr->dev_type) &&
-		    (low_count > dev_ptr->dev_caller_count)) {
-			low_count = dev_ptr->dev_caller_count;
-			low_indx = indx;
-		}
-	}
-	*device_nr_p = low_indx;
-	return low_indx;
-}
-
-static inline int
-send_to_crypto_device(struct work_element *we_p)
-{
-	struct caller *caller_p;
-	struct device *device_p;
-	int dev_nr;
-	int bytelen = ((struct ica_rsa_modexpo *)we_p->buffer)->inputdatalength;
-
-	if (!we_p->requestptr)
-		return SEN_FATAL_ERROR;
-	caller_p = (struct caller *)we_p->requestptr;
-	dev_nr = we_p->devindex;
-	if (select_device(&we_p->devtype, &dev_nr, bytelen) == -1) {
-		if (z90crypt.hdware_info->hdware_mask.st_count != 0)
-			return SEN_RETRY;
-		else
-			return SEN_NOT_AVAIL;
-	}
-	we_p->devindex = dev_nr;
-	device_p = z90crypt.device_p[dev_nr];
-	if (!device_p)
-		return SEN_NOT_AVAIL;
-	if (device_p->dev_type != we_p->devtype)
-		return SEN_RETRY;
-	if (device_p->dev_caller_count >= device_p->dev_q_depth)
-		return SEN_QUEUE_FULL;
-	PDEBUG("device number prior to send: %d\n", dev_nr);
-	switch (send_to_AP(dev_nr, z90crypt.cdx,
-			   caller_p->caller_dev_dep_req_l,
-			   caller_p->caller_dev_dep_req_p)) {
-	case DEV_SEN_EXCEPTION:
-		PRINTKC("Exception during send to device %d\n", dev_nr);
-		z90crypt.terminating = 1;
-		return SEN_FATAL_ERROR;
-	case DEV_GONE:
-		PRINTK("Device %d not available\n", dev_nr);
-		remove_device(device_p);
-		return SEN_NOT_AVAIL;
-	case DEV_EMPTY:
-		return SEN_NOT_AVAIL;
-	case DEV_NO_WORK:
-		return SEN_FATAL_ERROR;
-	case DEV_BAD_MESSAGE:
-		return SEN_USER_ERROR;
-	case DEV_QUEUE_FULL:
-		return SEN_QUEUE_FULL;
-	default:
-	case DEV_ONLINE:
-		break;
-	}
-	list_add_tail(&(caller_p->caller_liste), &(device_p->dev_caller_list));
-	device_p->dev_caller_count++;
-	return 0;
-}
-
-/**
- * Send puts the user's work on one of two queues:
- *   the pending queue if the send was successful
- *   the request queue if the send failed because device full or busy
- */
-static inline int
-z90crypt_send(struct work_element *we_p, const char *buf)
-{
-	int rv;
-
-	PDEBUG("PID %d\n", PID());
-
-	if (CHK_RDWRMASK(we_p->status[0]) != STAT_NOWORK) {
-		PDEBUG("PID %d tried to send more work but has outstanding "
-		       "work.\n", PID());
-		return -EWORKPEND;
-	}
-	we_p->devindex = -1; // Reset device number
-	spin_lock_irq(&queuespinlock);
-	rv = send_to_crypto_device(we_p);
-	switch (rv) {
-	case 0:
-		we_p->requestsent = jiffies;
-		we_p->audit[0] |= FP_SENT;
-		list_add_tail(&we_p->liste, &pending_list);
-		++pendingq_count;
-		we_p->audit[0] |= FP_PENDING;
-		break;
-	case SEN_BUSY:
-	case SEN_QUEUE_FULL:
-		rv = 0;
-		we_p->devindex = -1; // any device will do
-		we_p->requestsent = jiffies;
-		list_add_tail(&we_p->liste, &request_list);
-		++requestq_count;
-		we_p->audit[0] |= FP_REQUEST;
-		break;
-	case SEN_RETRY:
-		rv = -ERESTARTSYS;
-		break;
-	case SEN_NOT_AVAIL:
-		PRINTK("*** No devices available.\n");
-		rv = we_p->retcode = -ENODEV;
-		we_p->status[0] |= STAT_FAILED;
-		break;
-	case REC_OPERAND_INV:
-	case REC_OPERAND_SIZE:
-	case REC_EVEN_MOD:
-	case REC_INVALID_PAD:
-		rv = we_p->retcode = -EINVAL;
-		we_p->status[0] |= STAT_FAILED;
-		break;
-	default:
-		we_p->retcode = rv;
-		we_p->status[0] |= STAT_FAILED;
-		break;
-	}
-	if (rv != -ERESTARTSYS)
-		SET_RDWRMASK(we_p->status[0], STAT_WRITTEN);
-	spin_unlock_irq(&queuespinlock);
-	if (rv == 0)
-		tasklet_schedule(&reader_tasklet);
-	return rv;
-}
-
-/**
- * process_results copies the user's work from kernel space.
- */
-static inline int
-z90crypt_process_results(struct work_element *we_p, char __user *buf)
-{
-	int rv;
-
-	PDEBUG("we_p %p (PID %d)\n", we_p, PID());
-
-	LONG2DEVPTR(we_p->devindex)->dev_total_req_cnt++;
-	SET_RDWRMASK(we_p->status[0], STAT_READPEND);
-
-	rv = 0;
-	if (!we_p->buffer) {
-		PRINTK("we_p %p PID %d in STAT_READPEND: buffer NULL.\n",
-			we_p, PID());
-		rv = -ENOBUFF;
-	}
-
-	if (!rv)
-		if ((rv = copy_to_user(buf, we_p->buffer, we_p->buff_size))) {
-			PDEBUG("copy_to_user failed: rv = %d\n", rv);
-			rv = -EFAULT;
-		}
-
-	if (!rv)
-		rv = we_p->retcode;
-	if (!rv)
-		if (we_p->resp_buff_size
-		    &&	copy_to_user(we_p->resp_addr, we_p->resp_buff,
-				     we_p->resp_buff_size))
-			rv = -EFAULT;
-
-	SET_RDWRMASK(we_p->status[0], STAT_NOWORK);
-	return rv;
-}
-
-static unsigned char NULL_psmid[8] =
-{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00};
-
-/**
- * Used in device configuration functions
- */
-#define MAX_RESET 90
-
-/**
- * This is used only for PCICC support
- */
-static inline int
-is_PKCS11_padded(unsigned char *buffer, int length)
-{
-	int i;
-	if ((buffer[0] != 0x00) || (buffer[1] != 0x01))
-		return 0;
-	for (i = 2; i < length; i++)
-		if (buffer[i] != 0xFF)
-			break;
-	if ((i < 10) || (i == length))
-		return 0;
-	if (buffer[i] != 0x00)
-		return 0;
-	return 1;
-}
-
-/**
- * This is used only for PCICC support
- */
-static inline int
-is_PKCS12_padded(unsigned char *buffer, int length)
-{
-	int i;
-	if ((buffer[0] != 0x00) || (buffer[1] != 0x02))
-		return 0;
-	for (i = 2; i < length; i++)
-		if (buffer[i] == 0x00)
-			break;
-	if ((i < 10) || (i == length))
-		return 0;
-	if (buffer[i] != 0x00)
-		return 0;
-	return 1;
-}
-
-/**
- * builds struct caller and converts message from generic format to
- * device-dependent format
- * func is ICARSAMODEXPO or ICARSACRT
- * function is PCI_FUNC_KEY_ENCRYPT or PCI_FUNC_KEY_DECRYPT
- */
-static inline int
-build_caller(struct work_element *we_p, short function)
-{
-	int rv;
-	struct caller *caller_p = (struct caller *)we_p->requestptr;
-
-	if ((we_p->devtype != PCICC) && (we_p->devtype != PCICA) &&
-	    (we_p->devtype != PCIXCC_MCL2) && (we_p->devtype != PCIXCC_MCL3) &&
-	    (we_p->devtype != CEX2C) && (we_p->devtype != CEX2A))
-		return SEN_NOT_AVAIL;
-
-	memcpy(caller_p->caller_id, we_p->caller_id,
-	       sizeof(caller_p->caller_id));
-	caller_p->caller_dev_dep_req_p = caller_p->caller_dev_dep_req;
-	caller_p->caller_dev_dep_req_l = MAX_RESPONSE_SIZE;
-	caller_p->caller_buf_p = we_p->buffer;
-	INIT_LIST_HEAD(&(caller_p->caller_liste));
-
-	rv = convert_request(we_p->buffer, we_p->funccode, function,
-			     z90crypt.cdx, we_p->devtype,
-			     &caller_p->caller_dev_dep_req_l,
-			     caller_p->caller_dev_dep_req_p);
-	if (rv) {
-		if (rv == SEN_NOT_AVAIL)
-			PDEBUG("request can't be processed on hdwr avail\n");
-		else
-			PRINTK("Error from convert_request: %d\n", rv);
-	}
-	else
-		memcpy(&(caller_p->caller_dev_dep_req_p[4]), we_p->caller_id,8);
-	return rv;
-}
-
-static inline void
-unbuild_caller(struct device *device_p, struct caller *caller_p)
-{
-	if (!caller_p)
-		return;
-	if (caller_p->caller_liste.next && caller_p->caller_liste.prev)
-		if (!list_empty(&caller_p->caller_liste)) {
-			list_del_init(&caller_p->caller_liste);
-			device_p->dev_caller_count--;
-		}
-	memset(caller_p->caller_id, 0, sizeof(caller_p->caller_id));
-}
-
-static inline int
-get_crypto_request_buffer(struct work_element *we_p)
-{
-	struct ica_rsa_modexpo *mex_p;
-	struct ica_rsa_modexpo_crt *crt_p;
-	unsigned char *temp_buffer;
-	short function;
-	int rv;
-
-	mex_p =	(struct ica_rsa_modexpo *) we_p->buffer;
-	crt_p = (struct ica_rsa_modexpo_crt *) we_p->buffer;
-
-	PDEBUG("device type input = %d\n", we_p->devtype);
-
-	if (z90crypt.terminating)
-		return REC_NO_RESPONSE;
-	if (memcmp(we_p->caller_id, NULL_psmid, 8) == 0) {
-		PRINTK("psmid zeroes\n");
-		return SEN_FATAL_ERROR;
-	}
-	if (!we_p->buffer) {
-		PRINTK("buffer pointer NULL\n");
-		return SEN_USER_ERROR;
-	}
-	if (!we_p->requestptr) {
-		PRINTK("caller pointer NULL\n");
-		return SEN_USER_ERROR;
-	}
-
-	if ((we_p->devtype != PCICA) && (we_p->devtype != PCICC) &&
-	    (we_p->devtype != PCIXCC_MCL2) && (we_p->devtype != PCIXCC_MCL3) &&
-	    (we_p->devtype != CEX2C) && (we_p->devtype != CEX2A) &&
-	    (we_p->devtype != ANYDEV)) {
-		PRINTK("invalid device type\n");
-		return SEN_USER_ERROR;
-	}
-
-	if ((mex_p->inputdatalength < 1) ||
-	    (mex_p->inputdatalength > MAX_MOD_SIZE)) {
-		PRINTK("inputdatalength[%d] is not valid\n",
-		       mex_p->inputdatalength);
-		return SEN_USER_ERROR;
-	}
-
-	if (mex_p->outputdatalength < mex_p->inputdatalength) {
-		PRINTK("outputdatalength[%d] < inputdatalength[%d]\n",
-		       mex_p->outputdatalength, mex_p->inputdatalength);
-		return SEN_USER_ERROR;
-	}
-
-	if (!mex_p->inputdata || !mex_p->outputdata) {
-		PRINTK("inputdata[%p] or outputdata[%p] is NULL\n",
-		       mex_p->outputdata, mex_p->inputdata);
-		return SEN_USER_ERROR;
-	}
-
-	/**
-	 * As long as outputdatalength is big enough, we can set the
-	 * outputdatalength equal to the inputdatalength, since that is the
-	 * number of bytes we will copy in any case
-	 */
-	mex_p->outputdatalength = mex_p->inputdatalength;
-
-	rv = 0;
-	switch (we_p->funccode) {
-	case ICARSAMODEXPO:
-		if (!mex_p->b_key || !mex_p->n_modulus)
-			rv = SEN_USER_ERROR;
-		break;
-	case ICARSACRT:
-		if (!IS_EVEN(crt_p->inputdatalength)) {
-			PRINTK("inputdatalength[%d] is odd, CRT form\n",
-			       crt_p->inputdatalength);
-			rv = SEN_USER_ERROR;
-			break;
-		}
-		if (!crt_p->bp_key ||
-		    !crt_p->bq_key ||
-		    !crt_p->np_prime ||
-		    !crt_p->nq_prime ||
-		    !crt_p->u_mult_inv) {
-			PRINTK("CRT form, bad data: %p/%p/%p/%p/%p\n",
-			       crt_p->bp_key, crt_p->bq_key,
-			       crt_p->np_prime, crt_p->nq_prime,
-			       crt_p->u_mult_inv);
-			rv = SEN_USER_ERROR;
-		}
-		break;
-	default:
-		PRINTK("bad func = %d\n", we_p->funccode);
-		rv = SEN_USER_ERROR;
-		break;
-	}
-	if (rv != 0)
-		return rv;
-
-	if (select_device_type(&we_p->devtype, mex_p->inputdatalength) < 0)
-		return SEN_NOT_AVAIL;
-
-	temp_buffer = (unsigned char *)we_p + sizeof(struct work_element) +
-		      sizeof(struct caller);
-	if (copy_from_user(temp_buffer, mex_p->inputdata,
-			   mex_p->inputdatalength) != 0)
-		return SEN_RELEASED;
-
-	function = PCI_FUNC_KEY_ENCRYPT;
-	switch (we_p->devtype) {
-	/* PCICA and CEX2A do everything with a simple RSA mod-expo operation */
-	case PCICA:
-	case CEX2A:
-		function = PCI_FUNC_KEY_ENCRYPT;
-		break;
-	/**
-	 * PCIXCC_MCL2 does all Mod-Expo form with a simple RSA mod-expo
-	 * operation, and all CRT forms with a PKCS-1.2 format decrypt.
-	 * PCIXCC_MCL3 and CEX2C do all Mod-Expo and CRT forms with a simple RSA
-	 * mod-expo operation
-	 */
-	case PCIXCC_MCL2:
-		if (we_p->funccode == ICARSAMODEXPO)
-			function = PCI_FUNC_KEY_ENCRYPT;
-		else
-			function = PCI_FUNC_KEY_DECRYPT;
-		break;
-	case PCIXCC_MCL3:
-	case CEX2C:
-		if (we_p->funccode == ICARSAMODEXPO)
-			function = PCI_FUNC_KEY_ENCRYPT;
-		else
-			function = PCI_FUNC_KEY_DECRYPT;
-		break;
-	/**
-	 * PCICC does everything as a PKCS-1.2 format request
-	 */
-	case PCICC:
-		/* PCICC cannot handle input that is is PKCS#1.1 padded */
-		if (is_PKCS11_padded(temp_buffer, mex_p->inputdatalength)) {
-			return SEN_NOT_AVAIL;
-		}
-		if (we_p->funccode == ICARSAMODEXPO) {
-			if (is_PKCS12_padded(temp_buffer,
-					     mex_p->inputdatalength))
-				function = PCI_FUNC_KEY_ENCRYPT;
-			else
-				function = PCI_FUNC_KEY_DECRYPT;
-		} else
-			/* all CRT forms are decrypts */
-			function = PCI_FUNC_KEY_DECRYPT;
-		break;
-	}
-	PDEBUG("function: %04x\n", function);
-	rv = build_caller(we_p, function);
-	PDEBUG("rv from build_caller = %d\n", rv);
-	return rv;
-}
-
-static inline int
-z90crypt_prepare(struct work_element *we_p, unsigned int funccode,
-		 const char __user *buffer)
-{
-	int rv;
-
-	we_p->devindex = -1;
-	if (funccode == ICARSAMODEXPO)
-		we_p->buff_size = sizeof(struct ica_rsa_modexpo);
-	else
-		we_p->buff_size = sizeof(struct ica_rsa_modexpo_crt);
-
-	if (copy_from_user(we_p->buffer, buffer, we_p->buff_size))
-		return -EFAULT;
-
-	we_p->audit[0] |= FP_COPYFROM;
-	SET_RDWRMASK(we_p->status[0], STAT_WRITTEN);
-	we_p->funccode = funccode;
-	we_p->devtype = -1;
-	we_p->audit[0] |= FP_BUFFREQ;
-	rv = get_crypto_request_buffer(we_p);
-	switch (rv) {
-	case 0:
-		we_p->audit[0] |= FP_BUFFGOT;
-		break;
-	case SEN_USER_ERROR:
-		rv = -EINVAL;
-		break;
-	case SEN_QUEUE_FULL:
-		rv = 0;
-		break;
-	case SEN_RELEASED:
-		rv = -EFAULT;
-		break;
-	case REC_NO_RESPONSE:
-		rv = -ENODEV;
-		break;
-	case SEN_NOT_AVAIL:
-	case EGETBUFF:
-		rv = -EGETBUFF;
-		break;
-	default:
-		PRINTK("rv = %d\n", rv);
-		rv = -EGETBUFF;
-		break;
-	}
-	if (CHK_RDWRMASK(we_p->status[0]) == STAT_WRITTEN)
-		SET_RDWRMASK(we_p->status[0], STAT_DEFAULT);
-	return rv;
-}
-
-static inline void
-purge_work_element(struct work_element *we_p)
-{
-	struct list_head *lptr;
-
-	spin_lock_irq(&queuespinlock);
-	list_for_each(lptr, &request_list) {
-		if (lptr == &we_p->liste) {
-			list_del_init(lptr);
-			requestq_count--;
-			break;
-		}
-	}
-	list_for_each(lptr, &pending_list) {
-		if (lptr == &we_p->liste) {
-			list_del_init(lptr);
-			pendingq_count--;
-			break;
-		}
-	}
-	spin_unlock_irq(&queuespinlock);
-}
-
-/**
- * Build the request and send it.
- */
-static inline int
-z90crypt_rsa(struct priv_data *private_data_p, pid_t pid,
-	     unsigned int cmd, unsigned long arg)
-{
-	struct work_element *we_p;
-	int rv;
-
-	if ((rv = allocate_work_element(&we_p, private_data_p, pid))) {
-		PDEBUG("PID %d: allocate_work_element returned ENOMEM\n", pid);
-		return rv;
-	}
-	if ((rv = z90crypt_prepare(we_p, cmd, (const char __user *)arg)))
-		PDEBUG("PID %d: rv = %d from z90crypt_prepare\n", pid, rv);
-	if (!rv)
-		if ((rv = z90crypt_send(we_p, (const char *)arg)))
-			PDEBUG("PID %d: rv %d from z90crypt_send.\n", pid, rv);
-	if (!rv) {
-		we_p->audit[0] |= FP_ASLEEP;
-		wait_event(we_p->waitq, atomic_read(&we_p->alarmrung));
-		we_p->audit[0] |= FP_AWAKE;
-		rv = we_p->retcode;
-	}
-	if (!rv)
-		rv = z90crypt_process_results(we_p, (char __user *)arg);
-
-	if ((we_p->status[0] & STAT_FAILED)) {
-		switch (rv) {
-		/**
-		 * EINVAL *after* receive is almost always a padding error or
-		 * length error issued by a coprocessor (not an accelerator).
-		 * We convert this return value to -EGETBUFF which should
-		 * trigger a fallback to software.
-		 */
-		case -EINVAL:
-			if ((we_p->devtype != PCICA) &&
-			    (we_p->devtype != CEX2A))
-				rv = -EGETBUFF;
-			break;
-		case -ETIMEOUT:
-			if (z90crypt.mask.st_count > 0)
-				rv = -ERESTARTSYS; // retry with another
-			else
-				rv = -ENODEV; // no cards left
-		/* fall through to clean up request queue */
-		case -ERESTARTSYS:
-		case -ERELEASED:
-			switch (CHK_RDWRMASK(we_p->status[0])) {
-			case STAT_WRITTEN:
-				purge_work_element(we_p);
-				break;
-			case STAT_READPEND:
-			case STAT_NOWORK:
-			default:
-				break;
-			}
-			break;
-		default:
-			we_p->status[0] ^= STAT_FAILED;
-			break;
-		}
-	}
-	free_page((long)we_p);
-	return rv;
-}
-
-/**
- * This function is a little long, but it's really just one large switch
- * statement.
- */
-static long
-z90crypt_unlocked_ioctl(struct file *filp, unsigned int cmd, unsigned long arg)
-{
-	struct priv_data *private_data_p = filp->private_data;
-	unsigned char *status;
-	unsigned char *qdepth;
-	unsigned int *reqcnt;
-	struct ica_z90_status *pstat;
-	int ret, i, loopLim, tempstat;
-	static int deprecated_msg_count1 = 0;
-	static int deprecated_msg_count2 = 0;
-
-	PDEBUG("filp %p (PID %d), cmd 0x%08X\n", filp, PID(), cmd);
-	PDEBUG("cmd 0x%08X: dir %s, size 0x%04X, type 0x%02X, nr 0x%02X\n",
-		cmd,
-		!_IOC_DIR(cmd) ? "NO"
-		: ((_IOC_DIR(cmd) == (_IOC_READ|_IOC_WRITE)) ? "RW"
-		: ((_IOC_DIR(cmd) == _IOC_READ) ? "RD"
-		: "WR")),
-		_IOC_SIZE(cmd), _IOC_TYPE(cmd), _IOC_NR(cmd));
-
-	if (_IOC_TYPE(cmd) != Z90_IOCTL_MAGIC) {
-		PRINTK("cmd 0x%08X contains bad magic\n", cmd);
-		return -ENOTTY;
-	}
-
-	ret = 0;
-	switch (cmd) {
-	case ICARSAMODEXPO:
-	case ICARSACRT:
-		if (quiesce_z90crypt) {
-			ret = -EQUIESCE;
-			break;
-		}
-		ret = -ENODEV; // Default if no devices
-		loopLim = z90crypt.hdware_info->hdware_mask.st_count -
-			(z90crypt.hdware_info->hdware_mask.disabled_count +
-			 z90crypt.hdware_info->hdware_mask.user_disabled_count);
-		for (i = 0; i < loopLim; i++) {
-			ret = z90crypt_rsa(private_data_p, PID(), cmd, arg);
-			if (ret != -ERESTARTSYS)
-				break;
-		}
-		if (ret == -ERESTARTSYS)
-			ret = -ENODEV;
-		break;
-
-	case Z90STAT_TOTALCOUNT:
-		tempstat = get_status_totalcount();
-		if (copy_to_user((int __user *)arg, &tempstat,sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_PCICACOUNT:
-		tempstat = get_status_PCICAcount();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_PCICCCOUNT:
-		tempstat = get_status_PCICCcount();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_PCIXCCMCL2COUNT:
-		tempstat = get_status_PCIXCCMCL2count();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_PCIXCCMCL3COUNT:
-		tempstat = get_status_PCIXCCMCL3count();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_CEX2CCOUNT:
-		tempstat = get_status_CEX2Ccount();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_CEX2ACOUNT:
-		tempstat = get_status_CEX2Acount();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_REQUESTQ_COUNT:
-		tempstat = get_status_requestq_count();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_PENDINGQ_COUNT:
-		tempstat = get_status_pendingq_count();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_TOTALOPEN_COUNT:
-		tempstat = get_status_totalopen_count();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_DOMAIN_INDEX:
-		tempstat = get_status_domain_index();
-		if (copy_to_user((int __user *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90STAT_STATUS_MASK:
-		status = kmalloc(Z90CRYPT_NUM_APS, GFP_KERNEL);
-		if (!status) {
-			PRINTK("kmalloc for status failed!\n");
-			ret = -ENOMEM;
-			break;
-		}
-		get_status_status_mask(status);
-		if (copy_to_user((char __user *) arg, status, Z90CRYPT_NUM_APS)
-									!= 0)
-			ret = -EFAULT;
-		kfree(status);
-		break;
-
-	case Z90STAT_QDEPTH_MASK:
-		qdepth = kmalloc(Z90CRYPT_NUM_APS, GFP_KERNEL);
-		if (!qdepth) {
-			PRINTK("kmalloc for qdepth failed!\n");
-			ret = -ENOMEM;
-			break;
-		}
-		get_status_qdepth_mask(qdepth);
-		if (copy_to_user((char __user *) arg, qdepth, Z90CRYPT_NUM_APS) != 0)
-			ret = -EFAULT;
-		kfree(qdepth);
-		break;
-
-	case Z90STAT_PERDEV_REQCNT:
-		reqcnt = kmalloc(sizeof(int) * Z90CRYPT_NUM_APS, GFP_KERNEL);
-		if (!reqcnt) {
-			PRINTK("kmalloc for reqcnt failed!\n");
-			ret = -ENOMEM;
-			break;
-		}
-		get_status_perdevice_reqcnt(reqcnt);
-		if (copy_to_user((char __user *) arg, reqcnt,
-				 Z90CRYPT_NUM_APS * sizeof(int)) != 0)
-			ret = -EFAULT;
-		kfree(reqcnt);
-		break;
-
-		/* THIS IS DEPRECATED.	USE THE NEW STATUS CALLS */
-	case ICAZ90STATUS:
-		if (deprecated_msg_count1 < 20) {
-			PRINTK("deprecated call to ioctl (ICAZ90STATUS)!\n");
-			deprecated_msg_count1++;
-			if (deprecated_msg_count1 == 20)
-				PRINTK("No longer issuing messages related to "
-				       "deprecated call to ICAZ90STATUS.\n");
-		}
-
-		pstat = kmalloc(sizeof(struct ica_z90_status), GFP_KERNEL);
-		if (!pstat) {
-			PRINTK("kmalloc for pstat failed!\n");
-			ret = -ENOMEM;
-			break;
-		}
-
-		pstat->totalcount	 = get_status_totalcount();
-		pstat->leedslitecount	 = get_status_PCICAcount();
-		pstat->leeds2count	 = get_status_PCICCcount();
-		pstat->requestqWaitCount = get_status_requestq_count();
-		pstat->pendingqWaitCount = get_status_pendingq_count();
-		pstat->totalOpenCount	 = get_status_totalopen_count();
-		pstat->cryptoDomain	 = get_status_domain_index();
-		get_status_status_mask(pstat->status);
-		get_status_qdepth_mask(pstat->qdepth);
-
-		if (copy_to_user((struct ica_z90_status __user *) arg, pstat,
-				 sizeof(struct ica_z90_status)) != 0)
-			ret = -EFAULT;
-		kfree(pstat);
-		break;
-
-		/* THIS IS DEPRECATED.	USE THE NEW STATUS CALLS */
-	case Z90STAT_PCIXCCCOUNT:
-		if (deprecated_msg_count2 < 20) {
-			PRINTK("deprecated ioctl (Z90STAT_PCIXCCCOUNT)!\n");
-			deprecated_msg_count2++;
-			if (deprecated_msg_count2 == 20)
-				PRINTK("No longer issuing messages about depre"
-				       "cated ioctl Z90STAT_PCIXCCCOUNT.\n");
-		}
-
-		tempstat = get_status_PCIXCCcount();
-		if (copy_to_user((int *)arg, &tempstat, sizeof(int)) != 0)
-			ret = -EFAULT;
-		break;
-
-	case Z90QUIESCE:
-		if (current->euid != 0) {
-			PRINTK("QUIESCE fails: euid %d\n",
-			       current->euid);
-			ret = -EACCES;
-		} else {
-			PRINTK("QUIESCE device from PID %d\n", PID());
-			quiesce_z90crypt = 1;
-		}
-		break;
-
-	default:
-		/* user passed an invalid IOCTL number */
-		PDEBUG("cmd 0x%08X contains invalid ioctl code\n", cmd);
-		ret = -ENOTTY;
-		break;
-	}
-
-	return ret;
-}
-
-static inline int
-sprintcl(unsigned char *outaddr, unsigned char *addr, unsigned int len)
-{
-	int hl, i;
-
-	hl = 0;
-	for (i = 0; i < len; i++)
-		hl += sprintf(outaddr+hl, "%01x", (unsigned int) addr[i]);
-	hl += sprintf(outaddr+hl, " ");
-
-	return hl;
-}
-
-static inline int
-sprintrw(unsigned char *outaddr, unsigned char *addr, unsigned int len)
-{
-	int hl, inl, c, cx;
-
-	hl = sprintf(outaddr, "	   ");
-	inl = 0;
-	for (c = 0; c < (len / 16); c++) {
-		hl += sprintcl(outaddr+hl, addr+inl, 16);
-		inl += 16;
-	}
-
-	cx = len%16;
-	if (cx) {
-		hl += sprintcl(outaddr+hl, addr+inl, cx);
-		inl += cx;
-	}
-
-	hl += sprintf(outaddr+hl, "\n");
-
-	return hl;
-}
-
-static inline int
-sprinthx(unsigned char *title, unsigned char *outaddr,
-	 unsigned char *addr, unsigned int len)
-{
-	int hl, inl, r, rx;
-
-	hl = sprintf(outaddr, "\n%s\n", title);
-	inl = 0;
-	for (r = 0; r < (len / 64); r++) {
-		hl += sprintrw(outaddr+hl, addr+inl, 64);
-		inl += 64;
-	}
-	rx = len % 64;
-	if (rx) {
-		hl += sprintrw(outaddr+hl, addr+inl, rx);
-		inl += rx;
-	}
-
-	hl += sprintf(outaddr+hl, "\n");
-
-	return hl;
-}
-
-static inline int
-sprinthx4(unsigned char *title, unsigned char *outaddr,
-	  unsigned int *array, unsigned int len)
-{
-	int hl, r;
-
-	hl = sprintf(outaddr, "\n%s\n", title);
-
-	for (r = 0; r < len; r++) {
-		if ((r % 8) == 0)
-			hl += sprintf(outaddr+hl, "    ");
-		hl += sprintf(outaddr+hl, "%08X ", array[r]);
-		if ((r % 8) == 7)
-			hl += sprintf(outaddr+hl, "\n");
-	}
-
-	hl += sprintf(outaddr+hl, "\n");
-
-	return hl;
-}
-
-static int
-z90crypt_status(char *resp_buff, char **start, off_t offset,
-		int count, int *eof, void *data)
-{
-	unsigned char *workarea;
-	int len;
-
-	/* resp_buff is a page. Use the right half for a work area */
-	workarea = resp_buff+2000;
-	len = 0;
-	len += sprintf(resp_buff+len, "\nz90crypt version: %d.%d.%d\n",
-		z90crypt_VERSION, z90crypt_RELEASE, z90crypt_VARIANT);
-	len += sprintf(resp_buff+len, "Cryptographic domain: %d\n",
-		get_status_domain_index());
-	len += sprintf(resp_buff+len, "Total device count: %d\n",
-		get_status_totalcount());
-	len += sprintf(resp_buff+len, "PCICA count: %d\n",
-		get_status_PCICAcount());
-	len += sprintf(resp_buff+len, "PCICC count: %d\n",
-		get_status_PCICCcount());
-	len += sprintf(resp_buff+len, "PCIXCC MCL2 count: %d\n",
-		get_status_PCIXCCMCL2count());
-	len += sprintf(resp_buff+len, "PCIXCC MCL3 count: %d\n",
-		get_status_PCIXCCMCL3count());
-	len += sprintf(resp_buff+len, "CEX2C count: %d\n",
-		get_status_CEX2Ccount());
-	len += sprintf(resp_buff+len, "CEX2A count: %d\n",
-		get_status_CEX2Acount());
-	len += sprintf(resp_buff+len, "requestq count: %d\n",
-		get_status_requestq_count());
-	len += sprintf(resp_buff+len, "pendingq count: %d\n",
-		get_status_pendingq_count());
-	len += sprintf(resp_buff+len, "Total open handles: %d\n\n",
-		get_status_totalopen_count());
-	len += sprinthx(
-		"Online devices: 1=PCICA 2=PCICC 3=PCIXCC(MCL2) "
-		"4=PCIXCC(MCL3) 5=CEX2C 6=CEX2A",
-		resp_buff+len,
-		get_status_status_mask(workarea),
-		Z90CRYPT_NUM_APS);
-	len += sprinthx("Waiting work element counts",
-		resp_buff+len,
-		get_status_qdepth_mask(workarea),
-		Z90CRYPT_NUM_APS);
-	len += sprinthx4(
-		"Per-device successfully completed request counts",
-		resp_buff+len,
-		get_status_perdevice_reqcnt((unsigned int *)workarea),
-		Z90CRYPT_NUM_APS);
-	*eof = 1;
-	memset(workarea, 0, Z90CRYPT_NUM_APS * sizeof(unsigned int));
-	return len;
-}
-
-static inline void
-disable_card(int card_index)
-{
-	struct device *devp;
-
-	devp = LONG2DEVPTR(card_index);
-	if (!devp || devp->user_disabled)
-		return;
-	devp->user_disabled = 1;
-	z90crypt.hdware_info->hdware_mask.user_disabled_count++;
-	if (devp->dev_type == -1)
-		return;
-	z90crypt.hdware_info->type_mask[devp->dev_type].user_disabled_count++;
-}
-
-static inline void
-enable_card(int card_index)
-{
-	struct device *devp;
-
-	devp = LONG2DEVPTR(card_index);
-	if (!devp || !devp->user_disabled)
-		return;
-	devp->user_disabled = 0;
-	z90crypt.hdware_info->hdware_mask.user_disabled_count--;
-	if (devp->dev_type == -1)
-		return;
-	z90crypt.hdware_info->type_mask[devp->dev_type].user_disabled_count--;
-}
-
-static int
-z90crypt_status_write(struct file *file, const char __user *buffer,
-		      unsigned long count, void *data)
-{
-	int j, eol;
-	unsigned char *lbuf, *ptr;
-	unsigned int local_count;
-
-#define LBUFSIZE 1200
-	lbuf = kmalloc(LBUFSIZE, GFP_KERNEL);
-	if (!lbuf) {
-		PRINTK("kmalloc failed!\n");
-		return 0;
-	}
-
-	if (count <= 0)
-		return 0;
-
-	local_count = UMIN((unsigned int)count, LBUFSIZE-1);
-
-	if (copy_from_user(lbuf, buffer, local_count) != 0) {
-		kfree(lbuf);
-		return -EFAULT;
-	}
-
-	lbuf[local_count] = '\0';
-
-	ptr = strstr(lbuf, "Online devices");
-	if (ptr == 0) {
-		PRINTK("Unable to parse data (missing \"Online devices\")\n");
-		kfree(lbuf);
-		return count;
-	}
-
-	ptr = strstr(ptr, "\n");
-	if (ptr == 0) {
-		PRINTK("Unable to parse data (missing newline after \"Online devices\")\n");
-		kfree(lbuf);
-		return count;
-	}
-	ptr++;
-
-	if (strstr(ptr, "Waiting work element counts") == NULL) {
-		PRINTK("Unable to parse data (missing \"Waiting work element counts\")\n");
-		kfree(lbuf);
-		return count;
-	}
-
-	j = 0;
-	eol = 0;
-	while ((j < 64) && (*ptr != '\0')) {
-		switch (*ptr) {
-		case '\t':
-		case ' ':
-			break;
-		case '\n':
-		default:
-			eol = 1;
-			break;
-		case '0':	// no device
-		case '1':	// PCICA
-		case '2':	// PCICC
-		case '3':	// PCIXCC_MCL2
-		case '4':	// PCIXCC_MCL3
-		case '5':	// CEX2C
-		case '6':       // CEX2A
-			j++;
-			break;
-		case 'd':
-		case 'D':
-			disable_card(j);
-			j++;
-			break;
-		case 'e':
-		case 'E':
-			enable_card(j);
-			j++;
-			break;
-		}
-		if (eol)
-			break;
-		ptr++;
-	}
-
-	kfree(lbuf);
-	return count;
-}
-
-/**
- * Functions that run under a timer, with no process id
- *
- * The task functions:
- *     z90crypt_reader_task
- *	 helper_send_work
- *	 helper_handle_work_element
- *	 helper_receive_rc
- *     z90crypt_config_task
- *     z90crypt_cleanup_task
- *
- * Helper functions:
- *     z90crypt_schedule_reader_timer
- *     z90crypt_schedule_reader_task
- *     z90crypt_schedule_config_task
- *     z90crypt_schedule_cleanup_task
- */
-static inline int
-receive_from_crypto_device(int index, unsigned char *psmid, int *buff_len_p,
-			   unsigned char *buff, unsigned char __user **dest_p_p)
-{
-	int dv, rv;
-	struct device *dev_ptr;
-	struct caller *caller_p;
-	struct ica_rsa_modexpo *icaMsg_p;
-	struct list_head *ptr, *tptr;
-
-	memcpy(psmid, NULL_psmid, sizeof(NULL_psmid));
-
-	if (z90crypt.terminating)
-		return REC_FATAL_ERROR;
-
-	caller_p = 0;
-	dev_ptr = z90crypt.device_p[index];
-	rv = 0;
-	do {
-		if (!dev_ptr || dev_ptr->disabled) {
-			rv = REC_NO_WORK; // a disabled device can't return work
-			break;
-		}
-		if (dev_ptr->dev_self_x != index) {
-			PRINTKC("Corrupt dev ptr\n");
-			z90crypt.terminating = 1;
-			rv = REC_FATAL_ERROR;
-			break;
-		}
-		if (!dev_ptr->dev_resp_l || !dev_ptr->dev_resp_p) {
-			dv = DEV_REC_EXCEPTION;
-			PRINTK("dev_resp_l = %d, dev_resp_p = %p\n",
-			       dev_ptr->dev_resp_l, dev_ptr->dev_resp_p);
-		} else {
-			PDEBUG("Dequeue called for device %d\n", index);
-			dv = receive_from_AP(index, z90crypt.cdx,
-					     dev_ptr->dev_resp_l,
-					     dev_ptr->dev_resp_p, psmid);
-		}
-		switch (dv) {
-		case DEV_REC_EXCEPTION:
-			rv = REC_FATAL_ERROR;
-			z90crypt.terminating = 1;
-			PRINTKC("Exception in receive from device %d\n",
-				index);
-			break;
-		case DEV_ONLINE:
-			rv = 0;
-			break;
-		case DEV_EMPTY:
-			rv = REC_EMPTY;
-			break;
-		case DEV_NO_WORK:
-			rv = REC_NO_WORK;
-			break;
-		case DEV_BAD_MESSAGE:
-		case DEV_GONE:
-		case REC_HARDWAR_ERR:
-		default:
-			rv = REC_NO_RESPONSE;
-			break;
-		}
-		if (rv)
-			break;
-		if (dev_ptr->dev_caller_count <= 0) {
-			rv = REC_USER_GONE;
-			break;
-	        }
-
-		list_for_each_safe(ptr, tptr, &dev_ptr->dev_caller_list) {
-			caller_p = list_entry(ptr, struct caller, caller_liste);
-			if (!memcmp(caller_p->caller_id, psmid,
-				    sizeof(caller_p->caller_id))) {
-				if (!list_empty(&caller_p->caller_liste)) {
-					list_del_init(ptr);
-					dev_ptr->dev_caller_count--;
-					break;
-				}
-			}
-			caller_p = 0;
-		}
-		if (!caller_p) {
-			PRINTKW("Unable to locate PSMID %02X%02X%02X%02X%02X"
-				"%02X%02X%02X in device list\n",
-				psmid[0], psmid[1], psmid[2], psmid[3],
-				psmid[4], psmid[5], psmid[6], psmid[7]);
-			rv = REC_USER_GONE;
-			break;
-		}
-
-		PDEBUG("caller_p after successful receive: %p\n", caller_p);
-		rv = convert_response(dev_ptr->dev_resp_p,
-				      caller_p->caller_buf_p, buff_len_p, buff);
-		switch (rv) {
-		case REC_USE_PCICA:
-			break;
-		case REC_OPERAND_INV:
-		case REC_OPERAND_SIZE:
-		case REC_EVEN_MOD:
-		case REC_INVALID_PAD:
-			PDEBUG("device %d: 'user error' %d\n", index, rv);
-			break;
-		case WRONG_DEVICE_TYPE:
-		case REC_HARDWAR_ERR:
-		case REC_BAD_MESSAGE:
-			PRINTKW("device %d: hardware error %d\n", index, rv);
-			rv = REC_NO_RESPONSE;
-			break;
-		default:
-			PDEBUG("device %d: rv = %d\n", index, rv);
-			break;
-		}
-	} while (0);
-
-	switch (rv) {
-	case 0:
-		PDEBUG("Successful receive from device %d\n", index);
-		icaMsg_p = (struct ica_rsa_modexpo *)caller_p->caller_buf_p;
-		*dest_p_p = icaMsg_p->outputdata;
-		if (*buff_len_p == 0)
-			PRINTK("Zero *buff_len_p\n");
-		break;
-	case REC_NO_RESPONSE:
-		PRINTKW("Removing device %d from availability\n", index);
-		remove_device(dev_ptr);
-		break;
-	}
-
-	if (caller_p)
-		unbuild_caller(dev_ptr, caller_p);
-
-	return rv;
-}
-
-static inline void
-helper_send_work(int index)
-{
-	struct work_element *rq_p;
-	int rv;
-
-	if (list_empty(&request_list))
-		return;
-	requestq_count--;
-	rq_p = list_entry(request_list.next, struct work_element, liste);
-	list_del_init(&rq_p->liste);
-	rq_p->audit[1] |= FP_REMREQUEST;
-	if (rq_p->devtype == SHRT2DEVPTR(index)->dev_type) {
-		rq_p->devindex = SHRT2LONG(index);
-		rv = send_to_crypto_device(rq_p);
-		if (rv == 0) {
-			rq_p->requestsent = jiffies;
-			rq_p->audit[0] |= FP_SENT;
-			list_add_tail(&rq_p->liste, &pending_list);
-			++pendingq_count;
-			rq_p->audit[0] |= FP_PENDING;
-		} else {
-			switch (rv) {
-			case REC_OPERAND_INV:
-			case REC_OPERAND_SIZE:
-			case REC_EVEN_MOD:
-			case REC_INVALID_PAD:
-				rq_p->retcode = -EINVAL;
-				break;
-			case SEN_NOT_AVAIL:
-			case SEN_RETRY:
-			case REC_NO_RESPONSE:
-			default:
-				if (z90crypt.mask.st_count > 1)
-					rq_p->retcode =
-						-ERESTARTSYS;
-				else
-					rq_p->retcode = -ENODEV;
-				break;
-			}
-			rq_p->status[0] |= STAT_FAILED;
-			rq_p->audit[1] |= FP_AWAKENING;
-			atomic_set(&rq_p->alarmrung, 1);
-			wake_up(&rq_p->waitq);
-		}
-	} else {
-		if (z90crypt.mask.st_count > 1)
-			rq_p->retcode = -ERESTARTSYS;
-		else
-			rq_p->retcode = -ENODEV;
-		rq_p->status[0] |= STAT_FAILED;
-		rq_p->audit[1] |= FP_AWAKENING;
-		atomic_set(&rq_p->alarmrung, 1);
-		wake_up(&rq_p->waitq);
-	}
-}
-
-static inline void
-helper_handle_work_element(int index, unsigned char psmid[8], int rc,
-			   int buff_len, unsigned char *buff,
-			   unsigned char __user *resp_addr)
-{
-	struct work_element *pq_p;
-	struct list_head *lptr, *tptr;
-
-	pq_p = 0;
-	list_for_each_safe(lptr, tptr, &pending_list) {
-		pq_p = list_entry(lptr, struct work_element, liste);
-		if (!memcmp(pq_p->caller_id, psmid, sizeof(pq_p->caller_id))) {
-			list_del_init(lptr);
-			pendingq_count--;
-			pq_p->audit[1] |= FP_NOTPENDING;
-			break;
-		}
-		pq_p = 0;
-	}
-
-	if (!pq_p) {
-		PRINTK("device %d has work but no caller exists on pending Q\n",
-		       SHRT2LONG(index));
-		return;
-	}
-
-	switch (rc) {
-		case 0:
-			pq_p->resp_buff_size = buff_len;
-			pq_p->audit[1] |= FP_RESPSIZESET;
-			if (buff_len) {
-				pq_p->resp_addr = resp_addr;
-				pq_p->audit[1] |= FP_RESPADDRCOPIED;
-				memcpy(pq_p->resp_buff, buff, buff_len);
-				pq_p->audit[1] |= FP_RESPBUFFCOPIED;
-			}
-			break;
-		case REC_OPERAND_INV:
-		case REC_OPERAND_SIZE:
-		case REC_EVEN_MOD:
-		case REC_INVALID_PAD:
-			PDEBUG("-EINVAL after application error %d\n", rc);
-			pq_p->retcode = -EINVAL;
-			pq_p->status[0] |= STAT_FAILED;
-			break;
-		case REC_USE_PCICA:
-			pq_p->retcode = -ERESTARTSYS;
-			pq_p->status[0] |= STAT_FAILED;
-			break;
-		case REC_NO_RESPONSE:
-		default:
-			if (z90crypt.mask.st_count > 1)
-				pq_p->retcode = -ERESTARTSYS;
-			else
-				pq_p->retcode = -ENODEV;
-			pq_p->status[0] |= STAT_FAILED;
-			break;
-	}
-	if ((pq_p->status[0] != STAT_FAILED) || (pq_p->retcode != -ERELEASED)) {
-		pq_p->audit[1] |= FP_AWAKENING;
-		atomic_set(&pq_p->alarmrung, 1);
-		wake_up(&pq_p->waitq);
-	}
-}
-
-/**
- * return TRUE if the work element should be removed from the queue
- */
-static inline int
-helper_receive_rc(int index, int *rc_p)
-{
-	switch (*rc_p) {
-	case 0:
-	case REC_OPERAND_INV:
-	case REC_OPERAND_SIZE:
-	case REC_EVEN_MOD:
-	case REC_INVALID_PAD:
-	case REC_USE_PCICA:
-		break;
-
-	case REC_BUSY:
-	case REC_NO_WORK:
-	case REC_EMPTY:
-	case REC_RETRY_DEV:
-	case REC_FATAL_ERROR:
-		return 0;
-
-	case REC_NO_RESPONSE:
-		break;
-
-	default:
-		PRINTK("rc %d, device %d converted to REC_NO_RESPONSE\n",
-		       *rc_p, SHRT2LONG(index));
-		*rc_p = REC_NO_RESPONSE;
-		break;
-	}
-	return 1;
-}
-
-static inline void
-z90crypt_schedule_reader_timer(void)
-{
-	if (timer_pending(&reader_timer))
-		return;
-	if (mod_timer(&reader_timer, jiffies+(READERTIME*HZ/1000)) != 0)
-		PRINTK("Timer pending while modifying reader timer\n");
-}
-
-static void
-z90crypt_reader_task(unsigned long ptr)
-{
-	int workavail, index, rc, buff_len;
-	unsigned char	psmid[8];
-	unsigned char __user *resp_addr;
-	static unsigned char buff[1024];
-
-	/**
-	 * we use workavail = 2 to ensure 2 passes with nothing dequeued before
-	 * exiting the loop. If (pendingq_count+requestq_count) == 0 after the
-	 * loop, there is no work remaining on the queues.
-	 */
-	resp_addr = 0;
-	workavail = 2;
-	buff_len = 0;
-	while (workavail) {
-		workavail--;
-		rc = 0;
-		spin_lock_irq(&queuespinlock);
-		memset(buff, 0x00, sizeof(buff));
-
-		/* Dequeue once from each device in round robin. */
-		for (index = 0; index < z90crypt.mask.st_count; index++) {
-			PDEBUG("About to receive.\n");
-			rc = receive_from_crypto_device(SHRT2LONG(index),
-							psmid,
-							&buff_len,
-							buff,
-							&resp_addr);
-			PDEBUG("Dequeued: rc = %d.\n", rc);
-
-			if (helper_receive_rc(index, &rc)) {
-				if (rc != REC_NO_RESPONSE) {
-					helper_send_work(index);
-					workavail = 2;
-				}
-
-				helper_handle_work_element(index, psmid, rc,
-							   buff_len, buff,
-							   resp_addr);
-			}
-
-			if (rc == REC_FATAL_ERROR)
-				PRINTKW("REC_FATAL_ERROR from device %d!\n",
-					SHRT2LONG(index));
-		}
-		spin_unlock_irq(&queuespinlock);
-	}
-
-	if (pendingq_count + requestq_count)
-		z90crypt_schedule_reader_timer();
-}
-
-static inline void
-z90crypt_schedule_config_task(unsigned int expiration)
-{
-	if (timer_pending(&config_timer))
-		return;
-	if (mod_timer(&config_timer, jiffies+(expiration*HZ)) != 0)
-		PRINTK("Timer pending while modifying config timer\n");
-}
-
-static void
-z90crypt_config_task(unsigned long ptr)
-{
-	int rc;
-
-	PDEBUG("jiffies %ld\n", jiffies);
-
-	if ((rc = refresh_z90crypt(&z90crypt.cdx)))
-		PRINTK("Error %d detected in refresh_z90crypt.\n", rc);
-	/* If return was fatal, don't bother reconfiguring */
-	if ((rc != TSQ_FATAL_ERROR) && (rc != RSQ_FATAL_ERROR))
-		z90crypt_schedule_config_task(CONFIGTIME);
-}
-
-static inline void
-z90crypt_schedule_cleanup_task(void)
-{
-	if (timer_pending(&cleanup_timer))
-		return;
-	if (mod_timer(&cleanup_timer, jiffies+(CLEANUPTIME*HZ)) != 0)
-		PRINTK("Timer pending while modifying cleanup timer\n");
-}
-
-static inline void
-helper_drain_queues(void)
-{
-	struct work_element *pq_p;
-	struct list_head *lptr, *tptr;
-
-	list_for_each_safe(lptr, tptr, &pending_list) {
-		pq_p = list_entry(lptr, struct work_element, liste);
-		pq_p->retcode = -ENODEV;
-		pq_p->status[0] |= STAT_FAILED;
-		unbuild_caller(LONG2DEVPTR(pq_p->devindex),
-			       (struct caller *)pq_p->requestptr);
-		list_del_init(lptr);
-		pendingq_count--;
-		pq_p->audit[1] |= FP_NOTPENDING;
-		pq_p->audit[1] |= FP_AWAKENING;
-		atomic_set(&pq_p->alarmrung, 1);
-		wake_up(&pq_p->waitq);
-	}
-
-	list_for_each_safe(lptr, tptr, &request_list) {
-		pq_p = list_entry(lptr, struct work_element, liste);
-		pq_p->retcode = -ENODEV;
-		pq_p->status[0] |= STAT_FAILED;
-		list_del_init(lptr);
-		requestq_count--;
-		pq_p->audit[1] |= FP_REMREQUEST;
-		pq_p->audit[1] |= FP_AWAKENING;
-		atomic_set(&pq_p->alarmrung, 1);
-		wake_up(&pq_p->waitq);
-	}
-}
-
-static inline void
-helper_timeout_requests(void)
-{
-	struct work_element *pq_p;
-	struct list_head *lptr, *tptr;
-	long timelimit;
-
-	timelimit = jiffies - (CLEANUPTIME * HZ);
-	/* The list is in strict chronological order */
-	list_for_each_safe(lptr, tptr, &pending_list) {
-		pq_p = list_entry(lptr, struct work_element, liste);
-		if (pq_p->requestsent >= timelimit)
-			break;
-		PRINTKW("Purging(PQ) PSMID %02X%02X%02X%02X%02X%02X%02X%02X\n",
-		       ((struct caller *)pq_p->requestptr)->caller_id[0],
-		       ((struct caller *)pq_p->requestptr)->caller_id[1],
-		       ((struct caller *)pq_p->requestptr)->caller_id[2],
-		       ((struct caller *)pq_p->requestptr)->caller_id[3],
-		       ((struct caller *)pq_p->requestptr)->caller_id[4],
-		       ((struct caller *)pq_p->requestptr)->caller_id[5],
-		       ((struct caller *)pq_p->requestptr)->caller_id[6],
-		       ((struct caller *)pq_p->requestptr)->caller_id[7]);
-		pq_p->retcode = -ETIMEOUT;
-		pq_p->status[0] |= STAT_FAILED;
-		/* get this off any caller queue it may be on */
-		unbuild_caller(LONG2DEVPTR(pq_p->devindex),
-			       (struct caller *) pq_p->requestptr);
-		list_del_init(lptr);
-		pendingq_count--;
-		pq_p->audit[1] |= FP_TIMEDOUT;
-		pq_p->audit[1] |= FP_NOTPENDING;
-		pq_p->audit[1] |= FP_AWAKENING;
-		atomic_set(&pq_p->alarmrung, 1);
-		wake_up(&pq_p->waitq);
-	}
-
-	/**
-	 * If pending count is zero, items left on the request queue may
-	 * never be processed.
-	 */
-	if (pendingq_count <= 0) {
-		list_for_each_safe(lptr, tptr, &request_list) {
-			pq_p = list_entry(lptr, struct work_element, liste);
-			if (pq_p->requestsent >= timelimit)
-				break;
-		PRINTKW("Purging(RQ) PSMID %02X%02X%02X%02X%02X%02X%02X%02X\n",
-		       ((struct caller *)pq_p->requestptr)->caller_id[0],
-		       ((struct caller *)pq_p->requestptr)->caller_id[1],
-		       ((struct caller *)pq_p->requestptr)->caller_id[2],
-		       ((struct caller *)pq_p->requestptr)->caller_id[3],
-		       ((struct caller *)pq_p->requestptr)->caller_id[4],
-		       ((struct caller *)pq_p->requestptr)->caller_id[5],
-		       ((struct caller *)pq_p->requestptr)->caller_id[6],
-		       ((struct caller *)pq_p->requestptr)->caller_id[7]);
-			pq_p->retcode = -ETIMEOUT;
-			pq_p->status[0] |= STAT_FAILED;
-			list_del_init(lptr);
-			requestq_count--;
-			pq_p->audit[1] |= FP_TIMEDOUT;
-			pq_p->audit[1] |= FP_REMREQUEST;
-			pq_p->audit[1] |= FP_AWAKENING;
-			atomic_set(&pq_p->alarmrung, 1);
-			wake_up(&pq_p->waitq);
-		}
-	}
-}
-
-static void
-z90crypt_cleanup_task(unsigned long ptr)
-{
-	PDEBUG("jiffies %ld\n", jiffies);
-	spin_lock_irq(&queuespinlock);
-	if (z90crypt.mask.st_count <= 0) // no devices!
-		helper_drain_queues();
-	else
-		helper_timeout_requests();
-	spin_unlock_irq(&queuespinlock);
-	z90crypt_schedule_cleanup_task();
-}
-
-static void
-z90crypt_schedule_reader_task(unsigned long ptr)
-{
-	tasklet_schedule(&reader_tasklet);
-}
-
-/**
- * Lowlevel Functions:
- *
- *   create_z90crypt:  creates and initializes basic data structures
- *   refresh_z90crypt:	re-initializes basic data structures
- *   find_crypto_devices: returns a count and mask of hardware status
- *   create_crypto_device:  builds the descriptor for a device
- *   destroy_crypto_device:  unallocates the descriptor for a device
- *   destroy_z90crypt:	drains all work, unallocates structs
- */
-
-/**
- * build the z90crypt root structure using the given domain index
- */
-static int
-create_z90crypt(int *cdx_p)
-{
-	struct hdware_block *hdware_blk_p;
-
-	memset(&z90crypt, 0x00, sizeof(struct z90crypt));
-	z90crypt.domain_established = 0;
-	z90crypt.len = sizeof(struct z90crypt);
-	z90crypt.max_count = Z90CRYPT_NUM_DEVS;
-	z90crypt.cdx = *cdx_p;
-
-	hdware_blk_p = kzalloc(sizeof(struct hdware_block), GFP_ATOMIC);
-	if (!hdware_blk_p) {
-		PDEBUG("kmalloc for hardware block failed\n");
-		return ENOMEM;
-	}
-	z90crypt.hdware_info = hdware_blk_p;
-
-	return 0;
-}
-
-static inline int
-helper_scan_devices(int cdx_array[16], int *cdx_p, int *correct_cdx_found)
-{
-	enum hdstat hd_stat;
-	int q_depth, dev_type;
-	int indx, chkdom, numdomains;
-
-	q_depth = dev_type = numdomains = 0;
-	for (chkdom = 0; chkdom <= 15; cdx_array[chkdom++] = -1);
-	for (indx = 0; indx < z90crypt.max_count; indx++) {
-		hd_stat = HD_NOT_THERE;
-		numdomains = 0;
-		for (chkdom = 0; chkdom <= 15; chkdom++) {
-			hd_stat = query_online(indx, chkdom, MAX_RESET,
-					       &q_depth, &dev_type);
-			if (hd_stat == HD_TSQ_EXCEPTION) {
-				z90crypt.terminating = 1;
-				PRINTKC("exception taken!\n");
-				break;
-			}
-			if (hd_stat == HD_ONLINE) {
-				cdx_array[numdomains++] = chkdom;
-				if (*cdx_p == chkdom) {
-					*correct_cdx_found  = 1;
-					break;
-				}
-			}
-		}
-		if ((*correct_cdx_found == 1) || (numdomains != 0))
-			break;
-		if (z90crypt.terminating)
-			break;
-	}
-	return numdomains;
-}
-
-static inline int
-probe_crypto_domain(int *cdx_p)
-{
-	int cdx_array[16];
-	char cdx_array_text[53], temp[5];
-	int correct_cdx_found, numdomains;
-
-	correct_cdx_found = 0;
-	numdomains = helper_scan_devices(cdx_array, cdx_p, &correct_cdx_found);
-
-	if (z90crypt.terminating)
-		return TSQ_FATAL_ERROR;
-
-	if (correct_cdx_found)
-		return 0;
-
-	if (numdomains == 0) {
-		PRINTKW("Unable to find crypto domain: No devices found\n");
-		return Z90C_NO_DEVICES;
-	}
-
-	if (numdomains == 1) {
-		if (*cdx_p == -1) {
-			*cdx_p = cdx_array[0];
-			return 0;
-		}
-		PRINTKW("incorrect domain: specified = %d, found = %d\n",
-		       *cdx_p, cdx_array[0]);
-		return Z90C_INCORRECT_DOMAIN;
-	}
-
-	numdomains--;
-	sprintf(cdx_array_text, "%d", cdx_array[numdomains]);
-	while (numdomains) {
-		numdomains--;
-		sprintf(temp, ", %d", cdx_array[numdomains]);
-		strcat(cdx_array_text, temp);
-	}
-
-	PRINTKW("ambiguous domain detected: specified = %d, found array = %s\n",
-		*cdx_p, cdx_array_text);
-	return Z90C_AMBIGUOUS_DOMAIN;
-}
-
-static int
-refresh_z90crypt(int *cdx_p)
-{
-	int i, j, indx, rv;
-	static struct status local_mask;
-	struct device *devPtr;
-	unsigned char oldStat, newStat;
-	int return_unchanged;
-
-	if (z90crypt.len != sizeof(z90crypt))
-		return ENOTINIT;
-	if (z90crypt.terminating)
-		return TSQ_FATAL_ERROR;
-	rv = 0;
-	if (!z90crypt.hdware_info->hdware_mask.st_count &&
-	    !z90crypt.domain_established) {
-		rv = probe_crypto_domain(cdx_p);
-		if (z90crypt.terminating)
-			return TSQ_FATAL_ERROR;
-		if (rv == Z90C_NO_DEVICES)
-			return 0; // try later
-		if (rv)
-			return rv;
-		z90crypt.cdx = *cdx_p;
-		z90crypt.domain_established = 1;
-	}
-	rv = find_crypto_devices(&local_mask);
-	if (rv) {
-		PRINTK("find crypto devices returned %d\n", rv);
-		return rv;
-	}
-	if (!memcmp(&local_mask, &z90crypt.hdware_info->hdware_mask,
-		    sizeof(struct status))) {
-		return_unchanged = 1;
-		for (i = 0; i < Z90CRYPT_NUM_TYPES; i++) {
-			/**
-			 * Check for disabled cards.  If any device is marked
-			 * disabled, destroy it.
-			 */
-			for (j = 0;
-			     j < z90crypt.hdware_info->type_mask[i].st_count;
-			     j++) {
-				indx = z90crypt.hdware_info->type_x_addr[i].
-								device_index[j];
-				devPtr = z90crypt.device_p[indx];
-				if (devPtr && devPtr->disabled) {
-					local_mask.st_mask[indx] = HD_NOT_THERE;
-					return_unchanged = 0;
-				}
-			}
-		}
-		if (return_unchanged == 1)
-			return 0;
-	}
-
-	spin_lock_irq(&queuespinlock);
-	for (i = 0; i < z90crypt.max_count; i++) {
-		oldStat = z90crypt.hdware_info->hdware_mask.st_mask[i];
-		newStat = local_mask.st_mask[i];
-		if ((oldStat == HD_ONLINE) && (newStat != HD_ONLINE))
-			destroy_crypto_device(i);
-		else if ((oldStat != HD_ONLINE) && (newStat == HD_ONLINE)) {
-			rv = create_crypto_device(i);
-			if (rv >= REC_FATAL_ERROR)
-				return rv;
-			if (rv != 0) {
-				local_mask.st_mask[i] = HD_NOT_THERE;
-				local_mask.st_count--;
-			}
-		}
-	}
-	memcpy(z90crypt.hdware_info->hdware_mask.st_mask, local_mask.st_mask,
-	       sizeof(local_mask.st_mask));
-	z90crypt.hdware_info->hdware_mask.st_count = local_mask.st_count;
-	z90crypt.hdware_info->hdware_mask.disabled_count =
-						      local_mask.disabled_count;
-	refresh_index_array(&z90crypt.mask, &z90crypt.overall_device_x);
-	for (i = 0; i < Z90CRYPT_NUM_TYPES; i++)
-		refresh_index_array(&(z90crypt.hdware_info->type_mask[i]),
-				    &(z90crypt.hdware_info->type_x_addr[i]));
-	spin_unlock_irq(&queuespinlock);
-
-	return rv;
-}
-
-static int
-find_crypto_devices(struct status *deviceMask)
-{
-	int i, q_depth, dev_type;
-	enum hdstat hd_stat;
-
-	deviceMask->st_count = 0;
-	deviceMask->disabled_count = 0;
-	deviceMask->user_disabled_count = 0;
-
-	for (i = 0; i < z90crypt.max_count; i++) {
-		hd_stat = query_online(i, z90crypt.cdx, MAX_RESET, &q_depth,
-				       &dev_type);
-		if (hd_stat == HD_TSQ_EXCEPTION) {
-			z90crypt.terminating = 1;
-			PRINTKC("Exception during probe for crypto devices\n");
-			return TSQ_FATAL_ERROR;
-		}
-		deviceMask->st_mask[i] = hd_stat;
-		if (hd_stat == HD_ONLINE) {
-			PDEBUG("Got an online crypto!: %d\n", i);
-			PDEBUG("Got a queue depth of %d\n", q_depth);
-			PDEBUG("Got a device type of %d\n", dev_type);
-			if (q_depth <= 0)
-				return TSQ_FATAL_ERROR;
-			deviceMask->st_count++;
-			z90crypt.q_depth_array[i] = q_depth;
-			z90crypt.dev_type_array[i] = dev_type;
-		}
-	}
-
-	return 0;
-}
-
-static int
-refresh_index_array(struct status *status_str, struct device_x *index_array)
-{
-	int i, count;
-	enum devstat stat;
-
-	i = -1;
-	count = 0;
-	do {
-		stat = status_str->st_mask[++i];
-		if (stat == DEV_ONLINE)
-			index_array->device_index[count++] = i;
-	} while ((i < Z90CRYPT_NUM_DEVS) && (count < status_str->st_count));
-
-	return count;
-}
-
-static int
-create_crypto_device(int index)
-{
-	int rv, devstat, total_size;
-	struct device *dev_ptr;
-	struct status *type_str_p;
-	int deviceType;
-
-	dev_ptr = z90crypt.device_p[index];
-	if (!dev_ptr) {
-		total_size = sizeof(struct device) +
-			     z90crypt.q_depth_array[index] * sizeof(int);
-
-		dev_ptr = kzalloc(total_size, GFP_ATOMIC);
-		if (!dev_ptr) {
-			PRINTK("kmalloc device %d failed\n", index);
-			return ENOMEM;
-		}
-		dev_ptr->dev_resp_p = kmalloc(MAX_RESPONSE_SIZE, GFP_ATOMIC);
-		if (!dev_ptr->dev_resp_p) {
-			kfree(dev_ptr);
-			PRINTK("kmalloc device %d rec buffer failed\n", index);
-			return ENOMEM;
-		}
-		dev_ptr->dev_resp_l = MAX_RESPONSE_SIZE;
-		INIT_LIST_HEAD(&(dev_ptr->dev_caller_list));
-	}
-
-	devstat = reset_device(index, z90crypt.cdx, MAX_RESET);
-	if (devstat == DEV_RSQ_EXCEPTION) {
-		PRINTK("exception during reset device %d\n", index);
-		kfree(dev_ptr->dev_resp_p);
-		kfree(dev_ptr);
-		return RSQ_FATAL_ERROR;
-	}
-	if (devstat == DEV_ONLINE) {
-		dev_ptr->dev_self_x = index;
-		dev_ptr->dev_type = z90crypt.dev_type_array[index];
-		if (dev_ptr->dev_type == NILDEV) {
-			rv = probe_device_type(dev_ptr);
-			if (rv) {
-				PRINTK("rv = %d from probe_device_type %d\n",
-				       rv, index);
-				kfree(dev_ptr->dev_resp_p);
-				kfree(dev_ptr);
-				return rv;
-			}
-		}
-		if (dev_ptr->dev_type == PCIXCC_UNK) {
-			rv = probe_PCIXCC_type(dev_ptr);
-			if (rv) {
-				PRINTK("rv = %d from probe_PCIXCC_type %d\n",
-				       rv, index);
-				kfree(dev_ptr->dev_resp_p);
-				kfree(dev_ptr);
-				return rv;
-			}
-		}
-		deviceType = dev_ptr->dev_type;
-		z90crypt.dev_type_array[index] = deviceType;
-		if (deviceType == PCICA)
-			z90crypt.hdware_info->device_type_array[index] = 1;
-		else if (deviceType == PCICC)
-			z90crypt.hdware_info->device_type_array[index] = 2;
-		else if (deviceType == PCIXCC_MCL2)
-			z90crypt.hdware_info->device_type_array[index] = 3;
-		else if (deviceType == PCIXCC_MCL3)
-			z90crypt.hdware_info->device_type_array[index] = 4;
-		else if (deviceType == CEX2C)
-			z90crypt.hdware_info->device_type_array[index] = 5;
-		else if (deviceType == CEX2A)
-			z90crypt.hdware_info->device_type_array[index] = 6;
-		else // No idea how this would happen.
-			z90crypt.hdware_info->device_type_array[index] = -1;
-	}
-
-	/**
-	 * 'q_depth' returned by the hardware is one less than
-	 * the actual depth
-	 */
-	dev_ptr->dev_q_depth = z90crypt.q_depth_array[index];
-	dev_ptr->dev_type = z90crypt.dev_type_array[index];
-	dev_ptr->dev_stat = devstat;
-	dev_ptr->disabled = 0;
-	z90crypt.device_p[index] = dev_ptr;
-
-	if (devstat == DEV_ONLINE) {
-		if (z90crypt.mask.st_mask[index] != DEV_ONLINE) {
-			z90crypt.mask.st_mask[index] = DEV_ONLINE;
-			z90crypt.mask.st_count++;
-		}
-		deviceType = dev_ptr->dev_type;
-		type_str_p = &z90crypt.hdware_info->type_mask[deviceType];
-		if (type_str_p->st_mask[index] != DEV_ONLINE) {
-			type_str_p->st_mask[index] = DEV_ONLINE;
-			type_str_p->st_count++;
-		}
-	}
-
-	return 0;
-}
-
-static int
-destroy_crypto_device(int index)
-{
-	struct device *dev_ptr;
-	int t, disabledFlag;
-
-	dev_ptr = z90crypt.device_p[index];
-
-	/* remember device type; get rid of device struct */
-	if (dev_ptr) {
-		disabledFlag = dev_ptr->disabled;
-		t = dev_ptr->dev_type;
-		kfree(dev_ptr->dev_resp_p);
-		kfree(dev_ptr);
-	} else {
-		disabledFlag = 0;
-		t = -1;
-	}
-	z90crypt.device_p[index] = 0;
-
-	/* if the type is valid, remove the device from the type_mask */
-	if ((t != -1) && z90crypt.hdware_info->type_mask[t].st_mask[index]) {
-		  z90crypt.hdware_info->type_mask[t].st_mask[index] = 0x00;
-		  z90crypt.hdware_info->type_mask[t].st_count--;
-		  if (disabledFlag == 1)
-			z90crypt.hdware_info->type_mask[t].disabled_count--;
-	}
-	if (z90crypt.mask.st_mask[index] != DEV_GONE) {
-		z90crypt.mask.st_mask[index] = DEV_GONE;
-		z90crypt.mask.st_count--;
-	}
-	z90crypt.hdware_info->device_type_array[index] = 0;
-
-	return 0;
-}
-
-static void
-destroy_z90crypt(void)
-{
-	int i;
-
-	for (i = 0; i < z90crypt.max_count; i++)
-		if (z90crypt.device_p[i])
-			destroy_crypto_device(i);
-	kfree(z90crypt.hdware_info);
-	memset((void *)&z90crypt, 0, sizeof(z90crypt));
-}
-
-static unsigned char static_testmsg[384] = {
-0x00,0x00,0x00,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x00,0x06,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x58,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x01,0x00,0x43,0x43,
-0x41,0x2d,0x41,0x50,0x50,0x4c,0x20,0x20,0x20,0x01,0x01,0x01,0x00,0x00,0x00,0x00,
-0x50,0x4b,0x00,0x00,0x00,0x00,0x01,0x1c,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x05,0xb8,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x70,0x00,0x41,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x54,0x32,
-0x01,0x00,0xa0,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0xb8,0x05,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x0a,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x08,0x00,0x49,0x43,0x53,0x46,
-0x20,0x20,0x20,0x20,0x50,0x4b,0x0a,0x00,0x50,0x4b,0x43,0x53,0x2d,0x31,0x2e,0x32,
-0x37,0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x44,
-0x55,0x66,0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,
-0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x44,0x55,0x66,
-0x77,0x88,0x99,0x00,0x11,0x22,0x33,0x5d,0x00,0x5b,0x00,0x77,0x88,0x1e,0x00,0x00,
-0x57,0x00,0x00,0x00,0x00,0x04,0x00,0x00,0x4f,0x00,0x00,0x00,0x03,0x02,0x00,0x00,
-0x40,0x01,0x00,0x01,0xce,0x02,0x68,0x2d,0x5f,0xa9,0xde,0x0c,0xf6,0xd2,0x7b,0x58,
-0x4b,0xf9,0x28,0x68,0x3d,0xb4,0xf4,0xef,0x78,0xd5,0xbe,0x66,0x63,0x42,0xef,0xf8,
-0xfd,0xa4,0xf8,0xb0,0x8e,0x29,0xc2,0xc9,0x2e,0xd8,0x45,0xb8,0x53,0x8c,0x6f,0x4e,
-0x72,0x8f,0x6c,0x04,0x9c,0x88,0xfc,0x1e,0xc5,0x83,0x55,0x57,0xf7,0xdd,0xfd,0x4f,
-0x11,0x36,0x95,0x5d,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00
-};
-
-static int
-probe_device_type(struct device *devPtr)
-{
-	int rv, dv, i, index, length;
-	unsigned char psmid[8];
-	static unsigned char loc_testmsg[sizeof(static_testmsg)];
-
-	index = devPtr->dev_self_x;
-	rv = 0;
-	do {
-		memcpy(loc_testmsg, static_testmsg, sizeof(static_testmsg));
-		length = sizeof(static_testmsg) - 24;
-		/* the -24 allows for the header */
-		dv = send_to_AP(index, z90crypt.cdx, length, loc_testmsg);
-		if (dv) {
-			PDEBUG("dv returned by send during probe: %d\n", dv);
-			if (dv == DEV_SEN_EXCEPTION) {
-				rv = SEN_FATAL_ERROR;
-				PRINTKC("exception in send to AP %d\n", index);
-				break;
-			}
-			PDEBUG("return value from send_to_AP: %d\n", rv);
-			switch (dv) {
-			case DEV_GONE:
-				PDEBUG("dev %d not available\n", index);
-				rv = SEN_NOT_AVAIL;
-				break;
-			case DEV_ONLINE:
-				rv = 0;
-				break;
-			case DEV_EMPTY:
-				rv = SEN_NOT_AVAIL;
-				break;
-			case DEV_NO_WORK:
-				rv = SEN_FATAL_ERROR;
-				break;
-			case DEV_BAD_MESSAGE:
-				rv = SEN_USER_ERROR;
-				break;
-			case DEV_QUEUE_FULL:
-				rv = SEN_QUEUE_FULL;
-				break;
-			default:
-				PRINTK("unknown dv=%d for dev %d\n", dv, index);
-				rv = SEN_NOT_AVAIL;
-				break;
-			}
-		}
-
-		if (rv)
-			break;
-
-		for (i = 0; i < 6; i++) {
-			mdelay(300);
-			dv = receive_from_AP(index, z90crypt.cdx,
-					     devPtr->dev_resp_l,
-					     devPtr->dev_resp_p, psmid);
-			PDEBUG("dv returned by DQ = %d\n", dv);
-			if (dv == DEV_REC_EXCEPTION) {
-				rv = REC_FATAL_ERROR;
-				PRINTKC("exception in dequeue %d\n",
-					index);
-				break;
-			}
-			switch (dv) {
-			case DEV_ONLINE:
-				rv = 0;
-				break;
-			case DEV_EMPTY:
-				rv = REC_EMPTY;
-				break;
-			case DEV_NO_WORK:
-				rv = REC_NO_WORK;
-				break;
-			case DEV_BAD_MESSAGE:
-			case DEV_GONE:
-			default:
-				rv = REC_NO_RESPONSE;
-				break;
-			}
-			if ((rv != 0) && (rv != REC_NO_WORK))
-				break;
-			if (rv == 0)
-				break;
-		}
-		if (rv)
-			break;
-		rv = (devPtr->dev_resp_p[0] == 0x00) &&
-		     (devPtr->dev_resp_p[1] == 0x86);
-		if (rv)
-			devPtr->dev_type = PCICC;
-		else
-			devPtr->dev_type = PCICA;
-		rv = 0;
-	} while (0);
-	/* In a general error case, the card is not marked online */
-	return rv;
-}
-
-static unsigned char MCL3_testmsg[] = {
-0x00,0x00,0x00,0x00,0xEE,0xEE,0xEE,0xEE,0xEE,0xEE,0xEE,0xEE,
-0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x43,0x41,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x50,0x4B,0x00,0x00,0x00,0x00,0x01,0xC4,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x24,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xDC,0x02,0x00,0x00,0x00,0x54,0x32,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xE8,0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x24,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
-0x00,0x00,0x00,0x00,0x50,0x4B,0x00,0x0A,0x4D,0x52,0x50,0x20,0x20,0x20,0x20,0x20,
-0x00,0x42,0x00,0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,
-0x0E,0x0F,0x00,0x11,0x22,0x33,0x44,0x55,0x66,0x77,0x88,0x99,0xAA,0xBB,0xCC,0xDD,
-0xEE,0xFF,0xFF,0xEE,0xDD,0xCC,0xBB,0xAA,0x99,0x88,0x77,0x66,0x55,0x44,0x33,0x22,
-0x11,0x00,0x01,0x23,0x45,0x67,0x89,0xAB,0xCD,0xEF,0xFE,0xDC,0xBA,0x98,0x76,0x54,
-0x32,0x10,0x00,0x9A,0x00,0x98,0x00,0x00,0x1E,0x00,0x00,0x94,0x00,0x00,0x00,0x00,
-0x04,0x00,0x00,0x8C,0x00,0x00,0x00,0x40,0x02,0x00,0x00,0x40,0xBA,0xE8,0x23,0x3C,
-0x75,0xF3,0x91,0x61,0xD6,0x73,0x39,0xCF,0x7B,0x6D,0x8E,0x61,0x97,0x63,0x9E,0xD9,
-0x60,0x55,0xD6,0xC7,0xEF,0xF8,0x1E,0x63,0x95,0x17,0xCC,0x28,0x45,0x60,0x11,0xC5,
-0xC4,0x4E,0x66,0xC6,0xE6,0xC3,0xDE,0x8A,0x19,0x30,0xCF,0x0E,0xD7,0xAA,0xDB,0x01,
-0xD8,0x00,0xBB,0x8F,0x39,0x9F,0x64,0x28,0xF5,0x7A,0x77,0x49,0xCC,0x6B,0xA3,0x91,
-0x97,0x70,0xE7,0x60,0x1E,0x39,0xE1,0xE5,0x33,0xE1,0x15,0x63,0x69,0x08,0x80,0x4C,
-0x67,0xC4,0x41,0x8F,0x48,0xDF,0x26,0x98,0xF1,0xD5,0x8D,0x88,0xD9,0x6A,0xA4,0x96,
-0xC5,0x84,0xD9,0x30,0x49,0x67,0x7D,0x19,0xB1,0xB3,0x45,0x4D,0xB2,0x53,0x9A,0x47,
-0x3C,0x7C,0x55,0xBF,0xCC,0x85,0x00,0x36,0xF1,0x3D,0x93,0x53
-};
-
-static int
-probe_PCIXCC_type(struct device *devPtr)
-{
-	int rv, dv, i, index, length;
-	unsigned char psmid[8];
-	static unsigned char loc_testmsg[548];
-	struct CPRBX *cprbx_p;
-
-	index = devPtr->dev_self_x;
-	rv = 0;
-	do {
-		memcpy(loc_testmsg, MCL3_testmsg, sizeof(MCL3_testmsg));
-		length = sizeof(MCL3_testmsg) - 0x0C;
-		dv = send_to_AP(index, z90crypt.cdx, length, loc_testmsg);
-		if (dv) {
-			PDEBUG("dv returned = %d\n", dv);
-			if (dv == DEV_SEN_EXCEPTION) {
-				rv = SEN_FATAL_ERROR;
-				PRINTKC("exception in send to AP %d\n", index);
-				break;
-			}
-			PDEBUG("return value from send_to_AP: %d\n", rv);
-			switch (dv) {
-			case DEV_GONE:
-				PDEBUG("dev %d not available\n", index);
-				rv = SEN_NOT_AVAIL;
-				break;
-			case DEV_ONLINE:
-				rv = 0;
-				break;
-			case DEV_EMPTY:
-				rv = SEN_NOT_AVAIL;
-				break;
-			case DEV_NO_WORK:
-				rv = SEN_FATAL_ERROR;
-				break;
-			case DEV_BAD_MESSAGE:
-				rv = SEN_USER_ERROR;
-				break;
-			case DEV_QUEUE_FULL:
-				rv = SEN_QUEUE_FULL;
-				break;
-			default:
-				PRINTK("unknown dv=%d for dev %d\n", dv, index);
-				rv = SEN_NOT_AVAIL;
-				break;
-			}
-		}
-
-		if (rv)
-			break;
-
-		for (i = 0; i < 6; i++) {
-			mdelay(300);
-			dv = receive_from_AP(index, z90crypt.cdx,
-					     devPtr->dev_resp_l,
-					     devPtr->dev_resp_p, psmid);
-			PDEBUG("dv returned by DQ = %d\n", dv);
-			if (dv == DEV_REC_EXCEPTION) {
-				rv = REC_FATAL_ERROR;
-				PRINTKC("exception in dequeue %d\n",
-					index);
-				break;
-			}
-			switch (dv) {
-			case DEV_ONLINE:
-				rv = 0;
-				break;
-			case DEV_EMPTY:
-				rv = REC_EMPTY;
-				break;
-			case DEV_NO_WORK:
-				rv = REC_NO_WORK;
-				break;
-			case DEV_BAD_MESSAGE:
-			case DEV_GONE:
-			default:
-				rv = REC_NO_RESPONSE;
-				break;
-			}
-			if ((rv != 0) && (rv != REC_NO_WORK))
-				break;
-			if (rv == 0)
-				break;
-		}
-		if (rv)
-			break;
-		cprbx_p = (struct CPRBX *) (devPtr->dev_resp_p + 48);
-		if ((cprbx_p->ccp_rtcode == 8) && (cprbx_p->ccp_rscode == 33)) {
-			devPtr->dev_type = PCIXCC_MCL2;
-			PDEBUG("device %d is MCL2\n", index);
-		} else {
-			devPtr->dev_type = PCIXCC_MCL3;
-			PDEBUG("device %d is MCL3\n", index);
-		}
-	} while (0);
-	/* In a general error case, the card is not marked online */
-	return rv;
-}
-
-module_init(z90crypt_init_module);
-module_exit(z90crypt_cleanup_module);
diff --git a/drivers/s390/crypto/zcrypt_api.c b/drivers/s390/crypto/zcrypt_api.c
new file mode 100644
index 0000000..1edc10a
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_api.c
@@ -0,0 +1,1091 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_api.c
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *	       Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/proc_fs.h>
+#include <linux/compat.h>
+#include <asm/atomic.h>
+#include <asm/uaccess.h>
+
+#include "zcrypt_api.h"
+
+/**
+ * Module description.
+ */
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("Cryptographic Coprocessor interface, "
+		   "Copyright 2001, 2006 IBM Corporation");
+MODULE_LICENSE("GPL");
+
+static DEFINE_SPINLOCK(zcrypt_device_lock);
+static LIST_HEAD(zcrypt_device_list);
+static int zcrypt_device_count = 0;
+static atomic_t zcrypt_open_count = ATOMIC_INIT(0);
+
+/**
+ * Device attributes common for all crypto devices.
+ */
+static ssize_t zcrypt_type_show(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct zcrypt_device *zdev = to_ap_dev(dev)->private;
+	return snprintf(buf, PAGE_SIZE, "%s\n", zdev->type_string);
+}
+
+static DEVICE_ATTR(type, 0444, zcrypt_type_show, NULL);
+
+static ssize_t zcrypt_online_show(struct device *dev,
+				  struct device_attribute *attr, char *buf)
+{
+	struct zcrypt_device *zdev = to_ap_dev(dev)->private;
+	return snprintf(buf, PAGE_SIZE, "%d\n", zdev->online);
+}
+
+static ssize_t zcrypt_online_store(struct device *dev,
+				   struct device_attribute *attr,
+				   const char *buf, size_t count)
+{
+	struct zcrypt_device *zdev = to_ap_dev(dev)->private;
+	int online;
+
+	if (sscanf(buf, "%d\n", &online) != 1 || online < 0 || online > 1)
+		return -EINVAL;
+	zdev->online = online;
+	if (!online)
+		ap_flush_queue(zdev->ap_dev);
+	return count;
+}
+
+static DEVICE_ATTR(online, 0644, zcrypt_online_show, zcrypt_online_store);
+
+static struct attribute * zcrypt_device_attrs[] = {
+	&dev_attr_type.attr,
+	&dev_attr_online.attr,
+	NULL,
+};
+
+static struct attribute_group zcrypt_device_attr_group = {
+	.attrs = zcrypt_device_attrs,
+};
+
+/**
+ * Move the device towards the head of the device list.
+ * Need to be called while holding the zcrypt device list lock.
+ * Note: cards with speed_rating of 0 are kept at the end of the list.
+ */
+static void __zcrypt_increase_preference(struct zcrypt_device *zdev)
+{
+	struct zcrypt_device *tmp;
+	struct list_head *l;
+
+	if (zdev->speed_rating == 0)
+		return;
+	for (l = zdev->list.prev; l != &zcrypt_device_list; l = l->prev) {
+		tmp = list_entry(l, struct zcrypt_device, list);
+		if ((tmp->request_count + 1) * tmp->speed_rating <=
+		    (zdev->request_count + 1) * zdev->speed_rating &&
+		    tmp->speed_rating != 0)
+			break;
+	}
+	if (l == zdev->list.prev)
+		return;
+	/* Move zdev behind l */
+	list_del(&zdev->list);
+	list_add(&zdev->list, l);
+}
+
+/**
+ * Move the device towards the tail of the device list.
+ * Need to be called while holding the zcrypt device list lock.
+ * Note: cards with speed_rating of 0 are kept at the end of the list.
+ */
+static void __zcrypt_decrease_preference(struct zcrypt_device *zdev)
+{
+	struct zcrypt_device *tmp;
+	struct list_head *l;
+
+	if (zdev->speed_rating == 0)
+		return;
+	for (l = zdev->list.next; l != &zcrypt_device_list; l = l->next) {
+		tmp = list_entry(l, struct zcrypt_device, list);
+		if ((tmp->request_count + 1) * tmp->speed_rating >
+		    (zdev->request_count + 1) * zdev->speed_rating ||
+		    tmp->speed_rating == 0)
+			break;
+	}
+	if (l == zdev->list.next)
+		return;
+	/* Move zdev before l */
+	list_del(&zdev->list);
+	list_add_tail(&zdev->list, l);
+}
+
+static void zcrypt_device_release(struct kref *kref)
+{
+	struct zcrypt_device *zdev =
+		container_of(kref, struct zcrypt_device, refcount);
+	zcrypt_device_free(zdev);
+}
+
+void zcrypt_device_get(struct zcrypt_device *zdev)
+{
+	kref_get(&zdev->refcount);
+}
+EXPORT_SYMBOL(zcrypt_device_get);
+
+int zcrypt_device_put(struct zcrypt_device *zdev)
+{
+	return kref_put(&zdev->refcount, zcrypt_device_release);
+}
+EXPORT_SYMBOL(zcrypt_device_put);
+
+struct zcrypt_device *zcrypt_device_alloc(size_t max_response_size)
+{
+	struct zcrypt_device *zdev;
+
+	zdev = kzalloc(sizeof(struct zcrypt_device), GFP_KERNEL);
+	if (!zdev)
+		return NULL;
+	zdev->reply.message = kmalloc(max_response_size, GFP_KERNEL);
+	if (!zdev->reply.message)
+		goto out_free;
+	zdev->reply.length = max_response_size;
+	spin_lock_init(&zdev->lock);
+	INIT_LIST_HEAD(&zdev->list);
+	return zdev;
+
+out_free:
+	kfree(zdev);
+	return NULL;
+}
+EXPORT_SYMBOL(zcrypt_device_alloc);
+
+void zcrypt_device_free(struct zcrypt_device *zdev)
+{
+	kfree(zdev->reply.message);
+	kfree(zdev);
+}
+EXPORT_SYMBOL(zcrypt_device_free);
+
+/**
+ * Register a crypto device.
+ */
+int zcrypt_device_register(struct zcrypt_device *zdev)
+{
+	int rc;
+
+	rc = sysfs_create_group(&zdev->ap_dev->device.kobj,
+				&zcrypt_device_attr_group);
+	if (rc)
+		goto out;
+	get_device(&zdev->ap_dev->device);
+	kref_init(&zdev->refcount);
+	spin_lock_bh(&zcrypt_device_lock);
+	zdev->online = 1;	/* New devices are online by default. */
+	list_add_tail(&zdev->list, &zcrypt_device_list);
+	__zcrypt_increase_preference(zdev);
+	zcrypt_device_count++;
+	spin_unlock_bh(&zcrypt_device_lock);
+out:
+	return rc;
+}
+EXPORT_SYMBOL(zcrypt_device_register);
+
+/**
+ * Unregister a crypto device.
+ */
+void zcrypt_device_unregister(struct zcrypt_device *zdev)
+{
+	spin_lock_bh(&zcrypt_device_lock);
+	zcrypt_device_count--;
+	list_del_init(&zdev->list);
+	spin_unlock_bh(&zcrypt_device_lock);
+	sysfs_remove_group(&zdev->ap_dev->device.kobj,
+			   &zcrypt_device_attr_group);
+	put_device(&zdev->ap_dev->device);
+	zcrypt_device_put(zdev);
+}
+EXPORT_SYMBOL(zcrypt_device_unregister);
+
+/**
+ * zcrypt_read is not be supported beyond zcrypt 1.3.1
+ */
+static ssize_t zcrypt_read(struct file *filp, char __user *buf,
+			   size_t count, loff_t *f_pos)
+{
+	return -EPERM;
+}
+
+/**
+ * Write is is not allowed
+ */
+static ssize_t zcrypt_write(struct file *filp, const char __user *buf,
+			    size_t count, loff_t *f_pos)
+{
+	return -EPERM;
+}
+
+/**
+ * Device open/close functions to count number of users.
+ */
+static int zcrypt_open(struct inode *inode, struct file *filp)
+{
+	atomic_inc(&zcrypt_open_count);
+	return 0;
+}
+
+static int zcrypt_release(struct inode *inode, struct file *filp)
+{
+	atomic_dec(&zcrypt_open_count);
+	return 0;
+}
+
+/**
+ * zcrypt ioctls.
+ */
+static long zcrypt_rsa_modexpo(struct ica_rsa_modexpo *mex)
+{
+	struct zcrypt_device *zdev;
+	int rc;
+
+	if (mex->outputdatalength < mex->inputdatalength)
+		return -EINVAL;
+	/**
+	 * As long as outputdatalength is big enough, we can set the
+	 * outputdatalength equal to the inputdatalength, since that is the
+	 * number of bytes we will copy in any case
+	 */
+	mex->outputdatalength = mex->inputdatalength;
+
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list) {
+		if (!zdev->online ||
+		    !zdev->ops->rsa_modexpo ||
+		    zdev->min_mod_size > mex->inputdatalength ||
+		    zdev->max_mod_size < mex->inputdatalength)
+			continue;
+		zcrypt_device_get(zdev);
+		get_device(&zdev->ap_dev->device);
+		zdev->request_count++;
+		__zcrypt_decrease_preference(zdev);
+		spin_unlock_bh(&zcrypt_device_lock);
+		if (try_module_get(zdev->ap_dev->drv->driver.owner)) {
+			rc = zdev->ops->rsa_modexpo(zdev, mex);
+			module_put(zdev->ap_dev->drv->driver.owner);
+		}
+		else
+			rc = -EAGAIN;
+		spin_lock_bh(&zcrypt_device_lock);
+		zdev->request_count--;
+		__zcrypt_increase_preference(zdev);
+		put_device(&zdev->ap_dev->device);
+		zcrypt_device_put(zdev);
+		spin_unlock_bh(&zcrypt_device_lock);
+		return rc;
+	}
+	spin_unlock_bh(&zcrypt_device_lock);
+	return -ENODEV;
+}
+
+static long zcrypt_rsa_crt(struct ica_rsa_modexpo_crt *crt)
+{
+	struct zcrypt_device *zdev;
+	unsigned long long z1, z2, z3;
+	int rc, copied;
+
+	if (crt->outputdatalength < crt->inputdatalength ||
+	    (crt->inputdatalength & 1))
+		return -EINVAL;
+	/**
+	 * As long as outputdatalength is big enough, we can set the
+	 * outputdatalength equal to the inputdatalength, since that is the
+	 * number of bytes we will copy in any case
+	 */
+	crt->outputdatalength = crt->inputdatalength;
+
+	copied = 0;
+ restart:
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list) {
+		if (!zdev->online ||
+		    !zdev->ops->rsa_modexpo_crt ||
+		    zdev->min_mod_size > crt->inputdatalength ||
+		    zdev->max_mod_size < crt->inputdatalength)
+			continue;
+		if (zdev->short_crt && crt->inputdatalength > 240) {
+			/**
+			 * Check inputdata for leading zeros for cards
+			 * that can't handle np_prime, bp_key, or
+			 * u_mult_inv > 128 bytes.
+			 */
+			if (copied == 0) {
+				int len;
+				spin_unlock_bh(&zcrypt_device_lock);
+				/* len is max 256 / 2 - 120 = 8 */
+				len = crt->inputdatalength / 2 - 120;
+				z1 = z2 = z3 = 0;
+				if (copy_from_user(&z1, crt->np_prime, len) ||
+				    copy_from_user(&z2, crt->bp_key, len) ||
+				    copy_from_user(&z3, crt->u_mult_inv, len))
+					return -EFAULT;
+				copied = 1;
+				/**
+				 * We have to restart device lookup -
+				 * the device list may have changed by now.
+				 */
+				goto restart;
+			}
+			if (z1 != 0ULL || z2 != 0ULL || z3 != 0ULL)
+				/* The device can't handle this request. */
+				continue;
+		}
+		zcrypt_device_get(zdev);
+		get_device(&zdev->ap_dev->device);
+		zdev->request_count++;
+		__zcrypt_decrease_preference(zdev);
+		spin_unlock_bh(&zcrypt_device_lock);
+		if (try_module_get(zdev->ap_dev->drv->driver.owner)) {
+			rc = zdev->ops->rsa_modexpo_crt(zdev, crt);
+			module_put(zdev->ap_dev->drv->driver.owner);
+		}
+		else
+			rc = -EAGAIN;
+		spin_lock_bh(&zcrypt_device_lock);
+		zdev->request_count--;
+		__zcrypt_increase_preference(zdev);
+		put_device(&zdev->ap_dev->device);
+		zcrypt_device_put(zdev);
+		spin_unlock_bh(&zcrypt_device_lock);
+		return rc;
+	}
+	spin_unlock_bh(&zcrypt_device_lock);
+	return -ENODEV;
+}
+
+static long zcrypt_send_cprb(struct ica_xcRB *xcRB)
+{
+	struct zcrypt_device *zdev;
+	int rc;
+
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list) {
+		if (!zdev->online || !zdev->ops->send_cprb ||
+		    (xcRB->user_defined != AUTOSELECT &&
+			AP_QID_DEVICE(zdev->ap_dev->qid) != xcRB->user_defined)
+		    )
+			continue;
+		zcrypt_device_get(zdev);
+		get_device(&zdev->ap_dev->device);
+		zdev->request_count++;
+		__zcrypt_decrease_preference(zdev);
+		spin_unlock_bh(&zcrypt_device_lock);
+		if (try_module_get(zdev->ap_dev->drv->driver.owner)) {
+			rc = zdev->ops->send_cprb(zdev, xcRB);
+			module_put(zdev->ap_dev->drv->driver.owner);
+		}
+		else
+			rc = -EAGAIN;
+		spin_lock_bh(&zcrypt_device_lock);
+		zdev->request_count--;
+		__zcrypt_increase_preference(zdev);
+		put_device(&zdev->ap_dev->device);
+		zcrypt_device_put(zdev);
+		spin_unlock_bh(&zcrypt_device_lock);
+		return rc;
+	}
+	spin_unlock_bh(&zcrypt_device_lock);
+	return -ENODEV;
+}
+
+static void zcrypt_status_mask(char status[AP_DEVICES])
+{
+	struct zcrypt_device *zdev;
+
+	memset(status, 0, sizeof(char) * AP_DEVICES);
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list)
+		status[AP_QID_DEVICE(zdev->ap_dev->qid)] =
+			zdev->online ? zdev->user_space_type : 0x0d;
+	spin_unlock_bh(&zcrypt_device_lock);
+}
+
+static void zcrypt_qdepth_mask(char qdepth[AP_DEVICES])
+{
+	struct zcrypt_device *zdev;
+
+	memset(qdepth, 0, sizeof(char)	* AP_DEVICES);
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list) {
+		spin_lock(&zdev->ap_dev->lock);
+		qdepth[AP_QID_DEVICE(zdev->ap_dev->qid)] =
+			zdev->ap_dev->pendingq_count +
+			zdev->ap_dev->requestq_count;
+		spin_unlock(&zdev->ap_dev->lock);
+	}
+	spin_unlock_bh(&zcrypt_device_lock);
+}
+
+static void zcrypt_perdev_reqcnt(int reqcnt[AP_DEVICES])
+{
+	struct zcrypt_device *zdev;
+
+	memset(reqcnt, 0, sizeof(int) * AP_DEVICES);
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list) {
+		spin_lock(&zdev->ap_dev->lock);
+		reqcnt[AP_QID_DEVICE(zdev->ap_dev->qid)] =
+			zdev->ap_dev->total_request_count;
+		spin_unlock(&zdev->ap_dev->lock);
+	}
+	spin_unlock_bh(&zcrypt_device_lock);
+}
+
+static int zcrypt_pendingq_count(void)
+{
+	struct zcrypt_device *zdev;
+	int pendingq_count = 0;
+
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list) {
+		spin_lock(&zdev->ap_dev->lock);
+		pendingq_count += zdev->ap_dev->pendingq_count;
+		spin_unlock(&zdev->ap_dev->lock);
+	}
+	spin_unlock_bh(&zcrypt_device_lock);
+	return pendingq_count;
+}
+
+static int zcrypt_requestq_count(void)
+{
+	struct zcrypt_device *zdev;
+	int requestq_count = 0;
+
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list) {
+		spin_lock(&zdev->ap_dev->lock);
+		requestq_count += zdev->ap_dev->requestq_count;
+		spin_unlock(&zdev->ap_dev->lock);
+	}
+	spin_unlock_bh(&zcrypt_device_lock);
+	return requestq_count;
+}
+
+static int zcrypt_count_type(int type)
+{
+	struct zcrypt_device *zdev;
+	int device_count = 0;
+
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list)
+		if (zdev->user_space_type == type)
+			device_count++;
+	spin_unlock_bh(&zcrypt_device_lock);
+	return device_count;
+}
+
+/**
+ * Old, deprecated combi status call.
+ */
+static long zcrypt_ica_status(struct file *filp, unsigned long arg)
+{
+	struct ica_z90_status *pstat;
+	int ret;
+
+	pstat = kzalloc(sizeof(*pstat), GFP_KERNEL);
+	if (!pstat)
+		return -ENOMEM;
+	pstat->totalcount = zcrypt_device_count;
+	pstat->leedslitecount = zcrypt_count_type(ZCRYPT_PCICA);
+	pstat->leeds2count = zcrypt_count_type(ZCRYPT_PCICC);
+	pstat->requestqWaitCount = zcrypt_requestq_count();
+	pstat->pendingqWaitCount = zcrypt_pendingq_count();
+	pstat->totalOpenCount = atomic_read(&zcrypt_open_count);
+	pstat->cryptoDomain = ap_domain_index;
+	zcrypt_status_mask(pstat->status);
+	zcrypt_qdepth_mask(pstat->qdepth);
+	ret = 0;
+	if (copy_to_user((void __user *) arg, pstat, sizeof(*pstat)))
+		ret = -EFAULT;
+	kfree(pstat);
+	return ret;
+}
+
+static long zcrypt_unlocked_ioctl(struct file *filp, unsigned int cmd,
+				  unsigned long arg)
+{
+	int rc;
+
+	switch (cmd) {
+	case ICARSAMODEXPO: {
+		struct ica_rsa_modexpo __user *umex = (void __user *) arg;
+		struct ica_rsa_modexpo mex;
+		if (copy_from_user(&mex, umex, sizeof(mex)))
+			return -EFAULT;
+		do {
+			rc = zcrypt_rsa_modexpo(&mex);
+		} while (rc == -EAGAIN);
+		if (rc)
+			return rc;
+		return put_user(mex.outputdatalength, &umex->outputdatalength);
+	}
+	case ICARSACRT: {
+		struct ica_rsa_modexpo_crt __user *ucrt = (void __user *) arg;
+		struct ica_rsa_modexpo_crt crt;
+		if (copy_from_user(&crt, ucrt, sizeof(crt)))
+			return -EFAULT;
+		do {
+			rc = zcrypt_rsa_crt(&crt);
+		} while (rc == -EAGAIN);
+		if (rc)
+			return rc;
+		return put_user(crt.outputdatalength, &ucrt->outputdatalength);
+	}
+	case ZSECSENDCPRB: {
+		struct ica_xcRB __user *uxcRB = (void __user *) arg;
+		struct ica_xcRB xcRB;
+		if (copy_from_user(&xcRB, uxcRB, sizeof(xcRB)))
+			return -EFAULT;
+		do {
+			rc = zcrypt_send_cprb(&xcRB);
+		} while (rc == -EAGAIN);
+		if (copy_to_user(uxcRB, &xcRB, sizeof(xcRB)))
+			return -EFAULT;
+		return rc;
+	}
+	case Z90STAT_STATUS_MASK: {
+		char status[AP_DEVICES];
+		zcrypt_status_mask(status);
+		if (copy_to_user((char __user *) arg, status,
+				 sizeof(char) * AP_DEVICES))
+			return -EFAULT;
+		return 0;
+	}
+	case Z90STAT_QDEPTH_MASK: {
+		char qdepth[AP_DEVICES];
+		zcrypt_qdepth_mask(qdepth);
+		if (copy_to_user((char __user *) arg, qdepth,
+				 sizeof(char) * AP_DEVICES))
+			return -EFAULT;
+		return 0;
+	}
+	case Z90STAT_PERDEV_REQCNT: {
+		int reqcnt[AP_DEVICES];
+		zcrypt_perdev_reqcnt(reqcnt);
+		if (copy_to_user((int __user *) arg, reqcnt,
+				 sizeof(int) * AP_DEVICES))
+			return -EFAULT;
+		return 0;
+	}
+	case Z90STAT_REQUESTQ_COUNT:
+		return put_user(zcrypt_requestq_count(), (int __user *) arg);
+	case Z90STAT_PENDINGQ_COUNT:
+		return put_user(zcrypt_pendingq_count(), (int __user *) arg);
+	case Z90STAT_TOTALOPEN_COUNT:
+		return put_user(atomic_read(&zcrypt_open_count),
+				(int __user *) arg);
+	case Z90STAT_DOMAIN_INDEX:
+		return put_user(ap_domain_index, (int __user *) arg);
+	/**
+	 * Deprecated ioctls. Don't add another device count ioctl,
+	 * you can count them yourself in the user space with the
+	 * output of the Z90STAT_STATUS_MASK ioctl.
+	 */
+	case ICAZ90STATUS:
+		return zcrypt_ica_status(filp, arg);
+	case Z90STAT_TOTALCOUNT:
+		return put_user(zcrypt_device_count, (int __user *) arg);
+	case Z90STAT_PCICACOUNT:
+		return put_user(zcrypt_count_type(ZCRYPT_PCICA),
+				(int __user *) arg);
+	case Z90STAT_PCICCCOUNT:
+		return put_user(zcrypt_count_type(ZCRYPT_PCICC),
+				(int __user *) arg);
+	case Z90STAT_PCIXCCMCL2COUNT:
+		return put_user(zcrypt_count_type(ZCRYPT_PCIXCC_MCL2),
+				(int __user *) arg);
+	case Z90STAT_PCIXCCMCL3COUNT:
+		return put_user(zcrypt_count_type(ZCRYPT_PCIXCC_MCL3),
+				(int __user *) arg);
+	case Z90STAT_PCIXCCCOUNT:
+		return put_user(zcrypt_count_type(ZCRYPT_PCIXCC_MCL2) +
+				zcrypt_count_type(ZCRYPT_PCIXCC_MCL3),
+				(int __user *) arg);
+	case Z90STAT_CEX2CCOUNT:
+		return put_user(zcrypt_count_type(ZCRYPT_CEX2C),
+				(int __user *) arg);
+	case Z90STAT_CEX2ACOUNT:
+		return put_user(zcrypt_count_type(ZCRYPT_CEX2A),
+				(int __user *) arg);
+	default:
+		/* unknown ioctl number */
+		return -ENOIOCTLCMD;
+	}
+}
+
+#ifdef CONFIG_COMPAT
+/**
+ * ioctl32 conversion routines
+ */
+struct compat_ica_rsa_modexpo {
+	compat_uptr_t	inputdata;
+	unsigned int	inputdatalength;
+	compat_uptr_t	outputdata;
+	unsigned int	outputdatalength;
+	compat_uptr_t	b_key;
+	compat_uptr_t	n_modulus;
+};
+
+static long trans_modexpo32(struct file *filp, unsigned int cmd,
+			    unsigned long arg)
+{
+	struct compat_ica_rsa_modexpo __user *umex32 = compat_ptr(arg);
+	struct compat_ica_rsa_modexpo mex32;
+	struct ica_rsa_modexpo mex64;
+	long rc;
+
+	if (copy_from_user(&mex32, umex32, sizeof(mex32)))
+		return -EFAULT;
+	mex64.inputdata = compat_ptr(mex32.inputdata);
+	mex64.inputdatalength = mex32.inputdatalength;
+	mex64.outputdata = compat_ptr(mex32.outputdata);
+	mex64.outputdatalength = mex32.outputdatalength;
+	mex64.b_key = compat_ptr(mex32.b_key);
+	mex64.n_modulus = compat_ptr(mex32.n_modulus);
+	do {
+		rc = zcrypt_rsa_modexpo(&mex64);
+	} while (rc == -EAGAIN);
+	if (!rc)
+		rc = put_user(mex64.outputdatalength,
+			      &umex32->outputdatalength);
+	return rc;
+}
+
+struct compat_ica_rsa_modexpo_crt {
+	compat_uptr_t	inputdata;
+	unsigned int	inputdatalength;
+	compat_uptr_t	outputdata;
+	unsigned int	outputdatalength;
+	compat_uptr_t	bp_key;
+	compat_uptr_t	bq_key;
+	compat_uptr_t	np_prime;
+	compat_uptr_t	nq_prime;
+	compat_uptr_t	u_mult_inv;
+};
+
+static long trans_modexpo_crt32(struct file *filp, unsigned int cmd,
+				unsigned long arg)
+{
+	struct compat_ica_rsa_modexpo_crt __user *ucrt32 = compat_ptr(arg);
+	struct compat_ica_rsa_modexpo_crt crt32;
+	struct ica_rsa_modexpo_crt crt64;
+	long rc;
+
+	if (copy_from_user(&crt32, ucrt32, sizeof(crt32)))
+		return -EFAULT;
+	crt64.inputdata = compat_ptr(crt32.inputdata);
+	crt64.inputdatalength = crt32.inputdatalength;
+	crt64.outputdata=  compat_ptr(crt32.outputdata);
+	crt64.outputdatalength = crt32.outputdatalength;
+	crt64.bp_key = compat_ptr(crt32.bp_key);
+	crt64.bq_key = compat_ptr(crt32.bq_key);
+	crt64.np_prime = compat_ptr(crt32.np_prime);
+	crt64.nq_prime = compat_ptr(crt32.nq_prime);
+	crt64.u_mult_inv = compat_ptr(crt32.u_mult_inv);
+	do {
+		rc = zcrypt_rsa_crt(&crt64);
+	} while (rc == -EAGAIN);
+	if (!rc)
+		rc = put_user(crt64.outputdatalength,
+			      &ucrt32->outputdatalength);
+	return rc;
+}
+
+struct compat_ica_xcRB {
+	unsigned short	agent_ID;
+	unsigned int	user_defined;
+	unsigned short	request_ID;
+	unsigned int	request_control_blk_length;
+	unsigned char	padding1[16 - sizeof (compat_uptr_t)];
+	compat_uptr_t	request_control_blk_addr;
+	unsigned int	request_data_length;
+	char		padding2[16 - sizeof (compat_uptr_t)];
+	compat_uptr_t	request_data_address;
+	unsigned int	reply_control_blk_length;
+	char		padding3[16 - sizeof (compat_uptr_t)];
+	compat_uptr_t	reply_control_blk_addr;
+	unsigned int	reply_data_length;
+	char		padding4[16 - sizeof (compat_uptr_t)];
+	compat_uptr_t	reply_data_addr;
+	unsigned short	priority_window;
+	unsigned int	status;
+} __attribute__((packed));
+
+static long trans_xcRB32(struct file *filp, unsigned int cmd,
+			 unsigned long arg)
+{
+	struct compat_ica_xcRB __user *uxcRB32 = compat_ptr(arg);
+	struct compat_ica_xcRB xcRB32;
+	struct ica_xcRB xcRB64;
+	long rc;
+
+	if (copy_from_user(&xcRB32, uxcRB32, sizeof(xcRB32)))
+		return -EFAULT;
+	xcRB64.agent_ID = xcRB32.agent_ID;
+	xcRB64.user_defined = xcRB32.user_defined;
+	xcRB64.request_ID = xcRB32.request_ID;
+	xcRB64.request_control_blk_length =
+		xcRB32.request_control_blk_length;
+	xcRB64.request_control_blk_addr =
+		compat_ptr(xcRB32.request_control_blk_addr);
+	xcRB64.request_data_length =
+		xcRB32.request_data_length;
+	xcRB64.request_data_address =
+		compat_ptr(xcRB32.request_data_address);
+	xcRB64.reply_control_blk_length =
+		xcRB32.reply_control_blk_length;
+	xcRB64.reply_control_blk_addr =
+		compat_ptr(xcRB32.reply_control_blk_addr);
+	xcRB64.reply_data_length = xcRB32.reply_data_length;
+	xcRB64.reply_data_addr =
+		compat_ptr(xcRB32.reply_data_addr);
+	xcRB64.priority_window = xcRB32.priority_window;
+	xcRB64.status = xcRB32.status;
+	do {
+		rc = zcrypt_send_cprb(&xcRB64);
+	} while (rc == -EAGAIN);
+	xcRB32.reply_control_blk_length = xcRB64.reply_control_blk_length;
+	xcRB32.reply_data_length = xcRB64.reply_data_length;
+	xcRB32.status = xcRB64.status;
+	if (copy_to_user(uxcRB32, &xcRB32, sizeof(xcRB32)))
+			return -EFAULT;
+	return rc;
+}
+
+long zcrypt_compat_ioctl(struct file *filp, unsigned int cmd,
+			 unsigned long arg)
+{
+	if (cmd == ICARSAMODEXPO)
+		return trans_modexpo32(filp, cmd, arg);
+	if (cmd == ICARSACRT)
+		return trans_modexpo_crt32(filp, cmd, arg);
+	if (cmd == ZSECSENDCPRB)
+		return trans_xcRB32(filp, cmd, arg);
+	return zcrypt_unlocked_ioctl(filp, cmd, arg);
+}
+#endif
+
+/**
+ * Misc device file operations.
+ */
+static struct file_operations zcrypt_fops = {
+	.owner		= THIS_MODULE,
+	.read		= zcrypt_read,
+	.write		= zcrypt_write,
+	.unlocked_ioctl	= zcrypt_unlocked_ioctl,
+#ifdef CONFIG_COMPAT
+	.compat_ioctl	= zcrypt_compat_ioctl,
+#endif
+	.open		= zcrypt_open,
+	.release	= zcrypt_release
+};
+
+/**
+ * Misc device.
+ */
+static struct miscdevice zcrypt_misc_device = {
+	.minor	    = MISC_DYNAMIC_MINOR,
+	.name	    = "z90crypt",
+	.fops	    = &zcrypt_fops,
+};
+
+/**
+ * Deprecated /proc entry support.
+ */
+static struct proc_dir_entry *zcrypt_entry;
+
+static inline int sprintcl(unsigned char *outaddr, unsigned char *addr,
+			   unsigned int len)
+{
+	int hl, i;
+
+	hl = 0;
+	for (i = 0; i < len; i++)
+		hl += sprintf(outaddr+hl, "%01x", (unsigned int) addr[i]);
+	hl += sprintf(outaddr+hl, " ");
+	return hl;
+}
+
+static inline int sprintrw(unsigned char *outaddr, unsigned char *addr,
+			   unsigned int len)
+{
+	int hl, inl, c, cx;
+
+	hl = sprintf(outaddr, "	   ");
+	inl = 0;
+	for (c = 0; c < (len / 16); c++) {
+		hl += sprintcl(outaddr+hl, addr+inl, 16);
+		inl += 16;
+	}
+	cx = len%16;
+	if (cx) {
+		hl += sprintcl(outaddr+hl, addr+inl, cx);
+		inl += cx;
+	}
+	hl += sprintf(outaddr+hl, "\n");
+	return hl;
+}
+
+static inline int sprinthx(unsigned char *title, unsigned char *outaddr,
+			   unsigned char *addr, unsigned int len)
+{
+	int hl, inl, r, rx;
+
+	hl = sprintf(outaddr, "\n%s\n", title);
+	inl = 0;
+	for (r = 0; r < (len / 64); r++) {
+		hl += sprintrw(outaddr+hl, addr+inl, 64);
+		inl += 64;
+	}
+	rx = len % 64;
+	if (rx) {
+		hl += sprintrw(outaddr+hl, addr+inl, rx);
+		inl += rx;
+	}
+	hl += sprintf(outaddr+hl, "\n");
+	return hl;
+}
+
+static inline int sprinthx4(unsigned char *title, unsigned char *outaddr,
+			    unsigned int *array, unsigned int len)
+{
+	int hl, r;
+
+	hl = sprintf(outaddr, "\n%s\n", title);
+	for (r = 0; r < len; r++) {
+		if ((r % 8) == 0)
+			hl += sprintf(outaddr+hl, "    ");
+		hl += sprintf(outaddr+hl, "%08X ", array[r]);
+		if ((r % 8) == 7)
+			hl += sprintf(outaddr+hl, "\n");
+	}
+	hl += sprintf(outaddr+hl, "\n");
+	return hl;
+}
+
+static int zcrypt_status_read(char *resp_buff, char **start, off_t offset,
+			      int count, int *eof, void *data)
+{
+	unsigned char *workarea;
+	int len;
+
+	len = 0;
+
+	/* resp_buff is a page. Use the right half for a work area */
+	workarea = resp_buff + 2000;
+	len += sprintf(resp_buff + len, "\nzcrypt version: %d.%d.%d\n",
+		ZCRYPT_VERSION, ZCRYPT_RELEASE, ZCRYPT_VARIANT);
+	len += sprintf(resp_buff + len, "Cryptographic domain: %d\n",
+		       ap_domain_index);
+	len += sprintf(resp_buff + len, "Total device count: %d\n",
+		       zcrypt_device_count);
+	len += sprintf(resp_buff + len, "PCICA count: %d\n",
+		       zcrypt_count_type(ZCRYPT_PCICA));
+	len += sprintf(resp_buff + len, "PCICC count: %d\n",
+		       zcrypt_count_type(ZCRYPT_PCICC));
+	len += sprintf(resp_buff + len, "PCIXCC MCL2 count: %d\n",
+		       zcrypt_count_type(ZCRYPT_PCIXCC_MCL2));
+	len += sprintf(resp_buff + len, "PCIXCC MCL3 count: %d\n",
+		       zcrypt_count_type(ZCRYPT_PCIXCC_MCL3));
+	len += sprintf(resp_buff + len, "CEX2C count: %d\n",
+		       zcrypt_count_type(ZCRYPT_CEX2C));
+	len += sprintf(resp_buff + len, "CEX2A count: %d\n",
+		       zcrypt_count_type(ZCRYPT_CEX2A));
+	len += sprintf(resp_buff + len, "requestq count: %d\n",
+		       zcrypt_requestq_count());
+	len += sprintf(resp_buff + len, "pendingq count: %d\n",
+		       zcrypt_pendingq_count());
+	len += sprintf(resp_buff + len, "Total open handles: %d\n\n",
+		       atomic_read(&zcrypt_open_count));
+	zcrypt_status_mask(workarea);
+	len += sprinthx("Online devices: 1=PCICA 2=PCICC 3=PCIXCC(MCL2) "
+			"4=PCIXCC(MCL3) 5=CEX2C 6=CEX2A",
+			resp_buff+len, workarea, AP_DEVICES);
+	zcrypt_qdepth_mask(workarea);
+	len += sprinthx("Waiting work element counts",
+			resp_buff+len, workarea, AP_DEVICES);
+	zcrypt_perdev_reqcnt((unsigned int *) workarea);
+	len += sprinthx4("Per-device successfully completed request counts",
+			 resp_buff+len,(unsigned int *) workarea, AP_DEVICES);
+	*eof = 1;
+	memset((void *) workarea, 0x00, AP_DEVICES * sizeof(unsigned int));
+	return len;
+}
+
+static void zcrypt_disable_card(int index)
+{
+	struct zcrypt_device *zdev;
+
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list)
+		if (AP_QID_DEVICE(zdev->ap_dev->qid) == index) {
+			zdev->online = 0;
+			ap_flush_queue(zdev->ap_dev);
+			break;
+		}
+	spin_unlock_bh(&zcrypt_device_lock);
+}
+
+static void zcrypt_enable_card(int index)
+{
+	struct zcrypt_device *zdev;
+
+	spin_lock_bh(&zcrypt_device_lock);
+	list_for_each_entry(zdev, &zcrypt_device_list, list)
+		if (AP_QID_DEVICE(zdev->ap_dev->qid) == index) {
+			zdev->online = 1;
+			break;
+		}
+	spin_unlock_bh(&zcrypt_device_lock);
+}
+
+static int zcrypt_status_write(struct file *file, const char __user *buffer,
+			       unsigned long count, void *data)
+{
+	unsigned char *lbuf, *ptr;
+	unsigned long local_count;
+	int j;
+
+	if (count <= 0)
+		return 0;
+
+#define LBUFSIZE 1200UL
+	lbuf = kmalloc(LBUFSIZE, GFP_KERNEL);
+	if (!lbuf) {
+		PRINTK("kmalloc failed!\n");
+		return 0;
+	}
+
+	local_count = min(LBUFSIZE - 1, count);
+	if (copy_from_user(lbuf, buffer, local_count) != 0) {
+		kfree(lbuf);
+		return -EFAULT;
+	}
+	lbuf[local_count] = '\0';
+
+	ptr = strstr(lbuf, "Online devices");
+	if (!ptr) {
+		PRINTK("Unable to parse data (missing \"Online devices\")\n");
+		goto out;
+	}
+	ptr = strstr(ptr, "\n");
+	if (!ptr) {
+		PRINTK("Unable to parse data (missing newline "
+		       "after \"Online devices\")\n");
+		goto out;
+	}
+	ptr++;
+
+	if (strstr(ptr, "Waiting work element counts") == NULL) {
+		PRINTK("Unable to parse data (missing "
+		       "\"Waiting work element counts\")\n");
+		goto out;
+	}
+
+	for (j = 0; j < 64 && *ptr; ptr++) {
+		/**
+		 * '0' for no device, '1' for PCICA, '2' for PCICC,
+		 * '3' for PCIXCC_MCL2, '4' for PCIXCC_MCL3,
+		 * '5' for CEX2C and '6' for CEX2A'
+		 */
+		if (*ptr >= '0' && *ptr <= '6')
+			j++;
+		else if (*ptr == 'd' || *ptr == 'D')
+			zcrypt_disable_card(j++);
+		else if (*ptr == 'e' || *ptr == 'E')
+			zcrypt_enable_card(j++);
+		else if (*ptr != ' ' && *ptr != '\t')
+			break;
+	}
+out:
+	kfree(lbuf);
+	return count;
+}
+
+/**
+ * The module initialization code.
+ */
+int __init zcrypt_api_init(void)
+{
+	int rc;
+
+	/* Register the request sprayer. */
+	rc = misc_register(&zcrypt_misc_device);
+	if (rc < 0) {
+		PRINTKW(KERN_ERR "misc_register (minor %d) failed with %d\n",
+			zcrypt_misc_device.minor, rc);
+		goto out;
+	}
+
+	/* Set up the proc file system */
+	zcrypt_entry = create_proc_entry("driver/z90crypt", 0644, NULL);
+	if (!zcrypt_entry) {
+		PRINTK("Couldn't create z90crypt proc entry\n");
+		rc = -ENOMEM;
+		goto out_misc;
+	}
+	zcrypt_entry->nlink = 1;
+	zcrypt_entry->data = NULL;
+	zcrypt_entry->read_proc = zcrypt_status_read;
+	zcrypt_entry->write_proc = zcrypt_status_write;
+
+	return 0;
+
+out_misc:
+	misc_deregister(&zcrypt_misc_device);
+out:
+	return rc;
+}
+
+/**
+ * The module termination code.
+ */
+void zcrypt_api_exit(void)
+{
+	remove_proc_entry("driver/z90crypt", NULL);
+	misc_deregister(&zcrypt_misc_device);
+}
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+module_init(zcrypt_api_init);
+module_exit(zcrypt_api_exit);
+#endif
diff --git a/drivers/s390/crypto/zcrypt_api.h b/drivers/s390/crypto/zcrypt_api.h
new file mode 100644
index 0000000..de4877e
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_api.h
@@ -0,0 +1,141 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_api.h
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *	       Cornelia Huck <cornelia.huck@de.ibm.com>
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _ZCRYPT_API_H_
+#define _ZCRYPT_API_H_
+
+/**
+ * Macro definitions
+ *
+ * PDEBUG debugs in the form "zcrypt: function_name -> message"
+ *
+ * PRINTK is like PDEBUG, except that it is always enabled
+ * PRINTKN is like PRINTK, except that it does not include the function name
+ * PRINTKW is like PRINTK, except that it uses KERN_WARNING
+ * PRINTKC is like PRINTK, except that it uses KERN_CRIT
+ */
+#define DEV_NAME	"zcrypt"
+
+#define PRINTK(fmt, args...) \
+	printk(KERN_DEBUG DEV_NAME ": %s -> " fmt, __FUNCTION__ , ## args)
+#define PRINTKN(fmt, args...) \
+	printk(KERN_DEBUG DEV_NAME ": " fmt, ## args)
+#define PRINTKW(fmt, args...) \
+	printk(KERN_WARNING DEV_NAME ": %s -> " fmt, __FUNCTION__ , ## args)
+#define PRINTKC(fmt, args...) \
+	printk(KERN_CRIT DEV_NAME ": %s -> " fmt, __FUNCTION__ , ## args)
+
+#ifdef ZCRYPT_DEBUG
+#define PDEBUG(fmt, args...) \
+	printk(KERN_DEBUG DEV_NAME ": %s -> " fmt, __FUNCTION__ , ## args)
+#else
+#define PDEBUG(fmt, args...) do {} while (0)
+#endif
+
+#include "ap_bus.h"
+#include <asm/zcrypt.h>
+
+/* deprecated status calls */
+#define ICAZ90STATUS		_IOR(ZCRYPT_IOCTL_MAGIC, 0x10, struct ica_z90_status)
+#define Z90STAT_PCIXCCCOUNT	_IOR(ZCRYPT_IOCTL_MAGIC, 0x43, int)
+
+/**
+ * This structure is deprecated and the corresponding ioctl() has been
+ * replaced with individual ioctl()s for each piece of data!
+ */
+struct ica_z90_status {
+	int totalcount;
+	int leedslitecount; // PCICA
+	int leeds2count;    // PCICC
+	// int PCIXCCCount; is not in struct for backward compatibility
+	int requestqWaitCount;
+	int pendingqWaitCount;
+	int totalOpenCount;
+	int cryptoDomain;
+	// status: 0=not there, 1=PCICA, 2=PCICC, 3=PCIXCC_MCL2, 4=PCIXCC_MCL3,
+	//	   5=CEX2C
+	unsigned char status[64];
+	// qdepth: # work elements waiting for each device
+	unsigned char qdepth[64];
+};
+
+/**
+ * device type for an actual device is either PCICA, PCICC, PCIXCC_MCL2,
+ * PCIXCC_MCL3, CEX2C, or CEX2A
+ *
+ * NOTE: PCIXCC_MCL3 refers to a PCIXCC with May 2004 version of Licensed
+ *	 Internal Code (LIC) (EC J12220 level 29).
+ *	 PCIXCC_MCL2 refers to any LIC before this level.
+ */
+#define ZCRYPT_PCICA		1
+#define ZCRYPT_PCICC		2
+#define ZCRYPT_PCIXCC_MCL2	3
+#define ZCRYPT_PCIXCC_MCL3	4
+#define ZCRYPT_CEX2C		5
+#define ZCRYPT_CEX2A		6
+
+struct zcrypt_device;
+
+struct zcrypt_ops {
+	long (*rsa_modexpo)(struct zcrypt_device *, struct ica_rsa_modexpo *);
+	long (*rsa_modexpo_crt)(struct zcrypt_device *,
+				struct ica_rsa_modexpo_crt *);
+	long (*send_cprb)(struct zcrypt_device *, struct ica_xcRB *);
+};
+
+struct zcrypt_device {
+	struct list_head list;		/* Device list. */
+	spinlock_t lock;		/* Per device lock. */
+	struct kref refcount;		/* device refcounting */
+	struct ap_device *ap_dev;	/* The "real" ap device. */
+	struct zcrypt_ops *ops;		/* Crypto operations. */
+	int online;			/* User online/offline */
+
+	int user_space_type;		/* User space device id. */
+	char *type_string;		/* User space device name. */
+	int min_mod_size;		/* Min number of bits. */
+	int max_mod_size;		/* Max number of bits. */
+	int short_crt;			/* Card has crt length restriction. */
+	int speed_rating;		/* Speed of the crypto device. */
+
+	int request_count;		/* # current requests. */
+
+	struct ap_message reply;	/* Per-device reply structure. */
+};
+
+struct zcrypt_device *zcrypt_device_alloc(size_t);
+void zcrypt_device_free(struct zcrypt_device *);
+void zcrypt_device_get(struct zcrypt_device *);
+int zcrypt_device_put(struct zcrypt_device *);
+int zcrypt_device_register(struct zcrypt_device *);
+void zcrypt_device_unregister(struct zcrypt_device *);
+int zcrypt_api_init(void);
+void zcrypt_api_exit(void);
+
+#endif /* _ZCRYPT_API_H_ */
diff --git a/drivers/s390/crypto/zcrypt_cca_key.h b/drivers/s390/crypto/zcrypt_cca_key.h
new file mode 100644
index 0000000..8dbcf0e
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_cca_key.h
@@ -0,0 +1,350 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_cca_key.h
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _ZCRYPT_CCA_KEY_H_
+#define _ZCRYPT_CCA_KEY_H_
+
+struct T6_keyBlock_hdr {
+	unsigned short blen;
+	unsigned short ulen;
+	unsigned short flags;
+};
+
+/**
+ * mapping for the cca private ME key token.
+ * Three parts of interest here: the header, the private section and
+ * the public section.
+ *
+ * mapping for the cca key token header
+ */
+struct cca_token_hdr {
+	unsigned char  token_identifier;
+	unsigned char  version;
+	unsigned short token_length;
+	unsigned char  reserved[4];
+} __attribute__((packed));
+
+#define CCA_TKN_HDR_ID_EXT 0x1E
+
+/**
+ * mapping for the cca private ME section
+ */
+struct cca_private_ext_ME_sec {
+	unsigned char  section_identifier;
+	unsigned char  version;
+	unsigned short section_length;
+	unsigned char  private_key_hash[20];
+	unsigned char  reserved1[4];
+	unsigned char  key_format;
+	unsigned char  reserved2;
+	unsigned char  key_name_hash[20];
+	unsigned char  key_use_flags[4];
+	unsigned char  reserved3[6];
+	unsigned char  reserved4[24];
+	unsigned char  confounder[24];
+	unsigned char  exponent[128];
+	unsigned char  modulus[128];
+} __attribute__((packed));
+
+#define CCA_PVT_USAGE_ALL 0x80
+
+/**
+ * mapping for the cca public section
+ * In a private key, the modulus doesn't appear in the public
+ * section. So, an arbitrary public exponent of 0x010001 will be
+ * used, for a section length of 0x0F always.
+ */
+struct cca_public_sec {
+	unsigned char  section_identifier;
+	unsigned char  version;
+	unsigned short section_length;
+	unsigned char  reserved[2];
+	unsigned short exponent_len;
+	unsigned short modulus_bit_len;
+	unsigned short modulus_byte_len;    /* In a private key, this is 0 */
+} __attribute__((packed));
+
+/**
+ * mapping for the cca private CRT key 'token'
+ * The first three parts (the only parts considered in this release)
+ * are: the header, the private section and the public section.
+ * The header and public section are the same as for the
+ * struct cca_private_ext_ME
+ *
+ * Following the structure are the quantities p, q, dp, dq, u, pad,
+ * and modulus, in that order, where pad_len is the modulo 8
+ * complement of the residue modulo 8 of the sum of
+ * (p_len + q_len + dp_len + dq_len + u_len).
+ */
+struct cca_pvt_ext_CRT_sec {
+	unsigned char  section_identifier;
+	unsigned char  version;
+	unsigned short section_length;
+	unsigned char  private_key_hash[20];
+	unsigned char  reserved1[4];
+	unsigned char  key_format;
+	unsigned char  reserved2;
+	unsigned char  key_name_hash[20];
+	unsigned char  key_use_flags[4];
+	unsigned short p_len;
+	unsigned short q_len;
+	unsigned short dp_len;
+	unsigned short dq_len;
+	unsigned short u_len;
+	unsigned short mod_len;
+	unsigned char  reserved3[4];
+	unsigned short pad_len;
+	unsigned char  reserved4[52];
+	unsigned char  confounder[8];
+} __attribute__((packed));
+
+#define CCA_PVT_EXT_CRT_SEC_ID_PVT 0x08
+#define CCA_PVT_EXT_CRT_SEC_FMT_CL 0x40
+
+/**
+ * Set up private key fields of a type6 MEX message.
+ * Note that all numerics in the key token are big-endian,
+ * while the entries in the key block header are little-endian.
+ *
+ * @mex: pointer to user input data
+ * @p: pointer to memory area for the key
+ *
+ * Returns the size of the key area or -EFAULT
+ */
+static inline int zcrypt_type6_mex_key_de(struct ica_rsa_modexpo *mex,
+					  void *p, int big_endian)
+{
+	static struct cca_token_hdr static_pvt_me_hdr = {
+		.token_identifier	=  0x1E,
+		.token_length		=  0x0183,
+	};
+	static struct cca_private_ext_ME_sec static_pvt_me_sec = {
+		.section_identifier	=  0x02,
+		.section_length		=  0x016C,
+		.key_use_flags		= {0x80,0x00,0x00,0x00},
+	};
+	static struct cca_public_sec static_pub_me_sec = {
+		.section_identifier	=  0x04,
+		.section_length		=  0x000F,
+		.exponent_len		=  0x0003,
+	};
+	static char pk_exponent[3] = { 0x01, 0x00, 0x01 };
+	struct {
+		struct T6_keyBlock_hdr t6_hdr;
+		struct cca_token_hdr pvtMeHdr;
+		struct cca_private_ext_ME_sec pvtMeSec;
+		struct cca_public_sec pubMeSec;
+		char exponent[3];
+	} __attribute__((packed)) *key = p;
+	unsigned char *temp;
+
+	memset(key, 0, sizeof(*key));
+
+	if (big_endian) {
+		key->t6_hdr.blen = cpu_to_be16(0x189);
+		key->t6_hdr.ulen = cpu_to_be16(0x189 - 2);
+	} else {
+		key->t6_hdr.blen = cpu_to_le16(0x189);
+		key->t6_hdr.ulen = cpu_to_le16(0x189 - 2);
+	}
+	key->pvtMeHdr = static_pvt_me_hdr;
+	key->pvtMeSec = static_pvt_me_sec;
+	key->pubMeSec = static_pub_me_sec;
+	/**
+	 * In a private key, the modulus doesn't appear in the public
+	 * section. So, an arbitrary public exponent of 0x010001 will be
+	 * used.
+	 */
+	memcpy(key->exponent, pk_exponent, 3);
+
+	/* key parameter block */
+	temp = key->pvtMeSec.exponent +
+		sizeof(key->pvtMeSec.exponent) - mex->inputdatalength;
+	if (copy_from_user(temp, mex->b_key, mex->inputdatalength))
+		return -EFAULT;
+
+	/* modulus */
+	temp = key->pvtMeSec.modulus +
+		sizeof(key->pvtMeSec.modulus) - mex->inputdatalength;
+	if (copy_from_user(temp, mex->n_modulus, mex->inputdatalength))
+		return -EFAULT;
+	key->pubMeSec.modulus_bit_len = 8 * mex->inputdatalength;
+	return sizeof(*key);
+}
+
+/**
+ * Set up private key fields of a type6 MEX message. The _pad variant
+ * strips leading zeroes from the b_key.
+ * Note that all numerics in the key token are big-endian,
+ * while the entries in the key block header are little-endian.
+ *
+ * @mex: pointer to user input data
+ * @p: pointer to memory area for the key
+ *
+ * Returns the size of the key area or -EFAULT
+ */
+static inline int zcrypt_type6_mex_key_en(struct ica_rsa_modexpo *mex,
+					  void *p, int big_endian)
+{
+	static struct cca_token_hdr static_pub_hdr = {
+		.token_identifier	=  0x1E,
+	};
+	static struct cca_public_sec static_pub_sec = {
+		.section_identifier	=  0x04,
+	};
+	struct {
+		struct T6_keyBlock_hdr t6_hdr;
+		struct cca_token_hdr pubHdr;
+		struct cca_public_sec pubSec;
+		char exponent[0];
+	} __attribute__((packed)) *key = p;
+	unsigned char *temp;
+	int i;
+
+	memset(key, 0, sizeof(*key));
+
+	key->pubHdr = static_pub_hdr;
+	key->pubSec = static_pub_sec;
+
+	/* key parameter block */
+	temp = key->exponent;
+	if (copy_from_user(temp, mex->b_key, mex->inputdatalength))
+		return -EFAULT;
+	/* Strip leading zeroes from b_key. */
+	for (i = 0; i < mex->inputdatalength; i++)
+		if (temp[i])
+			break;
+	if (i >= mex->inputdatalength)
+		return -EINVAL;
+	memmove(temp, temp + i, mex->inputdatalength - i);
+	temp += mex->inputdatalength - i;
+	/* modulus */
+	if (copy_from_user(temp, mex->n_modulus, mex->inputdatalength))
+		return -EFAULT;
+
+	key->pubSec.modulus_bit_len = 8 * mex->inputdatalength;
+	key->pubSec.modulus_byte_len = mex->inputdatalength;
+	key->pubSec.exponent_len = mex->inputdatalength - i;
+	key->pubSec.section_length = sizeof(key->pubSec) +
+					2*mex->inputdatalength - i;
+	key->pubHdr.token_length =
+		key->pubSec.section_length + sizeof(key->pubHdr);
+	if (big_endian) {
+		key->t6_hdr.ulen = cpu_to_be16(key->pubHdr.token_length + 4);
+		key->t6_hdr.blen = cpu_to_be16(key->pubHdr.token_length + 6);
+	} else {
+		key->t6_hdr.ulen = cpu_to_le16(key->pubHdr.token_length + 4);
+		key->t6_hdr.blen = cpu_to_le16(key->pubHdr.token_length + 6);
+	}
+	return sizeof(*key) + 2*mex->inputdatalength - i;
+}
+
+/**
+ * Set up private key fields of a type6 CRT message.
+ * Note that all numerics in the key token are big-endian,
+ * while the entries in the key block header are little-endian.
+ *
+ * @mex: pointer to user input data
+ * @p: pointer to memory area for the key
+ *
+ * Returns the size of the key area or -EFAULT
+ */
+static inline int zcrypt_type6_crt_key(struct ica_rsa_modexpo_crt *crt,
+				       void *p, int big_endian)
+{
+	static struct cca_public_sec static_cca_pub_sec = {
+		.section_identifier = 4,
+		.section_length = 0x000f,
+		.exponent_len = 0x0003,
+	};
+	static char pk_exponent[3] = { 0x01, 0x00, 0x01 };
+	struct {
+		struct T6_keyBlock_hdr t6_hdr;
+		struct cca_token_hdr token;
+		struct cca_pvt_ext_CRT_sec pvt;
+		char key_parts[0];
+	} __attribute__((packed)) *key = p;
+	struct cca_public_sec *pub;
+	int short_len, long_len, pad_len, key_len, size;
+
+	memset(key, 0, sizeof(*key));
+
+	short_len = crt->inputdatalength / 2;
+	long_len = short_len + 8;
+	pad_len = -(3*long_len + 2*short_len) & 7;
+	key_len = 3*long_len + 2*short_len + pad_len + crt->inputdatalength;
+	size = sizeof(*key) + key_len + sizeof(*pub) + 3;
+
+	/* parameter block.key block */
+	if (big_endian) {
+		key->t6_hdr.blen = cpu_to_be16(size);
+		key->t6_hdr.ulen = cpu_to_be16(size - 2);
+	} else {
+		key->t6_hdr.blen = cpu_to_le16(size);
+		key->t6_hdr.ulen = cpu_to_le16(size - 2);
+	}
+
+	/* key token header */
+	key->token.token_identifier = CCA_TKN_HDR_ID_EXT;
+	key->token.token_length = size - 6;
+
+	/* private section */
+	key->pvt.section_identifier = CCA_PVT_EXT_CRT_SEC_ID_PVT;
+	key->pvt.section_length = sizeof(key->pvt) + key_len;
+	key->pvt.key_format = CCA_PVT_EXT_CRT_SEC_FMT_CL;
+	key->pvt.key_use_flags[0] = CCA_PVT_USAGE_ALL;
+	key->pvt.p_len = key->pvt.dp_len = key->pvt.u_len = long_len;
+	key->pvt.q_len = key->pvt.dq_len = short_len;
+	key->pvt.mod_len = crt->inputdatalength;
+	key->pvt.pad_len = pad_len;
+
+	/* key parts */
+	if (copy_from_user(key->key_parts, crt->np_prime, long_len) ||
+	    copy_from_user(key->key_parts + long_len,
+					crt->nq_prime, short_len) ||
+	    copy_from_user(key->key_parts + long_len + short_len,
+					crt->bp_key, long_len) ||
+	    copy_from_user(key->key_parts + 2*long_len + short_len,
+					crt->bq_key, short_len) ||
+	    copy_from_user(key->key_parts + 2*long_len + 2*short_len,
+					crt->u_mult_inv, long_len))
+		return -EFAULT;
+	memset(key->key_parts + 3*long_len + 2*short_len + pad_len,
+	       0xff, crt->inputdatalength);
+	pub = (struct cca_public_sec *)(key->key_parts + key_len);
+	*pub = static_cca_pub_sec;
+	pub->modulus_bit_len = 8 * crt->inputdatalength;
+	/**
+	 * In a private key, the modulus doesn't appear in the public
+	 * section. So, an arbitrary public exponent of 0x010001 will be
+	 * used.
+	 */
+	memcpy((char *) (pub + 1), pk_exponent, 3);
+	return size;
+}
+
+#endif /* _ZCRYPT_CCA_KEY_H_ */
diff --git a/drivers/s390/crypto/zcrypt_cex2a.c b/drivers/s390/crypto/zcrypt_cex2a.c
new file mode 100644
index 0000000..a62b000
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_cex2a.c
@@ -0,0 +1,435 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_cex2a.c
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <asm/atomic.h>
+#include <asm/uaccess.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_error.h"
+#include "zcrypt_cex2a.h"
+
+#define CEX2A_MIN_MOD_SIZE	  1	/*    8 bits	*/
+#define CEX2A_MAX_MOD_SIZE	256	/* 2048 bits	*/
+
+#define CEX2A_SPEED_RATING	970
+
+#define CEX2A_MAX_MESSAGE_SIZE	0x390	/* sizeof(struct type50_crb2_msg)    */
+#define CEX2A_MAX_RESPONSE_SIZE 0x110	/* max outputdatalength + type80_hdr */
+
+#define CEX2A_CLEANUP_TIME	(15*HZ)
+
+static struct ap_device_id zcrypt_cex2a_ids[] = {
+	{ AP_DEVICE(AP_DEVICE_TYPE_CEX2A) },
+	{ /* end of list */ },
+};
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+MODULE_DEVICE_TABLE(ap, zcrypt_cex2a_ids);
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("CEX2A Cryptographic Coprocessor device driver, "
+		   "Copyright 2001, 2006 IBM Corporation");
+MODULE_LICENSE("GPL");
+#endif
+
+static int zcrypt_cex2a_probe(struct ap_device *ap_dev);
+static void zcrypt_cex2a_remove(struct ap_device *ap_dev);
+static void zcrypt_cex2a_receive(struct ap_device *, struct ap_message *,
+				 struct ap_message *);
+
+static struct ap_driver zcrypt_cex2a_driver = {
+	.probe = zcrypt_cex2a_probe,
+	.remove = zcrypt_cex2a_remove,
+	.receive = zcrypt_cex2a_receive,
+	.ids = zcrypt_cex2a_ids,
+};
+
+/**
+ * Convert a ICAMEX message to a type50 MEX message.
+ *
+ * @zdev: crypto device pointer
+ * @zreq: crypto request pointer
+ * @mex: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICAMEX_msg_to_type50MEX_msg(struct zcrypt_device *zdev,
+				       struct ap_message *ap_msg,
+				       struct ica_rsa_modexpo *mex)
+{
+	unsigned char *mod, *exp, *inp;
+	int mod_len;
+
+	mod_len = mex->inputdatalength;
+
+	if (mod_len <= 128) {
+		struct type50_meb1_msg *meb1 = ap_msg->message;
+		memset(meb1, 0, sizeof(*meb1));
+		ap_msg->length = sizeof(*meb1);
+		meb1->header.msg_type_code = TYPE50_TYPE_CODE;
+		meb1->header.msg_len = sizeof(*meb1);
+		meb1->keyblock_type = TYPE50_MEB1_FMT;
+		mod = meb1->modulus + sizeof(meb1->modulus) - mod_len;
+		exp = meb1->exponent + sizeof(meb1->exponent) - mod_len;
+		inp = meb1->message + sizeof(meb1->message) - mod_len;
+	} else {
+		struct type50_meb2_msg *meb2 = ap_msg->message;
+		memset(meb2, 0, sizeof(*meb2));
+		ap_msg->length = sizeof(*meb2);
+		meb2->header.msg_type_code = TYPE50_TYPE_CODE;
+		meb2->header.msg_len = sizeof(*meb2);
+		meb2->keyblock_type = TYPE50_MEB2_FMT;
+		mod = meb2->modulus + sizeof(meb2->modulus) - mod_len;
+		exp = meb2->exponent + sizeof(meb2->exponent) - mod_len;
+		inp = meb2->message + sizeof(meb2->message) - mod_len;
+	}
+
+	if (copy_from_user(mod, mex->n_modulus, mod_len) ||
+	    copy_from_user(exp, mex->b_key, mod_len) ||
+	    copy_from_user(inp, mex->inputdata, mod_len))
+		return -EFAULT;
+	return 0;
+}
+
+/**
+ * Convert a ICACRT message to a type50 CRT message.
+ *
+ * @zdev: crypto device pointer
+ * @zreq: crypto request pointer
+ * @crt: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICACRT_msg_to_type50CRT_msg(struct zcrypt_device *zdev,
+				       struct ap_message *ap_msg,
+				       struct ica_rsa_modexpo_crt *crt)
+{
+	int mod_len, short_len, long_len, long_offset;
+	unsigned char *p, *q, *dp, *dq, *u, *inp;
+
+	mod_len = crt->inputdatalength;
+	short_len = mod_len / 2;
+	long_len = mod_len / 2 + 8;
+
+	/*
+	 * CEX2A cannot handle p, dp, or U > 128 bytes.
+	 * If we have one of these, we need to do extra checking.
+	 */
+	if (long_len > 128) {
+		/*
+		 * zcrypt_rsa_crt already checked for the leading
+		 * zeroes of np_prime, bp_key and u_mult_inc.
+		 */
+		long_offset = long_len - 128;
+		long_len = 128;
+	} else
+		long_offset = 0;
+
+	/*
+	 * Instead of doing extra work for p, dp, U > 64 bytes, we'll just use
+	 * the larger message structure.
+	 */
+	if (long_len <= 64) {
+		struct type50_crb1_msg *crb1 = ap_msg->message;
+		memset(crb1, 0, sizeof(*crb1));
+		ap_msg->length = sizeof(*crb1);
+		crb1->header.msg_type_code = TYPE50_TYPE_CODE;
+		crb1->header.msg_len = sizeof(*crb1);
+		crb1->keyblock_type = TYPE50_CRB1_FMT;
+		p = crb1->p + sizeof(crb1->p) - long_len;
+		q = crb1->q + sizeof(crb1->q) - short_len;
+		dp = crb1->dp + sizeof(crb1->dp) - long_len;
+		dq = crb1->dq + sizeof(crb1->dq) - short_len;
+		u = crb1->u + sizeof(crb1->u) - long_len;
+		inp = crb1->message + sizeof(crb1->message) - mod_len;
+	} else {
+		struct type50_crb2_msg *crb2 = ap_msg->message;
+		memset(crb2, 0, sizeof(*crb2));
+		ap_msg->length = sizeof(*crb2);
+		crb2->header.msg_type_code = TYPE50_TYPE_CODE;
+		crb2->header.msg_len = sizeof(*crb2);
+		crb2->keyblock_type = TYPE50_CRB2_FMT;
+		p = crb2->p + sizeof(crb2->p) - long_len;
+		q = crb2->q + sizeof(crb2->q) - short_len;
+		dp = crb2->dp + sizeof(crb2->dp) - long_len;
+		dq = crb2->dq + sizeof(crb2->dq) - short_len;
+		u = crb2->u + sizeof(crb2->u) - long_len;
+		inp = crb2->message + sizeof(crb2->message) - mod_len;
+	}
+
+	if (copy_from_user(p, crt->np_prime + long_offset, long_len) ||
+	    copy_from_user(q, crt->nq_prime, short_len) ||
+	    copy_from_user(dp, crt->bp_key + long_offset, long_len) ||
+	    copy_from_user(dq, crt->bq_key, short_len) ||
+	    copy_from_user(u, crt->u_mult_inv + long_offset, long_len) ||
+	    copy_from_user(inp, crt->inputdata, mod_len))
+		return -EFAULT;
+
+
+	return 0;
+}
+
+/**
+ * Copy results from a type 80 reply message back to user space.
+ *
+ * @zdev: crypto device pointer
+ * @reply: reply AP message.
+ * @data: pointer to user output data
+ * @length: size of user output data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int convert_type80(struct zcrypt_device *zdev,
+			  struct ap_message *reply,
+			  char __user *outputdata,
+			  unsigned int outputdatalength)
+{
+	struct type80_hdr *t80h = reply->message;
+	unsigned char *data;
+
+	if (t80h->len < sizeof(*t80h) + outputdatalength) {
+		/* The result is too short, the CEX2A card may not do that.. */
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+	BUG_ON(t80h->len > CEX2A_MAX_RESPONSE_SIZE);
+	data = reply->message + t80h->len - outputdatalength;
+	if (copy_to_user(outputdata, data, outputdatalength))
+		return -EFAULT;
+	return 0;
+}
+
+static int convert_response(struct zcrypt_device *zdev,
+			    struct ap_message *reply,
+			    char __user *outputdata,
+			    unsigned int outputdatalength)
+{
+	/* Response type byte is the second byte in the response. */
+	switch (((unsigned char *) reply->message)[1]) {
+	case TYPE82_RSP_CODE:
+	case TYPE88_RSP_CODE:
+		return convert_error(zdev, reply);
+	case TYPE80_RSP_CODE:
+		return convert_type80(zdev, reply,
+				      outputdata, outputdatalength);
+	default: /* Unknown response type, this should NEVER EVER happen */
+		PRINTK("Unrecognized Message Header: %08x%08x\n",
+		       *(unsigned int *) reply->message,
+		       *(unsigned int *) (reply->message+4));
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+}
+
+/**
+ * This function is called from the AP bus code after a crypto request
+ * "msg" has finished with the reply message "reply".
+ * It is called from tasklet context.
+ * @ap_dev: pointer to the AP device
+ * @msg: pointer to the AP message
+ * @reply: pointer to the AP reply message
+ */
+static void zcrypt_cex2a_receive(struct ap_device *ap_dev,
+				 struct ap_message *msg,
+				 struct ap_message *reply)
+{
+	static struct error_hdr error_reply = {
+		.type = TYPE82_RSP_CODE,
+		.reply_code = REP82_ERROR_MACHINE_FAILURE,
+	};
+	struct type80_hdr *t80h = reply->message;
+	int length;
+
+	/* Copy the reply message to the request message buffer. */
+	if (IS_ERR(reply))
+		memcpy(msg->message, &error_reply, sizeof(error_reply));
+	else if (t80h->type == TYPE80_RSP_CODE) {
+		length = min(CEX2A_MAX_RESPONSE_SIZE, (int) t80h->len);
+		memcpy(msg->message, reply->message, length);
+	} else
+		memcpy(msg->message, reply->message, sizeof error_reply);
+	complete((struct completion *) msg->private);
+}
+
+static atomic_t zcrypt_step = ATOMIC_INIT(0);
+
+/**
+ * The request distributor calls this function if it picked the CEX2A
+ * device to handle a modexpo request.
+ * @zdev: pointer to zcrypt_device structure that identifies the
+ *	  CEX2A device to the request distributor
+ * @mex: pointer to the modexpo request buffer
+ */
+static long zcrypt_cex2a_modexpo(struct zcrypt_device *zdev,
+				 struct ica_rsa_modexpo *mex)
+{
+	struct ap_message ap_msg;
+	struct completion work;
+	int rc;
+
+	ap_msg.message = (void *) kmalloc(CEX2A_MAX_MESSAGE_SIZE, GFP_KERNEL);
+	if (!ap_msg.message)
+		return -ENOMEM;
+	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg.private = &work;
+	rc = ICAMEX_msg_to_type50MEX_msg(zdev, &ap_msg, mex);
+	if (rc)
+		goto out_free;
+	init_completion(&work);
+	ap_queue_message(zdev->ap_dev, &ap_msg);
+	rc = wait_for_completion_interruptible_timeout(
+				&work, CEX2A_CLEANUP_TIME);
+	if (rc > 0)
+		rc = convert_response(zdev, &ap_msg, mex->outputdata,
+				      mex->outputdatalength);
+	else {
+		/* Signal pending or message timed out. */
+		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		if (rc == 0)
+			/* Message timed out. */
+			rc = -ETIME;
+	}
+out_free:
+	kfree(ap_msg.message);
+	return rc;
+}
+
+/**
+ * The request distributor calls this function if it picked the CEX2A
+ * device to handle a modexpo_crt request.
+ * @zdev: pointer to zcrypt_device structure that identifies the
+ *	  CEX2A device to the request distributor
+ * @crt: pointer to the modexpoc_crt request buffer
+ */
+static long zcrypt_cex2a_modexpo_crt(struct zcrypt_device *zdev,
+				     struct ica_rsa_modexpo_crt *crt)
+{
+	struct ap_message ap_msg;
+	struct completion work;
+	int rc;
+
+	ap_msg.message = (void *) kmalloc(CEX2A_MAX_MESSAGE_SIZE, GFP_KERNEL);
+	if (!ap_msg.message)
+		return -ENOMEM;
+	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg.private = &work;
+	rc = ICACRT_msg_to_type50CRT_msg(zdev, &ap_msg, crt);
+	if (rc)
+		goto out_free;
+	init_completion(&work);
+	ap_queue_message(zdev->ap_dev, &ap_msg);
+	rc = wait_for_completion_interruptible_timeout(
+				&work, CEX2A_CLEANUP_TIME);
+	if (rc > 0)
+		rc = convert_response(zdev, &ap_msg, crt->outputdata,
+				      crt->outputdatalength);
+	else {
+		/* Signal pending or message timed out. */
+		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		if (rc == 0)
+			/* Message timed out. */
+			rc = -ETIME;
+	}
+out_free:
+	kfree(ap_msg.message);
+	return rc;
+}
+
+/**
+ * The crypto operations for a CEX2A card.
+ */
+static struct zcrypt_ops zcrypt_cex2a_ops = {
+	.rsa_modexpo = zcrypt_cex2a_modexpo,
+	.rsa_modexpo_crt = zcrypt_cex2a_modexpo_crt,
+};
+
+/**
+ * Probe function for CEX2A cards. It always accepts the AP device
+ * since the bus_match already checked the hardware type.
+ * @ap_dev: pointer to the AP device.
+ */
+static int zcrypt_cex2a_probe(struct ap_device *ap_dev)
+{
+	struct zcrypt_device *zdev;
+	int rc;
+
+	zdev = zcrypt_device_alloc(CEX2A_MAX_RESPONSE_SIZE);
+	if (!zdev)
+		return -ENOMEM;
+	zdev->ap_dev = ap_dev;
+	zdev->ops = &zcrypt_cex2a_ops;
+	zdev->online = 1;
+	zdev->user_space_type = ZCRYPT_CEX2A;
+	zdev->type_string = "CEX2A";
+	zdev->min_mod_size = CEX2A_MIN_MOD_SIZE;
+	zdev->max_mod_size = CEX2A_MAX_MOD_SIZE;
+	zdev->short_crt = 1;
+	zdev->speed_rating = CEX2A_SPEED_RATING;
+	ap_dev->reply = &zdev->reply;
+	ap_dev->private = zdev;
+	rc = zcrypt_device_register(zdev);
+	if (rc)
+		goto out_free;
+	return 0;
+
+out_free:
+	ap_dev->private = NULL;
+	zcrypt_device_free(zdev);
+	return rc;
+}
+
+/**
+ * This is called to remove the extended CEX2A driver information
+ * if an AP device is removed.
+ */
+static void zcrypt_cex2a_remove(struct ap_device *ap_dev)
+{
+	struct zcrypt_device *zdev = ap_dev->private;
+
+	zcrypt_device_unregister(zdev);
+}
+
+int __init zcrypt_cex2a_init(void)
+{
+	return ap_driver_register(&zcrypt_cex2a_driver, THIS_MODULE, "cex2a");
+}
+
+void __exit zcrypt_cex2a_exit(void)
+{
+	ap_driver_unregister(&zcrypt_cex2a_driver);
+}
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+module_init(zcrypt_cex2a_init);
+module_exit(zcrypt_cex2a_exit);
+#endif
diff --git a/drivers/s390/crypto/zcrypt_cex2a.h b/drivers/s390/crypto/zcrypt_cex2a.h
new file mode 100644
index 0000000..8f69d1d
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_cex2a.h
@@ -0,0 +1,126 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_cex2a.h
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _ZCRYPT_CEX2A_H_
+#define _ZCRYPT_CEX2A_H_
+
+/**
+ * The type 50 message family is associated with a CEX2A card.
+ *
+ * The four members of the family are described below.
+ *
+ * Note that all unsigned char arrays are right-justified and left-padded
+ * with zeroes.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+struct type50_hdr {
+	unsigned char	reserved1;
+	unsigned char	msg_type_code;	/* 0x50 */
+	unsigned short	msg_len;
+	unsigned char	reserved2;
+	unsigned char	ignored;
+	unsigned short	reserved3;
+} __attribute__((packed));
+
+#define TYPE50_TYPE_CODE	0x50
+
+#define TYPE50_MEB1_FMT		0x0001
+#define TYPE50_MEB2_FMT		0x0002
+#define TYPE50_CRB1_FMT		0x0011
+#define TYPE50_CRB2_FMT		0x0012
+
+/* Mod-Exp, with a small modulus */
+struct type50_meb1_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0001 */
+	unsigned char	reserved[6];
+	unsigned char	exponent[128];
+	unsigned char	modulus[128];
+	unsigned char	message[128];
+} __attribute__((packed));
+
+/* Mod-Exp, with a large modulus */
+struct type50_meb2_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0002 */
+	unsigned char	reserved[6];
+	unsigned char	exponent[256];
+	unsigned char	modulus[256];
+	unsigned char	message[256];
+} __attribute__((packed));
+
+/* CRT, with a small modulus */
+struct type50_crb1_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0011 */
+	unsigned char	reserved[6];
+	unsigned char	p[64];
+	unsigned char	q[64];
+	unsigned char	dp[64];
+	unsigned char	dq[64];
+	unsigned char	u[64];
+	unsigned char	message[128];
+} __attribute__((packed));
+
+/* CRT, with a large modulus */
+struct type50_crb2_msg {
+	struct type50_hdr header;
+	unsigned short	keyblock_type;	/* 0x0012 */
+	unsigned char	reserved[6];
+	unsigned char	p[128];
+	unsigned char	q[128];
+	unsigned char	dp[128];
+	unsigned char	dq[128];
+	unsigned char	u[128];
+	unsigned char	message[256];
+} __attribute__((packed));
+
+/**
+ * The type 80 response family is associated with a CEX2A card.
+ *
+ * Note that all unsigned char arrays are right-justified and left-padded
+ * with zeroes.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+
+#define TYPE80_RSP_CODE 0x80
+
+struct type80_hdr {
+	unsigned char	reserved1;
+	unsigned char	type;		/* 0x80 */
+	unsigned short	len;
+	unsigned char	code;		/* 0x00 */
+	unsigned char	reserved2[3];
+	unsigned char	reserved3[8];
+} __attribute__((packed));
+
+int zcrypt_cex2a_init(void);
+void zcrypt_cex2a_exit(void);
+
+#endif /* _ZCRYPT_CEX2A_H_ */
diff --git a/drivers/s390/crypto/zcrypt_error.h b/drivers/s390/crypto/zcrypt_error.h
new file mode 100644
index 0000000..2cb616b
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_error.h
@@ -0,0 +1,133 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_error.h
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _ZCRYPT_ERROR_H_
+#define _ZCRYPT_ERROR_H_
+
+#include "zcrypt_api.h"
+
+/**
+ * Reply Messages
+ *
+ * Error reply messages are of two types:
+ *    82:  Error (see below)
+ *    88:  Error (see below)
+ * Both type 82 and type 88 have the same structure in the header.
+ *
+ * Request reply messages are of three known types:
+ *    80:  Reply from a Type 50 Request (see CEX2A-RELATED STRUCTS)
+ *    84:  Reply from a Type 4 Request (see PCICA-RELATED STRUCTS)
+ *    86:  Reply from a Type 6 Request (see PCICC/PCIXCC/CEX2C-RELATED STRUCTS)
+ *
+ */
+struct error_hdr {
+	unsigned char reserved1;	/* 0x00			*/
+	unsigned char type;		/* 0x82 or 0x88		*/
+	unsigned char reserved2[2];	/* 0x0000		*/
+	unsigned char reply_code;	/* reply code		*/
+	unsigned char reserved3[3];	/* 0x000000		*/
+};
+
+#define TYPE82_RSP_CODE 0x82
+#define TYPE88_RSP_CODE 0x88
+
+#define REP82_ERROR_MACHINE_FAILURE  0x10
+#define REP82_ERROR_PREEMPT_FAILURE  0x12
+#define REP82_ERROR_CHECKPT_FAILURE  0x14
+#define REP82_ERROR_MESSAGE_TYPE     0x20
+#define REP82_ERROR_INVALID_COMM_CD  0x21	/* Type 84	*/
+#define REP82_ERROR_INVALID_MSG_LEN  0x23
+#define REP82_ERROR_RESERVD_FIELD    0x24	/* was 0x50	*/
+#define REP82_ERROR_FORMAT_FIELD     0x29
+#define REP82_ERROR_INVALID_COMMAND  0x30
+#define REP82_ERROR_MALFORMED_MSG    0x40
+#define REP82_ERROR_RESERVED_FIELDO  0x50	/* old value	*/
+#define REP82_ERROR_WORD_ALIGNMENT   0x60
+#define REP82_ERROR_MESSAGE_LENGTH   0x80
+#define REP82_ERROR_OPERAND_INVALID  0x82
+#define REP82_ERROR_OPERAND_SIZE     0x84
+#define REP82_ERROR_EVEN_MOD_IN_OPND 0x85
+#define REP82_ERROR_RESERVED_FIELD   0x88
+#define REP82_ERROR_TRANSPORT_FAIL   0x90
+#define REP82_ERROR_PACKET_TRUNCATED 0xA0
+#define REP82_ERROR_ZERO_BUFFER_LEN  0xB0
+
+#define REP88_ERROR_MODULE_FAILURE   0x10
+
+#define REP88_ERROR_MESSAGE_TYPE     0x20
+#define REP88_ERROR_MESSAGE_MALFORMD 0x22
+#define REP88_ERROR_MESSAGE_LENGTH   0x23
+#define REP88_ERROR_RESERVED_FIELD   0x24
+#define REP88_ERROR_KEY_TYPE	     0x34
+#define REP88_ERROR_INVALID_KEY      0x82	/* CEX2A	*/
+#define REP88_ERROR_OPERAND	     0x84	/* CEX2A	*/
+#define REP88_ERROR_OPERAND_EVEN_MOD 0x85	/* CEX2A	*/
+
+static inline int convert_error(struct zcrypt_device *zdev,
+				struct ap_message *reply)
+{
+	struct error_hdr *ehdr = reply->message;
+
+	PRINTK("Hardware error : Type %02x Message Header: %08x%08x\n",
+	       ehdr->type, *(unsigned int *) reply->message,
+	       *(unsigned int *) (reply->message + 4));
+
+	switch (ehdr->reply_code) {
+	case REP82_ERROR_OPERAND_INVALID:
+	case REP82_ERROR_OPERAND_SIZE:
+	case REP82_ERROR_EVEN_MOD_IN_OPND:
+	case REP88_ERROR_MESSAGE_MALFORMD:
+	//   REP88_ERROR_INVALID_KEY		// '82' CEX2A
+	//   REP88_ERROR_OPERAND		// '84' CEX2A
+	//   REP88_ERROR_OPERAND_EVEN_MOD	// '85' CEX2A
+		/* Invalid input data. */
+		return -EINVAL;
+	case REP82_ERROR_MESSAGE_TYPE:
+	//   REP88_ERROR_MESSAGE_TYPE		// '20' CEX2A
+		/**
+		 * To sent a message of the wrong type is a bug in the
+		 * device driver. Warn about it, disable the device
+		 * and then repeat the request.
+		 */
+		WARN_ON(1);
+		zdev->online = 0;
+		return -EAGAIN;
+	case REP82_ERROR_TRANSPORT_FAIL:
+	case REP82_ERROR_MACHINE_FAILURE:
+	//   REP88_ERROR_MODULE_FAILURE		// '10' CEX2A
+		/* If a card fails disable it and repeat the request. */
+		zdev->online = 0;
+		return -EAGAIN;
+	default:
+		PRINTKW("unknown type %02x reply code = %d\n",
+			ehdr->type, ehdr->reply_code);
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+}
+
+#endif /* _ZCRYPT_ERROR_H_ */
diff --git a/drivers/s390/crypto/zcrypt_mono.c b/drivers/s390/crypto/zcrypt_mono.c
new file mode 100644
index 0000000..2a9349a
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_mono.c
@@ -0,0 +1,100 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_mono.c
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/interrupt.h>
+#include <linux/miscdevice.h>
+#include <linux/fs.h>
+#include <linux/proc_fs.h>
+#include <linux/compat.h>
+#include <asm/atomic.h>
+#include <asm/uaccess.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_pcica.h"
+#include "zcrypt_pcicc.h"
+#include "zcrypt_pcixcc.h"
+#include "zcrypt_cex2a.h"
+
+/**
+ * The module initialization code.
+ */
+int __init zcrypt_init(void)
+{
+	int rc;
+
+	rc = ap_module_init();
+	if (rc)
+		goto out;
+	rc = zcrypt_api_init();
+	if (rc)
+		goto out_ap;
+	rc = zcrypt_pcica_init();
+	if (rc)
+		goto out_api;
+	rc = zcrypt_pcicc_init();
+	if (rc)
+		goto out_pcica;
+	rc = zcrypt_pcixcc_init();
+	if (rc)
+		goto out_pcicc;
+	rc = zcrypt_cex2a_init();
+	if (rc)
+		goto out_pcixcc;
+	return 0;
+
+out_pcixcc:
+	zcrypt_pcixcc_exit();
+out_pcicc:
+	zcrypt_pcicc_exit();
+out_pcica:
+	zcrypt_pcica_exit();
+out_api:
+	zcrypt_api_exit();
+out_ap:
+	ap_module_exit();
+out:
+	return rc;
+}
+
+/**
+ * The module termination code.
+ */
+void __exit zcrypt_exit(void)
+{
+	zcrypt_cex2a_exit();
+	zcrypt_pcixcc_exit();
+	zcrypt_pcicc_exit();
+	zcrypt_pcica_exit();
+	zcrypt_api_exit();
+	ap_module_exit();
+}
+
+module_init(zcrypt_init);
+module_exit(zcrypt_exit);
diff --git a/drivers/s390/crypto/zcrypt_pcica.c b/drivers/s390/crypto/zcrypt_pcica.c
new file mode 100644
index 0000000..b6a4ecd
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_pcica.c
@@ -0,0 +1,418 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_pcica.c
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <asm/atomic.h>
+#include <asm/uaccess.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_error.h"
+#include "zcrypt_pcica.h"
+
+#define PCICA_MIN_MOD_SIZE	  1	/*    8 bits	*/
+#define PCICA_MAX_MOD_SIZE	256	/* 2048 bits	*/
+
+#define PCICA_SPEED_RATING	2800
+
+#define PCICA_MAX_MESSAGE_SIZE	0x3a0	/* sizeof(struct type4_lcr)	     */
+#define PCICA_MAX_RESPONSE_SIZE 0x110	/* max outputdatalength + type80_hdr */
+
+#define PCICA_CLEANUP_TIME	(15*HZ)
+
+static struct ap_device_id zcrypt_pcica_ids[] = {
+	{ AP_DEVICE(AP_DEVICE_TYPE_PCICA) },
+	{ /* end of list */ },
+};
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+MODULE_DEVICE_TABLE(ap, zcrypt_pcica_ids);
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("PCICA Cryptographic Coprocessor device driver, "
+		   "Copyright 2001, 2006 IBM Corporation");
+MODULE_LICENSE("GPL");
+#endif
+
+static int zcrypt_pcica_probe(struct ap_device *ap_dev);
+static void zcrypt_pcica_remove(struct ap_device *ap_dev);
+static void zcrypt_pcica_receive(struct ap_device *, struct ap_message *,
+				 struct ap_message *);
+
+static struct ap_driver zcrypt_pcica_driver = {
+	.probe = zcrypt_pcica_probe,
+	.remove = zcrypt_pcica_remove,
+	.receive = zcrypt_pcica_receive,
+	.ids = zcrypt_pcica_ids,
+};
+
+/**
+ * Convert a ICAMEX message to a type4 MEX message.
+ *
+ * @zdev: crypto device pointer
+ * @zreq: crypto request pointer
+ * @mex: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICAMEX_msg_to_type4MEX_msg(struct zcrypt_device *zdev,
+				      struct ap_message *ap_msg,
+				      struct ica_rsa_modexpo *mex)
+{
+	unsigned char *modulus, *exponent, *message;
+	int mod_len;
+
+	mod_len = mex->inputdatalength;
+
+	if (mod_len <= 128) {
+		struct type4_sme *sme = ap_msg->message;
+		memset(sme, 0, sizeof(*sme));
+		ap_msg->length = sizeof(*sme);
+		sme->header.msg_fmt = TYPE4_SME_FMT;
+		sme->header.msg_len = sizeof(*sme);
+		sme->header.msg_type_code = TYPE4_TYPE_CODE;
+		sme->header.request_code = TYPE4_REQU_CODE;
+		modulus = sme->modulus + sizeof(sme->modulus) - mod_len;
+		exponent = sme->exponent + sizeof(sme->exponent) - mod_len;
+		message = sme->message + sizeof(sme->message) - mod_len;
+	} else {
+		struct type4_lme *lme = ap_msg->message;
+		memset(lme, 0, sizeof(*lme));
+		ap_msg->length = sizeof(*lme);
+		lme->header.msg_fmt = TYPE4_LME_FMT;
+		lme->header.msg_len = sizeof(*lme);
+		lme->header.msg_type_code = TYPE4_TYPE_CODE;
+		lme->header.request_code = TYPE4_REQU_CODE;
+		modulus = lme->modulus + sizeof(lme->modulus) - mod_len;
+		exponent = lme->exponent + sizeof(lme->exponent) - mod_len;
+		message = lme->message + sizeof(lme->message) - mod_len;
+	}
+
+	if (copy_from_user(modulus, mex->n_modulus, mod_len) ||
+	    copy_from_user(exponent, mex->b_key, mod_len) ||
+	    copy_from_user(message, mex->inputdata, mod_len))
+		return -EFAULT;
+	return 0;
+}
+
+/**
+ * Convert a ICACRT message to a type4 CRT message.
+ *
+ * @zdev: crypto device pointer
+ * @zreq: crypto request pointer
+ * @crt: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICACRT_msg_to_type4CRT_msg(struct zcrypt_device *zdev,
+				      struct ap_message *ap_msg,
+				      struct ica_rsa_modexpo_crt *crt)
+{
+	unsigned char *p, *q, *dp, *dq, *u, *inp;
+	int mod_len, short_len, long_len;
+
+	mod_len = crt->inputdatalength;
+	short_len = mod_len / 2;
+	long_len = mod_len / 2 + 8;
+
+	if (mod_len <= 128) {
+		struct type4_scr *scr = ap_msg->message;
+		memset(scr, 0, sizeof(*scr));
+		ap_msg->length = sizeof(*scr);
+		scr->header.msg_type_code = TYPE4_TYPE_CODE;
+		scr->header.request_code = TYPE4_REQU_CODE;
+		scr->header.msg_fmt = TYPE4_SCR_FMT;
+		scr->header.msg_len = sizeof(*scr);
+		p = scr->p + sizeof(scr->p) - long_len;
+		q = scr->q + sizeof(scr->q) - short_len;
+		dp = scr->dp + sizeof(scr->dp) - long_len;
+		dq = scr->dq + sizeof(scr->dq) - short_len;
+		u = scr->u + sizeof(scr->u) - long_len;
+		inp = scr->message + sizeof(scr->message) - mod_len;
+	} else {
+		struct type4_lcr *lcr = ap_msg->message;
+		memset(lcr, 0, sizeof(*lcr));
+		ap_msg->length = sizeof(*lcr);
+		lcr->header.msg_type_code = TYPE4_TYPE_CODE;
+		lcr->header.request_code = TYPE4_REQU_CODE;
+		lcr->header.msg_fmt = TYPE4_LCR_FMT;
+		lcr->header.msg_len = sizeof(*lcr);
+		p = lcr->p + sizeof(lcr->p) - long_len;
+		q = lcr->q + sizeof(lcr->q) - short_len;
+		dp = lcr->dp + sizeof(lcr->dp) - long_len;
+		dq = lcr->dq + sizeof(lcr->dq) - short_len;
+		u = lcr->u + sizeof(lcr->u) - long_len;
+		inp = lcr->message + sizeof(lcr->message) - mod_len;
+	}
+
+	if (copy_from_user(p, crt->np_prime, long_len) ||
+	    copy_from_user(q, crt->nq_prime, short_len) ||
+	    copy_from_user(dp, crt->bp_key, long_len) ||
+	    copy_from_user(dq, crt->bq_key, short_len) ||
+	    copy_from_user(u, crt->u_mult_inv, long_len) ||
+	    copy_from_user(inp, crt->inputdata, mod_len))
+		return -EFAULT;
+	return 0;
+}
+
+/**
+ * Copy results from a type 84 reply message back to user space.
+ *
+ * @zdev: crypto device pointer
+ * @reply: reply AP message.
+ * @data: pointer to user output data
+ * @length: size of user output data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static inline int convert_type84(struct zcrypt_device *zdev,
+				 struct ap_message *reply,
+				 char __user *outputdata,
+				 unsigned int outputdatalength)
+{
+	struct type84_hdr *t84h = reply->message;
+	char *data;
+
+	if (t84h->len < sizeof(*t84h) + outputdatalength) {
+		/* The result is too short, the PCICA card may not do that.. */
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+	BUG_ON(t84h->len > PCICA_MAX_RESPONSE_SIZE);
+	data = reply->message + t84h->len - outputdatalength;
+	if (copy_to_user(outputdata, data, outputdatalength))
+		return -EFAULT;
+	return 0;
+}
+
+static int convert_response(struct zcrypt_device *zdev,
+			    struct ap_message *reply,
+			    char __user *outputdata,
+			    unsigned int outputdatalength)
+{
+	/* Response type byte is the second byte in the response. */
+	switch (((unsigned char *) reply->message)[1]) {
+	case TYPE82_RSP_CODE:
+	case TYPE88_RSP_CODE:
+		return convert_error(zdev, reply);
+	case TYPE84_RSP_CODE:
+		return convert_type84(zdev, reply,
+				      outputdata, outputdatalength);
+	default: /* Unknown response type, this should NEVER EVER happen */
+		PRINTK("Unrecognized Message Header: %08x%08x\n",
+		       *(unsigned int *) reply->message,
+		       *(unsigned int *) (reply->message+4));
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+}
+
+/**
+ * This function is called from the AP bus code after a crypto request
+ * "msg" has finished with the reply message "reply".
+ * It is called from tasklet context.
+ * @ap_dev: pointer to the AP device
+ * @msg: pointer to the AP message
+ * @reply: pointer to the AP reply message
+ */
+static void zcrypt_pcica_receive(struct ap_device *ap_dev,
+				 struct ap_message *msg,
+				 struct ap_message *reply)
+{
+	static struct error_hdr error_reply = {
+		.type = TYPE82_RSP_CODE,
+		.reply_code = REP82_ERROR_MACHINE_FAILURE,
+	};
+	struct type84_hdr *t84h = reply->message;
+	int length;
+
+	/* Copy the reply message to the request message buffer. */
+	if (IS_ERR(reply))
+		memcpy(msg->message, &error_reply, sizeof(error_reply));
+	else if (t84h->code == TYPE84_RSP_CODE) {
+		length = min(PCICA_MAX_RESPONSE_SIZE, (int) t84h->len);
+		memcpy(msg->message, reply->message, length);
+	} else
+		memcpy(msg->message, reply->message, sizeof error_reply);
+	complete((struct completion *) msg->private);
+}
+
+static atomic_t zcrypt_step = ATOMIC_INIT(0);
+
+/**
+ * The request distributor calls this function if it picked the PCICA
+ * device to handle a modexpo request.
+ * @zdev: pointer to zcrypt_device structure that identifies the
+ *	  PCICA device to the request distributor
+ * @mex: pointer to the modexpo request buffer
+ */
+static long zcrypt_pcica_modexpo(struct zcrypt_device *zdev,
+				 struct ica_rsa_modexpo *mex)
+{
+	struct ap_message ap_msg;
+	struct completion work;
+	int rc;
+
+	ap_msg.message = (void *) kmalloc(PCICA_MAX_MESSAGE_SIZE, GFP_KERNEL);
+	if (!ap_msg.message)
+		return -ENOMEM;
+	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg.private = &work;
+	rc = ICAMEX_msg_to_type4MEX_msg(zdev, &ap_msg, mex);
+	if (rc)
+		goto out_free;
+	init_completion(&work);
+	ap_queue_message(zdev->ap_dev, &ap_msg);
+	rc = wait_for_completion_interruptible_timeout(
+				&work, PCICA_CLEANUP_TIME);
+	if (rc > 0)
+		rc = convert_response(zdev, &ap_msg, mex->outputdata,
+				      mex->outputdatalength);
+	else {
+		/* Signal pending or message timed out. */
+		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		if (rc == 0)
+			/* Message timed out. */
+			rc = -ETIME;
+	}
+out_free:
+	kfree(ap_msg.message);
+	return rc;
+}
+
+/**
+ * The request distributor calls this function if it picked the PCICA
+ * device to handle a modexpo_crt request.
+ * @zdev: pointer to zcrypt_device structure that identifies the
+ *	  PCICA device to the request distributor
+ * @crt: pointer to the modexpoc_crt request buffer
+ */
+static long zcrypt_pcica_modexpo_crt(struct zcrypt_device *zdev,
+				     struct ica_rsa_modexpo_crt *crt)
+{
+	struct ap_message ap_msg;
+	struct completion work;
+	int rc;
+
+	ap_msg.message = (void *) kmalloc(PCICA_MAX_MESSAGE_SIZE, GFP_KERNEL);
+	if (!ap_msg.message)
+		return -ENOMEM;
+	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg.private = &work;
+	rc = ICACRT_msg_to_type4CRT_msg(zdev, &ap_msg, crt);
+	if (rc)
+		goto out_free;
+	init_completion(&work);
+	ap_queue_message(zdev->ap_dev, &ap_msg);
+	rc = wait_for_completion_interruptible_timeout(
+				&work, PCICA_CLEANUP_TIME);
+	if (rc > 0)
+		rc = convert_response(zdev, &ap_msg, crt->outputdata,
+				      crt->outputdatalength);
+	else {
+		/* Signal pending or message timed out. */
+		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		if (rc == 0)
+			/* Message timed out. */
+			rc = -ETIME;
+	}
+out_free:
+	kfree(ap_msg.message);
+	return rc;
+}
+
+/**
+ * The crypto operations for a PCICA card.
+ */
+static struct zcrypt_ops zcrypt_pcica_ops = {
+	.rsa_modexpo = zcrypt_pcica_modexpo,
+	.rsa_modexpo_crt = zcrypt_pcica_modexpo_crt,
+};
+
+/**
+ * Probe function for PCICA cards. It always accepts the AP device
+ * since the bus_match already checked the hardware type.
+ * @ap_dev: pointer to the AP device.
+ */
+static int zcrypt_pcica_probe(struct ap_device *ap_dev)
+{
+	struct zcrypt_device *zdev;
+	int rc;
+
+	zdev = zcrypt_device_alloc(PCICA_MAX_RESPONSE_SIZE);
+	if (!zdev)
+		return -ENOMEM;
+	zdev->ap_dev = ap_dev;
+	zdev->ops = &zcrypt_pcica_ops;
+	zdev->online = 1;
+	zdev->user_space_type = ZCRYPT_PCICA;
+	zdev->type_string = "PCICA";
+	zdev->min_mod_size = PCICA_MIN_MOD_SIZE;
+	zdev->max_mod_size = PCICA_MAX_MOD_SIZE;
+	zdev->speed_rating = PCICA_SPEED_RATING;
+	ap_dev->reply = &zdev->reply;
+	ap_dev->private = zdev;
+	rc = zcrypt_device_register(zdev);
+	if (rc)
+		goto out_free;
+	return 0;
+
+out_free:
+	ap_dev->private = NULL;
+	zcrypt_device_free(zdev);
+	return rc;
+}
+
+/**
+ * This is called to remove the extended PCICA driver information
+ * if an AP device is removed.
+ */
+static void zcrypt_pcica_remove(struct ap_device *ap_dev)
+{
+	struct zcrypt_device *zdev = ap_dev->private;
+
+	zcrypt_device_unregister(zdev);
+}
+
+int __init zcrypt_pcica_init(void)
+{
+	return ap_driver_register(&zcrypt_pcica_driver, THIS_MODULE, "pcica");
+}
+
+void zcrypt_pcica_exit(void)
+{
+	ap_driver_unregister(&zcrypt_pcica_driver);
+}
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+module_init(zcrypt_pcica_init);
+module_exit(zcrypt_pcica_exit);
+#endif
diff --git a/drivers/s390/crypto/zcrypt_pcica.h b/drivers/s390/crypto/zcrypt_pcica.h
new file mode 100644
index 0000000..3be1118
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_pcica.h
@@ -0,0 +1,117 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_pcica.h
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _ZCRYPT_PCICA_H_
+#define _ZCRYPT_PCICA_H_
+
+/**
+ * The type 4 message family is associated with a PCICA card.
+ *
+ * The four members of the family are described below.
+ *
+ * Note that all unsigned char arrays are right-justified and left-padded
+ * with zeroes.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+struct type4_hdr {
+	unsigned char  reserved1;
+	unsigned char  msg_type_code;	/* 0x04 */
+	unsigned short msg_len;
+	unsigned char  request_code;	/* 0x40 */
+	unsigned char  msg_fmt;
+	unsigned short reserved2;
+} __attribute__((packed));
+
+#define TYPE4_TYPE_CODE 0x04
+#define TYPE4_REQU_CODE 0x40
+
+#define TYPE4_SME_FMT 0x00
+#define TYPE4_LME_FMT 0x10
+#define TYPE4_SCR_FMT 0x40
+#define TYPE4_LCR_FMT 0x50
+
+/* Mod-Exp, with a small modulus */
+struct type4_sme {
+	struct type4_hdr header;
+	unsigned char	 message[128];
+	unsigned char	 exponent[128];
+	unsigned char	 modulus[128];
+} __attribute__((packed));
+
+/* Mod-Exp, with a large modulus */
+struct type4_lme {
+	struct type4_hdr header;
+	unsigned char	 message[256];
+	unsigned char	 exponent[256];
+	unsigned char	 modulus[256];
+} __attribute__((packed));
+
+/* CRT, with a small modulus */
+struct type4_scr {
+	struct type4_hdr header;
+	unsigned char	 message[128];
+	unsigned char	 dp[72];
+	unsigned char	 dq[64];
+	unsigned char	 p[72];
+	unsigned char	 q[64];
+	unsigned char	 u[72];
+} __attribute__((packed));
+
+/* CRT, with a large modulus */
+struct type4_lcr {
+	struct type4_hdr header;
+	unsigned char	 message[256];
+	unsigned char	 dp[136];
+	unsigned char	 dq[128];
+	unsigned char	 p[136];
+	unsigned char	 q[128];
+	unsigned char	 u[136];
+} __attribute__((packed));
+
+/**
+ * The type 84 response family is associated with a PCICA card.
+ *
+ * Note that all unsigned char arrays are right-justified and left-padded
+ * with zeroes.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+
+struct type84_hdr {
+	unsigned char  reserved1;
+	unsigned char  code;
+	unsigned short len;
+	unsigned char  reserved2[4];
+} __attribute__((packed));
+
+#define TYPE84_RSP_CODE 0x84
+
+int zcrypt_pcica_init(void);
+void zcrypt_pcica_exit(void);
+
+#endif /* _ZCRYPT_PCICA_H_ */
diff --git a/drivers/s390/crypto/zcrypt_pcicc.c b/drivers/s390/crypto/zcrypt_pcicc.c
new file mode 100644
index 0000000..f295a40
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_pcicc.c
@@ -0,0 +1,630 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_pcicc.c
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <asm/atomic.h>
+#include <asm/uaccess.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_error.h"
+#include "zcrypt_pcicc.h"
+#include "zcrypt_cca_key.h"
+
+#define PCICC_MIN_MOD_SIZE	 64	/*  512 bits */
+#define PCICC_MAX_MOD_SIZE_OLD	128	/* 1024 bits */
+#define PCICC_MAX_MOD_SIZE	256	/* 2048 bits */
+
+/**
+ * PCICC cards need a speed rating of 0. This keeps them at the end of
+ * the zcrypt device list (see zcrypt_api.c). PCICC cards are only
+ * used if no other cards are present because they are slow and can only
+ * cope with PKCS12 padded requests. The logic is queer. PKCS11 padded
+ * requests are rejected. The modexpo function encrypts PKCS12 padded data
+ * and decrypts any non-PKCS12 padded data (except PKCS11) in the assumption
+ * that it's encrypted PKCS12 data. The modexpo_crt function always decrypts
+ * the data in the assumption that its PKCS12 encrypted data.
+ */
+#define PCICC_SPEED_RATING	0
+
+#define PCICC_MAX_MESSAGE_SIZE 0x710	/* max size type6 v1 crt message */
+#define PCICC_MAX_RESPONSE_SIZE 0x710	/* max size type86 v1 reply	 */
+
+#define PCICC_CLEANUP_TIME	(15*HZ)
+
+static struct ap_device_id zcrypt_pcicc_ids[] = {
+	{ AP_DEVICE(AP_DEVICE_TYPE_PCICC) },
+	{ /* end of list */ },
+};
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+MODULE_DEVICE_TABLE(ap, zcrypt_pcicc_ids);
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("PCICC Cryptographic Coprocessor device driver, "
+		   "Copyright 2001, 2006 IBM Corporation");
+MODULE_LICENSE("GPL");
+#endif
+
+static int zcrypt_pcicc_probe(struct ap_device *ap_dev);
+static void zcrypt_pcicc_remove(struct ap_device *ap_dev);
+static void zcrypt_pcicc_receive(struct ap_device *, struct ap_message *,
+				 struct ap_message *);
+
+static struct ap_driver zcrypt_pcicc_driver = {
+	.probe = zcrypt_pcicc_probe,
+	.remove = zcrypt_pcicc_remove,
+	.receive = zcrypt_pcicc_receive,
+	.ids = zcrypt_pcicc_ids,
+};
+
+/**
+ * The following is used to initialize the CPRB passed to the PCICC card
+ * in a type6 message. The 3 fields that must be filled in at execution
+ * time are  req_parml, rpl_parml and usage_domain. Note that all three
+ * fields are *little*-endian. Actually, everything about this interface
+ * is ascii/little-endian, since the device has 'Intel inside'.
+ *
+ * The CPRB is followed immediately by the parm block.
+ * The parm block contains:
+ * - function code ('PD' 0x5044 or 'PK' 0x504B)
+ * - rule block (0x0A00 'PKCS-1.2' or 0x0A00 'ZERO-PAD')
+ * - VUD block
+ */
+static struct CPRB static_cprb = {
+	.cprb_len	= __constant_cpu_to_le16(0x0070),
+	.cprb_ver_id	=  0x41,
+	.func_id	= {0x54,0x32},
+	.checkpoint_flag=  0x01,
+	.svr_namel	= __constant_cpu_to_le16(0x0008),
+	.svr_name	= {'I','C','S','F',' ',' ',' ',' '}
+};
+
+/**
+ * Check the message for PKCS11 padding.
+ */
+static inline int is_PKCS11_padded(unsigned char *buffer, int length)
+{
+	int i;
+	if ((buffer[0] != 0x00) || (buffer[1] != 0x01))
+		return 0;
+	for (i = 2; i < length; i++)
+		if (buffer[i] != 0xFF)
+			break;
+	if (i < 10 || i == length)
+		return 0;
+	if (buffer[i] != 0x00)
+		return 0;
+	return 1;
+}
+
+/**
+ * Check the message for PKCS12 padding.
+ */
+static inline int is_PKCS12_padded(unsigned char *buffer, int length)
+{
+	int i;
+	if ((buffer[0] != 0x00) || (buffer[1] != 0x02))
+		return 0;
+	for (i = 2; i < length; i++)
+		if (buffer[i] == 0x00)
+			break;
+	if ((i < 10) || (i == length))
+		return 0;
+	if (buffer[i] != 0x00)
+		return 0;
+	return 1;
+}
+
+/**
+ * Convert a ICAMEX message to a type6 MEX message.
+ *
+ * @zdev: crypto device pointer
+ * @zreq: crypto request pointer
+ * @mex: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICAMEX_msg_to_type6MEX_msg(struct zcrypt_device *zdev,
+				      struct ap_message *ap_msg,
+				      struct ica_rsa_modexpo *mex)
+{
+	static struct type6_hdr static_type6_hdr = {
+		.type		=  0x06,
+		.offset1	=  0x00000058,
+		.agent_id	= {0x01,0x00,0x43,0x43,0x41,0x2D,0x41,0x50,
+				   0x50,0x4C,0x20,0x20,0x20,0x01,0x01,0x01},
+		.function_code	= {'P','K'},
+	};
+	static struct function_and_rules_block static_pke_function_and_rules ={
+		.function_code	= {'P','K'},
+		.ulen		= __constant_cpu_to_le16(10),
+		.only_rule	= {'P','K','C','S','-','1','.','2'}
+	};
+	struct {
+		struct type6_hdr hdr;
+		struct CPRB cprb;
+		struct function_and_rules_block fr;
+		unsigned short length;
+		char text[0];
+	} __attribute__((packed)) *msg = ap_msg->message;
+	int vud_len, pad_len, size;
+
+	/* VUD.ciphertext */
+	if (copy_from_user(msg->text, mex->inputdata, mex->inputdatalength))
+		return -EFAULT;
+
+	if (is_PKCS11_padded(msg->text, mex->inputdatalength))
+		return -EINVAL;
+
+	/* static message header and f&r */
+	msg->hdr = static_type6_hdr;
+	msg->fr = static_pke_function_and_rules;
+
+	if (is_PKCS12_padded(msg->text, mex->inputdatalength)) {
+		/* strip the padding and adjust the data length */
+		pad_len = strnlen(msg->text + 2, mex->inputdatalength - 2) + 3;
+		if (pad_len <= 9 || pad_len >= mex->inputdatalength)
+			return -ENODEV;
+		vud_len = mex->inputdatalength - pad_len;
+		memmove(msg->text, msg->text + pad_len, vud_len);
+		msg->length = cpu_to_le16(vud_len + 2);
+
+		/* Set up key after the variable length text. */
+		size = zcrypt_type6_mex_key_en(mex, msg->text + vud_len, 0);
+		if (size < 0)
+			return size;
+		size += sizeof(*msg) + vud_len;	/* total size of msg */
+	} else {
+		vud_len = mex->inputdatalength;
+		msg->length = cpu_to_le16(2 + vud_len);
+
+		msg->hdr.function_code[1] = 'D';
+		msg->fr.function_code[1] = 'D';
+
+		/* Set up key after the variable length text. */
+		size = zcrypt_type6_mex_key_de(mex, msg->text + vud_len, 0);
+		if (size < 0)
+			return size;
+		size += sizeof(*msg) + vud_len;	/* total size of msg */
+	}
+
+	/* message header, cprb and f&r */
+	msg->hdr.ToCardLen1 = (size - sizeof(msg->hdr) + 3) & -4;
+	msg->hdr.FromCardLen1 = PCICC_MAX_RESPONSE_SIZE - sizeof(msg->hdr);
+
+	msg->cprb = static_cprb;
+	msg->cprb.usage_domain[0]= AP_QID_QUEUE(zdev->ap_dev->qid);
+	msg->cprb.req_parml = cpu_to_le16(size - sizeof(msg->hdr) -
+					   sizeof(msg->cprb));
+	msg->cprb.rpl_parml = cpu_to_le16(msg->hdr.FromCardLen1);
+
+	ap_msg->length = (size + 3) & -4;
+	return 0;
+}
+
+/**
+ * Convert a ICACRT message to a type6 CRT message.
+ *
+ * @zdev: crypto device pointer
+ * @zreq: crypto request pointer
+ * @crt: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICACRT_msg_to_type6CRT_msg(struct zcrypt_device *zdev,
+				      struct ap_message *ap_msg,
+				      struct ica_rsa_modexpo_crt *crt)
+{
+	static struct type6_hdr static_type6_hdr = {
+		.type		=  0x06,
+		.offset1	=  0x00000058,
+		.agent_id	= {0x01,0x00,0x43,0x43,0x41,0x2D,0x41,0x50,
+				   0x50,0x4C,0x20,0x20,0x20,0x01,0x01,0x01},
+		.function_code	= {'P','D'},
+	};
+	static struct function_and_rules_block static_pkd_function_and_rules ={
+		.function_code	= {'P','D'},
+		.ulen		= __constant_cpu_to_le16(10),
+		.only_rule	= {'P','K','C','S','-','1','.','2'}
+	};
+	struct {
+		struct type6_hdr hdr;
+		struct CPRB cprb;
+		struct function_and_rules_block fr;
+		unsigned short length;
+		char text[0];
+	} __attribute__((packed)) *msg = ap_msg->message;
+	int size;
+
+	/* VUD.ciphertext */
+	msg->length = cpu_to_le16(2 + crt->inputdatalength);
+	if (copy_from_user(msg->text, crt->inputdata, crt->inputdatalength))
+		return -EFAULT;
+
+	if (is_PKCS11_padded(msg->text, crt->inputdatalength))
+		return -EINVAL;
+
+	/* Set up key after the variable length text. */
+	size = zcrypt_type6_crt_key(crt, msg->text + crt->inputdatalength, 0);
+	if (size < 0)
+		return size;
+	size += sizeof(*msg) + crt->inputdatalength;	/* total size of msg */
+
+	/* message header, cprb and f&r */
+	msg->hdr = static_type6_hdr;
+	msg->hdr.ToCardLen1 = (size -  sizeof(msg->hdr) + 3) & -4;
+	msg->hdr.FromCardLen1 = PCICC_MAX_RESPONSE_SIZE - sizeof(msg->hdr);
+
+	msg->cprb = static_cprb;
+	msg->cprb.usage_domain[0] = AP_QID_QUEUE(zdev->ap_dev->qid);
+	msg->cprb.req_parml = msg->cprb.rpl_parml =
+		cpu_to_le16(size - sizeof(msg->hdr) - sizeof(msg->cprb));
+
+	msg->fr = static_pkd_function_and_rules;
+
+	ap_msg->length = (size + 3) & -4;
+	return 0;
+}
+
+/**
+ * Copy results from a type 86 reply message back to user space.
+ *
+ * @zdev: crypto device pointer
+ * @reply: reply AP message.
+ * @data: pointer to user output data
+ * @length: size of user output data
+ *
+ * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error.
+ */
+struct type86_reply {
+	struct type86_hdr hdr;
+	struct type86_fmt2_ext fmt2;
+	struct CPRB cprb;
+	unsigned char pad[4];	/* 4 byte function code/rules block ? */
+	unsigned short length;
+	char text[0];
+} __attribute__((packed));
+
+static int convert_type86(struct zcrypt_device *zdev,
+			  struct ap_message *reply,
+			  char __user *outputdata,
+			  unsigned int outputdatalength)
+{
+	static unsigned char static_pad[] = {
+		0x00,0x02,
+		0x1B,0x7B,0x5D,0xB5,0x75,0x01,0x3D,0xFD,
+		0x8D,0xD1,0xC7,0x03,0x2D,0x09,0x23,0x57,
+		0x89,0x49,0xB9,0x3F,0xBB,0x99,0x41,0x5B,
+		0x75,0x21,0x7B,0x9D,0x3B,0x6B,0x51,0x39,
+		0xBB,0x0D,0x35,0xB9,0x89,0x0F,0x93,0xA5,
+		0x0B,0x47,0xF1,0xD3,0xBB,0xCB,0xF1,0x9D,
+		0x23,0x73,0x71,0xFF,0xF3,0xF5,0x45,0xFB,
+		0x61,0x29,0x23,0xFD,0xF1,0x29,0x3F,0x7F,
+		0x17,0xB7,0x1B,0xA9,0x19,0xBD,0x57,0xA9,
+		0xD7,0x95,0xA3,0xCB,0xED,0x1D,0xDB,0x45,
+		0x7D,0x11,0xD1,0x51,0x1B,0xED,0x71,0xE9,
+		0xB1,0xD1,0xAB,0xAB,0x21,0x2B,0x1B,0x9F,
+		0x3B,0x9F,0xF7,0xF7,0xBD,0x63,0xEB,0xAD,
+		0xDF,0xB3,0x6F,0x5B,0xDB,0x8D,0xA9,0x5D,
+		0xE3,0x7D,0x77,0x49,0x47,0xF5,0xA7,0xFD,
+		0xAB,0x2F,0x27,0x35,0x77,0xD3,0x49,0xC9,
+		0x09,0xEB,0xB1,0xF9,0xBF,0x4B,0xCB,0x2B,
+		0xEB,0xEB,0x05,0xFF,0x7D,0xC7,0x91,0x8B,
+		0x09,0x83,0xB9,0xB9,0x69,0x33,0x39,0x6B,
+		0x79,0x75,0x19,0xBF,0xBB,0x07,0x1D,0xBD,
+		0x29,0xBF,0x39,0x95,0x93,0x1D,0x35,0xC7,
+		0xC9,0x4D,0xE5,0x97,0x0B,0x43,0x9B,0xF1,
+		0x16,0x93,0x03,0x1F,0xA5,0xFB,0xDB,0xF3,
+		0x27,0x4F,0x27,0x61,0x05,0x1F,0xB9,0x23,
+		0x2F,0xC3,0x81,0xA9,0x23,0x71,0x55,0x55,
+		0xEB,0xED,0x41,0xE5,0xF3,0x11,0xF1,0x43,
+		0x69,0x03,0xBD,0x0B,0x37,0x0F,0x51,0x8F,
+		0x0B,0xB5,0x89,0x5B,0x67,0xA9,0xD9,0x4F,
+		0x01,0xF9,0x21,0x77,0x37,0x73,0x79,0xC5,
+		0x7F,0x51,0xC1,0xCF,0x97,0xA1,0x75,0xAD,
+		0x35,0x9D,0xD3,0xD3,0xA7,0x9D,0x5D,0x41,
+		0x6F,0x65,0x1B,0xCF,0xA9,0x87,0x91,0x09
+	};
+	struct type86_reply *msg = reply->message;
+	unsigned short service_rc, service_rs;
+	unsigned int reply_len, pad_len;
+	char *data;
+
+	service_rc = le16_to_cpu(msg->cprb.ccp_rtcode);
+	if (unlikely(service_rc != 0)) {
+		service_rs = le16_to_cpu(msg->cprb.ccp_rscode);
+		if (service_rc == 8 && service_rs == 66) {
+			PDEBUG("Bad block format on PCICC\n");
+			return -EINVAL;
+		}
+		if (service_rc == 8 && service_rs == 65) {
+			PDEBUG("Probably an even modulus on PCICC\n");
+			return -EINVAL;
+		}
+		if (service_rc == 8 && service_rs == 770) {
+			PDEBUG("Invalid key length on PCICC\n");
+			zdev->max_mod_size = PCICC_MAX_MOD_SIZE_OLD;
+			return -EAGAIN;
+		}
+		if (service_rc == 8 && service_rs == 783) {
+			PDEBUG("Extended bitlengths not enabled on PCICC\n");
+			zdev->max_mod_size = PCICC_MAX_MOD_SIZE_OLD;
+			return -EAGAIN;
+		}
+		PRINTK("Unknown service rc/rs (PCICC): %d/%d\n",
+		       service_rc, service_rs);
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+	data = msg->text;
+	reply_len = le16_to_cpu(msg->length) - 2;
+	if (reply_len > outputdatalength)
+		return -EINVAL;
+	/**
+	 * For all encipher requests, the length of the ciphertext (reply_len)
+	 * will always equal the modulus length. For MEX decipher requests
+	 * the output needs to get padded. Minimum pad size is 10.
+	 *
+	 * Currently, the cases where padding will be added is for:
+	 * - PCIXCC_MCL2 using a CRT form token (since PKD didn't support
+	 *   ZERO-PAD and CRT is only supported for PKD requests)
+	 * - PCICC, always
+	 */
+	pad_len = outputdatalength - reply_len;
+	if (pad_len > 0) {
+		if (pad_len < 10)
+			return -EINVAL;
+		/* 'restore' padding left in the PCICC/PCIXCC card. */
+		if (copy_to_user(outputdata, static_pad, pad_len - 1))
+			return -EFAULT;
+		if (put_user(0, outputdata + pad_len - 1))
+			return -EFAULT;
+	}
+	/* Copy the crypto response to user space. */
+	if (copy_to_user(outputdata + pad_len, data, reply_len))
+		return -EFAULT;
+	return 0;
+}
+
+static int convert_response(struct zcrypt_device *zdev,
+			    struct ap_message *reply,
+			    char __user *outputdata,
+			    unsigned int outputdatalength)
+{
+	struct type86_reply *msg = reply->message;
+
+	/* Response type byte is the second byte in the response. */
+	switch (msg->hdr.type) {
+	case TYPE82_RSP_CODE:
+	case TYPE88_RSP_CODE:
+		return convert_error(zdev, reply);
+	case TYPE86_RSP_CODE:
+		if (msg->hdr.reply_code)
+			return convert_error(zdev, reply);
+		if (msg->cprb.cprb_ver_id == 0x01)
+			return convert_type86(zdev, reply,
+					      outputdata, outputdatalength);
+		/* no break, incorrect cprb version is an unknown response */
+	default: /* Unknown response type, this should NEVER EVER happen */
+		PRINTK("Unrecognized Message Header: %08x%08x\n",
+		       *(unsigned int *) reply->message,
+		       *(unsigned int *) (reply->message+4));
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+}
+
+/**
+ * This function is called from the AP bus code after a crypto request
+ * "msg" has finished with the reply message "reply".
+ * It is called from tasklet context.
+ * @ap_dev: pointer to the AP device
+ * @msg: pointer to the AP message
+ * @reply: pointer to the AP reply message
+ */
+static void zcrypt_pcicc_receive(struct ap_device *ap_dev,
+				 struct ap_message *msg,
+				 struct ap_message *reply)
+{
+	static struct error_hdr error_reply = {
+		.type = TYPE82_RSP_CODE,
+		.reply_code = REP82_ERROR_MACHINE_FAILURE,
+	};
+	struct type86_reply *t86r = reply->message;
+	int length;
+
+	/* Copy the reply message to the request message buffer. */
+	if (IS_ERR(reply))
+		memcpy(msg->message, &error_reply, sizeof(error_reply));
+	else if (t86r->hdr.type == TYPE86_RSP_CODE &&
+		 t86r->cprb.cprb_ver_id == 0x01) {
+		length = sizeof(struct type86_reply) + t86r->length - 2;
+		length = min(PCICC_MAX_RESPONSE_SIZE, length);
+		memcpy(msg->message, reply->message, length);
+	} else
+		memcpy(msg->message, reply->message, sizeof error_reply);
+	complete((struct completion *) msg->private);
+}
+
+static atomic_t zcrypt_step = ATOMIC_INIT(0);
+
+/**
+ * The request distributor calls this function if it picked the PCICC
+ * device to handle a modexpo request.
+ * @zdev: pointer to zcrypt_device structure that identifies the
+ *	  PCICC device to the request distributor
+ * @mex: pointer to the modexpo request buffer
+ */
+static long zcrypt_pcicc_modexpo(struct zcrypt_device *zdev,
+				 struct ica_rsa_modexpo *mex)
+{
+	struct ap_message ap_msg;
+	struct completion work;
+	int rc;
+
+	ap_msg.message = (void *) get_zeroed_page(GFP_KERNEL);
+	if (!ap_msg.message)
+		return -ENOMEM;
+	ap_msg.length = PAGE_SIZE;
+	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg.private = &work;
+	rc = ICAMEX_msg_to_type6MEX_msg(zdev, &ap_msg, mex);
+	if (rc)
+		goto out_free;
+	init_completion(&work);
+	ap_queue_message(zdev->ap_dev, &ap_msg);
+	rc = wait_for_completion_interruptible_timeout(
+				&work, PCICC_CLEANUP_TIME);
+	if (rc > 0)
+		rc = convert_response(zdev, &ap_msg, mex->outputdata,
+				      mex->outputdatalength);
+	else {
+		/* Signal pending or message timed out. */
+		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		if (rc == 0)
+			/* Message timed out. */
+			rc = -ETIME;
+	}
+out_free:
+	free_page((unsigned long) ap_msg.message);
+	return rc;
+}
+
+/**
+ * The request distributor calls this function if it picked the PCICC
+ * device to handle a modexpo_crt request.
+ * @zdev: pointer to zcrypt_device structure that identifies the
+ *	  PCICC device to the request distributor
+ * @crt: pointer to the modexpoc_crt request buffer
+ */
+static long zcrypt_pcicc_modexpo_crt(struct zcrypt_device *zdev,
+				     struct ica_rsa_modexpo_crt *crt)
+{
+	struct ap_message ap_msg;
+	struct completion work;
+	int rc;
+
+	ap_msg.message = (void *) get_zeroed_page(GFP_KERNEL);
+	if (!ap_msg.message)
+		return -ENOMEM;
+	ap_msg.length = PAGE_SIZE;
+	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg.private = &work;
+	rc = ICACRT_msg_to_type6CRT_msg(zdev, &ap_msg, crt);
+	if (rc)
+		goto out_free;
+	init_completion(&work);
+	ap_queue_message(zdev->ap_dev, &ap_msg);
+	rc = wait_for_completion_interruptible_timeout(
+				&work, PCICC_CLEANUP_TIME);
+	if (rc > 0)
+		rc = convert_response(zdev, &ap_msg, crt->outputdata,
+				      crt->outputdatalength);
+	else {
+		/* Signal pending or message timed out. */
+		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		if (rc == 0)
+			/* Message timed out. */
+			rc = -ETIME;
+	}
+out_free:
+	free_page((unsigned long) ap_msg.message);
+	return rc;
+}
+
+/**
+ * The crypto operations for a PCICC card.
+ */
+static struct zcrypt_ops zcrypt_pcicc_ops = {
+	.rsa_modexpo = zcrypt_pcicc_modexpo,
+	.rsa_modexpo_crt = zcrypt_pcicc_modexpo_crt,
+};
+
+/**
+ * Probe function for PCICC cards. It always accepts the AP device
+ * since the bus_match already checked the hardware type.
+ * @ap_dev: pointer to the AP device.
+ */
+static int zcrypt_pcicc_probe(struct ap_device *ap_dev)
+{
+	struct zcrypt_device *zdev;
+	int rc;
+
+	zdev = zcrypt_device_alloc(PCICC_MAX_RESPONSE_SIZE);
+	if (!zdev)
+		return -ENOMEM;
+	zdev->ap_dev = ap_dev;
+	zdev->ops = &zcrypt_pcicc_ops;
+	zdev->online = 1;
+	zdev->user_space_type = ZCRYPT_PCICC;
+	zdev->type_string = "PCICC";
+	zdev->min_mod_size = PCICC_MIN_MOD_SIZE;
+	zdev->max_mod_size = PCICC_MAX_MOD_SIZE;
+	zdev->speed_rating = PCICC_SPEED_RATING;
+	ap_dev->reply = &zdev->reply;
+	ap_dev->private = zdev;
+	rc = zcrypt_device_register(zdev);
+	if (rc)
+		goto out_free;
+	return 0;
+
+ out_free:
+	ap_dev->private = NULL;
+	zcrypt_device_free(zdev);
+	return rc;
+}
+
+/**
+ * This is called to remove the extended PCICC driver information
+ * if an AP device is removed.
+ */
+static void zcrypt_pcicc_remove(struct ap_device *ap_dev)
+{
+	struct zcrypt_device *zdev = ap_dev->private;
+
+	zcrypt_device_unregister(zdev);
+}
+
+int __init zcrypt_pcicc_init(void)
+{
+	return ap_driver_register(&zcrypt_pcicc_driver, THIS_MODULE, "pcicc");
+}
+
+void zcrypt_pcicc_exit(void)
+{
+	ap_driver_unregister(&zcrypt_pcicc_driver);
+}
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+module_init(zcrypt_pcicc_init);
+module_exit(zcrypt_pcicc_exit);
+#endif
diff --git a/drivers/s390/crypto/zcrypt_pcicc.h b/drivers/s390/crypto/zcrypt_pcicc.h
new file mode 100644
index 0000000..6d44548
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_pcicc.h
@@ -0,0 +1,176 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_pcicc.h
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _ZCRYPT_PCICC_H_
+#define _ZCRYPT_PCICC_H_
+
+/**
+ * The type 6 message family is associated with PCICC or PCIXCC cards.
+ *
+ * It contains a message header followed by a CPRB, both of which
+ * are described below.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+struct type6_hdr {
+	unsigned char reserved1;	/* 0x00				*/
+	unsigned char type;		/* 0x06				*/
+	unsigned char reserved2[2];	/* 0x0000			*/
+	unsigned char right[4];		/* 0x00000000			*/
+	unsigned char reserved3[2];	/* 0x0000			*/
+	unsigned char reserved4[2];	/* 0x0000			*/
+	unsigned char apfs[4];		/* 0x00000000			*/
+	unsigned int  offset1;		/* 0x00000058 (offset to CPRB)	*/
+	unsigned int  offset2;		/* 0x00000000			*/
+	unsigned int  offset3;		/* 0x00000000			*/
+	unsigned int  offset4;		/* 0x00000000			*/
+	unsigned char agent_id[16];	/* PCICC:			*/
+					/*    0x0100			*/
+					/*    0x4343412d4150504c202020	*/
+					/*    0x010101			*/
+					/* PCIXCC:			*/
+					/*    0x4341000000000000	*/
+					/*    0x0000000000000000	*/
+	unsigned char rqid[2];		/* rqid.  internal to 603	*/
+	unsigned char reserved5[2];	/* 0x0000			*/
+	unsigned char function_code[2];	/* for PKD, 0x5044 (ascii 'PD')	*/
+	unsigned char reserved6[2];	/* 0x0000			*/
+	unsigned int  ToCardLen1;	/* (request CPRB len + 3) & -4	*/
+	unsigned int  ToCardLen2;	/* db len 0x00000000 for PKD	*/
+	unsigned int  ToCardLen3;	/* 0x00000000			*/
+	unsigned int  ToCardLen4;	/* 0x00000000			*/
+	unsigned int  FromCardLen1;	/* response buffer length	*/
+	unsigned int  FromCardLen2;	/* db len 0x00000000 for PKD	*/
+	unsigned int  FromCardLen3;	/* 0x00000000			*/
+	unsigned int  FromCardLen4;	/* 0x00000000			*/
+} __attribute__((packed));
+
+/**
+ * CPRB
+ *	  Note that all shorts, ints and longs are little-endian.
+ *	  All pointer fields are 32-bits long, and mean nothing
+ *
+ *	  A request CPRB is followed by a request_parameter_block.
+ *
+ *	  The request (or reply) parameter block is organized thus:
+ *	    function code
+ *	    VUD block
+ *	    key block
+ */
+struct CPRB {
+	unsigned short cprb_len;	/* CPRB length			 */
+	unsigned char cprb_ver_id;	/* CPRB version id.		 */
+	unsigned char pad_000;		/* Alignment pad byte.		 */
+	unsigned char srpi_rtcode[4];	/* SRPI return code LELONG	 */
+	unsigned char srpi_verb;	/* SRPI verb type		 */
+	unsigned char flags;		/* flags			 */
+	unsigned char func_id[2];	/* function id			 */
+	unsigned char checkpoint_flag;	/*				 */
+	unsigned char resv2;		/* reserved			 */
+	unsigned short req_parml;	/* request parameter buffer	 */
+					/* length 16-bit little endian	 */
+	unsigned char req_parmp[4];	/* request parameter buffer	 *
+					 * pointer (means nothing: the	 *
+					 * parameter buffer follows	 *
+					 * the CPRB).			 */
+	unsigned char req_datal[4];	/* request data buffer		 */
+					/* length	  ULELONG	 */
+	unsigned char req_datap[4];	/* request data buffer		 */
+					/* pointer			 */
+	unsigned short rpl_parml;	/* reply  parameter buffer	 */
+					/* length 16-bit little endian	 */
+	unsigned char pad_001[2];	/* Alignment pad bytes. ULESHORT */
+	unsigned char rpl_parmp[4];	/* reply parameter buffer	 *
+					 * pointer (means nothing: the	 *
+					 * parameter buffer follows	 *
+					 * the CPRB).			 */
+	unsigned char rpl_datal[4];	/* reply data buffer len ULELONG */
+	unsigned char rpl_datap[4];	/* reply data buffer		 */
+					/* pointer			 */
+	unsigned short ccp_rscode;	/* server reason code	ULESHORT */
+	unsigned short ccp_rtcode;	/* server return code	ULESHORT */
+	unsigned char repd_parml[2];	/* replied parameter len ULESHORT*/
+	unsigned char mac_data_len[2];	/* Mac Data Length	ULESHORT */
+	unsigned char repd_datal[4];	/* replied data length	ULELONG	 */
+	unsigned char req_pc[2];	/* PC identifier		 */
+	unsigned char res_origin[8];	/* resource origin		 */
+	unsigned char mac_value[8];	/* Mac Value			 */
+	unsigned char logon_id[8];	/* Logon Identifier		 */
+	unsigned char usage_domain[2];	/* cdx				 */
+	unsigned char resv3[18];	/* reserved for requestor	 */
+	unsigned short svr_namel;	/* server name length  ULESHORT	 */
+	unsigned char svr_name[8];	/* server name			 */
+} __attribute__((packed));
+
+/**
+ * The type 86 message family is associated with PCICC and PCIXCC cards.
+ *
+ * It contains a message header followed by a CPRB.  The CPRB is
+ * the same as the request CPRB, which is described above.
+ *
+ * If format is 1, an error condition exists and no data beyond
+ * the 8-byte message header is of interest.
+ *
+ * The non-error message is shown below.
+ *
+ * Note that all reserved fields must be zeroes.
+ */
+struct type86_hdr {
+	unsigned char reserved1;	/* 0x00				*/
+	unsigned char type;		/* 0x86				*/
+	unsigned char format;		/* 0x01 (error) or 0x02 (ok)	*/
+	unsigned char reserved2;	/* 0x00				*/
+	unsigned char reply_code;	/* reply code (see above)	*/
+	unsigned char reserved3[3];	/* 0x000000			*/
+} __attribute__((packed));
+
+#define TYPE86_RSP_CODE 0x86
+#define TYPE86_FMT2	0x02
+
+struct type86_fmt2_ext {
+	unsigned char	  reserved[4];	/* 0x00000000			*/
+	unsigned char	  apfs[4];	/* final status			*/
+	unsigned int	  count1;	/* length of CPRB + parameters	*/
+	unsigned int	  offset1;	/* offset to CPRB		*/
+	unsigned int	  count2;	/* 0x00000000			*/
+	unsigned int	  offset2;	/* db offset 0x00000000 for PKD	*/
+	unsigned int	  count3;	/* 0x00000000			*/
+	unsigned int	  offset3;	/* 0x00000000			*/
+	unsigned int	  count4;	/* 0x00000000			*/
+	unsigned int	  offset4;	/* 0x00000000			*/
+} __attribute__((packed));
+
+struct function_and_rules_block {
+	unsigned char function_code[2];
+	unsigned short ulen;
+	unsigned char only_rule[8];
+} __attribute__((packed));
+
+int zcrypt_pcicc_init(void);
+void zcrypt_pcicc_exit(void);
+
+#endif /* _ZCRYPT_PCICC_H_ */
diff --git a/drivers/s390/crypto/zcrypt_pcixcc.c b/drivers/s390/crypto/zcrypt_pcixcc.c
new file mode 100644
index 0000000..2da8b93
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_pcixcc.c
@@ -0,0 +1,951 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_pcixcc.c
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *				  Ralph Wuerthner <rwuerthn@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/err.h>
+#include <linux/delay.h>
+#include <asm/atomic.h>
+#include <asm/uaccess.h>
+
+#include "ap_bus.h"
+#include "zcrypt_api.h"
+#include "zcrypt_error.h"
+#include "zcrypt_pcicc.h"
+#include "zcrypt_pcixcc.h"
+#include "zcrypt_cca_key.h"
+
+#define PCIXCC_MIN_MOD_SIZE	 16	/*  128 bits	*/
+#define PCIXCC_MIN_MOD_SIZE_OLD	 64	/*  512 bits	*/
+#define PCIXCC_MAX_MOD_SIZE	256	/* 2048 bits	*/
+
+#define PCIXCC_MCL2_SPEED_RATING	7870	/* FIXME: needs finetuning */
+#define PCIXCC_MCL3_SPEED_RATING	7870
+#define CEX2C_SPEED_RATING		8540
+
+#define PCIXCC_MAX_ICA_MESSAGE_SIZE 0x77c  /* max size type6 v2 crt message */
+#define PCIXCC_MAX_ICA_RESPONSE_SIZE 0x77c /* max size type86 v2 reply	    */
+
+#define PCIXCC_MAX_XCRB_MESSAGE_SIZE (12*1024)
+#define PCIXCC_MAX_XCRB_RESPONSE_SIZE PCIXCC_MAX_XCRB_MESSAGE_SIZE
+#define PCIXCC_MAX_XCRB_DATA_SIZE (11*1024)
+#define PCIXCC_MAX_XCRB_REPLY_SIZE (5*1024)
+
+#define PCIXCC_MAX_RESPONSE_SIZE PCIXCC_MAX_XCRB_RESPONSE_SIZE
+
+#define PCIXCC_CLEANUP_TIME	(15*HZ)
+
+#define CEIL4(x) ((((x)+3)/4)*4)
+
+struct response_type {
+	struct completion work;
+	int type;
+};
+#define PCIXCC_RESPONSE_TYPE_ICA  0
+#define PCIXCC_RESPONSE_TYPE_XCRB 1
+
+static struct ap_device_id zcrypt_pcixcc_ids[] = {
+	{ AP_DEVICE(AP_DEVICE_TYPE_PCIXCC) },
+	{ AP_DEVICE(AP_DEVICE_TYPE_CEX2C) },
+	{ /* end of list */ },
+};
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+MODULE_DEVICE_TABLE(ap, zcrypt_pcixcc_ids);
+MODULE_AUTHOR("IBM Corporation");
+MODULE_DESCRIPTION("PCIXCC Cryptographic Coprocessor device driver, "
+		   "Copyright 2001, 2006 IBM Corporation");
+MODULE_LICENSE("GPL");
+#endif
+
+static int zcrypt_pcixcc_probe(struct ap_device *ap_dev);
+static void zcrypt_pcixcc_remove(struct ap_device *ap_dev);
+static void zcrypt_pcixcc_receive(struct ap_device *, struct ap_message *,
+				 struct ap_message *);
+
+static struct ap_driver zcrypt_pcixcc_driver = {
+	.probe = zcrypt_pcixcc_probe,
+	.remove = zcrypt_pcixcc_remove,
+	.receive = zcrypt_pcixcc_receive,
+	.ids = zcrypt_pcixcc_ids,
+};
+
+/**
+ * The following is used to initialize the CPRBX passed to the PCIXCC/CEX2C
+ * card in a type6 message. The 3 fields that must be filled in at execution
+ * time are  req_parml, rpl_parml and usage_domain.
+ * Everything about this interface is ascii/big-endian, since the
+ * device does *not* have 'Intel inside'.
+ *
+ * The CPRBX is followed immediately by the parm block.
+ * The parm block contains:
+ * - function code ('PD' 0x5044 or 'PK' 0x504B)
+ * - rule block (one of:)
+ *   + 0x000A 'PKCS-1.2' (MCL2 'PD')
+ *   + 0x000A 'ZERO-PAD' (MCL2 'PK')
+ *   + 0x000A 'ZERO-PAD' (MCL3 'PD' or CEX2C 'PD')
+ *   + 0x000A 'MRP     ' (MCL3 'PK' or CEX2C 'PK')
+ * - VUD block
+ */
+static struct CPRBX static_cprbx = {
+	.cprb_len	=  0x00DC,
+	.cprb_ver_id	=  0x02,
+	.func_id	= {0x54,0x32},
+};
+
+/**
+ * Convert a ICAMEX message to a type6 MEX message.
+ *
+ * @zdev: crypto device pointer
+ * @ap_msg: pointer to AP message
+ * @mex: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICAMEX_msg_to_type6MEX_msgX(struct zcrypt_device *zdev,
+				       struct ap_message *ap_msg,
+				       struct ica_rsa_modexpo *mex)
+{
+	static struct type6_hdr static_type6_hdrX = {
+		.type		=  0x06,
+		.offset1	=  0x00000058,
+		.agent_id	= {'C','A',},
+		.function_code	= {'P','K'},
+	};
+	static struct function_and_rules_block static_pke_fnr = {
+		.function_code	= {'P','K'},
+		.ulen		= 10,
+		.only_rule	= {'M','R','P',' ',' ',' ',' ',' '}
+	};
+	static struct function_and_rules_block static_pke_fnr_MCL2 = {
+		.function_code	= {'P','K'},
+		.ulen		= 10,
+		.only_rule	= {'Z','E','R','O','-','P','A','D'}
+	};
+	struct {
+		struct type6_hdr hdr;
+		struct CPRBX cprbx;
+		struct function_and_rules_block fr;
+		unsigned short length;
+		char text[0];
+	} __attribute__((packed)) *msg = ap_msg->message;
+	int size;
+
+	/* VUD.ciphertext */
+	msg->length = mex->inputdatalength + 2;
+	if (copy_from_user(msg->text, mex->inputdata, mex->inputdatalength))
+		return -EFAULT;
+
+	/* Set up key which is located after the variable length text. */
+	size = zcrypt_type6_mex_key_en(mex, msg->text+mex->inputdatalength, 1);
+	if (size < 0)
+		return size;
+	size += sizeof(*msg) + mex->inputdatalength;
+
+	/* message header, cprbx and f&r */
+	msg->hdr = static_type6_hdrX;
+	msg->hdr.ToCardLen1 = size - sizeof(msg->hdr);
+	msg->hdr.FromCardLen1 = PCIXCC_MAX_ICA_RESPONSE_SIZE - sizeof(msg->hdr);
+
+	msg->cprbx = static_cprbx;
+	msg->cprbx.domain = AP_QID_QUEUE(zdev->ap_dev->qid);
+	msg->cprbx.rpl_msgbl = msg->hdr.FromCardLen1;
+
+	msg->fr = (zdev->user_space_type == ZCRYPT_PCIXCC_MCL2) ?
+		static_pke_fnr_MCL2 : static_pke_fnr;
+
+	msg->cprbx.req_parml = size - sizeof(msg->hdr) - sizeof(msg->cprbx);
+
+	ap_msg->length = size;
+	return 0;
+}
+
+/**
+ * Convert a ICACRT message to a type6 CRT message.
+ *
+ * @zdev: crypto device pointer
+ * @ap_msg: pointer to AP message
+ * @crt: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+static int ICACRT_msg_to_type6CRT_msgX(struct zcrypt_device *zdev,
+				       struct ap_message *ap_msg,
+				       struct ica_rsa_modexpo_crt *crt)
+{
+	static struct type6_hdr static_type6_hdrX = {
+		.type		=  0x06,
+		.offset1	=  0x00000058,
+		.agent_id	= {'C','A',},
+		.function_code	= {'P','D'},
+	};
+	static struct function_and_rules_block static_pkd_fnr = {
+		.function_code	= {'P','D'},
+		.ulen		= 10,
+		.only_rule	= {'Z','E','R','O','-','P','A','D'}
+	};
+
+	static struct function_and_rules_block static_pkd_fnr_MCL2 = {
+		.function_code	= {'P','D'},
+		.ulen		= 10,
+		.only_rule	= {'P','K','C','S','-','1','.','2'}
+	};
+	struct {
+		struct type6_hdr hdr;
+		struct CPRBX cprbx;
+		struct function_and_rules_block fr;
+		unsigned short length;
+		char text[0];
+	} __attribute__((packed)) *msg = ap_msg->message;
+	int size;
+
+	/* VUD.ciphertext */
+	msg->length = crt->inputdatalength + 2;
+	if (copy_from_user(msg->text, crt->inputdata, crt->inputdatalength))
+		return -EFAULT;
+
+	/* Set up key which is located after the variable length text. */
+	size = zcrypt_type6_crt_key(crt, msg->text + crt->inputdatalength, 1);
+	if (size < 0)
+		return size;
+	size += sizeof(*msg) + crt->inputdatalength;	/* total size of msg */
+
+	/* message header, cprbx and f&r */
+	msg->hdr = static_type6_hdrX;
+	msg->hdr.ToCardLen1 = size -  sizeof(msg->hdr);
+	msg->hdr.FromCardLen1 = PCIXCC_MAX_ICA_RESPONSE_SIZE - sizeof(msg->hdr);
+
+	msg->cprbx = static_cprbx;
+	msg->cprbx.domain = AP_QID_QUEUE(zdev->ap_dev->qid);
+	msg->cprbx.req_parml = msg->cprbx.rpl_msgbl =
+		size - sizeof(msg->hdr) - sizeof(msg->cprbx);
+
+	msg->fr = (zdev->user_space_type == ZCRYPT_PCIXCC_MCL2) ?
+		static_pkd_fnr_MCL2 : static_pkd_fnr;
+
+	ap_msg->length = size;
+	return 0;
+}
+
+/**
+ * Convert a XCRB message to a type6 CPRB message.
+ *
+ * @zdev: crypto device pointer
+ * @ap_msg: pointer to AP message
+ * @xcRB: pointer to user input data
+ *
+ * Returns 0 on success or -EFAULT.
+ */
+struct type86_fmt2_msg {
+	struct type86_hdr hdr;
+	struct type86_fmt2_ext fmt2;
+} __attribute__((packed));
+
+static int XCRB_msg_to_type6CPRB_msgX(struct zcrypt_device *zdev,
+				       struct ap_message *ap_msg,
+				       struct ica_xcRB *xcRB)
+{
+	static struct type6_hdr static_type6_hdrX = {
+		.type		=  0x06,
+		.offset1	=  0x00000058,
+	};
+	struct {
+		struct type6_hdr hdr;
+		struct ica_CPRBX cprbx;
+	} __attribute__((packed)) *msg = ap_msg->message;
+
+	int rcblen = CEIL4(xcRB->request_control_blk_length);
+	int replylen;
+	char *req_data = ap_msg->message + sizeof(struct type6_hdr) + rcblen;
+	char *function_code;
+
+	/* length checks */
+	ap_msg->length = sizeof(struct type6_hdr) +
+		CEIL4(xcRB->request_control_blk_length) +
+		xcRB->request_data_length;
+	if (ap_msg->length > PCIXCC_MAX_XCRB_MESSAGE_SIZE) {
+		PRINTK("Combined message is too large (%ld/%d/%d).\n",
+		    sizeof(struct type6_hdr),
+		    xcRB->request_control_blk_length,
+		    xcRB->request_data_length);
+		return -EFAULT;
+	}
+	if (CEIL4(xcRB->reply_control_blk_length) >
+	    PCIXCC_MAX_XCRB_REPLY_SIZE) {
+		PDEBUG("Reply CPRB length is too large (%d).\n",
+		    xcRB->request_control_blk_length);
+		return -EFAULT;
+	}
+	if (CEIL4(xcRB->reply_data_length) > PCIXCC_MAX_XCRB_DATA_SIZE) {
+		PDEBUG("Reply data block length is too large (%d).\n",
+		    xcRB->reply_data_length);
+		return -EFAULT;
+	}
+	replylen = CEIL4(xcRB->reply_control_blk_length) +
+		CEIL4(xcRB->reply_data_length) +
+		sizeof(struct type86_fmt2_msg);
+	if (replylen > PCIXCC_MAX_XCRB_RESPONSE_SIZE) {
+		PDEBUG("Reply CPRB + data block > PCIXCC_MAX_XCRB_RESPONSE_SIZE"
+		       " (%d/%d/%d).\n",
+		       sizeof(struct type86_fmt2_msg),
+		       xcRB->reply_control_blk_length,
+		       xcRB->reply_data_length);
+		xcRB->reply_control_blk_length = PCIXCC_MAX_XCRB_RESPONSE_SIZE -
+			(sizeof(struct type86_fmt2_msg) +
+			    CEIL4(xcRB->reply_data_length));
+		PDEBUG("Capping Reply CPRB length at %d\n",
+		       xcRB->reply_control_blk_length);
+	}
+
+	/* prepare type6 header */
+	msg->hdr = static_type6_hdrX;
+	memcpy(msg->hdr.agent_id , &(xcRB->agent_ID), sizeof(xcRB->agent_ID));
+	msg->hdr.ToCardLen1 = xcRB->request_control_blk_length;
+	if (xcRB->request_data_length) {
+		msg->hdr.offset2 = msg->hdr.offset1 + rcblen;
+		msg->hdr.ToCardLen2 = xcRB->request_data_length;
+	}
+	msg->hdr.FromCardLen1 = xcRB->reply_control_blk_length;
+	msg->hdr.FromCardLen2 = xcRB->reply_data_length;
+
+	/* prepare CPRB */
+	if (copy_from_user(&(msg->cprbx), xcRB->request_control_blk_addr,
+		    xcRB->request_control_blk_length))
+		return -EFAULT;
+	if (msg->cprbx.cprb_len + sizeof(msg->hdr.function_code) >
+	    xcRB->request_control_blk_length) {
+		PDEBUG("cprb_len too large (%d/%d)\n", msg->cprbx.cprb_len,
+		    xcRB->request_control_blk_length);
+		return -EFAULT;
+	}
+	function_code = ((unsigned char *)&msg->cprbx) + msg->cprbx.cprb_len;
+	memcpy(msg->hdr.function_code, function_code, sizeof(msg->hdr.function_code));
+
+	/* copy data block */
+	if (xcRB->request_data_length &&
+	    copy_from_user(req_data, xcRB->request_data_address,
+		xcRB->request_data_length))
+		return -EFAULT;
+	return 0;
+}
+
+/**
+ * Copy results from a type 86 ICA reply message back to user space.
+ *
+ * @zdev: crypto device pointer
+ * @reply: reply AP message.
+ * @data: pointer to user output data
+ * @length: size of user output data
+ *
+ * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error.
+ */
+struct type86x_reply {
+	struct type86_hdr hdr;
+	struct type86_fmt2_ext fmt2;
+	struct CPRBX cprbx;
+	unsigned char pad[4];	/* 4 byte function code/rules block ? */
+	unsigned short length;
+	char text[0];
+} __attribute__((packed));
+
+static int convert_type86_ica(struct zcrypt_device *zdev,
+			  struct ap_message *reply,
+			  char __user *outputdata,
+			  unsigned int outputdatalength)
+{
+	static unsigned char static_pad[] = {
+		0x00,0x02,
+		0x1B,0x7B,0x5D,0xB5,0x75,0x01,0x3D,0xFD,
+		0x8D,0xD1,0xC7,0x03,0x2D,0x09,0x23,0x57,
+		0x89,0x49,0xB9,0x3F,0xBB,0x99,0x41,0x5B,
+		0x75,0x21,0x7B,0x9D,0x3B,0x6B,0x51,0x39,
+		0xBB,0x0D,0x35,0xB9,0x89,0x0F,0x93,0xA5,
+		0x0B,0x47,0xF1,0xD3,0xBB,0xCB,0xF1,0x9D,
+		0x23,0x73,0x71,0xFF,0xF3,0xF5,0x45,0xFB,
+		0x61,0x29,0x23,0xFD,0xF1,0x29,0x3F,0x7F,
+		0x17,0xB7,0x1B,0xA9,0x19,0xBD,0x57,0xA9,
+		0xD7,0x95,0xA3,0xCB,0xED,0x1D,0xDB,0x45,
+		0x7D,0x11,0xD1,0x51,0x1B,0xED,0x71,0xE9,
+		0xB1,0xD1,0xAB,0xAB,0x21,0x2B,0x1B,0x9F,
+		0x3B,0x9F,0xF7,0xF7,0xBD,0x63,0xEB,0xAD,
+		0xDF,0xB3,0x6F,0x5B,0xDB,0x8D,0xA9,0x5D,
+		0xE3,0x7D,0x77,0x49,0x47,0xF5,0xA7,0xFD,
+		0xAB,0x2F,0x27,0x35,0x77,0xD3,0x49,0xC9,
+		0x09,0xEB,0xB1,0xF9,0xBF,0x4B,0xCB,0x2B,
+		0xEB,0xEB,0x05,0xFF,0x7D,0xC7,0x91,0x8B,
+		0x09,0x83,0xB9,0xB9,0x69,0x33,0x39,0x6B,
+		0x79,0x75,0x19,0xBF,0xBB,0x07,0x1D,0xBD,
+		0x29,0xBF,0x39,0x95,0x93,0x1D,0x35,0xC7,
+		0xC9,0x4D,0xE5,0x97,0x0B,0x43,0x9B,0xF1,
+		0x16,0x93,0x03,0x1F,0xA5,0xFB,0xDB,0xF3,
+		0x27,0x4F,0x27,0x61,0x05,0x1F,0xB9,0x23,
+		0x2F,0xC3,0x81,0xA9,0x23,0x71,0x55,0x55,
+		0xEB,0xED,0x41,0xE5,0xF3,0x11,0xF1,0x43,
+		0x69,0x03,0xBD,0x0B,0x37,0x0F,0x51,0x8F,
+		0x0B,0xB5,0x89,0x5B,0x67,0xA9,0xD9,0x4F,
+		0x01,0xF9,0x21,0x77,0x37,0x73,0x79,0xC5,
+		0x7F,0x51,0xC1,0xCF,0x97,0xA1,0x75,0xAD,
+		0x35,0x9D,0xD3,0xD3,0xA7,0x9D,0x5D,0x41,
+		0x6F,0x65,0x1B,0xCF,0xA9,0x87,0x91,0x09
+	};
+	struct type86x_reply *msg = reply->message;
+	unsigned short service_rc, service_rs;
+	unsigned int reply_len, pad_len;
+	char *data;
+
+	service_rc = msg->cprbx.ccp_rtcode;
+	if (unlikely(service_rc != 0)) {
+		service_rs = msg->cprbx.ccp_rscode;
+		if (service_rc == 8 && service_rs == 66) {
+			PDEBUG("Bad block format on PCIXCC/CEX2C\n");
+			return -EINVAL;
+		}
+		if (service_rc == 8 && service_rs == 65) {
+			PDEBUG("Probably an even modulus on PCIXCC/CEX2C\n");
+			return -EINVAL;
+		}
+		if (service_rc == 8 && service_rs == 770) {
+			PDEBUG("Invalid key length on PCIXCC/CEX2C\n");
+			zdev->min_mod_size = PCIXCC_MIN_MOD_SIZE_OLD;
+			return -EAGAIN;
+		}
+		if (service_rc == 8 && service_rs == 783) {
+			PDEBUG("Extended bitlengths not enabled on PCIXCC/CEX2C\n");
+			zdev->min_mod_size = PCIXCC_MIN_MOD_SIZE_OLD;
+			return -EAGAIN;
+		}
+		PRINTK("Unknown service rc/rs (PCIXCC/CEX2C): %d/%d\n",
+		       service_rc, service_rs);
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+	data = msg->text;
+	reply_len = msg->length - 2;
+	if (reply_len > outputdatalength)
+		return -EINVAL;
+	/**
+	 * For all encipher requests, the length of the ciphertext (reply_len)
+	 * will always equal the modulus length. For MEX decipher requests
+	 * the output needs to get padded. Minimum pad size is 10.
+	 *
+	 * Currently, the cases where padding will be added is for:
+	 * - PCIXCC_MCL2 using a CRT form token (since PKD didn't support
+	 *   ZERO-PAD and CRT is only supported for PKD requests)
+	 * - PCICC, always
+	 */
+	pad_len = outputdatalength - reply_len;
+	if (pad_len > 0) {
+		if (pad_len < 10)
+			return -EINVAL;
+		/* 'restore' padding left in the PCICC/PCIXCC card. */
+		if (copy_to_user(outputdata, static_pad, pad_len - 1))
+			return -EFAULT;
+		if (put_user(0, outputdata + pad_len - 1))
+			return -EFAULT;
+	}
+	/* Copy the crypto response to user space. */
+	if (copy_to_user(outputdata + pad_len, data, reply_len))
+		return -EFAULT;
+	return 0;
+}
+
+/**
+ * Copy results from a type 86 XCRB reply message back to user space.
+ *
+ * @zdev: crypto device pointer
+ * @reply: reply AP message.
+ * @xcRB: pointer to XCRB
+ *
+ * Returns 0 on success or -EINVAL, -EFAULT, -EAGAIN in case of an error.
+ */
+static int convert_type86_xcrb(struct zcrypt_device *zdev,
+			       struct ap_message *reply,
+			       struct ica_xcRB *xcRB)
+{
+	struct type86_fmt2_msg *msg = reply->message;
+	char *data = reply->message;
+
+	/* Copy CPRB to user */
+	if (copy_to_user(xcRB->reply_control_blk_addr,
+		data + msg->fmt2.offset1, msg->fmt2.count1))
+		return -EFAULT;
+	xcRB->reply_control_blk_length = msg->fmt2.count1;
+
+	/* Copy data buffer to user */
+	if (msg->fmt2.count2)
+		if (copy_to_user(xcRB->reply_data_addr,
+			data + msg->fmt2.offset2, msg->fmt2.count2))
+			return -EFAULT;
+	xcRB->reply_data_length = msg->fmt2.count2;
+	return 0;
+}
+
+static int convert_response_ica(struct zcrypt_device *zdev,
+			    struct ap_message *reply,
+			    char __user *outputdata,
+			    unsigned int outputdatalength)
+{
+	struct type86x_reply *msg = reply->message;
+
+	/* Response type byte is the second byte in the response. */
+	switch (((unsigned char *) reply->message)[1]) {
+	case TYPE82_RSP_CODE:
+	case TYPE88_RSP_CODE:
+		return convert_error(zdev, reply);
+	case TYPE86_RSP_CODE:
+		if (msg->hdr.reply_code)
+			return convert_error(zdev, reply);
+		if (msg->cprbx.cprb_ver_id == 0x02)
+			return convert_type86_ica(zdev, reply,
+						  outputdata, outputdatalength);
+		/* no break, incorrect cprb version is an unknown response */
+	default: /* Unknown response type, this should NEVER EVER happen */
+		PRINTK("Unrecognized Message Header: %08x%08x\n",
+		       *(unsigned int *) reply->message,
+		       *(unsigned int *) (reply->message+4));
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+}
+
+static int convert_response_xcrb(struct zcrypt_device *zdev,
+			    struct ap_message *reply,
+			    struct ica_xcRB *xcRB)
+{
+	struct type86x_reply *msg = reply->message;
+
+	/* Response type byte is the second byte in the response. */
+	switch (((unsigned char *) reply->message)[1]) {
+	case TYPE82_RSP_CODE:
+	case TYPE88_RSP_CODE:
+		xcRB->status = 0x0008044DL; /* HDD_InvalidParm */
+		return convert_error(zdev, reply);
+	case TYPE86_RSP_CODE:
+		if (msg->hdr.reply_code) {
+			memcpy(&(xcRB->status), msg->fmt2.apfs, sizeof(u32));
+			return convert_error(zdev, reply);
+		}
+		if (msg->cprbx.cprb_ver_id == 0x02)
+			return convert_type86_xcrb(zdev, reply, xcRB);
+		/* no break, incorrect cprb version is an unknown response */
+	default: /* Unknown response type, this should NEVER EVER happen */
+		PRINTK("Unrecognized Message Header: %08x%08x\n",
+		       *(unsigned int *) reply->message,
+		       *(unsigned int *) (reply->message+4));
+		xcRB->status = 0x0008044DL; /* HDD_InvalidParm */
+		zdev->online = 0;
+		return -EAGAIN;	/* repeat the request on a different device. */
+	}
+}
+
+/**
+ * This function is called from the AP bus code after a crypto request
+ * "msg" has finished with the reply message "reply".
+ * It is called from tasklet context.
+ * @ap_dev: pointer to the AP device
+ * @msg: pointer to the AP message
+ * @reply: pointer to the AP reply message
+ */
+static void zcrypt_pcixcc_receive(struct ap_device *ap_dev,
+				  struct ap_message *msg,
+				  struct ap_message *reply)
+{
+	static struct error_hdr error_reply = {
+		.type = TYPE82_RSP_CODE,
+		.reply_code = REP82_ERROR_MACHINE_FAILURE,
+	};
+	struct response_type *resp_type =
+		(struct response_type *) msg->private;
+	struct type86x_reply *t86r = reply->message;
+	int length;
+
+	/* Copy the reply message to the request message buffer. */
+	if (IS_ERR(reply))
+		memcpy(msg->message, &error_reply, sizeof(error_reply));
+	else if (t86r->hdr.type == TYPE86_RSP_CODE &&
+		 t86r->cprbx.cprb_ver_id == 0x02) {
+		switch (resp_type->type) {
+		case PCIXCC_RESPONSE_TYPE_ICA:
+			length = sizeof(struct type86x_reply)
+				+ t86r->length - 2;
+			length = min(PCIXCC_MAX_ICA_RESPONSE_SIZE, length);
+			memcpy(msg->message, reply->message, length);
+			break;
+		case PCIXCC_RESPONSE_TYPE_XCRB:
+			length = t86r->fmt2.offset2 + t86r->fmt2.count2;
+			length = min(PCIXCC_MAX_XCRB_RESPONSE_SIZE, length);
+			memcpy(msg->message, reply->message, length);
+			break;
+		default:
+			PRINTK("Invalid internal response type: %i\n",
+			    resp_type->type);
+			memcpy(msg->message, &error_reply,
+			    sizeof error_reply);
+		}
+	} else
+		memcpy(msg->message, reply->message, sizeof error_reply);
+	complete(&(resp_type->work));
+}
+
+static atomic_t zcrypt_step = ATOMIC_INIT(0);
+
+/**
+ * The request distributor calls this function if it picked the PCIXCC/CEX2C
+ * device to handle a modexpo request.
+ * @zdev: pointer to zcrypt_device structure that identifies the
+ *	  PCIXCC/CEX2C device to the request distributor
+ * @mex: pointer to the modexpo request buffer
+ */
+static long zcrypt_pcixcc_modexpo(struct zcrypt_device *zdev,
+				  struct ica_rsa_modexpo *mex)
+{
+	struct ap_message ap_msg;
+	struct response_type resp_type = {
+		.type = PCIXCC_RESPONSE_TYPE_ICA,
+	};
+	int rc;
+
+	ap_msg.message = (void *) get_zeroed_page(GFP_KERNEL);
+	if (!ap_msg.message)
+		return -ENOMEM;
+	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg.private = &resp_type;
+	rc = ICAMEX_msg_to_type6MEX_msgX(zdev, &ap_msg, mex);
+	if (rc)
+		goto out_free;
+	init_completion(&resp_type.work);
+	ap_queue_message(zdev->ap_dev, &ap_msg);
+	rc = wait_for_completion_interruptible_timeout(
+				&resp_type.work, PCIXCC_CLEANUP_TIME);
+	if (rc > 0)
+		rc = convert_response_ica(zdev, &ap_msg, mex->outputdata,
+					  mex->outputdatalength);
+	else {
+		/* Signal pending or message timed out. */
+		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		if (rc == 0)
+			/* Message timed out. */
+			rc = -ETIME;
+	}
+out_free:
+	free_page((unsigned long) ap_msg.message);
+	return rc;
+}
+
+/**
+ * The request distributor calls this function if it picked the PCIXCC/CEX2C
+ * device to handle a modexpo_crt request.
+ * @zdev: pointer to zcrypt_device structure that identifies the
+ *	  PCIXCC/CEX2C device to the request distributor
+ * @crt: pointer to the modexpoc_crt request buffer
+ */
+static long zcrypt_pcixcc_modexpo_crt(struct zcrypt_device *zdev,
+				      struct ica_rsa_modexpo_crt *crt)
+{
+	struct ap_message ap_msg;
+	struct response_type resp_type = {
+		.type = PCIXCC_RESPONSE_TYPE_ICA,
+	};
+	int rc;
+
+	ap_msg.message = (void *) get_zeroed_page(GFP_KERNEL);
+	if (!ap_msg.message)
+		return -ENOMEM;
+	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg.private = &resp_type;
+	rc = ICACRT_msg_to_type6CRT_msgX(zdev, &ap_msg, crt);
+	if (rc)
+		goto out_free;
+	init_completion(&resp_type.work);
+	ap_queue_message(zdev->ap_dev, &ap_msg);
+	rc = wait_for_completion_interruptible_timeout(
+				&resp_type.work, PCIXCC_CLEANUP_TIME);
+	if (rc > 0)
+		rc = convert_response_ica(zdev, &ap_msg, crt->outputdata,
+					  crt->outputdatalength);
+	else {
+		/* Signal pending or message timed out. */
+		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		if (rc == 0)
+			/* Message timed out. */
+			rc = -ETIME;
+	}
+out_free:
+	free_page((unsigned long) ap_msg.message);
+	return rc;
+}
+
+/**
+ * The request distributor calls this function if it picked the PCIXCC/CEX2C
+ * device to handle a send_cprb request.
+ * @zdev: pointer to zcrypt_device structure that identifies the
+ *	  PCIXCC/CEX2C device to the request distributor
+ * @xcRB: pointer to the send_cprb request buffer
+ */
+long zcrypt_pcixcc_send_cprb(struct zcrypt_device *zdev, struct ica_xcRB *xcRB)
+{
+	struct ap_message ap_msg;
+	struct response_type resp_type = {
+		.type = PCIXCC_RESPONSE_TYPE_XCRB,
+	};
+	int rc;
+
+	ap_msg.message = (void *) kmalloc(PCIXCC_MAX_XCRB_MESSAGE_SIZE, GFP_KERNEL);
+	if (!ap_msg.message)
+		return -ENOMEM;
+	ap_msg.psmid = (((unsigned long long) current->pid) << 32) +
+				atomic_inc_return(&zcrypt_step);
+	ap_msg.private = &resp_type;
+	rc = XCRB_msg_to_type6CPRB_msgX(zdev, &ap_msg, xcRB);
+	if (rc)
+		goto out_free;
+	init_completion(&resp_type.work);
+	ap_queue_message(zdev->ap_dev, &ap_msg);
+	rc = wait_for_completion_interruptible_timeout(
+				&resp_type.work, PCIXCC_CLEANUP_TIME);
+	if (rc > 0)
+		rc = convert_response_xcrb(zdev, &ap_msg, xcRB);
+	else {
+		/* Signal pending or message timed out. */
+		ap_cancel_message(zdev->ap_dev, &ap_msg);
+		if (rc == 0)
+			/* Message timed out. */
+			rc = -ETIME;
+	}
+out_free:
+	memset(ap_msg.message, 0x0, ap_msg.length);
+	kfree(ap_msg.message);
+	return rc;
+}
+
+/**
+ * The crypto operations for a PCIXCC/CEX2C card.
+ */
+static struct zcrypt_ops zcrypt_pcixcc_ops = {
+	.rsa_modexpo = zcrypt_pcixcc_modexpo,
+	.rsa_modexpo_crt = zcrypt_pcixcc_modexpo_crt,
+	.send_cprb = zcrypt_pcixcc_send_cprb,
+};
+
+/**
+ * Micro-code detection function. Its sends a message to a pcixcc card
+ * to find out the microcode level.
+ * @ap_dev: pointer to the AP device.
+ */
+static int zcrypt_pcixcc_mcl(struct ap_device *ap_dev)
+{
+	static unsigned char msg[] = {
+		0x00,0x06,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x58,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x43,0x41,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x50,0x4B,0x00,0x00,
+		0x00,0x00,0x01,0xC4,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x07,0x24,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0xDC,0x02,0x00,0x00,0x00,0x54,0x32,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0xE8,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x07,0x24,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x04,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,
+		0x00,0x00,0x00,0x00,0x50,0x4B,0x00,0x0A,
+		0x4D,0x52,0x50,0x20,0x20,0x20,0x20,0x20,
+		0x00,0x42,0x00,0x01,0x02,0x03,0x04,0x05,
+		0x06,0x07,0x08,0x09,0x0A,0x0B,0x0C,0x0D,
+		0x0E,0x0F,0x00,0x11,0x22,0x33,0x44,0x55,
+		0x66,0x77,0x88,0x99,0xAA,0xBB,0xCC,0xDD,
+		0xEE,0xFF,0xFF,0xEE,0xDD,0xCC,0xBB,0xAA,
+		0x99,0x88,0x77,0x66,0x55,0x44,0x33,0x22,
+		0x11,0x00,0x01,0x23,0x45,0x67,0x89,0xAB,
+		0xCD,0xEF,0xFE,0xDC,0xBA,0x98,0x76,0x54,
+		0x32,0x10,0x00,0x9A,0x00,0x98,0x00,0x00,
+		0x1E,0x00,0x00,0x94,0x00,0x00,0x00,0x00,
+		0x04,0x00,0x00,0x8C,0x00,0x00,0x00,0x40,
+		0x02,0x00,0x00,0x40,0xBA,0xE8,0x23,0x3C,
+		0x75,0xF3,0x91,0x61,0xD6,0x73,0x39,0xCF,
+		0x7B,0x6D,0x8E,0x61,0x97,0x63,0x9E,0xD9,
+		0x60,0x55,0xD6,0xC7,0xEF,0xF8,0x1E,0x63,
+		0x95,0x17,0xCC,0x28,0x45,0x60,0x11,0xC5,
+		0xC4,0x4E,0x66,0xC6,0xE6,0xC3,0xDE,0x8A,
+		0x19,0x30,0xCF,0x0E,0xD7,0xAA,0xDB,0x01,
+		0xD8,0x00,0xBB,0x8F,0x39,0x9F,0x64,0x28,
+		0xF5,0x7A,0x77,0x49,0xCC,0x6B,0xA3,0x91,
+		0x97,0x70,0xE7,0x60,0x1E,0x39,0xE1,0xE5,
+		0x33,0xE1,0x15,0x63,0x69,0x08,0x80,0x4C,
+		0x67,0xC4,0x41,0x8F,0x48,0xDF,0x26,0x98,
+		0xF1,0xD5,0x8D,0x88,0xD9,0x6A,0xA4,0x96,
+		0xC5,0x84,0xD9,0x30,0x49,0x67,0x7D,0x19,
+		0xB1,0xB3,0x45,0x4D,0xB2,0x53,0x9A,0x47,
+		0x3C,0x7C,0x55,0xBF,0xCC,0x85,0x00,0x36,
+		0xF1,0x3D,0x93,0x53
+	};
+	unsigned long long psmid;
+	struct CPRBX *cprbx;
+	char *reply;
+	int rc, i;
+
+	reply = (void *) get_zeroed_page(GFP_KERNEL);
+	if (!reply)
+		return -ENOMEM;
+
+	rc = ap_send(ap_dev->qid, 0x0102030405060708ULL, msg, sizeof(msg));
+	if (rc)
+		goto out_free;
+
+	/* Wait for the test message to complete. */
+	for (i = 0; i < 6; i++) {
+		mdelay(300);
+		rc = ap_recv(ap_dev->qid, &psmid, reply, 4096);
+		if (rc == 0 && psmid == 0x0102030405060708ULL)
+			break;
+	}
+
+	if (i >= 6) {
+		/* Got no answer. */
+		rc = -ENODEV;
+		goto out_free;
+	}
+
+	cprbx = (struct CPRBX *) (reply + 48);
+	if (cprbx->ccp_rtcode == 8 && cprbx->ccp_rscode == 33)
+		rc = ZCRYPT_PCIXCC_MCL2;
+	else
+		rc = ZCRYPT_PCIXCC_MCL3;
+out_free:
+	free_page((unsigned long) reply);
+	return rc;
+}
+
+/**
+ * Probe function for PCIXCC/CEX2C cards. It always accepts the AP device
+ * since the bus_match already checked the hardware type. The PCIXCC
+ * cards come in two flavours: micro code level 2 and micro code level 3.
+ * This is checked by sending a test message to the device.
+ * @ap_dev: pointer to the AP device.
+ */
+static int zcrypt_pcixcc_probe(struct ap_device *ap_dev)
+{
+	struct zcrypt_device *zdev;
+	int rc;
+
+	zdev = zcrypt_device_alloc(PCIXCC_MAX_RESPONSE_SIZE);
+	if (!zdev)
+		return -ENOMEM;
+	zdev->ap_dev = ap_dev;
+	zdev->ops = &zcrypt_pcixcc_ops;
+	zdev->online = 1;
+	if (ap_dev->device_type == AP_DEVICE_TYPE_PCIXCC) {
+		rc = zcrypt_pcixcc_mcl(ap_dev);
+		if (rc < 0) {
+			zcrypt_device_free(zdev);
+			return rc;
+		}
+		zdev->user_space_type = rc;
+		if (rc == ZCRYPT_PCIXCC_MCL2) {
+			zdev->type_string = "PCIXCC_MCL2";
+			zdev->speed_rating = PCIXCC_MCL2_SPEED_RATING;
+			zdev->min_mod_size = PCIXCC_MIN_MOD_SIZE_OLD;
+			zdev->max_mod_size = PCIXCC_MAX_MOD_SIZE;
+		} else {
+			zdev->type_string = "PCIXCC_MCL3";
+			zdev->speed_rating = PCIXCC_MCL3_SPEED_RATING;
+			zdev->min_mod_size = PCIXCC_MIN_MOD_SIZE;
+			zdev->max_mod_size = PCIXCC_MAX_MOD_SIZE;
+		}
+	} else {
+		zdev->user_space_type = ZCRYPT_CEX2C;
+		zdev->type_string = "CEX2C";
+		zdev->speed_rating = CEX2C_SPEED_RATING;
+		zdev->min_mod_size = PCIXCC_MIN_MOD_SIZE;
+		zdev->max_mod_size = PCIXCC_MAX_MOD_SIZE;
+	}
+	ap_dev->reply = &zdev->reply;
+	ap_dev->private = zdev;
+	rc = zcrypt_device_register(zdev);
+	if (rc)
+		goto out_free;
+	return 0;
+
+ out_free:
+	ap_dev->private = NULL;
+	zcrypt_device_free(zdev);
+	return rc;
+}
+
+/**
+ * This is called to remove the extended PCIXCC/CEX2C driver information
+ * if an AP device is removed.
+ */
+static void zcrypt_pcixcc_remove(struct ap_device *ap_dev)
+{
+	struct zcrypt_device *zdev = ap_dev->private;
+
+	zcrypt_device_unregister(zdev);
+}
+
+int __init zcrypt_pcixcc_init(void)
+{
+	return ap_driver_register(&zcrypt_pcixcc_driver, THIS_MODULE, "pcixcc");
+}
+
+void zcrypt_pcixcc_exit(void)
+{
+	ap_driver_unregister(&zcrypt_pcixcc_driver);
+}
+
+#ifndef CONFIG_ZCRYPT_MONOLITHIC
+module_init(zcrypt_pcixcc_init);
+module_exit(zcrypt_pcixcc_exit);
+#endif
diff --git a/drivers/s390/crypto/zcrypt_pcixcc.h b/drivers/s390/crypto/zcrypt_pcixcc.h
new file mode 100644
index 0000000..a78ff30
--- /dev/null
+++ b/drivers/s390/crypto/zcrypt_pcixcc.h
@@ -0,0 +1,79 @@
+/*
+ *  linux/drivers/s390/crypto/zcrypt_pcixcc.h
+ *
+ *  zcrypt 2.1.0
+ *
+ *  Copyright (C)  2001, 2006 IBM Corporation
+ *  Author(s): Robert Burroughs
+ *	       Eric Rossman (edrossma@us.ibm.com)
+ *
+ *  Hotplug & misc device support: Jochen Roehrig (roehrig@de.ibm.com)
+ *  Major cleanup & driver split: Martin Schwidefsky <schwidefsky@de.ibm.com>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2, or (at your option)
+ * any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
+ */
+
+#ifndef _ZCRYPT_PCIXCC_H_
+#define _ZCRYPT_PCIXCC_H_
+
+/**
+ * CPRBX
+ *	  Note that all shorts and ints are big-endian.
+ *	  All pointer fields are 16 bytes long, and mean nothing.
+ *
+ *	  A request CPRB is followed by a request_parameter_block.
+ *
+ *	  The request (or reply) parameter block is organized thus:
+ *	    function code
+ *	    VUD block
+ *	    key block
+ */
+struct CPRBX {
+	unsigned short cprb_len;	/* CPRB length	      220	 */
+	unsigned char  cprb_ver_id;	/* CPRB version id.   0x02	 */
+	unsigned char  pad_000[3];	/* Alignment pad bytes		 */
+	unsigned char  func_id[2];	/* function id	      0x5432	 */
+	unsigned char  cprb_flags[4];	/* Flags			 */
+	unsigned int   req_parml;	/* request parameter buffer len	 */
+	unsigned int   req_datal;	/* request data buffer		 */
+	unsigned int   rpl_msgbl;	/* reply  message block length	 */
+	unsigned int   rpld_parml;	/* replied parameter block len	 */
+	unsigned int   rpl_datal;	/* reply data block len		 */
+	unsigned int   rpld_datal;	/* replied data block len	 */
+	unsigned int   req_extbl;	/* request extension block len	 */
+	unsigned char  pad_001[4];	/* reserved			 */
+	unsigned int   rpld_extbl;	/* replied extension block len	 */
+	unsigned char  req_parmb[16];	/* request parm block 'address'	 */
+	unsigned char  req_datab[16];	/* request data block 'address'	 */
+	unsigned char  rpl_parmb[16];	/* reply parm block 'address'	 */
+	unsigned char  rpl_datab[16];	/* reply data block 'address'	 */
+	unsigned char  req_extb[16];	/* request extension block 'addr'*/
+	unsigned char  rpl_extb[16];	/* reply extension block 'addres'*/
+	unsigned short ccp_rtcode;	/* server return code		 */
+	unsigned short ccp_rscode;	/* server reason code		 */
+	unsigned int   mac_data_len;	/* Mac Data Length		 */
+	unsigned char  logon_id[8];	/* Logon Identifier		 */
+	unsigned char  mac_value[8];	/* Mac Value			 */
+	unsigned char  mac_content_flgs;/* Mac content flag byte	 */
+	unsigned char  pad_002;		/* Alignment			 */
+	unsigned short domain;		/* Domain			 */
+	unsigned char  pad_003[12];	/* Domain masks			 */
+	unsigned char  pad_004[36];	/* reserved			 */
+} __attribute__((packed));
+
+int zcrypt_pcixcc_init(void);
+void zcrypt_pcixcc_exit(void);
+
+#endif /* _ZCRYPT_PCIXCC_H_ */
diff --git a/drivers/s390/s390mach.c b/drivers/s390/s390mach.c
index 5399c5d..a914129 100644
--- a/drivers/s390/s390mach.c
+++ b/drivers/s390/s390mach.c
@@ -19,9 +19,6 @@
 
 #include "s390mach.h"
 
-#define DBG printk
-// #define DBG(args,...) do {} while (0);
-
 static struct semaphore m_sem;
 
 extern int css_process_crw(int, int);
@@ -83,11 +80,11 @@ repeat:
 		ccode = stcrw(&crw[chain]);
 		if (ccode != 0)
 			break;
-		DBG(KERN_DEBUG "crw_info : CRW reports slct=%d, oflw=%d, "
-		    "chn=%d, rsc=%X, anc=%d, erc=%X, rsid=%X\n",
-		    crw[chain].slct, crw[chain].oflw, crw[chain].chn,
-		    crw[chain].rsc, crw[chain].anc, crw[chain].erc,
-		    crw[chain].rsid);
+		printk(KERN_DEBUG "crw_info : CRW reports slct=%d, oflw=%d, "
+		       "chn=%d, rsc=%X, anc=%d, erc=%X, rsid=%X\n",
+		       crw[chain].slct, crw[chain].oflw, crw[chain].chn,
+		       crw[chain].rsc, crw[chain].anc, crw[chain].erc,
+		       crw[chain].rsid);
 		/* Check for overflows. */
 		if (crw[chain].oflw) {
 			pr_debug("%s: crw overflow detected!\n", __FUNCTION__);
@@ -117,8 +114,8 @@ repeat:
 			 * reported to the common I/O layer.
 			 */
 			if (crw[chain].slct) {
-				DBG(KERN_INFO"solicited machine check for "
-				    "channel path %02X\n", crw[0].rsid);
+				pr_debug("solicited machine check for "
+					 "channel path %02X\n", crw[0].rsid);
 				break;
 			}
 			switch (crw[0].erc) {
diff --git a/drivers/s390/scsi/zfcp_aux.c b/drivers/s390/scsi/zfcp_aux.c
index adc9d8f..5d39b2d 100644
--- a/drivers/s390/scsi/zfcp_aux.c
+++ b/drivers/s390/scsi/zfcp_aux.c
@@ -189,6 +189,10 @@ struct zfcp_fsf_req *zfcp_reqlist_ismember(struct zfcp_adapter *adapter,
 	struct zfcp_fsf_req *request, *tmp;
 	unsigned int i;
 
+	/* 0 is reserved as an invalid req_id */
+	if (req_id == 0)
+		return NULL;
+
 	i = req_id % REQUEST_LIST_SIZE;
 
 	list_for_each_entry_safe(request, tmp, &adapter->req_list[i], list)
@@ -299,11 +303,45 @@ zfcp_init_device_configure(void)
 	return;
 }
 
+static int calc_alignment(int size)
+{
+	int align = 1;
+
+	if (!size)
+		return 0;
+
+	while ((size - align) > 0)
+		align <<= 1;
+
+	return align;
+}
+
 static int __init
 zfcp_module_init(void)
 {
+	int retval = -ENOMEM;
+	int size, align;
+
+	size = sizeof(struct zfcp_fsf_req_qtcb);
+	align = calc_alignment(size);
+	zfcp_data.fsf_req_qtcb_cache =
+		kmem_cache_create("zfcp_fsf", size, align, 0, NULL, NULL);
+	if (!zfcp_data.fsf_req_qtcb_cache)
+		goto out;
 
-	int retval = 0;
+	size = sizeof(struct fsf_status_read_buffer);
+	align = calc_alignment(size);
+	zfcp_data.sr_buffer_cache =
+		kmem_cache_create("zfcp_sr", size, align, 0, NULL, NULL);
+	if (!zfcp_data.sr_buffer_cache)
+		goto out_sr_cache;
+
+	size = sizeof(struct zfcp_gid_pn_data);
+	align = calc_alignment(size);
+	zfcp_data.gid_pn_cache =
+		kmem_cache_create("zfcp_gid", size, align, 0, NULL, NULL);
+	if (!zfcp_data.gid_pn_cache)
+		goto out_gid_cache;
 
 	atomic_set(&zfcp_data.loglevel, loglevel);
 
@@ -313,15 +351,16 @@ zfcp_module_init(void)
 	/* initialize adapters to be removed list head */
 	INIT_LIST_HEAD(&zfcp_data.adapter_remove_lh);
 
-	zfcp_transport_template = fc_attach_transport(&zfcp_transport_functions);
-	if (!zfcp_transport_template)
-		return -ENODEV;
+	zfcp_data.scsi_transport_template =
+		fc_attach_transport(&zfcp_transport_functions);
+	if (!zfcp_data.scsi_transport_template)
+		goto out_transport;
 
 	retval = misc_register(&zfcp_cfdc_misc);
 	if (retval != 0) {
 		ZFCP_LOG_INFO("registration of misc device "
 			      "zfcp_cfdc failed\n");
-		goto out;
+		goto out_misc;
 	}
 
 	ZFCP_LOG_TRACE("major/minor for zfcp_cfdc: %d/%d\n",
@@ -333,9 +372,6 @@ zfcp_module_init(void)
 	/* initialise configuration rw lock */
 	rwlock_init(&zfcp_data.config_lock);
 
-	/* save address of data structure managing the driver module */
-	zfcp_data.scsi_host_template.module = THIS_MODULE;
-
 	/* setup dynamic I/O */
 	retval = zfcp_ccw_register();
 	if (retval) {
@@ -350,6 +386,14 @@ zfcp_module_init(void)
 
  out_ccw_register:
 	misc_deregister(&zfcp_cfdc_misc);
+ out_misc:
+	fc_release_transport(zfcp_data.scsi_transport_template);
+ out_transport:
+	kmem_cache_destroy(zfcp_data.gid_pn_cache);
+ out_gid_cache:
+	kmem_cache_destroy(zfcp_data.sr_buffer_cache);
+ out_sr_cache:
+	kmem_cache_destroy(zfcp_data.fsf_req_qtcb_cache);
  out:
 	return retval;
 }
@@ -935,20 +979,20 @@ static int
 zfcp_allocate_low_mem_buffers(struct zfcp_adapter *adapter)
 {
 	adapter->pool.fsf_req_erp =
-		mempool_create_kmalloc_pool(ZFCP_POOL_FSF_REQ_ERP_NR,
-				sizeof(struct zfcp_fsf_req_pool_element));
+		mempool_create_slab_pool(ZFCP_POOL_FSF_REQ_ERP_NR,
+					 zfcp_data.fsf_req_qtcb_cache);
 	if (!adapter->pool.fsf_req_erp)
 		return -ENOMEM;
 
 	adapter->pool.fsf_req_scsi =
-		mempool_create_kmalloc_pool(ZFCP_POOL_FSF_REQ_SCSI_NR,
-				sizeof(struct zfcp_fsf_req_pool_element));
+		mempool_create_slab_pool(ZFCP_POOL_FSF_REQ_SCSI_NR,
+					 zfcp_data.fsf_req_qtcb_cache);
 	if (!adapter->pool.fsf_req_scsi)
 		return -ENOMEM;
 
 	adapter->pool.fsf_req_abort =
-		mempool_create_kmalloc_pool(ZFCP_POOL_FSF_REQ_ABORT_NR,
-				sizeof(struct zfcp_fsf_req_pool_element));
+		mempool_create_slab_pool(ZFCP_POOL_FSF_REQ_ABORT_NR,
+					 zfcp_data.fsf_req_qtcb_cache);
 	if (!adapter->pool.fsf_req_abort)
 		return -ENOMEM;
 
@@ -959,14 +1003,14 @@ zfcp_allocate_low_mem_buffers(struct zfcp_adapter *adapter)
 		return -ENOMEM;
 
 	adapter->pool.data_status_read =
-		mempool_create_kmalloc_pool(ZFCP_POOL_STATUS_READ_NR,
-					sizeof(struct fsf_status_read_buffer));
+		mempool_create_slab_pool(ZFCP_POOL_STATUS_READ_NR,
+					 zfcp_data.sr_buffer_cache);
 	if (!adapter->pool.data_status_read)
 		return -ENOMEM;
 
 	adapter->pool.data_gid_pn =
-		mempool_create_kmalloc_pool(ZFCP_POOL_DATA_GID_PN_NR,
-					    sizeof(struct zfcp_gid_pn_data));
+		mempool_create_slab_pool(ZFCP_POOL_DATA_GID_PN_NR,
+					 zfcp_data.gid_pn_cache);
 	if (!adapter->pool.data_gid_pn)
 		return -ENOMEM;
 
@@ -1091,9 +1135,6 @@ zfcp_adapter_enqueue(struct ccw_device *ccw_device)
 	/* initialize lock of associated request queue */
 	rwlock_init(&adapter->request_queue.queue_lock);
 
-	/* intitialise SCSI ER timer */
-	init_timer(&adapter->scsi_er_timer);
-
 	/* mark adapter unusable as long as sysfs registration is not complete */
 	atomic_set_mask(ZFCP_STATUS_COMMON_REMOVE, &adapter->status);
 
@@ -1609,7 +1650,6 @@ zfcp_ns_gid_pn_request(struct zfcp_erp_action *erp_action)
 	gid_pn->ct.handler = zfcp_ns_gid_pn_handler;
 	gid_pn->ct.handler_data = (unsigned long) gid_pn;
         gid_pn->ct.timeout = ZFCP_NS_GID_PN_TIMEOUT;
-        gid_pn->ct.timer = &erp_action->timer;
 	gid_pn->port = erp_action->port;
 
 	ret = zfcp_fsf_send_ct(&gid_pn->ct, adapter->pool.fsf_req_erp,
diff --git a/drivers/s390/scsi/zfcp_ccw.c b/drivers/s390/scsi/zfcp_ccw.c
index fdabade..81680ef 100644
--- a/drivers/s390/scsi/zfcp_ccw.c
+++ b/drivers/s390/scsi/zfcp_ccw.c
@@ -275,19 +275,6 @@ zfcp_ccw_register(void)
 }
 
 /**
- * zfcp_ccw_unregister - ccw unregister function
- *
- * Unregisters the driver from common i/o layer. Function will be called at
- * module unload/system shutdown.
- */
-void __exit
-zfcp_ccw_unregister(void)
-{
-	zfcp_sysfs_driver_remove_files(&zfcp_ccw_driver.driver);
-	ccw_driver_unregister(&zfcp_ccw_driver);
-}
-
-/**
  * zfcp_ccw_shutdown - gets called on reboot/shutdown
  *
  * Makes sure that QDIO queues are down when the system gets stopped.
diff --git a/drivers/s390/scsi/zfcp_dbf.c b/drivers/s390/scsi/zfcp_dbf.c
index c033145..0aa3b1a 100644
--- a/drivers/s390/scsi/zfcp_dbf.c
+++ b/drivers/s390/scsi/zfcp_dbf.c
@@ -707,7 +707,7 @@ _zfcp_scsi_dbf_event_common(const char *tag, const char *tag2, int level,
 			    struct zfcp_adapter *adapter,
 			    struct scsi_cmnd *scsi_cmnd,
 			    struct zfcp_fsf_req *fsf_req,
-			    struct zfcp_fsf_req *old_fsf_req)
+			    unsigned long old_req_id)
 {
 	struct zfcp_scsi_dbf_record *rec = &adapter->scsi_dbf_buf;
 	struct zfcp_dbf_dump *dump = (struct zfcp_dbf_dump *)rec;
@@ -768,8 +768,7 @@ _zfcp_scsi_dbf_event_common(const char *tag, const char *tag2, int level,
 				rec->fsf_seqno = fsf_req->seq_no;
 				rec->fsf_issued = fsf_req->issued;
 			}
-			rec->type.old_fsf_reqid =
-				    (unsigned long) old_fsf_req;
+			rec->type.old_fsf_reqid = old_req_id;
 		} else {
 			strncpy(dump->tag, "dump", ZFCP_DBF_TAG_SIZE);
 			dump->total_size = buflen;
@@ -794,17 +793,17 @@ zfcp_scsi_dbf_event_result(const char *tag, int level,
 			   struct zfcp_fsf_req *fsf_req)
 {
 	_zfcp_scsi_dbf_event_common("rslt", tag, level,
-			adapter, scsi_cmnd, fsf_req, NULL);
+			adapter, scsi_cmnd, fsf_req, 0);
 }
 
 inline void
 zfcp_scsi_dbf_event_abort(const char *tag, struct zfcp_adapter *adapter,
 			  struct scsi_cmnd *scsi_cmnd,
 			  struct zfcp_fsf_req *new_fsf_req,
-			  struct zfcp_fsf_req *old_fsf_req)
+			  unsigned long old_req_id)
 {
 	_zfcp_scsi_dbf_event_common("abrt", tag, 1,
-			adapter, scsi_cmnd, new_fsf_req, old_fsf_req);
+			adapter, scsi_cmnd, new_fsf_req, old_req_id);
 }
 
 inline void
@@ -814,7 +813,7 @@ zfcp_scsi_dbf_event_devreset(const char *tag, u8 flag, struct zfcp_unit *unit,
 	struct zfcp_adapter *adapter = unit->port->adapter;
 
 	_zfcp_scsi_dbf_event_common(flag == FCP_TARGET_RESET ? "trst" : "lrst",
-			tag, 1, adapter, scsi_cmnd, NULL, NULL);
+			tag, 1, adapter, scsi_cmnd, NULL, 0);
 }
 
 static int
diff --git a/drivers/s390/scsi/zfcp_def.h b/drivers/s390/scsi/zfcp_def.h
index 94d1b74d..8f88269 100644
--- a/drivers/s390/scsi/zfcp_def.h
+++ b/drivers/s390/scsi/zfcp_def.h
@@ -19,7 +19,6 @@
  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 
  */ 
 
-
 #ifndef ZFCP_DEF_H
 #define ZFCP_DEF_H
 
@@ -32,6 +31,10 @@
 #include <linux/blkdev.h>
 #include <linux/delay.h>
 #include <linux/timer.h>
+#include <linux/slab.h>
+#include <linux/mempool.h>
+#include <linux/syscalls.h>
+#include <linux/ioctl.h>
 #include <scsi/scsi.h>
 #include <scsi/scsi_tcq.h>
 #include <scsi/scsi_cmnd.h>
@@ -39,14 +42,11 @@
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_transport.h>
 #include <scsi/scsi_transport_fc.h>
-#include "zfcp_fsf.h"
 #include <asm/ccwdev.h>
 #include <asm/qdio.h>
 #include <asm/debug.h>
 #include <asm/ebcdic.h>
-#include <linux/mempool.h>
-#include <linux/syscalls.h>
-#include <linux/ioctl.h>
+#include "zfcp_fsf.h"
 
 
 /********************* GENERAL DEFINES *********************************/
@@ -137,7 +137,7 @@ zfcp_address_to_sg(void *address, struct scatterlist *list)
 #define ZFCP_EXCHANGE_CONFIG_DATA_RETRIES	7
 
 /* timeout value for "default timer" for fsf requests */
-#define ZFCP_FSF_REQUEST_TIMEOUT (60*HZ);
+#define ZFCP_FSF_REQUEST_TIMEOUT (60*HZ)
 
 /*************** FIBRE CHANNEL PROTOCOL SPECIFIC DEFINES ********************/
 
@@ -543,7 +543,7 @@ do { \
 } while (0)
 	
 #if ZFCP_LOG_LEVEL_LIMIT < ZFCP_LOG_LEVEL_NORMAL
-# define ZFCP_LOG_NORMAL(fmt, args...)
+# define ZFCP_LOG_NORMAL(fmt, args...)	do { } while (0)
 #else
 # define ZFCP_LOG_NORMAL(fmt, args...) \
 do { \
@@ -553,7 +553,7 @@ do { \
 #endif
 
 #if ZFCP_LOG_LEVEL_LIMIT < ZFCP_LOG_LEVEL_INFO
-# define ZFCP_LOG_INFO(fmt, args...)
+# define ZFCP_LOG_INFO(fmt, args...)	do { } while (0)
 #else
 # define ZFCP_LOG_INFO(fmt, args...) \
 do { \
@@ -563,14 +563,14 @@ do { \
 #endif
 
 #if ZFCP_LOG_LEVEL_LIMIT < ZFCP_LOG_LEVEL_DEBUG
-# define ZFCP_LOG_DEBUG(fmt, args...)
+# define ZFCP_LOG_DEBUG(fmt, args...)	do { } while (0)
 #else
 # define ZFCP_LOG_DEBUG(fmt, args...) \
 	ZFCP_LOG(ZFCP_LOG_LEVEL_DEBUG, fmt , ##args)
 #endif
 
 #if ZFCP_LOG_LEVEL_LIMIT < ZFCP_LOG_LEVEL_TRACE
-# define ZFCP_LOG_TRACE(fmt, args...)
+# define ZFCP_LOG_TRACE(fmt, args...)	do { } while (0)
 #else
 # define ZFCP_LOG_TRACE(fmt, args...) \
 	ZFCP_LOG(ZFCP_LOG_LEVEL_TRACE, fmt , ##args)
@@ -779,7 +779,6 @@ typedef void (*zfcp_send_ct_handler_t)(unsigned long);
  * @handler_data: data passed to handler function
  * @pool: pointer to memory pool for ct request structure
  * @timeout: FSF timeout for this request
- * @timer: timer (e.g. for request initiated by erp)
  * @completion: completion for synchronization purposes
  * @status: used to pass error status to calling function
  */
@@ -793,7 +792,6 @@ struct zfcp_send_ct {
 	unsigned long handler_data;
 	mempool_t *pool;
 	int timeout;
-	struct timer_list *timer;
 	struct completion *completion;
 	int status;
 };
@@ -821,7 +819,6 @@ typedef void (*zfcp_send_els_handler_t)(unsigned long);
  * @resp_count: number of elements in response scatter-gather list
  * @handler: handler function (called for response to the request)
  * @handler_data: data passed to handler function
- * @timer: timer (e.g. for request initiated by erp)
  * @completion: completion for synchronization purposes
  * @ls_code: hex code of ELS command
  * @status: used to pass error status to calling function
@@ -836,7 +833,6 @@ struct zfcp_send_els {
 	unsigned int resp_count;
 	zfcp_send_els_handler_t handler;
 	unsigned long handler_data;
-	struct timer_list *timer;
 	struct completion *completion;
 	int ls_code;
 	int status;
@@ -886,7 +882,6 @@ struct zfcp_adapter {
 	struct list_head        port_remove_lh;    /* head of ports to be
 						      removed */
 	u32			ports;	           /* number of remote ports */
-	struct timer_list	scsi_er_timer;     /* SCSI err recovery watch */
 	atomic_t		reqs_active;	   /* # active FSF reqs */
 	unsigned long		req_no;		   /* unique FSF req number */
 	struct list_head	*req_list;	   /* list of pending reqs */
@@ -1003,6 +998,7 @@ struct zfcp_fsf_req {
 	struct fsf_qtcb	       *qtcb;	       /* address of associated QTCB */
 	u32		       seq_no;         /* Sequence number of request */
         unsigned long          data;           /* private data of request */ 
+	struct timer_list      timer;	       /* used for erp or scsi er */
 	struct zfcp_erp_action *erp_action;    /* used if this request is
 						  issued on behalf of erp */
 	mempool_t	       *pool;	       /* used if request was alloacted
@@ -1016,6 +1012,7 @@ typedef void zfcp_fsf_req_handler_t(struct zfcp_fsf_req*);
 /* driver data */
 struct zfcp_data {
 	struct scsi_host_template scsi_host_template;
+	struct scsi_transport_template *scsi_transport_template;
         atomic_t                status;             /* Module status flags */
 	struct list_head	adapter_list_head;  /* head of adapter list */
 	struct list_head	adapter_remove_lh;  /* head of adapters to be
@@ -1031,6 +1028,9 @@ struct zfcp_data {
 	wwn_t                   init_wwpn;
 	fcp_lun_t               init_fcp_lun;
 	char 			*driver_version;
+	kmem_cache_t		*fsf_req_qtcb_cache;
+	kmem_cache_t		*sr_buffer_cache;
+	kmem_cache_t		*gid_pn_cache;
 };
 
 /**
@@ -1051,7 +1051,7 @@ struct zfcp_sg_list {
 #define ZFCP_POOL_DATA_GID_PN_NR	1
 
 /* struct used by memory pools for fsf_requests */
-struct zfcp_fsf_req_pool_element {
+struct zfcp_fsf_req_qtcb {
 	struct zfcp_fsf_req fsf_req;
 	struct fsf_qtcb qtcb;
 };
diff --git a/drivers/s390/scsi/zfcp_erp.c b/drivers/s390/scsi/zfcp_erp.c
index 7f60b6f..862a411 100644
--- a/drivers/s390/scsi/zfcp_erp.c
+++ b/drivers/s390/scsi/zfcp_erp.c
@@ -64,8 +64,6 @@ static int zfcp_erp_strategy_check_action(struct zfcp_erp_action *, int);
 static int zfcp_erp_adapter_strategy(struct zfcp_erp_action *);
 static int zfcp_erp_adapter_strategy_generic(struct zfcp_erp_action *, int);
 static int zfcp_erp_adapter_strategy_close(struct zfcp_erp_action *);
-static void zfcp_erp_adapter_strategy_close_qdio(struct zfcp_erp_action *);
-static void zfcp_erp_adapter_strategy_close_fsf(struct zfcp_erp_action *);
 static int zfcp_erp_adapter_strategy_open(struct zfcp_erp_action *);
 static int zfcp_erp_adapter_strategy_open_qdio(struct zfcp_erp_action *);
 static int zfcp_erp_adapter_strategy_open_fsf(struct zfcp_erp_action *);
@@ -93,6 +91,7 @@ static int zfcp_erp_unit_strategy_clearstati(struct zfcp_unit *);
 static int zfcp_erp_unit_strategy_close(struct zfcp_erp_action *);
 static int zfcp_erp_unit_strategy_open(struct zfcp_erp_action *);
 
+static void zfcp_erp_action_dismiss_adapter(struct zfcp_adapter *);
 static void zfcp_erp_action_dismiss_port(struct zfcp_port *);
 static void zfcp_erp_action_dismiss_unit(struct zfcp_unit *);
 static void zfcp_erp_action_dismiss(struct zfcp_erp_action *);
@@ -111,64 +110,86 @@ static inline void zfcp_erp_action_to_ready(struct zfcp_erp_action *);
 static inline void zfcp_erp_action_to_running(struct zfcp_erp_action *);
 
 static void zfcp_erp_memwait_handler(unsigned long);
-static void zfcp_erp_timeout_handler(unsigned long);
-static inline void zfcp_erp_timeout_init(struct zfcp_erp_action *);
 
 /**
- * zfcp_fsf_request_timeout_handler - called if a request timed out
- * @data: pointer to adapter for handler function
- *
- * This function needs to be called if requests (ELS, Generic Service,
- * or SCSI commands) exceed a certain time limit. The assumption is
- * that after the time limit the adapter get stuck. So we trigger a reopen of
- * the adapter. This should not be used for error recovery, SCSI abort
- * commands and SCSI requests from SCSI mid-layer.
+ * zfcp_close_qdio - close qdio queues for an adapter
  */
-void
-zfcp_fsf_request_timeout_handler(unsigned long data)
+static void zfcp_close_qdio(struct zfcp_adapter *adapter)
 {
-	struct zfcp_adapter *adapter;
+	struct zfcp_qdio_queue *req_queue;
+	int first, count;
 
-	adapter = (struct zfcp_adapter *) data;
+	if (!atomic_test_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &adapter->status))
+		return;
 
-	zfcp_erp_adapter_reopen(adapter, 0);
+	/* clear QDIOUP flag, thus do_QDIO is not called during qdio_shutdown */
+	req_queue = &adapter->request_queue;
+	write_lock_irq(&req_queue->queue_lock);
+	atomic_clear_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &adapter->status);
+	write_unlock_irq(&req_queue->queue_lock);
+
+	debug_text_event(adapter->erp_dbf, 3, "qdio_down2a");
+	while (qdio_shutdown(adapter->ccw_device,
+			     QDIO_FLAG_CLEANUP_USING_CLEAR) == -EINPROGRESS)
+		msleep(1000);
+	debug_text_event(adapter->erp_dbf, 3, "qdio_down2b");
+
+	/* cleanup used outbound sbals */
+	count = atomic_read(&req_queue->free_count);
+	if (count < QDIO_MAX_BUFFERS_PER_Q) {
+		first = (req_queue->free_index+count) % QDIO_MAX_BUFFERS_PER_Q;
+		count = QDIO_MAX_BUFFERS_PER_Q - count;
+		zfcp_qdio_zero_sbals(req_queue->buffer, first, count);
+	}
+	req_queue->free_index = 0;
+	atomic_set(&req_queue->free_count, 0);
+	req_queue->distance_from_int = 0;
+	adapter->response_queue.free_index = 0;
+	atomic_set(&adapter->response_queue.free_count, 0);
 }
 
 /**
- * zfcp_fsf_scsi_er_timeout_handler - timeout handler for scsi eh tasks
+ * zfcp_close_fsf - stop FSF operations for an adapter
  *
- * This function needs to be called whenever a SCSI error recovery
- * action (abort/reset) does not return.  Re-opening the adapter means
- * that the abort/reset command can be returned by zfcp. It won't complete
- * via the adapter anymore (because qdio queues are closed). If ERP is
- * already running on this adapter it will be stopped.
+ * Dismiss and cleanup all pending fsf_reqs (this wakes up all initiators of
+ * requests waiting for completion; especially this returns SCSI commands
+ * with error state).
  */
-void zfcp_fsf_scsi_er_timeout_handler(unsigned long data)
+static void zfcp_close_fsf(struct zfcp_adapter *adapter)
 {
-	struct zfcp_adapter *adapter = (struct zfcp_adapter *) data;
-	unsigned long flags;
-
-	ZFCP_LOG_NORMAL("warning: SCSI error recovery timed out. "
-			"Restarting all operations on the adapter %s\n",
-			zfcp_get_busid_by_adapter(adapter));
-	debug_text_event(adapter->erp_dbf, 1, "eh_lmem_tout");
+	/* close queues to ensure that buffers are not accessed by adapter */
+	zfcp_close_qdio(adapter);
+	zfcp_fsf_req_dismiss_all(adapter);
+	/* reset FSF request sequence number */
+	adapter->fsf_req_seq_no = 0;
+	/* all ports and units are closed */
+	zfcp_erp_modify_adapter_status(adapter,
+				       ZFCP_STATUS_COMMON_OPEN, ZFCP_CLEAR);
+}
 
-	write_lock_irqsave(&adapter->erp_lock, flags);
-	if (atomic_test_mask(ZFCP_STATUS_ADAPTER_ERP_PENDING,
-			     &adapter->status)) {
-		zfcp_erp_modify_adapter_status(adapter,
-		       ZFCP_STATUS_COMMON_UNBLOCKED|ZFCP_STATUS_COMMON_OPEN,
-		       ZFCP_CLEAR);
-		zfcp_erp_action_dismiss_adapter(adapter);
-		write_unlock_irqrestore(&adapter->erp_lock, flags);
-		/* dismiss all pending requests including requests for ERP */
-		zfcp_fsf_req_dismiss_all(adapter);
-		adapter->fsf_req_seq_no = 0;
-	} else
-		write_unlock_irqrestore(&adapter->erp_lock, flags);
+/**
+ * zfcp_fsf_request_timeout_handler - called if a request timed out
+ * @data: pointer to adapter for handler function
+ *
+ * This function needs to be called if requests (ELS, Generic Service,
+ * or SCSI commands) exceed a certain time limit. The assumption is
+ * that after the time limit the adapter get stuck. So we trigger a reopen of
+ * the adapter.
+ */
+static void zfcp_fsf_request_timeout_handler(unsigned long data)
+{
+	struct zfcp_adapter *adapter = (struct zfcp_adapter *) data;
 	zfcp_erp_adapter_reopen(adapter, 0);
 }
 
+void zfcp_fsf_start_timer(struct zfcp_fsf_req *fsf_req, unsigned long timeout)
+{
+	fsf_req->timer.function = zfcp_fsf_request_timeout_handler;
+	fsf_req->timer.data = (unsigned long) fsf_req->adapter;
+	fsf_req->timer.expires = timeout;
+	add_timer(&fsf_req->timer);
+}
+
 /*
  * function:	
  *
@@ -282,7 +303,6 @@ zfcp_erp_adisc(struct zfcp_port *port)
 	struct zfcp_ls_adisc *adisc;
 	void *address = NULL;
 	int retval = 0;
-	struct timer_list *timer;
 
 	send_els = kzalloc(sizeof(struct zfcp_send_els), GFP_ATOMIC);
 	if (send_els == NULL)
@@ -329,22 +349,11 @@ zfcp_erp_adisc(struct zfcp_port *port)
 		      (wwn_t) adisc->wwnn, adisc->hard_nport_id,
 		      adisc->nport_id);
 
-	timer = kmalloc(sizeof(struct timer_list), GFP_ATOMIC);
-	if (!timer)
-		goto nomem;
-
-	init_timer(timer);
-	timer->function = zfcp_fsf_request_timeout_handler;
-	timer->data = (unsigned long) adapter;
-	timer->expires = ZFCP_FSF_REQUEST_TIMEOUT;
-	send_els->timer = timer;
-
 	retval = zfcp_fsf_send_els(send_els);
 	if (retval != 0) {
 		ZFCP_LOG_NORMAL("error: initiation of Send ELS failed for port "
 				"0x%08x on adapter %s\n", send_els->d_id,
 				zfcp_get_busid_by_adapter(adapter));
-		del_timer(send_els->timer);
 		goto freemem;
 	}
 
@@ -356,7 +365,6 @@ zfcp_erp_adisc(struct zfcp_port *port)
 	if (address != NULL)
 		__free_pages(send_els->req->page, 0);
 	if (send_els != NULL) {
-		kfree(send_els->timer);
 		kfree(send_els->req);
 		kfree(send_els->resp);
 		kfree(send_els);
@@ -382,9 +390,6 @@ zfcp_erp_adisc_handler(unsigned long data)
 	struct zfcp_ls_adisc_acc *adisc;
 
 	send_els = (struct zfcp_send_els *) data;
-
-	del_timer(send_els->timer);
-
 	adapter = send_els->adapter;
 	port = send_els->port;
 	d_id = send_els->d_id;
@@ -433,7 +438,6 @@ zfcp_erp_adisc_handler(unsigned long data)
  out:
 	zfcp_port_put(port);
 	__free_pages(send_els->req->page, 0);
-	kfree(send_els->timer);
 	kfree(send_els->req);
 	kfree(send_els->resp);
 	kfree(send_els);
@@ -909,8 +913,6 @@ static void zfcp_erp_async_handler_nolock(struct zfcp_erp_action *erp_action,
 		debug_text_event(adapter->erp_dbf, 2, "a_asyh_ex");
 		debug_event(adapter->erp_dbf, 2, &erp_action->action,
 			    sizeof (int));
-		if (!(set_mask & ZFCP_STATUS_ERP_TIMEDOUT))
-			del_timer(&erp_action->timer);
 		erp_action->status |= set_mask;
 		zfcp_erp_action_ready(erp_action);
 	} else {
@@ -957,8 +959,7 @@ zfcp_erp_memwait_handler(unsigned long data)
  *		action gets an appropriate flag and will be processed
  *		accordingly
  */
-static void
-zfcp_erp_timeout_handler(unsigned long data)
+void zfcp_erp_timeout_handler(unsigned long data)
 {
 	struct zfcp_erp_action *erp_action = (struct zfcp_erp_action *) data;
 	struct zfcp_adapter *adapter = erp_action->adapter;
@@ -1934,8 +1935,7 @@ zfcp_erp_adapter_strategy_generic(struct zfcp_erp_action *erp_action, int close)
 			  &erp_action->adapter->status);
 
  failed_openfcp:
-	zfcp_erp_adapter_strategy_close_qdio(erp_action);
-	zfcp_erp_adapter_strategy_close_fsf(erp_action);
+	zfcp_close_fsf(erp_action->adapter);
  failed_qdio:
  out:
 	return retval;
@@ -2040,59 +2040,6 @@ zfcp_erp_adapter_strategy_open_qdio(struct zfcp_erp_action *erp_action)
 	return retval;
 }
 
-/**
- * zfcp_erp_adapter_strategy_close_qdio - close qdio queues for an adapter
- */
-static void
-zfcp_erp_adapter_strategy_close_qdio(struct zfcp_erp_action *erp_action)
-{
-	int first_used;
-	int used_count;
-	struct zfcp_adapter *adapter = erp_action->adapter;
-
-	if (!atomic_test_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &adapter->status)) {
-		ZFCP_LOG_DEBUG("error: attempt to shut down inactive QDIO "
-			       "queues on adapter %s\n",
-			       zfcp_get_busid_by_adapter(adapter));
-		return;
-	}
-
-	/*
-	 * Get queue_lock and clear QDIOUP flag. Thus it's guaranteed that
-	 * do_QDIO won't be called while qdio_shutdown is in progress.
-	 */
-	write_lock_irq(&adapter->request_queue.queue_lock);
-	atomic_clear_mask(ZFCP_STATUS_ADAPTER_QDIOUP, &adapter->status);
-	write_unlock_irq(&adapter->request_queue.queue_lock);
-
-	debug_text_event(adapter->erp_dbf, 3, "qdio_down2a");
-	while (qdio_shutdown(adapter->ccw_device,
-			     QDIO_FLAG_CLEANUP_USING_CLEAR) == -EINPROGRESS)
-		msleep(1000);
-	debug_text_event(adapter->erp_dbf, 3, "qdio_down2b");
-
-	/*
-	 * First we had to stop QDIO operation.
-	 * Now it is safe to take the following actions.
-	 */
-
-	/* Cleanup only necessary when there are unacknowledged buffers */
-	if (atomic_read(&adapter->request_queue.free_count)
-	    < QDIO_MAX_BUFFERS_PER_Q) {
-		first_used = (adapter->request_queue.free_index +
-			      atomic_read(&adapter->request_queue.free_count))
-			% QDIO_MAX_BUFFERS_PER_Q;
-		used_count = QDIO_MAX_BUFFERS_PER_Q -
-			atomic_read(&adapter->request_queue.free_count);
-		zfcp_qdio_zero_sbals(adapter->request_queue.buffer,
-				     first_used, used_count);
-	}
-	adapter->response_queue.free_index = 0;
-	atomic_set(&adapter->response_queue.free_count, 0);
-	adapter->request_queue.free_index = 0;
-	atomic_set(&adapter->request_queue.free_count, 0);
-	adapter->request_queue.distance_from_int = 0;
-}
 
 static int
 zfcp_erp_adapter_strategy_open_fsf(struct zfcp_erp_action *erp_action)
@@ -2127,7 +2074,6 @@ zfcp_erp_adapter_strategy_open_fsf_xconfig(struct zfcp_erp_action *erp_action)
 		write_lock_irq(&adapter->erp_lock);
 		zfcp_erp_action_to_running(erp_action);
 		write_unlock_irq(&adapter->erp_lock);
-		zfcp_erp_timeout_init(erp_action);
 		if (zfcp_fsf_exchange_config_data(erp_action)) {
 			retval = ZFCP_ERP_FAILED;
 			debug_text_event(adapter->erp_dbf, 5, "a_fstx_xf");
@@ -2196,7 +2142,6 @@ zfcp_erp_adapter_strategy_open_fsf_xport(struct zfcp_erp_action *erp_action)
 	zfcp_erp_action_to_running(erp_action);
 	write_unlock_irq(&adapter->erp_lock);
 
-	zfcp_erp_timeout_init(erp_action);
 	ret = zfcp_fsf_exchange_port_data(erp_action, adapter, NULL);
 	if (ret == -EOPNOTSUPP) {
 		debug_text_event(adapter->erp_dbf, 3, "a_xport_notsupp");
@@ -2248,27 +2193,6 @@ zfcp_erp_adapter_strategy_open_fsf_statusread(struct zfcp_erp_action
 	return retval;
 }
 
-/**
- * zfcp_erp_adapter_strategy_close_fsf - stop FSF operations for an adapter
- */
-static void
-zfcp_erp_adapter_strategy_close_fsf(struct zfcp_erp_action *erp_action)
-{
-	struct zfcp_adapter *adapter = erp_action->adapter;
-
-	/*
-	 * wake waiting initiators of requests,
-	 * return SCSI commands (with error status),
-	 * clean up all requests (synchronously)
-	 */
-	zfcp_fsf_req_dismiss_all(adapter);
-	/* reset FSF request sequence number */
-	adapter->fsf_req_seq_no = 0;
-	/* all ports and units are closed */
-	zfcp_erp_modify_adapter_status(adapter,
-				       ZFCP_STATUS_COMMON_OPEN, ZFCP_CLEAR);
-}
-
 /*
  * function:	
  *
@@ -2605,7 +2529,6 @@ zfcp_erp_port_forced_strategy_close(struct zfcp_erp_action *erp_action)
 	struct zfcp_adapter *adapter = erp_action->adapter;
 	struct zfcp_port *port = erp_action->port;
 
-	zfcp_erp_timeout_init(erp_action);
 	retval = zfcp_fsf_close_physical_port(erp_action);
 	if (retval == -ENOMEM) {
 		debug_text_event(adapter->erp_dbf, 5, "o_pfstc_nomem");
@@ -2662,7 +2585,6 @@ zfcp_erp_port_strategy_close(struct zfcp_erp_action *erp_action)
 	struct zfcp_adapter *adapter = erp_action->adapter;
 	struct zfcp_port *port = erp_action->port;
 
-	zfcp_erp_timeout_init(erp_action);
 	retval = zfcp_fsf_close_port(erp_action);
 	if (retval == -ENOMEM) {
 		debug_text_event(adapter->erp_dbf, 5, "p_pstc_nomem");
@@ -2700,7 +2622,6 @@ zfcp_erp_port_strategy_open_port(struct zfcp_erp_action *erp_action)
 	struct zfcp_adapter *adapter = erp_action->adapter;
 	struct zfcp_port *port = erp_action->port;
 
-	zfcp_erp_timeout_init(erp_action);
 	retval = zfcp_fsf_open_port(erp_action);
 	if (retval == -ENOMEM) {
 		debug_text_event(adapter->erp_dbf, 5, "p_psto_nomem");
@@ -2738,7 +2659,6 @@ zfcp_erp_port_strategy_open_common_lookup(struct zfcp_erp_action *erp_action)
 	struct zfcp_adapter *adapter = erp_action->adapter;
 	struct zfcp_port *port = erp_action->port;
 
-	zfcp_erp_timeout_init(erp_action);
 	retval = zfcp_ns_gid_pn_request(erp_action);
 	if (retval == -ENOMEM) {
 		debug_text_event(adapter->erp_dbf, 5, "p_pstn_nomem");
@@ -2864,7 +2784,6 @@ zfcp_erp_unit_strategy_close(struct zfcp_erp_action *erp_action)
 	struct zfcp_adapter *adapter = erp_action->adapter;
 	struct zfcp_unit *unit = erp_action->unit;
 
-	zfcp_erp_timeout_init(erp_action);
 	retval = zfcp_fsf_close_unit(erp_action);
 	if (retval == -ENOMEM) {
 		debug_text_event(adapter->erp_dbf, 5, "u_ustc_nomem");
@@ -2905,7 +2824,6 @@ zfcp_erp_unit_strategy_open(struct zfcp_erp_action *erp_action)
 	struct zfcp_adapter *adapter = erp_action->adapter;
 	struct zfcp_unit *unit = erp_action->unit;
 
-	zfcp_erp_timeout_init(erp_action);
 	retval = zfcp_fsf_open_unit(erp_action);
 	if (retval == -ENOMEM) {
 		debug_text_event(adapter->erp_dbf, 5, "u_usto_nomem");
@@ -2930,14 +2848,13 @@ zfcp_erp_unit_strategy_open(struct zfcp_erp_action *erp_action)
 	return retval;
 }
 
-static inline void
-zfcp_erp_timeout_init(struct zfcp_erp_action *erp_action)
+void zfcp_erp_start_timer(struct zfcp_fsf_req *fsf_req)
 {
-	init_timer(&erp_action->timer);
-	erp_action->timer.function = zfcp_erp_timeout_handler;
-	erp_action->timer.data = (unsigned long) erp_action;
-	/* jiffies will be added in zfcp_fsf_req_send */
-	erp_action->timer.expires = ZFCP_ERP_FSFREQ_TIMEOUT;
+	BUG_ON(!fsf_req->erp_action);
+	fsf_req->timer.function = zfcp_erp_timeout_handler;
+	fsf_req->timer.data = (unsigned long) fsf_req->erp_action;
+	fsf_req->timer.expires = jiffies + ZFCP_ERP_FSFREQ_TIMEOUT;
+	add_timer(&fsf_req->timer);
 }
 
 /*
@@ -3241,7 +3158,7 @@ zfcp_erp_action_cleanup(int action, struct zfcp_adapter *adapter,
 }
 
 
-void zfcp_erp_action_dismiss_adapter(struct zfcp_adapter *adapter)
+static void zfcp_erp_action_dismiss_adapter(struct zfcp_adapter *adapter)
 {
 	struct zfcp_port *port;
 
diff --git a/drivers/s390/scsi/zfcp_ext.h b/drivers/s390/scsi/zfcp_ext.h
index 146d7a2..b8794d7 100644
--- a/drivers/s390/scsi/zfcp_ext.h
+++ b/drivers/s390/scsi/zfcp_ext.h
@@ -55,7 +55,6 @@ extern void   zfcp_unit_dequeue(struct zfcp_unit *);
 
 /******************************* S/390 IO ************************************/
 extern int  zfcp_ccw_register(void);
-extern void zfcp_ccw_unregister(void);
 
 extern void zfcp_qdio_zero_sbals(struct qdio_buffer **, int, int);
 extern int  zfcp_qdio_allocate(struct zfcp_adapter *);
@@ -88,8 +87,8 @@ extern int  zfcp_fsf_exchange_port_data(struct zfcp_erp_action *,
 					struct fsf_qtcb_bottom_port *);
 extern int  zfcp_fsf_control_file(struct zfcp_adapter *, struct zfcp_fsf_req **,
 				  u32, u32, struct zfcp_sg_list *);
-extern void zfcp_fsf_request_timeout_handler(unsigned long);
-extern void zfcp_fsf_scsi_er_timeout_handler(unsigned long);
+extern void zfcp_fsf_start_timer(struct zfcp_fsf_req *, unsigned long);
+extern void zfcp_erp_start_timer(struct zfcp_fsf_req *);
 extern int  zfcp_fsf_req_dismiss_all(struct zfcp_adapter *);
 extern int  zfcp_fsf_status_read(struct zfcp_adapter *, int);
 extern int zfcp_fsf_req_create(struct zfcp_adapter *, u32, int, mempool_t *,
@@ -99,8 +98,7 @@ extern int zfcp_fsf_send_ct(struct zfcp_send_ct *, mempool_t *,
 extern int zfcp_fsf_send_els(struct zfcp_send_els *);
 extern int  zfcp_fsf_send_fcp_command_task(struct zfcp_adapter *,
 					   struct zfcp_unit *,
-					   struct scsi_cmnd *,
-					   struct timer_list*, int);
+					   struct scsi_cmnd *, int, int);
 extern int  zfcp_fsf_req_complete(struct zfcp_fsf_req *);
 extern void zfcp_fsf_incoming_els(struct zfcp_fsf_req *);
 extern void zfcp_fsf_req_free(struct zfcp_fsf_req *);
@@ -124,14 +122,11 @@ extern char *zfcp_get_fcp_rsp_info_ptr(struct fcp_rsp_iu *);
 extern void set_host_byte(u32 *, char);
 extern void set_driver_byte(u32 *, char);
 extern char *zfcp_get_fcp_sns_info_ptr(struct fcp_rsp_iu *);
-extern void zfcp_fsf_start_scsi_er_timer(struct zfcp_adapter *);
 extern fcp_dl_t zfcp_get_fcp_dl(struct fcp_cmnd_iu *);
 
 extern int zfcp_scsi_command_async(struct zfcp_adapter *,struct zfcp_unit *,
-				   struct scsi_cmnd *, struct timer_list *);
-extern int zfcp_scsi_command_sync(struct zfcp_unit *, struct scsi_cmnd *,
-				  struct timer_list *);
-extern struct scsi_transport_template *zfcp_transport_template;
+				   struct scsi_cmnd *, int);
+extern int zfcp_scsi_command_sync(struct zfcp_unit *, struct scsi_cmnd *, int);
 extern struct fc_function_template zfcp_transport_functions;
 
 /******************************** ERP ****************************************/
@@ -139,7 +134,6 @@ extern void zfcp_erp_modify_adapter_status(struct zfcp_adapter *, u32, int);
 extern int  zfcp_erp_adapter_reopen(struct zfcp_adapter *, int);
 extern int  zfcp_erp_adapter_shutdown(struct zfcp_adapter *, int);
 extern void zfcp_erp_adapter_failed(struct zfcp_adapter *);
-extern void zfcp_erp_action_dismiss_adapter(struct zfcp_adapter *);
 
 extern void zfcp_erp_modify_port_status(struct zfcp_port *, u32, int);
 extern int  zfcp_erp_port_reopen(struct zfcp_port *, int);
@@ -187,7 +181,7 @@ extern void zfcp_scsi_dbf_event_result(const char *, int, struct zfcp_adapter *,
 				       struct zfcp_fsf_req *);
 extern void zfcp_scsi_dbf_event_abort(const char *, struct zfcp_adapter *,
 				      struct scsi_cmnd *, struct zfcp_fsf_req *,
-				      struct zfcp_fsf_req *);
+				      unsigned long);
 extern void zfcp_scsi_dbf_event_devreset(const char *, u8, struct zfcp_unit *,
 					 struct scsi_cmnd *);
 extern void zfcp_reqlist_add(struct zfcp_adapter *, struct zfcp_fsf_req *);
diff --git a/drivers/s390/scsi/zfcp_fsf.c b/drivers/s390/scsi/zfcp_fsf.c
index ff2eacf..277826c 100644
--- a/drivers/s390/scsi/zfcp_fsf.c
+++ b/drivers/s390/scsi/zfcp_fsf.c
@@ -42,7 +42,7 @@ static inline int zfcp_fsf_req_sbal_check(
 static inline int zfcp_use_one_sbal(
 	struct scatterlist *, int, struct scatterlist *, int);
 static struct zfcp_fsf_req *zfcp_fsf_req_alloc(mempool_t *, int);
-static int zfcp_fsf_req_send(struct zfcp_fsf_req *, struct timer_list *);
+static int zfcp_fsf_req_send(struct zfcp_fsf_req *);
 static int zfcp_fsf_protstatus_eval(struct zfcp_fsf_req *);
 static int zfcp_fsf_fsfstatus_eval(struct zfcp_fsf_req *);
 static int zfcp_fsf_fsfstatus_qual_eval(struct zfcp_fsf_req *);
@@ -100,14 +100,19 @@ zfcp_fsf_req_alloc(mempool_t *pool, int req_flags)
 	if (req_flags & ZFCP_REQ_NO_QTCB)
 		size = sizeof(struct zfcp_fsf_req);
 	else
-		size = sizeof(struct zfcp_fsf_req_pool_element);
+		size = sizeof(struct zfcp_fsf_req_qtcb);
 
-	if (likely(pool != NULL))
+	if (likely(pool))
 		ptr = mempool_alloc(pool, GFP_ATOMIC);
-	else
-		ptr = kmalloc(size, GFP_ATOMIC);
+	else {
+		if (req_flags & ZFCP_REQ_NO_QTCB)
+			ptr = kmalloc(size, GFP_ATOMIC);
+		else
+			ptr = kmem_cache_alloc(zfcp_data.fsf_req_qtcb_cache,
+					       SLAB_ATOMIC);
+	}
 
-	if (unlikely(NULL == ptr))
+	if (unlikely(!ptr))
 		goto out;
 
 	memset(ptr, 0, size);
@@ -115,9 +120,8 @@ zfcp_fsf_req_alloc(mempool_t *pool, int req_flags)
 	if (req_flags & ZFCP_REQ_NO_QTCB) {
 		fsf_req = (struct zfcp_fsf_req *) ptr;
 	} else {
-		fsf_req = &((struct zfcp_fsf_req_pool_element *) ptr)->fsf_req;
-		fsf_req->qtcb =
-			&((struct zfcp_fsf_req_pool_element *) ptr)->qtcb;
+		fsf_req = &((struct zfcp_fsf_req_qtcb *) ptr)->fsf_req;
+		fsf_req->qtcb =	&((struct zfcp_fsf_req_qtcb *) ptr)->qtcb;
 	}
 
 	fsf_req->pool = pool;
@@ -139,10 +143,17 @@ zfcp_fsf_req_alloc(mempool_t *pool, int req_flags)
 void
 zfcp_fsf_req_free(struct zfcp_fsf_req *fsf_req)
 {
-	if (likely(fsf_req->pool != NULL))
+	if (likely(fsf_req->pool)) {
 		mempool_free(fsf_req, fsf_req->pool);
-	else
-		kfree(fsf_req);
+		return;
+	}
+
+	if (fsf_req->qtcb) {
+		kmem_cache_free(zfcp_data.fsf_req_qtcb_cache, fsf_req);
+		return;
+	}
+
+	kfree(fsf_req);
 }
 
 /**
@@ -214,8 +225,10 @@ zfcp_fsf_req_complete(struct zfcp_fsf_req *fsf_req)
 		 */
 		zfcp_fsf_status_read_handler(fsf_req);
 		goto out;
-	} else
+	} else {
+		del_timer(&fsf_req->timer);
 		zfcp_fsf_protstatus_eval(fsf_req);
+	}
 
 	/*
 	 * fsf_req may be deleted due to waking up functions, so 
@@ -774,8 +787,7 @@ zfcp_fsf_status_read(struct zfcp_adapter *adapter, int req_flags)
 	sbale->addr = (void *) status_buffer;
 	sbale->length = sizeof(struct fsf_status_read_buffer);
 
-	/* start QDIO request for this FSF request */
-	retval = zfcp_fsf_req_send(fsf_req, NULL);
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (retval) {
 		ZFCP_LOG_DEBUG("error: Could not set-up unsolicited status "
 			       "environment.\n");
@@ -1101,8 +1113,8 @@ zfcp_fsf_abort_fcp_command(unsigned long old_req_id,
 			   struct zfcp_unit *unit, int req_flags)
 {
 	volatile struct qdio_buffer_element *sbale;
-	unsigned long lock_flags;
 	struct zfcp_fsf_req *fsf_req = NULL;
+	unsigned long lock_flags;
 	int retval = 0;
 
 	/* setup new FSF request */
@@ -1132,12 +1144,9 @@ zfcp_fsf_abort_fcp_command(unsigned long old_req_id,
 	/* set handle of request which should be aborted */
 	fsf_req->qtcb->bottom.support.req_handle = (u64) old_req_id;
 
-	/* start QDIO request for this FSF request */
-
-	zfcp_fsf_start_scsi_er_timer(adapter);
-	retval = zfcp_fsf_req_send(fsf_req, NULL);
+	zfcp_fsf_start_timer(fsf_req, ZFCP_SCSI_ER_TIMEOUT);
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (retval) {
-		del_timer(&adapter->scsi_er_timer);
 		ZFCP_LOG_INFO("error: Failed to send abort command request "
 			      "on adapter %s, port 0x%016Lx, unit 0x%016Lx\n",
 			      zfcp_get_busid_by_adapter(adapter),
@@ -1173,8 +1182,6 @@ zfcp_fsf_abort_fcp_command_handler(struct zfcp_fsf_req *new_fsf_req)
 	unsigned char status_qual =
 	    new_fsf_req->qtcb->header.fsf_status_qual.word[0];
 
-	del_timer(&new_fsf_req->adapter->scsi_er_timer);
-
 	if (new_fsf_req->status & ZFCP_STATUS_FSFREQ_ERROR) {
 		/* do not set ZFCP_STATUS_FSFREQ_ABORTSUCCEEDED */
 		goto skip_fsfstatus;
@@ -1380,11 +1387,6 @@ zfcp_fsf_send_ct(struct zfcp_send_ct *ct, mempool_t *pool,
 		goto failed_req;
 	}
 
-        if (erp_action != NULL) {
-                erp_action->fsf_req = fsf_req;
-                fsf_req->erp_action = erp_action;
-        }
-
 	sbale = zfcp_qdio_sbale_req(fsf_req, fsf_req->sbal_curr, 0);
         if (zfcp_use_one_sbal(ct->req, ct->req_count,
                               ct->resp, ct->resp_count)){
@@ -1451,8 +1453,14 @@ zfcp_fsf_send_ct(struct zfcp_send_ct *ct, mempool_t *pool,
 
 	zfcp_san_dbf_event_ct_request(fsf_req);
 
-	/* start QDIO request for this FSF request */
-	ret = zfcp_fsf_req_send(fsf_req, ct->timer);
+	if (erp_action) {
+		erp_action->fsf_req = fsf_req;
+		fsf_req->erp_action = erp_action;
+		zfcp_erp_start_timer(fsf_req);
+	} else
+		zfcp_fsf_start_timer(fsf_req, ZFCP_FSF_REQUEST_TIMEOUT);
+
+	ret = zfcp_fsf_req_send(fsf_req);
 	if (ret) {
 		ZFCP_LOG_DEBUG("error: initiation of CT request failed "
 			       "(adapter %s, port 0x%016Lx)\n",
@@ -1749,8 +1757,8 @@ zfcp_fsf_send_els(struct zfcp_send_els *els)
 
 	zfcp_san_dbf_event_els_request(fsf_req);
 
-	/* start QDIO request for this FSF request */
-	ret = zfcp_fsf_req_send(fsf_req, els->timer);
+	zfcp_fsf_start_timer(fsf_req, ZFCP_FSF_REQUEST_TIMEOUT);
+	ret = zfcp_fsf_req_send(fsf_req);
 	if (ret) {
 		ZFCP_LOG_DEBUG("error: initiation of ELS request failed "
 			       "(adapter %s, port d_id: 0x%08x)\n",
@@ -1947,6 +1955,7 @@ int
 zfcp_fsf_exchange_config_data(struct zfcp_erp_action *erp_action)
 {
 	volatile struct qdio_buffer_element *sbale;
+	struct zfcp_fsf_req *fsf_req;
 	unsigned long lock_flags;
 	int retval = 0;
 
@@ -1955,7 +1964,7 @@ zfcp_fsf_exchange_config_data(struct zfcp_erp_action *erp_action)
 				     FSF_QTCB_EXCHANGE_CONFIG_DATA,
 				     ZFCP_REQ_AUTO_CLEANUP,
 				     erp_action->adapter->pool.fsf_req_erp,
-				     &lock_flags, &(erp_action->fsf_req));
+				     &lock_flags, &fsf_req);
 	if (retval < 0) {
 		ZFCP_LOG_INFO("error: Could not create exchange configuration "
 			      "data request for adapter %s.\n",
@@ -1963,26 +1972,26 @@ zfcp_fsf_exchange_config_data(struct zfcp_erp_action *erp_action)
 		goto out;
 	}
 
-	sbale = zfcp_qdio_sbale_req(erp_action->fsf_req,
-                                    erp_action->fsf_req->sbal_curr, 0);
+	sbale = zfcp_qdio_sbale_req(fsf_req, fsf_req->sbal_curr, 0);
         sbale[0].flags |= SBAL_FLAGS0_TYPE_READ;
         sbale[1].flags |= SBAL_FLAGS_LAST_ENTRY;
 
-	erp_action->fsf_req->erp_action = erp_action;
-	erp_action->fsf_req->qtcb->bottom.config.feature_selection =
+	fsf_req->qtcb->bottom.config.feature_selection =
 			FSF_FEATURE_CFDC |
 			FSF_FEATURE_LUN_SHARING |
 			FSF_FEATURE_NOTIFICATION_LOST |
 			FSF_FEATURE_UPDATE_ALERT;
+	fsf_req->erp_action = erp_action;
+	erp_action->fsf_req = fsf_req;
 
-	/* start QDIO request for this FSF request */
-	retval = zfcp_fsf_req_send(erp_action->fsf_req, &erp_action->timer);
+	zfcp_erp_start_timer(fsf_req);
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (retval) {
 		ZFCP_LOG_INFO
 		    ("error: Could not send exchange configuration data "
 		     "command on the adapter %s\n",
 		     zfcp_get_busid_by_adapter(erp_action->adapter));
-		zfcp_fsf_req_free(erp_action->fsf_req);
+		zfcp_fsf_req_free(fsf_req);
 		erp_action->fsf_req = NULL;
 		goto out;
 	}
@@ -2212,10 +2221,9 @@ zfcp_fsf_exchange_port_data(struct zfcp_erp_action *erp_action,
 			    struct fsf_qtcb_bottom_port *data)
 {
 	volatile struct qdio_buffer_element *sbale;
-	int retval = 0;
-	unsigned long lock_flags;
         struct zfcp_fsf_req *fsf_req;
-	struct timer_list *timer;
+	unsigned long lock_flags;
+	int retval = 0;
 
 	if (!(adapter->adapter_features & FSF_FEATURE_HBAAPI_MANAGEMENT)) {
 		ZFCP_LOG_INFO("error: exchange port data "
@@ -2248,22 +2256,11 @@ zfcp_fsf_exchange_port_data(struct zfcp_erp_action *erp_action,
 	if (erp_action) {
 		erp_action->fsf_req = fsf_req;
 		fsf_req->erp_action = erp_action;
-		timer = &erp_action->timer;
-	} else {
-		timer = kmalloc(sizeof(struct timer_list), GFP_ATOMIC);
-		if (!timer) {
-			write_unlock_irqrestore(&adapter->request_queue.queue_lock,
-						lock_flags);
-			zfcp_fsf_req_free(fsf_req);
-			return -ENOMEM;
-		}
-		init_timer(timer);
-		timer->function = zfcp_fsf_request_timeout_handler;
-		timer->data = (unsigned long) adapter;
-		timer->expires = ZFCP_FSF_REQUEST_TIMEOUT;
-	}
+		zfcp_erp_start_timer(fsf_req);
+	} else
+		zfcp_fsf_start_timer(fsf_req, ZFCP_FSF_REQUEST_TIMEOUT);
 
-	retval = zfcp_fsf_req_send(fsf_req, timer);
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (retval) {
 		ZFCP_LOG_INFO("error: Could not send an exchange port data "
                               "command on the adapter %s\n",
@@ -2271,8 +2268,6 @@ zfcp_fsf_exchange_port_data(struct zfcp_erp_action *erp_action,
 		zfcp_fsf_req_free(fsf_req);
 		if (erp_action)
 			erp_action->fsf_req = NULL;
-		else
-			kfree(timer);
 		write_unlock_irqrestore(&adapter->request_queue.queue_lock,
 					lock_flags);
 		return retval;
@@ -2283,9 +2278,7 @@ zfcp_fsf_exchange_port_data(struct zfcp_erp_action *erp_action,
 	if (!erp_action) {
 		wait_event(fsf_req->completion_wq,
 			   fsf_req->status & ZFCP_STATUS_FSFREQ_COMPLETED);
-		del_timer_sync(timer);
 		zfcp_fsf_req_free(fsf_req);
-		kfree(timer);
 	}
 	return retval;
 }
@@ -2367,6 +2360,7 @@ int
 zfcp_fsf_open_port(struct zfcp_erp_action *erp_action)
 {
 	volatile struct qdio_buffer_element *sbale;
+	struct zfcp_fsf_req *fsf_req;
 	unsigned long lock_flags;
 	int retval = 0;
 
@@ -2375,7 +2369,7 @@ zfcp_fsf_open_port(struct zfcp_erp_action *erp_action)
 				     FSF_QTCB_OPEN_PORT_WITH_DID,
 				     ZFCP_WAIT_FOR_SBAL | ZFCP_REQ_AUTO_CLEANUP,
 				     erp_action->adapter->pool.fsf_req_erp,
-				     &lock_flags, &(erp_action->fsf_req));
+				     &lock_flags, &fsf_req);
 	if (retval < 0) {
 		ZFCP_LOG_INFO("error: Could not create open port request "
 			      "for port 0x%016Lx on adapter %s.\n",
@@ -2384,24 +2378,24 @@ zfcp_fsf_open_port(struct zfcp_erp_action *erp_action)
 		goto out;
 	}
 
-	sbale = zfcp_qdio_sbale_req(erp_action->fsf_req,
-                                    erp_action->fsf_req->sbal_curr, 0);
+	sbale = zfcp_qdio_sbale_req(fsf_req, fsf_req->sbal_curr, 0);
         sbale[0].flags |= SBAL_FLAGS0_TYPE_READ;
         sbale[1].flags |= SBAL_FLAGS_LAST_ENTRY;
 
-	erp_action->fsf_req->qtcb->bottom.support.d_id = erp_action->port->d_id;
+	fsf_req->qtcb->bottom.support.d_id = erp_action->port->d_id;
 	atomic_set_mask(ZFCP_STATUS_COMMON_OPENING, &erp_action->port->status);
-	erp_action->fsf_req->data = (unsigned long) erp_action->port;
-	erp_action->fsf_req->erp_action = erp_action;
+	fsf_req->data = (unsigned long) erp_action->port;
+	fsf_req->erp_action = erp_action;
+	erp_action->fsf_req = fsf_req;
 
-	/* start QDIO request for this FSF request */
-	retval = zfcp_fsf_req_send(erp_action->fsf_req, &erp_action->timer);
+	zfcp_erp_start_timer(fsf_req);
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (retval) {
 		ZFCP_LOG_INFO("error: Could not send open port request for "
 			      "port 0x%016Lx on adapter %s.\n",
 			      erp_action->port->wwpn,
 			      zfcp_get_busid_by_adapter(erp_action->adapter));
-		zfcp_fsf_req_free(erp_action->fsf_req);
+		zfcp_fsf_req_free(fsf_req);
 		erp_action->fsf_req = NULL;
 		goto out;
 	}
@@ -2623,6 +2617,7 @@ int
 zfcp_fsf_close_port(struct zfcp_erp_action *erp_action)
 {
 	volatile struct qdio_buffer_element *sbale;
+	struct zfcp_fsf_req *fsf_req;
 	unsigned long lock_flags;
 	int retval = 0;
 
@@ -2631,7 +2626,7 @@ zfcp_fsf_close_port(struct zfcp_erp_action *erp_action)
 				     FSF_QTCB_CLOSE_PORT,
 				     ZFCP_WAIT_FOR_SBAL | ZFCP_REQ_AUTO_CLEANUP,
 				     erp_action->adapter->pool.fsf_req_erp,
-				     &lock_flags, &(erp_action->fsf_req));
+				     &lock_flags, &fsf_req);
 	if (retval < 0) {
 		ZFCP_LOG_INFO("error: Could not create a close port request "
 			      "for port 0x%016Lx on adapter %s.\n",
@@ -2640,25 +2635,25 @@ zfcp_fsf_close_port(struct zfcp_erp_action *erp_action)
 		goto out;
 	}
 
-	sbale = zfcp_qdio_sbale_req(erp_action->fsf_req,
-                                    erp_action->fsf_req->sbal_curr, 0);
+	sbale = zfcp_qdio_sbale_req(fsf_req, fsf_req->sbal_curr, 0);
         sbale[0].flags |= SBAL_FLAGS0_TYPE_READ;
         sbale[1].flags |= SBAL_FLAGS_LAST_ENTRY;
 
 	atomic_set_mask(ZFCP_STATUS_COMMON_CLOSING, &erp_action->port->status);
-	erp_action->fsf_req->data = (unsigned long) erp_action->port;
-	erp_action->fsf_req->erp_action = erp_action;
-	erp_action->fsf_req->qtcb->header.port_handle =
-	    erp_action->port->handle;
-
-	/* start QDIO request for this FSF request */
-	retval = zfcp_fsf_req_send(erp_action->fsf_req, &erp_action->timer);
+	fsf_req->data = (unsigned long) erp_action->port;
+	fsf_req->erp_action = erp_action;
+	fsf_req->qtcb->header.port_handle = erp_action->port->handle;
+	fsf_req->erp_action = erp_action;
+	erp_action->fsf_req = fsf_req;
+
+	zfcp_erp_start_timer(fsf_req);
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (retval) {
 		ZFCP_LOG_INFO("error: Could not send a close port request for "
 			      "port 0x%016Lx on adapter %s.\n",
 			      erp_action->port->wwpn,
 			      zfcp_get_busid_by_adapter(erp_action->adapter));
-		zfcp_fsf_req_free(erp_action->fsf_req);
+		zfcp_fsf_req_free(fsf_req);
 		erp_action->fsf_req = NULL;
 		goto out;
 	}
@@ -2755,16 +2750,17 @@ zfcp_fsf_close_port_handler(struct zfcp_fsf_req *fsf_req)
 int
 zfcp_fsf_close_physical_port(struct zfcp_erp_action *erp_action)
 {
-	int retval = 0;
-	unsigned long lock_flags;
 	volatile struct qdio_buffer_element *sbale;
+	struct zfcp_fsf_req *fsf_req;
+	unsigned long lock_flags;
+	int retval = 0;
 
 	/* setup new FSF request */
 	retval = zfcp_fsf_req_create(erp_action->adapter,
 				     FSF_QTCB_CLOSE_PHYSICAL_PORT,
 				     ZFCP_WAIT_FOR_SBAL | ZFCP_REQ_AUTO_CLEANUP,
 				     erp_action->adapter->pool.fsf_req_erp,
-				     &lock_flags, &erp_action->fsf_req);
+				     &lock_flags, &fsf_req);
 	if (retval < 0) {
 		ZFCP_LOG_INFO("error: Could not create close physical port "
 			      "request (adapter %s, port 0x%016Lx)\n",
@@ -2774,8 +2770,7 @@ zfcp_fsf_close_physical_port(struct zfcp_erp_action *erp_action)
 		goto out;
 	}
 
-	sbale = zfcp_qdio_sbale_req(erp_action->fsf_req,
-				    erp_action->fsf_req->sbal_curr, 0);
+	sbale = zfcp_qdio_sbale_req(fsf_req, fsf_req->sbal_curr, 0);
 	sbale[0].flags |= SBAL_FLAGS0_TYPE_READ;
 	sbale[1].flags |= SBAL_FLAGS_LAST_ENTRY;
 
@@ -2783,20 +2778,19 @@ zfcp_fsf_close_physical_port(struct zfcp_erp_action *erp_action)
 	atomic_set_mask(ZFCP_STATUS_PORT_PHYS_CLOSING,
 			&erp_action->port->status);
 	/* save a pointer to this port */
-	erp_action->fsf_req->data = (unsigned long) erp_action->port;
-	/* port to be closed */
-	erp_action->fsf_req->qtcb->header.port_handle =
-	    erp_action->port->handle;
-	erp_action->fsf_req->erp_action = erp_action;
-
-	/* start QDIO request for this FSF request */
-	retval = zfcp_fsf_req_send(erp_action->fsf_req, &erp_action->timer);
+	fsf_req->data = (unsigned long) erp_action->port;
+	fsf_req->qtcb->header.port_handle = erp_action->port->handle;
+	fsf_req->erp_action = erp_action;
+	erp_action->fsf_req = fsf_req;
+
+	zfcp_erp_start_timer(fsf_req);
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (retval) {
 		ZFCP_LOG_INFO("error: Could not send close physical port "
 			      "request (adapter %s, port 0x%016Lx)\n",
 			      zfcp_get_busid_by_adapter(erp_action->adapter),
 			      erp_action->port->wwpn);
-		zfcp_fsf_req_free(erp_action->fsf_req);
+		zfcp_fsf_req_free(fsf_req);
 		erp_action->fsf_req = NULL;
 		goto out;
 	}
@@ -2961,6 +2955,7 @@ int
 zfcp_fsf_open_unit(struct zfcp_erp_action *erp_action)
 {
 	volatile struct qdio_buffer_element *sbale;
+	struct zfcp_fsf_req *fsf_req;
 	unsigned long lock_flags;
 	int retval = 0;
 
@@ -2969,7 +2964,7 @@ zfcp_fsf_open_unit(struct zfcp_erp_action *erp_action)
 				     FSF_QTCB_OPEN_LUN,
 				     ZFCP_WAIT_FOR_SBAL | ZFCP_REQ_AUTO_CLEANUP,
 				     erp_action->adapter->pool.fsf_req_erp,
-				     &lock_flags, &(erp_action->fsf_req));
+				     &lock_flags, &fsf_req);
 	if (retval < 0) {
 		ZFCP_LOG_INFO("error: Could not create open unit request for "
 			      "unit 0x%016Lx on port 0x%016Lx on adapter %s.\n",
@@ -2979,24 +2974,22 @@ zfcp_fsf_open_unit(struct zfcp_erp_action *erp_action)
 		goto out;
 	}
 
-	sbale = zfcp_qdio_sbale_req(erp_action->fsf_req,
-                                    erp_action->fsf_req->sbal_curr, 0);
+	sbale = zfcp_qdio_sbale_req(fsf_req, fsf_req->sbal_curr, 0);
         sbale[0].flags |= SBAL_FLAGS0_TYPE_READ;
         sbale[1].flags |= SBAL_FLAGS_LAST_ENTRY;
 
-	erp_action->fsf_req->qtcb->header.port_handle =
-		erp_action->port->handle;
-	erp_action->fsf_req->qtcb->bottom.support.fcp_lun =
-		erp_action->unit->fcp_lun;
+	fsf_req->qtcb->header.port_handle = erp_action->port->handle;
+	fsf_req->qtcb->bottom.support.fcp_lun =	erp_action->unit->fcp_lun;
 	if (!(erp_action->adapter->connection_features & FSF_FEATURE_NPIV_MODE))
-		erp_action->fsf_req->qtcb->bottom.support.option =
+		fsf_req->qtcb->bottom.support.option =
 			FSF_OPEN_LUN_SUPPRESS_BOXING;
 	atomic_set_mask(ZFCP_STATUS_COMMON_OPENING, &erp_action->unit->status);
-	erp_action->fsf_req->data = (unsigned long) erp_action->unit;
-	erp_action->fsf_req->erp_action = erp_action;
+	fsf_req->data = (unsigned long) erp_action->unit;
+	fsf_req->erp_action = erp_action;
+	erp_action->fsf_req = fsf_req;
 
-	/* start QDIO request for this FSF request */
-	retval = zfcp_fsf_req_send(erp_action->fsf_req, &erp_action->timer);
+	zfcp_erp_start_timer(fsf_req);
+	retval = zfcp_fsf_req_send(erp_action->fsf_req);
 	if (retval) {
 		ZFCP_LOG_INFO("error: Could not send an open unit request "
 			      "on the adapter %s, port 0x%016Lx for "
@@ -3004,7 +2997,7 @@ zfcp_fsf_open_unit(struct zfcp_erp_action *erp_action)
 			      zfcp_get_busid_by_adapter(erp_action->adapter),
 			      erp_action->port->wwpn,
 			      erp_action->unit->fcp_lun);
-		zfcp_fsf_req_free(erp_action->fsf_req);
+		zfcp_fsf_req_free(fsf_req);
 		erp_action->fsf_req = NULL;
 		goto out;
 	}
@@ -3297,6 +3290,7 @@ int
 zfcp_fsf_close_unit(struct zfcp_erp_action *erp_action)
 {
 	volatile struct qdio_buffer_element *sbale;
+	struct zfcp_fsf_req *fsf_req;
 	unsigned long lock_flags;
 	int retval = 0;
 
@@ -3305,7 +3299,7 @@ zfcp_fsf_close_unit(struct zfcp_erp_action *erp_action)
 				     FSF_QTCB_CLOSE_LUN,
 				     ZFCP_WAIT_FOR_SBAL | ZFCP_REQ_AUTO_CLEANUP,
 				     erp_action->adapter->pool.fsf_req_erp,
-				     &lock_flags, &(erp_action->fsf_req));
+				     &lock_flags, &fsf_req);
 	if (retval < 0) {
 		ZFCP_LOG_INFO("error: Could not create close unit request for "
 			      "unit 0x%016Lx on port 0x%016Lx on adapter %s.\n",
@@ -3315,27 +3309,26 @@ zfcp_fsf_close_unit(struct zfcp_erp_action *erp_action)
 		goto out;
 	}
 
-	sbale = zfcp_qdio_sbale_req(erp_action->fsf_req,
-                                    erp_action->fsf_req->sbal_curr, 0);
+	sbale = zfcp_qdio_sbale_req(fsf_req, fsf_req->sbal_curr, 0);
         sbale[0].flags |= SBAL_FLAGS0_TYPE_READ;
         sbale[1].flags |= SBAL_FLAGS_LAST_ENTRY;
 
-	erp_action->fsf_req->qtcb->header.port_handle =
-	    erp_action->port->handle;
-	erp_action->fsf_req->qtcb->header.lun_handle = erp_action->unit->handle;
+	fsf_req->qtcb->header.port_handle = erp_action->port->handle;
+	fsf_req->qtcb->header.lun_handle = erp_action->unit->handle;
 	atomic_set_mask(ZFCP_STATUS_COMMON_CLOSING, &erp_action->unit->status);
-	erp_action->fsf_req->data = (unsigned long) erp_action->unit;
-	erp_action->fsf_req->erp_action = erp_action;
+	fsf_req->data = (unsigned long) erp_action->unit;
+	fsf_req->erp_action = erp_action;
+	erp_action->fsf_req = fsf_req;
 
-	/* start QDIO request for this FSF request */
-	retval = zfcp_fsf_req_send(erp_action->fsf_req, &erp_action->timer);
+	zfcp_erp_start_timer(fsf_req);
+	retval = zfcp_fsf_req_send(erp_action->fsf_req);
 	if (retval) {
 		ZFCP_LOG_INFO("error: Could not send a close unit request for "
 			      "unit 0x%016Lx on port 0x%016Lx onadapter %s.\n",
 			      erp_action->unit->fcp_lun,
 			      erp_action->port->wwpn,
 			      zfcp_get_busid_by_adapter(erp_action->adapter));
-		zfcp_fsf_req_free(erp_action->fsf_req);
+		zfcp_fsf_req_free(fsf_req);
 		erp_action->fsf_req = NULL;
 		goto out;
 	}
@@ -3488,7 +3481,7 @@ int
 zfcp_fsf_send_fcp_command_task(struct zfcp_adapter *adapter,
 			       struct zfcp_unit *unit,
 			       struct scsi_cmnd * scsi_cmnd,
-			       struct timer_list *timer, int req_flags)
+			       int use_timer, int req_flags)
 {
 	struct zfcp_fsf_req *fsf_req = NULL;
 	struct fcp_cmnd_iu *fcp_cmnd_iu;
@@ -3516,7 +3509,7 @@ zfcp_fsf_send_fcp_command_task(struct zfcp_adapter *adapter,
 	fsf_req->unit = unit;
 
 	/* associate FSF request with SCSI request (for look up on abort) */
-	scsi_cmnd->host_scribble = (char *) fsf_req;
+	scsi_cmnd->host_scribble = (unsigned char *) fsf_req->req_id;
 
 	/* associate SCSI command with FSF request */
 	fsf_req->data = (unsigned long) scsi_cmnd;
@@ -3629,11 +3622,10 @@ zfcp_fsf_send_fcp_command_task(struct zfcp_adapter *adapter,
 	ZFCP_HEX_DUMP(ZFCP_LOG_LEVEL_DEBUG,
 		      (char *) scsi_cmnd->cmnd, scsi_cmnd->cmd_len);
 
-	/*
-	 * start QDIO request for this FSF request
-	 *  covered by an SBALE)
-	 */
-	retval = zfcp_fsf_req_send(fsf_req, timer);
+	if (use_timer)
+		zfcp_fsf_start_timer(fsf_req, ZFCP_FSF_REQUEST_TIMEOUT);
+
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (unlikely(retval < 0)) {
 		ZFCP_LOG_INFO("error: Could not send FCP command request "
 			      "on adapter %s, port 0x%016Lx, unit 0x%016Lx\n",
@@ -3718,11 +3710,9 @@ zfcp_fsf_send_fcp_command_task_management(struct zfcp_adapter *adapter,
 	fcp_cmnd_iu->fcp_lun = unit->fcp_lun;
 	fcp_cmnd_iu->task_management_flags = tm_flags;
 
-	/* start QDIO request for this FSF request */
-	zfcp_fsf_start_scsi_er_timer(adapter);
-	retval = zfcp_fsf_req_send(fsf_req, NULL);
+	zfcp_fsf_start_timer(fsf_req, ZFCP_SCSI_ER_TIMEOUT);
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (retval) {
-		del_timer(&adapter->scsi_er_timer);
 		ZFCP_LOG_INFO("error: Could not send an FCP-command (task "
 			      "management) on adapter %s, port 0x%016Lx for "
 			      "unit LUN 0x%016Lx\n",
@@ -4226,7 +4216,6 @@ zfcp_fsf_send_fcp_command_task_management_handler(struct zfcp_fsf_req *fsf_req)
 	char *fcp_rsp_info = zfcp_get_fcp_rsp_info_ptr(fcp_rsp_iu);
 	struct zfcp_unit *unit = (struct zfcp_unit *) fsf_req->data;
 
-	del_timer(&fsf_req->adapter->scsi_er_timer);
 	if (fsf_req->status & ZFCP_STATUS_FSFREQ_ERROR) {
 		fsf_req->status |= ZFCP_STATUS_FSFREQ_TMFUNCFAILED;
 		goto skip_fsfstatus;
@@ -4295,7 +4284,6 @@ zfcp_fsf_control_file(struct zfcp_adapter *adapter,
 	struct zfcp_fsf_req *fsf_req;
 	struct fsf_qtcb_bottom_support *bottom;
 	volatile struct qdio_buffer_element *sbale;
-	struct timer_list *timer;
 	unsigned long lock_flags;
 	int req_flags = 0;
 	int direction;
@@ -4327,12 +4315,6 @@ zfcp_fsf_control_file(struct zfcp_adapter *adapter,
 		goto out;
 	}
 
-	timer = kmalloc(sizeof(struct timer_list), GFP_KERNEL);
-	if (!timer) {
-		retval = -ENOMEM;
-		goto out;
- 	}
-
 	retval = zfcp_fsf_req_create(adapter, fsf_command, req_flags,
 				     NULL, &lock_flags, &fsf_req);
 	if (retval < 0) {
@@ -4367,12 +4349,8 @@ zfcp_fsf_control_file(struct zfcp_adapter *adapter,
 	} else
 		sbale[1].flags |= SBAL_FLAGS_LAST_ENTRY;
 
-	init_timer(timer);
-	timer->function = zfcp_fsf_request_timeout_handler;
-	timer->data = (unsigned long) adapter;
-	timer->expires = ZFCP_FSF_REQUEST_TIMEOUT;
-
-	retval = zfcp_fsf_req_send(fsf_req, timer);
+	zfcp_fsf_start_timer(fsf_req, ZFCP_FSF_REQUEST_TIMEOUT);
+	retval = zfcp_fsf_req_send(fsf_req);
 	if (retval < 0) {
 		ZFCP_LOG_INFO("initiation of cfdc up/download failed"
 			      "(adapter %s)\n",
@@ -4392,15 +4370,12 @@ zfcp_fsf_control_file(struct zfcp_adapter *adapter,
 	           fsf_req->status & ZFCP_STATUS_FSFREQ_COMPLETED);
 
 	*fsf_req_ptr = fsf_req;
-	del_timer_sync(timer);
-	goto free_timer;
+	goto out;
 
  free_fsf_req:
 	zfcp_fsf_req_free(fsf_req);
  unlock_queue_lock:
 	write_unlock_irqrestore(&adapter->request_queue.queue_lock, lock_flags);
- free_timer:
-	kfree(timer);
  out:
 	return retval;
 }
@@ -4656,7 +4631,6 @@ zfcp_fsf_req_create(struct zfcp_adapter *adapter, u32 fsf_cmd, int req_flags,
 {
 	volatile struct qdio_buffer_element *sbale;
 	struct zfcp_fsf_req *fsf_req = NULL;
-	unsigned long flags;
 	int ret = 0;
 	struct zfcp_qdio_queue *req_queue = &adapter->request_queue;
 
@@ -4673,12 +4647,13 @@ zfcp_fsf_req_create(struct zfcp_adapter *adapter, u32 fsf_cmd, int req_flags,
 	fsf_req->fsf_command = fsf_cmd;
 	INIT_LIST_HEAD(&fsf_req->list);
 	
-	/* unique request id */
-	spin_lock_irqsave(&adapter->req_list_lock, flags);
+	/* this is serialized (we are holding req_queue-lock of adapter */
+	if (adapter->req_no == 0)
+		adapter->req_no++;
 	fsf_req->req_id = adapter->req_no++;
-	spin_unlock_irqrestore(&adapter->req_list_lock, flags);
 
-        zfcp_fsf_req_qtcb_init(fsf_req);
+	init_timer(&fsf_req->timer);
+	zfcp_fsf_req_qtcb_init(fsf_req);
 
 	/* initialize waitqueue which may be used to wait on 
 	   this request completion */
@@ -4748,8 +4723,7 @@ zfcp_fsf_req_create(struct zfcp_adapter *adapter, u32 fsf_cmd, int req_flags,
  * returns:	0 - request transfer succesfully started
  *		!0 - start of request transfer failed
  */
-static int
-zfcp_fsf_req_send(struct zfcp_fsf_req *fsf_req, struct timer_list *timer)
+static int zfcp_fsf_req_send(struct zfcp_fsf_req *fsf_req)
 {
 	struct zfcp_adapter *adapter;
 	struct zfcp_qdio_queue *req_queue;
@@ -4777,12 +4751,6 @@ zfcp_fsf_req_send(struct zfcp_fsf_req *fsf_req, struct timer_list *timer)
 
 	inc_seq_no = (fsf_req->qtcb != NULL);
 
-	/* figure out expiration time of timeout and start timeout */
-	if (unlikely(timer)) {
-		timer->expires += jiffies;
-		add_timer(timer);
-	}
-
 	ZFCP_LOG_TRACE("request queue of adapter %s: "
 		       "next free SBAL is %i, %i free SBALs\n",
 		       zfcp_get_busid_by_adapter(adapter),
@@ -4819,12 +4787,7 @@ zfcp_fsf_req_send(struct zfcp_fsf_req *fsf_req, struct timer_list *timer)
 	if (unlikely(retval)) {
 		/* Queues are down..... */
 		retval = -EIO;
-		/*
-		 * FIXME(potential race):
-		 * timer might be expired (absolutely unlikely)
-		 */
-		if (timer)
-			del_timer(timer);
+		del_timer(&fsf_req->timer);
 		spin_lock(&adapter->req_list_lock);
 		zfcp_reqlist_remove(adapter, fsf_req->req_id);
 		spin_unlock(&adapter->req_list_lock);
diff --git a/drivers/s390/scsi/zfcp_scsi.c b/drivers/s390/scsi/zfcp_scsi.c
index 1bb5508..7cafa34 100644
--- a/drivers/s390/scsi/zfcp_scsi.c
+++ b/drivers/s390/scsi/zfcp_scsi.c
@@ -39,11 +39,10 @@ static struct zfcp_unit *zfcp_unit_lookup(struct zfcp_adapter *, int,
 
 static struct device_attribute *zfcp_sysfs_sdev_attrs[];
 
-struct scsi_transport_template *zfcp_transport_template;
-
 struct zfcp_data zfcp_data = {
 	.scsi_host_template = {
 		.name			= ZFCP_NAME,
+		.module			= THIS_MODULE,
 		.proc_name		= "zfcp",
 		.slave_alloc		= zfcp_scsi_slave_alloc,
 		.slave_configure	= zfcp_scsi_slave_configure,
@@ -232,7 +231,7 @@ zfcp_scsi_command_fail(struct scsi_cmnd *scpnt, int result)
  */
 int
 zfcp_scsi_command_async(struct zfcp_adapter *adapter, struct zfcp_unit *unit,
-			struct scsi_cmnd *scpnt, struct timer_list *timer)
+			struct scsi_cmnd *scpnt, int use_timer)
 {
 	int tmp;
 	int retval;
@@ -268,7 +267,7 @@ zfcp_scsi_command_async(struct zfcp_adapter *adapter, struct zfcp_unit *unit,
 		goto out;
 	}
 
-	tmp = zfcp_fsf_send_fcp_command_task(adapter, unit, scpnt, timer,
+	tmp = zfcp_fsf_send_fcp_command_task(adapter, unit, scpnt, use_timer,
 					     ZFCP_REQ_AUTO_CLEANUP);
 
 	if (unlikely(tmp < 0)) {
@@ -292,21 +291,22 @@ zfcp_scsi_command_sync_handler(struct scsi_cmnd *scpnt)
  * zfcp_scsi_command_sync - send a SCSI command and wait for completion
  * @unit: unit where command is sent to
  * @scpnt: scsi command to be sent
- * @timer: timer to be started if request is successfully initiated
+ * @use_timer: indicates whether timer should be setup or not
  * Return: 0
  *
  * Errors are indicated in scpnt->result
  */
 int
 zfcp_scsi_command_sync(struct zfcp_unit *unit, struct scsi_cmnd *scpnt,
-		       struct timer_list *timer)
+		       int use_timer)
 {
 	int ret;
 	DECLARE_COMPLETION(wait);
 
 	scpnt->SCp.ptr = (void *) &wait;  /* silent re-use */
 	scpnt->scsi_done = zfcp_scsi_command_sync_handler;
-	ret = zfcp_scsi_command_async(unit->port->adapter, unit, scpnt, timer);
+	ret = zfcp_scsi_command_async(unit->port->adapter, unit, scpnt,
+				      use_timer);
 	if (ret == 0)
 		wait_for_completion(&wait);
 
@@ -342,7 +342,7 @@ zfcp_scsi_queuecommand(struct scsi_cmnd *scpnt,
 	adapter = (struct zfcp_adapter *) scpnt->device->host->hostdata[0];
 	unit = (struct zfcp_unit *) scpnt->device->hostdata;
 
-	return zfcp_scsi_command_async(adapter, unit, scpnt, NULL);
+	return zfcp_scsi_command_async(adapter, unit, scpnt, 0);
 }
 
 static struct zfcp_unit *
@@ -379,16 +379,15 @@ zfcp_unit_lookup(struct zfcp_adapter *adapter, int channel, unsigned int id,
  * will handle late commands.  (Usually, the normal completion of late
  * commands is ignored with respect to the running abort operation.)
  */
-int
-zfcp_scsi_eh_abort_handler(struct scsi_cmnd *scpnt)
+int zfcp_scsi_eh_abort_handler(struct scsi_cmnd *scpnt)
 {
  	struct Scsi_Host *scsi_host;
  	struct zfcp_adapter *adapter;
 	struct zfcp_unit *unit;
-	int retval = SUCCESS;
-	struct zfcp_fsf_req *new_fsf_req = NULL;
-	struct zfcp_fsf_req *old_fsf_req;
+	struct zfcp_fsf_req *fsf_req;
 	unsigned long flags;
+	unsigned long old_req_id;
+	int retval = SUCCESS;
 
 	scsi_host = scpnt->device->host;
 	adapter = (struct zfcp_adapter *) scsi_host->hostdata[0];
@@ -400,55 +399,47 @@ zfcp_scsi_eh_abort_handler(struct scsi_cmnd *scpnt)
 	/* avoid race condition between late normal completion and abort */
 	write_lock_irqsave(&adapter->abort_lock, flags);
 
-	/*
-	 * Check whether command has just completed and can not be aborted.
-	 * Even if the command has just been completed late, we can access
-	 * scpnt since the SCSI stack does not release it at least until
-	 * this routine returns. (scpnt is parameter passed to this routine
-	 * and must not disappear during abort even on late completion.)
-	 */
-	old_fsf_req = (struct zfcp_fsf_req *) scpnt->host_scribble;
-	if (!old_fsf_req) {
+	/* Check whether corresponding fsf_req is still pending */
+	spin_lock(&adapter->req_list_lock);
+	fsf_req = zfcp_reqlist_ismember(adapter, (unsigned long)
+					scpnt->host_scribble);
+	spin_unlock(&adapter->req_list_lock);
+	if (!fsf_req) {
 		write_unlock_irqrestore(&adapter->abort_lock, flags);
-		zfcp_scsi_dbf_event_abort("lte1", adapter, scpnt, NULL, NULL);
+		zfcp_scsi_dbf_event_abort("lte1", adapter, scpnt, NULL, 0);
 		retval = SUCCESS;
 		goto out;
 	}
-	old_fsf_req->data = 0;
-	old_fsf_req->status |= ZFCP_STATUS_FSFREQ_ABORTING;
+	fsf_req->data = 0;
+	fsf_req->status |= ZFCP_STATUS_FSFREQ_ABORTING;
+	old_req_id = fsf_req->req_id;
 
-	/* don't access old_fsf_req after releasing the abort_lock */
+	/* don't access old fsf_req after releasing the abort_lock */
 	write_unlock_irqrestore(&adapter->abort_lock, flags);
-	/* call FSF routine which does the abort */
-	new_fsf_req = zfcp_fsf_abort_fcp_command((unsigned long) old_fsf_req,
-						 adapter, unit, 0);
-	if (!new_fsf_req) {
+
+	fsf_req = zfcp_fsf_abort_fcp_command(old_req_id, adapter, unit, 0);
+	if (!fsf_req) {
 		ZFCP_LOG_INFO("error: initiation of Abort FCP Cmnd failed\n");
 		zfcp_scsi_dbf_event_abort("nres", adapter, scpnt, NULL,
-					  old_fsf_req);
+					  old_req_id);
 		retval = FAILED;
 		goto out;
 	}
 
-	/* wait for completion of abort */
-	__wait_event(new_fsf_req->completion_wq,
-		     new_fsf_req->status & ZFCP_STATUS_FSFREQ_COMPLETED);
+	__wait_event(fsf_req->completion_wq,
+		     fsf_req->status & ZFCP_STATUS_FSFREQ_COMPLETED);
 
-	/* status should be valid since signals were not permitted */
-	if (new_fsf_req->status & ZFCP_STATUS_FSFREQ_ABORTSUCCEEDED) {
-		zfcp_scsi_dbf_event_abort("okay", adapter, scpnt, new_fsf_req,
-					  NULL);
+	if (fsf_req->status & ZFCP_STATUS_FSFREQ_ABORTSUCCEEDED) {
+		zfcp_scsi_dbf_event_abort("okay", adapter, scpnt, fsf_req, 0);
 		retval = SUCCESS;
-	} else if (new_fsf_req->status & ZFCP_STATUS_FSFREQ_ABORTNOTNEEDED) {
-		zfcp_scsi_dbf_event_abort("lte2", adapter, scpnt, new_fsf_req,
-					  NULL);
+	} else if (fsf_req->status & ZFCP_STATUS_FSFREQ_ABORTNOTNEEDED) {
+		zfcp_scsi_dbf_event_abort("lte2", adapter, scpnt, fsf_req, 0);
 		retval = SUCCESS;
 	} else {
-		zfcp_scsi_dbf_event_abort("fail", adapter, scpnt, new_fsf_req,
-					  NULL);
+		zfcp_scsi_dbf_event_abort("fail", adapter, scpnt, fsf_req, 0);
 		retval = FAILED;
 	}
-	zfcp_fsf_req_free(new_fsf_req);
+	zfcp_fsf_req_free(fsf_req);
  out:
 	return retval;
 }
@@ -548,14 +539,11 @@ zfcp_task_management_function(struct zfcp_unit *unit, u8 tm_flags,
 
 /**
  * zfcp_scsi_eh_host_reset_handler - handler for host and bus reset
- *
- * If ERP is already running it will be stopped.
  */
 int zfcp_scsi_eh_host_reset_handler(struct scsi_cmnd *scpnt)
 {
 	struct zfcp_unit *unit;
 	struct zfcp_adapter *adapter;
-	unsigned long flags;
 
 	unit = (struct zfcp_unit*) scpnt->device->hostdata;
 	adapter = unit->port->adapter;
@@ -563,22 +551,8 @@ int zfcp_scsi_eh_host_reset_handler(struct scsi_cmnd *scpnt)
 	ZFCP_LOG_NORMAL("host/bus reset because of problems with "
 			"unit 0x%016Lx\n", unit->fcp_lun);
 
-	write_lock_irqsave(&adapter->erp_lock, flags);
-	if (atomic_test_mask(ZFCP_STATUS_ADAPTER_ERP_PENDING,
-			     &adapter->status)) {
-		zfcp_erp_modify_adapter_status(adapter,
-		       ZFCP_STATUS_COMMON_UNBLOCKED|ZFCP_STATUS_COMMON_OPEN,
-		       ZFCP_CLEAR);
-		zfcp_erp_action_dismiss_adapter(adapter);
-		write_unlock_irqrestore(&adapter->erp_lock, flags);
-		zfcp_fsf_req_dismiss_all(adapter);
-		adapter->fsf_req_seq_no = 0;
-		zfcp_erp_adapter_reopen(adapter, 0);
-	} else {
-		write_unlock_irqrestore(&adapter->erp_lock, flags);
-		zfcp_erp_adapter_reopen(adapter, 0);
-		zfcp_erp_wait(adapter);
-	}
+	zfcp_erp_adapter_reopen(adapter, 0);
+	zfcp_erp_wait(adapter);
 
 	return SUCCESS;
 }
@@ -607,7 +581,7 @@ zfcp_adapter_scsi_register(struct zfcp_adapter *adapter)
 	adapter->scsi_host->max_channel = 0;
 	adapter->scsi_host->unique_id = unique_id++;	/* FIXME */
 	adapter->scsi_host->max_cmd_len = ZFCP_MAX_SCSI_CMND_LENGTH;
-	adapter->scsi_host->transportt = zfcp_transport_template;
+	adapter->scsi_host->transportt = zfcp_data.scsi_transport_template;
 
 	/*
 	 * save a pointer to our own adapter data structure within
@@ -648,16 +622,6 @@ zfcp_adapter_scsi_unregister(struct zfcp_adapter *adapter)
 	return;
 }
 
-
-void
-zfcp_fsf_start_scsi_er_timer(struct zfcp_adapter *adapter)
-{
-	adapter->scsi_er_timer.function = zfcp_fsf_scsi_er_timeout_handler;
-	adapter->scsi_er_timer.data = (unsigned long) adapter;
-	adapter->scsi_er_timer.expires = jiffies + ZFCP_SCSI_ER_TIMEOUT;
-	add_timer(&adapter->scsi_er_timer);
-}
-
 /*
  * Support functions for FC transport class
  */
diff --git a/drivers/s390/sysinfo.c b/drivers/s390/sysinfo.c
index d1c1e75..1e788e8 100644
--- a/drivers/s390/sysinfo.c
+++ b/drivers/s390/sysinfo.c
@@ -11,19 +11,18 @@
 #include <linux/init.h>
 #include <asm/ebcdic.h>
 
-struct sysinfo_1_1_1
-{
+struct sysinfo_1_1_1 {
 	char reserved_0[32];
 	char manufacturer[16];
 	char type[4];
 	char reserved_1[12];
-	char model[16];
+	char model_capacity[16];
 	char sequence[16];
 	char plant[4];
+	char model[16];
 };
 
-struct sysinfo_1_2_1
-{
+struct sysinfo_1_2_1 {
 	char reserved_0[80];
 	char sequence[16];
 	char plant[4];
@@ -31,9 +30,12 @@ struct sysinfo_1_2_1
 	unsigned short cpu_address;
 };
 
-struct sysinfo_1_2_2
-{
-	char reserved_0[32];
+struct sysinfo_1_2_2 {
+	char format;
+	char reserved_0[1];
+	unsigned short acc_offset;
+	char reserved_1[24];
+	unsigned int secondary_capability;
 	unsigned int capability;
 	unsigned short cpus_total;
 	unsigned short cpus_configured;
@@ -42,8 +44,12 @@ struct sysinfo_1_2_2
 	unsigned short adjustment[0];
 };
 
-struct sysinfo_2_2_1
-{
+struct sysinfo_1_2_2_extension {
+	unsigned int alt_capability;
+	unsigned short alt_adjustment[0];
+};
+
+struct sysinfo_2_2_1 {
 	char reserved_0[80];
 	char sequence[16];
 	char plant[4];
@@ -51,15 +57,11 @@ struct sysinfo_2_2_1
 	unsigned short cpu_address;
 };
 
-struct sysinfo_2_2_2
-{
+struct sysinfo_2_2_2 {
 	char reserved_0[32];
 	unsigned short lpar_number;
 	char reserved_1;
 	unsigned char characteristics;
-	#define LPAR_CHAR_DEDICATED	(1 << 7)
-	#define LPAR_CHAR_SHARED	(1 << 6)
-	#define LPAR_CHAR_LIMITED	(1 << 5)
 	unsigned short cpus_total;
 	unsigned short cpus_configured;
 	unsigned short cpus_standby;
@@ -71,12 +73,14 @@ struct sysinfo_2_2_2
 	unsigned short cpus_shared;
 };
 
-struct sysinfo_3_2_2
-{
+#define LPAR_CHAR_DEDICATED	(1 << 7)
+#define LPAR_CHAR_SHARED	(1 << 6)
+#define LPAR_CHAR_LIMITED	(1 << 5)
+
+struct sysinfo_3_2_2 {
 	char reserved_0[31];
 	unsigned char count;
-	struct
-	{
+	struct {
 		char reserved_0[4];
 		unsigned short cpus_total;
 		unsigned short cpus_configured;
@@ -90,136 +94,223 @@ struct sysinfo_3_2_2
 	} vm[8];
 };
 
-union s390_sysinfo
+static inline int stsi(void *sysinfo, int fc, int sel1, int sel2)
 {
-	struct sysinfo_1_1_1 sysinfo_1_1_1;
-	struct sysinfo_1_2_1 sysinfo_1_2_1;
-	struct sysinfo_1_2_2 sysinfo_1_2_2;
-	struct sysinfo_2_2_1 sysinfo_2_2_1;
-	struct sysinfo_2_2_2 sysinfo_2_2_2;
-	struct sysinfo_3_2_2 sysinfo_3_2_2;
-};
-
-static inline int stsi (void *sysinfo, 
-                        int fc, int sel1, int sel2)
-{
-	int cc, retv;
-
-#ifndef CONFIG_64BIT
-	__asm__ __volatile__ (	"lr\t0,%2\n"
-				"\tlr\t1,%3\n"
-				"\tstsi\t0(%4)\n"
-				"0:\tipm\t%0\n"
-				"\tsrl\t%0,28\n"
-				"1:lr\t%1,0\n"
-				".section .fixup,\"ax\"\n"
-				"2:\tlhi\t%0,3\n"
-				"\tbras\t1,3f\n"
-				"\t.long 1b\n"
-				"3:\tl\t1,0(1)\n"
-				"\tbr\t1\n"
-				".previous\n"
-				".section __ex_table,\"a\"\n"
-				"\t.align 4\n"
-				"\t.long 0b,2b\n"
-				".previous\n"
-				: "=d" (cc), "=d" (retv)
-				: "d" ((fc << 28) | sel1), "d" (sel2), "a" (sysinfo) 
-				: "cc", "memory", "0", "1" );
-#else
-	__asm__ __volatile__ (	"lr\t0,%2\n"
-				"lr\t1,%3\n"
-				"\tstsi\t0(%4)\n"
-				"0:\tipm\t%0\n"
-				"\tsrl\t%0,28\n"
-				"1:lr\t%1,0\n"
-				".section .fixup,\"ax\"\n"
-				"2:\tlhi\t%0,3\n"
-				"\tjg\t1b\n"
-				".previous\n"
-				".section __ex_table,\"a\"\n"
-				"\t.align 8\n"
-				"\t.quad 0b,2b\n"
-				".previous\n"
-				: "=d" (cc), "=d" (retv)
-				: "d" ((fc << 28) | sel1), "d" (sel2), "a" (sysinfo) 
-				: "cc", "memory", "0", "1" );
-#endif
-
-	return cc? -1 : retv;
+	register int r0 asm("0") = (fc << 28) | sel1;
+	register int r1 asm("1") = sel2;
+
+	asm volatile(
+		"   stsi 0(%2)\n"
+		"0: jz   2f\n"
+		"1: lhi  %0,%3\n"
+		"2:\n"
+		EX_TABLE(0b,1b)
+		: "+d" (r0) : "d" (r1), "a" (sysinfo), "K" (-ENOSYS)
+		: "cc", "memory" );
+	return r0;
 }
 
-static inline int stsi_0 (void)
+static inline int stsi_0(void)
 {
 	int rc = stsi (NULL, 0, 0, 0);
-	return rc == -1 ? rc : (((unsigned int)rc) >> 28);
+	return rc == -ENOSYS ? rc : (((unsigned int) rc) >> 28);
 }
 
-static inline int stsi_1_1_1 (struct sysinfo_1_1_1 *info)
+static int stsi_1_1_1(struct sysinfo_1_1_1 *info, char *page, int len)
 {
-	int rc = stsi (info, 1, 1, 1);
-	if (rc != -1)
-	{
-		EBCASC (info->manufacturer, sizeof(info->manufacturer));
-		EBCASC (info->type, sizeof(info->type));
-		EBCASC (info->model, sizeof(info->model));
-		EBCASC (info->sequence, sizeof(info->sequence));
-		EBCASC (info->plant, sizeof(info->plant));
-	}
-	return rc == -1 ? rc : 0;
+	if (stsi(info, 1, 1, 1) == -ENOSYS)
+		return len;
+
+	EBCASC(info->manufacturer, sizeof(info->manufacturer));
+	EBCASC(info->type, sizeof(info->type));
+	EBCASC(info->model, sizeof(info->model));
+	EBCASC(info->sequence, sizeof(info->sequence));
+	EBCASC(info->plant, sizeof(info->plant));
+	EBCASC(info->model_capacity, sizeof(info->model_capacity));
+	len += sprintf(page + len, "Manufacturer:         %-16.16s\n",
+		       info->manufacturer);
+	len += sprintf(page + len, "Type:                 %-4.4s\n",
+		       info->type);
+	if (info->model[0] != '\0')
+		/*
+		 * Sigh: the model field has been renamed with System z9
+		 * to model_capacity and a new model field has been added
+		 * after the plant field. To avoid confusing older programs
+		 * the "Model:" prints "model_capacity model" or just
+		 * "model_capacity" if the model string is empty .
+		 */
+		len += sprintf(page + len,
+			       "Model:                %-16.16s %-16.16s\n",
+			       info->model_capacity, info->model);
+	else
+		len += sprintf(page + len, "Model:                %-16.16s\n",
+			       info->model_capacity);
+	len += sprintf(page + len, "Sequence Code:        %-16.16s\n",
+		       info->sequence);
+	len += sprintf(page + len, "Plant:                %-4.4s\n",
+		       info->plant);
+	len += sprintf(page + len, "Model Capacity:       %-16.16s\n",
+		       info->model_capacity);
+	return len;
 }
 
-static inline int stsi_1_2_1 (struct sysinfo_1_2_1 *info)
+#if 0 /* Currently unused */
+static int stsi_1_2_1(struct sysinfo_1_2_1 *info, char *page, int len)
 {
-	int rc = stsi (info, 1, 2, 1);
-	if (rc != -1)
-	{
-		EBCASC (info->sequence, sizeof(info->sequence));
-		EBCASC (info->plant, sizeof(info->plant));
-	}
-	return rc == -1 ? rc : 0;
+	if (stsi(info, 1, 2, 1) == -ENOSYS)
+		return len;
+
+	len += sprintf(page + len, "\n");
+	EBCASC(info->sequence, sizeof(info->sequence));
+	EBCASC(info->plant, sizeof(info->plant));
+	len += sprintf(page + len, "Sequence Code of CPU: %-16.16s\n",
+		       info->sequence);
+	len += sprintf(page + len, "Plant of CPU:         %-16.16s\n",
+		       info->plant);
+	return len;
 }
+#endif
 
-static inline int stsi_1_2_2 (struct sysinfo_1_2_2 *info)
+static int stsi_1_2_2(struct sysinfo_1_2_2 *info, char *page, int len)
 {
-	int rc = stsi (info, 1, 2, 2);
-	return rc == -1 ? rc : 0;
+	struct sysinfo_1_2_2_extension *ext;
+	int i;
+
+	if (stsi(info, 1, 2, 2) == -ENOSYS)
+		return len;
+	ext = (struct sysinfo_1_2_2_extension *)
+		((unsigned long) info + info->acc_offset);
+
+	len += sprintf(page + len, "\n");
+	len += sprintf(page + len, "CPUs Total:           %d\n",
+		       info->cpus_total);
+	len += sprintf(page + len, "CPUs Configured:      %d\n",
+		       info->cpus_configured);
+	len += sprintf(page + len, "CPUs Standby:         %d\n",
+		       info->cpus_standby);
+	len += sprintf(page + len, "CPUs Reserved:        %d\n",
+		       info->cpus_reserved);
+
+	if (info->format == 1) {
+		/*
+		 * Sigh 2. According to the specification the alternate
+		 * capability field is a 32 bit floating point number
+		 * if the higher order 8 bits are not zero. Printing
+		 * a floating point number in the kernel is a no-no,
+		 * always print the number as 32 bit unsigned integer.
+		 * The user-space needs to know about the stange
+		 * encoding of the alternate cpu capability.
+		 */
+		len += sprintf(page + len, "Capability:           %u %u\n",
+			       info->capability, ext->alt_capability);
+		for (i = 2; i <= info->cpus_total; i++)
+			len += sprintf(page + len,
+				       "Adjustment %02d-way:    %u %u\n",
+				       i, info->adjustment[i-2],
+				       ext->alt_adjustment[i-2]);
+
+	} else {
+		len += sprintf(page + len, "Capability:           %u\n",
+			       info->capability);
+		for (i = 2; i <= info->cpus_total; i++)
+			len += sprintf(page + len,
+				       "Adjustment %02d-way:    %u\n",
+				       i, info->adjustment[i-2]);
+	}
+
+	if (info->secondary_capability != 0)
+		len += sprintf(page + len, "Secondary Capability: %d\n",
+			       info->secondary_capability);
+
+	return len;
 }
 
-static inline int stsi_2_2_1 (struct sysinfo_2_2_1 *info)
+#if 0 /* Currently unused */
+static int stsi_2_2_1(struct sysinfo_2_2_1 *info, char *page, int len)
 {
-	int rc = stsi (info, 2, 2, 1);
-	if (rc != -1)
-	{
-		EBCASC (info->sequence, sizeof(info->sequence));
-		EBCASC (info->plant, sizeof(info->plant));
-	}
-	return rc == -1 ? rc : 0;
+	if (stsi(info, 2, 2, 1) == -ENOSYS)
+		return len;
+
+	len += sprintf(page + len, "\n");
+	EBCASC (info->sequence, sizeof(info->sequence));
+	EBCASC (info->plant, sizeof(info->plant));
+	len += sprintf(page + len, "Sequence Code of logical CPU: %-16.16s\n",
+		       info->sequence);
+	len += sprintf(page + len, "Plant of logical CPU: %-16.16s\n",
+		       info->plant);
+	return len;
 }
+#endif
 
-static inline int stsi_2_2_2 (struct sysinfo_2_2_2 *info)
+static int stsi_2_2_2(struct sysinfo_2_2_2 *info, char *page, int len)
 {
-	int rc = stsi (info, 2, 2, 2);
-	if (rc != -1)
-	{
-		EBCASC (info->name, sizeof(info->name));
-  	}
-	return rc == -1 ? rc : 0;
+	if (stsi(info, 2, 2, 2) == -ENOSYS)
+		return len;
+
+	EBCASC (info->name, sizeof(info->name));
+
+	len += sprintf(page + len, "\n");
+	len += sprintf(page + len, "LPAR Number:          %d\n",
+		       info->lpar_number);
+
+	len += sprintf(page + len, "LPAR Characteristics: ");
+	if (info->characteristics & LPAR_CHAR_DEDICATED)
+		len += sprintf(page + len, "Dedicated ");
+	if (info->characteristics & LPAR_CHAR_SHARED)
+		len += sprintf(page + len, "Shared ");
+	if (info->characteristics & LPAR_CHAR_LIMITED)
+		len += sprintf(page + len, "Limited ");
+	len += sprintf(page + len, "\n");
+
+	len += sprintf(page + len, "LPAR Name:            %-8.8s\n",
+		       info->name);
+
+	len += sprintf(page + len, "LPAR Adjustment:      %d\n",
+		       info->caf);
+
+	len += sprintf(page + len, "LPAR CPUs Total:      %d\n",
+		       info->cpus_total);
+	len += sprintf(page + len, "LPAR CPUs Configured: %d\n",
+		       info->cpus_configured);
+	len += sprintf(page + len, "LPAR CPUs Standby:    %d\n",
+		       info->cpus_standby);
+	len += sprintf(page + len, "LPAR CPUs Reserved:   %d\n",
+		       info->cpus_reserved);
+	len += sprintf(page + len, "LPAR CPUs Dedicated:  %d\n",
+		       info->cpus_dedicated);
+	len += sprintf(page + len, "LPAR CPUs Shared:     %d\n",
+		       info->cpus_shared);
+	return len;
 }
 
-static inline int stsi_3_2_2 (struct sysinfo_3_2_2 *info)
+static int stsi_3_2_2(struct sysinfo_3_2_2 *info, char *page, int len)
 {
-	int rc = stsi (info, 3, 2, 2);
-	if (rc != -1)
-	{
-		int i;
-		for (i = 0; i < info->count; i++)
-		{
-			EBCASC (info->vm[i].name, sizeof(info->vm[i].name));
-			EBCASC (info->vm[i].cpi, sizeof(info->vm[i].cpi));
-		}
+	int i;
+
+	if (stsi(info, 3, 2, 2) == -ENOSYS)
+		return len;
+	for (i = 0; i < info->count; i++) {
+		EBCASC (info->vm[i].name, sizeof(info->vm[i].name));
+		EBCASC (info->vm[i].cpi, sizeof(info->vm[i].cpi));
+		len += sprintf(page + len, "\n");
+		len += sprintf(page + len, "VM%02d Name:            %-8.8s\n",
+			       i, info->vm[i].name);
+		len += sprintf(page + len, "VM%02d Control Program: %-16.16s\n",
+			       i, info->vm[i].cpi);
+
+		len += sprintf(page + len, "VM%02d Adjustment:      %d\n",
+			       i, info->vm[i].caf);
+
+		len += sprintf(page + len, "VM%02d CPUs Total:      %d\n",
+			       i, info->vm[i].cpus_total);
+		len += sprintf(page + len, "VM%02d CPUs Configured: %d\n",
+			       i, info->vm[i].cpus_configured);
+		len += sprintf(page + len, "VM%02d CPUs Standby:    %d\n",
+			       i, info->vm[i].cpus_standby);
+		len += sprintf(page + len, "VM%02d CPUs Reserved:   %d\n",
+			       i, info->vm[i].cpus_reserved);
 	}
-	return rc == -1 ? rc : 0;
+	return len;
 }
 
 
@@ -227,118 +318,34 @@ static int proc_read_sysinfo(char *page, char **start,
                              off_t off, int count,
                              int *eof, void *data)
 {
-	unsigned long info_page = get_zeroed_page (GFP_KERNEL); 
-	union s390_sysinfo *info = (union s390_sysinfo *) info_page;
-	int len = 0;
-	int level;
-	int i;
+	unsigned long info = get_zeroed_page (GFP_KERNEL);
+	int level, len;
 	
 	if (!info)
 		return 0;
 
-	level = stsi_0 ();
-
-	if (level >= 1 && stsi_1_1_1 (&info->sysinfo_1_1_1) == 0)
-	{
-		len += sprintf (page+len, "Manufacturer:         %-16.16s\n",
-				info->sysinfo_1_1_1.manufacturer);
-		len += sprintf (page+len, "Type:                 %-4.4s\n",
-				info->sysinfo_1_1_1.type);
-		len += sprintf (page+len, "Model:                %-16.16s\n",
-				info->sysinfo_1_1_1.model);
-		len += sprintf (page+len, "Sequence Code:        %-16.16s\n",
-				info->sysinfo_1_1_1.sequence);
-		len += sprintf (page+len, "Plant:                %-4.4s\n",
-				info->sysinfo_1_1_1.plant);
-	}
-
-	if (level >= 1 && stsi_1_2_2 (&info->sysinfo_1_2_2) == 0)
-	{
-		len += sprintf (page+len, "\n");
-		len += sprintf (page+len, "CPUs Total:           %d\n",
-				info->sysinfo_1_2_2.cpus_total);
-		len += sprintf (page+len, "CPUs Configured:      %d\n",
-				info->sysinfo_1_2_2.cpus_configured);
-		len += sprintf (page+len, "CPUs Standby:         %d\n",
-				info->sysinfo_1_2_2.cpus_standby);
-		len += sprintf (page+len, "CPUs Reserved:        %d\n",
-				info->sysinfo_1_2_2.cpus_reserved);
-	
-		len += sprintf (page+len, "Capability:           %d\n",
-				info->sysinfo_1_2_2.capability);
+	len = 0;
+	level = stsi_0();
+	if (level >= 1)
+		len = stsi_1_1_1((struct sysinfo_1_1_1 *) info, page, len);
 
-		for (i = 2; i <= info->sysinfo_1_2_2.cpus_total; i++)
-			len += sprintf (page+len, "Adjustment %02d-way:    %d\n",
-					i, info->sysinfo_1_2_2.adjustment[i-2]);
-	}
+	if (level >= 1)
+		len = stsi_1_2_2((struct sysinfo_1_2_2 *) info, page, len);
 
-	if (level >= 2 && stsi_2_2_2 (&info->sysinfo_2_2_2) == 0)
-	{
-		len += sprintf (page+len, "\n");
-		len += sprintf (page+len, "LPAR Number:          %d\n",
-				info->sysinfo_2_2_2.lpar_number);
-
-		len += sprintf (page+len, "LPAR Characteristics: ");
-		if (info->sysinfo_2_2_2.characteristics & LPAR_CHAR_DEDICATED)
-			len += sprintf (page+len, "Dedicated ");
-		if (info->sysinfo_2_2_2.characteristics & LPAR_CHAR_SHARED)
-			len += sprintf (page+len, "Shared ");
-		if (info->sysinfo_2_2_2.characteristics & LPAR_CHAR_LIMITED)
-			len += sprintf (page+len, "Limited ");
-		len += sprintf (page+len, "\n");
-	
-		len += sprintf (page+len, "LPAR Name:            %-8.8s\n",
-				info->sysinfo_2_2_2.name);
-	
-		len += sprintf (page+len, "LPAR Adjustment:      %d\n",
-				info->sysinfo_2_2_2.caf);
-	
-		len += sprintf (page+len, "LPAR CPUs Total:      %d\n",
-				info->sysinfo_2_2_2.cpus_total);
-		len += sprintf (page+len, "LPAR CPUs Configured: %d\n",
-				info->sysinfo_2_2_2.cpus_configured);
-		len += sprintf (page+len, "LPAR CPUs Standby:    %d\n",
-				info->sysinfo_2_2_2.cpus_standby);
-		len += sprintf (page+len, "LPAR CPUs Reserved:   %d\n",
-				info->sysinfo_2_2_2.cpus_reserved);
-		len += sprintf (page+len, "LPAR CPUs Dedicated:  %d\n",
-				info->sysinfo_2_2_2.cpus_dedicated);
-		len += sprintf (page+len, "LPAR CPUs Shared:     %d\n",
-				info->sysinfo_2_2_2.cpus_shared);
-	}
+	if (level >= 2)
+		len = stsi_2_2_2((struct sysinfo_2_2_2 *) info, page, len);
 
-	if (level >= 3 && stsi_3_2_2 (&info->sysinfo_3_2_2) == 0)
-	{
-		for (i = 0; i < info->sysinfo_3_2_2.count; i++)
-		{
-			len += sprintf (page+len, "\n");
-			len += sprintf (page+len, "VM%02d Name:            %-8.8s\n",
-					i, info->sysinfo_3_2_2.vm[i].name);
-			len += sprintf (page+len, "VM%02d Control Program: %-16.16s\n",
-					i, info->sysinfo_3_2_2.vm[i].cpi);
-	
-			len += sprintf (page+len, "VM%02d Adjustment:      %d\n",
-					i, info->sysinfo_3_2_2.vm[i].caf);
-	
-			len += sprintf (page+len, "VM%02d CPUs Total:      %d\n",
-					i, info->sysinfo_3_2_2.vm[i].cpus_total);
-			len += sprintf (page+len, "VM%02d CPUs Configured: %d\n",
-					i, info->sysinfo_3_2_2.vm[i].cpus_configured);
-			len += sprintf (page+len, "VM%02d CPUs Standby:    %d\n",
-					i, info->sysinfo_3_2_2.vm[i].cpus_standby);
-			len += sprintf (page+len, "VM%02d CPUs Reserved:   %d\n",
-					i, info->sysinfo_3_2_2.vm[i].cpus_reserved);
-		}
-	}
+	if (level >= 3)
+		len = stsi_3_2_2((struct sysinfo_3_2_2 *) info, page, len);
 
-	free_page (info_page);
+	free_page (info);
         return len;
 }
 
 static __init int create_proc_sysinfo(void)
 {
-	create_proc_read_entry ("sysinfo", 0444, NULL, 
-				proc_read_sysinfo, NULL);
+	create_proc_read_entry("sysinfo", 0444, NULL,
+			       proc_read_sysinfo, NULL);
 	return 0;
 }
 
diff --git a/drivers/scsi/BusLogic.c b/drivers/scsi/BusLogic.c
index 16a12a3..4ea49fd 100644
--- a/drivers/scsi/BusLogic.c
+++ b/drivers/scsi/BusLogic.c
@@ -662,7 +662,7 @@ static int __init BusLogic_InitializeMultiMasterProbeInfo(struct BusLogic_HostAd
 	   particular standard ISA I/O Address need not be probed.
 	 */
 	PrimaryProbeInfo->IO_Address = 0;
-	while ((PCI_Device = pci_find_device(PCI_VENDOR_ID_BUSLOGIC, PCI_DEVICE_ID_BUSLOGIC_MULTIMASTER, PCI_Device)) != NULL) {
+	while ((PCI_Device = pci_get_device(PCI_VENDOR_ID_BUSLOGIC, PCI_DEVICE_ID_BUSLOGIC_MULTIMASTER, PCI_Device)) != NULL) {
 		struct BusLogic_HostAdapter *HostAdapter = PrototypeHostAdapter;
 		struct BusLogic_PCIHostAdapterInformation PCIHostAdapterInformation;
 		enum BusLogic_ISACompatibleIOPort ModifyIOAddressRequest;
@@ -762,7 +762,7 @@ static int __init BusLogic_InitializeMultiMasterProbeInfo(struct BusLogic_HostAd
 			PrimaryProbeInfo->Bus = Bus;
 			PrimaryProbeInfo->Device = Device;
 			PrimaryProbeInfo->IRQ_Channel = IRQ_Channel;
-			PrimaryProbeInfo->PCI_Device = PCI_Device;
+			PrimaryProbeInfo->PCI_Device = pci_dev_get(PCI_Device);
 			PCIMultiMasterCount++;
 		} else if (BusLogic_ProbeInfoCount < BusLogic_MaxHostAdapters) {
 			struct BusLogic_ProbeInfo *ProbeInfo = &BusLogic_ProbeInfoList[BusLogic_ProbeInfoCount++];
@@ -773,7 +773,7 @@ static int __init BusLogic_InitializeMultiMasterProbeInfo(struct BusLogic_HostAd
 			ProbeInfo->Bus = Bus;
 			ProbeInfo->Device = Device;
 			ProbeInfo->IRQ_Channel = IRQ_Channel;
-			ProbeInfo->PCI_Device = PCI_Device;
+			ProbeInfo->PCI_Device = pci_dev_get(PCI_Device);
 			NonPrimaryPCIMultiMasterCount++;
 			PCIMultiMasterCount++;
 		} else
@@ -823,7 +823,7 @@ static int __init BusLogic_InitializeMultiMasterProbeInfo(struct BusLogic_HostAd
 	   noting the PCI bus location and assigned IRQ Channel.
 	 */
 	PCI_Device = NULL;
-	while ((PCI_Device = pci_find_device(PCI_VENDOR_ID_BUSLOGIC, PCI_DEVICE_ID_BUSLOGIC_MULTIMASTER_NC, PCI_Device)) != NULL) {
+	while ((PCI_Device = pci_get_device(PCI_VENDOR_ID_BUSLOGIC, PCI_DEVICE_ID_BUSLOGIC_MULTIMASTER_NC, PCI_Device)) != NULL) {
 		unsigned char Bus;
 		unsigned char Device;
 		unsigned int IRQ_Channel;
@@ -850,7 +850,7 @@ static int __init BusLogic_InitializeMultiMasterProbeInfo(struct BusLogic_HostAd
 				ProbeInfo->Bus = Bus;
 				ProbeInfo->Device = Device;
 				ProbeInfo->IRQ_Channel = IRQ_Channel;
-				ProbeInfo->PCI_Device = PCI_Device;
+				ProbeInfo->PCI_Device = pci_dev_get(PCI_Device);
 				break;
 			}
 		}
@@ -874,7 +874,7 @@ static int __init BusLogic_InitializeFlashPointProbeInfo(struct BusLogic_HostAda
 	/*
 	   Interrogate PCI Configuration Space for any FlashPoint Host Adapters.
 	 */
-	while ((PCI_Device = pci_find_device(PCI_VENDOR_ID_BUSLOGIC, PCI_DEVICE_ID_BUSLOGIC_FLASHPOINT, PCI_Device)) != NULL) {
+	while ((PCI_Device = pci_get_device(PCI_VENDOR_ID_BUSLOGIC, PCI_DEVICE_ID_BUSLOGIC_FLASHPOINT, PCI_Device)) != NULL) {
 		unsigned char Bus;
 		unsigned char Device;
 		unsigned int IRQ_Channel;
@@ -923,7 +923,7 @@ static int __init BusLogic_InitializeFlashPointProbeInfo(struct BusLogic_HostAda
 			ProbeInfo->Bus = Bus;
 			ProbeInfo->Device = Device;
 			ProbeInfo->IRQ_Channel = IRQ_Channel;
-			ProbeInfo->PCI_Device = PCI_Device;
+			ProbeInfo->PCI_Device = pci_dev_get(PCI_Device);
 			FlashPointCount++;
 		} else
 			BusLogic_Warning("BusLogic: Too many Host Adapters " "detected\n", NULL);
@@ -1890,6 +1890,7 @@ static void BusLogic_ReleaseResources(struct BusLogic_HostAdapter *HostAdapter)
 	 */
 	if (HostAdapter->MailboxSpace)
 		pci_free_consistent(HostAdapter->PCI_Device, HostAdapter->MailboxSize, HostAdapter->MailboxSpace, HostAdapter->MailboxSpaceHandle);
+	pci_dev_put(HostAdapter->PCI_Device);
 	HostAdapter->MailboxSpace = NULL;
 	HostAdapter->MailboxSpaceHandle = 0;
 	HostAdapter->MailboxSize = 0;
@@ -2176,6 +2177,7 @@ static int __init BusLogic_init(void)
 {
 	int BusLogicHostAdapterCount = 0, DriverOptionsIndex = 0, ProbeIndex;
 	struct BusLogic_HostAdapter *PrototypeHostAdapter;
+	int ret = 0;
 
 #ifdef MODULE
 	if (BusLogic)
@@ -2282,25 +2284,49 @@ static int __init BusLogic_init(void)
 		   perform Target Device Inquiry.
 		 */
 		if (BusLogic_ReadHostAdapterConfiguration(HostAdapter) &&
-		    BusLogic_ReportHostAdapterConfiguration(HostAdapter) && BusLogic_AcquireResources(HostAdapter) && BusLogic_CreateInitialCCBs(HostAdapter) && BusLogic_InitializeHostAdapter(HostAdapter) && BusLogic_TargetDeviceInquiry(HostAdapter)) {
+		    BusLogic_ReportHostAdapterConfiguration(HostAdapter) &&
+		    BusLogic_AcquireResources(HostAdapter) &&
+		    BusLogic_CreateInitialCCBs(HostAdapter) &&
+		    BusLogic_InitializeHostAdapter(HostAdapter) &&
+		    BusLogic_TargetDeviceInquiry(HostAdapter)) {
 			/*
 			   Initialization has been completed successfully.  Release and
 			   re-register usage of the I/O Address range so that the Model
 			   Name of the Host Adapter will appear, and initialize the SCSI
 			   Host structure.
 			 */
-			release_region(HostAdapter->IO_Address, HostAdapter->AddressCount);
-			if (!request_region(HostAdapter->IO_Address, HostAdapter->AddressCount, HostAdapter->FullModelName)) {
-				printk(KERN_WARNING "BusLogic: Release and re-register of " "port 0x%04lx failed \n", (unsigned long) HostAdapter->IO_Address);
+			release_region(HostAdapter->IO_Address,
+				       HostAdapter->AddressCount);
+			if (!request_region(HostAdapter->IO_Address,
+					    HostAdapter->AddressCount,
+					    HostAdapter->FullModelName)) {
+				printk(KERN_WARNING
+					"BusLogic: Release and re-register of "
+					"port 0x%04lx failed \n",
+					(unsigned long)HostAdapter->IO_Address);
 				BusLogic_DestroyCCBs(HostAdapter);
 				BusLogic_ReleaseResources(HostAdapter);
 				list_del(&HostAdapter->host_list);
 				scsi_host_put(Host);
+				ret = -ENOMEM;
 			} else {
-				BusLogic_InitializeHostStructure(HostAdapter, Host);
-				scsi_add_host(Host, HostAdapter->PCI_Device ? &HostAdapter->PCI_Device->dev : NULL);
-				scsi_scan_host(Host);
-				BusLogicHostAdapterCount++;
+				BusLogic_InitializeHostStructure(HostAdapter,
+								 Host);
+				if (scsi_add_host(Host, HostAdapter->PCI_Device
+						? &HostAdapter->PCI_Device->dev
+						  : NULL)) {
+					printk(KERN_WARNING
+					       "BusLogic: scsi_add_host()"
+					       "failed!\n");
+					BusLogic_DestroyCCBs(HostAdapter);
+					BusLogic_ReleaseResources(HostAdapter);
+					list_del(&HostAdapter->host_list);
+					scsi_host_put(Host);
+					ret = -ENODEV;
+				} else {
+					scsi_scan_host(Host);
+					BusLogicHostAdapterCount++;
+				}
 			}
 		} else {
 			/*
@@ -2315,12 +2341,13 @@ static int __init BusLogic_init(void)
 			BusLogic_ReleaseResources(HostAdapter);
 			list_del(&HostAdapter->host_list);
 			scsi_host_put(Host);
+			ret = -ENODEV;
 		}
 	}
 	kfree(PrototypeHostAdapter);
 	kfree(BusLogic_ProbeInfoList);
 	BusLogic_ProbeInfoList = NULL;
-	return 0;
+	return ret;
 }
 
 
@@ -2954,6 +2981,7 @@ static int BusLogic_QueueCommand(struct scsi_cmnd *Command, void (*CompletionRou
 }
 
 
+#if 0
 /*
   BusLogic_AbortCommand aborts Command if possible.
 */
@@ -3024,6 +3052,7 @@ static int BusLogic_AbortCommand(struct scsi_cmnd *Command)
 	return SUCCESS;
 }
 
+#endif
 /*
   BusLogic_ResetHostAdapter resets Host Adapter if possible, marking all
   currently executing SCSI Commands as having been Reset.
diff --git a/drivers/scsi/Kconfig b/drivers/scsi/Kconfig
index 2df4d15..c4dfcc9 100644
--- a/drivers/scsi/Kconfig
+++ b/drivers/scsi/Kconfig
@@ -27,6 +27,11 @@ config SCSI
 	  However, do not compile this as a module if your root file system
 	  (the one containing the directory /) is located on a SCSI device.
 
+config SCSI_NETLINK
+	bool
+	default	n
+	select NET
+
 config SCSI_PROC_FS
 	bool "legacy /proc/scsi/ support"
 	depends on SCSI && PROC_FS
@@ -209,7 +214,7 @@ config SCSI_LOGGING
 	  there should be no noticeable performance impact as long as you have
 	  logging turned off.
 
-menu "SCSI Transport Attributes"
+menu "SCSI Transports"
 	depends on SCSI
 
 config SCSI_SPI_ATTRS
@@ -222,6 +227,7 @@ config SCSI_SPI_ATTRS
 config SCSI_FC_ATTRS
 	tristate "FiberChannel Transport Attributes"
 	depends on SCSI
+	select SCSI_NETLINK
 	help
 	  If you wish to export transport-specific information about
 	  each attached FiberChannel device to sysfs, say Y.
@@ -242,6 +248,8 @@ config SCSI_SAS_ATTRS
 	  If you wish to export transport-specific information about
 	  each attached SAS device to sysfs, say Y.
 
+source "drivers/scsi/libsas/Kconfig"
+
 endmenu
 
 menu "SCSI low-level drivers"
@@ -431,6 +439,7 @@ config SCSI_AIC7XXX_OLD
 	  module will be called aic7xxx_old.
 
 source "drivers/scsi/aic7xxx/Kconfig.aic79xx"
+source "drivers/scsi/aic94xx/Kconfig"
 
 # All the I2O code and drivers do not seem to be 64bit safe.
 config SCSI_DPT_I2O
@@ -469,6 +478,20 @@ config SCSI_IN2000
 	  To compile this driver as a module, choose M here: the
 	  module will be called in2000.
 
+config SCSI_ARCMSR
+	tristate "ARECA ARC11X0[PCI-X]/ARC12X0[PCI-EXPRESS] SATA-RAID support"
+	depends on PCI && SCSI
+	help
+	  This driver supports all of ARECA's SATA RAID controller cards.
+	  This is an ARECA-maintained driver by Erich Chen.
+	  If you have any problems, please mail to: < erich@areca.com.tw >
+	  Areca supports Linux RAID config tools.
+
+	  < http://www.areca.com.tw >
+
+	  To compile this driver as a module, choose M here: the
+	  module will be called arcmsr (modprobe arcmsr).
+
 source "drivers/scsi/megaraid/Kconfig.megaraid"
 
 config SCSI_HPTIOP
@@ -915,6 +938,13 @@ config 53C700_LE_ON_BE
 	depends on SCSI_LASI700
 	default y
 
+config SCSI_STEX
+	tristate "Promise SuperTrak EX Series support"
+	depends on PCI && SCSI
+	---help---
+	  This driver supports Promise SuperTrak EX8350/8300/16350/16300
+	  Storage controllers.
+
 config SCSI_SYM53C8XX_2
 	tristate "SYM53C8XX Version 2 SCSI support"
 	depends on PCI && SCSI
diff --git a/drivers/scsi/Makefile b/drivers/scsi/Makefile
index b678f95..1ef951b 100644
--- a/drivers/scsi/Makefile
+++ b/drivers/scsi/Makefile
@@ -32,6 +32,7 @@ obj-$(CONFIG_SCSI_SPI_ATTRS)	+= scsi_transport_spi.o
 obj-$(CONFIG_SCSI_FC_ATTRS) 	+= scsi_transport_fc.o
 obj-$(CONFIG_SCSI_ISCSI_ATTRS)	+= scsi_transport_iscsi.o
 obj-$(CONFIG_SCSI_SAS_ATTRS)	+= scsi_transport_sas.o
+obj-$(CONFIG_SCSI_SAS_LIBSAS)	+= libsas/
 
 obj-$(CONFIG_ISCSI_TCP) 	+= libiscsi.o	iscsi_tcp.o
 obj-$(CONFIG_INFINIBAND_ISER) 	+= libiscsi.o
@@ -59,6 +60,7 @@ obj-$(CONFIG_SCSI_PSI240I)	+= psi240i.o
 obj-$(CONFIG_SCSI_BUSLOGIC)	+= BusLogic.o
 obj-$(CONFIG_SCSI_DPT_I2O)	+= dpt_i2o.o
 obj-$(CONFIG_SCSI_U14_34F)	+= u14-34f.o
+obj-$(CONFIG_SCSI_ARCMSR)	+= arcmsr/
 obj-$(CONFIG_SCSI_ULTRASTOR)	+= ultrastor.o
 obj-$(CONFIG_SCSI_AHA152X)	+= aha152x.o
 obj-$(CONFIG_SCSI_AHA1542)	+= aha1542.o
@@ -67,6 +69,7 @@ obj-$(CONFIG_SCSI_AIC7XXX)	+= aic7xxx/
 obj-$(CONFIG_SCSI_AIC79XX)	+= aic7xxx/
 obj-$(CONFIG_SCSI_AACRAID)	+= aacraid/
 obj-$(CONFIG_SCSI_AIC7XXX_OLD)	+= aic7xxx_old.o
+obj-$(CONFIG_SCSI_AIC94XX)	+= aic94xx/
 obj-$(CONFIG_SCSI_IPS)		+= ips.o
 obj-$(CONFIG_SCSI_FD_MCS)	+= fd_mcs.o
 obj-$(CONFIG_SCSI_FUTURE_DOMAIN)+= fdomain.o
@@ -123,6 +126,7 @@ obj-$(CONFIG_SCSI_NSP32)	+= nsp32.o
 obj-$(CONFIG_SCSI_IPR)		+= ipr.o
 obj-$(CONFIG_SCSI_IBMVSCSI)	+= ibmvscsi/
 obj-$(CONFIG_SCSI_HPTIOP)	+= hptiop.o
+obj-$(CONFIG_SCSI_STEX)		+= stex.o
 
 obj-$(CONFIG_ARM)		+= arm/
 
@@ -140,6 +144,7 @@ scsi_mod-y			+= scsi.o hosts.o scsi_ioctl.o constants.o \
 				   scsicam.o scsi_error.o scsi_lib.o \
 				   scsi_scan.o scsi_sysfs.o \
 				   scsi_devinfo.o
+scsi_mod-$(CONFIG_SCSI_NETLINK)	+= scsi_netlink.o
 scsi_mod-$(CONFIG_SYSCTL)	+= scsi_sysctl.o
 scsi_mod-$(CONFIG_SCSI_PROC_FS)	+= scsi_proc.o
 
diff --git a/drivers/scsi/a2091.c b/drivers/scsi/a2091.c
index fddfa2e..0854069 100644
--- a/drivers/scsi/a2091.c
+++ b/drivers/scsi/a2091.c
@@ -40,7 +40,7 @@ static irqreturn_t a2091_intr (int irq, void *_instance, struct pt_regs *fp)
     return IRQ_HANDLED;
 }
 
-static int dma_setup (Scsi_Cmnd *cmd, int dir_in)
+static int dma_setup(struct scsi_cmnd *cmd, int dir_in)
 {
     unsigned short cntr = CNTR_PDMD | CNTR_INTEN;
     unsigned long addr = virt_to_bus(cmd->SCp.ptr);
@@ -115,7 +115,7 @@ static int dma_setup (Scsi_Cmnd *cmd, int dir_in)
     return 0;
 }
 
-static void dma_stop (struct Scsi_Host *instance, Scsi_Cmnd *SCpnt, 
+static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 		      int status)
 {
     /* disable SCSI interrupts */
@@ -217,7 +217,7 @@ int __init a2091_detect(struct scsi_host_template *tpnt)
     return num_a2091;
 }
 
-static int a2091_bus_reset(Scsi_Cmnd *cmd)
+static int a2091_bus_reset(struct scsi_cmnd *cmd)
 {
 	/* FIXME perform bus-specific reset */
 
diff --git a/drivers/scsi/a2091.h b/drivers/scsi/a2091.h
index 22d6a13..fe809bc 100644
--- a/drivers/scsi/a2091.h
+++ b/drivers/scsi/a2091.h
@@ -13,10 +13,6 @@
 
 int a2091_detect(struct scsi_host_template *);
 int a2091_release(struct Scsi_Host *);
-const char *wd33c93_info(void);
-int wd33c93_queuecommand(Scsi_Cmnd *, void (*done)(Scsi_Cmnd *));
-int wd33c93_abort(Scsi_Cmnd *);
-int wd33c93_reset(Scsi_Cmnd *, unsigned int);
 
 #ifndef CMD_PER_LUN
 #define CMD_PER_LUN 2
diff --git a/drivers/scsi/a3000.c b/drivers/scsi/a3000.c
index ae9ab4b..7bf46d4 100644
--- a/drivers/scsi/a3000.c
+++ b/drivers/scsi/a3000.c
@@ -44,7 +44,7 @@ static irqreturn_t a3000_intr (int irq, void *dummy, struct pt_regs *fp)
 	return IRQ_NONE;
 }
 
-static int dma_setup (Scsi_Cmnd *cmd, int dir_in)
+static int dma_setup(struct scsi_cmnd *cmd, int dir_in)
 {
     unsigned short cntr = CNTR_PDMD | CNTR_INTEN;
     unsigned long addr = virt_to_bus(cmd->SCp.ptr);
@@ -110,8 +110,8 @@ static int dma_setup (Scsi_Cmnd *cmd, int dir_in)
     return 0;
 }
 
-static void dma_stop (struct Scsi_Host *instance, Scsi_Cmnd *SCpnt,
-		      int status)
+static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
+		     int status)
 {
     /* disable SCSI interrupts */
     unsigned short cntr = CNTR_PDMD;
@@ -205,7 +205,7 @@ fail_register:
     return 0;
 }
 
-static int a3000_bus_reset(Scsi_Cmnd *cmd)
+static int a3000_bus_reset(struct scsi_cmnd *cmd)
 {
 	/* FIXME perform bus-specific reset */
 	
diff --git a/drivers/scsi/a3000.h b/drivers/scsi/a3000.h
index 5535a65..44a4ec7 100644
--- a/drivers/scsi/a3000.h
+++ b/drivers/scsi/a3000.h
@@ -13,10 +13,6 @@
 
 int a3000_detect(struct scsi_host_template *);
 int a3000_release(struct Scsi_Host *);
-const char *wd33c93_info(void);
-int wd33c93_queuecommand(Scsi_Cmnd *, void (*done)(Scsi_Cmnd *));
-int wd33c93_abort(Scsi_Cmnd *);
-int wd33c93_reset(Scsi_Cmnd *, unsigned int);
 
 #ifndef CMD_PER_LUN
 #define CMD_PER_LUN 2
diff --git a/drivers/scsi/aacraid/aachba.c b/drivers/scsi/aacraid/aachba.c
index 83b5c7d..ac108f9 100644
--- a/drivers/scsi/aacraid/aachba.c
+++ b/drivers/scsi/aacraid/aachba.c
@@ -169,13 +169,17 @@ MODULE_PARM_DESC(numacb, "Request a limit to the number of adapter control block
 int acbsize = -1;
 module_param(acbsize, int, S_IRUGO|S_IWUSR);
 MODULE_PARM_DESC(acbsize, "Request a specific adapter control block (FIB) size. Valid values are 512, 2048, 4096 and 8192. Default is to use suggestion from Firmware.");
+
+int expose_physicals = 0;
+module_param(expose_physicals, int, S_IRUGO|S_IWUSR);
+MODULE_PARM_DESC(expose_physicals, "Expose physical components of the arrays. 0=off, 1=on");
 /**
  *	aac_get_config_status	-	check the adapter configuration
  *	@common: adapter to query
  *
  *	Query config status, and commit the configuration if needed.
  */
-int aac_get_config_status(struct aac_dev *dev)
+int aac_get_config_status(struct aac_dev *dev, int commit_flag)
 {
 	int status = 0;
 	struct fib * fibptr;
@@ -219,7 +223,7 @@ int aac_get_config_status(struct aac_dev *dev)
 	aac_fib_complete(fibptr);
 	/* Send a CT_COMMIT_CONFIG to enable discovery of devices */
 	if (status >= 0) {
-		if (commit == 1) {
+		if ((commit == 1) || commit_flag) {
 			struct aac_commit_config * dinfo;
 			aac_fib_init(fibptr);
 			dinfo = (struct aac_commit_config *) fib_data(fibptr);
@@ -489,6 +493,8 @@ int aac_probe_container(struct aac_dev *dev, int cid)
 	unsigned instance;
 
 	fsa_dev_ptr = dev->fsa_dev;
+	if (!fsa_dev_ptr)
+		return -ENOMEM;
 	instance = dev->scsi_host_ptr->unique_id;
 
 	if (!(fibptr = aac_fib_alloc(dev)))
@@ -782,8 +788,9 @@ int aac_get_adapter_info(struct aac_dev* dev)
 		dev->maximum_num_channels = le32_to_cpu(bus_info->BusCount);
 	}
 
-	tmp = le32_to_cpu(dev->adapter_info.kernelrev);
-	printk(KERN_INFO "%s%d: kernel %d.%d-%d[%d] %.*s\n", 
+	if (!dev->in_reset) {
+		tmp = le32_to_cpu(dev->adapter_info.kernelrev);
+		printk(KERN_INFO "%s%d: kernel %d.%d-%d[%d] %.*s\n",
 			dev->name, 
 			dev->id,
 			tmp>>24,
@@ -792,20 +799,21 @@ int aac_get_adapter_info(struct aac_dev* dev)
 			le32_to_cpu(dev->adapter_info.kernelbuild),
 			(int)sizeof(dev->supplement_adapter_info.BuildDate),
 			dev->supplement_adapter_info.BuildDate);
-	tmp = le32_to_cpu(dev->adapter_info.monitorrev);
-	printk(KERN_INFO "%s%d: monitor %d.%d-%d[%d]\n", 
+		tmp = le32_to_cpu(dev->adapter_info.monitorrev);
+		printk(KERN_INFO "%s%d: monitor %d.%d-%d[%d]\n",
 			dev->name, dev->id,
 			tmp>>24,(tmp>>16)&0xff,tmp&0xff,
 			le32_to_cpu(dev->adapter_info.monitorbuild));
-	tmp = le32_to_cpu(dev->adapter_info.biosrev);
-	printk(KERN_INFO "%s%d: bios %d.%d-%d[%d]\n", 
+		tmp = le32_to_cpu(dev->adapter_info.biosrev);
+		printk(KERN_INFO "%s%d: bios %d.%d-%d[%d]\n",
 			dev->name, dev->id,
 			tmp>>24,(tmp>>16)&0xff,tmp&0xff,
 			le32_to_cpu(dev->adapter_info.biosbuild));
-	if (le32_to_cpu(dev->adapter_info.serial[0]) != 0xBAD0)
-		printk(KERN_INFO "%s%d: serial %x\n",
-			dev->name, dev->id,
-			le32_to_cpu(dev->adapter_info.serial[0]));
+		if (le32_to_cpu(dev->adapter_info.serial[0]) != 0xBAD0)
+			printk(KERN_INFO "%s%d: serial %x\n",
+				dev->name, dev->id,
+				le32_to_cpu(dev->adapter_info.serial[0]));
+	}
 
 	dev->nondasd_support = 0;
 	dev->raid_scsi_mode = 0;
@@ -1392,6 +1400,7 @@ static int aac_synchronize(struct scsi_cmnd *scsicmd, int cid)
 	struct scsi_cmnd *cmd;
 	struct scsi_device *sdev = scsicmd->device;
 	int active = 0;
+	struct aac_dev *aac;
 	unsigned long flags;
 
 	/*
@@ -1413,11 +1422,14 @@ static int aac_synchronize(struct scsi_cmnd *scsicmd, int cid)
 	if (active)
 		return SCSI_MLQUEUE_DEVICE_BUSY;
 
+	aac = (struct aac_dev *)scsicmd->device->host->hostdata;
+	if (aac->in_reset)
+		return SCSI_MLQUEUE_HOST_BUSY;
+
 	/*
 	 *	Allocate and initialize a Fib
 	 */
-	if (!(cmd_fibcontext = 
-	    aac_fib_alloc((struct aac_dev *)scsicmd->device->host->hostdata)))
+	if (!(cmd_fibcontext = aac_fib_alloc(aac)))
 		return SCSI_MLQUEUE_HOST_BUSY;
 
 	aac_fib_init(cmd_fibcontext);
@@ -1470,6 +1482,8 @@ int aac_scsi_cmd(struct scsi_cmnd * scsicmd)
 	struct aac_dev *dev = (struct aac_dev *)host->hostdata;
 	struct fsa_dev_info *fsa_dev_ptr = dev->fsa_dev;
 	
+	if (fsa_dev_ptr == NULL)
+		return -1;
 	/*
 	 *	If the bus, id or lun is out of range, return fail
 	 *	Test does not apply to ID 16, the pseudo id for the controller
@@ -1499,6 +1513,8 @@ int aac_scsi_cmd(struct scsi_cmnd * scsicmd)
 				case INQUIRY:
 				case READ_CAPACITY:
 				case TEST_UNIT_READY:
+					if (dev->in_reset)
+						return -1;
 					spin_unlock_irq(host->host_lock);
 					aac_probe_container(dev, cid);
 					if ((fsa_dev_ptr[cid].valid & 1) == 0)
@@ -1523,7 +1539,9 @@ int aac_scsi_cmd(struct scsi_cmnd * scsicmd)
 				return 0;
 			}
 		} else {  /* check for physical non-dasd devices */
-			if(dev->nondasd_support == 1){
+			if ((dev->nondasd_support == 1) || expose_physicals) {
+				if (dev->in_reset)
+					return -1;
 				return aac_send_srb_fib(scsicmd);
 			} else {
 				scsicmd->result = DID_NO_CONNECT << 16;
@@ -1579,6 +1597,8 @@ int aac_scsi_cmd(struct scsi_cmnd * scsicmd)
 			scsicmd->scsi_done(scsicmd);
 			return 0;
 		}
+		if (dev->in_reset)
+			return -1;
 		setinqstr(dev, (void *) (inq_data.inqd_vid), fsa_dev_ptr[cid].type);
 		inq_data.inqd_pdt = INQD_PDT_DA;	/* Direct/random access device */
 		aac_internal_transfer(scsicmd, &inq_data, 0, sizeof(inq_data));
@@ -1734,6 +1754,8 @@ int aac_scsi_cmd(struct scsi_cmnd * scsicmd)
 		case READ_10:
 		case READ_12:
 		case READ_16:
+			if (dev->in_reset)
+				return -1;
 			/*
 			 *	Hack to keep track of ordinal number of the device that
 			 *	corresponds to a container. Needed to convert
@@ -1752,6 +1774,8 @@ int aac_scsi_cmd(struct scsi_cmnd * scsicmd)
 		case WRITE_10:
 		case WRITE_12:
 		case WRITE_16:
+			if (dev->in_reset)
+				return -1;
 			return aac_write(scsicmd, cid);
 
 		case SYNCHRONIZE_CACHE:
@@ -1782,6 +1806,8 @@ static int query_disk(struct aac_dev *dev, void __user *arg)
 	struct fsa_dev_info *fsa_dev_ptr;
 
 	fsa_dev_ptr = dev->fsa_dev;
+	if (!fsa_dev_ptr)
+		return -EBUSY;
 	if (copy_from_user(&qd, arg, sizeof (struct aac_query_disk)))
 		return -EFAULT;
 	if (qd.cnum == -1)
@@ -1820,6 +1846,8 @@ static int force_delete_disk(struct aac_dev *dev, void __user *arg)
 	struct fsa_dev_info *fsa_dev_ptr;
 
 	fsa_dev_ptr = dev->fsa_dev;
+	if (!fsa_dev_ptr)
+		return -EBUSY;
 
 	if (copy_from_user(&dd, arg, sizeof (struct aac_delete_disk)))
 		return -EFAULT;
@@ -1843,6 +1871,8 @@ static int delete_disk(struct aac_dev *dev, void __user *arg)
 	struct fsa_dev_info *fsa_dev_ptr;
 
 	fsa_dev_ptr = dev->fsa_dev;
+	if (!fsa_dev_ptr)
+		return -EBUSY;
 
 	if (copy_from_user(&dd, arg, sizeof (struct aac_delete_disk)))
 		return -EFAULT;
diff --git a/drivers/scsi/aacraid/aacraid.h b/drivers/scsi/aacraid/aacraid.h
index d0eecd4..eb3ed91 100644
--- a/drivers/scsi/aacraid/aacraid.h
+++ b/drivers/scsi/aacraid/aacraid.h
@@ -494,6 +494,7 @@ struct adapter_ops
 	int  (*adapter_sync_cmd)(struct aac_dev *dev, u32 command, u32 p1, u32 p2, u32 p3, u32 p4, u32 p5, u32 p6, u32 *status, u32 *r1, u32 *r2, u32 *r3, u32 *r4);
 	int  (*adapter_check_health)(struct aac_dev *dev);
 	int  (*adapter_send)(struct fib * fib);
+	int  (*adapter_ioremap)(struct aac_dev * dev, u32 size);
 };
 
 /*
@@ -682,14 +683,6 @@ struct rx_inbound {
 	__le32	Mailbox[8];
 };
 
-#define	InboundMailbox0		IndexRegs.Mailbox[0]
-#define	InboundMailbox1		IndexRegs.Mailbox[1]
-#define	InboundMailbox2		IndexRegs.Mailbox[2]
-#define	InboundMailbox3		IndexRegs.Mailbox[3]
-#define	InboundMailbox4		IndexRegs.Mailbox[4]
-#define	InboundMailbox5		IndexRegs.Mailbox[5]
-#define	InboundMailbox6		IndexRegs.Mailbox[6]
-
 #define	INBOUNDDOORBELL_0	0x00000001
 #define INBOUNDDOORBELL_1	0x00000002
 #define INBOUNDDOORBELL_2	0x00000004
@@ -1010,6 +1003,8 @@ struct aac_dev
 		struct rx_registers __iomem *rx;
 		struct rkt_registers __iomem *rkt;
 	} regs;
+	volatile void __iomem *base;
+	volatile struct rx_inbound __iomem *IndexRegs;
 	u32			OIMR; /* Mask Register Cache */
 	/*
 	 *	AIF thread states
@@ -1029,6 +1024,7 @@ struct aac_dev
 	  init->InitStructRevision==cpu_to_le32(ADAPTER_INIT_STRUCT_REVISION_4)
 	u8			raw_io_64;
 	u8			printf_enabled;
+	u8			in_reset;
 };
 
 #define aac_adapter_interrupt(dev) \
@@ -1049,6 +1045,9 @@ struct aac_dev
 #define aac_adapter_send(fib) \
 	((fib)->dev)->a_ops.adapter_send(fib)
 
+#define aac_adapter_ioremap(dev, size) \
+	(dev)->a_ops.adapter_ioremap(dev, size)
+
 #define FIB_CONTEXT_FLAG_TIMED_OUT		(0x00000001)
 
 /*
@@ -1524,7 +1523,6 @@ struct aac_get_name {
 	__le32		count;	/* sizeof(((struct aac_get_name_resp *)NULL)->data) */
 };
 
-#define CT_OK        218
 struct aac_get_name_resp {
 	__le32		dummy0;
 	__le32		dummy1;
@@ -1670,6 +1668,7 @@ extern struct aac_common aac_config;
 #define RCV_TEMP_READINGS		0x00000025
 #define GET_COMM_PREFERRED_SETTINGS	0x00000026
 #define IOP_RESET			0x00001000
+#define IOP_RESET_ALWAYS		0x00001001
 #define RE_INIT_ADAPTER			0x000000ee
 
 /*
@@ -1788,7 +1787,7 @@ void aac_consumer_free(struct aac_dev * dev, struct aac_queue * q, u32 qnum);
 int aac_fib_complete(struct fib * context);
 #define fib_data(fibctx) ((void *)(fibctx)->hw_fib->data)
 struct aac_dev *aac_init_adapter(struct aac_dev *dev);
-int aac_get_config_status(struct aac_dev *dev);
+int aac_get_config_status(struct aac_dev *dev, int commit_flag);
 int aac_get_containers(struct aac_dev *dev);
 int aac_scsi_cmd(struct scsi_cmnd *cmd);
 int aac_dev_ioctl(struct aac_dev *dev, int cmd, void __user *arg);
@@ -1799,6 +1798,7 @@ int aac_sa_init(struct aac_dev *dev);
 unsigned int aac_response_normal(struct aac_queue * q);
 unsigned int aac_command_normal(struct aac_queue * q);
 unsigned int aac_intr_normal(struct aac_dev * dev, u32 Index);
+int aac_check_health(struct aac_dev * dev);
 int aac_command_thread(void *data);
 int aac_close_fib_context(struct aac_dev * dev, struct aac_fib_context *fibctx);
 int aac_fib_adapter_complete(struct fib * fibptr, unsigned short size);
diff --git a/drivers/scsi/aacraid/commctrl.c b/drivers/scsi/aacraid/commctrl.c
index 255421d..da1d3a9 100644
--- a/drivers/scsi/aacraid/commctrl.c
+++ b/drivers/scsi/aacraid/commctrl.c
@@ -38,7 +38,7 @@
 #include <linux/completion.h>
 #include <linux/dma-mapping.h>
 #include <linux/blkdev.h>
-#include <linux/delay.h>
+#include <linux/delay.h> /* ssleep prototype */
 #include <linux/kthread.h>
 #include <asm/semaphore.h>
 #include <asm/uaccess.h>
@@ -140,7 +140,8 @@ cleanup:
 		fibptr->hw_fib_pa = hw_fib_pa;
 		fibptr->hw_fib = hw_fib;
 	}
-	aac_fib_free(fibptr);
+	if (retval != -EINTR)
+		aac_fib_free(fibptr);
 	return retval;
 }
 
@@ -297,7 +298,7 @@ return_fib:
 		spin_unlock_irqrestore(&dev->fib_lock, flags);
 		/* If someone killed the AIF aacraid thread, restart it */
 		status = !dev->aif_thread;
-		if (status && dev->queues && dev->fsa_dev) {
+		if (status && !dev->in_reset && dev->queues && dev->fsa_dev) {
 			/* Be paranoid, be very paranoid! */
 			kthread_stop(dev->thread);
 			ssleep(1);
@@ -621,7 +622,13 @@ static int aac_send_raw_srb(struct aac_dev* dev, void __user * arg)
 
 		actual_fibsize = sizeof (struct aac_srb) + (((user_srbcmd->sg.count & 0xff) - 1) * sizeof (struct sgentry));
 		if(actual_fibsize != fibsize){ // User made a mistake - should not continue
-			dprintk((KERN_DEBUG"aacraid: Bad Size specified in Raw SRB command\n"));
+			dprintk((KERN_DEBUG"aacraid: Bad Size specified in "
+			  "Raw SRB command calculated fibsize=%d "
+			  "user_srbcmd->sg.count=%d aac_srb=%d sgentry=%d "
+			  "issued fibsize=%d\n",
+			  actual_fibsize, user_srbcmd->sg.count,
+			  sizeof(struct aac_srb), sizeof(struct sgentry),
+			  fibsize));
 			rcode = -EINVAL;
 			goto cleanup;
 		}
@@ -663,6 +670,10 @@ static int aac_send_raw_srb(struct aac_dev* dev, void __user * arg)
 		psg->count = cpu_to_le32(sg_indx+1);
 		status = aac_fib_send(ScsiPortCommand, srbfib, actual_fibsize, FsaNormal, 1, 1, NULL, NULL);
 	}
+	if (status == -EINTR) {
+		rcode = -EINTR;
+		goto cleanup;
+	}
 
 	if (status != 0){
 		dprintk((KERN_DEBUG"aacraid: Could not send raw srb fib to hba\n")); 
@@ -696,8 +707,10 @@ cleanup:
 	for(i=0; i <= sg_indx; i++){
 		kfree(sg_list[i]);
 	}
-	aac_fib_complete(srbfib);
-	aac_fib_free(srbfib);
+	if (rcode != -EINTR) {
+		aac_fib_complete(srbfib);
+		aac_fib_free(srbfib);
+	}
 
 	return rcode;
 }
diff --git a/drivers/scsi/aacraid/comminit.c b/drivers/scsi/aacraid/comminit.c
index 1cd3584..d5cf8b9 100644
--- a/drivers/scsi/aacraid/comminit.c
+++ b/drivers/scsi/aacraid/comminit.c
@@ -180,7 +180,7 @@ int aac_send_shutdown(struct aac_dev * dev)
 			  -2 /* Timeout silently */, 1,
 			  NULL, NULL);
 
-	if (status == 0)
+	if (status >= 0)
 		aac_fib_complete(fibctx);
 	aac_fib_free(fibctx);
 	return status;
@@ -307,17 +307,12 @@ struct aac_dev *aac_init_adapter(struct aac_dev *dev)
 		if (status[1] & AAC_OPT_NEW_COMM)
 			dev->new_comm_interface = dev->a_ops.adapter_send != 0;
 		if (dev->new_comm_interface && (status[2] > dev->base_size)) {
-			iounmap(dev->regs.sa);
+			aac_adapter_ioremap(dev, 0);
 			dev->base_size = status[2];
-			dprintk((KERN_DEBUG "ioremap(%lx,%d)\n",
-			  host->base, status[2]));
-			dev->regs.sa = ioremap(host->base, status[2]);
-			if (dev->regs.sa == NULL) {
+			if (aac_adapter_ioremap(dev, status[2])) {
 				/* remap failed, go back ... */
 				dev->new_comm_interface = 0;
-				dev->regs.sa = ioremap(host->base, 
-						AAC_MIN_FOOTPRINT_SIZE);
-				if (dev->regs.sa == NULL) {	
+				if (aac_adapter_ioremap(dev, AAC_MIN_FOOTPRINT_SIZE)) {
 					printk(KERN_WARNING
 					  "aacraid: unable to map adapter.\n");
 					return NULL;
diff --git a/drivers/scsi/aacraid/commsup.c b/drivers/scsi/aacraid/commsup.c
index 3f27419..8734a04 100644
--- a/drivers/scsi/aacraid/commsup.c
+++ b/drivers/scsi/aacraid/commsup.c
@@ -40,8 +40,10 @@
 #include <linux/blkdev.h>
 #include <linux/delay.h>
 #include <linux/kthread.h>
+#include <scsi/scsi.h>
 #include <scsi/scsi_host.h>
 #include <scsi/scsi_device.h>
+#include <scsi/scsi_cmnd.h>
 #include <asm/semaphore.h>
 
 #include "aacraid.h"
@@ -464,6 +466,8 @@ int aac_fib_send(u16 command, struct fib *fibptr, unsigned long size,
 	dprintk((KERN_DEBUG "  hw_fib pa being sent=%lx\n",(ulong)fibptr->hw_fib_pa));
 	dprintk((KERN_DEBUG "  fib being sent=%p\n",fibptr));
 
+	if (!dev->queues)
+		return -EBUSY;
 	q = &dev->queues->queue[AdapNormCmdQueue];
 
 	if(wait)
@@ -527,8 +531,15 @@ int aac_fib_send(u16 command, struct fib *fibptr, unsigned long size,
 				}
 				udelay(5);
 			}
-		} else
-			down(&fibptr->event_wait);
+		} else if (down_interruptible(&fibptr->event_wait)) {
+			spin_lock_irqsave(&fibptr->event_lock, flags);
+			if (fibptr->done == 0) {
+				fibptr->done = 2; /* Tell interrupt we aborted */
+				spin_unlock_irqrestore(&fibptr->event_lock, flags);
+				return -EINTR;
+			}
+			spin_unlock_irqrestore(&fibptr->event_lock, flags);
+		}
 		BUG_ON(fibptr->done == 0);
 			
 		if((fibptr->flags & FIB_CONTEXT_FLAG_TIMED_OUT)){
@@ -795,7 +806,7 @@ static void aac_handle_aif(struct aac_dev * dev, struct fib * fibptr)
 
 	/* Sniff for container changes */
 
-	if (!dev)
+	if (!dev || !dev->fsa_dev)
 		return;
 	container = (u32)-1;
 
@@ -1022,13 +1033,7 @@ static void aac_handle_aif(struct aac_dev * dev, struct fib * fibptr)
 	if (device) {
 		switch (device_config_needed) {
 		case DELETE:
-			scsi_remove_device(device);
-			break;
 		case CHANGE:
-			if (!dev->fsa_dev[container].valid) {
-				scsi_remove_device(device);
-				break;
-			}
 			scsi_rescan_device(&device->sdev_gendev);
 
 		default:
@@ -1045,6 +1050,262 @@ static void aac_handle_aif(struct aac_dev * dev, struct fib * fibptr)
 
 }
 
+static int _aac_reset_adapter(struct aac_dev *aac)
+{
+	int index, quirks;
+	u32 ret;
+	int retval;
+	struct Scsi_Host *host;
+	struct scsi_device *dev;
+	struct scsi_cmnd *command;
+	struct scsi_cmnd *command_list;
+
+	/*
+	 * Assumptions:
+	 *	- host is locked.
+	 *	- in_reset is asserted, so no new i/o is getting to the
+	 *	  card.
+	 *	- The card is dead.
+	 */
+	host = aac->scsi_host_ptr;
+	scsi_block_requests(host);
+	aac_adapter_disable_int(aac);
+	spin_unlock_irq(host->host_lock);
+	kthread_stop(aac->thread);
+
+	/*
+	 *	If a positive health, means in a known DEAD PANIC
+	 * state and the adapter could be reset to `try again'.
+	 */
+	retval = aac_adapter_check_health(aac);
+	if (retval == 0)
+		retval = aac_adapter_sync_cmd(aac, IOP_RESET_ALWAYS,
+		  0, 0, 0, 0, 0, 0, &ret, NULL, NULL, NULL, NULL);
+	if (retval)
+		retval = aac_adapter_sync_cmd(aac, IOP_RESET,
+		  0, 0, 0, 0, 0, 0, &ret, NULL, NULL, NULL, NULL);
+
+	if (retval)
+		goto out;
+	if (ret != 0x00000001) {
+		retval = -ENODEV;
+		goto out;
+	}
+
+	index = aac->cardtype;
+
+	/*
+	 * Re-initialize the adapter, first free resources, then carefully
+	 * apply the initialization sequence to come back again. Only risk
+	 * is a change in Firmware dropping cache, it is assumed the caller
+	 * will ensure that i/o is queisced and the card is flushed in that
+	 * case.
+	 */
+	aac_fib_map_free(aac);
+	aac->hw_fib_va = NULL;
+	aac->hw_fib_pa = 0;
+	pci_free_consistent(aac->pdev, aac->comm_size, aac->comm_addr, aac->comm_phys);
+	aac->comm_addr = NULL;
+	aac->comm_phys = 0;
+	kfree(aac->queues);
+	aac->queues = NULL;
+	free_irq(aac->pdev->irq, aac);
+	kfree(aac->fsa_dev);
+	aac->fsa_dev = NULL;
+	if (aac_get_driver_ident(index)->quirks & AAC_QUIRK_31BIT) {
+		if (((retval = pci_set_dma_mask(aac->pdev, DMA_32BIT_MASK))) ||
+		  ((retval = pci_set_consistent_dma_mask(aac->pdev, DMA_32BIT_MASK))))
+			goto out;
+	} else {
+		if (((retval = pci_set_dma_mask(aac->pdev, 0x7FFFFFFFULL))) ||
+		  ((retval = pci_set_consistent_dma_mask(aac->pdev, 0x7FFFFFFFULL))))
+			goto out;
+	}
+	if ((retval = (*(aac_get_driver_ident(index)->init))(aac)))
+		goto out;
+	if (aac_get_driver_ident(index)->quirks & AAC_QUIRK_31BIT)
+		if ((retval = pci_set_dma_mask(aac->pdev, DMA_32BIT_MASK)))
+			goto out;
+	aac->thread = kthread_run(aac_command_thread, aac, aac->name);
+	if (IS_ERR(aac->thread)) {
+		retval = PTR_ERR(aac->thread);
+		goto out;
+	}
+	(void)aac_get_adapter_info(aac);
+	quirks = aac_get_driver_ident(index)->quirks;
+	if ((quirks & AAC_QUIRK_34SG) && (host->sg_tablesize > 34)) {
+ 		host->sg_tablesize = 34;
+ 		host->max_sectors = (host->sg_tablesize * 8) + 112;
+ 	}
+ 	if ((quirks & AAC_QUIRK_17SG) && (host->sg_tablesize > 17)) {
+ 		host->sg_tablesize = 17;
+ 		host->max_sectors = (host->sg_tablesize * 8) + 112;
+ 	}
+	aac_get_config_status(aac, 1);
+	aac_get_containers(aac);
+	/*
+	 * This is where the assumption that the Adapter is quiesced
+	 * is important.
+	 */
+	command_list = NULL;
+	__shost_for_each_device(dev, host) {
+		unsigned long flags;
+		spin_lock_irqsave(&dev->list_lock, flags);
+		list_for_each_entry(command, &dev->cmd_list, list)
+			if (command->SCp.phase == AAC_OWNER_FIRMWARE) {
+				command->SCp.buffer = (struct scatterlist *)command_list;
+				command_list = command;
+			}
+		spin_unlock_irqrestore(&dev->list_lock, flags);
+	}
+	while ((command = command_list)) {
+		command_list = (struct scsi_cmnd *)command->SCp.buffer;
+		command->SCp.buffer = NULL;
+		command->result = DID_OK << 16
+		  | COMMAND_COMPLETE << 8
+		  | SAM_STAT_TASK_SET_FULL;
+		command->SCp.phase = AAC_OWNER_ERROR_HANDLER;
+		command->scsi_done(command);
+	}
+	retval = 0;
+
+out:
+	aac->in_reset = 0;
+	scsi_unblock_requests(host);
+	spin_lock_irq(host->host_lock);
+	return retval;
+}
+
+int aac_check_health(struct aac_dev * aac)
+{
+	int BlinkLED;
+	unsigned long time_now, flagv = 0;
+	struct list_head * entry;
+	struct Scsi_Host * host;
+
+	/* Extending the scope of fib_lock slightly to protect aac->in_reset */
+	if (spin_trylock_irqsave(&aac->fib_lock, flagv) == 0)
+		return 0;
+
+	if (aac->in_reset || !(BlinkLED = aac_adapter_check_health(aac))) {
+		spin_unlock_irqrestore(&aac->fib_lock, flagv);
+		return 0; /* OK */
+	}
+
+	aac->in_reset = 1;
+
+	/* Fake up an AIF:
+	 *	aac_aifcmd.command = AifCmdEventNotify = 1
+	 *	aac_aifcmd.seqnum = 0xFFFFFFFF
+	 *	aac_aifcmd.data[0] = AifEnExpEvent = 23
+	 *	aac_aifcmd.data[1] = AifExeFirmwarePanic = 3
+	 *	aac.aifcmd.data[2] = AifHighPriority = 3
+	 *	aac.aifcmd.data[3] = BlinkLED
+	 */
+
+	time_now = jiffies/HZ;
+	entry = aac->fib_list.next;
+
+	/*
+	 * For each Context that is on the
+	 * fibctxList, make a copy of the
+	 * fib, and then set the event to wake up the
+	 * thread that is waiting for it.
+	 */
+	while (entry != &aac->fib_list) {
+		/*
+		 * Extract the fibctx
+		 */
+		struct aac_fib_context *fibctx = list_entry(entry, struct aac_fib_context, next);
+		struct hw_fib * hw_fib;
+		struct fib * fib;
+		/*
+		 * Check if the queue is getting
+		 * backlogged
+		 */
+		if (fibctx->count > 20) {
+			/*
+			 * It's *not* jiffies folks,
+			 * but jiffies / HZ, so do not
+			 * panic ...
+			 */
+			u32 time_last = fibctx->jiffies;
+			/*
+			 * Has it been > 2 minutes
+			 * since the last read off
+			 * the queue?
+			 */
+			if ((time_now - time_last) > aif_timeout) {
+				entry = entry->next;
+				aac_close_fib_context(aac, fibctx);
+				continue;
+			}
+		}
+		/*
+		 * Warning: no sleep allowed while
+		 * holding spinlock
+		 */
+		hw_fib = kmalloc(sizeof(struct hw_fib), GFP_ATOMIC);
+		fib = kmalloc(sizeof(struct fib), GFP_ATOMIC);
+		if (fib && hw_fib) {
+			struct aac_aifcmd * aif;
+
+			memset(hw_fib, 0, sizeof(struct hw_fib));
+			memset(fib, 0, sizeof(struct fib));
+			fib->hw_fib = hw_fib;
+			fib->dev = aac;
+			aac_fib_init(fib);
+			fib->type = FSAFS_NTC_FIB_CONTEXT;
+			fib->size = sizeof (struct fib);
+			fib->data = hw_fib->data;
+			aif = (struct aac_aifcmd *)hw_fib->data;
+			aif->command = cpu_to_le32(AifCmdEventNotify);
+		 	aif->seqnum = cpu_to_le32(0xFFFFFFFF);
+		 	aif->data[0] = cpu_to_le32(AifEnExpEvent);
+			aif->data[1] = cpu_to_le32(AifExeFirmwarePanic);
+		 	aif->data[2] = cpu_to_le32(AifHighPriority);
+			aif->data[3] = cpu_to_le32(BlinkLED);
+
+			/*
+			 * Put the FIB onto the
+			 * fibctx's fibs
+			 */
+			list_add_tail(&fib->fiblink, &fibctx->fib_list);
+			fibctx->count++;
+			/*
+			 * Set the event to wake up the
+			 * thread that will waiting.
+			 */
+			up(&fibctx->wait_sem);
+		} else {
+			printk(KERN_WARNING "aifd: didn't allocate NewFib.\n");
+			kfree(fib);
+			kfree(hw_fib);
+		}
+		entry = entry->next;
+	}
+
+	spin_unlock_irqrestore(&aac->fib_lock, flagv);
+
+	if (BlinkLED < 0) {
+		printk(KERN_ERR "%s: Host adapter dead %d\n", aac->name, BlinkLED);
+		goto out;
+	}
+
+	printk(KERN_ERR "%s: Host adapter BLINK LED 0x%x\n", aac->name, BlinkLED);
+
+	host = aac->scsi_host_ptr;
+	spin_lock_irqsave(host->host_lock, flagv);
+	BlinkLED = _aac_reset_adapter(aac);
+	spin_unlock_irqrestore(host->host_lock, flagv);
+	return BlinkLED;
+
+out:
+	aac->in_reset = 0;
+	return BlinkLED;
+}
+
+
 /**
  *	aac_command_thread	-	command processing thread
  *	@dev: Adapter to monitor
diff --git a/drivers/scsi/aacraid/dpcsup.c b/drivers/scsi/aacraid/dpcsup.c
index b2a5c72..8335f07 100644
--- a/drivers/scsi/aacraid/dpcsup.c
+++ b/drivers/scsi/aacraid/dpcsup.c
@@ -124,10 +124,15 @@ unsigned int aac_response_normal(struct aac_queue * q)
 		} else {
 			unsigned long flagv;
 			spin_lock_irqsave(&fib->event_lock, flagv);
-			fib->done = 1;
+			if (!fib->done)
+				fib->done = 1;
 			up(&fib->event_wait);
 			spin_unlock_irqrestore(&fib->event_lock, flagv);
 			FIB_COUNTER_INCREMENT(aac_config.NormalRecved);
+			if (fib->done == 2) {
+				aac_fib_complete(fib);
+				aac_fib_free(fib);
+			}
 		}
 		consumed++;
 		spin_lock_irqsave(q->lock, flags);
@@ -316,7 +321,8 @@ unsigned int aac_intr_normal(struct aac_dev * dev, u32 Index)
 			unsigned long flagv;
 	  		dprintk((KERN_INFO "event_wait up\n"));
 			spin_lock_irqsave(&fib->event_lock, flagv);
-			fib->done = 1;
+			if (!fib->done)
+				fib->done = 1;
 			up(&fib->event_wait);
 			spin_unlock_irqrestore(&fib->event_lock, flagv);
 			FIB_COUNTER_INCREMENT(aac_config.NormalRecved);
diff --git a/drivers/scsi/aacraid/linit.c b/drivers/scsi/aacraid/linit.c
index e42a479..359e7dd 100644
--- a/drivers/scsi/aacraid/linit.c
+++ b/drivers/scsi/aacraid/linit.c
@@ -82,6 +82,8 @@ static LIST_HEAD(aac_devices);
 static int aac_cfg_major = -1;
 char aac_driver_version[] = AAC_DRIVER_FULL_VERSION;
 
+extern int expose_physicals;
+
 /*
  * Because of the way Linux names scsi devices, the order in this table has
  * become important.  Check for on-board Raid first, add-in cards second.
@@ -394,6 +396,7 @@ static int aac_slave_configure(struct scsi_device *sdev)
 		sdev->skip_ms_page_3f = 1;
 	}
 	if ((sdev->type == TYPE_DISK) &&
+			!expose_physicals &&
 			(sdev_channel(sdev) != CONTAINER_CHANNEL)) {
 		struct aac_dev *aac = (struct aac_dev *)sdev->host->hostdata;
 		if (!aac->raid_scsi_mode || (sdev_channel(sdev) != 2))
@@ -454,17 +457,17 @@ static int aac_eh_reset(struct scsi_cmnd* cmd)
 	printk(KERN_ERR "%s: Host adapter reset request. SCSI hang ?\n", 
 					AAC_DRIVERNAME);
 	aac = (struct aac_dev *)host->hostdata;
-	if (aac_adapter_check_health(aac)) {
-		printk(KERN_ERR "%s: Host adapter appears dead\n", 
-				AAC_DRIVERNAME);
-		return -ENODEV;
-	}
+
+	if ((count = aac_check_health(aac)))
+		return count;
 	/*
 	 * Wait for all commands to complete to this specific
 	 * target (block maximum 60 seconds).
 	 */
 	for (count = 60; count; --count) {
-		int active = 0;
+		int active = aac->in_reset;
+
+		if (active == 0)
 		__shost_for_each_device(dev, host) {
 			spin_lock_irqsave(&dev->list_lock, flags);
 			list_for_each_entry(command, &dev->cmd_list, list) {
@@ -864,13 +867,6 @@ static int __devinit aac_probe_one(struct pci_dev *pdev,
 	 *	Map in the registers from the adapter.
 	 */
 	aac->base_size = AAC_MIN_FOOTPRINT_SIZE;
-	if ((aac->regs.sa = ioremap(
-	  (unsigned long)aac->scsi_host_ptr->base, AAC_MIN_FOOTPRINT_SIZE))
-	  == NULL) {	
-		printk(KERN_WARNING "%s: unable to map adapter.\n",
-		  AAC_DRIVERNAME);
-		goto out_free_fibs;
-	}
 	if ((*aac_drivers[index].init)(aac))
 		goto out_unmap;
 
@@ -928,12 +924,12 @@ static int __devinit aac_probe_one(struct pci_dev *pdev,
 	 * all containers are on the virtual channel 0 (CONTAINER_CHANNEL)
 	 * physical channels are address by their actual physical number+1
 	 */
-	if (aac->nondasd_support == 1)
+	if ((aac->nondasd_support == 1) || expose_physicals)
 		shost->max_channel = aac->maximum_num_channels;
 	else
 		shost->max_channel = 0;
 
-	aac_get_config_status(aac);
+	aac_get_config_status(aac, 0);
 	aac_get_containers(aac);
 	list_add(&aac->entry, insert);
 
@@ -969,8 +965,7 @@ static int __devinit aac_probe_one(struct pci_dev *pdev,
 	aac_fib_map_free(aac);
 	pci_free_consistent(aac->pdev, aac->comm_size, aac->comm_addr, aac->comm_phys);
 	kfree(aac->queues);
-	iounmap(aac->regs.sa);
- out_free_fibs:
+	aac_adapter_ioremap(aac, 0);
 	kfree(aac->fibs);
 	kfree(aac->fsa_dev);
  out_free_host:
@@ -1005,7 +1000,7 @@ static void __devexit aac_remove_one(struct pci_dev *pdev)
 	kfree(aac->queues);
 
 	free_irq(pdev->irq, aac);
-	iounmap(aac->regs.sa);
+	aac_adapter_ioremap(aac, 0);
 	
 	kfree(aac->fibs);
 	kfree(aac->fsa_dev);
@@ -1013,6 +1008,10 @@ static void __devexit aac_remove_one(struct pci_dev *pdev)
 	list_del(&aac->entry);
 	scsi_host_put(shost);
 	pci_disable_device(pdev);
+	if (list_empty(&aac_devices)) {
+		unregister_chrdev(aac_cfg_major, "aac");
+		aac_cfg_major = -1;
+	}
 }
 
 static struct pci_driver aac_pci_driver = {
diff --git a/drivers/scsi/aacraid/rkt.c b/drivers/scsi/aacraid/rkt.c
index 458ea89..643f23b 100644
--- a/drivers/scsi/aacraid/rkt.c
+++ b/drivers/scsi/aacraid/rkt.c
@@ -28,370 +28,27 @@
  *
  */
 
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/types.h>
-#include <linux/sched.h>
-#include <linux/pci.h>
-#include <linux/spinlock.h>
-#include <linux/slab.h>
 #include <linux/blkdev.h>
-#include <linux/delay.h>
-#include <linux/completion.h>
-#include <linux/time.h>
-#include <linux/interrupt.h>
-#include <asm/semaphore.h>
 
 #include <scsi/scsi_host.h>
 
 #include "aacraid.h"
 
-static irqreturn_t aac_rkt_intr(int irq, void *dev_id, struct pt_regs *regs)
-{
-	struct aac_dev *dev = dev_id;
-
-	if (dev->new_comm_interface) {
-		u32 Index = rkt_readl(dev, MUnit.OutboundQueue);
-		if (Index == 0xFFFFFFFFL)
-			Index = rkt_readl(dev, MUnit.OutboundQueue);
-		if (Index != 0xFFFFFFFFL) {
-			do {
-				if (aac_intr_normal(dev, Index)) {
-					rkt_writel(dev, MUnit.OutboundQueue, Index);
-					rkt_writel(dev, MUnit.ODR, DoorBellAdapterNormRespReady);
-				}
-				Index = rkt_readl(dev, MUnit.OutboundQueue);
-			} while (Index != 0xFFFFFFFFL);
-			return IRQ_HANDLED;
-		}
-	} else {
-		unsigned long bellbits;
-		u8 intstat;
-		intstat = rkt_readb(dev, MUnit.OISR);
-		/*
-		 *	Read mask and invert because drawbridge is reversed.
-		 *	This allows us to only service interrupts that have 
-		 *	been enabled.
-		 *	Check to see if this is our interrupt.  If it isn't just return
-		 */
-		if (intstat & ~(dev->OIMR))
-		{
-			bellbits = rkt_readl(dev, OutboundDoorbellReg);
-			if (bellbits & DoorBellPrintfReady) {
-				aac_printf(dev, rkt_readl (dev, IndexRegs.Mailbox[5]));
-				rkt_writel(dev, MUnit.ODR,DoorBellPrintfReady);
-				rkt_writel(dev, InboundDoorbellReg,DoorBellPrintfDone);
-			}
-			else if (bellbits & DoorBellAdapterNormCmdReady) {
-				rkt_writel(dev, MUnit.ODR, DoorBellAdapterNormCmdReady);
-				aac_command_normal(&dev->queues->queue[HostNormCmdQueue]);
-//				rkt_writel(dev, MUnit.ODR, DoorBellAdapterNormCmdReady);
-			}
-			else if (bellbits & DoorBellAdapterNormRespReady) {
-				rkt_writel(dev, MUnit.ODR,DoorBellAdapterNormRespReady);
-				aac_response_normal(&dev->queues->queue[HostNormRespQueue]);
-			}
-			else if (bellbits & DoorBellAdapterNormCmdNotFull) {
-				rkt_writel(dev, MUnit.ODR, DoorBellAdapterNormCmdNotFull);
-			}
-			else if (bellbits & DoorBellAdapterNormRespNotFull) {
-				rkt_writel(dev, MUnit.ODR, DoorBellAdapterNormCmdNotFull);
-				rkt_writel(dev, MUnit.ODR, DoorBellAdapterNormRespNotFull);
-			}
-			return IRQ_HANDLED;
-		}
-	}
-	return IRQ_NONE;
-}
-
-/**
- *	aac_rkt_disable_interrupt	-	Disable interrupts
- *	@dev: Adapter
- */
-
-static void aac_rkt_disable_interrupt(struct aac_dev *dev)
-{
-	rkt_writeb(dev, MUnit.OIMR, dev->OIMR = 0xff);
-}
-
 /**
- *	rkt_sync_cmd	-	send a command and wait
- *	@dev: Adapter
- *	@command: Command to execute
- *	@p1: first parameter
- *	@ret: adapter status
+ *	aac_rkt_ioremap
+ *	@size: mapping resize request
  *
- *	This routine will send a synchronous command to the adapter and wait 
- *	for its	completion.
  */
-
-static int rkt_sync_cmd(struct aac_dev *dev, u32 command,
-	u32 p1, u32 p2, u32 p3, u32 p4, u32 p5, u32 p6,
-	u32 *status, u32 *r1, u32 *r2, u32 *r3, u32 *r4)
+static int aac_rkt_ioremap(struct aac_dev * dev, u32 size)
 {
-	unsigned long start;
-	int ok;
-	/*
-	 *	Write the command into Mailbox 0
-	 */
-	rkt_writel(dev, InboundMailbox0, command);
-	/*
-	 *	Write the parameters into Mailboxes 1 - 6
-	 */
-	rkt_writel(dev, InboundMailbox1, p1);
-	rkt_writel(dev, InboundMailbox2, p2);
-	rkt_writel(dev, InboundMailbox3, p3);
-	rkt_writel(dev, InboundMailbox4, p4);
-	/*
-	 *	Clear the synch command doorbell to start on a clean slate.
-	 */
-	rkt_writel(dev, OutboundDoorbellReg, OUTBOUNDDOORBELL_0);
-	/*
-	 *	Disable doorbell interrupts
-	 */
-	rkt_writeb(dev, MUnit.OIMR, dev->OIMR = 0xff);
-	/*
-	 *	Force the completion of the mask register write before issuing
-	 *	the interrupt.
-	 */
-	rkt_readb (dev, MUnit.OIMR);
-	/*
-	 *	Signal that there is a new synch command
-	 */
-	rkt_writel(dev, InboundDoorbellReg, INBOUNDDOORBELL_0);
-
-	ok = 0;
-	start = jiffies;
-
-	/*
-	 *	Wait up to 30 seconds
-	 */
-	while (time_before(jiffies, start+30*HZ)) 
-	{
-		udelay(5);	/* Delay 5 microseconds to let Mon960 get info. */
-		/*
-		 *	Mon960 will set doorbell0 bit when it has completed the command.
-		 */
-		if (rkt_readl(dev, OutboundDoorbellReg) & OUTBOUNDDOORBELL_0) {
-			/*
-			 *	Clear the doorbell.
-			 */
-			rkt_writel(dev, OutboundDoorbellReg, OUTBOUNDDOORBELL_0);
-			ok = 1;
-			break;
-		}
-		/*
-		 *	Yield the processor in case we are slow 
-		 */
-		msleep(1);
+	if (!size) {
+		iounmap(dev->regs.rkt);
+		return 0;
 	}
-	if (ok != 1) {
-		/*
-		 *	Restore interrupt mask even though we timed out
-		 */
-		if (dev->new_comm_interface)
-			rkt_writeb(dev, MUnit.OIMR, dev->OIMR = 0xf7);
-		else
-			rkt_writeb(dev, MUnit.OIMR, dev->OIMR = 0xfb);
-		return -ETIMEDOUT;
-	}
-	/*
-	 *	Pull the synch status from Mailbox 0.
-	 */
-	if (status)
-		*status = rkt_readl(dev, IndexRegs.Mailbox[0]);
-	if (r1)
-		*r1 = rkt_readl(dev, IndexRegs.Mailbox[1]);
-	if (r2)
-		*r2 = rkt_readl(dev, IndexRegs.Mailbox[2]);
-	if (r3)
-		*r3 = rkt_readl(dev, IndexRegs.Mailbox[3]);
-	if (r4)
-		*r4 = rkt_readl(dev, IndexRegs.Mailbox[4]);
-	/*
-	 *	Clear the synch command doorbell.
-	 */
-	rkt_writel(dev, OutboundDoorbellReg, OUTBOUNDDOORBELL_0);
-	/*
-	 *	Restore interrupt mask
-	 */
-	if (dev->new_comm_interface)
-		rkt_writeb(dev, MUnit.OIMR, dev->OIMR = 0xf7);
-	else
-		rkt_writeb(dev, MUnit.OIMR, dev->OIMR = 0xfb);
-	return 0;
-
-}
-
-/**
- *	aac_rkt_interrupt_adapter	-	interrupt adapter
- *	@dev: Adapter
- *
- *	Send an interrupt to the i960 and breakpoint it.
- */
-
-static void aac_rkt_interrupt_adapter(struct aac_dev *dev)
-{
-	rkt_sync_cmd(dev, BREAKPOINT_REQUEST, 0, 0, 0, 0, 0, 0,
-	  NULL, NULL, NULL, NULL, NULL);
-}
-
-/**
- *	aac_rkt_notify_adapter		-	send an event to the adapter
- *	@dev: Adapter
- *	@event: Event to send
- *
- *	Notify the i960 that something it probably cares about has
- *	happened.
- */
-
-static void aac_rkt_notify_adapter(struct aac_dev *dev, u32 event)
-{
-	switch (event) {
-
-	case AdapNormCmdQue:
-		rkt_writel(dev, MUnit.IDR,INBOUNDDOORBELL_1);
-		break;
-	case HostNormRespNotFull:
-		rkt_writel(dev, MUnit.IDR,INBOUNDDOORBELL_4);
-		break;
-	case AdapNormRespQue:
-		rkt_writel(dev, MUnit.IDR,INBOUNDDOORBELL_2);
-		break;
-	case HostNormCmdNotFull:
-		rkt_writel(dev, MUnit.IDR,INBOUNDDOORBELL_3);
-		break;
-	case HostShutdown:
-//		rkt_sync_cmd(dev, HOST_CRASHING, 0, 0, 0, 0, 0, 0,
-//		  NULL, NULL, NULL, NULL, NULL);
-		break;
-	case FastIo:
-		rkt_writel(dev, MUnit.IDR,INBOUNDDOORBELL_6);
-		break;
-	case AdapPrintfDone:
-		rkt_writel(dev, MUnit.IDR,INBOUNDDOORBELL_5);
-		break;
-	default:
-		BUG();
-		break;
-	}
-}
-
-/**
- *	aac_rkt_start_adapter		-	activate adapter
- *	@dev:	Adapter
- *
- *	Start up processing on an i960 based AAC adapter
- */
-
-static void aac_rkt_start_adapter(struct aac_dev *dev)
-{
-	struct aac_init *init;
-
-	init = dev->init;
-	init->HostElapsedSeconds = cpu_to_le32(get_seconds());
-	// We can only use a 32 bit address here
-	rkt_sync_cmd(dev, INIT_STRUCT_BASE_ADDRESS, (u32)(ulong)dev->init_pa,
-	  0, 0, 0, 0, 0, NULL, NULL, NULL, NULL, NULL);
-}
-
-/**
- *	aac_rkt_check_health
- *	@dev: device to check if healthy
- *
- *	Will attempt to determine if the specified adapter is alive and
- *	capable of handling requests, returning 0 if alive.
- */
-static int aac_rkt_check_health(struct aac_dev *dev)
-{
-	u32 status = rkt_readl(dev, MUnit.OMRx[0]);
-
-	/*
-	 *	Check to see if the board failed any self tests.
-	 */
-	if (status & SELF_TEST_FAILED)
+	dev->base = dev->regs.rkt = ioremap(dev->scsi_host_ptr->base, size);
+	if (dev->base == NULL)
 		return -1;
-	/*
-	 *	Check to see if the board panic'd.
-	 */
-	if (status & KERNEL_PANIC) {
-		char * buffer;
-		struct POSTSTATUS {
-			__le32 Post_Command;
-			__le32 Post_Address;
-		} * post;
-		dma_addr_t paddr, baddr;
-		int ret;
-
-		if ((status & 0xFF000000L) == 0xBC000000L)
-			return (status >> 16) & 0xFF;
-		buffer = pci_alloc_consistent(dev->pdev, 512, &baddr);
-		ret = -2;
-		if (buffer == NULL)
-			return ret;
-		post = pci_alloc_consistent(dev->pdev,
-		  sizeof(struct POSTSTATUS), &paddr);
-		if (post == NULL) {
-			pci_free_consistent(dev->pdev, 512, buffer, baddr);
-			return ret;
-		}
-                memset(buffer, 0, 512);
-		post->Post_Command = cpu_to_le32(COMMAND_POST_RESULTS);
-                post->Post_Address = cpu_to_le32(baddr);
-                rkt_writel(dev, MUnit.IMRx[0], paddr);
-                rkt_sync_cmd(dev, COMMAND_POST_RESULTS, baddr, 0, 0, 0, 0, 0,
-		  NULL, NULL, NULL, NULL, NULL);
-		pci_free_consistent(dev->pdev, sizeof(struct POSTSTATUS),
-		  post, paddr);
-                if ((buffer[0] == '0') && ((buffer[1] == 'x') || (buffer[1] == 'X'))) {
-                        ret = (buffer[2] <= '9') ? (buffer[2] - '0') : (buffer[2] - 'A' + 10);
-                        ret <<= 4;
-                        ret += (buffer[3] <= '9') ? (buffer[3] - '0') : (buffer[3] - 'A' + 10);
-                }
-		pci_free_consistent(dev->pdev, 512, buffer, baddr);
-                return ret;
-        }
-	/*
-	 *	Wait for the adapter to be up and running.
-	 */
-	if (!(status & KERNEL_UP_AND_RUNNING))
-		return -3;
-	/*
-	 *	Everything is OK
-	 */
-	return 0;
-}
-
-/**
- *	aac_rkt_send
- *	@fib: fib to issue
- *
- *	Will send a fib, returning 0 if successful.
- */
-static int aac_rkt_send(struct fib * fib)
-{
-	u64 addr = fib->hw_fib_pa;
-	struct aac_dev *dev = fib->dev;
-	volatile void __iomem *device = dev->regs.rkt;
-	u32 Index;
-
-	dprintk((KERN_DEBUG "%p->aac_rkt_send(%p->%llx)\n", dev, fib, addr));
-	Index = rkt_readl(dev, MUnit.InboundQueue);
-	if (Index == 0xFFFFFFFFL)
-		Index = rkt_readl(dev, MUnit.InboundQueue);
-	dprintk((KERN_DEBUG "Index = 0x%x\n", Index));
-	if (Index == 0xFFFFFFFFL)
-		return Index;
-	device += Index;
-	dprintk((KERN_DEBUG "entry = %x %x %u\n", (u32)(addr & 0xffffffff),
-	  (u32)(addr >> 32), (u32)le16_to_cpu(fib->hw_fib->header.Size)));
-	writel((u32)(addr & 0xffffffff), device);
-	device += sizeof(u32);
-	writel((u32)(addr >> 32), device);
-	device += sizeof(u32);
-	writel(le16_to_cpu(fib->hw_fib->header.Size), device);
-	rkt_writel(dev, MUnit.InboundQueue, Index);
-	dprintk((KERN_DEBUG "aac_rkt_send - return 0\n"));
+	dev->IndexRegs = &dev->regs.rkt->IndexRegs;
 	return 0;
 }
 
@@ -406,78 +63,18 @@ static int aac_rkt_send(struct fib * fib)
 
 int aac_rkt_init(struct aac_dev *dev)
 {
-	unsigned long start;
-	unsigned long status;
-	int instance;
-	const char * name;
-
-	instance = dev->id;
-	name     = dev->name;
+	int retval;
+	extern int _aac_rx_init(struct aac_dev *dev);
+	extern void aac_rx_start_adapter(struct aac_dev *dev);
 
 	/*
-	 *	Check to see if the board panic'd while booting.
-	 */
-	/*
-	 *	Check to see if the board failed any self tests.
-	 */
-	if (rkt_readl(dev, MUnit.OMRx[0]) & SELF_TEST_FAILED) {
-		printk(KERN_ERR "%s%d: adapter self-test failed.\n", dev->name, instance);
-		goto error_iounmap;
-	}
-	/*
-	 *	Check to see if the monitor panic'd while booting.
-	 */
-	if (rkt_readl(dev, MUnit.OMRx[0]) & MONITOR_PANIC) {
-		printk(KERN_ERR "%s%d: adapter monitor panic.\n", dev->name, instance);
-		goto error_iounmap;
-	}
-	/*
-	 *	Check to see if the board panic'd while booting.
-	 */
-	if (rkt_readl(dev, MUnit.OMRx[0]) & KERNEL_PANIC) {
-		printk(KERN_ERR "%s%d: adapter kernel panic'd.\n", dev->name, instance);
-		goto error_iounmap;
-	}
-	start = jiffies;
-	/*
-	 *	Wait for the adapter to be up and running. Wait up to 3 minutes
-	 */
-	while (!(rkt_readl(dev, MUnit.OMRx[0]) & KERNEL_UP_AND_RUNNING))
-	{
-		if(time_after(jiffies, start+startup_timeout*HZ))
-		{
-			status = rkt_readl(dev, MUnit.OMRx[0]);
-			printk(KERN_ERR "%s%d: adapter kernel failed to start, init status = %lx.\n", 
-					dev->name, instance, status);
-			goto error_iounmap;
-		}
-		msleep(1);
-	}
-	if (request_irq(dev->scsi_host_ptr->irq, aac_rkt_intr, IRQF_SHARED|IRQF_DISABLED, "aacraid", (void *)dev)<0)
-	{
-		printk(KERN_ERR "%s%d: Interrupt unavailable.\n", name, instance);
-		goto error_iounmap;
-	}
-	/*
 	 *	Fill in the function dispatch table.
 	 */
-	dev->a_ops.adapter_interrupt = aac_rkt_interrupt_adapter;
-	dev->a_ops.adapter_disable_int = aac_rkt_disable_interrupt;
-	dev->a_ops.adapter_notify = aac_rkt_notify_adapter;
-	dev->a_ops.adapter_sync_cmd = rkt_sync_cmd;
-	dev->a_ops.adapter_check_health = aac_rkt_check_health;
-	dev->a_ops.adapter_send = aac_rkt_send;
-
-	/*
-	 *	First clear out all interrupts.  Then enable the one's that we
-	 *	can handle.
-	 */
-	rkt_writeb(dev, MUnit.OIMR, 0xff);
-	rkt_writel(dev, MUnit.ODR, 0xffffffff);
-	rkt_writeb(dev, MUnit.OIMR, dev->OIMR = 0xfb);
+	dev->a_ops.adapter_ioremap = aac_rkt_ioremap;
 
-	if (aac_init_adapter(dev) == NULL)
-		goto error_irq;
+	retval = _aac_rx_init(dev);
+	if (retval)
+		return retval;
 	if (dev->new_comm_interface) {
 		/*
 		 * FIB Setup has already been done, but we can minimize the
@@ -494,20 +91,11 @@ int aac_rkt_init(struct aac_dev *dev)
 			dev->init->MaxIoCommands = cpu_to_le32(246);
 			dev->scsi_host_ptr->can_queue = 246 - AAC_NUM_MGT_FIB;
 		}
-		rkt_writeb(dev, MUnit.OIMR, dev->OIMR = 0xf7);
 	}
 	/*
 	 *	Tell the adapter that all is configured, and it can start
 	 *	accepting requests
 	 */
-	aac_rkt_start_adapter(dev);
+	aac_rx_start_adapter(dev);
 	return 0;
-
-error_irq:
-	rkt_writeb(dev, MUnit.OIMR, dev->OIMR = 0xff);
-	free_irq(dev->scsi_host_ptr->irq, (void *)dev);
-
-error_iounmap:
-
-	return -1;
 }
diff --git a/drivers/scsi/aacraid/rx.c b/drivers/scsi/aacraid/rx.c
index 035018d..a1d214d 100644
--- a/drivers/scsi/aacraid/rx.c
+++ b/drivers/scsi/aacraid/rx.c
@@ -79,7 +79,7 @@ static irqreturn_t aac_rx_intr(int irq, void *dev_id, struct pt_regs *regs)
 		{
 			bellbits = rx_readl(dev, OutboundDoorbellReg);
 			if (bellbits & DoorBellPrintfReady) {
-				aac_printf(dev, rx_readl (dev, IndexRegs.Mailbox[5]));
+				aac_printf(dev, readl (&dev->IndexRegs->Mailbox[5]));
 				rx_writel(dev, MUnit.ODR,DoorBellPrintfReady);
 				rx_writel(dev, InboundDoorbellReg,DoorBellPrintfDone);
 			}
@@ -134,14 +134,14 @@ static int rx_sync_cmd(struct aac_dev *dev, u32 command,
 	/*
 	 *	Write the command into Mailbox 0
 	 */
-	rx_writel(dev, InboundMailbox0, command);
+	writel(command, &dev->IndexRegs->Mailbox[0]);
 	/*
 	 *	Write the parameters into Mailboxes 1 - 6
 	 */
-	rx_writel(dev, InboundMailbox1, p1);
-	rx_writel(dev, InboundMailbox2, p2);
-	rx_writel(dev, InboundMailbox3, p3);
-	rx_writel(dev, InboundMailbox4, p4);
+	writel(p1, &dev->IndexRegs->Mailbox[1]);
+	writel(p2, &dev->IndexRegs->Mailbox[2]);
+	writel(p3, &dev->IndexRegs->Mailbox[3]);
+	writel(p4, &dev->IndexRegs->Mailbox[4]);
 	/*
 	 *	Clear the synch command doorbell to start on a clean slate.
 	 */
@@ -199,15 +199,15 @@ static int rx_sync_cmd(struct aac_dev *dev, u32 command,
 	 *	Pull the synch status from Mailbox 0.
 	 */
 	if (status)
-		*status = rx_readl(dev, IndexRegs.Mailbox[0]);
+		*status = readl(&dev->IndexRegs->Mailbox[0]);
 	if (r1)
-		*r1 = rx_readl(dev, IndexRegs.Mailbox[1]);
+		*r1 = readl(&dev->IndexRegs->Mailbox[1]);
 	if (r2)
-		*r2 = rx_readl(dev, IndexRegs.Mailbox[2]);
+		*r2 = readl(&dev->IndexRegs->Mailbox[2]);
 	if (r3)
-		*r3 = rx_readl(dev, IndexRegs.Mailbox[3]);
+		*r3 = readl(&dev->IndexRegs->Mailbox[3]);
 	if (r4)
-		*r4 = rx_readl(dev, IndexRegs.Mailbox[4]);
+		*r4 = readl(&dev->IndexRegs->Mailbox[4]);
 	/*
 	 *	Clear the synch command doorbell.
 	 */
@@ -261,8 +261,6 @@ static void aac_rx_notify_adapter(struct aac_dev *dev, u32 event)
 		rx_writel(dev, MUnit.IDR,INBOUNDDOORBELL_3);
 		break;
 	case HostShutdown:
-//		rx_sync_cmd(dev, HOST_CRASHING, 0, 0, 0, 0, 0, 0,
-//		  NULL, NULL, NULL, NULL, NULL);
 		break;
 	case FastIo:
 		rx_writel(dev, MUnit.IDR,INBOUNDDOORBELL_6);
@@ -283,7 +281,7 @@ static void aac_rx_notify_adapter(struct aac_dev *dev, u32 event)
  *	Start up processing on an i960 based AAC adapter
  */
 
-static void aac_rx_start_adapter(struct aac_dev *dev)
+void aac_rx_start_adapter(struct aac_dev *dev)
 {
 	struct aac_init *init;
 
@@ -381,7 +379,7 @@ static int aac_rx_send(struct fib * fib)
 	dprintk((KERN_DEBUG "Index = 0x%x\n", Index));
 	if (Index == 0xFFFFFFFFL)
 		return Index;
-	device += Index;
+	device = dev->base + Index;
 	dprintk((KERN_DEBUG "entry = %x %x %u\n", (u32)(addr & 0xffffffff),
 	  (u32)(addr >> 32), (u32)le16_to_cpu(fib->hw_fib->header.Size)));
 	writel((u32)(addr & 0xffffffff), device);
@@ -395,6 +393,43 @@ static int aac_rx_send(struct fib * fib)
 }
 
 /**
+ *	aac_rx_ioremap
+ *	@size: mapping resize request
+ *
+ */
+static int aac_rx_ioremap(struct aac_dev * dev, u32 size)
+{
+	if (!size) {
+		iounmap(dev->regs.rx);
+		return 0;
+	}
+	dev->base = dev->regs.rx = ioremap(dev->scsi_host_ptr->base, size);
+	if (dev->base == NULL)
+		return -1;
+	dev->IndexRegs = &dev->regs.rx->IndexRegs;
+	return 0;
+}
+
+static int aac_rx_restart_adapter(struct aac_dev *dev)
+{
+	u32 var;
+
+	printk(KERN_ERR "%s%d: adapter kernel panic'd.\n",
+			dev->name, dev->id);
+
+	if (aac_rx_check_health(dev) <= 0)
+		return 1;
+	if (rx_sync_cmd(dev, IOP_RESET, 0, 0, 0, 0, 0, 0,
+			&var, NULL, NULL, NULL, NULL))
+		return 1;
+	if (var != 0x00000001)
+		 return 1;
+	if (rx_readl(dev, MUnit.OMRx[0]) & KERNEL_PANIC)
+		return 1;
+	return 0;
+}
+
+/**
  *	aac_rx_init	-	initialize an i960 based AAC card
  *	@dev: device to configure
  *
@@ -403,7 +438,7 @@ static int aac_rx_send(struct fib * fib)
  *	to the comm region.
  */
 
-int aac_rx_init(struct aac_dev *dev)
+int _aac_rx_init(struct aac_dev *dev)
 {
 	unsigned long start;
 	unsigned long status;
@@ -413,27 +448,30 @@ int aac_rx_init(struct aac_dev *dev)
 	instance = dev->id;
 	name     = dev->name;
 
+	if (aac_adapter_ioremap(dev, dev->base_size)) {
+		printk(KERN_WARNING "%s: unable to map adapter.\n", name);
+		goto error_iounmap;
+	}
+
 	/*
 	 *	Check to see if the board panic'd while booting.
 	 */
+	status = rx_readl(dev, MUnit.OMRx[0]);
+	if (status & KERNEL_PANIC)
+		if (aac_rx_restart_adapter(dev))
+			goto error_iounmap;
 	/*
 	 *	Check to see if the board failed any self tests.
 	 */
-	if (rx_readl(dev, MUnit.OMRx[0]) & SELF_TEST_FAILED) {
+	status = rx_readl(dev, MUnit.OMRx[0]);
+	if (status & SELF_TEST_FAILED) {
 		printk(KERN_ERR "%s%d: adapter self-test failed.\n", dev->name, instance);
 		goto error_iounmap;
 	}
 	/*
-	 *	Check to see if the board panic'd while booting.
-	 */
-	if (rx_readl(dev, MUnit.OMRx[0]) & KERNEL_PANIC) {
-		printk(KERN_ERR "%s%d: adapter kernel panic.\n", dev->name, instance);
-		goto error_iounmap;
-	}
-	/*
 	 *	Check to see if the monitor panic'd while booting.
 	 */
-	if (rx_readl(dev, MUnit.OMRx[0]) & MONITOR_PANIC) {
+	if (status & MONITOR_PANIC) {
 		printk(KERN_ERR "%s%d: adapter monitor panic.\n", dev->name, instance);
 		goto error_iounmap;
 	}
@@ -441,12 +479,10 @@ int aac_rx_init(struct aac_dev *dev)
 	/*
 	 *	Wait for the adapter to be up and running. Wait up to 3 minutes
 	 */
-	while ((!(rx_readl(dev, IndexRegs.Mailbox[7]) & KERNEL_UP_AND_RUNNING))
-		|| (!(rx_readl(dev, MUnit.OMRx[0]) & KERNEL_UP_AND_RUNNING)))
+	while (!((status = rx_readl(dev, MUnit.OMRx[0])) & KERNEL_UP_AND_RUNNING))
 	{
 		if(time_after(jiffies, start+startup_timeout*HZ))
 		{
-			status = rx_readl(dev, IndexRegs.Mailbox[7]);
 			printk(KERN_ERR "%s%d: adapter kernel failed to start, init status = %lx.\n", 
 					dev->name, instance, status);
 			goto error_iounmap;
@@ -481,11 +517,6 @@ int aac_rx_init(struct aac_dev *dev)
 	if (dev->new_comm_interface)
 		rx_writeb(dev, MUnit.OIMR, dev->OIMR = 0xf7);
 
-	/*
-	 *	Tell the adapter that all is configured, and it can start
-	 *	accepting requests
-	 */
-	aac_rx_start_adapter(dev);
 	return 0;
 
 error_irq:
@@ -496,3 +527,23 @@ error_iounmap:
 
 	return -1;
 }
+
+int aac_rx_init(struct aac_dev *dev)
+{
+	int retval;
+
+	/*
+	 *	Fill in the function dispatch table.
+	 */
+	dev->a_ops.adapter_ioremap = aac_rx_ioremap;
+
+	retval = _aac_rx_init(dev);
+	if (!retval) {
+		/*
+		 *	Tell the adapter that all is configured, and it can
+		 * start accepting requests
+		 */
+		aac_rx_start_adapter(dev);
+	}
+	return retval;
+}
diff --git a/drivers/scsi/aacraid/sa.c b/drivers/scsi/aacraid/sa.c
index cd586cc..f906ead 100644
--- a/drivers/scsi/aacraid/sa.c
+++ b/drivers/scsi/aacraid/sa.c
@@ -281,6 +281,21 @@ static int aac_sa_check_health(struct aac_dev *dev)
 }
 
 /**
+ *	aac_sa_ioremap
+ *	@size: mapping resize request
+ *
+ */
+static int aac_sa_ioremap(struct aac_dev * dev, u32 size)
+{
+	if (!size) {
+		iounmap(dev->regs.sa);
+		return 0;
+	}
+	dev->base = dev->regs.sa = ioremap(dev->scsi_host_ptr->base, size);
+	return (dev->base == NULL) ? -1 : 0;
+}
+
+/**
  *	aac_sa_init	-	initialize an ARM based AAC card
  *	@dev: device to configure
  *
@@ -299,6 +314,11 @@ int aac_sa_init(struct aac_dev *dev)
 	instance = dev->id;
 	name     = dev->name;
 
+	if (aac_sa_ioremap(dev, dev->base_size)) {
+		printk(KERN_WARNING "%s: unable to map adapter.\n", name);
+		goto error_iounmap;
+	}
+
 	/*
 	 *	Check to see if the board failed any self tests.
 	 */
@@ -341,6 +361,7 @@ int aac_sa_init(struct aac_dev *dev)
 	dev->a_ops.adapter_notify = aac_sa_notify_adapter;
 	dev->a_ops.adapter_sync_cmd = sa_sync_cmd;
 	dev->a_ops.adapter_check_health = aac_sa_check_health;
+	dev->a_ops.adapter_ioremap = aac_sa_ioremap;
 
 	/*
 	 *	First clear out all interrupts.  Then enable the one's that 
diff --git a/drivers/scsi/advansys.c b/drivers/scsi/advansys.c
index e32b4ab..773f02e 100644
--- a/drivers/scsi/advansys.c
+++ b/drivers/scsi/advansys.c
@@ -888,10 +888,6 @@ typedef unsigned char uchar;
 #define ASC_PCI_ID2DEV(id)    (((id) >> 11) & 0x1F)
 #define ASC_PCI_ID2FUNC(id)   (((id) >> 8) & 0x7)
 #define ASC_PCI_MKID(bus, dev, func) ((((dev) & 0x1F) << 11) | (((func) & 0x7) << 8) | ((bus) & 0xFF))
-#define ASC_PCI_VENDORID                  0x10CD
-#define ASC_PCI_DEVICEID_1200A            0x1100
-#define ASC_PCI_DEVICEID_1200B            0x1200
-#define ASC_PCI_DEVICEID_ULTRA            0x1300
 #define ASC_PCI_REVISION_3150             0x02
 #define ASC_PCI_REVISION_3050             0x03
 
@@ -899,6 +895,14 @@ typedef unsigned char uchar;
 #define  ASC_DVCLIB_CALL_FAILED   (0)
 #define  ASC_DVCLIB_CALL_ERROR    (-1)
 
+#define PCI_VENDOR_ID_ASP		0x10cd
+#define PCI_DEVICE_ID_ASP_1200A		0x1100
+#define PCI_DEVICE_ID_ASP_ABP940	0x1200
+#define PCI_DEVICE_ID_ASP_ABP940U	0x1300
+#define PCI_DEVICE_ID_ASP_ABP940UW	0x2300
+#define PCI_DEVICE_ID_38C0800_REV1	0x2500
+#define PCI_DEVICE_ID_38C1600_REV1	0x2700
+
 /*
  * Enable CC_VERY_LONG_SG_LIST to support up to 64K element SG lists.
  * The SRB structure will have to be changed and the ASC_SRB2SCSIQ()
@@ -1492,8 +1496,6 @@ typedef struct asc_dvc_cfg {
 #define ASC_INIT_STATE_END_INQUIRY   0x0080
 #define ASC_INIT_RESET_SCSI_DONE     0x0100
 #define ASC_INIT_STATE_WITHOUT_EEP   0x8000
-#define ASC_PCI_DEVICE_ID_REV_A      0x1100
-#define ASC_PCI_DEVICE_ID_REV_B      0x1200
 #define ASC_BUG_FIX_IF_NOT_DWB       0x0001
 #define ASC_BUG_FIX_ASYN_USE_SYN     0x0002
 #define ASYN_SDTR_DATA_FIX_PCI_REV_AB 0x41
@@ -2100,12 +2102,6 @@ STATIC ASC_DCNT  AscGetMaxDmaCount(ushort);
 #define ADV_NUM_PAGE_CROSSING \
     ((ADV_SG_TOTAL_MEM_SIZE + (ADV_PAGE_SIZE - 1))/ADV_PAGE_SIZE)
 
-/* a_condor.h */
-#define ADV_PCI_VENDOR_ID               0x10CD
-#define ADV_PCI_DEVICE_ID_REV_A         0x2300
-#define ADV_PCI_DEVID_38C0800_REV1      0x2500
-#define ADV_PCI_DEVID_38C1600_REV1      0x2700
-
 #define ADV_EEP_DVC_CFG_BEGIN           (0x00)
 #define ADV_EEP_DVC_CFG_END             (0x15)
 #define ADV_EEP_DVC_CTL_BEGIN           (0x16)  /* location of OEM name */
@@ -3569,14 +3565,7 @@ typedef struct scsi_cmnd     REQ, *REQP;
 #define PCI_MAX_SLOT            0x1F
 #define PCI_MAX_BUS             0xFF
 #define PCI_IOADDRESS_MASK      0xFFFE
-#define ASC_PCI_VENDORID        0x10CD
 #define ASC_PCI_DEVICE_ID_CNT   6       /* PCI Device ID count. */
-#define ASC_PCI_DEVICE_ID_1100  0x1100
-#define ASC_PCI_DEVICE_ID_1200  0x1200
-#define ASC_PCI_DEVICE_ID_1300  0x1300
-#define ASC_PCI_DEVICE_ID_2300  0x2300  /* ASC-3550 */
-#define ASC_PCI_DEVICE_ID_2500  0x2500  /* ASC-38C0800 */
-#define ASC_PCI_DEVICE_ID_2700  0x2700  /* ASC-38C1600 */
 
 #ifndef ADVANSYS_STATS
 #define ASC_STATS(shp, counter)
@@ -4330,12 +4319,12 @@ advansys_detect(struct scsi_host_template *tpnt)
     struct pci_dev      *pci_devp = NULL;
     int                 pci_device_id_cnt = 0;
     unsigned int        pci_device_id[ASC_PCI_DEVICE_ID_CNT] = {
-                                    ASC_PCI_DEVICE_ID_1100,
-                                    ASC_PCI_DEVICE_ID_1200,
-                                    ASC_PCI_DEVICE_ID_1300,
-                                    ASC_PCI_DEVICE_ID_2300,
-                                    ASC_PCI_DEVICE_ID_2500,
-                                    ASC_PCI_DEVICE_ID_2700
+                                    PCI_DEVICE_ID_ASP_1200A,
+                                    PCI_DEVICE_ID_ASP_ABP940,
+                                    PCI_DEVICE_ID_ASP_ABP940U,
+                                    PCI_DEVICE_ID_ASP_ABP940UW,
+                                    PCI_DEVICE_ID_38C0800_REV1,
+                                    PCI_DEVICE_ID_38C1600_REV1
                         };
     ADV_PADDR           pci_memory_address;
 #endif /* CONFIG_PCI */
@@ -4471,7 +4460,7 @@ advansys_detect(struct scsi_host_template *tpnt)
 
                     /* Find all PCI cards. */
                     while (pci_device_id_cnt < ASC_PCI_DEVICE_ID_CNT) {
-                        if ((pci_devp = pci_find_device(ASC_PCI_VENDORID,
+                        if ((pci_devp = pci_find_device(PCI_VENDOR_ID_ASP,
                             pci_device_id[pci_device_id_cnt], pci_devp)) ==
                             NULL) {
                             pci_device_id_cnt++;
@@ -4575,9 +4564,9 @@ advansys_detect(struct scsi_host_template *tpnt)
              */
 #ifdef CONFIG_PCI
             if (asc_bus[bus] == ASC_IS_PCI &&
-                (pci_devp->device == ASC_PCI_DEVICE_ID_2300 ||
-                 pci_devp->device == ASC_PCI_DEVICE_ID_2500 ||
-                 pci_devp->device == ASC_PCI_DEVICE_ID_2700))
+                (pci_devp->device == PCI_DEVICE_ID_ASP_ABP940UW ||
+                 pci_devp->device == PCI_DEVICE_ID_38C0800_REV1 ||
+                 pci_devp->device == PCI_DEVICE_ID_38C1600_REV1))
             {
                 boardp->flags |= ASC_IS_WIDE_BOARD;
             }
@@ -4600,11 +4589,11 @@ advansys_detect(struct scsi_host_template *tpnt)
                 adv_dvc_varp->isr_callback = adv_isr_callback;
                 adv_dvc_varp->async_callback = adv_async_callback;
 #ifdef CONFIG_PCI
-                if (pci_devp->device == ASC_PCI_DEVICE_ID_2300)
+                if (pci_devp->device == PCI_DEVICE_ID_ASP_ABP940UW)
                 {
                     ASC_DBG(1, "advansys_detect: ASC-3550\n");
                     adv_dvc_varp->chip_type = ADV_CHIP_ASC3550;
-                } else if (pci_devp->device == ASC_PCI_DEVICE_ID_2500)
+                } else if (pci_devp->device == PCI_DEVICE_ID_38C0800_REV1)
                 {
                     ASC_DBG(1, "advansys_detect: ASC-38C0800\n");
                     adv_dvc_varp->chip_type = ADV_CHIP_ASC38C0800;
@@ -11922,7 +11911,7 @@ AscInitGetConfig(
         PCIRevisionID = DvcReadPCIConfigByte(asc_dvc,
                                     AscPCIConfigRevisionIDRegister);
 
-        if (PCIVendorID != ASC_PCI_VENDORID) {
+        if (PCIVendorID != PCI_VENDOR_ID_ASP) {
             warn_code |= ASC_WARN_SET_PCI_CONFIG_SPACE;
         }
         prevCmdRegBits = DvcReadPCIConfigByte(asc_dvc,
@@ -11942,15 +11931,15 @@ AscInitGetConfig(
                 warn_code |= ASC_WARN_SET_PCI_CONFIG_SPACE;
             }
         }
-        if ((PCIDeviceID == ASC_PCI_DEVICEID_1200A) ||
-            (PCIDeviceID == ASC_PCI_DEVICEID_1200B)) {
+        if ((PCIDeviceID == PCI_DEVICE_ID_ASP_1200A) ||
+            (PCIDeviceID == PCI_DEVICE_ID_ASP_ABP940)) {
             DvcWritePCIConfigByte(asc_dvc,
                             AscPCIConfigLatencyTimer, 0x00);
             if (DvcReadPCIConfigByte(asc_dvc, AscPCIConfigLatencyTimer)
                 != 0x00) {
                 warn_code |= ASC_WARN_SET_PCI_CONFIG_SPACE;
             }
-        } else if (PCIDeviceID == ASC_PCI_DEVICEID_ULTRA) {
+        } else if (PCIDeviceID == PCI_DEVICE_ID_ASP_ABP940U) {
             if (DvcReadPCIConfigByte(asc_dvc,
                                 AscPCIConfigLatencyTimer) < 0x20) {
                 DvcWritePCIConfigByte(asc_dvc,
@@ -12037,8 +12026,8 @@ AscInitFromAscDvcVar(
         AscSetChipCfgMsw(iop_base, cfg_msw);
         if ((asc_dvc->bus_type & ASC_IS_PCI_ULTRA) == ASC_IS_PCI_ULTRA) {
         } else {
-            if ((pci_device_id == ASC_PCI_DEVICE_ID_REV_A) ||
-                (pci_device_id == ASC_PCI_DEVICE_ID_REV_B)) {
+            if ((pci_device_id == PCI_DEVICE_ID_ASP_1200A) ||
+                (pci_device_id == PCI_DEVICE_ID_ASP_ABP940)) {
                 asc_dvc->bug_fix_cntl |= ASC_BUG_FIX_IF_NOT_DWB;
                 asc_dvc->bug_fix_cntl |= ASC_BUG_FIX_ASYN_USE_SYN;
             }
@@ -14275,8 +14264,8 @@ Default_38C0800_EEPROM_Config __initdata = {
     0,                          /* 55 reserved */
     0,                          /* 56 cisptr_lsw */
     0,                          /* 57 cisprt_msw */
-    ADV_PCI_VENDOR_ID,          /* 58 subsysvid */
-    ADV_PCI_DEVID_38C0800_REV1, /* 59 subsysid */
+    PCI_VENDOR_ID_ASP,          /* 58 subsysvid */
+    PCI_DEVICE_ID_38C0800_REV1, /* 59 subsysid */
     0,                          /* 60 reserved */
     0,                          /* 61 reserved */
     0,                          /* 62 reserved */
@@ -14405,8 +14394,8 @@ Default_38C1600_EEPROM_Config __initdata = {
     0,                          /* 55 reserved */
     0,                          /* 56 cisptr_lsw */
     0,                          /* 57 cisprt_msw */
-    ADV_PCI_VENDOR_ID,          /* 58 subsysvid */
-    ADV_PCI_DEVID_38C1600_REV1, /* 59 subsysid */
+    PCI_VENDOR_ID_ASP,          /* 58 subsysvid */
+    PCI_DEVICE_ID_38C1600_REV1, /* 59 subsysid */
     0,                          /* 60 reserved */
     0,                          /* 61 reserved */
     0,                          /* 62 reserved */
@@ -18225,3 +18214,22 @@ AdvInquiryHandling(
     }
 }
 MODULE_LICENSE("Dual BSD/GPL");
+
+/* PCI Devices supported by this driver */
+static struct pci_device_id advansys_pci_tbl[] __devinitdata = {
+	{ PCI_VENDOR_ID_ASP, PCI_DEVICE_ID_ASP_1200A,
+	PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{ PCI_VENDOR_ID_ASP, PCI_DEVICE_ID_ASP_ABP940,
+	PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{ PCI_VENDOR_ID_ASP, PCI_DEVICE_ID_ASP_ABP940U,
+	PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{ PCI_VENDOR_ID_ASP, PCI_DEVICE_ID_ASP_ABP940UW,
+	PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{ PCI_VENDOR_ID_ASP, PCI_DEVICE_ID_38C0800_REV1,
+	PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{ PCI_VENDOR_ID_ASP, PCI_DEVICE_ID_38C1600_REV1,
+	PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0},
+	{ }
+};
+MODULE_DEVICE_TABLE(pci, advansys_pci_tbl);
+
diff --git a/drivers/scsi/aha152x.c b/drivers/scsi/aha152x.c
index f974869..fb6a476 100644
--- a/drivers/scsi/aha152x.c
+++ b/drivers/scsi/aha152x.c
@@ -253,6 +253,7 @@
 #include <linux/isapnp.h>
 #include <linux/spinlock.h>
 #include <linux/workqueue.h>
+#include <linux/list.h>
 #include <asm/semaphore.h>
 #include <scsi/scsicam.h>
 
@@ -262,6 +263,8 @@
 #include <scsi/scsi_transport_spi.h>
 #include "aha152x.h"
 
+static LIST_HEAD(aha152x_host_list);
+
 
 /* DEFINES */
 
@@ -423,8 +426,6 @@ MODULE_DEVICE_TABLE(isapnp, id_table);
 
 #endif /* !PCMCIA */
 
-static int registered_count=0;
-static struct Scsi_Host *aha152x_host[2];
 static struct scsi_host_template aha152x_driver_template;
 
 /*
@@ -541,6 +542,7 @@ struct aha152x_hostdata {
 #ifdef __ISAPNP__
 	struct pnp_dev *pnpdev;
 #endif
+	struct list_head host_list;
 };
 
 
@@ -755,20 +757,9 @@ static inline Scsi_Cmnd *remove_SC(Scsi_Cmnd **SC, Scsi_Cmnd *SCp)
 	return ptr;
 }
 
-static inline struct Scsi_Host *lookup_irq(int irqno)
-{
-	int i;
-
-	for(i=0; i<ARRAY_SIZE(aha152x_host); i++)
-		if(aha152x_host[i] && aha152x_host[i]->irq==irqno)
-			return aha152x_host[i];
-
-	return NULL;
-}
-
 static irqreturn_t swintr(int irqno, void *dev_id, struct pt_regs *regs)
 {
-	struct Scsi_Host *shpnt = lookup_irq(irqno);
+	struct Scsi_Host *shpnt = (struct Scsi_Host *)dev_id;
 
 	if (!shpnt) {
         	printk(KERN_ERR "aha152x: catched software interrupt %d for unknown controller.\n", irqno);
@@ -791,10 +782,11 @@ struct Scsi_Host *aha152x_probe_one(struct aha152x_setup *setup)
 		return NULL;
 	}
 
-	/* need to have host registered before triggering any interrupt */
-	aha152x_host[registered_count] = shpnt;
-
 	memset(HOSTDATA(shpnt), 0, sizeof *HOSTDATA(shpnt));
+	INIT_LIST_HEAD(&HOSTDATA(shpnt)->host_list);
+
+	/* need to have host registered before triggering any interrupt */
+	list_add_tail(&HOSTDATA(shpnt)->host_list, &aha152x_host_list);
 
 	shpnt->io_port   = setup->io_port;
 	shpnt->n_io_port = IO_RANGE;
@@ -907,12 +899,10 @@ struct Scsi_Host *aha152x_probe_one(struct aha152x_setup *setup)
 
 	scsi_scan_host(shpnt);
 
-	registered_count++;
-
 	return shpnt;
 
 out_host_put:
-	aha152x_host[registered_count]=NULL;
+	list_del(&HOSTDATA(shpnt)->host_list);
 	scsi_host_put(shpnt);
 
 	return NULL;
@@ -937,6 +927,7 @@ void aha152x_release(struct Scsi_Host *shpnt)
 #endif
 
 	scsi_remove_host(shpnt);
+	list_del(&HOSTDATA(shpnt)->host_list);
 	scsi_host_put(shpnt);
 }
 
@@ -1459,9 +1450,12 @@ static struct work_struct aha152x_tq;
  */
 static void run(void)
 {
-	int i;
-	for (i = 0; i<ARRAY_SIZE(aha152x_host); i++) {
-		is_complete(aha152x_host[i]);
+	struct aha152x_hostdata *hd;
+
+	list_for_each_entry(hd, &aha152x_host_list, host_list) {
+		struct Scsi_Host *shost = container_of((void *)hd, struct Scsi_Host, hostdata);
+
+		is_complete(shost);
 	}
 }
 
@@ -1471,7 +1465,7 @@ static void run(void)
  */
 static irqreturn_t intr(int irqno, void *dev_id, struct pt_regs *regs)
 {
-	struct Scsi_Host *shpnt = lookup_irq(irqno);
+	struct Scsi_Host *shpnt = (struct Scsi_Host *)dev_id;
 	unsigned long flags;
 	unsigned char rev, dmacntrl0;
 
@@ -3953,16 +3947,17 @@ static int __init aha152x_init(void)
 #endif
 	}
 
-	return registered_count>0;
+	return 1;
 }
 
 static void __exit aha152x_exit(void)
 {
-	int i;
+	struct aha152x_hostdata *hd;
+
+	list_for_each_entry(hd, &aha152x_host_list, host_list) {
+		struct Scsi_Host *shost = container_of((void *)hd, struct Scsi_Host, hostdata);
 
-	for(i=0; i<ARRAY_SIZE(setup); i++) {
-		aha152x_release(aha152x_host[i]);
-		aha152x_host[i]=NULL;
+		aha152x_release(shost);
 	}
 }
 
diff --git a/drivers/scsi/aic7xxx/aic79xx_osm.c b/drivers/scsi/aic7xxx/aic79xx_osm.c
index 998999c..c7eeace 100644
--- a/drivers/scsi/aic7xxx/aic79xx_osm.c
+++ b/drivers/scsi/aic7xxx/aic79xx_osm.c
@@ -321,7 +321,7 @@ MODULE_LICENSE("Dual BSD/GPL");
 MODULE_VERSION(AIC79XX_DRIVER_VERSION);
 module_param(aic79xx, charp, 0444);
 MODULE_PARM_DESC(aic79xx,
-"period delimited, options string.\n"
+"period-delimited options string:\n"
 "	verbose			Enable verbose/diagnostic logging\n"
 "	allow_memio		Allow device registers to be memory mapped\n"
 "	debug			Bitmask of debug values to enable\n"
@@ -346,7 +346,7 @@ MODULE_PARM_DESC(aic79xx,
 "		Shorten the selection timeout to 128ms\n"
 "\n"
 "	options aic79xx 'aic79xx=verbose.tag_info:{{}.{}.{..10}}.seltime:1'\n"
-"\n");
+);
 
 static void ahd_linux_handle_scsi_status(struct ahd_softc *,
 					 struct scsi_device *,
diff --git a/drivers/scsi/aic7xxx/aic7xxx_osm.c b/drivers/scsi/aic7xxx/aic7xxx_osm.c
index aa4be8a..64c8b88 100644
--- a/drivers/scsi/aic7xxx/aic7xxx_osm.c
+++ b/drivers/scsi/aic7xxx/aic7xxx_osm.c
@@ -341,7 +341,7 @@ MODULE_LICENSE("Dual BSD/GPL");
 MODULE_VERSION(AIC7XXX_DRIVER_VERSION);
 module_param(aic7xxx, charp, 0444);
 MODULE_PARM_DESC(aic7xxx,
-"period delimited, options string.\n"
+"period-delimited options string:\n"
 "	verbose			Enable verbose/diagnostic logging\n"
 "	allow_memio		Allow device registers to be memory mapped\n"
 "	debug			Bitmask of debug values to enable\n"
@@ -2539,15 +2539,28 @@ static void ahc_linux_set_iu(struct scsi_target *starget, int iu)
 static void ahc_linux_get_signalling(struct Scsi_Host *shost)
 {
 	struct ahc_softc *ahc = *(struct ahc_softc **)shost->hostdata;
-	u8 mode = ahc_inb(ahc, SBLKCTL);
+	unsigned long flags;
+	u8 mode;
 
-	if (mode & ENAB40)
-		spi_signalling(shost) = SPI_SIGNAL_LVD;
-	else if (mode & ENAB20)
+	if (!(ahc->features & AHC_ULTRA2)) {
+		/* non-LVD chipset, may not have SBLKCTL reg */
 		spi_signalling(shost) = 
 			ahc->features & AHC_HVD ?
 			SPI_SIGNAL_HVD :
 			SPI_SIGNAL_SE;
+		return;
+	}
+
+	ahc_lock(ahc, &flags);
+	ahc_pause(ahc);
+	mode = ahc_inb(ahc, SBLKCTL);
+	ahc_unpause(ahc);
+	ahc_unlock(ahc, &flags);
+
+	if (mode & ENAB40)
+		spi_signalling(shost) = SPI_SIGNAL_LVD;
+	else if (mode & ENAB20)
+		spi_signalling(shost) = SPI_SIGNAL_SE;
 	else
 		spi_signalling(shost) = SPI_SIGNAL_UNKNOWN;
 }
diff --git a/drivers/scsi/aic7xxx_old.c b/drivers/scsi/aic7xxx_old.c
index 3f85b5e..5dcef48 100644
--- a/drivers/scsi/aic7xxx_old.c
+++ b/drivers/scsi/aic7xxx_old.c
@@ -249,8 +249,6 @@
 #include <linux/stat.h>
 #include <linux/slab.h>        /* for kmalloc() */
 
-#include <linux/config.h>        /* for CONFIG_PCI */
-
 #define AIC7XXX_C_VERSION  "5.2.6"
 
 #define ALL_TARGETS -1
@@ -9196,7 +9194,7 @@ aic7xxx_detect(struct scsi_host_template *template)
     for (i = 0; i < ARRAY_SIZE(aic_pdevs); i++)
     {
       pdev = NULL;
-      while ((pdev = pci_find_device(aic_pdevs[i].vendor_id,
+      while ((pdev = pci_get_device(aic_pdevs[i].vendor_id,
                                      aic_pdevs[i].device_id,
                                      pdev))) {
 	if (pci_enable_device(pdev))
@@ -9653,6 +9651,9 @@ aic7xxx_detect(struct scsi_host_template *template)
            */
           aic7xxx_configure_bugs(temp_p);
 
+          /* Hold a pci device reference */
+          pci_dev_get(temp_p->pdev);
+
           if ( list_p == NULL )
           {
             list_p = current_p = temp_p;
@@ -10989,8 +10990,10 @@ aic7xxx_release(struct Scsi_Host *host)
   if(!p->pdev)
     release_region(p->base, MAXREG - MINREG);
 #ifdef CONFIG_PCI
-  else
+  else {
     pci_release_regions(p->pdev);
+    pci_dev_put(p->pdev);
+  }
 #endif
   prev = NULL;
   next = first_aic7xxx;
diff --git a/drivers/scsi/aic94xx/Kconfig b/drivers/scsi/aic94xx/Kconfig
new file mode 100644
index 0000000..0ed391d
--- /dev/null
+++ b/drivers/scsi/aic94xx/Kconfig
@@ -0,0 +1,41 @@
+#
+# Kernel configuration file for aic94xx SAS/SATA driver.
+#
+# Copyright (c) 2005 Adaptec, Inc.  All rights reserved.
+# Copyright (c) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+#
+# This file is licensed under GPLv2.
+#
+# This file is part of the aic94xx driver.
+#
+# The aic94xx driver is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2 of the
+# License.
+#
+# The aic94xx driver is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Aic94xx Driver; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+#
+#
+
+config SCSI_AIC94XX
+	tristate "Adaptec AIC94xx SAS/SATA support"
+	depends on PCI
+	select SCSI_SAS_LIBSAS
+	help
+		This driver supports Adaptec's SAS/SATA 3Gb/s 64 bit PCI-X
+		AIC94xx chip based host adapters.
+
+config AIC94XX_DEBUG
+	bool "Compile in debug mode"
+	default y
+	depends on SCSI_AIC94XX
+	help
+		Compiles the aic94xx driver in debug mode.  In debug mode,
+		the driver prints some messages to the console.
diff --git a/drivers/scsi/aic94xx/Makefile b/drivers/scsi/aic94xx/Makefile
new file mode 100644
index 0000000..e6b7012
--- /dev/null
+++ b/drivers/scsi/aic94xx/Makefile
@@ -0,0 +1,39 @@
+#
+# Makefile for Adaptec aic94xx SAS/SATA driver.
+#
+# Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+# Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+#
+# This file is licensed under GPLv2.
+#
+# This file is part of the the aic94xx driver.
+#
+# The aic94xx driver is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2 of the
+# License.
+#
+# The aic94xx driver is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with the aic94xx driver; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+
+ifeq ($(CONFIG_AIC94XX_DEBUG),y)
+	EXTRA_CFLAGS += -DASD_DEBUG -DASD_ENTER_EXIT
+endif
+
+obj-$(CONFIG_SCSI_AIC94XX) += aic94xx.o
+aic94xx-y += aic94xx_init.o \
+	     aic94xx_hwi.o  \
+	     aic94xx_reg.o  \
+	     aic94xx_sds.o  \
+	     aic94xx_seq.o  \
+	     aic94xx_dump.o \
+	     aic94xx_scb.o  \
+	     aic94xx_dev.o  \
+	     aic94xx_tmf.o  \
+	     aic94xx_task.o
diff --git a/drivers/scsi/aic94xx/aic94xx.h b/drivers/scsi/aic94xx/aic94xx.h
new file mode 100644
index 0000000..1bd5b4e
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx.h
@@ -0,0 +1,114 @@
+/*
+ * Aic94xx SAS/SATA driver header file.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * $Id: //depot/aic94xx/aic94xx.h#31 $
+ */
+
+#ifndef _AIC94XX_H_
+#define _AIC94XX_H_
+
+#include <linux/slab.h>
+#include <linux/ctype.h>
+#include <scsi/libsas.h>
+
+#define ASD_DRIVER_NAME		"aic94xx"
+#define ASD_DRIVER_DESCRIPTION	"Adaptec aic94xx SAS/SATA driver"
+
+#define asd_printk(fmt, ...)	printk(KERN_NOTICE ASD_DRIVER_NAME ": " fmt, ## __VA_ARGS__)
+
+#ifdef ASD_ENTER_EXIT
+#define ENTER  printk(KERN_NOTICE "%s: ENTER %s\n", ASD_DRIVER_NAME, \
+		__FUNCTION__)
+#define EXIT   printk(KERN_NOTICE "%s: --EXIT %s\n", ASD_DRIVER_NAME, \
+		__FUNCTION__)
+#else
+#define ENTER
+#define EXIT
+#endif
+
+#ifdef ASD_DEBUG
+#define ASD_DPRINTK asd_printk
+#else
+#define ASD_DPRINTK(fmt, ...)
+#endif
+
+/* 2*ITNL timeout + 1 second */
+#define AIC94XX_SCB_TIMEOUT  (5*HZ)
+
+extern kmem_cache_t *asd_dma_token_cache;
+extern kmem_cache_t *asd_ascb_cache;
+extern char sas_addr_str[2*SAS_ADDR_SIZE + 1];
+
+static inline void asd_stringify_sas_addr(char *p, const u8 *sas_addr)
+{
+	int i;
+	for (i = 0; i < SAS_ADDR_SIZE; i++, p += 2)
+		snprintf(p, 3, "%02X", sas_addr[i]);
+	*p = '\0';
+}
+
+static inline void asd_destringify_sas_addr(u8 *sas_addr, const char *p)
+{
+	int i;
+	for (i = 0; i < SAS_ADDR_SIZE; i++) {
+		u8 h, l;
+		if (!*p)
+			break;
+		h = isdigit(*p) ? *p-'0' : *p-'A'+10;
+		p++;
+		l = isdigit(*p) ? *p-'0' : *p-'A'+10;
+		p++;
+		sas_addr[i] = (h<<4) | l;
+	}
+}
+
+struct asd_ha_struct;
+struct asd_ascb;
+
+int  asd_read_ocm(struct asd_ha_struct *asd_ha);
+int  asd_read_flash(struct asd_ha_struct *asd_ha);
+
+int  asd_dev_found(struct domain_device *dev);
+void asd_dev_gone(struct domain_device *dev);
+
+void asd_invalidate_edb(struct asd_ascb *ascb, int edb_id);
+
+int  asd_execute_task(struct sas_task *, int num, unsigned long gfp_flags);
+
+/* ---------- TMFs ---------- */
+int  asd_abort_task(struct sas_task *);
+int  asd_abort_task_set(struct domain_device *, u8 *lun);
+int  asd_clear_aca(struct domain_device *, u8 *lun);
+int  asd_clear_task_set(struct domain_device *, u8 *lun);
+int  asd_lu_reset(struct domain_device *, u8 *lun);
+int  asd_query_task(struct sas_task *);
+
+/* ---------- Adapter and Port management ---------- */
+int  asd_clear_nexus_port(struct asd_sas_port *port);
+int  asd_clear_nexus_ha(struct sas_ha_struct *sas_ha);
+
+/* ---------- Phy Management ---------- */
+int  asd_control_phy(struct asd_sas_phy *phy, enum phy_func func, void *arg);
+
+#endif
diff --git a/drivers/scsi/aic94xx/aic94xx_dev.c b/drivers/scsi/aic94xx/aic94xx_dev.c
new file mode 100644
index 0000000..6f8901b
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_dev.c
@@ -0,0 +1,353 @@
+/*
+ * Aic94xx SAS/SATA DDB management
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * $Id: //depot/aic94xx/aic94xx_dev.c#21 $
+ */
+
+#include "aic94xx.h"
+#include "aic94xx_hwi.h"
+#include "aic94xx_reg.h"
+#include "aic94xx_sas.h"
+
+#define FIND_FREE_DDB(_ha) find_first_zero_bit((_ha)->hw_prof.ddb_bitmap, \
+					       (_ha)->hw_prof.max_ddbs)
+#define SET_DDB(_ddb, _ha) set_bit(_ddb, (_ha)->hw_prof.ddb_bitmap)
+#define CLEAR_DDB(_ddb, _ha) clear_bit(_ddb, (_ha)->hw_prof.ddb_bitmap)
+
+static inline int asd_get_ddb(struct asd_ha_struct *asd_ha)
+{
+	unsigned long flags;
+	int ddb, i;
+
+	spin_lock_irqsave(&asd_ha->hw_prof.ddb_lock, flags);
+	ddb = FIND_FREE_DDB(asd_ha);
+	if (ddb >= asd_ha->hw_prof.max_ddbs) {
+		ddb = -ENOMEM;
+		spin_unlock_irqrestore(&asd_ha->hw_prof.ddb_lock, flags);
+		goto out;
+	}
+	SET_DDB(ddb, asd_ha);
+	spin_unlock_irqrestore(&asd_ha->hw_prof.ddb_lock, flags);
+
+	for (i = 0; i < sizeof(struct asd_ddb_ssp_smp_target_port); i+= 4)
+		asd_ddbsite_write_dword(asd_ha, ddb, i, 0);
+out:
+	return ddb;
+}
+
+#define INIT_CONN_TAG   offsetof(struct asd_ddb_ssp_smp_target_port, init_conn_tag)
+#define DEST_SAS_ADDR   offsetof(struct asd_ddb_ssp_smp_target_port, dest_sas_addr)
+#define SEND_QUEUE_HEAD offsetof(struct asd_ddb_ssp_smp_target_port, send_queue_head)
+#define DDB_TYPE        offsetof(struct asd_ddb_ssp_smp_target_port, ddb_type)
+#define CONN_MASK       offsetof(struct asd_ddb_ssp_smp_target_port, conn_mask)
+#define DDB_TARG_FLAGS  offsetof(struct asd_ddb_ssp_smp_target_port, flags)
+#define DDB_TARG_FLAGS2 offsetof(struct asd_ddb_stp_sata_target_port, flags2)
+#define EXEC_QUEUE_TAIL offsetof(struct asd_ddb_ssp_smp_target_port, exec_queue_tail)
+#define SEND_QUEUE_TAIL offsetof(struct asd_ddb_ssp_smp_target_port, send_queue_tail)
+#define SISTER_DDB      offsetof(struct asd_ddb_ssp_smp_target_port, sister_ddb)
+#define MAX_CCONN       offsetof(struct asd_ddb_ssp_smp_target_port, max_concurrent_conn)
+#define NUM_CTX         offsetof(struct asd_ddb_ssp_smp_target_port, num_contexts)
+#define ATA_CMD_SCBPTR  offsetof(struct asd_ddb_stp_sata_target_port, ata_cmd_scbptr)
+#define SATA_TAG_ALLOC_MASK offsetof(struct asd_ddb_stp_sata_target_port, sata_tag_alloc_mask)
+#define NUM_SATA_TAGS   offsetof(struct asd_ddb_stp_sata_target_port, num_sata_tags)
+#define SATA_STATUS     offsetof(struct asd_ddb_stp_sata_target_port, sata_status)
+#define NCQ_DATA_SCB_PTR offsetof(struct asd_ddb_stp_sata_target_port, ncq_data_scb_ptr)
+#define ITNL_TIMEOUT    offsetof(struct asd_ddb_ssp_smp_target_port, itnl_timeout)
+
+static inline void asd_free_ddb(struct asd_ha_struct *asd_ha, int ddb)
+{
+	unsigned long flags;
+
+	if (!ddb || ddb >= 0xFFFF)
+		return;
+	asd_ddbsite_write_byte(asd_ha, ddb, DDB_TYPE, DDB_TYPE_UNUSED);
+	spin_lock_irqsave(&asd_ha->hw_prof.ddb_lock, flags);
+	CLEAR_DDB(ddb, asd_ha);
+	spin_unlock_irqrestore(&asd_ha->hw_prof.ddb_lock, flags);
+}
+
+static inline void asd_set_ddb_type(struct domain_device *dev)
+{
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+	int ddb = (int) (unsigned long) dev->lldd_dev;
+
+	if (dev->dev_type == SATA_PM_PORT)
+		asd_ddbsite_write_byte(asd_ha,ddb, DDB_TYPE, DDB_TYPE_PM_PORT);
+	else if (dev->tproto)
+		asd_ddbsite_write_byte(asd_ha,ddb, DDB_TYPE, DDB_TYPE_TARGET);
+	else
+		asd_ddbsite_write_byte(asd_ha,ddb,DDB_TYPE,DDB_TYPE_INITIATOR);
+}
+
+static int asd_init_sata_tag_ddb(struct domain_device *dev)
+{
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+	int ddb, i;
+
+	ddb = asd_get_ddb(asd_ha);
+	if (ddb < 0)
+		return ddb;
+
+	for (i = 0; i < sizeof(struct asd_ddb_sata_tag); i += 2)
+		asd_ddbsite_write_word(asd_ha, ddb, i, 0xFFFF);
+
+	asd_ddbsite_write_word(asd_ha, (int) (unsigned long) dev->lldd_dev,
+			       SISTER_DDB, ddb);
+	return 0;
+}
+
+static inline int asd_init_sata(struct domain_device *dev)
+{
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+	int ddb = (int) (unsigned long) dev->lldd_dev;
+	u32 qdepth = 0;
+	int res = 0;
+
+	asd_ddbsite_write_word(asd_ha, ddb, ATA_CMD_SCBPTR, 0xFFFF);
+	if ((dev->dev_type == SATA_DEV || dev->dev_type == SATA_PM_PORT) &&
+	    dev->sata_dev.identify_device &&
+	    dev->sata_dev.identify_device[10] != 0) {
+		u16 w75 = le16_to_cpu(dev->sata_dev.identify_device[75]);
+		u16 w76 = le16_to_cpu(dev->sata_dev.identify_device[76]);
+
+		if (w76 & 0x100) /* NCQ? */
+			qdepth = (w75 & 0x1F) + 1;
+		asd_ddbsite_write_dword(asd_ha, ddb, SATA_TAG_ALLOC_MASK,
+					(1<<qdepth)-1);
+		asd_ddbsite_write_byte(asd_ha, ddb, NUM_SATA_TAGS, qdepth);
+	}
+	if (dev->dev_type == SATA_DEV || dev->dev_type == SATA_PM ||
+	    dev->dev_type == SATA_PM_PORT) {
+		struct dev_to_host_fis *fis = (struct dev_to_host_fis *)
+			dev->frame_rcvd;
+		asd_ddbsite_write_byte(asd_ha, ddb, SATA_STATUS, fis->status);
+	}
+	asd_ddbsite_write_word(asd_ha, ddb, NCQ_DATA_SCB_PTR, 0xFFFF);
+	if (qdepth > 0)
+		res = asd_init_sata_tag_ddb(dev);
+	return res;
+}
+
+static int asd_init_target_ddb(struct domain_device *dev)
+{
+	int ddb, i;
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+	u8 flags = 0;
+
+	ddb = asd_get_ddb(asd_ha);
+	if (ddb < 0)
+		return ddb;
+
+	dev->lldd_dev = (void *) (unsigned long) ddb;
+
+	asd_ddbsite_write_byte(asd_ha, ddb, 0, DDB_TP_CONN_TYPE);
+	asd_ddbsite_write_byte(asd_ha, ddb, 1, 0);
+	asd_ddbsite_write_word(asd_ha, ddb, INIT_CONN_TAG, 0xFFFF);
+	for (i = 0; i < SAS_ADDR_SIZE; i++)
+		asd_ddbsite_write_byte(asd_ha, ddb, DEST_SAS_ADDR+i,
+				       dev->sas_addr[i]);
+	asd_ddbsite_write_word(asd_ha, ddb, SEND_QUEUE_HEAD, 0xFFFF);
+	asd_set_ddb_type(dev);
+	asd_ddbsite_write_byte(asd_ha, ddb, CONN_MASK, dev->port->phy_mask);
+	if (dev->port->oob_mode != SATA_OOB_MODE) {
+		flags |= OPEN_REQUIRED;
+		if ((dev->dev_type == SATA_DEV) ||
+		    (dev->tproto & SAS_PROTO_STP)) {
+			struct smp_resp *rps_resp = &dev->sata_dev.rps_resp;
+			if (rps_resp->frame_type == SMP_RESPONSE &&
+			    rps_resp->function == SMP_REPORT_PHY_SATA &&
+			    rps_resp->result == SMP_RESP_FUNC_ACC) {
+				if (rps_resp->rps.affil_valid)
+					flags |= STP_AFFIL_POL;
+				if (rps_resp->rps.affil_supp)
+					flags |= SUPPORTS_AFFIL;
+			}
+		} else {
+			flags |= CONCURRENT_CONN_SUPP;
+			if (!dev->parent &&
+			    (dev->dev_type == EDGE_DEV ||
+			     dev->dev_type == FANOUT_DEV))
+				asd_ddbsite_write_byte(asd_ha, ddb, MAX_CCONN,
+						       4);
+			else
+				asd_ddbsite_write_byte(asd_ha, ddb, MAX_CCONN,
+						       dev->pathways);
+			asd_ddbsite_write_byte(asd_ha, ddb, NUM_CTX, 1);
+		}
+	}
+	if (dev->dev_type == SATA_PM)
+		flags |= SATA_MULTIPORT;
+	asd_ddbsite_write_byte(asd_ha, ddb, DDB_TARG_FLAGS, flags);
+
+	flags = 0;
+	if (dev->tproto & SAS_PROTO_STP)
+		flags |= STP_CL_POL_NO_TX;
+	asd_ddbsite_write_byte(asd_ha, ddb, DDB_TARG_FLAGS2, flags);
+
+	asd_ddbsite_write_word(asd_ha, ddb, EXEC_QUEUE_TAIL, 0xFFFF);
+	asd_ddbsite_write_word(asd_ha, ddb, SEND_QUEUE_TAIL, 0xFFFF);
+	asd_ddbsite_write_word(asd_ha, ddb, SISTER_DDB, 0xFFFF);
+
+	if (dev->dev_type == SATA_DEV || (dev->tproto & SAS_PROTO_STP)) {
+		i = asd_init_sata(dev);
+		if (i < 0) {
+			asd_free_ddb(asd_ha, ddb);
+			return i;
+		}
+	}
+
+	if (dev->dev_type == SAS_END_DEV) {
+		struct sas_end_device *rdev = rphy_to_end_device(dev->rphy);
+		if (rdev->I_T_nexus_loss_timeout > 0)
+			asd_ddbsite_write_word(asd_ha, ddb, ITNL_TIMEOUT,
+					       min(rdev->I_T_nexus_loss_timeout,
+						   (u16)ITNL_TIMEOUT_CONST));
+		else
+			asd_ddbsite_write_word(asd_ha, ddb, ITNL_TIMEOUT,
+					       (u16)ITNL_TIMEOUT_CONST);
+	}
+	return 0;
+}
+
+static int asd_init_sata_pm_table_ddb(struct domain_device *dev)
+{
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+	int ddb, i;
+
+	ddb = asd_get_ddb(asd_ha);
+	if (ddb < 0)
+		return ddb;
+
+	for (i = 0; i < 32; i += 2)
+		asd_ddbsite_write_word(asd_ha, ddb, i, 0xFFFF);
+
+	asd_ddbsite_write_word(asd_ha, (int) (unsigned long) dev->lldd_dev,
+			       SISTER_DDB, ddb);
+
+	return 0;
+}
+
+#define PM_PORT_FLAGS offsetof(struct asd_ddb_sata_pm_port, pm_port_flags)
+#define PARENT_DDB    offsetof(struct asd_ddb_sata_pm_port, parent_ddb)
+
+/**
+ * asd_init_sata_pm_port_ddb -- SATA Port Multiplier Port
+ * dev: pointer to domain device
+ *
+ * For SATA Port Multiplier Ports we need to allocate one SATA Port
+ * Multiplier Port DDB and depending on whether the target on it
+ * supports SATA II NCQ, one SATA Tag DDB.
+ */
+static int asd_init_sata_pm_port_ddb(struct domain_device *dev)
+{
+	int ddb, i, parent_ddb, pmtable_ddb;
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+	u8  flags;
+
+	ddb = asd_get_ddb(asd_ha);
+	if (ddb < 0)
+		return ddb;
+
+	asd_set_ddb_type(dev);
+	flags = (dev->sata_dev.port_no << 4) | PM_PORT_SET;
+	asd_ddbsite_write_byte(asd_ha, ddb, PM_PORT_FLAGS, flags);
+	asd_ddbsite_write_word(asd_ha, ddb, SISTER_DDB, 0xFFFF);
+	asd_ddbsite_write_word(asd_ha, ddb, ATA_CMD_SCBPTR, 0xFFFF);
+	asd_init_sata(dev);
+
+	parent_ddb = (int) (unsigned long) dev->parent->lldd_dev;
+	asd_ddbsite_write_word(asd_ha, ddb, PARENT_DDB, parent_ddb);
+	pmtable_ddb = asd_ddbsite_read_word(asd_ha, parent_ddb, SISTER_DDB);
+	asd_ddbsite_write_word(asd_ha, pmtable_ddb, dev->sata_dev.port_no,ddb);
+
+	if (asd_ddbsite_read_byte(asd_ha, ddb, NUM_SATA_TAGS) > 0) {
+		i = asd_init_sata_tag_ddb(dev);
+		if (i < 0) {
+			asd_free_ddb(asd_ha, ddb);
+			return i;
+		}
+	}
+	return 0;
+}
+
+static int asd_init_initiator_ddb(struct domain_device *dev)
+{
+	return -ENODEV;
+}
+
+/**
+ * asd_init_sata_pm_ddb -- SATA Port Multiplier
+ * dev: pointer to domain device
+ *
+ * For STP and direct-attached SATA Port Multipliers we need
+ * one target port DDB entry and one SATA PM table DDB entry.
+ */
+static int asd_init_sata_pm_ddb(struct domain_device *dev)
+{
+	int res = 0;
+
+	res = asd_init_target_ddb(dev);
+	if (res)
+		goto out;
+	res = asd_init_sata_pm_table_ddb(dev);
+	if (res)
+		asd_free_ddb(dev->port->ha->lldd_ha,
+			     (int) (unsigned long) dev->lldd_dev);
+out:
+	return res;
+}
+
+int asd_dev_found(struct domain_device *dev)
+{
+	int res = 0;
+
+	switch (dev->dev_type) {
+	case SATA_PM:
+		res = asd_init_sata_pm_ddb(dev);
+		break;
+	case SATA_PM_PORT:
+		res = asd_init_sata_pm_port_ddb(dev);
+		break;
+	default:
+		if (dev->tproto)
+			res = asd_init_target_ddb(dev);
+		else
+			res = asd_init_initiator_ddb(dev);
+	}
+	return res;
+}
+
+void asd_dev_gone(struct domain_device *dev)
+{
+	int ddb, sister_ddb;
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+
+	ddb = (int) (unsigned long) dev->lldd_dev;
+	sister_ddb = asd_ddbsite_read_word(asd_ha, ddb, SISTER_DDB);
+
+	if (sister_ddb != 0xFFFF)
+		asd_free_ddb(asd_ha, sister_ddb);
+	asd_free_ddb(asd_ha, ddb);
+	dev->lldd_dev = NULL;
+}
diff --git a/drivers/scsi/aic94xx/aic94xx_dump.c b/drivers/scsi/aic94xx/aic94xx_dump.c
new file mode 100644
index 0000000..e6ade59
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_dump.c
@@ -0,0 +1,959 @@
+/*
+ * Aic94xx SAS/SATA driver dump interface.
+ *
+ * Copyright (C) 2004 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2004 David Chaw <david_chaw@adaptec.com>
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * 2005/07/14/LT  Complete overhaul of this file.  Update pages, register
+ * locations, names, etc.  Make use of macros.  Print more information.
+ * Print all cseq and lseq mip and mdp.
+ *
+ */
+
+#include "linux/pci.h"
+#include "aic94xx.h"
+#include "aic94xx_reg.h"
+#include "aic94xx_reg_def.h"
+#include "aic94xx_sas.h"
+
+#include "aic94xx_dump.h"
+
+#ifdef ASD_DEBUG
+
+#define MD(x)	    (1 << (x))
+#define MODE_COMMON (1 << 31)
+#define MODE_0_7    (0xFF)
+
+static const struct lseq_cio_regs {
+	char	*name;
+	u32	offs;
+	u8	width;
+	u32	mode;
+} LSEQmCIOREGS[] = {
+	{"LmMnSCBPTR",    0x20, 16, MD(0)|MD(1)|MD(2)|MD(3)|MD(4) },
+	{"LmMnDDBPTR",    0x22, 16, MD(0)|MD(1)|MD(2)|MD(3)|MD(4) },
+	{"LmREQMBX",      0x30, 32, MODE_COMMON },
+	{"LmRSPMBX",      0x34, 32, MODE_COMMON },
+	{"LmMnINT",       0x38, 32, MODE_0_7 },
+	{"LmMnINTEN",     0x3C, 32, MODE_0_7 },
+	{"LmXMTPRIMD",    0x40, 32, MODE_COMMON },
+	{"LmXMTPRIMCS",   0x44,  8, MODE_COMMON },
+	{"LmCONSTAT",     0x45,  8, MODE_COMMON },
+	{"LmMnDMAERRS",   0x46,  8, MD(0)|MD(1) },
+	{"LmMnSGDMAERRS", 0x47,  8, MD(0)|MD(1) },
+	{"LmMnEXPHDRP",   0x48,  8, MD(0) },
+	{"LmMnSASAALIGN", 0x48,  8, MD(1) },
+	{"LmMnMSKHDRP",   0x49,  8, MD(0) },
+	{"LmMnSTPALIGN",  0x49,  8, MD(1) },
+	{"LmMnRCVHDRP",   0x4A,  8, MD(0) },
+	{"LmMnXMTHDRP",   0x4A,  8, MD(1) },
+	{"LmALIGNMODE",   0x4B,  8, MD(1) },
+	{"LmMnEXPRCVCNT", 0x4C, 32, MD(0) },
+	{"LmMnXMTCNT",    0x4C, 32, MD(1) },
+	{"LmMnCURRTAG",   0x54, 16, MD(0) },
+	{"LmMnPREVTAG",   0x56, 16, MD(0) },
+	{"LmMnACKOFS",    0x58,  8, MD(1) },
+	{"LmMnXFRLVL",    0x59,  8, MD(0)|MD(1) },
+	{"LmMnSGDMACTL",  0x5A,  8, MD(0)|MD(1) },
+	{"LmMnSGDMASTAT", 0x5B,  8, MD(0)|MD(1) },
+	{"LmMnDDMACTL",   0x5C,  8, MD(0)|MD(1) },
+	{"LmMnDDMASTAT",  0x5D,  8, MD(0)|MD(1) },
+	{"LmMnDDMAMODE",  0x5E, 16, MD(0)|MD(1) },
+	{"LmMnPIPECTL",   0x61,  8, MD(0)|MD(1) },
+	{"LmMnACTSCB",    0x62, 16, MD(0)|MD(1) },
+	{"LmMnSGBHADR",   0x64,  8, MD(0)|MD(1) },
+	{"LmMnSGBADR",    0x65,  8, MD(0)|MD(1) },
+	{"LmMnSGDCNT",    0x66,  8, MD(0)|MD(1) },
+	{"LmMnSGDMADR",   0x68, 32, MD(0)|MD(1) },
+	{"LmMnSGDMADR",   0x6C, 32, MD(0)|MD(1) },
+	{"LmMnXFRCNT",    0x70, 32, MD(0)|MD(1) },
+	{"LmMnXMTCRC",    0x74, 32, MD(1) },
+	{"LmCURRTAG",     0x74, 16, MD(0) },
+	{"LmPREVTAG",     0x76, 16, MD(0) },
+	{"LmMnDPSEL",     0x7B,  8, MD(0)|MD(1) },
+	{"LmDPTHSTAT",    0x7C,  8, MODE_COMMON },
+	{"LmMnHOLDLVL",   0x7D,  8, MD(0) },
+	{"LmMnSATAFS",    0x7E,  8, MD(1) },
+	{"LmMnCMPLTSTAT", 0x7F,  8, MD(0)|MD(1) },
+	{"LmPRMSTAT0",    0x80, 32, MODE_COMMON },
+	{"LmPRMSTAT1",    0x84, 32, MODE_COMMON },
+	{"LmGPRMINT",     0x88,  8, MODE_COMMON },
+        {"LmMnCURRSCB",   0x8A, 16, MD(0) },
+	{"LmPRMICODE",    0x8C, 32, MODE_COMMON },
+	{"LmMnRCVCNT",    0x90, 16, MD(0) },
+	{"LmMnBUFSTAT",   0x92, 16, MD(0) },
+	{"LmMnXMTHDRSIZE",0x92,  8, MD(1) },
+	{"LmMnXMTSIZE",   0x93,  8, MD(1) },
+	{"LmMnTGTXFRCNT", 0x94, 32, MD(0) },
+	{"LmMnEXPROFS",   0x98, 32, MD(0) },
+	{"LmMnXMTROFS",   0x98, 32, MD(1) },
+	{"LmMnRCVROFS",   0x9C, 32, MD(0) },
+	{"LmCONCTL",      0xA0, 16, MODE_COMMON },
+	{"LmBITLTIMER",   0xA2, 16, MODE_COMMON },
+	{"LmWWNLOW",      0xA8, 32, MODE_COMMON },
+	{"LmWWNHIGH",     0xAC, 32, MODE_COMMON },
+	{"LmMnFRMERR",    0xB0, 32, MD(0) },
+	{"LmMnFRMERREN",  0xB4, 32, MD(0) },
+	{"LmAWTIMER",     0xB8, 16, MODE_COMMON },
+	{"LmAWTCTL",      0xBA,  8, MODE_COMMON },
+	{"LmMnHDRCMPS",   0xC0, 32, MD(0) },
+	{"LmMnXMTSTAT",   0xC4,  8, MD(1) },
+	{"LmHWTSTATEN",   0xC5,  8, MODE_COMMON },
+	{"LmMnRRDYRC",    0xC6,  8, MD(0) },
+        {"LmMnRRDYTC",    0xC6,  8, MD(1) },
+	{"LmHWTSTAT",     0xC7,  8, MODE_COMMON },
+	{"LmMnDATABUFADR",0xC8, 16, MD(0)|MD(1) },
+	{"LmDWSSTATUS",   0xCB,  8, MODE_COMMON },
+	{"LmMnACTSTAT",   0xCE, 16, MD(0)|MD(1) },
+	{"LmMnREQSCB",    0xD2, 16, MD(0)|MD(1) },
+	{"LmXXXPRIM",     0xD4, 32, MODE_COMMON },
+	{"LmRCVASTAT",    0xD9,  8, MODE_COMMON },
+	{"LmINTDIS1",     0xDA,  8, MODE_COMMON },
+	{"LmPSTORESEL",   0xDB,  8, MODE_COMMON },
+	{"LmPSTORE",      0xDC, 32, MODE_COMMON },
+	{"LmPRIMSTAT0EN", 0xE0, 32, MODE_COMMON },
+	{"LmPRIMSTAT1EN", 0xE4, 32, MODE_COMMON },
+	{"LmDONETCTL",    0xF2, 16, MODE_COMMON },
+	{NULL, 0, 0, 0 }
+};
+/*
+static struct lseq_cio_regs LSEQmOOBREGS[] = {
+   {"OOB_BFLTR"        ,0x100, 8, MD(5)},
+   {"OOB_INIT_MIN"     ,0x102,16, MD(5)},
+   {"OOB_INIT_MAX"     ,0x104,16, MD(5)},
+   {"OOB_INIT_NEG"     ,0x106,16, MD(5)},
+   {"OOB_SAS_MIN"      ,0x108,16, MD(5)},
+   {"OOB_SAS_MAX"      ,0x10A,16, MD(5)},
+   {"OOB_SAS_NEG"      ,0x10C,16, MD(5)},
+   {"OOB_WAKE_MIN"     ,0x10E,16, MD(5)},
+   {"OOB_WAKE_MAX"     ,0x110,16, MD(5)},
+   {"OOB_WAKE_NEG"     ,0x112,16, MD(5)},
+   {"OOB_IDLE_MAX"     ,0x114,16, MD(5)},
+   {"OOB_BURST_MAX"    ,0x116,16, MD(5)},
+   {"OOB_XMIT_BURST"   ,0x118, 8, MD(5)},
+   {"OOB_SEND_PAIRS"   ,0x119, 8, MD(5)},
+   {"OOB_INIT_IDLE"    ,0x11A, 8, MD(5)},
+   {"OOB_INIT_NEGO"    ,0x11C, 8, MD(5)},
+   {"OOB_SAS_IDLE"     ,0x11E, 8, MD(5)},
+   {"OOB_SAS_NEGO"     ,0x120, 8, MD(5)},
+   {"OOB_WAKE_IDLE"    ,0x122, 8, MD(5)},
+   {"OOB_WAKE_NEGO"    ,0x124, 8, MD(5)},
+   {"OOB_DATA_KBITS"   ,0x126, 8, MD(5)},
+   {"OOB_BURST_DATA"   ,0x128,32, MD(5)},
+   {"OOB_ALIGN_0_DATA" ,0x12C,32, MD(5)},
+   {"OOB_ALIGN_1_DATA" ,0x130,32, MD(5)},
+   {"OOB_SYNC_DATA"    ,0x134,32, MD(5)},
+   {"OOB_D10_2_DATA"   ,0x138,32, MD(5)},
+   {"OOB_PHY_RST_CNT"  ,0x13C,32, MD(5)},
+   {"OOB_SIG_GEN"      ,0x140, 8, MD(5)},
+   {"OOB_XMIT"         ,0x141, 8, MD(5)},
+   {"FUNCTION_MAKS"    ,0x142, 8, MD(5)},
+   {"OOB_MODE"         ,0x143, 8, MD(5)},
+   {"CURRENT_STATUS"   ,0x144, 8, MD(5)},
+   {"SPEED_MASK"       ,0x145, 8, MD(5)},
+   {"PRIM_COUNT"       ,0x146, 8, MD(5)},
+   {"OOB_SIGNALS"      ,0x148, 8, MD(5)},
+   {"OOB_DATA_DET"     ,0x149, 8, MD(5)},
+   {"OOB_TIME_OUT"     ,0x14C, 8, MD(5)},
+   {"OOB_TIMER_ENABLE" ,0x14D, 8, MD(5)},
+   {"OOB_STATUS"       ,0x14E, 8, MD(5)},
+   {"HOT_PLUG_DELAY"   ,0x150, 8, MD(5)},
+   {"RCD_DELAY"        ,0x151, 8, MD(5)},
+   {"COMSAS_TIMER"     ,0x152, 8, MD(5)},
+   {"SNTT_DELAY"       ,0x153, 8, MD(5)},
+   {"SPD_CHNG_DELAY"   ,0x154, 8, MD(5)},
+   {"SNLT_DELAY"       ,0x155, 8, MD(5)},
+   {"SNWT_DELAY"       ,0x156, 8, MD(5)},
+   {"ALIGN_DELAY"      ,0x157, 8, MD(5)},
+   {"INT_ENABLE_0"     ,0x158, 8, MD(5)},
+   {"INT_ENABLE_1"     ,0x159, 8, MD(5)},
+   {"INT_ENABLE_2"     ,0x15A, 8, MD(5)},
+   {"INT_ENABLE_3"     ,0x15B, 8, MD(5)},
+   {"OOB_TEST_REG"     ,0x15C, 8, MD(5)},
+   {"PHY_CONTROL_0"    ,0x160, 8, MD(5)},
+   {"PHY_CONTROL_1"    ,0x161, 8, MD(5)},
+   {"PHY_CONTROL_2"    ,0x162, 8, MD(5)},
+   {"PHY_CONTROL_3"    ,0x163, 8, MD(5)},
+   {"PHY_OOB_CAL_TX"   ,0x164, 8, MD(5)},
+   {"PHY_OOB_CAL_RX"   ,0x165, 8, MD(5)},
+   {"OOB_PHY_CAL_TX"   ,0x166, 8, MD(5)},
+   {"OOB_PHY_CAL_RX"   ,0x167, 8, MD(5)},
+   {"PHY_CONTROL_4"    ,0x168, 8, MD(5)},
+   {"PHY_TEST"         ,0x169, 8, MD(5)},
+   {"PHY_PWR_CTL"      ,0x16A, 8, MD(5)},
+   {"PHY_PWR_DELAY"    ,0x16B, 8, MD(5)},
+   {"OOB_SM_CON"       ,0x16C, 8, MD(5)},
+   {"ADDR_TRAP_1"      ,0x16D, 8, MD(5)},
+   {"ADDR_NEXT_1"      ,0x16E, 8, MD(5)},
+   {"NEXT_ST_1"        ,0x16F, 8, MD(5)},
+   {"OOB_SM_STATE"     ,0x170, 8, MD(5)},
+   {"ADDR_TRAP_2"      ,0x171, 8, MD(5)},
+   {"ADDR_NEXT_2"      ,0x172, 8, MD(5)},
+   {"NEXT_ST_2"        ,0x173, 8, MD(5)},
+   {NULL, 0, 0, 0 }
+};
+*/
+#define STR_8BIT   "   %30s[0x%04x]:0x%02x\n"
+#define STR_16BIT  "   %30s[0x%04x]:0x%04x\n"
+#define STR_32BIT  "   %30s[0x%04x]:0x%08x\n"
+#define STR_64BIT  "   %30s[0x%04x]:0x%llx\n"
+
+#define PRINT_REG_8bit(_ha, _n, _r) asd_printk(STR_8BIT, #_n, _n,      \
+					     asd_read_reg_byte(_ha, _r))
+#define PRINT_REG_16bit(_ha, _n, _r) asd_printk(STR_16BIT, #_n, _n,     \
+					      asd_read_reg_word(_ha, _r))
+#define PRINT_REG_32bit(_ha, _n, _r) asd_printk(STR_32BIT, #_n, _n,      \
+					      asd_read_reg_dword(_ha, _r))
+
+#define PRINT_CREG_8bit(_ha, _n) asd_printk(STR_8BIT, #_n, _n,      \
+					     asd_read_reg_byte(_ha, C##_n))
+#define PRINT_CREG_16bit(_ha, _n) asd_printk(STR_16BIT, #_n, _n,     \
+					      asd_read_reg_word(_ha, C##_n))
+#define PRINT_CREG_32bit(_ha, _n) asd_printk(STR_32BIT, #_n, _n,      \
+					      asd_read_reg_dword(_ha, C##_n))
+
+#define MSTR_8BIT   "   Mode:%02d %30s[0x%04x]:0x%02x\n"
+#define MSTR_16BIT  "   Mode:%02d %30s[0x%04x]:0x%04x\n"
+#define MSTR_32BIT  "   Mode:%02d %30s[0x%04x]:0x%08x\n"
+
+#define PRINT_MREG_8bit(_ha, _m, _n, _r) asd_printk(MSTR_8BIT, _m, #_n, _n,   \
+					     asd_read_reg_byte(_ha, _r))
+#define PRINT_MREG_16bit(_ha, _m, _n, _r) asd_printk(MSTR_16BIT, _m, #_n, _n, \
+					      asd_read_reg_word(_ha, _r))
+#define PRINT_MREG_32bit(_ha, _m, _n, _r) asd_printk(MSTR_32BIT, _m, #_n, _n, \
+					      asd_read_reg_dword(_ha, _r))
+
+/* can also be used for MD when the register is mode aware already */
+#define PRINT_MIS_byte(_ha, _n) asd_printk(STR_8BIT, #_n,CSEQ_##_n-CMAPPEDSCR,\
+                                           asd_read_reg_byte(_ha, CSEQ_##_n))
+#define PRINT_MIS_word(_ha, _n) asd_printk(STR_16BIT,#_n,CSEQ_##_n-CMAPPEDSCR,\
+                                           asd_read_reg_word(_ha, CSEQ_##_n))
+#define PRINT_MIS_dword(_ha, _n)                      \
+        asd_printk(STR_32BIT,#_n,CSEQ_##_n-CMAPPEDSCR,\
+                   asd_read_reg_dword(_ha, CSEQ_##_n))
+#define PRINT_MIS_qword(_ha, _n)                                       \
+        asd_printk(STR_64BIT, #_n,CSEQ_##_n-CMAPPEDSCR,                \
+                   (unsigned long long)(((u64)asd_read_reg_dword(_ha, CSEQ_##_n))     \
+                 | (((u64)asd_read_reg_dword(_ha, (CSEQ_##_n)+4))<<32)))
+
+#define CMDP_REG(_n, _m) (_m*(CSEQ_PAGE_SIZE*2)+CSEQ_##_n)
+#define PRINT_CMDP_word(_ha, _n) \
+asd_printk("%20s 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x\n", \
+	#_n, \
+	asd_read_reg_word(_ha, CMDP_REG(_n, 0)), \
+	asd_read_reg_word(_ha, CMDP_REG(_n, 1)), \
+	asd_read_reg_word(_ha, CMDP_REG(_n, 2)), \
+	asd_read_reg_word(_ha, CMDP_REG(_n, 3)), \
+	asd_read_reg_word(_ha, CMDP_REG(_n, 4)), \
+	asd_read_reg_word(_ha, CMDP_REG(_n, 5)), \
+	asd_read_reg_word(_ha, CMDP_REG(_n, 6)), \
+	asd_read_reg_word(_ha, CMDP_REG(_n, 7)))
+
+#define PRINT_CMDP_byte(_ha, _n) \
+asd_printk("%20s 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x 0x%04x\n", \
+	#_n, \
+	asd_read_reg_byte(_ha, CMDP_REG(_n, 0)), \
+	asd_read_reg_byte(_ha, CMDP_REG(_n, 1)), \
+	asd_read_reg_byte(_ha, CMDP_REG(_n, 2)), \
+	asd_read_reg_byte(_ha, CMDP_REG(_n, 3)), \
+	asd_read_reg_byte(_ha, CMDP_REG(_n, 4)), \
+	asd_read_reg_byte(_ha, CMDP_REG(_n, 5)), \
+	asd_read_reg_byte(_ha, CMDP_REG(_n, 6)), \
+	asd_read_reg_byte(_ha, CMDP_REG(_n, 7)))
+
+static void asd_dump_cseq_state(struct asd_ha_struct *asd_ha)
+{
+	int mode;
+
+	asd_printk("CSEQ STATE\n");
+
+	asd_printk("ARP2 REGISTERS\n");
+
+	PRINT_CREG_32bit(asd_ha, ARP2CTL);
+	PRINT_CREG_32bit(asd_ha, ARP2INT);
+	PRINT_CREG_32bit(asd_ha, ARP2INTEN);
+	PRINT_CREG_8bit(asd_ha, MODEPTR);
+	PRINT_CREG_8bit(asd_ha, ALTMODE);
+	PRINT_CREG_8bit(asd_ha, FLAG);
+	PRINT_CREG_8bit(asd_ha, ARP2INTCTL);
+	PRINT_CREG_16bit(asd_ha, STACK);
+	PRINT_CREG_16bit(asd_ha, PRGMCNT);
+	PRINT_CREG_16bit(asd_ha, ACCUM);
+	PRINT_CREG_16bit(asd_ha, SINDEX);
+	PRINT_CREG_16bit(asd_ha, DINDEX);
+	PRINT_CREG_8bit(asd_ha, SINDIR);
+	PRINT_CREG_8bit(asd_ha, DINDIR);
+	PRINT_CREG_8bit(asd_ha, JUMLDIR);
+	PRINT_CREG_8bit(asd_ha, ARP2HALTCODE);
+	PRINT_CREG_16bit(asd_ha, CURRADDR);
+	PRINT_CREG_16bit(asd_ha, LASTADDR);
+	PRINT_CREG_16bit(asd_ha, NXTLADDR);
+
+	asd_printk("IOP REGISTERS\n");
+
+	PRINT_REG_32bit(asd_ha, BISTCTL1, CBISTCTL);
+	PRINT_CREG_32bit(asd_ha, MAPPEDSCR);
+
+	asd_printk("CIO REGISTERS\n");
+
+	for (mode = 0; mode < 9; mode++)
+		PRINT_MREG_16bit(asd_ha, mode, MnSCBPTR, CMnSCBPTR(mode));
+	PRINT_MREG_16bit(asd_ha, 15, MnSCBPTR, CMnSCBPTR(15));
+
+	for (mode = 0; mode < 9; mode++)
+		PRINT_MREG_16bit(asd_ha, mode, MnDDBPTR, CMnDDBPTR(mode));
+	PRINT_MREG_16bit(asd_ha, 15, MnDDBPTR, CMnDDBPTR(15));
+
+	for (mode = 0; mode < 8; mode++)
+		PRINT_MREG_32bit(asd_ha, mode, MnREQMBX, CMnREQMBX(mode));
+	for (mode = 0; mode < 8; mode++)
+		PRINT_MREG_32bit(asd_ha, mode, MnRSPMBX, CMnRSPMBX(mode));
+	for (mode = 0; mode < 8; mode++)
+		PRINT_MREG_32bit(asd_ha, mode, MnINT, CMnINT(mode));
+	for (mode = 0; mode < 8; mode++)
+		PRINT_MREG_32bit(asd_ha, mode, MnINTEN, CMnINTEN(mode));
+
+	PRINT_CREG_8bit(asd_ha, SCRATCHPAGE);
+	for (mode = 0; mode < 8; mode++)
+		PRINT_MREG_8bit(asd_ha, mode, MnSCRATCHPAGE,
+				CMnSCRATCHPAGE(mode));
+
+	PRINT_REG_32bit(asd_ha, CLINKCON, CLINKCON);
+	PRINT_REG_8bit(asd_ha, CCONMSK, CCONMSK);
+	PRINT_REG_8bit(asd_ha, CCONEXIST, CCONEXIST);
+	PRINT_REG_16bit(asd_ha, CCONMODE, CCONMODE);
+	PRINT_REG_32bit(asd_ha, CTIMERCALC, CTIMERCALC);
+	PRINT_REG_8bit(asd_ha, CINTDIS, CINTDIS);
+
+	asd_printk("SCRATCH MEMORY\n");
+
+	asd_printk("MIP 4 >>>>>\n");
+	PRINT_MIS_word(asd_ha, Q_EXE_HEAD);
+	PRINT_MIS_word(asd_ha, Q_EXE_TAIL);
+	PRINT_MIS_word(asd_ha, Q_DONE_HEAD);
+	PRINT_MIS_word(asd_ha, Q_DONE_TAIL);
+	PRINT_MIS_word(asd_ha, Q_SEND_HEAD);
+	PRINT_MIS_word(asd_ha, Q_SEND_TAIL);
+	PRINT_MIS_word(asd_ha, Q_DMA2CHIM_HEAD);
+	PRINT_MIS_word(asd_ha, Q_DMA2CHIM_TAIL);
+	PRINT_MIS_word(asd_ha, Q_COPY_HEAD);
+	PRINT_MIS_word(asd_ha, Q_COPY_TAIL);
+	PRINT_MIS_word(asd_ha, REG0);
+	PRINT_MIS_word(asd_ha, REG1);
+	PRINT_MIS_dword(asd_ha, REG2);
+	PRINT_MIS_byte(asd_ha, LINK_CTL_Q_MAP);
+	PRINT_MIS_byte(asd_ha, MAX_CSEQ_MODE);
+	PRINT_MIS_byte(asd_ha, FREE_LIST_HACK_COUNT);
+
+	asd_printk("MIP 5 >>>>\n");
+	PRINT_MIS_qword(asd_ha, EST_NEXUS_REQ_QUEUE);
+	PRINT_MIS_qword(asd_ha, EST_NEXUS_REQ_COUNT);
+	PRINT_MIS_word(asd_ha, Q_EST_NEXUS_HEAD);
+	PRINT_MIS_word(asd_ha, Q_EST_NEXUS_TAIL);
+	PRINT_MIS_word(asd_ha, NEED_EST_NEXUS_SCB);
+	PRINT_MIS_byte(asd_ha, EST_NEXUS_REQ_HEAD);
+	PRINT_MIS_byte(asd_ha, EST_NEXUS_REQ_TAIL);
+	PRINT_MIS_byte(asd_ha, EST_NEXUS_SCB_OFFSET);
+
+	asd_printk("MIP 6 >>>>\n");
+	PRINT_MIS_word(asd_ha, INT_ROUT_RET_ADDR0);
+	PRINT_MIS_word(asd_ha, INT_ROUT_RET_ADDR1);
+	PRINT_MIS_word(asd_ha, INT_ROUT_SCBPTR);
+	PRINT_MIS_byte(asd_ha, INT_ROUT_MODE);
+	PRINT_MIS_byte(asd_ha, ISR_SCRATCH_FLAGS);
+	PRINT_MIS_word(asd_ha, ISR_SAVE_SINDEX);
+	PRINT_MIS_word(asd_ha, ISR_SAVE_DINDEX);
+	PRINT_MIS_word(asd_ha, Q_MONIRTT_HEAD);
+	PRINT_MIS_word(asd_ha, Q_MONIRTT_TAIL);
+	PRINT_MIS_byte(asd_ha, FREE_SCB_MASK);
+	PRINT_MIS_word(asd_ha, BUILTIN_FREE_SCB_HEAD);
+	PRINT_MIS_word(asd_ha, BUILTIN_FREE_SCB_TAIL);
+	PRINT_MIS_word(asd_ha, EXTENDED_FREE_SCB_HEAD);
+	PRINT_MIS_word(asd_ha, EXTENDED_FREE_SCB_TAIL);
+
+	asd_printk("MIP 7 >>>>\n");
+	PRINT_MIS_qword(asd_ha, EMPTY_REQ_QUEUE);
+	PRINT_MIS_qword(asd_ha, EMPTY_REQ_COUNT);
+	PRINT_MIS_word(asd_ha, Q_EMPTY_HEAD);
+	PRINT_MIS_word(asd_ha, Q_EMPTY_TAIL);
+	PRINT_MIS_word(asd_ha, NEED_EMPTY_SCB);
+	PRINT_MIS_byte(asd_ha, EMPTY_REQ_HEAD);
+	PRINT_MIS_byte(asd_ha, EMPTY_REQ_TAIL);
+	PRINT_MIS_byte(asd_ha, EMPTY_SCB_OFFSET);
+	PRINT_MIS_word(asd_ha, PRIMITIVE_DATA);
+	PRINT_MIS_dword(asd_ha, TIMEOUT_CONST);
+
+	asd_printk("MDP 0 >>>>\n");
+	asd_printk("%-20s %6s %6s %6s %6s %6s %6s %6s %6s\n",
+		   "Mode: ", "0", "1", "2", "3", "4", "5", "6", "7");
+	PRINT_CMDP_word(asd_ha, LRM_SAVE_SINDEX);
+	PRINT_CMDP_word(asd_ha, LRM_SAVE_SCBPTR);
+	PRINT_CMDP_word(asd_ha, Q_LINK_HEAD);
+	PRINT_CMDP_word(asd_ha, Q_LINK_TAIL);
+	PRINT_CMDP_byte(asd_ha, LRM_SAVE_SCRPAGE);
+
+	asd_printk("MDP 0 Mode 8 >>>>\n");
+	PRINT_MIS_word(asd_ha, RET_ADDR);
+	PRINT_MIS_word(asd_ha, RET_SCBPTR);
+	PRINT_MIS_word(asd_ha, SAVE_SCBPTR);
+	PRINT_MIS_word(asd_ha, EMPTY_TRANS_CTX);
+	PRINT_MIS_word(asd_ha, RESP_LEN);
+	PRINT_MIS_word(asd_ha, TMF_SCBPTR);
+	PRINT_MIS_word(asd_ha, GLOBAL_PREV_SCB);
+	PRINT_MIS_word(asd_ha, GLOBAL_HEAD);
+	PRINT_MIS_word(asd_ha, CLEAR_LU_HEAD);
+	PRINT_MIS_byte(asd_ha, TMF_OPCODE);
+	PRINT_MIS_byte(asd_ha, SCRATCH_FLAGS);
+	PRINT_MIS_word(asd_ha, HSB_SITE);
+	PRINT_MIS_word(asd_ha, FIRST_INV_SCB_SITE);
+	PRINT_MIS_word(asd_ha, FIRST_INV_DDB_SITE);
+
+	asd_printk("MDP 1 Mode 8 >>>>\n");
+	PRINT_MIS_qword(asd_ha, LUN_TO_CLEAR);
+	PRINT_MIS_qword(asd_ha, LUN_TO_CHECK);
+
+	asd_printk("MDP 2 Mode 8 >>>>\n");
+	PRINT_MIS_qword(asd_ha, HQ_NEW_POINTER);
+	PRINT_MIS_qword(asd_ha, HQ_DONE_BASE);
+	PRINT_MIS_dword(asd_ha, HQ_DONE_POINTER);
+	PRINT_MIS_byte(asd_ha, HQ_DONE_PASS);
+}
+
+#define PRINT_LREG_8bit(_h, _lseq, _n) \
+        asd_printk(STR_8BIT, #_n, _n, asd_read_reg_byte(_h, Lm##_n(_lseq)))
+#define PRINT_LREG_16bit(_h, _lseq, _n) \
+        asd_printk(STR_16BIT, #_n, _n, asd_read_reg_word(_h, Lm##_n(_lseq)))
+#define PRINT_LREG_32bit(_h, _lseq, _n) \
+        asd_printk(STR_32BIT, #_n, _n, asd_read_reg_dword(_h, Lm##_n(_lseq)))
+
+#define PRINT_LMIP_byte(_h, _lseq, _n)                              \
+	asd_printk(STR_8BIT, #_n, LmSEQ_##_n(_lseq)-LmSCRATCH(_lseq), \
+		   asd_read_reg_byte(_h, LmSEQ_##_n(_lseq)))
+#define PRINT_LMIP_word(_h, _lseq, _n)                              \
+	asd_printk(STR_16BIT, #_n, LmSEQ_##_n(_lseq)-LmSCRATCH(_lseq), \
+		   asd_read_reg_word(_h, LmSEQ_##_n(_lseq)))
+#define PRINT_LMIP_dword(_h, _lseq, _n)                             \
+	asd_printk(STR_32BIT, #_n, LmSEQ_##_n(_lseq)-LmSCRATCH(_lseq), \
+		   asd_read_reg_dword(_h, LmSEQ_##_n(_lseq)))
+#define PRINT_LMIP_qword(_h, _lseq, _n)                                \
+	asd_printk(STR_64BIT, #_n, LmSEQ_##_n(_lseq)-LmSCRATCH(_lseq), \
+		 (unsigned long long)(((unsigned long long) \
+		 asd_read_reg_dword(_h, LmSEQ_##_n(_lseq))) \
+	          | (((unsigned long long) \
+		 asd_read_reg_dword(_h, LmSEQ_##_n(_lseq)+4))<<32)))
+
+static void asd_print_lseq_cio_reg(struct asd_ha_struct *asd_ha,
+				   u32 lseq_cio_addr, int i)
+{
+	switch (LSEQmCIOREGS[i].width) {
+	case 8:
+		asd_printk("%20s[0x%x]: 0x%02x\n", LSEQmCIOREGS[i].name,
+			   LSEQmCIOREGS[i].offs,
+			   asd_read_reg_byte(asd_ha, lseq_cio_addr +
+					     LSEQmCIOREGS[i].offs));
+
+		break;
+	case 16:
+		asd_printk("%20s[0x%x]: 0x%04x\n", LSEQmCIOREGS[i].name,
+			   LSEQmCIOREGS[i].offs,
+			   asd_read_reg_word(asd_ha, lseq_cio_addr +
+					     LSEQmCIOREGS[i].offs));
+
+		break;
+	case 32:
+		asd_printk("%20s[0x%x]: 0x%08x\n", LSEQmCIOREGS[i].name,
+			   LSEQmCIOREGS[i].offs,
+			   asd_read_reg_dword(asd_ha, lseq_cio_addr +
+					      LSEQmCIOREGS[i].offs));
+		break;
+	}
+}
+
+static void asd_dump_lseq_state(struct asd_ha_struct *asd_ha, int lseq)
+{
+	u32 moffs;
+	int mode;
+
+	asd_printk("LSEQ %d STATE\n", lseq);
+
+	asd_printk("LSEQ%d: ARP2 REGISTERS\n", lseq);
+	PRINT_LREG_32bit(asd_ha, lseq, ARP2CTL);
+	PRINT_LREG_32bit(asd_ha, lseq, ARP2INT);
+	PRINT_LREG_32bit(asd_ha, lseq, ARP2INTEN);
+	PRINT_LREG_8bit(asd_ha, lseq, MODEPTR);
+	PRINT_LREG_8bit(asd_ha, lseq, ALTMODE);
+	PRINT_LREG_8bit(asd_ha, lseq, FLAG);
+	PRINT_LREG_8bit(asd_ha, lseq, ARP2INTCTL);
+	PRINT_LREG_16bit(asd_ha, lseq, STACK);
+	PRINT_LREG_16bit(asd_ha, lseq, PRGMCNT);
+	PRINT_LREG_16bit(asd_ha, lseq, ACCUM);
+	PRINT_LREG_16bit(asd_ha, lseq, SINDEX);
+	PRINT_LREG_16bit(asd_ha, lseq, DINDEX);
+	PRINT_LREG_8bit(asd_ha, lseq, SINDIR);
+	PRINT_LREG_8bit(asd_ha, lseq, DINDIR);
+	PRINT_LREG_8bit(asd_ha, lseq, JUMLDIR);
+	PRINT_LREG_8bit(asd_ha, lseq, ARP2HALTCODE);
+	PRINT_LREG_16bit(asd_ha, lseq, CURRADDR);
+	PRINT_LREG_16bit(asd_ha, lseq, LASTADDR);
+	PRINT_LREG_16bit(asd_ha, lseq, NXTLADDR);
+
+	asd_printk("LSEQ%d: IOP REGISTERS\n", lseq);
+
+	PRINT_LREG_32bit(asd_ha, lseq, MODECTL);
+	PRINT_LREG_32bit(asd_ha, lseq, DBGMODE);
+	PRINT_LREG_32bit(asd_ha, lseq, CONTROL);
+	PRINT_REG_32bit(asd_ha, BISTCTL0, LmBISTCTL0(lseq));
+	PRINT_REG_32bit(asd_ha, BISTCTL1, LmBISTCTL1(lseq));
+
+	asd_printk("LSEQ%d: CIO REGISTERS\n", lseq);
+	asd_printk("Mode common:\n");
+
+	for (mode = 0; mode < 8; mode++) {
+		u32 lseq_cio_addr = LmSEQ_PHY_BASE(mode, lseq);
+		int i;
+
+		for (i = 0; LSEQmCIOREGS[i].name; i++)
+			if (LSEQmCIOREGS[i].mode == MODE_COMMON)
+				asd_print_lseq_cio_reg(asd_ha,lseq_cio_addr,i);
+	}
+
+	asd_printk("Mode unique:\n");
+	for (mode = 0; mode < 8; mode++) {
+		u32 lseq_cio_addr = LmSEQ_PHY_BASE(mode, lseq);
+		int i;
+
+		asd_printk("Mode %d\n", mode);
+		for  (i = 0; LSEQmCIOREGS[i].name; i++) {
+			if (!(LSEQmCIOREGS[i].mode & (1 << mode)))
+				continue;
+			asd_print_lseq_cio_reg(asd_ha, lseq_cio_addr, i);
+		}
+	}
+
+	asd_printk("SCRATCH MEMORY\n");
+
+	asd_printk("LSEQ%d MIP 0 >>>>\n", lseq);
+	PRINT_LMIP_word(asd_ha, lseq, Q_TGTXFR_HEAD);
+	PRINT_LMIP_word(asd_ha, lseq, Q_TGTXFR_TAIL);
+	PRINT_LMIP_byte(asd_ha, lseq, LINK_NUMBER);
+	PRINT_LMIP_byte(asd_ha, lseq, SCRATCH_FLAGS);
+	PRINT_LMIP_qword(asd_ha, lseq, CONNECTION_STATE);
+	PRINT_LMIP_word(asd_ha, lseq, CONCTL);
+	PRINT_LMIP_byte(asd_ha, lseq, CONSTAT);
+	PRINT_LMIP_byte(asd_ha, lseq, CONNECTION_MODES);
+	PRINT_LMIP_word(asd_ha, lseq, REG1_ISR);
+	PRINT_LMIP_word(asd_ha, lseq, REG2_ISR);
+	PRINT_LMIP_word(asd_ha, lseq, REG3_ISR);
+	PRINT_LMIP_qword(asd_ha, lseq,REG0_ISR);
+
+	asd_printk("LSEQ%d MIP 1 >>>>\n", lseq);
+	PRINT_LMIP_word(asd_ha, lseq, EST_NEXUS_SCBPTR0);
+	PRINT_LMIP_word(asd_ha, lseq, EST_NEXUS_SCBPTR1);
+	PRINT_LMIP_word(asd_ha, lseq, EST_NEXUS_SCBPTR2);
+	PRINT_LMIP_word(asd_ha, lseq, EST_NEXUS_SCBPTR3);
+	PRINT_LMIP_byte(asd_ha, lseq, EST_NEXUS_SCB_OPCODE0);
+	PRINT_LMIP_byte(asd_ha, lseq, EST_NEXUS_SCB_OPCODE1);
+	PRINT_LMIP_byte(asd_ha, lseq, EST_NEXUS_SCB_OPCODE2);
+	PRINT_LMIP_byte(asd_ha, lseq, EST_NEXUS_SCB_OPCODE3);
+	PRINT_LMIP_byte(asd_ha, lseq, EST_NEXUS_SCB_HEAD);
+	PRINT_LMIP_byte(asd_ha, lseq, EST_NEXUS_SCB_TAIL);
+	PRINT_LMIP_byte(asd_ha, lseq, EST_NEXUS_BUF_AVAIL);
+	PRINT_LMIP_dword(asd_ha, lseq, TIMEOUT_CONST);
+	PRINT_LMIP_word(asd_ha, lseq, ISR_SAVE_SINDEX);
+	PRINT_LMIP_word(asd_ha, lseq, ISR_SAVE_DINDEX);
+
+	asd_printk("LSEQ%d MIP 2 >>>>\n", lseq);
+	PRINT_LMIP_word(asd_ha, lseq, EMPTY_SCB_PTR0);
+	PRINT_LMIP_word(asd_ha, lseq, EMPTY_SCB_PTR1);
+	PRINT_LMIP_word(asd_ha, lseq, EMPTY_SCB_PTR2);
+	PRINT_LMIP_word(asd_ha, lseq, EMPTY_SCB_PTR3);
+	PRINT_LMIP_byte(asd_ha, lseq, EMPTY_SCB_OPCD0);
+	PRINT_LMIP_byte(asd_ha, lseq, EMPTY_SCB_OPCD1);
+	PRINT_LMIP_byte(asd_ha, lseq, EMPTY_SCB_OPCD2);
+	PRINT_LMIP_byte(asd_ha, lseq, EMPTY_SCB_OPCD3);
+	PRINT_LMIP_byte(asd_ha, lseq, EMPTY_SCB_HEAD);
+	PRINT_LMIP_byte(asd_ha, lseq, EMPTY_SCB_TAIL);
+	PRINT_LMIP_byte(asd_ha, lseq, EMPTY_BUFS_AVAIL);
+
+	asd_printk("LSEQ%d MIP 3 >>>>\n", lseq);
+	PRINT_LMIP_dword(asd_ha, lseq, DEV_PRES_TMR_TOUT_CONST);
+	PRINT_LMIP_dword(asd_ha, lseq, SATA_INTERLOCK_TIMEOUT);
+	PRINT_LMIP_dword(asd_ha, lseq, SRST_ASSERT_TIMEOUT);
+	PRINT_LMIP_dword(asd_ha, lseq, RCV_FIS_TIMEOUT);
+	PRINT_LMIP_dword(asd_ha, lseq, ONE_MILLISEC_TIMEOUT);
+	PRINT_LMIP_dword(asd_ha, lseq, TEN_MS_COMINIT_TIMEOUT);
+	PRINT_LMIP_dword(asd_ha, lseq, SMP_RCV_TIMEOUT);
+
+	for (mode = 0; mode < 3; mode++) {
+		asd_printk("LSEQ%d MDP 0 MODE %d >>>>\n", lseq, mode);
+		moffs = mode * LSEQ_MODE_SCRATCH_SIZE;
+
+		asd_printk(STR_16BIT, "RET_ADDR", 0,
+			   asd_read_reg_word(asd_ha, LmSEQ_RET_ADDR(lseq)
+					     + moffs));
+		asd_printk(STR_16BIT, "REG0_MODE", 2,
+			   asd_read_reg_word(asd_ha, LmSEQ_REG0_MODE(lseq)
+					     + moffs));
+		asd_printk(STR_16BIT, "MODE_FLAGS", 4,
+			   asd_read_reg_word(asd_ha, LmSEQ_MODE_FLAGS(lseq)
+					     + moffs));
+		asd_printk(STR_16BIT, "RET_ADDR2", 0x6,
+			   asd_read_reg_word(asd_ha, LmSEQ_RET_ADDR2(lseq)
+					     + moffs));
+		asd_printk(STR_16BIT, "RET_ADDR1", 0x8,
+			   asd_read_reg_word(asd_ha, LmSEQ_RET_ADDR1(lseq)
+					     + moffs));
+		asd_printk(STR_8BIT, "OPCODE_TO_CSEQ", 0xB,
+			   asd_read_reg_byte(asd_ha, LmSEQ_OPCODE_TO_CSEQ(lseq)
+					     + moffs));
+		asd_printk(STR_16BIT, "DATA_TO_CSEQ", 0xC,
+			   asd_read_reg_word(asd_ha, LmSEQ_DATA_TO_CSEQ(lseq)
+					     + moffs));
+	}
+
+	asd_printk("LSEQ%d MDP 0 MODE 5 >>>>\n", lseq);
+	moffs = LSEQ_MODE5_PAGE0_OFFSET;
+	asd_printk(STR_16BIT, "RET_ADDR", 0,
+		   asd_read_reg_word(asd_ha, LmSEQ_RET_ADDR(lseq) + moffs));
+	asd_printk(STR_16BIT, "REG0_MODE", 2,
+		   asd_read_reg_word(asd_ha, LmSEQ_REG0_MODE(lseq) + moffs));
+	asd_printk(STR_16BIT, "MODE_FLAGS", 4,
+		   asd_read_reg_word(asd_ha, LmSEQ_MODE_FLAGS(lseq) + moffs));
+	asd_printk(STR_16BIT, "RET_ADDR2", 0x6,
+		   asd_read_reg_word(asd_ha, LmSEQ_RET_ADDR2(lseq) + moffs));
+	asd_printk(STR_16BIT, "RET_ADDR1", 0x8,
+		   asd_read_reg_word(asd_ha, LmSEQ_RET_ADDR1(lseq) + moffs));
+	asd_printk(STR_8BIT, "OPCODE_TO_CSEQ", 0xB,
+	   asd_read_reg_byte(asd_ha, LmSEQ_OPCODE_TO_CSEQ(lseq) + moffs));
+	asd_printk(STR_16BIT, "DATA_TO_CSEQ", 0xC,
+	   asd_read_reg_word(asd_ha, LmSEQ_DATA_TO_CSEQ(lseq) + moffs));
+
+	asd_printk("LSEQ%d MDP 0 MODE 0 >>>>\n", lseq);
+	PRINT_LMIP_word(asd_ha, lseq, FIRST_INV_DDB_SITE);
+	PRINT_LMIP_word(asd_ha, lseq, EMPTY_TRANS_CTX);
+	PRINT_LMIP_word(asd_ha, lseq, RESP_LEN);
+	PRINT_LMIP_word(asd_ha, lseq, FIRST_INV_SCB_SITE);
+	PRINT_LMIP_dword(asd_ha, lseq, INTEN_SAVE);
+	PRINT_LMIP_byte(asd_ha, lseq, LINK_RST_FRM_LEN);
+	PRINT_LMIP_byte(asd_ha, lseq, LINK_RST_PROTOCOL);
+	PRINT_LMIP_byte(asd_ha, lseq, RESP_STATUS);
+	PRINT_LMIP_byte(asd_ha, lseq, LAST_LOADED_SGE);
+	PRINT_LMIP_byte(asd_ha, lseq, SAVE_SCBPTR);
+
+	asd_printk("LSEQ%d MDP 0 MODE 1 >>>>\n", lseq);
+	PRINT_LMIP_word(asd_ha, lseq, Q_XMIT_HEAD);
+	PRINT_LMIP_word(asd_ha, lseq, M1_EMPTY_TRANS_CTX);
+	PRINT_LMIP_word(asd_ha, lseq, INI_CONN_TAG);
+	PRINT_LMIP_byte(asd_ha, lseq, FAILED_OPEN_STATUS);
+	PRINT_LMIP_byte(asd_ha, lseq, XMIT_REQUEST_TYPE);
+	PRINT_LMIP_byte(asd_ha, lseq, M1_RESP_STATUS);
+	PRINT_LMIP_byte(asd_ha, lseq, M1_LAST_LOADED_SGE);
+	PRINT_LMIP_word(asd_ha, lseq, M1_SAVE_SCBPTR);
+
+	asd_printk("LSEQ%d MDP 0 MODE 2 >>>>\n", lseq);
+	PRINT_LMIP_word(asd_ha, lseq, PORT_COUNTER);
+	PRINT_LMIP_word(asd_ha, lseq, PM_TABLE_PTR);
+	PRINT_LMIP_word(asd_ha, lseq, SATA_INTERLOCK_TMR_SAVE);
+	PRINT_LMIP_word(asd_ha, lseq, IP_BITL);
+	PRINT_LMIP_word(asd_ha, lseq, COPY_SMP_CONN_TAG);
+	PRINT_LMIP_byte(asd_ha, lseq, P0M2_OFFS1AH);
+
+	asd_printk("LSEQ%d MDP 0 MODE 4/5 >>>>\n", lseq);
+	PRINT_LMIP_byte(asd_ha, lseq, SAVED_OOB_STATUS);
+	PRINT_LMIP_byte(asd_ha, lseq, SAVED_OOB_MODE);
+	PRINT_LMIP_word(asd_ha, lseq, Q_LINK_HEAD);
+	PRINT_LMIP_byte(asd_ha, lseq, LINK_RST_ERR);
+	PRINT_LMIP_byte(asd_ha, lseq, SAVED_OOB_SIGNALS);
+	PRINT_LMIP_byte(asd_ha, lseq, SAS_RESET_MODE);
+	PRINT_LMIP_byte(asd_ha, lseq, LINK_RESET_RETRY_COUNT);
+	PRINT_LMIP_byte(asd_ha, lseq, NUM_LINK_RESET_RETRIES);
+	PRINT_LMIP_word(asd_ha, lseq, OOB_INT_ENABLES);
+	PRINT_LMIP_word(asd_ha, lseq, NOTIFY_TIMER_TIMEOUT);
+	PRINT_LMIP_word(asd_ha, lseq, NOTIFY_TIMER_DOWN_COUNT);
+
+	asd_printk("LSEQ%d MDP 1 MODE 0 >>>>\n", lseq);
+	PRINT_LMIP_qword(asd_ha, lseq, SG_LIST_PTR_ADDR0);
+	PRINT_LMIP_qword(asd_ha, lseq, SG_LIST_PTR_ADDR1);
+
+	asd_printk("LSEQ%d MDP 1 MODE 1 >>>>\n", lseq);
+	PRINT_LMIP_qword(asd_ha, lseq, M1_SG_LIST_PTR_ADDR0);
+	PRINT_LMIP_qword(asd_ha, lseq, M1_SG_LIST_PTR_ADDR1);
+
+	asd_printk("LSEQ%d MDP 1 MODE 2 >>>>\n", lseq);
+	PRINT_LMIP_dword(asd_ha, lseq, INVALID_DWORD_COUNT);
+	PRINT_LMIP_dword(asd_ha, lseq, DISPARITY_ERROR_COUNT);
+	PRINT_LMIP_dword(asd_ha, lseq, LOSS_OF_SYNC_COUNT);
+
+	asd_printk("LSEQ%d MDP 1 MODE 4/5 >>>>\n", lseq);
+	PRINT_LMIP_dword(asd_ha, lseq, FRAME_TYPE_MASK);
+	PRINT_LMIP_dword(asd_ha, lseq, HASHED_SRC_ADDR_MASK_PRINT);
+	PRINT_LMIP_byte(asd_ha, lseq, NUM_FILL_BYTES_MASK);
+	PRINT_LMIP_word(asd_ha, lseq, TAG_MASK);
+	PRINT_LMIP_word(asd_ha, lseq, TARGET_PORT_XFER_TAG);
+	PRINT_LMIP_dword(asd_ha, lseq, DATA_OFFSET);
+
+	asd_printk("LSEQ%d MDP 2 MODE 0 >>>>\n", lseq);
+	PRINT_LMIP_dword(asd_ha, lseq, SMP_RCV_TIMER_TERM_TS);
+	PRINT_LMIP_byte(asd_ha, lseq, DEVICE_BITS);
+	PRINT_LMIP_word(asd_ha, lseq, SDB_DDB);
+	PRINT_LMIP_word(asd_ha, lseq, SDB_NUM_TAGS);
+	PRINT_LMIP_word(asd_ha, lseq, SDB_CURR_TAG);
+
+	asd_printk("LSEQ%d MDP 2 MODE 1 >>>>\n", lseq);
+	PRINT_LMIP_qword(asd_ha, lseq, TX_ID_ADDR_FRAME);
+	PRINT_LMIP_dword(asd_ha, lseq, OPEN_TIMER_TERM_TS);
+	PRINT_LMIP_dword(asd_ha, lseq, SRST_AS_TIMER_TERM_TS);
+	PRINT_LMIP_dword(asd_ha, lseq, LAST_LOADED_SG_EL);
+
+	asd_printk("LSEQ%d MDP 2 MODE 2 >>>>\n", lseq);
+	PRINT_LMIP_dword(asd_ha, lseq, CLOSE_TIMER_TERM_TS);
+	PRINT_LMIP_dword(asd_ha, lseq, BREAK_TIMER_TERM_TS);
+	PRINT_LMIP_dword(asd_ha, lseq, DWS_RESET_TIMER_TERM_TS);
+	PRINT_LMIP_dword(asd_ha, lseq, SATA_INTERLOCK_TIMER_TERM_TS);
+	PRINT_LMIP_dword(asd_ha, lseq, MCTL_TIMER_TERM_TS);
+
+	asd_printk("LSEQ%d MDP 2 MODE 4/5 >>>>\n", lseq);
+	PRINT_LMIP_dword(asd_ha, lseq, COMINIT_TIMER_TERM_TS);
+	PRINT_LMIP_dword(asd_ha, lseq, RCV_ID_TIMER_TERM_TS);
+	PRINT_LMIP_dword(asd_ha, lseq, RCV_FIS_TIMER_TERM_TS);
+	PRINT_LMIP_dword(asd_ha, lseq, DEV_PRES_TIMER_TERM_TS);
+}
+
+/**
+ * asd_dump_ddb_site -- dump a CSEQ DDB site
+ * @asd_ha: pointer to host adapter structure
+ * @site_no: site number of interest
+ */
+void asd_dump_target_ddb(struct asd_ha_struct *asd_ha, u16 site_no)
+{
+	if (site_no >= asd_ha->hw_prof.max_ddbs)
+		return;
+
+#define DDB_FIELDB(__name)                                        \
+	asd_ddbsite_read_byte(asd_ha, site_no,                    \
+			      offsetof(struct asd_ddb_ssp_smp_target_port, __name))
+#define DDB2_FIELDB(__name)                                       \
+	asd_ddbsite_read_byte(asd_ha, site_no,                    \
+			      offsetof(struct asd_ddb_stp_sata_target_port, __name))
+#define DDB_FIELDW(__name)                                        \
+	asd_ddbsite_read_word(asd_ha, site_no,                    \
+			      offsetof(struct asd_ddb_ssp_smp_target_port, __name))
+
+#define DDB_FIELDD(__name)                                         \
+	asd_ddbsite_read_dword(asd_ha, site_no,                    \
+			       offsetof(struct asd_ddb_ssp_smp_target_port, __name))
+
+	asd_printk("DDB: 0x%02x\n", site_no);
+	asd_printk("conn_type: 0x%02x\n", DDB_FIELDB(conn_type));
+	asd_printk("conn_rate: 0x%02x\n", DDB_FIELDB(conn_rate));
+	asd_printk("init_conn_tag: 0x%04x\n", be16_to_cpu(DDB_FIELDW(init_conn_tag)));
+	asd_printk("send_queue_head: 0x%04x\n", be16_to_cpu(DDB_FIELDW(send_queue_head)));
+	asd_printk("sq_suspended: 0x%02x\n", DDB_FIELDB(sq_suspended));
+	asd_printk("DDB Type: 0x%02x\n", DDB_FIELDB(ddb_type));
+	asd_printk("AWT Default: 0x%04x\n", DDB_FIELDW(awt_def));
+	asd_printk("compat_features: 0x%02x\n", DDB_FIELDB(compat_features));
+	asd_printk("Pathway Blocked Count: 0x%02x\n",
+		   DDB_FIELDB(pathway_blocked_count));
+	asd_printk("arb_wait_time: 0x%04x\n", DDB_FIELDW(arb_wait_time));
+	asd_printk("more_compat_features: 0x%08x\n",
+		   DDB_FIELDD(more_compat_features));
+	asd_printk("Conn Mask: 0x%02x\n", DDB_FIELDB(conn_mask));
+	asd_printk("flags: 0x%02x\n", DDB_FIELDB(flags));
+	asd_printk("flags2: 0x%02x\n", DDB2_FIELDB(flags2));
+	asd_printk("ExecQ Tail: 0x%04x\n",DDB_FIELDW(exec_queue_tail));
+	asd_printk("SendQ Tail: 0x%04x\n",DDB_FIELDW(send_queue_tail));
+	asd_printk("Active Task Count: 0x%04x\n",
+		   DDB_FIELDW(active_task_count));
+	asd_printk("ITNL Reason: 0x%02x\n", DDB_FIELDB(itnl_reason));
+	asd_printk("ITNL Timeout Const: 0x%04x\n", DDB_FIELDW(itnl_timeout));
+	asd_printk("ITNL timestamp: 0x%08x\n", DDB_FIELDD(itnl_timestamp));
+}
+
+void asd_dump_ddb_0(struct asd_ha_struct *asd_ha)
+{
+#define DDB0_FIELDB(__name)                                  \
+	asd_ddbsite_read_byte(asd_ha, 0,                     \
+			      offsetof(struct asd_ddb_seq_shared, __name))
+#define DDB0_FIELDW(__name)                                  \
+	asd_ddbsite_read_word(asd_ha, 0,                     \
+			      offsetof(struct asd_ddb_seq_shared, __name))
+
+#define DDB0_FIELDD(__name)                                  \
+	asd_ddbsite_read_dword(asd_ha,0 ,                    \
+			       offsetof(struct asd_ddb_seq_shared, __name))
+
+#define DDB0_FIELDA(__name, _o)                              \
+	asd_ddbsite_read_byte(asd_ha, 0,                     \
+			      offsetof(struct asd_ddb_seq_shared, __name)+_o)
+
+
+	asd_printk("DDB: 0\n");
+	asd_printk("q_free_ddb_head:%04x\n", DDB0_FIELDW(q_free_ddb_head));
+	asd_printk("q_free_ddb_tail:%04x\n", DDB0_FIELDW(q_free_ddb_tail));
+	asd_printk("q_free_ddb_cnt:%04x\n",  DDB0_FIELDW(q_free_ddb_cnt));
+	asd_printk("q_used_ddb_head:%04x\n", DDB0_FIELDW(q_used_ddb_head));
+	asd_printk("q_used_ddb_tail:%04x\n", DDB0_FIELDW(q_used_ddb_tail));
+	asd_printk("shared_mem_lock:%04x\n", DDB0_FIELDW(shared_mem_lock));
+	asd_printk("smp_conn_tag:%04x\n",    DDB0_FIELDW(smp_conn_tag));
+	asd_printk("est_nexus_buf_cnt:%04x\n", DDB0_FIELDW(est_nexus_buf_cnt));
+	asd_printk("est_nexus_buf_thresh:%04x\n",
+		   DDB0_FIELDW(est_nexus_buf_thresh));
+	asd_printk("conn_not_active:%02x\n", DDB0_FIELDB(conn_not_active));
+	asd_printk("phy_is_up:%02x\n",       DDB0_FIELDB(phy_is_up));
+	asd_printk("port_map_by_links:%02x %02x %02x %02x "
+		   "%02x %02x %02x %02x\n",
+		   DDB0_FIELDA(port_map_by_links, 0),
+		   DDB0_FIELDA(port_map_by_links, 1),
+		   DDB0_FIELDA(port_map_by_links, 2),
+		   DDB0_FIELDA(port_map_by_links, 3),
+		   DDB0_FIELDA(port_map_by_links, 4),
+		   DDB0_FIELDA(port_map_by_links, 5),
+		   DDB0_FIELDA(port_map_by_links, 6),
+		   DDB0_FIELDA(port_map_by_links, 7));
+}
+
+static void asd_dump_scb_site(struct asd_ha_struct *asd_ha, u16 site_no)
+{
+
+#define SCB_FIELDB(__name)                                                 \
+	asd_scbsite_read_byte(asd_ha, site_no, sizeof(struct scb_header)   \
+			      + offsetof(struct initiate_ssp_task, __name))
+#define SCB_FIELDW(__name)                                                 \
+	asd_scbsite_read_word(asd_ha, site_no, sizeof(struct scb_header)   \
+			      + offsetof(struct initiate_ssp_task, __name))
+#define SCB_FIELDD(__name)                                                 \
+	asd_scbsite_read_dword(asd_ha, site_no, sizeof(struct scb_header)  \
+			       + offsetof(struct initiate_ssp_task, __name))
+
+	asd_printk("Total Xfer Len: 0x%08x.\n", SCB_FIELDD(total_xfer_len));
+	asd_printk("Frame Type: 0x%02x.\n", SCB_FIELDB(ssp_frame.frame_type));
+	asd_printk("Tag: 0x%04x.\n", SCB_FIELDW(ssp_frame.tag));
+	asd_printk("Target Port Xfer Tag: 0x%04x.\n",
+		   SCB_FIELDW(ssp_frame.tptt));
+	asd_printk("Data Offset: 0x%08x.\n", SCB_FIELDW(ssp_frame.data_offs));
+	asd_printk("Retry Count: 0x%02x.\n", SCB_FIELDB(retry_count));
+}
+
+/**
+ * asd_dump_scb_sites -- dump currently used CSEQ SCB sites
+ * @asd_ha: pointer to host adapter struct
+ */
+void asd_dump_scb_sites(struct asd_ha_struct *asd_ha)
+{
+	u16	site_no;
+
+	for (site_no = 0; site_no < asd_ha->hw_prof.max_scbs; site_no++) {
+		u8 opcode;
+
+		if (!SCB_SITE_VALID(site_no))
+			continue;
+
+		/* We are only interested in SCB sites currently used.
+		 */
+		opcode = asd_scbsite_read_byte(asd_ha, site_no,
+					       offsetof(struct scb_header,
+							opcode));
+		if (opcode == 0xFF)
+			continue;
+
+		asd_printk("\nSCB: 0x%x\n", site_no);
+		asd_dump_scb_site(asd_ha, site_no);
+	}
+}
+
+/**
+ * ads_dump_seq_state -- dump CSEQ and LSEQ states
+ * @asd_ha: pointer to host adapter structure
+ * @lseq_mask: mask of LSEQs of interest
+ */
+void asd_dump_seq_state(struct asd_ha_struct *asd_ha, u8 lseq_mask)
+{
+	int lseq;
+
+	asd_dump_cseq_state(asd_ha);
+
+	if (lseq_mask != 0)
+		for_each_sequencer(lseq_mask, lseq_mask, lseq)
+			asd_dump_lseq_state(asd_ha, lseq);
+}
+
+void asd_dump_frame_rcvd(struct asd_phy *phy,
+			 struct done_list_struct *dl)
+{
+	unsigned long flags;
+	int i;
+
+	switch ((dl->status_block[1] & 0x70) >> 3) {
+	case SAS_PROTO_STP:
+		ASD_DPRINTK("STP proto device-to-host FIS:\n");
+		break;
+	default:
+	case SAS_PROTO_SSP:
+		ASD_DPRINTK("SAS proto IDENTIFY:\n");
+		break;
+	}
+	spin_lock_irqsave(&phy->sas_phy.frame_rcvd_lock, flags);
+	for (i = 0; i < phy->sas_phy.frame_rcvd_size; i+=4)
+		ASD_DPRINTK("%02x: %02x %02x %02x %02x\n",
+			    i,
+			    phy->frame_rcvd[i],
+			    phy->frame_rcvd[i+1],
+			    phy->frame_rcvd[i+2],
+			    phy->frame_rcvd[i+3]);
+	spin_unlock_irqrestore(&phy->sas_phy.frame_rcvd_lock, flags);
+}
+
+static inline void asd_dump_scb(struct asd_ascb *ascb, int ind)
+{
+	asd_printk("scb%d: vaddr: 0x%p, dma_handle: 0x%llx, next: 0x%llx, "
+		   "index:%d, opcode:0x%02x\n",
+		   ind, ascb->dma_scb.vaddr,
+		   (unsigned long long)ascb->dma_scb.dma_handle,
+		   (unsigned long long)
+		   le64_to_cpu(ascb->scb->header.next_scb),
+		   le16_to_cpu(ascb->scb->header.index),
+		   ascb->scb->header.opcode);
+}
+
+void asd_dump_scb_list(struct asd_ascb *ascb, int num)
+{
+	int i = 0;
+
+	asd_printk("dumping %d scbs:\n", num);
+
+	asd_dump_scb(ascb, i++);
+	--num;
+
+	if (num > 0 && !list_empty(&ascb->list)) {
+		struct list_head *el;
+
+		list_for_each(el, &ascb->list) {
+			struct asd_ascb *s = list_entry(el, struct asd_ascb,
+							list);
+			asd_dump_scb(s, i++);
+			if (--num <= 0)
+				break;
+		}
+	}
+}
+
+#endif /* ASD_DEBUG */
diff --git a/drivers/scsi/aic94xx/aic94xx_dump.h b/drivers/scsi/aic94xx/aic94xx_dump.h
new file mode 100644
index 0000000..0c388e7
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_dump.h
@@ -0,0 +1,52 @@
+/*
+ * Aic94xx SAS/SATA driver dump header file.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#ifndef _AIC94XX_DUMP_H_
+#define _AIC94XX_DUMP_H_
+
+#ifdef ASD_DEBUG
+
+void asd_dump_ddb_0(struct asd_ha_struct *asd_ha);
+void asd_dump_target_ddb(struct asd_ha_struct *asd_ha, u16 site_no);
+void asd_dump_scb_sites(struct asd_ha_struct *asd_ha);
+void asd_dump_seq_state(struct asd_ha_struct *asd_ha, u8 lseq_mask);
+void asd_dump_frame_rcvd(struct asd_phy *phy,
+			 struct done_list_struct *dl);
+void asd_dump_scb_list(struct asd_ascb *ascb, int num);
+#else /* ASD_DEBUG */
+
+static inline void asd_dump_ddb_0(struct asd_ha_struct *asd_ha) { }
+static inline void asd_dump_target_ddb(struct asd_ha_struct *asd_ha,
+				     u16 site_no) { }
+static inline void asd_dump_scb_sites(struct asd_ha_struct *asd_ha) { }
+static inline void asd_dump_seq_state(struct asd_ha_struct *asd_ha,
+				      u8 lseq_mask) { }
+static inline void asd_dump_frame_rcvd(struct asd_phy *phy,
+				       struct done_list_struct *dl) { }
+static inline void asd_dump_scb_list(struct asd_ascb *ascb, int num) { }
+#endif /* ASD_DEBUG */
+
+#endif /* _AIC94XX_DUMP_H_ */
diff --git a/drivers/scsi/aic94xx/aic94xx_hwi.c b/drivers/scsi/aic94xx/aic94xx_hwi.c
new file mode 100644
index 0000000..a242013
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_hwi.c
@@ -0,0 +1,1376 @@
+/*
+ * Aic94xx SAS/SATA driver hardware interface.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/delay.h>
+#include <linux/module.h>
+
+#include "aic94xx.h"
+#include "aic94xx_reg.h"
+#include "aic94xx_hwi.h"
+#include "aic94xx_seq.h"
+#include "aic94xx_dump.h"
+
+u32 MBAR0_SWB_SIZE;
+
+/* ---------- Initialization ---------- */
+
+static void asd_get_user_sas_addr(struct asd_ha_struct *asd_ha)
+{
+	extern char sas_addr_str[];
+	/* If the user has specified a WWN it overrides other settings
+	 */
+	if (sas_addr_str[0] != '\0')
+		asd_destringify_sas_addr(asd_ha->hw_prof.sas_addr,
+					 sas_addr_str);
+	else if (asd_ha->hw_prof.sas_addr[0] != 0)
+		asd_stringify_sas_addr(sas_addr_str, asd_ha->hw_prof.sas_addr);
+}
+
+static void asd_propagate_sas_addr(struct asd_ha_struct *asd_ha)
+{
+	int i;
+
+	for (i = 0; i < ASD_MAX_PHYS; i++) {
+		if (asd_ha->hw_prof.phy_desc[i].sas_addr[0] == 0)
+			continue;
+		/* Set a phy's address only if it has none.
+		 */
+		ASD_DPRINTK("setting phy%d addr to %llx\n", i,
+			    SAS_ADDR(asd_ha->hw_prof.sas_addr));
+		memcpy(asd_ha->hw_prof.phy_desc[i].sas_addr,
+		       asd_ha->hw_prof.sas_addr, SAS_ADDR_SIZE);
+	}
+}
+
+/* ---------- PHY initialization ---------- */
+
+static void asd_init_phy_identify(struct asd_phy *phy)
+{
+	phy->identify_frame = phy->id_frm_tok->vaddr;
+
+	memset(phy->identify_frame, 0, sizeof(*phy->identify_frame));
+
+	phy->identify_frame->dev_type = SAS_END_DEV;
+	if (phy->sas_phy.role & PHY_ROLE_INITIATOR)
+		phy->identify_frame->initiator_bits = phy->sas_phy.iproto;
+	if (phy->sas_phy.role & PHY_ROLE_TARGET)
+		phy->identify_frame->target_bits = phy->sas_phy.tproto;
+	memcpy(phy->identify_frame->sas_addr, phy->phy_desc->sas_addr,
+	       SAS_ADDR_SIZE);
+	phy->identify_frame->phy_id = phy->sas_phy.id;
+}
+
+static int asd_init_phy(struct asd_phy *phy)
+{
+	struct asd_ha_struct *asd_ha = phy->sas_phy.ha->lldd_ha;
+	struct asd_sas_phy *sas_phy = &phy->sas_phy;
+
+	sas_phy->enabled = 1;
+	sas_phy->class = SAS;
+	sas_phy->iproto = SAS_PROTO_ALL;
+	sas_phy->tproto = 0;
+	sas_phy->type = PHY_TYPE_PHYSICAL;
+	sas_phy->role = PHY_ROLE_INITIATOR;
+	sas_phy->oob_mode = OOB_NOT_CONNECTED;
+	sas_phy->linkrate = SAS_LINK_RATE_UNKNOWN;
+
+	phy->id_frm_tok = asd_alloc_coherent(asd_ha,
+					     sizeof(*phy->identify_frame),
+					     GFP_KERNEL);
+	if (!phy->id_frm_tok) {
+		asd_printk("no mem for IDENTIFY for phy%d\n", sas_phy->id);
+		return -ENOMEM;
+	} else
+		asd_init_phy_identify(phy);
+
+	memset(phy->frame_rcvd, 0, sizeof(phy->frame_rcvd));
+
+	return 0;
+}
+
+static int asd_init_phys(struct asd_ha_struct *asd_ha)
+{
+	u8 i;
+	u8 phy_mask = asd_ha->hw_prof.enabled_phys;
+
+	for (i = 0; i < ASD_MAX_PHYS; i++) {
+		struct asd_phy *phy = &asd_ha->phys[i];
+
+		phy->phy_desc = &asd_ha->hw_prof.phy_desc[i];
+
+		phy->sas_phy.enabled = 0;
+		phy->sas_phy.id = i;
+		phy->sas_phy.sas_addr = &phy->phy_desc->sas_addr[0];
+		phy->sas_phy.frame_rcvd = &phy->frame_rcvd[0];
+		phy->sas_phy.ha = &asd_ha->sas_ha;
+		phy->sas_phy.lldd_phy = phy;
+	}
+
+	/* Now enable and initialize only the enabled phys. */
+	for_each_phy(phy_mask, phy_mask, i) {
+		int err = asd_init_phy(&asd_ha->phys[i]);
+		if (err)
+			return err;
+	}
+
+	return 0;
+}
+
+/* ---------- Sliding windows ---------- */
+
+static int asd_init_sw(struct asd_ha_struct *asd_ha)
+{
+	struct pci_dev *pcidev = asd_ha->pcidev;
+	int err;
+	u32 v;
+
+	/* Unlock MBARs */
+	err = pci_read_config_dword(pcidev, PCI_CONF_MBAR_KEY, &v);
+	if (err) {
+		asd_printk("couldn't access conf. space of %s\n",
+			   pci_name(pcidev));
+		goto Err;
+	}
+	if (v)
+		err = pci_write_config_dword(pcidev, PCI_CONF_MBAR_KEY, v);
+	if (err) {
+		asd_printk("couldn't write to MBAR_KEY of %s\n",
+			   pci_name(pcidev));
+		goto Err;
+	}
+
+	/* Set sliding windows A, B and C to point to proper internal
+	 * memory regions.
+	 */
+	pci_write_config_dword(pcidev, PCI_CONF_MBAR0_SWA, REG_BASE_ADDR);
+	pci_write_config_dword(pcidev, PCI_CONF_MBAR0_SWB,
+			       REG_BASE_ADDR_CSEQCIO);
+	pci_write_config_dword(pcidev, PCI_CONF_MBAR0_SWC, REG_BASE_ADDR_EXSI);
+	asd_ha->io_handle[0].swa_base = REG_BASE_ADDR;
+	asd_ha->io_handle[0].swb_base = REG_BASE_ADDR_CSEQCIO;
+	asd_ha->io_handle[0].swc_base = REG_BASE_ADDR_EXSI;
+	MBAR0_SWB_SIZE = asd_ha->io_handle[0].len - 0x80;
+	if (!asd_ha->iospace) {
+		/* MBAR1 will point to OCM (On Chip Memory) */
+		pci_write_config_dword(pcidev, PCI_CONF_MBAR1, OCM_BASE_ADDR);
+		asd_ha->io_handle[1].swa_base = OCM_BASE_ADDR;
+	}
+	spin_lock_init(&asd_ha->iolock);
+Err:
+	return err;
+}
+
+/* ---------- SCB initialization ---------- */
+
+/**
+ * asd_init_scbs - manually allocate the first SCB.
+ * @asd_ha: pointer to host adapter structure
+ *
+ * This allocates the very first SCB which would be sent to the
+ * sequencer for execution.  Its bus address is written to
+ * CSEQ_Q_NEW_POINTER, mode page 2, mode 8.  Since the bus address of
+ * the _next_ scb to be DMA-ed to the host adapter is read from the last
+ * SCB DMA-ed to the host adapter, we have to always stay one step
+ * ahead of the sequencer and keep one SCB already allocated.
+ */
+static int asd_init_scbs(struct asd_ha_struct *asd_ha)
+{
+	struct asd_seq_data *seq = &asd_ha->seq;
+	int bitmap_bytes;
+
+	/* allocate the index array and bitmap */
+	asd_ha->seq.tc_index_bitmap_bits = asd_ha->hw_prof.max_scbs;
+	asd_ha->seq.tc_index_array = kzalloc(asd_ha->seq.tc_index_bitmap_bits*
+					     sizeof(void *), GFP_KERNEL);
+	if (!asd_ha->seq.tc_index_array)
+		return -ENOMEM;
+
+	bitmap_bytes = (asd_ha->seq.tc_index_bitmap_bits+7)/8;
+	bitmap_bytes = BITS_TO_LONGS(bitmap_bytes*8)*sizeof(unsigned long);
+	asd_ha->seq.tc_index_bitmap = kzalloc(bitmap_bytes, GFP_KERNEL);
+	if (!asd_ha->seq.tc_index_bitmap)
+		return -ENOMEM;
+
+	spin_lock_init(&seq->tc_index_lock);
+
+	seq->next_scb.size = sizeof(struct scb);
+	seq->next_scb.vaddr = dma_pool_alloc(asd_ha->scb_pool, GFP_KERNEL,
+					     &seq->next_scb.dma_handle);
+	if (!seq->next_scb.vaddr) {
+		kfree(asd_ha->seq.tc_index_bitmap);
+		kfree(asd_ha->seq.tc_index_array);
+		asd_ha->seq.tc_index_bitmap = NULL;
+		asd_ha->seq.tc_index_array = NULL;
+		return -ENOMEM;
+	}
+
+	seq->pending = 0;
+	spin_lock_init(&seq->pend_q_lock);
+	INIT_LIST_HEAD(&seq->pend_q);
+
+	return 0;
+}
+
+static inline void asd_get_max_scb_ddb(struct asd_ha_struct *asd_ha)
+{
+	asd_ha->hw_prof.max_scbs = asd_get_cmdctx_size(asd_ha)/ASD_SCB_SIZE;
+	asd_ha->hw_prof.max_ddbs = asd_get_devctx_size(asd_ha)/ASD_DDB_SIZE;
+	ASD_DPRINTK("max_scbs:%d, max_ddbs:%d\n",
+		    asd_ha->hw_prof.max_scbs,
+		    asd_ha->hw_prof.max_ddbs);
+}
+
+/* ---------- Done List initialization ---------- */
+
+static void asd_dl_tasklet_handler(unsigned long);
+
+static int asd_init_dl(struct asd_ha_struct *asd_ha)
+{
+	asd_ha->seq.actual_dl
+		= asd_alloc_coherent(asd_ha,
+			     ASD_DL_SIZE * sizeof(struct done_list_struct),
+				     GFP_KERNEL);
+	if (!asd_ha->seq.actual_dl)
+		return -ENOMEM;
+	asd_ha->seq.dl = asd_ha->seq.actual_dl->vaddr;
+	asd_ha->seq.dl_toggle = ASD_DEF_DL_TOGGLE;
+	asd_ha->seq.dl_next = 0;
+	tasklet_init(&asd_ha->seq.dl_tasklet, asd_dl_tasklet_handler,
+		     (unsigned long) asd_ha);
+
+	return 0;
+}
+
+/* ---------- EDB and ESCB init ---------- */
+
+static int asd_alloc_edbs(struct asd_ha_struct *asd_ha, unsigned int gfp_flags)
+{
+	struct asd_seq_data *seq = &asd_ha->seq;
+	int i;
+
+	seq->edb_arr = kmalloc(seq->num_edbs*sizeof(*seq->edb_arr), gfp_flags);
+	if (!seq->edb_arr)
+		return -ENOMEM;
+
+	for (i = 0; i < seq->num_edbs; i++) {
+		seq->edb_arr[i] = asd_alloc_coherent(asd_ha, ASD_EDB_SIZE,
+						     gfp_flags);
+		if (!seq->edb_arr[i])
+			goto Err_unroll;
+		memset(seq->edb_arr[i]->vaddr, 0, ASD_EDB_SIZE);
+	}
+
+	ASD_DPRINTK("num_edbs:%d\n", seq->num_edbs);
+
+	return 0;
+
+Err_unroll:
+	for (i-- ; i >= 0; i--)
+		asd_free_coherent(asd_ha, seq->edb_arr[i]);
+	kfree(seq->edb_arr);
+	seq->edb_arr = NULL;
+
+	return -ENOMEM;
+}
+
+static int asd_alloc_escbs(struct asd_ha_struct *asd_ha,
+			   unsigned int gfp_flags)
+{
+	struct asd_seq_data *seq = &asd_ha->seq;
+	struct asd_ascb *escb;
+	int i, escbs;
+
+	seq->escb_arr = kmalloc(seq->num_escbs*sizeof(*seq->escb_arr),
+				gfp_flags);
+	if (!seq->escb_arr)
+		return -ENOMEM;
+
+	escbs = seq->num_escbs;
+	escb = asd_ascb_alloc_list(asd_ha, &escbs, gfp_flags);
+	if (!escb) {
+		asd_printk("couldn't allocate list of escbs\n");
+		goto Err;
+	}
+	seq->num_escbs -= escbs;  /* subtract what was not allocated */
+	ASD_DPRINTK("num_escbs:%d\n", seq->num_escbs);
+
+	for (i = 0; i < seq->num_escbs; i++, escb = list_entry(escb->list.next,
+							       struct asd_ascb,
+							       list)) {
+		seq->escb_arr[i] = escb;
+		escb->scb->header.opcode = EMPTY_SCB;
+	}
+
+	return 0;
+Err:
+	kfree(seq->escb_arr);
+	seq->escb_arr = NULL;
+	return -ENOMEM;
+
+}
+
+static void asd_assign_edbs2escbs(struct asd_ha_struct *asd_ha)
+{
+	struct asd_seq_data *seq = &asd_ha->seq;
+	int i, k, z = 0;
+
+	for (i = 0; i < seq->num_escbs; i++) {
+		struct asd_ascb *ascb = seq->escb_arr[i];
+		struct empty_scb *escb = &ascb->scb->escb;
+
+		ascb->edb_index = z;
+
+		escb->num_valid = ASD_EDBS_PER_SCB;
+
+		for (k = 0; k < ASD_EDBS_PER_SCB; k++) {
+			struct sg_el *eb = &escb->eb[k];
+			struct asd_dma_tok *edb = seq->edb_arr[z++];
+
+			memset(eb, 0, sizeof(*eb));
+			eb->bus_addr = cpu_to_le64(((u64) edb->dma_handle));
+			eb->size = cpu_to_le32(((u32) edb->size));
+		}
+	}
+}
+
+/**
+ * asd_init_escbs -- allocate and initialize empty scbs
+ * @asd_ha: pointer to host adapter structure
+ *
+ * An empty SCB has sg_elements of ASD_EDBS_PER_SCB (7) buffers.
+ * They transport sense data, etc.
+ */
+static int asd_init_escbs(struct asd_ha_struct *asd_ha)
+{
+	struct asd_seq_data *seq = &asd_ha->seq;
+	int err = 0;
+
+	/* Allocate two empty data buffers (edb) per sequencer. */
+	int edbs = 2*(1+asd_ha->hw_prof.num_phys);
+
+	seq->num_escbs = (edbs+ASD_EDBS_PER_SCB-1)/ASD_EDBS_PER_SCB;
+	seq->num_edbs = seq->num_escbs * ASD_EDBS_PER_SCB;
+
+	err = asd_alloc_edbs(asd_ha, GFP_KERNEL);
+	if (err) {
+		asd_printk("couldn't allocate edbs\n");
+		return err;
+	}
+
+	err = asd_alloc_escbs(asd_ha, GFP_KERNEL);
+	if (err) {
+		asd_printk("couldn't allocate escbs\n");
+		return err;
+	}
+
+	asd_assign_edbs2escbs(asd_ha);
+	/* In order to insure that normal SCBs do not overfill sequencer
+	 * memory and leave no space for escbs (halting condition),
+	 * we increment pending here by the number of escbs.  However,
+	 * escbs are never pending.
+	 */
+	seq->pending   = seq->num_escbs;
+	seq->can_queue = 1 + (asd_ha->hw_prof.max_scbs - seq->pending)/2;
+
+	return 0;
+}
+
+/* ---------- HW initialization ---------- */
+
+/**
+ * asd_chip_hardrst -- hard reset the chip
+ * @asd_ha: pointer to host adapter structure
+ *
+ * This takes 16 cycles and is synchronous to CFCLK, which runs
+ * at 200 MHz, so this should take at most 80 nanoseconds.
+ */
+int asd_chip_hardrst(struct asd_ha_struct *asd_ha)
+{
+	int i;
+	int count = 100;
+	u32 reg;
+
+	for (i = 0 ; i < 4 ; i++) {
+		asd_write_reg_dword(asd_ha, COMBIST, HARDRST);
+	}
+
+	do {
+		udelay(1);
+		reg = asd_read_reg_dword(asd_ha, CHIMINT);
+		if (reg & HARDRSTDET) {
+			asd_write_reg_dword(asd_ha, CHIMINT,
+					    HARDRSTDET|PORRSTDET);
+			return 0;
+		}
+	} while (--count > 0);
+
+	return -ENODEV;
+}
+
+/**
+ * asd_init_chip -- initialize the chip
+ * @asd_ha: pointer to host adapter structure
+ *
+ * Hard resets the chip, disables HA interrupts, downloads the sequnecer
+ * microcode and starts the sequencers.  The caller has to explicitly
+ * enable HA interrupts with asd_enable_ints(asd_ha).
+ */
+static int asd_init_chip(struct asd_ha_struct *asd_ha)
+{
+	int err;
+
+	err = asd_chip_hardrst(asd_ha);
+	if (err) {
+		asd_printk("couldn't hard reset %s\n",
+			    pci_name(asd_ha->pcidev));
+		goto out;
+	}
+
+	asd_disable_ints(asd_ha);
+
+	err = asd_init_seqs(asd_ha);
+	if (err) {
+		asd_printk("couldn't init seqs for %s\n",
+			   pci_name(asd_ha->pcidev));
+		goto out;
+	}
+
+	err = asd_start_seqs(asd_ha);
+	if (err) {
+		asd_printk("coudln't start seqs for %s\n",
+			   pci_name(asd_ha->pcidev));
+		goto out;
+	}
+out:
+	return err;
+}
+
+#define MAX_DEVS ((OCM_MAX_SIZE) / (ASD_DDB_SIZE))
+
+static int max_devs = 0;
+module_param_named(max_devs, max_devs, int, S_IRUGO);
+MODULE_PARM_DESC(max_devs, "\n"
+	"\tMaximum number of SAS devices to support (not LUs).\n"
+	"\tDefault: 2176, Maximum: 65663.\n");
+
+static int max_cmnds = 0;
+module_param_named(max_cmnds, max_cmnds, int, S_IRUGO);
+MODULE_PARM_DESC(max_cmnds, "\n"
+	"\tMaximum number of commands queuable.\n"
+	"\tDefault: 512, Maximum: 66047.\n");
+
+static void asd_extend_devctx_ocm(struct asd_ha_struct *asd_ha)
+{
+	unsigned long dma_addr = OCM_BASE_ADDR;
+	u32 d;
+
+	dma_addr -= asd_ha->hw_prof.max_ddbs * ASD_DDB_SIZE;
+	asd_write_reg_addr(asd_ha, DEVCTXBASE, (dma_addr_t) dma_addr);
+	d = asd_read_reg_dword(asd_ha, CTXDOMAIN);
+	d |= 4;
+	asd_write_reg_dword(asd_ha, CTXDOMAIN, d);
+	asd_ha->hw_prof.max_ddbs += MAX_DEVS;
+}
+
+static int asd_extend_devctx(struct asd_ha_struct *asd_ha)
+{
+	dma_addr_t dma_handle;
+	unsigned long dma_addr;
+	u32 d;
+	int size;
+
+	asd_extend_devctx_ocm(asd_ha);
+
+	asd_ha->hw_prof.ddb_ext = NULL;
+	if (max_devs <= asd_ha->hw_prof.max_ddbs || max_devs > 0xFFFF) {
+		max_devs = asd_ha->hw_prof.max_ddbs;
+		return 0;
+	}
+
+	size = (max_devs - asd_ha->hw_prof.max_ddbs + 1) * ASD_DDB_SIZE;
+
+	asd_ha->hw_prof.ddb_ext = asd_alloc_coherent(asd_ha, size, GFP_KERNEL);
+	if (!asd_ha->hw_prof.ddb_ext) {
+		asd_printk("couldn't allocate memory for %d devices\n",
+			   max_devs);
+		max_devs = asd_ha->hw_prof.max_ddbs;
+		return -ENOMEM;
+	}
+	dma_handle = asd_ha->hw_prof.ddb_ext->dma_handle;
+	dma_addr = ALIGN((unsigned long) dma_handle, ASD_DDB_SIZE);
+	dma_addr -= asd_ha->hw_prof.max_ddbs * ASD_DDB_SIZE;
+	dma_handle = (dma_addr_t) dma_addr;
+	asd_write_reg_addr(asd_ha, DEVCTXBASE, dma_handle);
+	d = asd_read_reg_dword(asd_ha, CTXDOMAIN);
+	d &= ~4;
+	asd_write_reg_dword(asd_ha, CTXDOMAIN, d);
+
+	asd_ha->hw_prof.max_ddbs = max_devs;
+
+	return 0;
+}
+
+static int asd_extend_cmdctx(struct asd_ha_struct *asd_ha)
+{
+	dma_addr_t dma_handle;
+	unsigned long dma_addr;
+	u32 d;
+	int size;
+
+	asd_ha->hw_prof.scb_ext = NULL;
+	if (max_cmnds <= asd_ha->hw_prof.max_scbs || max_cmnds > 0xFFFF) {
+		max_cmnds = asd_ha->hw_prof.max_scbs;
+		return 0;
+	}
+
+	size = (max_cmnds - asd_ha->hw_prof.max_scbs + 1) * ASD_SCB_SIZE;
+
+	asd_ha->hw_prof.scb_ext = asd_alloc_coherent(asd_ha, size, GFP_KERNEL);
+	if (!asd_ha->hw_prof.scb_ext) {
+		asd_printk("couldn't allocate memory for %d commands\n",
+			   max_cmnds);
+		max_cmnds = asd_ha->hw_prof.max_scbs;
+		return -ENOMEM;
+	}
+	dma_handle = asd_ha->hw_prof.scb_ext->dma_handle;
+	dma_addr = ALIGN((unsigned long) dma_handle, ASD_SCB_SIZE);
+	dma_addr -= asd_ha->hw_prof.max_scbs * ASD_SCB_SIZE;
+	dma_handle = (dma_addr_t) dma_addr;
+	asd_write_reg_addr(asd_ha, CMDCTXBASE, dma_handle);
+	d = asd_read_reg_dword(asd_ha, CTXDOMAIN);
+	d &= ~1;
+	asd_write_reg_dword(asd_ha, CTXDOMAIN, d);
+
+	asd_ha->hw_prof.max_scbs = max_cmnds;
+
+	return 0;
+}
+
+/**
+ * asd_init_ctxmem -- initialize context memory
+ * asd_ha: pointer to host adapter structure
+ *
+ * This function sets the maximum number of SCBs and
+ * DDBs which can be used by the sequencer.  This is normally
+ * 512 and 128 respectively.  If support for more SCBs or more DDBs
+ * is required then CMDCTXBASE, DEVCTXBASE and CTXDOMAIN are
+ * initialized here to extend context memory to point to host memory,
+ * thus allowing unlimited support for SCBs and DDBs -- only limited
+ * by host memory.
+ */
+static int asd_init_ctxmem(struct asd_ha_struct *asd_ha)
+{
+	int bitmap_bytes;
+
+	asd_get_max_scb_ddb(asd_ha);
+	asd_extend_devctx(asd_ha);
+	asd_extend_cmdctx(asd_ha);
+
+	/* The kernel wants bitmaps to be unsigned long sized. */
+	bitmap_bytes = (asd_ha->hw_prof.max_ddbs+7)/8;
+	bitmap_bytes = BITS_TO_LONGS(bitmap_bytes*8)*sizeof(unsigned long);
+	asd_ha->hw_prof.ddb_bitmap = kzalloc(bitmap_bytes, GFP_KERNEL);
+	if (!asd_ha->hw_prof.ddb_bitmap)
+		return -ENOMEM;
+	spin_lock_init(&asd_ha->hw_prof.ddb_lock);
+
+	return 0;
+}
+
+int asd_init_hw(struct asd_ha_struct *asd_ha)
+{
+	int err;
+	u32 v;
+
+	err = asd_init_sw(asd_ha);
+	if (err)
+		return err;
+
+	err = pci_read_config_dword(asd_ha->pcidev, PCIC_HSTPCIX_CNTRL, &v);
+	if (err) {
+		asd_printk("couldn't read PCIC_HSTPCIX_CNTRL of %s\n",
+			   pci_name(asd_ha->pcidev));
+		return err;
+	}
+	pci_write_config_dword(asd_ha->pcidev, PCIC_HSTPCIX_CNTRL,
+					v | SC_TMR_DIS);
+	if (err) {
+		asd_printk("couldn't disable split completion timer of %s\n",
+			   pci_name(asd_ha->pcidev));
+		return err;
+	}
+
+	err = asd_read_ocm(asd_ha);
+	if (err) {
+		asd_printk("couldn't read ocm(%d)\n", err);
+		/* While suspicios, it is not an error that we
+		 * couldn't read the OCM. */
+	}
+
+	err = asd_read_flash(asd_ha);
+	if (err) {
+		asd_printk("couldn't read flash(%d)\n", err);
+		/* While suspicios, it is not an error that we
+		 * couldn't read FLASH memory.
+		 */
+	}
+
+	asd_init_ctxmem(asd_ha);
+
+	asd_get_user_sas_addr(asd_ha);
+	if (!asd_ha->hw_prof.sas_addr[0]) {
+		asd_printk("No SAS Address provided for %s\n",
+			   pci_name(asd_ha->pcidev));
+		err = -ENODEV;
+		goto Out;
+	}
+
+	asd_propagate_sas_addr(asd_ha);
+
+	err = asd_init_phys(asd_ha);
+	if (err) {
+		asd_printk("couldn't initialize phys for %s\n",
+			    pci_name(asd_ha->pcidev));
+		goto Out;
+	}
+
+	err = asd_init_scbs(asd_ha);
+	if (err) {
+		asd_printk("couldn't initialize scbs for %s\n",
+			    pci_name(asd_ha->pcidev));
+		goto Out;
+	}
+
+	err = asd_init_dl(asd_ha);
+	if (err) {
+		asd_printk("couldn't initialize the done list:%d\n",
+			    err);
+		goto Out;
+	}
+
+	err = asd_init_escbs(asd_ha);
+	if (err) {
+		asd_printk("couldn't initialize escbs\n");
+		goto Out;
+	}
+
+	err = asd_init_chip(asd_ha);
+	if (err) {
+		asd_printk("couldn't init the chip\n");
+		goto Out;
+	}
+Out:
+	return err;
+}
+
+/* ---------- Chip reset ---------- */
+
+/**
+ * asd_chip_reset -- reset the host adapter, etc
+ * @asd_ha: pointer to host adapter structure of interest
+ *
+ * Called from the ISR.  Hard reset the chip.  Let everything
+ * timeout.  This should be no different than hot-unplugging the
+ * host adapter.  Once everything times out we'll init the chip with
+ * a call to asd_init_chip() and enable interrupts with asd_enable_ints().
+ * XXX finish.
+ */
+static void asd_chip_reset(struct asd_ha_struct *asd_ha)
+{
+	struct sas_ha_struct *sas_ha = &asd_ha->sas_ha;
+
+	ASD_DPRINTK("chip reset for %s\n", pci_name(asd_ha->pcidev));
+	asd_chip_hardrst(asd_ha);
+	sas_ha->notify_ha_event(sas_ha, HAE_RESET);
+}
+
+/* ---------- Done List Routines ---------- */
+
+static void asd_dl_tasklet_handler(unsigned long data)
+{
+	struct asd_ha_struct *asd_ha = (struct asd_ha_struct *) data;
+	struct asd_seq_data *seq = &asd_ha->seq;
+	unsigned long flags;
+
+	while (1) {
+		struct done_list_struct *dl = &seq->dl[seq->dl_next];
+		struct asd_ascb *ascb;
+
+		if ((dl->toggle & DL_TOGGLE_MASK) != seq->dl_toggle)
+			break;
+
+		/* find the aSCB */
+		spin_lock_irqsave(&seq->tc_index_lock, flags);
+		ascb = asd_tc_index_find(seq, (int)le16_to_cpu(dl->index));
+		spin_unlock_irqrestore(&seq->tc_index_lock, flags);
+		if (unlikely(!ascb)) {
+			ASD_DPRINTK("BUG:sequencer:dl:no ascb?!\n");
+			goto next_1;
+		} else if (ascb->scb->header.opcode == EMPTY_SCB) {
+			goto out;
+		} else if (!ascb->uldd_timer && !del_timer(&ascb->timer)) {
+			goto next_1;
+		}
+		spin_lock_irqsave(&seq->pend_q_lock, flags);
+		list_del_init(&ascb->list);
+		seq->pending--;
+		spin_unlock_irqrestore(&seq->pend_q_lock, flags);
+	out:
+		ascb->tasklet_complete(ascb, dl);
+
+	next_1:
+		seq->dl_next = (seq->dl_next + 1) & (ASD_DL_SIZE-1);
+		if (!seq->dl_next)
+			seq->dl_toggle ^= DL_TOGGLE_MASK;
+	}
+}
+
+/* ---------- Interrupt Service Routines ---------- */
+
+/**
+ * asd_process_donelist_isr -- schedule processing of done list entries
+ * @asd_ha: pointer to host adapter structure
+ */
+static inline void asd_process_donelist_isr(struct asd_ha_struct *asd_ha)
+{
+	tasklet_schedule(&asd_ha->seq.dl_tasklet);
+}
+
+/**
+ * asd_com_sas_isr -- process device communication interrupt (COMINT)
+ * @asd_ha: pointer to host adapter structure
+ */
+static inline void asd_com_sas_isr(struct asd_ha_struct *asd_ha)
+{
+	u32 comstat = asd_read_reg_dword(asd_ha, COMSTAT);
+
+	/* clear COMSTAT int */
+	asd_write_reg_dword(asd_ha, COMSTAT, 0xFFFFFFFF);
+
+	if (comstat & CSBUFPERR) {
+		asd_printk("%s: command/status buffer dma parity error\n",
+			   pci_name(asd_ha->pcidev));
+	} else if (comstat & CSERR) {
+		int i;
+		u32 dmaerr = asd_read_reg_dword(asd_ha, DMAERR);
+		dmaerr &= 0xFF;
+		asd_printk("%s: command/status dma error, DMAERR: 0x%02x, "
+			   "CSDMAADR: 0x%04x, CSDMAADR+4: 0x%04x\n",
+			   pci_name(asd_ha->pcidev),
+			   dmaerr,
+			   asd_read_reg_dword(asd_ha, CSDMAADR),
+			   asd_read_reg_dword(asd_ha, CSDMAADR+4));
+		asd_printk("CSBUFFER:\n");
+		for (i = 0; i < 8; i++) {
+			asd_printk("%08x %08x %08x %08x\n",
+				   asd_read_reg_dword(asd_ha, CSBUFFER),
+				   asd_read_reg_dword(asd_ha, CSBUFFER+4),
+				   asd_read_reg_dword(asd_ha, CSBUFFER+8),
+				   asd_read_reg_dword(asd_ha, CSBUFFER+12));
+		}
+		asd_dump_seq_state(asd_ha, 0);
+	} else if (comstat & OVLYERR) {
+		u32 dmaerr = asd_read_reg_dword(asd_ha, DMAERR);
+		dmaerr = (dmaerr >> 8) & 0xFF;
+		asd_printk("%s: overlay dma error:0x%x\n",
+			   pci_name(asd_ha->pcidev),
+			   dmaerr);
+	}
+	asd_chip_reset(asd_ha);
+}
+
+static inline void asd_arp2_err(struct asd_ha_struct *asd_ha, u32 dchstatus)
+{
+	static const char *halt_code[256] = {
+		"UNEXPECTED_INTERRUPT0",
+		"UNEXPECTED_INTERRUPT1",
+		"UNEXPECTED_INTERRUPT2",
+		"UNEXPECTED_INTERRUPT3",
+		"UNEXPECTED_INTERRUPT4",
+		"UNEXPECTED_INTERRUPT5",
+		"UNEXPECTED_INTERRUPT6",
+		"UNEXPECTED_INTERRUPT7",
+		"UNEXPECTED_INTERRUPT8",
+		"UNEXPECTED_INTERRUPT9",
+		"UNEXPECTED_INTERRUPT10",
+		[11 ... 19] = "unknown[11,19]",
+		"NO_FREE_SCB_AVAILABLE",
+		"INVALID_SCB_OPCODE",
+		"INVALID_MBX_OPCODE",
+		"INVALID_ATA_STATE",
+		"ATA_QUEUE_FULL",
+		"ATA_TAG_TABLE_FAULT",
+		"ATA_TAG_MASK_FAULT",
+		"BAD_LINK_QUEUE_STATE",
+		"DMA2CHIM_QUEUE_ERROR",
+		"EMPTY_SCB_LIST_FULL",
+		"unknown[30]",
+		"IN_USE_SCB_ON_FREE_LIST",
+		"BAD_OPEN_WAIT_STATE",
+		"INVALID_STP_AFFILIATION",
+		"unknown[34]",
+		"EXEC_QUEUE_ERROR",
+		"TOO_MANY_EMPTIES_NEEDED",
+		"EMPTY_REQ_QUEUE_ERROR",
+		"Q_MONIRTT_MGMT_ERROR",
+		"TARGET_MODE_FLOW_ERROR",
+		"DEVICE_QUEUE_NOT_FOUND",
+		"START_IRTT_TIMER_ERROR",
+		"ABORT_TASK_ILLEGAL_REQ",
+		[43 ... 255] = "unknown[43,255]"
+	};
+
+	if (dchstatus & CSEQINT) {
+		u32 arp2int = asd_read_reg_dword(asd_ha, CARP2INT);
+
+		if (arp2int & (ARP2WAITTO|ARP2ILLOPC|ARP2PERR|ARP2CIOPERR)) {
+			asd_printk("%s: CSEQ arp2int:0x%x\n",
+				   pci_name(asd_ha->pcidev),
+				   arp2int);
+		} else if (arp2int & ARP2HALTC)
+			asd_printk("%s: CSEQ halted: %s\n",
+				   pci_name(asd_ha->pcidev),
+				   halt_code[(arp2int>>16)&0xFF]);
+		else
+			asd_printk("%s: CARP2INT:0x%x\n",
+				   pci_name(asd_ha->pcidev),
+				   arp2int);
+	}
+	if (dchstatus & LSEQINT_MASK) {
+		int lseq;
+		u8  lseq_mask = dchstatus & LSEQINT_MASK;
+
+		for_each_sequencer(lseq_mask, lseq_mask, lseq) {
+			u32 arp2int = asd_read_reg_dword(asd_ha,
+							 LmARP2INT(lseq));
+			if (arp2int & (ARP2WAITTO | ARP2ILLOPC | ARP2PERR
+				       | ARP2CIOPERR)) {
+				asd_printk("%s: LSEQ%d arp2int:0x%x\n",
+					   pci_name(asd_ha->pcidev),
+					   lseq, arp2int);
+				/* XXX we should only do lseq reset */
+			} else if (arp2int & ARP2HALTC)
+				asd_printk("%s: LSEQ%d halted: %s\n",
+					   pci_name(asd_ha->pcidev),
+					   lseq,halt_code[(arp2int>>16)&0xFF]);
+			else
+				asd_printk("%s: LSEQ%d ARP2INT:0x%x\n",
+					   pci_name(asd_ha->pcidev), lseq,
+					   arp2int);
+		}
+	}
+	asd_chip_reset(asd_ha);
+}
+
+/**
+ * asd_dch_sas_isr -- process device channel interrupt (DEVINT)
+ * @asd_ha: pointer to host adapter structure
+ */
+static inline void asd_dch_sas_isr(struct asd_ha_struct *asd_ha)
+{
+	u32 dchstatus = asd_read_reg_dword(asd_ha, DCHSTATUS);
+
+	if (dchstatus & CFIFTOERR) {
+		asd_printk("%s: CFIFTOERR\n", pci_name(asd_ha->pcidev));
+		asd_chip_reset(asd_ha);
+	} else
+		asd_arp2_err(asd_ha, dchstatus);
+}
+
+/**
+ * ads_rbi_exsi_isr -- process external system interface interrupt (INITERR)
+ * @asd_ha: pointer to host adapter structure
+ */
+static inline void asd_rbi_exsi_isr(struct asd_ha_struct *asd_ha)
+{
+	u32 stat0r = asd_read_reg_dword(asd_ha, ASISTAT0R);
+
+	if (!(stat0r & ASIERR)) {
+		asd_printk("hmm, EXSI interrupted but no error?\n");
+		return;
+	}
+
+	if (stat0r & ASIFMTERR) {
+		asd_printk("ASI SEEPROM format error for %s\n",
+			   pci_name(asd_ha->pcidev));
+	} else if (stat0r & ASISEECHKERR) {
+		u32 stat1r = asd_read_reg_dword(asd_ha, ASISTAT1R);
+		asd_printk("ASI SEEPROM checksum 0x%x error for %s\n",
+			   stat1r & CHECKSUM_MASK,
+			   pci_name(asd_ha->pcidev));
+	} else {
+		u32 statr = asd_read_reg_dword(asd_ha, ASIERRSTATR);
+
+		if (!(statr & CPI2ASIMSTERR_MASK)) {
+			ASD_DPRINTK("hmm, ASIERR?\n");
+			return;
+		} else {
+			u32 addr = asd_read_reg_dword(asd_ha, ASIERRADDR);
+			u32 data = asd_read_reg_dword(asd_ha, ASIERRDATAR);
+
+			asd_printk("%s: CPI2 xfer err: addr: 0x%x, wdata: 0x%x, "
+				   "count: 0x%x, byteen: 0x%x, targerr: 0x%x "
+				   "master id: 0x%x, master err: 0x%x\n",
+				   pci_name(asd_ha->pcidev),
+				   addr, data,
+				   (statr & CPI2ASIBYTECNT_MASK) >> 16,
+				   (statr & CPI2ASIBYTEEN_MASK) >> 12,
+				   (statr & CPI2ASITARGERR_MASK) >> 8,
+				   (statr & CPI2ASITARGMID_MASK) >> 4,
+				   (statr & CPI2ASIMSTERR_MASK));
+		}
+	}
+	asd_chip_reset(asd_ha);
+}
+
+/**
+ * asd_hst_pcix_isr -- process host interface interrupts
+ * @asd_ha: pointer to host adapter structure
+ *
+ * Asserted on PCIX errors: target abort, etc.
+ */
+static inline void asd_hst_pcix_isr(struct asd_ha_struct *asd_ha)
+{
+	u16 status;
+	u32 pcix_status;
+	u32 ecc_status;
+
+	pci_read_config_word(asd_ha->pcidev, PCI_STATUS, &status);
+	pci_read_config_dword(asd_ha->pcidev, PCIX_STATUS, &pcix_status);
+	pci_read_config_dword(asd_ha->pcidev, ECC_CTRL_STAT, &ecc_status);
+
+	if (status & PCI_STATUS_DETECTED_PARITY)
+		asd_printk("parity error for %s\n", pci_name(asd_ha->pcidev));
+	else if (status & PCI_STATUS_REC_MASTER_ABORT)
+		asd_printk("master abort for %s\n", pci_name(asd_ha->pcidev));
+	else if (status & PCI_STATUS_REC_TARGET_ABORT)
+		asd_printk("target abort for %s\n", pci_name(asd_ha->pcidev));
+	else if (status & PCI_STATUS_PARITY)
+		asd_printk("data parity for %s\n", pci_name(asd_ha->pcidev));
+	else if (pcix_status & RCV_SCE) {
+		asd_printk("received split completion error for %s\n",
+			   pci_name(asd_ha->pcidev));
+		pci_write_config_dword(asd_ha->pcidev,PCIX_STATUS,pcix_status);
+		/* XXX: Abort task? */
+		return;
+	} else if (pcix_status & UNEXP_SC) {
+		asd_printk("unexpected split completion for %s\n",
+			   pci_name(asd_ha->pcidev));
+		pci_write_config_dword(asd_ha->pcidev,PCIX_STATUS,pcix_status);
+		/* ignore */
+		return;
+	} else if (pcix_status & SC_DISCARD)
+		asd_printk("split completion discarded for %s\n",
+			   pci_name(asd_ha->pcidev));
+	else if (ecc_status & UNCOR_ECCERR)
+		asd_printk("uncorrectable ECC error for %s\n",
+			   pci_name(asd_ha->pcidev));
+	asd_chip_reset(asd_ha);
+}
+
+/**
+ * asd_hw_isr -- host adapter interrupt service routine
+ * @irq: ignored
+ * @dev_id: pointer to host adapter structure
+ * @regs: ignored
+ *
+ * The ISR processes done list entries and level 3 error handling.
+ */
+irqreturn_t asd_hw_isr(int irq, void *dev_id, struct pt_regs *regs)
+{
+	struct asd_ha_struct *asd_ha = dev_id;
+	u32 chimint = asd_read_reg_dword(asd_ha, CHIMINT);
+
+	if (!chimint)
+		return IRQ_NONE;
+
+	asd_write_reg_dword(asd_ha, CHIMINT, chimint);
+	(void) asd_read_reg_dword(asd_ha, CHIMINT);
+
+	if (chimint & DLAVAIL)
+		asd_process_donelist_isr(asd_ha);
+	if (chimint & COMINT)
+		asd_com_sas_isr(asd_ha);
+	if (chimint & DEVINT)
+		asd_dch_sas_isr(asd_ha);
+	if (chimint & INITERR)
+		asd_rbi_exsi_isr(asd_ha);
+	if (chimint & HOSTERR)
+		asd_hst_pcix_isr(asd_ha);
+
+	return IRQ_HANDLED;
+}
+
+/* ---------- SCB handling ---------- */
+
+static inline struct asd_ascb *asd_ascb_alloc(struct asd_ha_struct *asd_ha,
+					      unsigned int gfp_flags)
+{
+	extern kmem_cache_t *asd_ascb_cache;
+	struct asd_seq_data *seq = &asd_ha->seq;
+	struct asd_ascb *ascb;
+	unsigned long flags;
+
+	ascb = kmem_cache_alloc(asd_ascb_cache, gfp_flags);
+
+	if (ascb) {
+		memset(ascb, 0, sizeof(*ascb));
+		ascb->dma_scb.size = sizeof(struct scb);
+		ascb->dma_scb.vaddr = dma_pool_alloc(asd_ha->scb_pool,
+						     gfp_flags,
+						    &ascb->dma_scb.dma_handle);
+		if (!ascb->dma_scb.vaddr) {
+			kmem_cache_free(asd_ascb_cache, ascb);
+			return NULL;
+		}
+		memset(ascb->dma_scb.vaddr, 0, sizeof(struct scb));
+		asd_init_ascb(asd_ha, ascb);
+
+		spin_lock_irqsave(&seq->tc_index_lock, flags);
+		ascb->tc_index = asd_tc_index_get(seq, ascb);
+		spin_unlock_irqrestore(&seq->tc_index_lock, flags);
+		if (ascb->tc_index == -1)
+			goto undo;
+
+		ascb->scb->header.index = cpu_to_le16((u16)ascb->tc_index);
+	}
+
+	return ascb;
+undo:
+	dma_pool_free(asd_ha->scb_pool, ascb->dma_scb.vaddr,
+		      ascb->dma_scb.dma_handle);
+	kmem_cache_free(asd_ascb_cache, ascb);
+	ASD_DPRINTK("no index for ascb\n");
+	return NULL;
+}
+
+/**
+ * asd_ascb_alloc_list -- allocate a list of aSCBs
+ * @asd_ha: pointer to host adapter structure
+ * @num: pointer to integer number of aSCBs
+ * @gfp_flags: GFP_ flags.
+ *
+ * This is the only function which is used to allocate aSCBs.
+ * It can allocate one or many. If more than one, then they form
+ * a linked list in two ways: by their list field of the ascb struct
+ * and by the next_scb field of the scb_header.
+ *
+ * Returns NULL if no memory was available, else pointer to a list
+ * of ascbs.  When this function returns, @num would be the number
+ * of SCBs which were not able to be allocated, 0 if all requested
+ * were able to be allocated.
+ */
+struct asd_ascb *asd_ascb_alloc_list(struct asd_ha_struct
+				     *asd_ha, int *num,
+				     unsigned int gfp_flags)
+{
+	struct asd_ascb *first = NULL;
+
+	for ( ; *num > 0; --*num) {
+		struct asd_ascb *ascb = asd_ascb_alloc(asd_ha, gfp_flags);
+
+		if (!ascb)
+			break;
+		else if (!first)
+			first = ascb;
+		else {
+			struct asd_ascb *last = list_entry(first->list.prev,
+							   struct asd_ascb,
+							   list);
+			list_add_tail(&ascb->list, &first->list);
+			last->scb->header.next_scb =
+				cpu_to_le64(((u64)ascb->dma_scb.dma_handle));
+		}
+	}
+
+	return first;
+}
+
+/**
+ * asd_swap_head_scb -- swap the head scb
+ * @asd_ha: pointer to host adapter structure
+ * @ascb: pointer to the head of an ascb list
+ *
+ * The sequencer knows the DMA address of the next SCB to be DMAed to
+ * the host adapter, from initialization or from the last list DMAed.
+ * seq->next_scb keeps the address of this SCB.  The sequencer will
+ * DMA to the host adapter this list of SCBs.  But the head (first
+ * element) of this list is not known to the sequencer.  Here we swap
+ * the head of the list with the known SCB (memcpy()).
+ * Only one memcpy() is required per list so it is in our interest
+ * to keep the list of SCB as long as possible so that the ratio
+ * of number of memcpy calls to the number of SCB DMA-ed is as small
+ * as possible.
+ *
+ * LOCKING: called with the pending list lock held.
+ */
+static inline void asd_swap_head_scb(struct asd_ha_struct *asd_ha,
+				     struct asd_ascb *ascb)
+{
+	struct asd_seq_data *seq = &asd_ha->seq;
+	struct asd_ascb *last = list_entry(ascb->list.prev,
+					   struct asd_ascb,
+					   list);
+	struct asd_dma_tok t = ascb->dma_scb;
+
+	memcpy(seq->next_scb.vaddr, ascb->scb, sizeof(*ascb->scb));
+	ascb->dma_scb = seq->next_scb;
+	ascb->scb = ascb->dma_scb.vaddr;
+	seq->next_scb = t;
+	last->scb->header.next_scb =
+		cpu_to_le64(((u64)seq->next_scb.dma_handle));
+}
+
+/**
+ * asd_start_timers -- (add and) start timers of SCBs
+ * @list: pointer to struct list_head of the scbs
+ * @to: timeout in jiffies
+ *
+ * If an SCB in the @list has no timer function, assign the default
+ * one,  then start the timer of the SCB.  This function is
+ * intended to be called from asd_post_ascb_list(), just prior to
+ * posting the SCBs to the sequencer.
+ */
+static inline void asd_start_scb_timers(struct list_head *list)
+{
+	struct asd_ascb *ascb;
+	list_for_each_entry(ascb, list, list) {
+		if (!ascb->uldd_timer) {
+			ascb->timer.data = (unsigned long) ascb;
+			ascb->timer.function = asd_ascb_timedout;
+			ascb->timer.expires = jiffies + AIC94XX_SCB_TIMEOUT;
+			add_timer(&ascb->timer);
+		}
+	}
+}
+
+/**
+ * asd_post_ascb_list -- post a list of 1 or more aSCBs to the host adapter
+ * @asd_ha: pointer to a host adapter structure
+ * @ascb: pointer to the first aSCB in the list
+ * @num: number of aSCBs in the list (to be posted)
+ *
+ * See queueing comment in asd_post_escb_list().
+ *
+ * Additional note on queuing: In order to minimize the ratio of memcpy()
+ * to the number of ascbs sent, we try to batch-send as many ascbs as possible
+ * in one go.
+ * Two cases are possible:
+ *    A) can_queue >= num,
+ *    B) can_queue < num.
+ * Case A: we can send the whole batch at once.  Increment "pending"
+ * in the beginning of this function, when it is checked, in order to
+ * eliminate races when this function is called by multiple processes.
+ * Case B: should never happen if the managing layer considers
+ * lldd_queue_size.
+ */
+int asd_post_ascb_list(struct asd_ha_struct *asd_ha, struct asd_ascb *ascb,
+		       int num)
+{
+	unsigned long flags;
+	LIST_HEAD(list);
+	int can_queue;
+
+	spin_lock_irqsave(&asd_ha->seq.pend_q_lock, flags);
+	can_queue = asd_ha->hw_prof.max_scbs - asd_ha->seq.pending;
+	if (can_queue >= num)
+		asd_ha->seq.pending += num;
+	else
+		can_queue = 0;
+
+	if (!can_queue) {
+		spin_unlock_irqrestore(&asd_ha->seq.pend_q_lock, flags);
+		asd_printk("%s: scb queue full\n", pci_name(asd_ha->pcidev));
+		return -SAS_QUEUE_FULL;
+	}
+
+	asd_swap_head_scb(asd_ha, ascb);
+
+	__list_add(&list, ascb->list.prev, &ascb->list);
+
+	asd_start_scb_timers(&list);
+
+	asd_ha->seq.scbpro += num;
+	list_splice_init(&list, asd_ha->seq.pend_q.prev);
+	asd_write_reg_dword(asd_ha, SCBPRO, (u32)asd_ha->seq.scbpro);
+	spin_unlock_irqrestore(&asd_ha->seq.pend_q_lock, flags);
+
+	return 0;
+}
+
+/**
+ * asd_post_escb_list -- post a list of 1 or more empty scb
+ * @asd_ha: pointer to a host adapter structure
+ * @ascb: pointer to the first empty SCB in the list
+ * @num: number of aSCBs in the list (to be posted)
+ *
+ * This is essentially the same as asd_post_ascb_list, but we do not
+ * increment pending, add those to the pending list or get indexes.
+ * See asd_init_escbs() and asd_init_post_escbs().
+ *
+ * Since sending a list of ascbs is a superset of sending a single
+ * ascb, this function exists to generalize this.  More specifically,
+ * when sending a list of those, we want to do only a _single_
+ * memcpy() at swap head, as opposed to for each ascb sent (in the
+ * case of sending them one by one).  That is, we want to minimize the
+ * ratio of memcpy() operations to the number of ascbs sent.  The same
+ * logic applies to asd_post_ascb_list().
+ */
+int asd_post_escb_list(struct asd_ha_struct *asd_ha, struct asd_ascb *ascb,
+		       int num)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&asd_ha->seq.pend_q_lock, flags);
+	asd_swap_head_scb(asd_ha, ascb);
+	asd_ha->seq.scbpro += num;
+	asd_write_reg_dword(asd_ha, SCBPRO, (u32)asd_ha->seq.scbpro);
+	spin_unlock_irqrestore(&asd_ha->seq.pend_q_lock, flags);
+
+	return 0;
+}
+
+/* ---------- LED ---------- */
+
+/**
+ * asd_turn_led -- turn on/off an LED
+ * @asd_ha: pointer to host adapter structure
+ * @phy_id: the PHY id whose LED we want to manupulate
+ * @op: 1 to turn on, 0 to turn off
+ */
+void asd_turn_led(struct asd_ha_struct *asd_ha, int phy_id, int op)
+{
+	if (phy_id < ASD_MAX_PHYS) {
+		u32 v = asd_read_reg_dword(asd_ha, LmCONTROL(phy_id));
+		if (op)
+			v |= LEDPOL;
+		else
+			v &= ~LEDPOL;
+		asd_write_reg_dword(asd_ha, LmCONTROL(phy_id), v);
+	}
+}
+
+/**
+ * asd_control_led -- enable/disable an LED on the board
+ * @asd_ha: pointer to host adapter structure
+ * @phy_id: integer, the phy id
+ * @op: integer, 1 to enable, 0 to disable the LED
+ *
+ * First we output enable the LED, then we set the source
+ * to be an external module.
+ */
+void asd_control_led(struct asd_ha_struct *asd_ha, int phy_id, int op)
+{
+	if (phy_id < ASD_MAX_PHYS) {
+		u32 v;
+
+		v = asd_read_reg_dword(asd_ha, GPIOOER);
+		if (op)
+			v |= (1 << phy_id);
+		else
+			v &= ~(1 << phy_id);
+		asd_write_reg_dword(asd_ha, GPIOOER, v);
+
+		v = asd_read_reg_dword(asd_ha, GPIOCNFGR);
+		if (op)
+			v |= (1 << phy_id);
+		else
+			v &= ~(1 << phy_id);
+		asd_write_reg_dword(asd_ha, GPIOCNFGR, v);
+	}
+}
+
+/* ---------- PHY enable ---------- */
+
+static int asd_enable_phy(struct asd_ha_struct *asd_ha, int phy_id)
+{
+	struct asd_phy *phy = &asd_ha->phys[phy_id];
+
+	asd_write_reg_byte(asd_ha, LmSEQ_OOB_REG(phy_id, INT_ENABLE_2), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_OOB_REG(phy_id, HOT_PLUG_DELAY),
+			   HOTPLUG_DELAY_TIMEOUT);
+
+	/* Get defaults from manuf. sector */
+	/* XXX we need defaults for those in case MS is broken. */
+	asd_write_reg_byte(asd_ha, LmSEQ_OOB_REG(phy_id, PHY_CONTROL_0),
+			   phy->phy_desc->phy_control_0);
+	asd_write_reg_byte(asd_ha, LmSEQ_OOB_REG(phy_id, PHY_CONTROL_1),
+			   phy->phy_desc->phy_control_1);
+	asd_write_reg_byte(asd_ha, LmSEQ_OOB_REG(phy_id, PHY_CONTROL_2),
+			   phy->phy_desc->phy_control_2);
+	asd_write_reg_byte(asd_ha, LmSEQ_OOB_REG(phy_id, PHY_CONTROL_3),
+			   phy->phy_desc->phy_control_3);
+
+	asd_write_reg_dword(asd_ha, LmSEQ_TEN_MS_COMINIT_TIMEOUT(phy_id),
+			    ASD_COMINIT_TIMEOUT);
+
+	asd_write_reg_addr(asd_ha, LmSEQ_TX_ID_ADDR_FRAME(phy_id),
+			   phy->id_frm_tok->dma_handle);
+
+	asd_control_led(asd_ha, phy_id, 1);
+
+	return 0;
+}
+
+int asd_enable_phys(struct asd_ha_struct *asd_ha, const u8 phy_mask)
+{
+	u8  phy_m;
+	u8  i;
+	int num = 0, k;
+	struct asd_ascb *ascb;
+	struct asd_ascb *ascb_list;
+
+	if (!phy_mask) {
+		asd_printk("%s called with phy_mask of 0!?\n", __FUNCTION__);
+		return 0;
+	}
+
+	for_each_phy(phy_mask, phy_m, i) {
+		num++;
+		asd_enable_phy(asd_ha, i);
+	}
+
+	k = num;
+	ascb_list = asd_ascb_alloc_list(asd_ha, &k, GFP_KERNEL);
+	if (!ascb_list) {
+		asd_printk("no memory for control phy ascb list\n");
+		return -ENOMEM;
+	}
+	num -= k;
+
+	ascb = ascb_list;
+	for_each_phy(phy_mask, phy_m, i) {
+		asd_build_control_phy(ascb, i, ENABLE_PHY);
+		ascb = list_entry(ascb->list.next, struct asd_ascb, list);
+	}
+	ASD_DPRINTK("posting %d control phy scbs\n", num);
+	k = asd_post_ascb_list(asd_ha, ascb_list, num);
+	if (k)
+		asd_ascb_free_list(ascb_list);
+
+	return k;
+}
diff --git a/drivers/scsi/aic94xx/aic94xx_hwi.h b/drivers/scsi/aic94xx/aic94xx_hwi.h
new file mode 100644
index 0000000..c7d5053
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_hwi.h
@@ -0,0 +1,397 @@
+/*
+ * Aic94xx SAS/SATA driver hardware interface header file.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#ifndef _AIC94XX_HWI_H_
+#define _AIC94XX_HWI_H_
+
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+#include <linux/dma-mapping.h>
+
+#include <scsi/libsas.h>
+
+#include "aic94xx.h"
+#include "aic94xx_sas.h"
+
+/* Define ASD_MAX_PHYS to the maximum phys ever. Currently 8. */
+#define ASD_MAX_PHYS       8
+#define ASD_PCBA_SN_SIZE   12
+
+/* Those are to be further named properly, the "RAZORx" part, and
+ * subsequently included in include/linux/pci_ids.h.
+ */
+#define PCI_DEVICE_ID_ADAPTEC2_RAZOR10 0x410
+#define PCI_DEVICE_ID_ADAPTEC2_RAZOR12 0x412
+#define PCI_DEVICE_ID_ADAPTEC2_RAZOR1E 0x41E
+#define PCI_DEVICE_ID_ADAPTEC2_RAZOR30 0x430
+#define PCI_DEVICE_ID_ADAPTEC2_RAZOR32 0x432
+#define PCI_DEVICE_ID_ADAPTEC2_RAZOR3E 0x43E
+#define PCI_DEVICE_ID_ADAPTEC2_RAZOR3F 0x43F
+
+struct asd_ha_addrspace {
+	void __iomem  *addr;
+	unsigned long  start;       /* pci resource start */
+	unsigned long  len;         /* pci resource len */
+	unsigned long  flags;       /* pci resource flags */
+
+	/* addresses internal to the host adapter */
+	u32 swa_base; /* mmspace 1 (MBAR1) uses this only */
+	u32 swb_base;
+	u32 swc_base;
+};
+
+struct bios_struct {
+	int    present;
+	u8     maj;
+	u8     min;
+	u32    bld;
+};
+
+struct unit_element_struct {
+	u16    num;
+	u16    size;
+	void   *area;
+};
+
+struct flash_struct {
+	u32    bar;
+	int    present;
+	int    wide;
+	u8     manuf;
+	u8     dev_id;
+	u8     sec_prot;
+
+	u32    dir_offs;
+};
+
+struct asd_phy_desc {
+	/* From CTRL-A settings, then set to what is appropriate */
+	u8     sas_addr[SAS_ADDR_SIZE];
+	u8     max_sas_lrate;
+	u8     min_sas_lrate;
+	u8     max_sata_lrate;
+	u8     min_sata_lrate;
+	u8     flags;
+#define ASD_CRC_DIS  1
+#define ASD_SATA_SPINUP_HOLD 2
+
+	u8     phy_control_0; /* mode 5 reg 0x160 */
+	u8     phy_control_1; /* mode 5 reg 0x161 */
+	u8     phy_control_2; /* mode 5 reg 0x162 */
+	u8     phy_control_3; /* mode 5 reg 0x163 */
+};
+
+struct asd_dma_tok {
+	void *vaddr;
+	dma_addr_t dma_handle;
+	size_t size;
+};
+
+struct hw_profile {
+	struct bios_struct bios;
+	struct unit_element_struct ue;
+	struct flash_struct flash;
+
+	u8     sas_addr[SAS_ADDR_SIZE];
+	char   pcba_sn[ASD_PCBA_SN_SIZE+1];
+
+	u8     enabled_phys;	  /* mask of enabled phys */
+	struct asd_phy_desc phy_desc[ASD_MAX_PHYS];
+	u32    max_scbs;	  /* absolute sequencer scb queue size */
+	struct asd_dma_tok *scb_ext;
+	u32    max_ddbs;
+	struct asd_dma_tok *ddb_ext;
+
+	spinlock_t ddb_lock;
+	void  *ddb_bitmap;
+
+	int    num_phys;	  /* ENABLEABLE */
+	int    max_phys;	  /* REPORTED + ENABLEABLE */
+
+	unsigned addr_range;	  /* max # of addrs; max # of possible ports */
+	unsigned port_name_base;
+	unsigned dev_name_base;
+	unsigned sata_name_base;
+};
+
+struct asd_ascb {
+	struct list_head list;
+	struct asd_ha_struct *ha;
+
+	struct scb *scb;	  /* equals dma_scb->vaddr */
+	struct asd_dma_tok dma_scb;
+	struct asd_dma_tok *sg_arr;
+
+	void (*tasklet_complete)(struct asd_ascb *, struct done_list_struct *);
+	u8     uldd_timer:1;
+
+	/* internally generated command */
+	struct timer_list timer;
+	struct completion completion;
+	u8        tag_valid:1;
+	__be16    tag;		  /* error recovery only */
+
+	/* If this is an Empty SCB, index of first edb in seq->edb_arr. */
+	int    edb_index;
+
+	/* Used by the timer timeout function. */
+	int    tc_index;
+
+	void   *uldd_task;
+};
+
+#define ASD_DL_SIZE_BITS   0x8
+#define ASD_DL_SIZE        (1<<(2+ASD_DL_SIZE_BITS))
+#define ASD_DEF_DL_TOGGLE  0x01
+
+struct asd_seq_data {
+	spinlock_t pend_q_lock;
+	u16    scbpro;
+	int    pending;
+	struct list_head pend_q;
+	int    can_queue;	  /* per adapter */
+	struct asd_dma_tok next_scb; /* next scb to be delivered to CSEQ */
+
+	spinlock_t tc_index_lock;
+	void **tc_index_array;
+	void *tc_index_bitmap;
+	int   tc_index_bitmap_bits;
+
+	struct tasklet_struct dl_tasklet;
+	struct done_list_struct *dl; /* array of done list entries, equals */
+	struct asd_dma_tok *actual_dl; /* actual_dl->vaddr */
+	int    dl_toggle;
+	int    dl_next;
+
+	int    num_edbs;
+	struct asd_dma_tok **edb_arr;
+	int    num_escbs;
+	struct asd_ascb **escb_arr; /* array of pointers to escbs */
+};
+
+/* This is the Host Adapter structure.  It describes the hardware
+ * SAS adapter.
+ */
+struct asd_ha_struct {
+	struct pci_dev   *pcidev;
+	const char       *name;
+
+	struct sas_ha_struct sas_ha;
+
+	u8                revision_id;
+
+	int               iospace;
+	spinlock_t        iolock;
+	struct asd_ha_addrspace io_handle[2];
+
+	struct hw_profile hw_prof;
+
+	struct asd_phy    phys[ASD_MAX_PHYS];
+	struct asd_sas_port   ports[ASD_MAX_PHYS];
+
+	struct dma_pool  *scb_pool;
+
+	struct asd_seq_data  seq; /* sequencer related */
+};
+
+/* ---------- Common macros ---------- */
+
+#define ASD_BUSADDR_LO(__dma_handle) ((u32)(__dma_handle))
+#define ASD_BUSADDR_HI(__dma_handle) (((sizeof(dma_addr_t))==8)     \
+                                    ? ((u32)((__dma_handle) >> 32)) \
+                                    : ((u32)0))
+
+#define dev_to_asd_ha(__dev)  pci_get_drvdata(to_pci_dev(__dev))
+#define SCB_SITE_VALID(__site_no) (((__site_no) & 0xF0FF) != 0x00FF   \
+				 && ((__site_no) & 0xF0FF) > 0x001F)
+/* For each bit set in __lseq_mask, set __lseq to equal the bit
+ * position of the set bit and execute the statement following.
+ * __mc is the temporary mask, used as a mask "counter".
+ */
+#define for_each_sequencer(__lseq_mask, __mc, __lseq)                        \
+	for ((__mc)=(__lseq_mask),(__lseq)=0;(__mc)!=0;(__lseq++),(__mc)>>=1)\
+		if (((__mc) & 1))
+#define for_each_phy(__lseq_mask, __mc, __lseq)                              \
+	for ((__mc)=(__lseq_mask),(__lseq)=0;(__mc)!=0;(__lseq++),(__mc)>>=1)\
+		if (((__mc) & 1))
+
+#define PHY_ENABLED(_HA, _I) ((_HA)->hw_prof.enabled_phys & (1<<(_I)))
+
+/* ---------- DMA allocs ---------- */
+
+static inline struct asd_dma_tok *asd_dmatok_alloc(unsigned int flags)
+{
+	return kmem_cache_alloc(asd_dma_token_cache, flags);
+}
+
+static inline void asd_dmatok_free(struct asd_dma_tok *token)
+{
+	kmem_cache_free(asd_dma_token_cache, token);
+}
+
+static inline struct asd_dma_tok *asd_alloc_coherent(struct asd_ha_struct *
+						     asd_ha, size_t size,
+						     unsigned int flags)
+{
+	struct asd_dma_tok *token = asd_dmatok_alloc(flags);
+	if (token) {
+		token->size = size;
+		token->vaddr = dma_alloc_coherent(&asd_ha->pcidev->dev,
+						  token->size,
+						  &token->dma_handle,
+						  flags);
+		if (!token->vaddr) {
+			asd_dmatok_free(token);
+			token = NULL;
+		}
+	}
+	return token;
+}
+
+static inline void asd_free_coherent(struct asd_ha_struct *asd_ha,
+				     struct asd_dma_tok *token)
+{
+	if (token) {
+		dma_free_coherent(&asd_ha->pcidev->dev, token->size,
+				  token->vaddr, token->dma_handle);
+		asd_dmatok_free(token);
+	}
+}
+
+static inline void asd_init_ascb(struct asd_ha_struct *asd_ha,
+				 struct asd_ascb *ascb)
+{
+	INIT_LIST_HEAD(&ascb->list);
+	ascb->scb = ascb->dma_scb.vaddr;
+	ascb->ha = asd_ha;
+	ascb->timer.function = NULL;
+	init_timer(&ascb->timer);
+	ascb->tc_index = -1;
+	init_completion(&ascb->completion);
+}
+
+/* Must be called with the tc_index_lock held!
+ */
+static inline void asd_tc_index_release(struct asd_seq_data *seq, int index)
+{
+	seq->tc_index_array[index] = NULL;
+	clear_bit(index, seq->tc_index_bitmap);
+}
+
+/* Must be called with the tc_index_lock held!
+ */
+static inline int asd_tc_index_get(struct asd_seq_data *seq, void *ptr)
+{
+	int index;
+
+	index = find_first_zero_bit(seq->tc_index_bitmap,
+				    seq->tc_index_bitmap_bits);
+	if (index == seq->tc_index_bitmap_bits)
+		return -1;
+
+	seq->tc_index_array[index] = ptr;
+	set_bit(index, seq->tc_index_bitmap);
+
+	return index;
+}
+
+/* Must be called with the tc_index_lock held!
+ */
+static inline void *asd_tc_index_find(struct asd_seq_data *seq, int index)
+{
+	return seq->tc_index_array[index];
+}
+
+/**
+ * asd_ascb_free -- free a single aSCB after is has completed
+ * @ascb: pointer to the aSCB of interest
+ *
+ * This frees an aSCB after it has been executed/completed by
+ * the sequencer.
+ */
+static inline void asd_ascb_free(struct asd_ascb *ascb)
+{
+	if (ascb) {
+		struct asd_ha_struct *asd_ha = ascb->ha;
+		unsigned long flags;
+
+		BUG_ON(!list_empty(&ascb->list));
+		spin_lock_irqsave(&ascb->ha->seq.tc_index_lock, flags);
+		asd_tc_index_release(&ascb->ha->seq, ascb->tc_index);
+		spin_unlock_irqrestore(&ascb->ha->seq.tc_index_lock, flags);
+		dma_pool_free(asd_ha->scb_pool, ascb->dma_scb.vaddr,
+			      ascb->dma_scb.dma_handle);
+		kmem_cache_free(asd_ascb_cache, ascb);
+	}
+}
+
+/**
+ * asd_ascb_list_free -- free a list of ascbs
+ * @ascb_list: a list of ascbs
+ *
+ * This function will free a list of ascbs allocated by asd_ascb_alloc_list.
+ * It is used when say the scb queueing function returned QUEUE_FULL,
+ * and we do not need the ascbs any more.
+ */
+static inline void asd_ascb_free_list(struct asd_ascb *ascb_list)
+{
+	LIST_HEAD(list);
+	struct list_head *n, *pos;
+
+	__list_add(&list, ascb_list->list.prev, &ascb_list->list);
+	list_for_each_safe(pos, n, &list) {
+		list_del_init(pos);
+		asd_ascb_free(list_entry(pos, struct asd_ascb, list));
+	}
+}
+
+/* ---------- Function declarations ---------- */
+
+int  asd_init_hw(struct asd_ha_struct *asd_ha);
+irqreturn_t asd_hw_isr(int irq, void *dev_id, struct pt_regs *regs);
+
+
+struct asd_ascb *asd_ascb_alloc_list(struct asd_ha_struct
+				     *asd_ha, int *num,
+				     unsigned int gfp_mask);
+
+int  asd_post_ascb_list(struct asd_ha_struct *asd_ha, struct asd_ascb *ascb,
+			int num);
+int  asd_post_escb_list(struct asd_ha_struct *asd_ha, struct asd_ascb *ascb,
+			int num);
+
+int  asd_init_post_escbs(struct asd_ha_struct *asd_ha);
+void asd_build_control_phy(struct asd_ascb *ascb, int phy_id, u8 subfunc);
+void asd_control_led(struct asd_ha_struct *asd_ha, int phy_id, int op);
+void asd_turn_led(struct asd_ha_struct *asd_ha, int phy_id, int op);
+int  asd_enable_phys(struct asd_ha_struct *asd_ha, const u8 phy_mask);
+void asd_build_initiate_link_adm_task(struct asd_ascb *ascb, int phy_id,
+				      u8 subfunc);
+
+void asd_ascb_timedout(unsigned long data);
+int  asd_chip_hardrst(struct asd_ha_struct *asd_ha);
+
+#endif
diff --git a/drivers/scsi/aic94xx/aic94xx_init.c b/drivers/scsi/aic94xx/aic94xx_init.c
new file mode 100644
index 0000000..ee2ccad
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_init.c
@@ -0,0 +1,866 @@
+/*
+ * Aic94xx SAS/SATA driver initialization.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/config.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/pci.h>
+#include <linux/delay.h>
+
+#include <scsi/scsi_host.h>
+
+#include "aic94xx.h"
+#include "aic94xx_reg.h"
+#include "aic94xx_hwi.h"
+#include "aic94xx_seq.h"
+
+/* The format is "version.release.patchlevel" */
+#define ASD_DRIVER_VERSION "1.0.2"
+
+static int use_msi = 0;
+module_param_named(use_msi, use_msi, int, S_IRUGO);
+MODULE_PARM_DESC(use_msi, "\n"
+	"\tEnable(1) or disable(0) using PCI MSI.\n"
+	"\tDefault: 0");
+
+static int lldd_max_execute_num = 0;
+module_param_named(collector, lldd_max_execute_num, int, S_IRUGO);
+MODULE_PARM_DESC(collector, "\n"
+	"\tIf greater than one, tells the SAS Layer to run in Task Collector\n"
+	"\tMode.  If 1 or 0, tells the SAS Layer to run in Direct Mode.\n"
+	"\tThe aic94xx SAS LLDD supports both modes.\n"
+	"\tDefault: 0 (Direct Mode).\n");
+
+char sas_addr_str[2*SAS_ADDR_SIZE + 1] = "";
+
+static struct scsi_transport_template *aic94xx_transport_template;
+
+static struct scsi_host_template aic94xx_sht = {
+	.module			= THIS_MODULE,
+	/* .name is initialized */
+	.name			= "aic94xx",
+	.queuecommand		= sas_queuecommand,
+	.target_alloc		= sas_target_alloc,
+	.slave_configure	= sas_slave_configure,
+	.slave_destroy		= sas_slave_destroy,
+	.change_queue_depth	= sas_change_queue_depth,
+	.change_queue_type	= sas_change_queue_type,
+	.bios_param		= sas_bios_param,
+	.can_queue		= 1,
+	.cmd_per_lun		= 1,
+	.this_id		= -1,
+	.sg_tablesize		= SG_ALL,
+	.max_sectors		= SCSI_DEFAULT_MAX_SECTORS,
+	.use_clustering		= ENABLE_CLUSTERING,
+};
+
+static int __devinit asd_map_memio(struct asd_ha_struct *asd_ha)
+{
+	int err, i;
+	struct asd_ha_addrspace *io_handle;
+
+	asd_ha->iospace = 0;
+	for (i = 0; i < 3; i += 2) {
+		io_handle = &asd_ha->io_handle[i==0?0:1];
+		io_handle->start = pci_resource_start(asd_ha->pcidev, i);
+		io_handle->len   = pci_resource_len(asd_ha->pcidev, i);
+		io_handle->flags = pci_resource_flags(asd_ha->pcidev, i);
+		err = -ENODEV;
+		if (!io_handle->start || !io_handle->len) {
+			asd_printk("MBAR%d start or length for %s is 0.\n",
+				   i==0?0:1, pci_name(asd_ha->pcidev));
+			goto Err;
+		}
+		err = pci_request_region(asd_ha->pcidev, i, ASD_DRIVER_NAME);
+		if (err) {
+			asd_printk("couldn't reserve memory region for %s\n",
+				   pci_name(asd_ha->pcidev));
+			goto Err;
+		}
+		if (io_handle->flags & IORESOURCE_CACHEABLE)
+			io_handle->addr = ioremap(io_handle->start,
+						  io_handle->len);
+		else
+			io_handle->addr = ioremap_nocache(io_handle->start,
+							  io_handle->len);
+		if (!io_handle->addr) {
+			asd_printk("couldn't map MBAR%d of %s\n", i==0?0:1,
+				   pci_name(asd_ha->pcidev));
+			goto Err_unreq;
+		}
+	}
+
+	return 0;
+Err_unreq:
+	pci_release_region(asd_ha->pcidev, i);
+Err:
+	if (i > 0) {
+		io_handle = &asd_ha->io_handle[0];
+		iounmap(io_handle->addr);
+		pci_release_region(asd_ha->pcidev, 0);
+	}
+	return err;
+}
+
+static void __devexit asd_unmap_memio(struct asd_ha_struct *asd_ha)
+{
+	struct asd_ha_addrspace *io_handle;
+
+	io_handle = &asd_ha->io_handle[1];
+	iounmap(io_handle->addr);
+	pci_release_region(asd_ha->pcidev, 2);
+
+	io_handle = &asd_ha->io_handle[0];
+	iounmap(io_handle->addr);
+	pci_release_region(asd_ha->pcidev, 0);
+}
+
+static int __devinit asd_map_ioport(struct asd_ha_struct *asd_ha)
+{
+	int i = PCI_IOBAR_OFFSET, err;
+	struct asd_ha_addrspace *io_handle = &asd_ha->io_handle[0];
+
+	asd_ha->iospace = 1;
+	io_handle->start = pci_resource_start(asd_ha->pcidev, i);
+	io_handle->len   = pci_resource_len(asd_ha->pcidev, i);
+	io_handle->flags = pci_resource_flags(asd_ha->pcidev, i);
+	io_handle->addr  = (void __iomem *) io_handle->start;
+	if (!io_handle->start || !io_handle->len) {
+		asd_printk("couldn't get IO ports for %s\n",
+			   pci_name(asd_ha->pcidev));
+		return -ENODEV;
+	}
+	err = pci_request_region(asd_ha->pcidev, i, ASD_DRIVER_NAME);
+	if (err) {
+		asd_printk("couldn't reserve io space for %s\n",
+			   pci_name(asd_ha->pcidev));
+	}
+
+	return err;
+}
+
+static void __devexit asd_unmap_ioport(struct asd_ha_struct *asd_ha)
+{
+	pci_release_region(asd_ha->pcidev, PCI_IOBAR_OFFSET);
+}
+
+static int __devinit asd_map_ha(struct asd_ha_struct *asd_ha)
+{
+	int err;
+	u16 cmd_reg;
+
+	err = pci_read_config_word(asd_ha->pcidev, PCI_COMMAND, &cmd_reg);
+	if (err) {
+		asd_printk("couldn't read command register of %s\n",
+			   pci_name(asd_ha->pcidev));
+		goto Err;
+	}
+
+	err = -ENODEV;
+	if (cmd_reg & PCI_COMMAND_MEMORY) {
+		if ((err = asd_map_memio(asd_ha)))
+			goto Err;
+	} else if (cmd_reg & PCI_COMMAND_IO) {
+		if ((err = asd_map_ioport(asd_ha)))
+			goto Err;
+		asd_printk("%s ioport mapped -- upgrade your hardware\n",
+			   pci_name(asd_ha->pcidev));
+	} else {
+		asd_printk("no proper device access to %s\n",
+			   pci_name(asd_ha->pcidev));
+		goto Err;
+	}
+
+	return 0;
+Err:
+	return err;
+}
+
+static void __devexit asd_unmap_ha(struct asd_ha_struct *asd_ha)
+{
+	if (asd_ha->iospace)
+		asd_unmap_ioport(asd_ha);
+	else
+		asd_unmap_memio(asd_ha);
+}
+
+static const char *asd_dev_rev[30] = {
+	[0] = "A0",
+	[1] = "A1",
+	[8] = "B0",
+};
+
+static int __devinit asd_common_setup(struct asd_ha_struct *asd_ha)
+{
+	int err, i;
+
+	err = pci_read_config_byte(asd_ha->pcidev, PCI_REVISION_ID,
+				   &asd_ha->revision_id);
+	if (err) {
+		asd_printk("couldn't read REVISION ID register of %s\n",
+			   pci_name(asd_ha->pcidev));
+		goto Err;
+	}
+	err = -ENODEV;
+	if (asd_ha->revision_id < AIC9410_DEV_REV_B0) {
+		asd_printk("%s is revision %s (%X), which is not supported\n",
+			   pci_name(asd_ha->pcidev),
+			   asd_dev_rev[asd_ha->revision_id],
+			   asd_ha->revision_id);
+		goto Err;
+	}
+	/* Provide some sane default values. */
+	asd_ha->hw_prof.max_scbs = 512;
+	asd_ha->hw_prof.max_ddbs = 128;
+	asd_ha->hw_prof.num_phys = ASD_MAX_PHYS;
+	/* All phys are enabled, by default. */
+	asd_ha->hw_prof.enabled_phys = 0xFF;
+	for (i = 0; i < ASD_MAX_PHYS; i++) {
+		asd_ha->hw_prof.phy_desc[i].max_sas_lrate =
+			SAS_LINK_RATE_3_0_GBPS;
+		asd_ha->hw_prof.phy_desc[i].min_sas_lrate =
+			SAS_LINK_RATE_1_5_GBPS;
+		asd_ha->hw_prof.phy_desc[i].max_sata_lrate =
+			SAS_LINK_RATE_1_5_GBPS;
+		asd_ha->hw_prof.phy_desc[i].min_sata_lrate =
+			SAS_LINK_RATE_1_5_GBPS;
+	}
+
+	return 0;
+Err:
+	return err;
+}
+
+static int __devinit asd_aic9410_setup(struct asd_ha_struct *asd_ha)
+{
+	int err = asd_common_setup(asd_ha);
+
+	if (err)
+		return err;
+
+	asd_ha->hw_prof.addr_range = 8;
+	asd_ha->hw_prof.port_name_base = 0;
+	asd_ha->hw_prof.dev_name_base = 8;
+	asd_ha->hw_prof.sata_name_base = 16;
+
+	return 0;
+}
+
+static int __devinit asd_aic9405_setup(struct asd_ha_struct *asd_ha)
+{
+	int err = asd_common_setup(asd_ha);
+
+	if (err)
+		return err;
+
+	asd_ha->hw_prof.addr_range = 4;
+	asd_ha->hw_prof.port_name_base = 0;
+	asd_ha->hw_prof.dev_name_base = 4;
+	asd_ha->hw_prof.sata_name_base = 8;
+
+	return 0;
+}
+
+static ssize_t asd_show_dev_rev(struct device *dev,
+				struct device_attribute *attr, char *buf)
+{
+	struct asd_ha_struct *asd_ha = dev_to_asd_ha(dev);
+	return snprintf(buf, PAGE_SIZE, "%s\n",
+			asd_dev_rev[asd_ha->revision_id]);
+}
+static DEVICE_ATTR(revision, S_IRUGO, asd_show_dev_rev, NULL);
+
+static ssize_t asd_show_dev_bios_build(struct device *dev,
+				       struct device_attribute *attr,char *buf)
+{
+	struct asd_ha_struct *asd_ha = dev_to_asd_ha(dev);
+	return snprintf(buf, PAGE_SIZE, "%d\n", asd_ha->hw_prof.bios.bld);
+}
+static DEVICE_ATTR(bios_build, S_IRUGO, asd_show_dev_bios_build, NULL);
+
+static ssize_t asd_show_dev_pcba_sn(struct device *dev,
+				    struct device_attribute *attr, char *buf)
+{
+	struct asd_ha_struct *asd_ha = dev_to_asd_ha(dev);
+	return snprintf(buf, PAGE_SIZE, "%s\n", asd_ha->hw_prof.pcba_sn);
+}
+static DEVICE_ATTR(pcba_sn, S_IRUGO, asd_show_dev_pcba_sn, NULL);
+
+static void asd_create_dev_attrs(struct asd_ha_struct *asd_ha)
+{
+	device_create_file(&asd_ha->pcidev->dev, &dev_attr_revision);
+	device_create_file(&asd_ha->pcidev->dev, &dev_attr_bios_build);
+	device_create_file(&asd_ha->pcidev->dev, &dev_attr_pcba_sn);
+}
+
+static void asd_remove_dev_attrs(struct asd_ha_struct *asd_ha)
+{
+	device_remove_file(&asd_ha->pcidev->dev, &dev_attr_revision);
+	device_remove_file(&asd_ha->pcidev->dev, &dev_attr_bios_build);
+	device_remove_file(&asd_ha->pcidev->dev, &dev_attr_pcba_sn);
+}
+
+/* The first entry, 0, is used for dynamic ids, the rest for devices
+ * we know about.
+ */
+static struct asd_pcidev_struct {
+	const char * name;
+	int (*setup)(struct asd_ha_struct *asd_ha);
+} asd_pcidev_data[] = {
+	/* Id 0 is used for dynamic ids. */
+	{ .name  = "Adaptec AIC-94xx SAS/SATA Host Adapter",
+	  .setup = asd_aic9410_setup
+	},
+	{ .name  = "Adaptec AIC-9410W SAS/SATA Host Adapter",
+	  .setup = asd_aic9410_setup
+	},
+	{ .name  = "Adaptec AIC-9405W SAS/SATA Host Adapter",
+	  .setup = asd_aic9405_setup
+	},
+};
+
+static inline int asd_create_ha_caches(struct asd_ha_struct *asd_ha)
+{
+	asd_ha->scb_pool = dma_pool_create(ASD_DRIVER_NAME "_scb_pool",
+					   &asd_ha->pcidev->dev,
+					   sizeof(struct scb),
+					   8, 0);
+	if (!asd_ha->scb_pool) {
+		asd_printk("couldn't create scb pool\n");
+		return -ENOMEM;
+	}
+
+	return 0;
+}
+
+/**
+ * asd_free_edbs -- free empty data buffers
+ * asd_ha: pointer to host adapter structure
+ */
+static inline void asd_free_edbs(struct asd_ha_struct *asd_ha)
+{
+	struct asd_seq_data *seq = &asd_ha->seq;
+	int i;
+
+	for (i = 0; i < seq->num_edbs; i++)
+		asd_free_coherent(asd_ha, seq->edb_arr[i]);
+	kfree(seq->edb_arr);
+	seq->edb_arr = NULL;
+}
+
+static inline void asd_free_escbs(struct asd_ha_struct *asd_ha)
+{
+	struct asd_seq_data *seq = &asd_ha->seq;
+	int i;
+
+	for (i = 0; i < seq->num_escbs; i++) {
+		if (!list_empty(&seq->escb_arr[i]->list))
+			list_del_init(&seq->escb_arr[i]->list);
+
+		asd_ascb_free(seq->escb_arr[i]);
+	}
+	kfree(seq->escb_arr);
+	seq->escb_arr = NULL;
+}
+
+static inline void asd_destroy_ha_caches(struct asd_ha_struct *asd_ha)
+{
+	int i;
+
+	if (asd_ha->hw_prof.ddb_ext)
+		asd_free_coherent(asd_ha, asd_ha->hw_prof.ddb_ext);
+	if (asd_ha->hw_prof.scb_ext)
+		asd_free_coherent(asd_ha, asd_ha->hw_prof.scb_ext);
+
+	if (asd_ha->hw_prof.ddb_bitmap)
+		kfree(asd_ha->hw_prof.ddb_bitmap);
+	asd_ha->hw_prof.ddb_bitmap = NULL;
+
+	for (i = 0; i < ASD_MAX_PHYS; i++) {
+		struct asd_phy *phy = &asd_ha->phys[i];
+
+		asd_free_coherent(asd_ha, phy->id_frm_tok);
+	}
+	if (asd_ha->seq.escb_arr)
+		asd_free_escbs(asd_ha);
+	if (asd_ha->seq.edb_arr)
+		asd_free_edbs(asd_ha);
+	if (asd_ha->hw_prof.ue.area) {
+		kfree(asd_ha->hw_prof.ue.area);
+		asd_ha->hw_prof.ue.area = NULL;
+	}
+	if (asd_ha->seq.tc_index_array) {
+		kfree(asd_ha->seq.tc_index_array);
+		kfree(asd_ha->seq.tc_index_bitmap);
+		asd_ha->seq.tc_index_array = NULL;
+		asd_ha->seq.tc_index_bitmap = NULL;
+	}
+	if (asd_ha->seq.actual_dl) {
+			asd_free_coherent(asd_ha, asd_ha->seq.actual_dl);
+			asd_ha->seq.actual_dl = NULL;
+			asd_ha->seq.dl = NULL;
+	}
+	if (asd_ha->seq.next_scb.vaddr) {
+		dma_pool_free(asd_ha->scb_pool, asd_ha->seq.next_scb.vaddr,
+			      asd_ha->seq.next_scb.dma_handle);
+		asd_ha->seq.next_scb.vaddr = NULL;
+	}
+	dma_pool_destroy(asd_ha->scb_pool);
+	asd_ha->scb_pool = NULL;
+}
+
+kmem_cache_t *asd_dma_token_cache;
+kmem_cache_t *asd_ascb_cache;
+
+static int asd_create_global_caches(void)
+{
+	if (!asd_dma_token_cache) {
+		asd_dma_token_cache
+			= kmem_cache_create(ASD_DRIVER_NAME "_dma_token",
+					    sizeof(struct asd_dma_tok),
+					    0,
+					    SLAB_HWCACHE_ALIGN,
+					    NULL, NULL);
+		if (!asd_dma_token_cache) {
+			asd_printk("couldn't create dma token cache\n");
+			return -ENOMEM;
+		}
+	}
+
+	if (!asd_ascb_cache) {
+		asd_ascb_cache = kmem_cache_create(ASD_DRIVER_NAME "_ascb",
+						   sizeof(struct asd_ascb),
+						   0,
+						   SLAB_HWCACHE_ALIGN,
+						   NULL, NULL);
+		if (!asd_ascb_cache) {
+			asd_printk("couldn't create ascb cache\n");
+			goto Err;
+		}
+	}
+
+	return 0;
+Err:
+	kmem_cache_destroy(asd_dma_token_cache);
+	asd_dma_token_cache = NULL;
+	return -ENOMEM;
+}
+
+static void asd_destroy_global_caches(void)
+{
+	if (asd_dma_token_cache)
+		kmem_cache_destroy(asd_dma_token_cache);
+	asd_dma_token_cache = NULL;
+
+	if (asd_ascb_cache)
+		kmem_cache_destroy(asd_ascb_cache);
+	asd_ascb_cache = NULL;
+}
+
+static int asd_register_sas_ha(struct asd_ha_struct *asd_ha)
+{
+	int i;
+	struct asd_sas_phy   **sas_phys =
+		kmalloc(ASD_MAX_PHYS * sizeof(struct asd_sas_phy), GFP_KERNEL);
+	struct asd_sas_port  **sas_ports =
+		kmalloc(ASD_MAX_PHYS * sizeof(struct asd_sas_port), GFP_KERNEL);
+
+	if (!sas_phys || !sas_ports) {
+		kfree(sas_phys);
+		kfree(sas_ports);
+		return -ENOMEM;
+	}
+
+	asd_ha->sas_ha.sas_ha_name = (char *) asd_ha->name;
+	asd_ha->sas_ha.lldd_module = THIS_MODULE;
+	asd_ha->sas_ha.sas_addr = &asd_ha->hw_prof.sas_addr[0];
+
+	for (i = 0; i < ASD_MAX_PHYS; i++) {
+		sas_phys[i] = &asd_ha->phys[i].sas_phy;
+		sas_ports[i] = &asd_ha->ports[i];
+	}
+
+	asd_ha->sas_ha.sas_phy = sas_phys;
+	asd_ha->sas_ha.sas_port= sas_ports;
+	asd_ha->sas_ha.num_phys= ASD_MAX_PHYS;
+
+	asd_ha->sas_ha.lldd_queue_size = asd_ha->seq.can_queue;
+
+	return sas_register_ha(&asd_ha->sas_ha);
+}
+
+static int asd_unregister_sas_ha(struct asd_ha_struct *asd_ha)
+{
+	int err;
+
+	err = sas_unregister_ha(&asd_ha->sas_ha);
+
+	sas_remove_host(asd_ha->sas_ha.core.shost);
+	scsi_remove_host(asd_ha->sas_ha.core.shost);
+	scsi_host_put(asd_ha->sas_ha.core.shost);
+
+	kfree(asd_ha->sas_ha.sas_phy);
+	kfree(asd_ha->sas_ha.sas_port);
+
+	return err;
+}
+
+static int __devinit asd_pci_probe(struct pci_dev *dev,
+				   const struct pci_device_id *id)
+{
+	struct asd_pcidev_struct *asd_dev;
+	unsigned asd_id = (unsigned) id->driver_data;
+	struct asd_ha_struct *asd_ha;
+	struct Scsi_Host *shost;
+	int err;
+
+	if (asd_id >= ARRAY_SIZE(asd_pcidev_data)) {
+		asd_printk("wrong driver_data in PCI table\n");
+		return -ENODEV;
+	}
+
+	if ((err = pci_enable_device(dev))) {
+		asd_printk("couldn't enable device %s\n", pci_name(dev));
+		return err;
+	}
+
+	pci_set_master(dev);
+
+	err = -ENOMEM;
+
+	shost = scsi_host_alloc(&aic94xx_sht, sizeof(void *));
+	if (!shost)
+		goto Err;
+
+	asd_dev = &asd_pcidev_data[asd_id];
+
+	asd_ha = kzalloc(sizeof(*asd_ha), GFP_KERNEL);
+	if (!asd_ha) {
+		asd_printk("out of memory\n");
+		goto Err;
+	}
+	asd_ha->pcidev = dev;
+	asd_ha->sas_ha.pcidev = asd_ha->pcidev;
+	asd_ha->sas_ha.lldd_ha = asd_ha;
+
+	asd_ha->name = asd_dev->name;
+	asd_printk("found %s, device %s\n", asd_ha->name, pci_name(dev));
+
+	SHOST_TO_SAS_HA(shost) = &asd_ha->sas_ha;
+	asd_ha->sas_ha.core.shost = shost;
+	shost->transportt = aic94xx_transport_template;
+	shost->max_id = ~0;
+	shost->max_lun = ~0;
+	shost->max_cmd_len = 16;
+
+	err = scsi_add_host(shost, &dev->dev);
+	if (err) {
+		scsi_host_put(shost);
+		goto Err_free;
+	}
+
+
+
+	err = asd_dev->setup(asd_ha);
+	if (err)
+		goto Err_free;
+
+	err = -ENODEV;
+	if (!pci_set_dma_mask(dev, DMA_64BIT_MASK)
+	    && !pci_set_consistent_dma_mask(dev, DMA_64BIT_MASK))
+		;
+	else if (!pci_set_dma_mask(dev, DMA_32BIT_MASK)
+		 && !pci_set_consistent_dma_mask(dev, DMA_32BIT_MASK))
+		;
+	else {
+		asd_printk("no suitable DMA mask for %s\n", pci_name(dev));
+		goto Err_free;
+	}
+
+	pci_set_drvdata(dev, asd_ha);
+
+	err = asd_map_ha(asd_ha);
+	if (err)
+		goto Err_free;
+
+	err = asd_create_ha_caches(asd_ha);
+        if (err)
+		goto Err_unmap;
+
+	err = asd_init_hw(asd_ha);
+	if (err)
+		goto Err_free_cache;
+
+	asd_printk("device %s: SAS addr %llx, PCBA SN %s, %d phys, %d enabled "
+		   "phys, flash %s, BIOS %s%d\n",
+		   pci_name(dev), SAS_ADDR(asd_ha->hw_prof.sas_addr),
+		   asd_ha->hw_prof.pcba_sn, asd_ha->hw_prof.max_phys,
+		   asd_ha->hw_prof.num_phys,
+		   asd_ha->hw_prof.flash.present ? "present" : "not present",
+		   asd_ha->hw_prof.bios.present ? "build " : "not present",
+		   asd_ha->hw_prof.bios.bld);
+
+	shost->can_queue = asd_ha->seq.can_queue;
+
+	if (use_msi)
+		pci_enable_msi(asd_ha->pcidev);
+
+	err = request_irq(asd_ha->pcidev->irq, asd_hw_isr, SA_SHIRQ,
+			  ASD_DRIVER_NAME, asd_ha);
+	if (err) {
+		asd_printk("couldn't get irq %d for %s\n",
+			   asd_ha->pcidev->irq, pci_name(asd_ha->pcidev));
+		goto Err_irq;
+	}
+	asd_enable_ints(asd_ha);
+
+	err = asd_init_post_escbs(asd_ha);
+	if (err) {
+		asd_printk("couldn't post escbs for %s\n",
+			   pci_name(asd_ha->pcidev));
+		goto Err_escbs;
+	}
+	ASD_DPRINTK("escbs posted\n");
+
+	asd_create_dev_attrs(asd_ha);
+
+	err = asd_register_sas_ha(asd_ha);
+	if (err)
+		goto Err_reg_sas;
+
+	err = asd_enable_phys(asd_ha, asd_ha->hw_prof.enabled_phys);
+	if (err) {
+		asd_printk("coudln't enable phys, err:%d\n", err);
+		goto Err_en_phys;
+	}
+	ASD_DPRINTK("enabled phys\n");
+	/* give the phy enabling interrupt event time to come in (1s
+	 * is empirically about all it takes) */
+	ssleep(1);
+	/* Wait for discovery to finish */
+	scsi_flush_work(asd_ha->sas_ha.core.shost);
+
+	return 0;
+Err_en_phys:
+	asd_unregister_sas_ha(asd_ha);
+Err_reg_sas:
+	asd_remove_dev_attrs(asd_ha);
+Err_escbs:
+	asd_disable_ints(asd_ha);
+	free_irq(dev->irq, asd_ha);
+Err_irq:
+	if (use_msi)
+		pci_disable_msi(dev);
+	asd_chip_hardrst(asd_ha);
+Err_free_cache:
+	asd_destroy_ha_caches(asd_ha);
+Err_unmap:
+	asd_unmap_ha(asd_ha);
+Err_free:
+	kfree(asd_ha);
+	scsi_remove_host(shost);
+Err:
+	pci_disable_device(dev);
+	return err;
+}
+
+static void asd_free_queues(struct asd_ha_struct *asd_ha)
+{
+	unsigned long flags;
+	LIST_HEAD(pending);
+	struct list_head *n, *pos;
+
+	spin_lock_irqsave(&asd_ha->seq.pend_q_lock, flags);
+	asd_ha->seq.pending = 0;
+	list_splice_init(&asd_ha->seq.pend_q, &pending);
+	spin_unlock_irqrestore(&asd_ha->seq.pend_q_lock, flags);
+
+	if (!list_empty(&pending))
+		ASD_DPRINTK("Uh-oh! Pending is not empty!\n");
+
+	list_for_each_safe(pos, n, &pending) {
+		struct asd_ascb *ascb = list_entry(pos, struct asd_ascb, list);
+		list_del_init(pos);
+		ASD_DPRINTK("freeing from pending\n");
+		asd_ascb_free(ascb);
+	}
+}
+
+static void asd_turn_off_leds(struct asd_ha_struct *asd_ha)
+{
+	u8 phy_mask = asd_ha->hw_prof.enabled_phys;
+	u8 i;
+
+	for_each_phy(phy_mask, phy_mask, i) {
+		asd_turn_led(asd_ha, i, 0);
+		asd_control_led(asd_ha, i, 0);
+	}
+}
+
+static void __devexit asd_pci_remove(struct pci_dev *dev)
+{
+	struct asd_ha_struct *asd_ha = pci_get_drvdata(dev);
+
+	if (!asd_ha)
+		return;
+
+	asd_unregister_sas_ha(asd_ha);
+
+	asd_disable_ints(asd_ha);
+
+	asd_remove_dev_attrs(asd_ha);
+
+	/* XXX more here as needed */
+
+	free_irq(dev->irq, asd_ha);
+	if (use_msi)
+		pci_disable_msi(asd_ha->pcidev);
+	asd_turn_off_leds(asd_ha);
+	asd_chip_hardrst(asd_ha);
+	asd_free_queues(asd_ha);
+	asd_destroy_ha_caches(asd_ha);
+	asd_unmap_ha(asd_ha);
+	kfree(asd_ha);
+	pci_disable_device(dev);
+	return;
+}
+
+static ssize_t asd_version_show(struct device_driver *driver, char *buf)
+{
+	return snprintf(buf, PAGE_SIZE, "%s\n", ASD_DRIVER_VERSION);
+}
+static DRIVER_ATTR(version, S_IRUGO, asd_version_show, NULL);
+
+static void asd_create_driver_attrs(struct device_driver *driver)
+{
+	driver_create_file(driver, &driver_attr_version);
+}
+
+static void asd_remove_driver_attrs(struct device_driver *driver)
+{
+	driver_remove_file(driver, &driver_attr_version);
+}
+
+static struct sas_domain_function_template aic94xx_transport_functions = {
+	.lldd_port_formed	= asd_update_port_links,
+
+	.lldd_dev_found		= asd_dev_found,
+	.lldd_dev_gone		= asd_dev_gone,
+
+	.lldd_execute_task	= asd_execute_task,
+
+	.lldd_abort_task	= asd_abort_task,
+	.lldd_abort_task_set	= asd_abort_task_set,
+	.lldd_clear_aca		= asd_clear_aca,
+	.lldd_clear_task_set	= asd_clear_task_set,
+	.lldd_I_T_nexus_reset	= NULL,
+	.lldd_lu_reset		= asd_lu_reset,
+	.lldd_query_task	= asd_query_task,
+
+	.lldd_clear_nexus_port	= asd_clear_nexus_port,
+	.lldd_clear_nexus_ha	= asd_clear_nexus_ha,
+
+	.lldd_control_phy	= asd_control_phy,
+};
+
+static const struct pci_device_id aic94xx_pci_table[] __devinitdata = {
+	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_RAZOR10),
+	 0, 0, 1},
+	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_RAZOR12),
+	 0, 0, 1},
+	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_RAZOR1E),
+	 0, 0, 1},
+	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_RAZOR30),
+	 0, 0, 2},
+	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_RAZOR32),
+	 0, 0, 2},
+	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_RAZOR3E),
+	 0, 0, 2},
+	{PCI_DEVICE(PCI_VENDOR_ID_ADAPTEC2, PCI_DEVICE_ID_ADAPTEC2_RAZOR3F),
+	 0, 0, 2},
+	{}
+};
+
+MODULE_DEVICE_TABLE(pci, aic94xx_pci_table);
+
+static struct pci_driver aic94xx_pci_driver = {
+	.name		= ASD_DRIVER_NAME,
+	.id_table	= aic94xx_pci_table,
+	.probe		= asd_pci_probe,
+	.remove		= __devexit_p(asd_pci_remove),
+};
+
+static int __init aic94xx_init(void)
+{
+	int err;
+
+
+	asd_printk("%s version %s loaded\n", ASD_DRIVER_DESCRIPTION,
+		   ASD_DRIVER_VERSION);
+
+	err = asd_create_global_caches();
+	if (err)
+		return err;
+
+	aic94xx_transport_template =
+		sas_domain_attach_transport(&aic94xx_transport_functions);
+	if (!aic94xx_transport_template)
+		goto out_destroy_caches;
+
+	err = pci_register_driver(&aic94xx_pci_driver);
+	if (err)
+		goto out_release_transport;
+
+	asd_create_driver_attrs(&aic94xx_pci_driver.driver);
+
+	return err;
+
+ out_release_transport:
+	sas_release_transport(aic94xx_transport_template);
+ out_destroy_caches:
+	asd_destroy_global_caches();
+
+	return err;
+}
+
+static void __exit aic94xx_exit(void)
+{
+	asd_remove_driver_attrs(&aic94xx_pci_driver.driver);
+	pci_unregister_driver(&aic94xx_pci_driver);
+	sas_release_transport(aic94xx_transport_template);
+	asd_destroy_global_caches();
+	asd_printk("%s version %s unloaded\n", ASD_DRIVER_DESCRIPTION,
+		   ASD_DRIVER_VERSION);
+}
+
+module_init(aic94xx_init);
+module_exit(aic94xx_exit);
+
+MODULE_AUTHOR("Luben Tuikov <luben_tuikov@adaptec.com>");
+MODULE_DESCRIPTION(ASD_DRIVER_DESCRIPTION);
+MODULE_LICENSE("GPL v2");
+MODULE_VERSION(ASD_DRIVER_VERSION);
diff --git a/drivers/scsi/aic94xx/aic94xx_reg.c b/drivers/scsi/aic94xx/aic94xx_reg.c
new file mode 100644
index 0000000..f210dac
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_reg.c
@@ -0,0 +1,332 @@
+/*
+ * Aic94xx SAS/SATA driver register access.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/pci.h>
+#include "aic94xx_reg.h"
+#include "aic94xx.h"
+
+/* Writing to device address space.
+ * Offset comes before value to remind that the operation of
+ * this function is *offs = val.
+ */
+static inline void asd_write_byte(struct asd_ha_struct *asd_ha,
+				  unsigned long offs, u8 val)
+{
+	if (unlikely(asd_ha->iospace))
+		outb(val,
+		     (unsigned long)asd_ha->io_handle[0].addr + (offs & 0xFF));
+	else
+		writeb(val, asd_ha->io_handle[0].addr + offs);
+	wmb();
+}
+
+static inline void asd_write_word(struct asd_ha_struct *asd_ha,
+				  unsigned long offs, u16 val)
+{
+	if (unlikely(asd_ha->iospace))
+		outw(val,
+		     (unsigned long)asd_ha->io_handle[0].addr + (offs & 0xFF));
+	else
+		writew(val, asd_ha->io_handle[0].addr + offs);
+	wmb();
+}
+
+static inline void asd_write_dword(struct asd_ha_struct *asd_ha,
+				   unsigned long offs, u32 val)
+{
+	if (unlikely(asd_ha->iospace))
+		outl(val,
+		     (unsigned long)asd_ha->io_handle[0].addr + (offs & 0xFF));
+	else
+		writel(val, asd_ha->io_handle[0].addr + offs);
+	wmb();
+}
+
+/* Reading from device address space.
+ */
+static inline u8 asd_read_byte(struct asd_ha_struct *asd_ha,
+			       unsigned long offs)
+{
+	u8 val;
+	if (unlikely(asd_ha->iospace))
+		val = inb((unsigned long) asd_ha->io_handle[0].addr
+			  + (offs & 0xFF));
+	else
+		val = readb(asd_ha->io_handle[0].addr + offs);
+	rmb();
+	return val;
+}
+
+static inline u16 asd_read_word(struct asd_ha_struct *asd_ha,
+				unsigned long offs)
+{
+	u16 val;
+	if (unlikely(asd_ha->iospace))
+		val = inw((unsigned long)asd_ha->io_handle[0].addr
+			  + (offs & 0xFF));
+	else
+		val = readw(asd_ha->io_handle[0].addr + offs);
+	rmb();
+	return val;
+}
+
+static inline u32 asd_read_dword(struct asd_ha_struct *asd_ha,
+				 unsigned long offs)
+{
+	u32 val;
+	if (unlikely(asd_ha->iospace))
+		val = inl((unsigned long) asd_ha->io_handle[0].addr
+			  + (offs & 0xFF));
+	else
+		val = readl(asd_ha->io_handle[0].addr + offs);
+	rmb();
+	return val;
+}
+
+static inline u32 asd_mem_offs_swa(void)
+{
+	return 0;
+}
+
+static inline u32 asd_mem_offs_swc(void)
+{
+	return asd_mem_offs_swa() + MBAR0_SWA_SIZE;
+}
+
+static inline u32 asd_mem_offs_swb(void)
+{
+	return asd_mem_offs_swc() + MBAR0_SWC_SIZE + 0x20;
+}
+
+/* We know that the register wanted is in the range
+ * of the sliding window.
+ */
+#define ASD_READ_SW(ww, type, ord)                                     \
+static inline type asd_read_##ww##_##ord (struct asd_ha_struct *asd_ha,\
+					  u32 reg)                     \
+{                                                                      \
+	struct asd_ha_addrspace *io_handle = &asd_ha->io_handle[0];    \
+	u32 map_offs=(reg - io_handle-> ww##_base )+asd_mem_offs_##ww ();\
+	return asd_read_##ord (asd_ha, (unsigned long) map_offs);      \
+}
+
+#define ASD_WRITE_SW(ww, type, ord)                                    \
+static inline void asd_write_##ww##_##ord (struct asd_ha_struct *asd_ha,\
+				  u32 reg, type val)                   \
+{                                                                      \
+	struct asd_ha_addrspace *io_handle = &asd_ha->io_handle[0];    \
+	u32 map_offs=(reg - io_handle-> ww##_base )+asd_mem_offs_##ww ();\
+	asd_write_##ord (asd_ha, (unsigned long) map_offs, val);       \
+}
+
+ASD_READ_SW(swa, u8,  byte);
+ASD_READ_SW(swa, u16, word);
+ASD_READ_SW(swa, u32, dword);
+
+ASD_READ_SW(swb, u8,  byte);
+ASD_READ_SW(swb, u16, word);
+ASD_READ_SW(swb, u32, dword);
+
+ASD_READ_SW(swc, u8,  byte);
+ASD_READ_SW(swc, u16, word);
+ASD_READ_SW(swc, u32, dword);
+
+ASD_WRITE_SW(swa, u8,  byte);
+ASD_WRITE_SW(swa, u16, word);
+ASD_WRITE_SW(swa, u32, dword);
+
+ASD_WRITE_SW(swb, u8,  byte);
+ASD_WRITE_SW(swb, u16, word);
+ASD_WRITE_SW(swb, u32, dword);
+
+ASD_WRITE_SW(swc, u8,  byte);
+ASD_WRITE_SW(swc, u16, word);
+ASD_WRITE_SW(swc, u32, dword);
+
+/*
+ * A word about sliding windows:
+ * MBAR0 is divided into sliding windows A, C and B, in that order.
+ * SWA starts at offset 0 of MBAR0, up to 0x57, with size 0x58 bytes.
+ * SWC starts at offset 0x58 of MBAR0, up to 0x60, with size 0x8 bytes.
+ * From 0x60 to 0x7F, we have a copy of PCI config space 0x60-0x7F.
+ * SWB starts at offset 0x80 of MBAR0 and extends to the end of MBAR0.
+ * See asd_init_sw() in aic94xx_hwi.c
+ *
+ * We map the most common registers we'd access of the internal 4GB
+ * host adapter memory space.  If a register/internal memory location
+ * is wanted which is not mapped, we slide SWB, by paging it,
+ * see asd_move_swb() in aic94xx_reg.c.
+ */
+
+/**
+ * asd_move_swb -- move sliding window B
+ * @asd_ha: pointer to host adapter structure
+ * @reg: register desired to be within range of the new window
+ */
+static inline void asd_move_swb(struct asd_ha_struct *asd_ha, u32 reg)
+{
+	u32 base = reg & ~(MBAR0_SWB_SIZE-1);
+	pci_write_config_dword(asd_ha->pcidev, PCI_CONF_MBAR0_SWB, base);
+	asd_ha->io_handle[0].swb_base = base;
+}
+
+static void __asd_write_reg_byte(struct asd_ha_struct *asd_ha, u32 reg, u8 val)
+{
+	struct asd_ha_addrspace *io_handle=&asd_ha->io_handle[0];
+	BUG_ON(reg >= 0xC0000000 || reg < ALL_BASE_ADDR);
+	if (io_handle->swa_base <= reg
+	    && reg < io_handle->swa_base + MBAR0_SWA_SIZE)
+		asd_write_swa_byte (asd_ha, reg,val);
+	else if (io_handle->swb_base <= reg
+		 && reg < io_handle->swb_base + MBAR0_SWB_SIZE)
+		asd_write_swb_byte (asd_ha, reg, val);
+	else if (io_handle->swc_base <= reg
+		 && reg < io_handle->swc_base + MBAR0_SWC_SIZE)
+		asd_write_swc_byte (asd_ha, reg, val);
+	else {
+		/* Ok, we have to move SWB */
+		asd_move_swb(asd_ha, reg);
+		asd_write_swb_byte (asd_ha, reg, val);
+	}
+}
+
+#define ASD_WRITE_REG(type, ord)                                  \
+void asd_write_reg_##ord (struct asd_ha_struct *asd_ha, u32 reg, type val)\
+{                                                                 \
+	struct asd_ha_addrspace *io_handle=&asd_ha->io_handle[0]; \
+	unsigned long flags;                                      \
+	BUG_ON(reg >= 0xC0000000 || reg < ALL_BASE_ADDR);         \
+	spin_lock_irqsave(&asd_ha->iolock, flags);                \
+	if (io_handle->swa_base <= reg                            \
+	    && reg < io_handle->swa_base + MBAR0_SWA_SIZE)        \
+		asd_write_swa_##ord (asd_ha, reg,val);            \
+	else if (io_handle->swb_base <= reg                       \
+		 && reg < io_handle->swb_base + MBAR0_SWB_SIZE)   \
+		asd_write_swb_##ord (asd_ha, reg, val);           \
+	else if (io_handle->swc_base <= reg                       \
+		 && reg < io_handle->swc_base + MBAR0_SWC_SIZE)   \
+		asd_write_swc_##ord (asd_ha, reg, val);           \
+	else {                                                    \
+		/* Ok, we have to move SWB */                     \
+		asd_move_swb(asd_ha, reg);                        \
+		asd_write_swb_##ord (asd_ha, reg, val);           \
+	}                                                         \
+	spin_unlock_irqrestore(&asd_ha->iolock, flags);           \
+}
+
+ASD_WRITE_REG(u8, byte);
+ASD_WRITE_REG(u16,word);
+ASD_WRITE_REG(u32,dword);
+
+static u8 __asd_read_reg_byte(struct asd_ha_struct *asd_ha, u32 reg)
+{
+	struct asd_ha_addrspace *io_handle=&asd_ha->io_handle[0];
+	u8 val;
+	BUG_ON(reg >= 0xC0000000 || reg < ALL_BASE_ADDR);
+	if (io_handle->swa_base <= reg
+	    && reg < io_handle->swa_base + MBAR0_SWA_SIZE)
+		val = asd_read_swa_byte (asd_ha, reg);
+	else if (io_handle->swb_base <= reg
+		 && reg < io_handle->swb_base + MBAR0_SWB_SIZE)
+		val = asd_read_swb_byte (asd_ha, reg);
+	else if (io_handle->swc_base <= reg
+		 && reg < io_handle->swc_base + MBAR0_SWC_SIZE)
+		val = asd_read_swc_byte (asd_ha, reg);
+	else {
+		/* Ok, we have to move SWB */
+		asd_move_swb(asd_ha, reg);
+		val = asd_read_swb_byte (asd_ha, reg);
+	}
+	return val;
+}
+
+#define ASD_READ_REG(type, ord)                                   \
+type asd_read_reg_##ord (struct asd_ha_struct *asd_ha, u32 reg)   \
+{                                                                 \
+	struct asd_ha_addrspace *io_handle=&asd_ha->io_handle[0]; \
+	type val;                                                 \
+	unsigned long flags;                                      \
+	BUG_ON(reg >= 0xC0000000 || reg < ALL_BASE_ADDR);         \
+	spin_lock_irqsave(&asd_ha->iolock, flags);                \
+	if (io_handle->swa_base <= reg                            \
+	    && reg < io_handle->swa_base + MBAR0_SWA_SIZE)        \
+		val = asd_read_swa_##ord (asd_ha, reg);           \
+	else if (io_handle->swb_base <= reg                       \
+		 && reg < io_handle->swb_base + MBAR0_SWB_SIZE)   \
+		val = asd_read_swb_##ord (asd_ha, reg);           \
+	else if (io_handle->swc_base <= reg                       \
+		 && reg < io_handle->swc_base + MBAR0_SWC_SIZE)   \
+		val = asd_read_swc_##ord (asd_ha, reg);           \
+	else {                                                    \
+		/* Ok, we have to move SWB */                     \
+		asd_move_swb(asd_ha, reg);                        \
+		val = asd_read_swb_##ord (asd_ha, reg);           \
+	}                                                         \
+	spin_unlock_irqrestore(&asd_ha->iolock, flags);           \
+	return val;                                               \
+}
+
+ASD_READ_REG(u8, byte);
+ASD_READ_REG(u16,word);
+ASD_READ_REG(u32,dword);
+
+/**
+ * asd_read_reg_string -- read a string of bytes from io space memory
+ * @asd_ha: pointer to host adapter structure
+ * @dst: pointer to a destination buffer where data will be written to
+ * @offs: start offset (register) to read from
+ * @count: number of bytes to read
+ */
+void asd_read_reg_string(struct asd_ha_struct *asd_ha, void *dst,
+			 u32 offs, int count)
+{
+	u8 *p = dst;
+	unsigned long flags;
+
+	spin_lock_irqsave(&asd_ha->iolock, flags);
+	for ( ; count > 0; count--, offs++, p++)
+		*p = __asd_read_reg_byte(asd_ha, offs);
+	spin_unlock_irqrestore(&asd_ha->iolock, flags);
+}
+
+/**
+ * asd_write_reg_string -- write a string of bytes to io space memory
+ * @asd_ha: pointer to host adapter structure
+ * @src: pointer to source buffer where data will be read from
+ * @offs: start offset (register) to write to
+ * @count: number of bytes to write
+ */
+void asd_write_reg_string(struct asd_ha_struct *asd_ha, void *src,
+			  u32 offs, int count)
+{
+	u8 *p = src;
+	unsigned long flags;
+
+	spin_lock_irqsave(&asd_ha->iolock, flags);
+	for ( ; count > 0; count--, offs++, p++)
+		__asd_write_reg_byte(asd_ha, offs, *p);
+	spin_unlock_irqrestore(&asd_ha->iolock, flags);
+}
diff --git a/drivers/scsi/aic94xx/aic94xx_reg.h b/drivers/scsi/aic94xx/aic94xx_reg.h
new file mode 100644
index 0000000..2279307
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_reg.h
@@ -0,0 +1,302 @@
+/*
+ * Aic94xx SAS/SATA driver hardware registers definitions.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#ifndef _AIC94XX_REG_H_
+#define _AIC94XX_REG_H_
+
+#include <asm/io.h>
+#include "aic94xx_hwi.h"
+
+/* Values */
+#define AIC9410_DEV_REV_B0            0x8
+
+/* MBAR0, SWA, SWB, SWC, internal memory space addresses */
+#define REG_BASE_ADDR                 0xB8000000
+#define REG_BASE_ADDR_CSEQCIO         0xB8002000
+#define REG_BASE_ADDR_EXSI            0xB8042800
+
+#define MBAR0_SWA_SIZE                0x58
+extern  u32    MBAR0_SWB_SIZE;
+#define MBAR0_SWC_SIZE                0x8
+
+/* MBAR1, points to On Chip Memory */
+#define OCM_BASE_ADDR                 0xA0000000
+#define OCM_MAX_SIZE                  0x20000
+
+/* Smallest address possible to reference */
+#define ALL_BASE_ADDR                 OCM_BASE_ADDR
+
+/* PCI configuration space registers */
+#define PCI_IOBAR_OFFSET              4
+
+#define PCI_CONF_MBAR1                0x6C
+#define PCI_CONF_MBAR0_SWA            0x70
+#define PCI_CONF_MBAR0_SWB            0x74
+#define PCI_CONF_MBAR0_SWC            0x78
+#define PCI_CONF_MBAR_KEY             0x7C
+#define PCI_CONF_FLSH_BAR             0xB8
+
+#include "aic94xx_reg_def.h"
+
+u8  asd_read_reg_byte(struct asd_ha_struct *asd_ha, u32 reg);
+u16 asd_read_reg_word(struct asd_ha_struct *asd_ha, u32 reg);
+u32 asd_read_reg_dword(struct asd_ha_struct *asd_ha, u32 reg);
+
+void asd_write_reg_byte(struct asd_ha_struct *asd_ha, u32 reg, u8 val);
+void asd_write_reg_word(struct asd_ha_struct *asd_ha, u32 reg, u16 val);
+void asd_write_reg_dword(struct asd_ha_struct *asd_ha, u32 reg, u32 val);
+
+void asd_read_reg_string(struct asd_ha_struct *asd_ha, void *dst,
+			 u32 offs, int count);
+void asd_write_reg_string(struct asd_ha_struct *asd_ha, void *src,
+			  u32 offs, int count);
+
+#define ASD_READ_OCM(type, ord, S)                                    \
+static inline type asd_read_ocm_##ord (struct asd_ha_struct *asd_ha,  \
+					 u32 offs)                    \
+{                                                                     \
+	struct asd_ha_addrspace *io_handle = &asd_ha->io_handle[1];   \
+	type val = read##S (io_handle->addr + (unsigned long) offs);  \
+	rmb();                                                        \
+	return val;                                                   \
+}
+
+ASD_READ_OCM(u8, byte, b);
+ASD_READ_OCM(u16,word, w);
+ASD_READ_OCM(u32,dword,l);
+
+#define ASD_WRITE_OCM(type, ord, S)                                    \
+static inline void asd_write_ocm_##ord (struct asd_ha_struct *asd_ha,  \
+					 u32 offs, type val)          \
+{                                                                     \
+	struct asd_ha_addrspace *io_handle = &asd_ha->io_handle[1];   \
+	write##S (val, io_handle->addr + (unsigned long) offs);       \
+	return;                                                       \
+}
+
+ASD_WRITE_OCM(u8, byte, b);
+ASD_WRITE_OCM(u16,word, w);
+ASD_WRITE_OCM(u32,dword,l);
+
+#define ASD_DDBSITE_READ(type, ord)                                        \
+static inline type asd_ddbsite_read_##ord (struct asd_ha_struct *asd_ha,   \
+					   u16 ddb_site_no,                \
+					   u16 offs)                       \
+{                                                                          \
+	asd_write_reg_word(asd_ha, ALTCIOADR, MnDDB_SITE + offs);          \
+	asd_write_reg_word(asd_ha, ADDBPTR, ddb_site_no);                  \
+	return asd_read_reg_##ord (asd_ha, CTXACCESS);                     \
+}
+
+ASD_DDBSITE_READ(u32, dword);
+ASD_DDBSITE_READ(u16, word);
+
+static inline u8 asd_ddbsite_read_byte(struct asd_ha_struct *asd_ha,
+				       u16 ddb_site_no,
+				       u16 offs)
+{
+	if (offs & 1)
+		return asd_ddbsite_read_word(asd_ha, ddb_site_no,
+					     offs & ~1) >> 8;
+	else
+		return asd_ddbsite_read_word(asd_ha, ddb_site_no,
+					     offs) & 0xFF;
+}
+
+
+#define ASD_DDBSITE_WRITE(type, ord)                                       \
+static inline void asd_ddbsite_write_##ord (struct asd_ha_struct *asd_ha,  \
+					u16 ddb_site_no,                   \
+					u16 offs, type val)                \
+{                                                                          \
+	asd_write_reg_word(asd_ha, ALTCIOADR, MnDDB_SITE + offs);          \
+	asd_write_reg_word(asd_ha, ADDBPTR, ddb_site_no);                  \
+	asd_write_reg_##ord (asd_ha, CTXACCESS, val);                      \
+}
+
+ASD_DDBSITE_WRITE(u32, dword);
+ASD_DDBSITE_WRITE(u16, word);
+
+static inline void asd_ddbsite_write_byte(struct asd_ha_struct *asd_ha,
+					  u16 ddb_site_no,
+					  u16 offs, u8 val)
+{
+	u16 base = offs & ~1;
+	u16 rval = asd_ddbsite_read_word(asd_ha, ddb_site_no, base);
+	if (offs & 1)
+		rval = (val << 8) | (rval & 0xFF);
+	else
+		rval = (rval & 0xFF00) | val;
+	asd_ddbsite_write_word(asd_ha, ddb_site_no, base, rval);
+}
+
+
+#define ASD_SCBSITE_READ(type, ord)                                        \
+static inline type asd_scbsite_read_##ord (struct asd_ha_struct *asd_ha,   \
+					   u16 scb_site_no,                \
+					   u16 offs)                       \
+{                                                                          \
+	asd_write_reg_word(asd_ha, ALTCIOADR, MnSCB_SITE + offs);          \
+	asd_write_reg_word(asd_ha, ASCBPTR, scb_site_no);                  \
+	return asd_read_reg_##ord (asd_ha, CTXACCESS);                     \
+}
+
+ASD_SCBSITE_READ(u32, dword);
+ASD_SCBSITE_READ(u16, word);
+
+static inline u8 asd_scbsite_read_byte(struct asd_ha_struct *asd_ha,
+				       u16 scb_site_no,
+				       u16 offs)
+{
+	if (offs & 1)
+		return asd_scbsite_read_word(asd_ha, scb_site_no,
+					     offs & ~1) >> 8;
+	else
+		return asd_scbsite_read_word(asd_ha, scb_site_no,
+					     offs) & 0xFF;
+}
+
+
+#define ASD_SCBSITE_WRITE(type, ord)                                       \
+static inline void asd_scbsite_write_##ord (struct asd_ha_struct *asd_ha,  \
+					u16 scb_site_no,                   \
+					u16 offs, type val)                \
+{                                                                          \
+	asd_write_reg_word(asd_ha, ALTCIOADR, MnSCB_SITE + offs);          \
+	asd_write_reg_word(asd_ha, ASCBPTR, scb_site_no);                  \
+	asd_write_reg_##ord (asd_ha, CTXACCESS, val);                      \
+}
+
+ASD_SCBSITE_WRITE(u32, dword);
+ASD_SCBSITE_WRITE(u16, word);
+
+static inline void asd_scbsite_write_byte(struct asd_ha_struct *asd_ha,
+					  u16 scb_site_no,
+					  u16 offs, u8 val)
+{
+	u16 base = offs & ~1;
+	u16 rval = asd_scbsite_read_word(asd_ha, scb_site_no, base);
+	if (offs & 1)
+		rval = (val << 8) | (rval & 0xFF);
+	else
+		rval = (rval & 0xFF00) | val;
+	asd_scbsite_write_word(asd_ha, scb_site_no, base, rval);
+}
+
+/**
+ * asd_ddbsite_update_word -- atomically update a word in a ddb site
+ * @asd_ha: pointer to host adapter structure
+ * @ddb_site_no: the DDB site number
+ * @offs: the offset into the DDB
+ * @oldval: old value found in that offset
+ * @newval: the new value to replace it
+ *
+ * This function is used when the sequencers are running and we need to
+ * update a DDB site atomically without expensive pausing and upausing
+ * of the sequencers and accessing the DDB site through the CIO bus.
+ *
+ * Return 0 on success; -EFAULT on parity error; -EAGAIN if the old value
+ * is different than the current value at that offset.
+ */
+static inline int asd_ddbsite_update_word(struct asd_ha_struct *asd_ha,
+					  u16 ddb_site_no, u16 offs,
+					  u16 oldval, u16 newval)
+{
+	u8  done;
+	u16 oval = asd_ddbsite_read_word(asd_ha, ddb_site_no, offs);
+	if (oval != oldval)
+		return -EAGAIN;
+	asd_write_reg_word(asd_ha, AOLDDATA, oldval);
+	asd_write_reg_word(asd_ha, ANEWDATA, newval);
+	do {
+		done = asd_read_reg_byte(asd_ha, ATOMICSTATCTL);
+	} while (!(done & ATOMICDONE));
+	if (done & ATOMICERR)
+		return -EFAULT;	  /* parity error */
+	else if (done & ATOMICWIN)
+		return 0;	  /* success */
+	else
+		return -EAGAIN;	  /* oldval different than current value */
+}
+
+static inline int asd_ddbsite_update_byte(struct asd_ha_struct *asd_ha,
+					  u16 ddb_site_no, u16 offs,
+					  u8 _oldval, u8 _newval)
+{
+	u16 base = offs & ~1;
+	u16 oval;
+	u16 nval = asd_ddbsite_read_word(asd_ha, ddb_site_no, base);
+	if (offs & 1) {
+		if ((nval >> 8) != _oldval)
+			return -EAGAIN;
+		nval = (_newval << 8) | (nval & 0xFF);
+		oval = (_oldval << 8) | (nval & 0xFF);
+	} else {
+		if ((nval & 0xFF) != _oldval)
+			return -EAGAIN;
+		nval = (nval & 0xFF00) | _newval;
+		oval = (nval & 0xFF00) | _oldval;
+	}
+	return asd_ddbsite_update_word(asd_ha, ddb_site_no, base, oval, nval);
+}
+
+static inline void asd_write_reg_addr(struct asd_ha_struct *asd_ha, u32 reg,
+				      dma_addr_t dma_handle)
+{
+	asd_write_reg_dword(asd_ha, reg,   ASD_BUSADDR_LO(dma_handle));
+	asd_write_reg_dword(asd_ha, reg+4, ASD_BUSADDR_HI(dma_handle));
+}
+
+static inline u32 asd_get_cmdctx_size(struct asd_ha_struct *asd_ha)
+{
+	/* DCHREVISION returns 0, possibly broken */
+	u32 ctxmemsize = asd_read_reg_dword(asd_ha, LmMnINT(0,0)) & CTXMEMSIZE;
+	return ctxmemsize ? 65536 : 32768;
+}
+
+static inline u32 asd_get_devctx_size(struct asd_ha_struct *asd_ha)
+{
+	u32 ctxmemsize = asd_read_reg_dword(asd_ha, LmMnINT(0,0)) & CTXMEMSIZE;
+	return ctxmemsize ? 8192 : 4096;
+}
+
+static inline void asd_disable_ints(struct asd_ha_struct *asd_ha)
+{
+	asd_write_reg_dword(asd_ha, CHIMINTEN, RST_CHIMINTEN);
+}
+
+static inline void asd_enable_ints(struct asd_ha_struct *asd_ha)
+{
+	/* Enable COM SAS interrupt on errors, COMSTAT */
+	asd_write_reg_dword(asd_ha, COMSTATEN,
+			    EN_CSBUFPERR | EN_CSERR | EN_OVLYERR);
+	/* Enable DCH SAS CFIFTOERR */
+	asd_write_reg_dword(asd_ha, DCHSTATUS, EN_CFIFTOERR);
+	/* Enable Host Device interrupts */
+	asd_write_reg_dword(asd_ha, CHIMINTEN, SET_CHIMINTEN);
+}
+
+#endif
diff --git a/drivers/scsi/aic94xx/aic94xx_reg_def.h b/drivers/scsi/aic94xx/aic94xx_reg_def.h
new file mode 100644
index 0000000..b79f45f
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_reg_def.h
@@ -0,0 +1,2398 @@
+/*
+ * Aic94xx SAS/SATA driver hardware registers defintions.
+ *
+ * Copyright (C) 2004 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2004 David Chaw <david_chaw@adaptec.com>
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * Luben Tuikov: Some register value updates to make it work with the window
+ * agnostic register r/w functions.  Some register corrections, sizes,
+ * etc.
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ * $Id: //depot/aic94xx/aic94xx_reg_def.h#27 $
+ *
+ */
+
+#ifndef _ADP94XX_REG_DEF_H_
+#define _ADP94XX_REG_DEF_H_
+
+/*
+ * Common definitions.
+ */
+#define CSEQ_MODE_PAGE_SIZE	0x200		/* CSEQ mode page size */
+#define LmSEQ_MODE_PAGE_SIZE	0x200		/* LmSEQ mode page size */
+#define LmSEQ_HOST_REG_SIZE   	0x4000		/* LmSEQ Host Register size */
+
+/********************* COM_SAS registers definition *************************/
+
+/* The base is REG_BASE_ADDR, defined in aic94xx_reg.h.
+ */
+
+/*
+ * CHIM Registers, Address Range : (0x00-0xFF)
+ */
+#define COMBIST		(REG_BASE_ADDR + 0x00)
+
+/* bits 31:24 */
+#define		L7BLKRST		0x80000000
+#define		L6BLKRST		0x40000000
+#define		L5BLKRST		0x20000000
+#define		L4BLKRST		0x10000000
+#define		L3BLKRST		0x08000000
+#define		L2BLKRST		0x04000000
+#define		L1BLKRST		0x02000000
+#define		L0BLKRST		0x01000000
+#define		LmBLKRST		0xFF000000
+#define LmBLKRST_COMBIST(phyid)		(1 << (24 + phyid))
+
+#define		OCMBLKRST		0x00400000
+#define		CTXMEMBLKRST		0x00200000
+#define		CSEQBLKRST		0x00100000
+#define		EXSIBLKRST		0x00040000
+#define		DPIBLKRST		0x00020000
+#define		DFIFBLKRST		0x00010000
+#define		HARDRST			0x00000200
+#define		COMBLKRST		0x00000100
+#define		FRCDFPERR		0x00000080
+#define		FRCCIOPERR		0x00000020
+#define		FRCBISTERR		0x00000010
+#define		COMBISTEN		0x00000004
+#define		COMBISTDONE		0x00000002	/* ro */
+#define 	COMBISTFAIL		0x00000001	/* ro */
+
+#define COMSTAT		(REG_BASE_ADDR + 0x04)
+
+#define		REQMBXREAD		0x00000040
+#define 	RSPMBXAVAIL		0x00000020
+#define 	CSBUFPERR		0x00000008
+#define		OVLYERR			0x00000004
+#define 	CSERR			0x00000002
+#define		OVLYDMADONE		0x00000001
+
+#define		COMSTAT_MASK		(REQMBXREAD | RSPMBXAVAIL | \
+					 CSBUFPERR | OVLYERR | CSERR |\
+					 OVLYDMADONE)
+
+#define COMSTATEN	(REG_BASE_ADDR + 0x08)
+
+#define		EN_REQMBXREAD		0x00000040
+#define		EN_RSPMBXAVAIL		0x00000020
+#define		EN_CSBUFPERR		0x00000008
+#define		EN_OVLYERR		0x00000004
+#define		EN_CSERR		0x00000002
+#define		EN_OVLYDONE		0x00000001
+
+#define SCBPRO		(REG_BASE_ADDR + 0x0C)
+
+#define		SCBCONS_MASK		0xFFFF0000
+#define		SCBPRO_MASK		0x0000FFFF
+
+#define CHIMREQMBX	(REG_BASE_ADDR + 0x10)
+
+#define CHIMRSPMBX	(REG_BASE_ADDR + 0x14)
+
+#define CHIMINT		(REG_BASE_ADDR + 0x18)
+
+#define		EXT_INT0		0x00000800
+#define		EXT_INT1		0x00000400
+#define		PORRSTDET		0x00000200
+#define		HARDRSTDET		0x00000100
+#define		DLAVAILQ		0x00000080	/* ro */
+#define		HOSTERR			0x00000040
+#define		INITERR			0x00000020
+#define		DEVINT			0x00000010
+#define		COMINT			0x00000008
+#define		DEVTIMER2		0x00000004
+#define		DEVTIMER1		0x00000002
+#define		DLAVAIL			0x00000001
+
+#define		CHIMINT_MASK		(HOSTERR | INITERR | DEVINT | COMINT |\
+					 DEVTIMER2 | DEVTIMER1 | DLAVAIL)
+
+#define 	DEVEXCEPT_MASK		(HOSTERR | INITERR | DEVINT | COMINT)
+
+#define CHIMINTEN	(REG_BASE_ADDR + 0x1C)
+
+#define		RST_EN_EXT_INT1		0x01000000
+#define		RST_EN_EXT_INT0		0x00800000
+#define		RST_EN_HOSTERR		0x00400000
+#define		RST_EN_INITERR		0x00200000
+#define		RST_EN_DEVINT		0x00100000
+#define		RST_EN_COMINT		0x00080000
+#define		RST_EN_DEVTIMER2	0x00040000
+#define		RST_EN_DEVTIMER1	0x00020000
+#define		RST_EN_DLAVAIL		0x00010000
+#define		SET_EN_EXT_INT1		0x00000100
+#define		SET_EN_EXT_INT0		0x00000080
+#define		SET_EN_HOSTERR		0x00000040
+#define		SET_EN_INITERR		0x00000020
+#define		SET_EN_DEVINT		0x00000010
+#define		SET_EN_COMINT		0x00000008
+#define		SET_EN_DEVTIMER2	0x00000004
+#define		SET_EN_DEVTIMER1	0x00000002
+#define		SET_EN_DLAVAIL		0x00000001
+
+#define		RST_CHIMINTEN		(RST_EN_HOSTERR | RST_EN_INITERR | \
+					 RST_EN_DEVINT | RST_EN_COMINT | \
+					 RST_EN_DEVTIMER2 | RST_EN_DEVTIMER1 |\
+					 RST_EN_DLAVAIL)
+
+#define		SET_CHIMINTEN		(SET_EN_HOSTERR | SET_EN_INITERR |\
+					 SET_EN_DEVINT | SET_EN_COMINT |\
+					 SET_EN_DLAVAIL)
+
+#define OVLYDMACTL	(REG_BASE_ADDR + 0x20)
+
+#define		OVLYADR_MASK		0x07FF0000
+#define		OVLYLSEQ_MASK		0x0000FF00
+#define		OVLYCSEQ		0x00000080
+#define		OVLYHALTERR		0x00000040
+#define		PIOCMODE		0x00000020
+#define		RESETOVLYDMA		0x00000008	/* wo */
+#define		STARTOVLYDMA		0x00000004
+#define		STOPOVLYDMA		0x00000002	/* wo */
+#define		OVLYDMAACT		0x00000001	/* ro */
+
+#define OVLYDMACNT	(REG_BASE_ADDR + 0x24)
+
+#define		OVLYDOMAIN1		0x20000000	/* ro */
+#define		OVLYDOMAIN0		0x10000000
+#define		OVLYBUFADR_MASK		0x007F0000
+#define		OVLYDMACNT_MASK		0x00003FFF
+
+#define OVLYDMAADR	(REG_BASE_ADDR + 0x28)
+
+#define DMAERR		(REG_BASE_ADDR + 0x30)
+
+#define		OVLYERRSTAT_MASK	0x0000FF00	/* ro */
+#define		CSERRSTAT_MASK		0x000000FF	/* ro */
+
+#define SPIODATA	(REG_BASE_ADDR + 0x34)
+
+/* 0x38 - 0x3C are reserved  */
+
+#define T1CNTRLR	(REG_BASE_ADDR + 0x40)
+
+#define		T1DONE			0x00010000	/* ro */
+#define		TIMER64			0x00000400
+#define		T1ENABLE		0x00000200
+#define		T1RELOAD		0x00000100
+#define		T1PRESCALER_MASK	0x00000003
+
+#define	T1CMPR		(REG_BASE_ADDR + 0x44)
+
+#define T1CNTR		(REG_BASE_ADDR + 0x48)
+
+#define T2CNTRLR	(REG_BASE_ADDR + 0x4C)
+
+#define		T2DONE			0x00010000	/* ro */
+#define		T2ENABLE		0x00000200
+#define		T2RELOAD		0x00000100
+#define		T2PRESCALER_MASK	0x00000003
+
+#define	T2CMPR		(REG_BASE_ADDR + 0x50)
+
+#define T2CNTR		(REG_BASE_ADDR + 0x54)
+
+/* 0x58h - 0xFCh are reserved */
+
+/*
+ * DCH_SAS Registers, Address Range : (0x800-0xFFF)
+ */
+#define CMDCTXBASE	(REG_BASE_ADDR + 0x800)
+
+#define DEVCTXBASE	(REG_BASE_ADDR + 0x808)
+
+#define CTXDOMAIN	(REG_BASE_ADDR + 0x810)
+
+#define		DEVCTXDOMAIN1		0x00000008	/* ro */
+#define		DEVCTXDOMAIN0		0x00000004
+#define		CMDCTXDOMAIN1		0x00000002	/* ro */
+#define		CMDCTXDOMAIN0		0x00000001
+
+#define DCHCTL		(REG_BASE_ADDR + 0x814)
+
+#define		OCMBISTREPAIR		0x00080000
+#define		OCMBISTEN		0x00040000
+#define		OCMBISTDN		0x00020000	/* ro */
+#define		OCMBISTFAIL		0x00010000	/* ro */
+#define		DDBBISTEN		0x00004000
+#define		DDBBISTDN		0x00002000	/* ro */
+#define		DDBBISTFAIL		0x00001000	/* ro */
+#define		SCBBISTEN		0x00000400
+#define		SCBBISTDN		0x00000200	/* ro */
+#define		SCBBISTFAIL		0x00000100	/* ro */
+
+#define		MEMSEL_MASK		0x000000E0
+#define		MEMSEL_CCM_LSEQ		0x00000000
+#define		MEMSEL_CCM_IOP		0x00000020
+#define		MEMSEL_CCM_SASCTL	0x00000040
+#define		MEMSEL_DCM_LSEQ		0x00000060
+#define		MEMSEL_DCM_IOP		0x00000080
+#define		MEMSEL_OCM		0x000000A0
+
+#define		FRCERR			0x00000010
+#define		AUTORLS			0x00000001
+
+#define DCHREVISION	(REG_BASE_ADDR + 0x818)
+
+#define		DCHREVISION_MASK	0x000000FF
+
+#define DCHSTATUS	(REG_BASE_ADDR + 0x81C)
+
+#define		EN_CFIFTOERR		0x00020000
+#define		CFIFTOERR		0x00000200
+#define		CSEQINT			0x00000100	/* ro */
+#define		LSEQ7INT		0x00000080	/* ro */
+#define		LSEQ6INT		0x00000040	/* ro */
+#define		LSEQ5INT		0x00000020	/* ro */
+#define		LSEQ4INT		0x00000010	/* ro */
+#define		LSEQ3INT		0x00000008	/* ro */
+#define		LSEQ2INT		0x00000004	/* ro */
+#define		LSEQ1INT		0x00000002	/* ro */
+#define		LSEQ0INT		0x00000001	/* ro */
+
+#define		LSEQINT_MASK		(LSEQ7INT | LSEQ6INT | LSEQ5INT |\
+					 LSEQ4INT | LSEQ3INT | LSEQ2INT	|\
+					 LSEQ1INT | LSEQ0INT)
+
+#define DCHDFIFDEBUG	(REG_BASE_ADDR + 0x820)
+#define		ENFAIRMST		0x00FF0000
+#define		DISWRMST9		0x00000200
+#define		DISWRMST8		0x00000100
+#define		DISRDMST		0x000000FF
+
+#define ATOMICSTATCTL	(REG_BASE_ADDR + 0x824)
+/* 8 bit wide */
+#define		AUTOINC			0x80
+#define		ATOMICERR		0x04
+#define		ATOMICWIN		0x02
+#define		ATOMICDONE		0x01
+
+
+#define ALTCIOADR	(REG_BASE_ADDR + 0x828)
+/* 16 bit; bits 8:0 define CIO addr space of CSEQ */
+
+#define ASCBPTR		(REG_BASE_ADDR + 0x82C)
+/* 16 bit wide */
+
+#define ADDBPTR		(REG_BASE_ADDR + 0x82E)
+/* 16 bit wide */
+
+#define ANEWDATA	(REG_BASE_ADDR + 0x830)
+/* 16 bit */
+
+#define AOLDDATA	(REG_BASE_ADDR + 0x834)
+/* 16 bit */
+
+#define CTXACCESS	(REG_BASE_ADDR + 0x838)
+/* 32 bit */
+
+/* 0x83Ch - 0xFFCh are reserved */
+
+/*
+ * ARP2 External Processor Registers, Address Range : (0x00-0x1F)
+ */
+#define ARP2CTL		0x00
+
+#define		FRCSCRPERR		0x00040000
+#define		FRCARP2PERR		0x00020000
+#define		FRCARP2ILLOPC		0x00010000
+#define		ENWAITTO		0x00008000
+#define		PERRORDIS		0x00004000
+#define		FAILDIS			0x00002000
+#define		CIOPERRDIS		0x00001000
+#define		BREAKEN3		0x00000800
+#define		BREAKEN2		0x00000400
+#define		BREAKEN1		0x00000200
+#define		BREAKEN0		0x00000100
+#define		EPAUSE			0x00000008
+#define		PAUSED			0x00000004	/* ro */
+#define		STEP			0x00000002
+#define		ARP2RESET		0x00000001	/* wo */
+
+#define ARP2INT		0x04
+
+#define		HALTCODE_MASK		0x00FF0000	/* ro */
+#define		ARP2WAITTO		0x00000100
+#define		ARP2HALTC		0x00000080
+#define		ARP2ILLOPC		0x00000040
+#define		ARP2PERR		0x00000020
+#define		ARP2CIOPERR		0x00000010
+#define		ARP2BREAK3		0x00000008
+#define		ARP2BREAK2		0x00000004
+#define		ARP2BREAK1		0x00000002
+#define		ARP2BREAK0		0x00000001
+
+#define ARP2INTEN	0x08
+
+#define		EN_ARP2WAITTO		0x00000100
+#define		EN_ARP2HALTC		0x00000080
+#define		EN_ARP2ILLOPC		0x00000040
+#define		EN_ARP2PERR		0x00000020
+#define		EN_ARP2CIOPERR		0x00000010
+#define		EN_ARP2BREAK3		0x00000008
+#define		EN_ARP2BREAK2		0x00000004
+#define		EN_ARP2BREAK1		0x00000002
+#define		EN_ARP2BREAK0		0x00000001
+
+#define ARP2BREAKADR01	0x0C
+
+#define		BREAKADR1_MASK		0x0FFF0000
+#define		BREAKADR0_MASK		0x00000FFF
+
+#define	ARP2BREAKADR23	0x10
+
+#define		BREAKADR3_MASK		0x0FFF0000
+#define		BREAKADR2_MASK		0x00000FFF
+
+/* 0x14h - 0x1Ch are reserved */
+
+/*
+ * ARP2 Registers, Address Range : (0x00-0x1F)
+ * The definitions have the same address offset for CSEQ and LmSEQ
+ * CIO Bus Registers.
+ */
+#define MODEPTR		0x00
+
+#define		DSTMODE			0xF0
+#define		SRCMODE			0x0F
+
+#define ALTMODE		0x01
+
+#define		ALTDMODE		0xF0
+#define		ALTSMODE		0x0F
+
+#define ATOMICXCHG	0x02
+
+#define FLAG		0x04
+
+#define		INTCODE_MASK		0xF0
+#define		ALTMODEV2		0x04
+#define		CARRY_INT		0x02
+#define		CARRY			0x01
+
+#define ARP2INTCTL	0x05
+
+#define 	PAUSEDIS		0x80
+#define		RSTINTCTL		0x40
+#define		POPALTMODE		0x08
+#define		ALTMODEV		0x04
+#define		INTMASK			0x02
+#define		IRET			0x01
+
+#define STACK		0x06
+
+#define FUNCTION1	0x07
+
+#define PRGMCNT		0x08
+
+#define ACCUM		0x0A
+
+#define SINDEX		0x0C
+
+#define DINDEX		0x0E
+
+#define ALLONES		0x10
+
+#define ALLZEROS	0x11
+
+#define SINDIR		0x12
+
+#define DINDIR		0x13
+
+#define JUMLDIR		0x14
+
+#define ARP2HALTCODE	0x15
+
+#define CURRADDR	0x16
+
+#define LASTADDR	0x18
+
+#define NXTLADDR	0x1A
+
+#define DBGPORTPTR	0x1C
+
+#define DBGPORT		0x1D
+
+/*
+ * CIO Registers.
+ * The definitions have the same address offset for CSEQ and LmSEQ
+ * CIO Bus Registers.
+ */
+#define MnSCBPTR      	0x20
+
+#define MnDDBPTR      	0x22
+
+#define SCRATCHPAGE	0x24
+
+#define MnSCRATCHPAGE	0x25
+
+#define SCRATCHPAGESV	0x26
+
+#define MnSCRATCHPAGESV	0x27
+
+#define MnDMAERRS	0x46
+
+#define MnSGDMAERRS	0x47
+
+#define MnSGBUF		0x53
+
+#define MnSGDMASTAT	0x5b
+
+#define MnDDMACTL	0x5c	/* RAZOR.rspec.fm rev 1.5 is wrong */
+
+#define MnDDMASTAT	0x5d	/* RAZOR.rspec.fm rev 1.5 is wrong */
+
+#define MnDDMAMODE	0x5e	/* RAZOR.rspec.fm rev 1.5 is wrong */
+
+#define MnDMAENG	0x60
+
+#define MnPIPECTL	0x61
+
+#define MnSGBADR	0x65
+
+#define MnSCB_SITE	0x100
+
+#define MnDDB_SITE	0x180
+
+/*
+ * The common definitions below have the same address offset for both
+ * CSEQ and LmSEQ.
+ */
+#define BISTCTL0	0x4C
+
+#define BISTCTL1	0x50
+
+#define MAPPEDSCR	0x800
+
+/*
+ * CSEQ Host Register, Address Range : (0x000-0xFFC)
+ */
+#define CSEQ_HOST_REG_BASE_ADR		0xB8001000
+
+#define CARP2CTL			(CSEQ_HOST_REG_BASE_ADR	+ ARP2CTL)
+
+#define CARP2INT			(CSEQ_HOST_REG_BASE_ADR	+ ARP2INT)
+
+#define CARP2INTEN			(CSEQ_HOST_REG_BASE_ADR	+ ARP2INTEN)
+
+#define CARP2BREAKADR01			(CSEQ_HOST_REG_BASE_ADR+ARP2BREAKADR01)
+
+#define CARP2BREAKADR23			(CSEQ_HOST_REG_BASE_ADR+ARP2BREAKADR23)
+
+#define CBISTCTL			(CSEQ_HOST_REG_BASE_ADR	+ BISTCTL1)
+
+#define		CSEQRAMBISTEN		0x00000040
+#define		CSEQRAMBISTDN		0x00000020	/* ro */
+#define		CSEQRAMBISTFAIL		0x00000010	/* ro */
+#define		CSEQSCRBISTEN		0x00000004
+#define		CSEQSCRBISTDN		0x00000002	/* ro */
+#define		CSEQSCRBISTFAIL		0x00000001	/* ro */
+
+#define CMAPPEDSCR			(CSEQ_HOST_REG_BASE_ADR	+ MAPPEDSCR)
+
+/*
+ * CSEQ CIO Bus Registers, Address Range : (0x0000-0x1FFC)
+ * 16 modes, each mode is 512 bytes.
+ * Unless specified, the register should valid for all modes.
+ */
+#define CSEQ_CIO_REG_BASE_ADR		REG_BASE_ADDR_CSEQCIO
+
+#define CSEQm_CIO_REG(Mode, Reg) \
+		(CSEQ_CIO_REG_BASE_ADR  + \
+		((u32) (Mode) * CSEQ_MODE_PAGE_SIZE) + (u32) (Reg))
+
+#define CMODEPTR	(CSEQ_CIO_REG_BASE_ADR + MODEPTR)
+
+#define CALTMODE	(CSEQ_CIO_REG_BASE_ADR + ALTMODE)
+
+#define CATOMICXCHG	(CSEQ_CIO_REG_BASE_ADR + ATOMICXCHG)
+
+#define CFLAG		(CSEQ_CIO_REG_BASE_ADR + FLAG)
+
+#define CARP2INTCTL	(CSEQ_CIO_REG_BASE_ADR + ARP2INTCTL)
+
+#define CSTACK		(CSEQ_CIO_REG_BASE_ADR + STACK)
+
+#define CFUNCTION1	(CSEQ_CIO_REG_BASE_ADR + FUNCTION1)
+
+#define CPRGMCNT	(CSEQ_CIO_REG_BASE_ADR + PRGMCNT)
+
+#define CACCUM		(CSEQ_CIO_REG_BASE_ADR + ACCUM)
+
+#define CSINDEX		(CSEQ_CIO_REG_BASE_ADR + SINDEX)
+
+#define CDINDEX		(CSEQ_CIO_REG_BASE_ADR + DINDEX)
+
+#define CALLONES	(CSEQ_CIO_REG_BASE_ADR + ALLONES)
+
+#define CALLZEROS	(CSEQ_CIO_REG_BASE_ADR + ALLZEROS)
+
+#define CSINDIR		(CSEQ_CIO_REG_BASE_ADR + SINDIR)
+
+#define CDINDIR		(CSEQ_CIO_REG_BASE_ADR + DINDIR)
+
+#define CJUMLDIR	(CSEQ_CIO_REG_BASE_ADR + JUMLDIR)
+
+#define CARP2HALTCODE	(CSEQ_CIO_REG_BASE_ADR + ARP2HALTCODE)
+
+#define CCURRADDR	(CSEQ_CIO_REG_BASE_ADR + CURRADDR)
+
+#define CLASTADDR	(CSEQ_CIO_REG_BASE_ADR + LASTADDR)
+
+#define CNXTLADDR	(CSEQ_CIO_REG_BASE_ADR + NXTLADDR)
+
+#define CDBGPORTPTR	(CSEQ_CIO_REG_BASE_ADR + DBGPORTPTR)
+
+#define CDBGPORT	(CSEQ_CIO_REG_BASE_ADR + DBGPORT)
+
+#define CSCRATCHPAGE	(CSEQ_CIO_REG_BASE_ADR + SCRATCHPAGE)
+
+#define CMnSCBPTR(Mode)       CSEQm_CIO_REG(Mode, MnSCBPTR)
+
+#define CMnDDBPTR(Mode)       CSEQm_CIO_REG(Mode, MnDDBPTR)
+
+#define CMnSCRATCHPAGE(Mode)		CSEQm_CIO_REG(Mode, MnSCRATCHPAGE)
+
+#define CLINKCON	(CSEQ_CIO_REG_BASE_ADR + 0x28)
+
+#define	CCIOAACESS	(CSEQ_CIO_REG_BASE_ADR + 0x2C)
+
+/* mode 0-7 */
+#define MnREQMBX 0x30
+#define CMnREQMBX(Mode)			CSEQm_CIO_REG(Mode, 0x30)
+
+/* mode 8 */
+#define CSEQCON				CSEQm_CIO_REG(8, 0x30)
+
+/* mode 0-7 */
+#define MnRSPMBX 0x34
+#define CMnRSPMBX(Mode)			CSEQm_CIO_REG(Mode, 0x34)
+
+/* mode 8 */
+#define CSEQCOMCTL			CSEQm_CIO_REG(8, 0x34)
+
+/* mode 8 */
+#define CSEQCOMSTAT			CSEQm_CIO_REG(8, 0x35)
+
+/* mode 8 */
+#define CSEQCOMINTEN			CSEQm_CIO_REG(8, 0x36)
+
+/* mode 8 */
+#define CSEQCOMDMACTL			CSEQm_CIO_REG(8, 0x37)
+
+#define		CSHALTERR		0x10
+#define		RESETCSDMA		0x08		/* wo */
+#define		STARTCSDMA		0x04
+#define		STOPCSDMA		0x02		/* wo */
+#define		CSDMAACT		0x01		/* ro */
+
+/* mode 0-7 */
+#define MnINT 0x38
+#define CMnINT(Mode)			CSEQm_CIO_REG(Mode, 0x38)
+
+#define		CMnREQMBXE		0x02
+#define		CMnRSPMBXF		0x01
+#define		CMnINT_MASK		0x00000003
+
+/* mode 8 */
+#define CSEQREQMBX			CSEQm_CIO_REG(8, 0x38)
+
+/* mode 0-7 */
+#define MnINTEN 0x3C
+#define CMnINTEN(Mode)			CSEQm_CIO_REG(Mode, 0x3C)
+
+#define		EN_CMnRSPMBXF		0x01
+
+/* mode 8 */
+#define CSEQRSPMBX			CSEQm_CIO_REG(8, 0x3C)
+
+/* mode 8 */
+#define CSDMAADR			CSEQm_CIO_REG(8, 0x40)
+
+/* mode 8 */
+#define CSDMACNT			CSEQm_CIO_REG(8, 0x48)
+
+/* mode 8 */
+#define CSEQDLCTL			CSEQm_CIO_REG(8, 0x4D)
+
+#define		DONELISTEND		0x10
+#define 	DONELISTSIZE_MASK	0x0F
+#define		DONELISTSIZE_8ELEM	0x01
+#define		DONELISTSIZE_16ELEM	0x02
+#define		DONELISTSIZE_32ELEM	0x03
+#define		DONELISTSIZE_64ELEM	0x04
+#define		DONELISTSIZE_128ELEM	0x05
+#define		DONELISTSIZE_256ELEM	0x06
+#define		DONELISTSIZE_512ELEM	0x07
+#define		DONELISTSIZE_1024ELEM	0x08
+#define		DONELISTSIZE_2048ELEM	0x09
+#define		DONELISTSIZE_4096ELEM	0x0A
+#define		DONELISTSIZE_8192ELEM	0x0B
+#define		DONELISTSIZE_16384ELEM	0x0C
+
+/* mode 8 */
+#define CSEQDLOFFS			CSEQm_CIO_REG(8, 0x4E)
+
+/* mode 11 */
+#define CM11INTVEC0			CSEQm_CIO_REG(11, 0x50)
+
+/* mode 11 */
+#define CM11INTVEC1			CSEQm_CIO_REG(11, 0x52)
+
+/* mode 11 */
+#define CM11INTVEC2			CSEQm_CIO_REG(11, 0x54)
+
+#define	CCONMSK	  			(CSEQ_CIO_REG_BASE_ADR + 0x60)
+
+#define	CCONEXIST			(CSEQ_CIO_REG_BASE_ADR + 0x61)
+
+#define	CCONMODE			(CSEQ_CIO_REG_BASE_ADR + 0x62)
+
+#define CTIMERCALC			(CSEQ_CIO_REG_BASE_ADR + 0x64)
+
+#define CINTDIS				(CSEQ_CIO_REG_BASE_ADR + 0x68)
+
+/* mode 8, 32x32 bits, 128 bytes of mapped buffer */
+#define CSBUFFER			CSEQm_CIO_REG(8, 0x80)
+
+#define	CSCRATCH			(CSEQ_CIO_REG_BASE_ADR + 0x1C0)
+
+/* mode 0-8 */
+#define CMnSCRATCH(Mode)		CSEQm_CIO_REG(Mode, 0x1E0)
+
+/*
+ * CSEQ Mapped Instruction RAM Page, Address Range : (0x0000-0x1FFC)
+ */
+#define CSEQ_RAM_REG_BASE_ADR		0xB8004000
+
+/*
+ * The common definitions below have the same address offset for all the Link
+ * sequencers.
+ */
+#define MODECTL		0x40
+
+#define DBGMODE		0x44
+
+#define CONTROL		0x48
+#define LEDTIMER		0x00010000
+#define LEDTIMERS_10us		0x00000000
+#define LEDTIMERS_1ms		0x00000800
+#define LEDTIMERS_100ms		0x00001000
+#define LEDMODE_TXRX		0x00000000
+#define LEDMODE_CONNECTED	0x00000200
+#define LEDPOL			0x00000100
+
+#define LSEQRAM		0x1000
+
+/*
+ * LmSEQ Host Registers, Address Range : (0x0000-0x3FFC)
+ */
+#define LSEQ0_HOST_REG_BASE_ADR		0xB8020000
+#define LSEQ1_HOST_REG_BASE_ADR		0xB8024000
+#define LSEQ2_HOST_REG_BASE_ADR		0xB8028000
+#define LSEQ3_HOST_REG_BASE_ADR		0xB802C000
+#define LSEQ4_HOST_REG_BASE_ADR		0xB8030000
+#define LSEQ5_HOST_REG_BASE_ADR		0xB8034000
+#define LSEQ6_HOST_REG_BASE_ADR		0xB8038000
+#define LSEQ7_HOST_REG_BASE_ADR		0xB803C000
+
+#define LmARP2CTL(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					ARP2CTL)
+
+#define LmARP2INT(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					ARP2INT)
+
+#define LmARP2INTEN(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					ARP2INTEN)
+
+#define LmDBGMODE(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					DBGMODE)
+
+#define LmCONTROL(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					CONTROL)
+
+#define LmARP2BREAKADR01(LinkNum)	(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					ARP2BREAKADR01)
+
+#define LmARP2BREAKADR23(LinkNum)	(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					ARP2BREAKADR23)
+
+#define LmMODECTL(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					MODECTL)
+
+#define		LmAUTODISCI		0x08000000
+#define		LmDSBLBITLT		0x04000000
+#define		LmDSBLANTT		0x02000000
+#define		LmDSBLCRTT		0x01000000
+#define		LmDSBLCONT		0x00000100
+#define		LmPRIMODE		0x00000080
+#define		LmDSBLHOLD		0x00000040
+#define		LmDISACK		0x00000020
+#define		LmBLIND48		0x00000010
+#define		LmRCVMODE_MASK		0x0000000C
+#define		LmRCVMODE_PLD		0x00000000
+#define		LmRCVMODE_HPC		0x00000004
+
+#define LmDBGMODE(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					DBGMODE)
+
+#define		LmFRCPERR		0x80000000
+#define		LmMEMSEL_MASK		0x30000000
+#define		LmFRCRBPERR		0x00000000
+#define		LmFRCTBPERR		0x10000000
+#define		LmFRCSGBPERR		0x20000000
+#define		LmFRCARBPERR		0x30000000
+#define		LmRCVIDW		0x00080000
+#define		LmINVDWERR		0x00040000
+#define		LmRCVDISP		0x00004000
+#define		LmDISPERR		0x00002000
+#define		LmDSBLDSCR		0x00000800
+#define		LmDSBLSCR		0x00000400
+#define		LmFRCNAK		0x00000200
+#define		LmFRCROFS		0x00000100
+#define		LmFRCCRC		0x00000080
+#define		LmFRMTYPE_MASK		0x00000070
+#define		LmSG_DATA		0x00000000
+#define		LmSG_COMMAND		0x00000010
+#define		LmSG_TASK		0x00000020
+#define		LmSG_TGTXFER		0x00000030
+#define		LmSG_RESPONSE		0x00000040
+#define		LmSG_IDENADDR		0x00000050
+#define		LmSG_OPENADDR		0x00000060
+#define		LmDISCRCGEN		0x00000008
+#define		LmDISCRCCHK		0x00000004
+#define		LmSSXMTFRM		0x00000002
+#define		LmSSRCVFRM		0x00000001
+
+#define LmCONTROL(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					CONTROL)
+
+#define		LmSTEPXMTFRM		0x00000002
+#define		LmSTEPRCVFRM		0x00000001
+
+#define LmBISTCTL0(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	  \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) + \
+					BISTCTL0)
+
+#define		ARBBISTEN		0x40000000
+#define		ARBBISTDN		0x20000000	/* ro */
+#define		ARBBISTFAIL		0x10000000	/* ro */
+#define		TBBISTEN		0x00000400
+#define		TBBISTDN		0x00000200	/* ro */
+#define		TBBISTFAIL		0x00000100	/* ro */
+#define		RBBISTEN		0x00000040
+#define		RBBISTDN		0x00000020	/* ro */
+#define		RBBISTFAIL		0x00000010	/* ro */
+#define		SGBISTEN		0x00000004
+#define		SGBISTDN		0x00000002	/* ro */
+#define		SGBISTFAIL		0x00000001	/* ro */
+
+#define LmBISTCTL1(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	 \
+					((LinkNum)*LmSEQ_HOST_REG_SIZE) +\
+					BISTCTL1)
+
+#define		LmRAMPAGE1		0x00000200
+#define		LmRAMPAGE0		0x00000100
+#define		LmIMEMBISTEN		0x00000040
+#define		LmIMEMBISTDN		0x00000020	/* ro */
+#define		LmIMEMBISTFAIL		0x00000010	/* ro */
+#define		LmSCRBISTEN		0x00000004
+#define		LmSCRBISTDN		0x00000002	/* ro */
+#define		LmSCRBISTFAIL		0x00000001	/* ro */
+#define		LmRAMPAGE		(LmRAMPAGE1 + LmRAMPAGE0)
+#define		LmRAMPAGE_LSHIFT	0x8
+
+#define LmSCRATCH(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	   \
+					((LinkNum) * LmSEQ_HOST_REG_SIZE) +\
+					MAPPEDSCR)
+
+#define LmSEQRAM(LinkNum)		(LSEQ0_HOST_REG_BASE_ADR +	   \
+					((LinkNum) * LmSEQ_HOST_REG_SIZE) +\
+					LSEQRAM)
+
+/*
+ * LmSEQ CIO Bus Register, Address Range : (0x0000-0xFFC)
+ * 8 modes, each mode is 512 bytes.
+ * Unless specified, the register should valid for all modes.
+ */
+#define LmSEQ_CIOBUS_REG_BASE		0x2000
+
+#define  LmSEQ_PHY_BASE(Mode, LinkNum) \
+		(LSEQ0_HOST_REG_BASE_ADR + \
+		(LmSEQ_HOST_REG_SIZE * (u32) (LinkNum)) + \
+		LmSEQ_CIOBUS_REG_BASE + \
+		((u32) (Mode) * LmSEQ_MODE_PAGE_SIZE))
+
+#define  LmSEQ_PHY_REG(Mode, LinkNum, Reg) \
+                 (LmSEQ_PHY_BASE(Mode, LinkNum) + (u32) (Reg))
+
+#define LmMODEPTR(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, MODEPTR)
+
+#define LmALTMODE(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, ALTMODE)
+
+#define LmATOMICXCHG(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, ATOMICXCHG)
+
+#define LmFLAG(LinkNum)			LmSEQ_PHY_REG(0, LinkNum, FLAG)
+
+#define LmARP2INTCTL(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, ARP2INTCTL)
+
+#define LmSTACK(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, STACK)
+
+#define LmFUNCTION1(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, FUNCTION1)
+
+#define LmPRGMCNT(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, PRGMCNT)
+
+#define LmACCUM(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, ACCUM)
+
+#define LmSINDEX(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, SINDEX)
+
+#define LmDINDEX(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, DINDEX)
+
+#define LmALLONES(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, ALLONES)
+
+#define LmALLZEROS(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, ALLZEROS)
+
+#define LmSINDIR(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, SINDIR)
+
+#define LmDINDIR(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, DINDIR)
+
+#define LmJUMLDIR(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, JUMLDIR)
+
+#define LmARP2HALTCODE(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, ARP2HALTCODE)
+
+#define LmCURRADDR(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, CURRADDR)
+
+#define LmLASTADDR(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, LASTADDR)
+
+#define LmNXTLADDR(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, NXTLADDR)
+
+#define LmDBGPORTPTR(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, DBGPORTPTR)
+
+#define LmDBGPORT(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, DBGPORT)
+
+#define LmSCRATCHPAGE(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, SCRATCHPAGE)
+
+#define LmMnSCRATCHPAGE(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 	\
+						      MnSCRATCHPAGE)
+
+#define LmTIMERCALC(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x28)
+
+#define LmREQMBX(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x30)
+
+#define LmRSPMBX(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x34)
+
+#define LmMnINT(LinkNum, Mode)		LmSEQ_PHY_REG(Mode, LinkNum, 0x38)
+
+#define		CTXMEMSIZE		0x80000000	/* ro */
+#define		LmACKREQ		0x08000000
+#define		LmNAKREQ		0x04000000
+#define		LmMnXMTERR		0x02000000
+#define		LmM5OOBSVC		0x01000000
+#define		LmHWTINT		0x00800000
+#define		LmMnCTXDONE		0x00100000
+#define		LmM2REQMBXF		0x00080000
+#define		LmM2RSPMBXE		0x00040000
+#define		LmMnDMAERR		0x00020000
+#define		LmRCVPRIM		0x00010000
+#define		LmRCVERR		0x00008000
+#define		LmADDRRCV		0x00004000
+#define		LmMnHDRMISS		0x00002000
+#define		LmMnWAITSCB		0x00001000
+#define		LmMnRLSSCB		0x00000800
+#define		LmMnSAVECTX		0x00000400
+#define		LmMnFETCHSG		0x00000200
+#define		LmMnLOADCTX		0x00000100
+#define		LmMnCFGICL		0x00000080
+#define		LmMnCFGSATA		0x00000040
+#define		LmMnCFGEXPSATA		0x00000020
+#define		LmMnCFGCMPLT		0x00000010
+#define		LmMnCFGRBUF		0x00000008
+#define		LmMnSAVETTR		0x00000004
+#define		LmMnCFGRDAT		0x00000002
+#define		LmMnCFGHDR		0x00000001
+
+#define LmMnINTEN(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x3C)
+
+#define		EN_LmACKREQ		0x08000000
+#define		EN_LmNAKREQ		0x04000000
+#define		EN_LmMnXMTERR		0x02000000
+#define		EN_LmM5OOBSVC		0x01000000
+#define		EN_LmHWTINT		0x00800000
+#define		EN_LmMnCTXDONE		0x00100000
+#define		EN_LmM2REQMBXF		0x00080000
+#define		EN_LmM2RSPMBXE		0x00040000
+#define		EN_LmMnDMAERR		0x00020000
+#define		EN_LmRCVPRIM		0x00010000
+#define		EN_LmRCVERR		0x00008000
+#define		EN_LmADDRRCV		0x00004000
+#define		EN_LmMnHDRMISS		0x00002000
+#define		EN_LmMnWAITSCB		0x00001000
+#define		EN_LmMnRLSSCB		0x00000800
+#define		EN_LmMnSAVECTX		0x00000400
+#define		EN_LmMnFETCHSG		0x00000200
+#define		EN_LmMnLOADCTX		0x00000100
+#define		EN_LmMnCFGICL		0x00000080
+#define		EN_LmMnCFGSATA		0x00000040
+#define		EN_LmMnCFGEXPSATA	0x00000020
+#define		EN_LmMnCFGCMPLT		0x00000010
+#define		EN_LmMnCFGRBUF		0x00000008
+#define		EN_LmMnSAVETTR		0x00000004
+#define		EN_LmMnCFGRDAT		0x00000002
+#define		EN_LmMnCFGHDR		0x00000001
+
+#define		LmM0INTEN_MASK		(EN_LmMnCFGCMPLT | EN_LmMnCFGRBUF | \
+					 EN_LmMnSAVETTR | EN_LmMnCFGRDAT | \
+					 EN_LmMnCFGHDR | EN_LmRCVERR | \
+					 EN_LmADDRRCV | EN_LmMnHDRMISS | \
+					 EN_LmMnRLSSCB | EN_LmMnSAVECTX | \
+					 EN_LmMnFETCHSG | EN_LmMnLOADCTX | \
+					 EN_LmHWTINT | EN_LmMnCTXDONE | \
+					 EN_LmRCVPRIM | EN_LmMnCFGSATA | \
+					 EN_LmMnCFGEXPSATA | EN_LmMnDMAERR)
+
+#define		LmM1INTEN_MASK		(EN_LmMnCFGCMPLT | EN_LmADDRRCV | \
+					 EN_LmMnRLSSCB | EN_LmMnSAVECTX | \
+					 EN_LmMnFETCHSG | EN_LmMnLOADCTX | \
+					 EN_LmMnXMTERR | EN_LmHWTINT | \
+					 EN_LmMnCTXDONE | EN_LmRCVPRIM | \
+					 EN_LmRCVERR | EN_LmMnDMAERR)
+
+#define		LmM2INTEN_MASK		(EN_LmADDRRCV | EN_LmHWTINT | \
+					 EN_LmM2REQMBXF | EN_LmRCVPRIM | \
+					 EN_LmRCVERR)
+
+#define		LmM5INTEN_MASK		(EN_LmADDRRCV | EN_LmM5OOBSVC | \
+					 EN_LmHWTINT | EN_LmRCVPRIM | \
+					 EN_LmRCVERR)
+
+#define LmXMTPRIMD(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x40)
+
+#define LmXMTPRIMCS(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x44)
+
+#define LmCONSTAT(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x45)
+
+#define LmMnDMAERRS(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x46)
+
+#define LmMnSGDMAERRS(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x47)
+
+#define LmM0EXPHDRP(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x48)
+
+#define LmM1SASALIGN(LinkNum)		LmSEQ_PHY_REG(1, LinkNum, 0x48)
+#define SAS_ALIGN_DEFAULT		0xFF
+
+#define LmM0MSKHDRP(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x49)
+
+#define LmM1STPALIGN(LinkNum)		LmSEQ_PHY_REG(1, LinkNum, 0x49)
+#define STP_ALIGN_DEFAULT		0x1F
+
+#define LmM0RCVHDRP(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x4A)
+
+#define LmM1XMTHDRP(LinkNum)		LmSEQ_PHY_REG(1, LinkNum, 0x4A)
+
+#define LmM0ICLADR(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x4B)
+
+#define LmM1ALIGNMODE(LinkNum)		LmSEQ_PHY_REG(1, LinkNum, 0x4B)
+
+#define		LmDISALIGN		0x20
+#define		LmROTSTPALIGN		0x10
+#define		LmSTPALIGN		0x08
+#define		LmROTNOTIFY		0x04
+#define		LmDUALALIGN		0x02
+#define		LmROTALIGN		0x01
+
+#define LmM0EXPRCVNT(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x4C)
+
+#define LmM1XMTCNT(LinkNum)		LmSEQ_PHY_REG(1, LinkNum, 0x4C)
+
+#define LmMnBUFSTAT(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x4E)
+
+#define		LmMnBUFPERR		0x01
+
+/* mode 0-1 */
+#define LmMnXFRLVL(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x59)
+
+#define		LmMnXFRLVL_128		0x05
+#define		LmMnXFRLVL_256		0x04
+#define		LmMnXFRLVL_512		0x03
+#define		LmMnXFRLVL_1024		0x02
+#define		LmMnXFRLVL_1536		0x01
+#define		LmMnXFRLVL_2048		0x00
+
+ /* mode 0-1 */
+#define LmMnSGDMACTL(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x5A)
+
+#define 	LmMnRESETSG		0x04
+#define 	LmMnSTOPSG		0x02
+#define 	LmMnSTARTSG		0x01
+
+/* mode 0-1 */
+#define LmMnSGDMASTAT(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x5B)
+
+/* mode 0-1 */
+#define LmMnDDMACTL(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x5C)
+
+#define 	LmMnFLUSH		0x40		/* wo */
+#define 	LmMnRLSRTRY		0x20		/* wo */
+#define 	LmMnDISCARD		0x10		/* wo */
+#define 	LmMnRESETDAT		0x08		/* wo */
+#define 	LmMnSUSDAT		0x04		/* wo */
+#define 	LmMnSTOPDAT		0x02		/* wo */
+#define 	LmMnSTARTDAT		0x01		/* wo */
+
+/* mode 0-1 */
+#define LmMnDDMASTAT(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x5D)
+
+#define		LmMnDPEMPTY		0x80
+#define		LmMnFLUSHING		0x40
+#define		LmMnDDMAREQ		0x20
+#define		LmMnHDMAREQ		0x10
+#define		LmMnDATFREE		0x08
+#define		LmMnDATSUS		0x04
+#define		LmMnDATACT		0x02
+#define		LmMnDATEN		0x01
+
+/* mode 0-1 */
+#define LmMnDDMAMODE(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x5E)
+
+#define 	LmMnDMATYPE_NORMAL		0x0000
+#define 	LmMnDMATYPE_HOST_ONLY_TX	0x0001
+#define 	LmMnDMATYPE_DEVICE_ONLY_TX	0x0002
+#define 	LmMnDMATYPE_INVALID		0x0003
+#define 	LmMnDMATYPE_MASK	0x0003
+
+#define 	LmMnDMAWRAP		0x0004
+#define 	LmMnBITBUCKET		0x0008
+#define 	LmMnDISHDR		0x0010
+#define 	LmMnSTPCRC		0x0020
+#define 	LmXTEST			0x0040
+#define 	LmMnDISCRC		0x0080
+#define 	LmMnENINTLK		0x0100
+#define 	LmMnADDRFRM		0x0400
+#define 	LmMnENXMTCRC		0x0800
+
+/* mode 0-1 */
+#define LmMnXFRCNT(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x70)
+
+/* mode 0-1 */
+#define LmMnDPSEL(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x7B)
+#define 	LmMnDPSEL_MASK		0x07
+#define 	LmMnEOLPRE		0x40
+#define 	LmMnEOSPRE		0x80
+
+/* Registers used in conjunction with LmMnDPSEL and LmMnDPACC registers */
+/* Receive Mode n = 0 */
+#define LmMnHRADDR			0x00
+#define LmMnHBYTECNT			0x01
+#define LmMnHREWIND			0x02
+#define LmMnDWADDR			0x03
+#define LmMnDSPACECNT			0x04
+#define LmMnDFRMSIZE			0x05
+
+/* Registers used in conjunction with LmMnDPSEL and LmMnDPACC registers */
+/* Transmit Mode n = 1 */
+#define LmMnHWADDR			0x00
+#define LmMnHSPACECNT			0x01
+/* #define LmMnHREWIND			0x02 */
+#define LmMnDRADDR			0x03
+#define LmMnDBYTECNT			0x04
+/* #define LmMnDFRMSIZE			0x05 */
+
+/* mode 0-1 */
+#define LmMnDPACC(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x78)
+#define 	LmMnDPACC_MASK		0x00FFFFFF
+
+/* mode 0-1 */
+#define LmMnHOLDLVL(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x7D)
+
+#define LmPRMSTAT0(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x80)
+#define LmPRMSTAT0BYTE0			0x80
+#define LmPRMSTAT0BYTE1			0x81
+#define LmPRMSTAT0BYTE2			0x82
+#define LmPRMSTAT0BYTE3			0x83
+
+#define		LmFRAMERCVD		0x80000000
+#define		LmXFRRDYRCVD		0x40000000
+#define		LmUNKNOWNP		0x20000000
+#define		LmBREAK			0x10000000
+#define		LmDONE			0x08000000
+#define		LmOPENACPT		0x04000000
+#define		LmOPENRJCT		0x02000000
+#define		LmOPENRTRY		0x01000000
+#define		LmCLOSERV1		0x00800000
+#define		LmCLOSERV0		0x00400000
+#define		LmCLOSENORM		0x00200000
+#define		LmCLOSECLAF		0x00100000
+#define		LmNOTIFYRV2		0x00080000
+#define		LmNOTIFYRV1		0x00040000
+#define		LmNOTIFYRV0		0x00020000
+#define		LmNOTIFYSPIN		0x00010000
+#define		LmBROADRV4		0x00008000
+#define		LmBROADRV3		0x00004000
+#define		LmBROADRV2		0x00002000
+#define		LmBROADRV1		0x00001000
+#define		LmBROADSES		0x00000800
+#define		LmBROADRVCH1		0x00000400
+#define		LmBROADRVCH0		0x00000200
+#define		LmBROADCH		0x00000100
+#define		LmAIPRVWP		0x00000080
+#define		LmAIPWP			0x00000040
+#define		LmAIPWD			0x00000020
+#define		LmAIPWC			0x00000010
+#define		LmAIPRV2		0x00000008
+#define		LmAIPRV1		0x00000004
+#define		LmAIPRV0		0x00000002
+#define		LmAIPNRML		0x00000001
+
+#define		LmBROADCAST_MASK	(LmBROADCH | LmBROADRVCH0 | \
+					 LmBROADRVCH1)
+
+#define LmPRMSTAT1(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0x84)
+#define LmPRMSTAT1BYTE0			0x84
+#define LmPRMSTAT1BYTE1			0x85
+#define LmPRMSTAT1BYTE2			0x86
+#define LmPRMSTAT1BYTE3			0x87
+
+#define		LmFRMRCVDSTAT		0x80000000
+#define		LmBREAK_DET		0x04000000
+#define		LmCLOSE_DET		0x02000000
+#define		LmDONE_DET		0x01000000
+#define		LmXRDY			0x00040000
+#define 	LmSYNCSRST		0x00020000
+#define 	LmSYNC			0x00010000
+#define 	LmXHOLD			0x00008000
+#define 	LmRRDY			0x00004000
+#define 	LmHOLD			0x00002000
+#define 	LmROK			0x00001000
+#define 	LmRIP			0x00000800
+#define 	LmCRBLK			0x00000400
+#define 	LmACK			0x00000200
+#define 	LmNAK			0x00000100
+#define 	LmHARDRST		0x00000080
+#define 	LmERROR			0x00000040
+#define 	LmRERR			0x00000020
+#define 	LmPMREQP		0x00000010
+#define 	LmPMREQS		0x00000008
+#define 	LmPMACK			0x00000004
+#define 	LmPMNAK			0x00000002
+#define 	LmDMAT			0x00000001
+
+/* mode 1 */
+#define	LmMnSATAFS(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x7E)
+#define	LmMnXMTSIZE(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0x93)
+
+/* mode 0 */
+#define LmMnFRMERR(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0xB0)
+
+#define		LmACRCERR		0x00000800
+#define		LmPHYOVRN		0x00000400
+#define		LmOBOVRN		0x00000200
+#define 	LmMnZERODATA		0x00000100
+#define		LmSATAINTLK		0x00000080
+#define		LmMnCRCERR		0x00000020
+#define		LmRRDYOVRN		0x00000010
+#define		LmMISSSOAF		0x00000008
+#define		LmMISSSOF		0x00000004
+#define		LmMISSEOAF		0x00000002
+#define		LmMISSEOF		0x00000001
+
+#define LmFRMERREN(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0xB4)
+
+#define 	EN_LmACRCERR		0x00000800
+#define 	EN_LmPHYOVRN		0x00000400
+#define 	EN_LmOBOVRN		0x00000200
+#define 	EN_LmMnZERODATA		0x00000100
+#define 	EN_LmSATAINTLK		0x00000080
+#define 	EN_LmFRMBAD		0x00000040
+#define 	EN_LmMnCRCERR		0x00000020
+#define 	EN_LmRRDYOVRN		0x00000010
+#define 	EN_LmMISSSOAF		0x00000008
+#define 	EN_LmMISSSOF		0x00000004
+#define 	EN_LmMISSEOAF		0x00000002
+#define 	EN_LmMISSEOF		0x00000001
+
+#define 	LmFRMERREN_MASK  	(EN_LmSATAINTLK | EN_LmMnCRCERR | \
+					 EN_LmRRDYOVRN | EN_LmMISSSOF | \
+					 EN_LmMISSEOAF | EN_LmMISSEOF | \
+					 EN_LmACRCERR | LmPHYOVRN | \
+					 EN_LmOBOVRN | EN_LmMnZERODATA)
+
+#define LmHWTSTATEN(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0xC5)
+
+#define		EN_LmDONETO		0x80
+#define		EN_LmINVDISP		0x40
+#define		EN_LmINVDW		0x20
+#define		EN_LmDWSEVENT		0x08
+#define		EN_LmCRTTTO		0x04
+#define		EN_LmANTTTO		0x02
+#define		EN_LmBITLTTO		0x01
+
+#define		LmHWTSTATEN_MASK	(EN_LmINVDISP | EN_LmINVDW | \
+					 EN_LmDWSEVENT | EN_LmCRTTTO | \
+					 EN_LmANTTTO | EN_LmDONETO | \
+					 EN_LmBITLTTO)
+
+#define LmHWTSTAT(LinkNum) 		LmSEQ_PHY_REG(0, LinkNum, 0xC7)
+
+#define		LmDONETO		0x80
+#define		LmINVDISP		0x40
+#define		LmINVDW			0x20
+#define		LmDWSEVENT		0x08
+#define		LmCRTTTO		0x04
+#define		LmANTTTO		0x02
+#define		LmBITLTTO		0x01
+
+#define LmMnDATABUFADR(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0xC8)
+#define		LmDATABUFADR_MASK	0x0FFF
+
+#define LmMnDATABUF(LinkNum, Mode)	LmSEQ_PHY_REG(Mode, LinkNum, 0xCA)
+
+#define	LmPRIMSTAT0EN(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0xE0)
+
+#define 	EN_LmUNKNOWNP 		0x20000000
+#define 	EN_LmBREAK		0x10000000
+#define 	EN_LmDONE		0x08000000
+#define 	EN_LmOPENACPT		0x04000000
+#define 	EN_LmOPENRJCT		0x02000000
+#define 	EN_LmOPENRTRY		0x01000000
+#define 	EN_LmCLOSERV1		0x00800000
+#define 	EN_LmCLOSERV0		0x00400000
+#define 	EN_LmCLOSENORM		0x00200000
+#define 	EN_LmCLOSECLAF		0x00100000
+#define 	EN_LmNOTIFYRV2		0x00080000
+#define 	EN_LmNOTIFYRV1		0x00040000
+#define 	EN_LmNOTIFYRV0		0x00020000
+#define 	EN_LmNOTIFYSPIN		0x00010000
+#define 	EN_LmBROADRV4		0x00008000
+#define 	EN_LmBROADRV3		0x00004000
+#define 	EN_LmBROADRV2		0x00002000
+#define 	EN_LmBROADRV1		0x00001000
+#define 	EN_LmBROADRV0		0x00000800
+#define 	EN_LmBROADRVCH1		0x00000400
+#define 	EN_LmBROADRVCH0		0x00000200
+#define 	EN_LmBROADCH		0x00000100
+#define 	EN_LmAIPRVWP		0x00000080
+#define 	EN_LmAIPWP		0x00000040
+#define 	EN_LmAIPWD		0x00000020
+#define 	EN_LmAIPWC		0x00000010
+#define 	EN_LmAIPRV2		0x00000008
+#define 	EN_LmAIPRV1		0x00000004
+#define 	EN_LmAIPRV0		0x00000002
+#define 	EN_LmAIPNRML		0x00000001
+
+#define		LmPRIMSTAT0EN_MASK	(EN_LmBREAK | \
+					 EN_LmDONE | EN_LmOPENACPT | \
+					 EN_LmOPENRJCT | EN_LmOPENRTRY | \
+					 EN_LmCLOSERV1 | EN_LmCLOSERV0 | \
+					 EN_LmCLOSENORM | EN_LmCLOSECLAF | \
+					 EN_LmBROADRV4 | EN_LmBROADRV3 | \
+					 EN_LmBROADRV2 | EN_LmBROADRV1 | \
+					 EN_LmBROADRV0 | EN_LmBROADRVCH1 | \
+					 EN_LmBROADRVCH0 | EN_LmBROADCH | \
+					 EN_LmAIPRVWP | EN_LmAIPWP | \
+					 EN_LmAIPWD | EN_LmAIPWC | \
+					 EN_LmAIPRV2 | EN_LmAIPRV1 | \
+					 EN_LmAIPRV0 | EN_LmAIPNRML)
+
+#define LmPRIMSTAT1EN(LinkNum)		LmSEQ_PHY_REG(0, LinkNum, 0xE4)
+
+#define		EN_LmXRDY		0x00040000
+#define		EN_LmSYNCSRST		0x00020000
+#define		EN_LmSYNC		0x00010000
+#define 	EN_LmXHOLD		0x00008000
+#define 	EN_LmRRDY		0x00004000
+#define 	EN_LmHOLD		0x00002000
+#define 	EN_LmROK		0x00001000
+#define 	EN_LmRIP		0x00000800
+#define 	EN_LmCRBLK		0x00000400
+#define 	EN_LmACK		0x00000200
+#define 	EN_LmNAK		0x00000100
+#define 	EN_LmHARDRST		0x00000080
+#define 	EN_LmERROR		0x00000040
+#define 	EN_LmRERR		0x00000020
+#define 	EN_LmPMREQP		0x00000010
+#define 	EN_LmPMREQS		0x00000008
+#define 	EN_LmPMACK		0x00000004
+#define 	EN_LmPMNAK		0x00000002
+#define 	EN_LmDMAT		0x00000001
+
+#define LmPRIMSTAT1EN_MASK		(EN_LmHARDRST | \
+					 EN_LmSYNCSRST | \
+					 EN_LmPMREQP | EN_LmPMREQS | \
+					 EN_LmPMACK | EN_LmPMNAK)
+
+#define LmSMSTATE(LinkNum) 		LmSEQ_PHY_REG(0, LinkNum, 0xE8)
+
+#define LmSMSTATEBRK(LinkNum) 		LmSEQ_PHY_REG(0, LinkNum, 0xEC)
+
+#define LmSMDBGCTL(LinkNum) 		LmSEQ_PHY_REG(0, LinkNum, 0xF0)
+
+
+/*
+ * LmSEQ CIO Bus Mode 3 Register.
+ * Mode 3: Configuration and Setup, IOP Context SCB.
+ */
+#define LmM3SATATIMER(LinkNum) 		LmSEQ_PHY_REG(3, LinkNum, 0x48)
+
+#define LmM3INTVEC0(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0x90)
+
+#define LmM3INTVEC1(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0x92)
+
+#define LmM3INTVEC2(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0x94)
+
+#define LmM3INTVEC3(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0x96)
+
+#define LmM3INTVEC4(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0x98)
+
+#define LmM3INTVEC5(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0x9A)
+
+#define LmM3INTVEC6(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0x9C)
+
+#define LmM3INTVEC7(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0x9E)
+
+#define LmM3INTVEC8(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0xA4)
+
+#define LmM3INTVEC9(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0xA6)
+
+#define LmM3INTVEC10(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0xB0)
+
+#define LmM3FRMGAP(LinkNum)		LmSEQ_PHY_REG(3, LinkNum, 0xB4)
+
+#define LmBITL_TIMER(LinkNum) 		LmSEQ_PHY_REG(0, LinkNum, 0xA2)
+
+#define LmWWN(LinkNum) 			LmSEQ_PHY_REG(0, LinkNum, 0xA8)
+
+
+/*
+ * LmSEQ CIO Bus Mode 5 Registers.
+ * Mode 5: Phy/OOB Control and Status.
+ */
+#define LmSEQ_OOB_REG(phy_id, reg)	LmSEQ_PHY_REG(5, (phy_id), (reg))
+
+#define OOB_BFLTR	0x100
+
+#define		BFLTR_THR_MASK		0xF0
+#define		BFLTR_TC_MASK		0x0F
+
+#define OOB_INIT_MIN	0x102
+
+#define OOB_INIT_MAX	0x104
+
+#define OOB_INIT_NEG	0x106
+
+#define	OOB_SAS_MIN	0x108
+
+#define OOB_SAS_MAX	0x10A
+
+#define OOB_SAS_NEG	0x10C
+
+#define OOB_WAKE_MIN	0x10E
+
+#define OOB_WAKE_MAX	0x110
+
+#define OOB_WAKE_NEG	0x112
+
+#define OOB_IDLE_MAX	0x114
+
+#define OOB_BURST_MAX	0x116
+
+#define OOB_DATA_KBITS	0x126
+
+#define OOB_ALIGN_0_DATA	0x12C
+
+#define OOB_ALIGN_1_DATA	0x130
+
+#define D10_2_DATA_k		0x00
+#define SYNC_DATA_k		0x02
+#define ALIGN_1_DATA_k		0x04
+#define ALIGN_0_DATA_k		0x08
+#define BURST_DATA_k		0x10
+
+#define OOB_PHY_RESET_COUNT	0x13C
+
+#define OOB_SIG_GEN	0x140
+
+#define		START_OOB		0x80
+#define		START_DWS		0x40
+#define		ALIGN_CNT3		0x30
+#define 	ALIGN_CNT2		0x20
+#define 	ALIGN_CNT1		0x10
+#define 	ALIGN_CNT4		0x00
+#define		STOP_DWS		0x08
+#define		SEND_COMSAS		0x04
+#define		SEND_COMINIT		0x02
+#define		SEND_COMWAKE		0x01
+
+#define OOB_XMIT	0x141
+
+#define		TX_ENABLE		0x80
+#define		XMIT_OOB_BURST		0x10
+#define		XMIT_D10_2		0x08
+#define		XMIT_SYNC		0x04
+#define		XMIT_ALIGN_1		0x02
+#define		XMIT_ALIGN_0		0x01
+
+#define FUNCTION_MASK	0x142
+
+#define		SAS_MODE_DIS		0x80
+#define		SATA_MODE_DIS		0x40
+#define		SPINUP_HOLD_DIS		0x20
+#define		HOT_PLUG_DIS		0x10
+#define		SATA_PS_DIS		0x08
+#define		FUNCTION_MASK_DEFAULT	(SPINUP_HOLD_DIS | SATA_PS_DIS)
+
+#define OOB_MODE	0x143
+
+#define		SAS_MODE		0x80
+#define		SATA_MODE		0x40
+#define		SLOW_CLK		0x20
+#define		FORCE_XMIT_15		0x08
+#define		PHY_SPEED_60		0x04
+#define		PHY_SPEED_30		0x02
+#define		PHY_SPEED_15		0x01
+
+#define	CURRENT_STATUS	0x144
+
+#define		CURRENT_OOB_DONE	0x80
+#define		CURRENT_LOSS_OF_SIGNAL	0x40
+#define		CURRENT_SPINUP_HOLD	0x20
+#define		CURRENT_HOT_PLUG_CNCT	0x10
+#define		CURRENT_GTO_TIMEOUT	0x08
+#define		CURRENT_OOB_TIMEOUT	0x04
+#define		CURRENT_DEVICE_PRESENT	0x02
+#define		CURRENT_OOB_ERROR	0x01
+
+#define 	CURRENT_OOB1_ERROR	(CURRENT_HOT_PLUG_CNCT | \
+					 CURRENT_GTO_TIMEOUT)
+
+#define 	CURRENT_OOB2_ERROR	(CURRENT_HOT_PLUG_CNCT | \
+					 CURRENT_OOB_ERROR)
+
+#define		DEVICE_ADDED_W_CNT	(CURRENT_OOB_DONE | \
+					 CURRENT_HOT_PLUG_CNCT | \
+					 CURRENT_DEVICE_PRESENT)
+
+#define		DEVICE_ADDED_WO_CNT	(CURRENT_OOB_DONE | \
+					 CURRENT_DEVICE_PRESENT)
+
+#define 	DEVICE_REMOVED		CURRENT_LOSS_OF_SIGNAL
+
+#define		CURRENT_PHY_MASK	(CURRENT_OOB_DONE | \
+					 CURRENT_LOSS_OF_SIGNAL | \
+					 CURRENT_SPINUP_HOLD | \
+					 CURRENT_HOT_PLUG_CNCT | \
+					 CURRENT_GTO_TIMEOUT | \
+					 CURRENT_DEVICE_PRESENT | \
+					 CURRENT_OOB_ERROR )
+
+#define		CURRENT_ERR_MASK	(CURRENT_LOSS_OF_SIGNAL | \
+					 CURRENT_GTO_TIMEOUT | \
+					 CURRENT_OOB_TIMEOUT | \
+					 CURRENT_OOB_ERROR )
+
+#define SPEED_MASK	0x145
+
+#define		SATA_SPEED_30_DIS	0x10
+#define		SATA_SPEED_15_DIS	0x08
+#define		SAS_SPEED_60_DIS	0x04
+#define		SAS_SPEED_30_DIS	0x02
+#define		SAS_SPEED_15_DIS	0x01
+#define		SAS_SPEED_MASK_DEFAULT	0x00
+
+#define OOB_TIMER_ENABLE	0x14D
+
+#define		HOT_PLUG_EN		0x80
+#define		RCD_EN			0x40
+#define 	COMTIMER_EN		0x20
+#define		SNTT_EN			0x10
+#define		SNLT_EN			0x04
+#define		SNWT_EN			0x02
+#define		ALIGN_EN		0x01
+
+#define OOB_STATUS		0x14E
+
+#define		OOB_DONE		0x80
+#define		LOSS_OF_SIGNAL		0x40		/* ro */
+#define		SPINUP_HOLD		0x20
+#define		HOT_PLUG_CNCT		0x10		/* ro */
+#define		GTO_TIMEOUT		0x08		/* ro */
+#define		OOB_TIMEOUT		0x04		/* ro */
+#define		DEVICE_PRESENT		0x02		/* ro */
+#define		OOB_ERROR		0x01		/* ro */
+
+#define		OOB_STATUS_ERROR_MASK	(LOSS_OF_SIGNAL | GTO_TIMEOUT | \
+					 OOB_TIMEOUT | OOB_ERROR)
+
+#define OOB_STATUS_CLEAR	0x14F
+
+#define		OOB_DONE_CLR		0x80
+#define		LOSS_OF_SIGNAL_CLR 	0x40
+#define		SPINUP_HOLD_CLR		0x20
+#define		HOT_PLUG_CNCT_CLR     	0x10
+#define		GTO_TIMEOUT_CLR		0x08
+#define		OOB_TIMEOUT_CLR		0x04
+#define		OOB_ERROR_CLR		0x01
+
+#define HOT_PLUG_DELAY		0x150
+/* In 5 ms units. 20 = 100 ms. */
+#define	HOTPLUG_DELAY_TIMEOUT		20
+
+
+#define INT_ENABLE_2		0x15A
+
+#define		OOB_DONE_EN		0x80
+#define		LOSS_OF_SIGNAL_EN	0x40
+#define		SPINUP_HOLD_EN		0x20
+#define		HOT_PLUG_CNCT_EN	0x10
+#define		GTO_TIMEOUT_EN		0x08
+#define		OOB_TIMEOUT_EN		0x04
+#define		DEVICE_PRESENT_EN	0x02
+#define		OOB_ERROR_EN		0x01
+
+#define PHY_CONTROL_0		0x160
+
+#define		PHY_LOWPWREN_TX		0x80
+#define		PHY_LOWPWREN_RX		0x40
+#define		SPARE_REG_160_B5	0x20
+#define		OFFSET_CANCEL_RX	0x10
+
+/* bits 3:2 */
+#define		PHY_RXCOMCENTER_60V	0x00
+#define		PHY_RXCOMCENTER_70V	0x04
+#define		PHY_RXCOMCENTER_80V	0x08
+#define		PHY_RXCOMCENTER_90V	0x0C
+#define 	PHY_RXCOMCENTER_MASK	0x0C
+
+#define		PHY_RESET		0x02
+#define		SAS_DEFAULT_SEL		0x01
+
+#define PHY_CONTROL_1		0x161
+
+/* bits 2:0 */
+#define		SATA_PHY_DETLEVEL_50mv	0x00
+#define		SATA_PHY_DETLEVEL_75mv	0x01
+#define		SATA_PHY_DETLEVEL_100mv	0x02
+#define		SATA_PHY_DETLEVEL_125mv	0x03
+#define		SATA_PHY_DETLEVEL_150mv	0x04
+#define		SATA_PHY_DETLEVEL_175mv	0x05
+#define		SATA_PHY_DETLEVEL_200mv	0x06
+#define		SATA_PHY_DETLEVEL_225mv	0x07
+#define		SATA_PHY_DETLEVEL_MASK	0x07
+
+/* bits 5:3 */
+#define		SAS_PHY_DETLEVEL_50mv	0x00
+#define		SAS_PHY_DETLEVEL_75mv	0x08
+#define		SAS_PHY_DETLEVEL_100mv	0x10
+#define		SAS_PHY_DETLEVEL_125mv	0x11
+#define		SAS_PHY_DETLEVEL_150mv	0x20
+#define		SAS_PHY_DETLEVEL_175mv	0x21
+#define		SAS_PHY_DETLEVEL_200mv	0x30
+#define		SAS_PHY_DETLEVEL_225mv	0x31
+#define		SAS_PHY_DETLEVEL_MASK	0x38
+
+#define PHY_CONTROL_2		0x162
+
+/* bits 7:5 */
+#define 	SATA_PHY_DRV_400mv	0x00
+#define 	SATA_PHY_DRV_450mv	0x20
+#define 	SATA_PHY_DRV_500mv	0x40
+#define 	SATA_PHY_DRV_550mv	0x60
+#define 	SATA_PHY_DRV_600mv	0x80
+#define 	SATA_PHY_DRV_650mv	0xA0
+#define 	SATA_PHY_DRV_725mv	0xC0
+#define 	SATA_PHY_DRV_800mv	0xE0
+#define		SATA_PHY_DRV_MASK	0xE0
+
+/* bits 4:3 */
+#define 	SATA_PREEMP_0		0x00
+#define 	SATA_PREEMP_1		0x08
+#define 	SATA_PREEMP_2		0x10
+#define 	SATA_PREEMP_3		0x18
+#define 	SATA_PREEMP_MASK	0x18
+
+#define 	SATA_CMSH1P5		0x04
+
+/* bits 1:0 */
+#define 	SATA_SLEW_0		0x00
+#define 	SATA_SLEW_1		0x01
+#define 	SATA_SLEW_2		0x02
+#define 	SATA_SLEW_3		0x03
+#define 	SATA_SLEW_MASK		0x03
+
+#define PHY_CONTROL_3		0x163
+
+/* bits 7:5 */
+#define 	SAS_PHY_DRV_400mv	0x00
+#define 	SAS_PHY_DRV_450mv	0x20
+#define 	SAS_PHY_DRV_500mv	0x40
+#define 	SAS_PHY_DRV_550mv	0x60
+#define 	SAS_PHY_DRV_600mv	0x80
+#define 	SAS_PHY_DRV_650mv	0xA0
+#define 	SAS_PHY_DRV_725mv	0xC0
+#define 	SAS_PHY_DRV_800mv	0xE0
+#define		SAS_PHY_DRV_MASK	0xE0
+
+/* bits 4:3 */
+#define 	SAS_PREEMP_0		0x00
+#define 	SAS_PREEMP_1		0x08
+#define 	SAS_PREEMP_2		0x10
+#define 	SAS_PREEMP_3		0x18
+#define 	SAS_PREEMP_MASK		0x18
+
+#define 	SAS_CMSH1P5		0x04
+
+/* bits 1:0 */
+#define 	SAS_SLEW_0		0x00
+#define 	SAS_SLEW_1		0x01
+#define 	SAS_SLEW_2		0x02
+#define 	SAS_SLEW_3		0x03
+#define 	SAS_SLEW_MASK		0x03
+
+#define PHY_CONTROL_4		0x168
+
+#define		PHY_DONE_CAL_TX		0x80
+#define		PHY_DONE_CAL_RX		0x40
+#define		RX_TERM_LOAD_DIS	0x20
+#define		TX_TERM_LOAD_DIS	0x10
+#define		AUTO_TERM_CAL_DIS	0x08
+#define		PHY_SIGDET_FLTR_EN	0x04
+#define		OSC_FREQ		0x02
+#define		PHY_START_CAL		0x01
+
+/*
+ * HST_PCIX2 Registers, Addresss Range: (0x00-0xFC)
+ */
+#define PCIX_REG_BASE_ADR		0xB8040000
+
+#define PCIC_VENDOR_ID	0x00
+
+#define PCIC_DEVICE_ID	0x02
+
+#define PCIC_COMMAND	0x04
+
+#define		INT_DIS			0x0400
+#define		FBB_EN			0x0200		/* ro */
+#define		SERR_EN			0x0100
+#define		STEP_EN			0x0080		/* ro */
+#define		PERR_EN			0x0040
+#define		VGA_EN			0x0020		/* ro */
+#define		MWI_EN			0x0010
+#define		SPC_EN			0x0008
+#define		MST_EN			0x0004
+#define		MEM_EN			0x0002
+#define		IO_EN			0x0001
+
+#define	PCIC_STATUS	0x06
+
+#define		PERR_DET		0x8000
+#define		SERR_GEN		0x4000
+#define		MABT_DET		0x2000
+#define		TABT_DET		0x1000
+#define		TABT_GEN		0x0800
+#define		DPERR_DET		0x0100
+#define		CAP_LIST		0x0010
+#define		INT_STAT		0x0008
+
+#define	PCIC_DEVREV_ID	0x08
+
+#define	PCIC_CLASS_CODE	0x09
+
+#define	PCIC_CACHELINE_SIZE	0x0C
+
+#define	PCIC_MBAR0	0x10
+
+#define 	PCIC_MBAR0_OFFSET	0
+
+#define	PCIC_MBAR1	0x18
+
+#define 	PCIC_MBAR1_OFFSET	2
+
+#define	PCIC_IOBAR	0x20
+
+#define 	PCIC_IOBAR_OFFSET	4
+
+#define	PCIC_SUBVENDOR_ID	0x2C
+
+#define PCIC_SUBSYTEM_ID	0x2E
+
+#define PCIX_STATUS		0x44
+#define 	RCV_SCE		0x20000000
+#define 	UNEXP_SC	0x00080000
+#define 	SC_DISCARD	0x00040000
+
+#define ECC_CTRL_STAT		0x48
+#define 	UNCOR_ECCERR	0x00000008
+
+#define PCIC_PM_CSR		0x5C
+
+#define		PWR_STATE_D0		0
+#define		PWR_STATE_D1		1	/* not supported */
+#define		PWR_STATE_D2		2 	/* not supported */
+#define		PWR_STATE_D3		3
+
+#define PCIC_BASE1	0x6C	/* internal use only */
+
+#define		BASE1_RSVD		0xFFFFFFF8
+
+#define PCIC_BASEA	0x70	/* internal use only */
+
+#define		BASEA_RSVD		0xFFFFFFC0
+#define 	BASEA_START		0
+
+#define PCIC_BASEB	0x74	/* internal use only */
+
+#define		BASEB_RSVD		0xFFFFFF80
+#define		BASEB_IOMAP_MASK	0x7F
+#define 	BASEB_START		0x80
+
+#define PCIC_BASEC	0x78	/* internal use only */
+
+#define		BASEC_RSVD		0xFFFFFFFC
+#define 	BASEC_MASK		0x03
+#define 	BASEC_START		0x58
+
+#define PCIC_MBAR_KEY	0x7C	/* internal use only */
+
+#define 	MBAR_KEY_MASK		0xFFFFFFFF
+
+#define PCIC_HSTPCIX_CNTRL	0xA0
+
+#define 	REWIND_DIS		0x0800
+#define		SC_TMR_DIS		0x04000000
+
+#define PCIC_MBAR0_MASK	0xA8
+#define		PCIC_MBAR0_SIZE_MASK 	0x1FFFE000
+#define		PCIC_MBAR0_SIZE_SHIFT 	13
+#define		PCIC_MBAR0_SIZE(val)	\
+		    (((val) & PCIC_MBAR0_SIZE_MASK) >> PCIC_MBAR0_SIZE_SHIFT)
+
+#define PCIC_FLASH_MBAR	0xB8
+
+#define PCIC_INTRPT_STAT 0xD4
+
+#define PCIC_TP_CTRL	0xFC
+
+/*
+ * EXSI Registers, Addresss Range: (0x00-0xFC)
+ */
+#define EXSI_REG_BASE_ADR		REG_BASE_ADDR_EXSI
+
+#define	EXSICNFGR	(EXSI_REG_BASE_ADR + 0x00)
+
+#define		OCMINITIALIZED		0x80000000
+#define		ASIEN			0x00400000
+#define		HCMODE			0x00200000
+#define		PCIDEF			0x00100000
+#define		COMSTOCK		0x00080000
+#define		SEEPROMEND		0x00040000
+#define		MSTTIMEN		0x00020000
+#define		XREGEX			0x00000200
+#define		NVRAMW			0x00000100
+#define		NVRAMEX			0x00000080
+#define		SRAMW			0x00000040
+#define		SRAMEX			0x00000020
+#define		FLASHW			0x00000010
+#define		FLASHEX			0x00000008
+#define		SEEPROMCFG		0x00000004
+#define		SEEPROMTYP		0x00000002
+#define		SEEPROMEX		0x00000001
+
+
+#define EXSICNTRLR	(EXSI_REG_BASE_ADR + 0x04)
+
+#define		MODINT_EN		0x00000001
+
+
+#define PMSTATR		(EXSI_REG_BASE_ADR + 0x10)
+
+#define		FLASHRST		0x00000002
+#define		FLASHRDY		0x00000001
+
+
+#define FLCNFGR		(EXSI_REG_BASE_ADR + 0x14)
+
+#define		FLWEH_MASK		0x30000000
+#define		FLWESU_MASK		0x0C000000
+#define		FLWEPW_MASK		0x03F00000
+#define		FLOEH_MASK		0x000C0000
+#define 	FLOESU_MASK		0x00030000
+#define 	FLOEPW_MASK		0x0000FC00
+#define 	FLCSH_MASK		0x00000300
+#define 	FLCSSU_MASK		0x000000C0
+#define 	FLCSPW_MASK		0x0000003F
+
+#define SRCNFGR		(EXSI_REG_BASE_ADR + 0x18)
+
+#define		SRWEH_MASK		0x30000000
+#define		SRWESU_MASK		0x0C000000
+#define		SRWEPW_MASK		0x03F00000
+
+#define		SROEH_MASK		0x000C0000
+#define 	SROESU_MASK		0x00030000
+#define 	SROEPW_MASK		0x0000FC00
+#define		SRCSH_MASK		0x00000300
+#define		SRCSSU_MASK		0x000000C0
+#define		SRCSPW_MASK		0x0000003F
+
+#define NVCNFGR		(EXSI_REG_BASE_ADR + 0x1C)
+
+#define 	NVWEH_MASK		0x30000000
+#define 	NVWESU_MASK		0x0C000000
+#define 	NVWEPW_MASK		0x03F00000
+#define 	NVOEH_MASK		0x000C0000
+#define 	NVOESU_MASK		0x00030000
+#define 	NVOEPW_MASK		0x0000FC00
+#define 	NVCSH_MASK		0x00000300
+#define 	NVCSSU_MASK		0x000000C0
+#define 	NVCSPW_MASK		0x0000003F
+
+#define XRCNFGR		(EXSI_REG_BASE_ADR + 0x20)
+
+#define 	XRWEH_MASK		0x30000000
+#define 	XRWESU_MASK		0x0C000000
+#define 	XRWEPW_MASK		0x03F00000
+#define 	XROEH_MASK		0x000C0000
+#define 	XROESU_MASK		0x00030000
+#define 	XROEPW_MASK		0x0000FC00
+#define 	XRCSH_MASK		0x00000300
+#define 	XRCSSU_MASK		0x000000C0
+#define		XRCSPW_MASK		0x0000003F
+
+#define XREGADDR	(EXSI_REG_BASE_ADR + 0x24)
+
+#define 	XRADDRINCEN		0x80000000
+#define 	XREGADD_MASK		0x007FFFFF
+
+
+#define XREGDATAR	(EXSI_REG_BASE_ADR + 0x28)
+
+#define		XREGDATA_MASK 		0x0000FFFF
+
+#define GPIOOER		(EXSI_REG_BASE_ADR + 0x40)
+
+#define GPIOODENR	(EXSI_REG_BASE_ADR + 0x44)
+
+#define GPIOINVR	(EXSI_REG_BASE_ADR + 0x48)
+
+#define GPIODATAOR	(EXSI_REG_BASE_ADR + 0x4C)
+
+#define GPIODATAIR	(EXSI_REG_BASE_ADR + 0x50)
+
+#define GPIOCNFGR	(EXSI_REG_BASE_ADR + 0x54)
+
+#define		GPIO_EXTSRC		0x00000001
+
+#define SCNTRLR		(EXSI_REG_BASE_ADR + 0xA0)
+
+#define 	SXFERDONE		0x00000100
+#define 	SXFERCNT_MASK		0x000000E0
+#define 	SCMDTYP_MASK		0x0000001C
+#define 	SXFERSTART		0x00000002
+#define 	SXFEREN			0x00000001
+
+#define	SRATER		(EXSI_REG_BASE_ADR + 0xA4)
+
+#define	SADDRR		(EXSI_REG_BASE_ADR + 0xA8)
+
+#define 	SADDR_MASK		0x0000FFFF
+
+#define SDATAOR		(EXSI_REG_BASE_ADR + 0xAC)
+
+#define	SDATAOR0	(EXSI_REG_BASE_ADR + 0xAC)
+#define SDATAOR1	(EXSI_REG_BASE_ADR + 0xAD)
+#define SDATAOR2	(EXSI_REG_BASE_ADR + 0xAE)
+#define SDATAOR3	(EXSI_REG_BASE_ADR + 0xAF)
+
+#define SDATAIR		(EXSI_REG_BASE_ADR + 0xB0)
+
+#define SDATAIR0	(EXSI_REG_BASE_ADR + 0xB0)
+#define SDATAIR1	(EXSI_REG_BASE_ADR + 0xB1)
+#define SDATAIR2	(EXSI_REG_BASE_ADR + 0xB2)
+#define SDATAIR3	(EXSI_REG_BASE_ADR + 0xB3)
+
+#define ASISTAT0R	(EXSI_REG_BASE_ADR + 0xD0)
+#define 	ASIFMTERR		0x00000400
+#define 	ASISEECHKERR		0x00000200
+#define 	ASIERR			0x00000100
+
+#define ASISTAT1R	(EXSI_REG_BASE_ADR + 0xD4)
+#define 	CHECKSUM_MASK		0x0000FFFF
+
+#define ASIERRADDR	(EXSI_REG_BASE_ADR + 0xD8)
+#define ASIERRDATAR	(EXSI_REG_BASE_ADR + 0xDC)
+#define ASIERRSTATR	(EXSI_REG_BASE_ADR + 0xE0)
+#define 	CPI2ASIBYTECNT_MASK	0x00070000
+#define 	CPI2ASIBYTEEN_MASK      0x0000F000
+#define 	CPI2ASITARGERR_MASK	0x00000F00
+#define 	CPI2ASITARGMID_MASK	0x000000F0
+#define 	CPI2ASIMSTERR_MASK	0x0000000F
+
+/*
+ * XSRAM, External SRAM (DWord and any BE pattern accessible)
+ */
+#define XSRAM_REG_BASE_ADDR             0xB8100000
+#define XSRAM_SIZE                        0x100000
+
+/*
+ * NVRAM Registers, Address Range: (0x00000 - 0x3FFFF).
+ */
+#define		NVRAM_REG_BASE_ADR	0xBF800000
+#define		NVRAM_MAX_BASE_ADR	0x003FFFFF
+
+/* OCM base address */
+#define		OCM_BASE_ADDR		0xA0000000
+#define		OCM_MAX_SIZE		0x20000
+
+/*
+ * Sequencers (Central and Link) Scratch RAM page definitions.
+ */
+
+/*
+ * The Central Management Sequencer (CSEQ) Scratch Memory is a 1024
+ * byte memory.  It is dword accessible and has byte parity
+ * protection. The CSEQ accesses it in 32 byte windows, either as mode
+ * dependent or mode independent memory. Each mode has 96 bytes,
+ * (three 32 byte pages 0-2, not contiguous), leaving 128 bytes of
+ * Mode Independent memory (four 32 byte pages 3-7). Note that mode
+ * dependent scratch memory, Mode 8, page 0-3 overlaps mode
+ * independent scratch memory, pages 0-3.
+ * - 896 bytes of mode dependent scratch, 96 bytes per Modes 0-7, and
+ * 128 bytes in mode 8,
+ * - 259 bytes of mode independent scratch, common to modes 0-15.
+ *
+ * Sequencer scratch RAM is 1024 bytes.  This scratch memory is
+ * divided into mode dependent and mode independent scratch with this
+ * memory further subdivided into pages of size 32 bytes. There are 5
+ * pages (160 bytes) of mode independent scratch and 3 pages of
+ * dependent scratch memory for modes 0-7 (768 bytes). Mode 8 pages
+ * 0-2 dependent scratch overlap with pages 0-2 of mode independent
+ * scratch memory.
+ *
+ * The host accesses this scratch in a different manner from the
+ * central sequencer. The sequencer has to use CSEQ registers CSCRPAGE
+ * and CMnSCRPAGE to access the scratch memory. A flat mapping of the
+ * scratch memory is avaliable for software convenience and to prevent
+ * corruption while the sequencer is running. This memory is mapped
+ * onto addresses 800h - BFFh, total of 400h bytes.
+ *
+ * These addresses are mapped as follows:
+ *
+ *        800h-83Fh   Mode Dependent Scratch Mode 0 Pages 0-1
+ *        840h-87Fh   Mode Dependent Scratch Mode 1 Pages 0-1
+ *        880h-8BFh   Mode Dependent Scratch Mode 2 Pages 0-1
+ *        8C0h-8FFh   Mode Dependent Scratch Mode 3 Pages 0-1
+ *        900h-93Fh   Mode Dependent Scratch Mode 4 Pages 0-1
+ *        940h-97Fh   Mode Dependent Scratch Mode 5 Pages 0-1
+ *        980h-9BFh   Mode Dependent Scratch Mode 6 Pages 0-1
+ *        9C0h-9FFh   Mode Dependent Scratch Mode 7 Pages 0-1
+ *        A00h-A5Fh   Mode Dependent Scratch Mode 8 Pages 0-2
+ *                    Mode Independent Scratch Pages 0-2
+ *        A60h-A7Fh   Mode Dependent Scratch Mode 8 Page 3
+ *                    Mode Independent Scratch Page 3
+ *        A80h-AFFh   Mode Independent Scratch Pages 4-7
+ *        B00h-B1Fh   Mode Dependent Scratch Mode 0 Page 2
+ *        B20h-B3Fh   Mode Dependent Scratch Mode 1 Page 2
+ *        B40h-B5Fh   Mode Dependent Scratch Mode 2 Page 2
+ *        B60h-B7Fh   Mode Dependent Scratch Mode 3 Page 2
+ *        B80h-B9Fh   Mode Dependent Scratch Mode 4 Page 2
+ *        BA0h-BBFh   Mode Dependent Scratch Mode 5 Page 2
+ *        BC0h-BDFh   Mode Dependent Scratch Mode 6 Page 2
+ *        BE0h-BFFh   Mode Dependent Scratch Mode 7 Page 2
+ */
+
+/* General macros */
+#define CSEQ_PAGE_SIZE			32  /* Scratch page size (in bytes) */
+
+/* All macros start with offsets from base + 0x800 (CMAPPEDSCR).
+ * Mode dependent scratch page 0, mode 0.
+ * For modes 1-7 you have to do arithmetic. */
+#define CSEQ_LRM_SAVE_SINDEX		(CMAPPEDSCR + 0x0000)
+#define CSEQ_LRM_SAVE_SCBPTR		(CMAPPEDSCR + 0x0002)
+#define CSEQ_Q_LINK_HEAD		(CMAPPEDSCR + 0x0004)
+#define CSEQ_Q_LINK_TAIL		(CMAPPEDSCR + 0x0006)
+#define CSEQ_LRM_SAVE_SCRPAGE		(CMAPPEDSCR + 0x0008)
+
+/* Mode dependent scratch page 0 mode 8 macros. */
+#define CSEQ_RET_ADDR			(CMAPPEDSCR + 0x0200)
+#define CSEQ_RET_SCBPTR			(CMAPPEDSCR + 0x0202)
+#define CSEQ_SAVE_SCBPTR		(CMAPPEDSCR + 0x0204)
+#define CSEQ_EMPTY_TRANS_CTX		(CMAPPEDSCR + 0x0206)
+#define CSEQ_RESP_LEN			(CMAPPEDSCR + 0x0208)
+#define CSEQ_TMF_SCBPTR			(CMAPPEDSCR + 0x020A)
+#define CSEQ_GLOBAL_PREV_SCB		(CMAPPEDSCR + 0x020C)
+#define CSEQ_GLOBAL_HEAD		(CMAPPEDSCR + 0x020E)
+#define CSEQ_CLEAR_LU_HEAD		(CMAPPEDSCR + 0x0210)
+#define CSEQ_TMF_OPCODE			(CMAPPEDSCR + 0x0212)
+#define CSEQ_SCRATCH_FLAGS		(CMAPPEDSCR + 0x0213)
+#define CSEQ_HSB_SITE                   (CMAPPEDSCR + 0x021A)
+#define CSEQ_FIRST_INV_SCB_SITE		(CMAPPEDSCR + 0x021C)
+#define CSEQ_FIRST_INV_DDB_SITE		(CMAPPEDSCR + 0x021E)
+
+/* Mode dependent scratch page 1 mode 8 macros. */
+#define CSEQ_LUN_TO_CLEAR		(CMAPPEDSCR + 0x0220)
+#define CSEQ_LUN_TO_CHECK		(CMAPPEDSCR + 0x0228)
+
+/* Mode dependent scratch page 2 mode 8 macros */
+#define CSEQ_HQ_NEW_POINTER		(CMAPPEDSCR + 0x0240)
+#define CSEQ_HQ_DONE_BASE		(CMAPPEDSCR + 0x0248)
+#define CSEQ_HQ_DONE_POINTER		(CMAPPEDSCR + 0x0250)
+#define CSEQ_HQ_DONE_PASS		(CMAPPEDSCR + 0x0254)
+
+/* Mode independent scratch page 4 macros. */
+#define CSEQ_Q_EXE_HEAD			(CMAPPEDSCR + 0x0280)
+#define CSEQ_Q_EXE_TAIL			(CMAPPEDSCR + 0x0282)
+#define CSEQ_Q_DONE_HEAD                (CMAPPEDSCR + 0x0284)
+#define CSEQ_Q_DONE_TAIL                (CMAPPEDSCR + 0x0286)
+#define CSEQ_Q_SEND_HEAD		(CMAPPEDSCR + 0x0288)
+#define CSEQ_Q_SEND_TAIL		(CMAPPEDSCR + 0x028A)
+#define CSEQ_Q_DMA2CHIM_HEAD		(CMAPPEDSCR + 0x028C)
+#define CSEQ_Q_DMA2CHIM_TAIL		(CMAPPEDSCR + 0x028E)
+#define CSEQ_Q_COPY_HEAD		(CMAPPEDSCR + 0x0290)
+#define CSEQ_Q_COPY_TAIL		(CMAPPEDSCR + 0x0292)
+#define CSEQ_REG0			(CMAPPEDSCR + 0x0294)
+#define CSEQ_REG1			(CMAPPEDSCR + 0x0296)
+#define CSEQ_REG2			(CMAPPEDSCR + 0x0298)
+#define CSEQ_LINK_CTL_Q_MAP		(CMAPPEDSCR + 0x029C)
+#define CSEQ_MAX_CSEQ_MODE		(CMAPPEDSCR + 0x029D)
+#define CSEQ_FREE_LIST_HACK_COUNT	(CMAPPEDSCR + 0x029E)
+
+/* Mode independent scratch page 5 macros. */
+#define CSEQ_EST_NEXUS_REQ_QUEUE	(CMAPPEDSCR + 0x02A0)
+#define CSEQ_EST_NEXUS_REQ_COUNT	(CMAPPEDSCR + 0x02A8)
+#define CSEQ_Q_EST_NEXUS_HEAD		(CMAPPEDSCR + 0x02B0)
+#define CSEQ_Q_EST_NEXUS_TAIL		(CMAPPEDSCR + 0x02B2)
+#define CSEQ_NEED_EST_NEXUS_SCB		(CMAPPEDSCR + 0x02B4)
+#define CSEQ_EST_NEXUS_REQ_HEAD		(CMAPPEDSCR + 0x02B6)
+#define CSEQ_EST_NEXUS_REQ_TAIL		(CMAPPEDSCR + 0x02B7)
+#define CSEQ_EST_NEXUS_SCB_OFFSET	(CMAPPEDSCR + 0x02B8)
+
+/* Mode independent scratch page 6 macros. */
+#define CSEQ_INT_ROUT_RET_ADDR0		(CMAPPEDSCR + 0x02C0)
+#define CSEQ_INT_ROUT_RET_ADDR1		(CMAPPEDSCR + 0x02C2)
+#define CSEQ_INT_ROUT_SCBPTR		(CMAPPEDSCR + 0x02C4)
+#define CSEQ_INT_ROUT_MODE		(CMAPPEDSCR + 0x02C6)
+#define CSEQ_ISR_SCRATCH_FLAGS		(CMAPPEDSCR + 0x02C7)
+#define CSEQ_ISR_SAVE_SINDEX		(CMAPPEDSCR + 0x02C8)
+#define CSEQ_ISR_SAVE_DINDEX		(CMAPPEDSCR + 0x02CA)
+#define CSEQ_Q_MONIRTT_HEAD		(CMAPPEDSCR + 0x02D0)
+#define CSEQ_Q_MONIRTT_TAIL		(CMAPPEDSCR + 0x02D2)
+#define CSEQ_FREE_SCB_MASK		(CMAPPEDSCR + 0x02D5)
+#define CSEQ_BUILTIN_FREE_SCB_HEAD	(CMAPPEDSCR + 0x02D6)
+#define CSEQ_BUILTIN_FREE_SCB_TAIL	(CMAPPEDSCR + 0x02D8)
+#define CSEQ_EXTENDED_FREE_SCB_HEAD	(CMAPPEDSCR + 0x02DA)
+#define CSEQ_EXTENDED_FREE_SCB_TAIL	(CMAPPEDSCR + 0x02DC)
+
+/* Mode independent scratch page 7 macros. */
+#define CSEQ_EMPTY_REQ_QUEUE		(CMAPPEDSCR + 0x02E0)
+#define CSEQ_EMPTY_REQ_COUNT		(CMAPPEDSCR + 0x02E8)
+#define CSEQ_Q_EMPTY_HEAD		(CMAPPEDSCR + 0x02F0)
+#define CSEQ_Q_EMPTY_TAIL		(CMAPPEDSCR + 0x02F2)
+#define CSEQ_NEED_EMPTY_SCB		(CMAPPEDSCR + 0x02F4)
+#define CSEQ_EMPTY_REQ_HEAD		(CMAPPEDSCR + 0x02F6)
+#define CSEQ_EMPTY_REQ_TAIL		(CMAPPEDSCR + 0x02F7)
+#define CSEQ_EMPTY_SCB_OFFSET		(CMAPPEDSCR + 0x02F8)
+#define CSEQ_PRIMITIVE_DATA		(CMAPPEDSCR + 0x02FA)
+#define CSEQ_TIMEOUT_CONST		(CMAPPEDSCR + 0x02FC)
+
+/***************************************************************************
+* Link m Sequencer scratch RAM is 512 bytes.
+* This scratch memory is divided into mode dependent and mode
+* independent scratch with this memory further subdivided into
+* pages of size 32 bytes. There are 4 pages (128 bytes) of
+* mode independent scratch and 4 pages of dependent scratch
+* memory for modes 0-2 (384 bytes).
+*
+* The host accesses this scratch in a different manner from the
+* link sequencer. The sequencer has to use LSEQ registers
+* LmSCRPAGE and LmMnSCRPAGE to access the scratch memory. A flat
+* mapping of the scratch memory is avaliable for software
+* convenience and to prevent corruption while the sequencer is
+* running. This memory is mapped onto addresses 800h - 9FFh.
+*
+* These addresses are mapped as follows:
+*
+*        800h-85Fh   Mode Dependent Scratch Mode 0 Pages 0-2
+*        860h-87Fh   Mode Dependent Scratch Mode 0 Page 3
+*                    Mode Dependent Scratch Mode 5 Page 0
+*        880h-8DFh   Mode Dependent Scratch Mode 1 Pages 0-2
+*        8E0h-8FFh   Mode Dependent Scratch Mode 1 Page 3
+*                    Mode Dependent Scratch Mode 5 Page 1
+*        900h-95Fh   Mode Dependent Scratch Mode 2 Pages 0-2
+*        960h-97Fh   Mode Dependent Scratch Mode 2 Page 3
+*                    Mode Dependent Scratch Mode 5 Page 2
+*        980h-9DFh   Mode Independent Scratch Pages 0-3
+*        9E0h-9FFh   Mode Independent Scratch Page 3
+*                    Mode Dependent Scratch Mode 5 Page 3
+*
+****************************************************************************/
+/* General macros */
+#define LSEQ_MODE_SCRATCH_SIZE		0x80 /* Size of scratch RAM per mode */
+#define LSEQ_PAGE_SIZE			0x20 /* Scratch page size (in bytes) */
+#define LSEQ_MODE5_PAGE0_OFFSET 	0x60
+
+/* Common mode dependent scratch page 0 macros for modes 0,1,2, and 5 */
+/* Indexed using LSEQ_MODE_SCRATCH_SIZE * mode, for modes 0,1,2. */
+#define LmSEQ_RET_ADDR(LinkNum)		(LmSCRATCH(LinkNum) + 0x0000)
+#define LmSEQ_REG0_MODE(LinkNum)	(LmSCRATCH(LinkNum) + 0x0002)
+#define LmSEQ_MODE_FLAGS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0004)
+
+/* Mode flag macros (byte 0) */
+#define		SAS_SAVECTX_OCCURRED		0x80
+#define		SAS_OOBSVC_OCCURRED		0x40
+#define		SAS_OOB_DEVICE_PRESENT		0x20
+#define		SAS_CFGHDR_OCCURRED		0x10
+#define		SAS_RCV_INTS_ARE_DISABLED	0x08
+#define		SAS_OOB_HOT_PLUG_CNCT		0x04
+#define		SAS_AWAIT_OPEN_CONNECTION	0x02
+#define		SAS_CFGCMPLT_OCCURRED		0x01
+
+/* Mode flag macros (byte 1) */
+#define		SAS_RLSSCB_OCCURRED		0x80
+#define		SAS_FORCED_HEADER_MISS		0x40
+
+#define LmSEQ_RET_ADDR2(LinkNum)	(LmSCRATCH(LinkNum) + 0x0006)
+#define LmSEQ_RET_ADDR1(LinkNum)	(LmSCRATCH(LinkNum) + 0x0008)
+#define LmSEQ_OPCODE_TO_CSEQ(LinkNum)	(LmSCRATCH(LinkNum) + 0x000B)
+#define LmSEQ_DATA_TO_CSEQ(LinkNum)	(LmSCRATCH(LinkNum) + 0x000C)
+
+/* Mode dependent scratch page 0 macros for mode 0 (non-common) */
+/* Absolute offsets */
+#define LmSEQ_FIRST_INV_DDB_SITE(LinkNum)	(LmSCRATCH(LinkNum) + 0x000E)
+#define LmSEQ_EMPTY_TRANS_CTX(LinkNum)		(LmSCRATCH(LinkNum) + 0x0010)
+#define LmSEQ_RESP_LEN(LinkNum)			(LmSCRATCH(LinkNum) + 0x0012)
+#define LmSEQ_FIRST_INV_SCB_SITE(LinkNum)	(LmSCRATCH(LinkNum) + 0x0014)
+#define LmSEQ_INTEN_SAVE(LinkNum)		(LmSCRATCH(LinkNum) + 0x0016)
+#define LmSEQ_LINK_RST_FRM_LEN(LinkNum)		(LmSCRATCH(LinkNum) + 0x001A)
+#define LmSEQ_LINK_RST_PROTOCOL(LinkNum)	(LmSCRATCH(LinkNum) + 0x001B)
+#define LmSEQ_RESP_STATUS(LinkNum)		(LmSCRATCH(LinkNum) + 0x001C)
+#define LmSEQ_LAST_LOADED_SGE(LinkNum)		(LmSCRATCH(LinkNum) + 0x001D)
+#define LmSEQ_SAVE_SCBPTR(LinkNum)		(LmSCRATCH(LinkNum) + 0x001E)
+
+/* Mode dependent scratch page 0 macros for mode 1 (non-common) */
+/* Absolute offsets */
+#define LmSEQ_Q_XMIT_HEAD(LinkNum)		(LmSCRATCH(LinkNum) + 0x008E)
+#define LmSEQ_M1_EMPTY_TRANS_CTX(LinkNum)	(LmSCRATCH(LinkNum) + 0x0090)
+#define LmSEQ_INI_CONN_TAG(LinkNum)		(LmSCRATCH(LinkNum) + 0x0092)
+#define LmSEQ_FAILED_OPEN_STATUS(LinkNum)	(LmSCRATCH(LinkNum) + 0x009A)
+#define LmSEQ_XMIT_REQUEST_TYPE(LinkNum)	(LmSCRATCH(LinkNum) + 0x009B)
+#define LmSEQ_M1_RESP_STATUS(LinkNum)		(LmSCRATCH(LinkNum) + 0x009C)
+#define LmSEQ_M1_LAST_LOADED_SGE(LinkNum)	(LmSCRATCH(LinkNum) + 0x009D)
+#define LmSEQ_M1_SAVE_SCBPTR(LinkNum)		(LmSCRATCH(LinkNum) + 0x009E)
+
+/* Mode dependent scratch page 0 macros for mode 2 (non-common) */
+#define LmSEQ_PORT_COUNTER(LinkNum)		(LmSCRATCH(LinkNum) + 0x010E)
+#define LmSEQ_PM_TABLE_PTR(LinkNum)		(LmSCRATCH(LinkNum) + 0x0110)
+#define LmSEQ_SATA_INTERLOCK_TMR_SAVE(LinkNum)	(LmSCRATCH(LinkNum) + 0x0112)
+#define LmSEQ_IP_BITL(LinkNum)			(LmSCRATCH(LinkNum) + 0x0114)
+#define LmSEQ_COPY_SMP_CONN_TAG(LinkNum)	(LmSCRATCH(LinkNum) + 0x0116)
+#define LmSEQ_P0M2_OFFS1AH(LinkNum)		(LmSCRATCH(LinkNum) + 0x011A)
+
+/* Mode dependent scratch page 0 macros for modes 4/5 (non-common) */
+/* Absolute offsets */
+#define LmSEQ_SAVED_OOB_STATUS(LinkNum)		(LmSCRATCH(LinkNum) + 0x006E)
+#define LmSEQ_SAVED_OOB_MODE(LinkNum)		(LmSCRATCH(LinkNum) + 0x006F)
+#define LmSEQ_Q_LINK_HEAD(LinkNum)		(LmSCRATCH(LinkNum) + 0x0070)
+#define LmSEQ_LINK_RST_ERR(LinkNum)		(LmSCRATCH(LinkNum) + 0x0072)
+#define LmSEQ_SAVED_OOB_SIGNALS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0073)
+#define LmSEQ_SAS_RESET_MODE(LinkNum)		(LmSCRATCH(LinkNum) + 0x0074)
+#define LmSEQ_LINK_RESET_RETRY_COUNT(LinkNum)	(LmSCRATCH(LinkNum) + 0x0075)
+#define LmSEQ_NUM_LINK_RESET_RETRIES(LinkNum)	(LmSCRATCH(LinkNum) + 0x0076)
+#define LmSEQ_OOB_INT_ENABLES(LinkNum)		(LmSCRATCH(LinkNum) + 0x007A)
+#define LmSEQ_NOTIFY_TIMER_TIMEOUT(LinkNum)	(LmSCRATCH(LinkNum) + 0x007C)
+#define LmSEQ_NOTIFY_TIMER_DOWN_COUNT(LinkNum)	(LmSCRATCH(LinkNum) + 0x007E)
+
+/* Mode dependent scratch page 1, mode 0 and mode 1 */
+#define LmSEQ_SG_LIST_PTR_ADDR0(LinkNum)        (LmSCRATCH(LinkNum) + 0x0020)
+#define LmSEQ_SG_LIST_PTR_ADDR1(LinkNum)        (LmSCRATCH(LinkNum) + 0x0030)
+#define LmSEQ_M1_SG_LIST_PTR_ADDR0(LinkNum)     (LmSCRATCH(LinkNum) + 0x00A0)
+#define LmSEQ_M1_SG_LIST_PTR_ADDR1(LinkNum)     (LmSCRATCH(LinkNum) + 0x00B0)
+
+/* Mode dependent scratch page 1 macros for mode 2 */
+/* Absolute offsets */
+#define LmSEQ_INVALID_DWORD_COUNT(LinkNum)	(LmSCRATCH(LinkNum) + 0x0120)
+#define LmSEQ_DISPARITY_ERROR_COUNT(LinkNum) 	(LmSCRATCH(LinkNum) + 0x0124)
+#define LmSEQ_LOSS_OF_SYNC_COUNT(LinkNum)	(LmSCRATCH(LinkNum) + 0x0128)
+
+/* Mode dependent scratch page 1 macros for mode 4/5 */
+#define LmSEQ_FRAME_TYPE_MASK(LinkNum)	      (LmSCRATCH(LinkNum) + 0x00E0)
+#define LmSEQ_HASHED_DEST_ADDR_MASK(LinkNum)  (LmSCRATCH(LinkNum) + 0x00E1)
+#define LmSEQ_HASHED_SRC_ADDR_MASK_PRINT(LinkNum) (LmSCRATCH(LinkNum) + 0x00E4)
+#define LmSEQ_HASHED_SRC_ADDR_MASK(LinkNum)   (LmSCRATCH(LinkNum) + 0x00E5)
+#define LmSEQ_NUM_FILL_BYTES_MASK(LinkNum)    (LmSCRATCH(LinkNum) + 0x00EB)
+#define LmSEQ_TAG_MASK(LinkNum)		      (LmSCRATCH(LinkNum) + 0x00F0)
+#define LmSEQ_TARGET_PORT_XFER_TAG(LinkNum)   (LmSCRATCH(LinkNum) + 0x00F2)
+#define LmSEQ_DATA_OFFSET(LinkNum)	      (LmSCRATCH(LinkNum) + 0x00F4)
+
+/* Mode dependent scratch page 2 macros for mode 0 */
+/* Absolute offsets */
+#define LmSEQ_SMP_RCV_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0040)
+#define LmSEQ_DEVICE_BITS(LinkNum)		(LmSCRATCH(LinkNum) + 0x005B)
+#define LmSEQ_SDB_DDB(LinkNum)			(LmSCRATCH(LinkNum) + 0x005C)
+#define LmSEQ_SDB_NUM_TAGS(LinkNum)		(LmSCRATCH(LinkNum) + 0x005E)
+#define LmSEQ_SDB_CURR_TAG(LinkNum)		(LmSCRATCH(LinkNum) + 0x005F)
+
+/* Mode dependent scratch page 2 macros for mode 1 */
+/* Absolute offsets */
+/* byte 0 bits 1-0 are domain select. */
+#define LmSEQ_TX_ID_ADDR_FRAME(LinkNum)		(LmSCRATCH(LinkNum) + 0x00C0)
+#define LmSEQ_OPEN_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x00C8)
+#define LmSEQ_SRST_AS_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x00CC)
+#define LmSEQ_LAST_LOADED_SG_EL(LinkNum)	(LmSCRATCH(LinkNum) + 0x00D4)
+
+/* Mode dependent scratch page 2 macros for mode 2 */
+/* Absolute offsets */
+#define LmSEQ_STP_SHUTDOWN_TIMER_TERM_TS(LinkNum) (LmSCRATCH(LinkNum) + 0x0140)
+#define LmSEQ_CLOSE_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0144)
+#define LmSEQ_BREAK_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0148)
+#define LmSEQ_DWS_RESET_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x014C)
+#define LmSEQ_SATA_INTERLOCK_TIMER_TERM_TS(LinkNum) \
+						(LmSCRATCH(LinkNum) + 0x0150)
+#define LmSEQ_MCTL_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0154)
+
+/* Mode dependent scratch page 2 macros for mode 5 */
+#define LmSEQ_COMINIT_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0160)
+#define LmSEQ_RCV_ID_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0164)
+#define LmSEQ_RCV_FIS_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0168)
+#define LmSEQ_DEV_PRES_TIMER_TERM_TS(LinkNum)	(LmSCRATCH(LinkNum) + 0x016C)
+
+/* Mode dependent scratch page 3 macros for modes 0 and 1 */
+/* None defined */
+
+/* Mode dependent scratch page 3 macros for modes 2 and 5 */
+/* None defined */
+
+/* Mode Independent Scratch page 0 macros. */
+#define LmSEQ_Q_TGTXFR_HEAD(LinkNum)	(LmSCRATCH(LinkNum) + 0x0180)
+#define LmSEQ_Q_TGTXFR_TAIL(LinkNum)	(LmSCRATCH(LinkNum) + 0x0182)
+#define LmSEQ_LINK_NUMBER(LinkNum)	(LmSCRATCH(LinkNum) + 0x0186)
+#define LmSEQ_SCRATCH_FLAGS(LinkNum)	(LmSCRATCH(LinkNum) + 0x0187)
+/*
+ * Currently only bit 0, SAS_DWSAQD, is used.
+ */
+#define		SAS_DWSAQD			0x01  /*
+						       * DWSSTATUS: DWSAQD
+						       * bit las read in ISR.
+						       */
+#define  LmSEQ_CONNECTION_STATE(LinkNum) (LmSCRATCH(LinkNum) + 0x0188)
+/* Connection states (byte 0) */
+#define		SAS_WE_OPENED_CS		0x01
+#define		SAS_DEVICE_OPENED_CS		0x02
+#define		SAS_WE_SENT_DONE_CS		0x04
+#define		SAS_DEVICE_SENT_DONE_CS		0x08
+#define		SAS_WE_SENT_CLOSE_CS		0x10
+#define		SAS_DEVICE_SENT_CLOSE_CS	0x20
+#define		SAS_WE_SENT_BREAK_CS		0x40
+#define		SAS_DEVICE_SENT_BREAK_CS	0x80
+/* Connection states (byte 1) */
+#define		SAS_OPN_TIMEOUT_OR_OPN_RJCT_CS	0x01
+#define		SAS_AIP_RECEIVED_CS		0x02
+#define		SAS_CREDIT_TIMEOUT_OCCURRED_CS	0x04
+#define		SAS_ACKNAK_TIMEOUT_OCCURRED_CS	0x08
+#define		SAS_SMPRSP_TIMEOUT_OCCURRED_CS	0x10
+#define		SAS_DONE_TIMEOUT_OCCURRED_CS	0x20
+/* Connection states (byte 2) */
+#define		SAS_SMP_RESPONSE_RECEIVED_CS	0x01
+#define		SAS_INTLK_TIMEOUT_OCCURRED_CS	0x02
+#define		SAS_DEVICE_SENT_DMAT_CS		0x04
+#define		SAS_DEVICE_SENT_SYNCSRST_CS	0x08
+#define		SAS_CLEARING_AFFILIATION_CS	0x20
+#define		SAS_RXTASK_ACTIVE_CS		0x40
+#define		SAS_TXTASK_ACTIVE_CS		0x80
+/* Connection states (byte 3) */
+#define		SAS_PHY_LOSS_OF_SIGNAL_CS	0x01
+#define		SAS_DWS_TIMER_EXPIRED_CS	0x02
+#define		SAS_LINK_RESET_NOT_COMPLETE_CS	0x04
+#define		SAS_PHY_DISABLED_CS		0x08
+#define		SAS_LINK_CTL_TASK_ACTIVE_CS	0x10
+#define		SAS_PHY_EVENT_TASK_ACTIVE_CS	0x20
+#define		SAS_DEVICE_SENT_ID_FRAME_CS	0x40
+#define		SAS_DEVICE_SENT_REG_FIS_CS	0x40
+#define		SAS_DEVICE_SENT_HARD_RESET_CS	0x80
+#define  	SAS_PHY_IS_DOWN_FLAGS	(SAS_PHY_LOSS_OF_SIGNAL_CS|\
+					 SAS_DWS_TIMER_EXPIRED_CS |\
+					 SAS_LINK_RESET_NOT_COMPLETE_CS|\
+					 SAS_PHY_DISABLED_CS)
+
+#define		SAS_LINK_CTL_PHY_EVENT_FLAGS   (SAS_LINK_CTL_TASK_ACTIVE_CS |\
+						SAS_PHY_EVENT_TASK_ACTIVE_CS |\
+						SAS_DEVICE_SENT_ID_FRAME_CS  |\
+						SAS_DEVICE_SENT_HARD_RESET_CS)
+
+#define LmSEQ_CONCTL(LinkNum)		(LmSCRATCH(LinkNum) + 0x018C)
+#define LmSEQ_CONSTAT(LinkNum)		(LmSCRATCH(LinkNum) + 0x018E)
+#define LmSEQ_CONNECTION_MODES(LinkNum)	(LmSCRATCH(LinkNum) + 0x018F)
+#define LmSEQ_REG1_ISR(LinkNum)		(LmSCRATCH(LinkNum) + 0x0192)
+#define LmSEQ_REG2_ISR(LinkNum)		(LmSCRATCH(LinkNum) + 0x0194)
+#define LmSEQ_REG3_ISR(LinkNum)		(LmSCRATCH(LinkNum) + 0x0196)
+#define LmSEQ_REG0_ISR(LinkNum)		(LmSCRATCH(LinkNum) + 0x0198)
+
+/* Mode independent scratch page 1 macros. */
+#define LmSEQ_EST_NEXUS_SCBPTR0(LinkNum)	(LmSCRATCH(LinkNum) + 0x01A0)
+#define LmSEQ_EST_NEXUS_SCBPTR1(LinkNum)	(LmSCRATCH(LinkNum) + 0x01A2)
+#define LmSEQ_EST_NEXUS_SCBPTR2(LinkNum)	(LmSCRATCH(LinkNum) + 0x01A4)
+#define LmSEQ_EST_NEXUS_SCBPTR3(LinkNum)	(LmSCRATCH(LinkNum) + 0x01A6)
+#define LmSEQ_EST_NEXUS_SCB_OPCODE0(LinkNum)	(LmSCRATCH(LinkNum) + 0x01A8)
+#define LmSEQ_EST_NEXUS_SCB_OPCODE1(LinkNum)	(LmSCRATCH(LinkNum) + 0x01A9)
+#define LmSEQ_EST_NEXUS_SCB_OPCODE2(LinkNum)	(LmSCRATCH(LinkNum) + 0x01AA)
+#define LmSEQ_EST_NEXUS_SCB_OPCODE3(LinkNum)	(LmSCRATCH(LinkNum) + 0x01AB)
+#define LmSEQ_EST_NEXUS_SCB_HEAD(LinkNum)	(LmSCRATCH(LinkNum) + 0x01AC)
+#define LmSEQ_EST_NEXUS_SCB_TAIL(LinkNum)	(LmSCRATCH(LinkNum) + 0x01AD)
+#define LmSEQ_EST_NEXUS_BUF_AVAIL(LinkNum)	(LmSCRATCH(LinkNum) + 0x01AE)
+#define LmSEQ_TIMEOUT_CONST(LinkNum)		(LmSCRATCH(LinkNum) + 0x01B8)
+#define LmSEQ_ISR_SAVE_SINDEX(LinkNum)	        (LmSCRATCH(LinkNum) + 0x01BC)
+#define LmSEQ_ISR_SAVE_DINDEX(LinkNum)	        (LmSCRATCH(LinkNum) + 0x01BE)
+
+/* Mode independent scratch page 2 macros. */
+#define LmSEQ_EMPTY_SCB_PTR0(LinkNum)	(LmSCRATCH(LinkNum) + 0x01C0)
+#define LmSEQ_EMPTY_SCB_PTR1(LinkNum)	(LmSCRATCH(LinkNum) + 0x01C2)
+#define LmSEQ_EMPTY_SCB_PTR2(LinkNum)	(LmSCRATCH(LinkNum) + 0x01C4)
+#define LmSEQ_EMPTY_SCB_PTR3(LinkNum)	(LmSCRATCH(LinkNum) + 0x01C6)
+#define LmSEQ_EMPTY_SCB_OPCD0(LinkNum)	(LmSCRATCH(LinkNum) + 0x01C8)
+#define LmSEQ_EMPTY_SCB_OPCD1(LinkNum)	(LmSCRATCH(LinkNum) + 0x01C9)
+#define LmSEQ_EMPTY_SCB_OPCD2(LinkNum)	(LmSCRATCH(LinkNum) + 0x01CA)
+#define LmSEQ_EMPTY_SCB_OPCD3(LinkNum)	(LmSCRATCH(LinkNum) + 0x01CB)
+#define LmSEQ_EMPTY_SCB_HEAD(LinkNum)	(LmSCRATCH(LinkNum) + 0x01CC)
+#define LmSEQ_EMPTY_SCB_TAIL(LinkNum)	(LmSCRATCH(LinkNum) + 0x01CD)
+#define LmSEQ_EMPTY_BUFS_AVAIL(LinkNum)	(LmSCRATCH(LinkNum) + 0x01CE)
+#define LmSEQ_ATA_SCR_REGS(LinkNum)	(LmSCRATCH(LinkNum) + 0x01D4)
+
+/* Mode independent scratch page 3 macros. */
+#define LmSEQ_DEV_PRES_TMR_TOUT_CONST(LinkNum)	(LmSCRATCH(LinkNum) + 0x01E0)
+#define LmSEQ_SATA_INTERLOCK_TIMEOUT(LinkNum)	(LmSCRATCH(LinkNum) + 0x01E4)
+#define LmSEQ_STP_SHUTDOWN_TIMEOUT(LinkNum)	(LmSCRATCH(LinkNum) + 0x01E8)
+#define LmSEQ_SRST_ASSERT_TIMEOUT(LinkNum)	(LmSCRATCH(LinkNum) + 0x01EC)
+#define LmSEQ_RCV_FIS_TIMEOUT(LinkNum)		(LmSCRATCH(LinkNum) + 0x01F0)
+#define LmSEQ_ONE_MILLISEC_TIMEOUT(LinkNum)	(LmSCRATCH(LinkNum) + 0x01F4)
+#define LmSEQ_TEN_MS_COMINIT_TIMEOUT(LinkNum)	(LmSCRATCH(LinkNum) + 0x01F8)
+#define LmSEQ_SMP_RCV_TIMEOUT(LinkNum)		(LmSCRATCH(LinkNum) + 0x01FC)
+
+#endif
diff --git a/drivers/scsi/aic94xx/aic94xx_sas.h b/drivers/scsi/aic94xx/aic94xx_sas.h
new file mode 100644
index 0000000..64d2317
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_sas.h
@@ -0,0 +1,785 @@
+/*
+ * Aic94xx SAS/SATA driver SAS definitions and hardware interface header file.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#ifndef _AIC94XX_SAS_H_
+#define _AIC94XX_SAS_H_
+
+#include <scsi/libsas.h>
+
+/* ---------- DDBs ---------- */
+/* DDBs are device descriptor blocks which describe a device in the
+ * domain that this sequencer can maintain low-level connections for
+ * us.  They are be 64 bytes.
+ */
+
+struct asd_ddb_ssp_smp_target_port {
+	u8     conn_type;	  /* byte 0 */
+#define DDB_TP_CONN_TYPE 0x81	  /* Initiator port and addr frame type 0x01 */
+
+	u8     conn_rate;
+	__be16 init_conn_tag;
+	u8     dest_sas_addr[8];  /* bytes 4-11 */
+
+	__le16 send_queue_head;
+	u8     sq_suspended;
+	u8     ddb_type;	  /* DDB_TYPE_TARGET */
+#define DDB_TYPE_UNUSED    0xFF
+#define DDB_TYPE_TARGET    0xFE
+#define DDB_TYPE_INITIATOR 0xFD
+#define DDB_TYPE_PM_PORT   0xFC
+
+	__le16 _r_a;
+	__be16 awt_def;
+
+	u8     compat_features;	  /* byte 20 */
+	u8     pathway_blocked_count;
+	__be16 arb_wait_time;
+	__be32 more_compat_features; /* byte 24 */
+
+	u8     conn_mask;
+	u8     flags;	  /* concurrent conn:2,2 and open:0(1) */
+#define CONCURRENT_CONN_SUPP 0x04
+#define OPEN_REQUIRED        0x01
+
+	u16    _r_b;
+	__le16 exec_queue_tail;
+	__le16 send_queue_tail;
+	__le16 sister_ddb;
+
+	__le16 _r_c;
+
+	u8     max_concurrent_conn;
+	u8     num_concurrent_conn;
+	u8     num_contexts;
+
+	u8     _r_d;
+
+	__le16 active_task_count;
+
+	u8     _r_e[9];
+
+	u8     itnl_reason;	  /* I_T nexus loss reason */
+
+	__le16 _r_f;
+
+	__le16 itnl_timeout;
+#define ITNL_TIMEOUT_CONST 0x7D0 /* 2 seconds */
+
+	__le32 itnl_timestamp;
+} __attribute__ ((packed));
+
+struct asd_ddb_stp_sata_target_port {
+	u8     conn_type;	  /* byte 0 */
+	u8     conn_rate;
+	__be16 init_conn_tag;
+	u8     dest_sas_addr[8];  /* bytes 4-11 */
+
+	__le16 send_queue_head;
+	u8     sq_suspended;
+	u8     ddb_type;	  /* DDB_TYPE_TARGET */
+
+	__le16 _r_a;
+
+	__be16 awt_def;
+	u8     compat_features;	  /* byte 20 */
+	u8     pathway_blocked_count;
+	__be16 arb_wait_time;
+	__be32 more_compat_features; /* byte 24 */
+
+	u8     conn_mask;
+	u8     flags;	  /* concurrent conn:2,2 and open:0(1) */
+#define SATA_MULTIPORT     0x80
+#define SUPPORTS_AFFIL     0x40
+#define STP_AFFIL_POL      0x20
+
+	u8     _r_b;
+	u8     flags2;		  /* STP close policy:0 */
+#define STP_CL_POL_NO_TX    0x00
+#define STP_CL_POL_BTW_CMDS 0x01
+
+	__le16 exec_queue_tail;
+	__le16 send_queue_tail;
+	__le16 sister_ddb;
+	__le16 ata_cmd_scbptr;
+	__le32 sata_tag_alloc_mask;
+	__le16 active_task_count;
+	__le16 _r_c;
+	__le32 sata_sactive;
+	u8     num_sata_tags;
+	u8     sata_status;
+	u8     sata_ending_status;
+	u8     itnl_reason;	  /* I_T nexus loss reason */
+	__le16 ncq_data_scb_ptr;
+	__le16 itnl_timeout;
+	__le32 itnl_timestamp;
+} __attribute__ ((packed));
+
+/* This struct asd_ddb_init_port, describes the device descriptor block
+ * of an initiator port (when the sequencer is operating in target mode).
+ * Bytes [0,11] and [20,27] are from the OPEN address frame.
+ * The sequencer allocates an initiator port DDB entry.
+ */
+struct asd_ddb_init_port {
+	u8     conn_type;	  /* byte 0 */
+	u8     conn_rate;
+	__be16 init_conn_tag;     /* BE */
+	u8     dest_sas_addr[8];
+	__le16 send_queue_head;   /* LE, byte 12 */
+	u8     sq_suspended;
+	u8     ddb_type;	  /* DDB_TYPE_INITIATOR */
+	__le16 _r_a;
+	__be16 awt_def;		  /* BE */
+	u8     compat_features;
+	u8     pathway_blocked_count;
+	__be16 arb_wait_time;	  /* BE */
+	__be32 more_compat_features; /* BE */
+	u8     conn_mask;
+	u8     flags;		  /* == 5 */
+	u16    _r_b;
+	__le16 exec_queue_tail;	  /* execution queue tail */
+	__le16 send_queue_tail;
+	__le16 sister_ddb;
+	__le16 init_resp_timeout; /* initiator response timeout */
+	__le32 _r_c;
+	__le16 active_tasks;	  /* active task count */
+	__le16 init_list;	  /* initiator list link pointer */
+	__le32 _r_d;
+	u8     max_conn_to[3]; /* from Conn-Disc mode page, in us, LE */
+	u8     itnl_reason;	  /* I_T nexus loss reason */
+	__le16 bus_inact_to; /* from Conn-Disc mode page, in 100 us, LE */
+	__le16 itnl_to;		  /* from the Protocol Specific Port Ctrl MP */
+	__le32 itnl_timestamp;
+} __attribute__ ((packed));
+
+/* This struct asd_ddb_sata_tag, describes a look-up table to be used
+ * by the sequencers.  SATA II, IDENTIFY DEVICE data, word 76, bit 8:
+ * NCQ support.  This table is used by the sequencers to find the
+ * corresponding SCB, given a SATA II tag value.
+ */
+struct asd_ddb_sata_tag {
+	__le16 scb_pointer[32];
+} __attribute__ ((packed));
+
+/* This struct asd_ddb_sata_pm_table, describes a port number to
+ * connection handle look-up table.  SATA targets attached to a port
+ * multiplier require a 4-bit port number value.  There is one DDB
+ * entry of this type for each SATA port multiplier (sister DDB).
+ * Given a SATA PM port number, this table gives us the SATA PM Port
+ * DDB of the SATA port multiplier port (i.e. the SATA target
+ * discovered on the port).
+ */
+struct asd_ddb_sata_pm_table {
+	__le16 ddb_pointer[16];
+	__le16 _r_a[16];
+} __attribute__ ((packed));
+
+/* This struct asd_ddb_sata_pm_port, describes the SATA port multiplier
+ * port format DDB.
+ */
+struct asd_ddb_sata_pm_port {
+	u8     _r_a[15];
+	u8     ddb_type;
+	u8     _r_b[13];
+	u8     pm_port_flags;
+#define PM_PORT_MASK  0xF0
+#define PM_PORT_SET   0x02
+	u8     _r_c[6];
+	__le16 sister_ddb;
+	__le16 ata_cmd_scbptr;
+	__le32 sata_tag_alloc_mask;
+	__le16 active_task_count;
+	__le16 parent_ddb;
+	__le32 sata_sactive;
+	u8     num_sata_tags;
+	u8     sata_status;
+	u8     sata_ending_status;
+	u8     _r_d[9];
+} __attribute__ ((packed));
+
+/* This struct asd_ddb_seq_shared, describes a DDB shared by the
+ * central and link sequencers.  port_map_by_links is indexed phy
+ * number [0,7]; each byte is a bit mask of all the phys that are in
+ * the same port as the indexed phy.
+ */
+struct asd_ddb_seq_shared {
+	__le16 q_free_ddb_head;
+	__le16 q_free_ddb_tail;
+	__le16 q_free_ddb_cnt;
+	__le16 q_used_ddb_head;
+	__le16 q_used_ddb_tail;
+	__le16 shared_mem_lock;
+	__le16 smp_conn_tag;
+	__le16 est_nexus_buf_cnt;
+	__le16 est_nexus_buf_thresh;
+	u32    _r_a;
+	u8     settable_max_contexts;
+	u8     _r_b[23];
+	u8     conn_not_active;
+	u8     phy_is_up;
+	u8     _r_c[8];
+	u8     port_map_by_links[8];
+} __attribute__ ((packed));
+
+/* ---------- SG Element ---------- */
+
+/* This struct sg_el, describes the hardware scatter gather buffer
+ * element.  All entries are little endian.  In an SCB, there are 2 of
+ * this, plus one more, called a link element of this indicating a
+ * sublist if needed.
+ *
+ * A link element has only the bus address set and the flags (DS) bit
+ * valid.  The bus address points to the start of the sublist.
+ *
+ * If a sublist is needed, then that sublist should also include the 2
+ * sg_el embedded in the SCB, in which case next_sg_offset is 32,
+ * since sizeof(sg_el) = 16; EOS should be 1 and EOL 0 in this case.
+ */
+struct sg_el {
+	__le64 bus_addr;
+	__le32 size;
+	__le16 _r;
+	u8     next_sg_offs;
+	u8     flags;
+#define ASD_SG_EL_DS_MASK   0x30
+#define ASD_SG_EL_DS_OCM    0x10
+#define ASD_SG_EL_DS_HM     0x00
+#define ASD_SG_EL_LIST_MASK 0xC0
+#define ASD_SG_EL_LIST_EOL  0x40
+#define ASD_SG_EL_LIST_EOS  0x80
+} __attribute__ ((packed));
+
+/* ---------- SCBs ---------- */
+
+/* An SCB (sequencer control block) is comprised of a common header
+ * and a task part, for a total of 128 bytes.  All fields are in LE
+ * order, unless otherwise noted.
+ */
+
+/* This struct scb_header, defines the SCB header format.
+ */
+struct scb_header {
+	__le64 next_scb;
+	__le16 index;		  /* transaction context */
+	u8     opcode;
+} __attribute__ ((packed));
+
+/* SCB opcodes: Execution queue
+ */
+#define INITIATE_SSP_TASK       0x00
+#define INITIATE_LONG_SSP_TASK  0x01
+#define INITIATE_BIDIR_SSP_TASK 0x02
+#define ABORT_TASK              0x03
+#define INITIATE_SSP_TMF        0x04
+#define SSP_TARG_GET_DATA       0x05
+#define SSP_TARG_GET_DATA_GOOD  0x06
+#define SSP_TARG_SEND_RESP      0x07
+#define QUERY_SSP_TASK          0x08
+#define INITIATE_ATA_TASK       0x09
+#define INITIATE_ATAPI_TASK     0x0a
+#define CONTROL_ATA_DEV         0x0b
+#define INITIATE_SMP_TASK       0x0c
+#define SMP_TARG_SEND_RESP      0x0f
+
+/* SCB opcodes: Send Queue
+ */
+#define SSP_TARG_SEND_DATA      0x40
+#define SSP_TARG_SEND_DATA_GOOD 0x41
+
+/* SCB opcodes: Link Queue
+ */
+#define CONTROL_PHY             0x80
+#define SEND_PRIMITIVE          0x81
+#define INITIATE_LINK_ADM_TASK  0x82
+
+/* SCB opcodes: other
+ */
+#define EMPTY_SCB               0xc0
+#define INITIATE_SEQ_ADM_TASK   0xc1
+#define EST_ICL_TARG_WINDOW     0xc2
+#define COPY_MEM                0xc3
+#define CLEAR_NEXUS             0xc4
+#define INITIATE_DDB_ADM_TASK   0xc6
+#define ESTABLISH_NEXUS_ESCB    0xd0
+
+#define LUN_SIZE                8
+
+/* See SAS spec, task IU
+ */
+struct ssp_task_iu {
+	u8     lun[LUN_SIZE];	  /* BE */
+	u16    _r_a;
+	u8     tmf;
+	u8     _r_b;
+	__be16 tag;		  /* BE */
+	u8     _r_c[14];
+} __attribute__ ((packed));
+
+/* See SAS spec, command IU
+ */
+struct ssp_command_iu {
+	u8     lun[LUN_SIZE];
+	u8     _r_a;
+	u8     efb_prio_attr;	  /* enable first burst, task prio & attr */
+#define EFB_MASK        0x80
+#define TASK_PRIO_MASK	0x78
+#define TASK_ATTR_MASK  0x07
+
+	u8    _r_b;
+	u8     add_cdb_len;	  /* in dwords, since bit 0,1 are reserved */
+	union {
+		u8     cdb[16];
+		struct {
+			__le64 long_cdb_addr;	  /* bus address, LE */
+			__le32 long_cdb_size;	  /* LE */
+			u8     _r_c[3];
+			u8     eol_ds;		  /* eol:6,6, ds:5,4 */
+		} long_cdb;	  /* sequencer extension */
+	};
+} __attribute__ ((packed));
+
+struct xfer_rdy_iu {
+	__be32 requested_offset;  /* BE */
+	__be32 write_data_len;	  /* BE */
+	__be32 _r_a;
+} __attribute__ ((packed));
+
+/* ---------- SCB tasks ---------- */
+
+/* This is both ssp_task and long_ssp_task
+ */
+struct initiate_ssp_task {
+	u8     proto_conn_rate;	  /* proto:6,4, conn_rate:3,0 */
+	__le32 total_xfer_len;
+	struct ssp_frame_hdr  ssp_frame;
+	struct ssp_command_iu ssp_cmd;
+	__le16 sister_scb;	  /* 0xFFFF */
+	__le16 conn_handle;	  /* index to DDB for the intended target */
+	u8     data_dir;	  /* :1,0 */
+#define DATA_DIR_NONE   0x00
+#define DATA_DIR_IN     0x01
+#define DATA_DIR_OUT    0x02
+#define DATA_DIR_BYRECIPIENT 0x03
+
+	u8     _r_a;
+	u8     retry_count;
+	u8     _r_b[5];
+	struct sg_el sg_element[3]; /* 2 real and 1 link */
+} __attribute__ ((packed));
+
+/* This defines both ata_task and atapi_task.
+ * ata: C bit of FIS should be 1,
+ * atapi: C bit of FIS should be 1, and command register should be 0xA0,
+ * to indicate a packet command.
+ */
+struct initiate_ata_task {
+	u8     proto_conn_rate;
+	__le32 total_xfer_len;
+	struct host_to_dev_fis fis;
+	__le32 data_offs;
+	u8     atapi_packet[16];
+	u8     _r_a[12];
+	__le16 sister_scb;
+	__le16 conn_handle;
+	u8     ata_flags;	  /* CSMI:6,6, DTM:4,4, QT:3,3, data dir:1,0 */
+#define CSMI_TASK           0x40
+#define DATA_XFER_MODE_DMA  0x10
+#define ATA_Q_TYPE_MASK     0x08
+#define	ATA_Q_TYPE_UNTAGGED 0x00
+#define ATA_Q_TYPE_NCQ      0x08
+
+	u8     _r_b;
+	u8     retry_count;
+	u8     _r_c;
+	u8     flags;
+#define STP_AFFIL_POLICY   0x20
+#define SET_AFFIL_POLICY   0x10
+#define RET_PARTIAL_SGLIST 0x02
+
+	u8     _r_d[3];
+	struct sg_el sg_element[3];
+} __attribute__ ((packed));
+
+struct initiate_smp_task {
+	u8     proto_conn_rate;
+	u8     _r_a[40];
+	struct sg_el smp_req;
+	__le16 sister_scb;
+	__le16 conn_handle;
+	u8     _r_c[8];
+	struct sg_el smp_resp;
+	u8     _r_d[32];
+} __attribute__ ((packed));
+
+struct control_phy {
+	u8     phy_id;
+	u8     sub_func;
+#define DISABLE_PHY            0x00
+#define ENABLE_PHY             0x01
+#define RELEASE_SPINUP_HOLD    0x02
+#define ENABLE_PHY_NO_SAS_OOB  0x03
+#define ENABLE_PHY_NO_SATA_OOB 0x04
+#define PHY_NO_OP              0x05
+#define EXECUTE_HARD_RESET     0x81
+
+	u8     func_mask;
+	u8     speed_mask;
+	u8     hot_plug_delay;
+	u8     port_type;
+	u8     flags;
+#define DEV_PRES_TIMER_OVERRIDE_ENABLE 0x01
+#define DISABLE_PHY_IF_OOB_FAILS       0x02
+
+	__le32 timeout_override;
+	u8     link_reset_retries;
+	u8     _r_a[47];
+	__le16 conn_handle;
+	u8     _r_b[56];
+} __attribute__ ((packed));
+
+struct control_ata_dev {
+	u8     proto_conn_rate;
+	__le32 _r_a;
+	struct host_to_dev_fis fis;
+	u8     _r_b[32];
+	__le16 sister_scb;
+	__le16 conn_handle;
+	u8     ata_flags;	  /* 0 */
+	u8     _r_c[55];
+} __attribute__ ((packed));
+
+struct empty_scb {
+	u8     num_valid;
+	__le32 _r_a;
+#define ASD_EDBS_PER_SCB 7
+/* header+data+CRC+DMA suffix data */
+#define ASD_EDB_SIZE (24+1024+4+16)
+	struct sg_el eb[ASD_EDBS_PER_SCB];
+#define ELEMENT_NOT_VALID  0xC0
+} __attribute__ ((packed));
+
+struct initiate_link_adm {
+	u8     phy_id;
+	u8     sub_func;
+#define GET_LINK_ERROR_COUNT      0x00
+#define RESET_LINK_ERROR_COUNT    0x01
+#define ENABLE_NOTIFY_SPINUP_INTS 0x02
+
+	u8     _r_a[57];
+	__le16 conn_handle;
+	u8     _r_b[56];
+} __attribute__ ((packed));
+
+struct copy_memory {
+	u8     _r_a;
+	__le16 xfer_len;
+	__le16 _r_b;
+	__le64 src_busaddr;
+	u8     src_ds;		  /* See definition of sg_el */
+	u8     _r_c[45];
+	__le16 conn_handle;
+	__le64 _r_d;
+	__le64 dest_busaddr;
+	u8     dest_ds;		  /* See definition of sg_el */
+	u8     _r_e[39];
+} __attribute__ ((packed));
+
+struct abort_task {
+	u8     proto_conn_rate;
+	__le32 _r_a;
+	struct ssp_frame_hdr ssp_frame;
+	struct ssp_task_iu ssp_task;
+	__le16 sister_scb;
+	__le16 conn_handle;
+	u8     flags;	  /* ovrd_itnl_timer:3,3, suspend_data_trans:2,2 */
+#define SUSPEND_DATA_TRANS 0x04
+
+	u8     _r_b;
+	u8     retry_count;
+	u8     _r_c[5];
+	__le16 index;  /* Transaction context of task to be queried */
+	__le16 itnl_to;
+	u8     _r_d[44];
+} __attribute__ ((packed));
+
+struct clear_nexus {
+	u8     nexus;
+#define NEXUS_ADAPTER  0x00
+#define NEXUS_PORT     0x01
+#define NEXUS_I_T      0x02
+#define NEXUS_I_T_L    0x03
+#define NEXUS_TAG      0x04
+#define NEXUS_TRANS_CX 0x05
+#define NEXUS_SATA_TAG 0x06
+#define NEXUS_T_L      0x07
+#define NEXUS_L        0x08
+#define NEXUS_T_TAG    0x09
+
+	__le32 _r_a;
+	u8     flags;
+#define SUSPEND_TX     0x80
+#define RESUME_TX      0x40
+#define SEND_Q         0x04
+#define EXEC_Q         0x02
+#define NOTINQ         0x01
+
+	u8     _r_b[3];
+	u8     conn_mask;
+	u8     _r_c[19];
+	struct ssp_task_iu ssp_task; /* LUN and TAG */
+	__le16 _r_d;
+	__le16 conn_handle;
+	__le64 _r_e;
+	__le16 index;  /* Transaction context of task to be cleared */
+	__le16 context;		  /* Clear nexus context */
+	u8     _r_f[44];
+} __attribute__ ((packed));
+
+struct initiate_ssp_tmf {
+	u8     proto_conn_rate;
+	__le32 _r_a;
+	struct ssp_frame_hdr ssp_frame;
+	struct ssp_task_iu ssp_task;
+	__le16 sister_scb;
+	__le16 conn_handle;
+	u8     flags;	  /* itnl override and suspend data tx */
+#define OVERRIDE_ITNL_TIMER  8
+
+	u8     _r_b;
+	u8     retry_count;
+	u8     _r_c[5];
+	__le16 index;  /* Transaction context of task to be queried */
+	__le16 itnl_to;
+	u8     _r_d[44];
+} __attribute__ ((packed));
+
+/* Transmits an arbitrary primitive on the link.
+ * Used for NOTIFY and BROADCAST.
+ */
+struct send_prim {
+	u8     phy_id;
+	u8     wait_transmit; 	  /* :0,0 */
+	u8     xmit_flags;
+#define XMTPSIZE_MASK      0xF0
+#define XMTPSIZE_SINGLE    0x10
+#define XMTPSIZE_REPEATED  0x20
+#define XMTPSIZE_CONT      0x20
+#define XMTPSIZE_TRIPLE    0x30
+#define XMTPSIZE_REDUNDANT 0x60
+#define XMTPSIZE_INF       0
+
+#define XMTCONTEN          0x04
+#define XMTPFRM            0x02	  /* Transmit at the next frame boundary */
+#define XMTPIMM            0x01	  /* Transmit immediately */
+
+	__le16 _r_a;
+	u8     prim[4];		  /* K, D0, D1, D2 */
+	u8     _r_b[50];
+	__le16 conn_handle;
+	u8     _r_c[56];
+} __attribute__ ((packed));
+
+/* This describes both SSP Target Get Data and SSP Target Get Data And
+ * Send Good Response SCBs.  Used when the sequencer is operating in
+ * target mode...
+ */
+struct ssp_targ_get_data {
+	u8     proto_conn_rate;
+	__le32 total_xfer_len;
+	struct ssp_frame_hdr ssp_frame;
+	struct xfer_rdy_iu  xfer_rdy;
+	u8     lun[LUN_SIZE];
+	__le64 _r_a;
+	__le16 sister_scb;
+	__le16 conn_handle;
+	u8     data_dir;	  /* 01b */
+	u8     _r_b;
+	u8     retry_count;
+	u8     _r_c[5];
+	struct sg_el sg_element[3];
+} __attribute__ ((packed));
+
+/* ---------- The actual SCB struct ---------- */
+
+struct scb {
+	struct scb_header header;
+	union {
+		struct initiate_ssp_task ssp_task;
+		struct initiate_ata_task ata_task;
+		struct initiate_smp_task smp_task;
+		struct control_phy       control_phy;
+		struct control_ata_dev   control_ata_dev;
+		struct empty_scb         escb;
+		struct initiate_link_adm link_adm;
+		struct copy_memory       cp_mem;
+		struct abort_task        abort_task;
+		struct clear_nexus       clear_nexus;
+		struct initiate_ssp_tmf  ssp_tmf;
+	};
+} __attribute__ ((packed));
+
+/* ---------- Done List ---------- */
+/* The done list entry opcode field is defined below.
+ * The mnemonic encoding and meaning is as follows:
+ * TC - Task Complete, status was received and acknowledged
+ * TF - Task Failed, indicates an error prior to receiving acknowledgment
+ *   for the command:
+ *   - no conn,
+ *   - NACK or R_ERR received in response to this command,
+ *   - credit blocked or not available, or in the case of SMP request,
+ *   - no SMP response was received.
+ *   In these four cases it is known that the target didn't receive the
+ *   command.
+ * TI - Task Interrupted, error after the command was acknowledged.  It is
+ *   known that the command was received by the target.
+ * TU - Task Unacked, command was transmitted but neither ACK (R_OK) nor NAK
+ *   (R_ERR) was received due to loss of signal, broken connection, loss of
+ *   dword sync or other reason.  The application client should send the
+ *   appropriate task query.
+ * TA - Task Aborted, see TF.
+ * _RESP - The completion includes an empty buffer containing status.
+ * TO - Timeout.
+ */
+#define TC_NO_ERROR             0x00
+#define TC_UNDERRUN             0x01
+#define TC_OVERRUN              0x02
+#define TF_OPEN_TO              0x03
+#define TF_OPEN_REJECT          0x04
+#define TI_BREAK                0x05
+#define TI_PROTO_ERR            0x06
+#define TC_SSP_RESP             0x07
+#define TI_PHY_DOWN             0x08
+#define TF_PHY_DOWN             0x09
+#define TC_LINK_ADM_RESP        0x0a
+#define TC_CSMI                 0x0b
+#define TC_ATA_RESP             0x0c
+#define TU_PHY_DOWN             0x0d
+#define TU_BREAK                0x0e
+#define TI_SATA_TO              0x0f
+#define TI_NAK                  0x10
+#define TC_CONTROL_PHY          0x11
+#define TF_BREAK                0x12
+#define TC_RESUME               0x13
+#define TI_ACK_NAK_TO           0x14
+#define TF_SMPRSP_TO            0x15
+#define TF_SMP_XMIT_RCV_ERR     0x16
+#define TC_PARTIAL_SG_LIST      0x17
+#define TU_ACK_NAK_TO           0x18
+#define TU_SATA_TO              0x19
+#define TF_NAK_RECV             0x1a
+#define TA_I_T_NEXUS_LOSS       0x1b
+#define TC_ATA_R_ERR_RECV       0x1c
+#define TF_TMF_NO_CTX           0x1d
+#define TA_ON_REQ               0x1e
+#define TF_TMF_NO_TAG           0x1f
+#define TF_TMF_TAG_FREE         0x20
+#define TF_TMF_TASK_DONE        0x21
+#define TF_TMF_NO_CONN_HANDLE   0x22
+#define TC_TASK_CLEARED         0x23
+#define TI_SYNCS_RECV           0x24
+#define TU_SYNCS_RECV           0x25
+#define TF_IRTT_TO              0x26
+#define TF_NO_SMP_CONN          0x27
+#define TF_IU_SHORT             0x28
+#define TF_DATA_OFFS_ERR        0x29
+#define TF_INV_CONN_HANDLE      0x2a
+#define TF_REQUESTED_N_PENDING  0x2b
+
+/* 0xc1 - 0xc7: empty buffer received,
+   0xd1 - 0xd7: establish nexus empty buffer received
+*/
+/* This is the ESCB mask */
+#define ESCB_RECVD              0xC0
+
+
+/* This struct done_list_struct defines the done list entry.
+ * All fields are LE.
+ */
+struct done_list_struct {
+	__le16 index;		  /* aka transaction context */
+	u8     opcode;
+	u8     status_block[4];
+	u8     toggle;		  /* bit 0 */
+#define DL_TOGGLE_MASK     0x01
+} __attribute__ ((packed));
+
+/* ---------- PHYS ---------- */
+
+struct asd_phy {
+	struct asd_sas_phy        sas_phy;
+	struct asd_phy_desc   *phy_desc; /* hw profile */
+
+	struct sas_identify_frame *identify_frame;
+	struct asd_dma_tok  *id_frm_tok;
+
+	u8         frame_rcvd[ASD_EDB_SIZE];
+};
+
+
+#define ASD_SCB_SIZE sizeof(struct scb)
+#define ASD_DDB_SIZE sizeof(struct asd_ddb_ssp_smp_target_port)
+
+/* Define this to 0 if you do not want NOTIFY (ENABLE SPINIP) sent.
+ * Default: 0x10 (it's a mask)
+ */
+#define ASD_NOTIFY_ENABLE_SPINUP  0x10
+
+/* If enabled, set this to the interval between transmission
+ * of NOTIFY (ENABLE SPINUP). In units of 200 us.
+ */
+#define ASD_NOTIFY_TIMEOUT        2500
+
+/* Initial delay after OOB, before we transmit NOTIFY (ENABLE SPINUP).
+ * If 0, transmit immediately. In milliseconds.
+ */
+#define ASD_NOTIFY_DOWN_COUNT     0
+
+/* Device present timer timeout constant, 10 ms. */
+#define ASD_DEV_PRESENT_TIMEOUT   0x2710
+
+#define ASD_SATA_INTERLOCK_TIMEOUT 0
+
+/* How long to wait before shutting down an STP connection, unless
+ * an STP target sent frame(s). 50 usec.
+ * IGNORED by the sequencer (i.e. value 0 always).
+ */
+#define ASD_STP_SHUTDOWN_TIMEOUT  0x0
+
+/* ATA soft reset timer timeout. 5 usec. */
+#define ASD_SRST_ASSERT_TIMEOUT   0x05
+
+/* 31 sec */
+#define ASD_RCV_FIS_TIMEOUT       0x01D905C0
+
+#define ASD_ONE_MILLISEC_TIMEOUT  0x03e8
+
+/* COMINIT timer */
+#define ASD_TEN_MILLISEC_TIMEOUT  0x2710
+#define ASD_COMINIT_TIMEOUT ASD_TEN_MILLISEC_TIMEOUT
+
+/* 1 sec */
+#define ASD_SMP_RCV_TIMEOUT       0x000F4240
+
+#endif
diff --git a/drivers/scsi/aic94xx/aic94xx_scb.c b/drivers/scsi/aic94xx/aic94xx_scb.c
new file mode 100644
index 0000000..7ee49b5
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_scb.c
@@ -0,0 +1,758 @@
+/*
+ * Aic94xx SAS/SATA driver SCB management.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/pci.h>
+
+#include "aic94xx.h"
+#include "aic94xx_reg.h"
+#include "aic94xx_hwi.h"
+#include "aic94xx_seq.h"
+
+#include "aic94xx_dump.h"
+
+/* ---------- EMPTY SCB ---------- */
+
+#define DL_PHY_MASK      7
+#define BYTES_DMAED      0
+#define PRIMITIVE_RECVD  0x08
+#define PHY_EVENT        0x10
+#define LINK_RESET_ERROR 0x18
+#define TIMER_EVENT      0x20
+#define REQ_TASK_ABORT   0xF0
+#define REQ_DEVICE_RESET 0xF1
+#define SIGNAL_NCQ_ERROR 0xF2
+#define CLEAR_NCQ_ERROR  0xF3
+
+#define PHY_EVENTS_STATUS (CURRENT_LOSS_OF_SIGNAL | CURRENT_OOB_DONE   \
+			   | CURRENT_SPINUP_HOLD | CURRENT_GTO_TIMEOUT \
+			   | CURRENT_OOB_ERROR)
+
+static inline void get_lrate_mode(struct asd_phy *phy, u8 oob_mode)
+{
+	struct sas_phy *sas_phy = phy->sas_phy.phy;
+
+	switch (oob_mode & 7) {
+	case PHY_SPEED_60:
+		/* FIXME: sas transport class doesn't have this */
+		phy->sas_phy.linkrate = SAS_LINK_RATE_6_0_GBPS;
+		phy->sas_phy.phy->negotiated_linkrate = SAS_LINK_RATE_6_0_GBPS;
+		break;
+	case PHY_SPEED_30:
+		phy->sas_phy.linkrate = SAS_LINK_RATE_3_0_GBPS;
+		phy->sas_phy.phy->negotiated_linkrate = SAS_LINK_RATE_3_0_GBPS;
+		break;
+	case PHY_SPEED_15:
+		phy->sas_phy.linkrate = SAS_LINK_RATE_1_5_GBPS;
+		phy->sas_phy.phy->negotiated_linkrate = SAS_LINK_RATE_1_5_GBPS;
+		break;
+	}
+	sas_phy->negotiated_linkrate = phy->sas_phy.linkrate;
+	sas_phy->maximum_linkrate_hw = SAS_LINK_RATE_3_0_GBPS;
+	sas_phy->minimum_linkrate_hw = SAS_LINK_RATE_1_5_GBPS;
+	sas_phy->maximum_linkrate = phy->phy_desc->max_sas_lrate;
+	sas_phy->minimum_linkrate = phy->phy_desc->min_sas_lrate;
+
+	if (oob_mode & SAS_MODE)
+		phy->sas_phy.oob_mode = SAS_OOB_MODE;
+	else if (oob_mode & SATA_MODE)
+		phy->sas_phy.oob_mode = SATA_OOB_MODE;
+}
+
+static inline void asd_phy_event_tasklet(struct asd_ascb *ascb,
+					 struct done_list_struct *dl)
+{
+	struct asd_ha_struct *asd_ha = ascb->ha;
+	struct sas_ha_struct *sas_ha = &asd_ha->sas_ha;
+	int phy_id = dl->status_block[0] & DL_PHY_MASK;
+	struct asd_phy *phy = &asd_ha->phys[phy_id];
+
+	u8 oob_status = dl->status_block[1] & PHY_EVENTS_STATUS;
+	u8 oob_mode   = dl->status_block[2];
+
+	switch (oob_status) {
+	case CURRENT_LOSS_OF_SIGNAL:
+		/* directly attached device was removed */
+		ASD_DPRINTK("phy%d: device unplugged\n", phy_id);
+		asd_turn_led(asd_ha, phy_id, 0);
+		sas_phy_disconnected(&phy->sas_phy);
+		sas_ha->notify_phy_event(&phy->sas_phy, PHYE_LOSS_OF_SIGNAL);
+		break;
+	case CURRENT_OOB_DONE:
+		/* hot plugged device */
+		asd_turn_led(asd_ha, phy_id, 1);
+		get_lrate_mode(phy, oob_mode);
+		ASD_DPRINTK("phy%d device plugged: lrate:0x%x, proto:0x%x\n",
+			    phy_id, phy->sas_phy.linkrate, phy->sas_phy.iproto);
+		sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_DONE);
+		break;
+	case CURRENT_SPINUP_HOLD:
+		/* hot plug SATA, no COMWAKE sent */
+		asd_turn_led(asd_ha, phy_id, 1);
+		sas_ha->notify_phy_event(&phy->sas_phy, PHYE_SPINUP_HOLD);
+		break;
+	case CURRENT_GTO_TIMEOUT:
+	case CURRENT_OOB_ERROR:
+		ASD_DPRINTK("phy%d error while OOB: oob status:0x%x\n", phy_id,
+			    dl->status_block[1]);
+		asd_turn_led(asd_ha, phy_id, 0);
+		sas_phy_disconnected(&phy->sas_phy);
+		sas_ha->notify_phy_event(&phy->sas_phy, PHYE_OOB_ERROR);
+		break;
+	}
+}
+
+/* If phys are enabled sparsely, this will do the right thing. */
+static inline unsigned ord_phy(struct asd_ha_struct *asd_ha,
+			       struct asd_phy *phy)
+{
+	u8 enabled_mask = asd_ha->hw_prof.enabled_phys;
+	int i, k = 0;
+
+	for_each_phy(enabled_mask, enabled_mask, i) {
+		if (&asd_ha->phys[i] == phy)
+			return k;
+		k++;
+	}
+	return 0;
+}
+
+/**
+ * asd_get_attached_sas_addr -- extract/generate attached SAS address
+ * phy: pointer to asd_phy
+ * sas_addr: pointer to buffer where the SAS address is to be written
+ *
+ * This function extracts the SAS address from an IDENTIFY frame
+ * received.  If OOB is SATA, then a SAS address is generated from the
+ * HA tables.
+ *
+ * LOCKING: the frame_rcvd_lock needs to be held since this parses the frame
+ * buffer.
+ */
+static inline void asd_get_attached_sas_addr(struct asd_phy *phy, u8 *sas_addr)
+{
+	if (phy->sas_phy.frame_rcvd[0] == 0x34
+	    && phy->sas_phy.oob_mode == SATA_OOB_MODE) {
+		struct asd_ha_struct *asd_ha = phy->sas_phy.ha->lldd_ha;
+		/* FIS device-to-host */
+		u64 addr = be64_to_cpu(*(__be64 *)phy->phy_desc->sas_addr);
+
+		addr += asd_ha->hw_prof.sata_name_base + ord_phy(asd_ha, phy);
+		*(__be64 *)sas_addr = cpu_to_be64(addr);
+	} else {
+		struct sas_identify_frame *idframe =
+			(void *) phy->sas_phy.frame_rcvd;
+		memcpy(sas_addr, idframe->sas_addr, SAS_ADDR_SIZE);
+	}
+}
+
+static inline void asd_bytes_dmaed_tasklet(struct asd_ascb *ascb,
+					   struct done_list_struct *dl,
+					   int edb_id, int phy_id)
+{
+	unsigned long flags;
+	int edb_el = edb_id + ascb->edb_index;
+	struct asd_dma_tok *edb = ascb->ha->seq.edb_arr[edb_el];
+	struct asd_phy *phy = &ascb->ha->phys[phy_id];
+	struct sas_ha_struct *sas_ha = phy->sas_phy.ha;
+	u16 size = ((dl->status_block[3] & 7) << 8) | dl->status_block[2];
+
+	size = min(size, (u16) sizeof(phy->frame_rcvd));
+
+	spin_lock_irqsave(&phy->sas_phy.frame_rcvd_lock, flags);
+	memcpy(phy->sas_phy.frame_rcvd, edb->vaddr, size);
+	phy->sas_phy.frame_rcvd_size = size;
+	asd_get_attached_sas_addr(phy, phy->sas_phy.attached_sas_addr);
+	spin_unlock_irqrestore(&phy->sas_phy.frame_rcvd_lock, flags);
+	asd_dump_frame_rcvd(phy, dl);
+	sas_ha->notify_port_event(&phy->sas_phy, PORTE_BYTES_DMAED);
+}
+
+static inline void asd_link_reset_err_tasklet(struct asd_ascb *ascb,
+					      struct done_list_struct *dl,
+					      int phy_id)
+{
+	struct asd_ha_struct *asd_ha = ascb->ha;
+	struct sas_ha_struct *sas_ha = &asd_ha->sas_ha;
+	struct asd_sas_phy *sas_phy = sas_ha->sas_phy[phy_id];
+	u8 lr_error = dl->status_block[1];
+	u8 retries_left = dl->status_block[2];
+
+	switch (lr_error) {
+	case 0:
+		ASD_DPRINTK("phy%d: Receive ID timer expired\n", phy_id);
+		break;
+	case 1:
+		ASD_DPRINTK("phy%d: Loss of signal\n", phy_id);
+		break;
+	case 2:
+		ASD_DPRINTK("phy%d: Loss of dword sync\n", phy_id);
+		break;
+	case 3:
+		ASD_DPRINTK("phy%d: Receive FIS timeout\n", phy_id);
+		break;
+	default:
+		ASD_DPRINTK("phy%d: unknown link reset error code: 0x%x\n",
+			    phy_id, lr_error);
+		break;
+	}
+
+	asd_turn_led(asd_ha, phy_id, 0);
+	sas_phy_disconnected(sas_phy);
+	sas_ha->notify_port_event(sas_phy, PORTE_LINK_RESET_ERR);
+
+	if (retries_left == 0) {
+		int num = 1;
+		struct asd_ascb *cp = asd_ascb_alloc_list(ascb->ha, &num,
+							  GFP_ATOMIC);
+		if (!cp) {
+			asd_printk("%s: out of memory\n", __FUNCTION__);
+			goto out;
+		}
+		ASD_DPRINTK("phy%d: retries:0 performing link reset seq\n",
+			    phy_id);
+		asd_build_control_phy(cp, phy_id, ENABLE_PHY);
+		if (asd_post_ascb_list(ascb->ha, cp, 1) != 0)
+			asd_ascb_free(cp);
+	}
+out:
+	;
+}
+
+static inline void asd_primitive_rcvd_tasklet(struct asd_ascb *ascb,
+					      struct done_list_struct *dl,
+					      int phy_id)
+{
+	unsigned long flags;
+	struct sas_ha_struct *sas_ha = &ascb->ha->sas_ha;
+	struct asd_sas_phy *sas_phy = sas_ha->sas_phy[phy_id];
+	u8  reg  = dl->status_block[1];
+	u32 cont = dl->status_block[2] << ((reg & 3)*8);
+
+	reg &= ~3;
+	switch (reg) {
+	case LmPRMSTAT0BYTE0:
+		switch (cont) {
+		case LmBROADCH:
+		case LmBROADRVCH0:
+		case LmBROADRVCH1:
+		case LmBROADSES:
+			ASD_DPRINTK("phy%d: BROADCAST change received:%d\n",
+				    phy_id, cont);
+			spin_lock_irqsave(&sas_phy->sas_prim_lock, flags);
+			sas_phy->sas_prim = ffs(cont);
+			spin_unlock_irqrestore(&sas_phy->sas_prim_lock, flags);
+			sas_ha->notify_port_event(sas_phy,PORTE_BROADCAST_RCVD);
+			break;
+
+		case LmUNKNOWNP:
+			ASD_DPRINTK("phy%d: unknown BREAK\n", phy_id);
+			break;
+
+		default:
+			ASD_DPRINTK("phy%d: primitive reg:0x%x, cont:0x%04x\n",
+				    phy_id, reg, cont);
+			break;
+		}
+		break;
+	case LmPRMSTAT1BYTE0:
+		switch (cont) {
+		case LmHARDRST:
+			ASD_DPRINTK("phy%d: HARD_RESET primitive rcvd\n",
+				    phy_id);
+			/* The sequencer disables all phys on that port.
+			 * We have to re-enable the phys ourselves. */
+			sas_ha->notify_port_event(sas_phy, PORTE_HARD_RESET);
+			break;
+
+		default:
+			ASD_DPRINTK("phy%d: primitive reg:0x%x, cont:0x%04x\n",
+				    phy_id, reg, cont);
+			break;
+		}
+		break;
+	default:
+		ASD_DPRINTK("unknown primitive register:0x%x\n",
+			    dl->status_block[1]);
+		break;
+	}
+}
+
+/**
+ * asd_invalidate_edb -- invalidate an EDB and if necessary post the ESCB
+ * @ascb: pointer to Empty SCB
+ * @edb_id: index [0,6] to the empty data buffer which is to be invalidated
+ *
+ * After an EDB has been invalidated, if all EDBs in this ESCB have been
+ * invalidated, the ESCB is posted back to the sequencer.
+ * Context is tasklet/IRQ.
+ */
+void asd_invalidate_edb(struct asd_ascb *ascb, int edb_id)
+{
+	struct asd_seq_data *seq = &ascb->ha->seq;
+	struct empty_scb *escb = &ascb->scb->escb;
+	struct sg_el     *eb   = &escb->eb[edb_id];
+	struct asd_dma_tok *edb = seq->edb_arr[ascb->edb_index + edb_id];
+
+	memset(edb->vaddr, 0, ASD_EDB_SIZE);
+	eb->flags |= ELEMENT_NOT_VALID;
+	escb->num_valid--;
+
+	if (escb->num_valid == 0) {
+		int i;
+		/* ASD_DPRINTK("reposting escb: vaddr: 0x%p, "
+			    "dma_handle: 0x%08llx, next: 0x%08llx, "
+			    "index:%d, opcode:0x%02x\n",
+			    ascb->dma_scb.vaddr,
+			    (u64)ascb->dma_scb.dma_handle,
+			    le64_to_cpu(ascb->scb->header.next_scb),
+			    le16_to_cpu(ascb->scb->header.index),
+			    ascb->scb->header.opcode);
+		*/
+		escb->num_valid = ASD_EDBS_PER_SCB;
+		for (i = 0; i < ASD_EDBS_PER_SCB; i++)
+			escb->eb[i].flags = 0;
+		if (!list_empty(&ascb->list))
+			list_del_init(&ascb->list);
+		i = asd_post_escb_list(ascb->ha, ascb, 1);
+		if (i)
+			asd_printk("couldn't post escb, err:%d\n", i);
+	}
+}
+
+static void escb_tasklet_complete(struct asd_ascb *ascb,
+				  struct done_list_struct *dl)
+{
+	struct asd_ha_struct *asd_ha = ascb->ha;
+	struct sas_ha_struct *sas_ha = &asd_ha->sas_ha;
+	int edb = (dl->opcode & DL_PHY_MASK) - 1; /* [0xc1,0xc7] -> [0,6] */
+	u8  sb_opcode = dl->status_block[0];
+	int phy_id = sb_opcode & DL_PHY_MASK;
+	struct asd_sas_phy *sas_phy = sas_ha->sas_phy[phy_id];
+
+	if (edb > 6 || edb < 0) {
+		ASD_DPRINTK("edb is 0x%x! dl->opcode is 0x%x\n",
+			    edb, dl->opcode);
+		ASD_DPRINTK("sb_opcode : 0x%x, phy_id: 0x%x\n",
+			    sb_opcode, phy_id);
+		ASD_DPRINTK("escb: vaddr: 0x%p, "
+			    "dma_handle: 0x%llx, next: 0x%llx, "
+			    "index:%d, opcode:0x%02x\n",
+			    ascb->dma_scb.vaddr,
+			    (unsigned long long)ascb->dma_scb.dma_handle,
+			    (unsigned long long)
+			    le64_to_cpu(ascb->scb->header.next_scb),
+			    le16_to_cpu(ascb->scb->header.index),
+			    ascb->scb->header.opcode);
+	}
+
+	sb_opcode &= ~DL_PHY_MASK;
+
+	switch (sb_opcode) {
+	case BYTES_DMAED:
+		ASD_DPRINTK("%s: phy%d: BYTES_DMAED\n", __FUNCTION__, phy_id);
+		asd_bytes_dmaed_tasklet(ascb, dl, edb, phy_id);
+		break;
+	case PRIMITIVE_RECVD:
+		ASD_DPRINTK("%s: phy%d: PRIMITIVE_RECVD\n", __FUNCTION__,
+			    phy_id);
+		asd_primitive_rcvd_tasklet(ascb, dl, phy_id);
+		break;
+	case PHY_EVENT:
+		ASD_DPRINTK("%s: phy%d: PHY_EVENT\n", __FUNCTION__, phy_id);
+		asd_phy_event_tasklet(ascb, dl);
+		break;
+	case LINK_RESET_ERROR:
+		ASD_DPRINTK("%s: phy%d: LINK_RESET_ERROR\n", __FUNCTION__,
+			    phy_id);
+		asd_link_reset_err_tasklet(ascb, dl, phy_id);
+		break;
+	case TIMER_EVENT:
+		ASD_DPRINTK("%s: phy%d: TIMER_EVENT, lost dw sync\n",
+			    __FUNCTION__, phy_id);
+		asd_turn_led(asd_ha, phy_id, 0);
+		/* the device is gone */
+		sas_phy_disconnected(sas_phy);
+		sas_ha->notify_port_event(sas_phy, PORTE_TIMER_EVENT);
+		break;
+	case REQ_TASK_ABORT:
+		ASD_DPRINTK("%s: phy%d: REQ_TASK_ABORT\n", __FUNCTION__,
+			    phy_id);
+		break;
+	case REQ_DEVICE_RESET:
+		ASD_DPRINTK("%s: phy%d: REQ_DEVICE_RESET\n", __FUNCTION__,
+			    phy_id);
+		break;
+	case SIGNAL_NCQ_ERROR:
+		ASD_DPRINTK("%s: phy%d: SIGNAL_NCQ_ERROR\n", __FUNCTION__,
+			    phy_id);
+		break;
+	case CLEAR_NCQ_ERROR:
+		ASD_DPRINTK("%s: phy%d: CLEAR_NCQ_ERROR\n", __FUNCTION__,
+			    phy_id);
+		break;
+	default:
+		ASD_DPRINTK("%s: phy%d: unknown event:0x%x\n", __FUNCTION__,
+			    phy_id, sb_opcode);
+		ASD_DPRINTK("edb is 0x%x! dl->opcode is 0x%x\n",
+			    edb, dl->opcode);
+		ASD_DPRINTK("sb_opcode : 0x%x, phy_id: 0x%x\n",
+			    sb_opcode, phy_id);
+		ASD_DPRINTK("escb: vaddr: 0x%p, "
+			    "dma_handle: 0x%llx, next: 0x%llx, "
+			    "index:%d, opcode:0x%02x\n",
+			    ascb->dma_scb.vaddr,
+			    (unsigned long long)ascb->dma_scb.dma_handle,
+			    (unsigned long long)
+			    le64_to_cpu(ascb->scb->header.next_scb),
+			    le16_to_cpu(ascb->scb->header.index),
+			    ascb->scb->header.opcode);
+
+		break;
+	}
+
+	asd_invalidate_edb(ascb, edb);
+}
+
+int asd_init_post_escbs(struct asd_ha_struct *asd_ha)
+{
+	struct asd_seq_data *seq = &asd_ha->seq;
+	int i;
+
+	for (i = 0; i < seq->num_escbs; i++)
+		seq->escb_arr[i]->tasklet_complete = escb_tasklet_complete;
+
+	ASD_DPRINTK("posting %d escbs\n", i);
+	return asd_post_escb_list(asd_ha, seq->escb_arr[0], seq->num_escbs);
+}
+
+/* ---------- CONTROL PHY ---------- */
+
+#define CONTROL_PHY_STATUS (CURRENT_DEVICE_PRESENT | CURRENT_OOB_DONE   \
+			    | CURRENT_SPINUP_HOLD | CURRENT_GTO_TIMEOUT \
+			    | CURRENT_OOB_ERROR)
+
+/**
+ * control_phy_tasklet_complete -- tasklet complete for CONTROL PHY ascb
+ * @ascb: pointer to an ascb
+ * @dl: pointer to the done list entry
+ *
+ * This function completes a CONTROL PHY scb and frees the ascb.
+ * A note on LEDs:
+ *  - an LED blinks if there is IO though it,
+ *  - if a device is connected to the LED, it is lit,
+ *  - if no device is connected to the LED, is is dimmed (off).
+ */
+static void control_phy_tasklet_complete(struct asd_ascb *ascb,
+					 struct done_list_struct *dl)
+{
+	struct asd_ha_struct *asd_ha = ascb->ha;
+	struct scb *scb = ascb->scb;
+	struct control_phy *control_phy = &scb->control_phy;
+	u8 phy_id = control_phy->phy_id;
+	struct asd_phy *phy = &ascb->ha->phys[phy_id];
+
+	u8 status     = dl->status_block[0];
+	u8 oob_status = dl->status_block[1];
+	u8 oob_mode   = dl->status_block[2];
+	/* u8 oob_signals= dl->status_block[3]; */
+
+	if (status != 0) {
+		ASD_DPRINTK("%s: phy%d status block opcode:0x%x\n",
+			    __FUNCTION__, phy_id, status);
+		goto out;
+	}
+
+	switch (control_phy->sub_func) {
+	case DISABLE_PHY:
+		asd_ha->hw_prof.enabled_phys &= ~(1 << phy_id);
+		asd_turn_led(asd_ha, phy_id, 0);
+		asd_control_led(asd_ha, phy_id, 0);
+		ASD_DPRINTK("%s: disable phy%d\n", __FUNCTION__, phy_id);
+		break;
+
+	case ENABLE_PHY:
+		asd_control_led(asd_ha, phy_id, 1);
+		if (oob_status & CURRENT_OOB_DONE) {
+			asd_ha->hw_prof.enabled_phys |= (1 << phy_id);
+			get_lrate_mode(phy, oob_mode);
+			asd_turn_led(asd_ha, phy_id, 1);
+			ASD_DPRINTK("%s: phy%d, lrate:0x%x, proto:0x%x\n",
+				    __FUNCTION__, phy_id,phy->sas_phy.linkrate,
+				    phy->sas_phy.iproto);
+		} else if (oob_status & CURRENT_SPINUP_HOLD) {
+			asd_ha->hw_prof.enabled_phys |= (1 << phy_id);
+			asd_turn_led(asd_ha, phy_id, 1);
+			ASD_DPRINTK("%s: phy%d, spinup hold\n", __FUNCTION__,
+				    phy_id);
+		} else if (oob_status & CURRENT_ERR_MASK) {
+			asd_turn_led(asd_ha, phy_id, 0);
+			ASD_DPRINTK("%s: phy%d: error: oob status:0x%02x\n",
+				    __FUNCTION__, phy_id, oob_status);
+		} else if (oob_status & (CURRENT_HOT_PLUG_CNCT
+					 | CURRENT_DEVICE_PRESENT))  {
+			asd_ha->hw_prof.enabled_phys |= (1 << phy_id);
+			asd_turn_led(asd_ha, phy_id, 1);
+			ASD_DPRINTK("%s: phy%d: hot plug or device present\n",
+				    __FUNCTION__, phy_id);
+		} else {
+			asd_ha->hw_prof.enabled_phys |= (1 << phy_id);
+			asd_turn_led(asd_ha, phy_id, 0);
+			ASD_DPRINTK("%s: phy%d: no device present: "
+				    "oob_status:0x%x\n",
+				    __FUNCTION__, phy_id, oob_status);
+		}
+		break;
+	case RELEASE_SPINUP_HOLD:
+	case PHY_NO_OP:
+	case EXECUTE_HARD_RESET:
+		ASD_DPRINTK("%s: phy%d: sub_func:0x%x\n", __FUNCTION__,
+			    phy_id, control_phy->sub_func);
+		/* XXX finish */
+		break;
+	default:
+		ASD_DPRINTK("%s: phy%d: sub_func:0x%x?\n", __FUNCTION__,
+			    phy_id, control_phy->sub_func);
+		break;
+	}
+out:
+	asd_ascb_free(ascb);
+}
+
+static inline void set_speed_mask(u8 *speed_mask, struct asd_phy_desc *pd)
+{
+	/* disable all speeds, then enable defaults */
+	*speed_mask = SAS_SPEED_60_DIS | SAS_SPEED_30_DIS | SAS_SPEED_15_DIS
+		| SATA_SPEED_30_DIS | SATA_SPEED_15_DIS;
+
+	switch (pd->max_sas_lrate) {
+	case SAS_LINK_RATE_6_0_GBPS:
+		*speed_mask &= ~SAS_SPEED_60_DIS;
+	default:
+	case SAS_LINK_RATE_3_0_GBPS:
+		*speed_mask &= ~SAS_SPEED_30_DIS;
+	case SAS_LINK_RATE_1_5_GBPS:
+		*speed_mask &= ~SAS_SPEED_15_DIS;
+	}
+
+	switch (pd->min_sas_lrate) {
+	case SAS_LINK_RATE_6_0_GBPS:
+		*speed_mask |= SAS_SPEED_30_DIS;
+	case SAS_LINK_RATE_3_0_GBPS:
+		*speed_mask |= SAS_SPEED_15_DIS;
+	default:
+	case SAS_LINK_RATE_1_5_GBPS:
+		/* nothing to do */
+		;
+	}
+
+	switch (pd->max_sata_lrate) {
+	case SAS_LINK_RATE_3_0_GBPS:
+		*speed_mask &= ~SATA_SPEED_30_DIS;
+	default:
+	case SAS_LINK_RATE_1_5_GBPS:
+		*speed_mask &= ~SATA_SPEED_15_DIS;
+	}
+
+	switch (pd->min_sata_lrate) {
+	case SAS_LINK_RATE_3_0_GBPS:
+		*speed_mask |= SATA_SPEED_15_DIS;
+	default:
+	case SAS_LINK_RATE_1_5_GBPS:
+		/* nothing to do */
+		;
+	}
+}
+
+/**
+ * asd_build_control_phy -- build a CONTROL PHY SCB
+ * @ascb: pointer to an ascb
+ * @phy_id: phy id to control, integer
+ * @subfunc: subfunction, what to actually to do the phy
+ *
+ * This function builds a CONTROL PHY scb.  No allocation of any kind
+ * is performed. @ascb is allocated with the list function.
+ * The caller can override the ascb->tasklet_complete to point
+ * to its own callback function.  It must call asd_ascb_free()
+ * at its tasklet complete function.
+ * See the default implementation.
+ */
+void asd_build_control_phy(struct asd_ascb *ascb, int phy_id, u8 subfunc)
+{
+	struct asd_phy *phy = &ascb->ha->phys[phy_id];
+	struct scb *scb = ascb->scb;
+	struct control_phy *control_phy = &scb->control_phy;
+
+	scb->header.opcode = CONTROL_PHY;
+	control_phy->phy_id = (u8) phy_id;
+	control_phy->sub_func = subfunc;
+
+	switch (subfunc) {
+	case EXECUTE_HARD_RESET:  /* 0x81 */
+	case ENABLE_PHY:          /* 0x01 */
+		/* decide hot plug delay */
+		control_phy->hot_plug_delay = HOTPLUG_DELAY_TIMEOUT;
+
+		/* decide speed mask */
+		set_speed_mask(&control_phy->speed_mask, phy->phy_desc);
+
+		/* initiator port settings are in the hi nibble */
+		if (phy->sas_phy.role == PHY_ROLE_INITIATOR)
+			control_phy->port_type = SAS_PROTO_ALL << 4;
+		else if (phy->sas_phy.role == PHY_ROLE_TARGET)
+			control_phy->port_type = SAS_PROTO_ALL;
+		else
+			control_phy->port_type =
+				(SAS_PROTO_ALL << 4) | SAS_PROTO_ALL;
+
+		/* link reset retries, this should be nominal */
+		control_phy->link_reset_retries = 10;
+
+	case RELEASE_SPINUP_HOLD: /* 0x02 */
+		/* decide the func_mask */
+		control_phy->func_mask = FUNCTION_MASK_DEFAULT;
+		if (phy->phy_desc->flags & ASD_SATA_SPINUP_HOLD)
+			control_phy->func_mask &= ~SPINUP_HOLD_DIS;
+		else
+			control_phy->func_mask |= SPINUP_HOLD_DIS;
+	}
+
+	control_phy->conn_handle = cpu_to_le16(0xFFFF);
+
+	ascb->tasklet_complete = control_phy_tasklet_complete;
+}
+
+/* ---------- INITIATE LINK ADM TASK ---------- */
+
+static void link_adm_tasklet_complete(struct asd_ascb *ascb,
+				      struct done_list_struct *dl)
+{
+	u8 opcode = dl->opcode;
+	struct initiate_link_adm *link_adm = &ascb->scb->link_adm;
+	u8 phy_id = link_adm->phy_id;
+
+	if (opcode != TC_NO_ERROR) {
+		asd_printk("phy%d: link adm task 0x%x completed with error "
+			   "0x%x\n", phy_id, link_adm->sub_func, opcode);
+	}
+	ASD_DPRINTK("phy%d: link adm task 0x%x: 0x%x\n",
+		    phy_id, link_adm->sub_func, opcode);
+
+	asd_ascb_free(ascb);
+}
+
+void asd_build_initiate_link_adm_task(struct asd_ascb *ascb, int phy_id,
+				      u8 subfunc)
+{
+	struct scb *scb = ascb->scb;
+	struct initiate_link_adm *link_adm = &scb->link_adm;
+
+	scb->header.opcode = INITIATE_LINK_ADM_TASK;
+
+	link_adm->phy_id = phy_id;
+	link_adm->sub_func = subfunc;
+	link_adm->conn_handle = cpu_to_le16(0xFFFF);
+
+	ascb->tasklet_complete = link_adm_tasklet_complete;
+}
+
+/* ---------- SCB timer ---------- */
+
+/**
+ * asd_ascb_timedout -- called when a pending SCB's timer has expired
+ * @data: unsigned long, a pointer to the ascb in question
+ *
+ * This is the default timeout function which does the most necessary.
+ * Upper layers can implement their own timeout function, say to free
+ * resources they have with this SCB, and then call this one at the
+ * end of their timeout function.  To do this, one should initialize
+ * the ascb->timer.{function, data, expires} prior to calling the post
+ * funcion.  The timer is started by the post function.
+ */
+void asd_ascb_timedout(unsigned long data)
+{
+	struct asd_ascb *ascb = (void *) data;
+	struct asd_seq_data *seq = &ascb->ha->seq;
+	unsigned long flags;
+
+	ASD_DPRINTK("scb:0x%x timed out\n", ascb->scb->header.opcode);
+
+	spin_lock_irqsave(&seq->pend_q_lock, flags);
+	seq->pending--;
+	list_del_init(&ascb->list);
+	spin_unlock_irqrestore(&seq->pend_q_lock, flags);
+
+	asd_ascb_free(ascb);
+}
+
+/* ---------- CONTROL PHY ---------- */
+
+/* Given the spec value, return a driver value. */
+static const int phy_func_table[] = {
+	[PHY_FUNC_NOP]        = PHY_NO_OP,
+	[PHY_FUNC_LINK_RESET] = ENABLE_PHY,
+	[PHY_FUNC_HARD_RESET] = EXECUTE_HARD_RESET,
+	[PHY_FUNC_DISABLE]    = DISABLE_PHY,
+	[PHY_FUNC_RELEASE_SPINUP_HOLD] = RELEASE_SPINUP_HOLD,
+};
+
+int asd_control_phy(struct asd_sas_phy *phy, enum phy_func func, void *arg)
+{
+	struct asd_ha_struct *asd_ha = phy->ha->lldd_ha;
+	struct asd_phy_desc *pd = asd_ha->phys[phy->id].phy_desc;
+	struct asd_ascb *ascb;
+	struct sas_phy_linkrates *rates;
+	int res = 1;
+
+	switch (func) {
+	case PHY_FUNC_CLEAR_ERROR_LOG:
+		return -ENOSYS;
+	case PHY_FUNC_SET_LINK_RATE:
+		rates = arg;
+		if (rates->minimum_linkrate) {
+			pd->min_sas_lrate = rates->minimum_linkrate;
+			pd->min_sata_lrate = rates->minimum_linkrate;
+		}
+		if (rates->maximum_linkrate) {
+			pd->max_sas_lrate = rates->maximum_linkrate;
+			pd->max_sata_lrate = rates->maximum_linkrate;
+		}
+		func = PHY_FUNC_LINK_RESET;
+		break;
+	default:
+		break;
+	}
+
+	ascb = asd_ascb_alloc_list(asd_ha, &res, GFP_KERNEL);
+	if (!ascb)
+		return -ENOMEM;
+
+	asd_build_control_phy(ascb, phy->id, phy_func_table[func]);
+	res = asd_post_ascb_list(asd_ha, ascb , 1);
+	if (res)
+		asd_ascb_free(ascb);
+
+	return res;
+}
diff --git a/drivers/scsi/aic94xx/aic94xx_sds.c b/drivers/scsi/aic94xx/aic94xx_sds.c
new file mode 100644
index 0000000..83574b5
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_sds.c
@@ -0,0 +1,1089 @@
+/*
+ * Aic94xx SAS/SATA driver access to shared data structures and memory
+ * maps.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/delay.h>
+
+#include "aic94xx.h"
+#include "aic94xx_reg.h"
+
+/* ---------- OCM stuff ---------- */
+
+struct asd_ocm_dir_ent {
+	u8 type;
+	u8 offs[3];
+	u8 _r1;
+	u8 size[3];
+} __attribute__ ((packed));
+
+struct asd_ocm_dir {
+	char sig[2];
+	u8   _r1[2];
+	u8   major;          /* 0 */
+	u8   minor;          /* 0 */
+	u8   _r2;
+	u8   num_de;
+	struct asd_ocm_dir_ent entry[15];
+} __attribute__ ((packed));
+
+#define	OCM_DE_OCM_DIR			0x00
+#define	OCM_DE_WIN_DRVR			0x01
+#define	OCM_DE_BIOS_CHIM		0x02
+#define	OCM_DE_RAID_ENGN		0x03
+#define	OCM_DE_BIOS_INTL		0x04
+#define	OCM_DE_BIOS_CHIM_OSM		0x05
+#define	OCM_DE_BIOS_CHIM_DYNAMIC	0x06
+#define	OCM_DE_ADDC2C_RES0		0x07
+#define	OCM_DE_ADDC2C_RES1		0x08
+#define	OCM_DE_ADDC2C_RES2		0x09
+#define	OCM_DE_ADDC2C_RES3		0x0A
+
+#define OCM_INIT_DIR_ENTRIES	5
+/***************************************************************************
+*  OCM dircetory default
+***************************************************************************/
+static struct asd_ocm_dir OCMDirInit =
+{
+	.sig = {0x4D, 0x4F},	/* signature */
+	.num_de = OCM_INIT_DIR_ENTRIES,	/* no. of directory entries */
+};
+
+/***************************************************************************
+*  OCM dircetory Entries default
+***************************************************************************/
+static struct asd_ocm_dir_ent OCMDirEntriesInit[OCM_INIT_DIR_ENTRIES] =
+{
+	{
+		.type = (OCM_DE_ADDC2C_RES0),	/* Entry type  */
+		.offs = {128},			/* Offset */
+		.size = {0, 4},			/* size */
+	},
+	{
+		.type = (OCM_DE_ADDC2C_RES1),	/* Entry type  */
+		.offs = {128, 4},		/* Offset */
+		.size = {0, 4},			/* size */
+	},
+	{
+		.type = (OCM_DE_ADDC2C_RES2),	/* Entry type  */
+		.offs = {128, 8},		/* Offset */
+		.size = {0, 4},			/* size */
+	},
+	{
+		.type = (OCM_DE_ADDC2C_RES3),	/* Entry type  */
+		.offs = {128, 12},		/* Offset */
+		.size = {0, 4},			/* size */
+	},
+	{
+		.type = (OCM_DE_WIN_DRVR),	/* Entry type  */
+		.offs = {128, 16},		/* Offset */
+		.size = {128, 235, 1},		/* size */
+	},
+};
+
+struct asd_bios_chim_struct {
+	char sig[4];
+	u8   major;          /* 1 */
+	u8   minor;          /* 0 */
+	u8   bios_major;
+	u8   bios_minor;
+	__le32  bios_build;
+	u8   flags;
+	u8   pci_slot;
+	__le16  ue_num;
+	__le16  ue_size;
+	u8  _r[14];
+	/* The unit element array is right here.
+	 */
+} __attribute__ ((packed));
+
+/**
+ * asd_read_ocm_seg - read an on chip memory (OCM) segment
+ * @asd_ha: pointer to the host adapter structure
+ * @buffer: where to write the read data
+ * @offs: offset into OCM where to read from
+ * @size: how many bytes to read
+ *
+ * Return the number of bytes not read. Return 0 on success.
+ */
+static int asd_read_ocm_seg(struct asd_ha_struct *asd_ha, void *buffer,
+			    u32 offs, int size)
+{
+	u8 *p = buffer;
+	if (unlikely(asd_ha->iospace))
+		asd_read_reg_string(asd_ha, buffer, offs+OCM_BASE_ADDR, size);
+	else {
+		for ( ; size > 0; size--, offs++, p++)
+			*p = asd_read_ocm_byte(asd_ha, offs);
+	}
+	return size;
+}
+
+static int asd_read_ocm_dir(struct asd_ha_struct *asd_ha,
+			    struct asd_ocm_dir *dir, u32 offs)
+{
+	int err = asd_read_ocm_seg(asd_ha, dir, offs, sizeof(*dir));
+	if (err) {
+		ASD_DPRINTK("couldn't read ocm segment\n");
+		return err;
+	}
+
+	if (dir->sig[0] != 'M' || dir->sig[1] != 'O') {
+		ASD_DPRINTK("no valid dir signature(%c%c) at start of OCM\n",
+			    dir->sig[0], dir->sig[1]);
+		return -ENOENT;
+	}
+	if (dir->major != 0) {
+		asd_printk("unsupported major version of ocm dir:0x%x\n",
+			   dir->major);
+		return -ENOENT;
+	}
+	dir->num_de &= 0xf;
+	return 0;
+}
+
+/**
+ * asd_write_ocm_seg - write an on chip memory (OCM) segment
+ * @asd_ha: pointer to the host adapter structure
+ * @buffer: where to read the write data
+ * @offs: offset into OCM to write to
+ * @size: how many bytes to write
+ *
+ * Return the number of bytes not written. Return 0 on success.
+ */
+static void asd_write_ocm_seg(struct asd_ha_struct *asd_ha, void *buffer,
+			    u32 offs, int size)
+{
+	u8 *p = buffer;
+	if (unlikely(asd_ha->iospace))
+		asd_write_reg_string(asd_ha, buffer, offs+OCM_BASE_ADDR, size);
+	else {
+		for ( ; size > 0; size--, offs++, p++)
+			asd_write_ocm_byte(asd_ha, offs, *p);
+	}
+	return;
+}
+
+#define THREE_TO_NUM(X) ((X)[0] | ((X)[1] << 8) | ((X)[2] << 16))
+
+static int asd_find_dir_entry(struct asd_ocm_dir *dir, u8 type,
+			      u32 *offs, u32 *size)
+{
+	int i;
+	struct asd_ocm_dir_ent *ent;
+
+	for (i = 0; i < dir->num_de; i++) {
+		if (dir->entry[i].type == type)
+			break;
+	}
+	if (i >= dir->num_de)
+		return -ENOENT;
+	ent = &dir->entry[i];
+	*offs = (u32) THREE_TO_NUM(ent->offs);
+	*size = (u32) THREE_TO_NUM(ent->size);
+	return 0;
+}
+
+#define OCM_BIOS_CHIM_DE  2
+#define BC_BIOS_PRESENT   1
+
+static int asd_get_bios_chim(struct asd_ha_struct *asd_ha,
+			     struct asd_ocm_dir *dir)
+{
+	int err;
+	struct asd_bios_chim_struct *bc_struct;
+	u32 offs, size;
+
+	err = asd_find_dir_entry(dir, OCM_BIOS_CHIM_DE, &offs, &size);
+	if (err) {
+		ASD_DPRINTK("couldn't find BIOS_CHIM dir ent\n");
+		goto out;
+	}
+	err = -ENOMEM;
+	bc_struct = kmalloc(sizeof(*bc_struct), GFP_KERNEL);
+	if (!bc_struct) {
+		asd_printk("no memory for bios_chim struct\n");
+		goto out;
+	}
+	err = asd_read_ocm_seg(asd_ha, (void *)bc_struct, offs,
+			       sizeof(*bc_struct));
+	if (err) {
+		ASD_DPRINTK("couldn't read ocm segment\n");
+		goto out2;
+	}
+	if (strncmp(bc_struct->sig, "SOIB", 4)
+	    && strncmp(bc_struct->sig, "IPSA", 4)) {
+		ASD_DPRINTK("BIOS_CHIM entry has no valid sig(%c%c%c%c)\n",
+			    bc_struct->sig[0], bc_struct->sig[1],
+			    bc_struct->sig[2], bc_struct->sig[3]);
+		err = -ENOENT;
+		goto out2;
+	}
+	if (bc_struct->major != 1) {
+		asd_printk("BIOS_CHIM unsupported major version:0x%x\n",
+			   bc_struct->major);
+		err = -ENOENT;
+		goto out2;
+	}
+	if (bc_struct->flags & BC_BIOS_PRESENT) {
+		asd_ha->hw_prof.bios.present = 1;
+		asd_ha->hw_prof.bios.maj = bc_struct->bios_major;
+		asd_ha->hw_prof.bios.min = bc_struct->bios_minor;
+		asd_ha->hw_prof.bios.bld = le32_to_cpu(bc_struct->bios_build);
+		ASD_DPRINTK("BIOS present (%d,%d), %d\n",
+			    asd_ha->hw_prof.bios.maj,
+			    asd_ha->hw_prof.bios.min,
+			    asd_ha->hw_prof.bios.bld);
+	}
+	asd_ha->hw_prof.ue.num = le16_to_cpu(bc_struct->ue_num);
+	asd_ha->hw_prof.ue.size= le16_to_cpu(bc_struct->ue_size);
+	ASD_DPRINTK("ue num:%d, ue size:%d\n", asd_ha->hw_prof.ue.num,
+		    asd_ha->hw_prof.ue.size);
+	size = asd_ha->hw_prof.ue.num * asd_ha->hw_prof.ue.size;
+	if (size > 0) {
+		err = -ENOMEM;
+		asd_ha->hw_prof.ue.area = kmalloc(size, GFP_KERNEL);
+		if (!asd_ha->hw_prof.ue.area)
+			goto out2;
+		err = asd_read_ocm_seg(asd_ha, (void *)asd_ha->hw_prof.ue.area,
+				       offs + sizeof(*bc_struct), size);
+		if (err) {
+			kfree(asd_ha->hw_prof.ue.area);
+			asd_ha->hw_prof.ue.area = NULL;
+			asd_ha->hw_prof.ue.num  = 0;
+			asd_ha->hw_prof.ue.size = 0;
+			ASD_DPRINTK("couldn't read ue entries(%d)\n", err);
+		}
+	}
+out2:
+	kfree(bc_struct);
+out:
+	return err;
+}
+
+static void
+asd_hwi_initialize_ocm_dir (struct asd_ha_struct *asd_ha)
+{
+	int i;
+
+	/* Zero OCM */
+	for (i = 0; i < OCM_MAX_SIZE; i += 4)
+		asd_write_ocm_dword(asd_ha, i, 0);
+
+	/* Write Dir */
+	asd_write_ocm_seg(asd_ha, &OCMDirInit, 0,
+			  sizeof(struct asd_ocm_dir));
+
+	/* Write Dir Entries */
+	for (i = 0; i < OCM_INIT_DIR_ENTRIES; i++)
+		asd_write_ocm_seg(asd_ha, &OCMDirEntriesInit[i],
+				  sizeof(struct asd_ocm_dir) +
+				  (i * sizeof(struct asd_ocm_dir_ent))
+				  , sizeof(struct asd_ocm_dir_ent));
+
+}
+
+static int
+asd_hwi_check_ocm_access (struct asd_ha_struct *asd_ha)
+{
+	struct pci_dev *pcidev = asd_ha->pcidev;
+	u32 reg;
+	int err = 0;
+	u32 v;
+
+	/* check if OCM has been initialized by BIOS */
+	reg = asd_read_reg_dword(asd_ha, EXSICNFGR);
+
+	if (!(reg & OCMINITIALIZED)) {
+		err = pci_read_config_dword(pcidev, PCIC_INTRPT_STAT, &v);
+		if (err) {
+			asd_printk("couldn't access PCIC_INTRPT_STAT of %s\n",
+					pci_name(pcidev));
+			goto out;
+		}
+
+		printk(KERN_INFO "OCM is not initialized by BIOS,"
+		       "reinitialize it and ignore it, current IntrptStatus"
+		       "is 0x%x\n", v);
+
+		if (v)
+			err = pci_write_config_dword(pcidev,
+						     PCIC_INTRPT_STAT, v);
+		if (err) {
+			asd_printk("couldn't write PCIC_INTRPT_STAT of %s\n",
+					pci_name(pcidev));
+			goto out;
+		}
+
+		asd_hwi_initialize_ocm_dir(asd_ha);
+
+	}
+out:
+	return err;
+}
+
+/**
+ * asd_read_ocm - read on chip memory (OCM)
+ * @asd_ha: pointer to the host adapter structure
+ */
+int asd_read_ocm(struct asd_ha_struct *asd_ha)
+{
+	int err;
+	struct asd_ocm_dir *dir;
+
+	if (asd_hwi_check_ocm_access(asd_ha))
+		return -1;
+
+	dir = kmalloc(sizeof(*dir), GFP_KERNEL);
+	if (!dir) {
+		asd_printk("no memory for ocm dir\n");
+		return -ENOMEM;
+	}
+
+	err = asd_read_ocm_dir(asd_ha, dir, 0);
+	if (err)
+		goto out;
+
+	err = asd_get_bios_chim(asd_ha, dir);
+out:
+	kfree(dir);
+	return err;
+}
+
+/* ---------- FLASH stuff ---------- */
+
+#define FLASH_RESET			0xF0
+
+#define FLASH_SIZE                      0x200000
+#define FLASH_DIR_COOKIE                "*** ADAPTEC FLASH DIRECTORY *** "
+#define FLASH_NEXT_ENTRY_OFFS		0x2000
+#define FLASH_MAX_DIR_ENTRIES		32
+
+#define FLASH_DE_TYPE_MASK              0x3FFFFFFF
+#define FLASH_DE_MS                     0x120
+#define FLASH_DE_CTRL_A_USER            0xE0
+
+struct asd_flash_de {
+	__le32   type;
+	__le32   offs;
+	__le32   pad_size;
+	__le32   image_size;
+	__le32   chksum;
+	u8       _r[12];
+	u8       version[32];
+} __attribute__ ((packed));
+
+struct asd_flash_dir {
+	u8    cookie[32];
+	__le32   rev;		  /* 2 */
+	__le32   chksum;
+	__le32   chksum_antidote;
+	__le32   bld;
+	u8    bld_id[32];	  /* build id data */
+	u8    ver_data[32];	  /* date and time of build */
+	__le32   ae_mask;
+	__le32   v_mask;
+	__le32   oc_mask;
+	u8    _r[20];
+	struct asd_flash_de dir_entry[FLASH_MAX_DIR_ENTRIES];
+} __attribute__ ((packed));
+
+struct asd_manuf_sec {
+	char  sig[2];		  /* 'S', 'M' */
+	u16   offs_next;
+	u8    maj;           /* 0 */
+	u8    min;           /* 0 */
+	u16   chksum;
+	u16   size;
+	u8    _r[6];
+	u8    sas_addr[SAS_ADDR_SIZE];
+	u8    pcba_sn[ASD_PCBA_SN_SIZE];
+	/* Here start the other segments */
+	u8    linked_list[0];
+} __attribute__ ((packed));
+
+struct asd_manuf_phy_desc {
+	u8    state;         /* low 4 bits */
+#define MS_PHY_STATE_ENABLEABLE 0
+#define MS_PHY_STATE_REPORTED   1
+#define MS_PHY_STATE_HIDDEN     2
+	u8    phy_id;
+	u16   _r;
+	u8    phy_control_0; /* mode 5 reg 0x160 */
+	u8    phy_control_1; /* mode 5 reg 0x161 */
+	u8    phy_control_2; /* mode 5 reg 0x162 */
+	u8    phy_control_3; /* mode 5 reg 0x163 */
+} __attribute__ ((packed));
+
+struct asd_manuf_phy_param {
+	char  sig[2];		  /* 'P', 'M' */
+	u16   next;
+	u8    maj;           /* 0 */
+	u8    min;           /* 2 */
+	u8    num_phy_desc;  /* 8 */
+	u8    phy_desc_size; /* 8 */
+	u8    _r[3];
+	u8    usage_model_id;
+	u32   _r2;
+	struct asd_manuf_phy_desc phy_desc[ASD_MAX_PHYS];
+} __attribute__ ((packed));
+
+#if 0
+static const char *asd_sb_type[] = {
+	"unknown",
+	"SGPIO",
+	[2 ... 0x7F] = "unknown",
+	[0x80] = "ADPT_I2C",
+	[0x81 ... 0xFF] = "VENDOR_UNIQUExx"
+};
+#endif
+
+struct asd_ms_sb_desc {
+	u8    type;
+	u8    node_desc_index;
+	u8    conn_desc_index;
+	u8    _recvd[0];
+} __attribute__ ((packed));
+
+#if 0
+static const char *asd_conn_type[] = {
+	[0 ... 7] = "unknown",
+	"SFF8470",
+	"SFF8482",
+	"SFF8484",
+	[0x80] = "PCIX_DAUGHTER0",
+	[0x81] = "SAS_DAUGHTER0",
+	[0x82 ... 0xFF] = "VENDOR_UNIQUExx"
+};
+
+static const char *asd_conn_location[] = {
+	"unknown",
+	"internal",
+	"external",
+	"board_to_board",
+};
+#endif
+
+struct asd_ms_conn_desc {
+	u8    type;
+	u8    location;
+	u8    num_sideband_desc;
+	u8    size_sideband_desc;
+	u32   _resvd;
+	u8    name[16];
+	struct asd_ms_sb_desc sb_desc[0];
+} __attribute__ ((packed));
+
+struct asd_nd_phy_desc {
+	u8    vp_attch_type;
+	u8    attch_specific[0];
+} __attribute__ ((packed));
+
+#if 0
+static const char *asd_node_type[] = {
+	"IOP",
+	"IO_CONTROLLER",
+	"EXPANDER",
+	"PORT_MULTIPLIER",
+	"PORT_MULTIPLEXER",
+	"MULTI_DROP_I2C_BUS",
+};
+#endif
+
+struct asd_ms_node_desc {
+	u8    type;
+	u8    num_phy_desc;
+	u8    size_phy_desc;
+	u8    _resvd;
+	u8    name[16];
+	struct asd_nd_phy_desc phy_desc[0];
+} __attribute__ ((packed));
+
+struct asd_ms_conn_map {
+	char  sig[2];		  /* 'M', 'C' */
+	__le16 next;
+	u8    maj;		  /* 0 */
+	u8    min;		  /* 0 */
+	__le16 cm_size;		  /* size of this struct */
+	u8    num_conn;
+	u8    conn_size;
+	u8    num_nodes;
+	u8    usage_model_id;
+	u32   _resvd;
+	struct asd_ms_conn_desc conn_desc[0];
+	struct asd_ms_node_desc node_desc[0];
+} __attribute__ ((packed));
+
+struct asd_ctrla_phy_entry {
+	u8    sas_addr[SAS_ADDR_SIZE];
+	u8    sas_link_rates;  /* max in hi bits, min in low bits */
+	u8    flags;
+	u8    sata_link_rates;
+	u8    _r[5];
+} __attribute__ ((packed));
+
+struct asd_ctrla_phy_settings {
+	u8    id0;		  /* P'h'y */
+	u8    _r;
+	u16   next;
+	u8    num_phys;	      /* number of PHYs in the PCI function */
+	u8    _r2[3];
+	struct asd_ctrla_phy_entry phy_ent[ASD_MAX_PHYS];
+} __attribute__ ((packed));
+
+struct asd_ll_el {
+	u8   id0;
+	u8   id1;
+	__le16  next;
+	u8   something_here[0];
+} __attribute__ ((packed));
+
+static int asd_poll_flash(struct asd_ha_struct *asd_ha)
+{
+	int c;
+	u8 d;
+
+	for (c = 5000; c > 0; c--) {
+		d  = asd_read_reg_byte(asd_ha, asd_ha->hw_prof.flash.bar);
+		d ^= asd_read_reg_byte(asd_ha, asd_ha->hw_prof.flash.bar);
+		if (!d)
+			return 0;
+		udelay(5);
+	}
+	return -ENOENT;
+}
+
+static int asd_reset_flash(struct asd_ha_struct *asd_ha)
+{
+	int err;
+
+	err = asd_poll_flash(asd_ha);
+	if (err)
+		return err;
+	asd_write_reg_byte(asd_ha, asd_ha->hw_prof.flash.bar, FLASH_RESET);
+	err = asd_poll_flash(asd_ha);
+
+	return err;
+}
+
+static inline int asd_read_flash_seg(struct asd_ha_struct *asd_ha,
+				     void *buffer, u32 offs, int size)
+{
+	asd_read_reg_string(asd_ha, buffer, asd_ha->hw_prof.flash.bar+offs,
+			    size);
+	return 0;
+}
+
+/**
+ * asd_find_flash_dir - finds and reads the flash directory
+ * @asd_ha: pointer to the host adapter structure
+ * @flash_dir: pointer to flash directory structure
+ *
+ * If found, the flash directory segment will be copied to
+ * @flash_dir.  Return 1 if found, 0 if not.
+ */
+static int asd_find_flash_dir(struct asd_ha_struct *asd_ha,
+			      struct asd_flash_dir *flash_dir)
+{
+	u32 v;
+	for (v = 0; v < FLASH_SIZE; v += FLASH_NEXT_ENTRY_OFFS) {
+		asd_read_flash_seg(asd_ha, flash_dir, v,
+				   sizeof(FLASH_DIR_COOKIE)-1);
+		if (memcmp(flash_dir->cookie, FLASH_DIR_COOKIE,
+			   sizeof(FLASH_DIR_COOKIE)-1) == 0) {
+			asd_ha->hw_prof.flash.dir_offs = v;
+			asd_read_flash_seg(asd_ha, flash_dir, v,
+					   sizeof(*flash_dir));
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static int asd_flash_getid(struct asd_ha_struct *asd_ha)
+{
+	int err = 0;
+	u32 reg;
+
+	reg = asd_read_reg_dword(asd_ha, EXSICNFGR);
+
+	if (!(reg & FLASHEX)) {
+		ASD_DPRINTK("flash doesn't exist\n");
+		return -ENOENT;
+	}
+	if (pci_read_config_dword(asd_ha->pcidev, PCI_CONF_FLSH_BAR,
+				  &asd_ha->hw_prof.flash.bar)) {
+		asd_printk("couldn't read PCI_CONF_FLSH_BAR of %s\n",
+			   pci_name(asd_ha->pcidev));
+		return -ENOENT;
+	}
+	asd_ha->hw_prof.flash.present = 1;
+	asd_ha->hw_prof.flash.wide = reg & FLASHW ? 1 : 0;
+	err = asd_reset_flash(asd_ha);
+	if (err) {
+		ASD_DPRINTK("couldn't reset flash(%d)\n", err);
+		return err;
+	}
+	return 0;
+}
+
+static u16 asd_calc_flash_chksum(u16 *p, int size)
+{
+	u16 chksum = 0;
+
+	while (size-- > 0)
+		chksum += *p++;
+
+	return chksum;
+}
+
+
+static int asd_find_flash_de(struct asd_flash_dir *flash_dir, u32 entry_type,
+			     u32 *offs, u32 *size)
+{
+	int i;
+	struct asd_flash_de *de;
+
+	for (i = 0; i < FLASH_MAX_DIR_ENTRIES; i++) {
+		u32 type = le32_to_cpu(flash_dir->dir_entry[i].type);
+
+		type &= FLASH_DE_TYPE_MASK;
+		if (type == entry_type)
+			break;
+	}
+	if (i >= FLASH_MAX_DIR_ENTRIES)
+		return -ENOENT;
+	de = &flash_dir->dir_entry[i];
+	*offs = le32_to_cpu(de->offs);
+	*size = le32_to_cpu(de->pad_size);
+	return 0;
+}
+
+static int asd_validate_ms(struct asd_manuf_sec *ms)
+{
+	if (ms->sig[0] != 'S' || ms->sig[1] != 'M') {
+		ASD_DPRINTK("manuf sec: no valid sig(%c%c)\n",
+			    ms->sig[0], ms->sig[1]);
+		return -ENOENT;
+	}
+	if (ms->maj != 0) {
+		asd_printk("unsupported manuf. sector. major version:%x\n",
+			   ms->maj);
+		return -ENOENT;
+	}
+	ms->offs_next = le16_to_cpu((__force __le16) ms->offs_next);
+	ms->chksum = le16_to_cpu((__force __le16) ms->chksum);
+	ms->size = le16_to_cpu((__force __le16) ms->size);
+
+	if (asd_calc_flash_chksum((u16 *)ms, ms->size/2)) {
+		asd_printk("failed manuf sector checksum\n");
+	}
+
+	return 0;
+}
+
+static int asd_ms_get_sas_addr(struct asd_ha_struct *asd_ha,
+			       struct asd_manuf_sec *ms)
+{
+	memcpy(asd_ha->hw_prof.sas_addr, ms->sas_addr, SAS_ADDR_SIZE);
+	return 0;
+}
+
+static int asd_ms_get_pcba_sn(struct asd_ha_struct *asd_ha,
+			      struct asd_manuf_sec *ms)
+{
+	memcpy(asd_ha->hw_prof.pcba_sn, ms->pcba_sn, ASD_PCBA_SN_SIZE);
+	asd_ha->hw_prof.pcba_sn[ASD_PCBA_SN_SIZE] = '\0';
+	return 0;
+}
+
+/**
+ * asd_find_ll_by_id - find a linked list entry by its id
+ * @start: void pointer to the first element in the linked list
+ * @id0: the first byte of the id  (offs 0)
+ * @id1: the second byte of the id (offs 1)
+ *
+ * @start has to be the _base_ element start, since the
+ * linked list entries's offset is from this pointer.
+ * Some linked list entries use only the first id, in which case
+ * you can pass 0xFF for the second.
+ */
+static void *asd_find_ll_by_id(void * const start, const u8 id0, const u8 id1)
+{
+	struct asd_ll_el *el = start;
+
+	do {
+		switch (id1) {
+		default:
+			if (el->id1 == id1)
+		case 0xFF:
+				if (el->id0 == id0)
+					return el;
+		}
+		el = start + le16_to_cpu(el->next);
+	} while (el != start);
+
+	return NULL;
+}
+
+/**
+ * asd_ms_get_phy_params - get phy parameters from the manufacturing sector
+ * @asd_ha: pointer to the host adapter structure
+ * @manuf_sec: pointer to the manufacturing sector
+ *
+ * The manufacturing sector contans also the linked list of sub-segments,
+ * since when it was read, its size was taken from the flash directory,
+ * not from the structure size.
+ *
+ * HIDDEN phys do not count in the total count.  REPORTED phys cannot
+ * be enabled but are reported and counted towards the total.
+ * ENEBLEABLE phys are enabled by default and count towards the total.
+ * The absolute total phy number is ASD_MAX_PHYS.  hw_prof->num_phys
+ * merely specifies the number of phys the host adapter decided to
+ * report.  E.g., it is possible for phys 0, 1 and 2 to be HIDDEN,
+ * phys 3, 4 and 5 to be REPORTED and phys 6 and 7 to be ENEBLEABLE.
+ * In this case ASD_MAX_PHYS is 8, hw_prof->num_phys is 5, and only 2
+ * are actually enabled (enabled by default, max number of phys
+ * enableable in this case).
+ */
+static int asd_ms_get_phy_params(struct asd_ha_struct *asd_ha,
+				 struct asd_manuf_sec *manuf_sec)
+{
+	int i;
+	int en_phys = 0;
+	int rep_phys = 0;
+	struct asd_manuf_phy_param *phy_param;
+	struct asd_manuf_phy_param dflt_phy_param;
+
+	phy_param = asd_find_ll_by_id(manuf_sec, 'P', 'M');
+	if (!phy_param) {
+		ASD_DPRINTK("ms: no phy parameters found\n");
+		ASD_DPRINTK("ms: Creating default phy parameters\n");
+		dflt_phy_param.sig[0] = 'P';
+		dflt_phy_param.sig[1] = 'M';
+		dflt_phy_param.maj = 0;
+		dflt_phy_param.min = 2;
+		dflt_phy_param.num_phy_desc = 8;
+		dflt_phy_param.phy_desc_size = sizeof(struct asd_manuf_phy_desc);
+		for (i =0; i < ASD_MAX_PHYS; i++) {
+			dflt_phy_param.phy_desc[i].state = 0;
+			dflt_phy_param.phy_desc[i].phy_id = i;
+			dflt_phy_param.phy_desc[i].phy_control_0 = 0xf6;
+			dflt_phy_param.phy_desc[i].phy_control_1 = 0x10;
+			dflt_phy_param.phy_desc[i].phy_control_2 = 0x43;
+			dflt_phy_param.phy_desc[i].phy_control_3 = 0xeb;
+		}
+
+		phy_param = &dflt_phy_param;
+
+	}
+
+	if (phy_param->maj != 0) {
+		asd_printk("unsupported manuf. phy param major version:0x%x\n",
+			   phy_param->maj);
+		return -ENOENT;
+	}
+
+	ASD_DPRINTK("ms: num_phy_desc: %d\n", phy_param->num_phy_desc);
+	asd_ha->hw_prof.enabled_phys = 0;
+	for (i = 0; i < phy_param->num_phy_desc; i++) {
+		struct asd_manuf_phy_desc *pd = &phy_param->phy_desc[i];
+		switch (pd->state & 0xF) {
+		case MS_PHY_STATE_HIDDEN:
+			ASD_DPRINTK("ms: phy%d: HIDDEN\n", i);
+			continue;
+		case MS_PHY_STATE_REPORTED:
+			ASD_DPRINTK("ms: phy%d: REPORTED\n", i);
+			asd_ha->hw_prof.enabled_phys &= ~(1 << i);
+			rep_phys++;
+			continue;
+		case MS_PHY_STATE_ENABLEABLE:
+			ASD_DPRINTK("ms: phy%d: ENEBLEABLE\n", i);
+			asd_ha->hw_prof.enabled_phys |= (1 << i);
+			en_phys++;
+			break;
+		}
+		asd_ha->hw_prof.phy_desc[i].phy_control_0 = pd->phy_control_0;
+		asd_ha->hw_prof.phy_desc[i].phy_control_1 = pd->phy_control_1;
+		asd_ha->hw_prof.phy_desc[i].phy_control_2 = pd->phy_control_2;
+		asd_ha->hw_prof.phy_desc[i].phy_control_3 = pd->phy_control_3;
+	}
+	asd_ha->hw_prof.max_phys = rep_phys + en_phys;
+	asd_ha->hw_prof.num_phys = en_phys;
+	ASD_DPRINTK("ms: max_phys:0x%x, num_phys:0x%x\n",
+		    asd_ha->hw_prof.max_phys, asd_ha->hw_prof.num_phys);
+	ASD_DPRINTK("ms: enabled_phys:0x%x\n", asd_ha->hw_prof.enabled_phys);
+	return 0;
+}
+
+static int asd_ms_get_connector_map(struct asd_ha_struct *asd_ha,
+				    struct asd_manuf_sec *manuf_sec)
+{
+	struct asd_ms_conn_map *cm;
+
+	cm = asd_find_ll_by_id(manuf_sec, 'M', 'C');
+	if (!cm) {
+		ASD_DPRINTK("ms: no connector map found\n");
+		return 0;
+	}
+
+	if (cm->maj != 0) {
+		ASD_DPRINTK("ms: unsupported: connector map major version 0x%x"
+			    "\n", cm->maj);
+		return -ENOENT;
+	}
+
+	/* XXX */
+
+	return 0;
+}
+
+
+/**
+ * asd_process_ms - find and extract information from the manufacturing sector
+ * @asd_ha: pointer to the host adapter structure
+ * @flash_dir: pointer to the flash directory
+ */
+static int asd_process_ms(struct asd_ha_struct *asd_ha,
+			  struct asd_flash_dir *flash_dir)
+{
+	int err;
+	struct asd_manuf_sec *manuf_sec;
+	u32 offs, size;
+
+	err = asd_find_flash_de(flash_dir, FLASH_DE_MS, &offs, &size);
+	if (err) {
+		ASD_DPRINTK("Couldn't find the manuf. sector\n");
+		goto out;
+	}
+
+	if (size == 0)
+		goto out;
+
+	err = -ENOMEM;
+	manuf_sec = kmalloc(size, GFP_KERNEL);
+	if (!manuf_sec) {
+		ASD_DPRINTK("no mem for manuf sector\n");
+		goto out;
+	}
+
+	err = asd_read_flash_seg(asd_ha, (void *)manuf_sec, offs, size);
+	if (err) {
+		ASD_DPRINTK("couldn't read manuf sector at 0x%x, size 0x%x\n",
+			    offs, size);
+		goto out2;
+	}
+
+	err = asd_validate_ms(manuf_sec);
+	if (err) {
+		ASD_DPRINTK("couldn't validate manuf sector\n");
+		goto out2;
+	}
+
+	err = asd_ms_get_sas_addr(asd_ha, manuf_sec);
+	if (err) {
+		ASD_DPRINTK("couldn't read the SAS_ADDR\n");
+		goto out2;
+	}
+	ASD_DPRINTK("manuf sect SAS_ADDR %llx\n",
+		    SAS_ADDR(asd_ha->hw_prof.sas_addr));
+
+	err = asd_ms_get_pcba_sn(asd_ha, manuf_sec);
+	if (err) {
+		ASD_DPRINTK("couldn't read the PCBA SN\n");
+		goto out2;
+	}
+	ASD_DPRINTK("manuf sect PCBA SN %s\n", asd_ha->hw_prof.pcba_sn);
+
+	err = asd_ms_get_phy_params(asd_ha, manuf_sec);
+	if (err) {
+		ASD_DPRINTK("ms: couldn't get phy parameters\n");
+		goto out2;
+	}
+
+	err = asd_ms_get_connector_map(asd_ha, manuf_sec);
+	if (err) {
+		ASD_DPRINTK("ms: couldn't get connector map\n");
+		goto out2;
+	}
+
+out2:
+	kfree(manuf_sec);
+out:
+	return err;
+}
+
+static int asd_process_ctrla_phy_settings(struct asd_ha_struct *asd_ha,
+					  struct asd_ctrla_phy_settings *ps)
+{
+	int i;
+	for (i = 0; i < ps->num_phys; i++) {
+		struct asd_ctrla_phy_entry *pe = &ps->phy_ent[i];
+
+		if (!PHY_ENABLED(asd_ha, i))
+			continue;
+		if (*(u64 *)pe->sas_addr == 0) {
+			asd_ha->hw_prof.enabled_phys &= ~(1 << i);
+			continue;
+		}
+		/* This is the SAS address which should be sent in IDENTIFY. */
+		memcpy(asd_ha->hw_prof.phy_desc[i].sas_addr, pe->sas_addr,
+		       SAS_ADDR_SIZE);
+		asd_ha->hw_prof.phy_desc[i].max_sas_lrate =
+			(pe->sas_link_rates & 0xF0) >> 4;
+		asd_ha->hw_prof.phy_desc[i].min_sas_lrate =
+			(pe->sas_link_rates & 0x0F);
+		asd_ha->hw_prof.phy_desc[i].max_sata_lrate =
+			(pe->sata_link_rates & 0xF0) >> 4;
+		asd_ha->hw_prof.phy_desc[i].min_sata_lrate =
+			(pe->sata_link_rates & 0x0F);
+		asd_ha->hw_prof.phy_desc[i].flags = pe->flags;
+		ASD_DPRINTK("ctrla: phy%d: sas_addr: %llx, sas rate:0x%x-0x%x,"
+			    " sata rate:0x%x-0x%x, flags:0x%x\n",
+			    i,
+			    SAS_ADDR(asd_ha->hw_prof.phy_desc[i].sas_addr),
+			    asd_ha->hw_prof.phy_desc[i].max_sas_lrate,
+			    asd_ha->hw_prof.phy_desc[i].min_sas_lrate,
+			    asd_ha->hw_prof.phy_desc[i].max_sata_lrate,
+			    asd_ha->hw_prof.phy_desc[i].min_sata_lrate,
+			    asd_ha->hw_prof.phy_desc[i].flags);
+	}
+
+	return 0;
+}
+
+/**
+ * asd_process_ctrl_a_user - process CTRL-A user settings
+ * @asd_ha: pointer to the host adapter structure
+ * @flash_dir: pointer to the flash directory
+ */
+static int asd_process_ctrl_a_user(struct asd_ha_struct *asd_ha,
+				   struct asd_flash_dir *flash_dir)
+{
+	int err, i;
+	u32 offs, size;
+	struct asd_ll_el *el;
+	struct asd_ctrla_phy_settings *ps;
+	struct asd_ctrla_phy_settings dflt_ps;
+
+	err = asd_find_flash_de(flash_dir, FLASH_DE_CTRL_A_USER, &offs, &size);
+	if (err) {
+		ASD_DPRINTK("couldn't find CTRL-A user settings section\n");
+		ASD_DPRINTK("Creating default CTRL-A user settings section\n");
+
+		dflt_ps.id0 = 'h';
+		dflt_ps.num_phys = 8;
+		for (i =0; i < ASD_MAX_PHYS; i++) {
+			memcpy(dflt_ps.phy_ent[i].sas_addr,
+			       asd_ha->hw_prof.sas_addr, SAS_ADDR_SIZE);
+			dflt_ps.phy_ent[i].sas_link_rates = 0x98;
+			dflt_ps.phy_ent[i].flags = 0x0;
+			dflt_ps.phy_ent[i].sata_link_rates = 0x0;
+		}
+
+		size = sizeof(struct asd_ctrla_phy_settings);
+		ps = &dflt_ps;
+	}
+
+	if (size == 0)
+		goto out;
+
+	err = -ENOMEM;
+	el = kmalloc(size, GFP_KERNEL);
+	if (!el) {
+		ASD_DPRINTK("no mem for ctrla user settings section\n");
+		goto out;
+	}
+
+	err = asd_read_flash_seg(asd_ha, (void *)el, offs, size);
+	if (err) {
+		ASD_DPRINTK("couldn't read ctrla phy settings section\n");
+		goto out2;
+	}
+
+	err = -ENOENT;
+	ps = asd_find_ll_by_id(el, 'h', 0xFF);
+	if (!ps) {
+		ASD_DPRINTK("couldn't find ctrla phy settings struct\n");
+		goto out2;
+	}
+
+	err = asd_process_ctrla_phy_settings(asd_ha, ps);
+	if (err) {
+		ASD_DPRINTK("couldn't process ctrla phy settings\n");
+		goto out2;
+	}
+out2:
+	kfree(el);
+out:
+	return err;
+}
+
+/**
+ * asd_read_flash - read flash memory
+ * @asd_ha: pointer to the host adapter structure
+ */
+int asd_read_flash(struct asd_ha_struct *asd_ha)
+{
+	int err;
+	struct asd_flash_dir *flash_dir;
+
+	err = asd_flash_getid(asd_ha);
+	if (err)
+		return err;
+
+	flash_dir = kmalloc(sizeof(*flash_dir), GFP_KERNEL);
+	if (!flash_dir)
+		return -ENOMEM;
+
+	err = -ENOENT;
+	if (!asd_find_flash_dir(asd_ha, flash_dir)) {
+		ASD_DPRINTK("couldn't find flash directory\n");
+		goto out;
+	}
+
+	if (le32_to_cpu(flash_dir->rev) != 2) {
+		asd_printk("unsupported flash dir version:0x%x\n",
+			   le32_to_cpu(flash_dir->rev));
+		goto out;
+	}
+
+	err = asd_process_ms(asd_ha, flash_dir);
+	if (err) {
+		ASD_DPRINTK("couldn't process manuf sector settings\n");
+		goto out;
+	}
+
+	err = asd_process_ctrl_a_user(asd_ha, flash_dir);
+	if (err) {
+		ASD_DPRINTK("couldn't process CTRL-A user settings\n");
+		goto out;
+	}
+
+out:
+	kfree(flash_dir);
+	return err;
+}
diff --git a/drivers/scsi/aic94xx/aic94xx_seq.c b/drivers/scsi/aic94xx/aic94xx_seq.c
new file mode 100644
index 0000000..d9b6da5f
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_seq.c
@@ -0,0 +1,1404 @@
+/*
+ * Aic94xx SAS/SATA driver sequencer interface.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * Parts of this code adapted from David Chaw's adp94xx_seq.c.
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/delay.h>
+#include <linux/pci.h>
+#include <linux/module.h>
+#include <linux/firmware.h>
+#include "aic94xx_reg.h"
+#include "aic94xx_hwi.h"
+
+#include "aic94xx_seq.h"
+#include "aic94xx_dump.h"
+
+/* It takes no more than 0.05 us for an instruction
+ * to complete. So waiting for 1 us should be more than
+ * plenty.
+ */
+#define PAUSE_DELAY 1
+#define PAUSE_TRIES 1000
+
+static const struct firmware *sequencer_fw;
+static const char *sequencer_version;
+static u16 cseq_vecs[CSEQ_NUM_VECS], lseq_vecs[LSEQ_NUM_VECS], mode2_task,
+	cseq_idle_loop, lseq_idle_loop;
+static u8 *cseq_code, *lseq_code;
+static u32 cseq_code_size, lseq_code_size;
+
+static u16 first_scb_site_no = 0xFFFF;
+static u16 last_scb_site_no;
+
+/* ---------- Pause/Unpause CSEQ/LSEQ ---------- */
+
+/**
+ * asd_pause_cseq - pause the central sequencer
+ * @asd_ha: pointer to host adapter structure
+ *
+ * Return 0 on success, negative on failure.
+ */
+int asd_pause_cseq(struct asd_ha_struct *asd_ha)
+{
+	int	count = PAUSE_TRIES;
+	u32	arp2ctl;
+
+	arp2ctl = asd_read_reg_dword(asd_ha, CARP2CTL);
+	if (arp2ctl & PAUSED)
+		return 0;
+
+	asd_write_reg_dword(asd_ha, CARP2CTL, arp2ctl | EPAUSE);
+	do {
+		arp2ctl = asd_read_reg_dword(asd_ha, CARP2CTL);
+		if (arp2ctl & PAUSED)
+			return 0;
+		udelay(PAUSE_DELAY);
+	} while (--count > 0);
+
+	ASD_DPRINTK("couldn't pause CSEQ\n");
+	return -1;
+}
+
+/**
+ * asd_unpause_cseq - unpause the central sequencer.
+ * @asd_ha: pointer to host adapter structure.
+ *
+ * Return 0 on success, negative on error.
+ */
+int asd_unpause_cseq(struct asd_ha_struct *asd_ha)
+{
+	u32	arp2ctl;
+	int	count = PAUSE_TRIES;
+
+	arp2ctl = asd_read_reg_dword(asd_ha, CARP2CTL);
+	if (!(arp2ctl & PAUSED))
+		return 0;
+
+	asd_write_reg_dword(asd_ha, CARP2CTL, arp2ctl & ~EPAUSE);
+	do {
+		arp2ctl = asd_read_reg_dword(asd_ha, CARP2CTL);
+		if (!(arp2ctl & PAUSED))
+			return 0;
+		udelay(PAUSE_DELAY);
+	} while (--count > 0);
+
+	ASD_DPRINTK("couldn't unpause the CSEQ\n");
+	return -1;
+}
+
+/**
+ * asd_seq_pause_lseq - pause a link sequencer
+ * @asd_ha: pointer to a host adapter structure
+ * @lseq: link sequencer of interest
+ *
+ * Return 0 on success, negative on error.
+ */
+static inline int asd_seq_pause_lseq(struct asd_ha_struct *asd_ha, int lseq)
+{
+	u32    arp2ctl;
+	int    count = PAUSE_TRIES;
+
+	arp2ctl = asd_read_reg_dword(asd_ha, LmARP2CTL(lseq));
+	if (arp2ctl & PAUSED)
+		return 0;
+
+	asd_write_reg_dword(asd_ha, LmARP2CTL(lseq), arp2ctl | EPAUSE);
+	do {
+		arp2ctl = asd_read_reg_dword(asd_ha, LmARP2CTL(lseq));
+		if (arp2ctl & PAUSED)
+			return 0;
+		udelay(PAUSE_DELAY);
+	} while (--count > 0);
+
+	ASD_DPRINTK("couldn't pause LSEQ %d\n", lseq);
+	return -1;
+}
+
+/**
+ * asd_pause_lseq - pause the link sequencer(s)
+ * @asd_ha: pointer to host adapter structure
+ * @lseq_mask: mask of link sequencers of interest
+ *
+ * Return 0 on success, negative on failure.
+ */
+int asd_pause_lseq(struct asd_ha_struct *asd_ha, u8 lseq_mask)
+{
+	int lseq;
+	int err = 0;
+
+	for_each_sequencer(lseq_mask, lseq_mask, lseq) {
+		err = asd_seq_pause_lseq(asd_ha, lseq);
+		if (err)
+			return err;
+	}
+
+	return err;
+}
+
+/**
+ * asd_seq_unpause_lseq - unpause a link sequencer
+ * @asd_ha: pointer to host adapter structure
+ * @lseq: link sequencer of interest
+ *
+ * Return 0 on success, negative on error.
+ */
+static inline int asd_seq_unpause_lseq(struct asd_ha_struct *asd_ha, int lseq)
+{
+	u32 arp2ctl;
+	int count = PAUSE_TRIES;
+
+	arp2ctl = asd_read_reg_dword(asd_ha, LmARP2CTL(lseq));
+	if (!(arp2ctl & PAUSED))
+		return 0;
+
+	asd_write_reg_dword(asd_ha, LmARP2CTL(lseq), arp2ctl & ~EPAUSE);
+	do {
+		arp2ctl = asd_read_reg_dword(asd_ha, LmARP2CTL(lseq));
+		if (!(arp2ctl & PAUSED))
+			return 0;
+		udelay(PAUSE_DELAY);
+	} while (--count > 0);
+
+	ASD_DPRINTK("couldn't unpause LSEQ %d\n", lseq);
+	return 0;
+}
+
+
+/**
+ * asd_unpause_lseq - unpause the link sequencer(s)
+ * @asd_ha: pointer to host adapter structure
+ * @lseq_mask: mask of link sequencers of interest
+ *
+ * Return 0 on success, negative on failure.
+ */
+int asd_unpause_lseq(struct asd_ha_struct *asd_ha, u8 lseq_mask)
+{
+	int lseq;
+	int err = 0;
+
+	for_each_sequencer(lseq_mask, lseq_mask, lseq) {
+		err = asd_seq_unpause_lseq(asd_ha, lseq);
+		if (err)
+			return err;
+	}
+
+	return err;
+}
+
+/* ---------- Downloading CSEQ/LSEQ microcode ---------- */
+
+static int asd_verify_cseq(struct asd_ha_struct *asd_ha, const u8 *_prog,
+			   u32 size)
+{
+	u32 addr = CSEQ_RAM_REG_BASE_ADR;
+	const u32 *prog = (u32 *) _prog;
+	u32 i;
+
+	for (i = 0; i < size; i += 4, prog++, addr += 4) {
+		u32 val = asd_read_reg_dword(asd_ha, addr);
+
+		if (le32_to_cpu(*prog) != val) {
+			asd_printk("%s: cseq verify failed at %u "
+				   "read:0x%x, wanted:0x%x\n",
+				   pci_name(asd_ha->pcidev),
+				   i, val, le32_to_cpu(*prog));
+			return -1;
+		}
+	}
+	ASD_DPRINTK("verified %d bytes, passed\n", size);
+	return 0;
+}
+
+/**
+ * asd_verify_lseq - verify the microcode of a link sequencer
+ * @asd_ha: pointer to host adapter structure
+ * @_prog: pointer to the microcode
+ * @size: size of the microcode in bytes
+ * @lseq: link sequencer of interest
+ *
+ * The link sequencer code is accessed in 4 KB pages, which are selected
+ * by setting LmRAMPAGE (bits 8 and 9) of the LmBISTCTL1 register.
+ * The 10 KB LSEQm instruction code is mapped, page at a time, at
+ * LmSEQRAM address.
+ */
+static int asd_verify_lseq(struct asd_ha_struct *asd_ha, const u8 *_prog,
+			   u32 size, int lseq)
+{
+#define LSEQ_CODEPAGE_SIZE 4096
+	int pages =  (size + LSEQ_CODEPAGE_SIZE - 1) / LSEQ_CODEPAGE_SIZE;
+	u32 page;
+	const u32 *prog = (u32 *) _prog;
+
+	for (page = 0; page < pages; page++) {
+		u32 i;
+
+		asd_write_reg_dword(asd_ha, LmBISTCTL1(lseq),
+				    page << LmRAMPAGE_LSHIFT);
+		for (i = 0; size > 0 && i < LSEQ_CODEPAGE_SIZE;
+		     i += 4, prog++, size-=4) {
+
+			u32 val = asd_read_reg_dword(asd_ha, LmSEQRAM(lseq)+i);
+
+			if (le32_to_cpu(*prog) != val) {
+				asd_printk("%s: LSEQ%d verify failed "
+					   "page:%d, offs:%d\n",
+					   pci_name(asd_ha->pcidev),
+					   lseq, page, i);
+				return -1;
+			}
+		}
+	}
+	ASD_DPRINTK("LSEQ%d verified %d bytes, passed\n", lseq,
+		    (int)((u8 *)prog-_prog));
+	return 0;
+}
+
+/**
+ * asd_verify_seq -- verify CSEQ/LSEQ microcode
+ * @asd_ha: pointer to host adapter structure
+ * @prog: pointer to microcode
+ * @size: size of the microcode
+ * @lseq_mask: if 0, verify CSEQ microcode, else mask of LSEQs of interest
+ *
+ * Return 0 if microcode is correct, negative on mismatch.
+ */
+static int asd_verify_seq(struct asd_ha_struct *asd_ha, const u8 *prog,
+			      u32 size, u8 lseq_mask)
+{
+	if (lseq_mask == 0)
+		return asd_verify_cseq(asd_ha, prog, size);
+	else {
+		int lseq, err;
+
+		for_each_sequencer(lseq_mask, lseq_mask, lseq) {
+			err = asd_verify_lseq(asd_ha, prog, size, lseq);
+			if (err)
+				return err;
+		}
+	}
+
+	return 0;
+}
+#define ASD_DMA_MODE_DOWNLOAD
+#ifdef ASD_DMA_MODE_DOWNLOAD
+/* This is the size of the CSEQ Mapped instruction page */
+#define MAX_DMA_OVLY_COUNT ((1U << 14)-1)
+static int asd_download_seq(struct asd_ha_struct *asd_ha,
+			    const u8 * const prog, u32 size, u8 lseq_mask)
+{
+	u32 comstaten;
+	u32 reg;
+	int page;
+	const int pages = (size + MAX_DMA_OVLY_COUNT - 1) / MAX_DMA_OVLY_COUNT;
+	struct asd_dma_tok *token;
+	int err = 0;
+
+	if (size % 4) {
+		asd_printk("sequencer program not multiple of 4\n");
+		return -1;
+	}
+
+	asd_pause_cseq(asd_ha);
+	asd_pause_lseq(asd_ha, 0xFF);
+
+	/* save, disable and clear interrupts */
+	comstaten = asd_read_reg_dword(asd_ha, COMSTATEN);
+	asd_write_reg_dword(asd_ha, COMSTATEN, 0);
+	asd_write_reg_dword(asd_ha, COMSTAT, COMSTAT_MASK);
+
+	asd_write_reg_dword(asd_ha, CHIMINTEN, RST_CHIMINTEN);
+	asd_write_reg_dword(asd_ha, CHIMINT, CHIMINT_MASK);
+
+	token = asd_alloc_coherent(asd_ha, MAX_DMA_OVLY_COUNT, GFP_KERNEL);
+	if (!token) {
+		asd_printk("out of memory for dma SEQ download\n");
+		err = -ENOMEM;
+		goto out;
+	}
+	ASD_DPRINTK("dma-ing %d bytes\n", size);
+
+	for (page = 0; page < pages; page++) {
+		int i;
+		u32 left = min(size-page*MAX_DMA_OVLY_COUNT,
+			       (u32)MAX_DMA_OVLY_COUNT);
+
+		memcpy(token->vaddr, prog + page*MAX_DMA_OVLY_COUNT, left);
+		asd_write_reg_addr(asd_ha, OVLYDMAADR, token->dma_handle);
+		asd_write_reg_dword(asd_ha, OVLYDMACNT, left);
+		reg = !page ? RESETOVLYDMA : 0;
+		reg |= (STARTOVLYDMA | OVLYHALTERR);
+		reg |= (lseq_mask ? (((u32)lseq_mask) << 8) : OVLYCSEQ);
+		/* Start DMA. */
+		asd_write_reg_dword(asd_ha, OVLYDMACTL, reg);
+
+		for (i = PAUSE_TRIES*100; i > 0; i--) {
+			u32 dmadone = asd_read_reg_dword(asd_ha, OVLYDMACTL);
+			if (!(dmadone & OVLYDMAACT))
+				break;
+			udelay(PAUSE_DELAY);
+		}
+	}
+
+	reg = asd_read_reg_dword(asd_ha, COMSTAT);
+	if (!(reg & OVLYDMADONE) || (reg & OVLYERR)
+	    || (asd_read_reg_dword(asd_ha, CHIMINT) & DEVEXCEPT_MASK)){
+		asd_printk("%s: error DMA-ing sequencer code\n",
+			   pci_name(asd_ha->pcidev));
+		err = -ENODEV;
+	}
+
+	asd_free_coherent(asd_ha, token);
+ out:
+	asd_write_reg_dword(asd_ha, COMSTATEN, comstaten);
+
+	return err ? : asd_verify_seq(asd_ha, prog, size, lseq_mask);
+}
+#else /* ASD_DMA_MODE_DOWNLOAD */
+static int asd_download_seq(struct asd_ha_struct *asd_ha, const u8 *_prog,
+			    u32 size, u8 lseq_mask)
+{
+	int i;
+	u32 reg = 0;
+	const u32 *prog = (u32 *) _prog;
+
+	if (size % 4) {
+		asd_printk("sequencer program not multiple of 4\n");
+		return -1;
+	}
+
+	asd_pause_cseq(asd_ha);
+	asd_pause_lseq(asd_ha, 0xFF);
+
+	reg |= (lseq_mask ? (((u32)lseq_mask) << 8) : OVLYCSEQ);
+	reg |= PIOCMODE;
+
+	asd_write_reg_dword(asd_ha, OVLYDMACNT, size);
+	asd_write_reg_dword(asd_ha, OVLYDMACTL, reg);
+
+	ASD_DPRINTK("downloading %s sequencer%s in PIO mode...\n",
+		    lseq_mask ? "LSEQ" : "CSEQ", lseq_mask ? "s" : "");
+
+	for (i = 0; i < size; i += 4, prog++)
+		asd_write_reg_dword(asd_ha, SPIODATA, *prog);
+
+	reg = (reg & ~PIOCMODE) | OVLYHALTERR;
+	asd_write_reg_dword(asd_ha, OVLYDMACTL, reg);
+
+	return asd_verify_seq(asd_ha, _prog, size, lseq_mask);
+}
+#endif /* ASD_DMA_MODE_DOWNLOAD */
+
+/**
+ * asd_seq_download_seqs - download the sequencer microcode
+ * @asd_ha: pointer to host adapter structure
+ *
+ * Download the central and link sequencer microcode.
+ */
+static int asd_seq_download_seqs(struct asd_ha_struct *asd_ha)
+{
+	int 	err;
+
+	if (!asd_ha->hw_prof.enabled_phys) {
+		asd_printk("%s: no enabled phys!\n", pci_name(asd_ha->pcidev));
+		return -ENODEV;
+	}
+
+	/* Download the CSEQ */
+	ASD_DPRINTK("downloading CSEQ...\n");
+	err = asd_download_seq(asd_ha, cseq_code, cseq_code_size, 0);
+	if (err) {
+		asd_printk("CSEQ download failed:%d\n", err);
+		return err;
+	}
+
+	/* Download the Link Sequencers code. All of the Link Sequencers
+	 * microcode can be downloaded at the same time.
+	 */
+	ASD_DPRINTK("downloading LSEQs...\n");
+	err = asd_download_seq(asd_ha, lseq_code, lseq_code_size,
+			       asd_ha->hw_prof.enabled_phys);
+	if (err) {
+		/* Try it one at a time */
+		u8 lseq;
+		u8 lseq_mask = asd_ha->hw_prof.enabled_phys;
+
+		for_each_sequencer(lseq_mask, lseq_mask, lseq) {
+			err = asd_download_seq(asd_ha, lseq_code,
+					       lseq_code_size, 1<<lseq);
+			if (err)
+				break;
+		}
+	}
+	if (err)
+		asd_printk("LSEQs download failed:%d\n", err);
+
+	return err;
+}
+
+/* ---------- Initializing the chip, chip memory, etc. ---------- */
+
+/**
+ * asd_init_cseq_mip - initialize CSEQ mode independent pages 4-7
+ * @asd_ha: pointer to host adapter structure
+ */
+static void asd_init_cseq_mip(struct asd_ha_struct *asd_ha)
+{
+	/* CSEQ Mode Independent, page 4 setup. */
+	asd_write_reg_word(asd_ha, CSEQ_Q_EXE_HEAD, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_EXE_TAIL, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_DONE_HEAD, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_DONE_TAIL, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_SEND_HEAD, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_SEND_TAIL, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_DMA2CHIM_HEAD, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_DMA2CHIM_TAIL, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_COPY_HEAD, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_COPY_TAIL, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_REG0, 0);
+	asd_write_reg_word(asd_ha, CSEQ_REG1, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_REG2, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_LINK_CTL_Q_MAP, 0);
+	{
+		u8 con = asd_read_reg_byte(asd_ha, CCONEXIST);
+		u8 val = hweight8(con);
+		asd_write_reg_byte(asd_ha, CSEQ_MAX_CSEQ_MODE, (val<<4)|val);
+	}
+	asd_write_reg_word(asd_ha, CSEQ_FREE_LIST_HACK_COUNT, 0);
+
+	/* CSEQ Mode independent, page 5 setup. */
+	asd_write_reg_dword(asd_ha, CSEQ_EST_NEXUS_REQ_QUEUE, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_EST_NEXUS_REQ_QUEUE+4, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_EST_NEXUS_REQ_COUNT, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_EST_NEXUS_REQ_COUNT+4, 0);
+	asd_write_reg_word(asd_ha, CSEQ_Q_EST_NEXUS_HEAD, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_EST_NEXUS_TAIL, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_NEED_EST_NEXUS_SCB, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_EST_NEXUS_REQ_HEAD, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_EST_NEXUS_REQ_TAIL, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_EST_NEXUS_SCB_OFFSET, 0);
+
+	/* CSEQ Mode independent, page 6 setup. */
+	asd_write_reg_word(asd_ha, CSEQ_INT_ROUT_RET_ADDR0, 0);
+	asd_write_reg_word(asd_ha, CSEQ_INT_ROUT_RET_ADDR1, 0);
+	asd_write_reg_word(asd_ha, CSEQ_INT_ROUT_SCBPTR, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_INT_ROUT_MODE, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_ISR_SCRATCH_FLAGS, 0);
+	asd_write_reg_word(asd_ha, CSEQ_ISR_SAVE_SINDEX, 0);
+	asd_write_reg_word(asd_ha, CSEQ_ISR_SAVE_DINDEX, 0);
+	asd_write_reg_word(asd_ha, CSEQ_Q_MONIRTT_HEAD, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_MONIRTT_TAIL, 0xFFFF);
+	/* Calculate the free scb mask. */
+	{
+		u16 cmdctx = asd_get_cmdctx_size(asd_ha);
+		cmdctx = (~((cmdctx/128)-1)) >> 8;
+		asd_write_reg_byte(asd_ha, CSEQ_FREE_SCB_MASK, (u8)cmdctx);
+	}
+	asd_write_reg_word(asd_ha, CSEQ_BUILTIN_FREE_SCB_HEAD,
+			   first_scb_site_no);
+	asd_write_reg_word(asd_ha, CSEQ_BUILTIN_FREE_SCB_TAIL,
+			   last_scb_site_no);
+	asd_write_reg_word(asd_ha, CSEQ_EXTENDED_FREE_SCB_HEAD, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_EXTENDED_FREE_SCB_TAIL, 0xFFFF);
+
+	/* CSEQ Mode independent, page 7 setup. */
+	asd_write_reg_dword(asd_ha, CSEQ_EMPTY_REQ_QUEUE, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_EMPTY_REQ_QUEUE+4, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_EMPTY_REQ_COUNT, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_EMPTY_REQ_COUNT+4, 0);
+	asd_write_reg_word(asd_ha, CSEQ_Q_EMPTY_HEAD, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_Q_EMPTY_TAIL, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_NEED_EMPTY_SCB, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_EMPTY_REQ_HEAD, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_EMPTY_REQ_TAIL, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_EMPTY_SCB_OFFSET, 0);
+	asd_write_reg_word(asd_ha, CSEQ_PRIMITIVE_DATA, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_TIMEOUT_CONST, 0);
+}
+
+/**
+ * asd_init_cseq_mdp - initialize CSEQ Mode dependent pages
+ * @asd_ha: pointer to host adapter structure
+ */
+static void asd_init_cseq_mdp(struct asd_ha_struct *asd_ha)
+{
+	int	i;
+	int	moffs;
+
+	moffs = CSEQ_PAGE_SIZE * 2;
+
+	/* CSEQ Mode dependent, modes 0-7, page 0 setup. */
+	for (i = 0; i < 8; i++) {
+		asd_write_reg_word(asd_ha, i*moffs+CSEQ_LRM_SAVE_SINDEX, 0);
+		asd_write_reg_word(asd_ha, i*moffs+CSEQ_LRM_SAVE_SCBPTR, 0);
+		asd_write_reg_word(asd_ha, i*moffs+CSEQ_Q_LINK_HEAD, 0xFFFF);
+		asd_write_reg_word(asd_ha, i*moffs+CSEQ_Q_LINK_TAIL, 0xFFFF);
+		asd_write_reg_byte(asd_ha, i*moffs+CSEQ_LRM_SAVE_SCRPAGE, 0);
+	}
+
+	/* CSEQ Mode dependent, mode 0-7, page 1 and 2 shall be ignored. */
+
+	/* CSEQ Mode dependent, mode 8, page 0 setup. */
+	asd_write_reg_word(asd_ha, CSEQ_RET_ADDR, 0xFFFF);
+	asd_write_reg_word(asd_ha, CSEQ_RET_SCBPTR, 0);
+	asd_write_reg_word(asd_ha, CSEQ_SAVE_SCBPTR, 0);
+	asd_write_reg_word(asd_ha, CSEQ_EMPTY_TRANS_CTX, 0);
+	asd_write_reg_word(asd_ha, CSEQ_RESP_LEN, 0);
+	asd_write_reg_word(asd_ha, CSEQ_TMF_SCBPTR, 0);
+	asd_write_reg_word(asd_ha, CSEQ_GLOBAL_PREV_SCB, 0);
+	asd_write_reg_word(asd_ha, CSEQ_GLOBAL_HEAD, 0);
+	asd_write_reg_word(asd_ha, CSEQ_CLEAR_LU_HEAD, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_TMF_OPCODE, 0);
+	asd_write_reg_byte(asd_ha, CSEQ_SCRATCH_FLAGS, 0);
+	asd_write_reg_word(asd_ha, CSEQ_HSB_SITE, 0);
+	asd_write_reg_word(asd_ha, CSEQ_FIRST_INV_SCB_SITE,
+			   (u16)last_scb_site_no+1);
+	asd_write_reg_word(asd_ha, CSEQ_FIRST_INV_DDB_SITE,
+			   (u16)asd_ha->hw_prof.max_ddbs);
+
+	/* CSEQ Mode dependent, mode 8, page 1 setup. */
+	asd_write_reg_dword(asd_ha, CSEQ_LUN_TO_CLEAR, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_LUN_TO_CLEAR + 4, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_LUN_TO_CHECK, 0);
+	asd_write_reg_dword(asd_ha, CSEQ_LUN_TO_CHECK + 4, 0);
+
+	/* CSEQ Mode dependent, mode 8, page 2 setup. */
+	/* Tell the sequencer the bus address of the first SCB. */
+	asd_write_reg_addr(asd_ha, CSEQ_HQ_NEW_POINTER,
+			   asd_ha->seq.next_scb.dma_handle);
+	ASD_DPRINTK("First SCB dma_handle: 0x%llx\n",
+		    (unsigned long long)asd_ha->seq.next_scb.dma_handle);
+
+	/* Tell the sequencer the first Done List entry address. */
+	asd_write_reg_addr(asd_ha, CSEQ_HQ_DONE_BASE,
+			   asd_ha->seq.actual_dl->dma_handle);
+
+	/* Initialize the Q_DONE_POINTER with the least significant
+	 * 4 bytes of the first Done List address. */
+	asd_write_reg_dword(asd_ha, CSEQ_HQ_DONE_POINTER,
+			    ASD_BUSADDR_LO(asd_ha->seq.actual_dl->dma_handle));
+
+	asd_write_reg_byte(asd_ha, CSEQ_HQ_DONE_PASS, ASD_DEF_DL_TOGGLE);
+
+	/* CSEQ Mode dependent, mode 8, page 3 shall be ignored. */
+}
+
+/**
+ * asd_init_cseq_scratch -- setup and init CSEQ
+ * @asd_ha: pointer to host adapter structure
+ *
+ * Setup and initialize Central sequencers. Initialiaze the mode
+ * independent and dependent scratch page to the default settings.
+ */
+static void asd_init_cseq_scratch(struct asd_ha_struct *asd_ha)
+{
+	asd_init_cseq_mip(asd_ha);
+	asd_init_cseq_mdp(asd_ha);
+}
+
+/**
+ * asd_init_lseq_mip -- initialize LSEQ Mode independent pages 0-3
+ * @asd_ha: pointer to host adapter structure
+ */
+static void asd_init_lseq_mip(struct asd_ha_struct *asd_ha, u8 lseq)
+{
+	int i;
+
+	/* LSEQ Mode independent page 0 setup. */
+	asd_write_reg_word(asd_ha, LmSEQ_Q_TGTXFR_HEAD(lseq), 0xFFFF);
+	asd_write_reg_word(asd_ha, LmSEQ_Q_TGTXFR_TAIL(lseq), 0xFFFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_LINK_NUMBER(lseq), lseq);
+	asd_write_reg_byte(asd_ha, LmSEQ_SCRATCH_FLAGS(lseq),
+			   ASD_NOTIFY_ENABLE_SPINUP);
+	asd_write_reg_dword(asd_ha, LmSEQ_CONNECTION_STATE(lseq),0x08000000);
+	asd_write_reg_word(asd_ha, LmSEQ_CONCTL(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_CONSTAT(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_CONNECTION_MODES(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_REG1_ISR(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_REG2_ISR(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_REG3_ISR(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_REG0_ISR(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_REG0_ISR(lseq)+4, 0);
+
+	/* LSEQ Mode independent page 1 setup. */
+	asd_write_reg_word(asd_ha, LmSEQ_EST_NEXUS_SCBPTR0(lseq), 0xFFFF);
+	asd_write_reg_word(asd_ha, LmSEQ_EST_NEXUS_SCBPTR1(lseq), 0xFFFF);
+	asd_write_reg_word(asd_ha, LmSEQ_EST_NEXUS_SCBPTR2(lseq), 0xFFFF);
+	asd_write_reg_word(asd_ha, LmSEQ_EST_NEXUS_SCBPTR3(lseq), 0xFFFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_EST_NEXUS_SCB_OPCODE0(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EST_NEXUS_SCB_OPCODE1(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EST_NEXUS_SCB_OPCODE2(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EST_NEXUS_SCB_OPCODE3(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EST_NEXUS_SCB_HEAD(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EST_NEXUS_SCB_TAIL(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EST_NEXUS_BUF_AVAIL(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_TIMEOUT_CONST(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_ISR_SAVE_SINDEX(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_ISR_SAVE_DINDEX(lseq), 0);
+
+	/* LSEQ Mode Independent page 2 setup. */
+	asd_write_reg_word(asd_ha, LmSEQ_EMPTY_SCB_PTR0(lseq), 0xFFFF);
+	asd_write_reg_word(asd_ha, LmSEQ_EMPTY_SCB_PTR1(lseq), 0xFFFF);
+	asd_write_reg_word(asd_ha, LmSEQ_EMPTY_SCB_PTR2(lseq), 0xFFFF);
+	asd_write_reg_word(asd_ha, LmSEQ_EMPTY_SCB_PTR3(lseq), 0xFFFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_EMPTY_SCB_OPCD0(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EMPTY_SCB_OPCD1(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EMPTY_SCB_OPCD2(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EMPTY_SCB_OPCD3(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EMPTY_SCB_HEAD(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EMPTY_SCB_TAIL(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_EMPTY_BUFS_AVAIL(lseq), 0);
+	for (i = 0; i < 12; i += 4)
+		asd_write_reg_dword(asd_ha, LmSEQ_ATA_SCR_REGS(lseq) + i, 0);
+
+	/* LSEQ Mode Independent page 3 setup. */
+
+	/* Device present timer timeout */
+	asd_write_reg_dword(asd_ha, LmSEQ_DEV_PRES_TMR_TOUT_CONST(lseq),
+			    ASD_DEV_PRESENT_TIMEOUT);
+
+	/* SATA interlock timer disabled */
+	asd_write_reg_dword(asd_ha, LmSEQ_SATA_INTERLOCK_TIMEOUT(lseq),
+			    ASD_SATA_INTERLOCK_TIMEOUT);
+
+	/* STP shutdown timer timeout constant, IGNORED by the sequencer,
+	 * always 0. */
+	asd_write_reg_dword(asd_ha, LmSEQ_STP_SHUTDOWN_TIMEOUT(lseq),
+			    ASD_STP_SHUTDOWN_TIMEOUT);
+
+	asd_write_reg_dword(asd_ha, LmSEQ_SRST_ASSERT_TIMEOUT(lseq),
+			    ASD_SRST_ASSERT_TIMEOUT);
+
+	asd_write_reg_dword(asd_ha, LmSEQ_RCV_FIS_TIMEOUT(lseq),
+			    ASD_RCV_FIS_TIMEOUT);
+
+	asd_write_reg_dword(asd_ha, LmSEQ_ONE_MILLISEC_TIMEOUT(lseq),
+			    ASD_ONE_MILLISEC_TIMEOUT);
+
+	/* COM_INIT timer */
+	asd_write_reg_dword(asd_ha, LmSEQ_TEN_MS_COMINIT_TIMEOUT(lseq),
+			    ASD_TEN_MILLISEC_TIMEOUT);
+
+	asd_write_reg_dword(asd_ha, LmSEQ_SMP_RCV_TIMEOUT(lseq),
+			    ASD_SMP_RCV_TIMEOUT);
+}
+
+/**
+ * asd_init_lseq_mdp -- initialize LSEQ mode dependent pages.
+ * @asd_ha: pointer to host adapter structure
+ */
+static void asd_init_lseq_mdp(struct asd_ha_struct *asd_ha,  int lseq)
+{
+	int    i;
+	u32    moffs;
+	u16 ret_addr[] = {
+		0xFFFF,		  /* mode 0 */
+		0xFFFF,		  /* mode 1 */
+		mode2_task,	  /* mode 2 */
+		0,
+		0xFFFF,		  /* mode 4/5 */
+		0xFFFF,		  /* mode 4/5 */
+	};
+
+	/*
+	 * Mode 0,1,2 and 4/5 have common field on page 0 for the first
+	 * 14 bytes.
+	 */
+	for (i = 0; i < 3; i++) {
+		moffs = i * LSEQ_MODE_SCRATCH_SIZE;
+		asd_write_reg_word(asd_ha, LmSEQ_RET_ADDR(lseq)+moffs,
+				   ret_addr[i]);
+		asd_write_reg_word(asd_ha, LmSEQ_REG0_MODE(lseq)+moffs, 0);
+		asd_write_reg_word(asd_ha, LmSEQ_MODE_FLAGS(lseq)+moffs, 0);
+		asd_write_reg_word(asd_ha, LmSEQ_RET_ADDR2(lseq)+moffs,0xFFFF);
+		asd_write_reg_word(asd_ha, LmSEQ_RET_ADDR1(lseq)+moffs,0xFFFF);
+		asd_write_reg_byte(asd_ha, LmSEQ_OPCODE_TO_CSEQ(lseq)+moffs,0);
+		asd_write_reg_word(asd_ha, LmSEQ_DATA_TO_CSEQ(lseq)+moffs,0);
+	}
+	/*
+	 *  Mode 5 page 0 overlaps the same scratch page with Mode 0 page 3.
+	 */
+	asd_write_reg_word(asd_ha,
+			 LmSEQ_RET_ADDR(lseq)+LSEQ_MODE5_PAGE0_OFFSET,
+			   ret_addr[5]);
+	asd_write_reg_word(asd_ha,
+			 LmSEQ_REG0_MODE(lseq)+LSEQ_MODE5_PAGE0_OFFSET,0);
+	asd_write_reg_word(asd_ha,
+			 LmSEQ_MODE_FLAGS(lseq)+LSEQ_MODE5_PAGE0_OFFSET, 0);
+	asd_write_reg_word(asd_ha,
+			 LmSEQ_RET_ADDR2(lseq)+LSEQ_MODE5_PAGE0_OFFSET,0xFFFF);
+	asd_write_reg_word(asd_ha,
+			 LmSEQ_RET_ADDR1(lseq)+LSEQ_MODE5_PAGE0_OFFSET,0xFFFF);
+	asd_write_reg_byte(asd_ha,
+		         LmSEQ_OPCODE_TO_CSEQ(lseq)+LSEQ_MODE5_PAGE0_OFFSET,0);
+	asd_write_reg_word(asd_ha,
+		         LmSEQ_DATA_TO_CSEQ(lseq)+LSEQ_MODE5_PAGE0_OFFSET, 0);
+
+	/* LSEQ Mode dependent 0, page 0 setup. */
+	asd_write_reg_word(asd_ha, LmSEQ_FIRST_INV_DDB_SITE(lseq),
+			   (u16)asd_ha->hw_prof.max_ddbs);
+	asd_write_reg_word(asd_ha, LmSEQ_EMPTY_TRANS_CTX(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_RESP_LEN(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_FIRST_INV_SCB_SITE(lseq),
+			   (u16)last_scb_site_no+1);
+	asd_write_reg_word(asd_ha, LmSEQ_INTEN_SAVE(lseq),
+			    (u16) LmM0INTEN_MASK & 0xFFFF0000 >> 16);
+	asd_write_reg_word(asd_ha, LmSEQ_INTEN_SAVE(lseq) + 2,
+			    (u16) LmM0INTEN_MASK & 0xFFFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_LINK_RST_FRM_LEN(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_LINK_RST_PROTOCOL(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_RESP_STATUS(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_LAST_LOADED_SGE(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_SAVE_SCBPTR(lseq), 0);
+
+	/* LSEQ mode dependent, mode 1, page 0 setup. */
+	asd_write_reg_word(asd_ha, LmSEQ_Q_XMIT_HEAD(lseq), 0xFFFF);
+	asd_write_reg_word(asd_ha, LmSEQ_M1_EMPTY_TRANS_CTX(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_INI_CONN_TAG(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_FAILED_OPEN_STATUS(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_XMIT_REQUEST_TYPE(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_M1_RESP_STATUS(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_M1_LAST_LOADED_SGE(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_M1_SAVE_SCBPTR(lseq), 0);
+
+	/* LSEQ Mode dependent mode 2, page 0 setup */
+	asd_write_reg_word(asd_ha, LmSEQ_PORT_COUNTER(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_PM_TABLE_PTR(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_SATA_INTERLOCK_TMR_SAVE(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_IP_BITL(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_COPY_SMP_CONN_TAG(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_P0M2_OFFS1AH(lseq), 0);
+
+	/* LSEQ Mode dependent, mode 4/5, page 0 setup. */
+	asd_write_reg_byte(asd_ha, LmSEQ_SAVED_OOB_STATUS(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_SAVED_OOB_MODE(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_Q_LINK_HEAD(lseq), 0xFFFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_LINK_RST_ERR(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_SAVED_OOB_SIGNALS(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_SAS_RESET_MODE(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_LINK_RESET_RETRY_COUNT(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_NUM_LINK_RESET_RETRIES(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_OOB_INT_ENABLES(lseq), 0);
+	/*
+	 * Set the desired interval between transmissions of the NOTIFY
+	 * (ENABLE SPINUP) primitive.  Must be initilized to val - 1.
+	 */
+	asd_write_reg_word(asd_ha, LmSEQ_NOTIFY_TIMER_TIMEOUT(lseq),
+			   ASD_NOTIFY_TIMEOUT - 1);
+	/* No delay for the first NOTIFY to be sent to the attached target. */
+	asd_write_reg_word(asd_ha, LmSEQ_NOTIFY_TIMER_DOWN_COUNT(lseq),
+			   ASD_NOTIFY_DOWN_COUNT);
+
+	/* LSEQ Mode dependent, mode 0 and 1, page 1 setup. */
+	for (i = 0; i < 2; i++)	{
+		int j;
+		/* Start from Page 1 of Mode 0 and 1. */
+		moffs = LSEQ_PAGE_SIZE + i*LSEQ_MODE_SCRATCH_SIZE;
+		/* All the fields of page 1 can be intialized to 0. */
+		for (j = 0; j < LSEQ_PAGE_SIZE; j += 4)
+			asd_write_reg_dword(asd_ha, LmSCRATCH(lseq)+moffs+j,0);
+	}
+
+	/* LSEQ Mode dependent, mode 2, page 1 setup. */
+	asd_write_reg_dword(asd_ha, LmSEQ_INVALID_DWORD_COUNT(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_DISPARITY_ERROR_COUNT(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_LOSS_OF_SYNC_COUNT(lseq), 0);
+
+	/* LSEQ Mode dependent, mode 4/5, page 1. */
+	for (i = 0; i < LSEQ_PAGE_SIZE; i+=4)
+		asd_write_reg_dword(asd_ha, LmSEQ_FRAME_TYPE_MASK(lseq)+i, 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_FRAME_TYPE_MASK(lseq), 0xFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_HASHED_DEST_ADDR_MASK(lseq), 0xFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_HASHED_DEST_ADDR_MASK(lseq)+1,0xFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_HASHED_DEST_ADDR_MASK(lseq)+2,0xFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_HASHED_SRC_ADDR_MASK(lseq), 0xFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_HASHED_SRC_ADDR_MASK(lseq)+1, 0xFF);
+	asd_write_reg_byte(asd_ha, LmSEQ_HASHED_SRC_ADDR_MASK(lseq)+2, 0xFF);
+	asd_write_reg_dword(asd_ha, LmSEQ_DATA_OFFSET(lseq), 0xFFFFFFFF);
+
+	/* LSEQ Mode dependent, mode 0, page 2 setup. */
+	asd_write_reg_dword(asd_ha, LmSEQ_SMP_RCV_TIMER_TERM_TS(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_DEVICE_BITS(lseq), 0);
+	asd_write_reg_word(asd_ha, LmSEQ_SDB_DDB(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_SDB_NUM_TAGS(lseq), 0);
+	asd_write_reg_byte(asd_ha, LmSEQ_SDB_CURR_TAG(lseq), 0);
+
+	/* LSEQ Mode Dependent 1, page 2 setup. */
+	asd_write_reg_dword(asd_ha, LmSEQ_TX_ID_ADDR_FRAME(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_TX_ID_ADDR_FRAME(lseq)+4, 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_OPEN_TIMER_TERM_TS(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_SRST_AS_TIMER_TERM_TS(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_LAST_LOADED_SG_EL(lseq), 0);
+
+	/* LSEQ Mode Dependent 2, page 2 setup. */
+	/* The LmSEQ_STP_SHUTDOWN_TIMER_TERM_TS is IGNORED by the sequencer,
+	 * i.e. always 0. */
+	asd_write_reg_dword(asd_ha, LmSEQ_STP_SHUTDOWN_TIMER_TERM_TS(lseq),0);
+	asd_write_reg_dword(asd_ha, LmSEQ_CLOSE_TIMER_TERM_TS(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_BREAK_TIMER_TERM_TS(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_DWS_RESET_TIMER_TERM_TS(lseq), 0);
+	asd_write_reg_dword(asd_ha,LmSEQ_SATA_INTERLOCK_TIMER_TERM_TS(lseq),0);
+	asd_write_reg_dword(asd_ha, LmSEQ_MCTL_TIMER_TERM_TS(lseq), 0);
+
+	/* LSEQ Mode Dependent 4/5, page 2 setup. */
+	asd_write_reg_dword(asd_ha, LmSEQ_COMINIT_TIMER_TERM_TS(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_RCV_ID_TIMER_TERM_TS(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_RCV_FIS_TIMER_TERM_TS(lseq), 0);
+	asd_write_reg_dword(asd_ha, LmSEQ_DEV_PRES_TIMER_TERM_TS(lseq),	0);
+}
+
+/**
+ * asd_init_lseq_scratch -- setup and init link sequencers
+ * @asd_ha: pointer to host adapter struct
+ */
+static void asd_init_lseq_scratch(struct asd_ha_struct *asd_ha)
+{
+	u8 lseq;
+	u8 lseq_mask;
+
+	lseq_mask = asd_ha->hw_prof.enabled_phys;
+	for_each_sequencer(lseq_mask, lseq_mask, lseq) {
+		asd_init_lseq_mip(asd_ha, lseq);
+		asd_init_lseq_mdp(asd_ha, lseq);
+	}
+}
+
+/**
+ * asd_init_scb_sites -- initialize sequencer SCB sites (memory).
+ * @asd_ha: pointer to host adapter structure
+ *
+ * This should be done before initializing common CSEQ and LSEQ
+ * scratch since those areas depend on some computed values here,
+ * last_scb_site_no, etc.
+ */
+static void asd_init_scb_sites(struct asd_ha_struct *asd_ha)
+{
+	u16	site_no;
+	u16     max_scbs = 0;
+
+	for (site_no = asd_ha->hw_prof.max_scbs-1;
+	     site_no != (u16) -1;
+	     site_no--) {
+		u16	i;
+
+		/* Initialize all fields in the SCB site to 0. */
+		for (i = 0; i < ASD_SCB_SIZE; i += 4)
+			asd_scbsite_write_dword(asd_ha, site_no, i, 0);
+
+		/* Workaround needed by SEQ to fix a SATA issue is to exclude
+		 * certain SCB sites from the free list. */
+		if (!SCB_SITE_VALID(site_no))
+			continue;
+
+		if (last_scb_site_no == 0)
+			last_scb_site_no = site_no;
+
+		/* For every SCB site, we need to initialize the
+		 * following fields: Q_NEXT, SCB_OPCODE, SCB_FLAGS,
+		 * and SG Element Flag. */
+
+		/* Q_NEXT field of the last SCB is invalidated. */
+		asd_scbsite_write_word(asd_ha, site_no, 0, first_scb_site_no);
+
+		/* Initialize SCB Site Opcode field to invalid. */
+		asd_scbsite_write_byte(asd_ha, site_no,
+				       offsetof(struct scb_header, opcode),
+				       0xFF);
+
+		/* Initialize SCB Site Flags field to mean a response
+		 * frame has been received.  This means inadvertent
+		 * frames received to be dropped. */
+		asd_scbsite_write_byte(asd_ha, site_no, 0x49, 0x01);
+
+		first_scb_site_no = site_no;
+		max_scbs++;
+	}
+	asd_ha->hw_prof.max_scbs = max_scbs;
+	ASD_DPRINTK("max_scbs:%d\n", asd_ha->hw_prof.max_scbs);
+	ASD_DPRINTK("first_scb_site_no:0x%x\n", first_scb_site_no);
+	ASD_DPRINTK("last_scb_site_no:0x%x\n", last_scb_site_no);
+}
+
+/**
+ * asd_init_cseq_cio - initialize CSEQ CIO registers
+ * @asd_ha: pointer to host adapter structure
+ */
+static void asd_init_cseq_cio(struct asd_ha_struct *asd_ha)
+{
+	int i;
+
+	asd_write_reg_byte(asd_ha, CSEQCOMINTEN, 0);
+	asd_write_reg_byte(asd_ha, CSEQDLCTL, ASD_DL_SIZE_BITS);
+	asd_write_reg_byte(asd_ha, CSEQDLOFFS, 0);
+	asd_write_reg_byte(asd_ha, CSEQDLOFFS+1, 0);
+	asd_ha->seq.scbpro = 0;
+	asd_write_reg_dword(asd_ha, SCBPRO, 0);
+	asd_write_reg_dword(asd_ha, CSEQCON, 0);
+
+	/* Intialize CSEQ Mode 11 Interrupt Vectors.
+	 * The addresses are 16 bit wide and in dword units.
+	 * The values of their macros are in byte units.
+	 * Thus we have to divide by 4. */
+	asd_write_reg_word(asd_ha, CM11INTVEC0, cseq_vecs[0]);
+	asd_write_reg_word(asd_ha, CM11INTVEC1, cseq_vecs[1]);
+	asd_write_reg_word(asd_ha, CM11INTVEC2, cseq_vecs[2]);
+
+	/* Enable ARP2HALTC (ARP2 Halted from Halt Code Write). */
+	asd_write_reg_byte(asd_ha, CARP2INTEN, EN_ARP2HALTC);
+
+	/* Initialize CSEQ Scratch Page to 0x04. */
+	asd_write_reg_byte(asd_ha, CSCRATCHPAGE, 0x04);
+
+	/* Initialize CSEQ Mode[0-8] Dependent registers. */
+	/* Initialize Scratch Page to 0. */
+	for (i = 0; i < 9; i++)
+		asd_write_reg_byte(asd_ha, CMnSCRATCHPAGE(i), 0);
+
+	/* Reset the ARP2 Program Count. */
+	asd_write_reg_word(asd_ha, CPRGMCNT, cseq_idle_loop);
+
+	for (i = 0; i < 8; i++) {
+		/* Intialize Mode n Link m Interrupt Enable. */
+		asd_write_reg_dword(asd_ha, CMnINTEN(i), EN_CMnRSPMBXF);
+		/* Initialize Mode n Request Mailbox. */
+		asd_write_reg_dword(asd_ha, CMnREQMBX(i), 0);
+	}
+}
+
+/**
+ * asd_init_lseq_cio -- initialize LmSEQ CIO registers
+ * @asd_ha: pointer to host adapter structure
+ */
+static void asd_init_lseq_cio(struct asd_ha_struct *asd_ha, int lseq)
+{
+	u8  *sas_addr;
+	int  i;
+
+	/* Enable ARP2HALTC (ARP2 Halted from Halt Code Write). */
+	asd_write_reg_dword(asd_ha, LmARP2INTEN(lseq), EN_ARP2HALTC);
+
+	asd_write_reg_byte(asd_ha, LmSCRATCHPAGE(lseq), 0);
+
+	/* Initialize Mode 0,1, and 2 SCRATCHPAGE to 0. */
+	for (i = 0; i < 3; i++)
+		asd_write_reg_byte(asd_ha, LmMnSCRATCHPAGE(lseq, i), 0);
+
+	/* Initialize Mode 5 SCRATCHPAGE to 0. */
+	asd_write_reg_byte(asd_ha, LmMnSCRATCHPAGE(lseq, 5), 0);
+
+	asd_write_reg_dword(asd_ha, LmRSPMBX(lseq), 0);
+	/* Initialize Mode 0,1,2 and 5 Interrupt Enable and
+	 * Interrupt registers. */
+	asd_write_reg_dword(asd_ha, LmMnINTEN(lseq, 0), LmM0INTEN_MASK);
+	asd_write_reg_dword(asd_ha, LmMnINT(lseq, 0), 0xFFFFFFFF);
+	/* Mode 1 */
+	asd_write_reg_dword(asd_ha, LmMnINTEN(lseq, 1), LmM1INTEN_MASK);
+	asd_write_reg_dword(asd_ha, LmMnINT(lseq, 1), 0xFFFFFFFF);
+	/* Mode 2 */
+	asd_write_reg_dword(asd_ha, LmMnINTEN(lseq, 2), LmM2INTEN_MASK);
+	asd_write_reg_dword(asd_ha, LmMnINT(lseq, 2), 0xFFFFFFFF);
+	/* Mode 5 */
+	asd_write_reg_dword(asd_ha, LmMnINTEN(lseq, 5), LmM5INTEN_MASK);
+	asd_write_reg_dword(asd_ha, LmMnINT(lseq, 5), 0xFFFFFFFF);
+
+	/* Enable HW Timer status. */
+	asd_write_reg_byte(asd_ha, LmHWTSTATEN(lseq), LmHWTSTATEN_MASK);
+
+	/* Enable Primitive Status 0 and 1. */
+	asd_write_reg_dword(asd_ha, LmPRIMSTAT0EN(lseq), LmPRIMSTAT0EN_MASK);
+	asd_write_reg_dword(asd_ha, LmPRIMSTAT1EN(lseq), LmPRIMSTAT1EN_MASK);
+
+	/* Enable Frame Error. */
+	asd_write_reg_dword(asd_ha, LmFRMERREN(lseq), LmFRMERREN_MASK);
+	asd_write_reg_byte(asd_ha, LmMnHOLDLVL(lseq, 0), 0x50);
+
+	/* Initialize Mode 0 Transfer Level to 512. */
+	asd_write_reg_byte(asd_ha,  LmMnXFRLVL(lseq, 0), LmMnXFRLVL_512);
+	/* Initialize Mode 1 Transfer Level to 256. */
+	asd_write_reg_byte(asd_ha, LmMnXFRLVL(lseq, 1), LmMnXFRLVL_256);
+
+	/* Initialize Program Count. */
+	asd_write_reg_word(asd_ha, LmPRGMCNT(lseq), lseq_idle_loop);
+
+	/* Enable Blind SG Move. */
+	asd_write_reg_dword(asd_ha, LmMODECTL(lseq), LmBLIND48);
+	asd_write_reg_word(asd_ha, LmM3SATATIMER(lseq),
+			   ASD_SATA_INTERLOCK_TIMEOUT);
+
+	(void) asd_read_reg_dword(asd_ha, LmREQMBX(lseq));
+
+	/* Clear Primitive Status 0 and 1. */
+	asd_write_reg_dword(asd_ha, LmPRMSTAT0(lseq), 0xFFFFFFFF);
+	asd_write_reg_dword(asd_ha, LmPRMSTAT1(lseq), 0xFFFFFFFF);
+
+	/* Clear HW Timer status. */
+	asd_write_reg_byte(asd_ha, LmHWTSTAT(lseq), 0xFF);
+
+	/* Clear DMA Errors for Mode 0 and 1. */
+	asd_write_reg_byte(asd_ha, LmMnDMAERRS(lseq, 0), 0xFF);
+	asd_write_reg_byte(asd_ha, LmMnDMAERRS(lseq, 1), 0xFF);
+
+	/* Clear SG DMA Errors for Mode 0 and 1. */
+	asd_write_reg_byte(asd_ha, LmMnSGDMAERRS(lseq, 0), 0xFF);
+	asd_write_reg_byte(asd_ha, LmMnSGDMAERRS(lseq, 1), 0xFF);
+
+	/* Clear Mode 0 Buffer Parity Error. */
+	asd_write_reg_byte(asd_ha, LmMnBUFSTAT(lseq, 0), LmMnBUFPERR);
+
+	/* Clear Mode 0 Frame Error register. */
+	asd_write_reg_dword(asd_ha, LmMnFRMERR(lseq, 0), 0xFFFFFFFF);
+
+	/* Reset LSEQ external interrupt arbiter. */
+	asd_write_reg_byte(asd_ha, LmARP2INTCTL(lseq), RSTINTCTL);
+
+	/* Set the Phy SAS for the LmSEQ WWN. */
+	sas_addr = asd_ha->phys[lseq].phy_desc->sas_addr;
+	for (i = 0; i < SAS_ADDR_SIZE; i++)
+		asd_write_reg_byte(asd_ha, LmWWN(lseq) + i, sas_addr[i]);
+
+	/* Set the Transmit Size to 1024 bytes, 0 = 256 Dwords. */
+	asd_write_reg_byte(asd_ha, LmMnXMTSIZE(lseq, 1), 0);
+
+	/* Set the Bus Inactivity Time Limit Timer. */
+	asd_write_reg_word(asd_ha, LmBITL_TIMER(lseq), 9);
+
+	/* Enable SATA Port Multiplier. */
+	asd_write_reg_byte(asd_ha, LmMnSATAFS(lseq, 1), 0x80);
+
+	/* Initialize Interrupt Vector[0-10] address in Mode 3.
+	 * See the comment on CSEQ_INT_* */
+	asd_write_reg_word(asd_ha, LmM3INTVEC0(lseq), lseq_vecs[0]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC1(lseq), lseq_vecs[1]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC2(lseq), lseq_vecs[2]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC3(lseq), lseq_vecs[3]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC4(lseq), lseq_vecs[4]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC5(lseq), lseq_vecs[5]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC6(lseq), lseq_vecs[6]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC7(lseq), lseq_vecs[7]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC8(lseq), lseq_vecs[8]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC9(lseq), lseq_vecs[9]);
+	asd_write_reg_word(asd_ha, LmM3INTVEC10(lseq), lseq_vecs[10]);
+	/*
+	 * Program the Link LED control, applicable only for
+	 * Chip Rev. B or later.
+	 */
+	asd_write_reg_dword(asd_ha, LmCONTROL(lseq),
+			    (LEDTIMER | LEDMODE_TXRX | LEDTIMERS_100ms));
+
+	/* Set the Align Rate for SAS and STP mode. */
+	asd_write_reg_byte(asd_ha, LmM1SASALIGN(lseq), SAS_ALIGN_DEFAULT);
+	asd_write_reg_byte(asd_ha, LmM1STPALIGN(lseq), STP_ALIGN_DEFAULT);
+}
+
+
+/**
+ * asd_post_init_cseq -- clear CSEQ Mode n Int. status and Response mailbox
+ * @asd_ha: pointer to host adapter struct
+ */
+static void asd_post_init_cseq(struct asd_ha_struct *asd_ha)
+{
+	int i;
+
+	for (i = 0; i < 8; i++)
+		asd_write_reg_dword(asd_ha, CMnINT(i), 0xFFFFFFFF);
+	for (i = 0; i < 8; i++)
+		asd_read_reg_dword(asd_ha, CMnRSPMBX(i));
+	/* Reset the external interrupt arbiter. */
+	asd_write_reg_byte(asd_ha, CARP2INTCTL, RSTINTCTL);
+}
+
+/**
+ * asd_init_ddb_0 -- initialize DDB 0
+ * @asd_ha: pointer to host adapter structure
+ *
+ * Initialize DDB site 0 which is used internally by the sequencer.
+ */
+static void asd_init_ddb_0(struct asd_ha_struct *asd_ha)
+{
+	int	i;
+
+	/* Zero out the DDB explicitly */
+	for (i = 0; i < sizeof(struct asd_ddb_seq_shared); i+=4)
+		asd_ddbsite_write_dword(asd_ha, 0, i, 0);
+
+	asd_ddbsite_write_word(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, q_free_ddb_head), 0);
+	asd_ddbsite_write_word(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, q_free_ddb_tail),
+			       asd_ha->hw_prof.max_ddbs-1);
+	asd_ddbsite_write_word(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, q_free_ddb_cnt), 0);
+	asd_ddbsite_write_word(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, q_used_ddb_head), 0xFFFF);
+	asd_ddbsite_write_word(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, q_used_ddb_tail), 0xFFFF);
+	asd_ddbsite_write_word(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, shared_mem_lock), 0);
+	asd_ddbsite_write_word(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, smp_conn_tag), 0);
+	asd_ddbsite_write_word(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, est_nexus_buf_cnt), 0);
+	asd_ddbsite_write_word(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, est_nexus_buf_thresh),
+			       asd_ha->hw_prof.num_phys * 2);
+	asd_ddbsite_write_byte(asd_ha, 0,
+		 offsetof(struct asd_ddb_seq_shared, settable_max_contexts),0);
+	asd_ddbsite_write_byte(asd_ha, 0,
+	       offsetof(struct asd_ddb_seq_shared, conn_not_active), 0xFF);
+	asd_ddbsite_write_byte(asd_ha, 0,
+	       offsetof(struct asd_ddb_seq_shared, phy_is_up), 0x00);
+	/* DDB 0 is reserved */
+	set_bit(0, asd_ha->hw_prof.ddb_bitmap);
+}
+
+/**
+ * asd_seq_setup_seqs -- setup and initialize central and link sequencers
+ * @asd_ha: pointer to host adapter structure
+ */
+static void asd_seq_setup_seqs(struct asd_ha_struct *asd_ha)
+{
+	int 		lseq;
+	u8		lseq_mask;
+
+	/* Initialize SCB sites. Done first to compute some values which
+	 * the rest of the init code depends on. */
+	asd_init_scb_sites(asd_ha);
+
+	/* Initialize CSEQ Scratch RAM registers. */
+	asd_init_cseq_scratch(asd_ha);
+
+	/* Initialize LmSEQ Scratch RAM registers. */
+	asd_init_lseq_scratch(asd_ha);
+
+	/* Initialize CSEQ CIO registers. */
+	asd_init_cseq_cio(asd_ha);
+
+	asd_init_ddb_0(asd_ha);
+
+	/* Initialize LmSEQ CIO registers. */
+	lseq_mask = asd_ha->hw_prof.enabled_phys;
+	for_each_sequencer(lseq_mask, lseq_mask, lseq)
+		asd_init_lseq_cio(asd_ha, lseq);
+	asd_post_init_cseq(asd_ha);
+}
+
+
+/**
+ * asd_seq_start_cseq -- start the central sequencer, CSEQ
+ * @asd_ha: pointer to host adapter structure
+ */
+static int asd_seq_start_cseq(struct asd_ha_struct *asd_ha)
+{
+	/* Reset the ARP2 instruction to location zero. */
+	asd_write_reg_word(asd_ha, CPRGMCNT, cseq_idle_loop);
+
+	/* Unpause the CSEQ  */
+	return asd_unpause_cseq(asd_ha);
+}
+
+/**
+ * asd_seq_start_lseq -- start a link sequencer
+ * @asd_ha: pointer to host adapter structure
+ * @lseq: the link sequencer of interest
+ */
+static int asd_seq_start_lseq(struct asd_ha_struct *asd_ha, int lseq)
+{
+	/* Reset the ARP2 instruction to location zero. */
+	asd_write_reg_word(asd_ha, LmPRGMCNT(lseq), lseq_idle_loop);
+
+	/* Unpause the LmSEQ  */
+	return asd_seq_unpause_lseq(asd_ha, lseq);
+}
+
+static int asd_request_firmware(struct asd_ha_struct *asd_ha)
+{
+	int err, i;
+	struct sequencer_file_header header, *hdr_ptr;
+	u32 csum = 0;
+	u16 *ptr_cseq_vecs, *ptr_lseq_vecs;
+
+	if (sequencer_fw)
+		/* already loaded */
+		return 0;
+
+	err = request_firmware(&sequencer_fw,
+			       SAS_RAZOR_SEQUENCER_FW_FILE,
+			       &asd_ha->pcidev->dev);
+	if (err)
+		return err;
+
+	hdr_ptr = (struct sequencer_file_header *)sequencer_fw->data;
+
+	header.csum = le32_to_cpu(hdr_ptr->csum);
+	header.major = le32_to_cpu(hdr_ptr->major);
+	header.minor = le32_to_cpu(hdr_ptr->minor);
+	sequencer_version = hdr_ptr->version;
+	header.cseq_table_offset = le32_to_cpu(hdr_ptr->cseq_table_offset);
+	header.cseq_table_size = le32_to_cpu(hdr_ptr->cseq_table_size);
+	header.lseq_table_offset = le32_to_cpu(hdr_ptr->lseq_table_offset);
+	header.lseq_table_size = le32_to_cpu(hdr_ptr->lseq_table_size);
+	header.cseq_code_offset = le32_to_cpu(hdr_ptr->cseq_code_offset);
+	header.cseq_code_size = le32_to_cpu(hdr_ptr->cseq_code_size);
+	header.lseq_code_offset = le32_to_cpu(hdr_ptr->lseq_code_offset);
+	header.lseq_code_size = le32_to_cpu(hdr_ptr->lseq_code_size);
+	header.mode2_task = le16_to_cpu(hdr_ptr->mode2_task);
+	header.cseq_idle_loop = le16_to_cpu(hdr_ptr->cseq_idle_loop);
+	header.lseq_idle_loop = le16_to_cpu(hdr_ptr->lseq_idle_loop);
+
+	for (i = sizeof(header.csum); i < sequencer_fw->size; i++)
+		csum += sequencer_fw->data[i];
+
+	if (csum != header.csum) {
+		asd_printk("Firmware file checksum mismatch\n");
+		return -EINVAL;
+	}
+
+	if (header.cseq_table_size != CSEQ_NUM_VECS ||
+	    header.lseq_table_size != LSEQ_NUM_VECS) {
+		asd_printk("Firmware file table size mismatch\n");
+		return -EINVAL;
+	}
+
+	ptr_cseq_vecs = (u16 *)&sequencer_fw->data[header.cseq_table_offset];
+	ptr_lseq_vecs = (u16 *)&sequencer_fw->data[header.lseq_table_offset];
+	mode2_task = header.mode2_task;
+	cseq_idle_loop = header.cseq_idle_loop;
+	lseq_idle_loop = header.lseq_idle_loop;
+
+	for (i = 0; i < CSEQ_NUM_VECS; i++)
+		cseq_vecs[i] = le16_to_cpu(ptr_cseq_vecs[i]);
+
+	for (i = 0; i < LSEQ_NUM_VECS; i++)
+		lseq_vecs[i] = le16_to_cpu(ptr_lseq_vecs[i]);
+
+	cseq_code = &sequencer_fw->data[header.cseq_code_offset];
+	cseq_code_size = header.cseq_code_size;
+	lseq_code = &sequencer_fw->data[header.lseq_code_offset];
+	lseq_code_size = header.lseq_code_size;
+
+	return 0;
+}
+
+int asd_init_seqs(struct asd_ha_struct *asd_ha)
+{
+	int err;
+
+	err = asd_request_firmware(asd_ha);
+
+	if (err) {
+		asd_printk("Failed to load sequencer firmware file %s, error %d\n",
+			   SAS_RAZOR_SEQUENCER_FW_FILE, err);
+		return err;
+	}
+
+	asd_printk("using sequencer %s\n", sequencer_version);
+	err = asd_seq_download_seqs(asd_ha);
+	if (err) {
+		asd_printk("couldn't download sequencers for %s\n",
+			   pci_name(asd_ha->pcidev));
+		return err;
+	}
+
+	asd_seq_setup_seqs(asd_ha);
+
+	return 0;
+}
+
+int asd_start_seqs(struct asd_ha_struct *asd_ha)
+{
+	int err;
+	u8  lseq_mask;
+	int lseq;
+
+	err = asd_seq_start_cseq(asd_ha);
+	if (err) {
+		asd_printk("couldn't start CSEQ for %s\n",
+			   pci_name(asd_ha->pcidev));
+		return err;
+	}
+
+	lseq_mask = asd_ha->hw_prof.enabled_phys;
+	for_each_sequencer(lseq_mask, lseq_mask, lseq) {
+		err = asd_seq_start_lseq(asd_ha, lseq);
+		if (err) {
+			asd_printk("coudln't start LSEQ %d for %s\n", lseq,
+				   pci_name(asd_ha->pcidev));
+			return err;
+		}
+	}
+
+	return 0;
+}
+
+/**
+ * asd_update_port_links -- update port_map_by_links and phy_is_up
+ * @sas_phy: pointer to the phy which has been added to a port
+ *
+ * 1) When a link reset has completed and we got BYTES DMAED with a
+ * valid frame we call this function for that phy, to indicate that
+ * the phy is up, i.e. we update the phy_is_up in DDB 0.  The
+ * sequencer checks phy_is_up when pending SCBs are to be sent, and
+ * when an open address frame has been received.
+ *
+ * 2) When we know of ports, we call this function to update the map
+ * of phys participaing in that port, i.e. we update the
+ * port_map_by_links in DDB 0.  When a HARD_RESET primitive has been
+ * received, the sequencer disables all phys in that port.
+ * port_map_by_links is also used as the conn_mask byte in the
+ * initiator/target port DDB.
+ */
+void asd_update_port_links(struct asd_sas_phy *sas_phy)
+{
+	struct asd_ha_struct *asd_ha = sas_phy->ha->lldd_ha;
+	const u8 phy_mask = (u8) sas_phy->port->phy_mask;
+	u8  phy_is_up;
+	u8  mask;
+	int i, err;
+
+	for_each_phy(phy_mask, mask, i)
+		asd_ddbsite_write_byte(asd_ha, 0,
+				       offsetof(struct asd_ddb_seq_shared,
+						port_map_by_links)+i,phy_mask);
+
+	for (i = 0; i < 12; i++) {
+		phy_is_up = asd_ddbsite_read_byte(asd_ha, 0,
+			  offsetof(struct asd_ddb_seq_shared, phy_is_up));
+		err = asd_ddbsite_update_byte(asd_ha, 0,
+				offsetof(struct asd_ddb_seq_shared, phy_is_up),
+				phy_is_up,
+				phy_is_up | phy_mask);
+		if (!err)
+			break;
+		else if (err == -EFAULT) {
+			asd_printk("phy_is_up: parity error in DDB 0\n");
+			break;
+		}
+	}
+
+	if (err)
+		asd_printk("couldn't update DDB 0:error:%d\n", err);
+}
+
+MODULE_FIRMWARE(SAS_RAZOR_SEQUENCER_FW_FILE);
diff --git a/drivers/scsi/aic94xx/aic94xx_seq.h b/drivers/scsi/aic94xx/aic94xx_seq.h
new file mode 100644
index 0000000..42281c3
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_seq.h
@@ -0,0 +1,70 @@
+/*
+ * Aic94xx SAS/SATA driver sequencer interface header file.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#ifndef _AIC94XX_SEQ_H_
+#define _AIC94XX_SEQ_H_
+
+#define CSEQ_NUM_VECS	3
+#define LSEQ_NUM_VECS	11
+
+#define SAS_RAZOR_SEQUENCER_FW_FILE "aic94xx-seq.fw"
+
+/* Note:  All quantites in the sequencer file are little endian */
+struct sequencer_file_header {
+	/* Checksum of the entire contents of the sequencer excluding
+	 * these four bytes */
+	u32	csum;
+	/* numeric major version */
+	u32	major;
+	/* numeric minor version */
+	u32	minor;
+	/* version string printed by driver */
+	char	version[16];
+	u32	cseq_table_offset;
+	u32	cseq_table_size;
+	u32	lseq_table_offset;
+	u32	lseq_table_size;
+	u32	cseq_code_offset;
+	u32	cseq_code_size;
+	u32	lseq_code_offset;
+	u32	lseq_code_size;
+	u16	mode2_task;
+	u16	cseq_idle_loop;
+	u16	lseq_idle_loop;
+} __attribute__((packed));
+
+#ifdef __KERNEL__
+int asd_pause_cseq(struct asd_ha_struct *asd_ha);
+int asd_unpause_cseq(struct asd_ha_struct *asd_ha);
+int asd_pause_lseq(struct asd_ha_struct *asd_ha, u8 lseq_mask);
+int asd_unpause_lseq(struct asd_ha_struct *asd_ha, u8 lseq_mask);
+int asd_init_seqs(struct asd_ha_struct *asd_ha);
+int asd_start_seqs(struct asd_ha_struct *asd_ha);
+
+void asd_update_port_links(struct asd_sas_phy *phy);
+#endif
+
+#endif
diff --git a/drivers/scsi/aic94xx/aic94xx_task.c b/drivers/scsi/aic94xx/aic94xx_task.c
new file mode 100644
index 0000000..285e70d
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_task.c
@@ -0,0 +1,642 @@
+/*
+ * Aic94xx SAS/SATA Tasks
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/spinlock.h>
+#include "aic94xx.h"
+#include "aic94xx_sas.h"
+#include "aic94xx_hwi.h"
+
+static void asd_unbuild_ata_ascb(struct asd_ascb *a);
+static void asd_unbuild_smp_ascb(struct asd_ascb *a);
+static void asd_unbuild_ssp_ascb(struct asd_ascb *a);
+
+static inline void asd_can_dequeue(struct asd_ha_struct *asd_ha, int num)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&asd_ha->seq.pend_q_lock, flags);
+	asd_ha->seq.can_queue += num;
+	spin_unlock_irqrestore(&asd_ha->seq.pend_q_lock, flags);
+}
+
+/* PCI_DMA_... to our direction translation.
+ */
+static const u8 data_dir_flags[] = {
+	[PCI_DMA_BIDIRECTIONAL] = DATA_DIR_BYRECIPIENT,	/* UNSPECIFIED */
+	[PCI_DMA_TODEVICE]      = DATA_DIR_OUT, /* OUTBOUND */
+	[PCI_DMA_FROMDEVICE]    = DATA_DIR_IN, /* INBOUND */
+	[PCI_DMA_NONE]          = DATA_DIR_NONE, /* NO TRANSFER */
+};
+
+static inline int asd_map_scatterlist(struct sas_task *task,
+				      struct sg_el *sg_arr,
+				      unsigned long gfp_flags)
+{
+	struct asd_ascb *ascb = task->lldd_task;
+	struct asd_ha_struct *asd_ha = ascb->ha;
+	struct scatterlist *sc;
+	int num_sg, res;
+
+	if (task->data_dir == PCI_DMA_NONE)
+		return 0;
+
+	if (task->num_scatter == 0) {
+		void *p = task->scatter;
+		dma_addr_t dma = pci_map_single(asd_ha->pcidev, p,
+						task->total_xfer_len,
+						task->data_dir);
+		sg_arr[0].bus_addr = cpu_to_le64((u64)dma);
+		sg_arr[0].size = cpu_to_le32(task->total_xfer_len);
+		sg_arr[0].flags |= ASD_SG_EL_LIST_EOL;
+		return 0;
+	}
+
+	num_sg = pci_map_sg(asd_ha->pcidev, task->scatter, task->num_scatter,
+			    task->data_dir);
+	if (num_sg == 0)
+		return -ENOMEM;
+
+	if (num_sg > 3) {
+		int i;
+
+		ascb->sg_arr = asd_alloc_coherent(asd_ha,
+						  num_sg*sizeof(struct sg_el),
+						  gfp_flags);
+		if (!ascb->sg_arr) {
+			res = -ENOMEM;
+			goto err_unmap;
+		}
+		for (sc = task->scatter, i = 0; i < num_sg; i++, sc++) {
+			struct sg_el *sg =
+				&((struct sg_el *)ascb->sg_arr->vaddr)[i];
+			sg->bus_addr = cpu_to_le64((u64)sg_dma_address(sc));
+			sg->size = cpu_to_le32((u32)sg_dma_len(sc));
+			if (i == num_sg-1)
+				sg->flags |= ASD_SG_EL_LIST_EOL;
+		}
+
+		for (sc = task->scatter, i = 0; i < 2; i++, sc++) {
+			sg_arr[i].bus_addr =
+				cpu_to_le64((u64)sg_dma_address(sc));
+			sg_arr[i].size = cpu_to_le32((u32)sg_dma_len(sc));
+		}
+		sg_arr[1].next_sg_offs = 2 * sizeof(*sg_arr);
+		sg_arr[1].flags |= ASD_SG_EL_LIST_EOS;
+
+		memset(&sg_arr[2], 0, sizeof(*sg_arr));
+		sg_arr[2].bus_addr=cpu_to_le64((u64)ascb->sg_arr->dma_handle);
+	} else {
+		int i;
+		for (sc = task->scatter, i = 0; i < num_sg; i++, sc++) {
+			sg_arr[i].bus_addr =
+				cpu_to_le64((u64)sg_dma_address(sc));
+			sg_arr[i].size = cpu_to_le32((u32)sg_dma_len(sc));
+		}
+		sg_arr[i-1].flags |= ASD_SG_EL_LIST_EOL;
+	}
+
+	return 0;
+err_unmap:
+	pci_unmap_sg(asd_ha->pcidev, task->scatter, task->num_scatter,
+		     task->data_dir);
+	return res;
+}
+
+static inline void asd_unmap_scatterlist(struct asd_ascb *ascb)
+{
+	struct asd_ha_struct *asd_ha = ascb->ha;
+	struct sas_task *task = ascb->uldd_task;
+
+	if (task->data_dir == PCI_DMA_NONE)
+		return;
+
+	if (task->num_scatter == 0) {
+		dma_addr_t dma = (dma_addr_t)
+		       le64_to_cpu(ascb->scb->ssp_task.sg_element[0].bus_addr);
+		pci_unmap_single(ascb->ha->pcidev, dma, task->total_xfer_len,
+				 task->data_dir);
+		return;
+	}
+
+	asd_free_coherent(asd_ha, ascb->sg_arr);
+	pci_unmap_sg(asd_ha->pcidev, task->scatter, task->num_scatter,
+		     task->data_dir);
+}
+
+/* ---------- Task complete tasklet ---------- */
+
+static void asd_get_response_tasklet(struct asd_ascb *ascb,
+				     struct done_list_struct *dl)
+{
+	struct asd_ha_struct *asd_ha = ascb->ha;
+	struct sas_task *task = ascb->uldd_task;
+	struct task_status_struct *ts = &task->task_status;
+	unsigned long flags;
+	struct tc_resp_sb_struct {
+		__le16 index_escb;
+		u8     len_lsb;
+		u8     flags;
+	} __attribute__ ((packed)) *resp_sb = (void *) dl->status_block;
+
+/* 	int  size   = ((resp_sb->flags & 7) << 8) | resp_sb->len_lsb; */
+	int  edb_id = ((resp_sb->flags & 0x70) >> 4)-1;
+	struct asd_ascb *escb;
+	struct asd_dma_tok *edb;
+	void *r;
+
+	spin_lock_irqsave(&asd_ha->seq.tc_index_lock, flags);
+	escb = asd_tc_index_find(&asd_ha->seq,
+				 (int)le16_to_cpu(resp_sb->index_escb));
+	spin_unlock_irqrestore(&asd_ha->seq.tc_index_lock, flags);
+
+	if (!escb) {
+		ASD_DPRINTK("Uh-oh! No escb for this dl?!\n");
+		return;
+	}
+
+	ts->buf_valid_size = 0;
+	edb = asd_ha->seq.edb_arr[edb_id + escb->edb_index];
+	r = edb->vaddr;
+	if (task->task_proto == SAS_PROTO_SSP) {
+		struct ssp_response_iu *iu =
+			r + 16 + sizeof(struct ssp_frame_hdr);
+
+		ts->residual = le32_to_cpu(*(__le32 *)r);
+		ts->resp = SAS_TASK_COMPLETE;
+		if (iu->datapres == 0)
+			ts->stat = iu->status;
+		else if (iu->datapres == 1)
+			ts->stat = iu->resp_data[3];
+		else if (iu->datapres == 2) {
+			ts->stat = SAM_CHECK_COND;
+			ts->buf_valid_size = min((u32) SAS_STATUS_BUF_SIZE,
+					 be32_to_cpu(iu->sense_data_len));
+			memcpy(ts->buf, iu->sense_data, ts->buf_valid_size);
+			if (iu->status != SAM_CHECK_COND) {
+				ASD_DPRINTK("device %llx sent sense data, but "
+					    "stat(0x%x) is not CHECK_CONDITION"
+					    "\n",
+					    SAS_ADDR(task->dev->sas_addr),
+					    ts->stat);
+			}
+		}
+	}  else {
+		struct ata_task_resp *resp = (void *) &ts->buf[0];
+
+		ts->residual = le32_to_cpu(*(__le32 *)r);
+
+		if (SAS_STATUS_BUF_SIZE >= sizeof(*resp)) {
+			resp->frame_len = le16_to_cpu(*(__le16 *)(r+6));
+			memcpy(&resp->ending_fis[0], r+16, 24);
+			ts->buf_valid_size = sizeof(*resp);
+		}
+	}
+
+	asd_invalidate_edb(escb, edb_id);
+}
+
+static void asd_task_tasklet_complete(struct asd_ascb *ascb,
+				      struct done_list_struct *dl)
+{
+	struct sas_task *task = ascb->uldd_task;
+	struct task_status_struct *ts = &task->task_status;
+	unsigned long flags;
+	u8 opcode = dl->opcode;
+
+	asd_can_dequeue(ascb->ha, 1);
+
+Again:
+	switch (opcode) {
+	case TC_NO_ERROR:
+		ts->resp = SAS_TASK_COMPLETE;
+		ts->stat = SAM_GOOD;
+		break;
+	case TC_UNDERRUN:
+		ts->resp = SAS_TASK_COMPLETE;
+		ts->stat = SAS_DATA_UNDERRUN;
+		ts->residual = le32_to_cpu(*(__le32 *)dl->status_block);
+		break;
+	case TC_OVERRUN:
+		ts->resp = SAS_TASK_COMPLETE;
+		ts->stat = SAS_DATA_OVERRUN;
+		ts->residual = 0;
+		break;
+	case TC_SSP_RESP:
+	case TC_ATA_RESP:
+		ts->resp = SAS_TASK_COMPLETE;
+		ts->stat = SAS_PROTO_RESPONSE;
+		asd_get_response_tasklet(ascb, dl);
+		break;
+	case TF_OPEN_REJECT:
+		ts->resp = SAS_TASK_UNDELIVERED;
+		ts->stat = SAS_OPEN_REJECT;
+		if (dl->status_block[1] & 2)
+			ts->open_rej_reason = 1 + dl->status_block[2];
+		else if (dl->status_block[1] & 1)
+			ts->open_rej_reason = (dl->status_block[2] >> 4)+10;
+		else
+			ts->open_rej_reason = SAS_OREJ_UNKNOWN;
+		break;
+	case TF_OPEN_TO:
+		ts->resp = SAS_TASK_UNDELIVERED;
+		ts->stat = SAS_OPEN_TO;
+		break;
+	case TF_PHY_DOWN:
+	case TU_PHY_DOWN:
+		ts->resp = SAS_TASK_UNDELIVERED;
+		ts->stat = SAS_PHY_DOWN;
+		break;
+	case TI_PHY_DOWN:
+		ts->resp = SAS_TASK_COMPLETE;
+		ts->stat = SAS_PHY_DOWN;
+		break;
+	case TI_BREAK:
+	case TI_PROTO_ERR:
+	case TI_NAK:
+	case TI_ACK_NAK_TO:
+	case TF_SMP_XMIT_RCV_ERR:
+	case TC_ATA_R_ERR_RECV:
+		ts->resp = SAS_TASK_COMPLETE;
+		ts->stat = SAS_INTERRUPTED;
+		break;
+	case TF_BREAK:
+	case TU_BREAK:
+	case TU_ACK_NAK_TO:
+	case TF_SMPRSP_TO:
+		ts->resp = SAS_TASK_UNDELIVERED;
+		ts->stat = SAS_DEV_NO_RESPONSE;
+		break;
+	case TF_NAK_RECV:
+		ts->resp = SAS_TASK_COMPLETE;
+		ts->stat = SAS_NAK_R_ERR;
+		break;
+	case TA_I_T_NEXUS_LOSS:
+		opcode = dl->status_block[0];
+		goto Again;
+		break;
+	case TF_INV_CONN_HANDLE:
+		ts->resp = SAS_TASK_UNDELIVERED;
+		ts->stat = SAS_DEVICE_UNKNOWN;
+		break;
+	case TF_REQUESTED_N_PENDING:
+		ts->resp = SAS_TASK_UNDELIVERED;
+		ts->stat = SAS_PENDING;
+		break;
+	case TC_TASK_CLEARED:
+	case TA_ON_REQ:
+		ts->resp = SAS_TASK_COMPLETE;
+		ts->stat = SAS_ABORTED_TASK;
+		break;
+
+	case TF_NO_SMP_CONN:
+	case TF_TMF_NO_CTX:
+	case TF_TMF_NO_TAG:
+	case TF_TMF_TAG_FREE:
+	case TF_TMF_TASK_DONE:
+	case TF_TMF_NO_CONN_HANDLE:
+	case TF_IRTT_TO:
+	case TF_IU_SHORT:
+	case TF_DATA_OFFS_ERR:
+		ts->resp = SAS_TASK_UNDELIVERED;
+		ts->stat = SAS_DEV_NO_RESPONSE;
+		break;
+
+	case TC_LINK_ADM_RESP:
+	case TC_CONTROL_PHY:
+	case TC_RESUME:
+	case TC_PARTIAL_SG_LIST:
+	default:
+		ASD_DPRINTK("%s: dl opcode: 0x%x?\n", __FUNCTION__, opcode);
+		break;
+	}
+
+	switch (task->task_proto) {
+	case SATA_PROTO:
+	case SAS_PROTO_STP:
+		asd_unbuild_ata_ascb(ascb);
+		break;
+	case SAS_PROTO_SMP:
+		asd_unbuild_smp_ascb(ascb);
+		break;
+	case SAS_PROTO_SSP:
+		asd_unbuild_ssp_ascb(ascb);
+	default:
+		break;
+	}
+
+	spin_lock_irqsave(&task->task_state_lock, flags);
+	task->task_state_flags &= ~SAS_TASK_STATE_PENDING;
+	task->task_state_flags |= SAS_TASK_STATE_DONE;
+	if (unlikely((task->task_state_flags & SAS_TASK_STATE_ABORTED))) {
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
+		ASD_DPRINTK("task 0x%p done with opcode 0x%x resp 0x%x "
+			    "stat 0x%x but aborted by upper layer!\n",
+			    task, opcode, ts->resp, ts->stat);
+		complete(&ascb->completion);
+	} else {
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
+		task->lldd_task = NULL;
+		asd_ascb_free(ascb);
+		mb();
+		task->task_done(task);
+	}
+}
+
+/* ---------- ATA ---------- */
+
+static int asd_build_ata_ascb(struct asd_ascb *ascb, struct sas_task *task,
+			      unsigned long gfp_flags)
+{
+	struct domain_device *dev = task->dev;
+	struct scb *scb;
+	u8     flags;
+	int    res = 0;
+
+	scb = ascb->scb;
+
+	if (unlikely(task->ata_task.device_control_reg_update))
+		scb->header.opcode = CONTROL_ATA_DEV;
+	else if (dev->sata_dev.command_set == ATA_COMMAND_SET)
+		scb->header.opcode = INITIATE_ATA_TASK;
+	else
+		scb->header.opcode = INITIATE_ATAPI_TASK;
+
+	scb->ata_task.proto_conn_rate = (1 << 5); /* STP */
+	if (dev->port->oob_mode == SAS_OOB_MODE)
+		scb->ata_task.proto_conn_rate |= dev->linkrate;
+
+	scb->ata_task.total_xfer_len = cpu_to_le32(task->total_xfer_len);
+	scb->ata_task.fis = task->ata_task.fis;
+	scb->ata_task.fis.fis_type = 0x27;
+	if (likely(!task->ata_task.device_control_reg_update))
+		scb->ata_task.fis.flags |= 0x80; /* C=1: update ATA cmd reg */
+	scb->ata_task.fis.flags &= 0xF0; /* PM_PORT field shall be 0 */
+	if (dev->sata_dev.command_set == ATAPI_COMMAND_SET)
+		memcpy(scb->ata_task.atapi_packet, task->ata_task.atapi_packet,
+		       16);
+	scb->ata_task.sister_scb = cpu_to_le16(0xFFFF);
+	scb->ata_task.conn_handle = cpu_to_le16(
+		(u16)(unsigned long)dev->lldd_dev);
+
+	if (likely(!task->ata_task.device_control_reg_update)) {
+		flags = 0;
+		if (task->ata_task.dma_xfer)
+			flags |= DATA_XFER_MODE_DMA;
+		if (task->ata_task.use_ncq &&
+		    dev->sata_dev.command_set != ATAPI_COMMAND_SET)
+			flags |= ATA_Q_TYPE_NCQ;
+		flags |= data_dir_flags[task->data_dir];
+		scb->ata_task.ata_flags = flags;
+
+		scb->ata_task.retry_count = task->ata_task.retry_count;
+
+		flags = 0;
+		if (task->ata_task.set_affil_pol)
+			flags |= SET_AFFIL_POLICY;
+		if (task->ata_task.stp_affil_pol)
+			flags |= STP_AFFIL_POLICY;
+		scb->ata_task.flags = flags;
+	}
+	ascb->tasklet_complete = asd_task_tasklet_complete;
+
+	if (likely(!task->ata_task.device_control_reg_update))
+		res = asd_map_scatterlist(task, scb->ata_task.sg_element,
+					  gfp_flags);
+
+	return res;
+}
+
+static void asd_unbuild_ata_ascb(struct asd_ascb *a)
+{
+	asd_unmap_scatterlist(a);
+}
+
+/* ---------- SMP ---------- */
+
+static int asd_build_smp_ascb(struct asd_ascb *ascb, struct sas_task *task,
+			      unsigned long gfp_flags)
+{
+	struct asd_ha_struct *asd_ha = ascb->ha;
+	struct domain_device *dev = task->dev;
+	struct scb *scb;
+
+	pci_map_sg(asd_ha->pcidev, &task->smp_task.smp_req, 1,
+		   PCI_DMA_FROMDEVICE);
+	pci_map_sg(asd_ha->pcidev, &task->smp_task.smp_resp, 1,
+		   PCI_DMA_FROMDEVICE);
+
+	scb = ascb->scb;
+
+	scb->header.opcode = INITIATE_SMP_TASK;
+
+	scb->smp_task.proto_conn_rate = dev->linkrate;
+
+	scb->smp_task.smp_req.bus_addr =
+		cpu_to_le64((u64)sg_dma_address(&task->smp_task.smp_req));
+	scb->smp_task.smp_req.size =
+		cpu_to_le32((u32)sg_dma_len(&task->smp_task.smp_req)-4);
+
+	scb->smp_task.smp_resp.bus_addr =
+		cpu_to_le64((u64)sg_dma_address(&task->smp_task.smp_resp));
+	scb->smp_task.smp_resp.size =
+		cpu_to_le32((u32)sg_dma_len(&task->smp_task.smp_resp)-4);
+
+	scb->smp_task.sister_scb = cpu_to_le16(0xFFFF);
+	scb->smp_task.conn_handle = cpu_to_le16((u16)
+						(unsigned long)dev->lldd_dev);
+
+	ascb->tasklet_complete = asd_task_tasklet_complete;
+
+	return 0;
+}
+
+static void asd_unbuild_smp_ascb(struct asd_ascb *a)
+{
+	struct sas_task *task = a->uldd_task;
+
+	BUG_ON(!task);
+	pci_unmap_sg(a->ha->pcidev, &task->smp_task.smp_req, 1,
+		     PCI_DMA_FROMDEVICE);
+	pci_unmap_sg(a->ha->pcidev, &task->smp_task.smp_resp, 1,
+		     PCI_DMA_FROMDEVICE);
+}
+
+/* ---------- SSP ---------- */
+
+static int asd_build_ssp_ascb(struct asd_ascb *ascb, struct sas_task *task,
+			      unsigned long gfp_flags)
+{
+	struct domain_device *dev = task->dev;
+	struct scb *scb;
+	int    res = 0;
+
+	scb = ascb->scb;
+
+	scb->header.opcode = INITIATE_SSP_TASK;
+
+	scb->ssp_task.proto_conn_rate  = (1 << 4); /* SSP */
+	scb->ssp_task.proto_conn_rate |= dev->linkrate;
+	scb->ssp_task.total_xfer_len = cpu_to_le32(task->total_xfer_len);
+	scb->ssp_task.ssp_frame.frame_type = SSP_DATA;
+	memcpy(scb->ssp_task.ssp_frame.hashed_dest_addr, dev->hashed_sas_addr,
+	       HASHED_SAS_ADDR_SIZE);
+	memcpy(scb->ssp_task.ssp_frame.hashed_src_addr,
+	       dev->port->ha->hashed_sas_addr, HASHED_SAS_ADDR_SIZE);
+	scb->ssp_task.ssp_frame.tptt = cpu_to_be16(0xFFFF);
+
+	memcpy(scb->ssp_task.ssp_cmd.lun, task->ssp_task.LUN, 8);
+	if (task->ssp_task.enable_first_burst)
+		scb->ssp_task.ssp_cmd.efb_prio_attr |= EFB_MASK;
+	scb->ssp_task.ssp_cmd.efb_prio_attr |= (task->ssp_task.task_prio << 3);
+	scb->ssp_task.ssp_cmd.efb_prio_attr |= (task->ssp_task.task_attr & 7);
+	memcpy(scb->ssp_task.ssp_cmd.cdb, task->ssp_task.cdb, 16);
+
+	scb->ssp_task.sister_scb = cpu_to_le16(0xFFFF);
+	scb->ssp_task.conn_handle = cpu_to_le16(
+		(u16)(unsigned long)dev->lldd_dev);
+	scb->ssp_task.data_dir = data_dir_flags[task->data_dir];
+	scb->ssp_task.retry_count = scb->ssp_task.retry_count;
+
+	ascb->tasklet_complete = asd_task_tasklet_complete;
+
+	res = asd_map_scatterlist(task, scb->ssp_task.sg_element, gfp_flags);
+
+	return res;
+}
+
+static void asd_unbuild_ssp_ascb(struct asd_ascb *a)
+{
+	asd_unmap_scatterlist(a);
+}
+
+/* ---------- Execute Task ---------- */
+
+static inline int asd_can_queue(struct asd_ha_struct *asd_ha, int num)
+{
+	int res = 0;
+	unsigned long flags;
+
+	spin_lock_irqsave(&asd_ha->seq.pend_q_lock, flags);
+	if ((asd_ha->seq.can_queue - num) < 0)
+		res = -SAS_QUEUE_FULL;
+	else
+		asd_ha->seq.can_queue -= num;
+	spin_unlock_irqrestore(&asd_ha->seq.pend_q_lock, flags);
+
+	return res;
+}
+
+int asd_execute_task(struct sas_task *task, const int num,
+		     unsigned long gfp_flags)
+{
+	int res = 0;
+	LIST_HEAD(alist);
+	struct sas_task *t = task;
+	struct asd_ascb *ascb = NULL, *a;
+	struct asd_ha_struct *asd_ha = task->dev->port->ha->lldd_ha;
+
+	res = asd_can_queue(asd_ha, num);
+	if (res)
+		return res;
+
+	res = num;
+	ascb = asd_ascb_alloc_list(asd_ha, &res, gfp_flags);
+	if (res) {
+		res = -ENOMEM;
+		goto out_err;
+	}
+
+	__list_add(&alist, ascb->list.prev, &ascb->list);
+	list_for_each_entry(a, &alist, list) {
+		a->uldd_task = t;
+		t->lldd_task = a;
+		t = list_entry(t->list.next, struct sas_task, list);
+	}
+	list_for_each_entry(a, &alist, list) {
+		t = a->uldd_task;
+		a->uldd_timer = 1;
+		if (t->task_proto & SAS_PROTO_STP)
+			t->task_proto = SAS_PROTO_STP;
+		switch (t->task_proto) {
+		case SATA_PROTO:
+		case SAS_PROTO_STP:
+			res = asd_build_ata_ascb(a, t, gfp_flags);
+			break;
+		case SAS_PROTO_SMP:
+			res = asd_build_smp_ascb(a, t, gfp_flags);
+			break;
+		case SAS_PROTO_SSP:
+			res = asd_build_ssp_ascb(a, t, gfp_flags);
+			break;
+		default:
+			asd_printk("unknown sas_task proto: 0x%x\n",
+				   t->task_proto);
+			res = -ENOMEM;
+			break;
+		}
+		if (res)
+			goto out_err_unmap;
+	}
+	list_del_init(&alist);
+
+	res = asd_post_ascb_list(asd_ha, ascb, num);
+	if (unlikely(res)) {
+		a = NULL;
+		__list_add(&alist, ascb->list.prev, &ascb->list);
+		goto out_err_unmap;
+	}
+
+	return 0;
+out_err_unmap:
+	{
+		struct asd_ascb *b = a;
+		list_for_each_entry(a, &alist, list) {
+			if (a == b)
+				break;
+			t = a->uldd_task;
+			switch (t->task_proto) {
+			case SATA_PROTO:
+			case SAS_PROTO_STP:
+				asd_unbuild_ata_ascb(a);
+				break;
+			case SAS_PROTO_SMP:
+				asd_unbuild_smp_ascb(a);
+				break;
+			case SAS_PROTO_SSP:
+				asd_unbuild_ssp_ascb(a);
+			default:
+				break;
+			}
+			t->lldd_task = NULL;
+		}
+	}
+	list_del_init(&alist);
+out_err:
+	if (ascb)
+		asd_ascb_free_list(ascb);
+	asd_can_dequeue(asd_ha, num);
+	return res;
+}
diff --git a/drivers/scsi/aic94xx/aic94xx_tmf.c b/drivers/scsi/aic94xx/aic94xx_tmf.c
new file mode 100644
index 0000000..6123438
--- /dev/null
+++ b/drivers/scsi/aic94xx/aic94xx_tmf.c
@@ -0,0 +1,636 @@
+/*
+ * Aic94xx Task Management Functions
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This file is part of the aic94xx driver.
+ *
+ * The aic94xx driver is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; version 2 of the
+ * License.
+ *
+ * The aic94xx driver is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with the aic94xx driver; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/spinlock.h>
+#include "aic94xx.h"
+#include "aic94xx_sas.h"
+#include "aic94xx_hwi.h"
+
+/* ---------- Internal enqueue ---------- */
+
+static int asd_enqueue_internal(struct asd_ascb *ascb,
+		void (*tasklet_complete)(struct asd_ascb *,
+					 struct done_list_struct *),
+				void (*timed_out)(unsigned long))
+{
+	int res;
+
+	ascb->tasklet_complete = tasklet_complete;
+	ascb->uldd_timer = 1;
+
+	ascb->timer.data = (unsigned long) ascb;
+	ascb->timer.function = timed_out;
+	ascb->timer.expires = jiffies + AIC94XX_SCB_TIMEOUT;
+
+	add_timer(&ascb->timer);
+
+	res = asd_post_ascb_list(ascb->ha, ascb, 1);
+	if (unlikely(res))
+		del_timer(&ascb->timer);
+	return res;
+}
+
+static inline void asd_timedout_common(unsigned long data)
+{
+	struct asd_ascb *ascb = (void *) data;
+	struct asd_seq_data *seq = &ascb->ha->seq;
+        unsigned long flags;
+
+	spin_lock_irqsave(&seq->pend_q_lock, flags);
+        seq->pending--;
+        list_del_init(&ascb->list);
+        spin_unlock_irqrestore(&seq->pend_q_lock, flags);
+}
+
+/* ---------- CLEAR NEXUS ---------- */
+
+static void asd_clear_nexus_tasklet_complete(struct asd_ascb *ascb,
+					     struct done_list_struct *dl)
+{
+	ASD_DPRINTK("%s: here\n", __FUNCTION__);
+	if (!del_timer(&ascb->timer)) {
+		ASD_DPRINTK("%s: couldn't delete timer\n", __FUNCTION__);
+		return;
+	}
+	ASD_DPRINTK("%s: opcode: 0x%x\n", __FUNCTION__, dl->opcode);
+	ascb->uldd_task = (void *) (unsigned long) dl->opcode;
+	complete(&ascb->completion);
+}
+
+static void asd_clear_nexus_timedout(unsigned long data)
+{
+	struct asd_ascb *ascb = (void *) data;
+
+	ASD_DPRINTK("%s: here\n", __FUNCTION__);
+	asd_timedout_common(data);
+	ascb->uldd_task = (void *) TMF_RESP_FUNC_FAILED;
+	complete(&ascb->completion);
+}
+
+#define CLEAR_NEXUS_PRE         \
+	ASD_DPRINTK("%s: PRE\n", __FUNCTION__); \
+        res = 1;                \
+	ascb = asd_ascb_alloc_list(asd_ha, &res, GFP_KERNEL); \
+	if (!ascb)              \
+		return -ENOMEM; \
+                                \
+	scb = ascb->scb;        \
+	scb->header.opcode = CLEAR_NEXUS
+
+#define CLEAR_NEXUS_POST        \
+	ASD_DPRINTK("%s: POST\n", __FUNCTION__); \
+	res = asd_enqueue_internal(ascb, asd_clear_nexus_tasklet_complete, \
+				   asd_clear_nexus_timedout);              \
+	if (res)                \
+		goto out_err;   \
+	ASD_DPRINTK("%s: clear nexus posted, waiting...\n", __FUNCTION__); \
+	wait_for_completion(&ascb->completion); \
+	res = (int) (unsigned long) ascb->uldd_task; \
+	if (res == TC_NO_ERROR) \
+		res = TMF_RESP_FUNC_COMPLETE;   \
+out_err:                        \
+	asd_ascb_free(ascb);    \
+	return res
+
+int asd_clear_nexus_ha(struct sas_ha_struct *sas_ha)
+{
+	struct asd_ha_struct *asd_ha = sas_ha->lldd_ha;
+	struct asd_ascb *ascb;
+	struct scb *scb;
+	int res;
+
+	CLEAR_NEXUS_PRE;
+	scb->clear_nexus.nexus = NEXUS_ADAPTER;
+	CLEAR_NEXUS_POST;
+}
+
+int asd_clear_nexus_port(struct asd_sas_port *port)
+{
+	struct asd_ha_struct *asd_ha = port->ha->lldd_ha;
+	struct asd_ascb *ascb;
+	struct scb *scb;
+	int res;
+
+	CLEAR_NEXUS_PRE;
+	scb->clear_nexus.nexus = NEXUS_PORT;
+	scb->clear_nexus.conn_mask = port->phy_mask;
+	CLEAR_NEXUS_POST;
+}
+
+#if 0
+static int asd_clear_nexus_I_T(struct domain_device *dev)
+{
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+	struct asd_ascb *ascb;
+	struct scb *scb;
+	int res;
+
+	CLEAR_NEXUS_PRE;
+	scb->clear_nexus.nexus = NEXUS_I_T;
+	scb->clear_nexus.flags = SEND_Q | EXEC_Q | NOTINQ;
+	if (dev->tproto)
+		scb->clear_nexus.flags |= SUSPEND_TX;
+	scb->clear_nexus.conn_handle = cpu_to_le16((u16)(unsigned long)
+						   dev->lldd_dev);
+	CLEAR_NEXUS_POST;
+}
+#endif
+
+static int asd_clear_nexus_I_T_L(struct domain_device *dev, u8 *lun)
+{
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+	struct asd_ascb *ascb;
+	struct scb *scb;
+	int res;
+
+	CLEAR_NEXUS_PRE;
+	scb->clear_nexus.nexus = NEXUS_I_T_L;
+	scb->clear_nexus.flags = SEND_Q | EXEC_Q | NOTINQ;
+	if (dev->tproto)
+		scb->clear_nexus.flags |= SUSPEND_TX;
+	memcpy(scb->clear_nexus.ssp_task.lun, lun, 8);
+	scb->clear_nexus.conn_handle = cpu_to_le16((u16)(unsigned long)
+						   dev->lldd_dev);
+	CLEAR_NEXUS_POST;
+}
+
+static int asd_clear_nexus_tag(struct sas_task *task)
+{
+	struct asd_ha_struct *asd_ha = task->dev->port->ha->lldd_ha;
+	struct asd_ascb *tascb = task->lldd_task;
+	struct asd_ascb *ascb;
+	struct scb *scb;
+	int res;
+
+	CLEAR_NEXUS_PRE;
+	scb->clear_nexus.nexus = NEXUS_TAG;
+	memcpy(scb->clear_nexus.ssp_task.lun, task->ssp_task.LUN, 8);
+	scb->clear_nexus.ssp_task.tag = tascb->tag;
+	if (task->dev->tproto)
+		scb->clear_nexus.conn_handle = cpu_to_le16((u16)(unsigned long)
+							  task->dev->lldd_dev);
+	CLEAR_NEXUS_POST;
+}
+
+static int asd_clear_nexus_index(struct sas_task *task)
+{
+	struct asd_ha_struct *asd_ha = task->dev->port->ha->lldd_ha;
+	struct asd_ascb *tascb = task->lldd_task;
+	struct asd_ascb *ascb;
+	struct scb *scb;
+	int res;
+
+	CLEAR_NEXUS_PRE;
+	scb->clear_nexus.nexus = NEXUS_TRANS_CX;
+	if (task->dev->tproto)
+		scb->clear_nexus.conn_handle = cpu_to_le16((u16)(unsigned long)
+							  task->dev->lldd_dev);
+	scb->clear_nexus.index = cpu_to_le16(tascb->tc_index);
+	CLEAR_NEXUS_POST;
+}
+
+/* ---------- TMFs ---------- */
+
+static void asd_tmf_timedout(unsigned long data)
+{
+	struct asd_ascb *ascb = (void *) data;
+
+	ASD_DPRINTK("tmf timed out\n");
+	asd_timedout_common(data);
+	ascb->uldd_task = (void *) TMF_RESP_FUNC_FAILED;
+	complete(&ascb->completion);
+}
+
+static int asd_get_tmf_resp_tasklet(struct asd_ascb *ascb,
+				    struct done_list_struct *dl)
+{
+	struct asd_ha_struct *asd_ha = ascb->ha;
+	unsigned long flags;
+	struct tc_resp_sb_struct {
+		__le16 index_escb;
+		u8     len_lsb;
+		u8     flags;
+	} __attribute__ ((packed)) *resp_sb = (void *) dl->status_block;
+
+	int  edb_id = ((resp_sb->flags & 0x70) >> 4)-1;
+	struct asd_ascb *escb;
+	struct asd_dma_tok *edb;
+	struct ssp_frame_hdr *fh;
+	struct ssp_response_iu   *ru;
+	int res = TMF_RESP_FUNC_FAILED;
+
+	ASD_DPRINTK("tmf resp tasklet\n");
+
+	spin_lock_irqsave(&asd_ha->seq.tc_index_lock, flags);
+	escb = asd_tc_index_find(&asd_ha->seq,
+				 (int)le16_to_cpu(resp_sb->index_escb));
+	spin_unlock_irqrestore(&asd_ha->seq.tc_index_lock, flags);
+
+	if (!escb) {
+		ASD_DPRINTK("Uh-oh! No escb for this dl?!\n");
+		return res;
+	}
+
+	edb = asd_ha->seq.edb_arr[edb_id + escb->edb_index];
+	ascb->tag = *(__be16 *)(edb->vaddr+4);
+	fh = edb->vaddr + 16;
+	ru = edb->vaddr + 16 + sizeof(*fh);
+	res = ru->status;
+	if (ru->datapres == 1)	  /* Response data present */
+		res = ru->resp_data[3];
+#if 0
+	ascb->tag = fh->tag;
+#endif
+	ascb->tag_valid = 1;
+
+	asd_invalidate_edb(escb, edb_id);
+	return res;
+}
+
+static void asd_tmf_tasklet_complete(struct asd_ascb *ascb,
+				     struct done_list_struct *dl)
+{
+	if (!del_timer(&ascb->timer))
+		return;
+
+	ASD_DPRINTK("tmf tasklet complete\n");
+
+	if (dl->opcode == TC_SSP_RESP)
+		ascb->uldd_task = (void *) (unsigned long)
+			asd_get_tmf_resp_tasklet(ascb, dl);
+	else
+		ascb->uldd_task = (void *) 0xFF00 + (unsigned long) dl->opcode;
+
+	complete(&ascb->completion);
+}
+
+static inline int asd_clear_nexus(struct sas_task *task)
+{
+	int res = TMF_RESP_FUNC_FAILED;
+	struct asd_ascb *tascb = task->lldd_task;
+	unsigned long flags;
+
+	ASD_DPRINTK("task not done, clearing nexus\n");
+	if (tascb->tag_valid)
+		res = asd_clear_nexus_tag(task);
+	else
+		res = asd_clear_nexus_index(task);
+	wait_for_completion_timeout(&tascb->completion,
+				    AIC94XX_SCB_TIMEOUT);
+	ASD_DPRINTK("came back from clear nexus\n");
+	spin_lock_irqsave(&task->task_state_lock, flags);
+	if (task->task_state_flags & SAS_TASK_STATE_DONE)
+		res = TMF_RESP_FUNC_COMPLETE;
+	spin_unlock_irqrestore(&task->task_state_lock, flags);
+
+	return res;
+}
+
+/**
+ * asd_abort_task -- ABORT TASK TMF
+ * @task: the task to be aborted
+ *
+ * Before calling ABORT TASK the task state flags should be ORed with
+ * SAS_TASK_STATE_ABORTED (unless SAS_TASK_STATE_DONE is set) under
+ * the task_state_lock IRQ spinlock, then ABORT TASK *must* be called.
+ *
+ * Implements the ABORT TASK TMF, I_T_L_Q nexus.
+ * Returns: SAS TMF responses (see sas_task.h),
+ *          -ENOMEM,
+ *          -SAS_QUEUE_FULL.
+ *
+ * When ABORT TASK returns, the caller of ABORT TASK checks first the
+ * task->task_state_flags, and then the return value of ABORT TASK.
+ *
+ * If the task has task state bit SAS_TASK_STATE_DONE set, then the
+ * task was completed successfully prior to it being aborted.  The
+ * caller of ABORT TASK has responsibility to call task->task_done()
+ * xor free the task, depending on their framework.  The return code
+ * is TMF_RESP_FUNC_FAILED in this case.
+ *
+ * Else the SAS_TASK_STATE_DONE bit is not set,
+ * 	If the return code is TMF_RESP_FUNC_COMPLETE, then
+ * 		the task was aborted successfully.  The caller of
+ * 		ABORT TASK has responsibility to call task->task_done()
+ *              to finish the task, xor free the task depending on their
+ *		framework.
+ *	else
+ * 		the ABORT TASK returned some kind of error. The task
+ *              was _not_ cancelled.  Nothing can be assumed.
+ *		The caller of ABORT TASK may wish to retry.
+ */
+int asd_abort_task(struct sas_task *task)
+{
+	struct asd_ascb *tascb = task->lldd_task;
+	struct asd_ha_struct *asd_ha = tascb->ha;
+	int res = 1;
+	unsigned long flags;
+	struct asd_ascb *ascb = NULL;
+	struct scb *scb;
+
+	spin_lock_irqsave(&task->task_state_lock, flags);
+	if (task->task_state_flags & SAS_TASK_STATE_DONE) {
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
+		res = TMF_RESP_FUNC_COMPLETE;
+		ASD_DPRINTK("%s: task 0x%p done\n", __FUNCTION__, task);
+		goto out_done;
+	}
+	spin_unlock_irqrestore(&task->task_state_lock, flags);
+
+	ascb = asd_ascb_alloc_list(asd_ha, &res, GFP_KERNEL);
+	if (!ascb)
+		return -ENOMEM;
+	scb = ascb->scb;
+
+	scb->header.opcode = ABORT_TASK;
+
+	switch (task->task_proto) {
+	case SATA_PROTO:
+	case SAS_PROTO_STP:
+		scb->abort_task.proto_conn_rate = (1 << 5); /* STP */
+		break;
+	case SAS_PROTO_SSP:
+		scb->abort_task.proto_conn_rate  = (1 << 4); /* SSP */
+		scb->abort_task.proto_conn_rate |= task->dev->linkrate;
+		break;
+	case SAS_PROTO_SMP:
+		break;
+	default:
+		break;
+	}
+
+	if (task->task_proto == SAS_PROTO_SSP) {
+		scb->abort_task.ssp_frame.frame_type = SSP_TASK;
+		memcpy(scb->abort_task.ssp_frame.hashed_dest_addr,
+		       task->dev->hashed_sas_addr, HASHED_SAS_ADDR_SIZE);
+		memcpy(scb->abort_task.ssp_frame.hashed_src_addr,
+		       task->dev->port->ha->hashed_sas_addr,
+		       HASHED_SAS_ADDR_SIZE);
+		scb->abort_task.ssp_frame.tptt = cpu_to_be16(0xFFFF);
+
+		memcpy(scb->abort_task.ssp_task.lun, task->ssp_task.LUN, 8);
+		scb->abort_task.ssp_task.tmf = TMF_ABORT_TASK;
+		scb->abort_task.ssp_task.tag = cpu_to_be16(0xFFFF);
+	}
+
+	scb->abort_task.sister_scb = cpu_to_le16(0xFFFF);
+	scb->abort_task.conn_handle = cpu_to_le16(
+		(u16)(unsigned long)task->dev->lldd_dev);
+	scb->abort_task.retry_count = 1;
+	scb->abort_task.index = cpu_to_le16((u16)tascb->tc_index);
+	scb->abort_task.itnl_to = cpu_to_le16(ITNL_TIMEOUT_CONST);
+
+	res = asd_enqueue_internal(ascb, asd_tmf_tasklet_complete,
+				   asd_tmf_timedout);
+	if (res)
+		goto out;
+	wait_for_completion(&ascb->completion);
+	ASD_DPRINTK("tmf came back\n");
+
+	res = (int) (unsigned long) ascb->uldd_task;
+	tascb->tag = ascb->tag;
+	tascb->tag_valid = ascb->tag_valid;
+
+	spin_lock_irqsave(&task->task_state_lock, flags);
+	if (task->task_state_flags & SAS_TASK_STATE_DONE) {
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
+		res = TMF_RESP_FUNC_COMPLETE;
+		ASD_DPRINTK("%s: task 0x%p done\n", __FUNCTION__, task);
+		goto out_done;
+	}
+	spin_unlock_irqrestore(&task->task_state_lock, flags);
+
+	switch (res) {
+	/* The task to be aborted has been sent to the device.
+	 * We got a Response IU for the ABORT TASK TMF. */
+	case TC_NO_ERROR + 0xFF00:
+	case TMF_RESP_FUNC_COMPLETE:
+	case TMF_RESP_FUNC_FAILED:
+		res = asd_clear_nexus(task);
+		break;
+	case TMF_RESP_INVALID_FRAME:
+	case TMF_RESP_OVERLAPPED_TAG:
+	case TMF_RESP_FUNC_ESUPP:
+	case TMF_RESP_NO_LUN:
+		goto out_done; break;
+	}
+	/* In the following we assume that the managing layer
+	 * will _never_ make a mistake, when issuing ABORT TASK.
+	 */
+	switch (res) {
+	default:
+		res = asd_clear_nexus(task);
+		/* fallthrough */
+	case TC_NO_ERROR + 0xFF00:
+	case TMF_RESP_FUNC_COMPLETE:
+		break;
+	/* The task hasn't been sent to the device xor we never got
+	 * a (sane) Response IU for the ABORT TASK TMF.
+	 */
+	case TF_NAK_RECV + 0xFF00:
+		res = TMF_RESP_INVALID_FRAME;
+		break;
+	case TF_TMF_TASK_DONE + 0xFF00:	/* done but not reported yet */
+		res = TMF_RESP_FUNC_FAILED;
+		wait_for_completion_timeout(&tascb->completion,
+					    AIC94XX_SCB_TIMEOUT);
+		spin_lock_irqsave(&task->task_state_lock, flags);
+		if (task->task_state_flags & SAS_TASK_STATE_DONE)
+			res = TMF_RESP_FUNC_COMPLETE;
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
+		goto out_done;
+	case TF_TMF_NO_TAG + 0xFF00:
+	case TF_TMF_TAG_FREE + 0xFF00: /* the tag is in the free list */
+	case TF_TMF_NO_CONN_HANDLE + 0xFF00: /* no such device */
+		res = TMF_RESP_FUNC_COMPLETE;
+		goto out_done;
+	case TF_TMF_NO_CTX + 0xFF00: /* not in seq, or proto != SSP */
+		res = TMF_RESP_FUNC_ESUPP;
+		goto out;
+	}
+out_done:
+	if (res == TMF_RESP_FUNC_COMPLETE) {
+		task->lldd_task = NULL;
+		mb();
+		asd_ascb_free(tascb);
+	}
+out:
+	asd_ascb_free(ascb);
+	ASD_DPRINTK("task 0x%p aborted, res: 0x%x\n", task, res);
+	return res;
+}
+
+/**
+ * asd_initiate_ssp_tmf -- send a TMF to an I_T_L or I_T_L_Q nexus
+ * @dev: pointer to struct domain_device of interest
+ * @lun: pointer to u8[8] which is the LUN
+ * @tmf: the TMF to be performed (see sas_task.h or the SAS spec)
+ * @index: the transaction context of the task to be queried if QT TMF
+ *
+ * This function is used to send ABORT TASK SET, CLEAR ACA,
+ * CLEAR TASK SET, LU RESET and QUERY TASK TMFs.
+ *
+ * No SCBs should be queued to the I_T_L nexus when this SCB is
+ * pending.
+ *
+ * Returns: TMF response code (see sas_task.h or the SAS spec)
+ */
+static int asd_initiate_ssp_tmf(struct domain_device *dev, u8 *lun,
+				int tmf, int index)
+{
+	struct asd_ha_struct *asd_ha = dev->port->ha->lldd_ha;
+	struct asd_ascb *ascb;
+	int res = 1;
+	struct scb *scb;
+
+	if (!(dev->tproto & SAS_PROTO_SSP))
+		return TMF_RESP_FUNC_ESUPP;
+
+	ascb = asd_ascb_alloc_list(asd_ha, &res, GFP_KERNEL);
+	if (!ascb)
+		return -ENOMEM;
+	scb = ascb->scb;
+
+	if (tmf == TMF_QUERY_TASK)
+		scb->header.opcode = QUERY_SSP_TASK;
+	else
+		scb->header.opcode = INITIATE_SSP_TMF;
+
+	scb->ssp_tmf.proto_conn_rate  = (1 << 4); /* SSP */
+	scb->ssp_tmf.proto_conn_rate |= dev->linkrate;
+	/* SSP frame header */
+	scb->ssp_tmf.ssp_frame.frame_type = SSP_TASK;
+	memcpy(scb->ssp_tmf.ssp_frame.hashed_dest_addr,
+	       dev->hashed_sas_addr, HASHED_SAS_ADDR_SIZE);
+	memcpy(scb->ssp_tmf.ssp_frame.hashed_src_addr,
+	       dev->port->ha->hashed_sas_addr, HASHED_SAS_ADDR_SIZE);
+	scb->ssp_tmf.ssp_frame.tptt = cpu_to_be16(0xFFFF);
+	/* SSP Task IU */
+	memcpy(scb->ssp_tmf.ssp_task.lun, lun, 8);
+	scb->ssp_tmf.ssp_task.tmf = tmf;
+
+	scb->ssp_tmf.sister_scb = cpu_to_le16(0xFFFF);
+	scb->ssp_tmf.conn_handle= cpu_to_le16((u16)(unsigned long)
+					      dev->lldd_dev);
+	scb->ssp_tmf.retry_count = 1;
+	scb->ssp_tmf.itnl_to = cpu_to_le16(ITNL_TIMEOUT_CONST);
+	if (tmf == TMF_QUERY_TASK)
+		scb->ssp_tmf.index = cpu_to_le16(index);
+
+	res = asd_enqueue_internal(ascb, asd_tmf_tasklet_complete,
+				   asd_tmf_timedout);
+	if (res)
+		goto out_err;
+	wait_for_completion(&ascb->completion);
+	res = (int) (unsigned long) ascb->uldd_task;
+
+	switch (res) {
+	case TC_NO_ERROR + 0xFF00:
+		res = TMF_RESP_FUNC_COMPLETE;
+		break;
+	case TF_NAK_RECV + 0xFF00:
+		res = TMF_RESP_INVALID_FRAME;
+		break;
+	case TF_TMF_TASK_DONE + 0xFF00:
+		res = TMF_RESP_FUNC_FAILED;
+		break;
+	case TF_TMF_NO_TAG + 0xFF00:
+	case TF_TMF_TAG_FREE + 0xFF00: /* the tag is in the free list */
+	case TF_TMF_NO_CONN_HANDLE + 0xFF00: /* no such device */
+		res = TMF_RESP_FUNC_COMPLETE;
+		break;
+	case TF_TMF_NO_CTX + 0xFF00: /* not in seq, or proto != SSP */
+		res = TMF_RESP_FUNC_ESUPP;
+		break;
+	default:
+		ASD_DPRINTK("%s: converting result 0x%x to TMF_RESP_FUNC_FAILED\n",
+			    __FUNCTION__, res);
+		res = TMF_RESP_FUNC_FAILED;
+		break;
+	}
+out_err:
+	asd_ascb_free(ascb);
+	return res;
+}
+
+int asd_abort_task_set(struct domain_device *dev, u8 *lun)
+{
+	int res = asd_initiate_ssp_tmf(dev, lun, TMF_ABORT_TASK_SET, 0);
+
+	if (res == TMF_RESP_FUNC_COMPLETE)
+		asd_clear_nexus_I_T_L(dev, lun);
+	return res;
+}
+
+int asd_clear_aca(struct domain_device *dev, u8 *lun)
+{
+	int res = asd_initiate_ssp_tmf(dev, lun, TMF_CLEAR_ACA, 0);
+
+	if (res == TMF_RESP_FUNC_COMPLETE)
+		asd_clear_nexus_I_T_L(dev, lun);
+	return res;
+}
+
+int asd_clear_task_set(struct domain_device *dev, u8 *lun)
+{
+	int res = asd_initiate_ssp_tmf(dev, lun, TMF_CLEAR_TASK_SET, 0);
+
+	if (res == TMF_RESP_FUNC_COMPLETE)
+		asd_clear_nexus_I_T_L(dev, lun);
+	return res;
+}
+
+int asd_lu_reset(struct domain_device *dev, u8 *lun)
+{
+	int res = asd_initiate_ssp_tmf(dev, lun, TMF_LU_RESET, 0);
+
+	if (res == TMF_RESP_FUNC_COMPLETE)
+		asd_clear_nexus_I_T_L(dev, lun);
+	return res;
+}
+
+/**
+ * asd_query_task -- send a QUERY TASK TMF to an I_T_L_Q nexus
+ * task: pointer to sas_task struct of interest
+ *
+ * Returns: TMF_RESP_FUNC_COMPLETE if the task is not in the task set,
+ * or TMF_RESP_FUNC_SUCC if the task is in the task set.
+ *
+ * Normally the management layer sets the task to aborted state,
+ * and then calls query task and then abort task.
+ */
+int asd_query_task(struct sas_task *task)
+{
+	struct asd_ascb *ascb = task->lldd_task;
+	int index;
+
+	if (ascb) {
+		index = ascb->tc_index;
+		return asd_initiate_ssp_tmf(task->dev, task->ssp_task.LUN,
+					    TMF_QUERY_TASK, index);
+	}
+	return TMF_RESP_FUNC_COMPLETE;
+}
diff --git a/drivers/scsi/arcmsr/Makefile b/drivers/scsi/arcmsr/Makefile
new file mode 100644
index 0000000..721aced
--- /dev/null
+++ b/drivers/scsi/arcmsr/Makefile
@@ -0,0 +1,6 @@
+# File: drivers/arcmsr/Makefile
+# Makefile for the ARECA PCI-X PCI-EXPRESS SATA RAID controllers SCSI driver.
+
+arcmsr-objs := arcmsr_attr.o arcmsr_hba.o
+
+obj-$(CONFIG_SCSI_ARCMSR) := arcmsr.o
diff --git a/drivers/scsi/arcmsr/arcmsr.h b/drivers/scsi/arcmsr/arcmsr.h
new file mode 100644
index 0000000..aff96db
--- /dev/null
+++ b/drivers/scsi/arcmsr/arcmsr.h
@@ -0,0 +1,472 @@
+/*
+*******************************************************************************
+**        O.S   : Linux
+**   FILE NAME  : arcmsr.h
+**        BY    : Erich Chen
+**   Description: SCSI RAID Device Driver for
+**                ARECA RAID Host adapter
+*******************************************************************************
+** Copyright (C) 2002 - 2005, Areca Technology Corporation All rights reserved.
+**
+**     Web site: www.areca.com.tw
+**       E-mail: erich@areca.com.tw
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU General Public License version 2 as
+** published by the Free Software Foundation.
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+*******************************************************************************
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING, BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION)HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+**(INCLUDING NEGLIGENCE OR OTHERWISE)ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************
+*/
+#include <linux/interrupt.h>
+
+struct class_device_attribute;
+
+#define ARCMSR_MAX_OUTSTANDING_CMD 						256
+#define ARCMSR_MAX_FREECCB_NUM							288
+#define ARCMSR_DRIVER_VERSION				"Driver Version 1.20.00.13"
+#define ARCMSR_SCSI_INITIATOR_ID						255
+#define ARCMSR_MAX_XFER_SECTORS							512
+#define ARCMSR_MAX_TARGETID							 17
+#define ARCMSR_MAX_TARGETLUN							  8
+#define ARCMSR_MAX_CMD_PERLUN				 ARCMSR_MAX_OUTSTANDING_CMD
+#define ARCMSR_MAX_QBUFFER						       4096
+#define ARCMSR_MAX_SG_ENTRIES							 38
+
+/*
+*******************************************************************************
+**        split 64bits dma addressing
+*******************************************************************************
+*/
+#define dma_addr_hi32(addr)               (uint32_t) ((addr>>16)>>16)
+#define dma_addr_lo32(addr)               (uint32_t) (addr & 0xffffffff)
+/*
+*******************************************************************************
+**        MESSAGE CONTROL CODE
+*******************************************************************************
+*/
+struct CMD_MESSAGE
+{
+      uint32_t HeaderLength;
+      uint8_t  Signature[8];
+      uint32_t Timeout;
+      uint32_t ControlCode;
+      uint32_t ReturnCode;
+      uint32_t Length;
+};
+/*
+*******************************************************************************
+**        IOP Message Transfer Data for user space
+*******************************************************************************
+*/
+struct CMD_MESSAGE_FIELD
+{
+    struct CMD_MESSAGE			cmdmessage;
+    uint8_t				messagedatabuffer[1032];
+};
+/* IOP message transfer */
+#define ARCMSR_MESSAGE_FAIL             0x0001
+/* DeviceType */
+#define ARECA_SATA_RAID				0x90000000
+/* FunctionCode */
+#define FUNCTION_READ_RQBUFFER			0x0801
+#define FUNCTION_WRITE_WQBUFFER			0x0802
+#define FUNCTION_CLEAR_RQBUFFER			0x0803
+#define FUNCTION_CLEAR_WQBUFFER			0x0804
+#define FUNCTION_CLEAR_ALLQBUFFER		0x0805
+#define FUNCTION_RETURN_CODE_3F			0x0806
+#define FUNCTION_SAY_HELLO			0x0807
+#define FUNCTION_SAY_GOODBYE			0x0808
+#define FUNCTION_FLUSH_ADAPTER_CACHE		0x0809
+/* ARECA IO CONTROL CODE*/
+#define ARCMSR_MESSAGE_READ_RQBUFFER       \
+	ARECA_SATA_RAID | FUNCTION_READ_RQBUFFER
+#define ARCMSR_MESSAGE_WRITE_WQBUFFER      \
+	ARECA_SATA_RAID | FUNCTION_WRITE_WQBUFFER
+#define ARCMSR_MESSAGE_CLEAR_RQBUFFER      \
+	ARECA_SATA_RAID | FUNCTION_CLEAR_RQBUFFER
+#define ARCMSR_MESSAGE_CLEAR_WQBUFFER      \
+	ARECA_SATA_RAID | FUNCTION_CLEAR_WQBUFFER
+#define ARCMSR_MESSAGE_CLEAR_ALLQBUFFER    \
+	ARECA_SATA_RAID | FUNCTION_CLEAR_ALLQBUFFER
+#define ARCMSR_MESSAGE_RETURN_CODE_3F      \
+	ARECA_SATA_RAID | FUNCTION_RETURN_CODE_3F
+#define ARCMSR_MESSAGE_SAY_HELLO           \
+	ARECA_SATA_RAID | FUNCTION_SAY_HELLO
+#define ARCMSR_MESSAGE_SAY_GOODBYE         \
+	ARECA_SATA_RAID | FUNCTION_SAY_GOODBYE
+#define ARCMSR_MESSAGE_FLUSH_ADAPTER_CACHE \
+	ARECA_SATA_RAID | FUNCTION_FLUSH_ADAPTER_CACHE
+/* ARECA IOCTL ReturnCode */
+#define ARCMSR_MESSAGE_RETURNCODE_OK              0x00000001
+#define ARCMSR_MESSAGE_RETURNCODE_ERROR           0x00000006
+#define ARCMSR_MESSAGE_RETURNCODE_3F              0x0000003F
+/*
+*************************************************************
+**   structure for holding DMA address data
+*************************************************************
+*/
+#define IS_SG64_ADDR                0x01000000 /* bit24 */
+struct  SG32ENTRY
+{
+	uint32_t					length;
+	uint32_t					address;
+};
+struct  SG64ENTRY
+{
+ 	uint32_t					length;
+ 	uint32_t					address;
+ 	uint32_t					addresshigh;
+};
+struct SGENTRY_UNION
+{
+	union
+	{
+		struct SG32ENTRY            sg32entry;
+		struct SG64ENTRY            sg64entry;
+	}u;
+};
+/*
+********************************************************************
+**      Q Buffer of IOP Message Transfer
+********************************************************************
+*/
+struct QBUFFER
+{
+	uint32_t      data_len;
+	uint8_t       data[124];
+};
+/*
+*******************************************************************************
+**      FIRMWARE INFO
+*******************************************************************************
+*/
+struct FIRMWARE_INFO
+{
+	uint32_t      signature;                /*0, 00-03*/
+	uint32_t      request_len;              /*1, 04-07*/
+	uint32_t      numbers_queue;            /*2, 08-11*/
+	uint32_t      sdram_size;               /*3, 12-15*/
+	uint32_t      ide_channels;             /*4, 16-19*/
+	char          vendor[40];               /*5, 20-59*/
+	char          model[8];                 /*15, 60-67*/
+	char          firmware_ver[16];         /*17, 68-83*/
+	char          device_map[16];           /*21, 84-99*/
+};
+/* signature of set and get firmware config */
+#define ARCMSR_SIGNATURE_GET_CONFIG                   0x87974060
+#define ARCMSR_SIGNATURE_SET_CONFIG                   0x87974063
+/* message code of inbound message register */
+#define ARCMSR_INBOUND_MESG0_NOP                      0x00000000
+#define ARCMSR_INBOUND_MESG0_GET_CONFIG               0x00000001
+#define ARCMSR_INBOUND_MESG0_SET_CONFIG               0x00000002
+#define ARCMSR_INBOUND_MESG0_ABORT_CMD                0x00000003
+#define ARCMSR_INBOUND_MESG0_STOP_BGRB                0x00000004
+#define ARCMSR_INBOUND_MESG0_FLUSH_CACHE              0x00000005
+#define ARCMSR_INBOUND_MESG0_START_BGRB               0x00000006
+#define ARCMSR_INBOUND_MESG0_CHK331PENDING            0x00000007
+#define ARCMSR_INBOUND_MESG0_SYNC_TIMER               0x00000008
+/* doorbell interrupt generator */
+#define ARCMSR_INBOUND_DRIVER_DATA_WRITE_OK           0x00000001
+#define ARCMSR_INBOUND_DRIVER_DATA_READ_OK            0x00000002
+#define ARCMSR_OUTBOUND_IOP331_DATA_WRITE_OK          0x00000001
+#define ARCMSR_OUTBOUND_IOP331_DATA_READ_OK           0x00000002
+/* ccb areca cdb flag */
+#define ARCMSR_CCBPOST_FLAG_SGL_BSIZE                 0x80000000
+#define ARCMSR_CCBPOST_FLAG_IAM_BIOS                  0x40000000
+#define ARCMSR_CCBREPLY_FLAG_IAM_BIOS                 0x40000000
+#define ARCMSR_CCBREPLY_FLAG_ERROR                    0x10000000
+/* outbound firmware ok */
+#define ARCMSR_OUTBOUND_MESG1_FIRMWARE_OK             0x80000000
+/*
+*******************************************************************************
+**    ARECA SCSI COMMAND DESCRIPTOR BLOCK size 0x1F8 (504)
+*******************************************************************************
+*/
+struct ARCMSR_CDB
+{
+	uint8_t							Bus;
+	uint8_t							TargetID;
+	uint8_t							LUN;
+	uint8_t							Function;
+
+	uint8_t							CdbLength;
+	uint8_t							sgcount;
+	uint8_t							Flags;
+#define ARCMSR_CDB_FLAG_SGL_BSIZE          0x01
+#define ARCMSR_CDB_FLAG_BIOS               0x02
+#define ARCMSR_CDB_FLAG_WRITE              0x04
+#define ARCMSR_CDB_FLAG_SIMPLEQ            0x00
+#define ARCMSR_CDB_FLAG_HEADQ              0x08
+#define ARCMSR_CDB_FLAG_ORDEREDQ           0x10
+	uint8_t							Reserved1;
+
+	uint32_t						Context;
+	uint32_t						DataLength;
+
+	uint8_t							Cdb[16];
+
+	uint8_t							DeviceStatus;
+#define ARCMSR_DEV_CHECK_CONDITION          0x02
+#define ARCMSR_DEV_SELECT_TIMEOUT			0xF0
+#define ARCMSR_DEV_ABORTED				0xF1
+#define ARCMSR_DEV_INIT_FAIL				0xF2
+	uint8_t							SenseData[15];
+
+	union
+	{
+		struct SG32ENTRY                sg32entry[ARCMSR_MAX_SG_ENTRIES];
+		struct SG64ENTRY                sg64entry[ARCMSR_MAX_SG_ENTRIES];
+	} u;
+};
+/*
+*******************************************************************************
+**     Messaging Unit (MU) of the Intel R 80331 I/O processor (80331)
+*******************************************************************************
+*/
+struct MessageUnit
+{
+	uint32_t	resrved0[4];			/*0000 000F*/
+	uint32_t	inbound_msgaddr0;		/*0010 0013*/
+	uint32_t	inbound_msgaddr1;		/*0014 0017*/
+	uint32_t	outbound_msgaddr0;		/*0018 001B*/
+	uint32_t	outbound_msgaddr1;		/*001C 001F*/
+	uint32_t	inbound_doorbell;		/*0020 0023*/
+	uint32_t	inbound_intstatus;		/*0024 0027*/
+	uint32_t	inbound_intmask;		/*0028 002B*/
+	uint32_t	outbound_doorbell;		/*002C 002F*/
+	uint32_t	outbound_intstatus;		/*0030 0033*/
+	uint32_t	outbound_intmask;		/*0034 0037*/
+	uint32_t	reserved1[2];			/*0038 003F*/
+	uint32_t	inbound_queueport;		/*0040 0043*/
+	uint32_t	outbound_queueport;     	/*0044 0047*/
+	uint32_t	reserved2[2];			/*0048 004F*/
+	uint32_t	reserved3[492];			/*0050 07FF 492*/
+	uint32_t	reserved4[128];			/*0800 09FF 128*/
+	uint32_t	message_rwbuffer[256];		/*0a00 0DFF 256*/
+	uint32_t	message_wbuffer[32];		/*0E00 0E7F  32*/
+	uint32_t	reserved5[32];			/*0E80 0EFF  32*/
+	uint32_t	message_rbuffer[32];		/*0F00 0F7F  32*/
+	uint32_t	reserved6[32];			/*0F80 0FFF  32*/
+};
+/*
+*******************************************************************************
+**                 Adapter Control Block
+*******************************************************************************
+*/
+struct AdapterControlBlock
+{
+	struct pci_dev *		pdev;
+	struct Scsi_Host *		host;
+	unsigned long			vir2phy_offset;
+	/* Offset is used in making arc cdb physical to virtual calculations */
+	uint32_t			outbound_int_enable;
+
+	struct MessageUnit __iomem *		pmu;
+	/* message unit ATU inbound base address0 */
+
+	uint32_t			acb_flags;
+#define ACB_F_SCSISTOPADAPTER         0x0001
+#define ACB_F_MSG_STOP_BGRB           0x0002
+	/* stop RAID background rebuild */
+#define ACB_F_MSG_START_BGRB          0x0004
+	/* stop RAID background rebuild */
+#define ACB_F_IOPDATA_OVERFLOW        0x0008
+	/* iop message data rqbuffer overflow */
+#define ACB_F_MESSAGE_WQBUFFER_CLEARED  0x0010
+	/* message clear wqbuffer */
+#define ACB_F_MESSAGE_RQBUFFER_CLEARED  0x0020
+	/* message clear rqbuffer */
+#define ACB_F_MESSAGE_WQBUFFER_READED   0x0040
+#define ACB_F_BUS_RESET               0x0080
+#define ACB_F_IOP_INITED              0x0100
+	/* iop init */
+
+	struct CommandControlBlock *			pccb_pool[ARCMSR_MAX_FREECCB_NUM];
+	/* used for memory free */
+	struct list_head		ccb_free_list;
+	/* head of free ccb list */
+	atomic_t			ccboutstandingcount;
+
+	void *				dma_coherent;
+	/* dma_coherent used for memory free */
+	dma_addr_t			dma_coherent_handle;
+	/* dma_coherent_handle used for memory free */
+
+	uint8_t				rqbuffer[ARCMSR_MAX_QBUFFER];
+	/* data collection buffer for read from 80331 */
+	int32_t				rqbuf_firstindex;
+	/* first of read buffer  */
+	int32_t				rqbuf_lastindex;
+	/* last of read buffer   */
+	uint8_t				wqbuffer[ARCMSR_MAX_QBUFFER];
+	/* data collection buffer for write to 80331  */
+	int32_t				wqbuf_firstindex;
+	/* first of write buffer */
+	int32_t				wqbuf_lastindex;
+	/* last of write buffer  */
+	uint8_t				devstate[ARCMSR_MAX_TARGETID][ARCMSR_MAX_TARGETLUN];
+	/* id0 ..... id15, lun0...lun7 */
+#define ARECA_RAID_GONE               0x55
+#define ARECA_RAID_GOOD               0xaa
+	uint32_t			num_resets;
+	uint32_t			num_aborts;
+	uint32_t			firm_request_len;
+	uint32_t			firm_numbers_queue;
+	uint32_t			firm_sdram_size;
+	uint32_t			firm_hd_channels;
+	char				firm_model[12];
+	char				firm_version[20];
+};/* HW_DEVICE_EXTENSION */
+/*
+*******************************************************************************
+**                   Command Control Block
+**             this CCB length must be 32 bytes boundary
+*******************************************************************************
+*/
+struct CommandControlBlock
+{
+	struct ARCMSR_CDB		arcmsr_cdb;
+	/*
+	** 0-503 (size of CDB=504):
+	** arcmsr messenger scsi command descriptor size 504 bytes
+	*/
+	uint32_t			cdb_shifted_phyaddr;
+	/* 504-507 */
+	uint32_t			reserved1;
+	/* 508-511 */
+#if BITS_PER_LONG == 64
+	/*  ======================512+64 bytes========================  */
+	struct list_head		list;
+	/* 512-527 16 bytes next/prev ptrs for ccb lists */
+	struct scsi_cmnd *		pcmd;
+	/* 528-535 8 bytes pointer of linux scsi command */
+	struct AdapterControlBlock *	acb;
+	/* 536-543 8 bytes pointer of acb */
+
+	uint16_t			ccb_flags;
+	/* 544-545 */
+	#define		CCB_FLAG_READ			0x0000
+	#define		CCB_FLAG_WRITE			0x0001
+	#define		CCB_FLAG_ERROR			0x0002
+	#define		CCB_FLAG_FLUSHCACHE		0x0004
+	#define		CCB_FLAG_MASTER_ABORTED		0x0008
+	uint16_t			startdone;
+	/* 546-547 */
+	#define		ARCMSR_CCB_DONE			0x0000
+	#define		ARCMSR_CCB_START		0x55AA
+	#define		ARCMSR_CCB_ABORTED		0xAA55
+	#define		ARCMSR_CCB_ILLEGAL		0xFFFF
+	uint32_t			reserved2[7];
+	/* 548-551 552-555 556-559 560-563 564-567 568-571 572-575 */
+#else
+	/*  ======================512+32 bytes========================  */
+	struct list_head		list;
+	/* 512-519 8 bytes next/prev ptrs for ccb lists */
+	struct scsi_cmnd *		pcmd;
+	/* 520-523 4 bytes pointer of linux scsi command */
+	struct AdapterControlBlock *	acb;
+	/* 524-527 4 bytes pointer of acb */
+
+	uint16_t			ccb_flags;
+	/* 528-529 */
+	#define		CCB_FLAG_READ			0x0000
+	#define		CCB_FLAG_WRITE			0x0001
+	#define		CCB_FLAG_ERROR			0x0002
+	#define		CCB_FLAG_FLUSHCACHE		0x0004
+	#define		CCB_FLAG_MASTER_ABORTED		0x0008
+	uint16_t			startdone;
+	/* 530-531 */
+	#define		ARCMSR_CCB_DONE			0x0000
+	#define		ARCMSR_CCB_START		0x55AA
+	#define		ARCMSR_CCB_ABORTED		0xAA55
+	#define		ARCMSR_CCB_ILLEGAL		0xFFFF
+	uint32_t			reserved2[3];
+	/* 532-535 536-539 540-543 */
+#endif
+	/*  ==========================================================  */
+};
+/*
+*******************************************************************************
+**    ARECA SCSI sense data
+*******************************************************************************
+*/
+struct SENSE_DATA
+{
+	uint8_t				ErrorCode:7;
+#define SCSI_SENSE_CURRENT_ERRORS	0x70
+#define SCSI_SENSE_DEFERRED_ERRORS	0x71
+	uint8_t				Valid:1;
+	uint8_t				SegmentNumber;
+	uint8_t				SenseKey:4;
+	uint8_t				Reserved:1;
+	uint8_t				IncorrectLength:1;
+	uint8_t				EndOfMedia:1;
+	uint8_t				FileMark:1;
+	uint8_t				Information[4];
+	uint8_t				AdditionalSenseLength;
+	uint8_t				CommandSpecificInformation[4];
+	uint8_t				AdditionalSenseCode;
+	uint8_t				AdditionalSenseCodeQualifier;
+	uint8_t				FieldReplaceableUnitCode;
+	uint8_t				SenseKeySpecific[3];
+};
+/*
+*******************************************************************************
+**  Outbound Interrupt Status Register - OISR
+*******************************************************************************
+*/
+#define     ARCMSR_MU_OUTBOUND_INTERRUPT_STATUS_REG                 0x30
+#define     ARCMSR_MU_OUTBOUND_PCI_INT                              0x10
+#define     ARCMSR_MU_OUTBOUND_POSTQUEUE_INT                        0x08
+#define     ARCMSR_MU_OUTBOUND_DOORBELL_INT                         0x04
+#define     ARCMSR_MU_OUTBOUND_MESSAGE1_INT                         0x02
+#define     ARCMSR_MU_OUTBOUND_MESSAGE0_INT                         0x01
+#define     ARCMSR_MU_OUTBOUND_HANDLE_INT                 \
+                    (ARCMSR_MU_OUTBOUND_MESSAGE0_INT      \
+                     |ARCMSR_MU_OUTBOUND_MESSAGE1_INT     \
+                     |ARCMSR_MU_OUTBOUND_DOORBELL_INT     \
+                     |ARCMSR_MU_OUTBOUND_POSTQUEUE_INT    \
+                     |ARCMSR_MU_OUTBOUND_PCI_INT)
+/*
+*******************************************************************************
+**  Outbound Interrupt Mask Register - OIMR
+*******************************************************************************
+*/
+#define     ARCMSR_MU_OUTBOUND_INTERRUPT_MASK_REG                   0x34
+#define     ARCMSR_MU_OUTBOUND_PCI_INTMASKENABLE                    0x10
+#define     ARCMSR_MU_OUTBOUND_POSTQUEUE_INTMASKENABLE              0x08
+#define     ARCMSR_MU_OUTBOUND_DOORBELL_INTMASKENABLE               0x04
+#define     ARCMSR_MU_OUTBOUND_MESSAGE1_INTMASKENABLE               0x02
+#define     ARCMSR_MU_OUTBOUND_MESSAGE0_INTMASKENABLE               0x01
+#define     ARCMSR_MU_OUTBOUND_ALL_INTMASKENABLE                    0x1F
+
+extern void arcmsr_post_Qbuffer(struct AdapterControlBlock *acb);
+extern struct class_device_attribute *arcmsr_host_attrs[];
+extern int arcmsr_alloc_sysfs_attr(struct AdapterControlBlock *acb);
+void arcmsr_free_sysfs_attr(struct AdapterControlBlock *acb);
+
diff --git a/drivers/scsi/arcmsr/arcmsr_attr.c b/drivers/scsi/arcmsr/arcmsr_attr.c
new file mode 100644
index 0000000..12497da
--- /dev/null
+++ b/drivers/scsi/arcmsr/arcmsr_attr.c
@@ -0,0 +1,381 @@
+/*
+*******************************************************************************
+**        O.S   : Linux
+**   FILE NAME  : arcmsr_attr.c
+**        BY    : Erich Chen
+**   Description: attributes exported to sysfs and device host
+*******************************************************************************
+** Copyright (C) 2002 - 2005, Areca Technology Corporation All rights reserved
+**
+**     Web site: www.areca.com.tw
+**       E-mail: erich@areca.com.tw
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU General Public License version 2 as
+** published by the Free Software Foundation.
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+*******************************************************************************
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING,BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION)HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE)ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************
+** For history of changes, see Documentation/scsi/ChangeLog.arcmsr
+**     Firmware Specification, see Documentation/scsi/arcmsr_spec.txt
+*******************************************************************************
+*/
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/delay.h>
+#include <linux/pci.h>
+
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_transport.h>
+#include "arcmsr.h"
+
+struct class_device_attribute *arcmsr_host_attrs[];
+
+static ssize_t
+arcmsr_sysfs_iop_message_read(struct kobject *kobj, char *buf, loff_t off,
+    size_t count)
+{
+	struct class_device *cdev = container_of(kobj,struct class_device,kobj);
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+	struct MessageUnit __iomem *reg = acb->pmu;
+	uint8_t *pQbuffer,*ptmpQbuffer;
+	int32_t allxfer_len = 0;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EACCES;
+
+	/* do message unit read. */
+	ptmpQbuffer = (uint8_t *)buf;
+	while ((acb->rqbuf_firstindex != acb->rqbuf_lastindex)
+		&& (allxfer_len < 1031)) {
+		pQbuffer = &acb->rqbuffer[acb->rqbuf_firstindex];
+		memcpy(ptmpQbuffer, pQbuffer, 1);
+		acb->rqbuf_firstindex++;
+		acb->rqbuf_firstindex %= ARCMSR_MAX_QBUFFER;
+		ptmpQbuffer++;
+		allxfer_len++;
+	}
+	if (acb->acb_flags & ACB_F_IOPDATA_OVERFLOW) {
+		struct QBUFFER __iomem * prbuffer = (struct QBUFFER __iomem *)
+					&reg->message_rbuffer;
+		uint8_t __iomem * iop_data = (uint8_t __iomem *)prbuffer->data;
+		int32_t iop_len;
+
+		acb->acb_flags &= ~ACB_F_IOPDATA_OVERFLOW;
+		iop_len = readl(&prbuffer->data_len);
+		while (iop_len > 0) {
+			acb->rqbuffer[acb->rqbuf_lastindex] = readb(iop_data);
+			acb->rqbuf_lastindex++;
+			acb->rqbuf_lastindex %= ARCMSR_MAX_QBUFFER;
+			iop_data++;
+			iop_len--;
+		}
+		writel(ARCMSR_INBOUND_DRIVER_DATA_READ_OK,
+				&reg->inbound_doorbell);
+	}
+	return (allxfer_len);
+}
+
+static ssize_t
+arcmsr_sysfs_iop_message_write(struct kobject *kobj, char *buf, loff_t off,
+    size_t count)
+{
+	struct class_device *cdev = container_of(kobj,struct class_device,kobj);
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+	int32_t my_empty_len, user_len, wqbuf_firstindex, wqbuf_lastindex;
+	uint8_t *pQbuffer, *ptmpuserbuffer;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EACCES;
+	if (count > 1032)
+		return -EINVAL;
+	/* do message unit write. */
+	ptmpuserbuffer = (uint8_t *)buf;
+	user_len = (int32_t)count;
+	wqbuf_lastindex = acb->wqbuf_lastindex;
+	wqbuf_firstindex = acb->wqbuf_firstindex;
+	if (wqbuf_lastindex != wqbuf_firstindex) {
+		arcmsr_post_Qbuffer(acb);
+		return 0;	/*need retry*/
+	} else {
+		my_empty_len = (wqbuf_firstindex-wqbuf_lastindex - 1)
+				&(ARCMSR_MAX_QBUFFER - 1);
+		if (my_empty_len >= user_len) {
+			while (user_len > 0) {
+				pQbuffer =
+				&acb->wqbuffer[acb->wqbuf_lastindex];
+				memcpy(pQbuffer, ptmpuserbuffer, 1);
+				acb->wqbuf_lastindex++;
+				acb->wqbuf_lastindex %= ARCMSR_MAX_QBUFFER;
+				ptmpuserbuffer++;
+				user_len--;
+			}
+			if (acb->acb_flags & ACB_F_MESSAGE_WQBUFFER_CLEARED) {
+				acb->acb_flags &=
+					~ACB_F_MESSAGE_WQBUFFER_CLEARED;
+				arcmsr_post_Qbuffer(acb);
+			}
+			return count;
+		} else {
+			return 0;	/*need retry*/
+		}
+	}
+}
+
+static ssize_t
+arcmsr_sysfs_iop_message_clear(struct kobject *kobj, char *buf, loff_t off,
+    size_t count)
+{
+	struct class_device *cdev = container_of(kobj,struct class_device,kobj);
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+	struct MessageUnit __iomem *reg = acb->pmu;
+	uint8_t *pQbuffer;
+
+	if (!capable(CAP_SYS_ADMIN))
+		return -EACCES;
+
+	if (acb->acb_flags & ACB_F_IOPDATA_OVERFLOW) {
+		acb->acb_flags &= ~ACB_F_IOPDATA_OVERFLOW;
+		writel(ARCMSR_INBOUND_DRIVER_DATA_READ_OK
+				, &reg->inbound_doorbell);
+	}
+	acb->acb_flags |=
+		(ACB_F_MESSAGE_WQBUFFER_CLEARED
+		| ACB_F_MESSAGE_RQBUFFER_CLEARED
+		| ACB_F_MESSAGE_WQBUFFER_READED);
+	acb->rqbuf_firstindex = 0;
+	acb->rqbuf_lastindex = 0;
+	acb->wqbuf_firstindex = 0;
+	acb->wqbuf_lastindex = 0;
+	pQbuffer = acb->rqbuffer;
+	memset(pQbuffer, 0, sizeof (struct QBUFFER));
+	pQbuffer = acb->wqbuffer;
+	memset(pQbuffer, 0, sizeof (struct QBUFFER));
+	return 1;
+}
+
+static struct bin_attribute arcmsr_sysfs_message_read_attr = {
+	.attr = {
+		.name = "mu_read",
+		.mode = S_IRUSR ,
+		.owner = THIS_MODULE,
+	},
+	.size = 1032,
+	.read = arcmsr_sysfs_iop_message_read,
+};
+
+static struct bin_attribute arcmsr_sysfs_message_write_attr = {
+	.attr = {
+		.name = "mu_write",
+		.mode = S_IWUSR,
+		.owner = THIS_MODULE,
+	},
+	.size = 1032,
+	.write = arcmsr_sysfs_iop_message_write,
+};
+
+static struct bin_attribute arcmsr_sysfs_message_clear_attr = {
+	.attr = {
+		.name = "mu_clear",
+		.mode = S_IWUSR,
+		.owner = THIS_MODULE,
+	},
+	.size = 1,
+	.write = arcmsr_sysfs_iop_message_clear,
+};
+
+int arcmsr_alloc_sysfs_attr(struct AdapterControlBlock *acb)
+{
+	struct Scsi_Host *host = acb->host;
+	int error;
+
+	error = sysfs_create_bin_file(&host->shost_classdev.kobj,
+				&arcmsr_sysfs_message_read_attr);
+	if (error) {
+		printk(KERN_ERR "arcmsr: alloc sysfs mu_read failed\n");
+		goto error_bin_file_message_read;
+	}
+	error = sysfs_create_bin_file(&host->shost_classdev.kobj,
+				&arcmsr_sysfs_message_write_attr);
+	if (error) {
+		printk(KERN_ERR "arcmsr: alloc sysfs mu_write failed\n");
+		goto error_bin_file_message_write;
+	}
+	error = sysfs_create_bin_file(&host->shost_classdev.kobj,
+				&arcmsr_sysfs_message_clear_attr);
+	if (error) {
+		printk(KERN_ERR "arcmsr: alloc sysfs mu_clear failed\n");
+		goto error_bin_file_message_clear;
+	}
+	return 0;
+error_bin_file_message_clear:
+	sysfs_remove_bin_file(&host->shost_classdev.kobj,
+				&arcmsr_sysfs_message_write_attr);
+error_bin_file_message_write:
+	sysfs_remove_bin_file(&host->shost_classdev.kobj,
+				&arcmsr_sysfs_message_read_attr);
+error_bin_file_message_read:
+	return error;
+}
+
+void
+arcmsr_free_sysfs_attr(struct AdapterControlBlock *acb) {
+	struct Scsi_Host *host = acb->host;
+
+	sysfs_remove_bin_file(&host->shost_classdev.kobj,
+				&arcmsr_sysfs_message_clear_attr);
+	sysfs_remove_bin_file(&host->shost_classdev.kobj,
+				&arcmsr_sysfs_message_write_attr);
+	sysfs_remove_bin_file(&host->shost_classdev.kobj,
+				&arcmsr_sysfs_message_read_attr);
+}
+
+
+static ssize_t
+arcmsr_attr_host_driver_version(struct class_device *cdev, char *buf) {
+	return snprintf(buf, PAGE_SIZE,
+			"%s\n",
+			ARCMSR_DRIVER_VERSION);
+}
+
+static ssize_t
+arcmsr_attr_host_driver_posted_cmd(struct class_device *cdev, char *buf) {
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+	return snprintf(buf, PAGE_SIZE,
+			"%4d\n",
+			atomic_read(&acb->ccboutstandingcount));
+}
+
+static ssize_t
+arcmsr_attr_host_driver_reset(struct class_device *cdev, char *buf) {
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+	return snprintf(buf, PAGE_SIZE,
+			"%4d\n",
+			acb->num_resets);
+}
+
+static ssize_t
+arcmsr_attr_host_driver_abort(struct class_device *cdev, char *buf) {
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+	return snprintf(buf, PAGE_SIZE,
+			"%4d\n",
+			acb->num_aborts);
+}
+
+static ssize_t
+arcmsr_attr_host_fw_model(struct class_device *cdev, char *buf) {
+    struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+	return snprintf(buf, PAGE_SIZE,
+			"%s\n",
+			acb->firm_model);
+}
+
+static ssize_t
+arcmsr_attr_host_fw_version(struct class_device *cdev, char *buf) {
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+
+	return snprintf(buf, PAGE_SIZE,
+			"%s\n",
+			acb->firm_version);
+}
+
+static ssize_t
+arcmsr_attr_host_fw_request_len(struct class_device *cdev, char *buf) {
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+
+	return snprintf(buf, PAGE_SIZE,
+			"%4d\n",
+			acb->firm_request_len);
+}
+
+static ssize_t
+arcmsr_attr_host_fw_numbers_queue(struct class_device *cdev, char *buf) {
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+
+	return snprintf(buf, PAGE_SIZE,
+			"%4d\n",
+			acb->firm_numbers_queue);
+}
+
+static ssize_t
+arcmsr_attr_host_fw_sdram_size(struct class_device *cdev, char *buf) {
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+
+	return snprintf(buf, PAGE_SIZE,
+			"%4d\n",
+			acb->firm_sdram_size);
+}
+
+static ssize_t
+arcmsr_attr_host_fw_hd_channels(struct class_device *cdev, char *buf) {
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct AdapterControlBlock *acb = (struct AdapterControlBlock *) host->hostdata;
+
+	return snprintf(buf, PAGE_SIZE,
+			"%4d\n",
+			acb->firm_hd_channels);
+}
+
+static CLASS_DEVICE_ATTR(host_driver_version, S_IRUGO, arcmsr_attr_host_driver_version, NULL);
+static CLASS_DEVICE_ATTR(host_driver_posted_cmd, S_IRUGO, arcmsr_attr_host_driver_posted_cmd, NULL);
+static CLASS_DEVICE_ATTR(host_driver_reset, S_IRUGO, arcmsr_attr_host_driver_reset, NULL);
+static CLASS_DEVICE_ATTR(host_driver_abort, S_IRUGO, arcmsr_attr_host_driver_abort, NULL);
+static CLASS_DEVICE_ATTR(host_fw_model, S_IRUGO, arcmsr_attr_host_fw_model, NULL);
+static CLASS_DEVICE_ATTR(host_fw_version, S_IRUGO, arcmsr_attr_host_fw_version, NULL);
+static CLASS_DEVICE_ATTR(host_fw_request_len, S_IRUGO, arcmsr_attr_host_fw_request_len, NULL);
+static CLASS_DEVICE_ATTR(host_fw_numbers_queue, S_IRUGO, arcmsr_attr_host_fw_numbers_queue, NULL);
+static CLASS_DEVICE_ATTR(host_fw_sdram_size, S_IRUGO, arcmsr_attr_host_fw_sdram_size, NULL);
+static CLASS_DEVICE_ATTR(host_fw_hd_channels, S_IRUGO, arcmsr_attr_host_fw_hd_channels, NULL);
+
+struct class_device_attribute *arcmsr_host_attrs[] = {
+	&class_device_attr_host_driver_version,
+	&class_device_attr_host_driver_posted_cmd,
+	&class_device_attr_host_driver_reset,
+	&class_device_attr_host_driver_abort,
+	&class_device_attr_host_fw_model,
+	&class_device_attr_host_fw_version,
+	&class_device_attr_host_fw_request_len,
+	&class_device_attr_host_fw_numbers_queue,
+	&class_device_attr_host_fw_sdram_size,
+	&class_device_attr_host_fw_hd_channels,
+	NULL,
+};
diff --git a/drivers/scsi/arcmsr/arcmsr_hba.c b/drivers/scsi/arcmsr/arcmsr_hba.c
new file mode 100644
index 0000000..475f978
--- /dev/null
+++ b/drivers/scsi/arcmsr/arcmsr_hba.c
@@ -0,0 +1,1496 @@
+/*
+*******************************************************************************
+**        O.S   : Linux
+**   FILE NAME  : arcmsr_hba.c
+**        BY    : Erich Chen
+**   Description: SCSI RAID Device Driver for
+**                ARECA RAID Host adapter
+*******************************************************************************
+** Copyright (C) 2002 - 2005, Areca Technology Corporation All rights reserved
+**
+**     Web site: www.areca.com.tw
+**       E-mail: erich@areca.com.tw
+**
+** This program is free software; you can redistribute it and/or modify
+** it under the terms of the GNU General Public License version 2 as
+** published by the Free Software Foundation.
+** This program is distributed in the hope that it will be useful,
+** but WITHOUT ANY WARRANTY; without even the implied warranty of
+** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+** GNU General Public License for more details.
+*******************************************************************************
+** Redistribution and use in source and binary forms, with or without
+** modification, are permitted provided that the following conditions
+** are met:
+** 1. Redistributions of source code must retain the above copyright
+**    notice, this list of conditions and the following disclaimer.
+** 2. Redistributions in binary form must reproduce the above copyright
+**    notice, this list of conditions and the following disclaimer in the
+**    documentation and/or other materials provided with the distribution.
+** 3. The name of the author may not be used to endorse or promote products
+**    derived from this software without specific prior written permission.
+**
+** THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+** IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+** OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+** IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+** INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES(INCLUDING,BUT
+** NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+** DATA, OR PROFITS; OR BUSINESS INTERRUPTION)HOWEVER CAUSED AND ON ANY
+** THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+** (INCLUDING NEGLIGENCE OR OTHERWISE)ARISING IN ANY WAY OUT OF THE USE OF
+** THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*******************************************************************************
+** For history of changes, see Documentation/scsi/ChangeLog.arcmsr
+**     Firmware Specification, see Documentation/scsi/arcmsr_spec.txt
+*******************************************************************************
+*/
+#include <linux/module.h>
+#include <linux/reboot.h>
+#include <linux/spinlock.h>
+#include <linux/pci_ids.h>
+#include <linux/interrupt.h>
+#include <linux/moduleparam.h>
+#include <linux/errno.h>
+#include <linux/types.h>
+#include <linux/delay.h>
+#include <linux/dma-mapping.h>
+#include <linux/timer.h>
+#include <linux/pci.h>
+#include <asm/dma.h>
+#include <asm/io.h>
+#include <asm/system.h>
+#include <asm/uaccess.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_tcq.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_transport.h>
+#include <scsi/scsicam.h>
+#include "arcmsr.h"
+
+MODULE_AUTHOR("Erich Chen <erich@areca.com.tw>");
+MODULE_DESCRIPTION("ARECA (ARC11xx/12xx) SATA RAID HOST Adapter");
+MODULE_LICENSE("Dual BSD/GPL");
+MODULE_VERSION(ARCMSR_DRIVER_VERSION);
+
+static int arcmsr_iop_message_xfer(struct AdapterControlBlock *acb, struct scsi_cmnd *cmd);
+static int arcmsr_abort(struct scsi_cmnd *);
+static int arcmsr_bus_reset(struct scsi_cmnd *);
+static int arcmsr_bios_param(struct scsi_device *sdev,
+				struct block_device *bdev, sector_t capacity, int *info);
+static int arcmsr_queue_command(struct scsi_cmnd * cmd,
+				void (*done) (struct scsi_cmnd *));
+static int arcmsr_probe(struct pci_dev *pdev,
+				const struct pci_device_id *id);
+static void arcmsr_remove(struct pci_dev *pdev);
+static void arcmsr_shutdown(struct pci_dev *pdev);
+static void arcmsr_iop_init(struct AdapterControlBlock *acb);
+static void arcmsr_free_ccb_pool(struct AdapterControlBlock *acb);
+static void arcmsr_stop_adapter_bgrb(struct AdapterControlBlock *acb);
+static void arcmsr_flush_adapter_cache(struct AdapterControlBlock *acb);
+static uint8_t arcmsr_wait_msgint_ready(struct AdapterControlBlock *acb);
+static const char *arcmsr_info(struct Scsi_Host *);
+static irqreturn_t arcmsr_interrupt(struct AdapterControlBlock *acb);
+
+static int arcmsr_adjust_disk_queue_depth(struct scsi_device *sdev, int queue_depth)
+{
+	if (queue_depth > ARCMSR_MAX_CMD_PERLUN)
+		queue_depth = ARCMSR_MAX_CMD_PERLUN;
+	scsi_adjust_queue_depth(sdev, MSG_ORDERED_TAG, queue_depth);
+	return queue_depth;
+}
+
+static struct scsi_host_template arcmsr_scsi_host_template = {
+	.module			= THIS_MODULE,
+	.name			= "ARCMSR ARECA SATA RAID HOST Adapter" ARCMSR_DRIVER_VERSION,
+	.info			= arcmsr_info,
+	.queuecommand		= arcmsr_queue_command,
+	.eh_abort_handler	= arcmsr_abort,
+	.eh_bus_reset_handler	= arcmsr_bus_reset,
+	.bios_param		= arcmsr_bios_param,
+	.change_queue_depth	= arcmsr_adjust_disk_queue_depth,
+	.can_queue		= ARCMSR_MAX_OUTSTANDING_CMD,
+	.this_id		= ARCMSR_SCSI_INITIATOR_ID,
+	.sg_tablesize		= ARCMSR_MAX_SG_ENTRIES,
+	.max_sectors    	= ARCMSR_MAX_XFER_SECTORS,
+	.cmd_per_lun		= ARCMSR_MAX_CMD_PERLUN,
+	.use_clustering		= ENABLE_CLUSTERING,
+	.shost_attrs		= arcmsr_host_attrs,
+};
+
+static struct pci_device_id arcmsr_device_id_table[] = {
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1110)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1120)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1130)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1160)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1170)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1210)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1220)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1230)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1260)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1270)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1280)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1380)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1381)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1680)},
+	{PCI_DEVICE(PCI_VENDOR_ID_ARECA, PCI_DEVICE_ID_ARECA_1681)},
+	{0, 0}, /* Terminating entry */
+};
+MODULE_DEVICE_TABLE(pci, arcmsr_device_id_table);
+static struct pci_driver arcmsr_pci_driver = {
+	.name			= "arcmsr",
+	.id_table		= arcmsr_device_id_table,
+	.probe			= arcmsr_probe,
+	.remove			= arcmsr_remove,
+	.shutdown		= arcmsr_shutdown
+};
+
+static irqreturn_t arcmsr_do_interrupt(int irq, void *dev_id,
+	struct pt_regs *regs)
+{
+	irqreturn_t handle_state;
+	struct AdapterControlBlock *acb;
+	unsigned long flags;
+
+	acb = (struct AdapterControlBlock *)dev_id;
+
+	spin_lock_irqsave(acb->host->host_lock, flags);
+	handle_state = arcmsr_interrupt(acb);
+	spin_unlock_irqrestore(acb->host->host_lock, flags);
+	return handle_state;
+}
+
+static int arcmsr_bios_param(struct scsi_device *sdev,
+		struct block_device *bdev, sector_t capacity, int *geom)
+{
+	int ret, heads, sectors, cylinders, total_capacity;
+	unsigned char *buffer;/* return copy of block device's partition table */
+
+	buffer = scsi_bios_ptable(bdev);
+	if (buffer) {
+		ret = scsi_partsize(buffer, capacity, &geom[2], &geom[0], &geom[1]);
+		kfree(buffer);
+		if (ret != -1)
+			return ret;
+	}
+	total_capacity = capacity;
+	heads = 64;
+	sectors = 32;
+	cylinders = total_capacity / (heads * sectors);
+	if (cylinders > 1024) {
+		heads = 255;
+		sectors = 63;
+		cylinders = total_capacity / (heads * sectors);
+	}
+	geom[0] = heads;
+	geom[1] = sectors;
+	geom[2] = cylinders;
+	return 0;
+}
+
+static int arcmsr_alloc_ccb_pool(struct AdapterControlBlock *acb)
+{
+	struct pci_dev *pdev = acb->pdev;
+	struct MessageUnit __iomem *reg = acb->pmu;
+	u32 ccb_phyaddr_hi32;
+	void *dma_coherent;
+	dma_addr_t dma_coherent_handle, dma_addr;
+	struct CommandControlBlock *ccb_tmp;
+	int i, j;
+
+	dma_coherent = dma_alloc_coherent(&pdev->dev,
+			ARCMSR_MAX_FREECCB_NUM *
+			sizeof (struct CommandControlBlock) + 0x20,
+			&dma_coherent_handle, GFP_KERNEL);
+	if (!dma_coherent)
+		return -ENOMEM;
+
+	acb->dma_coherent = dma_coherent;
+	acb->dma_coherent_handle = dma_coherent_handle;
+
+	if (((unsigned long)dma_coherent & 0x1F)) {
+		dma_coherent = dma_coherent +
+			(0x20 - ((unsigned long)dma_coherent & 0x1F));
+		dma_coherent_handle = dma_coherent_handle +
+			(0x20 - ((unsigned long)dma_coherent_handle & 0x1F));
+	}
+
+	dma_addr = dma_coherent_handle;
+	ccb_tmp = (struct CommandControlBlock *)dma_coherent;
+	for (i = 0; i < ARCMSR_MAX_FREECCB_NUM; i++) {
+		ccb_tmp->cdb_shifted_phyaddr = dma_addr >> 5;
+		ccb_tmp->acb = acb;
+		acb->pccb_pool[i] = ccb_tmp;
+		list_add_tail(&ccb_tmp->list, &acb->ccb_free_list);
+		dma_addr = dma_addr + sizeof (struct CommandControlBlock);
+		ccb_tmp++;
+	}
+
+	acb->vir2phy_offset = (unsigned long)ccb_tmp -
+			      (unsigned long)dma_addr;
+	for (i = 0; i < ARCMSR_MAX_TARGETID; i++)
+		for (j = 0; j < ARCMSR_MAX_TARGETLUN; j++)
+			acb->devstate[i][j] = ARECA_RAID_GOOD;
+
+	/*
+	** here we need to tell iop 331 our ccb_tmp.HighPart
+	** if ccb_tmp.HighPart is not zero
+	*/
+	ccb_phyaddr_hi32 = (uint32_t) ((dma_coherent_handle >> 16) >> 16);
+	if (ccb_phyaddr_hi32 != 0) {
+		writel(ARCMSR_SIGNATURE_SET_CONFIG, &reg->message_rwbuffer[0]);
+		writel(ccb_phyaddr_hi32, &reg->message_rwbuffer[1]);
+		writel(ARCMSR_INBOUND_MESG0_SET_CONFIG, &reg->inbound_msgaddr0);
+		if (arcmsr_wait_msgint_ready(acb))
+			printk(KERN_NOTICE "arcmsr%d: "
+			       "'set ccb high part physical address' timeout\n",
+				acb->host->host_no);
+	}
+
+	writel(readl(&reg->outbound_intmask) |
+			ARCMSR_MU_OUTBOUND_ALL_INTMASKENABLE,
+	       &reg->outbound_intmask);
+	return 0;
+}
+
+static int arcmsr_probe(struct pci_dev *pdev,
+	const struct pci_device_id *id)
+{
+	struct Scsi_Host *host;
+	struct AdapterControlBlock *acb;
+	uint8_t bus, dev_fun;
+	int error;
+
+	error = pci_enable_device(pdev);
+	if (error)
+		goto out;
+	pci_set_master(pdev);
+
+	host = scsi_host_alloc(&arcmsr_scsi_host_template,
+			sizeof(struct AdapterControlBlock));
+	if (!host) {
+		error = -ENOMEM;
+		goto out_disable_device;
+	}
+	acb = (struct AdapterControlBlock *)host->hostdata;
+	memset(acb, 0, sizeof (struct AdapterControlBlock));
+
+	error = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
+	if (error) {
+		error = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
+		if (error) {
+			printk(KERN_WARNING
+			       "scsi%d: No suitable DMA mask available\n",
+			       host->host_no);
+			goto out_host_put;
+		}
+	}
+	bus = pdev->bus->number;
+	dev_fun = pdev->devfn;
+	acb->host = host;
+	acb->pdev = pdev;
+	host->max_sectors = ARCMSR_MAX_XFER_SECTORS;
+	host->max_lun = ARCMSR_MAX_TARGETLUN;
+	host->max_id = ARCMSR_MAX_TARGETID;/*16:8*/
+	host->max_cmd_len = 16;    /*this is issue of 64bit LBA, over 2T byte*/
+	host->sg_tablesize = ARCMSR_MAX_SG_ENTRIES;
+	host->can_queue = ARCMSR_MAX_FREECCB_NUM; /* max simultaneous cmds */
+	host->cmd_per_lun = ARCMSR_MAX_CMD_PERLUN;
+	host->this_id = ARCMSR_SCSI_INITIATOR_ID;
+	host->unique_id = (bus << 8) | dev_fun;
+	host->irq = pdev->irq;
+	error = pci_request_regions(pdev, "arcmsr");
+	if (error)
+		goto out_host_put;
+
+	acb->pmu = ioremap(pci_resource_start(pdev, 0),
+			   pci_resource_len(pdev, 0));
+	if (!acb->pmu) {
+		printk(KERN_NOTICE "arcmsr%d: memory"
+			" mapping region fail \n", acb->host->host_no);
+		goto out_release_regions;
+	}
+	acb->acb_flags |= (ACB_F_MESSAGE_WQBUFFER_CLEARED |
+			   ACB_F_MESSAGE_RQBUFFER_CLEARED |
+			   ACB_F_MESSAGE_WQBUFFER_READED);
+	acb->acb_flags &= ~ACB_F_SCSISTOPADAPTER;
+	INIT_LIST_HEAD(&acb->ccb_free_list);
+
+	error = arcmsr_alloc_ccb_pool(acb);
+	if (error)
+		goto out_iounmap;
+
+	error = request_irq(pdev->irq, arcmsr_do_interrupt,
+			SA_INTERRUPT | SA_SHIRQ, "arcmsr", acb);
+	if (error)
+		goto out_free_ccb_pool;
+
+	arcmsr_iop_init(acb);
+	pci_set_drvdata(pdev, host);
+
+	error = scsi_add_host(host, &pdev->dev);
+	if (error)
+		goto out_free_irq;
+
+	error = arcmsr_alloc_sysfs_attr(acb);
+	if (error)
+		goto out_free_sysfs;
+
+	scsi_scan_host(host);
+	return 0;
+ out_free_sysfs:
+ out_free_irq:
+	free_irq(pdev->irq, acb);
+ out_free_ccb_pool:
+	arcmsr_free_ccb_pool(acb);
+ out_iounmap:
+	iounmap(acb->pmu);
+ out_release_regions:
+	pci_release_regions(pdev);
+ out_host_put:
+	scsi_host_put(host);
+ out_disable_device:
+	pci_disable_device(pdev);
+ out:
+	return error;
+}
+
+static void arcmsr_abort_allcmd(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+
+	writel(ARCMSR_INBOUND_MESG0_ABORT_CMD, &reg->inbound_msgaddr0);
+	if (arcmsr_wait_msgint_ready(acb))
+		printk(KERN_NOTICE
+			"arcmsr%d: wait 'abort all outstanding command' timeout \n"
+			, acb->host->host_no);
+}
+
+static void arcmsr_pci_unmap_dma(struct CommandControlBlock *ccb)
+{
+	struct AdapterControlBlock *acb = ccb->acb;
+	struct scsi_cmnd *pcmd = ccb->pcmd;
+
+	if (pcmd->use_sg != 0) {
+		struct scatterlist *sl;
+
+		sl = (struct scatterlist *)pcmd->request_buffer;
+		pci_unmap_sg(acb->pdev, sl, pcmd->use_sg, pcmd->sc_data_direction);
+	}
+	else if (pcmd->request_bufflen != 0)
+		pci_unmap_single(acb->pdev,
+			pcmd->SCp.dma_handle,
+			pcmd->request_bufflen, pcmd->sc_data_direction);
+}
+
+static void arcmsr_ccb_complete(struct CommandControlBlock *ccb, int stand_flag)
+{
+	struct AdapterControlBlock *acb = ccb->acb;
+	struct scsi_cmnd *pcmd = ccb->pcmd;
+
+	arcmsr_pci_unmap_dma(ccb);
+	if (stand_flag == 1)
+		atomic_dec(&acb->ccboutstandingcount);
+	ccb->startdone = ARCMSR_CCB_DONE;
+	ccb->ccb_flags = 0;
+	list_add_tail(&ccb->list, &acb->ccb_free_list);
+	pcmd->scsi_done(pcmd);
+}
+
+static void arcmsr_remove(struct pci_dev *pdev)
+{
+	struct Scsi_Host *host = pci_get_drvdata(pdev);
+	struct AdapterControlBlock *acb =
+		(struct AdapterControlBlock *) host->hostdata;
+	struct MessageUnit __iomem *reg = acb->pmu;
+	int poll_count = 0;
+
+	arcmsr_free_sysfs_attr(acb);
+	scsi_remove_host(host);
+	arcmsr_stop_adapter_bgrb(acb);
+	arcmsr_flush_adapter_cache(acb);
+	writel(readl(&reg->outbound_intmask) |
+		ARCMSR_MU_OUTBOUND_ALL_INTMASKENABLE,
+		&reg->outbound_intmask);
+	acb->acb_flags |= ACB_F_SCSISTOPADAPTER;
+	acb->acb_flags &= ~ACB_F_IOP_INITED;
+
+	for (poll_count = 0; poll_count < 256; poll_count++) {
+		if (!atomic_read(&acb->ccboutstandingcount))
+			break;
+		arcmsr_interrupt(acb);
+		msleep(25);
+	}
+
+	if (atomic_read(&acb->ccboutstandingcount)) {
+		int i;
+
+		arcmsr_abort_allcmd(acb);
+		for (i = 0; i < ARCMSR_MAX_OUTSTANDING_CMD; i++)
+			readl(&reg->outbound_queueport);
+		for (i = 0; i < ARCMSR_MAX_FREECCB_NUM; i++) {
+			struct CommandControlBlock *ccb = acb->pccb_pool[i];
+			if (ccb->startdone == ARCMSR_CCB_START) {
+				ccb->startdone = ARCMSR_CCB_ABORTED;
+				ccb->pcmd->result = DID_ABORT << 16;
+				arcmsr_ccb_complete(ccb, 1);
+			}
+		}
+	}
+
+	free_irq(pdev->irq, acb);
+	iounmap(acb->pmu);
+	arcmsr_free_ccb_pool(acb);
+	pci_release_regions(pdev);
+
+	scsi_host_put(host);
+
+	pci_disable_device(pdev);
+	pci_set_drvdata(pdev, NULL);
+}
+
+static void arcmsr_shutdown(struct pci_dev *pdev)
+{
+	struct Scsi_Host *host = pci_get_drvdata(pdev);
+	struct AdapterControlBlock *acb =
+		(struct AdapterControlBlock *)host->hostdata;
+
+	arcmsr_stop_adapter_bgrb(acb);
+	arcmsr_flush_adapter_cache(acb);
+}
+
+static int arcmsr_module_init(void)
+{
+	int error = 0;
+
+	error = pci_register_driver(&arcmsr_pci_driver);
+	return error;
+}
+
+static void arcmsr_module_exit(void)
+{
+	pci_unregister_driver(&arcmsr_pci_driver);
+}
+module_init(arcmsr_module_init);
+module_exit(arcmsr_module_exit);
+
+static u32 arcmsr_disable_outbound_ints(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	u32 orig_mask = readl(&reg->outbound_intmask);
+
+	writel(orig_mask | ARCMSR_MU_OUTBOUND_ALL_INTMASKENABLE,
+			&reg->outbound_intmask);
+	return orig_mask;
+}
+
+static void arcmsr_enable_outbound_ints(struct AdapterControlBlock *acb,
+		u32 orig_mask)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	u32 mask;
+
+	mask = orig_mask & ~(ARCMSR_MU_OUTBOUND_POSTQUEUE_INTMASKENABLE |
+			     ARCMSR_MU_OUTBOUND_DOORBELL_INTMASKENABLE);
+	writel(mask, &reg->outbound_intmask);
+}
+
+static void arcmsr_flush_adapter_cache(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg=acb->pmu;
+
+	writel(ARCMSR_INBOUND_MESG0_FLUSH_CACHE, &reg->inbound_msgaddr0);
+	if (arcmsr_wait_msgint_ready(acb))
+		printk(KERN_NOTICE
+			"arcmsr%d: wait 'flush adapter cache' timeout \n"
+			, acb->host->host_no);
+}
+
+static void arcmsr_report_sense_info(struct CommandControlBlock *ccb)
+{
+	struct scsi_cmnd *pcmd = ccb->pcmd;
+	struct SENSE_DATA *sensebuffer = (struct SENSE_DATA *)pcmd->sense_buffer;
+
+	pcmd->result = DID_OK << 16;
+	if (sensebuffer) {
+		int sense_data_length =
+			sizeof (struct SENSE_DATA) < sizeof (pcmd->sense_buffer)
+			? sizeof (struct SENSE_DATA) : sizeof (pcmd->sense_buffer);
+		memset(sensebuffer, 0, sizeof (pcmd->sense_buffer));
+		memcpy(sensebuffer, ccb->arcmsr_cdb.SenseData, sense_data_length);
+		sensebuffer->ErrorCode = SCSI_SENSE_CURRENT_ERRORS;
+		sensebuffer->Valid = 1;
+	}
+}
+
+static uint8_t arcmsr_wait_msgint_ready(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	uint32_t Index;
+	uint8_t Retries = 0x00;
+
+	do {
+		for (Index = 0; Index < 100; Index++) {
+			if (readl(&reg->outbound_intstatus)
+				& ARCMSR_MU_OUTBOUND_MESSAGE0_INT) {
+				writel(ARCMSR_MU_OUTBOUND_MESSAGE0_INT
+					, &reg->outbound_intstatus);
+				return 0x00;
+			}
+			msleep_interruptible(10);
+		}/*max 1 seconds*/
+	} while (Retries++ < 20);/*max 20 sec*/
+	return 0xff;
+}
+
+static void arcmsr_build_ccb(struct AdapterControlBlock *acb,
+	struct CommandControlBlock *ccb, struct scsi_cmnd *pcmd)
+{
+	struct ARCMSR_CDB *arcmsr_cdb = (struct ARCMSR_CDB *)&ccb->arcmsr_cdb;
+	int8_t *psge = (int8_t *)&arcmsr_cdb->u;
+	uint32_t address_lo, address_hi;
+	int arccdbsize = 0x30;
+
+	ccb->pcmd = pcmd;
+	memset(arcmsr_cdb, 0, sizeof (struct ARCMSR_CDB));
+	arcmsr_cdb->Bus = 0;
+	arcmsr_cdb->TargetID = pcmd->device->id;
+	arcmsr_cdb->LUN = pcmd->device->lun;
+	arcmsr_cdb->Function = 1;
+	arcmsr_cdb->CdbLength = (uint8_t)pcmd->cmd_len;
+	arcmsr_cdb->Context = (unsigned long)arcmsr_cdb;
+	memcpy(arcmsr_cdb->Cdb, pcmd->cmnd, pcmd->cmd_len);
+	if (pcmd->use_sg) {
+		int length, sgcount, i, cdb_sgcount = 0;
+		struct scatterlist *sl;
+
+		/* Get Scatter Gather List from scsiport. */
+		sl = (struct scatterlist *) pcmd->request_buffer;
+		sgcount = pci_map_sg(acb->pdev, sl, pcmd->use_sg,
+				pcmd->sc_data_direction);
+		/* map stor port SG list to our iop SG List. */
+		for (i = 0; i < sgcount; i++) {
+			/* Get the physical address of the current data pointer */
+			length = cpu_to_le32(sg_dma_len(sl));
+			address_lo = cpu_to_le32(dma_addr_lo32(sg_dma_address(sl)));
+			address_hi = cpu_to_le32(dma_addr_hi32(sg_dma_address(sl)));
+			if (address_hi == 0) {
+				struct SG32ENTRY *pdma_sg = (struct SG32ENTRY *)psge;
+
+				pdma_sg->address = address_lo;
+				pdma_sg->length = length;
+				psge += sizeof (struct SG32ENTRY);
+				arccdbsize += sizeof (struct SG32ENTRY);
+			} else {
+				struct SG64ENTRY *pdma_sg = (struct SG64ENTRY *)psge;
+
+				pdma_sg->addresshigh = address_hi;
+				pdma_sg->address = address_lo;
+				pdma_sg->length = length|IS_SG64_ADDR;
+				psge += sizeof (struct SG64ENTRY);
+				arccdbsize += sizeof (struct SG64ENTRY);
+			}
+			sl++;
+			cdb_sgcount++;
+		}
+		arcmsr_cdb->sgcount = (uint8_t)cdb_sgcount;
+		arcmsr_cdb->DataLength = pcmd->request_bufflen;
+		if ( arccdbsize > 256)
+			arcmsr_cdb->Flags |= ARCMSR_CDB_FLAG_SGL_BSIZE;
+	} else if (pcmd->request_bufflen) {
+		dma_addr_t dma_addr;
+		dma_addr = pci_map_single(acb->pdev, pcmd->request_buffer,
+				pcmd->request_bufflen, pcmd->sc_data_direction);
+		pcmd->SCp.dma_handle = dma_addr;
+		address_lo = cpu_to_le32(dma_addr_lo32(dma_addr));
+		address_hi = cpu_to_le32(dma_addr_hi32(dma_addr));
+		if (address_hi == 0) {
+			struct  SG32ENTRY *pdma_sg = (struct SG32ENTRY *)psge;
+			pdma_sg->address = address_lo;
+			pdma_sg->length = pcmd->request_bufflen;
+		} else {
+			struct SG64ENTRY *pdma_sg = (struct SG64ENTRY *)psge;
+			pdma_sg->addresshigh = address_hi;
+			pdma_sg->address = address_lo;
+			pdma_sg->length = pcmd->request_bufflen|IS_SG64_ADDR;
+		}
+		arcmsr_cdb->sgcount = 1;
+		arcmsr_cdb->DataLength = pcmd->request_bufflen;
+	}
+	if (pcmd->sc_data_direction == DMA_TO_DEVICE ) {
+		arcmsr_cdb->Flags |= ARCMSR_CDB_FLAG_WRITE;
+		ccb->ccb_flags |= CCB_FLAG_WRITE;
+	}
+}
+
+static void arcmsr_post_ccb(struct AdapterControlBlock *acb, struct CommandControlBlock *ccb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	uint32_t cdb_shifted_phyaddr = ccb->cdb_shifted_phyaddr;
+	struct ARCMSR_CDB *arcmsr_cdb = (struct ARCMSR_CDB *)&ccb->arcmsr_cdb;
+
+	atomic_inc(&acb->ccboutstandingcount);
+	ccb->startdone = ARCMSR_CCB_START;
+	if (arcmsr_cdb->Flags & ARCMSR_CDB_FLAG_SGL_BSIZE)
+		writel(cdb_shifted_phyaddr | ARCMSR_CCBPOST_FLAG_SGL_BSIZE,
+			&reg->inbound_queueport);
+	else
+		writel(cdb_shifted_phyaddr, &reg->inbound_queueport);
+}
+
+void arcmsr_post_Qbuffer(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	struct QBUFFER __iomem *pwbuffer = (struct QBUFFER __iomem *) &reg->message_wbuffer;
+	uint8_t __iomem *iop_data = (uint8_t __iomem *) pwbuffer->data;
+	int32_t allxfer_len = 0;
+
+	if (acb->acb_flags & ACB_F_MESSAGE_WQBUFFER_READED) {
+		acb->acb_flags &= (~ACB_F_MESSAGE_WQBUFFER_READED);
+		while ((acb->wqbuf_firstindex != acb->wqbuf_lastindex)
+			&& (allxfer_len < 124)) {
+			writeb(acb->wqbuffer[acb->wqbuf_firstindex], iop_data);
+			acb->wqbuf_firstindex++;
+			acb->wqbuf_firstindex %= ARCMSR_MAX_QBUFFER;
+			iop_data++;
+			allxfer_len++;
+		}
+		writel(allxfer_len, &pwbuffer->data_len);
+		writel(ARCMSR_INBOUND_DRIVER_DATA_WRITE_OK
+			, &reg->inbound_doorbell);
+	}
+}
+
+static void arcmsr_stop_adapter_bgrb(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+
+	acb->acb_flags &= ~ACB_F_MSG_START_BGRB;
+	writel(ARCMSR_INBOUND_MESG0_STOP_BGRB, &reg->inbound_msgaddr0);
+	if (arcmsr_wait_msgint_ready(acb))
+		printk(KERN_NOTICE
+			"arcmsr%d: wait 'stop adapter background rebulid' timeout \n"
+			, acb->host->host_no);
+}
+
+static void arcmsr_free_ccb_pool(struct AdapterControlBlock *acb)
+{
+	dma_free_coherent(&acb->pdev->dev,
+		ARCMSR_MAX_FREECCB_NUM * sizeof (struct CommandControlBlock) + 0x20,
+		acb->dma_coherent,
+		acb->dma_coherent_handle);
+}
+
+static irqreturn_t arcmsr_interrupt(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	struct CommandControlBlock *ccb;
+	uint32_t flag_ccb, outbound_intstatus, outbound_doorbell;
+
+	outbound_intstatus = readl(&reg->outbound_intstatus)
+		& acb->outbound_int_enable;
+	writel(outbound_intstatus, &reg->outbound_intstatus);
+	if (outbound_intstatus & ARCMSR_MU_OUTBOUND_DOORBELL_INT) {
+		outbound_doorbell = readl(&reg->outbound_doorbell);
+		writel(outbound_doorbell, &reg->outbound_doorbell);
+		if (outbound_doorbell & ARCMSR_OUTBOUND_IOP331_DATA_WRITE_OK) {
+			struct QBUFFER __iomem * prbuffer =
+				(struct QBUFFER __iomem *) &reg->message_rbuffer;
+			uint8_t __iomem * iop_data = (uint8_t __iomem *)prbuffer->data;
+			int32_t my_empty_len, iop_len, rqbuf_firstindex, rqbuf_lastindex;
+
+			rqbuf_lastindex = acb->rqbuf_lastindex;
+			rqbuf_firstindex = acb->rqbuf_firstindex;
+			iop_len = readl(&prbuffer->data_len);
+			my_empty_len = (rqbuf_firstindex - rqbuf_lastindex - 1)
+					&(ARCMSR_MAX_QBUFFER - 1);
+			if (my_empty_len >= iop_len) {
+				while (iop_len > 0) {
+					acb->rqbuffer[acb->rqbuf_lastindex] = readb(iop_data);
+					acb->rqbuf_lastindex++;
+					acb->rqbuf_lastindex %= ARCMSR_MAX_QBUFFER;
+					iop_data++;
+					iop_len--;
+				}
+				writel(ARCMSR_INBOUND_DRIVER_DATA_READ_OK,
+					&reg->inbound_doorbell);
+			} else
+				acb->acb_flags |= ACB_F_IOPDATA_OVERFLOW;
+		}
+		if (outbound_doorbell & ARCMSR_OUTBOUND_IOP331_DATA_READ_OK) {
+			acb->acb_flags |= ACB_F_MESSAGE_WQBUFFER_READED;
+			if (acb->wqbuf_firstindex != acb->wqbuf_lastindex) {
+				struct QBUFFER __iomem * pwbuffer =
+						(struct QBUFFER __iomem *) &reg->message_wbuffer;
+				uint8_t __iomem * iop_data = (uint8_t __iomem *) pwbuffer->data;
+				int32_t allxfer_len = 0;
+
+				acb->acb_flags &= (~ACB_F_MESSAGE_WQBUFFER_READED);
+				while ((acb->wqbuf_firstindex != acb->wqbuf_lastindex)
+					&& (allxfer_len < 124)) {
+					writeb(acb->wqbuffer[acb->wqbuf_firstindex], iop_data);
+					acb->wqbuf_firstindex++;
+					acb->wqbuf_firstindex %= ARCMSR_MAX_QBUFFER;
+					iop_data++;
+					allxfer_len++;
+				}
+				writel(allxfer_len, &pwbuffer->data_len);
+				writel(ARCMSR_INBOUND_DRIVER_DATA_WRITE_OK,
+					&reg->inbound_doorbell);
+			}
+			if (acb->wqbuf_firstindex == acb->wqbuf_lastindex)
+				acb->acb_flags |= ACB_F_MESSAGE_WQBUFFER_CLEARED;
+		}
+	}
+	if (outbound_intstatus & ARCMSR_MU_OUTBOUND_POSTQUEUE_INT) {
+		int id, lun;
+		/*
+		****************************************************************
+		**               areca cdb command done
+		****************************************************************
+		*/
+		while (1) {
+			if ((flag_ccb = readl(&reg->outbound_queueport)) == 0xFFFFFFFF)
+				break;/*chip FIFO no ccb for completion already*/
+			/* check if command done with no error*/
+			ccb = (struct CommandControlBlock *)(acb->vir2phy_offset +
+				(flag_ccb << 5));
+			if ((ccb->acb != acb) || (ccb->startdone != ARCMSR_CCB_START)) {
+				if (ccb->startdone == ARCMSR_CCB_ABORTED) {
+					struct scsi_cmnd *abortcmd=ccb->pcmd;
+					if (abortcmd) {
+					abortcmd->result |= DID_ABORT >> 16;
+					arcmsr_ccb_complete(ccb, 1);
+					printk(KERN_NOTICE
+						"arcmsr%d: ccb='0x%p' isr got aborted command \n"
+						, acb->host->host_no, ccb);
+					}
+					continue;
+				}
+				printk(KERN_NOTICE
+					"arcmsr%d: isr get an illegal ccb command done acb='0x%p'"
+					"ccb='0x%p' ccbacb='0x%p' startdone = 0x%x"
+					" ccboutstandingcount=%d \n"
+					, acb->host->host_no
+					, acb
+					, ccb
+					, ccb->acb
+					, ccb->startdone
+					, atomic_read(&acb->ccboutstandingcount));
+				continue;
+			}
+			id = ccb->pcmd->device->id;
+			lun = ccb->pcmd->device->lun;
+			if (!(flag_ccb & ARCMSR_CCBREPLY_FLAG_ERROR)) {
+				if (acb->devstate[id][lun] == ARECA_RAID_GONE)
+					acb->devstate[id][lun] = ARECA_RAID_GOOD;
+				ccb->pcmd->result = DID_OK << 16;
+				arcmsr_ccb_complete(ccb, 1);
+			} else {
+				switch(ccb->arcmsr_cdb.DeviceStatus) {
+				case ARCMSR_DEV_SELECT_TIMEOUT: {
+						acb->devstate[id][lun] = ARECA_RAID_GONE;
+						ccb->pcmd->result = DID_TIME_OUT << 16;
+						arcmsr_ccb_complete(ccb, 1);
+					}
+					break;
+				case ARCMSR_DEV_ABORTED:
+				case ARCMSR_DEV_INIT_FAIL: {
+						acb->devstate[id][lun] = ARECA_RAID_GONE;
+						ccb->pcmd->result = DID_BAD_TARGET << 16;
+						arcmsr_ccb_complete(ccb, 1);
+					}
+					break;
+				case ARCMSR_DEV_CHECK_CONDITION: {
+						acb->devstate[id][lun] = ARECA_RAID_GOOD;
+						arcmsr_report_sense_info(ccb);
+						arcmsr_ccb_complete(ccb, 1);
+					}
+					break;
+				default:
+					printk(KERN_NOTICE
+						"arcmsr%d: scsi id=%d lun=%d"
+						" isr get command error done,"
+						"but got unknown DeviceStatus = 0x%x \n"
+						, acb->host->host_no
+						, id
+						, lun
+						, ccb->arcmsr_cdb.DeviceStatus);
+						acb->devstate[id][lun] = ARECA_RAID_GONE;
+						ccb->pcmd->result = DID_NO_CONNECT << 16;
+						arcmsr_ccb_complete(ccb, 1);
+					break;
+				}
+			}
+		}/*drain reply FIFO*/
+	}
+	if (!(outbound_intstatus & ARCMSR_MU_OUTBOUND_HANDLE_INT))
+		return IRQ_NONE;
+	return IRQ_HANDLED;
+}
+
+static void arcmsr_iop_parking(struct AdapterControlBlock *acb)
+{
+	if (acb) {
+		/* stop adapter background rebuild */
+		if (acb->acb_flags & ACB_F_MSG_START_BGRB) {
+			acb->acb_flags &= ~ACB_F_MSG_START_BGRB;
+			arcmsr_stop_adapter_bgrb(acb);
+			arcmsr_flush_adapter_cache(acb);
+		}
+	}
+}
+
+static int arcmsr_iop_message_xfer(struct AdapterControlBlock *acb, struct scsi_cmnd *cmd)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	struct CMD_MESSAGE_FIELD *pcmdmessagefld;
+	int retvalue = 0, transfer_len = 0;
+	char *buffer;
+	uint32_t controlcode = (uint32_t ) cmd->cmnd[5] << 24 |
+						(uint32_t ) cmd->cmnd[6] << 16 |
+						(uint32_t ) cmd->cmnd[7] << 8  |
+						(uint32_t ) cmd->cmnd[8];
+					/* 4 bytes: Areca io control code */
+	if (cmd->use_sg) {
+		struct scatterlist *sg = (struct scatterlist *)cmd->request_buffer;
+
+		buffer = kmap_atomic(sg->page, KM_IRQ0) + sg->offset;
+		if (cmd->use_sg > 1) {
+			retvalue = ARCMSR_MESSAGE_FAIL;
+			goto message_out;
+		}
+		transfer_len += sg->length;
+	} else {
+		buffer = cmd->request_buffer;
+		transfer_len = cmd->request_bufflen;
+	}
+	if (transfer_len > sizeof(struct CMD_MESSAGE_FIELD)) {
+		retvalue = ARCMSR_MESSAGE_FAIL;
+		goto message_out;
+	}
+	pcmdmessagefld = (struct CMD_MESSAGE_FIELD *) buffer;
+	switch(controlcode) {
+	case ARCMSR_MESSAGE_READ_RQBUFFER: {
+			unsigned long *ver_addr;
+			dma_addr_t buf_handle;
+			uint8_t *pQbuffer, *ptmpQbuffer;
+			int32_t allxfer_len = 0;
+
+			ver_addr = pci_alloc_consistent(acb->pdev, 1032, &buf_handle);
+			if (!ver_addr) {
+				retvalue = ARCMSR_MESSAGE_FAIL;
+				goto message_out;
+			}
+			ptmpQbuffer = (uint8_t *) ver_addr;
+			while ((acb->rqbuf_firstindex != acb->rqbuf_lastindex)
+				&& (allxfer_len < 1031)) {
+				pQbuffer = &acb->rqbuffer[acb->rqbuf_firstindex];
+				memcpy(ptmpQbuffer, pQbuffer, 1);
+				acb->rqbuf_firstindex++;
+				acb->rqbuf_firstindex %= ARCMSR_MAX_QBUFFER;
+				ptmpQbuffer++;
+				allxfer_len++;
+			}
+			if (acb->acb_flags & ACB_F_IOPDATA_OVERFLOW) {
+				struct QBUFFER __iomem * prbuffer = (struct QBUFFER __iomem *)
+							&reg->message_rbuffer;
+				uint8_t __iomem * iop_data = (uint8_t __iomem *)prbuffer->data;
+				int32_t iop_len;
+
+				acb->acb_flags &= ~ACB_F_IOPDATA_OVERFLOW;
+				iop_len = readl(&prbuffer->data_len);
+				while (iop_len > 0) {
+					acb->rqbuffer[acb->rqbuf_lastindex] = readb(iop_data);
+					acb->rqbuf_lastindex++;
+					acb->rqbuf_lastindex %= ARCMSR_MAX_QBUFFER;
+					iop_data++;
+					iop_len--;
+				}
+				writel(ARCMSR_INBOUND_DRIVER_DATA_READ_OK,
+						&reg->inbound_doorbell);
+			}
+			memcpy(pcmdmessagefld->messagedatabuffer,
+				(uint8_t *)ver_addr, allxfer_len);
+			pcmdmessagefld->cmdmessage.Length = allxfer_len;
+			pcmdmessagefld->cmdmessage.ReturnCode = ARCMSR_MESSAGE_RETURNCODE_OK;
+			pci_free_consistent(acb->pdev, 1032, ver_addr, buf_handle);
+		}
+		break;
+	case ARCMSR_MESSAGE_WRITE_WQBUFFER: {
+			unsigned long *ver_addr;
+			dma_addr_t buf_handle;
+			int32_t my_empty_len, user_len, wqbuf_firstindex, wqbuf_lastindex;
+			uint8_t *pQbuffer, *ptmpuserbuffer;
+
+			ver_addr = pci_alloc_consistent(acb->pdev, 1032, &buf_handle);
+			if (!ver_addr) {
+				retvalue = ARCMSR_MESSAGE_FAIL;
+				goto message_out;
+			}
+			ptmpuserbuffer = (uint8_t *)ver_addr;
+			user_len = pcmdmessagefld->cmdmessage.Length;
+			memcpy(ptmpuserbuffer, pcmdmessagefld->messagedatabuffer, user_len);
+			wqbuf_lastindex = acb->wqbuf_lastindex;
+			wqbuf_firstindex = acb->wqbuf_firstindex;
+			if (wqbuf_lastindex != wqbuf_firstindex) {
+				struct SENSE_DATA *sensebuffer =
+					(struct SENSE_DATA *)cmd->sense_buffer;
+				arcmsr_post_Qbuffer(acb);
+				/* has error report sensedata */
+				sensebuffer->ErrorCode = 0x70;
+				sensebuffer->SenseKey = ILLEGAL_REQUEST;
+				sensebuffer->AdditionalSenseLength = 0x0A;
+				sensebuffer->AdditionalSenseCode = 0x20;
+				sensebuffer->Valid = 1;
+				retvalue = ARCMSR_MESSAGE_FAIL;
+			} else {
+				my_empty_len = (wqbuf_firstindex-wqbuf_lastindex - 1)
+						&(ARCMSR_MAX_QBUFFER - 1);
+				if (my_empty_len >= user_len) {
+					while (user_len > 0) {
+						pQbuffer =
+						&acb->wqbuffer[acb->wqbuf_lastindex];
+						memcpy(pQbuffer, ptmpuserbuffer, 1);
+						acb->wqbuf_lastindex++;
+						acb->wqbuf_lastindex %= ARCMSR_MAX_QBUFFER;
+						ptmpuserbuffer++;
+						user_len--;
+					}
+					if (acb->acb_flags & ACB_F_MESSAGE_WQBUFFER_CLEARED) {
+						acb->acb_flags &=
+							~ACB_F_MESSAGE_WQBUFFER_CLEARED;
+						arcmsr_post_Qbuffer(acb);
+					}
+				} else {
+					/* has error report sensedata */
+					struct SENSE_DATA *sensebuffer =
+						(struct SENSE_DATA *)cmd->sense_buffer;
+					sensebuffer->ErrorCode = 0x70;
+					sensebuffer->SenseKey = ILLEGAL_REQUEST;
+					sensebuffer->AdditionalSenseLength = 0x0A;
+					sensebuffer->AdditionalSenseCode = 0x20;
+					sensebuffer->Valid = 1;
+					retvalue = ARCMSR_MESSAGE_FAIL;
+				}
+			}
+			pci_free_consistent(acb->pdev, 1032, ver_addr, buf_handle);
+		}
+		break;
+	case ARCMSR_MESSAGE_CLEAR_RQBUFFER: {
+			uint8_t *pQbuffer = acb->rqbuffer;
+
+			if (acb->acb_flags & ACB_F_IOPDATA_OVERFLOW) {
+				acb->acb_flags &= ~ACB_F_IOPDATA_OVERFLOW;
+				writel(ARCMSR_INBOUND_DRIVER_DATA_READ_OK,
+					&reg->inbound_doorbell);
+			}
+			acb->acb_flags |= ACB_F_MESSAGE_RQBUFFER_CLEARED;
+			acb->rqbuf_firstindex = 0;
+			acb->rqbuf_lastindex = 0;
+			memset(pQbuffer, 0, ARCMSR_MAX_QBUFFER);
+			pcmdmessagefld->cmdmessage.ReturnCode =
+				ARCMSR_MESSAGE_RETURNCODE_OK;
+		}
+		break;
+	case ARCMSR_MESSAGE_CLEAR_WQBUFFER: {
+			uint8_t *pQbuffer = acb->wqbuffer;
+
+			if (acb->acb_flags & ACB_F_IOPDATA_OVERFLOW) {
+				acb->acb_flags &= ~ACB_F_IOPDATA_OVERFLOW;
+				writel(ARCMSR_INBOUND_DRIVER_DATA_READ_OK
+						, &reg->inbound_doorbell);
+			}
+			acb->acb_flags |=
+				(ACB_F_MESSAGE_WQBUFFER_CLEARED |
+					ACB_F_MESSAGE_WQBUFFER_READED);
+			acb->wqbuf_firstindex = 0;
+			acb->wqbuf_lastindex = 0;
+			memset(pQbuffer, 0, ARCMSR_MAX_QBUFFER);
+			pcmdmessagefld->cmdmessage.ReturnCode =
+				ARCMSR_MESSAGE_RETURNCODE_OK;
+		}
+		break;
+	case ARCMSR_MESSAGE_CLEAR_ALLQBUFFER: {
+			uint8_t *pQbuffer;
+
+			if (acb->acb_flags & ACB_F_IOPDATA_OVERFLOW) {
+				acb->acb_flags &= ~ACB_F_IOPDATA_OVERFLOW;
+				writel(ARCMSR_INBOUND_DRIVER_DATA_READ_OK
+						, &reg->inbound_doorbell);
+			}
+			acb->acb_flags |=
+				(ACB_F_MESSAGE_WQBUFFER_CLEARED
+				| ACB_F_MESSAGE_RQBUFFER_CLEARED
+				| ACB_F_MESSAGE_WQBUFFER_READED);
+			acb->rqbuf_firstindex = 0;
+			acb->rqbuf_lastindex = 0;
+			acb->wqbuf_firstindex = 0;
+			acb->wqbuf_lastindex = 0;
+			pQbuffer = acb->rqbuffer;
+			memset(pQbuffer, 0, sizeof (struct QBUFFER));
+			pQbuffer = acb->wqbuffer;
+			memset(pQbuffer, 0, sizeof (struct QBUFFER));
+			pcmdmessagefld->cmdmessage.ReturnCode = ARCMSR_MESSAGE_RETURNCODE_OK;
+		}
+		break;
+	case ARCMSR_MESSAGE_RETURN_CODE_3F: {
+			pcmdmessagefld->cmdmessage.ReturnCode = ARCMSR_MESSAGE_RETURNCODE_3F;
+		}
+		break;
+	case ARCMSR_MESSAGE_SAY_HELLO: {
+			int8_t * hello_string = "Hello! I am ARCMSR";
+
+			memcpy(pcmdmessagefld->messagedatabuffer, hello_string
+				, (int16_t)strlen(hello_string));
+			pcmdmessagefld->cmdmessage.ReturnCode = ARCMSR_MESSAGE_RETURNCODE_OK;
+		}
+		break;
+	case ARCMSR_MESSAGE_SAY_GOODBYE:
+		arcmsr_iop_parking(acb);
+		break;
+	case ARCMSR_MESSAGE_FLUSH_ADAPTER_CACHE:
+		arcmsr_flush_adapter_cache(acb);
+		break;
+	default:
+		retvalue = ARCMSR_MESSAGE_FAIL;
+	}
+ message_out:
+	if (cmd->use_sg) {
+		struct scatterlist *sg;
+
+		sg = (struct scatterlist *) cmd->request_buffer;
+		kunmap_atomic(buffer - sg->offset, KM_IRQ0);
+	}
+	return retvalue;
+}
+
+static struct CommandControlBlock *arcmsr_get_freeccb(struct AdapterControlBlock *acb)
+{
+	struct list_head *head = &acb->ccb_free_list;
+	struct CommandControlBlock *ccb = NULL;
+
+	if (!list_empty(head)) {
+		ccb = list_entry(head->next, struct CommandControlBlock, list);
+		list_del(head->next);
+	}
+	return ccb;
+}
+
+static void arcmsr_handle_virtual_command(struct AdapterControlBlock *acb,
+		struct scsi_cmnd *cmd)
+{
+	switch (cmd->cmnd[0]) {
+	case INQUIRY: {
+		unsigned char inqdata[36];
+		char *buffer;
+
+		if (cmd->device->lun) {
+			cmd->result = (DID_TIME_OUT << 16);
+			cmd->scsi_done(cmd);
+			return;
+		}
+		inqdata[0] = TYPE_PROCESSOR;
+		/* Periph Qualifier & Periph Dev Type */
+		inqdata[1] = 0;
+		/* rem media bit & Dev Type Modifier */
+		inqdata[2] = 0;
+		/* ISO,ECMA,& ANSI versions */
+		inqdata[4] = 31;
+		/* length of additional data */
+		strncpy(&inqdata[8], "Areca   ", 8);
+		/* Vendor Identification */
+		strncpy(&inqdata[16], "RAID controller ", 16);
+		/* Product Identification */
+		strncpy(&inqdata[32], "R001", 4); /* Product Revision */
+		if (cmd->use_sg) {
+			struct scatterlist *sg;
+
+			sg = (struct scatterlist *) cmd->request_buffer;
+			buffer = kmap_atomic(sg->page, KM_IRQ0) + sg->offset;
+		} else {
+			buffer = cmd->request_buffer;
+		}
+		memcpy(buffer, inqdata, sizeof(inqdata));
+		if (cmd->use_sg) {
+			struct scatterlist *sg;
+
+			sg = (struct scatterlist *) cmd->request_buffer;
+			kunmap_atomic(buffer - sg->offset, KM_IRQ0);
+		}
+		cmd->scsi_done(cmd);
+	}
+	break;
+	case WRITE_BUFFER:
+	case READ_BUFFER: {
+		if (arcmsr_iop_message_xfer(acb, cmd))
+			cmd->result = (DID_ERROR << 16);
+		cmd->scsi_done(cmd);
+	}
+	break;
+	default:
+		cmd->scsi_done(cmd);
+	}
+}
+
+static int arcmsr_queue_command(struct scsi_cmnd *cmd,
+	void (* done)(struct scsi_cmnd *))
+{
+	struct Scsi_Host *host = cmd->device->host;
+	struct AdapterControlBlock *acb =
+		(struct AdapterControlBlock *) host->hostdata;
+	struct CommandControlBlock *ccb;
+	int target = cmd->device->id;
+	int lun = cmd->device->lun;
+
+	cmd->scsi_done = done;
+	cmd->host_scribble = NULL;
+	cmd->result = 0;
+	if (acb->acb_flags & ACB_F_BUS_RESET) {
+		printk(KERN_NOTICE "arcmsr%d: bus reset"
+			" and return busy \n"
+			, acb->host->host_no);
+		return SCSI_MLQUEUE_HOST_BUSY;
+	}
+	if(target == 16) {
+		/* virtual device for iop message transfer */
+		arcmsr_handle_virtual_command(acb, cmd);
+		return 0;
+	}
+	if (acb->devstate[target][lun] == ARECA_RAID_GONE) {
+		uint8_t block_cmd;
+
+		block_cmd = cmd->cmnd[0] & 0x0f;
+		if (block_cmd == 0x08 || block_cmd == 0x0a) {
+			printk(KERN_NOTICE
+				"arcmsr%d: block 'read/write'"
+				"command with gone raid volume"
+				" Cmd=%2x, TargetId=%d, Lun=%d \n"
+				, acb->host->host_no
+				, cmd->cmnd[0]
+				, target, lun);
+			cmd->result = (DID_NO_CONNECT << 16);
+			cmd->scsi_done(cmd);
+			return 0;
+		}
+	}
+	if (atomic_read(&acb->ccboutstandingcount) >=
+			ARCMSR_MAX_OUTSTANDING_CMD)
+		return SCSI_MLQUEUE_HOST_BUSY;
+
+	ccb = arcmsr_get_freeccb(acb);
+	if (!ccb)
+		return SCSI_MLQUEUE_HOST_BUSY;
+	arcmsr_build_ccb(acb, ccb, cmd);
+	arcmsr_post_ccb(acb, ccb);
+	return 0;
+}
+
+static void arcmsr_get_firmware_spec(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	char *acb_firm_model = acb->firm_model;
+	char *acb_firm_version = acb->firm_version;
+	char __iomem *iop_firm_model = (char __iomem *) &reg->message_rwbuffer[15];
+	char __iomem *iop_firm_version = (char __iomem *) &reg->message_rwbuffer[17];
+	int count;
+
+	writel(ARCMSR_INBOUND_MESG0_GET_CONFIG, &reg->inbound_msgaddr0);
+	if (arcmsr_wait_msgint_ready(acb))
+		printk(KERN_NOTICE
+			"arcmsr%d: wait "
+			"'get adapter firmware miscellaneous data' timeout \n"
+			, acb->host->host_no);
+	count = 8;
+	while (count) {
+		*acb_firm_model = readb(iop_firm_model);
+		acb_firm_model++;
+		iop_firm_model++;
+		count--;
+	}
+	count = 16;
+	while (count) {
+		*acb_firm_version = readb(iop_firm_version);
+		acb_firm_version++;
+		iop_firm_version++;
+		count--;
+	}
+	printk(KERN_INFO
+		"ARECA RAID ADAPTER%d: FIRMWARE VERSION %s \n"
+		, acb->host->host_no
+		, acb->firm_version);
+	acb->firm_request_len = readl(&reg->message_rwbuffer[1]);
+	acb->firm_numbers_queue = readl(&reg->message_rwbuffer[2]);
+	acb->firm_sdram_size = readl(&reg->message_rwbuffer[3]);
+	acb->firm_hd_channels = readl(&reg->message_rwbuffer[4]);
+}
+
+static void arcmsr_polling_ccbdone(struct AdapterControlBlock *acb,
+	struct CommandControlBlock *poll_ccb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	struct CommandControlBlock *ccb;
+	uint32_t flag_ccb, outbound_intstatus, poll_ccb_done = 0, poll_count = 0;
+	int id, lun;
+
+ polling_ccb_retry:
+	poll_count++;
+	outbound_intstatus = readl(&reg->outbound_intstatus)
+					& acb->outbound_int_enable;
+	writel(outbound_intstatus, &reg->outbound_intstatus);/*clear interrupt*/
+	while (1) {
+		if ((flag_ccb = readl(&reg->outbound_queueport)) == 0xFFFFFFFF) {
+			if (poll_ccb_done)
+				break;
+			else {
+				msleep(25);
+				if (poll_count > 100)
+					break;
+				goto polling_ccb_retry;
+			}
+		}
+		ccb = (struct CommandControlBlock *)
+			(acb->vir2phy_offset + (flag_ccb << 5));
+		if ((ccb->acb != acb) ||
+			(ccb->startdone != ARCMSR_CCB_START)) {
+			if ((ccb->startdone == ARCMSR_CCB_ABORTED) ||
+				(ccb == poll_ccb)) {
+				printk(KERN_NOTICE
+					"arcmsr%d: scsi id=%d lun=%d ccb='0x%p'"
+					" poll command abort successfully \n"
+					, acb->host->host_no
+					, ccb->pcmd->device->id
+					, ccb->pcmd->device->lun
+					, ccb);
+				ccb->pcmd->result = DID_ABORT << 16;
+				arcmsr_ccb_complete(ccb, 1);
+				poll_ccb_done = 1;
+				continue;
+			}
+			printk(KERN_NOTICE
+				"arcmsr%d: polling get an illegal ccb"
+				" command done ccb='0x%p'"
+				"ccboutstandingcount=%d \n"
+				, acb->host->host_no
+				, ccb
+				, atomic_read(&acb->ccboutstandingcount));
+			continue;
+		}
+		id = ccb->pcmd->device->id;
+		lun = ccb->pcmd->device->lun;
+		if (!(flag_ccb & ARCMSR_CCBREPLY_FLAG_ERROR)) {
+			if (acb->devstate[id][lun] == ARECA_RAID_GONE)
+				acb->devstate[id][lun] = ARECA_RAID_GOOD;
+			ccb->pcmd->result = DID_OK << 16;
+			arcmsr_ccb_complete(ccb, 1);
+		} else {
+			switch(ccb->arcmsr_cdb.DeviceStatus) {
+			case ARCMSR_DEV_SELECT_TIMEOUT: {
+					acb->devstate[id][lun] = ARECA_RAID_GONE;
+					ccb->pcmd->result = DID_TIME_OUT << 16;
+					arcmsr_ccb_complete(ccb, 1);
+				}
+				break;
+			case ARCMSR_DEV_ABORTED:
+			case ARCMSR_DEV_INIT_FAIL: {
+					acb->devstate[id][lun] = ARECA_RAID_GONE;
+					ccb->pcmd->result = DID_BAD_TARGET << 16;
+					arcmsr_ccb_complete(ccb, 1);
+				}
+				break;
+			case ARCMSR_DEV_CHECK_CONDITION: {
+					acb->devstate[id][lun] = ARECA_RAID_GOOD;
+					arcmsr_report_sense_info(ccb);
+					arcmsr_ccb_complete(ccb, 1);
+				}
+				break;
+			default:
+				printk(KERN_NOTICE
+					"arcmsr%d: scsi id=%d lun=%d"
+					" polling and getting command error done"
+					"but got unknown DeviceStatus = 0x%x \n"
+					, acb->host->host_no
+					, id
+					, lun
+					, ccb->arcmsr_cdb.DeviceStatus);
+				acb->devstate[id][lun] = ARECA_RAID_GONE;
+				ccb->pcmd->result = DID_BAD_TARGET << 16;
+				arcmsr_ccb_complete(ccb, 1);
+				break;
+			}
+		}
+	}
+}
+
+static void arcmsr_iop_init(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	uint32_t intmask_org, mask, outbound_doorbell, firmware_state = 0;
+
+	do {
+		firmware_state = readl(&reg->outbound_msgaddr1);
+	} while (!(firmware_state & ARCMSR_OUTBOUND_MESG1_FIRMWARE_OK));
+	intmask_org = readl(&reg->outbound_intmask)
+			| ARCMSR_MU_OUTBOUND_MESSAGE0_INTMASKENABLE;
+	arcmsr_get_firmware_spec(acb);
+
+	acb->acb_flags |= ACB_F_MSG_START_BGRB;
+	writel(ARCMSR_INBOUND_MESG0_START_BGRB, &reg->inbound_msgaddr0);
+	if (arcmsr_wait_msgint_ready(acb)) {
+		printk(KERN_NOTICE "arcmsr%d: "
+			"wait 'start adapter background rebulid' timeout\n",
+			acb->host->host_no);
+	}
+
+	outbound_doorbell = readl(&reg->outbound_doorbell);
+	writel(outbound_doorbell, &reg->outbound_doorbell);
+	writel(ARCMSR_INBOUND_DRIVER_DATA_READ_OK, &reg->inbound_doorbell);
+	mask = ~(ARCMSR_MU_OUTBOUND_POSTQUEUE_INTMASKENABLE
+			| ARCMSR_MU_OUTBOUND_DOORBELL_INTMASKENABLE);
+	writel(intmask_org & mask, &reg->outbound_intmask);
+	acb->outbound_int_enable = ~(intmask_org & mask) & 0x000000ff;
+	acb->acb_flags |= ACB_F_IOP_INITED;
+}
+
+static void arcmsr_iop_reset(struct AdapterControlBlock *acb)
+{
+	struct MessageUnit __iomem *reg = acb->pmu;
+	struct CommandControlBlock *ccb;
+	uint32_t intmask_org;
+	int i = 0;
+
+	if (atomic_read(&acb->ccboutstandingcount) != 0) {
+		/* talk to iop 331 outstanding command aborted */
+		arcmsr_abort_allcmd(acb);
+		/* wait for 3 sec for all command aborted*/
+		msleep_interruptible(3000);
+		/* disable all outbound interrupt */
+		intmask_org = arcmsr_disable_outbound_ints(acb);
+		/* clear all outbound posted Q */
+		for (i = 0; i < ARCMSR_MAX_OUTSTANDING_CMD; i++)
+			readl(&reg->outbound_queueport);
+		for (i = 0; i < ARCMSR_MAX_FREECCB_NUM; i++) {
+			ccb = acb->pccb_pool[i];
+			if ((ccb->startdone == ARCMSR_CCB_START) ||
+				(ccb->startdone == ARCMSR_CCB_ABORTED)) {
+				ccb->startdone = ARCMSR_CCB_ABORTED;
+				ccb->pcmd->result = DID_ABORT << 16;
+				arcmsr_ccb_complete(ccb, 1);
+			}
+		}
+		/* enable all outbound interrupt */
+		arcmsr_enable_outbound_ints(acb, intmask_org);
+	}
+	atomic_set(&acb->ccboutstandingcount, 0);
+}
+
+static int arcmsr_bus_reset(struct scsi_cmnd *cmd)
+{
+	struct AdapterControlBlock *acb =
+		(struct AdapterControlBlock *)cmd->device->host->hostdata;
+	int i;
+
+	acb->num_resets++;
+	acb->acb_flags |= ACB_F_BUS_RESET;
+	for (i = 0; i < 400; i++) {
+		if (!atomic_read(&acb->ccboutstandingcount))
+			break;
+		arcmsr_interrupt(acb);
+		msleep(25);
+	}
+	arcmsr_iop_reset(acb);
+	acb->acb_flags &= ~ACB_F_BUS_RESET;
+	return SUCCESS;
+}
+
+static void arcmsr_abort_one_cmd(struct AdapterControlBlock *acb,
+		struct CommandControlBlock *ccb)
+{
+	u32 intmask;
+
+	ccb->startdone = ARCMSR_CCB_ABORTED;
+
+	/*
+	** Wait for 3 sec for all command done.
+	*/
+	msleep_interruptible(3000);
+
+	intmask = arcmsr_disable_outbound_ints(acb);
+	arcmsr_polling_ccbdone(acb, ccb);
+	arcmsr_enable_outbound_ints(acb, intmask);
+}
+
+static int arcmsr_abort(struct scsi_cmnd *cmd)
+{
+	struct AdapterControlBlock *acb =
+		(struct AdapterControlBlock *)cmd->device->host->hostdata;
+	int i = 0;
+
+	printk(KERN_NOTICE
+		"arcmsr%d: abort device command of scsi id=%d lun=%d \n",
+		acb->host->host_no, cmd->device->id, cmd->device->lun);
+	acb->num_aborts++;
+
+	/*
+	************************************************
+	** the all interrupt service routine is locked
+	** we need to handle it as soon as possible and exit
+	************************************************
+	*/
+	if (!atomic_read(&acb->ccboutstandingcount))
+		return SUCCESS;
+
+	for (i = 0; i < ARCMSR_MAX_FREECCB_NUM; i++) {
+		struct CommandControlBlock *ccb = acb->pccb_pool[i];
+		if (ccb->startdone == ARCMSR_CCB_START && ccb->pcmd == cmd) {
+			arcmsr_abort_one_cmd(acb, ccb);
+			break;
+		}
+	}
+
+	return SUCCESS;
+}
+
+static const char *arcmsr_info(struct Scsi_Host *host)
+{
+	struct AdapterControlBlock *acb =
+		(struct AdapterControlBlock *) host->hostdata;
+	static char buf[256];
+	char *type;
+	int raid6 = 1;
+
+	switch (acb->pdev->device) {
+	case PCI_DEVICE_ID_ARECA_1110:
+	case PCI_DEVICE_ID_ARECA_1210:
+		raid6 = 0;
+		/*FALLTHRU*/
+	case PCI_DEVICE_ID_ARECA_1120:
+	case PCI_DEVICE_ID_ARECA_1130:
+	case PCI_DEVICE_ID_ARECA_1160:
+	case PCI_DEVICE_ID_ARECA_1170:
+	case PCI_DEVICE_ID_ARECA_1220:
+	case PCI_DEVICE_ID_ARECA_1230:
+	case PCI_DEVICE_ID_ARECA_1260:
+	case PCI_DEVICE_ID_ARECA_1270:
+	case PCI_DEVICE_ID_ARECA_1280:
+		type = "SATA";
+		break;
+	case PCI_DEVICE_ID_ARECA_1380:
+	case PCI_DEVICE_ID_ARECA_1381:
+	case PCI_DEVICE_ID_ARECA_1680:
+	case PCI_DEVICE_ID_ARECA_1681:
+		type = "SAS";
+		break;
+	default:
+		type = "X-TYPE";
+		break;
+	}
+	sprintf(buf, "Areca %s Host Adapter RAID Controller%s\n        %s",
+			type, raid6 ? "( RAID6 capable)" : "",
+			ARCMSR_DRIVER_VERSION);
+	return buf;
+}
+
+
diff --git a/drivers/scsi/dpt_i2o.c b/drivers/scsi/dpt_i2o.c
index e133733..7b3bd34 100644
--- a/drivers/scsi/dpt_i2o.c
+++ b/drivers/scsi/dpt_i2o.c
@@ -46,7 +46,6 @@ MODULE_DESCRIPTION("Adaptec I2O RAID Driver");
 
 #include <linux/stat.h>
 #include <linux/slab.h>		/* for kmalloc() */
-#include <linux/config.h>	/* for CONFIG_PCI */
 #include <linux/pci.h>		/* for PCI support */
 #include <linux/proc_fs.h>
 #include <linux/blkdev.h>
@@ -185,7 +184,7 @@ static int adpt_detect(struct scsi_host_template* sht)
 	PINFO("Detecting Adaptec I2O RAID controllers...\n");
 
         /* search for all Adatpec I2O RAID cards */
-	while ((pDev = pci_find_device( PCI_DPT_VENDOR_ID, PCI_ANY_ID, pDev))) {
+	while ((pDev = pci_get_device( PCI_DPT_VENDOR_ID, PCI_ANY_ID, pDev))) {
 		if(pDev->device == PCI_DPT_DEVICE_ID ||
 		   pDev->device == PCI_DPT_RAPTOR_DEVICE_ID){
 			if(adpt_install_hba(sht, pDev) ){
@@ -193,8 +192,11 @@ static int adpt_detect(struct scsi_host_template* sht)
 				PERROR("Will not try to detect others.\n");
 				return hba_count-1;
 			}
+			pci_dev_get(pDev);
 		}
 	}
+	if (pDev)
+		pci_dev_put(pDev);
 
 	/* In INIT state, Activate IOPs */
 	for (pHba = hba_chain; pHba; pHba = pHba->next) {
@@ -1076,6 +1078,7 @@ static void adpt_i2o_delete_hba(adpt_hba* pHba)
 			}
 		}
 	}
+	pci_dev_put(pHba->pDev);
 	kfree(pHba);
 
 	if(hba_count <= 0){
diff --git a/drivers/scsi/eata_generic.h b/drivers/scsi/eata_generic.h
index 34bce2c..635c148 100644
--- a/drivers/scsi/eata_generic.h
+++ b/drivers/scsi/eata_generic.h
@@ -364,6 +364,7 @@ typedef struct hstd {
     __u8   moresupport;		 /* HBA supports MORE flag     */
     struct Scsi_Host *next;	    
     struct Scsi_Host *prev;
+    struct pci_dev *pdev;	/* PCI device or NULL for non PCI */
     struct eata_sp sp;		 /* status packet	       */ 
     struct eata_ccb ccb[0];	 /* ccb array begins here      */
 }hostdata;
diff --git a/drivers/scsi/eata_pio.c b/drivers/scsi/eata_pio.c
index 771b019..d312633 100644
--- a/drivers/scsi/eata_pio.c
+++ b/drivers/scsi/eata_pio.c
@@ -71,11 +71,11 @@
 #include "eata_pio.h"
 
 
-static uint ISAbases[MAXISA] =	{
+static unsigned int ISAbases[MAXISA] =	{
 	 0x1F0, 0x170, 0x330, 0x230
 };
 
-static uint ISAirqs[MAXISA] = {
+static unsigned int ISAirqs[MAXISA] = {
 	14, 12, 15, 11
 };
 
@@ -84,7 +84,7 @@ static unsigned char EISAbases[] = {
 	1, 1, 1, 1, 1, 1, 1, 1 
 };
 
-static uint registered_HBAs;
+static unsigned int registered_HBAs;
 static struct Scsi_Host *last_HBA;
 static struct Scsi_Host *first_HBA;
 static unsigned char reg_IRQ[16];
@@ -165,6 +165,7 @@ static int eata_pio_proc_info(struct Scsi_Host *shost, char *buffer, char **star
 
 static int eata_pio_release(struct Scsi_Host *sh)
 {
+	hostdata *hd = SD(sh);
 	if (sh->irq && reg_IRQ[sh->irq] == 1)
 		free_irq(sh->irq, NULL);
 	else
@@ -173,10 +174,13 @@ static int eata_pio_release(struct Scsi_Host *sh)
 		if (sh->io_port && sh->n_io_port)
 			release_region(sh->io_port, sh->n_io_port);
 	}
+	/* At this point the PCI reference can go */
+	if (hd->pdev)
+		pci_dev_put(hd->pdev);
 	return 1;
 }
 
-static void IncStat(struct scsi_pointer *SCp, uint Increment)
+static void IncStat(struct scsi_pointer *SCp, unsigned int Increment)
 {
 	SCp->ptr += Increment;
 	if ((SCp->this_residual -= Increment) == 0) {
@@ -190,46 +194,49 @@ static void IncStat(struct scsi_pointer *SCp, uint Increment)
 	}
 }
 
-static void eata_pio_int_handler(int irq, void *dev_id, struct pt_regs *regs);
+static irqreturn_t eata_pio_int_handler(int irq, void *dev_id, struct pt_regs *regs);
 
 static irqreturn_t do_eata_pio_int_handler(int irq, void *dev_id,
 						struct pt_regs *regs)
 {
 	unsigned long flags;
 	struct Scsi_Host *dev = dev_id;
+	irqreturn_t ret;
 
 	spin_lock_irqsave(dev->host_lock, flags);
-	eata_pio_int_handler(irq, dev_id, regs);
+	ret = eata_pio_int_handler(irq, dev_id, regs);
 	spin_unlock_irqrestore(dev->host_lock, flags);
-	return IRQ_HANDLED;
+	return ret;
 }
 
-static void eata_pio_int_handler(int irq, void *dev_id, struct pt_regs *regs)
+static irqreturn_t eata_pio_int_handler(int irq, void *dev_id, struct pt_regs *regs)
 {
-	uint eata_stat = 0xfffff;
+	unsigned int eata_stat = 0xfffff;
 	struct scsi_cmnd *cmd;
 	hostdata *hd;
 	struct eata_ccb *cp;
-	uint base;
-	uint x, z;
+	unsigned long base;
+	unsigned int x, z;
 	struct Scsi_Host *sh;
 	unsigned short zwickel = 0;
 	unsigned char stat, odd;
+	irqreturn_t ret = IRQ_NONE;
 
 	for (x = 1, sh = first_HBA; x <= registered_HBAs; x++, sh = SD(sh)->prev) 
 	{
 		if (sh->irq != irq)
 			continue;
-		if (inb((uint) sh->base + HA_RSTATUS) & HA_SBUSY)
+		if (inb(sh->base + HA_RSTATUS) & HA_SBUSY)
 			continue;
 
 		int_counter++;
+		ret = IRQ_HANDLED;
 
 		hd = SD(sh);
 
 		cp = &hd->ccb[0];
 		cmd = cp->cmd;
-		base = (uint) cmd->device->host->base;
+		base = cmd->device->host->base;
 
 		do {
 			stat = inb(base + HA_RSTATUS);
@@ -304,7 +311,7 @@ static void eata_pio_int_handler(int irq, void *dev_id, struct pt_regs *regs)
 		if (!(inb(base + HA_RSTATUS) & HA_SERROR)) {
 			cmd->result = (DID_OK << 16);
 			hd->devflags |= (1 << cp->cp_id);
-		} else if (hd->devflags & 1 << cp->cp_id)
+		} else if (hd->devflags & (1 << cp->cp_id))
 			cmd->result = (DID_OK << 16) + 0x02;
 		else
 			cmd->result = (DID_NO_CONNECT << 16);
@@ -313,7 +320,7 @@ static void eata_pio_int_handler(int irq, void *dev_id, struct pt_regs *regs)
 			cp->status = FREE;
 			eata_stat = inb(base + HA_RSTATUS);
 			printk(KERN_CRIT "eata_pio: int_handler, freeing locked " "queueslot\n");
-			return;
+			return ret;
 		}
 #if DBG_INTR2
 		if (stat != 0x50)
@@ -325,12 +332,12 @@ static void eata_pio_int_handler(int irq, void *dev_id, struct pt_regs *regs)
 		cmd->scsi_done(cmd);
 	}
 
-	return;
+	return ret;
 }
 
-static inline uint eata_pio_send_command(uint base, unsigned char command)
+static inline unsigned int eata_pio_send_command(unsigned long base, unsigned char command)
 {
-	uint loop = HZ / 2;
+	unsigned int loop = 50;
 
 	while (inb(base + HA_RSTATUS) & HA_SBUSY)
 		if (--loop == 0)
@@ -349,8 +356,8 @@ static inline uint eata_pio_send_command(uint base, unsigned char command)
 static int eata_pio_queue(struct scsi_cmnd *cmd,
 		void (*done)(struct scsi_cmnd *))
 {
-	uint x, y;
-	uint base;
+	unsigned int x, y;
+	unsigned long base;
 
 	hostdata *hd;
 	struct Scsi_Host *sh;
@@ -360,7 +367,7 @@ static int eata_pio_queue(struct scsi_cmnd *cmd,
 
 	hd = HD(cmd);
 	sh = cmd->device->host;
-	base = (uint) sh->base;
+	base = sh->base;
 
 	/* use only slot 0, as 2001 can handle only one cmd at a time */
 
@@ -395,9 +402,9 @@ static int eata_pio_queue(struct scsi_cmnd *cmd,
 		cp->DataIn = 0;	/* Input mode  */
 
 	cp->Interpret = (cmd->device->id == hd->hostid);
-	cp->cp_datalen = htonl((unsigned long) cmd->request_bufflen);
+	cp->cp_datalen = cpu_to_be32(cmd->request_bufflen);
 	cp->Auto_Req_Sen = 0;
-	cp->cp_reqDMA = htonl(0);
+	cp->cp_reqDMA = 0;
 	cp->reqlen = 0;
 
 	cp->cp_id = cmd->device->id;
@@ -406,7 +413,7 @@ static int eata_pio_queue(struct scsi_cmnd *cmd,
 	cp->cp_identify = 1;
 	memcpy(cp->cp_cdb, cmd->cmnd, COMMAND_SIZE(*cmd->cmnd));
 
-	cp->cp_statDMA = htonl(0);
+	cp->cp_statDMA = 0;
 
 	cp->cp_viraddr = cp;
 	cp->cmd = cmd;
@@ -445,14 +452,14 @@ static int eata_pio_queue(struct scsi_cmnd *cmd,
 
 	DBG(DBG_QUEUE, scmd_printk(KERN_DEBUG, cmd,
 		"Queued base %#.4lx pid: %ld "
-		"slot %d irq %d\n", (long) sh->base, cmd->pid, y, sh->irq));
+		"slot %d irq %d\n", sh->base, cmd->pid, y, sh->irq));
 
 	return (0);
 }
 
 static int eata_pio_abort(struct scsi_cmnd *cmd)
 {
-	uint loop = HZ;
+	unsigned int loop = 100;
 
 	DBG(DBG_ABNORM, scmd_printk(KERN_WARNING, cmd,
 		"eata_pio_abort called pid: %ld\n",
@@ -485,7 +492,7 @@ static int eata_pio_abort(struct scsi_cmnd *cmd)
 
 static int eata_pio_host_reset(struct scsi_cmnd *cmd)
 {
-	uint x, limit = 0;
+	unsigned int x, limit = 0;
 	unsigned char success = 0;
 	struct scsi_cmnd *sp;
 	struct Scsi_Host *host = cmd->device->host;
@@ -518,7 +525,7 @@ static int eata_pio_host_reset(struct scsi_cmnd *cmd)
 	}
 
 	/* hard reset the HBA  */
-	outb(EATA_CMD_RESET, (uint) cmd->device->host->base + HA_WCOMMAND);
+	outb(EATA_CMD_RESET, cmd->device->host->base + HA_WCOMMAND);
 
 	DBG(DBG_ABNORM, printk(KERN_WARNING "eata_pio_reset: board reset done.\n"));
 	HD(cmd)->state = RESET;
@@ -558,7 +565,7 @@ static int eata_pio_host_reset(struct scsi_cmnd *cmd)
 	}
 }
 
-static char *get_pio_board_data(unsigned long base, uint irq, uint id, unsigned long cplen, unsigned short cppadlen)
+static char *get_pio_board_data(unsigned long base, unsigned int irq, unsigned int id, unsigned long cplen, unsigned short cppadlen)
 {
 	struct eata_ccb cp;
 	static char buff[256];
@@ -570,8 +577,8 @@ static char *get_pio_board_data(unsigned long base, uint irq, uint id, unsigned
 	cp.DataIn = 1;
 	cp.Interpret = 1;	/* Interpret command */
 
-	cp.cp_datalen = htonl(254);
-	cp.cp_dataDMA = htonl(0);
+	cp.cp_datalen = cpu_to_be32(254);
+	cp.cp_dataDMA = cpu_to_be32(0);
 
 	cp.cp_id = id;
 	cp.cp_lun = 0;
@@ -583,7 +590,7 @@ static char *get_pio_board_data(unsigned long base, uint irq, uint id, unsigned
 	cp.cp_cdb[4] = 254;
 	cp.cp_cdb[5] = 0;
 
-	if (eata_pio_send_command((uint) base, EATA_CMD_PIO_SEND_CP))
+	if (eata_pio_send_command(base, EATA_CMD_PIO_SEND_CP))
 		return (NULL);
 	while (!(inb(base + HA_RSTATUS) & HA_SDRQ));
 	outsw(base + HA_RDATA, &cp, cplen);
@@ -604,7 +611,7 @@ static char *get_pio_board_data(unsigned long base, uint irq, uint id, unsigned
 	}
 }
 
-static int get_pio_conf_PIO(u32 base, struct get_conf *buf)
+static int get_pio_conf_PIO(unsigned long base, struct get_conf *buf)
 {
 	unsigned long loop = HZ / 2;
 	int z;
@@ -619,30 +626,30 @@ static int get_pio_conf_PIO(u32 base, struct get_conf *buf)
 		if (--loop == 0)
 			goto fail;
 
-	DBG(DBG_PIO && DBG_PROBE, printk(KERN_DEBUG "Issuing PIO READ CONFIG to HBA at %#x\n", base));
+	DBG(DBG_PIO && DBG_PROBE, printk(KERN_DEBUG "Issuing PIO READ CONFIG to HBA at %#lx\n", base));
 	eata_pio_send_command(base, EATA_CMD_PIO_READ_CONFIG);
 
-	loop = HZ / 2;
+	loop = 50;
 	for (p = (unsigned short *) buf; (long) p <= ((long) buf + (sizeof(struct get_conf) / 2)); p++) {
 		while (!(inb(base + HA_RSTATUS) & HA_SDRQ))
 			if (--loop == 0)
 				goto fail;
 
-		loop = HZ / 2;
+		loop = 50;
 		*p = inw(base + HA_RDATA);
 	}
 	if (inb(base + HA_RSTATUS) & HA_SERROR) {
 		DBG(DBG_PROBE, printk("eata_dma: get_conf_PIO, error during "
-					"transfer for HBA at %x\n", base));
+					"transfer for HBA at %lx\n", base));
 		goto fail;
 	}
 
-	if (htonl(EATA_SIGNATURE) != buf->signature)
+	if (cpu_to_be32(EATA_SIGNATURE) != buf->signature)
 		goto fail;
 
 	DBG(DBG_PIO && DBG_PROBE, printk(KERN_NOTICE "EATA Controller found "
-				"at %#4x EATA Level: %x\n",
-				base, (uint) (buf->version)));
+				"at %#4lx EATA Level: %x\n",
+				base, (unsigned int) (buf->version)));
 
 	while (inb(base + HA_RSTATUS) & HA_SDRQ)
 		inw(base + HA_RDATA);
@@ -665,12 +672,12 @@ static int get_pio_conf_PIO(u32 base, struct get_conf *buf)
 static void print_pio_config(struct get_conf *gc)
 {
 	printk("Please check values: (read config data)\n");
-	printk("LEN: %d ver:%d OCS:%d TAR:%d TRNXFR:%d MORES:%d\n", (uint) ntohl(gc->len), gc->version, gc->OCS_enabled, gc->TAR_support, gc->TRNXFR, gc->MORE_support);
-	printk("HAAV:%d SCSIID0:%d ID1:%d ID2:%d QUEUE:%d SG:%d SEC:%d\n", gc->HAA_valid, gc->scsi_id[3], gc->scsi_id[2], gc->scsi_id[1], ntohs(gc->queuesiz), ntohs(gc->SGsiz), gc->SECOND);
+	printk("LEN: %d ver:%d OCS:%d TAR:%d TRNXFR:%d MORES:%d\n", be32_to_cpu(gc->len), gc->version, gc->OCS_enabled, gc->TAR_support, gc->TRNXFR, gc->MORE_support);
+	printk("HAAV:%d SCSIID0:%d ID1:%d ID2:%d QUEUE:%d SG:%d SEC:%d\n", gc->HAA_valid, gc->scsi_id[3], gc->scsi_id[2], gc->scsi_id[1], be16_to_cpu(gc->queuesiz), be16_to_cpu(gc->SGsiz), gc->SECOND);
 	printk("IRQ:%d IRQT:%d FORCADR:%d MCH:%d RIDQ:%d\n", gc->IRQ, gc->IRQ_TR, gc->FORCADR, gc->MAX_CHAN, gc->ID_qest);
 }
 
-static uint print_selftest(uint base)
+static unsigned int print_selftest(unsigned int base)
 {
 	unsigned char buffer[512];
 #ifdef VERBOSE_SETUP
@@ -697,7 +704,7 @@ static uint print_selftest(uint base)
 	return (!(inb(base + HA_RSTATUS) & HA_SERROR));
 }
 
-static int register_pio_HBA(long base, struct get_conf *gc)
+static int register_pio_HBA(long base, struct get_conf *gc, struct pci_dev *pdev)
 {
 	unsigned long size = 0;
 	char *buff;
@@ -714,17 +721,17 @@ static int register_pio_HBA(long base, struct get_conf *gc)
 			return 0;
 	}
 
-	if ((buff = get_pio_board_data((uint) base, gc->IRQ, gc->scsi_id[3], cplen = (htonl(gc->cplen) + 1) / 2, cppadlen = (htons(gc->cppadlen) + 1) / 2)) == NULL) {
-		printk("HBA at %#lx didn't react on INQUIRY. Sorry.\n", (unsigned long) base);
+	if ((buff = get_pio_board_data(base, gc->IRQ, gc->scsi_id[3], cplen = (cpu_to_be32(gc->cplen) + 1) / 2, cppadlen = (cpu_to_be16(gc->cppadlen) + 1) / 2)) == NULL) {
+		printk("HBA at %#lx didn't react on INQUIRY. Sorry.\n", base);
 		return 0;
 	}
 
 	if (!print_selftest(base) && !ALLOW_DMA_BOARDS) {
-		printk("HBA at %#lx failed while performing self test & setup.\n", (unsigned long) base);
+		printk("HBA at %#lx failed while performing self test & setup.\n", base);
 		return 0;
 	}
 
-	size = sizeof(hostdata) + (sizeof(struct eata_ccb) * ntohs(gc->queuesiz));
+	size = sizeof(hostdata) + (sizeof(struct eata_ccb) * be16_to_cpu(gc->queuesiz));
 
 	sh = scsi_register(&driver_template, size);
 	if (sh == NULL)
@@ -749,8 +756,8 @@ static int register_pio_HBA(long base, struct get_conf *gc)
 
 	hd = SD(sh);
 
-	memset(hd->ccb, 0, (sizeof(struct eata_ccb) * ntohs(gc->queuesiz)));
-	memset(hd->reads, 0, sizeof(unsigned long) * 26);
+	memset(hd->ccb, 0, (sizeof(struct eata_ccb) * be16_to_cpu(gc->queuesiz)));
+	memset(hd->reads, 0, sizeof(hd->reads));
 
 	strlcpy(SD(sh)->vendor, &buff[8], sizeof(SD(sh)->vendor));
 	strlcpy(SD(sh)->name, &buff[16], sizeof(SD(sh)->name));
@@ -761,7 +768,7 @@ static int register_pio_HBA(long base, struct get_conf *gc)
 	SD(sh)->revision[4] = buff[35];
 	SD(sh)->revision[5] = 0;
 
-	switch (ntohl(gc->len)) {
+	switch (be32_to_cpu(gc->len)) {
 	case 0x1c:
 		SD(sh)->EATA_revision = 'a';
 		break;
@@ -777,7 +784,7 @@ static int register_pio_HBA(long base, struct get_conf *gc)
 		SD(sh)->EATA_revision = '?';
 	}
 
-	if (ntohl(gc->len) >= 0x22) {
+	if (be32_to_cpu(gc->len) >= 0x22) {
 		if (gc->is_PCI)
 			hd->bustype = IS_PCI;
 		else if (gc->is_EISA)
@@ -811,6 +818,8 @@ static int register_pio_HBA(long base, struct get_conf *gc)
 
 	hd->channel = 0;
 
+	hd->pdev = pci_dev_get(pdev);	/* Keep a PCI reference */
+
 	sh->max_id = 8;
 	sh->max_lun = 8;
 
@@ -841,7 +850,7 @@ static void find_pio_ISA(struct get_conf *buf)
 			continue;
 		if (!get_pio_conf_PIO(ISAbases[i], buf))
 			continue;
-		if (!register_pio_HBA(ISAbases[i], buf))
+		if (!register_pio_HBA(ISAbases[i], buf, NULL))
 			release_region(ISAbases[i], 9);
 		else
 			ISAbases[i] = 0;
@@ -873,7 +882,7 @@ static void find_pio_EISA(struct get_conf *buf)
 				if (get_pio_conf_PIO(base, buf)) {
 					DBG(DBG_PROBE && DBG_EISA, print_pio_config(buf));
 					if (buf->IRQ) {
-						if (!register_pio_HBA(base, buf))
+						if (!register_pio_HBA(base, buf, NULL))
 							release_region(base, 9);
 					} else {
 						printk(KERN_NOTICE "eata_dma: No valid IRQ. HBA " "removed from list\n");
@@ -896,9 +905,9 @@ static void find_pio_PCI(struct get_conf *buf)
 	printk("eata_dma: kernel PCI support not enabled. Skipping scan for PCI HBAs.\n");
 #else
 	struct pci_dev *dev = NULL;
-	u32 base, x;
+	unsigned long base, x;
 
-	while ((dev = pci_find_device(PCI_VENDOR_ID_DPT, PCI_DEVICE_ID_DPT, dev)) != NULL) {
+	while ((dev = pci_get_device(PCI_VENDOR_ID_DPT, PCI_DEVICE_ID_DPT, dev)) != NULL) {
 		DBG(DBG_PROBE && DBG_PCI, printk("eata_pio: find_PCI, HBA at %s\n", pci_name(dev)));
 		if (pci_enable_device(dev))
 			continue;
@@ -926,7 +935,7 @@ static void find_pio_PCI(struct get_conf *buf)
 				 * eventually remove it from the EISA and ISA list 
 				 */
 
-				if (!register_pio_HBA(base, buf)) {
+				if (!register_pio_HBA(base, buf, dev)) {
 					release_region(base, 9);
 					continue;
 				}
@@ -976,12 +985,12 @@ static int eata_pio_detect(struct scsi_host_template *tpnt)
 		printk("Registered HBAs:\n");
 		printk("HBA no. Boardtype: Revis: EATA: Bus: BaseIO: IRQ: Ch: ID: Pr:" " QS: SG: CPL:\n");
 		for (i = 1; i <= registered_HBAs; i++) {
-			printk("scsi%-2d: %.10s v%s 2.0%c  %s %#.4x   %2d   %d   %d   %c"
+			printk("scsi%-2d: %.10s v%s 2.0%c  %s %#.4lx   %2d   %d   %d   %c"
 			       "  %2d  %2d  %2d\n",
 			       HBA_ptr->host_no, SD(HBA_ptr)->name, SD(HBA_ptr)->revision,
 			       SD(HBA_ptr)->EATA_revision, (SD(HBA_ptr)->bustype == 'P') ?
 			       "PCI " : (SD(HBA_ptr)->bustype == 'E') ? "EISA" : "ISA ",
-			       (uint) HBA_ptr->base, HBA_ptr->irq, SD(HBA_ptr)->channel, HBA_ptr->this_id,
+			       HBA_ptr->base, HBA_ptr->irq, SD(HBA_ptr)->channel, HBA_ptr->this_id,
 			       SD(HBA_ptr)->primary ? 'Y' : 'N', HBA_ptr->can_queue,
 			       HBA_ptr->sg_tablesize, HBA_ptr->cmd_per_lun);
 			HBA_ptr = SD(HBA_ptr)->next;
diff --git a/drivers/scsi/fcal.c b/drivers/scsi/fcal.c
index 7f89102..c4e16c0 100644
--- a/drivers/scsi/fcal.c
+++ b/drivers/scsi/fcal.c
@@ -248,8 +248,7 @@ int fcal_proc_info (struct Scsi_Host *host, char *buffer, char **start, off_t of
 				if (scd->id == target) {
 					SPRINTF ("  [AL-PA: %02x, Id: %02d, Port WWN: %08x%08x, Node WWN: %08x%08x]  ",
 						alpa, target, u1[0], u1[1], u2[0], u2[1]);
-					SPRINTF ("%s ", (scd->type < MAX_SCSI_DEVICE_CODE) ?
-						scsi_device_types[(short) scd->type] : "Unknown device");
+					SPRINTF ("%s ", scsi_device_type(scd->type));
 
 					for (j = 0; (j < 8) && (scd->vendor[j] >= 0x20); j++)
 						SPRINTF ("%c", scd->vendor[j]);
diff --git a/drivers/scsi/g_NCR5380.c b/drivers/scsi/g_NCR5380.c
index 67f1100f..cdd893b 100644
--- a/drivers/scsi/g_NCR5380.c
+++ b/drivers/scsi/g_NCR5380.c
@@ -811,7 +811,6 @@ static int generic_NCR5380_proc_info(struct Scsi_Host *scsi_ptr, char *buffer, c
 	struct NCR5380_hostdata *hostdata;
 #ifdef NCR5380_STATS
 	struct scsi_device *dev;
-	extern const char *const scsi_device_types[MAX_SCSI_DEVICE_CODE];
 #endif
 
 	NCR5380_setup(scsi_ptr);
@@ -851,7 +850,7 @@ static int generic_NCR5380_proc_info(struct Scsi_Host *scsi_ptr, char *buffer, c
 		long tr = hostdata->time_read[dev->id] / HZ;
 		long tw = hostdata->time_write[dev->id] / HZ;
 
-		PRINTP("  T:%d %s " ANDP dev->id ANDP(dev->type < MAX_SCSI_DEVICE_CODE) ? scsi_device_types[(int) dev->type] : "Unknown");
+		PRINTP("  T:%d %s " ANDP dev->id ANDP scsi_device_type(dev->type));
 		for (i = 0; i < 8; i++)
 			if (dev->vendor[i] >= 0x20)
 				*(buffer + (len++)) = dev->vendor[i];
diff --git a/drivers/scsi/gvp11.c b/drivers/scsi/gvp11.c
index a0d831b..18dbe5c 100644
--- a/drivers/scsi/gvp11.c
+++ b/drivers/scsi/gvp11.c
@@ -47,7 +47,7 @@ void gvp11_setup (char *str, int *ints)
     gvp11_xfer_mask = ints[1];
 }
 
-static int dma_setup (Scsi_Cmnd *cmd, int dir_in)
+static int dma_setup(struct scsi_cmnd *cmd, int dir_in)
 {
     unsigned short cntr = GVP11_DMAC_INT_ENABLE;
     unsigned long addr = virt_to_bus(cmd->SCp.ptr);
@@ -142,8 +142,8 @@ static int dma_setup (Scsi_Cmnd *cmd, int dir_in)
     return 0;
 }
 
-static void dma_stop (struct Scsi_Host *instance, Scsi_Cmnd *SCpnt,
-		      int status)
+static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
+		     int status)
 {
     /* stop DMA */
     DMA(instance)->SP_DMA = 1;
@@ -341,7 +341,7 @@ release:
     return num_gvp11;
 }
 
-static int gvp11_bus_reset(Scsi_Cmnd *cmd)
+static int gvp11_bus_reset(struct scsi_cmnd *cmd)
 {
 	/* FIXME perform bus-specific reset */
 
diff --git a/drivers/scsi/gvp11.h b/drivers/scsi/gvp11.h
index 575d219d..bf22859 100644
--- a/drivers/scsi/gvp11.h
+++ b/drivers/scsi/gvp11.h
@@ -13,10 +13,6 @@
 
 int gvp11_detect(struct scsi_host_template *);
 int gvp11_release(struct Scsi_Host *);
-const char *wd33c93_info(void);
-int wd33c93_queuecommand(Scsi_Cmnd *, void (*done)(Scsi_Cmnd *));
-int wd33c93_abort(Scsi_Cmnd *);
-int wd33c93_reset(Scsi_Cmnd *, unsigned int);
 
 #ifndef CMD_PER_LUN
 #define CMD_PER_LUN 2
diff --git a/drivers/scsi/hosts.c b/drivers/scsi/hosts.c
index dfcb96f..68ef163 100644
--- a/drivers/scsi/hosts.c
+++ b/drivers/scsi/hosts.c
@@ -265,6 +265,9 @@ static void scsi_host_dev_release(struct device *dev)
 		destroy_workqueue(shost->work_q);
 
 	scsi_destroy_command_freelist(shost);
+	if (shost->bqt)
+		blk_free_tags(shost->bqt);
+
 	kfree(shost->shost_data);
 
 	if (parent)
@@ -487,7 +490,9 @@ EXPORT_SYMBOL(scsi_is_host_device);
  * @work:	Work to queue for execution.
  *
  * Return value:
- * 	0 on success / != 0 for error
+ * 	1 - work queued for execution
+ *	0 - work is already queued
+ *	-EINVAL - work queue doesn't exist
  **/
 int scsi_queue_work(struct Scsi_Host *shost, struct work_struct *work)
 {
diff --git a/drivers/scsi/hptiop.c b/drivers/scsi/hptiop.c
index bcb3444..28bfb8f9 100644
--- a/drivers/scsi/hptiop.c
+++ b/drivers/scsi/hptiop.c
@@ -15,7 +15,6 @@
  *
  * For more information, visit http://www.highpoint-tech.com
  */
-#include <linux/config.h>
 #include <linux/module.h>
 #include <linux/types.h>
 #include <linux/string.h>
diff --git a/drivers/scsi/ibmvscsi/rpa_vscsi.c b/drivers/scsi/ibmvscsi/rpa_vscsi.c
index ed22b96..01b8ac6 100644
--- a/drivers/scsi/ibmvscsi/rpa_vscsi.c
+++ b/drivers/scsi/ibmvscsi/rpa_vscsi.c
@@ -156,8 +156,8 @@ static void gather_partition_info(void)
 {
 	struct device_node *rootdn;
 
-	char *ppartition_name;
-	unsigned int *p_number_ptr;
+	const char *ppartition_name;
+	const unsigned int *p_number_ptr;
 
 	/* Retrieve information about this partition */
 	rootdn = find_path_device("/");
@@ -165,14 +165,11 @@ static void gather_partition_info(void)
 		return;
 	}
 
-	ppartition_name =
-		get_property(rootdn, "ibm,partition-name", NULL);
+	ppartition_name = get_property(rootdn, "ibm,partition-name", NULL);
 	if (ppartition_name)
 		strncpy(partition_name, ppartition_name,
 				sizeof(partition_name));
-	p_number_ptr =
-		(unsigned int *)get_property(rootdn, "ibm,partition-no",
-					     NULL);
+	p_number_ptr = get_property(rootdn, "ibm,partition-no", NULL);
 	if (p_number_ptr)
 		partition_number = *p_number_ptr;
 }
diff --git a/drivers/scsi/ipr.c b/drivers/scsi/ipr.c
index 01080b3..7ed4eef 100644
--- a/drivers/scsi/ipr.c
+++ b/drivers/scsi/ipr.c
@@ -175,6 +175,8 @@ struct ipr_error_table_t ipr_error_table[] = {
 	"Qualified success"},
 	{0x01080000, 1, 1,
 	"FFFE: Soft device bus error recovered by the IOA"},
+	{0x01088100, 0, 1,
+	"4101: Soft device bus fabric error"},
 	{0x01170600, 0, 1,
 	"FFF9: Device sector reassign successful"},
 	{0x01170900, 0, 1,
@@ -225,6 +227,8 @@ struct ipr_error_table_t ipr_error_table[] = {
 	"3109: IOA timed out a device command"},
 	{0x04088000, 0, 0,
 	"3120: SCSI bus is not operational"},
+	{0x04088100, 0, 1,
+	"4100: Hard device bus fabric error"},
 	{0x04118000, 0, 1,
 	"9000: IOA reserved area data check"},
 	{0x04118100, 0, 1,
@@ -273,6 +277,14 @@ struct ipr_error_table_t ipr_error_table[] = {
 	"9091: Incorrect hardware configuration change has been detected"},
 	{0x04678000, 0, 1,
 	"9073: Invalid multi-adapter configuration"},
+	{0x04678100, 0, 1,
+	"4010: Incorrect connection between cascaded expanders"},
+	{0x04678200, 0, 1,
+	"4020: Connections exceed IOA design limits"},
+	{0x04678300, 0, 1,
+	"4030: Incorrect multipath connection"},
+	{0x04679000, 0, 1,
+	"4110: Unsupported enclosure function"},
 	{0x046E0000, 0, 1,
 	"FFF4: Command to logical unit failed"},
 	{0x05240000, 1, 0,
@@ -297,6 +309,8 @@ struct ipr_error_table_t ipr_error_table[] = {
 	"9031: Array protection temporarily suspended, protection resuming"},
 	{0x06040600, 0, 1,
 	"9040: Array protection temporarily suspended, protection resuming"},
+	{0x06288000, 0, 1,
+	"3140: Device bus not ready to ready transition"},
 	{0x06290000, 0, 1,
 	"FFFB: SCSI bus was reset"},
 	{0x06290500, 0, 0,
@@ -319,6 +333,16 @@ struct ipr_error_table_t ipr_error_table[] = {
 	"3150: SCSI bus configuration error"},
 	{0x06678100, 0, 1,
 	"9074: Asymmetric advanced function disk configuration"},
+	{0x06678300, 0, 1,
+	"4040: Incomplete multipath connection between IOA and enclosure"},
+	{0x06678400, 0, 1,
+	"4041: Incomplete multipath connection between enclosure and device"},
+	{0x06678500, 0, 1,
+	"9075: Incomplete multipath connection between IOA and remote IOA"},
+	{0x06678600, 0, 1,
+	"9076: Configuration error, missing remote IOA"},
+	{0x06679100, 0, 1,
+	"4050: Enclosure does not support a required multipath function"},
 	{0x06690200, 0, 1,
 	"9041: Array protection temporarily suspended"},
 	{0x06698200, 0, 1,
@@ -331,6 +355,10 @@ struct ipr_error_table_t ipr_error_table[] = {
 	"9072: Link not operational transition"},
 	{0x066B8200, 0, 1,
 	"9032: Array exposed but still protected"},
+	{0x066B9100, 0, 1,
+	"4061: Multipath redundancy level got better"},
+	{0x066B9200, 0, 1,
+	"4060: Multipath redundancy level got worse"},
 	{0x07270000, 0, 0,
 	"Failure due to other device"},
 	{0x07278000, 0, 1,
@@ -4099,8 +4127,7 @@ static int ipr_get_autosense(struct ipr_cmnd *ipr_cmd)
 {
 	struct ipr_ioasa *ioasa = &ipr_cmd->ioasa;
 
-	if ((be32_to_cpu(ioasa->ioasc_specific) &
-	     (IPR_ADDITIONAL_STATUS_FMT | IPR_AUTOSENSE_VALID)) == 0)
+	if ((be32_to_cpu(ioasa->ioasc_specific) & IPR_AUTOSENSE_VALID) == 0)
 		return 0;
 
 	memcpy(ipr_cmd->scsi_cmd->sense_buffer, ioasa->auto_sense.data,
@@ -4190,7 +4217,8 @@ static void ipr_erp_start(struct ipr_ioa_cfg *ioa_cfg,
 	case IPR_IOASC_NR_INIT_CMD_REQUIRED:
 		break;
 	default:
-		scsi_cmd->result |= (DID_ERROR << 16);
+		if (IPR_IOASC_SENSE_KEY(ioasc) > RECOVERED_ERROR)
+			scsi_cmd->result |= (DID_ERROR << 16);
 		if (!ipr_is_vset_device(res) && !ipr_is_naca_model(res))
 			res->needs_sync_complete = 1;
 		break;
diff --git a/drivers/scsi/ipr.h b/drivers/scsi/ipr.h
index 1ad24df..11eaff52 100644
--- a/drivers/scsi/ipr.h
+++ b/drivers/scsi/ipr.h
@@ -36,8 +36,8 @@
 /*
  * Literals
  */
-#define IPR_DRIVER_VERSION "2.1.3"
-#define IPR_DRIVER_DATE "(March 29, 2006)"
+#define IPR_DRIVER_VERSION "2.1.4"
+#define IPR_DRIVER_DATE "(August 2, 2006)"
 
 /*
  * IPR_MAX_CMD_PER_LUN: This defines the maximum number of outstanding
@@ -45,6 +45,7 @@
  *	This can be adjusted at runtime through sysfs device attributes.
  */
 #define IPR_MAX_CMD_PER_LUN				6
+#define IPR_MAX_CMD_PER_ATA_LUN			1
 
 /*
  * IPR_NUM_BASE_CMD_BLKS: This defines the maximum number of
@@ -106,7 +107,7 @@
 #define IPR_IOA_BUS						0xff
 #define IPR_IOA_TARGET					0xff
 #define IPR_IOA_LUN						0xff
-#define IPR_MAX_NUM_BUSES				8
+#define IPR_MAX_NUM_BUSES				16
 #define IPR_MAX_BUS_TO_SCAN				IPR_MAX_NUM_BUSES
 
 #define IPR_NUM_RESET_RELOAD_RETRIES		3
@@ -145,6 +146,7 @@
 #define	IPR_LUN_RESET					0x40
 #define	IPR_TARGET_RESET					0x20
 #define	IPR_BUS_RESET					0x10
+#define	IPR_ATA_PHY_RESET					0x80
 #define IPR_ID_HOST_RR_Q				0xC4
 #define IPR_QUERY_IOA_CONFIG				0xC5
 #define IPR_CANCEL_ALL_REQUESTS			0xCE
@@ -295,7 +297,11 @@ struct ipr_std_inq_data {
 }__attribute__ ((packed));
 
 struct ipr_config_table_entry {
-	u8 service_level;
+	u8 proto;
+#define IPR_PROTO_SATA			0x02
+#define IPR_PROTO_SATA_ATAPI		0x03
+#define IPR_PROTO_SAS_STP		0x06
+#define IPR_PROTO_SAS_STP_ATAPI	0x07
 	u8 array_id;
 	u8 flags;
 #define IPR_IS_IOA_RESOURCE	0x80
@@ -307,6 +313,7 @@ struct ipr_config_table_entry {
 #define IPR_SUBTYPE_AF_DASD			0
 #define IPR_SUBTYPE_GENERIC_SCSI	1
 #define IPR_SUBTYPE_VOLUME_SET		2
+#define IPR_SUBTYPE_GENERIC_ATA	4
 
 #define IPR_QUEUEING_MODEL(res)	((((res)->cfgte.flags) & 0x70) >> 4)
 #define IPR_QUEUE_FROZEN_MODEL	0
@@ -350,6 +357,7 @@ struct ipr_cmd_pkt {
 #define IPR_RQTYPE_SCSICDB		0x00
 #define IPR_RQTYPE_IOACMD		0x01
 #define IPR_RQTYPE_HCAM			0x02
+#define IPR_RQTYPE_ATA_PASSTHRU	0x04
 
 	u8 luntar_luntrn;
 
@@ -373,6 +381,37 @@ struct ipr_cmd_pkt {
 	__be16 timeout;
 }__attribute__ ((packed, aligned(4)));
 
+struct ipr_ioarcb_ata_regs {
+	u8 flags;
+#define IPR_ATA_FLAG_PACKET_CMD			0x80
+#define IPR_ATA_FLAG_XFER_TYPE_DMA			0x40
+#define IPR_ATA_FLAG_STATUS_ON_GOOD_COMPLETION	0x20
+	u8 reserved[3];
+
+	__be16 data;
+	u8 feature;
+	u8 nsect;
+	u8 lbal;
+	u8 lbam;
+	u8 lbah;
+	u8 device;
+	u8 command;
+	u8 reserved2[3];
+	u8 hob_feature;
+	u8 hob_nsect;
+	u8 hob_lbal;
+	u8 hob_lbam;
+	u8 hob_lbah;
+	u8 ctl;
+}__attribute__ ((packed, aligned(4)));
+
+struct ipr_ioarcb_add_data {
+	union {
+		struct ipr_ioarcb_ata_regs regs;
+		__be32 add_cmd_parms[10];
+	}u;
+}__attribute__ ((packed, aligned(4)));
+
 /* IOA Request Control Block    128 bytes  */
 struct ipr_ioarcb {
 	__be32 ioarcb_host_pci_addr;
@@ -397,7 +436,7 @@ struct ipr_ioarcb {
 	struct ipr_cmd_pkt cmd_pkt;
 
 	__be32 add_cmd_parms_len;
-	__be32 add_cmd_parms[10];
+	struct ipr_ioarcb_add_data add_data;
 }__attribute__((packed, aligned (4)));
 
 struct ipr_ioadl_desc {
@@ -433,6 +472,21 @@ struct ipr_ioasa_gpdd {
 	__be32 ioa_data[2];
 }__attribute__((packed, aligned (4)));
 
+struct ipr_ioasa_gata {
+	u8 error;
+	u8 nsect;		/* Interrupt reason */
+	u8 lbal;
+	u8 lbam;
+	u8 lbah;
+	u8 device;
+	u8 status;
+	u8 alt_status;	/* ATA CTL */
+	u8 hob_nsect;
+	u8 hob_lbal;
+	u8 hob_lbam;
+	u8 hob_lbah;
+}__attribute__((packed, aligned (4)));
+
 struct ipr_auto_sense {
 	__be16 auto_sense_len;
 	__be16 ioa_data_len;
@@ -466,6 +520,7 @@ struct ipr_ioasa {
 	__be32 ioasc_specific;	/* status code specific field */
 #define IPR_ADDITIONAL_STATUS_FMT		0x80000000
 #define IPR_AUTOSENSE_VALID			0x40000000
+#define IPR_ATA_DEVICE_WAS_RESET		0x20000000
 #define IPR_IOASC_SPECIFIC_MASK		0x00ffffff
 #define IPR_FIELD_POINTER_VALID		(0x80000000 >> 8)
 #define IPR_FIELD_POINTER_MASK		0x0000ffff
@@ -474,6 +529,7 @@ struct ipr_ioasa {
 		struct ipr_ioasa_vset vset;
 		struct ipr_ioasa_af_dasd dasd;
 		struct ipr_ioasa_gpdd gpdd;
+		struct ipr_ioasa_gata gata;
 	} u;
 
 	struct ipr_auto_sense auto_sense;
@@ -1308,6 +1364,22 @@ static inline int ipr_is_scsi_disk(struct ipr_resource_entry *res)
 }
 
 /**
+ * ipr_is_gata - Determine if a resource is a generic ATA resource
+ * @res:	resource entry struct
+ *
+ * Return value:
+ * 	1 if GATA / 0 if not GATA
+ **/
+static inline int ipr_is_gata(struct ipr_resource_entry *res)
+{
+	if (!ipr_is_ioa_resource(res) &&
+	    IPR_RES_SUBTYPE(res) == IPR_SUBTYPE_GENERIC_ATA)
+		return 1;
+	else
+		return 0;
+}
+
+/**
  * ipr_is_naca_model - Determine if a resource is using NACA queueing model
  * @res:	resource entry struct
  *
diff --git a/drivers/scsi/iscsi_tcp.c b/drivers/scsi/iscsi_tcp.c
index 058f094..0a9dbc5 100644
--- a/drivers/scsi/iscsi_tcp.c
+++ b/drivers/scsi/iscsi_tcp.c
@@ -108,8 +108,8 @@ iscsi_hdr_digest(struct iscsi_conn *conn, struct iscsi_buf *buf,
 {
 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 
-	crypto_digest_digest(tcp_conn->tx_tfm, &buf->sg, 1, crc);
-	buf->sg.length += sizeof(uint32_t);
+	crypto_hash_digest(&tcp_conn->tx_hash, &buf->sg, buf->sg.length, crc);
+	buf->sg.length = tcp_conn->hdr_size;
 }
 
 static inline int
@@ -281,7 +281,6 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 {
 	struct iscsi_data *hdr;
 	struct scsi_cmnd *sc = ctask->sc;
-	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
 
 	hdr = &r2t->dtask.hdr;
 	memset(hdr, 0, sizeof(struct iscsi_data));
@@ -336,10 +335,12 @@ iscsi_solicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 			sg_count += sg->length;
 		}
 		BUG_ON(r2t->sg == NULL);
-	} else
-		iscsi_buf_init_iov(&tcp_ctask->sendbuf,
+	} else {
+		iscsi_buf_init_iov(&r2t->sendbuf,
 			    (char*)sc->request_buffer + r2t->data_offset,
 			    r2t->data_count);
+		r2t->sg = NULL;
+	}
 }
 
 /**
@@ -358,8 +359,11 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 	int r2tsn = be32_to_cpu(rhdr->r2tsn);
 	int rc;
 
-	if (tcp_conn->in.datalen)
+	if (tcp_conn->in.datalen) {
+		printk(KERN_ERR "iscsi_tcp: invalid R2t with datalen %d\n",
+		       tcp_conn->in.datalen);
 		return ISCSI_ERR_DATALEN;
+	}
 
 	if (tcp_ctask->exp_r2tsn && tcp_ctask->exp_r2tsn != r2tsn)
 		return ISCSI_ERR_R2TSN;
@@ -385,15 +389,23 @@ iscsi_r2t_rsp(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 
 	r2t->exp_statsn = rhdr->statsn;
 	r2t->data_length = be32_to_cpu(rhdr->data_length);
-	if (r2t->data_length == 0 ||
-	    r2t->data_length > session->max_burst) {
+	if (r2t->data_length == 0) {
+		printk(KERN_ERR "iscsi_tcp: invalid R2T with zero data len\n");
 		spin_unlock(&session->lock);
 		return ISCSI_ERR_DATALEN;
 	}
 
+	if (r2t->data_length > session->max_burst)
+		debug_scsi("invalid R2T with data len %u and max burst %u."
+			   "Attempting to execute request.\n",
+			    r2t->data_length, session->max_burst);
+
 	r2t->data_offset = be32_to_cpu(rhdr->data_offset);
 	if (r2t->data_offset + r2t->data_length > ctask->total_length) {
 		spin_unlock(&session->lock);
+		printk(KERN_ERR "iscsi_tcp: invalid R2T with data len %u at "
+		       "offset %u and total length %d\n", r2t->data_length,
+		       r2t->data_offset, ctask->total_length);
 		return ISCSI_ERR_DATALEN;
 	}
 
@@ -456,7 +468,8 @@ iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
 
 		sg_init_one(&sg, (u8 *)hdr,
 			    sizeof(struct iscsi_hdr) + ahslen);
-		crypto_digest_digest(tcp_conn->rx_tfm, &sg, 1, (u8 *)&cdgst);
+		crypto_hash_digest(&tcp_conn->rx_hash, &sg, sg.length,
+				   (u8 *)&cdgst);
 		rdgst = *(uint32_t*)((char*)hdr + sizeof(struct iscsi_hdr) +
 				     ahslen);
 		if (cdgst != rdgst) {
@@ -492,7 +505,6 @@ iscsi_tcp_hdr_recv(struct iscsi_conn *conn)
 			goto copy_hdr;
 
 		spin_lock(&session->lock);
-		iscsi_tcp_cleanup_ctask(conn, tcp_conn->in.ctask);
 		rc = __iscsi_complete_pdu(conn, hdr, NULL, 0);
 		spin_unlock(&session->lock);
 		break;
@@ -637,10 +649,9 @@ iscsi_ctask_copy(struct iscsi_tcp_conn *tcp_conn, struct iscsi_cmd_task *ctask,
  *	byte counters.
  **/
 static inline int
-iscsi_tcp_copy(struct iscsi_conn *conn)
+iscsi_tcp_copy(struct iscsi_conn *conn, int buf_size)
 {
 	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
-	int buf_size = tcp_conn->in.datalen;
 	int buf_left = buf_size - tcp_conn->data_copied;
 	int size = min(tcp_conn->in.copy, buf_left);
 	int rc;
@@ -665,15 +676,15 @@ iscsi_tcp_copy(struct iscsi_conn *conn)
 }
 
 static inline void
-partial_sg_digest_update(struct iscsi_tcp_conn *tcp_conn,
-			 struct scatterlist *sg, int offset, int length)
+partial_sg_digest_update(struct hash_desc *desc, struct scatterlist *sg,
+			 int offset, int length)
 {
 	struct scatterlist temp;
 
 	memcpy(&temp, sg, sizeof(struct scatterlist));
 	temp.offset = offset;
 	temp.length = length;
-	crypto_digest_update(tcp_conn->data_rx_tfm, &temp, 1);
+	crypto_hash_update(desc, &temp, length);
 }
 
 static void
@@ -682,7 +693,7 @@ iscsi_recv_digest_update(struct iscsi_tcp_conn *tcp_conn, char* buf, int len)
 	struct scatterlist tmp;
 
 	sg_init_one(&tmp, buf, len);
-	crypto_digest_update(tcp_conn->data_rx_tfm, &tmp, 1);
+	crypto_hash_update(&tcp_conn->rx_hash, &tmp, len);
 }
 
 static int iscsi_scsi_data_in(struct iscsi_conn *conn)
@@ -736,11 +747,12 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
 		if (!rc) {
 			if (conn->datadgst_en) {
 				if (!offset)
-					crypto_digest_update(
-							tcp_conn->data_rx_tfm,
+					crypto_hash_update(
+							&tcp_conn->rx_hash,
 							&sg[i], 1);
 				else
-					partial_sg_digest_update(tcp_conn,
+					partial_sg_digest_update(
+							&tcp_conn->rx_hash,
 							&sg[i],
 							sg[i].offset + offset,
 							sg[i].length - offset);
@@ -754,8 +766,10 @@ static int iscsi_scsi_data_in(struct iscsi_conn *conn)
 				/*
 				 * data-in is complete, but buffer not...
 				 */
-				partial_sg_digest_update(tcp_conn, &sg[i],
-						sg[i].offset, sg[i].length-rc);
+				partial_sg_digest_update(&tcp_conn->rx_hash,
+							 &sg[i],
+							 sg[i].offset,
+							 sg[i].length-rc);
 			rc = 0;
 			break;
 		}
@@ -772,7 +786,6 @@ done:
 			   (long)sc, sc->result, ctask->itt,
 			   tcp_conn->in.hdr->flags);
 		spin_lock(&conn->session->lock);
-		iscsi_tcp_cleanup_ctask(conn, ctask);
 		__iscsi_complete_pdu(conn, tcp_conn->in.hdr, NULL, 0);
 		spin_unlock(&conn->session->lock);
 	}
@@ -792,9 +805,6 @@ iscsi_data_recv(struct iscsi_conn *conn)
 		rc = iscsi_scsi_data_in(conn);
 		break;
 	case ISCSI_OP_SCSI_CMD_RSP:
-		spin_lock(&conn->session->lock);
-		iscsi_tcp_cleanup_ctask(conn, tcp_conn->in.ctask);
-		spin_unlock(&conn->session->lock);
 	case ISCSI_OP_TEXT_RSP:
 	case ISCSI_OP_LOGIN_RSP:
 	case ISCSI_OP_ASYNC_EVENT:
@@ -803,7 +813,7 @@ iscsi_data_recv(struct iscsi_conn *conn)
 		 * Collect data segment to the connection's data
 		 * placeholder
 		 */
-		if (iscsi_tcp_copy(conn)) {
+		if (iscsi_tcp_copy(conn, tcp_conn->in.datalen)) {
 			rc = -EAGAIN;
 			goto exit;
 		}
@@ -876,10 +886,8 @@ more:
 		 */
 		rc = iscsi_tcp_hdr_recv(conn);
 		if (!rc && tcp_conn->in.datalen) {
-			if (conn->datadgst_en) {
-				BUG_ON(!tcp_conn->data_rx_tfm);
-				crypto_digest_init(tcp_conn->data_rx_tfm);
-			}
+			if (conn->datadgst_en)
+				crypto_hash_init(&tcp_conn->rx_hash);
 			tcp_conn->in_progress = IN_PROGRESS_DATA_RECV;
 		} else if (rc) {
 			iscsi_conn_failure(conn, rc);
@@ -892,10 +900,15 @@ more:
 
 		debug_tcp("extra data_recv offset %d copy %d\n",
 			  tcp_conn->in.offset, tcp_conn->in.copy);
-		skb_copy_bits(tcp_conn->in.skb, tcp_conn->in.offset,
-				&recv_digest, 4);
-		tcp_conn->in.offset += 4;
-		tcp_conn->in.copy -= 4;
+		rc = iscsi_tcp_copy(conn, sizeof(uint32_t));
+		if (rc) {
+			if (rc == -EAGAIN)
+				goto again;
+			iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED);
+			return 0;
+		}
+
+		memcpy(&recv_digest, conn->data, sizeof(uint32_t));
 		if (recv_digest != tcp_conn->in.datadgst) {
 			debug_tcp("iscsi_tcp: data digest error!"
 				  "0x%x != 0x%x\n", recv_digest,
@@ -931,13 +944,14 @@ more:
 					  tcp_conn->in.padding);
 				memset(pad, 0, tcp_conn->in.padding);
 				sg_init_one(&sg, pad, tcp_conn->in.padding);
-				crypto_digest_update(tcp_conn->data_rx_tfm,
-						     &sg, 1);
+				crypto_hash_update(&tcp_conn->rx_hash,
+						   &sg, sg.length);
 			}
-			crypto_digest_final(tcp_conn->data_rx_tfm,
-					    (u8 *) & tcp_conn->in.datadgst);
+			crypto_hash_final(&tcp_conn->rx_hash,
+					  (u8 *) &tcp_conn->in.datadgst);
 			debug_tcp("rx digest 0x%x\n", tcp_conn->in.datadgst);
 			tcp_conn->in_progress = IN_PROGRESS_DDIGEST_RECV;
+			tcp_conn->data_copied = 0;
 		} else
 			tcp_conn->in_progress = IN_PROGRESS_WAIT_HEADER;
 	}
@@ -1177,37 +1191,12 @@ iscsi_sendpage(struct iscsi_conn *conn, struct iscsi_buf *buf,
 
 static inline void
 iscsi_data_digest_init(struct iscsi_tcp_conn *tcp_conn,
-		      struct iscsi_cmd_task *ctask)
+		      struct iscsi_tcp_cmd_task *tcp_ctask)
 {
-	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
-
-	BUG_ON(!tcp_conn->data_tx_tfm);
-	crypto_digest_init(tcp_conn->data_tx_tfm);
+	crypto_hash_init(&tcp_conn->tx_hash);
 	tcp_ctask->digest_count = 4;
 }
 
-static int
-iscsi_digest_final_send(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
-			struct iscsi_buf *buf, uint32_t *digest, int final)
-{
-	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
-	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
-	int rc = 0;
-	int sent = 0;
-
-	if (final)
-		crypto_digest_final(tcp_conn->data_tx_tfm, (u8*)digest);
-
-	iscsi_buf_init_iov(buf, (char*)digest, 4);
-	rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
-	if (rc) {
-		tcp_ctask->datadigest = *digest;
-		tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
-	} else
-		tcp_ctask->digest_count = 4;
-	return rc;
-}
-
 /**
  * iscsi_solicit_data_cont - initialize next Data-Out
  * @conn: iscsi connection
@@ -1225,7 +1214,6 @@ static void
 iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 			struct iscsi_r2t_info *r2t, int left)
 {
-	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
 	struct iscsi_data *hdr;
 	struct scsi_cmnd *sc = ctask->sc;
 	int new_offset;
@@ -1254,27 +1242,30 @@ iscsi_solicit_data_cont(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 	iscsi_buf_init_iov(&r2t->headbuf, (char*)hdr,
 			   sizeof(struct iscsi_hdr));
 
-	if (sc->use_sg && !iscsi_buf_left(&r2t->sendbuf)) {
-		BUG_ON(tcp_ctask->bad_sg == r2t->sg);
+	if (iscsi_buf_left(&r2t->sendbuf))
+		return;
+
+	if (sc->use_sg) {
 		iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
 		r2t->sg += 1;
-	} else
-		iscsi_buf_init_iov(&tcp_ctask->sendbuf,
+	} else {
+		iscsi_buf_init_iov(&r2t->sendbuf,
 			    (char*)sc->request_buffer + new_offset,
 			    r2t->data_count);
+		r2t->sg = NULL;
+	}
 }
 
-static void
-iscsi_unsolicit_data_init(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+static void iscsi_set_padding(struct iscsi_tcp_cmd_task *tcp_ctask,
+			      unsigned long len)
 {
-	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
-	struct iscsi_data_task *dtask;
+	tcp_ctask->pad_count = len & (ISCSI_PAD_LEN - 1);
+	if (!tcp_ctask->pad_count)
+		return;
 
-	dtask = tcp_ctask->dtask = &tcp_ctask->unsol_dtask;
-	iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr,
-				      tcp_ctask->r2t_data_count);
-	iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
-			   sizeof(struct iscsi_hdr));
+	tcp_ctask->pad_count = ISCSI_PAD_LEN - tcp_ctask->pad_count;
+	debug_scsi("write padding %d bytes\n", tcp_ctask->pad_count);
+	tcp_ctask->xmstate |= XMSTATE_W_PAD;
 }
 
 /**
@@ -1302,38 +1293,20 @@ iscsi_tcp_cmd_init(struct iscsi_cmd_task *ctask)
 		if (sc->use_sg) {
 			struct scatterlist *sg = sc->request_buffer;
 
-			iscsi_buf_init_sg(&tcp_ctask->sendbuf,
-					  &sg[tcp_ctask->sg_count++]);
-			tcp_ctask->sg = sg;
+			iscsi_buf_init_sg(&tcp_ctask->sendbuf, sg);
+			tcp_ctask->sg = sg + 1;
 			tcp_ctask->bad_sg = sg + sc->use_sg;
-		} else
+		} else {
 			iscsi_buf_init_iov(&tcp_ctask->sendbuf,
 					   sc->request_buffer,
 					   sc->request_bufflen);
-
-		if (ctask->imm_count)
-			tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
-
-		tcp_ctask->pad_count = ctask->total_length & (ISCSI_PAD_LEN-1);
-		if (tcp_ctask->pad_count) {
-			tcp_ctask->pad_count = ISCSI_PAD_LEN -
-							tcp_ctask->pad_count;
-			debug_scsi("write padding %d bytes\n",
-				   tcp_ctask->pad_count);
-			tcp_ctask->xmstate |= XMSTATE_W_PAD;
+			tcp_ctask->sg = NULL;
+			tcp_ctask->bad_sg = NULL;
 		}
-
-		if (ctask->unsol_count)
-			tcp_ctask->xmstate |= XMSTATE_UNS_HDR |
-						XMSTATE_UNS_INIT;
-		tcp_ctask->r2t_data_count = ctask->total_length -
-				    ctask->imm_count -
-				    ctask->unsol_count;
-
-		debug_scsi("cmd [itt 0x%x total %d imm %d imm_data %d "
-			   "r2t_data %d]\n",
+		debug_scsi("cmd [itt 0x%x total %d imm_data %d "
+			   "unsol count %d, unsol offset %d]\n",
 			   ctask->itt, ctask->total_length, ctask->imm_count,
-			   ctask->unsol_count, tcp_ctask->r2t_data_count);
+			   ctask->unsol_count, ctask->unsol_offset);
 	} else
 		tcp_ctask->xmstate = XMSTATE_R_HDR;
 
@@ -1415,8 +1388,8 @@ iscsi_tcp_mtask_xmit(struct iscsi_conn *conn, struct iscsi_mgmt_task *mtask)
 }
 
 static inline int
-handle_xmstate_r_hdr(struct iscsi_conn *conn,
-		     struct iscsi_tcp_cmd_task *tcp_ctask)
+iscsi_send_read_hdr(struct iscsi_conn *conn,
+		    struct iscsi_tcp_cmd_task *tcp_ctask)
 {
 	int rc;
 
@@ -1434,7 +1407,7 @@ handle_xmstate_r_hdr(struct iscsi_conn *conn,
 }
 
 static inline int
-handle_xmstate_w_hdr(struct iscsi_conn *conn,
+iscsi_send_write_hdr(struct iscsi_conn *conn,
 		     struct iscsi_cmd_task *ctask)
 {
 	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
@@ -1445,85 +1418,126 @@ handle_xmstate_w_hdr(struct iscsi_conn *conn,
 		iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
 				 (u8*)tcp_ctask->hdrext);
 	rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->imm_count);
-	if (rc)
+	if (rc) {
 		tcp_ctask->xmstate |= XMSTATE_W_HDR;
-	return rc;
+		return rc;
+	}
+
+	if (ctask->imm_count) {
+		tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
+		iscsi_set_padding(tcp_ctask, ctask->imm_count);
+
+		if (ctask->conn->datadgst_en) {
+			iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
+			tcp_ctask->immdigest = 0;
+		}
+	}
+
+	if (ctask->unsol_count)
+		tcp_ctask->xmstate |= XMSTATE_UNS_HDR | XMSTATE_UNS_INIT;
+	return 0;
 }
 
-static inline int
-handle_xmstate_data_digest(struct iscsi_conn *conn,
-			   struct iscsi_cmd_task *ctask)
+static int
+iscsi_send_padding(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 {
 	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
-	int rc;
+	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+	int sent = 0, rc;
 
-	tcp_ctask->xmstate &= ~XMSTATE_DATA_DIGEST;
-	debug_tcp("resent data digest 0x%x\n", tcp_ctask->datadigest);
-	rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
-				    &tcp_ctask->datadigest, 0);
+	if (tcp_ctask->xmstate & XMSTATE_W_PAD) {
+		iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
+				   tcp_ctask->pad_count);
+		if (conn->datadgst_en)
+			crypto_hash_update(&tcp_conn->tx_hash,
+					   &tcp_ctask->sendbuf.sg,
+					   tcp_ctask->sendbuf.sg.length);
+	} else if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_PAD))
+		return 0;
+
+	tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
+	tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_PAD;
+	debug_scsi("sending %d pad bytes for itt 0x%x\n",
+		   tcp_ctask->pad_count, ctask->itt);
+	rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
+			   &sent);
 	if (rc) {
-		tcp_ctask->xmstate |= XMSTATE_DATA_DIGEST;
-		debug_tcp("resent data digest 0x%x fail!\n",
-			  tcp_ctask->datadigest);
+		debug_scsi("padding send failed %d\n", rc);
+		tcp_ctask->xmstate |= XMSTATE_W_RESEND_PAD;
 	}
-
 	return rc;
 }
 
-static inline int
-handle_xmstate_imm_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+static int
+iscsi_send_digest(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
+			struct iscsi_buf *buf, uint32_t *digest)
 {
-	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
-	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
-	int rc;
+	struct iscsi_tcp_cmd_task *tcp_ctask;
+	struct iscsi_tcp_conn *tcp_conn;
+	int rc, sent = 0;
 
-	BUG_ON(!ctask->imm_count);
-	tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
+	if (!conn->datadgst_en)
+		return 0;
 
-	if (conn->datadgst_en) {
-		iscsi_data_digest_init(tcp_conn, ctask);
-		tcp_ctask->immdigest = 0;
-	}
+	tcp_ctask = ctask->dd_data;
+	tcp_conn = conn->dd_data;
 
-	for (;;) {
-		rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf,
-				   &ctask->imm_count, &tcp_ctask->sent);
-		if (rc) {
-			tcp_ctask->xmstate |= XMSTATE_IMM_DATA;
-			if (conn->datadgst_en) {
-				crypto_digest_final(tcp_conn->data_tx_tfm,
-						(u8*)&tcp_ctask->immdigest);
-				debug_tcp("tx imm sendpage fail 0x%x\n",
-					  tcp_ctask->datadigest);
-			}
-			return rc;
-		}
-		if (conn->datadgst_en)
-			crypto_digest_update(tcp_conn->data_tx_tfm,
-					     &tcp_ctask->sendbuf.sg, 1);
+	if (!(tcp_ctask->xmstate & XMSTATE_W_RESEND_DATA_DIGEST)) {
+		crypto_hash_final(&tcp_conn->tx_hash, (u8*)digest);
+		iscsi_buf_init_iov(buf, (char*)digest, 4);
+	}
+	tcp_ctask->xmstate &= ~XMSTATE_W_RESEND_DATA_DIGEST;
 
-		if (!ctask->imm_count)
-			break;
-		iscsi_buf_init_sg(&tcp_ctask->sendbuf,
-				  &tcp_ctask->sg[tcp_ctask->sg_count++]);
+	rc = iscsi_sendpage(conn, buf, &tcp_ctask->digest_count, &sent);
+	if (!rc)
+		debug_scsi("sent digest 0x%x for itt 0x%x\n", *digest,
+			  ctask->itt);
+	else {
+		debug_scsi("sending digest 0x%x failed for itt 0x%x!\n",
+			  *digest, ctask->itt);
+		tcp_ctask->xmstate |= XMSTATE_W_RESEND_DATA_DIGEST;
 	}
+	return rc;
+}
 
-	if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
-		rc = iscsi_digest_final_send(conn, ctask, &tcp_ctask->immbuf,
-				            &tcp_ctask->immdigest, 1);
-		if (rc) {
-			debug_tcp("sending imm digest 0x%x fail!\n",
-				  tcp_ctask->immdigest);
-			return rc;
+static int
+iscsi_send_data(struct iscsi_cmd_task *ctask, struct iscsi_buf *sendbuf,
+		struct scatterlist **sg, int *sent, int *count,
+		struct iscsi_buf *digestbuf, uint32_t *digest)
+{
+	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
+	struct iscsi_conn *conn = ctask->conn;
+	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
+	int rc, buf_sent, offset;
+
+	while (*count) {
+		buf_sent = 0;
+		offset = sendbuf->sent;
+
+		rc = iscsi_sendpage(conn, sendbuf, count, &buf_sent);
+		*sent = *sent + buf_sent;
+		if (buf_sent && conn->datadgst_en)
+			partial_sg_digest_update(&tcp_conn->tx_hash,
+				&sendbuf->sg, sendbuf->sg.offset + offset,
+				buf_sent);
+		if (!iscsi_buf_left(sendbuf) && *sg != tcp_ctask->bad_sg) {
+			iscsi_buf_init_sg(sendbuf, *sg);
+			*sg = *sg + 1;
 		}
-		debug_tcp("sending imm digest 0x%x\n", tcp_ctask->immdigest);
+
+		if (rc)
+			return rc;
 	}
 
-	return 0;
+	rc = iscsi_send_padding(conn, ctask);
+	if (rc)
+		return rc;
+
+	return iscsi_send_digest(conn, ctask, digestbuf, digest);
 }
 
-static inline int
-handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+static int
+iscsi_send_unsol_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 {
 	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
 	struct iscsi_data_task *dtask;
@@ -1531,12 +1545,17 @@ handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 
 	tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
 	if (tcp_ctask->xmstate & XMSTATE_UNS_INIT) {
-		iscsi_unsolicit_data_init(conn, ctask);
-		dtask = tcp_ctask->dtask;
+		dtask = &tcp_ctask->unsol_dtask;
+
+		iscsi_prep_unsolicit_data_pdu(ctask, &dtask->hdr);
+		iscsi_buf_init_iov(&tcp_ctask->headbuf, (char*)&dtask->hdr,
+				   sizeof(struct iscsi_hdr));
 		if (conn->hdrdgst_en)
 			iscsi_hdr_digest(conn, &tcp_ctask->headbuf,
 					(u8*)dtask->hdrext);
+
 		tcp_ctask->xmstate &= ~XMSTATE_UNS_INIT;
+		iscsi_set_padding(tcp_ctask, ctask->data_count);
 	}
 
 	rc = iscsi_sendhdr(conn, &tcp_ctask->headbuf, ctask->data_count);
@@ -1546,254 +1565,138 @@ handle_xmstate_uns_hdr(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 		return rc;
 	}
 
+	if (conn->datadgst_en) {
+		dtask = &tcp_ctask->unsol_dtask;
+		iscsi_data_digest_init(ctask->conn->dd_data, tcp_ctask);
+		dtask->digest = 0;
+	}
+
 	debug_scsi("uns dout [itt 0x%x dlen %d sent %d]\n",
 		   ctask->itt, ctask->unsol_count, tcp_ctask->sent);
 	return 0;
 }
 
-static inline int
-handle_xmstate_uns_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+static int
+iscsi_send_unsol_pdu(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 {
 	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
-	struct iscsi_data_task *dtask = tcp_ctask->dtask;
-	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 	int rc;
 
-	BUG_ON(!ctask->data_count);
-	tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
-
-	if (conn->datadgst_en) {
-		iscsi_data_digest_init(tcp_conn, ctask);
-		dtask->digest = 0;
+	if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
+		BUG_ON(!ctask->unsol_count);
+		tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
+send_hdr:
+		rc = iscsi_send_unsol_hdr(conn, ctask);
+		if (rc)
+			return rc;
 	}
 
-	for (;;) {
+	if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
+		struct iscsi_data_task *dtask = &tcp_ctask->unsol_dtask;
 		int start = tcp_ctask->sent;
 
-		rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf,
-				   &ctask->data_count, &tcp_ctask->sent);
-		if (rc) {
-			ctask->unsol_count -= tcp_ctask->sent - start;
-			tcp_ctask->xmstate |= XMSTATE_UNS_DATA;
-			/* will continue with this ctask later.. */
-			if (conn->datadgst_en) {
-				crypto_digest_final(tcp_conn->data_tx_tfm,
-						(u8 *)&dtask->digest);
-				debug_tcp("tx uns data fail 0x%x\n",
-					  dtask->digest);
-			}
-			return rc;
-		}
-
-		BUG_ON(tcp_ctask->sent > ctask->total_length);
+		rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
+				     &tcp_ctask->sent, &ctask->data_count,
+				     &dtask->digestbuf, &dtask->digest);
 		ctask->unsol_count -= tcp_ctask->sent - start;
-
+		if (rc)
+			return rc;
+		tcp_ctask->xmstate &= ~XMSTATE_UNS_DATA;
 		/*
-		 * XXX:we may run here with un-initial sendbuf.
-		 * so pass it
+		 * Done with the Data-Out. Next, check if we need
+		 * to send another unsolicited Data-Out.
 		 */
-		if (conn->datadgst_en && tcp_ctask->sent - start > 0)
-			crypto_digest_update(tcp_conn->data_tx_tfm,
-					     &tcp_ctask->sendbuf.sg, 1);
-
-		if (!ctask->data_count)
-			break;
-		iscsi_buf_init_sg(&tcp_ctask->sendbuf,
-				  &tcp_ctask->sg[tcp_ctask->sg_count++]);
-	}
-	BUG_ON(ctask->unsol_count < 0);
-
-	/*
-	 * Done with the Data-Out. Next, check if we need
-	 * to send another unsolicited Data-Out.
-	 */
-	if (ctask->unsol_count) {
-		if (conn->datadgst_en) {
-			rc = iscsi_digest_final_send(conn, ctask,
-						    &dtask->digestbuf,
-						    &dtask->digest, 1);
-			if (rc) {
-				debug_tcp("send uns digest 0x%x fail\n",
-					  dtask->digest);
-				return rc;
-			}
-			debug_tcp("sending uns digest 0x%x, more uns\n",
-				  dtask->digest);
+		if (ctask->unsol_count) {
+			debug_scsi("sending more uns\n");
+			tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
+			goto send_hdr;
 		}
-		tcp_ctask->xmstate |= XMSTATE_UNS_INIT;
-		return 1;
 	}
-
-	if (conn->datadgst_en && !(tcp_ctask->xmstate & XMSTATE_W_PAD)) {
-		rc = iscsi_digest_final_send(conn, ctask,
-					    &dtask->digestbuf,
-					    &dtask->digest, 1);
-		if (rc) {
-			debug_tcp("send last uns digest 0x%x fail\n",
-				   dtask->digest);
-			return rc;
-		}
-		debug_tcp("sending uns digest 0x%x\n",dtask->digest);
-	}
-
 	return 0;
 }
 
-static inline int
-handle_xmstate_sol_data(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
+static int iscsi_send_sol_pdu(struct iscsi_conn *conn,
+			      struct iscsi_cmd_task *ctask)
 {
-	struct iscsi_session *session = conn->session;
-	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
-	struct iscsi_r2t_info *r2t = tcp_ctask->r2t;
-	struct iscsi_data_task *dtask = &r2t->dtask;
+	struct iscsi_session *session = conn->session;
+	struct iscsi_r2t_info *r2t;
+	struct iscsi_data_task *dtask;
 	int left, rc;
 
-	tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
-	tcp_ctask->dtask = dtask;
-
-	if (conn->datadgst_en) {
-		iscsi_data_digest_init(tcp_conn, ctask);
-		dtask->digest = 0;
-	}
-solicit_again:
-	/*
-	 * send Data-Out within this R2T sequence.
-	 */
-	if (!r2t->data_count)
-		goto data_out_done;
-
-	rc = iscsi_sendpage(conn, &r2t->sendbuf, &r2t->data_count, &r2t->sent);
-	if (rc) {
+	if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
+		tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
 		tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
-		/* will continue with this ctask later.. */
-		if (conn->datadgst_en) {
-			crypto_digest_final(tcp_conn->data_tx_tfm,
-					  (u8 *)&dtask->digest);
-			debug_tcp("r2t data send fail 0x%x\n", dtask->digest);
-		}
-		return rc;
-	}
+		if (!tcp_ctask->r2t)
+			__kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
+				    sizeof(void*));
+send_hdr:
+		r2t = tcp_ctask->r2t;
+		dtask = &r2t->dtask;
 
-	BUG_ON(r2t->data_count < 0);
-	if (conn->datadgst_en)
-		crypto_digest_update(tcp_conn->data_tx_tfm, &r2t->sendbuf.sg,
-				     1);
-
-	if (r2t->data_count) {
-		BUG_ON(ctask->sc->use_sg == 0);
-		if (!iscsi_buf_left(&r2t->sendbuf)) {
-			BUG_ON(tcp_ctask->bad_sg == r2t->sg);
-			iscsi_buf_init_sg(&r2t->sendbuf, r2t->sg);
-			r2t->sg += 1;
+		if (conn->hdrdgst_en)
+			iscsi_hdr_digest(conn, &r2t->headbuf,
+					(u8*)dtask->hdrext);
+		rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
+		if (rc) {
+			tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
+			tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
+			return rc;
 		}
-		goto solicit_again;
-	}
 
-data_out_done:
-	/*
-	 * Done with this Data-Out. Next, check if we have
-	 * to send another Data-Out for this R2T.
-	 */
-	BUG_ON(r2t->data_length - r2t->sent < 0);
-	left = r2t->data_length - r2t->sent;
-	if (left) {
 		if (conn->datadgst_en) {
-			rc = iscsi_digest_final_send(conn, ctask,
-						    &dtask->digestbuf,
-						    &dtask->digest, 1);
-			if (rc) {
-				debug_tcp("send r2t data digest 0x%x"
-					  "fail\n", dtask->digest);
-				return rc;
-			}
-			debug_tcp("r2t data send digest 0x%x\n",
-				  dtask->digest);
+			iscsi_data_digest_init(conn->dd_data, tcp_ctask);
+			dtask->digest = 0;
 		}
-		iscsi_solicit_data_cont(conn, ctask, r2t, left);
-		tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
-		tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
-		return 1;
-	}
 
-	/*
-	 * Done with this R2T. Check if there are more
-	 * outstanding R2Ts ready to be processed.
-	 */
-	BUG_ON(tcp_ctask->r2t_data_count - r2t->data_length < 0);
-	if (conn->datadgst_en) {
-		rc = iscsi_digest_final_send(conn, ctask, &dtask->digestbuf,
-					    &dtask->digest, 1);
-		if (rc) {
-			debug_tcp("send last r2t data digest 0x%x"
-				  "fail\n", dtask->digest);
-			return rc;
-		}
-		debug_tcp("r2t done dout digest 0x%x\n", dtask->digest);
-	}
-
-	tcp_ctask->r2t_data_count -= r2t->data_length;
-	tcp_ctask->r2t = NULL;
-	spin_lock_bh(&session->lock);
-	__kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t, sizeof(void*));
-	spin_unlock_bh(&session->lock);
-	if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t, sizeof(void*))) {
-		tcp_ctask->r2t = r2t;
-		tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
-		tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
-		return 1;
+		iscsi_set_padding(tcp_ctask, r2t->data_count);
+		debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
+			r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
+			r2t->sent);
 	}
 
-	return 0;
-}
+	if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
+		r2t = tcp_ctask->r2t;
+		dtask = &r2t->dtask;
 
-static inline int
-handle_xmstate_w_pad(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
-{
-	struct iscsi_tcp_cmd_task *tcp_ctask = ctask->dd_data;
-	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
-	struct iscsi_data_task *dtask = tcp_ctask->dtask;
-	int sent = 0, rc;
+		rc = iscsi_send_data(ctask, &r2t->sendbuf, &r2t->sg,
+				     &r2t->sent, &r2t->data_count,
+				     &dtask->digestbuf, &dtask->digest);
+		if (rc)
+			return rc;
+		tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
 
-	tcp_ctask->xmstate &= ~XMSTATE_W_PAD;
-	iscsi_buf_init_iov(&tcp_ctask->sendbuf, (char*)&tcp_ctask->pad,
-			    tcp_ctask->pad_count);
-	rc = iscsi_sendpage(conn, &tcp_ctask->sendbuf, &tcp_ctask->pad_count,
-			   &sent);
-	if (rc) {
-		tcp_ctask->xmstate |= XMSTATE_W_PAD;
-		return rc;
-	}
+		/*
+		 * Done with this Data-Out. Next, check if we have
+		 * to send another Data-Out for this R2T.
+		 */
+		BUG_ON(r2t->data_length - r2t->sent < 0);
+		left = r2t->data_length - r2t->sent;
+		if (left) {
+			iscsi_solicit_data_cont(conn, ctask, r2t, left);
+			tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
+			tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
+			goto send_hdr;
+		}
 
-	if (conn->datadgst_en) {
-		crypto_digest_update(tcp_conn->data_tx_tfm,
-				     &tcp_ctask->sendbuf.sg, 1);
-		/* imm data? */
-		if (!dtask) {
-			rc = iscsi_digest_final_send(conn, ctask,
-						    &tcp_ctask->immbuf,
-						    &tcp_ctask->immdigest, 1);
-			if (rc) {
-				debug_tcp("send padding digest 0x%x"
-					  "fail!\n", tcp_ctask->immdigest);
-				return rc;
-			}
-			debug_tcp("done with padding, digest 0x%x\n",
-				  tcp_ctask->datadigest);
-		} else {
-			rc = iscsi_digest_final_send(conn, ctask,
-						    &dtask->digestbuf,
-						    &dtask->digest, 1);
-			if (rc) {
-				debug_tcp("send padding digest 0x%x"
-				          "fail\n", dtask->digest);
-				return rc;
-			}
-			debug_tcp("done with padding, digest 0x%x\n",
-				  dtask->digest);
+		/*
+		 * Done with this R2T. Check if there are more
+		 * outstanding R2Ts ready to be processed.
+		 */
+		spin_lock_bh(&session->lock);
+		tcp_ctask->r2t = NULL;
+		__kfifo_put(tcp_ctask->r2tpool.queue, (void*)&r2t,
+			    sizeof(void*));
+		if (__kfifo_get(tcp_ctask->r2tqueue, (void*)&r2t,
+				sizeof(void*))) {
+			tcp_ctask->r2t = r2t;
+			tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
+			tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
+			spin_unlock_bh(&session->lock);
+			goto send_hdr;
 		}
+		spin_unlock_bh(&session->lock);
 	}
-
 	return 0;
 }
 
@@ -1813,85 +1716,30 @@ iscsi_tcp_ctask_xmit(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask)
 		return rc;
 
 	if (tcp_ctask->xmstate & XMSTATE_R_HDR)
-		return handle_xmstate_r_hdr(conn, tcp_ctask);
+		return iscsi_send_read_hdr(conn, tcp_ctask);
 
 	if (tcp_ctask->xmstate & XMSTATE_W_HDR) {
-		rc = handle_xmstate_w_hdr(conn, ctask);
-		if (rc)
-			return rc;
-	}
-
-	/* XXX: for data digest xmit recover */
-	if (tcp_ctask->xmstate & XMSTATE_DATA_DIGEST) {
-		rc = handle_xmstate_data_digest(conn, ctask);
+		rc = iscsi_send_write_hdr(conn, ctask);
 		if (rc)
 			return rc;
 	}
 
 	if (tcp_ctask->xmstate & XMSTATE_IMM_DATA) {
-		rc = handle_xmstate_imm_data(conn, ctask);
+		rc = iscsi_send_data(ctask, &tcp_ctask->sendbuf, &tcp_ctask->sg,
+				     &tcp_ctask->sent, &ctask->imm_count,
+				     &tcp_ctask->immbuf, &tcp_ctask->immdigest);
 		if (rc)
 			return rc;
+		tcp_ctask->xmstate &= ~XMSTATE_IMM_DATA;
 	}
 
-	if (tcp_ctask->xmstate & XMSTATE_UNS_HDR) {
-		BUG_ON(!ctask->unsol_count);
-		tcp_ctask->xmstate &= ~XMSTATE_UNS_HDR;
-unsolicit_head_again:
-		rc = handle_xmstate_uns_hdr(conn, ctask);
-		if (rc)
-			return rc;
-	}
-
-	if (tcp_ctask->xmstate & XMSTATE_UNS_DATA) {
-		rc = handle_xmstate_uns_data(conn, ctask);
-		if (rc == 1)
-			goto unsolicit_head_again;
-		else if (rc)
-			return rc;
-		goto done;
-	}
-
-	if (tcp_ctask->xmstate & XMSTATE_SOL_HDR) {
-		struct iscsi_r2t_info *r2t;
-
-		tcp_ctask->xmstate &= ~XMSTATE_SOL_HDR;
-		tcp_ctask->xmstate |= XMSTATE_SOL_DATA;
-		if (!tcp_ctask->r2t)
-			__kfifo_get(tcp_ctask->r2tqueue, (void*)&tcp_ctask->r2t,
-				    sizeof(void*));
-solicit_head_again:
-		r2t = tcp_ctask->r2t;
-		if (conn->hdrdgst_en)
-			iscsi_hdr_digest(conn, &r2t->headbuf,
-					(u8*)r2t->dtask.hdrext);
-		rc = iscsi_sendhdr(conn, &r2t->headbuf, r2t->data_count);
-		if (rc) {
-			tcp_ctask->xmstate &= ~XMSTATE_SOL_DATA;
-			tcp_ctask->xmstate |= XMSTATE_SOL_HDR;
-			return rc;
-		}
-
-		debug_scsi("sol dout [dsn %d itt 0x%x dlen %d sent %d]\n",
-			r2t->solicit_datasn - 1, ctask->itt, r2t->data_count,
-			r2t->sent);
-	}
-
-	if (tcp_ctask->xmstate & XMSTATE_SOL_DATA) {
-		rc = handle_xmstate_sol_data(conn, ctask);
-		if (rc == 1)
-			goto solicit_head_again;
-		if (rc)
-			return rc;
-	}
+	rc = iscsi_send_unsol_pdu(conn, ctask);
+	if (rc)
+		return rc;
 
-done:
-	/*
-	 * Last thing to check is whether we need to send write
-	 * padding. Note that we check for xmstate equality, not just the bit.
-	 */
-	if (tcp_ctask->xmstate == XMSTATE_W_PAD)
-		rc = handle_xmstate_w_pad(conn, ctask);
+	rc = iscsi_send_sol_pdu(conn, ctask);
+	if (rc)
+		return rc;
 
 	return rc;
 }
@@ -1923,8 +1771,24 @@ iscsi_tcp_conn_create(struct iscsi_cls_session *cls_session, uint32_t conn_idx)
 	/* initial operational parameters */
 	tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
 
+	tcp_conn->tx_hash.tfm = crypto_alloc_hash("crc32c", 0,
+						  CRYPTO_ALG_ASYNC);
+	tcp_conn->tx_hash.flags = 0;
+	if (!tcp_conn->tx_hash.tfm)
+		goto free_tcp_conn;
+
+	tcp_conn->rx_hash.tfm = crypto_alloc_hash("crc32c", 0,
+						  CRYPTO_ALG_ASYNC);
+	tcp_conn->rx_hash.flags = 0;
+	if (!tcp_conn->rx_hash.tfm)
+		goto free_tx_tfm;
+
 	return cls_conn;
 
+free_tx_tfm:
+	crypto_free_hash(tcp_conn->tx_hash.tfm);
+free_tcp_conn:
+	kfree(tcp_conn);
 tcp_conn_alloc_fail:
 	iscsi_conn_teardown(cls_conn);
 	return NULL;
@@ -1962,14 +1826,10 @@ iscsi_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
 
 	/* now free tcp_conn */
 	if (digest) {
-		if (tcp_conn->tx_tfm)
-			crypto_free_tfm(tcp_conn->tx_tfm);
-		if (tcp_conn->rx_tfm)
-			crypto_free_tfm(tcp_conn->rx_tfm);
-		if (tcp_conn->data_tx_tfm)
-			crypto_free_tfm(tcp_conn->data_tx_tfm);
-		if (tcp_conn->data_rx_tfm)
-			crypto_free_tfm(tcp_conn->data_rx_tfm);
+		if (tcp_conn->tx_hash.tfm)
+			crypto_free_hash(tcp_conn->tx_hash.tfm);
+		if (tcp_conn->rx_hash.tfm)
+			crypto_free_hash(tcp_conn->rx_hash.tfm);
 	}
 
 	kfree(tcp_conn);
@@ -1979,9 +1839,11 @@ static void
 iscsi_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
 {
 	struct iscsi_conn *conn = cls_conn->dd_data;
+	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 
 	iscsi_conn_stop(cls_conn, flag);
 	iscsi_tcp_release_conn(conn);
+	tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
 }
 
 static int
@@ -2127,48 +1989,11 @@ iscsi_conn_set_param(struct iscsi_cls_conn *cls_conn, enum iscsi_param param,
 	case ISCSI_PARAM_HDRDGST_EN:
 		iscsi_set_param(cls_conn, param, buf, buflen);
 		tcp_conn->hdr_size = sizeof(struct iscsi_hdr);
-		if (conn->hdrdgst_en) {
+		if (conn->hdrdgst_en)
 			tcp_conn->hdr_size += sizeof(__u32);
-			if (!tcp_conn->tx_tfm)
-				tcp_conn->tx_tfm = crypto_alloc_tfm("crc32c",
-								    0);
-			if (!tcp_conn->tx_tfm)
-				return -ENOMEM;
-			if (!tcp_conn->rx_tfm)
-				tcp_conn->rx_tfm = crypto_alloc_tfm("crc32c",
-								    0);
-			if (!tcp_conn->rx_tfm) {
-				crypto_free_tfm(tcp_conn->tx_tfm);
-				return -ENOMEM;
-			}
-		} else {
-			if (tcp_conn->tx_tfm)
-				crypto_free_tfm(tcp_conn->tx_tfm);
-			if (tcp_conn->rx_tfm)
-				crypto_free_tfm(tcp_conn->rx_tfm);
-		}
 		break;
 	case ISCSI_PARAM_DATADGST_EN:
 		iscsi_set_param(cls_conn, param, buf, buflen);
-		if (conn->datadgst_en) {
-			if (!tcp_conn->data_tx_tfm)
-				tcp_conn->data_tx_tfm =
-				    crypto_alloc_tfm("crc32c", 0);
-			if (!tcp_conn->data_tx_tfm)
-				return -ENOMEM;
-			if (!tcp_conn->data_rx_tfm)
-				tcp_conn->data_rx_tfm =
-				    crypto_alloc_tfm("crc32c", 0);
-			if (!tcp_conn->data_rx_tfm) {
-				crypto_free_tfm(tcp_conn->data_tx_tfm);
-				return -ENOMEM;
-			}
-		} else {
-			if (tcp_conn->data_tx_tfm)
-				crypto_free_tfm(tcp_conn->data_tx_tfm);
-			if (tcp_conn->data_rx_tfm)
-				crypto_free_tfm(tcp_conn->data_rx_tfm);
-		}
 		tcp_conn->sendpage = conn->datadgst_en ?
 			sock_no_sendpage : tcp_conn->sock->ops->sendpage;
 		break;
diff --git a/drivers/scsi/iscsi_tcp.h b/drivers/scsi/iscsi_tcp.h
index 6a4ee70..3273683 100644
--- a/drivers/scsi/iscsi_tcp.h
+++ b/drivers/scsi/iscsi_tcp.h
@@ -31,26 +31,25 @@
 #define IN_PROGRESS_DDIGEST_RECV	0x3
 
 /* xmit state machine */
-#define	XMSTATE_IDLE			0x0
-#define	XMSTATE_R_HDR			0x1
-#define	XMSTATE_W_HDR			0x2
-#define	XMSTATE_IMM_HDR			0x4
-#define	XMSTATE_IMM_DATA		0x8
-#define	XMSTATE_UNS_INIT		0x10
-#define	XMSTATE_UNS_HDR			0x20
-#define	XMSTATE_UNS_DATA		0x40
-#define	XMSTATE_SOL_HDR			0x80
-#define	XMSTATE_SOL_DATA		0x100
-#define	XMSTATE_W_PAD			0x200
-#define XMSTATE_DATA_DIGEST		0x400
-
-#define ISCSI_CONN_RCVBUF_MIN		262144
-#define ISCSI_CONN_SNDBUF_MIN		262144
+#define XMSTATE_IDLE			0x0
+#define XMSTATE_R_HDR			0x1
+#define XMSTATE_W_HDR			0x2
+#define XMSTATE_IMM_HDR			0x4
+#define XMSTATE_IMM_DATA		0x8
+#define XMSTATE_UNS_INIT		0x10
+#define XMSTATE_UNS_HDR			0x20
+#define XMSTATE_UNS_DATA		0x40
+#define XMSTATE_SOL_HDR			0x80
+#define XMSTATE_SOL_DATA		0x100
+#define XMSTATE_W_PAD			0x200
+#define XMSTATE_W_RESEND_PAD		0x400
+#define XMSTATE_W_RESEND_DATA_DIGEST	0x800
+
 #define ISCSI_PAD_LEN			4
-#define ISCSI_R2T_MAX			16
 #define ISCSI_SG_TABLESIZE		SG_ALL
 #define ISCSI_TCP_MAX_CMD_LEN		16
 
+struct crypto_hash;
 struct socket;
 
 /* Socket connection recieve helper */
@@ -84,9 +83,6 @@ struct iscsi_tcp_conn {
 	/* iSCSI connection-wide sequencing */
 	int			hdr_size;	/* PDU header size */
 
-	struct crypto_tfm	*rx_tfm;	/* CRC32C (Rx) */
-	struct crypto_tfm	*data_rx_tfm;	/* CRC32C (Rx) for data */
-
 	/* control data */
 	struct iscsi_tcp_recv	in;		/* TCP receive context */
 	int			in_progress;	/* connection state machine */
@@ -96,9 +92,9 @@ struct iscsi_tcp_conn {
 	void			(*old_state_change)(struct sock *);
 	void			(*old_write_space)(struct sock *);
 
-	/* xmit */
-	struct crypto_tfm	*tx_tfm;	/* CRC32C (Tx) */
-	struct crypto_tfm	*data_tx_tfm;	/* CRC32C (Tx) for data */
+	/* data and header digests */
+	struct hash_desc	tx_hash;	/* CRC32C (Tx) */
+	struct hash_desc	rx_hash;	/* CRC32C (Rx) */
 
 	/* MIB custom statistics */
 	uint32_t		sendpage_failures_cnt;
@@ -157,19 +153,15 @@ struct iscsi_tcp_cmd_task {
 	struct scatterlist	*bad_sg;		/* assert statement */
 	int			sg_count;		/* SG's to process  */
 	uint32_t		exp_r2tsn;
-	int			r2t_data_count;		/* R2T Data-Out bytes */
 	int			data_offset;
 	struct iscsi_r2t_info	*r2t;			/* in progress R2T    */
 	struct iscsi_queue	r2tpool;
 	struct kfifo		*r2tqueue;
 	struct iscsi_r2t_info	**r2ts;
-	uint32_t		datadigest;		/* for recover digest */
 	int			digest_count;
 	uint32_t		immdigest;		/* for imm data */
 	struct iscsi_buf	immbuf;			/* for imm data digest */
-	struct iscsi_data_task	*dtask;		/* data task in progress*/
 	struct iscsi_data_task	unsol_dtask;	/* unsol data task */
-	int			digest_offset;	/* for partial buff digest */
 };
 
 #endif /* ISCSI_H */
diff --git a/drivers/scsi/libiscsi.c b/drivers/scsi/libiscsi.c
index 5884cd2..c542d0e 100644
--- a/drivers/scsi/libiscsi.c
+++ b/drivers/scsi/libiscsi.c
@@ -68,8 +68,7 @@ iscsi_check_assign_cmdsn(struct iscsi_session *session, struct iscsi_nopin *hdr)
 EXPORT_SYMBOL_GPL(iscsi_check_assign_cmdsn);
 
 void iscsi_prep_unsolicit_data_pdu(struct iscsi_cmd_task *ctask,
-				   struct iscsi_data *hdr,
-				   int transport_data_cnt)
+				   struct iscsi_data *hdr)
 {
 	struct iscsi_conn *conn = ctask->conn;
 
@@ -82,14 +81,12 @@ void iscsi_prep_unsolicit_data_pdu(struct iscsi_cmd_task *ctask,
 
 	hdr->itt = ctask->hdr->itt;
 	hdr->exp_statsn = cpu_to_be32(conn->exp_statsn);
-
-	hdr->offset = cpu_to_be32(ctask->total_length -
-				  transport_data_cnt -
-				  ctask->unsol_count);
+	hdr->offset = cpu_to_be32(ctask->unsol_offset);
 
 	if (ctask->unsol_count > conn->max_xmit_dlength) {
 		hton24(hdr->dlength, conn->max_xmit_dlength);
 		ctask->data_count = conn->max_xmit_dlength;
+		ctask->unsol_offset += ctask->data_count;
 		hdr->flags = 0;
 	} else {
 		hton24(hdr->dlength, ctask->unsol_count);
@@ -125,6 +122,7 @@ static void iscsi_prep_scsi_cmd_pdu(struct iscsi_cmd_task *ctask)
         memcpy(hdr->cdb, sc->cmnd, sc->cmd_len);
         memset(&hdr->cdb[sc->cmd_len], 0, MAX_COMMAND_SIZE - sc->cmd_len);
 
+	ctask->data_count = 0;
 	if (sc->sc_data_direction == DMA_TO_DEVICE) {
 		hdr->flags |= ISCSI_FLAG_CMD_WRITE;
 		/*
@@ -143,6 +141,7 @@ static void iscsi_prep_scsi_cmd_pdu(struct iscsi_cmd_task *ctask)
 		 */
 		ctask->imm_count = 0;
 		ctask->unsol_count = 0;
+		ctask->unsol_offset = 0;
 		ctask->unsol_datasn = 0;
 
 		if (session->imm_data_en) {
@@ -156,9 +155,12 @@ static void iscsi_prep_scsi_cmd_pdu(struct iscsi_cmd_task *ctask)
 		} else
 			zero_data(ctask->hdr->dlength);
 
-		if (!session->initial_r2t_en)
+		if (!session->initial_r2t_en) {
 			ctask->unsol_count = min(session->first_burst,
 				ctask->total_length) - ctask->imm_count;
+			ctask->unsol_offset = ctask->imm_count;
+		}
+
 		if (!ctask->unsol_count)
 			/* No unsolicit Data-Out's */
 			ctask->hdr->flags |= ISCSI_FLAG_CMD_FINAL;
@@ -177,25 +179,51 @@ EXPORT_SYMBOL_GPL(iscsi_prep_scsi_cmd_pdu);
 
 /**
  * iscsi_complete_command - return command back to scsi-ml
- * @session: iscsi session
  * @ctask: iscsi cmd task
  *
  * Must be called with session lock.
  * This function returns the scsi command to scsi-ml and returns
  * the cmd task to the pool of available cmd tasks.
  */
-static void iscsi_complete_command(struct iscsi_session *session,
-				   struct iscsi_cmd_task *ctask)
+static void iscsi_complete_command(struct iscsi_cmd_task *ctask)
 {
+	struct iscsi_session *session = ctask->conn->session;
 	struct scsi_cmnd *sc = ctask->sc;
 
 	ctask->state = ISCSI_TASK_COMPLETED;
 	ctask->sc = NULL;
+	/* SCSI eh reuses commands to verify us */
+	sc->SCp.ptr = NULL;
 	list_del_init(&ctask->running);
 	__kfifo_put(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
 	sc->scsi_done(sc);
 }
 
+static void __iscsi_get_ctask(struct iscsi_cmd_task *ctask)
+{
+	atomic_inc(&ctask->refcount);
+}
+
+static void iscsi_get_ctask(struct iscsi_cmd_task *ctask)
+{
+	spin_lock_bh(&ctask->conn->session->lock);
+	__iscsi_get_ctask(ctask);
+	spin_unlock_bh(&ctask->conn->session->lock);
+}
+
+static void __iscsi_put_ctask(struct iscsi_cmd_task *ctask)
+{
+	if (atomic_dec_and_test(&ctask->refcount))
+		iscsi_complete_command(ctask);
+}
+
+static void iscsi_put_ctask(struct iscsi_cmd_task *ctask)
+{
+	spin_lock_bh(&ctask->conn->session->lock);
+	__iscsi_put_ctask(ctask);
+	spin_unlock_bh(&ctask->conn->session->lock);
+}
+
 /**
  * iscsi_cmd_rsp - SCSI Command Response processing
  * @conn: iscsi connection
@@ -272,7 +300,7 @@ out:
 		   (long)sc, sc->result, ctask->itt);
 	conn->scsirsp_pdus_cnt++;
 
-	iscsi_complete_command(conn->session, ctask);
+	__iscsi_put_ctask(ctask);
 	return rc;
 }
 
@@ -295,6 +323,30 @@ static void iscsi_tmf_rsp(struct iscsi_conn *conn, struct iscsi_hdr *hdr)
 	wake_up(&conn->ehwait);
 }
 
+static int iscsi_handle_reject(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
+			       char *data, int datalen)
+{
+	struct iscsi_reject *reject = (struct iscsi_reject *)hdr;
+	struct iscsi_hdr rejected_pdu;
+	uint32_t itt;
+
+	conn->exp_statsn = be32_to_cpu(reject->statsn) + 1;
+
+	if (reject->reason == ISCSI_REASON_DATA_DIGEST_ERROR) {
+		if (ntoh24(reject->dlength) > datalen)
+			return ISCSI_ERR_PROTO;
+
+		if (ntoh24(reject->dlength) >= sizeof(struct iscsi_hdr)) {
+			memcpy(&rejected_pdu, data, sizeof(struct iscsi_hdr));
+			itt = rejected_pdu.itt & ISCSI_ITT_MASK;
+			printk(KERN_ERR "itt 0x%x had pdu (op 0x%x) rejected "
+				"due to DataDigest error.\n", itt,
+				rejected_pdu.opcode);
+		}
+	}
+	return 0;
+}
+
 /**
  * __iscsi_complete_pdu - complete pdu
  * @conn: iscsi conn
@@ -336,7 +388,7 @@ int __iscsi_complete_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 			BUG_ON((void*)ctask != ctask->sc->SCp.ptr);
 			if (hdr->flags & ISCSI_FLAG_DATA_STATUS) {
 				conn->scsirsp_pdus_cnt++;
-				iscsi_complete_command(session, ctask);
+				__iscsi_put_ctask(ctask);
 			}
 			break;
 		case ISCSI_OP_R2T:
@@ -406,6 +458,11 @@ int __iscsi_complete_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 			break;
 		}
 	} else if (itt == ISCSI_RESERVED_TAG) {
+		rc = iscsi_check_assign_cmdsn(session,
+					     (struct iscsi_nopin*)hdr);
+		if (rc)
+			goto done;
+
 		switch(opcode) {
 		case ISCSI_OP_NOOP_IN:
 			if (datalen) {
@@ -413,11 +470,6 @@ int __iscsi_complete_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 				break;
 			}
 
-			rc = iscsi_check_assign_cmdsn(session,
-						 (struct iscsi_nopin*)hdr);
-			if (rc)
-				break;
-
 			if (hdr->ttt == ISCSI_RESERVED_TAG)
 				break;
 
@@ -425,7 +477,8 @@ int __iscsi_complete_pdu(struct iscsi_conn *conn, struct iscsi_hdr *hdr,
 				rc = ISCSI_ERR_CONN_FAILED;
 			break;
 		case ISCSI_OP_REJECT:
-			/* we need sth like iscsi_reject_rsp()*/
+			rc = iscsi_handle_reject(conn, hdr, data, datalen);
+			break;
 		case ISCSI_OP_ASYNC_EVENT:
 			conn->exp_statsn = be32_to_cpu(hdr->statsn) + 1;
 			/* we need sth like iscsi_async_event_rsp() */
@@ -561,7 +614,9 @@ static int iscsi_data_xmit(struct iscsi_conn *conn)
 	BUG_ON(conn->ctask && conn->mtask);
 
 	if (conn->ctask) {
+		iscsi_get_ctask(conn->ctask);
 		rc = tt->xmit_cmd_task(conn, conn->ctask);
+		iscsi_put_ctask(conn->ctask);
 		if (rc)
 			goto again;
 		/* done with this in-progress ctask */
@@ -602,12 +657,19 @@ static int iscsi_data_xmit(struct iscsi_conn *conn)
 					 struct iscsi_cmd_task, running);
 		conn->ctask->state = ISCSI_TASK_RUNNING;
 		list_move_tail(conn->xmitqueue.next, &conn->run_list);
+		__iscsi_get_ctask(conn->ctask);
 		spin_unlock_bh(&conn->session->lock);
 
 		rc = tt->xmit_cmd_task(conn, conn->ctask);
 		if (rc)
 			goto again;
+
 		spin_lock_bh(&conn->session->lock);
+		__iscsi_put_ctask(conn->ctask);
+		if (rc) {
+			spin_unlock_bh(&conn->session->lock);
+			goto again;
+		}
 	}
 	spin_unlock_bh(&conn->session->lock);
 	/* done with this ctask */
@@ -657,6 +719,7 @@ enum {
 	FAILURE_SESSION_FAILED,
 	FAILURE_SESSION_FREED,
 	FAILURE_WINDOW_CLOSED,
+	FAILURE_OOM,
 	FAILURE_SESSION_TERMINATE,
 	FAILURE_SESSION_IN_RECOVERY,
 	FAILURE_SESSION_RECOVERY_TIMEOUT,
@@ -672,6 +735,7 @@ int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
 
 	sc->scsi_done = done;
 	sc->result = 0;
+	sc->SCp.ptr = NULL;
 
 	host = sc->device->host;
 	session = iscsi_hostdata(host->hostdata);
@@ -715,10 +779,15 @@ int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
 
 	conn = session->leadconn;
 
-	__kfifo_get(session->cmdpool.queue, (void*)&ctask, sizeof(void*));
+	if (!__kfifo_get(session->cmdpool.queue, (void*)&ctask,
+			 sizeof(void*))) {
+		reason = FAILURE_OOM;
+		goto reject;
+	}
 	sc->SCp.phase = session->age;
 	sc->SCp.ptr = (char *)ctask;
 
+	atomic_set(&ctask->refcount, 1);
 	ctask->state = ISCSI_TASK_PENDING;
 	ctask->mtask = NULL;
 	ctask->conn = conn;
@@ -731,9 +800,10 @@ int iscsi_queuecommand(struct scsi_cmnd *sc, void (*done)(struct scsi_cmnd *))
 
 	list_add_tail(&ctask->running, &conn->xmitqueue);
 	debug_scsi(
-	       "ctask enq [%s cid %d sc %lx itt 0x%x len %d cmdsn %d win %d]\n",
+	       "ctask enq [%s cid %d sc %p cdb 0x%x itt 0x%x len %d cmdsn %d "
+		"win %d]\n",
 		sc->sc_data_direction == DMA_TO_DEVICE ? "write" : "read",
-		conn->id, (long)sc, ctask->itt, sc->request_bufflen,
+		conn->id, sc, sc->cmnd[0], ctask->itt, sc->request_bufflen,
 		session->cmdsn, session->max_cmdsn - session->exp_cmdsn + 1);
 	spin_unlock(&session->lock);
 
@@ -1061,16 +1131,30 @@ static void fail_command(struct iscsi_conn *conn, struct iscsi_cmd_task *ctask,
 
 	sc->result = err;
 	sc->resid = sc->request_bufflen;
-	iscsi_complete_command(conn->session, ctask);
+	/* release ref from queuecommand */
+	__iscsi_put_ctask(ctask);
 }
 
 int iscsi_eh_abort(struct scsi_cmnd *sc)
 {
-	struct iscsi_cmd_task *ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
-	struct iscsi_conn *conn = ctask->conn;
-	struct iscsi_session *session = conn->session;
+	struct iscsi_cmd_task *ctask;
+	struct iscsi_conn *conn;
+	struct iscsi_session *session;
 	int rc;
 
+	/*
+	 * if session was ISCSI_STATE_IN_RECOVERY then we may not have
+	 * got the command.
+	 */
+	if (!sc->SCp.ptr) {
+		debug_scsi("sc never reached iscsi layer or it completed.\n");
+		return SUCCESS;
+	}
+
+	ctask = (struct iscsi_cmd_task *)sc->SCp.ptr;
+	conn = ctask->conn;
+	session = conn->session;
+
 	conn->eh_abort_cnt++;
 	debug_scsi("aborting [sc %p itt 0x%x]\n", sc, ctask->itt);
 
@@ -1520,11 +1604,19 @@ int iscsi_conn_start(struct iscsi_cls_conn *cls_conn)
 	struct iscsi_conn *conn = cls_conn->dd_data;
 	struct iscsi_session *session = conn->session;
 
-	if (session == NULL) {
+	if (!session) {
 		printk(KERN_ERR "iscsi: can't start unbound connection\n");
 		return -EPERM;
 	}
 
+	if ((session->imm_data_en || !session->initial_r2t_en) &&
+	     session->first_burst > session->max_burst) {
+		printk("iscsi: invalid burst lengths: "
+		       "first_burst %d max_burst %d\n",
+		       session->first_burst, session->max_burst);
+		return -EINVAL;
+	}
+
 	spin_lock_bh(&session->lock);
 	conn->c_stage = ISCSI_CONN_STARTED;
 	session->state = ISCSI_STATE_LOGGED_IN;
diff --git a/drivers/scsi/libsas/Kconfig b/drivers/scsi/libsas/Kconfig
new file mode 100644
index 0000000..aafdc92f
--- /dev/null
+++ b/drivers/scsi/libsas/Kconfig
@@ -0,0 +1,39 @@
+#
+# Kernel configuration file for the SAS Class
+#
+# Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+# Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+#
+# This file is licensed under GPLv2.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2 of the
+# License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+
+config SCSI_SAS_LIBSAS
+	tristate "SAS Domain Transport Attributes"
+	depends on SCSI
+	select SCSI_SAS_ATTRS
+	help
+	  This provides transport specific helpers for SAS drivers which
+	  use the domain device construct (like the aic94xxx).
+
+config SCSI_SAS_LIBSAS_DEBUG
+	bool "Compile the SAS Domain Transport Attributes in debug mode"
+	default y
+	depends on SCSI_SAS_LIBSAS
+	help
+		Compiles the SAS Layer in debug mode.  In debug mode, the
+		SAS Layer prints diagnostic and debug messages.
diff --git a/drivers/scsi/libsas/Makefile b/drivers/scsi/libsas/Makefile
new file mode 100644
index 0000000..44d972a
--- /dev/null
+++ b/drivers/scsi/libsas/Makefile
@@ -0,0 +1,36 @@
+#
+# Kernel Makefile for the libsas helpers
+#
+# Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+# Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+#
+# This file is licensed under GPLv2.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; version 2 of the
+# License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+
+ifeq ($(CONFIG_SCSI_SAS_LIBSAS_DEBUG),y)
+	EXTRA_CFLAGS += -DSAS_DEBUG
+endif
+
+obj-$(CONFIG_SCSI_SAS_LIBSAS) += libsas.o
+libsas-y +=  sas_init.o     \
+		sas_phy.o      \
+		sas_port.o     \
+		sas_event.o    \
+		sas_dump.o     \
+		sas_discover.o \
+		sas_expander.o \
+		sas_scsi_host.o
diff --git a/drivers/scsi/libsas/sas_discover.c b/drivers/scsi/libsas/sas_discover.c
new file mode 100644
index 0000000..d977bd4
--- /dev/null
+++ b/drivers/scsi/libsas/sas_discover.c
@@ -0,0 +1,749 @@
+/*
+ * Serial Attached SCSI (SAS) Discover process
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/scatterlist.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_eh.h>
+#include "sas_internal.h"
+
+#include <scsi/scsi_transport.h>
+#include <scsi/scsi_transport_sas.h>
+#include "../scsi_sas_internal.h"
+
+/* ---------- Basic task processing for discovery purposes ---------- */
+
+void sas_init_dev(struct domain_device *dev)
+{
+        INIT_LIST_HEAD(&dev->siblings);
+        INIT_LIST_HEAD(&dev->dev_list_node);
+        switch (dev->dev_type) {
+        case SAS_END_DEV:
+                break;
+        case EDGE_DEV:
+        case FANOUT_DEV:
+                INIT_LIST_HEAD(&dev->ex_dev.children);
+                break;
+        case SATA_DEV:
+        case SATA_PM:
+        case SATA_PM_PORT:
+                INIT_LIST_HEAD(&dev->sata_dev.children);
+                break;
+        default:
+                break;
+        }
+}
+
+static void sas_task_timedout(unsigned long _task)
+{
+	struct sas_task *task = (void *) _task;
+	unsigned long flags;
+
+	spin_lock_irqsave(&task->task_state_lock, flags);
+	if (!(task->task_state_flags & SAS_TASK_STATE_DONE))
+		task->task_state_flags |= SAS_TASK_STATE_ABORTED;
+	spin_unlock_irqrestore(&task->task_state_lock, flags);
+
+	complete(&task->completion);
+}
+
+static void sas_disc_task_done(struct sas_task *task)
+{
+	if (!del_timer(&task->timer))
+		return;
+	complete(&task->completion);
+}
+
+#define SAS_DEV_TIMEOUT 10
+
+/**
+ * sas_execute_task -- Basic task processing for discovery
+ * @task: the task to be executed
+ * @buffer: pointer to buffer to do I/O
+ * @size: size of @buffer
+ * @pci_dma_dir: PCI_DMA_...
+ */
+static int sas_execute_task(struct sas_task *task, void *buffer, int size,
+			    int pci_dma_dir)
+{
+	int res = 0;
+	struct scatterlist *scatter = NULL;
+	struct task_status_struct *ts = &task->task_status;
+	int num_scatter = 0;
+	int retries = 0;
+	struct sas_internal *i =
+		to_sas_internal(task->dev->port->ha->core.shost->transportt);
+
+	if (pci_dma_dir != PCI_DMA_NONE) {
+		scatter = kzalloc(sizeof(*scatter), GFP_KERNEL);
+		if (!scatter)
+			goto out;
+
+		sg_init_one(scatter, buffer, size);
+		num_scatter = 1;
+	}
+
+	task->task_proto = task->dev->tproto;
+	task->scatter = scatter;
+	task->num_scatter = num_scatter;
+	task->total_xfer_len = size;
+	task->data_dir = pci_dma_dir;
+	task->task_done = sas_disc_task_done;
+
+	for (retries = 0; retries < 5; retries++) {
+		task->task_state_flags = SAS_TASK_STATE_PENDING;
+		init_completion(&task->completion);
+
+		task->timer.data = (unsigned long) task;
+		task->timer.function = sas_task_timedout;
+		task->timer.expires = jiffies + SAS_DEV_TIMEOUT*HZ;
+		add_timer(&task->timer);
+
+		res = i->dft->lldd_execute_task(task, 1, GFP_KERNEL);
+		if (res) {
+			del_timer(&task->timer);
+			SAS_DPRINTK("executing SAS discovery task failed:%d\n",
+				    res);
+			goto ex_err;
+		}
+		wait_for_completion(&task->completion);
+		res = -ETASK;
+		if (task->task_state_flags & SAS_TASK_STATE_ABORTED) {
+			int res2;
+			SAS_DPRINTK("task aborted, flags:0x%x\n",
+				    task->task_state_flags);
+			res2 = i->dft->lldd_abort_task(task);
+			SAS_DPRINTK("came back from abort task\n");
+			if (!(task->task_state_flags & SAS_TASK_STATE_DONE)) {
+				if (res2 == TMF_RESP_FUNC_COMPLETE)
+					continue; /* Retry the task */
+				else
+					goto ex_err;
+			}
+		}
+		if (task->task_status.stat == SAM_BUSY ||
+			   task->task_status.stat == SAM_TASK_SET_FULL ||
+			   task->task_status.stat == SAS_QUEUE_FULL) {
+			SAS_DPRINTK("task: q busy, sleeping...\n");
+			schedule_timeout_interruptible(HZ);
+		} else if (task->task_status.stat == SAM_CHECK_COND) {
+			struct scsi_sense_hdr shdr;
+
+			if (!scsi_normalize_sense(ts->buf, ts->buf_valid_size,
+						  &shdr)) {
+				SAS_DPRINTK("couldn't normalize sense\n");
+				continue;
+			}
+			if ((shdr.sense_key == 6 && shdr.asc == 0x29) ||
+			    (shdr.sense_key == 2 && shdr.asc == 4 &&
+			     shdr.ascq == 1)) {
+				SAS_DPRINTK("device %016llx LUN: %016llx "
+					    "powering up or not ready yet, "
+					    "sleeping...\n",
+					    SAS_ADDR(task->dev->sas_addr),
+					    SAS_ADDR(task->ssp_task.LUN));
+
+				schedule_timeout_interruptible(5*HZ);
+			} else if (shdr.sense_key == 1) {
+				res = 0;
+				break;
+			} else if (shdr.sense_key == 5) {
+				break;
+			} else {
+				SAS_DPRINTK("dev %016llx LUN: %016llx "
+					    "sense key:0x%x ASC:0x%x ASCQ:0x%x"
+					    "\n",
+					    SAS_ADDR(task->dev->sas_addr),
+					    SAS_ADDR(task->ssp_task.LUN),
+					    shdr.sense_key,
+					    shdr.asc, shdr.ascq);
+			}
+		} else if (task->task_status.resp != SAS_TASK_COMPLETE ||
+			   task->task_status.stat != SAM_GOOD) {
+			SAS_DPRINTK("task finished with resp:0x%x, "
+				    "stat:0x%x\n",
+				    task->task_status.resp,
+				    task->task_status.stat);
+			goto ex_err;
+		} else {
+			res = 0;
+			break;
+		}
+	}
+ex_err:
+	if (pci_dma_dir != PCI_DMA_NONE)
+		kfree(scatter);
+out:
+	return res;
+}
+
+/* ---------- Domain device discovery ---------- */
+
+/**
+ * sas_get_port_device -- Discover devices which caused port creation
+ * @port: pointer to struct sas_port of interest
+ *
+ * Devices directly attached to a HA port, have no parent.  This is
+ * how we know they are (domain) "root" devices.  All other devices
+ * do, and should have their "parent" pointer set appropriately as
+ * soon as a child device is discovered.
+ */
+static int sas_get_port_device(struct asd_sas_port *port)
+{
+	unsigned long flags;
+	struct asd_sas_phy *phy;
+	struct sas_rphy *rphy;
+	struct domain_device *dev;
+
+	dev = kzalloc(sizeof(*dev), GFP_KERNEL);
+	if (!dev)
+		return -ENOMEM;
+
+	spin_lock_irqsave(&port->phy_list_lock, flags);
+	if (list_empty(&port->phy_list)) {
+		spin_unlock_irqrestore(&port->phy_list_lock, flags);
+		kfree(dev);
+		return -ENODEV;
+	}
+	phy = container_of(port->phy_list.next, struct asd_sas_phy, port_phy_el);
+	spin_lock(&phy->frame_rcvd_lock);
+	memcpy(dev->frame_rcvd, phy->frame_rcvd, min(sizeof(dev->frame_rcvd),
+					     (size_t)phy->frame_rcvd_size));
+	spin_unlock(&phy->frame_rcvd_lock);
+	spin_unlock_irqrestore(&port->phy_list_lock, flags);
+
+	if (dev->frame_rcvd[0] == 0x34 && port->oob_mode == SATA_OOB_MODE) {
+		struct dev_to_host_fis *fis =
+			(struct dev_to_host_fis *) dev->frame_rcvd;
+		if (fis->interrupt_reason == 1 && fis->lbal == 1 &&
+		    fis->byte_count_low==0x69 && fis->byte_count_high == 0x96
+		    && (fis->device & ~0x10) == 0)
+			dev->dev_type = SATA_PM;
+		else
+			dev->dev_type = SATA_DEV;
+		dev->tproto = SATA_PROTO;
+	} else {
+		struct sas_identify_frame *id =
+			(struct sas_identify_frame *) dev->frame_rcvd;
+		dev->dev_type = id->dev_type;
+		dev->iproto = id->initiator_bits;
+		dev->tproto = id->target_bits;
+	}
+
+	sas_init_dev(dev);
+
+	switch (dev->dev_type) {
+	case SAS_END_DEV:
+		rphy = sas_end_device_alloc(port->port);
+		break;
+	case EDGE_DEV:
+		rphy = sas_expander_alloc(port->port,
+					  SAS_EDGE_EXPANDER_DEVICE);
+		break;
+	case FANOUT_DEV:
+		rphy = sas_expander_alloc(port->port,
+					  SAS_FANOUT_EXPANDER_DEVICE);
+		break;
+	case SATA_DEV:
+	default:
+		printk("ERROR: Unidentified device type %d\n", dev->dev_type);
+		rphy = NULL;
+		break;
+	}
+
+	if (!rphy) {
+		kfree(dev);
+		return -ENODEV;
+	}
+	rphy->identify.phy_identifier = phy->phy->identify.phy_identifier;
+	memcpy(dev->sas_addr, port->attached_sas_addr, SAS_ADDR_SIZE);
+	sas_fill_in_rphy(dev, rphy);
+	sas_hash_addr(dev->hashed_sas_addr, dev->sas_addr);
+	port->port_dev = dev;
+	dev->port = port;
+	dev->linkrate = port->linkrate;
+	dev->min_linkrate = port->linkrate;
+	dev->max_linkrate = port->linkrate;
+	dev->pathways = port->num_phys;
+	memset(port->disc.fanout_sas_addr, 0, SAS_ADDR_SIZE);
+	memset(port->disc.eeds_a, 0, SAS_ADDR_SIZE);
+	memset(port->disc.eeds_b, 0, SAS_ADDR_SIZE);
+	port->disc.max_level = 0;
+
+	dev->rphy = rphy;
+	spin_lock(&port->dev_list_lock);
+	list_add_tail(&dev->dev_list_node, &port->dev_list);
+	spin_unlock(&port->dev_list_lock);
+
+	return 0;
+}
+
+/* ---------- Discover and Revalidate ---------- */
+
+/* ---------- SATA ---------- */
+
+static void sas_get_ata_command_set(struct domain_device *dev)
+{
+	struct dev_to_host_fis *fis =
+		(struct dev_to_host_fis *) dev->frame_rcvd;
+
+	if ((fis->sector_count == 1 && /* ATA */
+	     fis->lbal         == 1 &&
+	     fis->lbam         == 0 &&
+	     fis->lbah         == 0 &&
+	     fis->device       == 0)
+	    ||
+	    (fis->sector_count == 0 && /* CE-ATA (mATA) */
+	     fis->lbal         == 0 &&
+	     fis->lbam         == 0xCE &&
+	     fis->lbah         == 0xAA &&
+	     (fis->device & ~0x10) == 0))
+
+		dev->sata_dev.command_set = ATA_COMMAND_SET;
+
+	else if ((fis->interrupt_reason == 1 &&	/* ATAPI */
+		  fis->lbal             == 1 &&
+		  fis->byte_count_low   == 0x14 &&
+		  fis->byte_count_high  == 0xEB &&
+		  (fis->device & ~0x10) == 0))
+
+		dev->sata_dev.command_set = ATAPI_COMMAND_SET;
+
+	else if ((fis->sector_count == 1 && /* SEMB */
+		  fis->lbal         == 1 &&
+		  fis->lbam         == 0x3C &&
+		  fis->lbah         == 0xC3 &&
+		  fis->device       == 0)
+		||
+		 (fis->interrupt_reason == 1 &&	/* SATA PM */
+		  fis->lbal             == 1 &&
+		  fis->byte_count_low   == 0x69 &&
+		  fis->byte_count_high  == 0x96 &&
+		  (fis->device & ~0x10) == 0))
+
+		/* Treat it as a superset? */
+		dev->sata_dev.command_set = ATAPI_COMMAND_SET;
+}
+
+/**
+ * sas_issue_ata_cmd -- Basic SATA command processing for discovery
+ * @dev: the device to send the command to
+ * @command: the command register
+ * @features: the features register
+ * @buffer: pointer to buffer to do I/O
+ * @size: size of @buffer
+ * @pci_dma_dir: PCI_DMA_...
+ */
+static int sas_issue_ata_cmd(struct domain_device *dev, u8 command,
+			     u8 features, void *buffer, int size,
+			     int pci_dma_dir)
+{
+	int res = 0;
+	struct sas_task *task;
+	struct dev_to_host_fis *d2h_fis = (struct dev_to_host_fis *)
+		&dev->frame_rcvd[0];
+
+	res = -ENOMEM;
+	task = sas_alloc_task(GFP_KERNEL);
+	if (!task)
+		goto out;
+
+	task->dev = dev;
+
+	task->ata_task.fis.command = command;
+	task->ata_task.fis.features = features;
+	task->ata_task.fis.device = d2h_fis->device;
+	task->ata_task.retry_count = 1;
+
+	res = sas_execute_task(task, buffer, size, pci_dma_dir);
+
+	sas_free_task(task);
+out:
+	return res;
+}
+
+static void sas_sata_propagate_sas_addr(struct domain_device *dev)
+{
+	unsigned long flags;
+	struct asd_sas_port *port = dev->port;
+	struct asd_sas_phy  *phy;
+
+	BUG_ON(dev->parent);
+
+	memcpy(port->attached_sas_addr, dev->sas_addr, SAS_ADDR_SIZE);
+	spin_lock_irqsave(&port->phy_list_lock, flags);
+	list_for_each_entry(phy, &port->phy_list, port_phy_el)
+		memcpy(phy->attached_sas_addr, dev->sas_addr, SAS_ADDR_SIZE);
+	spin_unlock_irqrestore(&port->phy_list_lock, flags);
+}
+
+#define ATA_IDENTIFY_DEV         0xEC
+#define ATA_IDENTIFY_PACKET_DEV  0xA1
+#define ATA_SET_FEATURES         0xEF
+#define ATA_FEATURE_PUP_STBY_SPIN_UP 0x07
+
+/**
+ * sas_discover_sata_dev -- discover a STP/SATA device (SATA_DEV)
+ * @dev: STP/SATA device of interest (ATA/ATAPI)
+ *
+ * The LLDD has already been notified of this device, so that we can
+ * send FISes to it.  Here we try to get IDENTIFY DEVICE or IDENTIFY
+ * PACKET DEVICE, if ATAPI device, so that the LLDD can fine-tune its
+ * performance for this device.
+ */
+static int sas_discover_sata_dev(struct domain_device *dev)
+{
+	int     res;
+	__le16  *identify_x;
+	u8      command;
+
+	identify_x = kzalloc(512, GFP_KERNEL);
+	if (!identify_x)
+		return -ENOMEM;
+
+	if (dev->sata_dev.command_set == ATA_COMMAND_SET) {
+		dev->sata_dev.identify_device = identify_x;
+		command = ATA_IDENTIFY_DEV;
+	} else {
+		dev->sata_dev.identify_packet_device = identify_x;
+		command = ATA_IDENTIFY_PACKET_DEV;
+	}
+
+	res = sas_issue_ata_cmd(dev, command, 0, identify_x, 512,
+				PCI_DMA_FROMDEVICE);
+	if (res)
+		goto out_err;
+
+	/* lives on the media? */
+	if (le16_to_cpu(identify_x[0]) & 4) {
+		/* incomplete response */
+		SAS_DPRINTK("sending SET FEATURE/PUP_STBY_SPIN_UP to "
+			    "dev %llx\n", SAS_ADDR(dev->sas_addr));
+		if (!le16_to_cpu(identify_x[83] & (1<<6)))
+			goto cont1;
+		res = sas_issue_ata_cmd(dev, ATA_SET_FEATURES,
+					ATA_FEATURE_PUP_STBY_SPIN_UP,
+					NULL, 0, PCI_DMA_NONE);
+		if (res)
+			goto cont1;
+
+		schedule_timeout_interruptible(5*HZ); /* More time? */
+		res = sas_issue_ata_cmd(dev, command, 0, identify_x, 512,
+					PCI_DMA_FROMDEVICE);
+		if (res)
+			goto out_err;
+	}
+cont1:
+	/* Get WWN */
+	if (dev->port->oob_mode != SATA_OOB_MODE) {
+		memcpy(dev->sas_addr, dev->sata_dev.rps_resp.rps.stp_sas_addr,
+		       SAS_ADDR_SIZE);
+	} else if (dev->sata_dev.command_set == ATA_COMMAND_SET &&
+		   (le16_to_cpu(dev->sata_dev.identify_device[108]) & 0xF000)
+		   == 0x5000) {
+		int i;
+
+		for (i = 0; i < 4; i++) {
+			dev->sas_addr[2*i] =
+	     (le16_to_cpu(dev->sata_dev.identify_device[108+i]) & 0xFF00) >> 8;
+			dev->sas_addr[2*i+1] =
+	      le16_to_cpu(dev->sata_dev.identify_device[108+i]) & 0x00FF;
+		}
+	}
+	sas_hash_addr(dev->hashed_sas_addr, dev->sas_addr);
+	if (!dev->parent)
+		sas_sata_propagate_sas_addr(dev);
+
+	/* XXX Hint: register this SATA device with SATL.
+	   When this returns, dev->sata_dev->lu is alive and
+	   present.
+	sas_satl_register_dev(dev);
+	*/
+	return 0;
+out_err:
+	dev->sata_dev.identify_packet_device = NULL;
+	dev->sata_dev.identify_device = NULL;
+	kfree(identify_x);
+	return res;
+}
+
+static int sas_discover_sata_pm(struct domain_device *dev)
+{
+	return -ENODEV;
+}
+
+int sas_notify_lldd_dev_found(struct domain_device *dev)
+{
+	int res = 0;
+	struct sas_ha_struct *sas_ha = dev->port->ha;
+	struct Scsi_Host *shost = sas_ha->core.shost;
+	struct sas_internal *i = to_sas_internal(shost->transportt);
+
+	if (i->dft->lldd_dev_found) {
+		res = i->dft->lldd_dev_found(dev);
+		if (res) {
+			printk("sas: driver on pcidev %s cannot handle "
+			       "device %llx, error:%d\n",
+			       pci_name(sas_ha->pcidev),
+			       SAS_ADDR(dev->sas_addr), res);
+		}
+	}
+	return res;
+}
+
+
+void sas_notify_lldd_dev_gone(struct domain_device *dev)
+{
+	struct sas_ha_struct *sas_ha = dev->port->ha;
+	struct Scsi_Host *shost = sas_ha->core.shost;
+	struct sas_internal *i = to_sas_internal(shost->transportt);
+
+	if (i->dft->lldd_dev_gone)
+		i->dft->lldd_dev_gone(dev);
+}
+
+/* ---------- Common/dispatchers ---------- */
+
+/**
+ * sas_discover_sata -- discover an STP/SATA domain device
+ * @dev: pointer to struct domain_device of interest
+ *
+ * First we notify the LLDD of this device, so we can send frames to
+ * it.  Then depending on the type of device we call the appropriate
+ * discover functions.  Once device discover is done, we notify the
+ * LLDD so that it can fine-tune its parameters for the device, by
+ * removing it and then adding it.  That is, the second time around,
+ * the driver would have certain fields, that it is looking at, set.
+ * Finally we initialize the kobj so that the device can be added to
+ * the system at registration time.  Devices directly attached to a HA
+ * port, have no parents.  All other devices do, and should have their
+ * "parent" pointer set appropriately before calling this function.
+ */
+int sas_discover_sata(struct domain_device *dev)
+{
+	int res;
+
+	sas_get_ata_command_set(dev);
+
+	res = sas_notify_lldd_dev_found(dev);
+	if (res)
+		return res;
+
+	switch (dev->dev_type) {
+	case SATA_DEV:
+		res = sas_discover_sata_dev(dev);
+		break;
+	case SATA_PM:
+		res = sas_discover_sata_pm(dev);
+		break;
+	default:
+		break;
+	}
+
+	sas_notify_lldd_dev_gone(dev);
+	if (!res) {
+		sas_notify_lldd_dev_found(dev);
+	}
+	return res;
+}
+
+/**
+ * sas_discover_end_dev -- discover an end device (SSP, etc)
+ * @end: pointer to domain device of interest
+ *
+ * See comment in sas_discover_sata().
+ */
+int sas_discover_end_dev(struct domain_device *dev)
+{
+	int res;
+
+	res = sas_notify_lldd_dev_found(dev);
+	if (res)
+		return res;
+
+	res = sas_rphy_add(dev->rphy);
+	if (res)
+		goto out_err;
+
+	/* do this to get the end device port attributes which will have
+	 * been scanned in sas_rphy_add */
+	sas_notify_lldd_dev_gone(dev);
+	sas_notify_lldd_dev_found(dev);
+
+	return 0;
+
+out_err:
+	sas_notify_lldd_dev_gone(dev);
+	return res;
+}
+
+/* ---------- Device registration and unregistration ---------- */
+
+static inline void sas_unregister_common_dev(struct domain_device *dev)
+{
+	sas_notify_lldd_dev_gone(dev);
+	if (!dev->parent)
+		dev->port->port_dev = NULL;
+	else
+		list_del_init(&dev->siblings);
+	list_del_init(&dev->dev_list_node);
+}
+
+void sas_unregister_dev(struct domain_device *dev)
+{
+	if (dev->rphy) {
+		sas_remove_children(&dev->rphy->dev);
+		sas_rphy_delete(dev->rphy);
+		dev->rphy = NULL;
+	}
+	if (dev->dev_type == EDGE_DEV || dev->dev_type == FANOUT_DEV) {
+		/* remove the phys and ports, everything else should be gone */
+		kfree(dev->ex_dev.ex_phy);
+		dev->ex_dev.ex_phy = NULL;
+	}
+	sas_unregister_common_dev(dev);
+}
+
+void sas_unregister_domain_devices(struct asd_sas_port *port)
+{
+	struct domain_device *dev, *n;
+
+	list_for_each_entry_safe_reverse(dev,n,&port->dev_list,dev_list_node)
+		sas_unregister_dev(dev);
+
+	port->port->rphy = NULL;
+
+}
+
+/* ---------- Discovery and Revalidation ---------- */
+
+/**
+ * sas_discover_domain -- discover the domain
+ * @port: port to the domain of interest
+ *
+ * NOTE: this process _must_ quit (return) as soon as any connection
+ * errors are encountered.  Connection recovery is done elsewhere.
+ * Discover process only interrogates devices in order to discover the
+ * domain.
+ */
+static void sas_discover_domain(void *data)
+{
+	int error = 0;
+	struct asd_sas_port *port = data;
+
+	sas_begin_event(DISCE_DISCOVER_DOMAIN, &port->disc.disc_event_lock,
+			&port->disc.pending);
+
+	if (port->port_dev)
+		return ;
+	else {
+		error = sas_get_port_device(port);
+		if (error)
+			return;
+	}
+
+	SAS_DPRINTK("DOING DISCOVERY on port %d, pid:%d\n", port->id,
+		    current->pid);
+
+	switch (port->port_dev->dev_type) {
+	case SAS_END_DEV:
+		error = sas_discover_end_dev(port->port_dev);
+		break;
+	case EDGE_DEV:
+	case FANOUT_DEV:
+		error = sas_discover_root_expander(port->port_dev);
+		break;
+	case SATA_DEV:
+	case SATA_PM:
+		error = sas_discover_sata(port->port_dev);
+		break;
+	default:
+		SAS_DPRINTK("unhandled device %d\n", port->port_dev->dev_type);
+		break;
+	}
+
+	if (error) {
+		kfree(port->port_dev); /* not kobject_register-ed yet */
+		port->port_dev = NULL;
+	}
+
+	SAS_DPRINTK("DONE DISCOVERY on port %d, pid:%d, result:%d\n", port->id,
+		    current->pid, error);
+}
+
+static void sas_revalidate_domain(void *data)
+{
+	int res = 0;
+	struct asd_sas_port *port = data;
+
+	sas_begin_event(DISCE_REVALIDATE_DOMAIN, &port->disc.disc_event_lock,
+			&port->disc.pending);
+
+	SAS_DPRINTK("REVALIDATING DOMAIN on port %d, pid:%d\n", port->id,
+		    current->pid);
+	if (port->port_dev)
+		res = sas_ex_revalidate_domain(port->port_dev);
+
+	SAS_DPRINTK("done REVALIDATING DOMAIN on port %d, pid:%d, res 0x%x\n",
+		    port->id, current->pid, res);
+}
+
+/* ---------- Events ---------- */
+
+int sas_discover_event(struct asd_sas_port *port, enum discover_event ev)
+{
+	struct sas_discovery *disc;
+
+	if (!port)
+		return 0;
+	disc = &port->disc;
+
+	BUG_ON(ev >= DISC_NUM_EVENTS);
+
+	sas_queue_event(ev, &disc->disc_event_lock, &disc->pending,
+			&disc->disc_work[ev], port->ha->core.shost);
+
+	return 0;
+}
+
+/**
+ * sas_init_disc -- initialize the discovery struct in the port
+ * @port: pointer to struct port
+ *
+ * Called when the ports are being initialized.
+ */
+void sas_init_disc(struct sas_discovery *disc, struct asd_sas_port *port)
+{
+	int i;
+
+	static void (*sas_event_fns[DISC_NUM_EVENTS])(void *) = {
+		[DISCE_DISCOVER_DOMAIN] = sas_discover_domain,
+		[DISCE_REVALIDATE_DOMAIN] = sas_revalidate_domain,
+	};
+
+	spin_lock_init(&disc->disc_event_lock);
+	disc->pending = 0;
+	for (i = 0; i < DISC_NUM_EVENTS; i++)
+		INIT_WORK(&disc->disc_work[i], sas_event_fns[i], port);
+}
diff --git a/drivers/scsi/libsas/sas_dump.c b/drivers/scsi/libsas/sas_dump.c
new file mode 100644
index 0000000..f1246d2
--- /dev/null
+++ b/drivers/scsi/libsas/sas_dump.c
@@ -0,0 +1,76 @@
+/*
+ * Serial Attached SCSI (SAS) Dump/Debugging routines
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include "sas_dump.h"
+
+#ifdef SAS_DEBUG
+
+static const char *sas_hae_str[] = {
+	[0] = "HAE_RESET",
+};
+
+static const char *sas_porte_str[] = {
+	[0] = "PORTE_BYTES_DMAED",
+	[1] = "PORTE_BROADCAST_RCVD",
+	[2] = "PORTE_LINK_RESET_ERR",
+	[3] = "PORTE_TIMER_EVENT",
+	[4] = "PORTE_HARD_RESET",
+};
+
+static const char *sas_phye_str[] = {
+	[0] = "PHYE_LOSS_OF_SIGNAL",
+	[1] = "PHYE_OOB_DONE",
+	[2] = "PHYE_OOB_ERROR",
+	[3] = "PHYE_SPINUP_HOLD",
+};
+
+void sas_dprint_porte(int phyid, enum port_event pe)
+{
+	SAS_DPRINTK("phy%d: port event: %s\n", phyid, sas_porte_str[pe]);
+}
+void sas_dprint_phye(int phyid, enum phy_event pe)
+{
+	SAS_DPRINTK("phy%d: phy event: %s\n", phyid, sas_phye_str[pe]);
+}
+
+void sas_dprint_hae(struct sas_ha_struct *sas_ha, enum ha_event he)
+{
+	SAS_DPRINTK("ha %s: %s event\n", pci_name(sas_ha->pcidev),
+		    sas_hae_str[he]);
+}
+
+void sas_dump_port(struct asd_sas_port *port)
+{
+	SAS_DPRINTK("port%d: class:0x%x\n", port->id, port->class);
+	SAS_DPRINTK("port%d: sas_addr:%llx\n", port->id,
+		    SAS_ADDR(port->sas_addr));
+	SAS_DPRINTK("port%d: attached_sas_addr:%llx\n", port->id,
+		    SAS_ADDR(port->attached_sas_addr));
+	SAS_DPRINTK("port%d: iproto:0x%x\n", port->id, port->iproto);
+	SAS_DPRINTK("port%d: tproto:0x%x\n", port->id, port->tproto);
+	SAS_DPRINTK("port%d: oob_mode:0x%x\n", port->id, port->oob_mode);
+	SAS_DPRINTK("port%d: num_phys:%d\n", port->id, port->num_phys);
+}
+
+#endif /* SAS_DEBUG */
diff --git a/drivers/scsi/libsas/sas_dump.h b/drivers/scsi/libsas/sas_dump.h
new file mode 100644
index 0000000..47b45d4
--- /dev/null
+++ b/drivers/scsi/libsas/sas_dump.h
@@ -0,0 +1,42 @@
+/*
+ * Serial Attached SCSI (SAS) Dump/Debugging routines header file
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include "sas_internal.h"
+
+#ifdef SAS_DEBUG
+
+void sas_dprint_porte(int phyid, enum port_event pe);
+void sas_dprint_phye(int phyid, enum phy_event pe);
+void sas_dprint_hae(struct sas_ha_struct *sas_ha, enum ha_event he);
+void sas_dump_port(struct asd_sas_port *port);
+
+#else /* SAS_DEBUG */
+
+static inline void sas_dprint_porte(int phyid, enum port_event pe) { }
+static inline void sas_dprint_phye(int phyid, enum phy_event pe) { }
+static inline void sas_dprint_hae(struct sas_ha_struct *sas_ha,
+				  enum ha_event he) { }
+static inline void sas_dump_port(struct asd_sas_port *port) { }
+
+#endif /* SAS_DEBUG */
diff --git a/drivers/scsi/libsas/sas_event.c b/drivers/scsi/libsas/sas_event.c
new file mode 100644
index 0000000..19110ed
--- /dev/null
+++ b/drivers/scsi/libsas/sas_event.c
@@ -0,0 +1,75 @@
+/*
+ * Serial Attached SCSI (SAS) Event processing
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <scsi/scsi_host.h>
+#include "sas_internal.h"
+#include "sas_dump.h"
+
+static void notify_ha_event(struct sas_ha_struct *sas_ha, enum ha_event event)
+{
+	BUG_ON(event >= HA_NUM_EVENTS);
+
+	sas_queue_event(event, &sas_ha->event_lock, &sas_ha->pending,
+			&sas_ha->ha_events[event], sas_ha->core.shost);
+}
+
+static void notify_port_event(struct asd_sas_phy *phy, enum port_event event)
+{
+	struct sas_ha_struct *ha = phy->ha;
+
+	BUG_ON(event >= PORT_NUM_EVENTS);
+
+	sas_queue_event(event, &ha->event_lock, &phy->port_events_pending,
+			&phy->port_events[event], ha->core.shost);
+}
+
+static void notify_phy_event(struct asd_sas_phy *phy, enum phy_event event)
+{
+	struct sas_ha_struct *ha = phy->ha;
+
+	BUG_ON(event >= PHY_NUM_EVENTS);
+
+	sas_queue_event(event, &ha->event_lock, &phy->phy_events_pending,
+			&phy->phy_events[event], ha->core.shost);
+}
+
+int sas_init_events(struct sas_ha_struct *sas_ha)
+{
+	static void (*sas_ha_event_fns[HA_NUM_EVENTS])(void *) = {
+		[HAE_RESET] = sas_hae_reset,
+	};
+
+	int i;
+
+	spin_lock_init(&sas_ha->event_lock);
+
+	for (i = 0; i < HA_NUM_EVENTS; i++)
+		INIT_WORK(&sas_ha->ha_events[i], sas_ha_event_fns[i], sas_ha);
+
+	sas_ha->notify_ha_event = notify_ha_event;
+	sas_ha->notify_port_event = notify_port_event;
+	sas_ha->notify_phy_event = notify_phy_event;
+
+	return 0;
+}
diff --git a/drivers/scsi/libsas/sas_expander.c b/drivers/scsi/libsas/sas_expander.c
new file mode 100644
index 0000000..30b8014
--- /dev/null
+++ b/drivers/scsi/libsas/sas_expander.c
@@ -0,0 +1,1855 @@
+/*
+ * Serial Attached SCSI (SAS) Expander discovery and configuration
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include <linux/pci.h>
+#include <linux/scatterlist.h>
+
+#include "sas_internal.h"
+
+#include <scsi/scsi_transport.h>
+#include <scsi/scsi_transport_sas.h>
+#include "../scsi_sas_internal.h"
+
+static int sas_discover_expander(struct domain_device *dev);
+static int sas_configure_routing(struct domain_device *dev, u8 *sas_addr);
+static int sas_configure_phy(struct domain_device *dev, int phy_id,
+			     u8 *sas_addr, int include);
+static int sas_disable_routing(struct domain_device *dev,  u8 *sas_addr);
+
+#if 0
+/* FIXME: smp needs to migrate into the sas class */
+static ssize_t smp_portal_read(struct kobject *, char *, loff_t, size_t);
+static ssize_t smp_portal_write(struct kobject *, char *, loff_t, size_t);
+#endif
+
+/* ---------- SMP task management ---------- */
+
+static void smp_task_timedout(unsigned long _task)
+{
+	struct sas_task *task = (void *) _task;
+	unsigned long flags;
+
+	spin_lock_irqsave(&task->task_state_lock, flags);
+	if (!(task->task_state_flags & SAS_TASK_STATE_DONE))
+		task->task_state_flags |= SAS_TASK_STATE_ABORTED;
+	spin_unlock_irqrestore(&task->task_state_lock, flags);
+
+	complete(&task->completion);
+}
+
+static void smp_task_done(struct sas_task *task)
+{
+	if (!del_timer(&task->timer))
+		return;
+	complete(&task->completion);
+}
+
+/* Give it some long enough timeout. In seconds. */
+#define SMP_TIMEOUT 10
+
+static int smp_execute_task(struct domain_device *dev, void *req, int req_size,
+			    void *resp, int resp_size)
+{
+	int res;
+	struct sas_task *task = sas_alloc_task(GFP_KERNEL);
+	struct sas_internal *i =
+		to_sas_internal(dev->port->ha->core.shost->transportt);
+
+	if (!task)
+		return -ENOMEM;
+
+	task->dev = dev;
+	task->task_proto = dev->tproto;
+	sg_init_one(&task->smp_task.smp_req, req, req_size);
+	sg_init_one(&task->smp_task.smp_resp, resp, resp_size);
+
+	task->task_done = smp_task_done;
+
+	task->timer.data = (unsigned long) task;
+	task->timer.function = smp_task_timedout;
+	task->timer.expires = jiffies + SMP_TIMEOUT*HZ;
+	add_timer(&task->timer);
+
+	res = i->dft->lldd_execute_task(task, 1, GFP_KERNEL);
+
+	if (res) {
+		del_timer(&task->timer);
+		SAS_DPRINTK("executing SMP task failed:%d\n", res);
+		goto ex_err;
+	}
+
+	wait_for_completion(&task->completion);
+	res = -ETASK;
+	if ((task->task_state_flags & SAS_TASK_STATE_ABORTED)) {
+		SAS_DPRINTK("smp task timed out or aborted\n");
+		i->dft->lldd_abort_task(task);
+		if (!(task->task_state_flags & SAS_TASK_STATE_DONE)) {
+			SAS_DPRINTK("SMP task aborted and not done\n");
+			goto ex_err;
+		}
+	}
+	if (task->task_status.resp == SAS_TASK_COMPLETE &&
+	    task->task_status.stat == SAM_GOOD)
+		res = 0;
+	else
+		SAS_DPRINTK("%s: task to dev %016llx response: 0x%x "
+			    "status 0x%x\n", __FUNCTION__,
+			    SAS_ADDR(dev->sas_addr),
+			    task->task_status.resp,
+			    task->task_status.stat);
+ex_err:
+	sas_free_task(task);
+	return res;
+}
+
+/* ---------- Allocations ---------- */
+
+static inline void *alloc_smp_req(int size)
+{
+	u8 *p = kzalloc(size, GFP_KERNEL);
+	if (p)
+		p[0] = SMP_REQUEST;
+	return p;
+}
+
+static inline void *alloc_smp_resp(int size)
+{
+	return kzalloc(size, GFP_KERNEL);
+}
+
+/* ---------- Expander configuration ---------- */
+
+static void sas_set_ex_phy(struct domain_device *dev, int phy_id,
+			   void *disc_resp)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	struct ex_phy *phy = &ex->ex_phy[phy_id];
+	struct smp_resp *resp = disc_resp;
+	struct discover_resp *dr = &resp->disc;
+	struct sas_rphy *rphy = dev->rphy;
+	int rediscover = (phy->phy != NULL);
+
+	if (!rediscover) {
+		phy->phy = sas_phy_alloc(&rphy->dev, phy_id);
+
+		/* FIXME: error_handling */
+		BUG_ON(!phy->phy);
+	}
+
+	switch (resp->result) {
+	case SMP_RESP_PHY_VACANT:
+		phy->phy_state = PHY_VACANT;
+		return;
+	default:
+		phy->phy_state = PHY_NOT_PRESENT;
+		return;
+	case SMP_RESP_FUNC_ACC:
+		phy->phy_state = PHY_EMPTY; /* do not know yet */
+		break;
+	}
+
+	phy->phy_id = phy_id;
+	phy->attached_dev_type = dr->attached_dev_type;
+	phy->linkrate = dr->linkrate;
+	phy->attached_sata_host = dr->attached_sata_host;
+	phy->attached_sata_dev  = dr->attached_sata_dev;
+	phy->attached_sata_ps   = dr->attached_sata_ps;
+	phy->attached_iproto = dr->iproto << 1;
+	phy->attached_tproto = dr->tproto << 1;
+	memcpy(phy->attached_sas_addr, dr->attached_sas_addr, SAS_ADDR_SIZE);
+	phy->attached_phy_id = dr->attached_phy_id;
+	phy->phy_change_count = dr->change_count;
+	phy->routing_attr = dr->routing_attr;
+	phy->virtual = dr->virtual;
+	phy->last_da_index = -1;
+
+	phy->phy->identify.initiator_port_protocols = phy->attached_iproto;
+	phy->phy->identify.target_port_protocols = phy->attached_tproto;
+	phy->phy->identify.phy_identifier = phy_id;
+	phy->phy->minimum_linkrate_hw = dr->hmin_linkrate;
+	phy->phy->maximum_linkrate_hw = dr->hmax_linkrate;
+	phy->phy->minimum_linkrate = dr->pmin_linkrate;
+	phy->phy->maximum_linkrate = dr->pmax_linkrate;
+	phy->phy->negotiated_linkrate = phy->linkrate;
+
+	if (!rediscover)
+		sas_phy_add(phy->phy);
+
+	SAS_DPRINTK("ex %016llx phy%02d:%c attached: %016llx\n",
+		    SAS_ADDR(dev->sas_addr), phy->phy_id,
+		    phy->routing_attr == TABLE_ROUTING ? 'T' :
+		    phy->routing_attr == DIRECT_ROUTING ? 'D' :
+		    phy->routing_attr == SUBTRACTIVE_ROUTING ? 'S' : '?',
+		    SAS_ADDR(phy->attached_sas_addr));
+
+	return;
+}
+
+#define DISCOVER_REQ_SIZE  16
+#define DISCOVER_RESP_SIZE 56
+
+static int sas_ex_phy_discover(struct domain_device *dev, int single)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	int  res = 0;
+	u8   *disc_req;
+	u8   *disc_resp;
+
+	disc_req = alloc_smp_req(DISCOVER_REQ_SIZE);
+	if (!disc_req)
+		return -ENOMEM;
+
+	disc_resp = alloc_smp_req(DISCOVER_RESP_SIZE);
+	if (!disc_resp) {
+		kfree(disc_req);
+		return -ENOMEM;
+	}
+
+	disc_req[1] = SMP_DISCOVER;
+
+	if (0 <= single && single < ex->num_phys) {
+		disc_req[9] = single;
+		res = smp_execute_task(dev, disc_req, DISCOVER_REQ_SIZE,
+				       disc_resp, DISCOVER_RESP_SIZE);
+		if (res)
+			goto out_err;
+		sas_set_ex_phy(dev, single, disc_resp);
+	} else {
+		int i;
+
+		for (i = 0; i < ex->num_phys; i++) {
+			disc_req[9] = i;
+			res = smp_execute_task(dev, disc_req,
+					       DISCOVER_REQ_SIZE, disc_resp,
+					       DISCOVER_RESP_SIZE);
+			if (res)
+				goto out_err;
+			sas_set_ex_phy(dev, i, disc_resp);
+		}
+	}
+out_err:
+	kfree(disc_resp);
+	kfree(disc_req);
+	return res;
+}
+
+static int sas_expander_discover(struct domain_device *dev)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	int res = -ENOMEM;
+
+	ex->ex_phy = kzalloc(sizeof(*ex->ex_phy)*ex->num_phys, GFP_KERNEL);
+	if (!ex->ex_phy)
+		return -ENOMEM;
+
+	res = sas_ex_phy_discover(dev, -1);
+	if (res)
+		goto out_err;
+
+	return 0;
+ out_err:
+	kfree(ex->ex_phy);
+	ex->ex_phy = NULL;
+	return res;
+}
+
+#define MAX_EXPANDER_PHYS 128
+
+static void ex_assign_report_general(struct domain_device *dev,
+					    struct smp_resp *resp)
+{
+	struct report_general_resp *rg = &resp->rg;
+
+	dev->ex_dev.ex_change_count = be16_to_cpu(rg->change_count);
+	dev->ex_dev.max_route_indexes = be16_to_cpu(rg->route_indexes);
+	dev->ex_dev.num_phys = min(rg->num_phys, (u8)MAX_EXPANDER_PHYS);
+	dev->ex_dev.conf_route_table = rg->conf_route_table;
+	dev->ex_dev.configuring = rg->configuring;
+	memcpy(dev->ex_dev.enclosure_logical_id, rg->enclosure_logical_id, 8);
+}
+
+#define RG_REQ_SIZE   8
+#define RG_RESP_SIZE 32
+
+static int sas_ex_general(struct domain_device *dev)
+{
+	u8 *rg_req;
+	struct smp_resp *rg_resp;
+	int res;
+	int i;
+
+	rg_req = alloc_smp_req(RG_REQ_SIZE);
+	if (!rg_req)
+		return -ENOMEM;
+
+	rg_resp = alloc_smp_resp(RG_RESP_SIZE);
+	if (!rg_resp) {
+		kfree(rg_req);
+		return -ENOMEM;
+	}
+
+	rg_req[1] = SMP_REPORT_GENERAL;
+
+	for (i = 0; i < 5; i++) {
+		res = smp_execute_task(dev, rg_req, RG_REQ_SIZE, rg_resp,
+				       RG_RESP_SIZE);
+
+		if (res) {
+			SAS_DPRINTK("RG to ex %016llx failed:0x%x\n",
+				    SAS_ADDR(dev->sas_addr), res);
+			goto out;
+		} else if (rg_resp->result != SMP_RESP_FUNC_ACC) {
+			SAS_DPRINTK("RG:ex %016llx returned SMP result:0x%x\n",
+				    SAS_ADDR(dev->sas_addr), rg_resp->result);
+			res = rg_resp->result;
+			goto out;
+		}
+
+		ex_assign_report_general(dev, rg_resp);
+
+		if (dev->ex_dev.configuring) {
+			SAS_DPRINTK("RG: ex %llx self-configuring...\n",
+				    SAS_ADDR(dev->sas_addr));
+			schedule_timeout_interruptible(5*HZ);
+		} else
+			break;
+	}
+out:
+	kfree(rg_req);
+	kfree(rg_resp);
+	return res;
+}
+
+static void ex_assign_manuf_info(struct domain_device *dev, void
+					*_mi_resp)
+{
+	u8 *mi_resp = _mi_resp;
+	struct sas_rphy *rphy = dev->rphy;
+	struct sas_expander_device *edev = rphy_to_expander_device(rphy);
+
+	memcpy(edev->vendor_id, mi_resp + 12, SAS_EXPANDER_VENDOR_ID_LEN);
+	memcpy(edev->product_id, mi_resp + 20, SAS_EXPANDER_PRODUCT_ID_LEN);
+	memcpy(edev->product_rev, mi_resp + 36,
+	       SAS_EXPANDER_PRODUCT_REV_LEN);
+
+	if (mi_resp[8] & 1) {
+		memcpy(edev->component_vendor_id, mi_resp + 40,
+		       SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN);
+		edev->component_id = mi_resp[48] << 8 | mi_resp[49];
+		edev->component_revision_id = mi_resp[50];
+	}
+}
+
+#define MI_REQ_SIZE   8
+#define MI_RESP_SIZE 64
+
+static int sas_ex_manuf_info(struct domain_device *dev)
+{
+	u8 *mi_req;
+	u8 *mi_resp;
+	int res;
+
+	mi_req = alloc_smp_req(MI_REQ_SIZE);
+	if (!mi_req)
+		return -ENOMEM;
+
+	mi_resp = alloc_smp_resp(MI_RESP_SIZE);
+	if (!mi_resp) {
+		kfree(mi_req);
+		return -ENOMEM;
+	}
+
+	mi_req[1] = SMP_REPORT_MANUF_INFO;
+
+	res = smp_execute_task(dev, mi_req, MI_REQ_SIZE, mi_resp,MI_RESP_SIZE);
+	if (res) {
+		SAS_DPRINTK("MI: ex %016llx failed:0x%x\n",
+			    SAS_ADDR(dev->sas_addr), res);
+		goto out;
+	} else if (mi_resp[2] != SMP_RESP_FUNC_ACC) {
+		SAS_DPRINTK("MI ex %016llx returned SMP result:0x%x\n",
+			    SAS_ADDR(dev->sas_addr), mi_resp[2]);
+		goto out;
+	}
+
+	ex_assign_manuf_info(dev, mi_resp);
+out:
+	kfree(mi_req);
+	kfree(mi_resp);
+	return res;
+}
+
+#define PC_REQ_SIZE  44
+#define PC_RESP_SIZE 8
+
+int sas_smp_phy_control(struct domain_device *dev, int phy_id,
+			enum phy_func phy_func,
+			struct sas_phy_linkrates *rates)
+{
+	u8 *pc_req;
+	u8 *pc_resp;
+	int res;
+
+	pc_req = alloc_smp_req(PC_REQ_SIZE);
+	if (!pc_req)
+		return -ENOMEM;
+
+	pc_resp = alloc_smp_resp(PC_RESP_SIZE);
+	if (!pc_resp) {
+		kfree(pc_req);
+		return -ENOMEM;
+	}
+
+	pc_req[1] = SMP_PHY_CONTROL;
+	pc_req[9] = phy_id;
+	pc_req[10]= phy_func;
+	if (rates) {
+		pc_req[32] = rates->minimum_linkrate << 4;
+		pc_req[33] = rates->maximum_linkrate << 4;
+	}
+
+	res = smp_execute_task(dev, pc_req, PC_REQ_SIZE, pc_resp,PC_RESP_SIZE);
+
+	kfree(pc_resp);
+	kfree(pc_req);
+	return res;
+}
+
+static void sas_ex_disable_phy(struct domain_device *dev, int phy_id)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	struct ex_phy *phy = &ex->ex_phy[phy_id];
+
+	sas_smp_phy_control(dev, phy_id, PHY_FUNC_DISABLE, NULL);
+	phy->linkrate = SAS_PHY_DISABLED;
+}
+
+static void sas_ex_disable_port(struct domain_device *dev, u8 *sas_addr)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	int i;
+
+	for (i = 0; i < ex->num_phys; i++) {
+		struct ex_phy *phy = &ex->ex_phy[i];
+
+		if (phy->phy_state == PHY_VACANT ||
+		    phy->phy_state == PHY_NOT_PRESENT)
+			continue;
+
+		if (SAS_ADDR(phy->attached_sas_addr) == SAS_ADDR(sas_addr))
+			sas_ex_disable_phy(dev, i);
+	}
+}
+
+static int sas_dev_present_in_domain(struct asd_sas_port *port,
+					    u8 *sas_addr)
+{
+	struct domain_device *dev;
+
+	if (SAS_ADDR(port->sas_addr) == SAS_ADDR(sas_addr))
+		return 1;
+	list_for_each_entry(dev, &port->dev_list, dev_list_node) {
+		if (SAS_ADDR(dev->sas_addr) == SAS_ADDR(sas_addr))
+			return 1;
+	}
+	return 0;
+}
+
+#define RPEL_REQ_SIZE	16
+#define RPEL_RESP_SIZE	32
+int sas_smp_get_phy_events(struct sas_phy *phy)
+{
+	int res;
+	struct sas_rphy *rphy = dev_to_rphy(phy->dev.parent);
+	struct domain_device *dev = sas_find_dev_by_rphy(rphy);
+	u8 *req = alloc_smp_req(RPEL_REQ_SIZE);
+	u8 *resp = kzalloc(RPEL_RESP_SIZE, GFP_KERNEL);
+
+	if (!resp)
+		return -ENOMEM;
+
+	req[1] = SMP_REPORT_PHY_ERR_LOG;
+	req[9] = phy->number;
+
+	res = smp_execute_task(dev, req, RPEL_REQ_SIZE,
+			            resp, RPEL_RESP_SIZE);
+
+	if (!res)
+		goto out;
+
+	phy->invalid_dword_count = scsi_to_u32(&resp[12]);
+	phy->running_disparity_error_count = scsi_to_u32(&resp[16]);
+	phy->loss_of_dword_sync_count = scsi_to_u32(&resp[20]);
+	phy->phy_reset_problem_count = scsi_to_u32(&resp[24]);
+
+ out:
+	kfree(resp);
+	return res;
+
+}
+
+#define RPS_REQ_SIZE  16
+#define RPS_RESP_SIZE 60
+
+static int sas_get_report_phy_sata(struct domain_device *dev,
+					  int phy_id,
+					  struct smp_resp *rps_resp)
+{
+	int res;
+	u8 *rps_req = alloc_smp_req(RPS_REQ_SIZE);
+
+	if (!rps_req)
+		return -ENOMEM;
+
+	rps_req[1] = SMP_REPORT_PHY_SATA;
+	rps_req[9] = phy_id;
+
+	res = smp_execute_task(dev, rps_req, RPS_REQ_SIZE,
+			            rps_resp, RPS_RESP_SIZE);
+
+	kfree(rps_req);
+	return 0;
+}
+
+static void sas_ex_get_linkrate(struct domain_device *parent,
+				       struct domain_device *child,
+				       struct ex_phy *parent_phy)
+{
+	struct expander_device *parent_ex = &parent->ex_dev;
+	struct sas_port *port;
+	int i;
+
+	child->pathways = 0;
+
+	port = parent_phy->port;
+
+	for (i = 0; i < parent_ex->num_phys; i++) {
+		struct ex_phy *phy = &parent_ex->ex_phy[i];
+
+		if (phy->phy_state == PHY_VACANT ||
+		    phy->phy_state == PHY_NOT_PRESENT)
+			continue;
+
+		if (SAS_ADDR(phy->attached_sas_addr) ==
+		    SAS_ADDR(child->sas_addr)) {
+
+			child->min_linkrate = min(parent->min_linkrate,
+						  phy->linkrate);
+			child->max_linkrate = max(parent->max_linkrate,
+						  phy->linkrate);
+			child->pathways++;
+			sas_port_add_phy(port, phy->phy);
+		}
+	}
+	child->linkrate = min(parent_phy->linkrate, child->max_linkrate);
+	child->pathways = min(child->pathways, parent->pathways);
+}
+
+static struct domain_device *sas_ex_discover_end_dev(
+	struct domain_device *parent, int phy_id)
+{
+	struct expander_device *parent_ex = &parent->ex_dev;
+	struct ex_phy *phy = &parent_ex->ex_phy[phy_id];
+	struct domain_device *child = NULL;
+	struct sas_rphy *rphy;
+	int res;
+
+	if (phy->attached_sata_host || phy->attached_sata_ps)
+		return NULL;
+
+	child = kzalloc(sizeof(*child), GFP_KERNEL);
+	if (!child)
+		return NULL;
+
+	child->parent = parent;
+	child->port   = parent->port;
+	child->iproto = phy->attached_iproto;
+	memcpy(child->sas_addr, phy->attached_sas_addr, SAS_ADDR_SIZE);
+	sas_hash_addr(child->hashed_sas_addr, child->sas_addr);
+	phy->port = sas_port_alloc(&parent->rphy->dev, phy_id);
+	BUG_ON(!phy->port);
+	/* FIXME: better error handling*/
+	BUG_ON(sas_port_add(phy->port) != 0);
+	sas_ex_get_linkrate(parent, child, phy);
+
+	if ((phy->attached_tproto & SAS_PROTO_STP) || phy->attached_sata_dev) {
+		child->dev_type = SATA_DEV;
+		if (phy->attached_tproto & SAS_PROTO_STP)
+			child->tproto = phy->attached_tproto;
+		if (phy->attached_sata_dev)
+			child->tproto |= SATA_DEV;
+		res = sas_get_report_phy_sata(parent, phy_id,
+					      &child->sata_dev.rps_resp);
+		if (res) {
+			SAS_DPRINTK("report phy sata to %016llx:0x%x returned "
+				    "0x%x\n", SAS_ADDR(parent->sas_addr),
+				    phy_id, res);
+			kfree(child);
+			return NULL;
+		}
+		memcpy(child->frame_rcvd, &child->sata_dev.rps_resp.rps.fis,
+		       sizeof(struct dev_to_host_fis));
+		sas_init_dev(child);
+		res = sas_discover_sata(child);
+		if (res) {
+			SAS_DPRINTK("sas_discover_sata() for device %16llx at "
+				    "%016llx:0x%x returned 0x%x\n",
+				    SAS_ADDR(child->sas_addr),
+				    SAS_ADDR(parent->sas_addr), phy_id, res);
+			kfree(child);
+			return NULL;
+		}
+	} else if (phy->attached_tproto & SAS_PROTO_SSP) {
+		child->dev_type = SAS_END_DEV;
+		rphy = sas_end_device_alloc(phy->port);
+		/* FIXME: error handling */
+		BUG_ON(!rphy);
+		child->tproto = phy->attached_tproto;
+		sas_init_dev(child);
+
+		child->rphy = rphy;
+		sas_fill_in_rphy(child, rphy);
+
+		spin_lock(&parent->port->dev_list_lock);
+		list_add_tail(&child->dev_list_node, &parent->port->dev_list);
+		spin_unlock(&parent->port->dev_list_lock);
+
+		res = sas_discover_end_dev(child);
+		if (res) {
+			SAS_DPRINTK("sas_discover_end_dev() for device %16llx "
+				    "at %016llx:0x%x returned 0x%x\n",
+				    SAS_ADDR(child->sas_addr),
+				    SAS_ADDR(parent->sas_addr), phy_id, res);
+			/* FIXME: this kfrees list elements without removing them */
+			//kfree(child);
+			return NULL;
+		}
+	} else {
+		SAS_DPRINTK("target proto 0x%x at %016llx:0x%x not handled\n",
+			    phy->attached_tproto, SAS_ADDR(parent->sas_addr),
+			    phy_id);
+	}
+
+	list_add_tail(&child->siblings, &parent_ex->children);
+	return child;
+}
+
+static struct domain_device *sas_ex_discover_expander(
+	struct domain_device *parent, int phy_id)
+{
+	struct sas_expander_device *parent_ex = rphy_to_expander_device(parent->rphy);
+	struct ex_phy *phy = &parent->ex_dev.ex_phy[phy_id];
+	struct domain_device *child = NULL;
+	struct sas_rphy *rphy;
+	struct sas_expander_device *edev;
+	struct asd_sas_port *port;
+	int res;
+
+	if (phy->routing_attr == DIRECT_ROUTING) {
+		SAS_DPRINTK("ex %016llx:0x%x:D <--> ex %016llx:0x%x is not "
+			    "allowed\n",
+			    SAS_ADDR(parent->sas_addr), phy_id,
+			    SAS_ADDR(phy->attached_sas_addr),
+			    phy->attached_phy_id);
+		return NULL;
+	}
+	child = kzalloc(sizeof(*child), GFP_KERNEL);
+	if (!child)
+		return NULL;
+
+	phy->port = sas_port_alloc(&parent->rphy->dev, phy_id);
+	/* FIXME: better error handling */
+	BUG_ON(sas_port_add(phy->port) != 0);
+
+
+	switch (phy->attached_dev_type) {
+	case EDGE_DEV:
+		rphy = sas_expander_alloc(phy->port,
+					  SAS_EDGE_EXPANDER_DEVICE);
+		break;
+	case FANOUT_DEV:
+		rphy = sas_expander_alloc(phy->port,
+					  SAS_FANOUT_EXPANDER_DEVICE);
+		break;
+	default:
+		rphy = NULL;	/* shut gcc up */
+		BUG();
+	}
+	port = parent->port;
+	child->rphy = rphy;
+	edev = rphy_to_expander_device(rphy);
+	child->dev_type = phy->attached_dev_type;
+	child->parent = parent;
+	child->port = port;
+	child->iproto = phy->attached_iproto;
+	child->tproto = phy->attached_tproto;
+	memcpy(child->sas_addr, phy->attached_sas_addr, SAS_ADDR_SIZE);
+	sas_hash_addr(child->hashed_sas_addr, child->sas_addr);
+	sas_ex_get_linkrate(parent, child, phy);
+	edev->level = parent_ex->level + 1;
+	parent->port->disc.max_level = max(parent->port->disc.max_level,
+					   edev->level);
+	sas_init_dev(child);
+	sas_fill_in_rphy(child, rphy);
+	sas_rphy_add(rphy);
+
+	spin_lock(&parent->port->dev_list_lock);
+	list_add_tail(&child->dev_list_node, &parent->port->dev_list);
+	spin_unlock(&parent->port->dev_list_lock);
+
+	res = sas_discover_expander(child);
+	if (res) {
+		kfree(child);
+		return NULL;
+	}
+	list_add_tail(&child->siblings, &parent->ex_dev.children);
+	return child;
+}
+
+static int sas_ex_discover_dev(struct domain_device *dev, int phy_id)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	struct ex_phy *ex_phy = &ex->ex_phy[phy_id];
+	struct domain_device *child = NULL;
+	int res = 0;
+
+	/* Phy state */
+	if (ex_phy->linkrate == SAS_SATA_SPINUP_HOLD) {
+		if (!sas_smp_phy_control(dev, phy_id, PHY_FUNC_LINK_RESET, NULL))
+			res = sas_ex_phy_discover(dev, phy_id);
+		if (res)
+			return res;
+	}
+
+	/* Parent and domain coherency */
+	if (!dev->parent && (SAS_ADDR(ex_phy->attached_sas_addr) ==
+			     SAS_ADDR(dev->port->sas_addr))) {
+		sas_add_parent_port(dev, phy_id);
+		return 0;
+	}
+	if (dev->parent && (SAS_ADDR(ex_phy->attached_sas_addr) ==
+			    SAS_ADDR(dev->parent->sas_addr))) {
+		sas_add_parent_port(dev, phy_id);
+		if (ex_phy->routing_attr == TABLE_ROUTING)
+			sas_configure_phy(dev, phy_id, dev->port->sas_addr, 1);
+		return 0;
+	}
+
+	if (sas_dev_present_in_domain(dev->port, ex_phy->attached_sas_addr))
+		sas_ex_disable_port(dev, ex_phy->attached_sas_addr);
+
+	if (ex_phy->attached_dev_type == NO_DEVICE) {
+		if (ex_phy->routing_attr == DIRECT_ROUTING) {
+			memset(ex_phy->attached_sas_addr, 0, SAS_ADDR_SIZE);
+			sas_configure_routing(dev, ex_phy->attached_sas_addr);
+		}
+		return 0;
+	} else if (ex_phy->linkrate == SAS_LINK_RATE_UNKNOWN)
+		return 0;
+
+	if (ex_phy->attached_dev_type != SAS_END_DEV &&
+	    ex_phy->attached_dev_type != FANOUT_DEV &&
+	    ex_phy->attached_dev_type != EDGE_DEV) {
+		SAS_DPRINTK("unknown device type(0x%x) attached to ex %016llx "
+			    "phy 0x%x\n", ex_phy->attached_dev_type,
+			    SAS_ADDR(dev->sas_addr),
+			    phy_id);
+		return 0;
+	}
+
+	res = sas_configure_routing(dev, ex_phy->attached_sas_addr);
+	if (res) {
+		SAS_DPRINTK("configure routing for dev %016llx "
+			    "reported 0x%x. Forgotten\n",
+			    SAS_ADDR(ex_phy->attached_sas_addr), res);
+		sas_disable_routing(dev, ex_phy->attached_sas_addr);
+		return res;
+	}
+
+	switch (ex_phy->attached_dev_type) {
+	case SAS_END_DEV:
+		child = sas_ex_discover_end_dev(dev, phy_id);
+		break;
+	case FANOUT_DEV:
+		if (SAS_ADDR(dev->port->disc.fanout_sas_addr)) {
+			SAS_DPRINTK("second fanout expander %016llx phy 0x%x "
+				    "attached to ex %016llx phy 0x%x\n",
+				    SAS_ADDR(ex_phy->attached_sas_addr),
+				    ex_phy->attached_phy_id,
+				    SAS_ADDR(dev->sas_addr),
+				    phy_id);
+			sas_ex_disable_phy(dev, phy_id);
+			break;
+		} else
+			memcpy(dev->port->disc.fanout_sas_addr,
+			       ex_phy->attached_sas_addr, SAS_ADDR_SIZE);
+		/* fallthrough */
+	case EDGE_DEV:
+		child = sas_ex_discover_expander(dev, phy_id);
+		break;
+	default:
+		break;
+	}
+
+	if (child) {
+		int i;
+
+		for (i = 0; i < ex->num_phys; i++) {
+			if (ex->ex_phy[i].phy_state == PHY_VACANT ||
+			    ex->ex_phy[i].phy_state == PHY_NOT_PRESENT)
+				continue;
+
+			if (SAS_ADDR(ex->ex_phy[i].attached_sas_addr) ==
+			    SAS_ADDR(child->sas_addr))
+				ex->ex_phy[i].phy_state= PHY_DEVICE_DISCOVERED;
+		}
+	}
+
+	return res;
+}
+
+static int sas_find_sub_addr(struct domain_device *dev, u8 *sub_addr)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	int i;
+
+	for (i = 0; i < ex->num_phys; i++) {
+		struct ex_phy *phy = &ex->ex_phy[i];
+
+		if (phy->phy_state == PHY_VACANT ||
+		    phy->phy_state == PHY_NOT_PRESENT)
+			continue;
+
+		if ((phy->attached_dev_type == EDGE_DEV ||
+		     phy->attached_dev_type == FANOUT_DEV) &&
+		    phy->routing_attr == SUBTRACTIVE_ROUTING) {
+
+			memcpy(sub_addr, phy->attached_sas_addr,SAS_ADDR_SIZE);
+
+			return 1;
+		}
+	}
+	return 0;
+}
+
+static int sas_check_level_subtractive_boundary(struct domain_device *dev)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	struct domain_device *child;
+	u8 sub_addr[8] = {0, };
+
+	list_for_each_entry(child, &ex->children, siblings) {
+		if (child->dev_type != EDGE_DEV &&
+		    child->dev_type != FANOUT_DEV)
+			continue;
+		if (sub_addr[0] == 0) {
+			sas_find_sub_addr(child, sub_addr);
+			continue;
+		} else {
+			u8 s2[8];
+
+			if (sas_find_sub_addr(child, s2) &&
+			    (SAS_ADDR(sub_addr) != SAS_ADDR(s2))) {
+
+				SAS_DPRINTK("ex %016llx->%016llx-?->%016llx "
+					    "diverges from subtractive "
+					    "boundary %016llx\n",
+					    SAS_ADDR(dev->sas_addr),
+					    SAS_ADDR(child->sas_addr),
+					    SAS_ADDR(s2),
+					    SAS_ADDR(sub_addr));
+
+				sas_ex_disable_port(child, s2);
+			}
+		}
+	}
+	return 0;
+}
+/**
+ * sas_ex_discover_devices -- discover devices attached to this expander
+ * dev: pointer to the expander domain device
+ * single: if you want to do a single phy, else set to -1;
+ *
+ * Configure this expander for use with its devices and register the
+ * devices of this expander.
+ */
+static int sas_ex_discover_devices(struct domain_device *dev, int single)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	int i = 0, end = ex->num_phys;
+	int res = 0;
+
+	if (0 <= single && single < end) {
+		i = single;
+		end = i+1;
+	}
+
+	for ( ; i < end; i++) {
+		struct ex_phy *ex_phy = &ex->ex_phy[i];
+
+		if (ex_phy->phy_state == PHY_VACANT ||
+		    ex_phy->phy_state == PHY_NOT_PRESENT ||
+		    ex_phy->phy_state == PHY_DEVICE_DISCOVERED)
+			continue;
+
+		switch (ex_phy->linkrate) {
+		case SAS_PHY_DISABLED:
+		case SAS_PHY_RESET_PROBLEM:
+		case SAS_SATA_PORT_SELECTOR:
+			continue;
+		default:
+			res = sas_ex_discover_dev(dev, i);
+			if (res)
+				break;
+			continue;
+		}
+	}
+
+	if (!res)
+		sas_check_level_subtractive_boundary(dev);
+
+	return res;
+}
+
+static int sas_check_ex_subtractive_boundary(struct domain_device *dev)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	int i;
+	u8  *sub_sas_addr = NULL;
+
+	if (dev->dev_type != EDGE_DEV)
+		return 0;
+
+	for (i = 0; i < ex->num_phys; i++) {
+		struct ex_phy *phy = &ex->ex_phy[i];
+
+		if (phy->phy_state == PHY_VACANT ||
+		    phy->phy_state == PHY_NOT_PRESENT)
+			continue;
+
+		if ((phy->attached_dev_type == FANOUT_DEV ||
+		     phy->attached_dev_type == EDGE_DEV) &&
+		    phy->routing_attr == SUBTRACTIVE_ROUTING) {
+
+			if (!sub_sas_addr)
+				sub_sas_addr = &phy->attached_sas_addr[0];
+			else if (SAS_ADDR(sub_sas_addr) !=
+				 SAS_ADDR(phy->attached_sas_addr)) {
+
+				SAS_DPRINTK("ex %016llx phy 0x%x "
+					    "diverges(%016llx) on subtractive "
+					    "boundary(%016llx). Disabled\n",
+					    SAS_ADDR(dev->sas_addr), i,
+					    SAS_ADDR(phy->attached_sas_addr),
+					    SAS_ADDR(sub_sas_addr));
+				sas_ex_disable_phy(dev, i);
+			}
+		}
+	}
+	return 0;
+}
+
+static void sas_print_parent_topology_bug(struct domain_device *child,
+						 struct ex_phy *parent_phy,
+						 struct ex_phy *child_phy)
+{
+	static const char ra_char[] = {
+		[DIRECT_ROUTING] = 'D',
+		[SUBTRACTIVE_ROUTING] = 'S',
+		[TABLE_ROUTING] = 'T',
+	};
+	static const char *ex_type[] = {
+		[EDGE_DEV] = "edge",
+		[FANOUT_DEV] = "fanout",
+	};
+	struct domain_device *parent = child->parent;
+
+	sas_printk("%s ex %016llx phy 0x%x <--> %s ex %016llx phy 0x%x "
+		   "has %c:%c routing link!\n",
+
+		   ex_type[parent->dev_type],
+		   SAS_ADDR(parent->sas_addr),
+		   parent_phy->phy_id,
+
+		   ex_type[child->dev_type],
+		   SAS_ADDR(child->sas_addr),
+		   child_phy->phy_id,
+
+		   ra_char[parent_phy->routing_attr],
+		   ra_char[child_phy->routing_attr]);
+}
+
+static int sas_check_eeds(struct domain_device *child,
+				 struct ex_phy *parent_phy,
+				 struct ex_phy *child_phy)
+{
+	int res = 0;
+	struct domain_device *parent = child->parent;
+
+	if (SAS_ADDR(parent->port->disc.fanout_sas_addr) != 0) {
+		res = -ENODEV;
+		SAS_DPRINTK("edge ex %016llx phy S:0x%x <--> edge ex %016llx "
+			    "phy S:0x%x, while there is a fanout ex %016llx\n",
+			    SAS_ADDR(parent->sas_addr),
+			    parent_phy->phy_id,
+			    SAS_ADDR(child->sas_addr),
+			    child_phy->phy_id,
+			    SAS_ADDR(parent->port->disc.fanout_sas_addr));
+	} else if (SAS_ADDR(parent->port->disc.eeds_a) == 0) {
+		memcpy(parent->port->disc.eeds_a, parent->sas_addr,
+		       SAS_ADDR_SIZE);
+		memcpy(parent->port->disc.eeds_b, child->sas_addr,
+		       SAS_ADDR_SIZE);
+	} else if (((SAS_ADDR(parent->port->disc.eeds_a) ==
+		    SAS_ADDR(parent->sas_addr)) ||
+		   (SAS_ADDR(parent->port->disc.eeds_a) ==
+		    SAS_ADDR(child->sas_addr)))
+		   &&
+		   ((SAS_ADDR(parent->port->disc.eeds_b) ==
+		     SAS_ADDR(parent->sas_addr)) ||
+		    (SAS_ADDR(parent->port->disc.eeds_b) ==
+		     SAS_ADDR(child->sas_addr))))
+		;
+	else {
+		res = -ENODEV;
+		SAS_DPRINTK("edge ex %016llx phy 0x%x <--> edge ex %016llx "
+			    "phy 0x%x link forms a third EEDS!\n",
+			    SAS_ADDR(parent->sas_addr),
+			    parent_phy->phy_id,
+			    SAS_ADDR(child->sas_addr),
+			    child_phy->phy_id);
+	}
+
+	return res;
+}
+
+/* Here we spill over 80 columns.  It is intentional.
+ */
+static int sas_check_parent_topology(struct domain_device *child)
+{
+	struct expander_device *child_ex = &child->ex_dev;
+	struct expander_device *parent_ex;
+	int i;
+	int res = 0;
+
+	if (!child->parent)
+		return 0;
+
+	if (child->parent->dev_type != EDGE_DEV &&
+	    child->parent->dev_type != FANOUT_DEV)
+		return 0;
+
+	parent_ex = &child->parent->ex_dev;
+
+	for (i = 0; i < parent_ex->num_phys; i++) {
+		struct ex_phy *parent_phy = &parent_ex->ex_phy[i];
+		struct ex_phy *child_phy;
+
+		if (parent_phy->phy_state == PHY_VACANT ||
+		    parent_phy->phy_state == PHY_NOT_PRESENT)
+			continue;
+
+		if (SAS_ADDR(parent_phy->attached_sas_addr) != SAS_ADDR(child->sas_addr))
+			continue;
+
+		child_phy = &child_ex->ex_phy[parent_phy->attached_phy_id];
+
+		switch (child->parent->dev_type) {
+		case EDGE_DEV:
+			if (child->dev_type == FANOUT_DEV) {
+				if (parent_phy->routing_attr != SUBTRACTIVE_ROUTING ||
+				    child_phy->routing_attr != TABLE_ROUTING) {
+					sas_print_parent_topology_bug(child, parent_phy, child_phy);
+					res = -ENODEV;
+				}
+			} else if (parent_phy->routing_attr == SUBTRACTIVE_ROUTING) {
+				if (child_phy->routing_attr == SUBTRACTIVE_ROUTING) {
+					res = sas_check_eeds(child, parent_phy, child_phy);
+				} else if (child_phy->routing_attr != TABLE_ROUTING) {
+					sas_print_parent_topology_bug(child, parent_phy, child_phy);
+					res = -ENODEV;
+				}
+			} else if (parent_phy->routing_attr == TABLE_ROUTING &&
+				   child_phy->routing_attr != SUBTRACTIVE_ROUTING) {
+				sas_print_parent_topology_bug(child, parent_phy, child_phy);
+				res = -ENODEV;
+			}
+			break;
+		case FANOUT_DEV:
+			if (parent_phy->routing_attr != TABLE_ROUTING ||
+			    child_phy->routing_attr != SUBTRACTIVE_ROUTING) {
+				sas_print_parent_topology_bug(child, parent_phy, child_phy);
+				res = -ENODEV;
+			}
+			break;
+		default:
+			break;
+		}
+	}
+
+	return res;
+}
+
+#define RRI_REQ_SIZE  16
+#define RRI_RESP_SIZE 44
+
+static int sas_configure_present(struct domain_device *dev, int phy_id,
+				 u8 *sas_addr, int *index, int *present)
+{
+	int i, res = 0;
+	struct expander_device *ex = &dev->ex_dev;
+	struct ex_phy *phy = &ex->ex_phy[phy_id];
+	u8 *rri_req;
+	u8 *rri_resp;
+
+	*present = 0;
+	*index = 0;
+
+	rri_req = alloc_smp_req(RRI_REQ_SIZE);
+	if (!rri_req)
+		return -ENOMEM;
+
+	rri_resp = alloc_smp_resp(RRI_RESP_SIZE);
+	if (!rri_resp) {
+		kfree(rri_req);
+		return -ENOMEM;
+	}
+
+	rri_req[1] = SMP_REPORT_ROUTE_INFO;
+	rri_req[9] = phy_id;
+
+	for (i = 0; i < ex->max_route_indexes ; i++) {
+		*(__be16 *)(rri_req+6) = cpu_to_be16(i);
+		res = smp_execute_task(dev, rri_req, RRI_REQ_SIZE, rri_resp,
+				       RRI_RESP_SIZE);
+		if (res)
+			goto out;
+		res = rri_resp[2];
+		if (res == SMP_RESP_NO_INDEX) {
+			SAS_DPRINTK("overflow of indexes: dev %016llx "
+				    "phy 0x%x index 0x%x\n",
+				    SAS_ADDR(dev->sas_addr), phy_id, i);
+			goto out;
+		} else if (res != SMP_RESP_FUNC_ACC) {
+			SAS_DPRINTK("%s: dev %016llx phy 0x%x index 0x%x "
+				    "result 0x%x\n", __FUNCTION__,
+				    SAS_ADDR(dev->sas_addr), phy_id, i, res);
+			goto out;
+		}
+		if (SAS_ADDR(sas_addr) != 0) {
+			if (SAS_ADDR(rri_resp+16) == SAS_ADDR(sas_addr)) {
+				*index = i;
+				if ((rri_resp[12] & 0x80) == 0x80)
+					*present = 0;
+				else
+					*present = 1;
+				goto out;
+			} else if (SAS_ADDR(rri_resp+16) == 0) {
+				*index = i;
+				*present = 0;
+				goto out;
+			}
+		} else if (SAS_ADDR(rri_resp+16) == 0 &&
+			   phy->last_da_index < i) {
+			phy->last_da_index = i;
+			*index = i;
+			*present = 0;
+			goto out;
+		}
+	}
+	res = -1;
+out:
+	kfree(rri_req);
+	kfree(rri_resp);
+	return res;
+}
+
+#define CRI_REQ_SIZE  44
+#define CRI_RESP_SIZE  8
+
+static int sas_configure_set(struct domain_device *dev, int phy_id,
+			     u8 *sas_addr, int index, int include)
+{
+	int res;
+	u8 *cri_req;
+	u8 *cri_resp;
+
+	cri_req = alloc_smp_req(CRI_REQ_SIZE);
+	if (!cri_req)
+		return -ENOMEM;
+
+	cri_resp = alloc_smp_resp(CRI_RESP_SIZE);
+	if (!cri_resp) {
+		kfree(cri_req);
+		return -ENOMEM;
+	}
+
+	cri_req[1] = SMP_CONF_ROUTE_INFO;
+	*(__be16 *)(cri_req+6) = cpu_to_be16(index);
+	cri_req[9] = phy_id;
+	if (SAS_ADDR(sas_addr) == 0 || !include)
+		cri_req[12] |= 0x80;
+	memcpy(cri_req+16, sas_addr, SAS_ADDR_SIZE);
+
+	res = smp_execute_task(dev, cri_req, CRI_REQ_SIZE, cri_resp,
+			       CRI_RESP_SIZE);
+	if (res)
+		goto out;
+	res = cri_resp[2];
+	if (res == SMP_RESP_NO_INDEX) {
+		SAS_DPRINTK("overflow of indexes: dev %016llx phy 0x%x "
+			    "index 0x%x\n",
+			    SAS_ADDR(dev->sas_addr), phy_id, index);
+	}
+out:
+	kfree(cri_req);
+	kfree(cri_resp);
+	return res;
+}
+
+static int sas_configure_phy(struct domain_device *dev, int phy_id,
+				    u8 *sas_addr, int include)
+{
+	int index;
+	int present;
+	int res;
+
+	res = sas_configure_present(dev, phy_id, sas_addr, &index, &present);
+	if (res)
+		return res;
+	if (include ^ present)
+		return sas_configure_set(dev, phy_id, sas_addr, index,include);
+
+	return res;
+}
+
+/**
+ * sas_configure_parent -- configure routing table of parent
+ * parent: parent expander
+ * child: child expander
+ * sas_addr: SAS port identifier of device directly attached to child
+ */
+static int sas_configure_parent(struct domain_device *parent,
+				struct domain_device *child,
+				u8 *sas_addr, int include)
+{
+	struct expander_device *ex_parent = &parent->ex_dev;
+	int res = 0;
+	int i;
+
+	if (parent->parent) {
+		res = sas_configure_parent(parent->parent, parent, sas_addr,
+					   include);
+		if (res)
+			return res;
+	}
+
+	if (ex_parent->conf_route_table == 0) {
+		SAS_DPRINTK("ex %016llx has self-configuring routing table\n",
+			    SAS_ADDR(parent->sas_addr));
+		return 0;
+	}
+
+	for (i = 0; i < ex_parent->num_phys; i++) {
+		struct ex_phy *phy = &ex_parent->ex_phy[i];
+
+		if ((phy->routing_attr == TABLE_ROUTING) &&
+		    (SAS_ADDR(phy->attached_sas_addr) ==
+		     SAS_ADDR(child->sas_addr))) {
+			res = sas_configure_phy(parent, i, sas_addr, include);
+			if (res)
+				return res;
+		}
+	}
+
+	return res;
+}
+
+/**
+ * sas_configure_routing -- configure routing
+ * dev: expander device
+ * sas_addr: port identifier of device directly attached to the expander device
+ */
+static int sas_configure_routing(struct domain_device *dev, u8 *sas_addr)
+{
+	if (dev->parent)
+		return sas_configure_parent(dev->parent, dev, sas_addr, 1);
+	return 0;
+}
+
+static int sas_disable_routing(struct domain_device *dev,  u8 *sas_addr)
+{
+	if (dev->parent)
+		return sas_configure_parent(dev->parent, dev, sas_addr, 0);
+	return 0;
+}
+
+#if 0
+#define SMP_BIN_ATTR_NAME "smp_portal"
+
+static void sas_ex_smp_hook(struct domain_device *dev)
+{
+	struct expander_device *ex_dev = &dev->ex_dev;
+	struct bin_attribute *bin_attr = &ex_dev->smp_bin_attr;
+
+	memset(bin_attr, 0, sizeof(*bin_attr));
+
+	bin_attr->attr.name = SMP_BIN_ATTR_NAME;
+	bin_attr->attr.owner = THIS_MODULE;
+	bin_attr->attr.mode = 0600;
+
+	bin_attr->size = 0;
+	bin_attr->private = NULL;
+	bin_attr->read = smp_portal_read;
+	bin_attr->write= smp_portal_write;
+	bin_attr->mmap = NULL;
+
+	ex_dev->smp_portal_pid = -1;
+	init_MUTEX(&ex_dev->smp_sema);
+}
+#endif
+
+/**
+ * sas_discover_expander -- expander discovery
+ * @ex: pointer to expander domain device
+ *
+ * See comment in sas_discover_sata().
+ */
+static int sas_discover_expander(struct domain_device *dev)
+{
+	int res;
+
+	res = sas_notify_lldd_dev_found(dev);
+	if (res)
+		return res;
+
+	res = sas_ex_general(dev);
+	if (res)
+		goto out_err;
+	res = sas_ex_manuf_info(dev);
+	if (res)
+		goto out_err;
+
+	res = sas_expander_discover(dev);
+	if (res) {
+		SAS_DPRINTK("expander %016llx discovery failed(0x%x)\n",
+			    SAS_ADDR(dev->sas_addr), res);
+		goto out_err;
+	}
+
+	sas_check_ex_subtractive_boundary(dev);
+	res = sas_check_parent_topology(dev);
+	if (res)
+		goto out_err;
+	return 0;
+out_err:
+	sas_notify_lldd_dev_gone(dev);
+	return res;
+}
+
+static int sas_ex_level_discovery(struct asd_sas_port *port, const int level)
+{
+	int res = 0;
+	struct domain_device *dev;
+
+	list_for_each_entry(dev, &port->dev_list, dev_list_node) {
+		if (dev->dev_type == EDGE_DEV ||
+		    dev->dev_type == FANOUT_DEV) {
+			struct sas_expander_device *ex =
+				rphy_to_expander_device(dev->rphy);
+
+			if (level == ex->level)
+				res = sas_ex_discover_devices(dev, -1);
+			else if (level > 0)
+				res = sas_ex_discover_devices(port->port_dev, -1);
+
+		}
+	}
+
+	return res;
+}
+
+static int sas_ex_bfs_disc(struct asd_sas_port *port)
+{
+	int res;
+	int level;
+
+	do {
+		level = port->disc.max_level;
+		res = sas_ex_level_discovery(port, level);
+		mb();
+	} while (level < port->disc.max_level);
+
+	return res;
+}
+
+int sas_discover_root_expander(struct domain_device *dev)
+{
+	int res;
+	struct sas_expander_device *ex = rphy_to_expander_device(dev->rphy);
+
+	sas_rphy_add(dev->rphy);
+
+	ex->level = dev->port->disc.max_level; /* 0 */
+	res = sas_discover_expander(dev);
+	if (!res)
+		sas_ex_bfs_disc(dev->port);
+
+	return res;
+}
+
+/* ---------- Domain revalidation ---------- */
+
+static int sas_get_phy_discover(struct domain_device *dev,
+				int phy_id, struct smp_resp *disc_resp)
+{
+	int res;
+	u8 *disc_req;
+
+	disc_req = alloc_smp_req(DISCOVER_REQ_SIZE);
+	if (!disc_req)
+		return -ENOMEM;
+
+	disc_req[1] = SMP_DISCOVER;
+	disc_req[9] = phy_id;
+
+	res = smp_execute_task(dev, disc_req, DISCOVER_REQ_SIZE,
+			       disc_resp, DISCOVER_RESP_SIZE);
+	if (res)
+		goto out;
+	else if (disc_resp->result != SMP_RESP_FUNC_ACC) {
+		res = disc_resp->result;
+		goto out;
+	}
+out:
+	kfree(disc_req);
+	return res;
+}
+
+static int sas_get_phy_change_count(struct domain_device *dev,
+				    int phy_id, int *pcc)
+{
+	int res;
+	struct smp_resp *disc_resp;
+
+	disc_resp = alloc_smp_resp(DISCOVER_RESP_SIZE);
+	if (!disc_resp)
+		return -ENOMEM;
+
+	res = sas_get_phy_discover(dev, phy_id, disc_resp);
+	if (!res)
+		*pcc = disc_resp->disc.change_count;
+
+	kfree(disc_resp);
+	return res;
+}
+
+static int sas_get_phy_attached_sas_addr(struct domain_device *dev,
+					 int phy_id, u8 *attached_sas_addr)
+{
+	int res;
+	struct smp_resp *disc_resp;
+	struct discover_resp *dr;
+
+	disc_resp = alloc_smp_resp(DISCOVER_RESP_SIZE);
+	if (!disc_resp)
+		return -ENOMEM;
+	dr = &disc_resp->disc;
+
+	res = sas_get_phy_discover(dev, phy_id, disc_resp);
+	if (!res) {
+		memcpy(attached_sas_addr,disc_resp->disc.attached_sas_addr,8);
+		if (dr->attached_dev_type == 0)
+			memset(attached_sas_addr, 0, 8);
+	}
+	kfree(disc_resp);
+	return res;
+}
+
+static int sas_find_bcast_phy(struct domain_device *dev, int *phy_id,
+			      int from_phy)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	int res = 0;
+	int i;
+
+	for (i = from_phy; i < ex->num_phys; i++) {
+		int phy_change_count = 0;
+
+		res = sas_get_phy_change_count(dev, i, &phy_change_count);
+		if (res)
+			goto out;
+		else if (phy_change_count != ex->ex_phy[i].phy_change_count) {
+			ex->ex_phy[i].phy_change_count = phy_change_count;
+			*phy_id = i;
+			return 0;
+		}
+	}
+out:
+	return res;
+}
+
+static int sas_get_ex_change_count(struct domain_device *dev, int *ecc)
+{
+	int res;
+	u8  *rg_req;
+	struct smp_resp  *rg_resp;
+
+	rg_req = alloc_smp_req(RG_REQ_SIZE);
+	if (!rg_req)
+		return -ENOMEM;
+
+	rg_resp = alloc_smp_resp(RG_RESP_SIZE);
+	if (!rg_resp) {
+		kfree(rg_req);
+		return -ENOMEM;
+	}
+
+	rg_req[1] = SMP_REPORT_GENERAL;
+
+	res = smp_execute_task(dev, rg_req, RG_REQ_SIZE, rg_resp,
+			       RG_RESP_SIZE);
+	if (res)
+		goto out;
+	if (rg_resp->result != SMP_RESP_FUNC_ACC) {
+		res = rg_resp->result;
+		goto out;
+	}
+
+	*ecc = be16_to_cpu(rg_resp->rg.change_count);
+out:
+	kfree(rg_resp);
+	kfree(rg_req);
+	return res;
+}
+
+static int sas_find_bcast_dev(struct domain_device *dev,
+			      struct domain_device **src_dev)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	int ex_change_count = -1;
+	int res;
+
+	res = sas_get_ex_change_count(dev, &ex_change_count);
+	if (res)
+		goto out;
+	if (ex_change_count != -1 &&
+	    ex_change_count != ex->ex_change_count) {
+		*src_dev = dev;
+		ex->ex_change_count = ex_change_count;
+	} else {
+		struct domain_device *ch;
+
+		list_for_each_entry(ch, &ex->children, siblings) {
+			if (ch->dev_type == EDGE_DEV ||
+			    ch->dev_type == FANOUT_DEV) {
+				res = sas_find_bcast_dev(ch, src_dev);
+				if (src_dev)
+					return res;
+			}
+		}
+	}
+out:
+	return res;
+}
+
+static void sas_unregister_ex_tree(struct domain_device *dev)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	struct domain_device *child, *n;
+
+	list_for_each_entry_safe(child, n, &ex->children, siblings) {
+		if (child->dev_type == EDGE_DEV ||
+		    child->dev_type == FANOUT_DEV)
+			sas_unregister_ex_tree(child);
+		else
+			sas_unregister_dev(child);
+	}
+	sas_unregister_dev(dev);
+}
+
+static void sas_unregister_devs_sas_addr(struct domain_device *parent,
+					 int phy_id)
+{
+	struct expander_device *ex_dev = &parent->ex_dev;
+	struct ex_phy *phy = &ex_dev->ex_phy[phy_id];
+	struct domain_device *child, *n;
+
+	list_for_each_entry_safe(child, n, &ex_dev->children, siblings) {
+		if (SAS_ADDR(child->sas_addr) ==
+		    SAS_ADDR(phy->attached_sas_addr)) {
+			if (child->dev_type == EDGE_DEV ||
+			    child->dev_type == FANOUT_DEV)
+				sas_unregister_ex_tree(child);
+			else
+				sas_unregister_dev(child);
+			break;
+		}
+	}
+	sas_disable_routing(parent, phy->attached_sas_addr);
+	memset(phy->attached_sas_addr, 0, SAS_ADDR_SIZE);
+	sas_port_delete_phy(phy->port, phy->phy);
+	if (phy->port->num_phys == 0)
+		sas_port_delete(phy->port);
+	phy->port = NULL;
+}
+
+static int sas_discover_bfs_by_root_level(struct domain_device *root,
+					  const int level)
+{
+	struct expander_device *ex_root = &root->ex_dev;
+	struct domain_device *child;
+	int res = 0;
+
+	list_for_each_entry(child, &ex_root->children, siblings) {
+		if (child->dev_type == EDGE_DEV ||
+		    child->dev_type == FANOUT_DEV) {
+			struct sas_expander_device *ex =
+				rphy_to_expander_device(child->rphy);
+
+			if (level > ex->level)
+				res = sas_discover_bfs_by_root_level(child,
+								     level);
+			else if (level == ex->level)
+				res = sas_ex_discover_devices(child, -1);
+		}
+	}
+	return res;
+}
+
+static int sas_discover_bfs_by_root(struct domain_device *dev)
+{
+	int res;
+	struct sas_expander_device *ex = rphy_to_expander_device(dev->rphy);
+	int level = ex->level+1;
+
+	res = sas_ex_discover_devices(dev, -1);
+	if (res)
+		goto out;
+	do {
+		res = sas_discover_bfs_by_root_level(dev, level);
+		mb();
+		level += 1;
+	} while (level <= dev->port->disc.max_level);
+out:
+	return res;
+}
+
+static int sas_discover_new(struct domain_device *dev, int phy_id)
+{
+	struct ex_phy *ex_phy = &dev->ex_dev.ex_phy[phy_id];
+	struct domain_device *child;
+	int res;
+
+	SAS_DPRINTK("ex %016llx phy%d new device attached\n",
+		    SAS_ADDR(dev->sas_addr), phy_id);
+	res = sas_ex_phy_discover(dev, phy_id);
+	if (res)
+		goto out;
+	res = sas_ex_discover_devices(dev, phy_id);
+	if (res)
+		goto out;
+	list_for_each_entry(child, &dev->ex_dev.children, siblings) {
+		if (SAS_ADDR(child->sas_addr) ==
+		    SAS_ADDR(ex_phy->attached_sas_addr)) {
+			if (child->dev_type == EDGE_DEV ||
+			    child->dev_type == FANOUT_DEV)
+				res = sas_discover_bfs_by_root(child);
+			break;
+		}
+	}
+out:
+	return res;
+}
+
+static int sas_rediscover_dev(struct domain_device *dev, int phy_id)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	struct ex_phy *phy = &ex->ex_phy[phy_id];
+	u8 attached_sas_addr[8];
+	int res;
+
+	res = sas_get_phy_attached_sas_addr(dev, phy_id, attached_sas_addr);
+	switch (res) {
+	case SMP_RESP_NO_PHY:
+		phy->phy_state = PHY_NOT_PRESENT;
+		sas_unregister_devs_sas_addr(dev, phy_id);
+		goto out; break;
+	case SMP_RESP_PHY_VACANT:
+		phy->phy_state = PHY_VACANT;
+		sas_unregister_devs_sas_addr(dev, phy_id);
+		goto out; break;
+	case SMP_RESP_FUNC_ACC:
+		break;
+	}
+
+	if (SAS_ADDR(attached_sas_addr) == 0) {
+		phy->phy_state = PHY_EMPTY;
+		sas_unregister_devs_sas_addr(dev, phy_id);
+	} else if (SAS_ADDR(attached_sas_addr) ==
+		   SAS_ADDR(phy->attached_sas_addr)) {
+		SAS_DPRINTK("ex %016llx phy 0x%x broadcast flutter\n",
+			    SAS_ADDR(dev->sas_addr), phy_id);
+		sas_ex_phy_discover(dev, phy_id);
+	} else
+		res = sas_discover_new(dev, phy_id);
+out:
+	return res;
+}
+
+static int sas_rediscover(struct domain_device *dev, const int phy_id)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	struct ex_phy *changed_phy = &ex->ex_phy[phy_id];
+	int res = 0;
+	int i;
+
+	SAS_DPRINTK("ex %016llx phy%d originated BROADCAST(CHANGE)\n",
+		    SAS_ADDR(dev->sas_addr), phy_id);
+
+	if (SAS_ADDR(changed_phy->attached_sas_addr) != 0) {
+		for (i = 0; i < ex->num_phys; i++) {
+			struct ex_phy *phy = &ex->ex_phy[i];
+
+			if (i == phy_id)
+				continue;
+			if (SAS_ADDR(phy->attached_sas_addr) ==
+			    SAS_ADDR(changed_phy->attached_sas_addr)) {
+				SAS_DPRINTK("phy%d part of wide port with "
+					    "phy%d\n", phy_id, i);
+				goto out;
+			}
+		}
+		res = sas_rediscover_dev(dev, phy_id);
+	} else
+		res = sas_discover_new(dev, phy_id);
+out:
+	return res;
+}
+
+/**
+ * sas_revalidate_domain -- revalidate the domain
+ * @port: port to the domain of interest
+ *
+ * NOTE: this process _must_ quit (return) as soon as any connection
+ * errors are encountered.  Connection recovery is done elsewhere.
+ * Discover process only interrogates devices in order to discover the
+ * domain.
+ */
+int sas_ex_revalidate_domain(struct domain_device *port_dev)
+{
+	int res;
+	struct domain_device *dev = NULL;
+
+	res = sas_find_bcast_dev(port_dev, &dev);
+	if (res)
+		goto out;
+	if (dev) {
+		struct expander_device *ex = &dev->ex_dev;
+		int i = 0, phy_id;
+
+		do {
+			phy_id = -1;
+			res = sas_find_bcast_phy(dev, &phy_id, i);
+			if (phy_id == -1)
+				break;
+			res = sas_rediscover(dev, phy_id);
+			i = phy_id + 1;
+		} while (i < ex->num_phys);
+	}
+out:
+	return res;
+}
+
+#if 0
+/* ---------- SMP portal ---------- */
+
+static ssize_t smp_portal_write(struct kobject *kobj, char *buf, loff_t offs,
+				size_t size)
+{
+	struct domain_device *dev = to_dom_device(kobj);
+	struct expander_device *ex = &dev->ex_dev;
+
+	if (offs != 0)
+		return -EFBIG;
+	else if (size == 0)
+		return 0;
+
+	down_interruptible(&ex->smp_sema);
+	if (ex->smp_req)
+		kfree(ex->smp_req);
+	ex->smp_req = kzalloc(size, GFP_USER);
+	if (!ex->smp_req) {
+		up(&ex->smp_sema);
+		return -ENOMEM;
+	}
+	memcpy(ex->smp_req, buf, size);
+	ex->smp_req_size = size;
+	ex->smp_portal_pid = current->pid;
+	up(&ex->smp_sema);
+
+	return size;
+}
+
+static ssize_t smp_portal_read(struct kobject *kobj, char *buf, loff_t offs,
+			       size_t size)
+{
+	struct domain_device *dev = to_dom_device(kobj);
+	struct expander_device *ex = &dev->ex_dev;
+	u8 *smp_resp;
+	int res = -EINVAL;
+
+	/* XXX: sysfs gives us an offset of 0x10 or 0x8 while in fact
+	 *  it should be 0.
+	 */
+
+	down_interruptible(&ex->smp_sema);
+	if (!ex->smp_req || ex->smp_portal_pid != current->pid)
+		goto out;
+
+	res = 0;
+	if (size == 0)
+		goto out;
+
+	res = -ENOMEM;
+	smp_resp = alloc_smp_resp(size);
+	if (!smp_resp)
+		goto out;
+	res = smp_execute_task(dev, ex->smp_req, ex->smp_req_size,
+			       smp_resp, size);
+	if (!res) {
+		memcpy(buf, smp_resp, size);
+		res = size;
+	}
+
+	kfree(smp_resp);
+out:
+	kfree(ex->smp_req);
+	ex->smp_req = NULL;
+	ex->smp_req_size = 0;
+	ex->smp_portal_pid = -1;
+	up(&ex->smp_sema);
+	return res;
+}
+#endif
diff --git a/drivers/scsi/libsas/sas_init.c b/drivers/scsi/libsas/sas_init.c
new file mode 100644
index 0000000..c836a23
--- /dev/null
+++ b/drivers/scsi/libsas/sas_init.c
@@ -0,0 +1,267 @@
+/*
+ * Serial Attached SCSI (SAS) Transport Layer initialization
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ *
+ */
+
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/device.h>
+#include <linux/spinlock.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_transport.h>
+#include <scsi/scsi_transport_sas.h>
+
+#include "sas_internal.h"
+
+#include "../scsi_sas_internal.h"
+
+kmem_cache_t *sas_task_cache;
+
+/*------------ SAS addr hash -----------*/
+void sas_hash_addr(u8 *hashed, const u8 *sas_addr)
+{
+        const u32 poly = 0x00DB2777;
+        u32     r = 0;
+        int     i;
+
+        for (i = 0; i < 8; i++) {
+                int b;
+                for (b = 7; b >= 0; b--) {
+                        r <<= 1;
+                        if ((1 << b) & sas_addr[i]) {
+                                if (!(r & 0x01000000))
+                                        r ^= poly;
+                        } else if (r & 0x01000000)
+                                r ^= poly;
+                }
+        }
+
+        hashed[0] = (r >> 16) & 0xFF;
+        hashed[1] = (r >> 8) & 0xFF ;
+        hashed[2] = r & 0xFF;
+}
+
+
+/* ---------- HA events ---------- */
+
+void sas_hae_reset(void *data)
+{
+	struct sas_ha_struct *ha = data;
+
+	sas_begin_event(HAE_RESET, &ha->event_lock,
+			&ha->pending);
+}
+
+int sas_register_ha(struct sas_ha_struct *sas_ha)
+{
+	int error = 0;
+
+	spin_lock_init(&sas_ha->phy_port_lock);
+	sas_hash_addr(sas_ha->hashed_sas_addr, sas_ha->sas_addr);
+
+	if (sas_ha->lldd_queue_size == 0)
+		sas_ha->lldd_queue_size = 1;
+	else if (sas_ha->lldd_queue_size == -1)
+		sas_ha->lldd_queue_size = 128; /* Sanity */
+
+	error = sas_register_phys(sas_ha);
+	if (error) {
+		printk(KERN_NOTICE "couldn't register sas phys:%d\n", error);
+		return error;
+	}
+
+	error = sas_register_ports(sas_ha);
+	if (error) {
+		printk(KERN_NOTICE "couldn't register sas ports:%d\n", error);
+		goto Undo_phys;
+	}
+
+	error = sas_init_events(sas_ha);
+	if (error) {
+		printk(KERN_NOTICE "couldn't start event thread:%d\n", error);
+		goto Undo_ports;
+	}
+
+	if (sas_ha->lldd_max_execute_num > 1) {
+		error = sas_init_queue(sas_ha);
+		if (error) {
+			printk(KERN_NOTICE "couldn't start queue thread:%d, "
+			       "running in direct mode\n", error);
+			sas_ha->lldd_max_execute_num = 1;
+		}
+	}
+
+	return 0;
+
+Undo_ports:
+	sas_unregister_ports(sas_ha);
+Undo_phys:
+
+	return error;
+}
+
+int sas_unregister_ha(struct sas_ha_struct *sas_ha)
+{
+	if (sas_ha->lldd_max_execute_num > 1) {
+		sas_shutdown_queue(sas_ha);
+	}
+
+	sas_unregister_ports(sas_ha);
+
+	return 0;
+}
+
+static int sas_get_linkerrors(struct sas_phy *phy)
+{
+	if (scsi_is_sas_phy_local(phy))
+		/* FIXME: we have no local phy stats
+		 * gathering at this time */
+		return -EINVAL;
+
+	return sas_smp_get_phy_events(phy);
+}
+
+static int sas_phy_reset(struct sas_phy *phy, int hard_reset)
+{
+	int ret;
+	enum phy_func reset_type;
+
+	if (hard_reset)
+		reset_type = PHY_FUNC_HARD_RESET;
+	else
+		reset_type = PHY_FUNC_LINK_RESET;
+
+	if (scsi_is_sas_phy_local(phy)) {
+		struct Scsi_Host *shost = dev_to_shost(phy->dev.parent);
+		struct sas_ha_struct *sas_ha = SHOST_TO_SAS_HA(shost);
+		struct asd_sas_phy *asd_phy = sas_ha->sas_phy[phy->number];
+		struct sas_internal *i =
+			to_sas_internal(sas_ha->core.shost->transportt);
+
+		ret = i->dft->lldd_control_phy(asd_phy, reset_type, NULL);
+	} else {
+		struct sas_rphy *rphy = dev_to_rphy(phy->dev.parent);
+		struct domain_device *ddev = sas_find_dev_by_rphy(rphy);
+		ret = sas_smp_phy_control(ddev, phy->number, reset_type, NULL);
+	}
+	return ret;
+}
+
+static int sas_set_phy_speed(struct sas_phy *phy,
+			     struct sas_phy_linkrates *rates)
+{
+	int ret;
+
+	if ((rates->minimum_linkrate &&
+	     rates->minimum_linkrate > phy->maximum_linkrate) ||
+	    (rates->maximum_linkrate &&
+	     rates->maximum_linkrate < phy->minimum_linkrate))
+		return -EINVAL;
+
+	if (rates->minimum_linkrate &&
+	    rates->minimum_linkrate < phy->minimum_linkrate_hw)
+		rates->minimum_linkrate = phy->minimum_linkrate_hw;
+
+	if (rates->maximum_linkrate &&
+	    rates->maximum_linkrate > phy->maximum_linkrate_hw)
+		rates->maximum_linkrate = phy->maximum_linkrate_hw;
+
+	if (scsi_is_sas_phy_local(phy)) {
+		struct Scsi_Host *shost = dev_to_shost(phy->dev.parent);
+		struct sas_ha_struct *sas_ha = SHOST_TO_SAS_HA(shost);
+		struct asd_sas_phy *asd_phy = sas_ha->sas_phy[phy->number];
+		struct sas_internal *i =
+			to_sas_internal(sas_ha->core.shost->transportt);
+
+		ret = i->dft->lldd_control_phy(asd_phy, PHY_FUNC_SET_LINK_RATE,
+					       rates);
+	} else {
+		struct sas_rphy *rphy = dev_to_rphy(phy->dev.parent);
+		struct domain_device *ddev = sas_find_dev_by_rphy(rphy);
+		ret = sas_smp_phy_control(ddev, phy->number,
+					  PHY_FUNC_LINK_RESET, rates);
+
+	}
+
+	return ret;
+}
+
+static struct sas_function_template sft = {
+	.phy_reset = sas_phy_reset,
+	.set_phy_speed = sas_set_phy_speed,
+	.get_linkerrors = sas_get_linkerrors,
+};
+
+struct scsi_transport_template *
+sas_domain_attach_transport(struct sas_domain_function_template *dft)
+{
+	struct scsi_transport_template *stt = sas_attach_transport(&sft);
+	struct sas_internal *i;
+
+	if (!stt)
+		return stt;
+
+	i = to_sas_internal(stt);
+	i->dft = dft;
+	stt->create_work_queue = 1;
+	stt->eh_timed_out = sas_scsi_timed_out;
+	stt->eh_strategy_handler = sas_scsi_recover_host;
+
+	return stt;
+}
+EXPORT_SYMBOL_GPL(sas_domain_attach_transport);
+
+
+void sas_domain_release_transport(struct scsi_transport_template *stt)
+{
+	sas_release_transport(stt);
+}
+EXPORT_SYMBOL_GPL(sas_domain_release_transport);
+
+/* ---------- SAS Class register/unregister ---------- */
+
+static int __init sas_class_init(void)
+{
+	sas_task_cache = kmem_cache_create("sas_task", sizeof(struct sas_task),
+					   0, SLAB_HWCACHE_ALIGN, NULL, NULL);
+	if (!sas_task_cache)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void __exit sas_class_exit(void)
+{
+	kmem_cache_destroy(sas_task_cache);
+}
+
+MODULE_AUTHOR("Luben Tuikov <luben_tuikov@adaptec.com>");
+MODULE_DESCRIPTION("SAS Transport Layer");
+MODULE_LICENSE("GPL v2");
+
+module_init(sas_class_init);
+module_exit(sas_class_exit);
+
+EXPORT_SYMBOL_GPL(sas_register_ha);
+EXPORT_SYMBOL_GPL(sas_unregister_ha);
diff --git a/drivers/scsi/libsas/sas_internal.h b/drivers/scsi/libsas/sas_internal.h
new file mode 100644
index 0000000..bffcee4
--- /dev/null
+++ b/drivers/scsi/libsas/sas_internal.h
@@ -0,0 +1,146 @@
+/*
+ * Serial Attached SCSI (SAS) class internal header file
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ *
+ */
+
+#ifndef _SAS_INTERNAL_H_
+#define _SAS_INTERNAL_H_
+
+#include <scsi/scsi.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_transport_sas.h>
+#include <scsi/libsas.h>
+
+#define sas_printk(fmt, ...) printk(KERN_NOTICE "sas: " fmt, ## __VA_ARGS__)
+
+#ifdef SAS_DEBUG
+#define SAS_DPRINTK(fmt, ...) printk(KERN_NOTICE "sas: " fmt, ## __VA_ARGS__)
+#else
+#define SAS_DPRINTK(fmt, ...)
+#endif
+
+void sas_scsi_recover_host(struct Scsi_Host *shost);
+
+int sas_show_class(enum sas_class class, char *buf);
+int sas_show_proto(enum sas_proto proto, char *buf);
+int sas_show_linkrate(enum sas_linkrate linkrate, char *buf);
+int sas_show_oob_mode(enum sas_oob_mode oob_mode, char *buf);
+
+int  sas_register_phys(struct sas_ha_struct *sas_ha);
+void sas_unregister_phys(struct sas_ha_struct *sas_ha);
+
+int  sas_register_ports(struct sas_ha_struct *sas_ha);
+void sas_unregister_ports(struct sas_ha_struct *sas_ha);
+
+enum scsi_eh_timer_return sas_scsi_timed_out(struct scsi_cmnd *);
+
+int  sas_init_queue(struct sas_ha_struct *sas_ha);
+int  sas_init_events(struct sas_ha_struct *sas_ha);
+void sas_shutdown_queue(struct sas_ha_struct *sas_ha);
+
+void sas_deform_port(struct asd_sas_phy *phy);
+
+void sas_porte_bytes_dmaed(void *);
+void sas_porte_broadcast_rcvd(void *);
+void sas_porte_link_reset_err(void *);
+void sas_porte_timer_event(void *);
+void sas_porte_hard_reset(void *);
+
+int sas_notify_lldd_dev_found(struct domain_device *);
+void sas_notify_lldd_dev_gone(struct domain_device *);
+
+int sas_smp_phy_control(struct domain_device *dev, int phy_id,
+			enum phy_func phy_func, struct sas_phy_linkrates *);
+int sas_smp_get_phy_events(struct sas_phy *phy);
+
+struct domain_device *sas_find_dev_by_rphy(struct sas_rphy *rphy);
+
+void sas_hae_reset(void *);
+
+static inline void sas_queue_event(int event, spinlock_t *lock,
+				   unsigned long *pending,
+				   struct work_struct *work,
+				   struct Scsi_Host *shost)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(lock, flags);
+	if (test_bit(event, pending)) {
+		spin_unlock_irqrestore(lock, flags);
+		return;
+	}
+	__set_bit(event, pending);
+	spin_unlock_irqrestore(lock, flags);
+	scsi_queue_work(shost, work);
+}
+
+static inline void sas_begin_event(int event, spinlock_t *lock,
+				   unsigned long *pending)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(lock, flags);
+	__clear_bit(event, pending);
+	spin_unlock_irqrestore(lock, flags);
+}
+
+static inline void sas_fill_in_rphy(struct domain_device *dev,
+				    struct sas_rphy *rphy)
+{
+	rphy->identify.sas_address = SAS_ADDR(dev->sas_addr);
+	rphy->identify.initiator_port_protocols = dev->iproto;
+	rphy->identify.target_port_protocols = dev->tproto;
+	switch (dev->dev_type) {
+	case SATA_DEV:
+		/* FIXME: need sata device type */
+	case SAS_END_DEV:
+		rphy->identify.device_type = SAS_END_DEVICE;
+		break;
+	case EDGE_DEV:
+		rphy->identify.device_type = SAS_EDGE_EXPANDER_DEVICE;
+		break;
+	case FANOUT_DEV:
+		rphy->identify.device_type = SAS_FANOUT_EXPANDER_DEVICE;
+		break;
+	default:
+		rphy->identify.device_type = SAS_PHY_UNUSED;
+		break;
+	}
+}
+
+static inline void sas_add_parent_port(struct domain_device *dev, int phy_id)
+{
+	struct expander_device *ex = &dev->ex_dev;
+	struct ex_phy *ex_phy = &ex->ex_phy[phy_id];
+
+	if (!ex->parent_port) {
+		ex->parent_port = sas_port_alloc(&dev->rphy->dev, phy_id);
+		/* FIXME: error handling */
+		BUG_ON(!ex->parent_port);
+		BUG_ON(sas_port_add(ex->parent_port));
+		sas_port_mark_backlink(ex->parent_port);
+	}
+	sas_port_add_phy(ex->parent_port, ex_phy->phy);
+}
+
+#endif /* _SAS_INTERNAL_H_ */
diff --git a/drivers/scsi/libsas/sas_phy.c b/drivers/scsi/libsas/sas_phy.c
new file mode 100644
index 0000000..9340cdb
--- /dev/null
+++ b/drivers/scsi/libsas/sas_phy.c
@@ -0,0 +1,158 @@
+/*
+ * Serial Attached SCSI (SAS) Phy class
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include "sas_internal.h"
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_transport.h>
+#include <scsi/scsi_transport_sas.h>
+#include "../scsi_sas_internal.h"
+
+/* ---------- Phy events ---------- */
+
+static void sas_phye_loss_of_signal(void *data)
+{
+	struct asd_sas_phy *phy = data;
+
+	sas_begin_event(PHYE_LOSS_OF_SIGNAL, &phy->ha->event_lock,
+			&phy->phy_events_pending);
+	phy->error = 0;
+	sas_deform_port(phy);
+}
+
+static void sas_phye_oob_done(void *data)
+{
+	struct asd_sas_phy *phy = data;
+
+	sas_begin_event(PHYE_OOB_DONE, &phy->ha->event_lock,
+			&phy->phy_events_pending);
+	phy->error = 0;
+}
+
+static void sas_phye_oob_error(void *data)
+{
+	struct asd_sas_phy *phy = data;
+	struct sas_ha_struct *sas_ha = phy->ha;
+	struct asd_sas_port *port = phy->port;
+	struct sas_internal *i =
+		to_sas_internal(sas_ha->core.shost->transportt);
+
+	sas_begin_event(PHYE_OOB_ERROR, &phy->ha->event_lock,
+			&phy->phy_events_pending);
+
+	sas_deform_port(phy);
+
+	if (!port && phy->enabled && i->dft->lldd_control_phy) {
+		phy->error++;
+		switch (phy->error) {
+		case 1:
+		case 2:
+			i->dft->lldd_control_phy(phy, PHY_FUNC_HARD_RESET,
+						 NULL);
+			break;
+		case 3:
+		default:
+			phy->error = 0;
+			phy->enabled = 0;
+			i->dft->lldd_control_phy(phy, PHY_FUNC_DISABLE, NULL);
+			break;
+		}
+	}
+}
+
+static void sas_phye_spinup_hold(void *data)
+{
+	struct asd_sas_phy *phy = data;
+	struct sas_ha_struct *sas_ha = phy->ha;
+	struct sas_internal *i =
+		to_sas_internal(sas_ha->core.shost->transportt);
+
+	sas_begin_event(PHYE_SPINUP_HOLD, &phy->ha->event_lock,
+			&phy->phy_events_pending);
+
+	phy->error = 0;
+	i->dft->lldd_control_phy(phy, PHY_FUNC_RELEASE_SPINUP_HOLD, NULL);
+}
+
+/* ---------- Phy class registration ---------- */
+
+int sas_register_phys(struct sas_ha_struct *sas_ha)
+{
+	int i;
+
+	static void (*sas_phy_event_fns[PHY_NUM_EVENTS])(void *) = {
+		[PHYE_LOSS_OF_SIGNAL] = sas_phye_loss_of_signal,
+		[PHYE_OOB_DONE] = sas_phye_oob_done,
+		[PHYE_OOB_ERROR] = sas_phye_oob_error,
+		[PHYE_SPINUP_HOLD] = sas_phye_spinup_hold,
+	};
+
+	static void (*sas_port_event_fns[PORT_NUM_EVENTS])(void *) = {
+		[PORTE_BYTES_DMAED] = sas_porte_bytes_dmaed,
+		[PORTE_BROADCAST_RCVD] = sas_porte_broadcast_rcvd,
+		[PORTE_LINK_RESET_ERR] = sas_porte_link_reset_err,
+		[PORTE_TIMER_EVENT] = sas_porte_timer_event,
+		[PORTE_HARD_RESET] = sas_porte_hard_reset,
+	};
+
+	/* Now register the phys. */
+	for (i = 0; i < sas_ha->num_phys; i++) {
+		int k;
+		struct asd_sas_phy *phy = sas_ha->sas_phy[i];
+
+		phy->error = 0;
+		INIT_LIST_HEAD(&phy->port_phy_el);
+		for (k = 0; k < PORT_NUM_EVENTS; k++)
+			INIT_WORK(&phy->port_events[k], sas_port_event_fns[k],
+				  phy);
+
+		for (k = 0; k < PHY_NUM_EVENTS; k++)
+			INIT_WORK(&phy->phy_events[k], sas_phy_event_fns[k],
+				  phy);
+		phy->port = NULL;
+		phy->ha = sas_ha;
+		spin_lock_init(&phy->frame_rcvd_lock);
+		spin_lock_init(&phy->sas_prim_lock);
+		phy->frame_rcvd_size = 0;
+
+		phy->phy = sas_phy_alloc(&sas_ha->core.shost->shost_gendev,
+					 i);
+		if (!phy->phy)
+			return -ENOMEM;
+
+		phy->phy->identify.initiator_port_protocols =
+			phy->iproto;
+		phy->phy->identify.target_port_protocols = phy->tproto;
+		phy->phy->identify.sas_address = SAS_ADDR(sas_ha->sas_addr);
+		phy->phy->identify.phy_identifier = i;
+		phy->phy->minimum_linkrate_hw = SAS_LINK_RATE_UNKNOWN;
+		phy->phy->maximum_linkrate_hw = SAS_LINK_RATE_UNKNOWN;
+		phy->phy->minimum_linkrate = SAS_LINK_RATE_UNKNOWN;
+		phy->phy->maximum_linkrate = SAS_LINK_RATE_UNKNOWN;
+		phy->phy->negotiated_linkrate = SAS_LINK_RATE_UNKNOWN;
+
+		sas_phy_add(phy->phy);
+	}
+
+	return 0;
+}
diff --git a/drivers/scsi/libsas/sas_port.c b/drivers/scsi/libsas/sas_port.c
new file mode 100644
index 0000000..253cdcf
--- /dev/null
+++ b/drivers/scsi/libsas/sas_port.c
@@ -0,0 +1,279 @@
+/*
+ * Serial Attached SCSI (SAS) Port class
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include "sas_internal.h"
+
+#include <scsi/scsi_transport.h>
+#include <scsi/scsi_transport_sas.h>
+#include "../scsi_sas_internal.h"
+
+/**
+ * sas_form_port -- add this phy to a port
+ * @phy: the phy of interest
+ *
+ * This function adds this phy to an existing port, thus creating a wide
+ * port, or it creates a port and adds the phy to the port.
+ */
+static void sas_form_port(struct asd_sas_phy *phy)
+{
+	int i;
+	struct sas_ha_struct *sas_ha = phy->ha;
+	struct asd_sas_port *port = phy->port;
+	struct sas_internal *si =
+		to_sas_internal(sas_ha->core.shost->transportt);
+
+	if (port) {
+		if (memcmp(port->attached_sas_addr, phy->attached_sas_addr,
+			   SAS_ADDR_SIZE) == 0)
+			sas_deform_port(phy);
+		else {
+			SAS_DPRINTK("%s: phy%d belongs to port%d already(%d)!\n",
+				    __FUNCTION__, phy->id, phy->port->id,
+				    phy->port->num_phys);
+			return;
+		}
+	}
+
+	/* find a port */
+	spin_lock(&sas_ha->phy_port_lock);
+	for (i = 0; i < sas_ha->num_phys; i++) {
+		port = sas_ha->sas_port[i];
+		spin_lock(&port->phy_list_lock);
+		if (*(u64 *) port->sas_addr &&
+		    memcmp(port->attached_sas_addr,
+			   phy->attached_sas_addr, SAS_ADDR_SIZE) == 0 &&
+		    port->num_phys > 0) {
+			/* wide port */
+			SAS_DPRINTK("phy%d matched wide port%d\n", phy->id,
+				    port->id);
+			break;
+		} else if (*(u64 *) port->sas_addr == 0 && port->num_phys==0) {
+			memcpy(port->sas_addr, phy->sas_addr, SAS_ADDR_SIZE);
+			break;
+		}
+		spin_unlock(&port->phy_list_lock);
+	}
+
+	if (i >= sas_ha->num_phys) {
+		printk(KERN_NOTICE "%s: couldn't find a free port, bug?\n",
+		       __FUNCTION__);
+		spin_unlock(&sas_ha->phy_port_lock);
+		return;
+	}
+
+	/* add the phy to the port */
+	list_add_tail(&phy->port_phy_el, &port->phy_list);
+	phy->port = port;
+	port->num_phys++;
+	port->phy_mask |= (1U << phy->id);
+
+	if (!port->phy)
+		port->phy = phy->phy;
+
+	SAS_DPRINTK("phy%d added to port%d, phy_mask:0x%x\n", phy->id,
+		    port->id, port->phy_mask);
+
+	if (*(u64 *)port->attached_sas_addr == 0) {
+		port->class = phy->class;
+		memcpy(port->attached_sas_addr, phy->attached_sas_addr,
+		       SAS_ADDR_SIZE);
+		port->iproto = phy->iproto;
+		port->tproto = phy->tproto;
+		port->oob_mode = phy->oob_mode;
+		port->linkrate = phy->linkrate;
+	} else
+		port->linkrate = max(port->linkrate, phy->linkrate);
+	spin_unlock(&port->phy_list_lock);
+	spin_unlock(&sas_ha->phy_port_lock);
+
+	if (!port->port) {
+		port->port = sas_port_alloc(phy->phy->dev.parent, port->id);
+		BUG_ON(!port->port);
+		sas_port_add(port->port);
+	}
+	sas_port_add_phy(port->port, phy->phy);
+
+	if (port->port_dev)
+		port->port_dev->pathways = port->num_phys;
+
+	/* Tell the LLDD about this port formation. */
+	if (si->dft->lldd_port_formed)
+		si->dft->lldd_port_formed(phy);
+
+	sas_discover_event(phy->port, DISCE_DISCOVER_DOMAIN);
+}
+
+/**
+ * sas_deform_port -- remove this phy from the port it belongs to
+ * @phy: the phy of interest
+ *
+ * This is called when the physical link to the other phy has been
+ * lost (on this phy), in Event thread context. We cannot delay here.
+ */
+void sas_deform_port(struct asd_sas_phy *phy)
+{
+	struct sas_ha_struct *sas_ha = phy->ha;
+	struct asd_sas_port *port = phy->port;
+	struct sas_internal *si =
+		to_sas_internal(sas_ha->core.shost->transportt);
+
+	if (!port)
+		return;		  /* done by a phy event */
+
+	if (port->port_dev)
+		port->port_dev->pathways--;
+
+	if (port->num_phys == 1) {
+		sas_unregister_domain_devices(port);
+		sas_port_delete(port->port);
+		port->port = NULL;
+	} else
+		sas_port_delete_phy(port->port, phy->phy);
+
+
+	if (si->dft->lldd_port_deformed)
+		si->dft->lldd_port_deformed(phy);
+
+	spin_lock(&sas_ha->phy_port_lock);
+	spin_lock(&port->phy_list_lock);
+
+	list_del_init(&phy->port_phy_el);
+	phy->port = NULL;
+	port->num_phys--;
+	port->phy_mask &= ~(1U << phy->id);
+
+	if (port->num_phys == 0) {
+		INIT_LIST_HEAD(&port->phy_list);
+		memset(port->sas_addr, 0, SAS_ADDR_SIZE);
+		memset(port->attached_sas_addr, 0, SAS_ADDR_SIZE);
+		port->class = 0;
+		port->iproto = 0;
+		port->tproto = 0;
+		port->oob_mode = 0;
+		port->phy_mask = 0;
+	}
+	spin_unlock(&port->phy_list_lock);
+	spin_unlock(&sas_ha->phy_port_lock);
+
+	return;
+}
+
+/* ---------- SAS port events ---------- */
+
+void sas_porte_bytes_dmaed(void *data)
+{
+	struct asd_sas_phy *phy = data;
+
+	sas_begin_event(PORTE_BYTES_DMAED, &phy->ha->event_lock,
+			&phy->port_events_pending);
+
+	sas_form_port(phy);
+}
+
+void sas_porte_broadcast_rcvd(void *data)
+{
+	unsigned long flags;
+	u32 prim;
+	struct asd_sas_phy *phy = data;
+
+	sas_begin_event(PORTE_BROADCAST_RCVD, &phy->ha->event_lock,
+			&phy->port_events_pending);
+
+	spin_lock_irqsave(&phy->sas_prim_lock, flags);
+	prim = phy->sas_prim;
+	spin_unlock_irqrestore(&phy->sas_prim_lock, flags);
+
+	SAS_DPRINTK("broadcast received: %d\n", prim);
+	sas_discover_event(phy->port, DISCE_REVALIDATE_DOMAIN);
+}
+
+void sas_porte_link_reset_err(void *data)
+{
+	struct asd_sas_phy *phy = data;
+
+	sas_begin_event(PORTE_LINK_RESET_ERR, &phy->ha->event_lock,
+			&phy->port_events_pending);
+
+	sas_deform_port(phy);
+}
+
+void sas_porte_timer_event(void *data)
+{
+	struct asd_sas_phy *phy = data;
+
+	sas_begin_event(PORTE_TIMER_EVENT, &phy->ha->event_lock,
+			&phy->port_events_pending);
+
+	sas_deform_port(phy);
+}
+
+void sas_porte_hard_reset(void *data)
+{
+	struct asd_sas_phy *phy = data;
+
+	sas_begin_event(PORTE_HARD_RESET, &phy->ha->event_lock,
+			&phy->port_events_pending);
+
+	sas_deform_port(phy);
+}
+
+/* ---------- SAS port registration ---------- */
+
+static void sas_init_port(struct asd_sas_port *port,
+			  struct sas_ha_struct *sas_ha, int i)
+{
+	port->id = i;
+	INIT_LIST_HEAD(&port->dev_list);
+	spin_lock_init(&port->phy_list_lock);
+	INIT_LIST_HEAD(&port->phy_list);
+	port->num_phys = 0;
+	port->phy_mask = 0;
+	port->ha = sas_ha;
+
+	spin_lock_init(&port->dev_list_lock);
+}
+
+int sas_register_ports(struct sas_ha_struct *sas_ha)
+{
+	int i;
+
+	/* initialize the ports and discovery */
+	for (i = 0; i < sas_ha->num_phys; i++) {
+		struct asd_sas_port *port = sas_ha->sas_port[i];
+
+		sas_init_port(port, sas_ha, i);
+		sas_init_disc(&port->disc, port);
+	}
+	return 0;
+}
+
+void sas_unregister_ports(struct sas_ha_struct *sas_ha)
+{
+	int i;
+
+	for (i = 0; i < sas_ha->num_phys; i++)
+		if (sas_ha->sas_phy[i]->port)
+			sas_deform_port(sas_ha->sas_phy[i]);
+
+}
diff --git a/drivers/scsi/libsas/sas_scsi_host.c b/drivers/scsi/libsas/sas_scsi_host.c
new file mode 100644
index 0000000..43e0e4e
--- /dev/null
+++ b/drivers/scsi/libsas/sas_scsi_host.c
@@ -0,0 +1,786 @@
+/*
+ * Serial Attached SCSI (SAS) class SCSI Host glue.
+ *
+ * Copyright (C) 2005 Adaptec, Inc.  All rights reserved.
+ * Copyright (C) 2005 Luben Tuikov <luben_tuikov@adaptec.com>
+ *
+ * This file is licensed under GPLv2.
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+ * USA
+ *
+ */
+
+#include "sas_internal.h"
+
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_tcq.h>
+#include <scsi/scsi.h>
+#include <scsi/scsi_transport.h>
+#include <scsi/scsi_transport_sas.h>
+#include "../scsi_sas_internal.h"
+
+#include <linux/err.h>
+#include <linux/blkdev.h>
+#include <linux/scatterlist.h>
+
+/* ---------- SCSI Host glue ---------- */
+
+#define TO_SAS_TASK(_scsi_cmd)  ((void *)(_scsi_cmd)->host_scribble)
+#define ASSIGN_SAS_TASK(_sc, _t) do { (_sc)->host_scribble = (void *) _t; } while (0)
+
+static void sas_scsi_task_done(struct sas_task *task)
+{
+	struct task_status_struct *ts = &task->task_status;
+	struct scsi_cmnd *sc = task->uldd_task;
+	unsigned ts_flags = task->task_state_flags;
+	int hs = 0, stat = 0;
+
+	if (unlikely(!sc)) {
+		SAS_DPRINTK("task_done called with non existing SCSI cmnd!\n");
+		list_del_init(&task->list);
+		sas_free_task(task);
+		return;
+	}
+
+	if (ts->resp == SAS_TASK_UNDELIVERED) {
+		/* transport error */
+		hs = DID_NO_CONNECT;
+	} else { /* ts->resp == SAS_TASK_COMPLETE */
+		/* task delivered, what happened afterwards? */
+		switch (ts->stat) {
+		case SAS_DEV_NO_RESPONSE:
+		case SAS_INTERRUPTED:
+		case SAS_PHY_DOWN:
+		case SAS_NAK_R_ERR:
+		case SAS_OPEN_TO:
+			hs = DID_NO_CONNECT;
+			break;
+		case SAS_DATA_UNDERRUN:
+			sc->resid = ts->residual;
+			if (sc->request_bufflen - sc->resid < sc->underflow)
+				hs = DID_ERROR;
+			break;
+		case SAS_DATA_OVERRUN:
+			hs = DID_ERROR;
+			break;
+		case SAS_QUEUE_FULL:
+			hs = DID_SOFT_ERROR; /* retry */
+			break;
+		case SAS_DEVICE_UNKNOWN:
+			hs = DID_BAD_TARGET;
+			break;
+		case SAS_SG_ERR:
+			hs = DID_PARITY;
+			break;
+		case SAS_OPEN_REJECT:
+			if (ts->open_rej_reason == SAS_OREJ_RSVD_RETRY)
+				hs = DID_SOFT_ERROR; /* retry */
+			else
+				hs = DID_ERROR;
+			break;
+		case SAS_PROTO_RESPONSE:
+			SAS_DPRINTK("LLDD:%s sent SAS_PROTO_RESP for an SSP "
+				    "task; please report this\n",
+				    task->dev->port->ha->sas_ha_name);
+			break;
+		case SAS_ABORTED_TASK:
+			hs = DID_ABORT;
+			break;
+		case SAM_CHECK_COND:
+			memcpy(sc->sense_buffer, ts->buf,
+			       max(SCSI_SENSE_BUFFERSIZE, ts->buf_valid_size));
+			stat = SAM_CHECK_COND;
+			break;
+		default:
+			stat = ts->stat;
+			break;
+		}
+	}
+	ASSIGN_SAS_TASK(sc, NULL);
+	sc->result = (hs << 16) | stat;
+	list_del_init(&task->list);
+	sas_free_task(task);
+	/* This is very ugly but this is how SCSI Core works. */
+	if (ts_flags & SAS_TASK_STATE_ABORTED)
+		scsi_finish_command(sc);
+	else
+		sc->scsi_done(sc);
+}
+
+static enum task_attribute sas_scsi_get_task_attr(struct scsi_cmnd *cmd)
+{
+	enum task_attribute ta = TASK_ATTR_SIMPLE;
+	if (cmd->request && blk_rq_tagged(cmd->request)) {
+		if (cmd->device->ordered_tags &&
+		    (cmd->request->flags & REQ_HARDBARRIER))
+			ta = TASK_ATTR_HOQ;
+	}
+	return ta;
+}
+
+static struct sas_task *sas_create_task(struct scsi_cmnd *cmd,
+					       struct domain_device *dev,
+					       unsigned long gfp_flags)
+{
+	struct sas_task *task = sas_alloc_task(gfp_flags);
+	struct scsi_lun lun;
+
+	if (!task)
+		return NULL;
+
+	*(u32 *)cmd->sense_buffer = 0;
+	task->uldd_task = cmd;
+	ASSIGN_SAS_TASK(cmd, task);
+
+	task->dev = dev;
+	task->task_proto = task->dev->tproto; /* BUG_ON(!SSP) */
+
+	task->ssp_task.retry_count = 1;
+	int_to_scsilun(cmd->device->lun, &lun);
+	memcpy(task->ssp_task.LUN, &lun.scsi_lun, 8);
+	task->ssp_task.task_attr = sas_scsi_get_task_attr(cmd);
+	memcpy(task->ssp_task.cdb, cmd->cmnd, 16);
+
+	task->scatter = cmd->request_buffer;
+	task->num_scatter = cmd->use_sg;
+	task->total_xfer_len = cmd->request_bufflen;
+	task->data_dir = cmd->sc_data_direction;
+
+	task->task_done = sas_scsi_task_done;
+
+	return task;
+}
+
+static int sas_queue_up(struct sas_task *task)
+{
+	struct sas_ha_struct *sas_ha = task->dev->port->ha;
+	struct scsi_core *core = &sas_ha->core;
+	unsigned long flags;
+	LIST_HEAD(list);
+
+	spin_lock_irqsave(&core->task_queue_lock, flags);
+	if (sas_ha->lldd_queue_size < core->task_queue_size + 1) {
+		spin_unlock_irqrestore(&core->task_queue_lock, flags);
+		return -SAS_QUEUE_FULL;
+	}
+	list_add_tail(&task->list, &core->task_queue);
+	core->task_queue_size += 1;
+	spin_unlock_irqrestore(&core->task_queue_lock, flags);
+	up(&core->queue_thread_sema);
+
+	return 0;
+}
+
+/**
+ * sas_queuecommand -- Enqueue a command for processing
+ * @parameters: See SCSI Core documentation
+ *
+ * Note: XXX: Remove the host unlock/lock pair when SCSI Core can
+ * call us without holding an IRQ spinlock...
+ */
+int sas_queuecommand(struct scsi_cmnd *cmd,
+		     void (*scsi_done)(struct scsi_cmnd *))
+{
+	int res = 0;
+	struct domain_device *dev = cmd_to_domain_dev(cmd);
+	struct Scsi_Host *host = cmd->device->host;
+	struct sas_internal *i = to_sas_internal(host->transportt);
+
+	spin_unlock_irq(host->host_lock);
+
+	{
+		struct sas_ha_struct *sas_ha = dev->port->ha;
+		struct sas_task *task;
+
+		res = -ENOMEM;
+		task = sas_create_task(cmd, dev, GFP_ATOMIC);
+		if (!task)
+			goto out;
+
+		cmd->scsi_done = scsi_done;
+		/* Queue up, Direct Mode or Task Collector Mode. */
+		if (sas_ha->lldd_max_execute_num < 2)
+			res = i->dft->lldd_execute_task(task, 1, GFP_ATOMIC);
+		else
+			res = sas_queue_up(task);
+
+		/* Examine */
+		if (res) {
+			SAS_DPRINTK("lldd_execute_task returned: %d\n", res);
+			ASSIGN_SAS_TASK(cmd, NULL);
+			sas_free_task(task);
+			if (res == -SAS_QUEUE_FULL) {
+				cmd->result = DID_SOFT_ERROR << 16; /* retry */
+				res = 0;
+				scsi_done(cmd);
+			}
+			goto out;
+		}
+	}
+out:
+	spin_lock_irq(host->host_lock);
+	return res;
+}
+
+static void sas_scsi_clear_queue_lu(struct list_head *error_q, struct scsi_cmnd *my_cmd)
+{
+	struct scsi_cmnd *cmd, *n;
+
+	list_for_each_entry_safe(cmd, n, error_q, eh_entry) {
+		if (cmd == my_cmd)
+			list_del_init(&cmd->eh_entry);
+	}
+}
+
+static void sas_scsi_clear_queue_I_T(struct list_head *error_q,
+				     struct domain_device *dev)
+{
+	struct scsi_cmnd *cmd, *n;
+
+	list_for_each_entry_safe(cmd, n, error_q, eh_entry) {
+		struct domain_device *x = cmd_to_domain_dev(cmd);
+
+		if (x == dev)
+			list_del_init(&cmd->eh_entry);
+	}
+}
+
+static void sas_scsi_clear_queue_port(struct list_head *error_q,
+				      struct asd_sas_port *port)
+{
+	struct scsi_cmnd *cmd, *n;
+
+	list_for_each_entry_safe(cmd, n, error_q, eh_entry) {
+		struct domain_device *dev = cmd_to_domain_dev(cmd);
+		struct asd_sas_port *x = dev->port;
+
+		if (x == port)
+			list_del_init(&cmd->eh_entry);
+	}
+}
+
+enum task_disposition {
+	TASK_IS_DONE,
+	TASK_IS_ABORTED,
+	TASK_IS_AT_LU,
+	TASK_IS_NOT_AT_LU,
+};
+
+static enum task_disposition sas_scsi_find_task(struct sas_task *task)
+{
+	struct sas_ha_struct *ha = task->dev->port->ha;
+	unsigned long flags;
+	int i, res;
+	struct sas_internal *si =
+		to_sas_internal(task->dev->port->ha->core.shost->transportt);
+
+	if (ha->lldd_max_execute_num > 1) {
+		struct scsi_core *core = &ha->core;
+		struct sas_task *t, *n;
+
+		spin_lock_irqsave(&core->task_queue_lock, flags);
+		list_for_each_entry_safe(t, n, &core->task_queue, list) {
+			if (task == t) {
+				list_del_init(&t->list);
+				spin_unlock_irqrestore(&core->task_queue_lock,
+						       flags);
+				SAS_DPRINTK("%s: task 0x%p aborted from "
+					    "task_queue\n",
+					    __FUNCTION__, task);
+				return TASK_IS_ABORTED;
+			}
+		}
+		spin_unlock_irqrestore(&core->task_queue_lock, flags);
+	}
+
+	for (i = 0; i < 5; i++) {
+		SAS_DPRINTK("%s: aborting task 0x%p\n", __FUNCTION__, task);
+		res = si->dft->lldd_abort_task(task);
+
+		spin_lock_irqsave(&task->task_state_lock, flags);
+		if (task->task_state_flags & SAS_TASK_STATE_DONE) {
+			spin_unlock_irqrestore(&task->task_state_lock, flags);
+			SAS_DPRINTK("%s: task 0x%p is done\n", __FUNCTION__,
+				    task);
+			return TASK_IS_DONE;
+		}
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
+
+		if (res == TMF_RESP_FUNC_COMPLETE) {
+			SAS_DPRINTK("%s: task 0x%p is aborted\n",
+				    __FUNCTION__, task);
+			return TASK_IS_ABORTED;
+		} else if (si->dft->lldd_query_task) {
+			SAS_DPRINTK("%s: querying task 0x%p\n",
+				    __FUNCTION__, task);
+			res = si->dft->lldd_query_task(task);
+			if (res == TMF_RESP_FUNC_SUCC) {
+				SAS_DPRINTK("%s: task 0x%p at LU\n",
+					    __FUNCTION__, task);
+				return TASK_IS_AT_LU;
+			} else if (res == TMF_RESP_FUNC_COMPLETE) {
+				SAS_DPRINTK("%s: task 0x%p not at LU\n",
+					    __FUNCTION__, task);
+				return TASK_IS_NOT_AT_LU;
+			}
+		}
+	}
+	return res;
+}
+
+static int sas_recover_lu(struct domain_device *dev, struct scsi_cmnd *cmd)
+{
+	int res = TMF_RESP_FUNC_FAILED;
+	struct scsi_lun lun;
+	struct sas_internal *i =
+		to_sas_internal(dev->port->ha->core.shost->transportt);
+
+	int_to_scsilun(cmd->device->lun, &lun);
+
+	SAS_DPRINTK("eh: device %llx LUN %x has the task\n",
+		    SAS_ADDR(dev->sas_addr),
+		    cmd->device->lun);
+
+	if (i->dft->lldd_abort_task_set)
+		res = i->dft->lldd_abort_task_set(dev, lun.scsi_lun);
+
+	if (res == TMF_RESP_FUNC_FAILED) {
+		if (i->dft->lldd_clear_task_set)
+			res = i->dft->lldd_clear_task_set(dev, lun.scsi_lun);
+	}
+
+	if (res == TMF_RESP_FUNC_FAILED) {
+		if (i->dft->lldd_lu_reset)
+			res = i->dft->lldd_lu_reset(dev, lun.scsi_lun);
+	}
+
+	return res;
+}
+
+static int sas_recover_I_T(struct domain_device *dev)
+{
+	int res = TMF_RESP_FUNC_FAILED;
+	struct sas_internal *i =
+		to_sas_internal(dev->port->ha->core.shost->transportt);
+
+	SAS_DPRINTK("I_T nexus reset for dev %016llx\n",
+		    SAS_ADDR(dev->sas_addr));
+
+	if (i->dft->lldd_I_T_nexus_reset)
+		res = i->dft->lldd_I_T_nexus_reset(dev);
+
+	return res;
+}
+
+void sas_scsi_recover_host(struct Scsi_Host *shost)
+{
+	struct sas_ha_struct *ha = SHOST_TO_SAS_HA(shost);
+	unsigned long flags;
+	LIST_HEAD(error_q);
+	struct scsi_cmnd *cmd, *n;
+	enum task_disposition res = TASK_IS_DONE;
+	int tmf_resp;
+	struct sas_internal *i = to_sas_internal(shost->transportt);
+
+	spin_lock_irqsave(shost->host_lock, flags);
+	list_splice_init(&shost->eh_cmd_q, &error_q);
+	spin_unlock_irqrestore(shost->host_lock, flags);
+
+	SAS_DPRINTK("Enter %s\n", __FUNCTION__);
+
+	/* All tasks on this list were marked SAS_TASK_STATE_ABORTED
+	 * by sas_scsi_timed_out() callback.
+	 */
+Again:
+	SAS_DPRINTK("going over list...\n");
+	list_for_each_entry_safe(cmd, n, &error_q, eh_entry) {
+		struct sas_task *task = TO_SAS_TASK(cmd);
+
+		SAS_DPRINTK("trying to find task 0x%p\n", task);
+		list_del_init(&cmd->eh_entry);
+		res = sas_scsi_find_task(task);
+
+		cmd->eh_eflags = 0;
+		shost->host_failed--;
+
+		switch (res) {
+		case TASK_IS_DONE:
+			SAS_DPRINTK("%s: task 0x%p is done\n", __FUNCTION__,
+				    task);
+			task->task_done(task);
+			continue;
+		case TASK_IS_ABORTED:
+			SAS_DPRINTK("%s: task 0x%p is aborted\n",
+				    __FUNCTION__, task);
+			task->task_done(task);
+			continue;
+		case TASK_IS_AT_LU:
+			SAS_DPRINTK("task 0x%p is at LU: lu recover\n", task);
+			tmf_resp = sas_recover_lu(task->dev, cmd);
+			if (tmf_resp == TMF_RESP_FUNC_COMPLETE) {
+				SAS_DPRINTK("dev %016llx LU %x is "
+					    "recovered\n",
+					    SAS_ADDR(task->dev),
+					    cmd->device->lun);
+				task->task_done(task);
+				sas_scsi_clear_queue_lu(&error_q, cmd);
+				goto Again;
+			}
+			/* fallthrough */
+		case TASK_IS_NOT_AT_LU:
+			SAS_DPRINTK("task 0x%p is not at LU: I_T recover\n",
+				    task);
+			tmf_resp = sas_recover_I_T(task->dev);
+			if (tmf_resp == TMF_RESP_FUNC_COMPLETE) {
+				SAS_DPRINTK("I_T %016llx recovered\n",
+					    SAS_ADDR(task->dev->sas_addr));
+				task->task_done(task);
+				sas_scsi_clear_queue_I_T(&error_q, task->dev);
+				goto Again;
+			}
+			/* Hammer time :-) */
+			if (i->dft->lldd_clear_nexus_port) {
+				struct asd_sas_port *port = task->dev->port;
+				SAS_DPRINTK("clearing nexus for port:%d\n",
+					    port->id);
+				res = i->dft->lldd_clear_nexus_port(port);
+				if (res == TMF_RESP_FUNC_COMPLETE) {
+					SAS_DPRINTK("clear nexus port:%d "
+						    "succeeded\n", port->id);
+					task->task_done(task);
+					sas_scsi_clear_queue_port(&error_q,
+								  port);
+					goto Again;
+				}
+			}
+			if (i->dft->lldd_clear_nexus_ha) {
+				SAS_DPRINTK("clear nexus ha\n");
+				res = i->dft->lldd_clear_nexus_ha(ha);
+				if (res == TMF_RESP_FUNC_COMPLETE) {
+					SAS_DPRINTK("clear nexus ha "
+						    "succeeded\n");
+					task->task_done(task);
+					goto out;
+				}
+			}
+			/* If we are here -- this means that no amount
+			 * of effort could recover from errors.  Quite
+			 * possibly the HA just disappeared.
+			 */
+			SAS_DPRINTK("error from  device %llx, LUN %x "
+				    "couldn't be recovered in any way\n",
+				    SAS_ADDR(task->dev->sas_addr),
+				    cmd->device->lun);
+
+			task->task_done(task);
+			goto clear_q;
+		}
+	}
+out:
+	SAS_DPRINTK("--- Exit %s\n", __FUNCTION__);
+	return;
+clear_q:
+	SAS_DPRINTK("--- Exit %s -- clear_q\n", __FUNCTION__);
+	list_for_each_entry_safe(cmd, n, &error_q, eh_entry) {
+		struct sas_task *task = TO_SAS_TASK(cmd);
+		list_del_init(&cmd->eh_entry);
+		task->task_done(task);
+	}
+}
+
+enum scsi_eh_timer_return sas_scsi_timed_out(struct scsi_cmnd *cmd)
+{
+	struct sas_task *task = TO_SAS_TASK(cmd);
+	unsigned long flags;
+
+	if (!task) {
+		SAS_DPRINTK("command 0x%p, task 0x%p, timed out: EH_HANDLED\n",
+			    cmd, task);
+		return EH_HANDLED;
+	}
+
+	spin_lock_irqsave(&task->task_state_lock, flags);
+	if (task->task_state_flags & SAS_TASK_STATE_DONE) {
+		spin_unlock_irqrestore(&task->task_state_lock, flags);
+		SAS_DPRINTK("command 0x%p, task 0x%p, timed out: EH_HANDLED\n",
+			    cmd, task);
+		return EH_HANDLED;
+	}
+	task->task_state_flags |= SAS_TASK_STATE_ABORTED;
+	spin_unlock_irqrestore(&task->task_state_lock, flags);
+
+	SAS_DPRINTK("command 0x%p, task 0x%p, timed out: EH_NOT_HANDLED\n",
+		    cmd, task);
+
+	return EH_NOT_HANDLED;
+}
+
+struct domain_device *sas_find_dev_by_rphy(struct sas_rphy *rphy)
+{
+	struct Scsi_Host *shost = dev_to_shost(rphy->dev.parent);
+	struct sas_ha_struct *ha = SHOST_TO_SAS_HA(shost);
+	struct domain_device *found_dev = NULL;
+	int i;
+
+	spin_lock(&ha->phy_port_lock);
+	for (i = 0; i < ha->num_phys; i++) {
+		struct asd_sas_port *port = ha->sas_port[i];
+		struct domain_device *dev;
+
+		spin_lock(&port->dev_list_lock);
+		list_for_each_entry(dev, &port->dev_list, dev_list_node) {
+			if (rphy == dev->rphy) {
+				found_dev = dev;
+				spin_unlock(&port->dev_list_lock);
+				goto found;
+			}
+		}
+		spin_unlock(&port->dev_list_lock);
+	}
+ found:
+	spin_unlock(&ha->phy_port_lock);
+
+	return found_dev;
+}
+
+static inline struct domain_device *sas_find_target(struct scsi_target *starget)
+{
+	struct sas_rphy *rphy = dev_to_rphy(starget->dev.parent);
+
+	return sas_find_dev_by_rphy(rphy);
+}
+
+int sas_target_alloc(struct scsi_target *starget)
+{
+	struct domain_device *found_dev = sas_find_target(starget);
+
+	if (!found_dev)
+		return -ENODEV;
+
+	starget->hostdata = found_dev;
+	return 0;
+}
+
+#define SAS_DEF_QD 32
+#define SAS_MAX_QD 64
+
+int sas_slave_configure(struct scsi_device *scsi_dev)
+{
+	struct domain_device *dev = sdev_to_domain_dev(scsi_dev);
+	struct sas_ha_struct *sas_ha;
+
+	BUG_ON(dev->rphy->identify.device_type != SAS_END_DEVICE);
+
+	sas_ha = dev->port->ha;
+
+	sas_read_port_mode_page(scsi_dev);
+
+	if (scsi_dev->tagged_supported) {
+		scsi_set_tag_type(scsi_dev, MSG_SIMPLE_TAG);
+		scsi_activate_tcq(scsi_dev, SAS_DEF_QD);
+	} else {
+		SAS_DPRINTK("device %llx, LUN %x doesn't support "
+			    "TCQ\n", SAS_ADDR(dev->sas_addr),
+			    scsi_dev->lun);
+		scsi_dev->tagged_supported = 0;
+		scsi_set_tag_type(scsi_dev, 0);
+		scsi_deactivate_tcq(scsi_dev, 1);
+	}
+
+	return 0;
+}
+
+void sas_slave_destroy(struct scsi_device *scsi_dev)
+{
+}
+
+int sas_change_queue_depth(struct scsi_device *scsi_dev, int new_depth)
+{
+	int res = min(new_depth, SAS_MAX_QD);
+
+	if (scsi_dev->tagged_supported)
+		scsi_adjust_queue_depth(scsi_dev, scsi_get_tag_type(scsi_dev),
+					res);
+	else {
+		struct domain_device *dev = sdev_to_domain_dev(scsi_dev);
+		sas_printk("device %llx LUN %x queue depth changed to 1\n",
+			   SAS_ADDR(dev->sas_addr),
+			   scsi_dev->lun);
+		scsi_adjust_queue_depth(scsi_dev, 0, 1);
+		res = 1;
+	}
+
+	return res;
+}
+
+int sas_change_queue_type(struct scsi_device *scsi_dev, int qt)
+{
+	if (!scsi_dev->tagged_supported)
+		return 0;
+
+	scsi_deactivate_tcq(scsi_dev, 1);
+
+	scsi_set_tag_type(scsi_dev, qt);
+	scsi_activate_tcq(scsi_dev, scsi_dev->queue_depth);
+
+	return qt;
+}
+
+int sas_bios_param(struct scsi_device *scsi_dev,
+			  struct block_device *bdev,
+			  sector_t capacity, int *hsc)
+{
+	hsc[0] = 255;
+	hsc[1] = 63;
+	sector_div(capacity, 255*63);
+	hsc[2] = capacity;
+
+	return 0;
+}
+
+/* ---------- Task Collector Thread implementation ---------- */
+
+static void sas_queue(struct sas_ha_struct *sas_ha)
+{
+	struct scsi_core *core = &sas_ha->core;
+	unsigned long flags;
+	LIST_HEAD(q);
+	int can_queue;
+	int res;
+	struct sas_internal *i = to_sas_internal(core->shost->transportt);
+
+	spin_lock_irqsave(&core->task_queue_lock, flags);
+	while (!core->queue_thread_kill &&
+	       !list_empty(&core->task_queue)) {
+
+		can_queue = sas_ha->lldd_queue_size - core->task_queue_size;
+		if (can_queue >= 0) {
+			can_queue = core->task_queue_size;
+			list_splice_init(&core->task_queue, &q);
+		} else {
+			struct list_head *a, *n;
+
+			can_queue = sas_ha->lldd_queue_size;
+			list_for_each_safe(a, n, &core->task_queue) {
+				list_move_tail(a, &q);
+				if (--can_queue == 0)
+					break;
+			}
+			can_queue = sas_ha->lldd_queue_size;
+		}
+		core->task_queue_size -= can_queue;
+		spin_unlock_irqrestore(&core->task_queue_lock, flags);
+		{
+			struct sas_task *task = list_entry(q.next,
+							   struct sas_task,
+							   list);
+			list_del_init(&q);
+			res = i->dft->lldd_execute_task(task, can_queue,
+							GFP_KERNEL);
+			if (unlikely(res))
+				__list_add(&q, task->list.prev, &task->list);
+		}
+		spin_lock_irqsave(&core->task_queue_lock, flags);
+		if (res) {
+			list_splice_init(&q, &core->task_queue); /*at head*/
+			core->task_queue_size += can_queue;
+		}
+	}
+	spin_unlock_irqrestore(&core->task_queue_lock, flags);
+}
+
+static DECLARE_COMPLETION(queue_th_comp);
+
+/**
+ * sas_queue_thread -- The Task Collector thread
+ * @_sas_ha: pointer to struct sas_ha
+ */
+static int sas_queue_thread(void *_sas_ha)
+{
+	struct sas_ha_struct *sas_ha = _sas_ha;
+	struct scsi_core *core = &sas_ha->core;
+
+	daemonize("sas_queue_%d", core->shost->host_no);
+	current->flags |= PF_NOFREEZE;
+
+	complete(&queue_th_comp);
+
+	while (1) {
+		down_interruptible(&core->queue_thread_sema);
+		sas_queue(sas_ha);
+		if (core->queue_thread_kill)
+			break;
+	}
+
+	complete(&queue_th_comp);
+
+	return 0;
+}
+
+int sas_init_queue(struct sas_ha_struct *sas_ha)
+{
+	int res;
+	struct scsi_core *core = &sas_ha->core;
+
+	spin_lock_init(&core->task_queue_lock);
+	core->task_queue_size = 0;
+	INIT_LIST_HEAD(&core->task_queue);
+	init_MUTEX_LOCKED(&core->queue_thread_sema);
+
+	res = kernel_thread(sas_queue_thread, sas_ha, 0);
+	if (res >= 0)
+		wait_for_completion(&queue_th_comp);
+
+	return res < 0 ? res : 0;
+}
+
+void sas_shutdown_queue(struct sas_ha_struct *sas_ha)
+{
+	unsigned long flags;
+	struct scsi_core *core = &sas_ha->core;
+	struct sas_task *task, *n;
+
+	init_completion(&queue_th_comp);
+	core->queue_thread_kill = 1;
+	up(&core->queue_thread_sema);
+	wait_for_completion(&queue_th_comp);
+
+	if (!list_empty(&core->task_queue))
+		SAS_DPRINTK("HA: %llx: scsi core task queue is NOT empty!?\n",
+			    SAS_ADDR(sas_ha->sas_addr));
+
+	spin_lock_irqsave(&core->task_queue_lock, flags);
+	list_for_each_entry_safe(task, n, &core->task_queue, list) {
+		struct scsi_cmnd *cmd = task->uldd_task;
+
+		list_del_init(&task->list);
+
+		ASSIGN_SAS_TASK(cmd, NULL);
+		sas_free_task(task);
+		cmd->result = DID_ABORT << 16;
+		cmd->scsi_done(cmd);
+	}
+	spin_unlock_irqrestore(&core->task_queue_lock, flags);
+}
+
+EXPORT_SYMBOL_GPL(sas_queuecommand);
+EXPORT_SYMBOL_GPL(sas_target_alloc);
+EXPORT_SYMBOL_GPL(sas_slave_configure);
+EXPORT_SYMBOL_GPL(sas_slave_destroy);
+EXPORT_SYMBOL_GPL(sas_change_queue_depth);
+EXPORT_SYMBOL_GPL(sas_change_queue_type);
+EXPORT_SYMBOL_GPL(sas_bios_param);
diff --git a/drivers/scsi/lpfc/lpfc.h b/drivers/scsi/lpfc/lpfc.h
index d44f9aa..3f7f5f8 100644
--- a/drivers/scsi/lpfc/lpfc.h
+++ b/drivers/scsi/lpfc/lpfc.h
@@ -285,6 +285,7 @@ struct lpfc_hba {
 	uint32_t cfg_log_verbose;
 	uint32_t cfg_lun_queue_depth;
 	uint32_t cfg_nodev_tmo;
+	uint32_t cfg_devloss_tmo;
 	uint32_t cfg_hba_queue_depth;
 	uint32_t cfg_fcp_class;
 	uint32_t cfg_use_adisc;
@@ -302,6 +303,9 @@ struct lpfc_hba {
 	uint32_t cfg_poll_tmo;
 	uint32_t cfg_sg_seg_cnt;
 	uint32_t cfg_sg_dma_buf_size;
+	uint64_t cfg_soft_wwpn;
+
+	uint32_t dev_loss_tmo_changed;
 
 	lpfc_vpd_t vpd;		/* vital product data */
 
@@ -351,6 +355,8 @@ struct lpfc_hba {
 #define VPD_PORT            0x8         /* valid vpd port data */
 #define VPD_MASK            0xf         /* mask for any vpd data */
 
+	uint8_t soft_wwpn_enable;
+
 	struct timer_list fcp_poll_timer;
 	struct timer_list els_tmofunc;
 
@@ -391,3 +397,5 @@ struct rnidrsp {
 	struct list_head list;
 	uint32_t data;
 };
+
+#define FC_REG_DUMP_EVENT	0x10	/* Register for Dump events */
diff --git a/drivers/scsi/lpfc/lpfc_attr.c b/drivers/scsi/lpfc/lpfc_attr.c
index d384c16..9496e87 100644
--- a/drivers/scsi/lpfc/lpfc_attr.c
+++ b/drivers/scsi/lpfc/lpfc_attr.c
@@ -39,6 +39,9 @@
 #include "lpfc_compat.h"
 #include "lpfc_crtn.h"
 
+#define LPFC_DEF_DEVLOSS_TMO 30
+#define LPFC_MIN_DEVLOSS_TMO 1
+#define LPFC_MAX_DEVLOSS_TMO 255
 
 static void
 lpfc_jedec_to_ascii(int incr, char hdw[])
@@ -548,6 +551,119 @@ static CLASS_DEVICE_ATTR(board_mode, S_IRUGO | S_IWUSR,
 			 lpfc_board_mode_show, lpfc_board_mode_store);
 static CLASS_DEVICE_ATTR(issue_reset, S_IWUSR, NULL, lpfc_issue_reset);
 
+
+static char *lpfc_soft_wwpn_key = "C99G71SL8032A";
+
+static ssize_t
+lpfc_soft_wwpn_enable_store(struct class_device *cdev, const char *buf,
+				size_t count)
+{
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct lpfc_hba *phba = (struct lpfc_hba*)host->hostdata;
+	unsigned int cnt = count;
+
+	/*
+	 * We're doing a simple sanity check for soft_wwpn setting.
+	 * We require that the user write a specific key to enable
+	 * the soft_wwpn attribute to be settable. Once the attribute
+	 * is written, the enable key resets. If further updates are
+	 * desired, the key must be written again to re-enable the
+	 * attribute.
+	 *
+	 * The "key" is not secret - it is a hardcoded string shown
+	 * here. The intent is to protect against the random user or
+	 * application that is just writing attributes.
+	 */
+
+	/* count may include a LF at end of string */
+	if (buf[cnt-1] == '\n')
+		cnt--;
+
+	if ((cnt != strlen(lpfc_soft_wwpn_key)) ||
+	    (strncmp(buf, lpfc_soft_wwpn_key, strlen(lpfc_soft_wwpn_key)) != 0))
+		return -EINVAL;
+
+	phba->soft_wwpn_enable = 1;
+	return count;
+}
+static CLASS_DEVICE_ATTR(lpfc_soft_wwpn_enable, S_IWUSR, NULL,
+				lpfc_soft_wwpn_enable_store);
+
+static ssize_t
+lpfc_soft_wwpn_show(struct class_device *cdev, char *buf)
+{
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct lpfc_hba *phba = (struct lpfc_hba*)host->hostdata;
+	return snprintf(buf, PAGE_SIZE, "0x%llx\n", phba->cfg_soft_wwpn);
+}
+
+
+static ssize_t
+lpfc_soft_wwpn_store(struct class_device *cdev, const char *buf, size_t count)
+{
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct lpfc_hba *phba = (struct lpfc_hba*)host->hostdata;
+	struct completion online_compl;
+	int stat1=0, stat2=0;
+	unsigned int i, j, cnt=count;
+	u8 wwpn[8];
+
+	/* count may include a LF at end of string */
+	if (buf[cnt-1] == '\n')
+		cnt--;
+
+	if (!phba->soft_wwpn_enable || (cnt < 16) || (cnt > 18) ||
+	    ((cnt == 17) && (*buf++ != 'x')) ||
+	    ((cnt == 18) && ((*buf++ != '0') || (*buf++ != 'x'))))
+		return -EINVAL;
+
+	phba->soft_wwpn_enable = 0;
+
+	memset(wwpn, 0, sizeof(wwpn));
+
+	/* Validate and store the new name */
+	for (i=0, j=0; i < 16; i++) {
+		if ((*buf >= 'a') && (*buf <= 'f'))
+			j = ((j << 4) | ((*buf++ -'a') + 10));
+		else if ((*buf >= 'A') && (*buf <= 'F'))
+			j = ((j << 4) | ((*buf++ -'A') + 10));
+		else if ((*buf >= '0') && (*buf <= '9'))
+			j = ((j << 4) | (*buf++ -'0'));
+		else
+			return -EINVAL;
+		if (i % 2) {
+			wwpn[i/2] = j & 0xff;
+			j = 0;
+		}
+	}
+	phba->cfg_soft_wwpn = wwn_to_u64(wwpn);
+	fc_host_port_name(host) = phba->cfg_soft_wwpn;
+
+	dev_printk(KERN_NOTICE, &phba->pcidev->dev,
+		   "lpfc%d: Reinitializing to use soft_wwpn\n", phba->brd_no);
+
+	init_completion(&online_compl);
+	lpfc_workq_post_event(phba, &stat1, &online_compl, LPFC_EVT_OFFLINE);
+	wait_for_completion(&online_compl);
+	if (stat1)
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+			"%d:0463 lpfc_soft_wwpn attribute set failed to reinit "
+			"adapter - %d\n", phba->brd_no, stat1);
+
+	init_completion(&online_compl);
+	lpfc_workq_post_event(phba, &stat2, &online_compl, LPFC_EVT_ONLINE);
+	wait_for_completion(&online_compl);
+	if (stat2)
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+			"%d:0464 lpfc_soft_wwpn attribute set failed to reinit "
+			"adapter - %d\n", phba->brd_no, stat2);
+
+	return (stat1 || stat2) ? -EIO : count;
+}
+static CLASS_DEVICE_ATTR(lpfc_soft_wwpn, S_IRUGO | S_IWUSR,\
+			 lpfc_soft_wwpn_show, lpfc_soft_wwpn_store);
+
+
 static int lpfc_poll = 0;
 module_param(lpfc_poll, int, 0);
 MODULE_PARM_DESC(lpfc_poll, "FCP ring polling mode control:"
@@ -559,6 +675,123 @@ static CLASS_DEVICE_ATTR(lpfc_poll, S_IRUGO | S_IWUSR,
 			 lpfc_poll_show, lpfc_poll_store);
 
 /*
+# lpfc_nodev_tmo: If set, it will hold all I/O errors on devices that disappear
+# until the timer expires. Value range is [0,255]. Default value is 30.
+*/
+static int lpfc_nodev_tmo = LPFC_DEF_DEVLOSS_TMO;
+static int lpfc_devloss_tmo = LPFC_DEF_DEVLOSS_TMO;
+module_param(lpfc_nodev_tmo, int, 0);
+MODULE_PARM_DESC(lpfc_nodev_tmo,
+		 "Seconds driver will hold I/O waiting "
+		 "for a device to come back");
+static ssize_t
+lpfc_nodev_tmo_show(struct class_device *cdev, char *buf)
+{
+	struct Scsi_Host *host = class_to_shost(cdev);
+	struct lpfc_hba *phba = (struct lpfc_hba*)host->hostdata;
+	int val = 0;
+	val = phba->cfg_devloss_tmo;
+	return snprintf(buf, PAGE_SIZE, "%d\n",
+			phba->cfg_devloss_tmo);
+}
+
+static int
+lpfc_nodev_tmo_init(struct lpfc_hba *phba, int val)
+{
+	static int warned;
+	if (phba->cfg_devloss_tmo !=  LPFC_DEF_DEVLOSS_TMO) {
+		phba->cfg_nodev_tmo = phba->cfg_devloss_tmo;
+		if (!warned && val != LPFC_DEF_DEVLOSS_TMO) {
+			warned = 1;
+			lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+					"%d:0402 Ignoring nodev_tmo module "
+					"parameter because devloss_tmo is"
+					" set.\n",
+					phba->brd_no);
+		}
+		return 0;
+	}
+
+	if (val >= LPFC_MIN_DEVLOSS_TMO && val <= LPFC_MAX_DEVLOSS_TMO) {
+		phba->cfg_nodev_tmo = val;
+		phba->cfg_devloss_tmo = val;
+		return 0;
+	}
+	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+			"%d:0400 lpfc_nodev_tmo attribute cannot be set to %d, "
+			"allowed range is [%d, %d]\n",
+			phba->brd_no, val,
+			LPFC_MIN_DEVLOSS_TMO, LPFC_MAX_DEVLOSS_TMO);
+	phba->cfg_nodev_tmo = LPFC_DEF_DEVLOSS_TMO;
+	return -EINVAL;
+}
+
+static int
+lpfc_nodev_tmo_set(struct lpfc_hba *phba, int val)
+{
+	if (phba->dev_loss_tmo_changed ||
+		(lpfc_devloss_tmo != LPFC_DEF_DEVLOSS_TMO)) {
+		lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+				"%d:0401 Ignoring change to nodev_tmo "
+				"because devloss_tmo is set.\n",
+				phba->brd_no);
+		return 0;
+	}
+
+	if (val >= LPFC_MIN_DEVLOSS_TMO && val <= LPFC_MAX_DEVLOSS_TMO) {
+		phba->cfg_nodev_tmo = val;
+		phba->cfg_devloss_tmo = val;
+		return 0;
+	}
+
+	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+			"%d:0403 lpfc_nodev_tmo attribute cannot be set to %d, "
+			"allowed range is [%d, %d]\n",
+			phba->brd_no, val, LPFC_MIN_DEVLOSS_TMO,
+			LPFC_MAX_DEVLOSS_TMO);
+	return -EINVAL;
+}
+
+lpfc_param_store(nodev_tmo)
+
+static CLASS_DEVICE_ATTR(lpfc_nodev_tmo, S_IRUGO | S_IWUSR,
+			 lpfc_nodev_tmo_show, lpfc_nodev_tmo_store);
+
+/*
+# lpfc_devloss_tmo: If set, it will hold all I/O errors on devices that
+# disappear until the timer expires. Value range is [0,255]. Default
+# value is 30.
+*/
+module_param(lpfc_devloss_tmo, int, 0);
+MODULE_PARM_DESC(lpfc_devloss_tmo,
+		 "Seconds driver will hold I/O waiting "
+		 "for a device to come back");
+lpfc_param_init(devloss_tmo, LPFC_DEF_DEVLOSS_TMO,
+		LPFC_MIN_DEVLOSS_TMO, LPFC_MAX_DEVLOSS_TMO)
+lpfc_param_show(devloss_tmo)
+static int
+lpfc_devloss_tmo_set(struct lpfc_hba *phba, int val)
+{
+	if (val >= LPFC_MIN_DEVLOSS_TMO && val <= LPFC_MAX_DEVLOSS_TMO) {
+		phba->cfg_nodev_tmo = val;
+		phba->cfg_devloss_tmo = val;
+		phba->dev_loss_tmo_changed = 1;
+		return 0;
+	}
+
+	lpfc_printf_log(phba, KERN_ERR, LOG_INIT,
+			"%d:0404 lpfc_devloss_tmo attribute cannot be set to"
+			" %d, allowed range is [%d, %d]\n",
+			phba->brd_no, val, LPFC_MIN_DEVLOSS_TMO,
+			LPFC_MAX_DEVLOSS_TMO);
+	return -EINVAL;
+}
+
+lpfc_param_store(devloss_tmo)
+static CLASS_DEVICE_ATTR(lpfc_devloss_tmo, S_IRUGO | S_IWUSR,
+	lpfc_devloss_tmo_show, lpfc_devloss_tmo_store);
+
+/*
 # lpfc_log_verbose: Only turn this flag on if you are willing to risk being
 # deluged with LOTS of information.
 # You can set a bit mask to record specific types of verbose messages:
@@ -617,14 +850,6 @@ LPFC_ATTR_R(scan_down, 1, 0, 1,
 	     "Start scanning for devices from highest ALPA to lowest");
 
 /*
-# lpfc_nodev_tmo: If set, it will hold all I/O errors on devices that disappear
-# until the timer expires. Value range is [0,255]. Default value is 30.
-# NOTE: this MUST be less then the SCSI Layer command timeout - 1.
-*/
-LPFC_ATTR_RW(nodev_tmo, 30, 0, 255,
-	     "Seconds driver will hold I/O waiting for a device to come back");
-
-/*
 # lpfc_topology:  link topology for init link
 #            0x0  = attempt loop mode then point-to-point
 #            0x01 = internal loopback mode
@@ -720,6 +945,7 @@ LPFC_ATTR_R(max_luns, 255, 0, 65535,
 LPFC_ATTR_RW(poll_tmo, 10, 1, 255,
 	     "Milliseconds driver will wait between polling FCP ring");
 
+
 struct class_device_attribute *lpfc_host_attrs[] = {
 	&class_device_attr_info,
 	&class_device_attr_serialnum,
@@ -737,6 +963,7 @@ struct class_device_attribute *lpfc_host_attrs[] = {
 	&class_device_attr_lpfc_lun_queue_depth,
 	&class_device_attr_lpfc_hba_queue_depth,
 	&class_device_attr_lpfc_nodev_tmo,
+	&class_device_attr_lpfc_devloss_tmo,
 	&class_device_attr_lpfc_fcp_class,
 	&class_device_attr_lpfc_use_adisc,
 	&class_device_attr_lpfc_ack0,
@@ -754,6 +981,8 @@ struct class_device_attribute *lpfc_host_attrs[] = {
 	&class_device_attr_issue_reset,
 	&class_device_attr_lpfc_poll,
 	&class_device_attr_lpfc_poll_tmo,
+	&class_device_attr_lpfc_soft_wwpn,
+	&class_device_attr_lpfc_soft_wwpn_enable,
 	NULL,
 };
 
@@ -1204,6 +1433,15 @@ lpfc_get_host_fabric_name (struct Scsi_Host *shost)
 	fc_host_fabric_name(shost) = node_name;
 }
 
+static void
+lpfc_get_host_symbolic_name (struct Scsi_Host *shost)
+{
+	struct lpfc_hba *phba = (struct lpfc_hba*)shost->hostdata;
+
+	spin_lock_irq(shost->host_lock);
+	lpfc_get_hba_sym_node_name(phba, fc_host_symbolic_name(shost));
+	spin_unlock_irq(shost->host_lock);
+}
 
 static struct fc_host_statistics *
 lpfc_get_stats(struct Scsi_Host *shost)
@@ -1441,27 +1679,12 @@ lpfc_get_starget_port_name(struct scsi_target *starget)
 }
 
 static void
-lpfc_get_rport_loss_tmo(struct fc_rport *rport)
-{
-	/*
-	 * Return the driver's global value for device loss timeout plus
-	 * five seconds to allow the driver's nodev timer to run.
-	 */
-	rport->dev_loss_tmo = lpfc_nodev_tmo + 5;
-}
-
-static void
 lpfc_set_rport_loss_tmo(struct fc_rport *rport, uint32_t timeout)
 {
-	/*
-	 * The driver doesn't have a per-target timeout setting.  Set
-	 * this value globally. lpfc_nodev_tmo should be greater then 0.
-	 */
 	if (timeout)
-		lpfc_nodev_tmo = timeout;
+		rport->dev_loss_tmo = timeout;
 	else
-		lpfc_nodev_tmo = 1;
-	rport->dev_loss_tmo = lpfc_nodev_tmo + 5;
+		rport->dev_loss_tmo = 1;
 }
 
 
@@ -1486,7 +1709,6 @@ struct fc_function_template lpfc_transport_functions = {
 	.show_host_port_name = 1,
 	.show_host_supported_classes = 1,
 	.show_host_supported_fc4s = 1,
-	.show_host_symbolic_name = 1,
 	.show_host_supported_speeds = 1,
 	.show_host_maxframe_size = 1,
 
@@ -1509,6 +1731,9 @@ struct fc_function_template lpfc_transport_functions = {
 	.get_host_fabric_name = lpfc_get_host_fabric_name,
 	.show_host_fabric_name = 1,
 
+	.get_host_symbolic_name = lpfc_get_host_symbolic_name,
+	.show_host_symbolic_name = 1,
+
 	/*
 	 * The LPFC driver treats linkdown handling as target loss events
 	 * so there are no sysfs handlers for link_down_tmo.
@@ -1521,7 +1746,6 @@ struct fc_function_template lpfc_transport_functions = {
 	.show_rport_maxframe_size = 1,
 	.show_rport_supported_classes = 1,
 
-	.get_rport_dev_loss_tmo = lpfc_get_rport_loss_tmo,
 	.set_rport_dev_loss_tmo = lpfc_set_rport_loss_tmo,
 	.show_rport_dev_loss_tmo = 1,
 
@@ -1535,6 +1759,8 @@ struct fc_function_template lpfc_transport_functions = {
 	.show_starget_port_name = 1,
 
 	.issue_fc_host_lip = lpfc_issue_lip,
+	.dev_loss_tmo_callbk = lpfc_dev_loss_tmo_callbk,
+	.terminate_rport_io = lpfc_terminate_rport_io,
 };
 
 void
@@ -1550,14 +1776,15 @@ lpfc_get_cfgparam(struct lpfc_hba *phba)
 	lpfc_ack0_init(phba, lpfc_ack0);
 	lpfc_topology_init(phba, lpfc_topology);
 	lpfc_scan_down_init(phba, lpfc_scan_down);
-	lpfc_nodev_tmo_init(phba, lpfc_nodev_tmo);
 	lpfc_link_speed_init(phba, lpfc_link_speed);
 	lpfc_fdmi_on_init(phba, lpfc_fdmi_on);
 	lpfc_discovery_threads_init(phba, lpfc_discovery_threads);
 	lpfc_max_luns_init(phba, lpfc_max_luns);
 	lpfc_poll_tmo_init(phba, lpfc_poll_tmo);
-
+	lpfc_devloss_tmo_init(phba, lpfc_devloss_tmo);
+	lpfc_nodev_tmo_init(phba, lpfc_nodev_tmo);
 	phba->cfg_poll = lpfc_poll;
+	phba->cfg_soft_wwpn = 0L;
 
 	/*
 	 * The total number of segments is the configuration value plus 2
diff --git a/drivers/scsi/lpfc/lpfc_crtn.h b/drivers/scsi/lpfc/lpfc_crtn.h
index 2a17646..3d68449 100644
--- a/drivers/scsi/lpfc/lpfc_crtn.h
+++ b/drivers/scsi/lpfc/lpfc_crtn.h
@@ -18,6 +18,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
+struct fc_rport;
 void lpfc_dump_mem(struct lpfc_hba *, LPFC_MBOXQ_t *, uint16_t);
 void lpfc_read_nv(struct lpfc_hba *, LPFC_MBOXQ_t *);
 int lpfc_read_la(struct lpfc_hba * phba, LPFC_MBOXQ_t * pmb,
@@ -200,6 +201,8 @@ extern struct scsi_host_template lpfc_template;
 extern struct fc_function_template lpfc_transport_functions;
 
 void lpfc_get_hba_sym_node_name(struct lpfc_hba * phba, uint8_t * symbp);
+void lpfc_terminate_rport_io(struct fc_rport *);
+void lpfc_dev_loss_tmo_callbk(struct fc_rport *rport);
 
 #define ScsiResult(host_code, scsi_code) (((host_code) << 16) | scsi_code)
 #define HBA_EVENT_RSCN                   5
diff --git a/drivers/scsi/lpfc/lpfc_ct.c b/drivers/scsi/lpfc/lpfc_ct.c
index bbb7310..ae41064 100644
--- a/drivers/scsi/lpfc/lpfc_ct.c
+++ b/drivers/scsi/lpfc/lpfc_ct.c
@@ -324,7 +324,6 @@ lpfc_ns_rsp(struct lpfc_hba * phba, struct lpfc_dmabuf * mp, uint32_t Size)
 	struct lpfc_sli_ct_request *Response =
 		(struct lpfc_sli_ct_request *) mp->virt;
 	struct lpfc_nodelist *ndlp = NULL;
-	struct lpfc_nodelist *next_ndlp;
 	struct lpfc_dmabuf *mlast, *next_mp;
 	uint32_t *ctptr = (uint32_t *) & Response->un.gid.PortType;
 	uint32_t Did;
@@ -399,30 +398,6 @@ nsout1:
  	 * current driver state.
  	 */
 	if (phba->hba_state == LPFC_HBA_READY) {
-
-		/*
-		 * Switch ports that connect a loop of multiple targets need
-		 * special consideration.  The driver wants to unregister the
-		 * rpi only on the target that was pulled from the loop.  On
-		 * RSCN, the driver wants to rediscover an NPort only if the
-		 * driver flagged it as NLP_NPR_2B_DISC.  Provided adisc is
-		 * not enabled and the NPort is not capable of retransmissions
-		 * (FC Tape) prevent timing races with the scsi error handler by
-		 * unregistering the Nport's RPI.  This action causes all
-		 * outstanding IO to flush back to the midlayer.
-		 */
-		list_for_each_entry_safe(ndlp, next_ndlp, &phba->fc_npr_list,
-					 nlp_listp) {
-			if (!(ndlp->nlp_flag & NLP_NPR_2B_DISC) &&
-			    (lpfc_rscn_payload_check(phba, ndlp->nlp_DID))) {
-				if ((phba->cfg_use_adisc == 0) &&
-				    !(ndlp->nlp_fcp_info &
-				      NLP_FCP_2_DEVICE)) {
-					lpfc_unreg_rpi(phba, ndlp);
-					ndlp->nlp_flag &= ~NLP_NPR_ADISC;
-				}
-			}
-		}
 		lpfc_els_flush_rscn(phba);
 		spin_lock_irq(phba->host->host_lock);
 		phba->fc_flag |= FC_RSCN_MODE; /* we are still in RSCN mode */
diff --git a/drivers/scsi/lpfc/lpfc_disc.h b/drivers/scsi/lpfc/lpfc_disc.h
index 41cf5d3..9766f90 100644
--- a/drivers/scsi/lpfc/lpfc_disc.h
+++ b/drivers/scsi/lpfc/lpfc_disc.h
@@ -30,7 +30,6 @@
 
 /* worker thread events */
 enum lpfc_work_type {
-	LPFC_EVT_NODEV_TMO,
 	LPFC_EVT_ONLINE,
 	LPFC_EVT_OFFLINE,
 	LPFC_EVT_WARM_START,
@@ -74,11 +73,9 @@ struct lpfc_nodelist {
 #define NLP_FCP_2_DEVICE   0x10			/* FCP-2 device */
 
 	struct timer_list   nlp_delayfunc;	/* Used for delayed ELS cmds */
-	struct timer_list   nlp_tmofunc;	/* Used for nodev tmo */
 	struct fc_rport *rport;			/* Corresponding FC transport
 						   port structure */
 	struct lpfc_hba      *nlp_phba;
-	struct lpfc_work_evt nodev_timeout_evt;
 	struct lpfc_work_evt els_retry_evt;
 	unsigned long last_ramp_up_time;        /* jiffy of last ramp up */
 	unsigned long last_q_full_time;		/* jiffy of last queue full */
@@ -102,7 +99,6 @@ struct lpfc_nodelist {
 #define NLP_LOGO_SND       0x100	/* sent LOGO request for this entry */
 #define NLP_RNID_SND       0x400	/* sent RNID request for this entry */
 #define NLP_ELS_SND_MASK   0x7e0	/* sent ELS request for this entry */
-#define NLP_NODEV_TMO      0x10000	/* nodev timeout is running for node */
 #define NLP_DELAY_TMO      0x20000	/* delay timeout is running for node */
 #define NLP_NPR_2B_DISC    0x40000	/* node is included in num_disc_nodes */
 #define NLP_RCV_PLOGI      0x80000	/* Rcv'ed PLOGI from remote system */
@@ -169,7 +165,7 @@ struct lpfc_nodelist {
  */
 /*
  * For a Link Down, all nodes on the ADISC, PLOGI, unmapped or mapped
- * lists will receive a DEVICE_RECOVERY event. If the linkdown or nodev timers
+ * lists will receive a DEVICE_RECOVERY event. If the linkdown or devloss timers
  * expire, all effected nodes will receive a DEVICE_RM event.
  */
 /*
diff --git a/drivers/scsi/lpfc/lpfc_els.c b/drivers/scsi/lpfc/lpfc_els.c
index 3567de61..71864cdc 100644
--- a/drivers/scsi/lpfc/lpfc_els.c
+++ b/drivers/scsi/lpfc/lpfc_els.c
@@ -2506,6 +2506,7 @@ lpfc_els_rcv_rscn(struct lpfc_hba * phba,
 	uint32_t *lp;
 	IOCB_t *icmd;
 	uint32_t payload_len, cmd;
+	int i;
 
 	icmd = &cmdiocb->iocb;
 	pcmd = (struct lpfc_dmabuf *) cmdiocb->context2;
@@ -2524,6 +2525,10 @@ lpfc_els_rcv_rscn(struct lpfc_hba * phba,
 			phba->brd_no,
 			phba->fc_flag, payload_len, *lp, phba->fc_rscn_id_cnt);
 
+	for (i = 0; i < payload_len/sizeof(uint32_t); i++)
+		fc_host_post_event(phba->host, fc_get_event_number(),
+			FCH_EVT_RSCN, lp[i]);
+
 	/* If we are about to begin discovery, just ACC the RSCN.
 	 * Discovery processing will satisfy it.
 	 */
diff --git a/drivers/scsi/lpfc/lpfc_hbadisc.c b/drivers/scsi/lpfc/lpfc_hbadisc.c
index b2f1552..d586c3d 100644
--- a/drivers/scsi/lpfc/lpfc_hbadisc.c
+++ b/drivers/scsi/lpfc/lpfc_hbadisc.c
@@ -56,28 +56,63 @@ static uint8_t lpfcAlpaArray[] = {
 
 static void lpfc_disc_timeout_handler(struct lpfc_hba *);
 
-static void
-lpfc_process_nodev_timeout(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
+void
+lpfc_terminate_rport_io(struct fc_rport *rport)
 {
-	uint8_t *name = (uint8_t *)&ndlp->nlp_portname;
-	int warn_on = 0;
+	struct lpfc_rport_data *rdata;
+	struct lpfc_nodelist * ndlp;
+	struct lpfc_hba *phba;
 
-	spin_lock_irq(phba->host->host_lock);
-	if (!(ndlp->nlp_flag & NLP_NODEV_TMO)) {
-		spin_unlock_irq(phba->host->host_lock);
+	rdata = rport->dd_data;
+	ndlp = rdata->pnode;
+
+	if (!ndlp) {
+		if (rport->roles & FC_RPORT_ROLE_FCP_TARGET)
+			printk(KERN_ERR "Cannot find remote node"
+			" to terminate I/O Data x%x\n",
+			rport->port_id);
 		return;
 	}
 
-	/*
-	 * If a discovery event readded nodev_timer after timer
-	 * firing and before processing the timer, cancel the
-	 * nlp_tmofunc.
-	 */
-	spin_unlock_irq(phba->host->host_lock);
-	del_timer_sync(&ndlp->nlp_tmofunc);
+	phba = ndlp->nlp_phba;
+
 	spin_lock_irq(phba->host->host_lock);
+	if (ndlp->nlp_sid != NLP_NO_SID) {
+		lpfc_sli_abort_iocb(phba, &phba->sli.ring[phba->sli.fcp_ring],
+			ndlp->nlp_sid, 0, 0, LPFC_CTX_TGT);
+	}
+	spin_unlock_irq(phba->host->host_lock);
 
-	ndlp->nlp_flag &= ~NLP_NODEV_TMO;
+	return;
+}
+
+/*
+ * This function will be called when dev_loss_tmo fire.
+ */
+void
+lpfc_dev_loss_tmo_callbk(struct fc_rport *rport)
+{
+	struct lpfc_rport_data *rdata;
+	struct lpfc_nodelist * ndlp;
+	uint8_t *name;
+	int warn_on = 0;
+	struct lpfc_hba *phba;
+
+	rdata = rport->dd_data;
+	ndlp = rdata->pnode;
+
+	if (!ndlp) {
+		if (rport->roles & FC_RPORT_ROLE_FCP_TARGET)
+			printk(KERN_ERR "Cannot find remote node"
+			" for rport in dev_loss_tmo_callbk x%x\n",
+			rport->port_id);
+		return;
+	}
+
+	name = (uint8_t *)&ndlp->nlp_portname;
+	phba = ndlp->nlp_phba;
+
+	spin_lock_irq(phba->host->host_lock);
 
 	if (ndlp->nlp_sid != NLP_NO_SID) {
 		warn_on = 1;
@@ -85,11 +120,14 @@ lpfc_process_nodev_timeout(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
 		lpfc_sli_abort_iocb(phba, &phba->sli.ring[phba->sli.fcp_ring],
 			ndlp->nlp_sid, 0, 0, LPFC_CTX_TGT);
 	}
+	if (phba->fc_flag & FC_UNLOADING)
+		warn_on = 0;
+
 	spin_unlock_irq(phba->host->host_lock);
 
 	if (warn_on) {
 		lpfc_printf_log(phba, KERN_ERR, LOG_DISCOVERY,
-				"%d:0203 Nodev timeout on "
+				"%d:0203 Devloss timeout on "
 				"WWPN %x:%x:%x:%x:%x:%x:%x:%x "
 				"NPort x%x Data: x%x x%x x%x\n",
 				phba->brd_no,
@@ -99,7 +137,7 @@ lpfc_process_nodev_timeout(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
 				ndlp->nlp_state, ndlp->nlp_rpi);
 	} else {
 		lpfc_printf_log(phba, KERN_INFO, LOG_DISCOVERY,
-				"%d:0204 Nodev timeout on "
+				"%d:0204 Devloss timeout on "
 				"WWPN %x:%x:%x:%x:%x:%x:%x:%x "
 				"NPort x%x Data: x%x x%x x%x\n",
 				phba->brd_no,
@@ -109,7 +147,12 @@ lpfc_process_nodev_timeout(struct lpfc_hba *phba, struct lpfc_nodelist *ndlp)
 				ndlp->nlp_state, ndlp->nlp_rpi);
 	}
 
-	lpfc_disc_state_machine(phba, ndlp, NULL, NLP_EVT_DEVICE_RM);
+	ndlp->rport = NULL;
+	rdata->pnode = NULL;
+
+	if (!(phba->fc_flag & FC_UNLOADING))
+		lpfc_disc_state_machine(phba, ndlp, NULL, NLP_EVT_DEVICE_RM);
+
 	return;
 }
 
@@ -127,11 +170,6 @@ lpfc_work_list_done(struct lpfc_hba * phba)
 		spin_unlock_irq(phba->host->host_lock);
 		free_evt = 1;
 		switch (evtp->evt) {
-		case LPFC_EVT_NODEV_TMO:
-			ndlp = (struct lpfc_nodelist *)(evtp->evt_arg1);
-			lpfc_process_nodev_timeout(phba, ndlp);
-			free_evt = 0;
-			break;
 		case LPFC_EVT_ELS_RETRY:
 			ndlp = (struct lpfc_nodelist *)(evtp->evt_arg1);
 			lpfc_els_retry_delay_handler(ndlp);
@@ -340,6 +378,9 @@ lpfc_linkdown(struct lpfc_hba * phba)
 		spin_unlock_irq(phba->host->host_lock);
 	}
 
+	fc_host_post_event(phba->host, fc_get_event_number(),
+			FCH_EVT_LINKDOWN, 0);
+
 	/* Clean up any firmware default rpi's */
 	if ((mb = mempool_alloc(phba->mbox_mem_pool, GFP_KERNEL))) {
 		lpfc_unreg_did(phba, 0xffffffff, mb);
@@ -374,16 +415,6 @@ lpfc_linkdown(struct lpfc_hba * phba)
 			rc = lpfc_disc_state_machine(phba, ndlp, NULL,
 					     NLP_EVT_DEVICE_RECOVERY);
 
-			/* Check config parameter use-adisc or FCP-2 */
-			if ((rc != NLP_STE_FREED_NODE) &&
-				(phba->cfg_use_adisc == 0) &&
-				!(ndlp->nlp_fcp_info & NLP_FCP_2_DEVICE)) {
-				/* We know we will have to relogin, so
-				 * unreglogin the rpi right now to fail
-				 * any outstanding I/Os quickly.
-				 */
-				lpfc_unreg_rpi(phba, ndlp);
-			}
 		}
 	}
 
@@ -427,6 +458,9 @@ lpfc_linkup(struct lpfc_hba * phba)
 	struct list_head *listp, *node_list[7];
 	int i;
 
+	fc_host_post_event(phba->host, fc_get_event_number(),
+			FCH_EVT_LINKUP, 0);
+
 	spin_lock_irq(phba->host->host_lock);
 	phba->hba_state = LPFC_LINK_UP;
 	phba->fc_flag &= ~(FC_PT2PT | FC_PT2PT_PLOGI | FC_ABORT_DISCOVERY |
@@ -638,6 +672,8 @@ lpfc_mbx_cmpl_read_sparam(struct lpfc_hba * phba, LPFC_MBOXQ_t * pmb)
 
 	memcpy((uint8_t *) & phba->fc_sparam, (uint8_t *) mp->virt,
 	       sizeof (struct serv_parm));
+	if (phba->cfg_soft_wwpn)
+		u64_to_wwn(phba->cfg_soft_wwpn, phba->fc_sparam.portName.u.wwn);
 	memcpy((uint8_t *) & phba->fc_nodename,
 	       (uint8_t *) & phba->fc_sparam.nodeName,
 	       sizeof (struct lpfc_name));
@@ -1098,8 +1134,11 @@ lpfc_unregister_remote_port(struct lpfc_hba * phba,
 	struct fc_rport *rport = ndlp->rport;
 	struct lpfc_rport_data *rdata = rport->dd_data;
 
-	ndlp->rport = NULL;
-	rdata->pnode = NULL;
+	if (rport->scsi_target_id == -1) {
+		ndlp->rport = NULL;
+		rdata->pnode = NULL;
+	}
+
 	fc_remote_port_delete(rport);
 
 	return;
@@ -1227,17 +1266,6 @@ lpfc_nlp_list(struct lpfc_hba * phba, struct lpfc_nodelist * nlp, int list)
 		list_add_tail(&nlp->nlp_listp, &phba->fc_nlpunmap_list);
 		phba->fc_unmap_cnt++;
 		phba->nport_event_cnt++;
-		/* stop nodev tmo if running */
-		if (nlp->nlp_flag & NLP_NODEV_TMO) {
-			nlp->nlp_flag &= ~NLP_NODEV_TMO;
-			spin_unlock_irq(phba->host->host_lock);
-			del_timer_sync(&nlp->nlp_tmofunc);
-			spin_lock_irq(phba->host->host_lock);
-			if (!list_empty(&nlp->nodev_timeout_evt.evt_listp))
-				list_del_init(&nlp->nodev_timeout_evt.
-						evt_listp);
-
-		}
 		nlp->nlp_flag &= ~NLP_NODEV_REMOVE;
 		nlp->nlp_type |= NLP_FC_NODE;
 		break;
@@ -1248,17 +1276,6 @@ lpfc_nlp_list(struct lpfc_hba * phba, struct lpfc_nodelist * nlp, int list)
 		list_add_tail(&nlp->nlp_listp, &phba->fc_nlpmap_list);
 		phba->fc_map_cnt++;
 		phba->nport_event_cnt++;
-		/* stop nodev tmo if running */
-		if (nlp->nlp_flag & NLP_NODEV_TMO) {
-			nlp->nlp_flag &= ~NLP_NODEV_TMO;
-			spin_unlock_irq(phba->host->host_lock);
-			del_timer_sync(&nlp->nlp_tmofunc);
-			spin_lock_irq(phba->host->host_lock);
-			if (!list_empty(&nlp->nodev_timeout_evt.evt_listp))
-				list_del_init(&nlp->nodev_timeout_evt.
-						evt_listp);
-
-		}
 		nlp->nlp_flag &= ~NLP_NODEV_REMOVE;
 		break;
 	case NLP_NPR_LIST:
@@ -1267,11 +1284,6 @@ lpfc_nlp_list(struct lpfc_hba * phba, struct lpfc_nodelist * nlp, int list)
 		list_add_tail(&nlp->nlp_listp, &phba->fc_npr_list);
 		phba->fc_npr_cnt++;
 
-		if (!(nlp->nlp_flag & NLP_NODEV_TMO))
-			mod_timer(&nlp->nlp_tmofunc,
-		 			jiffies + HZ * phba->cfg_nodev_tmo);
-
-		nlp->nlp_flag |= NLP_NODEV_TMO;
 		nlp->nlp_flag &= ~NLP_RCV_PLOGI;
 		break;
 	case NLP_JUST_DQ:
@@ -1301,7 +1313,8 @@ lpfc_nlp_list(struct lpfc_hba * phba, struct lpfc_nodelist * nlp, int list)
 			 * already. If we have, and it's a scsi entity, be
 			 * sure to unblock any attached scsi devices
 			 */
-			if (!nlp->rport)
+			if ((!nlp->rport) || (nlp->rport->port_state ==
+					FC_PORTSTATE_BLOCKED))
 				lpfc_register_remote_port(phba, nlp);
 
 			/*
@@ -1575,15 +1588,12 @@ lpfc_freenode(struct lpfc_hba * phba, struct lpfc_nodelist * ndlp)
 
 	lpfc_els_abort(phba,ndlp,0);
 	spin_lock_irq(phba->host->host_lock);
-	ndlp->nlp_flag &= ~(NLP_NODEV_TMO|NLP_DELAY_TMO);
+	ndlp->nlp_flag &= ~NLP_DELAY_TMO;
 	spin_unlock_irq(phba->host->host_lock);
-	del_timer_sync(&ndlp->nlp_tmofunc);
 
 	ndlp->nlp_last_elscmd = 0;
 	del_timer_sync(&ndlp->nlp_delayfunc);
 
-	if (!list_empty(&ndlp->nodev_timeout_evt.evt_listp))
-		list_del_init(&ndlp->nodev_timeout_evt.evt_listp);
 	if (!list_empty(&ndlp->els_retry_evt.evt_listp))
 		list_del_init(&ndlp->els_retry_evt.evt_listp);
 
@@ -1600,16 +1610,6 @@ lpfc_freenode(struct lpfc_hba * phba, struct lpfc_nodelist * ndlp)
 int
 lpfc_nlp_remove(struct lpfc_hba * phba, struct lpfc_nodelist * ndlp)
 {
-	if (ndlp->nlp_flag & NLP_NODEV_TMO) {
-		spin_lock_irq(phba->host->host_lock);
-		ndlp->nlp_flag &= ~NLP_NODEV_TMO;
-		spin_unlock_irq(phba->host->host_lock);
-		del_timer_sync(&ndlp->nlp_tmofunc);
-		if (!list_empty(&ndlp->nodev_timeout_evt.evt_listp))
-			list_del_init(&ndlp->nodev_timeout_evt.evt_listp);
-
-	}
-
 
 	if (ndlp->nlp_flag & NLP_DELAY_TMO) {
 		lpfc_cancel_retry_delay_tmo(phba, ndlp);
@@ -2424,34 +2424,6 @@ lpfc_disc_timeout_handler(struct lpfc_hba *phba)
 	return;
 }
 
-static void
-lpfc_nodev_timeout(unsigned long ptr)
-{
-	struct lpfc_hba *phba;
-	struct lpfc_nodelist *ndlp;
-	unsigned long iflag;
-	struct lpfc_work_evt  *evtp;
-
-	ndlp = (struct lpfc_nodelist *)ptr;
-	phba = ndlp->nlp_phba;
-	evtp = &ndlp->nodev_timeout_evt;
-	spin_lock_irqsave(phba->host->host_lock, iflag);
-
-	if (!list_empty(&evtp->evt_listp)) {
-		spin_unlock_irqrestore(phba->host->host_lock, iflag);
-		return;
-	}
-	evtp->evt_arg1  = ndlp;
-	evtp->evt       = LPFC_EVT_NODEV_TMO;
-	list_add_tail(&evtp->evt_listp, &phba->work_list);
-	if (phba->work_wait)
-		wake_up(phba->work_wait);
-
-	spin_unlock_irqrestore(phba->host->host_lock, iflag);
-	return;
-}
-
-
 /*
  * This routine handles processing a NameServer REG_LOGIN mailbox
  * command upon completion. It is setup in the LPFC_MBOXQ
@@ -2575,11 +2547,7 @@ lpfc_nlp_init(struct lpfc_hba * phba, struct lpfc_nodelist * ndlp,
 		 uint32_t did)
 {
 	memset(ndlp, 0, sizeof (struct lpfc_nodelist));
-	INIT_LIST_HEAD(&ndlp->nodev_timeout_evt.evt_listp);
 	INIT_LIST_HEAD(&ndlp->els_retry_evt.evt_listp);
-	init_timer(&ndlp->nlp_tmofunc);
-	ndlp->nlp_tmofunc.function = lpfc_nodev_timeout;
-	ndlp->nlp_tmofunc.data = (unsigned long)ndlp;
 	init_timer(&ndlp->nlp_delayfunc);
 	ndlp->nlp_delayfunc.function = lpfc_els_retry_delay;
 	ndlp->nlp_delayfunc.data = (unsigned long)ndlp;
diff --git a/drivers/scsi/lpfc/lpfc_init.c b/drivers/scsi/lpfc/lpfc_init.c
index f6948ff..4cdf346 100644
--- a/drivers/scsi/lpfc/lpfc_init.c
+++ b/drivers/scsi/lpfc/lpfc_init.c
@@ -268,6 +268,8 @@ lpfc_config_port_post(struct lpfc_hba * phba)
 	kfree(mp);
 	pmb->context1 = NULL;
 
+	if (phba->cfg_soft_wwpn)
+		u64_to_wwn(phba->cfg_soft_wwpn, phba->fc_sparam.portName.u.wwn);
 	memcpy(&phba->fc_nodename, &phba->fc_sparam.nodeName,
 	       sizeof (struct lpfc_name));
 	memcpy(&phba->fc_portname, &phba->fc_sparam.portName,
@@ -511,6 +513,7 @@ lpfc_handle_eratt(struct lpfc_hba * phba)
 {
 	struct lpfc_sli *psli = &phba->sli;
 	struct lpfc_sli_ring  *pring;
+	uint32_t event_data;
 
 	if (phba->work_hs & HS_FFER6) {
 		/* Re-establishing Link */
@@ -555,6 +558,11 @@ lpfc_handle_eratt(struct lpfc_hba * phba)
 				phba->brd_no, phba->work_hs,
 				phba->work_status[0], phba->work_status[1]);
 
+		event_data = FC_REG_DUMP_EVENT;
+		fc_host_post_vendor_event(phba->host, fc_get_event_number(),
+				sizeof(event_data), (char *) &event_data,
+				SCSI_NL_VID_TYPE_PCI | PCI_VENDOR_ID_EMULEX);
+
 		psli->sli_flag &= ~LPFC_SLI2_ACTIVE;
 		lpfc_offline(phba);
 		phba->hba_state = LPFC_HBA_ERROR;
diff --git a/drivers/scsi/lpfc/lpfc_nportdisc.c b/drivers/scsi/lpfc/lpfc_nportdisc.c
index 20449a8..d5f4150 100644
--- a/drivers/scsi/lpfc/lpfc_nportdisc.c
+++ b/drivers/scsi/lpfc/lpfc_nportdisc.c
@@ -1813,7 +1813,7 @@ lpfc_device_recov_npr_node(struct lpfc_hba * phba,
  */
 /*
  * For a Link Down, all nodes on the ADISC, PLOGI, unmapped or mapped
- * lists will receive a DEVICE_RECOVERY event. If the linkdown or nodev timers
+ * lists will receive a DEVICE_RECOVERY event. If the linkdown or devloss timers
  * expire, all effected nodes will receive a DEVICE_RM event.
  */
 /*
diff --git a/drivers/scsi/lpfc/lpfc_scsi.c b/drivers/scsi/lpfc/lpfc_scsi.c
index a8816a8..97ae98d 100644
--- a/drivers/scsi/lpfc/lpfc_scsi.c
+++ b/drivers/scsi/lpfc/lpfc_scsi.c
@@ -935,7 +935,7 @@ lpfc_abort_handler(struct scsi_cmnd *cmnd)
 			schedule_timeout_uninterruptible(LPFC_ABORT_WAIT*HZ);
 		spin_lock_irq(phba->host->host_lock);
 		if (++loop_count
-		    > (2 * phba->cfg_nodev_tmo)/LPFC_ABORT_WAIT)
+		    > (2 * phba->cfg_devloss_tmo)/LPFC_ABORT_WAIT)
 			break;
 	}
 
@@ -978,7 +978,7 @@ lpfc_reset_lun_handler(struct scsi_cmnd *cmnd)
 	spin_lock_irq(shost->host_lock);
 	/*
 	 * If target is not in a MAPPED state, delay the reset until
-	 * target is rediscovered or nodev timeout expires.
+	 * target is rediscovered or devloss timeout expires.
 	 */
 	while ( 1 ) {
 		if (!pnode)
@@ -1050,7 +1050,7 @@ lpfc_reset_lun_handler(struct scsi_cmnd *cmnd)
 		spin_lock_irq(phba->host->host_lock);
 
 		if (++loopcnt
-		    > (2 * phba->cfg_nodev_tmo)/LPFC_RESET_WAIT)
+		    > (2 * phba->cfg_devloss_tmo)/LPFC_RESET_WAIT)
 			break;
 
 		cnt = lpfc_sli_sum_iocb(phba,
@@ -1151,7 +1151,7 @@ lpfc_reset_bus_handler(struct scsi_cmnd *cmnd)
 		spin_lock_irq(phba->host->host_lock);
 
 		if (++loopcnt
-		    > (2 * phba->cfg_nodev_tmo)/LPFC_RESET_WAIT)
+		    > (2 * phba->cfg_devloss_tmo)/LPFC_RESET_WAIT)
 			break;
 
 		cnt = lpfc_sli_sum_iocb(phba,
@@ -1249,7 +1249,7 @@ lpfc_slave_configure(struct scsi_device *sdev)
 	 * target pointer is stored in the starget_data for the
 	 * driver's sysfs entry point functions.
 	 */
-	rport->dev_loss_tmo = phba->cfg_nodev_tmo + 5;
+	rport->dev_loss_tmo = phba->cfg_devloss_tmo;
 
 	if (phba->cfg_poll & ENABLE_FCP_RING_POLLING) {
 		lpfc_sli_poll_fcp_ring(phba);
diff --git a/drivers/scsi/lpfc/lpfc_version.h b/drivers/scsi/lpfc/lpfc_version.h
index c7091ea..ac41790 100644
--- a/drivers/scsi/lpfc/lpfc_version.h
+++ b/drivers/scsi/lpfc/lpfc_version.h
@@ -18,7 +18,7 @@
  * included with this package.                                     *
  *******************************************************************/
 
-#define LPFC_DRIVER_VERSION "8.1.9"
+#define LPFC_DRIVER_VERSION "8.1.10"
 
 #define LPFC_DRIVER_NAME "lpfc"
 
diff --git a/drivers/scsi/mac53c94.c b/drivers/scsi/mac53c94.c
index 89ef34d..6422de7 100644
--- a/drivers/scsi/mac53c94.c
+++ b/drivers/scsi/mac53c94.c
@@ -431,7 +431,7 @@ static int mac53c94_probe(struct macio_dev *mdev, const struct of_device_id *mat
 	struct fsc_state *state;
 	struct Scsi_Host *host;
 	void *dma_cmd_space;
-	unsigned char *clkprop;
+	const unsigned char *clkprop;
 	int proplen, rc = -ENODEV;
 
 	if (macio_resource_count(mdev) != 2 || macio_irq_count(mdev) != 2) {
diff --git a/drivers/scsi/megaraid.c b/drivers/scsi/megaraid.c
index 76edbb6..b87bef6 100644
--- a/drivers/scsi/megaraid.c
+++ b/drivers/scsi/megaraid.c
@@ -2822,9 +2822,7 @@ mega_print_inquiry(char *page, char *scsi_inq)
 
 	i = scsi_inq[0] & 0x1f;
 
-	len += sprintf(page+len, "  Type:   %s ",
-		i < MAX_SCSI_DEVICE_CODE ? scsi_device_types[i] :
-		   "Unknown          ");
+	len += sprintf(page+len, "  Type:   %s ", scsi_device_type(i));
 
 	len += sprintf(page+len,
 	"                 ANSI SCSI revision: %02x", scsi_inq[2] & 0x07);
@@ -3658,8 +3656,9 @@ megadev_ioctl(struct inode *inode, struct file *filep, unsigned int cmd,
 			 * Send the request sense data also, irrespective of
 			 * whether the user has asked for it or not.
 			 */
-			copy_to_user(upthru->reqsensearea,
-					pthru->reqsensearea, 14);
+			if (copy_to_user(upthru->reqsensearea,
+					pthru->reqsensearea, 14))
+				rval = -EFAULT;
 
 freemem_and_return:
 			if( pthru->dataxferlen ) {
diff --git a/drivers/scsi/megaraid/megaraid_mbox.c b/drivers/scsi/megaraid/megaraid_mbox.c
index cd982c8..266b391 100644
--- a/drivers/scsi/megaraid/megaraid_mbox.c
+++ b/drivers/scsi/megaraid/megaraid_mbox.c
@@ -330,6 +330,21 @@ static struct device_attribute *megaraid_sdev_attrs[] = {
 	NULL,
 };
 
+/**
+ * megaraid_change_queue_depth - Change the device's queue depth
+ * @sdev:	scsi device struct
+ * @qdepth:	depth to set
+ *
+ * Return value:
+ * 	actual depth set
+ **/
+static int megaraid_change_queue_depth(struct scsi_device *sdev, int qdepth)
+{
+	if (qdepth > MBOX_MAX_SCSI_CMDS)
+		qdepth = MBOX_MAX_SCSI_CMDS;
+	scsi_adjust_queue_depth(sdev, 0, qdepth);
+	return sdev->queue_depth;
+}
 
 /*
  * Scsi host template for megaraid unified driver
@@ -343,6 +358,7 @@ static struct scsi_host_template megaraid_template_g = {
 	.eh_device_reset_handler	= megaraid_reset_handler,
 	.eh_bus_reset_handler		= megaraid_reset_handler,
 	.eh_host_reset_handler		= megaraid_reset_handler,
+	.change_queue_depth		= megaraid_change_queue_depth,
 	.use_clustering			= ENABLE_CLUSTERING,
 	.sdev_attrs			= megaraid_sdev_attrs,
 	.shost_attrs			= megaraid_shost_attrs,
diff --git a/drivers/scsi/megaraid/megaraid_sas.c b/drivers/scsi/megaraid/megaraid_sas.c
index a8c9627..4cab5b5 100644
--- a/drivers/scsi/megaraid/megaraid_sas.c
+++ b/drivers/scsi/megaraid/megaraid_sas.c
@@ -53,31 +53,15 @@ MODULE_DESCRIPTION("LSI Logic MegaRAID SAS Driver");
  */
 static struct pci_device_id megasas_pci_table[] = {
 
-	{
-	 PCI_VENDOR_ID_LSI_LOGIC,
-	 PCI_DEVICE_ID_LSI_SAS1064R, /* xscale IOP */
-	 PCI_ANY_ID,
-	 PCI_ANY_ID,
-	 },
-	{
-	 PCI_VENDOR_ID_LSI_LOGIC,
-	 PCI_DEVICE_ID_LSI_SAS1078R, /* ppc IOP */
-	 PCI_ANY_ID,
-	 PCI_ANY_ID,
-	},
-	{
-	 PCI_VENDOR_ID_LSI_LOGIC,
-	 PCI_DEVICE_ID_LSI_VERDE_ZCR,	/* xscale IOP, vega */
-	 PCI_ANY_ID,
-	 PCI_ANY_ID,
-	 },
-	{
-	 PCI_VENDOR_ID_DELL,
-	 PCI_DEVICE_ID_DELL_PERC5, /* xscale IOP */
-	 PCI_ANY_ID,
-	 PCI_ANY_ID,
-	 },
-	{0}			/* Terminating entry */
+	{PCI_DEVICE(PCI_VENDOR_ID_LSI_LOGIC, PCI_DEVICE_ID_LSI_SAS1064R)},
+	/* xscale IOP */
+	{PCI_DEVICE(PCI_VENDOR_ID_LSI_LOGIC, PCI_DEVICE_ID_LSI_SAS1078R)},
+	/* ppc IOP */
+	{PCI_DEVICE(PCI_VENDOR_ID_LSI_LOGIC, PCI_DEVICE_ID_LSI_VERDE_ZCR)},
+	/* xscale IOP, vega */
+	{PCI_DEVICE(PCI_VENDOR_ID_DELL, PCI_DEVICE_ID_DELL_PERC5)},
+	/* xscale IOP */
+	{}
 };
 
 MODULE_DEVICE_TABLE(pci, megasas_pci_table);
@@ -2854,7 +2838,7 @@ static int __init megasas_init(void)
 	/*
 	 * Register ourselves as PCI hotplug module
 	 */
-	rval = pci_module_init(&megasas_pci_driver);
+	rval = pci_register_driver(&megasas_pci_driver);
 
 	if (rval) {
 		printk(KERN_DEBUG "megasas: PCI hotplug regisration failed \n");
diff --git a/drivers/scsi/mesh.c b/drivers/scsi/mesh.c
index 5572981..592b52a 100644
--- a/drivers/scsi/mesh.c
+++ b/drivers/scsi/mesh.c
@@ -1850,7 +1850,8 @@ static int mesh_probe(struct macio_dev *mdev, const struct of_device_id *match)
 {
 	struct device_node *mesh = macio_get_of_node(mdev);
 	struct pci_dev* pdev = macio_get_pci_dev(mdev);
-	int tgt, *cfp, minper;
+	int tgt, minper;
+	const int *cfp;
 	struct mesh_state *ms;
 	struct Scsi_Host *mesh_host;
 	void *dma_cmd_space;
@@ -1939,7 +1940,7 @@ static int mesh_probe(struct macio_dev *mdev, const struct of_device_id *match)
 	       	ms->tgts[tgt].current_req = NULL;
        	}
 
-	if ((cfp = (int *) get_property(mesh, "clock-frequency", NULL)))
+	if ((cfp = get_property(mesh, "clock-frequency", NULL)))
        		ms->clk_freq = *cfp;
 	else {
        		printk(KERN_INFO "mesh: assuming 50MHz clock frequency\n");
diff --git a/drivers/scsi/mvme147.c b/drivers/scsi/mvme147.c
index cb367c2..9b991b7 100644
--- a/drivers/scsi/mvme147.c
+++ b/drivers/scsi/mvme147.c
@@ -29,7 +29,7 @@ static irqreturn_t mvme147_intr (int irq, void *dummy, struct pt_regs *fp)
     return IRQ_HANDLED;
 }
 
-static int dma_setup (Scsi_Cmnd *cmd, int dir_in)
+static int dma_setup(struct scsi_cmnd *cmd, int dir_in)
 {
     unsigned char flags = 0x01;
     unsigned long addr = virt_to_bus(cmd->SCp.ptr);
@@ -57,7 +57,7 @@ static int dma_setup (Scsi_Cmnd *cmd, int dir_in)
     return 0;
 }
 
-static void dma_stop (struct Scsi_Host *instance, Scsi_Cmnd *SCpnt,
+static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 		      int status)
 {
     m147_pcc->dma_cntrl = 0;
@@ -112,7 +112,7 @@ int mvme147_detect(struct scsi_host_template *tpnt)
     return 0;
 }
 
-static int mvme147_bus_reset(Scsi_Cmnd *cmd)
+static int mvme147_bus_reset(struct scsi_cmnd *cmd)
 {
 	/* FIXME perform bus-specific reset */
 
diff --git a/drivers/scsi/mvme147.h b/drivers/scsi/mvme147.h
index 2f56d69..32aee854 100644
--- a/drivers/scsi/mvme147.h
+++ b/drivers/scsi/mvme147.h
@@ -12,10 +12,6 @@
 
 int mvme147_detect(struct scsi_host_template *);
 int mvme147_release(struct Scsi_Host *);
-const char *wd33c93_info(void);
-int wd33c93_queuecommand(Scsi_Cmnd *, void (*done)(Scsi_Cmnd *));
-int wd33c93_abort(Scsi_Cmnd *);
-int wd33c93_reset(Scsi_Cmnd *, unsigned int);
 
 #ifndef CMD_PER_LUN
 #define CMD_PER_LUN 2
diff --git a/drivers/scsi/scsi.c b/drivers/scsi/scsi.c
index b332cad..c51b576 100644
--- a/drivers/scsi/scsi.c
+++ b/drivers/scsi/scsi.c
@@ -96,24 +96,40 @@ unsigned int scsi_logging_level;
 EXPORT_SYMBOL(scsi_logging_level);
 #endif
 
-const char *const scsi_device_types[MAX_SCSI_DEVICE_CODE] = {
-	"Direct-Access    ",
-	"Sequential-Access",
+static const char *const scsi_device_types[] = {
+	"Direct access    ",
+	"Sequential access",
 	"Printer          ",
 	"Processor        ",
 	"WORM             ",
-	"CD-ROM           ",
+	"CD/DVD           ",
 	"Scanner          ",
-	"Optical Device   ",
-	"Medium Changer   ",
+	"Optical memory   ",
+	"Media changer    ",
 	"Communications   ",
-	"Unknown          ",
-	"Unknown          ",
+	"ASC IT8          ",
+	"ASC IT8          ",
 	"RAID             ",
 	"Enclosure        ",
-	"Direct-Access-RBC",
+	"Direct access RBC",
+	"Optical card     ",
+	"Bridge controller",
+	"Object storage   ",
+	"Automation/Drive ",
 };
-EXPORT_SYMBOL(scsi_device_types);
+
+const char * scsi_device_type(unsigned type)
+{
+	if (type == 0x1e)
+		return "Well-known LUN   ";
+	if (type == 0x1f)
+		return "No Device        ";
+	if (type > ARRAY_SIZE(scsi_device_types))
+		return "Unknown          ";
+	return scsi_device_types[type];
+}
+
+EXPORT_SYMBOL(scsi_device_type);
 
 struct scsi_host_cmd_pool {
 	kmem_cache_t	*slab;
@@ -835,14 +851,14 @@ EXPORT_SYMBOL(scsi_track_queue_full);
  */
 int scsi_device_get(struct scsi_device *sdev)
 {
-	if (sdev->sdev_state == SDEV_DEL || sdev->sdev_state == SDEV_CANCEL)
+	if (sdev->sdev_state == SDEV_DEL)
 		return -ENXIO;
 	if (!get_device(&sdev->sdev_gendev))
 		return -ENXIO;
-	if (!try_module_get(sdev->host->hostt->module)) {
-		put_device(&sdev->sdev_gendev);
-		return -ENXIO;
-	}
+	/* We can fail this if we're doing SCSI operations
+	 * from module exit (like cache flush) */
+	try_module_get(sdev->host->hostt->module);
+
 	return 0;
 }
 EXPORT_SYMBOL(scsi_device_get);
@@ -857,7 +873,14 @@ EXPORT_SYMBOL(scsi_device_get);
  */
 void scsi_device_put(struct scsi_device *sdev)
 {
-	module_put(sdev->host->hostt->module);
+	struct module *module = sdev->host->hostt->module;
+
+#ifdef CONFIG_MODULE_UNLOAD
+	/* The module refcount will be zero if scsi_device_get()
+	 * was called from a module removal routine */
+	if (module && module_refcount(module) != 0)
+		module_put(module);
+#endif
 	put_device(&sdev->sdev_gendev);
 }
 EXPORT_SYMBOL(scsi_device_put);
@@ -1099,6 +1122,8 @@ static int __init init_scsi(void)
 	for_each_possible_cpu(i)
 		INIT_LIST_HEAD(&per_cpu(scsi_done_q, i));
 
+	scsi_netlink_init();
+
 	printk(KERN_NOTICE "SCSI subsystem initialized\n");
 	return 0;
 
@@ -1119,6 +1144,7 @@ cleanup_queue:
 
 static void __exit exit_scsi(void)
 {
+	scsi_netlink_exit();
 	scsi_sysfs_unregister();
 	scsi_exit_sysctl();
 	scsi_exit_hosts();
diff --git a/drivers/scsi/scsi.h b/drivers/scsi/scsi.h
index f51e466..d5a55fa 100644
--- a/drivers/scsi/scsi.h
+++ b/drivers/scsi/scsi.h
@@ -20,8 +20,6 @@
 #ifndef _SCSI_H
 #define _SCSI_H
 
-#include <linux/config.h>	    /* for CONFIG_SCSI_LOGGING */
-
 #include <scsi/scsi_cmnd.h>
 #include <scsi/scsi_device.h>
 #include <scsi/scsi_eh.h>
diff --git a/drivers/scsi/scsi_debug.c b/drivers/scsi/scsi_debug.c
index a80303c..9c0f358 100644
--- a/drivers/scsi/scsi_debug.c
+++ b/drivers/scsi/scsi_debug.c
@@ -1,5 +1,4 @@
 /*
- *  linux/kernel/scsi_debug.c
  * vvvvvvvvvvvvvvvvvvvvvvv Original vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv
  *  Copyright (C) 1992  Eric Youngdale
  *  Simulate a host adapter with 2 disks attached.  Do a lot of checking
@@ -8,7 +7,9 @@
  * ^^^^^^^^^^^^^^^^^^^^^^^ Original ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  *
  *  This version is more generic, simulating a variable number of disk
- *  (or disk like devices) sharing a common amount of RAM
+ *  (or disk like devices) sharing a common amount of RAM. To be more
+ *  realistic, the simulated devices have the transport attributes of
+ *  SAS disks.
  *
  *
  *  For documentation see http://www.torque.net/sg/sdebug26.html
@@ -50,8 +51,8 @@
 #include "scsi_logging.h"
 #include "scsi_debug.h"
 
-#define SCSI_DEBUG_VERSION "1.79"
-static const char * scsi_debug_version_date = "20060604";
+#define SCSI_DEBUG_VERSION "1.80"
+static const char * scsi_debug_version_date = "20060914";
 
 /* Additional Sense Code (ASC) used */
 #define NO_ADDITIONAL_SENSE 0x0
@@ -86,6 +87,8 @@ static const char * scsi_debug_version_date = "20060604";
 #define DEF_D_SENSE   0
 #define DEF_NO_LUN_0   0
 #define DEF_VIRTUAL_GB   0
+#define DEF_FAKE_RW	0
+#define DEF_VPD_USE_HOSTNO 1
 
 /* bit mask values for scsi_debug_opts */
 #define SCSI_DEBUG_OPT_NOISE   1
@@ -127,6 +130,8 @@ static int scsi_debug_ptype = DEF_PTYPE; /* SCSI peripheral type (0==disk) */
 static int scsi_debug_dsense = DEF_D_SENSE;
 static int scsi_debug_no_lun_0 = DEF_NO_LUN_0;
 static int scsi_debug_virtual_gb = DEF_VIRTUAL_GB;
+static int scsi_debug_fake_rw = DEF_FAKE_RW;
+static int scsi_debug_vpd_use_hostno = DEF_VPD_USE_HOSTNO;
 
 static int scsi_debug_cmnd_count = 0;
 
@@ -423,6 +428,8 @@ int scsi_debug_queuecommand(struct scsi_cmnd * SCpnt, done_funct_t done)
 	case READ_6:
 		if ((errsts = check_readiness(SCpnt, 0, devip)))
 			break;
+		if (scsi_debug_fake_rw)
+			break;
 		if ((*cmd) == READ_16) {
 			for (lba = 0, j = 0; j < 8; ++j) {
 				if (j > 0)
@@ -465,6 +472,8 @@ int scsi_debug_queuecommand(struct scsi_cmnd * SCpnt, done_funct_t done)
 	case WRITE_6:
 		if ((errsts = check_readiness(SCpnt, 0, devip)))
 			break;
+		if (scsi_debug_fake_rw)
+			break;
 		if ((*cmd) == WRITE_16) {
 			for (lba = 0, j = 0; j < 8; ++j) {
 				if (j > 0)
@@ -941,6 +950,8 @@ static int resp_inquiry(struct scsi_cmnd * scp, int target,
 		char lu_id_str[6];
 		int host_no = devip->sdbg_host->shost->host_no;
 		
+		if (0 == scsi_debug_vpd_use_hostno)
+			host_no = 0;
 		lu_id_num = devip->wlun ? -1 : (((host_no + 1) * 2000) +
 			    (devip->target * 1000) + devip->lun);
 		target_dev_id = ((host_no + 1) * 2000) +
@@ -1059,19 +1070,6 @@ static int resp_requests(struct scsi_cmnd * scp,
 			arr[12] = THRESHOLD_EXCEEDED;
 			arr[13] = 0xff;		/* TEST set and MRIE==6 */
 		}
-	} else if (devip->stopped) {
-		if (want_dsense) {
-			arr[0] = 0x72;
-			arr[1] = 0x0;		/* NO_SENSE in sense_key */
-			arr[2] = LOW_POWER_COND_ON;
-			arr[3] = 0x0;		/* TEST set and MRIE==6 */
-		} else {
-			arr[0] = 0x70;
-			arr[2] = 0x0;		/* NO_SENSE in sense_key */
-			arr[7] = 0xa;   	/* 18 byte sense buffer */
-			arr[12] = LOW_POWER_COND_ON;
-			arr[13] = 0x0;		/* TEST set and MRIE==6 */
-		}
 	} else {
 		memcpy(arr, sbuff, SDEBUG_SENSE_LEN);
 		if ((cmd[1] & 1) && (! scsi_debug_dsense)) {
@@ -1325,21 +1323,26 @@ static int resp_sas_sha_m_spg(unsigned char * p, int pcontrol)
 static int resp_mode_sense(struct scsi_cmnd * scp, int target,
 			   struct sdebug_dev_info * devip)
 {
-	unsigned char dbd;
-	int pcontrol, pcode, subpcode;
+	unsigned char dbd, llbaa;
+	int pcontrol, pcode, subpcode, bd_len;
 	unsigned char dev_spec;
-	int alloc_len, msense_6, offset, len, errsts, target_dev_id;
+	int k, alloc_len, msense_6, offset, len, errsts, target_dev_id;
 	unsigned char * ap;
 	unsigned char arr[SDEBUG_MAX_MSENSE_SZ];
 	unsigned char *cmd = (unsigned char *)scp->cmnd;
 
 	if ((errsts = check_readiness(scp, 1, devip)))
 		return errsts;
-	dbd = cmd[1] & 0x8;
+	dbd = !!(cmd[1] & 0x8);
 	pcontrol = (cmd[2] & 0xc0) >> 6;
 	pcode = cmd[2] & 0x3f;
 	subpcode = cmd[3];
 	msense_6 = (MODE_SENSE == cmd[0]);
+	llbaa = msense_6 ? 0 : !!(cmd[1] & 0x10);
+	if ((0 == scsi_debug_ptype) && (0 == dbd))
+		bd_len = llbaa ? 16 : 8;
+	else
+		bd_len = 0;
 	alloc_len = msense_6 ? cmd[4] : ((cmd[7] << 8) | cmd[8]);
 	memset(arr, 0, SDEBUG_MAX_MSENSE_SZ);
 	if (0x3 == pcontrol) {  /* Saving values not supported */
@@ -1349,15 +1352,58 @@ static int resp_mode_sense(struct scsi_cmnd * scp, int target,
 	}
 	target_dev_id = ((devip->sdbg_host->shost->host_no + 1) * 2000) +
 			(devip->target * 1000) - 3;
-	dev_spec = DEV_READONLY(target) ? 0x80 : 0x0;
+	/* set DPOFUA bit for disks */
+	if (0 == scsi_debug_ptype)
+		dev_spec = (DEV_READONLY(target) ? 0x80 : 0x0) | 0x10;
+	else
+		dev_spec = 0x0;
 	if (msense_6) {
 		arr[2] = dev_spec;
+		arr[3] = bd_len;
 		offset = 4;
 	} else {
 		arr[3] = dev_spec;
+		if (16 == bd_len)
+			arr[4] = 0x1;	/* set LONGLBA bit */
+		arr[7] = bd_len;	/* assume 255 or less */
 		offset = 8;
 	}
 	ap = arr + offset;
+	if ((bd_len > 0) && (0 == sdebug_capacity)) {
+		if (scsi_debug_virtual_gb > 0) {
+			sdebug_capacity = 2048 * 1024;
+			sdebug_capacity *= scsi_debug_virtual_gb;
+		} else
+			sdebug_capacity = sdebug_store_sectors;
+	}
+	if (8 == bd_len) {
+		if (sdebug_capacity > 0xfffffffe) {
+			ap[0] = 0xff;
+			ap[1] = 0xff;
+			ap[2] = 0xff;
+			ap[3] = 0xff;
+		} else {
+			ap[0] = (sdebug_capacity >> 24) & 0xff;
+			ap[1] = (sdebug_capacity >> 16) & 0xff;
+			ap[2] = (sdebug_capacity >> 8) & 0xff;
+			ap[3] = sdebug_capacity & 0xff;
+		}
+        	ap[6] = (SECT_SIZE_PER(target) >> 8) & 0xff;
+        	ap[7] = SECT_SIZE_PER(target) & 0xff;
+		offset += bd_len;
+		ap = arr + offset;
+	} else if (16 == bd_len) {
+		unsigned long long capac = sdebug_capacity;
+
+        	for (k = 0; k < 8; ++k, capac >>= 8)
+                	ap[7 - k] = capac & 0xff;
+        	ap[12] = (SECT_SIZE_PER(target) >> 24) & 0xff;
+        	ap[13] = (SECT_SIZE_PER(target) >> 16) & 0xff;
+        	ap[14] = (SECT_SIZE_PER(target) >> 8) & 0xff;
+        	ap[15] = SECT_SIZE_PER(target) & 0xff;
+		offset += bd_len;
+		ap = arr + offset;
+	}
 
 	if ((subpcode > 0x0) && (subpcode < 0xff) && (0x19 != pcode)) {
 		/* TODO: Control Extension page */
@@ -1471,7 +1517,7 @@ static int resp_mode_select(struct scsi_cmnd * scp, int mselect6,
                        " IO sent=%d bytes\n", param_len, res);
 	md_len = mselect6 ? (arr[0] + 1) : ((arr[0] << 8) + arr[1] + 2);
 	bd_len = mselect6 ? arr[3] : ((arr[6] << 8) + arr[7]);
-	if ((md_len > 2) || (0 != bd_len)) {
+	if (md_len > 2) {
 		mk_sense_buffer(devip, ILLEGAL_REQUEST,
 				INVALID_FIELD_IN_PARAM_LIST, 0);
 		return check_condition_result;
@@ -1544,7 +1590,7 @@ static int resp_ie_l_pg(unsigned char * arr)
 static int resp_log_sense(struct scsi_cmnd * scp,
                           struct sdebug_dev_info * devip)
 {
-	int ppc, sp, pcontrol, pcode, alloc_len, errsts, len, n;
+	int ppc, sp, pcontrol, pcode, subpcode, alloc_len, errsts, len, n;
 	unsigned char arr[SDEBUG_MAX_LSENSE_SZ];
 	unsigned char *cmd = (unsigned char *)scp->cmnd;
 
@@ -1560,23 +1606,63 @@ static int resp_log_sense(struct scsi_cmnd * scp,
 	}
 	pcontrol = (cmd[2] & 0xc0) >> 6;
 	pcode = cmd[2] & 0x3f;
+	subpcode = cmd[3] & 0xff;
 	alloc_len = (cmd[7] << 8) + cmd[8];
 	arr[0] = pcode;
-	switch (pcode) {
-	case 0x0:	/* Supported log pages log page */
-		n = 4;
-		arr[n++] = 0x0;		/* this page */
-		arr[n++] = 0xd;		/* Temperature */
-		arr[n++] = 0x2f;	/* Informational exceptions */
-		arr[3] = n - 4;
-		break;
-	case 0xd:	/* Temperature log page */
-		arr[3] = resp_temp_l_pg(arr + 4);
-		break;
-	case 0x2f:	/* Informational exceptions log page */
-		arr[3] = resp_ie_l_pg(arr + 4);
-		break;
-	default:
+	if (0 == subpcode) {
+		switch (pcode) {
+		case 0x0:	/* Supported log pages log page */
+			n = 4;
+			arr[n++] = 0x0;		/* this page */
+			arr[n++] = 0xd;		/* Temperature */
+			arr[n++] = 0x2f;	/* Informational exceptions */
+			arr[3] = n - 4;
+			break;
+		case 0xd:	/* Temperature log page */
+			arr[3] = resp_temp_l_pg(arr + 4);
+			break;
+		case 0x2f:	/* Informational exceptions log page */
+			arr[3] = resp_ie_l_pg(arr + 4);
+			break;
+		default:
+			mk_sense_buffer(devip, ILLEGAL_REQUEST,
+					INVALID_FIELD_IN_CDB, 0);
+			return check_condition_result;
+		}
+	} else if (0xff == subpcode) {
+		arr[0] |= 0x40;
+		arr[1] = subpcode;
+		switch (pcode) {
+		case 0x0:	/* Supported log pages and subpages log page */
+			n = 4;
+			arr[n++] = 0x0;
+			arr[n++] = 0x0;		/* 0,0 page */
+			arr[n++] = 0x0;
+			arr[n++] = 0xff;	/* this page */
+			arr[n++] = 0xd;
+			arr[n++] = 0x0;		/* Temperature */
+			arr[n++] = 0x2f;
+			arr[n++] = 0x0;	/* Informational exceptions */
+			arr[3] = n - 4;
+			break;
+		case 0xd:	/* Temperature subpages */
+			n = 4;
+			arr[n++] = 0xd;
+			arr[n++] = 0x0;		/* Temperature */
+			arr[3] = n - 4;
+			break;
+		case 0x2f:	/* Informational exceptions subpages */
+			n = 4;
+			arr[n++] = 0x2f;
+			arr[n++] = 0x0;		/* Informational exceptions */
+			arr[3] = n - 4;
+			break;
+		default:
+			mk_sense_buffer(devip, ILLEGAL_REQUEST,
+					INVALID_FIELD_IN_CDB, 0);
+			return check_condition_result;
+		}
+	} else {
 		mk_sense_buffer(devip, ILLEGAL_REQUEST,
 				INVALID_FIELD_IN_CDB, 0);
 		return check_condition_result;
@@ -2151,11 +2237,18 @@ static int schedule_resp(struct scsi_cmnd * cmnd,
 	}
 }
 
+/* Note: The following macros create attribute files in the
+   /sys/module/scsi_debug/parameters directory. Unfortunately this
+   driver is unaware of a change and cannot trigger auxiliary actions
+   as it can when the corresponding attribute in the
+   /sys/bus/pseudo/drivers/scsi_debug directory is changed.
+ */
 module_param_named(add_host, scsi_debug_add_host, int, S_IRUGO | S_IWUSR);
 module_param_named(delay, scsi_debug_delay, int, S_IRUGO | S_IWUSR);
 module_param_named(dev_size_mb, scsi_debug_dev_size_mb, int, S_IRUGO);
 module_param_named(dsense, scsi_debug_dsense, int, S_IRUGO | S_IWUSR);
 module_param_named(every_nth, scsi_debug_every_nth, int, S_IRUGO | S_IWUSR);
+module_param_named(fake_rw, scsi_debug_fake_rw, int, S_IRUGO | S_IWUSR);
 module_param_named(max_luns, scsi_debug_max_luns, int, S_IRUGO | S_IWUSR);
 module_param_named(no_lun_0, scsi_debug_no_lun_0, int, S_IRUGO | S_IWUSR);
 module_param_named(num_parts, scsi_debug_num_parts, int, S_IRUGO);
@@ -2164,6 +2257,8 @@ module_param_named(opts, scsi_debug_opts, int, S_IRUGO | S_IWUSR);
 module_param_named(ptype, scsi_debug_ptype, int, S_IRUGO | S_IWUSR);
 module_param_named(scsi_level, scsi_debug_scsi_level, int, S_IRUGO);
 module_param_named(virtual_gb, scsi_debug_virtual_gb, int, S_IRUGO | S_IWUSR);
+module_param_named(vpd_use_hostno, scsi_debug_vpd_use_hostno, int,
+		   S_IRUGO | S_IWUSR);
 
 MODULE_AUTHOR("Eric Youngdale + Douglas Gilbert");
 MODULE_DESCRIPTION("SCSI debug adapter driver");
@@ -2175,6 +2270,7 @@ MODULE_PARM_DESC(delay, "# of jiffies to delay response(def=1)");
 MODULE_PARM_DESC(dev_size_mb, "size in MB of ram shared by devs(def=8)");
 MODULE_PARM_DESC(dsense, "use descriptor sense format(def=0 -> fixed)");
 MODULE_PARM_DESC(every_nth, "timeout every nth command(def=100)");
+MODULE_PARM_DESC(fake_rw, "fake reads/writes instead of copying (def=0)");
 MODULE_PARM_DESC(max_luns, "number of LUNs per target to simulate(def=1)");
 MODULE_PARM_DESC(no_lun_0, "no LU number 0 (def=0 -> have lun 0)");
 MODULE_PARM_DESC(num_parts, "number of partitions(def=0)");
@@ -2183,6 +2279,7 @@ MODULE_PARM_DESC(opts, "1->noise, 2->medium_error, 4->... (def=0)");
 MODULE_PARM_DESC(ptype, "SCSI peripheral type(def=0[disk])");
 MODULE_PARM_DESC(scsi_level, "SCSI level to simulate(def=5[SPC-3])");
 MODULE_PARM_DESC(virtual_gb, "virtual gigabyte size (def=0 -> use dev_size_mb)");
+MODULE_PARM_DESC(vpd_use_hostno, "0 -> dev ids ignore hostno (def=1 -> unique dev ids)");
 
 
 static char sdebug_info[256];
@@ -2334,6 +2431,24 @@ static ssize_t sdebug_dsense_store(struct device_driver * ddp,
 DRIVER_ATTR(dsense, S_IRUGO | S_IWUSR, sdebug_dsense_show,
 	    sdebug_dsense_store);
 
+static ssize_t sdebug_fake_rw_show(struct device_driver * ddp, char * buf)
+{
+        return scnprintf(buf, PAGE_SIZE, "%d\n", scsi_debug_fake_rw);
+}
+static ssize_t sdebug_fake_rw_store(struct device_driver * ddp,
+				    const char * buf, size_t count)
+{
+        int n;
+
+	if ((count > 0) && (1 == sscanf(buf, "%d", &n)) && (n >= 0)) {
+		scsi_debug_fake_rw = n;
+		return count;
+	}
+	return -EINVAL;
+}
+DRIVER_ATTR(fake_rw, S_IRUGO | S_IWUSR, sdebug_fake_rw_show,
+	    sdebug_fake_rw_store);
+
 static ssize_t sdebug_no_lun_0_show(struct device_driver * ddp, char * buf)
 {
         return scnprintf(buf, PAGE_SIZE, "%d\n", scsi_debug_no_lun_0);
@@ -2487,6 +2602,31 @@ static ssize_t sdebug_add_host_store(struct device_driver * ddp,
 DRIVER_ATTR(add_host, S_IRUGO | S_IWUSR, sdebug_add_host_show, 
 	    sdebug_add_host_store);
 
+static ssize_t sdebug_vpd_use_hostno_show(struct device_driver * ddp,
+					  char * buf)
+{
+	return scnprintf(buf, PAGE_SIZE, "%d\n", scsi_debug_vpd_use_hostno);
+}
+static ssize_t sdebug_vpd_use_hostno_store(struct device_driver * ddp,
+					   const char * buf, size_t count)
+{
+	int n;
+
+	if ((count > 0) && (1 == sscanf(buf, "%d", &n)) && (n >= 0)) {
+		scsi_debug_vpd_use_hostno = n;
+		return count;
+	}
+	return -EINVAL;
+}
+DRIVER_ATTR(vpd_use_hostno, S_IRUGO | S_IWUSR, sdebug_vpd_use_hostno_show,
+	    sdebug_vpd_use_hostno_store);
+
+/* Note: The following function creates attribute files in the
+   /sys/bus/pseudo/drivers/scsi_debug directory. The advantage of these
+   files (over those found in the /sys/module/scsi_debug/parameters
+   directory) is that auxiliary actions can be triggered when an attribute
+   is changed. For example see: sdebug_add_host_store() above.
+ */
 static int do_create_driverfs_files(void)
 {
 	int ret;
@@ -2496,23 +2636,31 @@ static int do_create_driverfs_files(void)
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_dev_size_mb);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_dsense);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_every_nth);
+	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_fake_rw);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_max_luns);
-	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_num_tgts);
+	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_no_lun_0);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_num_parts);
+	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_num_tgts);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_ptype);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_opts);
 	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_scsi_level);
+	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_virtual_gb);
+	ret |= driver_create_file(&sdebug_driverfs_driver, &driver_attr_vpd_use_hostno);
 	return ret;
 }
 
 static void do_remove_driverfs_files(void)
 {
+	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_vpd_use_hostno);
+	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_virtual_gb);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_scsi_level);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_opts);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_ptype);
-	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_num_parts);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_num_tgts);
+	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_num_parts);
+	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_no_lun_0);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_max_luns);
+	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_fake_rw);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_every_nth);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_dsense);
 	driver_remove_file(&sdebug_driverfs_driver, &driver_attr_dev_size_mb);
diff --git a/drivers/scsi/scsi_lib.c b/drivers/scsi/scsi_lib.c
index 077c1c6..d6743b9 100644
--- a/drivers/scsi/scsi_lib.c
+++ b/drivers/scsi/scsi_lib.c
@@ -551,7 +551,15 @@ static void scsi_run_queue(struct request_queue *q)
 		list_del_init(&sdev->starved_entry);
 		spin_unlock_irqrestore(shost->host_lock, flags);
 
-		blk_run_queue(sdev->request_queue);
+
+		if (test_bit(QUEUE_FLAG_REENTER, &q->queue_flags) &&
+		    !test_and_set_bit(QUEUE_FLAG_REENTER,
+				      &sdev->request_queue->queue_flags)) {
+			blk_run_queue(sdev->request_queue);
+			clear_bit(QUEUE_FLAG_REENTER,
+				  &sdev->request_queue->queue_flags);
+		} else
+			blk_run_queue(sdev->request_queue);
 
 		spin_lock_irqsave(shost->host_lock, flags);
 		if (unlikely(!list_empty(&sdev->starved_entry)))
diff --git a/drivers/scsi/scsi_netlink.c b/drivers/scsi/scsi_netlink.c
new file mode 100644
index 0000000..1b59b27
--- /dev/null
+++ b/drivers/scsi/scsi_netlink.c
@@ -0,0 +1,199 @@
+/*
+ *  scsi_netlink.c  - SCSI Transport Netlink Interface
+ *
+ *  Copyright (C) 2006   James Smart, Emulex Corporation
+ *
+ *  This program is free software; you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation; either version 2 of the License, or
+ *  (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+ *
+ */
+#include <linux/time.h>
+#include <linux/jiffies.h>
+#include <linux/security.h>
+#include <net/sock.h>
+#include <net/netlink.h>
+
+#include <scsi/scsi_netlink.h>
+#include "scsi_priv.h"
+
+struct sock *scsi_nl_sock = NULL;
+EXPORT_SYMBOL_GPL(scsi_nl_sock);
+
+
+/**
+ * scsi_nl_rcv_msg -
+ *    Receive message handler. Extracts message from a receive buffer.
+ *    Validates message header and calls appropriate transport message handler
+ *
+ * @skb:		socket receive buffer
+ *
+ **/
+static void
+scsi_nl_rcv_msg(struct sk_buff *skb)
+{
+	struct nlmsghdr *nlh;
+	struct scsi_nl_hdr *hdr;
+	uint32_t rlen;
+	int err;
+
+	while (skb->len >= NLMSG_SPACE(0)) {
+		err = 0;
+
+		nlh = (struct nlmsghdr *) skb->data;
+		if ((nlh->nlmsg_len < (sizeof(*nlh) + sizeof(*hdr))) ||
+		    (skb->len < nlh->nlmsg_len)) {
+			printk(KERN_WARNING "%s: discarding partial skb\n",
+				 __FUNCTION__);
+			return;
+		}
+
+		rlen = NLMSG_ALIGN(nlh->nlmsg_len);
+		if (rlen > skb->len)
+			rlen = skb->len;
+
+		if (nlh->nlmsg_type != SCSI_TRANSPORT_MSG) {
+			err = -EBADMSG;
+			goto next_msg;
+		}
+
+		hdr = NLMSG_DATA(nlh);
+		if ((hdr->version != SCSI_NL_VERSION) ||
+		    (hdr->magic != SCSI_NL_MAGIC)) {
+			err = -EPROTOTYPE;
+			goto next_msg;
+		}
+
+		if (security_netlink_recv(skb, CAP_SYS_ADMIN)) {
+			err = -EPERM;
+			goto next_msg;
+		}
+
+		if (nlh->nlmsg_len < (sizeof(*nlh) + hdr->msglen)) {
+			printk(KERN_WARNING "%s: discarding partial message\n",
+				 __FUNCTION__);
+			return;
+		}
+
+		/*
+		 * We currently don't support anyone sending us a message
+		 */
+
+next_msg:
+		if ((err) || (nlh->nlmsg_flags & NLM_F_ACK))
+			netlink_ack(skb, nlh, err);
+
+		skb_pull(skb, rlen);
+	}
+}
+
+
+/**
+ * scsi_nl_rcv_msg -
+ *    Receive handler for a socket. Extracts a received message buffer from
+ *    the socket, and starts message processing.
+ *
+ * @sk:		socket
+ * @len:	unused
+ *
+ **/
+static void
+scsi_nl_rcv(struct sock *sk, int len)
+{
+	struct sk_buff *skb;
+
+	while ((skb = skb_dequeue(&sk->sk_receive_queue))) {
+		scsi_nl_rcv_msg(skb);
+		kfree_skb(skb);
+	}
+}
+
+
+/**
+ * scsi_nl_rcv_event -
+ *    Event handler for a netlink socket.
+ *
+ * @this:		event notifier block
+ * @event:		event type
+ * @ptr:		event payload
+ *
+ **/
+static int
+scsi_nl_rcv_event(struct notifier_block *this, unsigned long event, void *ptr)
+{
+	struct netlink_notify *n = ptr;
+
+	if (n->protocol != NETLINK_SCSITRANSPORT)
+		return NOTIFY_DONE;
+
+	/*
+	 * Currently, we are not tracking PID's, etc. There is nothing
+	 * to handle.
+	 */
+
+	return NOTIFY_DONE;
+}
+
+static struct notifier_block scsi_netlink_notifier = {
+	.notifier_call  = scsi_nl_rcv_event,
+};
+
+
+/**
+ * scsi_netlink_init -
+ *    Called by SCSI subsystem to intialize the SCSI transport netlink
+ *    interface
+ *
+ **/
+void
+scsi_netlink_init(void)
+{
+	int error;
+
+	error = netlink_register_notifier(&scsi_netlink_notifier);
+	if (error) {
+		printk(KERN_ERR "%s: register of event handler failed - %d\n",
+				__FUNCTION__, error);
+		return;
+	}
+
+	scsi_nl_sock = netlink_kernel_create(NETLINK_SCSITRANSPORT,
+				SCSI_NL_GRP_CNT, scsi_nl_rcv, THIS_MODULE);
+	if (!scsi_nl_sock) {
+		printk(KERN_ERR "%s: register of recieve handler failed\n",
+				__FUNCTION__);
+		netlink_unregister_notifier(&scsi_netlink_notifier);
+	}
+
+	return;
+}
+
+
+/**
+ * scsi_netlink_exit -
+ *    Called by SCSI subsystem to disable the SCSI transport netlink
+ *    interface
+ *
+ **/
+void
+scsi_netlink_exit(void)
+{
+	if (scsi_nl_sock) {
+		sock_release(scsi_nl_sock->sk_socket);
+		netlink_unregister_notifier(&scsi_netlink_notifier);
+	}
+
+	return;
+}
+
+
diff --git a/drivers/scsi/scsi_priv.h b/drivers/scsi/scsi_priv.h
index ae24c85..5d023d4 100644
--- a/drivers/scsi/scsi_priv.h
+++ b/drivers/scsi/scsi_priv.h
@@ -8,6 +8,7 @@ struct scsi_cmnd;
 struct scsi_device;
 struct scsi_host_template;
 struct Scsi_Host;
+struct scsi_nl_hdr;
 
 
 /*
@@ -110,6 +111,16 @@ extern void __scsi_remove_device(struct scsi_device *);
 
 extern struct bus_type scsi_bus_type;
 
+/* scsi_netlink.c */
+#ifdef CONFIG_SCSI_NETLINK
+extern void scsi_netlink_init(void);
+extern void scsi_netlink_exit(void);
+extern struct sock *scsi_nl_sock;
+#else
+static inline void scsi_netlink_init(void) {}
+static inline void scsi_netlink_exit(void) {}
+#endif
+
 /* 
  * internal scsi timeout functions: for use by mid-layer and transport
  * classes.
diff --git a/drivers/scsi/scsi_proc.c b/drivers/scsi/scsi_proc.c
index 55200e4..524a5f7 100644
--- a/drivers/scsi/scsi_proc.c
+++ b/drivers/scsi/scsi_proc.c
@@ -178,9 +178,7 @@ static int proc_print_scsidevice(struct device *dev, void *data)
 
 	seq_printf(s, "\n");
 
-	seq_printf(s, "  Type:   %s ",
-		     sdev->type < MAX_SCSI_DEVICE_CODE ?
-	       scsi_device_types[(int) sdev->type] : "Unknown          ");
+	seq_printf(s, "  Type:   %s ", scsi_device_type(sdev->type));
 	seq_printf(s, "               ANSI"
 		     " SCSI revision: %02x", (sdev->scsi_level - 1) ?
 		     sdev->scsi_level - 1 : 1);
diff --git a/drivers/scsi/scsi_scan.c b/drivers/scsi/scsi_scan.c
index 1bd92b9..fd9e281 100644
--- a/drivers/scsi/scsi_scan.c
+++ b/drivers/scsi/scsi_scan.c
@@ -134,59 +134,6 @@ static void scsi_unlock_floptical(struct scsi_device *sdev,
 }
 
 /**
- * print_inquiry - printk the inquiry information
- * @inq_result:	printk this SCSI INQUIRY
- *
- * Description:
- *     printk the vendor, model, and other information found in the
- *     INQUIRY data in @inq_result.
- *
- * Notes:
- *     Remove this, and replace with a hotplug event that logs any
- *     relevant information.
- **/
-static void print_inquiry(unsigned char *inq_result)
-{
-	int i;
-
-	printk(KERN_NOTICE "  Vendor: ");
-	for (i = 8; i < 16; i++)
-		if (inq_result[i] >= 0x20 && i < inq_result[4] + 5)
-			printk("%c", inq_result[i]);
-		else
-			printk(" ");
-
-	printk("  Model: ");
-	for (i = 16; i < 32; i++)
-		if (inq_result[i] >= 0x20 && i < inq_result[4] + 5)
-			printk("%c", inq_result[i]);
-		else
-			printk(" ");
-
-	printk("  Rev: ");
-	for (i = 32; i < 36; i++)
-		if (inq_result[i] >= 0x20 && i < inq_result[4] + 5)
-			printk("%c", inq_result[i]);
-		else
-			printk(" ");
-
-	printk("\n");
-
-	i = inq_result[0] & 0x1f;
-
-	printk(KERN_NOTICE "  Type:   %s ",
-	       i <
-	       MAX_SCSI_DEVICE_CODE ? scsi_device_types[i] :
-	       "Unknown          ");
-	printk("                 ANSI SCSI revision: %02x",
-	       inq_result[2] & 0x07);
-	if ((inq_result[2] & 0x07) == 1 && (inq_result[3] & 0x0f) == 1)
-		printk(" CCS\n");
-	else
-		printk("\n");
-}
-
-/**
  * scsi_alloc_sdev - allocate and setup a scsi_Device
  *
  * Description:
@@ -319,6 +266,18 @@ static struct scsi_target *__scsi_find_target(struct device *parent,
 	return found_starget;
 }
 
+/**
+ * scsi_alloc_target - allocate a new or find an existing target
+ * @parent:	parent of the target (need not be a scsi host)
+ * @channel:	target channel number (zero if no channels)
+ * @id:		target id number
+ *
+ * Return an existing target if one exists, provided it hasn't already
+ * gone into STARGET_DEL state, otherwise allocate a new target.
+ *
+ * The target is returned with an incremented reference, so the caller
+ * is responsible for both reaping and doing a last put
+ */
 static struct scsi_target *scsi_alloc_target(struct device *parent,
 					     int channel, uint id)
 {
@@ -384,14 +343,15 @@ static struct scsi_target *scsi_alloc_target(struct device *parent,
 			return NULL;
 		}
 	}
+	get_device(dev);
 
 	return starget;
 
  found:
 	found_target->reap_ref++;
 	spin_unlock_irqrestore(shost->host_lock, flags);
-	put_device(parent);
 	if (found_target->state != STARGET_DEL) {
+		put_device(parent);
 		kfree(starget);
 		return found_target;
 	}
@@ -450,6 +410,32 @@ void scsi_target_reap(struct scsi_target *starget)
 }
 
 /**
+ * sanitize_inquiry_string - remove non-graphical chars from an INQUIRY result string
+ * @s: INQUIRY result string to sanitize
+ * @len: length of the string
+ *
+ * Description:
+ *	The SCSI spec says that INQUIRY vendor, product, and revision
+ *	strings must consist entirely of graphic ASCII characters,
+ *	padded on the right with spaces.  Since not all devices obey
+ *	this rule, we will replace non-graphic or non-ASCII characters
+ *	with spaces.  Exception: a NUL character is interpreted as a
+ *	string terminator, so all the following characters are set to
+ *	spaces.
+ **/
+static void sanitize_inquiry_string(unsigned char *s, int len)
+{
+	int terminated = 0;
+
+	for (; len > 0; (--len, ++s)) {
+		if (*s == 0)
+			terminated = 1;
+		if (terminated || *s < 0x20 || *s > 0x7e)
+			*s = ' ';
+	}
+}
+
+/**
  * scsi_probe_lun - probe a single LUN using a SCSI INQUIRY
  * @sdev:	scsi_device to probe
  * @inq_result:	area to store the INQUIRY result
@@ -463,7 +449,7 @@ void scsi_target_reap(struct scsi_target *starget)
  *     INQUIRY data is in @inq_result; the scsi_level and INQUIRY length
  *     are copied to the scsi_device any flags value is stored in *@bflags.
  **/
-static int scsi_probe_lun(struct scsi_device *sdev, char *inq_result,
+static int scsi_probe_lun(struct scsi_device *sdev, unsigned char *inq_result,
 			  int result_len, int *bflags)
 {
 	unsigned char scsi_cmd[MAX_COMMAND_SIZE];
@@ -522,7 +508,11 @@ static int scsi_probe_lun(struct scsi_device *sdev, char *inq_result,
 	}
 
 	if (result == 0) {
-		response_len = (unsigned char) inq_result[4] + 5;
+		sanitize_inquiry_string(&inq_result[8], 8);
+		sanitize_inquiry_string(&inq_result[16], 16);
+		sanitize_inquiry_string(&inq_result[32], 4);
+
+		response_len = inq_result[4] + 5;
 		if (response_len > 255)
 			response_len = first_inquiry_len;	/* sanity */
 
@@ -628,7 +618,8 @@ static int scsi_probe_lun(struct scsi_device *sdev, char *inq_result,
  *     SCSI_SCAN_NO_RESPONSE: could not allocate or setup a scsi_device
  *     SCSI_SCAN_LUN_PRESENT: a new scsi_device was allocated and initialized
  **/
-static int scsi_add_lun(struct scsi_device *sdev, char *inq_result, int *bflags)
+static int scsi_add_lun(struct scsi_device *sdev, unsigned char *inq_result,
+		int *bflags)
 {
 	/*
 	 * XXX do not save the inquiry, since it can change underneath us,
@@ -653,9 +644,8 @@ static int scsi_add_lun(struct scsi_device *sdev, char *inq_result, int *bflags)
 	if (*bflags & BLIST_ISROM) {
 		/*
 		 * It would be better to modify sdev->type, and set
-		 * sdev->removable, but then the print_inquiry() output
-		 * would not show TYPE_ROM; if print_inquiry() is removed
-		 * the issue goes away.
+		 * sdev->removable; this can now be done since
+		 * print_inquiry has gone away.
 		 */
 		inq_result[0] = TYPE_ROM;
 		inq_result[1] |= 0x80;	/* removable */
@@ -684,8 +674,6 @@ static int scsi_add_lun(struct scsi_device *sdev, char *inq_result, int *bflags)
 		printk(KERN_INFO "scsi: unknown device type %d\n", sdev->type);
 	}
 
-	print_inquiry(inq_result);
-
 	/*
 	 * For a peripheral qualifier (PQ) value of 1 (001b), the SCSI
 	 * spec says: The device server is capable of supporting the
@@ -715,6 +703,12 @@ static int scsi_add_lun(struct scsi_device *sdev, char *inq_result, int *bflags)
 	if (inq_result[7] & 0x10)
 		sdev->sdtr = 1;
 
+	sdev_printk(KERN_NOTICE, sdev, "%s %.8s %.16s %.4s PQ: %d "
+			"ANSI: %d%s\n", scsi_device_type(sdev->type),
+			sdev->vendor, sdev->model, sdev->rev,
+			sdev->inq_periph_qual, inq_result[2] & 0x07,
+			(inq_result[3] & 0x0f) == 1 ? " CCS" : "");
+
 	/*
 	 * End sysfs code.
 	 */
@@ -943,11 +937,26 @@ static int scsi_probe_and_add_lun(struct scsi_target *starget,
 	}
 
 	/*
-	 * Non-standard SCSI targets may set the PDT to 0x1f (unknown or
-	 * no device type) instead of using the Peripheral Qualifier to
-	 * indicate that no LUN is present.  For example, USB UFI does this.
+	 * Some targets may set slight variations of PQ and PDT to signal
+	 * that no LUN is present, so don't add sdev in these cases.
+	 * Two specific examples are:
+	 * 1) NetApp targets: return PQ=1, PDT=0x1f
+	 * 2) USB UFI: returns PDT=0x1f, with the PQ bits being "reserved"
+	 *    in the UFI 1.0 spec (we cannot rely on reserved bits).
+	 *
+	 * References:
+	 * 1) SCSI SPC-3, pp. 145-146
+	 * PQ=1: "A peripheral device having the specified peripheral
+	 * device type is not connected to this logical unit. However, the
+	 * device server is capable of supporting the specified peripheral
+	 * device type on this logical unit."
+	 * PDT=0x1f: "Unknown or no device type"
+	 * 2) USB UFI 1.0, p. 20
+	 * PDT=00h Direct-access device (floppy)
+	 * PDT=1Fh none (no FDD connected to the requested logical unit)
 	 */
-	if (starget->pdt_1f_for_no_lun && (result[0] & 0x1f) == 0x1f) {
+	if (((result[0] >> 5) == 1 || starget->pdt_1f_for_no_lun) &&
+	     (result[0] & 0x1f) == 0x1f) {
 		SCSI_LOG_SCAN_BUS(3, printk(KERN_INFO
 					"scsi scan: peripheral device type"
 					" of 31, no device added\n"));
@@ -1345,7 +1354,6 @@ struct scsi_device *__scsi_add_device(struct Scsi_Host *shost, uint channel,
 	if (!starget)
 		return ERR_PTR(-ENOMEM);
 
-	get_device(&starget->dev);
 	mutex_lock(&shost->scan_mutex);
 	if (scsi_host_scan_allowed(shost))
 		scsi_probe_and_add_lun(starget, lun, NULL, &sdev, 1, hostdata);
@@ -1404,7 +1412,6 @@ static void __scsi_scan_target(struct device *parent, unsigned int channel,
 	if (!starget)
 		return;
 
-	get_device(&starget->dev);
 	if (lun != SCAN_WILD_CARD) {
 		/*
 		 * Scan for a specific host/chan/id/lun.
@@ -1586,7 +1593,8 @@ struct scsi_device *scsi_get_host_dev(struct Scsi_Host *shost)
 	if (sdev) {
 		sdev->sdev_gendev.parent = get_device(&starget->dev);
 		sdev->borken = 0;
-	}
+	} else
+		scsi_target_reap(starget);
 	put_device(&starget->dev);
  out:
 	mutex_unlock(&shost->scan_mutex);
diff --git a/drivers/scsi/scsi_transport_fc.c b/drivers/scsi/scsi_transport_fc.c
index b03aa85..38c215a 100644
--- a/drivers/scsi/scsi_transport_fc.c
+++ b/drivers/scsi/scsi_transport_fc.c
@@ -32,6 +32,9 @@
 #include <scsi/scsi_transport.h>
 #include <scsi/scsi_transport_fc.h>
 #include <scsi/scsi_cmnd.h>
+#include <linux/netlink.h>
+#include <net/netlink.h>
+#include <scsi/scsi_netlink_fc.h>
 #include "scsi_priv.h"
 
 static int fc_queue_work(struct Scsi_Host *, struct work_struct *);
@@ -93,6 +96,29 @@ fc_enum_name_search(port_type, fc_port_type, fc_port_type_names)
 #define FC_PORTTYPE_MAX_NAMELEN		50
 
 
+/* Convert fc_host_event_code values to ascii string name */
+static const struct {
+	enum fc_host_event_code		value;
+	char				*name;
+} fc_host_event_code_names[] = {
+	{ FCH_EVT_LIP,			"lip" },
+	{ FCH_EVT_LINKUP,		"link_up" },
+	{ FCH_EVT_LINKDOWN,		"link_down" },
+	{ FCH_EVT_LIPRESET,		"lip_reset" },
+	{ FCH_EVT_RSCN,			"rscn" },
+	{ FCH_EVT_ADAPTER_CHANGE,	"adapter_chg" },
+	{ FCH_EVT_PORT_UNKNOWN,		"port_unknown" },
+	{ FCH_EVT_PORT_ONLINE,		"port_online" },
+	{ FCH_EVT_PORT_OFFLINE,		"port_offline" },
+	{ FCH_EVT_PORT_FABRIC,		"port_fabric" },
+	{ FCH_EVT_LINK_UNKNOWN,		"link_unknown" },
+	{ FCH_EVT_VENDOR_UNIQUE,	"vendor_unique" },
+};
+fc_enum_name_search(host_event_code, fc_host_event_code,
+		fc_host_event_code_names)
+#define FC_HOST_EVENT_CODE_MAX_NAMELEN	30
+
+
 /* Convert fc_port_state values to ascii string name */
 static struct {
 	enum fc_port_state	value;
@@ -216,6 +242,7 @@ fc_bitfield_name_search(remote_port_roles, fc_remote_port_role_names)
 
 
 static void fc_timeout_deleted_rport(void *data);
+static void fc_timeout_fail_rport_io(void *data);
 static void fc_scsi_scan_rport(void *data);
 
 /*
@@ -223,7 +250,7 @@ static void fc_scsi_scan_rport(void *data);
  * Increase these values if you add attributes
  */
 #define FC_STARGET_NUM_ATTRS 	3
-#define FC_RPORT_NUM_ATTRS	9
+#define FC_RPORT_NUM_ATTRS	10
 #define FC_HOST_NUM_ATTRS	17
 
 struct fc_internal {
@@ -301,8 +328,6 @@ static int fc_host_setup(struct transport_container *tc, struct device *dev,
 	fc_host->supported_classes = FC_COS_UNSPECIFIED;
 	memset(fc_host->supported_fc4s, 0,
 		sizeof(fc_host->supported_fc4s));
-	memset(fc_host->symbolic_name, 0,
-		sizeof(fc_host->symbolic_name));
 	fc_host->supported_speeds = FC_PORTSPEED_UNKNOWN;
 	fc_host->maxframe_size = -1;
 	memset(fc_host->serial_number, 0,
@@ -315,6 +340,8 @@ static int fc_host_setup(struct transport_container *tc, struct device *dev,
 		sizeof(fc_host->active_fc4s));
 	fc_host->speed = FC_PORTSPEED_UNKNOWN;
 	fc_host->fabric_name = -1;
+	memset(fc_host->symbolic_name, 0, sizeof(fc_host->symbolic_name));
+	memset(fc_host->system_hostname, 0, sizeof(fc_host->system_hostname));
 
 	fc_host->tgtid_bind_type = FC_TGTID_BIND_BY_WWPN;
 
@@ -377,10 +404,184 @@ MODULE_PARM_DESC(dev_loss_tmo,
 		 " exceeded, the scsi target is removed. Value should be"
 		 " between 1 and SCSI_DEVICE_BLOCK_MAX_TIMEOUT.");
 
+/**
+ * Netlink Infrastructure
+ **/
+
+static atomic_t fc_event_seq;
+
+/**
+ * fc_get_event_number - Obtain the next sequential FC event number
+ *
+ * Notes:
+ *   We could have inline'd this, but it would have required fc_event_seq to
+ *   be exposed. For now, live with the subroutine call.
+ *   Atomic used to avoid lock/unlock...
+ **/
+u32
+fc_get_event_number(void)
+{
+	return atomic_add_return(1, &fc_event_seq);
+}
+EXPORT_SYMBOL(fc_get_event_number);
+
+
+/**
+ * fc_host_post_event - called to post an even on an fc_host.
+ *
+ * @shost:		host the event occurred on
+ * @event_number:	fc event number obtained from get_fc_event_number()
+ * @event_code:		fc_host event being posted
+ * @event_data:		32bits of data for the event being posted
+ *
+ * Notes:
+ *	This routine assumes no locks are held on entry.
+ **/
+void
+fc_host_post_event(struct Scsi_Host *shost, u32 event_number,
+		enum fc_host_event_code event_code, u32 event_data)
+{
+	struct sk_buff *skb;
+	struct nlmsghdr	*nlh;
+	struct fc_nl_event *event;
+	const char *name;
+	u32 len, skblen;
+	int err;
+
+	if (!scsi_nl_sock) {
+		err = -ENOENT;
+		goto send_fail;
+	}
+
+	len = FC_NL_MSGALIGN(sizeof(*event));
+	skblen = NLMSG_SPACE(len);
+
+	skb = alloc_skb(skblen, GFP_KERNEL);
+	if (!skb) {
+		err = -ENOBUFS;
+		goto send_fail;
+	}
+
+	nlh = nlmsg_put(skb, 0, 0, SCSI_TRANSPORT_MSG,
+				skblen - sizeof(*nlh), 0);
+	if (!nlh) {
+		err = -ENOBUFS;
+		goto send_fail_skb;
+	}
+	event = NLMSG_DATA(nlh);
+
+	INIT_SCSI_NL_HDR(&event->snlh, SCSI_NL_TRANSPORT_FC,
+				FC_NL_ASYNC_EVENT, len);
+	event->seconds = get_seconds();
+	event->vendor_id = 0;
+	event->host_no = shost->host_no;
+	event->event_datalen = sizeof(u32);	/* bytes */
+	event->event_num = event_number;
+	event->event_code = event_code;
+	event->event_data = event_data;
+
+	err = nlmsg_multicast(scsi_nl_sock, skb, 0, SCSI_NL_GRP_FC_EVENTS,
+			      GFP_KERNEL);
+	if (err && (err != -ESRCH))	/* filter no recipient errors */
+		/* nlmsg_multicast already kfree_skb'd */
+		goto send_fail;
+
+	return;
+
+send_fail_skb:
+	kfree_skb(skb);
+send_fail:
+	name = get_fc_host_event_code_name(event_code);
+	printk(KERN_WARNING
+		"%s: Dropped Event : host %d %s data 0x%08x - err %d\n",
+		__FUNCTION__, shost->host_no,
+		(name) ? name : "<unknown>", event_data, err);
+	return;
+}
+EXPORT_SYMBOL(fc_host_post_event);
+
+
+/**
+ * fc_host_post_vendor_event - called to post a vendor unique event on
+ *                             a fc_host
+ *
+ * @shost:		host the event occurred on
+ * @event_number:	fc event number obtained from get_fc_event_number()
+ * @data_len:		amount, in bytes, of vendor unique data
+ * @data_buf:		pointer to vendor unique data
+ *
+ * Notes:
+ *	This routine assumes no locks are held on entry.
+ **/
+void
+fc_host_post_vendor_event(struct Scsi_Host *shost, u32 event_number,
+		u32 data_len, char * data_buf, u64 vendor_id)
+{
+	struct sk_buff *skb;
+	struct nlmsghdr	*nlh;
+	struct fc_nl_event *event;
+	u32 len, skblen;
+	int err;
+
+	if (!scsi_nl_sock) {
+		err = -ENOENT;
+		goto send_vendor_fail;
+	}
+
+	len = FC_NL_MSGALIGN(sizeof(*event) + data_len);
+	skblen = NLMSG_SPACE(len);
+
+	skb = alloc_skb(skblen, GFP_KERNEL);
+	if (!skb) {
+		err = -ENOBUFS;
+		goto send_vendor_fail;
+	}
+
+	nlh = nlmsg_put(skb, 0, 0, SCSI_TRANSPORT_MSG,
+				skblen - sizeof(*nlh), 0);
+	if (!nlh) {
+		err = -ENOBUFS;
+		goto send_vendor_fail_skb;
+	}
+	event = NLMSG_DATA(nlh);
+
+	INIT_SCSI_NL_HDR(&event->snlh, SCSI_NL_TRANSPORT_FC,
+				FC_NL_ASYNC_EVENT, len);
+	event->seconds = get_seconds();
+	event->vendor_id = vendor_id;
+	event->host_no = shost->host_no;
+	event->event_datalen = data_len;	/* bytes */
+	event->event_num = event_number;
+	event->event_code = FCH_EVT_VENDOR_UNIQUE;
+	memcpy(&event->event_data, data_buf, data_len);
+
+	err = nlmsg_multicast(scsi_nl_sock, skb, 0, SCSI_NL_GRP_FC_EVENTS,
+			      GFP_KERNEL);
+	if (err && (err != -ESRCH))	/* filter no recipient errors */
+		/* nlmsg_multicast already kfree_skb'd */
+		goto send_vendor_fail;
+
+	return;
+
+send_vendor_fail_skb:
+	kfree_skb(skb);
+send_vendor_fail:
+	printk(KERN_WARNING
+		"%s: Dropped Event : host %d vendor_unique - err %d\n",
+		__FUNCTION__, shost->host_no, err);
+	return;
+}
+EXPORT_SYMBOL(fc_host_post_vendor_event);
+
+
 
 static __init int fc_transport_init(void)
 {
-	int error = transport_class_register(&fc_host_class);
+	int error;
+
+	atomic_set(&fc_event_seq, 0);
+
+	error = transport_class_register(&fc_host_class);
 	if (error)
 		return error;
 	error = transport_class_register(&fc_rport_class);
@@ -424,11 +625,14 @@ store_fc_rport_##field(struct class_device *cdev, const char *buf,	\
 	struct fc_rport *rport = transport_class_to_rport(cdev);	\
 	struct Scsi_Host *shost = rport_to_shost(rport);		\
 	struct fc_internal *i = to_fc_internal(shost->transportt);	\
+	char *cp;							\
 	if ((rport->port_state == FC_PORTSTATE_BLOCKED) ||		\
 	    (rport->port_state == FC_PORTSTATE_DELETED) ||		\
 	    (rport->port_state == FC_PORTSTATE_NOTPRESENT))		\
 		return -EBUSY;						\
-	val = simple_strtoul(buf, NULL, 0);				\
+	val = simple_strtoul(buf, &cp, 0);				\
+	if (*cp && (*cp != '\n'))					\
+		return -EINVAL;						\
 	i->f->set_rport_##field(rport, val);				\
 	return count;							\
 }
@@ -510,6 +714,13 @@ static FC_CLASS_DEVICE_ATTR(rport, title, S_IRUGO,			\
 	if (i->f->show_rport_##field)					\
 		count++
 
+#define SETUP_PRIVATE_RPORT_ATTRIBUTE_RW(field)				\
+{									\
+	i->private_rport_attrs[count] = class_device_attr_rport_##field; \
+	i->rport_attrs[count] = &i->private_rport_attrs[count];		\
+	count++;							\
+}
+
 
 /* The FC Transport Remote Port Attributes: */
 
@@ -542,12 +753,14 @@ store_fc_rport_dev_loss_tmo(struct class_device *cdev, const char *buf,
 	struct fc_rport *rport = transport_class_to_rport(cdev);
 	struct Scsi_Host *shost = rport_to_shost(rport);
 	struct fc_internal *i = to_fc_internal(shost->transportt);
+	char *cp;
 	if ((rport->port_state == FC_PORTSTATE_BLOCKED) ||
 	    (rport->port_state == FC_PORTSTATE_DELETED) ||
 	    (rport->port_state == FC_PORTSTATE_NOTPRESENT))
 		return -EBUSY;
-	val = simple_strtoul(buf, NULL, 0);
-	if ((val < 0) || (val > SCSI_DEVICE_BLOCK_MAX_TIMEOUT))
+	val = simple_strtoul(buf, &cp, 0);
+	if ((*cp && (*cp != '\n')) ||
+	    (val < 0) || (val > SCSI_DEVICE_BLOCK_MAX_TIMEOUT))
 		return -EINVAL;
 	i->f->set_rport_dev_loss_tmo(rport, val);
 	return count;
@@ -597,6 +810,44 @@ static FC_CLASS_DEVICE_ATTR(rport, roles, S_IRUGO,
 fc_private_rport_rd_enum_attr(port_state, FC_PORTSTATE_MAX_NAMELEN);
 fc_private_rport_rd_attr(scsi_target_id, "%d\n", 20);
 
+/*
+ * fast_io_fail_tmo attribute
+ */
+static ssize_t
+show_fc_rport_fast_io_fail_tmo (struct class_device *cdev, char *buf)
+{
+	struct fc_rport *rport = transport_class_to_rport(cdev);
+
+	if (rport->fast_io_fail_tmo == -1)
+		return snprintf(buf, 5, "off\n");
+	return snprintf(buf, 20, "%d\n", rport->fast_io_fail_tmo);
+}
+
+static ssize_t
+store_fc_rport_fast_io_fail_tmo(struct class_device *cdev, const char *buf,
+			   size_t count)
+{
+	int val;
+	char *cp;
+	struct fc_rport *rport = transport_class_to_rport(cdev);
+
+	if ((rport->port_state == FC_PORTSTATE_BLOCKED) ||
+	    (rport->port_state == FC_PORTSTATE_DELETED) ||
+	    (rport->port_state == FC_PORTSTATE_NOTPRESENT))
+		return -EBUSY;
+	if (strncmp(buf, "off", 3) == 0)
+		rport->fast_io_fail_tmo = -1;
+	else {
+		val = simple_strtoul(buf, &cp, 0);
+		if ((*cp && (*cp != '\n')) ||
+		    (val < 0) || (val >= rport->dev_loss_tmo))
+			return -EINVAL;
+		rport->fast_io_fail_tmo = val;
+	}
+	return count;
+}
+static FC_CLASS_DEVICE_ATTR(rport, fast_io_fail_tmo, S_IRUGO | S_IWUSR,
+	show_fc_rport_fast_io_fail_tmo, store_fc_rport_fast_io_fail_tmo);
 
 
 /*
@@ -682,12 +933,34 @@ store_fc_host_##field(struct class_device *cdev, const char *buf,	\
 	int val;							\
 	struct Scsi_Host *shost = transport_class_to_shost(cdev);	\
 	struct fc_internal *i = to_fc_internal(shost->transportt);	\
+	char *cp;							\
 									\
-	val = simple_strtoul(buf, NULL, 0);				\
+	val = simple_strtoul(buf, &cp, 0);				\
+	if (*cp && (*cp != '\n'))					\
+		return -EINVAL;						\
 	i->f->set_host_##field(shost, val);				\
 	return count;							\
 }
 
+#define fc_host_store_str_function(field, slen)				\
+static ssize_t								\
+store_fc_host_##field(struct class_device *cdev, const char *buf,	\
+			   size_t count)				\
+{									\
+	struct Scsi_Host *shost = transport_class_to_shost(cdev);	\
+	struct fc_internal *i = to_fc_internal(shost->transportt);	\
+	unsigned int cnt=count;						\
+									\
+	/* count may include a LF at end of string */			\
+	if (buf[cnt-1] == '\n')						\
+		cnt--;							\
+	if (cnt > ((slen) - 1))						\
+		return -EINVAL;						\
+	memcpy(fc_host_##field(shost), buf, cnt);			\
+	i->f->set_host_##field(shost);					\
+	return count;							\
+}
+
 #define fc_host_rd_attr(field, format_string, sz)			\
 	fc_host_show_function(field, format_string, sz, )		\
 static FC_CLASS_DEVICE_ATTR(host, field, S_IRUGO,			\
@@ -815,7 +1088,6 @@ fc_private_host_rd_attr_cast(node_name, "0x%llx\n", 20, unsigned long long);
 fc_private_host_rd_attr_cast(port_name, "0x%llx\n", 20, unsigned long long);
 fc_private_host_rd_attr_cast(permanent_port_name, "0x%llx\n", 20,
 			     unsigned long long);
-fc_private_host_rd_attr(symbolic_name, "%s\n", (FC_SYMBOLIC_NAME_SIZE +1));
 fc_private_host_rd_attr(maxframe_size, "%u bytes\n", 20);
 fc_private_host_rd_attr(serial_number, "%s\n", (FC_SERIAL_NUMBER_SIZE +1));
 
@@ -858,6 +1130,13 @@ fc_host_rd_attr(port_id, "0x%06x\n", 20);
 fc_host_rd_enum_attr(port_type, FC_PORTTYPE_MAX_NAMELEN);
 fc_host_rd_enum_attr(port_state, FC_PORTSTATE_MAX_NAMELEN);
 fc_host_rd_attr_cast(fabric_name, "0x%llx\n", 20, unsigned long long);
+fc_host_rd_attr(symbolic_name, "%s\n", FC_SYMBOLIC_NAME_SIZE + 1);
+
+fc_private_host_show_function(system_hostname, "%s\n",
+		FC_SYMBOLIC_NAME_SIZE + 1, )
+fc_host_store_str_function(system_hostname, FC_SYMBOLIC_NAME_SIZE)
+static FC_CLASS_DEVICE_ATTR(host, system_hostname, S_IRUGO | S_IWUSR,
+		show_fc_host_system_hostname, store_fc_host_system_hostname);
 
 
 /* Private Host Attributes */
@@ -1223,7 +1502,6 @@ fc_attach_transport(struct fc_function_template *ft)
 	SETUP_HOST_ATTRIBUTE_RD(permanent_port_name);
 	SETUP_HOST_ATTRIBUTE_RD(supported_classes);
 	SETUP_HOST_ATTRIBUTE_RD(supported_fc4s);
-	SETUP_HOST_ATTRIBUTE_RD(symbolic_name);
 	SETUP_HOST_ATTRIBUTE_RD(supported_speeds);
 	SETUP_HOST_ATTRIBUTE_RD(maxframe_size);
 	SETUP_HOST_ATTRIBUTE_RD(serial_number);
@@ -1234,6 +1512,8 @@ fc_attach_transport(struct fc_function_template *ft)
 	SETUP_HOST_ATTRIBUTE_RD(active_fc4s);
 	SETUP_HOST_ATTRIBUTE_RD(speed);
 	SETUP_HOST_ATTRIBUTE_RD(fabric_name);
+	SETUP_HOST_ATTRIBUTE_RD(symbolic_name);
+	SETUP_HOST_ATTRIBUTE_RW(system_hostname);
 
 	/* Transport-managed attributes */
 	SETUP_PRIVATE_HOST_ATTRIBUTE_RW(tgtid_bind_type);
@@ -1257,6 +1537,8 @@ fc_attach_transport(struct fc_function_template *ft)
 	SETUP_PRIVATE_RPORT_ATTRIBUTE_RD(roles);
 	SETUP_PRIVATE_RPORT_ATTRIBUTE_RD(port_state);
 	SETUP_PRIVATE_RPORT_ATTRIBUTE_RD(scsi_target_id);
+	if (ft->terminate_rport_io)
+		SETUP_PRIVATE_RPORT_ATTRIBUTE_RW(fast_io_fail_tmo);
 
 	BUG_ON(count > FC_RPORT_NUM_ATTRS);
 
@@ -1328,7 +1610,7 @@ fc_flush_work(struct Scsi_Host *shost)
  * @delay:	jiffies to delay the work queuing
  *
  * Return value:
- * 	0 on success / != 0 for error
+ * 	1 on success / 0 already queued / < 0 for error
  **/
 static int
 fc_queue_devloss_work(struct Scsi_Host *shost, struct work_struct *work,
@@ -1343,6 +1625,9 @@ fc_queue_devloss_work(struct Scsi_Host *shost, struct work_struct *work,
 		return -EINVAL;
 	}
 
+	if (delay == 0)
+		return queue_work(fc_host_devloss_work_q(shost), work);
+
 	return queue_delayed_work(fc_host_devloss_work_q(shost), work, delay);
 }
 
@@ -1435,10 +1720,23 @@ fc_starget_delete(void *data)
 	struct fc_rport *rport = (struct fc_rport *)data;
 	struct Scsi_Host *shost = rport_to_shost(rport);
 	unsigned long flags;
+	struct fc_internal *i = to_fc_internal(shost->transportt);
+
+	/*
+	 * Involve the LLDD if possible. All io on the rport is to
+	 * be terminated, either as part of the dev_loss_tmo callback
+	 * processing, or via the terminate_rport_io function.
+	 */
+	if (i->f->dev_loss_tmo_callbk)
+		i->f->dev_loss_tmo_callbk(rport);
+	else if (i->f->terminate_rport_io)
+		i->f->terminate_rport_io(rport);
 
 	spin_lock_irqsave(shost->host_lock, flags);
 	if (rport->flags & FC_RPORT_DEVLOSS_PENDING) {
 		spin_unlock_irqrestore(shost->host_lock, flags);
+		if (!cancel_delayed_work(&rport->fail_io_work))
+			fc_flush_devloss(shost);
 		if (!cancel_delayed_work(&rport->dev_loss_work))
 			fc_flush_devloss(shost);
 		spin_lock_irqsave(shost->host_lock, flags);
@@ -1461,10 +1759,7 @@ fc_rport_final_delete(void *data)
 	struct fc_rport *rport = (struct fc_rport *)data;
 	struct device *dev = &rport->dev;
 	struct Scsi_Host *shost = rport_to_shost(rport);
-
-	/* Delete SCSI target and sdevs */
-	if (rport->scsi_target_id != -1)
-		fc_starget_delete(data);
+	struct fc_internal *i = to_fc_internal(shost->transportt);
 
 	/*
 	 * if a scan is pending, flush the SCSI Host work_q so that 
@@ -1473,6 +1768,14 @@ fc_rport_final_delete(void *data)
 	if (rport->flags & FC_RPORT_SCAN_PENDING)
 		scsi_flush_work(shost);
 
+	/* Delete SCSI target and sdevs */
+	if (rport->scsi_target_id != -1)
+		fc_starget_delete(data);
+	else if (i->f->dev_loss_tmo_callbk)
+		i->f->dev_loss_tmo_callbk(rport);
+	else if (i->f->terminate_rport_io)
+		i->f->terminate_rport_io(rport);
+
 	transport_remove_device(dev);
 	device_del(dev);
 	transport_destroy_device(dev);
@@ -1524,8 +1827,10 @@ fc_rport_create(struct Scsi_Host *shost, int channel,
 	if (fci->f->dd_fcrport_size)
 		rport->dd_data = &rport[1];
 	rport->channel = channel;
+	rport->fast_io_fail_tmo = -1;
 
 	INIT_WORK(&rport->dev_loss_work, fc_timeout_deleted_rport, rport);
+	INIT_WORK(&rport->fail_io_work, fc_timeout_fail_rport_io, rport);
 	INIT_WORK(&rport->scan_work, fc_scsi_scan_rport, rport);
 	INIT_WORK(&rport->stgt_delete_work, fc_starget_delete, rport);
 	INIT_WORK(&rport->rport_delete_work, fc_rport_final_delete, rport);
@@ -1689,11 +1994,13 @@ fc_remote_port_add(struct Scsi_Host *shost, int channel,
 				/* restart the target */
 
 				/*
-				 * Stop the target timer first. Take no action
+				 * Stop the target timers first. Take no action
 				 * on the del_timer failure as the state
 				 * machine state change will validate the
 				 * transaction.
 				 */
+				if (!cancel_delayed_work(&rport->fail_io_work))
+					fc_flush_devloss(shost);
 				if (!cancel_delayed_work(work))
 					fc_flush_devloss(shost);
 
@@ -1837,6 +2144,7 @@ void
 fc_remote_port_delete(struct fc_rport  *rport)
 {
 	struct Scsi_Host *shost = rport_to_shost(rport);
+	struct fc_internal *i = to_fc_internal(shost->transportt);
 	int timeout = rport->dev_loss_tmo;
 	unsigned long flags;
 
@@ -1867,6 +2175,12 @@ fc_remote_port_delete(struct fc_rport  *rport)
 
 	scsi_target_block(&rport->dev);
 
+	/* see if we need to kill io faster than waiting for device loss */
+	if ((rport->fast_io_fail_tmo != -1) &&
+	    (rport->fast_io_fail_tmo < timeout) && (i->f->terminate_rport_io))
+		fc_queue_devloss_work(shost, &rport->fail_io_work,
+					rport->fast_io_fail_tmo * HZ);
+
 	/* cap the length the devices can be blocked until they are deleted */
 	fc_queue_devloss_work(shost, &rport->dev_loss_work, timeout * HZ);
 }
@@ -1926,6 +2240,8 @@ fc_remote_port_rolechg(struct fc_rport  *rport, u32 roles)
 		 * machine state change will validate the
 		 * transaction.
 		 */
+		if (!cancel_delayed_work(&rport->fail_io_work))
+			fc_flush_devloss(shost);
 		if (!cancel_delayed_work(&rport->dev_loss_work))
 			fc_flush_devloss(shost);
 
@@ -2047,6 +2363,28 @@ fc_timeout_deleted_rport(void  *data)
 }
 
 /**
+ * fc_timeout_fail_rport_io - Timeout handler for a fast io failing on a
+ *                       disconnected SCSI target.
+ *
+ * @data:	rport to terminate io on.
+ *
+ * Notes: Only requests the failure of the io, not that all are flushed
+ *    prior to returning.
+ **/
+static void
+fc_timeout_fail_rport_io(void  *data)
+{
+	struct fc_rport *rport = (struct fc_rport *)data;
+	struct Scsi_Host *shost = rport_to_shost(rport);
+	struct fc_internal *i = to_fc_internal(shost->transportt);
+
+	if (rport->port_state != FC_PORTSTATE_BLOCKED)
+		return;
+
+	i->f->terminate_rport_io(rport);
+}
+
+/**
  * fc_scsi_scan_rport - called to perform a scsi scan on a remote port.
  *
  * @data:	remote port to be scanned.
diff --git a/drivers/scsi/scsi_transport_iscsi.c b/drivers/scsi/scsi_transport_iscsi.c
index 2ecd141..7b0019c 100644
--- a/drivers/scsi/scsi_transport_iscsi.c
+++ b/drivers/scsi/scsi_transport_iscsi.c
@@ -34,7 +34,7 @@
 #define ISCSI_SESSION_ATTRS 11
 #define ISCSI_CONN_ATTRS 11
 #define ISCSI_HOST_ATTRS 0
-#define ISCSI_TRANSPORT_VERSION "1.1-646"
+#define ISCSI_TRANSPORT_VERSION "2.0-685"
 
 struct iscsi_internal {
 	int daemon_pid;
diff --git a/drivers/scsi/scsi_transport_sas.c b/drivers/scsi/scsi_transport_sas.c
index 5a625c3..b5b0c2c 100644
--- a/drivers/scsi/scsi_transport_sas.c
+++ b/drivers/scsi/scsi_transport_sas.c
@@ -77,6 +77,24 @@ get_sas_##title##_names(u32 table_key, char *buf)		\
 	return len;						\
 }
 
+#define sas_bitfield_name_set(title, table)			\
+static ssize_t							\
+set_sas_##title##_names(u32 *table_key, const char *buf)	\
+{								\
+	ssize_t len = 0;					\
+	int i;							\
+								\
+	for (i = 0; i < ARRAY_SIZE(table); i++) {		\
+		len = strlen(table[i].name);			\
+		if (strncmp(buf, table[i].name, len) == 0 &&	\
+		    (buf[len] == '\n' || buf[len] == '\0')) {	\
+			*table_key = table[i].value;		\
+			return 0;				\
+		}						\
+	}							\
+	return -EINVAL;						\
+}
+
 #define sas_bitfield_name_search(title, table)			\
 static ssize_t							\
 get_sas_##title##_names(u32 table_key, char *buf)		\
@@ -131,7 +149,7 @@ static struct {
 	{ SAS_LINK_RATE_6_0_GBPS,	"6.0 Gbit" },
 };
 sas_bitfield_name_search(linkspeed, sas_linkspeed_names)
-
+sas_bitfield_name_set(linkspeed, sas_linkspeed_names)
 
 /*
  * SAS host attributes
@@ -253,10 +271,39 @@ show_sas_phy_##field(struct class_device *cdev, char *buf)		\
 	return get_sas_linkspeed_names(phy->field, buf);		\
 }
 
+/* Fudge to tell if we're minimum or maximum */
+#define sas_phy_store_linkspeed(field)					\
+static ssize_t								\
+store_sas_phy_##field(struct class_device *cdev, const char *buf,	\
+		      size_t count)					\
+{									\
+	struct sas_phy *phy = transport_class_to_phy(cdev);		\
+	struct Scsi_Host *shost = dev_to_shost(phy->dev.parent);	\
+	struct sas_internal *i = to_sas_internal(shost->transportt);	\
+	u32 value;							\
+	struct sas_phy_linkrates rates = {0};				\
+	int error;							\
+									\
+	error = set_sas_linkspeed_names(&value, buf);			\
+	if (error)							\
+		return error;						\
+	rates.field = value;						\
+	error = i->f->set_phy_speed(phy, &rates);			\
+									\
+	return error ? error : count;					\
+}
+
+#define sas_phy_linkspeed_rw_attr(field)				\
+	sas_phy_show_linkspeed(field)					\
+	sas_phy_store_linkspeed(field)					\
+static CLASS_DEVICE_ATTR(field, S_IRUGO, show_sas_phy_##field,		\
+	store_sas_phy_##field)
+
 #define sas_phy_linkspeed_attr(field)					\
 	sas_phy_show_linkspeed(field)					\
 static CLASS_DEVICE_ATTR(field, S_IRUGO, show_sas_phy_##field, NULL)
 
+
 #define sas_phy_show_linkerror(field)					\
 static ssize_t								\
 show_sas_phy_##field(struct class_device *cdev, char *buf)		\
@@ -266,9 +313,6 @@ show_sas_phy_##field(struct class_device *cdev, char *buf)		\
 	struct sas_internal *i = to_sas_internal(shost->transportt);	\
 	int error;							\
 									\
-	if (!phy->local_attached)					\
-		return -EINVAL;						\
-									\
 	error = i->f->get_linkerrors ? i->f->get_linkerrors(phy) : 0;	\
 	if (error)							\
 		return error;						\
@@ -299,9 +343,6 @@ static ssize_t do_sas_phy_reset(struct class_device *cdev,
 	struct sas_internal *i = to_sas_internal(shost->transportt);
 	int error;
 
-	if (!phy->local_attached)
-		return -EINVAL;
-
 	error = i->f->phy_reset(phy, hard_reset);
 	if (error)
 		return error;
@@ -332,9 +373,9 @@ sas_phy_simple_attr(identify.phy_identifier, phy_identifier, "%d\n", u8);
 //sas_phy_simple_attr(port_identifier, port_identifier, "%d\n", int);
 sas_phy_linkspeed_attr(negotiated_linkrate);
 sas_phy_linkspeed_attr(minimum_linkrate_hw);
-sas_phy_linkspeed_attr(minimum_linkrate);
+sas_phy_linkspeed_rw_attr(minimum_linkrate);
 sas_phy_linkspeed_attr(maximum_linkrate_hw);
-sas_phy_linkspeed_attr(maximum_linkrate);
+sas_phy_linkspeed_rw_attr(maximum_linkrate);
 sas_phy_linkerror_attr(invalid_dword_count);
 sas_phy_linkerror_attr(running_disparity_error_count);
 sas_phy_linkerror_attr(loss_of_dword_sync_count);
@@ -849,7 +890,7 @@ show_sas_rphy_enclosure_identifier(struct class_device *cdev, char *buf)
 	 * Only devices behind an expander are supported, because the
 	 * enclosure identifier is a SMP feature.
 	 */
-	if (phy->local_attached)
+	if (scsi_is_sas_phy_local(phy))
 		return -EINVAL;
 
 	error = i->f->get_enclosure_identifier(rphy, &identifier);
@@ -870,7 +911,7 @@ show_sas_rphy_bay_identifier(struct class_device *cdev, char *buf)
 	struct sas_internal *i = to_sas_internal(shost->transportt);
 	int val;
 
-	if (phy->local_attached)
+	if (scsi_is_sas_phy_local(phy))
 		return -EINVAL;
 
 	val = i->f->get_bay_identifier(rphy);
@@ -1316,13 +1357,23 @@ static int sas_user_scan(struct Scsi_Host *shost, uint channel,
  * Setup / Teardown code
  */
 
-#define SETUP_TEMPLATE(attrb, field, perm, test)				\
+#define SETUP_TEMPLATE(attrb, field, perm, test)			\
 	i->private_##attrb[count] = class_device_attr_##field;		\
 	i->private_##attrb[count].attr.mode = perm;			\
 	i->attrb[count] = &i->private_##attrb[count];			\
 	if (test)							\
 		count++
 
+#define SETUP_TEMPLATE_RW(attrb, field, perm, test, ro_test, ro_perm)	\
+	i->private_##attrb[count] = class_device_attr_##field;		\
+	i->private_##attrb[count].attr.mode = perm;			\
+	if (ro_test) {							\
+		i->private_##attrb[count].attr.mode = ro_perm;		\
+		i->private_##attrb[count].store = NULL;			\
+	}								\
+	i->attrb[count] = &i->private_##attrb[count];			\
+	if (test)							\
+		count++
 
 #define SETUP_RPORT_ATTRIBUTE(field) 					\
 	SETUP_TEMPLATE(rphy_attrs, field, S_IRUGO, 1)
@@ -1333,6 +1384,10 @@ static int sas_user_scan(struct Scsi_Host *shost, uint channel,
 #define SETUP_PHY_ATTRIBUTE(field)					\
 	SETUP_TEMPLATE(phy_attrs, field, S_IRUGO, 1)
 
+#define SETUP_PHY_ATTRIBUTE_RW(field)					\
+	SETUP_TEMPLATE_RW(phy_attrs, field, S_IRUGO | S_IWUSR, 1,	\
+			!i->f->set_phy_speed, S_IRUGO)
+
 #define SETUP_PORT_ATTRIBUTE(field)					\
 	SETUP_TEMPLATE(port_attrs, field, S_IRUGO, 1)
 
@@ -1413,9 +1468,9 @@ sas_attach_transport(struct sas_function_template *ft)
 	//SETUP_PHY_ATTRIBUTE(port_identifier);
 	SETUP_PHY_ATTRIBUTE(negotiated_linkrate);
 	SETUP_PHY_ATTRIBUTE(minimum_linkrate_hw);
-	SETUP_PHY_ATTRIBUTE(minimum_linkrate);
+	SETUP_PHY_ATTRIBUTE_RW(minimum_linkrate);
 	SETUP_PHY_ATTRIBUTE(maximum_linkrate_hw);
-	SETUP_PHY_ATTRIBUTE(maximum_linkrate);
+	SETUP_PHY_ATTRIBUTE_RW(maximum_linkrate);
 
 	SETUP_PHY_ATTRIBUTE(invalid_dword_count);
 	SETUP_PHY_ATTRIBUTE(running_disparity_error_count);
diff --git a/drivers/scsi/scsi_transport_spi.c b/drivers/scsi/scsi_transport_spi.c
index 29a9a53..9f070f0 100644
--- a/drivers/scsi/scsi_transport_spi.c
+++ b/drivers/scsi/scsi_transport_spi.c
@@ -47,6 +47,7 @@
 
 /* Private data accessors (keep these out of the header file) */
 #define spi_dv_pending(x) (((struct spi_transport_attrs *)&(x)->starget_data)->dv_pending)
+#define spi_dv_in_progress(x) (((struct spi_transport_attrs *)&(x)->starget_data)->dv_in_progress)
 #define spi_dv_mutex(x) (((struct spi_transport_attrs *)&(x)->starget_data)->dv_mutex)
 
 struct spi_internal {
@@ -240,6 +241,7 @@ static int spi_setup_transport_attrs(struct transport_container *tc,
 	spi_pcomp_en(starget) = 0;
 	spi_hold_mcs(starget) = 0;
 	spi_dv_pending(starget) = 0;
+	spi_dv_in_progress(starget) = 0;
 	spi_initial_dv(starget) = 0;
 	mutex_init(&spi_dv_mutex(starget));
 
@@ -830,28 +832,37 @@ spi_dv_device_internal(struct scsi_device *sdev, u8 *buffer)
 	DV_SET(period, spi_min_period(starget));
 	/* try QAS requests; this should be harmless to set if the
 	 * target supports it */
-	if (scsi_device_qas(sdev))
+	if (scsi_device_qas(sdev)) {
 		DV_SET(qas, 1);
-	/* Also try IU transfers */
-	if (scsi_device_ius(sdev))
+	} else {
+		DV_SET(qas, 0);
+	}
+
+	if (scsi_device_ius(sdev) && spi_min_period(starget) < 9) {
+		/* This u320 (or u640). Set IU transfers */
 		DV_SET(iu, 1);
-	if (spi_min_period(starget) < 9) {
-		/* This u320 (or u640). Ignore the coupled parameters
-		 * like DT and IU, but set the optional ones */
+		/* Then set the optional parameters */
 		DV_SET(rd_strm, 1);
 		DV_SET(wr_flow, 1);
 		DV_SET(rti, 1);
 		if (spi_min_period(starget) == 8)
 			DV_SET(pcomp_en, 1);
+	} else {
+		DV_SET(iu, 0);
 	}
+
 	/* now that we've done all this, actually check the bus
 	 * signal type (if known).  Some devices are stupid on
 	 * a SE bus and still claim they can try LVD only settings */
 	if (i->f->get_signalling)
 		i->f->get_signalling(shost);
 	if (spi_signalling(shost) == SPI_SIGNAL_SE ||
-	    spi_signalling(shost) == SPI_SIGNAL_HVD)
+	    spi_signalling(shost) == SPI_SIGNAL_HVD ||
+	    !scsi_device_dt(sdev)) {
 		DV_SET(dt, 0);
+	} else {
+		DV_SET(dt, 1);
+	}
 	/* Do the read only INQUIRY tests */
 	spi_dv_retrain(sdev, buffer, buffer + sdev->inquiry_len,
 		       spi_dv_device_compare_inquiry);
@@ -907,6 +918,10 @@ spi_dv_device(struct scsi_device *sdev)
 	if (unlikely(scsi_device_get(sdev)))
 		return;
 
+	if (unlikely(spi_dv_in_progress(starget)))
+		return;
+	spi_dv_in_progress(starget) = 1;
+
 	buffer = kzalloc(len, GFP_KERNEL);
 
 	if (unlikely(!buffer))
@@ -938,6 +953,7 @@ spi_dv_device(struct scsi_device *sdev)
  out_free:
 	kfree(buffer);
  out_put:
+	spi_dv_in_progress(starget) = 0;
 	scsi_device_put(sdev);
 }
 EXPORT_SYMBOL(spi_dv_device);
diff --git a/drivers/scsi/sd.c b/drivers/scsi/sd.c
index 98bd3aa..638cff4 100644
--- a/drivers/scsi/sd.c
+++ b/drivers/scsi/sd.c
@@ -1215,7 +1215,7 @@ repeat:
 		/* Either no media are present but the drive didn't tell us,
 		   or they are present but the read capacity command fails */
 		/* sdkp->media_present = 0; -- not always correct */
-		sdkp->capacity = 0x200000; /* 1 GB - random */
+		sdkp->capacity = 0; /* unknown mapped to zero - as usual */
 
 		return;
 	} else if (the_result && longrc) {
diff --git a/drivers/scsi/sgiwd93.c b/drivers/scsi/sgiwd93.c
index 7cd366f..4f1db6f 100644
--- a/drivers/scsi/sgiwd93.c
+++ b/drivers/scsi/sgiwd93.c
@@ -97,7 +97,7 @@ static irqreturn_t sgiwd93_intr(int irq, void *dev_id, struct pt_regs *regs)
 }
 
 static inline
-void fill_hpc_entries(struct hpc_chunk *hcp, Scsi_Cmnd *cmd, int datainp)
+void fill_hpc_entries(struct hpc_chunk *hcp, struct scsi_cmnd *cmd, int datainp)
 {
 	unsigned long len = cmd->SCp.this_residual;
 	void *addr = cmd->SCp.ptr;
@@ -129,7 +129,7 @@ void fill_hpc_entries(struct hpc_chunk *hcp, Scsi_Cmnd *cmd, int datainp)
 	hcp->desc.cntinfo = HPCDMA_EOX;
 }
 
-static int dma_setup(Scsi_Cmnd *cmd, int datainp)
+static int dma_setup(struct scsi_cmnd *cmd, int datainp)
 {
 	struct ip22_hostdata *hdata = HDATA(cmd->device->host);
 	struct hpc3_scsiregs *hregs =
@@ -163,7 +163,7 @@ static int dma_setup(Scsi_Cmnd *cmd, int datainp)
 	return 0;
 }
 
-static void dma_stop(struct Scsi_Host *instance, Scsi_Cmnd *SCpnt,
+static void dma_stop(struct Scsi_Host *instance, struct scsi_cmnd *SCpnt,
 		     int status)
 {
 	struct ip22_hostdata *hdata = HDATA(instance);
@@ -305,7 +305,7 @@ static int sgiwd93_release(struct Scsi_Host *instance)
 	return 1;
 }
 
-static int sgiwd93_bus_reset(Scsi_Cmnd *cmd)
+static int sgiwd93_bus_reset(struct scsi_cmnd *cmd)
 {
 	/* FIXME perform bus-specific reset */
 
diff --git a/drivers/scsi/stex.c b/drivers/scsi/stex.c
new file mode 100644
index 0000000..3cf3106
--- /dev/null
+++ b/drivers/scsi/stex.c
@@ -0,0 +1,1252 @@
+/*
+ * SuperTrak EX Series Storage Controller driver for Linux
+ *
+ *	Copyright (C) 2005, 2006 Promise Technology Inc.
+ *
+ *	This program is free software; you can redistribute it and/or
+ *	modify it under the terms of the GNU General Public License
+ *	as published by the Free Software Foundation; either version
+ *	2 of the License, or (at your option) any later version.
+ *
+ *	Written By:
+ *		Ed Lin <promise_linux@promise.com>
+ *
+ *	Version: 2.9.0.13
+ *
+ */
+
+#include <linux/init.h>
+#include <linux/errno.h>
+#include <linux/kernel.h>
+#include <linux/delay.h>
+#include <linux/sched.h>
+#include <linux/time.h>
+#include <linux/pci.h>
+#include <linux/blkdev.h>
+#include <linux/interrupt.h>
+#include <linux/types.h>
+#include <linux/module.h>
+#include <linux/spinlock.h>
+#include <asm/io.h>
+#include <asm/irq.h>
+#include <asm/byteorder.h>
+#include <scsi/scsi.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_tcq.h>
+
+#define DRV_NAME "stex"
+#define ST_DRIVER_VERSION "2.9.0.13"
+#define ST_VER_MAJOR 		2
+#define ST_VER_MINOR 		9
+#define ST_OEM 			0
+#define ST_BUILD_VER 		13
+
+enum {
+	/* MU register offset */
+	IMR0	= 0x10,	/* MU_INBOUND_MESSAGE_REG0 */
+	IMR1	= 0x14,	/* MU_INBOUND_MESSAGE_REG1 */
+	OMR0	= 0x18,	/* MU_OUTBOUND_MESSAGE_REG0 */
+	OMR1	= 0x1c,	/* MU_OUTBOUND_MESSAGE_REG1 */
+	IDBL	= 0x20,	/* MU_INBOUND_DOORBELL */
+	IIS	= 0x24,	/* MU_INBOUND_INTERRUPT_STATUS */
+	IIM	= 0x28,	/* MU_INBOUND_INTERRUPT_MASK */
+	ODBL	= 0x2c,	/* MU_OUTBOUND_DOORBELL */
+	OIS	= 0x30,	/* MU_OUTBOUND_INTERRUPT_STATUS */
+	OIM	= 0x3c,	/* MU_OUTBOUND_INTERRUPT_MASK */
+
+	/* MU register value */
+	MU_INBOUND_DOORBELL_HANDSHAKE		= 1,
+	MU_INBOUND_DOORBELL_REQHEADCHANGED	= 2,
+	MU_INBOUND_DOORBELL_STATUSTAILCHANGED	= 4,
+	MU_INBOUND_DOORBELL_HMUSTOPPED		= 8,
+	MU_INBOUND_DOORBELL_RESET		= 16,
+
+	MU_OUTBOUND_DOORBELL_HANDSHAKE		= 1,
+	MU_OUTBOUND_DOORBELL_REQUESTTAILCHANGED	= 2,
+	MU_OUTBOUND_DOORBELL_STATUSHEADCHANGED	= 4,
+	MU_OUTBOUND_DOORBELL_BUSCHANGE		= 8,
+	MU_OUTBOUND_DOORBELL_HASEVENT		= 16,
+
+	/* MU status code */
+	MU_STATE_STARTING			= 1,
+	MU_STATE_FMU_READY_FOR_HANDSHAKE	= 2,
+	MU_STATE_SEND_HANDSHAKE_FRAME		= 3,
+	MU_STATE_STARTED			= 4,
+	MU_STATE_RESETTING			= 5,
+
+	MU_MAX_DELAY_TIME			= 240000,
+	MU_HANDSHAKE_SIGNATURE			= 0x55aaaa55,
+	HMU_PARTNER_TYPE			= 2,
+
+	/* firmware returned values */
+	SRB_STATUS_SUCCESS			= 0x01,
+	SRB_STATUS_ERROR			= 0x04,
+	SRB_STATUS_BUSY				= 0x05,
+	SRB_STATUS_INVALID_REQUEST		= 0x06,
+	SRB_STATUS_SELECTION_TIMEOUT		= 0x0A,
+	SRB_SEE_SENSE 				= 0x80,
+
+	/* task attribute */
+	TASK_ATTRIBUTE_SIMPLE			= 0x0,
+	TASK_ATTRIBUTE_HEADOFQUEUE		= 0x1,
+	TASK_ATTRIBUTE_ORDERED			= 0x2,
+	TASK_ATTRIBUTE_ACA			= 0x4,
+
+	/* request count, etc. */
+	MU_MAX_REQUEST				= 32,
+
+	/* one message wasted, use MU_MAX_REQUEST+1
+		to handle MU_MAX_REQUEST messages */
+	MU_REQ_COUNT				= (MU_MAX_REQUEST + 1),
+	MU_STATUS_COUNT				= (MU_MAX_REQUEST + 1),
+
+	STEX_CDB_LENGTH				= MAX_COMMAND_SIZE,
+	REQ_VARIABLE_LEN			= 1024,
+	STATUS_VAR_LEN				= 128,
+	ST_CAN_QUEUE				= MU_MAX_REQUEST,
+	ST_CMD_PER_LUN				= MU_MAX_REQUEST,
+	ST_MAX_SG				= 32,
+
+	/* sg flags */
+	SG_CF_EOT				= 0x80,	/* end of table */
+	SG_CF_64B				= 0x40,	/* 64 bit item */
+	SG_CF_HOST				= 0x20,	/* sg in host memory */
+
+	ST_MAX_ARRAY_SUPPORTED			= 16,
+	ST_MAX_TARGET_NUM			= (ST_MAX_ARRAY_SUPPORTED+1),
+	ST_MAX_LUN_PER_TARGET			= 16,
+
+	st_shasta				= 0,
+	st_vsc					= 1,
+
+	PASSTHRU_REQ_TYPE			= 0x00000001,
+	PASSTHRU_REQ_NO_WAKEUP			= 0x00000100,
+	ST_INTERNAL_TIMEOUT			= 30,
+
+	/* vendor specific commands of Promise */
+	ARRAY_CMD				= 0xe0,
+	CONTROLLER_CMD				= 0xe1,
+	DEBUGGING_CMD				= 0xe2,
+	PASSTHRU_CMD				= 0xe3,
+
+	PASSTHRU_GET_ADAPTER			= 0x05,
+	PASSTHRU_GET_DRVVER			= 0x10,
+	CTLR_POWER_STATE_CHANGE			= 0x0e,
+	CTLR_POWER_SAVING			= 0x01,
+
+	PASSTHRU_SIGNATURE			= 0x4e415041,
+
+	INQUIRY_EVPD				= 0x01,
+};
+
+struct st_sgitem {
+	u8 ctrl;	/* SG_CF_xxx */
+	u8 reserved[3];
+	__le32 count;
+	__le32 addr;
+	__le32 addr_hi;
+};
+
+struct st_sgtable {
+	__le16 sg_count;
+	__le16 max_sg_count;
+	__le32 sz_in_byte;
+	struct st_sgitem table[ST_MAX_SG];
+};
+
+struct handshake_frame {
+	__le32 rb_phy;		/* request payload queue physical address */
+	__le32 rb_phy_hi;
+	__le16 req_sz;		/* size of each request payload */
+	__le16 req_cnt;		/* count of reqs the buffer can hold */
+	__le16 status_sz;	/* size of each status payload */
+	__le16 status_cnt;	/* count of status the buffer can hold */
+	__le32 hosttime;	/* seconds from Jan 1, 1970 (GMT) */
+	__le32 hosttime_hi;
+	u8 partner_type;	/* who sends this frame */
+	u8 reserved0[7];
+	__le32 partner_ver_major;
+	__le32 partner_ver_minor;
+	__le32 partner_ver_oem;
+	__le32 partner_ver_build;
+	u32 reserved1[4];
+};
+
+struct req_msg {
+	__le16 tag;
+	u8 lun;
+	u8 target;
+	u8 task_attr;
+	u8 task_manage;
+	u8 prd_entry;
+	u8 payload_sz;		/* payload size in 4-byte */
+	u8 cdb[STEX_CDB_LENGTH];
+	u8 variable[REQ_VARIABLE_LEN];
+};
+
+struct status_msg {
+	__le16 tag;
+	u8 lun;
+	u8 target;
+	u8 srb_status;
+	u8 scsi_status;
+	u8 reserved;
+	u8 payload_sz;		/* payload size in 4-byte */
+	u8 variable[STATUS_VAR_LEN];
+};
+
+struct ver_info {
+	u32 major;
+	u32 minor;
+	u32 oem;
+	u32 build;
+	u32 reserved[2];
+};
+
+struct st_frame {
+	u32 base[6];
+	u32 rom_addr;
+
+	struct ver_info drv_ver;
+	struct ver_info bios_ver;
+
+	u32 bus;
+	u32 slot;
+	u32 irq_level;
+	u32 irq_vec;
+	u32 id;
+	u32 subid;
+
+	u32 dimm_size;
+	u8 dimm_type;
+	u8 reserved[3];
+
+	u32 channel;
+	u32 reserved1;
+};
+
+struct st_drvver {
+	u32 major;
+	u32 minor;
+	u32 oem;
+	u32 build;
+	u32 signature[2];
+	u8 console_id;
+	u8 host_no;
+	u8 reserved0[2];
+	u32 reserved[3];
+};
+
+#define MU_REQ_BUFFER_SIZE	(MU_REQ_COUNT * sizeof(struct req_msg))
+#define MU_STATUS_BUFFER_SIZE	(MU_STATUS_COUNT * sizeof(struct status_msg))
+#define MU_BUFFER_SIZE		(MU_REQ_BUFFER_SIZE + MU_STATUS_BUFFER_SIZE)
+#define STEX_BUFFER_SIZE	(MU_BUFFER_SIZE + sizeof(struct st_frame))
+
+struct st_ccb {
+	struct req_msg *req;
+	struct scsi_cmnd *cmd;
+
+	void *sense_buffer;
+	unsigned int sense_bufflen;
+	int sg_count;
+
+	u32 req_type;
+	u8 srb_status;
+	u8 scsi_status;
+};
+
+struct st_hba {
+	void __iomem *mmio_base;	/* iomapped PCI memory space */
+	void *dma_mem;
+	dma_addr_t dma_handle;
+
+	struct Scsi_Host *host;
+	struct pci_dev *pdev;
+
+	u32 req_head;
+	u32 req_tail;
+	u32 status_head;
+	u32 status_tail;
+
+	struct status_msg *status_buffer;
+	void *copy_buffer; /* temp buffer for driver-handled commands */
+	struct st_ccb ccb[MU_MAX_REQUEST];
+	struct st_ccb *wait_ccb;
+	wait_queue_head_t waitq;
+
+	unsigned int mu_status;
+	int out_req_cnt;
+
+	unsigned int cardtype;
+};
+
+static const char console_inq_page[] =
+{
+	0x03,0x00,0x03,0x03,0xFA,0x00,0x00,0x30,
+	0x50,0x72,0x6F,0x6D,0x69,0x73,0x65,0x20,	/* "Promise " */
+	0x52,0x41,0x49,0x44,0x20,0x43,0x6F,0x6E,	/* "RAID Con" */
+	0x73,0x6F,0x6C,0x65,0x20,0x20,0x20,0x20,	/* "sole    " */
+	0x31,0x2E,0x30,0x30,0x20,0x20,0x20,0x20,	/* "1.00    " */
+	0x53,0x58,0x2F,0x52,0x53,0x41,0x46,0x2D,	/* "SX/RSAF-" */
+	0x54,0x45,0x31,0x2E,0x30,0x30,0x20,0x20,	/* "TE1.00  " */
+	0x0C,0x20,0x20,0x20,0x20,0x20,0x20,0x20
+};
+
+MODULE_AUTHOR("Ed Lin");
+MODULE_DESCRIPTION("Promise Technology SuperTrak EX Controllers");
+MODULE_LICENSE("GPL");
+MODULE_VERSION(ST_DRIVER_VERSION);
+
+static void stex_gettime(__le32 *time)
+{
+	struct timeval tv;
+	do_gettimeofday(&tv);
+
+	*time = cpu_to_le32(tv.tv_sec & 0xffffffff);
+	*(time + 1) = cpu_to_le32((tv.tv_sec >> 16) >> 16);
+}
+
+static struct status_msg *stex_get_status(struct st_hba *hba)
+{
+	struct status_msg *status =
+		hba->status_buffer + hba->status_tail;
+
+	++hba->status_tail;
+	hba->status_tail %= MU_STATUS_COUNT;
+
+	return status;
+}
+
+static void stex_set_sense(struct scsi_cmnd *cmd, u8 sk, u8 asc, u8 ascq)
+{
+	cmd->result = (DRIVER_SENSE << 24) | SAM_STAT_CHECK_CONDITION;
+
+	cmd->sense_buffer[0] = 0x70;    /* fixed format, current */
+	cmd->sense_buffer[2] = sk;
+	cmd->sense_buffer[7] = 18 - 8;  /* additional sense length */
+	cmd->sense_buffer[12] = asc;
+	cmd->sense_buffer[13] = ascq;
+}
+
+static void stex_invalid_field(struct scsi_cmnd *cmd,
+			       void (*done)(struct scsi_cmnd *))
+{
+	/* "Invalid field in cbd" */
+	stex_set_sense(cmd, ILLEGAL_REQUEST, 0x24, 0x0);
+	done(cmd);
+}
+
+static struct req_msg *stex_alloc_req(struct st_hba *hba)
+{
+	struct req_msg *req = ((struct req_msg *)hba->dma_mem) +
+		hba->req_head;
+
+	++hba->req_head;
+	hba->req_head %= MU_REQ_COUNT;
+
+	return req;
+}
+
+static int stex_map_sg(struct st_hba *hba,
+	struct req_msg *req, struct st_ccb *ccb)
+{
+	struct pci_dev *pdev = hba->pdev;
+	struct scsi_cmnd *cmd;
+	dma_addr_t dma_handle;
+	struct scatterlist *src;
+	struct st_sgtable *dst;
+	int i;
+
+	cmd = ccb->cmd;
+	dst = (struct st_sgtable *)req->variable;
+	dst->max_sg_count = cpu_to_le16(ST_MAX_SG);
+	dst->sz_in_byte = cpu_to_le32(cmd->request_bufflen);
+
+	if (cmd->use_sg) {
+		int n_elem;
+
+		src = (struct scatterlist *) cmd->request_buffer;
+		n_elem = pci_map_sg(pdev, src,
+			cmd->use_sg, cmd->sc_data_direction);
+		if (n_elem <= 0)
+			return -EIO;
+
+		ccb->sg_count = n_elem;
+		dst->sg_count = cpu_to_le16((u16)n_elem);
+
+		for (i = 0; i < n_elem; i++, src++) {
+			dst->table[i].count = cpu_to_le32((u32)sg_dma_len(src));
+			dst->table[i].addr =
+				cpu_to_le32(sg_dma_address(src) & 0xffffffff);
+			dst->table[i].addr_hi =
+				cpu_to_le32((sg_dma_address(src) >> 16) >> 16);
+			dst->table[i].ctrl = SG_CF_64B | SG_CF_HOST;
+		}
+		dst->table[--i].ctrl |= SG_CF_EOT;
+		return 0;
+	}
+
+	dma_handle = pci_map_single(pdev, cmd->request_buffer,
+		cmd->request_bufflen, cmd->sc_data_direction);
+	cmd->SCp.dma_handle = dma_handle;
+
+	ccb->sg_count = 1;
+	dst->sg_count = cpu_to_le16(1);
+	dst->table[0].addr = cpu_to_le32(dma_handle & 0xffffffff);
+	dst->table[0].addr_hi = cpu_to_le32((dma_handle >> 16) >> 16);
+	dst->table[0].count = cpu_to_le32((u32)cmd->request_bufflen);
+	dst->table[0].ctrl = SG_CF_EOT | SG_CF_64B | SG_CF_HOST;
+
+	return 0;
+}
+
+static void stex_internal_copy(struct scsi_cmnd *cmd,
+	const void *src, size_t *count, int sg_count)
+{
+	size_t lcount;
+	size_t len;
+	void *s, *d, *base = NULL;
+	if (*count > cmd->request_bufflen)
+		*count = cmd->request_bufflen;
+	lcount = *count;
+	while (lcount) {
+		len = lcount;
+		s = (void *)src;
+		if (cmd->use_sg) {
+			size_t offset = *count - lcount;
+			s += offset;
+			base = scsi_kmap_atomic_sg(cmd->request_buffer,
+				sg_count, &offset, &len);
+			if (base == NULL) {
+				*count -= lcount;
+				return;
+			}
+			d = base + offset;
+		} else
+			d = cmd->request_buffer;
+
+		memcpy(d, s, len);
+
+		lcount -= len;
+		if (cmd->use_sg)
+			scsi_kunmap_atomic_sg(base);
+	}
+}
+
+static int stex_direct_copy(struct scsi_cmnd *cmd,
+	const void *src, size_t count)
+{
+	struct st_hba *hba = (struct st_hba *) &cmd->device->host->hostdata[0];
+	size_t cp_len = count;
+	int n_elem = 0;
+
+	if (cmd->use_sg) {
+		n_elem = pci_map_sg(hba->pdev, cmd->request_buffer,
+			cmd->use_sg, cmd->sc_data_direction);
+		if (n_elem <= 0)
+			return 0;
+	}
+
+	stex_internal_copy(cmd, src, &cp_len, n_elem);
+
+	if (cmd->use_sg)
+		pci_unmap_sg(hba->pdev, cmd->request_buffer,
+			cmd->use_sg, cmd->sc_data_direction);
+	return cp_len == count;
+}
+
+static void stex_controller_info(struct st_hba *hba, struct st_ccb *ccb)
+{
+	struct st_frame *p;
+	size_t count = sizeof(struct st_frame);
+
+	p = hba->copy_buffer;
+	memset(p->base, 0, sizeof(u32)*6);
+	*(unsigned long *)(p->base) = pci_resource_start(hba->pdev, 0);
+	p->rom_addr = 0;
+
+	p->drv_ver.major = ST_VER_MAJOR;
+	p->drv_ver.minor = ST_VER_MINOR;
+	p->drv_ver.oem = ST_OEM;
+	p->drv_ver.build = ST_BUILD_VER;
+
+	p->bus = hba->pdev->bus->number;
+	p->slot = hba->pdev->devfn;
+	p->irq_level = 0;
+	p->irq_vec = hba->pdev->irq;
+	p->id = hba->pdev->vendor << 16 | hba->pdev->device;
+	p->subid =
+		hba->pdev->subsystem_vendor << 16 | hba->pdev->subsystem_device;
+
+	stex_internal_copy(ccb->cmd, p, &count, ccb->sg_count);
+}
+
+static void
+stex_send_cmd(struct st_hba *hba, struct req_msg *req, u16 tag)
+{
+	req->tag = cpu_to_le16(tag);
+	req->task_attr = TASK_ATTRIBUTE_SIMPLE;
+	req->task_manage = 0; /* not supported yet */
+	req->payload_sz = (u8)(sizeof(struct req_msg)/sizeof(u32));
+
+	hba->ccb[tag].req = req;
+	hba->out_req_cnt++;
+
+	writel(hba->req_head, hba->mmio_base + IMR0);
+	writel(MU_INBOUND_DOORBELL_REQHEADCHANGED, hba->mmio_base + IDBL);
+	readl(hba->mmio_base + IDBL); /* flush */
+}
+
+static int
+stex_slave_alloc(struct scsi_device *sdev)
+{
+	/* Cheat: usually extracted from Inquiry data */
+	sdev->tagged_supported = 1;
+
+	scsi_activate_tcq(sdev, sdev->host->can_queue);
+
+	return 0;
+}
+
+static int
+stex_slave_config(struct scsi_device *sdev)
+{
+	sdev->use_10_for_rw = 1;
+	sdev->use_10_for_ms = 1;
+	sdev->timeout = 60 * HZ;
+	sdev->tagged_supported = 1;
+
+	return 0;
+}
+
+static void
+stex_slave_destroy(struct scsi_device *sdev)
+{
+	scsi_deactivate_tcq(sdev, 1);
+}
+
+static int
+stex_queuecommand(struct scsi_cmnd *cmd, void (* done)(struct scsi_cmnd *))
+{
+	struct st_hba *hba;
+	struct Scsi_Host *host;
+	unsigned int id,lun;
+	struct req_msg *req;
+	u16 tag;
+	host = cmd->device->host;
+	id = cmd->device->id;
+	lun = cmd->device->channel; /* firmware lun issue work around */
+	hba = (struct st_hba *) &host->hostdata[0];
+
+	switch (cmd->cmnd[0]) {
+	case MODE_SENSE_10:
+	{
+		static char ms10_caching_page[12] =
+			{ 0, 0x12, 0, 0, 0, 0, 0, 0, 0x8, 0xa, 0x4, 0 };
+		unsigned char page;
+		page = cmd->cmnd[2] & 0x3f;
+		if (page == 0x8 || page == 0x3f) {
+			stex_direct_copy(cmd, ms10_caching_page,
+					sizeof(ms10_caching_page));
+			cmd->result = DID_OK << 16 | COMMAND_COMPLETE << 8;
+			done(cmd);
+		} else
+			stex_invalid_field(cmd, done);
+		return 0;
+	}
+	case INQUIRY:
+		if (id != ST_MAX_ARRAY_SUPPORTED)
+			break;
+		if (lun == 0 && (cmd->cmnd[1] & INQUIRY_EVPD) == 0) {
+			stex_direct_copy(cmd, console_inq_page,
+				sizeof(console_inq_page));
+			cmd->result = DID_OK << 16 | COMMAND_COMPLETE << 8;
+			done(cmd);
+		} else
+			stex_invalid_field(cmd, done);
+		return 0;
+	case PASSTHRU_CMD:
+		if (cmd->cmnd[1] == PASSTHRU_GET_DRVVER) {
+			struct st_drvver ver;
+			ver.major = ST_VER_MAJOR;
+			ver.minor = ST_VER_MINOR;
+			ver.oem = ST_OEM;
+			ver.build = ST_BUILD_VER;
+			ver.signature[0] = PASSTHRU_SIGNATURE;
+			ver.console_id = ST_MAX_ARRAY_SUPPORTED;
+			ver.host_no = hba->host->host_no;
+			cmd->result = stex_direct_copy(cmd, &ver, sizeof(ver)) ?
+				DID_OK << 16 | COMMAND_COMPLETE << 8 :
+				DID_ERROR << 16 | COMMAND_COMPLETE << 8;
+			done(cmd);
+			return 0;
+		}
+	default:
+		break;
+	}
+
+	cmd->scsi_done = done;
+
+	tag = cmd->request->tag;
+
+	if (unlikely(tag >= host->can_queue))
+		return SCSI_MLQUEUE_HOST_BUSY;
+
+	req = stex_alloc_req(hba);
+	req->lun = lun;
+	req->target = id;
+
+	/* cdb */
+	memcpy(req->cdb, cmd->cmnd, STEX_CDB_LENGTH);
+
+	hba->ccb[tag].cmd = cmd;
+	hba->ccb[tag].sense_bufflen = SCSI_SENSE_BUFFERSIZE;
+	hba->ccb[tag].sense_buffer = cmd->sense_buffer;
+	hba->ccb[tag].req_type = 0;
+
+	if (cmd->sc_data_direction != DMA_NONE)
+		stex_map_sg(hba, req, &hba->ccb[tag]);
+
+	stex_send_cmd(hba, req, tag);
+	return 0;
+}
+
+static void stex_unmap_sg(struct st_hba *hba, struct scsi_cmnd *cmd)
+{
+	if (cmd->sc_data_direction != DMA_NONE) {
+		if (cmd->use_sg)
+			pci_unmap_sg(hba->pdev, cmd->request_buffer,
+				cmd->use_sg, cmd->sc_data_direction);
+		else
+			pci_unmap_single(hba->pdev, cmd->SCp.dma_handle,
+				cmd->request_bufflen, cmd->sc_data_direction);
+	}
+}
+
+static void stex_scsi_done(struct st_ccb *ccb)
+{
+	struct scsi_cmnd *cmd = ccb->cmd;
+	int result;
+
+	if (ccb->srb_status == SRB_STATUS_SUCCESS ||  ccb->srb_status == 0) {
+		result = ccb->scsi_status;
+		switch (ccb->scsi_status) {
+		case SAM_STAT_GOOD:
+			result |= DID_OK << 16 | COMMAND_COMPLETE << 8;
+			break;
+		case SAM_STAT_CHECK_CONDITION:
+			result |= DRIVER_SENSE << 24;
+			break;
+		case SAM_STAT_BUSY:
+			result |= DID_BUS_BUSY << 16 | COMMAND_COMPLETE << 8;
+			break;
+		default:
+			result |= DID_ERROR << 16 | COMMAND_COMPLETE << 8;
+			break;
+		}
+	}
+	else if (ccb->srb_status & SRB_SEE_SENSE)
+		result = DRIVER_SENSE << 24 | SAM_STAT_CHECK_CONDITION;
+	else switch (ccb->srb_status) {
+		case SRB_STATUS_SELECTION_TIMEOUT:
+			result = DID_NO_CONNECT << 16 | COMMAND_COMPLETE << 8;
+			break;
+		case SRB_STATUS_BUSY:
+			result = DID_BUS_BUSY << 16 | COMMAND_COMPLETE << 8;
+			break;
+		case SRB_STATUS_INVALID_REQUEST:
+		case SRB_STATUS_ERROR:
+		default:
+			result = DID_ERROR << 16 | COMMAND_COMPLETE << 8;
+			break;
+	}
+
+	cmd->result = result;
+	cmd->scsi_done(cmd);
+}
+
+static void stex_copy_data(struct st_ccb *ccb,
+	struct status_msg *resp, unsigned int variable)
+{
+	size_t count = variable;
+	if (resp->scsi_status != SAM_STAT_GOOD) {
+		if (ccb->sense_buffer != NULL)
+			memcpy(ccb->sense_buffer, resp->variable,
+				min(variable, ccb->sense_bufflen));
+		return;
+	}
+
+	if (ccb->cmd == NULL)
+		return;
+	stex_internal_copy(ccb->cmd, resp->variable, &count, ccb->sg_count);
+}
+
+static void stex_mu_intr(struct st_hba *hba, u32 doorbell)
+{
+	void __iomem *base = hba->mmio_base;
+	struct status_msg *resp;
+	struct st_ccb *ccb;
+	unsigned int size;
+	u16 tag;
+
+	if (!(doorbell & MU_OUTBOUND_DOORBELL_STATUSHEADCHANGED))
+		return;
+
+	/* status payloads */
+	hba->status_head = readl(base + OMR1);
+	if (unlikely(hba->status_head >= MU_STATUS_COUNT)) {
+		printk(KERN_WARNING DRV_NAME "(%s): invalid status head\n",
+			pci_name(hba->pdev));
+		return;
+	}
+
+	if (unlikely(hba->mu_status != MU_STATE_STARTED ||
+		hba->out_req_cnt <= 0)) {
+		hba->status_tail = hba->status_head;
+		goto update_status;
+	}
+
+	while (hba->status_tail != hba->status_head) {
+		resp = stex_get_status(hba);
+		tag = le16_to_cpu(resp->tag);
+		if (unlikely(tag >= hba->host->can_queue)) {
+			printk(KERN_WARNING DRV_NAME
+				"(%s): invalid tag\n", pci_name(hba->pdev));
+			continue;
+		}
+
+		ccb = &hba->ccb[tag];
+		if (hba->wait_ccb == ccb)
+			hba->wait_ccb = NULL;
+		if (unlikely(ccb->req == NULL)) {
+			printk(KERN_WARNING DRV_NAME
+				"(%s): lagging req\n", pci_name(hba->pdev));
+			continue;
+		}
+
+		size = resp->payload_sz * sizeof(u32); /* payload size */
+		if (unlikely(size < sizeof(*resp) - STATUS_VAR_LEN ||
+			size > sizeof(*resp))) {
+			printk(KERN_WARNING DRV_NAME "(%s): bad status size\n",
+				pci_name(hba->pdev));
+		} else {
+			size -= sizeof(*resp) - STATUS_VAR_LEN; /* copy size */
+			if (size)
+				stex_copy_data(ccb, resp, size);
+		}
+
+		ccb->srb_status = resp->srb_status;
+		ccb->scsi_status = resp->scsi_status;
+
+		if (likely(ccb->cmd != NULL)) {
+			if (unlikely(ccb->cmd->cmnd[0] == PASSTHRU_CMD &&
+				ccb->cmd->cmnd[1] == PASSTHRU_GET_ADAPTER))
+				stex_controller_info(hba, ccb);
+			stex_unmap_sg(hba, ccb->cmd);
+			stex_scsi_done(ccb);
+			hba->out_req_cnt--;
+		} else if (ccb->req_type & PASSTHRU_REQ_TYPE) {
+			hba->out_req_cnt--;
+			if (ccb->req_type & PASSTHRU_REQ_NO_WAKEUP) {
+				ccb->req_type = 0;
+				continue;
+			}
+			ccb->req_type = 0;
+			if (waitqueue_active(&hba->waitq))
+				wake_up(&hba->waitq);
+		}
+	}
+
+update_status:
+	writel(hba->status_head, base + IMR1);
+	readl(base + IMR1); /* flush */
+}
+
+static irqreturn_t stex_intr(int irq, void *__hba, struct pt_regs *regs)
+{
+	struct st_hba *hba = __hba;
+	void __iomem *base = hba->mmio_base;
+	u32 data;
+	unsigned long flags;
+	int handled = 0;
+
+	spin_lock_irqsave(hba->host->host_lock, flags);
+
+	data = readl(base + ODBL);
+
+	if (data && data != 0xffffffff) {
+		/* clear the interrupt */
+		writel(data, base + ODBL);
+		readl(base + ODBL); /* flush */
+		stex_mu_intr(hba, data);
+		handled = 1;
+	}
+
+	spin_unlock_irqrestore(hba->host->host_lock, flags);
+
+	return IRQ_RETVAL(handled);
+}
+
+static int stex_handshake(struct st_hba *hba)
+{
+	void __iomem *base = hba->mmio_base;
+	struct handshake_frame *h;
+	dma_addr_t status_phys;
+	int i;
+
+	if (readl(base + OMR0) != MU_HANDSHAKE_SIGNATURE) {
+		writel(MU_INBOUND_DOORBELL_HANDSHAKE, base + IDBL);
+		readl(base + IDBL);
+		for (i = 0; readl(base + OMR0) != MU_HANDSHAKE_SIGNATURE
+			&& i < MU_MAX_DELAY_TIME; i++) {
+			rmb();
+			msleep(1);
+		}
+
+		if (i == MU_MAX_DELAY_TIME) {
+			printk(KERN_ERR DRV_NAME
+				"(%s): no handshake signature\n",
+				pci_name(hba->pdev));
+			return -1;
+		}
+	}
+
+	udelay(10);
+
+	h = (struct handshake_frame *)(hba->dma_mem + MU_REQ_BUFFER_SIZE);
+	h->rb_phy = cpu_to_le32(hba->dma_handle);
+	h->rb_phy_hi = cpu_to_le32((hba->dma_handle >> 16) >> 16);
+	h->req_sz = cpu_to_le16(sizeof(struct req_msg));
+	h->req_cnt = cpu_to_le16(MU_REQ_COUNT);
+	h->status_sz = cpu_to_le16(sizeof(struct status_msg));
+	h->status_cnt = cpu_to_le16(MU_STATUS_COUNT);
+	stex_gettime(&h->hosttime);
+	h->partner_type = HMU_PARTNER_TYPE;
+
+	status_phys = hba->dma_handle + MU_REQ_BUFFER_SIZE;
+	writel(status_phys, base + IMR0);
+	readl(base + IMR0);
+	writel((status_phys >> 16) >> 16, base + IMR1);
+	readl(base + IMR1);
+
+	writel((status_phys >> 16) >> 16, base + OMR0); /* old fw compatible */
+	readl(base + OMR0);
+	writel(MU_INBOUND_DOORBELL_HANDSHAKE, base + IDBL);
+	readl(base + IDBL); /* flush */
+
+	udelay(10);
+	for (i = 0; readl(base + OMR0) != MU_HANDSHAKE_SIGNATURE
+		&& i < MU_MAX_DELAY_TIME; i++) {
+		rmb();
+		msleep(1);
+	}
+
+	if (i == MU_MAX_DELAY_TIME) {
+		printk(KERN_ERR DRV_NAME
+			"(%s): no signature after handshake frame\n",
+			pci_name(hba->pdev));
+		return -1;
+	}
+
+	writel(0, base + IMR0);
+	readl(base + IMR0);
+	writel(0, base + OMR0);
+	readl(base + OMR0);
+	writel(0, base + IMR1);
+	readl(base + IMR1);
+	writel(0, base + OMR1);
+	readl(base + OMR1); /* flush */
+	hba->mu_status = MU_STATE_STARTED;
+	return 0;
+}
+
+static int stex_abort(struct scsi_cmnd *cmd)
+{
+	struct Scsi_Host *host = cmd->device->host;
+	struct st_hba *hba = (struct st_hba *)host->hostdata;
+	u16 tag = cmd->request->tag;
+	void __iomem *base;
+	u32 data;
+	int result = SUCCESS;
+	unsigned long flags;
+	base = hba->mmio_base;
+	spin_lock_irqsave(host->host_lock, flags);
+	if (tag < host->can_queue && hba->ccb[tag].cmd == cmd)
+		hba->wait_ccb = &hba->ccb[tag];
+	else {
+		for (tag = 0; tag < host->can_queue; tag++)
+			if (hba->ccb[tag].cmd == cmd) {
+				hba->wait_ccb = &hba->ccb[tag];
+				break;
+			}
+		if (tag >= host->can_queue)
+			goto out;
+	}
+
+	data = readl(base + ODBL);
+	if (data == 0 || data == 0xffffffff)
+		goto fail_out;
+
+	writel(data, base + ODBL);
+	readl(base + ODBL); /* flush */
+
+	stex_mu_intr(hba, data);
+
+	if (hba->wait_ccb == NULL) {
+		printk(KERN_WARNING DRV_NAME
+			"(%s): lost interrupt\n", pci_name(hba->pdev));
+		goto out;
+	}
+
+fail_out:
+	stex_unmap_sg(hba, cmd);
+	hba->wait_ccb->req = NULL; /* nullify the req's future return */
+	hba->wait_ccb = NULL;
+	result = FAILED;
+out:
+	spin_unlock_irqrestore(host->host_lock, flags);
+	return result;
+}
+
+static void stex_hard_reset(struct st_hba *hba)
+{
+	struct pci_bus *bus;
+	int i;
+	u16 pci_cmd;
+	u8 pci_bctl;
+
+	for (i = 0; i < 16; i++)
+		pci_read_config_dword(hba->pdev, i * 4,
+			&hba->pdev->saved_config_space[i]);
+
+	/* Reset secondary bus. Our controller(MU/ATU) is the only device on
+	   secondary bus. Consult Intel 80331/3 developer's manual for detail */
+	bus = hba->pdev->bus;
+	pci_read_config_byte(bus->self, PCI_BRIDGE_CONTROL, &pci_bctl);
+	pci_bctl |= PCI_BRIDGE_CTL_BUS_RESET;
+	pci_write_config_byte(bus->self, PCI_BRIDGE_CONTROL, pci_bctl);
+	msleep(1);
+	pci_bctl &= ~PCI_BRIDGE_CTL_BUS_RESET;
+	pci_write_config_byte(bus->self, PCI_BRIDGE_CONTROL, pci_bctl);
+
+	for (i = 0; i < MU_MAX_DELAY_TIME; i++) {
+		pci_read_config_word(hba->pdev, PCI_COMMAND, &pci_cmd);
+		if (pci_cmd & PCI_COMMAND_MASTER)
+			break;
+		msleep(1);
+	}
+
+	ssleep(5);
+	for (i = 0; i < 16; i++)
+		pci_write_config_dword(hba->pdev, i * 4,
+			hba->pdev->saved_config_space[i]);
+}
+
+static int stex_reset(struct scsi_cmnd *cmd)
+{
+	struct st_hba *hba;
+	unsigned long flags;
+	hba = (struct st_hba *) &cmd->device->host->hostdata[0];
+
+	hba->mu_status = MU_STATE_RESETTING;
+
+	if (hba->cardtype == st_shasta)
+		stex_hard_reset(hba);
+
+	if (stex_handshake(hba)) {
+		printk(KERN_WARNING DRV_NAME
+			"(%s): resetting: handshake failed\n",
+			pci_name(hba->pdev));
+		return FAILED;
+	}
+	spin_lock_irqsave(hba->host->host_lock, flags);
+	hba->req_head = 0;
+	hba->req_tail = 0;
+	hba->status_head = 0;
+	hba->status_tail = 0;
+	hba->out_req_cnt = 0;
+	spin_unlock_irqrestore(hba->host->host_lock, flags);
+
+	return SUCCESS;
+}
+
+static int stex_biosparam(struct scsi_device *sdev,
+	struct block_device *bdev, sector_t capacity, int geom[])
+{
+	int heads = 255, sectors = 63, cylinders;
+
+	if (capacity < 0x200000) {
+		heads = 64;
+		sectors = 32;
+	}
+
+	cylinders = sector_div(capacity, heads * sectors);
+
+	geom[0] = heads;
+	geom[1] = sectors;
+	geom[2] = cylinders;
+
+	return 0;
+}
+
+static struct scsi_host_template driver_template = {
+	.module				= THIS_MODULE,
+	.name				= DRV_NAME,
+	.proc_name			= DRV_NAME,
+	.bios_param			= stex_biosparam,
+	.queuecommand			= stex_queuecommand,
+	.slave_alloc			= stex_slave_alloc,
+	.slave_configure		= stex_slave_config,
+	.slave_destroy			= stex_slave_destroy,
+	.eh_abort_handler		= stex_abort,
+	.eh_host_reset_handler		= stex_reset,
+	.can_queue			= ST_CAN_QUEUE,
+	.this_id			= -1,
+	.sg_tablesize			= ST_MAX_SG,
+	.cmd_per_lun			= ST_CMD_PER_LUN,
+};
+
+static int stex_set_dma_mask(struct pci_dev * pdev)
+{
+	int ret;
+	if (!pci_set_dma_mask(pdev, DMA_64BIT_MASK)
+		&& !pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK))
+		return 0;
+	ret = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
+	if (!ret)
+		ret = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
+	return ret;
+}
+
+static int __devinit
+stex_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct st_hba *hba;
+	struct Scsi_Host *host;
+	int err;
+
+	err = pci_enable_device(pdev);
+	if (err)
+		return err;
+
+	pci_set_master(pdev);
+
+	host = scsi_host_alloc(&driver_template, sizeof(struct st_hba));
+
+	if (!host) {
+		printk(KERN_ERR DRV_NAME "(%s): scsi_host_alloc failed\n",
+			pci_name(pdev));
+		err = -ENOMEM;
+		goto out_disable;
+	}
+
+	hba = (struct st_hba *)host->hostdata;
+	memset(hba, 0, sizeof(struct st_hba));
+
+	err = pci_request_regions(pdev, DRV_NAME);
+	if (err < 0) {
+		printk(KERN_ERR DRV_NAME "(%s): request regions failed\n",
+			pci_name(pdev));
+		goto out_scsi_host_put;
+	}
+
+	hba->mmio_base = ioremap(pci_resource_start(pdev, 0),
+		pci_resource_len(pdev, 0));
+	if ( !hba->mmio_base) {
+		printk(KERN_ERR DRV_NAME "(%s): memory map failed\n",
+			pci_name(pdev));
+		err = -ENOMEM;
+		goto out_release_regions;
+	}
+
+	err = stex_set_dma_mask(pdev);
+	if (err) {
+		printk(KERN_ERR DRV_NAME "(%s): set dma mask failed\n",
+			pci_name(pdev));
+		goto out_iounmap;
+	}
+
+	hba->dma_mem = dma_alloc_coherent(&pdev->dev,
+		STEX_BUFFER_SIZE, &hba->dma_handle, GFP_KERNEL);
+	if (!hba->dma_mem) {
+		err = -ENOMEM;
+		printk(KERN_ERR DRV_NAME "(%s): dma mem alloc failed\n",
+			pci_name(pdev));
+		goto out_iounmap;
+	}
+
+	hba->status_buffer =
+		(struct status_msg *)(hba->dma_mem + MU_REQ_BUFFER_SIZE);
+	hba->copy_buffer = hba->dma_mem + MU_BUFFER_SIZE;
+	hba->mu_status = MU_STATE_STARTING;
+
+	hba->cardtype = (unsigned int) id->driver_data;
+
+	/* firmware uses id/lun pair for a logical drive, but lun would be
+	   always 0 if CONFIG_SCSI_MULTI_LUN not configured, so we use
+	   channel to map lun here */
+	host->max_channel = ST_MAX_LUN_PER_TARGET - 1;
+	host->max_id = ST_MAX_TARGET_NUM;
+	host->max_lun = 1;
+	host->unique_id = host->host_no;
+	host->max_cmd_len = STEX_CDB_LENGTH;
+
+	hba->host = host;
+	hba->pdev = pdev;
+	init_waitqueue_head(&hba->waitq);
+
+	err = request_irq(pdev->irq, stex_intr, IRQF_SHARED, DRV_NAME, hba);
+	if (err) {
+		printk(KERN_ERR DRV_NAME "(%s): request irq failed\n",
+			pci_name(pdev));
+		goto out_pci_free;
+	}
+
+	err = stex_handshake(hba);
+	if (err)
+		goto out_free_irq;
+
+	err = scsi_init_shared_tag_map(host, ST_CAN_QUEUE);
+	if (err) {
+		printk(KERN_ERR DRV_NAME "(%s): init shared queue failed\n",
+			pci_name(pdev));
+		goto out_free_irq;
+	}
+
+	pci_set_drvdata(pdev, hba);
+
+	err = scsi_add_host(host, &pdev->dev);
+	if (err) {
+		printk(KERN_ERR DRV_NAME "(%s): scsi_add_host failed\n",
+			pci_name(pdev));
+		goto out_free_irq;
+	}
+
+	scsi_scan_host(host);
+
+	return 0;
+
+out_free_irq:
+	free_irq(pdev->irq, hba);
+out_pci_free:
+	dma_free_coherent(&pdev->dev, STEX_BUFFER_SIZE,
+			  hba->dma_mem, hba->dma_handle);
+out_iounmap:
+	iounmap(hba->mmio_base);
+out_release_regions:
+	pci_release_regions(pdev);
+out_scsi_host_put:
+	scsi_host_put(host);
+out_disable:
+	pci_disable_device(pdev);
+
+	return err;
+}
+
+static void stex_hba_stop(struct st_hba *hba)
+{
+	struct req_msg *req;
+	unsigned long flags;
+	unsigned long before;
+	u16 tag = 0;
+
+	spin_lock_irqsave(hba->host->host_lock, flags);
+	req = stex_alloc_req(hba);
+	memset(req->cdb, 0, STEX_CDB_LENGTH);
+
+	req->cdb[0] = CONTROLLER_CMD;
+	req->cdb[1] = CTLR_POWER_STATE_CHANGE;
+	req->cdb[2] = CTLR_POWER_SAVING;
+
+	hba->ccb[tag].cmd = NULL;
+	hba->ccb[tag].sg_count = 0;
+	hba->ccb[tag].sense_bufflen = 0;
+	hba->ccb[tag].sense_buffer = NULL;
+	hba->ccb[tag].req_type |= PASSTHRU_REQ_TYPE;
+
+	stex_send_cmd(hba, req, tag);
+	spin_unlock_irqrestore(hba->host->host_lock, flags);
+
+	before = jiffies;
+	while (hba->ccb[tag].req_type & PASSTHRU_REQ_TYPE) {
+		if (time_after(jiffies, before + ST_INTERNAL_TIMEOUT * HZ))
+			return;
+		msleep(10);
+	}
+}
+
+static void stex_hba_free(struct st_hba *hba)
+{
+	free_irq(hba->pdev->irq, hba);
+
+	iounmap(hba->mmio_base);
+
+	pci_release_regions(hba->pdev);
+
+	dma_free_coherent(&hba->pdev->dev, STEX_BUFFER_SIZE,
+			  hba->dma_mem, hba->dma_handle);
+}
+
+static void stex_remove(struct pci_dev *pdev)
+{
+	struct st_hba *hba = pci_get_drvdata(pdev);
+
+	scsi_remove_host(hba->host);
+
+	pci_set_drvdata(pdev, NULL);
+
+	stex_hba_stop(hba);
+
+	stex_hba_free(hba);
+
+	scsi_host_put(hba->host);
+
+	pci_disable_device(pdev);
+}
+
+static void stex_shutdown(struct pci_dev *pdev)
+{
+	struct st_hba *hba = pci_get_drvdata(pdev);
+
+	stex_hba_stop(hba);
+}
+
+static struct pci_device_id stex_pci_tbl[] = {
+	{ 0x105a, 0x8350, PCI_ANY_ID, PCI_ANY_ID, 0, 0, st_shasta },
+	{ 0x105a, 0xc350, PCI_ANY_ID, PCI_ANY_ID, 0, 0, st_shasta },
+	{ 0x105a, 0xf350, PCI_ANY_ID, PCI_ANY_ID, 0, 0, st_shasta },
+	{ 0x105a, 0x4301, PCI_ANY_ID, PCI_ANY_ID, 0, 0, st_shasta },
+	{ 0x105a, 0x4302, PCI_ANY_ID, PCI_ANY_ID, 0, 0, st_shasta },
+	{ 0x105a, 0x8301, PCI_ANY_ID, PCI_ANY_ID, 0, 0, st_shasta },
+	{ 0x105a, 0x8302, PCI_ANY_ID, PCI_ANY_ID, 0, 0, st_shasta },
+	{ 0x1725, 0x7250, PCI_ANY_ID, PCI_ANY_ID, 0, 0, st_vsc },
+	{ }	/* terminate list */
+};
+MODULE_DEVICE_TABLE(pci, stex_pci_tbl);
+
+static struct pci_driver stex_pci_driver = {
+	.name		= DRV_NAME,
+	.id_table	= stex_pci_tbl,
+	.probe		= stex_probe,
+	.remove		= __devexit_p(stex_remove),
+	.shutdown	= stex_shutdown,
+};
+
+static int __init stex_init(void)
+{
+	printk(KERN_INFO DRV_NAME
+		": Promise SuperTrak EX Driver version: %s\n",
+		 ST_DRIVER_VERSION);
+
+	return pci_register_driver(&stex_pci_driver);
+}
+
+static void __exit stex_exit(void)
+{
+	pci_unregister_driver(&stex_pci_driver);
+}
+
+module_init(stex_init);
+module_exit(stex_exit);
diff --git a/drivers/scsi/ultrastor.c b/drivers/scsi/ultrastor.c
index e681681..0372aa9 100644
--- a/drivers/scsi/ultrastor.c
+++ b/drivers/scsi/ultrastor.c
@@ -196,8 +196,8 @@ struct mscp {
   u32 sense_data PACKED;
   /* The following fields are for software only.  They are included in
      the MSCP structure because they are associated with SCSI requests.  */
-  void (*done)(Scsi_Cmnd *);
-  Scsi_Cmnd *SCint;
+  void (*done) (struct scsi_cmnd *);
+  struct scsi_cmnd *SCint;
   ultrastor_sg_list sglist[ULTRASTOR_24F_MAX_SG]; /* use larger size for 24F */
 };
 
@@ -289,7 +289,7 @@ static const unsigned short ultrastor_ports_14f[] = {
 
 static void ultrastor_interrupt(int, void *, struct pt_regs *);
 static irqreturn_t do_ultrastor_interrupt(int, void *, struct pt_regs *);
-static inline void build_sg_list(struct mscp *, Scsi_Cmnd *SCpnt);
+static inline void build_sg_list(struct mscp *, struct scsi_cmnd *SCpnt);
 
 
 /* Always called with host lock held */
@@ -673,7 +673,7 @@ static const char *ultrastor_info(struct Scsi_Host * shpnt)
     return buf;
 }
 
-static inline void build_sg_list(struct mscp *mscp, Scsi_Cmnd *SCpnt)
+static inline void build_sg_list(struct mscp *mscp, struct scsi_cmnd *SCpnt)
 {
 	struct scatterlist *sl;
 	long transfer_length = 0;
@@ -694,7 +694,8 @@ static inline void build_sg_list(struct mscp *mscp, Scsi_Cmnd *SCpnt)
 	mscp->transfer_data_length = transfer_length;
 }
 
-static int ultrastor_queuecommand(Scsi_Cmnd *SCpnt, void (*done)(Scsi_Cmnd *))
+static int ultrastor_queuecommand(struct scsi_cmnd *SCpnt,
+				void (*done) (struct scsi_cmnd *))
 {
     struct mscp *my_mscp;
 #if ULTRASTOR_MAX_CMDS > 1
@@ -833,7 +834,7 @@ retry:
 
  */
 
-static int ultrastor_abort(Scsi_Cmnd *SCpnt)
+static int ultrastor_abort(struct scsi_cmnd *SCpnt)
 {
 #if ULTRASTOR_DEBUG & UD_ABORT
     char out[108];
@@ -843,7 +844,7 @@ static int ultrastor_abort(Scsi_Cmnd *SCpnt)
     unsigned int mscp_index;
     unsigned char old_aborted;
     unsigned long flags;
-    void (*done)(Scsi_Cmnd *);
+    void (*done)(struct scsi_cmnd *);
     struct Scsi_Host *host = SCpnt->device->host;
 
     if(config.slot) 
@@ -960,7 +961,7 @@ static int ultrastor_abort(Scsi_Cmnd *SCpnt)
     return SUCCESS;
 }
 
-static int ultrastor_host_reset(Scsi_Cmnd * SCpnt)
+static int ultrastor_host_reset(struct scsi_cmnd * SCpnt)
 {
     unsigned long flags;
     int i;
@@ -1045,8 +1046,8 @@ static void ultrastor_interrupt(int irq, void *dev_id, struct pt_regs *regs)
     unsigned int mscp_index;
 #endif
     struct mscp *mscp;
-    void (*done)(Scsi_Cmnd *);
-    Scsi_Cmnd *SCtmp;
+    void (*done) (struct scsi_cmnd *);
+    struct scsi_cmnd *SCtmp;
 
 #if ULTRASTOR_MAX_CMDS == 1
     mscp = &config.mscp[0];
@@ -1079,7 +1080,7 @@ static void ultrastor_interrupt(int irq, void *dev_id, struct pt_regs *regs)
 	    return;
 	}
 	if (icm_status == 3) {
-	    void (*done)(Scsi_Cmnd *) = mscp->done;
+	    void (*done)(struct scsi_cmnd *) = mscp->done;
 	    if (done) {
 		mscp->done = NULL;
 		mscp->SCint->result = DID_ABORT << 16;
diff --git a/drivers/scsi/ultrastor.h b/drivers/scsi/ultrastor.h
index da759a1..a692905 100644
--- a/drivers/scsi/ultrastor.h
+++ b/drivers/scsi/ultrastor.h
@@ -14,11 +14,13 @@
 #define _ULTRASTOR_H
 
 static int ultrastor_detect(struct scsi_host_template *);
-static const char *ultrastor_info(struct Scsi_Host * shpnt);
-static int ultrastor_queuecommand(Scsi_Cmnd *, void (*done)(Scsi_Cmnd *));
-static int ultrastor_abort(Scsi_Cmnd *);
-static int ultrastor_host_reset(Scsi_Cmnd *);
-static int ultrastor_biosparam(struct scsi_device *, struct block_device *, sector_t, int *);
+static const char *ultrastor_info(struct Scsi_Host *shpnt);
+static int ultrastor_queuecommand(struct scsi_cmnd *,
+				void (*done)(struct scsi_cmnd *));
+static int ultrastor_abort(struct scsi_cmnd *);
+static int ultrastor_host_reset(struct scsi_cmnd *);
+static int ultrastor_biosparam(struct scsi_device *, struct block_device *,
+				sector_t, int *);
 
 
 #define ULTRASTOR_14F_MAX_SG 16
diff --git a/drivers/serial/pmac_zilog.c b/drivers/serial/pmac_zilog.c
index bfd2a22..a3b99ca 100644
--- a/drivers/serial/pmac_zilog.c
+++ b/drivers/serial/pmac_zilog.c
@@ -1400,8 +1400,8 @@ static struct uart_ops pmz_pops = {
 static int __init pmz_init_port(struct uart_pmac_port *uap)
 {
 	struct device_node *np = uap->node;
-	char *conn;
-	struct slot_names_prop {
+	const char *conn;
+	const struct slot_names_prop {
 		int	count;
 		char	name[1];
 	} *slots;
@@ -1458,7 +1458,7 @@ no_dma:
 		uap->flags |= PMACZILOG_FLAG_IS_IRDA;
 	uap->port_type = PMAC_SCC_ASYNC;
 	/* 1999 Powerbook G3 has slot-names property instead */
-	slots = (struct slot_names_prop *)get_property(np, "slot-names", &len);
+	slots = get_property(np, "slot-names", &len);
 	if (slots && slots->count > 0) {
 		if (strcmp(slots->name, "IrDA") == 0)
 			uap->flags |= PMACZILOG_FLAG_IS_IRDA;
@@ -1470,7 +1470,8 @@ no_dma:
 	if (ZS_IS_INTMODEM(uap)) {
 		struct device_node* i2c_modem = find_devices("i2c-modem");
 		if (i2c_modem) {
-			char* mid = get_property(i2c_modem, "modem-id", NULL);
+			const char* mid =
+				get_property(i2c_modem, "modem-id", NULL);
 			if (mid) switch(*mid) {
 			case 0x04 :
 			case 0x05 :
diff --git a/drivers/usb/input/hid-core.c b/drivers/usb/input/hid-core.c
index a2c56b2..3305fb6 100644
--- a/drivers/usb/input/hid-core.c
+++ b/drivers/usb/input/hid-core.c
@@ -1818,7 +1818,7 @@ static struct hid_device *usb_hid_configure(struct usb_interface *intf)
 	int n, len, insize = 0;
 
         /* Ignore all Wacom devices */
-        if (dev->descriptor.idVendor == USB_VENDOR_ID_WACOM)
+        if (le16_to_cpu(dev->descriptor.idVendor) == USB_VENDOR_ID_WACOM)
                 return NULL;
 
 	for (n = 0; hid_blacklist[n].idVendor; n++)
diff --git a/drivers/video/S3triofb.c b/drivers/video/S3triofb.c
index afd146f..397005e 100644
--- a/drivers/video/S3triofb.c
+++ b/drivers/video/S3triofb.c
@@ -349,30 +349,30 @@ static void __init s3triofb_of_init(struct device_node *dp)
     s3trio_name[sizeof(s3trio_name)-1] = '\0';
     strcpy(fb_fix.id, s3trio_name);
 
-    if((pp = (int *)get_property(dp, "vendor-id", &len)) != NULL
+    if((pp = get_property(dp, "vendor-id", &len)) != NULL
 	&& *pp!=PCI_VENDOR_ID_S3) {
 	printk("%s: can't find S3 Trio board\n", dp->full_name);
 	return;
     }
 
-    if((pp = (int *)get_property(dp, "device-id", &len)) != NULL
+    if((pp = get_property(dp, "device-id", &len)) != NULL
 	&& *pp!=PCI_DEVICE_ID_S3_TRIO) {
 	printk("%s: can't find S3 Trio board\n", dp->full_name);
 	return;
     }
 
-    if ((pp = (int *)get_property(dp, "depth", &len)) != NULL
+    if ((pp = get_property(dp, "depth", &len)) != NULL
 	&& len == sizeof(int) && *pp != 8) {
 	printk("%s: can't use depth = %d\n", dp->full_name, *pp);
 	return;
     }
-    if ((pp = (int *)get_property(dp, "width", &len)) != NULL
+    if ((pp = get_property(dp, "width", &len)) != NULL
 	&& len == sizeof(int))
 	fb_var.xres = fb_var.xres_virtual = *pp;
-    if ((pp = (int *)get_property(dp, "height", &len)) != NULL
+    if ((pp = get_property(dp, "height", &len)) != NULL
 	&& len == sizeof(int))
 	fb_var.yres = fb_var.yres_virtual = *pp;
-    if ((pp = (int *)get_property(dp, "linebytes", &len)) != NULL
+    if ((pp = get_property(dp, "linebytes", &len)) != NULL
 	&& len == sizeof(int))
 	fb_fix.line_length = *pp;
     else
diff --git a/drivers/video/aty/radeon_base.c b/drivers/video/aty/radeon_base.c
index 8e3400d..0ed577e 100644
--- a/drivers/video/aty/radeon_base.c
+++ b/drivers/video/aty/radeon_base.c
@@ -413,11 +413,11 @@ static int  __devinit radeon_find_mem_vbios(struct radeonfb_info *rinfo)
 static int __devinit radeon_read_xtal_OF (struct radeonfb_info *rinfo)
 {
 	struct device_node *dp = rinfo->of_node;
-	u32 *val;
+	const u32 *val;
 
 	if (dp == NULL)
 		return -ENODEV;
-	val = (u32 *) get_property(dp, "ATY,RefCLK", NULL);
+	val = get_property(dp, "ATY,RefCLK", NULL);
 	if (!val || !*val) {
 		printk(KERN_WARNING "radeonfb: No ATY,RefCLK property !\n");
 		return -EINVAL;
@@ -425,11 +425,11 @@ static int __devinit radeon_read_xtal_OF (struct radeonfb_info *rinfo)
 
 	rinfo->pll.ref_clk = (*val) / 10;
 
-	val = (u32 *) get_property(dp, "ATY,SCLK", NULL);
+	val = get_property(dp, "ATY,SCLK", NULL);
 	if (val && *val)
 		rinfo->pll.sclk = (*val) / 10;
 
-	val = (u32 *) get_property(dp, "ATY,MCLK", NULL);
+	val = get_property(dp, "ATY,MCLK", NULL);
 	if (val && *val)
 		rinfo->pll.mclk = (*val) / 10;
 
diff --git a/drivers/video/aty/radeon_monitor.c b/drivers/video/aty/radeon_monitor.c
index 98c05bc..ea531a6 100644
--- a/drivers/video/aty/radeon_monitor.c
+++ b/drivers/video/aty/radeon_monitor.c
@@ -64,13 +64,13 @@ static int __devinit radeon_parse_montype_prop(struct device_node *dp, u8 **out_
 {
         static char *propnames[] = { "DFP,EDID", "LCD,EDID", "EDID",
 				     "EDID1", "EDID2",  NULL };
-	u8 *pedid = NULL;
-	u8 *pmt = NULL;
+	const u8 *pedid = NULL;
+	const u8 *pmt = NULL;
 	u8 *tmp;
         int i, mt = MT_NONE;  
 	
 	RTRACE("analyzing OF properties...\n");
-	pmt = (u8 *)get_property(dp, "display-type", NULL);
+	pmt = get_property(dp, "display-type", NULL);
 	if (!pmt)
 		return MT_NONE;
 	RTRACE("display-type: %s\n", pmt);
@@ -89,7 +89,7 @@ static int __devinit radeon_parse_montype_prop(struct device_node *dp, u8 **out_
 	}
 
 	for (i = 0; propnames[i] != NULL; ++i) {
-		pedid = (u8 *)get_property(dp, propnames[i], NULL);
+		pedid = get_property(dp, propnames[i], NULL);
 		if (pedid != NULL)
 			break;
 	}
@@ -124,14 +124,14 @@ static int __devinit radeon_probe_OF_head(struct radeonfb_info *rinfo, int head_
 		return MT_NONE;
 
 	if (rinfo->has_CRTC2) {
-		char *pname;
+		const char *pname;
 		int len, second = 0;
 
 		dp = dp->child;
 		do {
 			if (!dp)
 				return MT_NONE;
-			pname = (char *)get_property(dp, "name", NULL);
+			pname = get_property(dp, "name", NULL);
 			if (!pname)
 				return MT_NONE;
 			len = strlen(pname);
diff --git a/drivers/video/aty/radeon_pm.c b/drivers/video/aty/radeon_pm.c
index f31e606..e308ed2 100644
--- a/drivers/video/aty/radeon_pm.c
+++ b/drivers/video/aty/radeon_pm.c
@@ -1268,7 +1268,7 @@ static void radeon_pm_full_reset_sdram(struct radeonfb_info *rinfo)
 			  0x21320032, 0xa1320032, 0x21320032, 0xffffffff,
 			  0x31320032 };
 
-		u32 *mrtable = default_mrtable;
+		const u32 *mrtable = default_mrtable;
 		int i, mrtable_size = ARRAY_SIZE(default_mrtable);
 
 		mdelay(30);
@@ -1287,7 +1287,7 @@ static void radeon_pm_full_reset_sdram(struct radeonfb_info *rinfo)
 		if (rinfo->of_node != NULL) {
 			int size;
 
-			mrtable = (u32 *)get_property(rinfo->of_node, "ATY,MRT", &size);
+			mrtable = get_property(rinfo->of_node, "ATY,MRT", &size);
 			if (mrtable)
 				mrtable_size = size >> 2;
 			else
diff --git a/drivers/video/console/fbcon.c b/drivers/video/console/fbcon.c
index 390439b..1b4f75d 100644
--- a/drivers/video/console/fbcon.c
+++ b/drivers/video/console/fbcon.c
@@ -3197,11 +3197,11 @@ static void fbcon_exit(void)
 		return;
 
 #ifdef CONFIG_ATARI
-	free_irq(IRQ_AUTO_4, fbcon_vbl_handler);
+	free_irq(IRQ_AUTO_4, fb_vbl_handler);
 #endif
 #ifdef CONFIG_MAC
 	if (MACH_IS_MAC && vbl_detected)
-		free_irq(IRQ_MAC_VBL, fbcon_vbl_handler);
+		free_irq(IRQ_MAC_VBL, fb_vbl_handler);
 #endif
 
 	kfree((void *)softback_buf);
diff --git a/drivers/video/nvidia/nv_of.c b/drivers/video/nvidia/nv_of.c
index 8209106..d9af88c 100644
--- a/drivers/video/nvidia/nv_of.c
+++ b/drivers/video/nvidia/nv_of.c
@@ -32,7 +32,7 @@ int nvidia_probe_of_connector(struct fb_info *info, int conn, u8 **out_edid)
 {
 	struct nvidia_par *par = info->par;
 	struct device_node *parent, *dp;
-	unsigned char *pedid = NULL;
+	const unsigned char *pedid = NULL;
 	static char *propnames[] = {
 		"DFP,EDID", "LCD,EDID", "EDID", "EDID1",
 		"EDID,B", "EDID,A", NULL };
@@ -42,20 +42,19 @@ int nvidia_probe_of_connector(struct fb_info *info, int conn, u8 **out_edid)
 	if (parent == NULL)
 		return -1;
 	if (par->twoHeads) {
-		char *pname;
+		const char *pname;
 		int len;
 
 		for (dp = NULL;
 		     (dp = of_get_next_child(parent, dp)) != NULL;) {
-			pname = (char *)get_property(dp, "name", NULL);
+			pname = get_property(dp, "name", NULL);
 			if (!pname)
 				continue;
 			len = strlen(pname);
 			if ((pname[len-1] == 'A' && conn == 1) ||
 			    (pname[len-1] == 'B' && conn == 2)) {
 				for (i = 0; propnames[i] != NULL; ++i) {
-					pedid = (unsigned char *)
-						get_property(dp, propnames[i],
+					pedid = get_property(dp, propnames[i],
 							     NULL);
 					if (pedid != NULL)
 						break;
@@ -67,8 +66,7 @@ int nvidia_probe_of_connector(struct fb_info *info, int conn, u8 **out_edid)
 	}
 	if (pedid == NULL) {
 		for (i = 0; propnames[i] != NULL; ++i) {
-			pedid = (unsigned char *)
-				get_property(parent, propnames[i], NULL);
+			pedid = get_property(parent, propnames[i], NULL);
 			if (pedid != NULL)
 				break;
 		}
diff --git a/drivers/video/offb.c b/drivers/video/offb.c
index 0013311..bad0e98 100644
--- a/drivers/video/offb.c
+++ b/drivers/video/offb.c
@@ -409,30 +409,30 @@ static void __init offb_init_nodriver(struct device_node *dp, int no_real_node)
 	unsigned int flags, rsize, addr_prop = 0;
 	unsigned long max_size = 0;
 	u64 rstart, address = OF_BAD_ADDR;
-	u32 *pp, *addrp, *up;
+	const u32 *pp, *addrp, *up;
 	u64 asize;
 
-	pp = (u32 *)get_property(dp, "linux,bootx-depth", &len);
+	pp = get_property(dp, "linux,bootx-depth", &len);
 	if (pp == NULL)
-		pp = (u32 *)get_property(dp, "depth", &len);
+		pp = get_property(dp, "depth", &len);
 	if (pp && len == sizeof(u32))
 		depth = *pp;
 
-	pp = (u32 *)get_property(dp, "linux,bootx-width", &len);
+	pp = get_property(dp, "linux,bootx-width", &len);
 	if (pp == NULL)
-		pp = (u32 *)get_property(dp, "width", &len);
+		pp = get_property(dp, "width", &len);
 	if (pp && len == sizeof(u32))
 		width = *pp;
 
-	pp = (u32 *)get_property(dp, "linux,bootx-height", &len);
+	pp = get_property(dp, "linux,bootx-height", &len);
 	if (pp == NULL)
-		pp = (u32 *)get_property(dp, "height", &len);
+		pp = get_property(dp, "height", &len);
 	if (pp && len == sizeof(u32))
 		height = *pp;
 
-	pp = (u32 *)get_property(dp, "linux,bootx-linebytes", &len);
+	pp = get_property(dp, "linux,bootx-linebytes", &len);
 	if (pp == NULL)
-		pp = (u32 *)get_property(dp, "linebytes", &len);
+		pp = get_property(dp, "linebytes", &len);
 	if (pp && len == sizeof(u32))
 		pitch = *pp;
 	else
@@ -450,9 +450,9 @@ static void __init offb_init_nodriver(struct device_node *dp, int no_real_node)
 	 * ranges and pick one that is both big enough and if possible encloses
 	 * the "address" property. If none match, we pick the biggest
 	 */
-	up = (u32 *)get_property(dp, "linux,bootx-addr", &len);
+	up = get_property(dp, "linux,bootx-addr", &len);
 	if (up == NULL)
-		up = (u32 *)get_property(dp, "address", &len);
+		up = get_property(dp, "address", &len);
 	if (up && len == sizeof(u32))
 		addr_prop = *up;
 
diff --git a/drivers/video/riva/fbdev.c b/drivers/video/riva/fbdev.c
index 8ddb47a..4acde4f 100644
--- a/drivers/video/riva/fbdev.c
+++ b/drivers/video/riva/fbdev.c
@@ -1826,8 +1826,8 @@ static int __devinit riva_get_EDID_OF(struct fb_info *info, struct pci_dev *pd)
 {
 	struct riva_par *par = info->par;
 	struct device_node *dp;
-	unsigned char *pedid = NULL;
-	unsigned char *disptype = NULL;
+	const unsigned char *pedid = NULL;
+	const unsigned char *disptype = NULL;
 	static char *propnames[] = {
 		"DFP,EDID", "LCD,EDID", "EDID", "EDID1", "EDID,B", "EDID,A", NULL };
 	int i;
@@ -1835,14 +1835,13 @@ static int __devinit riva_get_EDID_OF(struct fb_info *info, struct pci_dev *pd)
 	NVTRACE_ENTER();
 	dp = pci_device_to_OF_node(pd);
 	for (; dp != NULL; dp = dp->child) {
-		disptype = (unsigned char *)get_property(dp, "display-type", NULL);
+		disptype = get_property(dp, "display-type", NULL);
 		if (disptype == NULL)
 			continue;
 		if (strncmp(disptype, "LCD", 3) != 0)
 			continue;
 		for (i = 0; propnames[i] != NULL; ++i) {
-			pedid = (unsigned char *)
-				get_property(dp, propnames[i], NULL);
+			pedid = get_property(dp, propnames[i], NULL);
 			if (pedid != NULL) {
 				par->EDID = pedid;
 				NVTRACE("LCD found.\n");