1 files changed, 81 insertions, 175 deletions

diff --git a/drivers/infiniband/hw/mthca/mthca_cq.c b/drivers/infiniband/hw/mthca/mthca_cq.c
index 5687c30..8600b6c 100644
--- a/drivers/infiniband/hw/mthca/mthca_cq.c
+++ b/drivers/infiniband/hw/mthca/mthca_cq.c
@@ -2,6 +2,8 @@
  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
  * Copyright (c) 2005 Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
+ * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -37,7 +39,7 @@
 #include <linux/init.h>
 #include <linux/hardirq.h>
 
-#include <ib_pack.h>
+#include <rdma/ib_pack.h>
 
 #include "mthca_dev.h"
 #include "mthca_cmd.h"
@@ -55,21 +57,21 @@ enum {
  * Must be packed because start is 64 bits but only aligned to 32 bits.
  */
 struct mthca_cq_context {
-	u32 flags;
-	u64 start;
-	u32 logsize_usrpage;
-	u32 error_eqn;		/* Tavor only */
-	u32 comp_eqn;
-	u32 pd;
-	u32 lkey;
-	u32 last_notified_index;
-	u32 solicit_producer_index;
-	u32 consumer_index;
-	u32 producer_index;
-	u32 cqn;
-	u32 ci_db;		/* Arbel only */
-	u32 state_db;		/* Arbel only */
-	u32 reserved;
+	__be32 flags;
+	__be64 start;
+	__be32 logsize_usrpage;
+	__be32 error_eqn;	/* Tavor only */
+	__be32 comp_eqn;
+	__be32 pd;
+	__be32 lkey;
+	__be32 last_notified_index;
+	__be32 solicit_producer_index;
+	__be32 consumer_index;
+	__be32 producer_index;
+	__be32 cqn;
+	__be32 ci_db;		/* Arbel only */
+	__be32 state_db;	/* Arbel only */
+	u32    reserved;
 } __attribute__((packed));
 
 #define MTHCA_CQ_STATUS_OK          ( 0 << 28)
@@ -108,31 +110,31 @@ enum {
 };
 
 struct mthca_cqe {
-	u32 my_qpn;
-	u32 my_ee;
-	u32 rqpn;
-	u16 sl_g_mlpath;
-	u16 rlid;
-	u32 imm_etype_pkey_eec;
-	u32 byte_cnt;
-	u32 wqe;
-	u8  opcode;
-	u8  is_send;
-	u8  reserved;
-	u8  owner;
+	__be32 my_qpn;
+	__be32 my_ee;
+	__be32 rqpn;
+	__be16 sl_g_mlpath;
+	__be16 rlid;
+	__be32 imm_etype_pkey_eec;
+	__be32 byte_cnt;
+	__be32 wqe;
+	u8     opcode;
+	u8     is_send;
+	u8     reserved;
+	u8     owner;
 };
 
 struct mthca_err_cqe {
-	u32 my_qpn;
-	u32 reserved1[3];
-	u8  syndrome;
-	u8  reserved2;
-	u16 db_cnt;
-	u32 reserved3;
-	u32 wqe;
-	u8  opcode;
-	u8  reserved4[2];
-	u8  owner;
+	__be32 my_qpn;
+	u32    reserved1[3];
+	u8     syndrome;
+	u8     reserved2;
+	__be16 db_cnt;
+	u32    reserved3;
+	__be32 wqe;
+	u8     opcode;
+	u8     reserved4[2];
+	u8     owner;
 };
 
 #define MTHCA_CQ_ENTRY_OWNER_SW      (0 << 7)
@@ -191,7 +193,7 @@ static void dump_cqe(struct mthca_dev *dev, void *cqe_ptr)
 static inline void update_cons_index(struct mthca_dev *dev, struct mthca_cq *cq,
 				     int incr)
 {
-	u32 doorbell[2];
+	__be32 doorbell[2];
 
 	if (mthca_is_memfree(dev)) {
 		*cq->set_ci_db = cpu_to_be32(cq->cons_index);
@@ -222,7 +224,8 @@ void mthca_cq_event(struct mthca_dev *dev, u32 cqn)
 	cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
 }
 
-void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn)
+void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn,
+		    struct mthca_srq *srq)
 {
 	struct mthca_cq *cq;
 	struct mthca_cqe *cqe;
@@ -263,8 +266,11 @@ void mthca_cq_clean(struct mthca_dev *dev, u32 cqn, u32 qpn)
 	 */
 	while (prod_index > cq->cons_index) {
 		cqe = get_cqe(cq, (prod_index - 1) & cq->ibcq.cqe);
-		if (cqe->my_qpn == cpu_to_be32(qpn))
+		if (cqe->my_qpn == cpu_to_be32(qpn)) {
+			if (srq)
+				mthca_free_srq_wqe(srq, be32_to_cpu(cqe->wqe));
 			++nfreed;
+		}
 		else if (nfreed)
 			memcpy(get_cqe(cq, (prod_index - 1 + nfreed) &
 				       cq->ibcq.cqe),
@@ -291,7 +297,7 @@ static int handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,
 {
 	int err;
 	int dbd;
-	u32 new_wqe;
+	__be32 new_wqe;
 
 	if (cqe->syndrome == SYNDROME_LOCAL_QP_OP_ERR) {
 		mthca_dbg(dev, "local QP operation err "
@@ -365,6 +371,13 @@ static int handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,
 		break;
 	}
 
+	/*
+	 * Mem-free HCAs always generate one CQE per WQE, even in the
+	 * error case, so we don't have to check the doorbell count, etc.
+	 */
+	if (mthca_is_memfree(dev))
+		return 0;
+
 	err = mthca_free_err_wqe(dev, qp, is_send, wqe_index, &dbd, &new_wqe);
 	if (err)
 		return err;
@@ -373,12 +386,8 @@ static int handle_error_cqe(struct mthca_dev *dev, struct mthca_cq *cq,
 	 * If we're at the end of the WQE chain, or we've used up our
 	 * doorbell count, free the CQE.  Otherwise just update it for
 	 * the next poll operation.
-	 *
-	 * This does not apply to mem-free HCAs: they don't use the
-	 * doorbell count field, and so we should always free the CQE.
 	 */
-	if (mthca_is_memfree(dev) ||
-	    !(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd))
+	if (!(new_wqe & cpu_to_be32(0x3f)) || (!cqe->db_cnt && dbd))
 		return 0;
 
 	cqe->db_cnt   = cpu_to_be16(be16_to_cpu(cqe->db_cnt) - dbd);
@@ -450,23 +459,27 @@ static inline int mthca_poll_one(struct mthca_dev *dev,
 			     >> wq->wqe_shift);
 		entry->wr_id = (*cur_qp)->wrid[wqe_index +
 					       (*cur_qp)->rq.max];
+	} else if ((*cur_qp)->ibqp.srq) {
+		struct mthca_srq *srq = to_msrq((*cur_qp)->ibqp.srq);
+		u32 wqe = be32_to_cpu(cqe->wqe);
+		wq = NULL;
+		wqe_index = wqe >> srq->wqe_shift;
+		entry->wr_id = srq->wrid[wqe_index];
+		mthca_free_srq_wqe(srq, wqe);
 	} else {
 		wq = &(*cur_qp)->rq;
 		wqe_index = be32_to_cpu(cqe->wqe) >> wq->wqe_shift;
 		entry->wr_id = (*cur_qp)->wrid[wqe_index];
 	}
 
-	if (wq->last_comp < wqe_index)
-		wq->tail += wqe_index - wq->last_comp;
-	else
-		wq->tail += wqe_index + wq->max - wq->last_comp;
-
-	wq->last_comp = wqe_index;
+	if (wq) {
+		if (wq->last_comp < wqe_index)
+			wq->tail += wqe_index - wq->last_comp;
+		else
+			wq->tail += wqe_index + wq->max - wq->last_comp;
 
-	if (0)
-		mthca_dbg(dev, "%s completion for QP %06x, index %d (nr %d)\n",
-			  is_send ? "Send" : "Receive",
-			  (*cur_qp)->qpn, wqe_index, wq->max);
+		wq->last_comp = wqe_index;
+	}
 
 	if (is_error) {
 		err = handle_error_cqe(dev, cq, *cur_qp, wqe_index, is_send,
@@ -584,13 +597,13 @@ int mthca_poll_cq(struct ib_cq *ibcq, int num_entries,
 
 int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify)
 {
-	u32 doorbell[2];
+	__be32 doorbell[2];
 
 	doorbell[0] = cpu_to_be32((notify == IB_CQ_SOLICITED ?
 				   MTHCA_TAVOR_CQ_DB_REQ_NOT_SOL :
 				   MTHCA_TAVOR_CQ_DB_REQ_NOT)      |
 				  to_mcq(cq)->cqn);
-	doorbell[1] = 0xffffffff;
+	doorbell[1] = (__force __be32) 0xffffffff;
 
 	mthca_write64(doorbell,
 		      to_mdev(cq->device)->kar + MTHCA_CQ_DOORBELL,
@@ -602,9 +615,9 @@ int mthca_tavor_arm_cq(struct ib_cq *cq, enum ib_cq_notify notify)
 int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
 {
 	struct mthca_cq *cq = to_mcq(ibcq);
-	u32 doorbell[2];
+	__be32 doorbell[2];
 	u32 sn;
-	u32 ci;
+	__be32 ci;
 
 	sn = cq->arm_sn & 3;
 	ci = cpu_to_be32(cq->cons_index);
@@ -637,113 +650,8 @@ int mthca_arbel_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify notify)
 
 static void mthca_free_cq_buf(struct mthca_dev *dev, struct mthca_cq *cq)
 {
-	int i;
-	int size;
-
-	if (cq->is_direct)
-		dma_free_coherent(&dev->pdev->dev,
-				  (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE,
-				  cq->queue.direct.buf,
-				  pci_unmap_addr(&cq->queue.direct,
-						 mapping));
-	else {
-		size = (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE;
-		for (i = 0; i < (size + PAGE_SIZE - 1) / PAGE_SIZE; ++i)
-			if (cq->queue.page_list[i].buf)
-				dma_free_coherent(&dev->pdev->dev, PAGE_SIZE,
-						  cq->queue.page_list[i].buf,
-						  pci_unmap_addr(&cq->queue.page_list[i],
-								 mapping));
-
-		kfree(cq->queue.page_list);
-	}
-}
-
-static int mthca_alloc_cq_buf(struct mthca_dev *dev, int size,
-			      struct mthca_cq *cq)
-{
-	int err = -ENOMEM;
-	int npages, shift;
-	u64 *dma_list = NULL;
-	dma_addr_t t;
-	int i;
-
-	if (size <= MTHCA_MAX_DIRECT_CQ_SIZE) {
-		cq->is_direct = 1;
-		npages        = 1;
-		shift         = get_order(size) + PAGE_SHIFT;
-
-		cq->queue.direct.buf = dma_alloc_coherent(&dev->pdev->dev,
-							  size, &t, GFP_KERNEL);
-		if (!cq->queue.direct.buf)
-			return -ENOMEM;
-
-		pci_unmap_addr_set(&cq->queue.direct, mapping, t);
-
-		memset(cq->queue.direct.buf, 0, size);
-
-		while (t & ((1 << shift) - 1)) {
-			--shift;
-			npages *= 2;
-		}
-
-		dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
-		if (!dma_list)
-			goto err_free;
-
-		for (i = 0; i < npages; ++i)
-			dma_list[i] = t + i * (1 << shift);
-	} else {
-		cq->is_direct = 0;
-		npages        = (size + PAGE_SIZE - 1) / PAGE_SIZE;
-		shift         = PAGE_SHIFT;
-
-		dma_list = kmalloc(npages * sizeof *dma_list, GFP_KERNEL);
-		if (!dma_list)
-			return -ENOMEM;
-
-		cq->queue.page_list = kmalloc(npages * sizeof *cq->queue.page_list,
-					      GFP_KERNEL);
-		if (!cq->queue.page_list)
-			goto err_out;
-
-		for (i = 0; i < npages; ++i)
-			cq->queue.page_list[i].buf = NULL;
-
-		for (i = 0; i < npages; ++i) {
-			cq->queue.page_list[i].buf =
-				dma_alloc_coherent(&dev->pdev->dev, PAGE_SIZE,
-						   &t, GFP_KERNEL);
-			if (!cq->queue.page_list[i].buf)
-				goto err_free;
-
-			dma_list[i] = t;
-			pci_unmap_addr_set(&cq->queue.page_list[i], mapping, t);
-
-			memset(cq->queue.page_list[i].buf, 0, PAGE_SIZE);
-		}
-	}
-
-	err = mthca_mr_alloc_phys(dev, dev->driver_pd.pd_num,
-				  dma_list, shift, npages,
-				  0, size,
-				  MTHCA_MPT_FLAG_LOCAL_WRITE |
-				  MTHCA_MPT_FLAG_LOCAL_READ,
-				  &cq->mr);
-	if (err)
-		goto err_free;
-
-	kfree(dma_list);
-
-	return 0;
-
-err_free:
-	mthca_free_cq_buf(dev, cq);
-
-err_out:
-	kfree(dma_list);
-
-	return err;
+	mthca_buf_free(dev, (cq->ibcq.cqe + 1) * MTHCA_CQ_ENTRY_SIZE,
+		       &cq->queue, cq->is_direct, &cq->mr);
 }
 
 int mthca_init_cq(struct mthca_dev *dev, int nent,
@@ -795,7 +703,9 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
 	cq_context = mailbox->buf;
 
 	if (cq->is_kernel) {
-		err = mthca_alloc_cq_buf(dev, size, cq);
+		err = mthca_buf_alloc(dev, size, MTHCA_MAX_DIRECT_CQ_SIZE,
+				      &cq->queue, &cq->is_direct,
+				      &dev->driver_pd, 1, &cq->mr);
 		if (err)
 			goto err_out_mailbox;
 
@@ -811,7 +721,6 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
 	cq_context->flags           = cpu_to_be32(MTHCA_CQ_STATUS_OK      |
 						  MTHCA_CQ_STATE_DISARMED |
 						  MTHCA_CQ_FLAG_TR);
-	cq_context->start           = cpu_to_be64(0);
 	cq_context->logsize_usrpage = cpu_to_be32((ffs(nent) - 1) << 24);
 	if (ctx)
 		cq_context->logsize_usrpage |= cpu_to_be32(ctx->uar.index);
@@ -857,10 +766,8 @@ int mthca_init_cq(struct mthca_dev *dev, int nent,
 	return 0;
 
 err_out_free_mr:
-	if (cq->is_kernel) {
-		mthca_free_mr(dev, &cq->mr);
+	if (cq->is_kernel)
 		mthca_free_cq_buf(dev, cq);
-	}
 
 err_out_mailbox:
 	mthca_free_mailbox(dev, mailbox);
@@ -904,7 +811,7 @@ void mthca_free_cq(struct mthca_dev *dev,
 		mthca_warn(dev, "HW2SW_CQ returned status 0x%02x\n", status);
 
 	if (0) {
-		u32 *ctx = mailbox->buf;
+		__be32 *ctx = mailbox->buf;
 		int j;
 
 		printk(KERN_ERR "context for CQN %x (cons index %x, next sw %d)\n",
@@ -928,7 +835,6 @@ void mthca_free_cq(struct mthca_dev *dev,
 	wait_event(cq->wait, !atomic_read(&cq->refcount));
 
 	if (cq->is_kernel) {
-		mthca_free_mr(dev, &cq->mr);
 		mthca_free_cq_buf(dev, cq);
 		if (mthca_is_memfree(dev)) {
 			mthca_free_db(dev, MTHCA_DB_TYPE_CQ_ARM,    cq->arm_db_index);