IB/umad: Add P_Key index support

Add support for setting the P_Key index of sent MADs and getting the
P_Key index of received MADs.  This requires a change to the layout of
the ABI structure struct ib_user_mad_hdr, so to avoid breaking
compatibility, we default to the old (unchanged) ABI and add a new
ioctl IB_USER_MAD_ENABLE_PKEY that allows applications that are aware
of the new ABI to opt into using it.

We plan on switching to the new ABI by default in a year or so, and
this patch adds a warning that is printed when an application uses the
old ABI, to push people towards converting to the new ABI.

Signed-off-by: Roland Dreier <rolandd@cisco.com>
Reviewed-by: Sean Hefty <sean.hefty@intel.com>
Reviewed-by: Hal Rosenstock <hal@xsigo.com>

1 files changed, 73 insertions, 29 deletions

diff --git a/drivers/infiniband/core/user_mad.c b/drivers/infiniband/core/user_mad.c
index d97ded2..aee2913 100644
--- a/drivers/infiniband/core/user_mad.c
+++ b/drivers/infiniband/core/user_mad.c
@@ -118,6 +118,8 @@ struct ib_umad_file {
 	wait_queue_head_t	recv_wait;
 	struct ib_mad_agent    *agent[IB_UMAD_MAX_AGENTS];
 	int			agents_dead;
+	u8			use_pkey_index;
+	u8			already_used;
 };
 
 struct ib_umad_packet {
@@ -147,6 +149,12 @@ static void ib_umad_release_dev(struct kref *ref)
 	kfree(dev);
 }
 
+static int hdr_size(struct ib_umad_file *file)
+{
+	return file->use_pkey_index ? sizeof (struct ib_user_mad_hdr) :
+		sizeof (struct ib_user_mad_hdr_old);
+}
+
 /* caller must hold port->mutex at least for reading */
 static struct ib_mad_agent *__get_agent(struct ib_umad_file *file, int id)
 {
@@ -221,13 +229,13 @@ static void recv_handler(struct ib_mad_agent *agent,
 	packet->length = mad_recv_wc->mad_len;
 	packet->recv_wc = mad_recv_wc;
 
-	packet->mad.hdr.status    = 0;
-	packet->mad.hdr.length    = sizeof (struct ib_user_mad) +
-				    mad_recv_wc->mad_len;
-	packet->mad.hdr.qpn 	  = cpu_to_be32(mad_recv_wc->wc->src_qp);
-	packet->mad.hdr.lid 	  = cpu_to_be16(mad_recv_wc->wc->slid);
-	packet->mad.hdr.sl  	  = mad_recv_wc->wc->sl;
-	packet->mad.hdr.path_bits = mad_recv_wc->wc->dlid_path_bits;
+	packet->mad.hdr.status	   = 0;
+	packet->mad.hdr.length	   = hdr_size(file) + mad_recv_wc->mad_len;
+	packet->mad.hdr.qpn	   = cpu_to_be32(mad_recv_wc->wc->src_qp);
+	packet->mad.hdr.lid	   = cpu_to_be16(mad_recv_wc->wc->slid);
+	packet->mad.hdr.sl	   = mad_recv_wc->wc->sl;
+	packet->mad.hdr.path_bits  = mad_recv_wc->wc->dlid_path_bits;
+	packet->mad.hdr.pkey_index = mad_recv_wc->wc->pkey_index;
 	packet->mad.hdr.grh_present = !!(mad_recv_wc->wc->wc_flags & IB_WC_GRH);
 	if (packet->mad.hdr.grh_present) {
 		struct ib_ah_attr ah_attr;
@@ -253,8 +261,8 @@ err1:
 	ib_free_recv_mad(mad_recv_wc);
 }
 
-static ssize_t copy_recv_mad(char __user *buf, struct ib_umad_packet *packet,
-			     size_t count)
+static ssize_t copy_recv_mad(struct ib_umad_file *file, char __user *buf,
+			     struct ib_umad_packet *packet, size_t count)
 {
 	struct ib_mad_recv_buf *recv_buf;
 	int left, seg_payload, offset, max_seg_payload;
@@ -262,15 +270,15 @@ static ssize_t copy_recv_mad(char __user *buf, struct ib_umad_packet *packet,
 	/* We need enough room to copy the first (or only) MAD segment. */
 	recv_buf = &packet->recv_wc->recv_buf;
 	if ((packet->length <= sizeof (*recv_buf->mad) &&
-	     count < sizeof (packet->mad) + packet->length) ||
+	     count < hdr_size(file) + packet->length) ||
 	    (packet->length > sizeof (*recv_buf->mad) &&
-	     count < sizeof (packet->mad) + sizeof (*recv_buf->mad)))
+	     count < hdr_size(file) + sizeof (*recv_buf->mad)))
 		return -EINVAL;
 
-	if (copy_to_user(buf, &packet->mad, sizeof (packet->mad)))
+	if (copy_to_user(buf, &packet->mad, hdr_size(file)))
 		return -EFAULT;
 
-	buf += sizeof (packet->mad);
+	buf += hdr_size(file);
 	seg_payload = min_t(int, packet->length, sizeof (*recv_buf->mad));
 	if (copy_to_user(buf, recv_buf->mad, seg_payload))
 		return -EFAULT;
@@ -280,7 +288,7 @@ static ssize_t copy_recv_mad(char __user *buf, struct ib_umad_packet *packet,
 		 * Multipacket RMPP MAD message. Copy remainder of message.
 		 * Note that last segment may have a shorter payload.
 		 */
-		if (count < sizeof (packet->mad) + packet->length) {
+		if (count < hdr_size(file) + packet->length) {
 			/*
 			 * The buffer is too small, return the first RMPP segment,
 			 * which includes the RMPP message length.
@@ -300,18 +308,23 @@ static ssize_t copy_recv_mad(char __user *buf, struct ib_umad_packet *packet,
 				return -EFAULT;
 		}
 	}
-	return sizeof (packet->mad) + packet->length;
+	return hdr_size(file) + packet->length;
 }
 
-static ssize_t copy_send_mad(char __user *buf, struct ib_umad_packet *packet,
-			     size_t count)
+static ssize_t copy_send_mad(struct ib_umad_file *file, char __user *buf,
+			     struct ib_umad_packet *packet, size_t count)
 {
-	ssize_t size = sizeof (packet->mad) + packet->length;
+	ssize_t size = hdr_size(file) + packet->length;
 
 	if (count < size)
 		return -EINVAL;
 
-	if (copy_to_user(buf, &packet->mad, size))
+	if (copy_to_user(buf, &packet->mad, hdr_size(file)))
+		return -EFAULT;
+
+	buf += hdr_size(file);
+
+	if (copy_to_user(buf, packet->mad.data, packet->length))
 		return -EFAULT;
 
 	return size;
@@ -324,7 +337,7 @@ static ssize_t ib_umad_read(struct file *filp, char __user *buf,
 	struct ib_umad_packet *packet;
 	ssize_t ret;
 
-	if (count < sizeof (struct ib_user_mad))
+	if (count < hdr_size(file))
 		return -EINVAL;
 
 	spin_lock_irq(&file->recv_lock);
@@ -348,9 +361,9 @@ static ssize_t ib_umad_read(struct file *filp, char __user *buf,
 	spin_unlock_irq(&file->recv_lock);
 
 	if (packet->recv_wc)
-		ret = copy_recv_mad(buf, packet, count);
+		ret = copy_recv_mad(file, buf, packet, count);
 	else
-		ret = copy_send_mad(buf, packet, count);
+		ret = copy_send_mad(file, buf, packet, count);
 
 	if (ret < 0) {
 		/* Requeue packet */
@@ -442,15 +455,14 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
 	__be64 *tid;
 	int ret, data_len, hdr_len, copy_offset, rmpp_active;
 
-	if (count < sizeof (struct ib_user_mad) + IB_MGMT_RMPP_HDR)
+	if (count < hdr_size(file) + IB_MGMT_RMPP_HDR)
 		return -EINVAL;
 
 	packet = kzalloc(sizeof *packet + IB_MGMT_RMPP_HDR, GFP_KERNEL);
 	if (!packet)
 		return -ENOMEM;
 
-	if (copy_from_user(&packet->mad, buf,
-			    sizeof (struct ib_user_mad) + IB_MGMT_RMPP_HDR)) {
+	if (copy_from_user(&packet->mad, buf, hdr_size(file))) {
 		ret = -EFAULT;
 		goto err;
 	}
@@ -461,6 +473,13 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
 		goto err;
 	}
 
+	buf += hdr_size(file);
+
+	if (copy_from_user(packet->mad.data, buf, IB_MGMT_RMPP_HDR)) {
+		ret = -EFAULT;
+		goto err;
+	}
+
 	down_read(&file->port->mutex);
 
 	agent = __get_agent(file, packet->mad.hdr.id);
@@ -500,11 +519,11 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
 			      IB_MGMT_RMPP_FLAG_ACTIVE;
 	}
 
-	data_len = count - sizeof (struct ib_user_mad) - hdr_len;
+	data_len = count - hdr_size(file) - hdr_len;
 	packet->msg = ib_create_send_mad(agent,
 					 be32_to_cpu(packet->mad.hdr.qpn),
-					 0, rmpp_active, hdr_len,
-					 data_len, GFP_KERNEL);
+					 packet->mad.hdr.pkey_index, rmpp_active,
+					 hdr_len, data_len, GFP_KERNEL);
 	if (IS_ERR(packet->msg)) {
 		ret = PTR_ERR(packet->msg);
 		goto err_ah;
@@ -517,7 +536,6 @@ static ssize_t ib_umad_write(struct file *filp, const char __user *buf,
 
 	/* Copy MAD header.  Any RMPP header is already in place. */
 	memcpy(packet->msg->mad, packet->mad.data, IB_MGMT_MAD_HDR);
-	buf += sizeof (struct ib_user_mad);
 
 	if (!rmpp_active) {
 		if (copy_from_user(packet->msg->mad + copy_offset,
@@ -646,6 +664,16 @@ found:
 		goto out;
 	}
 
+	if (!file->already_used) {
+		file->already_used = 1;
+		if (!file->use_pkey_index) {
+			printk(KERN_WARNING "user_mad: process %s did not enable "
+			       "P_Key index support.\n", current->comm);
+			printk(KERN_WARNING "user_mad:   Documentation/infiniband/user_mad.txt "
+			       "has info on the new ABI.\n");
+		}
+	}
+
 	file->agent[agent_id] = agent;
 	ret = 0;
 
@@ -682,6 +710,20 @@ out:
 	return ret;
 }
 
+static long ib_umad_enable_pkey(struct ib_umad_file *file)
+{
+	int ret = 0;
+
+	down_write(&file->port->mutex);
+	if (file->already_used)
+		ret = -EINVAL;
+	else
+		file->use_pkey_index = 1;
+	up_write(&file->port->mutex);
+
+	return ret;
+}
+
 static long ib_umad_ioctl(struct file *filp, unsigned int cmd,
 			  unsigned long arg)
 {
@@ -690,6 +732,8 @@ static long ib_umad_ioctl(struct file *filp, unsigned int cmd,
 		return ib_umad_reg_agent(filp->private_data, arg);
 	case IB_USER_MAD_UNREGISTER_AGENT:
 		return ib_umad_unreg_agent(filp->private_data, arg);
+	case IB_USER_MAD_ENABLE_PKEY:
+		return ib_umad_enable_pkey(filp->private_data);
 	default:
 		return -ENOIOCTLCMD;
 	}