MFC 331248: Set the proper vnet in IPsec callback functions.

When using hardware crypto engines, the callback functions used to handle
an IPsec packet after it has been encrypted or decrypted can be invoked
asynchronously from a worker thread that is not associated with a vnet.
Extend 'struct xform_data' to include a vnet pointer and save the current
vnet in this new member when queueing crypto requests in IPsec.  In the
IPsec callback routines, use the new member to set the current vnet while
processing the modified packet.

This fixes a panic when using hardware offload such as ccr(4) with IPsec
after VIMAGE was enabled in GENERIC.

Sponsored by:	Chelsio Communications

4 files changed, 31 insertions, 0 deletions

diff --git a/sys/netipsec/xform.h b/sys/netipsec/xform.h
index 8e6f8bd..2720f72 100644
--- a/sys/netipsec/xform.h
+++ b/sys/netipsec/xform.h
@@ -76,6 +76,7 @@ struct xform_data {
 	int			protoff;	/* current protocol offset */
 	int			skip;		/* data offset */
 	uint8_t			nxt;		/* next protocol, e.g. IPV4 */
+	struct vnet		*vnet;
 };
 
 #define	XF_IP4		1	/* unused */
diff --git a/sys/netipsec/xform_ah.c b/sys/netipsec/xform_ah.c
index 6716e70..8749e20 100644
--- a/sys/netipsec/xform_ah.c
+++ b/sys/netipsec/xform_ah.c
@@ -665,6 +665,7 @@ ah_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	xd->protoff = protoff;
 	xd->skip = skip;
 	xd->cryptoid = cryptoid;
+	xd->vnet = curvnet;
 	return (crypto_dispatch(crp));
 bad:
 	m_freem(m);
@@ -694,6 +695,7 @@ ah_input_cb(struct cryptop *crp)
 	crd = crp->crp_desc;
 	m = (struct mbuf *) crp->crp_buf;
 	xd = (struct xform_data *) crp->crp_opaque;
+	CURVNET_SET(xd->vnet);
 	sav = xd->sav;
 	skip = xd->skip;
 	nxt = xd->nxt;
@@ -713,6 +715,7 @@ ah_input_cb(struct cryptop *crp)
 			if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_sid;
+			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		AHSTAT_INC(ahs_noxform);
@@ -808,8 +811,10 @@ ah_input_cb(struct cryptop *crp)
 		panic("%s: Unexpected address family: %d saidx=%p", __func__,
 		    saidx->dst.sa.sa_family, saidx);
 	}
+	CURVNET_RESTORE();
 	return error;
 bad:
+	CURVNET_RESTORE();
 	if (sav)
 		key_freesav(&sav);
 	if (m != NULL)
@@ -1041,6 +1046,7 @@ ah_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	xd->skip = skip;
 	xd->idx = idx;
 	xd->cryptoid = cryptoid;
+	xd->vnet = curvnet;
 
 	return crypto_dispatch(crp);
 bad:
@@ -1068,6 +1074,7 @@ ah_output_cb(struct cryptop *crp)
 
 	m = (struct mbuf *) crp->crp_buf;
 	xd = (struct xform_data *) crp->crp_opaque;
+	CURVNET_SET(xd->vnet);
 	sp = xd->sp;
 	sav = xd->sav;
 	skip = xd->skip;
@@ -1082,6 +1089,7 @@ ah_output_cb(struct cryptop *crp)
 			if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_sid;
+			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		AHSTAT_INC(ahs_noxform);
@@ -1123,8 +1131,10 @@ ah_output_cb(struct cryptop *crp)
 
 	/* NB: m is reclaimed by ipsec_process_done. */
 	error = ipsec_process_done(m, sp, sav, idx);
+	CURVNET_RESTORE();
 	return (error);
 bad:
+	CURVNET_RESTORE();
 	free(xd, M_XDATA);
 	crypto_freereq(crp);
 	key_freesav(&sav);
diff --git a/sys/netipsec/xform_esp.c b/sys/netipsec/xform_esp.c
index 39d5b8c..c5f4768 100644
--- a/sys/netipsec/xform_esp.c
+++ b/sys/netipsec/xform_esp.c
@@ -395,6 +395,7 @@ esp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	xd->protoff = protoff;
 	xd->skip = skip;
 	xd->cryptoid = cryptoid;
+	xd->vnet = curvnet;
 
 	/* Decryption descriptor */
 	IPSEC_ASSERT(crde != NULL, ("null esp crypto descriptor"));
@@ -454,6 +455,7 @@ esp_input_cb(struct cryptop *crp)
 
 	m = (struct mbuf *) crp->crp_buf;
 	xd = (struct xform_data *) crp->crp_opaque;
+	CURVNET_SET(xd->vnet);
 	sav = xd->sav;
 	skip = xd->skip;
 	protoff = xd->protoff;
@@ -469,6 +471,7 @@ esp_input_cb(struct cryptop *crp)
 			if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_sid;
+			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		ESPSTAT_INC(esps_noxform);
@@ -603,8 +606,10 @@ esp_input_cb(struct cryptop *crp)
 		panic("%s: Unexpected address family: %d saidx=%p", __func__,
 		    saidx->dst.sa.sa_family, saidx);
 	}
+	CURVNET_RESTORE();
 	return error;
 bad:
+	CURVNET_RESTORE();
 	if (sav != NULL)
 		key_freesav(&sav);
 	if (m != NULL)
@@ -837,6 +842,7 @@ esp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	xd->sav = sav;
 	xd->idx = idx;
 	xd->cryptoid = cryptoid;
+	xd->vnet = curvnet;
 
 	/* Crypto operation descriptor. */
 	crp->crp_ilen = m->m_pkthdr.len; /* Total input length. */
@@ -880,6 +886,7 @@ esp_output_cb(struct cryptop *crp)
 	int error;
 
 	xd = (struct xform_data *) crp->crp_opaque;
+	CURVNET_SET(xd->vnet);
 	m = (struct mbuf *) crp->crp_buf;
 	sp = xd->sp;
 	sav = xd->sav;
@@ -893,6 +900,7 @@ esp_output_cb(struct cryptop *crp)
 			if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_sid;
+			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		ESPSTAT_INC(esps_noxform);
@@ -938,8 +946,10 @@ esp_output_cb(struct cryptop *crp)
 
 	/* NB: m is reclaimed by ipsec_process_done. */
 	error = ipsec_process_done(m, sp, sav, idx);
+	CURVNET_RESTORE();
 	return (error);
 bad:
+	CURVNET_RESTORE();
 	free(xd, M_XDATA);
 	crypto_freereq(crp);
 	key_freesav(&sav);
diff --git a/sys/netipsec/xform_ipcomp.c b/sys/netipsec/xform_ipcomp.c
index 1e8fc59..660818e 100644
--- a/sys/netipsec/xform_ipcomp.c
+++ b/sys/netipsec/xform_ipcomp.c
@@ -253,6 +253,7 @@ ipcomp_input(struct mbuf *m, struct secasvar *sav, int skip, int protoff)
 	xd->sav = sav;
 	xd->protoff = protoff;
 	xd->skip = skip;
+	xd->vnet = curvnet;
 
 	SECASVAR_LOCK(sav);
 	crp->crp_sid = xd->cryptoid = sav->tdb_cryptoid;
@@ -287,6 +288,7 @@ ipcomp_input_cb(struct cryptop *crp)
 
 	m = (struct mbuf *) crp->crp_buf;
 	xd = (struct xform_data *) crp->crp_opaque;
+	CURVNET_SET(xd->vnet);
 	sav = xd->sav;
 	skip = xd->skip;
 	protoff = xd->protoff;
@@ -303,6 +305,7 @@ ipcomp_input_cb(struct cryptop *crp)
 			if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_sid;
+			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		IPCOMPSTAT_INC(ipcomps_noxform);
@@ -367,8 +370,10 @@ ipcomp_input_cb(struct cryptop *crp)
 		panic("%s: Unexpected address family: %d saidx=%p", __func__,
 		    saidx->dst.sa.sa_family, saidx);
 	}
+	CURVNET_RESTORE();
 	return error;
 bad:
+	CURVNET_RESTORE();
 	if (sav != NULL)
 		key_freesav(&sav);
 	if (m != NULL)
@@ -494,6 +499,7 @@ ipcomp_output(struct mbuf *m, struct secpolicy *sp, struct secasvar *sav,
 	xd->idx = idx;
 	xd->skip = skip;
 	xd->protoff = protoff;
+	xd->vnet = curvnet;
 
 	/* Crypto operation descriptor */
 	crp->crp_ilen = m->m_pkthdr.len;	/* Total input length */
@@ -532,6 +538,7 @@ ipcomp_output_cb(struct cryptop *crp)
 
 	m = (struct mbuf *) crp->crp_buf;
 	xd = (struct xform_data *) crp->crp_opaque;
+	CURVNET_SET(xd->vnet);
 	idx = xd->idx;
 	sp = xd->sp;
 	sav = xd->sav;
@@ -546,6 +553,7 @@ ipcomp_output_cb(struct cryptop *crp)
 			if (ipsec_updateid(sav, &crp->crp_sid, &cryptoid) != 0)
 				crypto_freesession(cryptoid);
 			xd->cryptoid = crp->crp_sid;
+			CURVNET_RESTORE();
 			return (crypto_dispatch(crp));
 		}
 		IPCOMPSTAT_INC(ipcomps_noxform);
@@ -641,10 +649,12 @@ ipcomp_output_cb(struct cryptop *crp)
 
 	/* NB: m is reclaimed by ipsec_process_done. */
 	error = ipsec_process_done(m, sp, sav, idx);
+	CURVNET_RESTORE();
 	return (error);
 bad:
 	if (m)
 		m_freem(m);
+	CURVNET_RESTORE();
 	free(xd, M_XDATA);
 	crypto_freereq(crp);
 	key_freesav(&sav);