summaryrefslogtreecommitdiffstats
path: root/sys/netipsec/key.c
diff options
context:
space:
mode:
Diffstat (limited to 'sys/netipsec/key.c')
-rw-r--r--sys/netipsec/key.c422
1 files changed, 224 insertions, 198 deletions
diff --git a/sys/netipsec/key.c b/sys/netipsec/key.c
index a3e3d76..4e21706 100644
--- a/sys/netipsec/key.c
+++ b/sys/netipsec/key.c
@@ -139,20 +139,37 @@ static VNET_DEFINE(int, key_preferred_oldsa) = 1;
static VNET_DEFINE(u_int32_t, acq_seq) = 0;
#define V_acq_seq VNET(acq_seq)
+ /* SPD cache */
+struct secpolicycache {
+ struct secpolicy *policy;
+ struct secpolicyindex spidx;
+ u_int32_t genid;
+};
+#define SPCACHESIZE 8192
+static VNET_DEFINE(struct secpolicycache, spcache[2][SPCACHESIZE]);
+#define V_spcache VNET(spcache)
+static VNET_DEFINE(u_int32_t, spcache_genid) = 0;
+#define V_spcache_genid VNET(spcache_genid)
+
/* SPD */
static VNET_DEFINE(LIST_HEAD(_sptree, secpolicy), sptree[IPSEC_DIR_MAX]);
#define V_sptree VNET(sptree)
-static struct mtx sptree_lock;
-#define SPTREE_LOCK_INIT() \
- mtx_init(&sptree_lock, "sptree", \
+static struct mtx sptree_lock[IPSEC_DIR_MAX];
+#define SPTREE_LOCK_INIT(dir) \
+ mtx_init(&sptree_lock[dir], "sptree", \
"fast ipsec security policy database", MTX_DEF)
-#define SPTREE_LOCK_DESTROY() mtx_destroy(&sptree_lock)
-#define SPTREE_LOCK() mtx_lock(&sptree_lock)
-#define SPTREE_UNLOCK() mtx_unlock(&sptree_lock)
-#define SPTREE_LOCK_ASSERT() mtx_assert(&sptree_lock, MA_OWNED)
+#define SPTREE_LOCK_DESTROY(dir) mtx_destroy(&sptree_lock[dir])
+#define SPTREE_LOCK(dir) mtx_lock(&sptree_lock[dir])
+#define SPTREE_UNLOCK(dir) mtx_unlock(&sptree_lock[dir])
+#define SPTREE_LOCK_ASSERT(dir) mtx_assert(&sptree_lock[dir], MA_OWNED)
+
+#define SPIHASHSIZE 1024
+#define SPIHASH(x) (((x) + ((x) >> 16)) % SPIHASHSIZE)
static VNET_DEFINE(LIST_HEAD(_sahtree, secashead), sahtree); /* SAD */
#define V_sahtree VNET(sahtree)
+static VNET_DEFINE(LIST_HEAD(_spihash, secasvar), spihash[SPIHASHSIZE]);
+#define V_spihash VNET(spihash)
static struct mtx sahtree_lock;
#define SAHTREE_LOCK_INIT() \
mtx_init(&sahtree_lock, "sahtree", \
@@ -329,6 +346,8 @@ SYSCTL_VNET_INT(_net_key, KEYCTL_PREFERED_OLDSA,
#define __LIST_CHAINED(elm) \
(!((elm)->chain.le_next == NULL && (elm)->chain.le_prev == NULL))
+#define __LIST_SPIHASHED(elm) \
+ (!((elm)->spihash.le_next == NULL && (elm)->spihash.le_prev == NULL))
#define LIST_INSERT_TAIL(head, elm, type, field) \
do {\
struct type *curelm = LIST_FIRST(head); \
@@ -437,11 +456,11 @@ static u_int key_getspreqmsglen(struct secpolicy *);
static int key_spdexpire(struct secpolicy *);
static struct secashead *key_newsah(struct secasindex *);
static void key_delsah(struct secashead *);
-static struct secasvar *key_newsav(struct mbuf *,
+static struct secasvar *key_newsav(u_int32_t, struct mbuf *,
const struct sadb_msghdr *, struct secashead *, int *,
const char*, int);
-#define KEY_NEWSAV(m, sadb, sah, e) \
- key_newsav(m, sadb, sah, e, __FILE__, __LINE__)
+#define KEY_NEWSAV(spi, m, sadb, sah, e) \
+ key_newsav(spi, m, sadb, sah, e, __FILE__, __LINE__)
static void key_delsav(struct secasvar *);
static struct secashead *key_getsah(struct secasindex *);
static struct secasvar *key_checkspidup(struct secasindex *, u_int32_t);
@@ -586,9 +605,9 @@ sa_delref(struct secasvar *sav)
void
key_addref(struct secpolicy *sp)
{
- SPTREE_LOCK();
+ SPTREE_LOCK(sp->spidx.dir);
SP_ADDREF(sp);
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(sp->spidx.dir);
}
/*
@@ -611,16 +630,12 @@ key_havesp(u_int dir)
* OUT: NULL: not found
* others: found and return the pointer.
*/
-struct secpolicy *
-key_allocsp(struct secpolicyindex *spidx, u_int dir, const char* where,
+static struct secpolicy *
+key_allocsp_slow(struct secpolicyindex *spidx, u_int dir, const char* where,
int tag)
{
struct secpolicy *sp;
- IPSEC_ASSERT(spidx != NULL, ("null spidx"));
- IPSEC_ASSERT(dir == IPSEC_DIR_INBOUND || dir == IPSEC_DIR_OUTBOUND,
- ("invalid direction %u", dir));
-
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s from %s:%u\n", __func__, where, tag));
@@ -629,7 +644,7 @@ key_allocsp(struct secpolicyindex *spidx, u_int dir, const char* where,
printf("*** objects\n");
kdebug_secpolicyindex(spidx));
- SPTREE_LOCK();
+ SPTREE_LOCK_ASSERT(dir);
LIST_FOREACH(sp, &V_sptree[dir], chain) {
KEYDEBUG(KEYDEBUG_IPSEC_DATA,
printf("*** in SPD\n");
@@ -642,6 +657,45 @@ key_allocsp(struct secpolicyindex *spidx, u_int dir, const char* where,
}
sp = NULL;
found:
+
+ KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
+ printf("DP %s return SP:%p (ID=%u) refcnt %u\n", __func__,
+ sp, sp ? sp->id : 0, sp ? sp->refcnt : 0));
+ return sp;
+}
+
+static Fnv32_t key_hash_spidx(struct secpolicyindex *spidx)
+{
+ Fnv32_t hash = FNV1_32_INIT;
+ hash = fnv_32_buf(&spidx->src.sa, spidx->src.sa.sa_len, hash);
+ hash = fnv_32_buf(&spidx->dst.sa, spidx->dst.sa.sa_len, hash);
+ hash = fnv_32_buf(&spidx->ul_proto, sizeof(spidx->ul_proto), hash);
+ return hash;
+}
+
+struct secpolicy *
+key_allocsp(struct secpolicyindex *spidx, u_int dir, const char* where, int tag)
+{
+ struct secpolicy *sp;
+ Fnv32_t hash;
+ int hdir;
+
+ IPSEC_ASSERT(spidx != NULL, ("null spidx"));
+ IPSEC_ASSERT(dir == IPSEC_DIR_INBOUND || dir == IPSEC_DIR_OUTBOUND,
+ ("invalid direction %u", dir));
+
+ SPTREE_LOCK(dir);
+ hdir = (dir == IPSEC_DIR_INBOUND);
+ hash = key_hash_spidx(spidx) % SPCACHESIZE;
+ if (V_spcache[hdir][hash].genid == V_spcache_genid &&
+ key_cmpspidx_exactly(&V_spcache[hdir][hash].spidx, spidx)) {
+ sp = V_spcache[hdir][hash].policy;
+ } else {
+ sp = key_allocsp_slow(spidx, dir, where, tag);
+ V_spcache[hdir][hash].policy = sp;
+ V_spcache[hdir][hash].spidx = *spidx;
+ V_spcache[hdir][hash].genid = V_spcache_genid;
+ }
if (sp) {
/* sanity check */
KEY_CHKSPDIR(sp->spidx.dir, dir, __func__);
@@ -650,11 +704,8 @@ found:
sp->lastused = time_second;
SP_ADDREF(sp);
}
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(dir);
- KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
- printf("DP %s return SP:%p (ID=%u) refcnt %u\n", __func__,
- sp, sp ? sp->id : 0, sp ? sp->refcnt : 0));
return sp;
}
@@ -683,7 +734,7 @@ key_allocsp2(u_int32_t spi, union sockaddr_union *dst, u_int8_t proto,
printf("spi %u proto %u dir %u\n", spi, proto, dir);
kdebug_sockaddr(&dst->sa));
- SPTREE_LOCK();
+ SPTREE_LOCK(dir);
LIST_FOREACH(sp, &V_sptree[dir], chain) {
KEYDEBUG(KEYDEBUG_IPSEC_DATA,
printf("*** in SPD\n");
@@ -710,7 +761,7 @@ found:
sp->lastused = time_second;
SP_ADDREF(sp);
}
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(dir);
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s return SP:%p (ID=%u) refcnt %u\n", __func__,
@@ -745,7 +796,7 @@ key_gettunnel(const struct sockaddr *osrc,
goto done;
}
- SPTREE_LOCK();
+ SPTREE_LOCK(dir);
LIST_FOREACH(sp, &V_sptree[dir], chain) {
if (sp->state == IPSEC_SPSTATE_DEAD)
continue;
@@ -787,7 +838,7 @@ found:
sp->lastused = time_second;
SP_ADDREF(sp);
}
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(dir);
done:
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s return SP:%p (ID=%u) refcnt %u\n", __func__,
@@ -1077,10 +1128,7 @@ struct secasvar *
key_allocsa(union sockaddr_union *dst, u_int proto, u_int32_t spi,
const char* where, int tag)
{
- struct secashead *sah;
struct secasvar *sav;
- u_int stateidx, arraysize, state;
- const u_int *saorder_state_valid;
#ifdef IPSEC_NAT_T
int natt_chkport;
#endif
@@ -1103,54 +1151,32 @@ key_allocsa(union sockaddr_union *dst, u_int proto, u_int32_t spi,
* encrypted so we can't check internal IP header.
*/
SAHTREE_LOCK();
- if (V_key_preferred_oldsa) {
- saorder_state_valid = saorder_state_valid_prefer_old;
- arraysize = _ARRAYLEN(saorder_state_valid_prefer_old);
- } else {
- saorder_state_valid = saorder_state_valid_prefer_new;
- arraysize = _ARRAYLEN(saorder_state_valid_prefer_new);
- }
- LIST_FOREACH(sah, &V_sahtree, chain) {
+ LIST_FOREACH(sav, &spihash[SPIHASH(spi)], spihash) {
int checkport;
- /* search valid state */
- for (stateidx = 0; stateidx < arraysize; stateidx++) {
- state = saorder_state_valid[stateidx];
- LIST_FOREACH(sav, &sah->savtree[state], chain) {
- /* sanity check */
- KEY_CHKSASTATE(sav->state, state, __func__);
- /* do not return entries w/ unusable state */
- if (sav->state != SADB_SASTATE_MATURE &&
- sav->state != SADB_SASTATE_DYING)
- continue;
- if (proto != sav->sah->saidx.proto)
- continue;
- if (spi != sav->spi)
- continue;
- checkport = 0;
+ if (sav->state != SADB_SASTATE_MATURE &&
+ sav->state != SADB_SASTATE_DYING)
+ continue;
+ if (proto != sav->sah->saidx.proto)
+ continue;
+ if (spi != sav->spi)
+ continue;
+
+ checkport = 0;
#ifdef IPSEC_NAT_T
- /*
- * Really only check ports when this is a NAT-T
- * SA. Otherwise other lookups providing ports
- * might suffer.
- */
- if (sav->natt_type && natt_chkport)
- checkport = 1;
-#endif
-#if 0 /* don't check src */
- /* check src address */
- if (key_sockaddrcmp(&src->sa,
- &sav->sah->saidx.src.sa, checkport) != 0)
- continue;
+ /*
+ * Really only check ports when this is a NAT-T
+ * SA. Otherwise other lookups providing ports
+ * might suffer.
+ */
+ if (sav->natt_type && natt_chkport)
+ checkport = 1;
#endif
- /* check dst address */
- if (key_sockaddrcmp(&dst->sa,
- &sav->sah->saidx.dst.sa, checkport) != 0)
- continue;
- sa_addref(sav);
- goto done;
- }
- }
+ /* check dst address */
+ if (key_sockaddrcmp(&dst->sa, &sav->sah->saidx.dst.sa, checkport) != 0)
+ continue;
+ sa_addref(sav);
+ goto done;
}
sav = NULL;
done:
@@ -1173,7 +1199,7 @@ _key_freesp(struct secpolicy **spp, const char* where, int tag)
IPSEC_ASSERT(sp != NULL, ("null sp"));
- SPTREE_LOCK();
+ SPTREE_LOCK(sp->spidx.dir);
SP_DELREF(sp);
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
@@ -1184,7 +1210,7 @@ _key_freesp(struct secpolicy **spp, const char* where, int tag)
*spp = NULL;
key_delsp(sp);
}
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(sp->spidx.dir);
}
/*
@@ -1281,8 +1307,10 @@ key_delsp(struct secpolicy *sp)
struct ipsecrequest *isr, *nextisr;
IPSEC_ASSERT(sp != NULL, ("null sp"));
- SPTREE_LOCK_ASSERT();
+ SPTREE_LOCK_ASSERT(sp->spidx.dir);
+ if (sp->state != IPSEC_SPSTATE_DEAD)
+ V_spcache_genid++;
sp->state = IPSEC_SPSTATE_DEAD;
IPSEC_ASSERT(sp->refcnt == 0,
@@ -1316,7 +1344,7 @@ key_getsp(struct secpolicyindex *spidx)
IPSEC_ASSERT(spidx != NULL, ("null spidx"));
- SPTREE_LOCK();
+ SPTREE_LOCK(spidx->dir);
LIST_FOREACH(sp, &V_sptree[spidx->dir], chain) {
if (sp->state == IPSEC_SPSTATE_DEAD)
continue;
@@ -1325,7 +1353,7 @@ key_getsp(struct secpolicyindex *spidx)
break;
}
}
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(spidx->dir);
return sp;
}
@@ -1339,9 +1367,11 @@ static struct secpolicy *
key_getspbyid(u_int32_t id)
{
struct secpolicy *sp;
+ int dir;
- SPTREE_LOCK();
- LIST_FOREACH(sp, &V_sptree[IPSEC_DIR_INBOUND], chain) {
+ dir = IPSEC_DIR_INBOUND;
+ SPTREE_LOCK(dir);
+ LIST_FOREACH(sp, &V_sptree[dir], chain) {
if (sp->state == IPSEC_SPSTATE_DEAD)
continue;
if (sp->id == id) {
@@ -1349,8 +1379,11 @@ key_getspbyid(u_int32_t id)
goto done;
}
}
+ SPTREE_UNLOCK(dir);
- LIST_FOREACH(sp, &V_sptree[IPSEC_DIR_OUTBOUND], chain) {
+ dir = IPSEC_DIR_OUTBOUND;
+ SPTREE_LOCK(dir);
+ LIST_FOREACH(sp, &V_sptree[dir], chain) {
if (sp->state == IPSEC_SPSTATE_DEAD)
continue;
if (sp->id == id) {
@@ -1359,7 +1392,7 @@ key_getspbyid(u_int32_t id)
}
}
done:
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(dir);
return sp;
}
@@ -1869,9 +1902,9 @@ key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
newsp = key_getsp(&spidx);
if (mhp->msg->sadb_msg_type == SADB_X_SPDUPDATE) {
if (newsp) {
- SPTREE_LOCK();
+ SPTREE_LOCK(spidx.dir);
newsp->state = IPSEC_SPSTATE_DEAD;
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(spidx.dir);
KEY_FREESP(&newsp);
}
} else {
@@ -1930,6 +1963,7 @@ key_spdadd(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
newsp->refcnt = 1; /* do not reclaim until I say I do */
newsp->state = IPSEC_SPSTATE_ALIVE;
LIST_INSERT_TAIL(&V_sptree[newsp->spidx.dir], newsp, secpolicy, chain);
+ V_spcache_genid++;
/* delete the entry in spacqtree */
if (mhp->msg->sadb_msg_type == SADB_X_SPDUPDATE) {
@@ -2104,9 +2138,9 @@ key_spddelete(struct socket *so, struct mbuf *m,
/* save policy id to buffer to be returned. */
xpl0->sadb_x_policy_id = sp->id;
- SPTREE_LOCK();
+ SPTREE_LOCK(spidx.dir);
sp->state = IPSEC_SPSTATE_DEAD;
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(spidx.dir);
KEY_FREESP(&sp);
{
@@ -2171,9 +2205,9 @@ key_spddelete2(struct socket *so, struct mbuf *m,
return key_senderror(so, m, EINVAL);
}
- SPTREE_LOCK();
+ SPTREE_LOCK(sp->spidx.dir);
sp->state = IPSEC_SPSTATE_DEAD;
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(sp->spidx.dir);
KEY_FREESP(&sp);
{
@@ -2365,10 +2399,11 @@ key_spdflush(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
return key_senderror(so, m, EINVAL);
for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
- SPTREE_LOCK();
+ SPTREE_LOCK(dir);
LIST_FOREACH(sp, &V_sptree[dir], chain)
sp->state = IPSEC_SPSTATE_DEAD;
- SPTREE_UNLOCK();
+ V_spcache_genid++;
+ SPTREE_UNLOCK(dir);
}
if (sizeof(struct sadb_msg) > m->m_len + M_TRAILINGSPACE(m)) {
@@ -2404,7 +2439,7 @@ key_spddump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
struct secpolicy *sp;
int cnt;
u_int dir;
- struct mbuf *n;
+ struct mbuf *n, *rh, *rt;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
@@ -2413,32 +2448,44 @@ key_spddump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
/* search SPD entry and get buffer size. */
cnt = 0;
- SPTREE_LOCK();
for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
+ SPTREE_LOCK(dir);
LIST_FOREACH(sp, &V_sptree[dir], chain) {
cnt++;
}
+ SPTREE_UNLOCK(dir);
}
if (cnt == 0) {
- SPTREE_UNLOCK();
return key_senderror(so, m, ENOENT);
}
+ rh = rt = NULL;
+ /* XXX: SP tree may change from above! */
for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
+ SPTREE_LOCK(dir);
LIST_FOREACH(sp, &V_sptree[dir], chain) {
--cnt;
n = key_setdumpsp(sp, SADB_X_SPDDUMP, cnt,
mhp->msg->sadb_msg_pid);
+ if (!n) {
+ SPTREE_UNLOCK(dir);
+ m_freem(rh);
+ return key_senderror(so, m, ENOBUFS);
+ }
+
+ if (rt)
+ rt->m_nextpkt = n;
+ else
+ rh = n;
+ rt = n;
- if (n)
- key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
}
+ SPTREE_UNLOCK(dir);
}
- SPTREE_UNLOCK();
m_freem(m);
- return 0;
+ return key_sendup_mbuf(so, rh, KEY_SENDUP_ONESHOT);
}
static struct mbuf *
@@ -2747,16 +2794,16 @@ key_delsah(struct secashead *sah)
* does not modify mbuf. does not free mbuf on error.
*/
static struct secasvar *
-key_newsav(struct mbuf *m, const struct sadb_msghdr *mhp,
+key_newsav(u_int32_t spi, struct mbuf *m, const struct sadb_msghdr *mhp,
struct secashead *sah, int *errp, const char *where, int tag)
{
struct secasvar *newsav;
- const struct sadb_sa *xsa;
IPSEC_ASSERT(m != NULL, ("null mbuf"));
IPSEC_ASSERT(mhp != NULL, ("null msghdr"));
IPSEC_ASSERT(mhp->msg != NULL, ("null msg"));
IPSEC_ASSERT(sah != NULL, ("null secashead"));
+ SAHTREE_LOCK_ASSERT();
newsav = malloc(sizeof(struct secasvar), M_IPSEC_SA, M_NOWAIT|M_ZERO);
if (newsav == NULL) {
@@ -2765,41 +2812,20 @@ key_newsav(struct mbuf *m, const struct sadb_msghdr *mhp,
goto done;
}
- switch (mhp->msg->sadb_msg_type) {
- case SADB_GETSPI:
- newsav->spi = 0;
#ifdef IPSEC_DOSEQCHECK
- /* sync sequence number */
- if (mhp->msg->sadb_msg_seq == 0)
- newsav->seq =
- (V_acq_seq = (V_acq_seq == ~0 ? 1 : ++V_acq_seq));
- else
+ /* sync sequence number */
+ if (mhp->msg->sadb_msg_type == SADB_GETSPI &&
+ mhp->msg->sadb_msg_seq == 0)
+ newsav->seq =
+ (V_acq_seq = (V_acq_seq == ~0 ? 1 : ++V_acq_seq));
+ else
#endif
- newsav->seq = mhp->msg->sadb_msg_seq;
- break;
-
- case SADB_ADD:
- /* sanity check */
- if (mhp->ext[SADB_EXT_SA] == NULL) {
- free(newsav, M_IPSEC_SA);
- newsav = NULL;
- ipseclog((LOG_DEBUG, "%s: invalid message is passed.\n",
- __func__));
- *errp = EINVAL;
- goto done;
- }
- xsa = (const struct sadb_sa *)mhp->ext[SADB_EXT_SA];
- newsav->spi = xsa->sadb_sa_spi;
newsav->seq = mhp->msg->sadb_msg_seq;
- break;
- default:
- free(newsav, M_IPSEC_SA);
- newsav = NULL;
- *errp = EINVAL;
- goto done;
- }
+ newsav->created = time_second;
+ newsav->pid = mhp->msg->sadb_msg_pid;
+ newsav->spi = spi;
/* copy sav values */
if (mhp->msg->sadb_msg_type != SADB_GETSPI) {
@@ -2811,21 +2837,20 @@ key_newsav(struct mbuf *m, const struct sadb_msghdr *mhp,
}
}
- SECASVAR_LOCK_INIT(newsav);
-
/* reset created */
- newsav->created = time_second;
newsav->pid = mhp->msg->sadb_msg_pid;
+ newsav->spi = spi;
/* add to satree */
newsav->sah = sah;
sa_initref(newsav);
newsav->state = SADB_SASTATE_LARVAL;
- SAHTREE_LOCK();
LIST_INSERT_TAIL(&sah->savtree[SADB_SASTATE_LARVAL], newsav,
secasvar, chain);
- SAHTREE_UNLOCK();
+ if (spi)
+ LIST_INSERT_HEAD(&spihash[SPIHASH(spi)], newsav, spihash);
+
done:
KEYDEBUG(KEYDEBUG_IPSEC_STAMP,
printf("DP %s from %s:%u return SP:%p\n", __func__,
@@ -2901,8 +2926,9 @@ key_delsav(struct secasvar *sav)
/* remove from SA header */
if (__LIST_CHAINED(sav))
LIST_REMOVE(sav, chain);
+ if (__LIST_SPIHASHED(sav))
+ LIST_REMOVE(sav, spihash);
key_cleansav(sav);
- SECASVAR_LOCK_DESTROY(sav);
free(sav, M_IPSEC_SA);
}
@@ -2931,7 +2957,6 @@ key_getsah(struct secasindex *saidx)
/*
* check not to be duplicated SPI.
- * NOTE: this function is too slow due to searching all SAD.
* OUT:
* NULL : not found
* others : found, pointer to a SA.
@@ -2939,9 +2964,10 @@ key_getsah(struct secasindex *saidx)
static struct secasvar *
key_checkspidup(struct secasindex *saidx, u_int32_t spi)
{
- struct secashead *sah;
struct secasvar *sav;
+ SAHTREE_LOCK_ASSERT();
+
/* check address family */
if (saidx->src.sa.sa_family != saidx->dst.sa.sa_family) {
ipseclog((LOG_DEBUG, "%s: address family mismatched.\n",
@@ -2950,16 +2976,11 @@ key_checkspidup(struct secasindex *saidx, u_int32_t spi)
}
sav = NULL;
- /* check all SAD */
- SAHTREE_LOCK();
- LIST_FOREACH(sah, &V_sahtree, chain) {
- if (!key_ismyaddr((struct sockaddr *)&sah->saidx.dst))
- continue;
- sav = key_getsavbyspi(sah, spi);
- if (sav != NULL)
+ LIST_FOREACH(sav, &spihash[SPIHASH(spi)], spihash) {
+ if (sav->spi == spi &&
+ key_ismyaddr((struct sockaddr *)&sav->sah->saidx.dst))
break;
}
- SAHTREE_UNLOCK();
return sav;
}
@@ -2974,27 +2995,20 @@ static struct secasvar *
key_getsavbyspi(struct secashead *sah, u_int32_t spi)
{
struct secasvar *sav;
- u_int stateidx, state;
+ u_int stateidx;
sav = NULL;
SAHTREE_LOCK_ASSERT();
- /* search all status */
- for (stateidx = 0;
- stateidx < _ARRAYLEN(saorder_state_alive);
- stateidx++) {
-
- state = saorder_state_alive[stateidx];
- LIST_FOREACH(sav, &sah->savtree[state], chain) {
-
- /* sanity check */
- if (sav->state != state) {
- ipseclog((LOG_DEBUG, "%s: "
- "invalid sav->state (queue: %d SA: %d)\n",
- __func__, state, sav->state));
- continue;
- }
+ LIST_FOREACH(sav, &spihash[SPIHASH(spi)], spihash) {
+ if (sav->sah != sah)
+ continue;
+ if (sav->spi != spi)
+ continue;
- if (sav->spi == spi)
+ for (stateidx = 0;
+ stateidx < _ARRAYLEN(saorder_state_alive);
+ stateidx++) {
+ if (sav->state == saorder_state_alive[stateidx])
return sav;
}
}
@@ -3336,7 +3350,7 @@ static struct mbuf *
key_setdumpsa(struct secasvar *sav, u_int8_t type, u_int8_t satype,
u_int32_t seq, u_int32_t pid)
{
- struct mbuf *result = NULL, *tres = NULL, *m;
+ struct mbuf *result = NULL, *m;
int i;
int dumporder[] = {
SADB_EXT_SA, SADB_X_EXT_SA2,
@@ -3358,7 +3372,7 @@ key_setdumpsa(struct secasvar *sav, u_int8_t type, u_int8_t satype,
goto fail;
result = m;
- for (i = sizeof(dumporder)/sizeof(dumporder[0]) - 1; i >= 0; i--) {
+ for (i = 0; i < sizeof(dumporder)/sizeof(dumporder[0]); i++) {
m = NULL;
switch (dumporder[i]) {
case SADB_EXT_SA:
@@ -3476,13 +3490,9 @@ key_setdumpsa(struct secasvar *sav, u_int8_t type, u_int8_t satype,
if (!m)
goto fail;
- if (tres)
- m_cat(m, tres);
- tres = m;
-
+ m_cat(result, m);
}
- m_cat(result, tres);
if (result->m_len < sizeof(struct sadb_msg)) {
result = m_pullup(result, sizeof(struct sadb_msg));
if (result == NULL)
@@ -3500,7 +3510,6 @@ key_setdumpsa(struct secasvar *sav, u_int8_t type, u_int8_t satype,
fail:
m_freem(result);
- m_freem(tres);
return NULL;
}
@@ -4230,7 +4239,7 @@ key_flush_spd(time_t now)
/* SPD */
for (dir = 0; dir < IPSEC_DIR_MAX; dir++) {
restart:
- SPTREE_LOCK();
+ SPTREE_LOCK(dir);
LIST_FOREACH(sp, &V_sptree[dir], chain) {
if (sp->scangen == gen) /* previously handled */
continue;
@@ -4249,7 +4258,7 @@ restart:
*/
SP_DELREF(sp);
key_delsp(sp);
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(dir);
goto restart;
}
if (sp->lifetime == 0 && sp->validtime == 0)
@@ -4257,12 +4266,13 @@ restart:
if ((sp->lifetime && now - sp->created > sp->lifetime)
|| (sp->validtime && now - sp->lastused > sp->validtime)) {
sp->state = IPSEC_SPSTATE_DEAD;
- SPTREE_UNLOCK();
+ V_spcache_genid++;
+ SPTREE_UNLOCK(dir);
key_spdexpire(sp);
goto restart;
}
}
- SPTREE_UNLOCK();
+ SPTREE_UNLOCK(dir);
}
}
@@ -4678,12 +4688,6 @@ key_getspi(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
}
#endif
- /* SPI allocation */
- spi = key_do_getnewspi((struct sadb_spirange *)mhp->ext[SADB_EXT_SPIRANGE],
- &saidx);
- if (spi == 0)
- return key_senderror(so, m, EINVAL);
-
/* get a SA index */
if ((newsah = key_getsah(&saidx)) == NULL) {
/* create a new SA index */
@@ -4693,16 +4697,24 @@ key_getspi(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
}
}
+ /* SPI allocation */
+ SAHTREE_LOCK();
+ spi = key_do_getnewspi((struct sadb_spirange *)mhp->ext[SADB_EXT_SPIRANGE],
+ &saidx);
+ if (spi == 0) {
+ SAHTREE_UNLOCK();
+ return key_senderror(so, m, EINVAL);
+ }
+
/* get a new SA */
/* XXX rewrite */
- newsav = KEY_NEWSAV(m, mhp, newsah, &error);
+ newsav = KEY_NEWSAV(spi, m, mhp, newsah, &error);
if (newsav == NULL) {
/* XXX don't free new SA index allocated in above. */
+ SAHTREE_UNLOCK();
return key_senderror(so, m, error);
}
-
- /* set spi */
- newsav->spi = htonl(spi);
+ SAHTREE_UNLOCK();
/* delete the entry in acqtree */
if (mhp->msg->sadb_msg_seq != 0) {
@@ -4745,7 +4757,7 @@ key_getspi(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
m_sa = (struct sadb_sa *)(mtod(n, caddr_t) + off);
m_sa->sadb_sa_len = PFKEY_UNIT64(sizeof(struct sadb_sa));
m_sa->sadb_sa_exttype = SADB_EXT_SA;
- m_sa->sadb_sa_spi = htonl(spi);
+ m_sa->sadb_sa_spi = spi;
off += PFKEY_ALIGN8(sizeof(struct sadb_sa));
IPSEC_ASSERT(off == len,
@@ -4792,6 +4804,8 @@ key_do_getnewspi(struct sadb_spirange *spirange, struct secasindex *saidx)
u_int32_t min, max;
int count = V_key_spi_trycnt;
+ SAHTREE_LOCK_ASSERT();
+
/* set spi range to allocate */
if (spirange != NULL) {
min = spirange->sadb_spirange_min;
@@ -4813,15 +4827,15 @@ key_do_getnewspi(struct sadb_spirange *spirange, struct secasindex *saidx)
}
if (min == max) {
- if (key_checkspidup(saidx, min) != NULL) {
+ newspi = htonl(min);
+
+ if (key_checkspidup(saidx, newspi) != NULL) {
ipseclog((LOG_DEBUG, "%s: SPI %u exists already.\n",
__func__, min));
return 0;
}
count--; /* taking one cost. */
- newspi = min;
-
} else {
/* init SPI */
@@ -4830,7 +4844,7 @@ key_do_getnewspi(struct sadb_spirange *spirange, struct secasindex *saidx)
/* when requesting to allocate spi ranged */
while (count--) {
/* generate pseudo-random SPI value ranged. */
- newspi = min + (key_random() % (max - min + 1));
+ newspi = htonl(min + (key_random() % (max - min + 1)));
if (key_checkspidup(saidx, newspi) == NULL)
break;
@@ -5303,12 +5317,14 @@ key_add(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
/* We can create new SA only if SPI is differenct. */
SAHTREE_LOCK();
newsav = key_getsavbyspi(newsah, sa0->sadb_sa_spi);
- SAHTREE_UNLOCK();
if (newsav != NULL) {
+ SAHTREE_UNLOCK();
ipseclog((LOG_DEBUG, "%s: SA already exists.\n", __func__));
return key_senderror(so, m, EEXIST);
}
- newsav = KEY_NEWSAV(m, mhp, newsah, &error);
+ newsav = KEY_NEWSAV(sa0->sadb_sa_spi, m, mhp, newsah, &error);
+ SAHTREE_UNLOCK();
+
if (newsav == NULL) {
return key_senderror(so, m, error);
}
@@ -7054,7 +7070,7 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
u_int8_t state;
int cnt;
struct sadb_msg *newmsg;
- struct mbuf *n;
+ struct mbuf *rh, *rt, *n;
IPSEC_ASSERT(so != NULL, ("null socket"));
IPSEC_ASSERT(m != NULL, ("null mbuf"));
@@ -7093,6 +7109,7 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
/* send this to the userland, one at a time. */
newmsg = NULL;
+ rh = rt = NULL;
LIST_FOREACH(sah, &V_sahtree, chain) {
if (mhp->msg->sadb_msg_satype != SADB_SATYPE_UNSPEC
&& proto != sah->saidx.proto)
@@ -7101,6 +7118,7 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
/* map proto to satype */
if ((satype = key_proto2satype(sah->saidx.proto)) == 0) {
SAHTREE_UNLOCK();
+ m_freem(rh);
ipseclog((LOG_DEBUG, "%s: there was invalid proto in "
"SAD.\n", __func__));
return key_senderror(so, m, EINVAL);
@@ -7115,16 +7133,21 @@ key_dump(struct socket *so, struct mbuf *m, const struct sadb_msghdr *mhp)
--cnt, mhp->msg->sadb_msg_pid);
if (!n) {
SAHTREE_UNLOCK();
+ m_freem(rh);
return key_senderror(so, m, ENOBUFS);
}
- key_sendup_mbuf(so, n, KEY_SENDUP_ONE);
+ if (rt)
+ rt->m_nextpkt = n;
+ else
+ rh = n;
+ rt = n;
}
}
}
SAHTREE_UNLOCK();
m_freem(m);
- return 0;
+ return key_sendup_mbuf(so, rh, KEY_SENDUP_ONESHOT);
}
/*
@@ -7658,6 +7681,8 @@ key_init(void)
LIST_INIT(&V_sptree[i]);
LIST_INIT(&V_sahtree);
+ for (i = 0; i < SPIHASHSIZE; i++)
+ LIST_INIT(&V_spihash[i]);
for (i = 0; i <= SADB_SATYPE_MAX; i++)
LIST_INIT(&V_regtree[i]);
@@ -7672,7 +7697,8 @@ key_init(void)
if (!IS_DEFAULT_VNET(curvnet))
return;
- SPTREE_LOCK_INIT();
+ for (i = 0; i < IPSEC_DIR_MAX; i++)
+ SPTREE_LOCK_INIT(i);
REGTREE_LOCK_INIT();
SAHTREE_LOCK_INIT();
ACQ_LOCK_INIT();
@@ -7699,8 +7725,8 @@ key_destroy(void)
struct secreg *reg;
int i;
- SPTREE_LOCK();
for (i = 0; i < IPSEC_DIR_MAX; i++) {
+ SPTREE_LOCK(i);
for (sp = LIST_FIRST(&V_sptree[i]);
sp != NULL; sp = nextsp) {
nextsp = LIST_NEXT(sp, chain);
@@ -7709,8 +7735,8 @@ key_destroy(void)
free(sp, M_IPSEC_SP);
}
}
+ SPTREE_UNLOCK(i);
}
- SPTREE_UNLOCK();
SAHTREE_LOCK();
for (sah = LIST_FIRST(&V_sahtree); sah != NULL; sah = nextsah) {
OpenPOWER on IntegriCloud