diff options
53 files changed, 4515 insertions, 1512 deletions
diff --git a/arch/s390/crypto/des_s390.c b/arch/s390/crypto/des_s390.c index 4aba83b..2bc479a 100644 --- a/arch/s390/crypto/des_s390.c +++ b/arch/s390/crypto/des_s390.c @@ -250,8 +250,9 @@ static int des3_128_setkey(struct crypto_tfm *tfm, const u8 *key, const u8 *temp_key = key; u32 *flags = &tfm->crt_flags; - if (!(memcmp(key, &key[DES_KEY_SIZE], DES_KEY_SIZE))) { - *flags |= CRYPTO_TFM_RES_BAD_KEY_SCHED; + if (!(memcmp(key, &key[DES_KEY_SIZE], DES_KEY_SIZE)) && + (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } for (i = 0; i < 2; i++, temp_key += DES_KEY_SIZE) { @@ -411,9 +412,9 @@ static int des3_192_setkey(struct crypto_tfm *tfm, const u8 *key, if (!(memcmp(key, &key[DES_KEY_SIZE], DES_KEY_SIZE) && memcmp(&key[DES_KEY_SIZE], &key[DES_KEY_SIZE * 2], - DES_KEY_SIZE))) { - - *flags |= CRYPTO_TFM_RES_BAD_KEY_SCHED; + DES_KEY_SIZE)) && + (*flags & CRYPTO_TFM_REQ_WEAK_KEY)) { + *flags |= CRYPTO_TFM_RES_WEAK_KEY; return -EINVAL; } for (i = 0; i < 3; i++, temp_key += DES_KEY_SIZE) { diff --git a/arch/s390/crypto/sha1_s390.c b/arch/s390/crypto/sha1_s390.c index e85ba34..f6de782 100644 --- a/arch/s390/crypto/sha1_s390.c +++ b/arch/s390/crypto/sha1_s390.c @@ -46,12 +46,38 @@ static int sha1_init(struct shash_desc *desc) return 0; } +static int sha1_export(struct shash_desc *desc, void *out) +{ + struct s390_sha_ctx *sctx = shash_desc_ctx(desc); + struct sha1_state *octx = out; + + octx->count = sctx->count; + memcpy(octx->state, sctx->state, sizeof(octx->state)); + memcpy(octx->buffer, sctx->buf, sizeof(octx->buffer)); + return 0; +} + +static int sha1_import(struct shash_desc *desc, const void *in) +{ + struct s390_sha_ctx *sctx = shash_desc_ctx(desc); + const struct sha1_state *ictx = in; + + sctx->count = ictx->count; + memcpy(sctx->state, ictx->state, sizeof(ictx->state)); + memcpy(sctx->buf, ictx->buffer, sizeof(ictx->buffer)); + sctx->func = KIMD_SHA_1; + return 0; +} + static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_init, .update = s390_sha_update, .final = s390_sha_final, + .export = sha1_export, + .import = sha1_import, .descsize = sizeof(struct s390_sha_ctx), + .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name= "sha1-s390", diff --git a/arch/s390/crypto/sha256_s390.c b/arch/s390/crypto/sha256_s390.c index f9fefc5..61a7db3 100644 --- a/arch/s390/crypto/sha256_s390.c +++ b/arch/s390/crypto/sha256_s390.c @@ -42,12 +42,38 @@ static int sha256_init(struct shash_desc *desc) return 0; } +static int sha256_export(struct shash_desc *desc, void *out) +{ + struct s390_sha_ctx *sctx = shash_desc_ctx(desc); + struct sha256_state *octx = out; + + octx->count = sctx->count; + memcpy(octx->state, sctx->state, sizeof(octx->state)); + memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); + return 0; +} + +static int sha256_import(struct shash_desc *desc, const void *in) +{ + struct s390_sha_ctx *sctx = shash_desc_ctx(desc); + const struct sha256_state *ictx = in; + + sctx->count = ictx->count; + memcpy(sctx->state, ictx->state, sizeof(ictx->state)); + memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); + sctx->func = KIMD_SHA_256; + return 0; +} + static struct shash_alg alg = { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_init, .update = s390_sha_update, .final = s390_sha_final, + .export = sha256_export, + .import = sha256_import, .descsize = sizeof(struct s390_sha_ctx), + .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-s390", diff --git a/arch/s390/crypto/sha512_s390.c b/arch/s390/crypto/sha512_s390.c index 83192bf..4bf73d0 100644 --- a/arch/s390/crypto/sha512_s390.c +++ b/arch/s390/crypto/sha512_s390.c @@ -13,7 +13,10 @@ * */ #include <crypto/internal/hash.h> +#include <crypto/sha.h> +#include <linux/errno.h> #include <linux/init.h> +#include <linux/kernel.h> #include <linux/module.h> #include "sha.h" @@ -37,12 +40,42 @@ static int sha512_init(struct shash_desc *desc) return 0; } +static int sha512_export(struct shash_desc *desc, void *out) +{ + struct s390_sha_ctx *sctx = shash_desc_ctx(desc); + struct sha512_state *octx = out; + + octx->count[0] = sctx->count; + octx->count[1] = 0; + memcpy(octx->state, sctx->state, sizeof(octx->state)); + memcpy(octx->buf, sctx->buf, sizeof(octx->buf)); + return 0; +} + +static int sha512_import(struct shash_desc *desc, const void *in) +{ + struct s390_sha_ctx *sctx = shash_desc_ctx(desc); + const struct sha512_state *ictx = in; + + if (unlikely(ictx->count[1])) + return -ERANGE; + sctx->count = ictx->count[0]; + + memcpy(sctx->state, ictx->state, sizeof(ictx->state)); + memcpy(sctx->buf, ictx->buf, sizeof(ictx->buf)); + sctx->func = KIMD_SHA_512; + return 0; +} + static struct shash_alg sha512_alg = { .digestsize = SHA512_DIGEST_SIZE, .init = sha512_init, .update = s390_sha_update, .final = s390_sha_final, + .export = sha512_export, + .import = sha512_import, .descsize = sizeof(struct s390_sha_ctx), + .statesize = sizeof(struct sha512_state), .base = { .cra_name = "sha512", .cra_driver_name= "sha512-s390", @@ -78,7 +111,10 @@ static struct shash_alg sha384_alg = { .init = sha384_init, .update = s390_sha_update, .final = s390_sha_final, + .export = sha512_export, + .import = sha512_import, .descsize = sizeof(struct s390_sha_ctx), + .statesize = sizeof(struct sha512_state), .base = { .cra_name = "sha384", .cra_driver_name= "sha384-s390", diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index c580c5e..d3ec8d5 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -636,7 +636,7 @@ static int __init aesni_init(void) int err; if (!cpu_has_aes) { - printk(KERN_ERR "Intel AES-NI instructions are not detected.\n"); + printk(KERN_INFO "Intel AES-NI instructions are not detected.\n"); return -ENODEV; } if ((err = crypto_register_alg(&aesni_alg))) diff --git a/crypto/Kconfig b/crypto/Kconfig index 4dfdd03..26b5dd0 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -23,11 +23,13 @@ comment "Crypto core or helper" config CRYPTO_FIPS bool "FIPS 200 compliance" + depends on CRYPTO_ANSI_CPRNG help This options enables the fips boot option which is required if you want to system to operate in a FIPS 200 certification. You should say no unless you know what - this is. + this is. Note that CRYPTO_ANSI_CPRNG is requred if this + option is selected config CRYPTO_ALGAPI tristate @@ -156,7 +158,7 @@ config CRYPTO_GCM tristate "GCM/GMAC support" select CRYPTO_CTR select CRYPTO_AEAD - select CRYPTO_GF128MUL + select CRYPTO_GHASH help Support for Galois/Counter Mode (GCM) and Galois Message Authentication Code (GMAC). Required for IPSec. @@ -267,6 +269,18 @@ config CRYPTO_XCBC http://csrc.nist.gov/encryption/modes/proposedmodes/ xcbc-mac/xcbc-mac-spec.pdf +config CRYPTO_VMAC + tristate "VMAC support" + depends on EXPERIMENTAL + select CRYPTO_HASH + select CRYPTO_MANAGER + help + VMAC is a message authentication algorithm designed for + very high speed on 64-bit architectures. + + See also: + <http://fastcrypto.org/vmac> + comment "Digest" config CRYPTO_CRC32C @@ -289,6 +303,13 @@ config CRYPTO_CRC32C_INTEL gain performance compared with software implementation. Module will be crc32c-intel. +config CRYPTO_GHASH + tristate "GHASH digest algorithm" + select CRYPTO_SHASH + select CRYPTO_GF128MUL + help + GHASH is message digest algorithm for GCM (Galois/Counter Mode). + config CRYPTO_MD4 tristate "MD4 digest algorithm" select CRYPTO_HASH @@ -780,13 +801,14 @@ comment "Random Number Generation" config CRYPTO_ANSI_CPRNG tristate "Pseudo Random Number Generation for Cryptographic modules" + default m select CRYPTO_AES select CRYPTO_RNG - select CRYPTO_FIPS help This option enables the generic pseudo random number generator for cryptographic modules. Uses the Algorithm specified in - ANSI X9.31 A.2.4 + ANSI X9.31 A.2.4. Not this option must be enabled if CRYPTO_FIPS + is selected source "drivers/crypto/Kconfig" diff --git a/crypto/Makefile b/crypto/Makefile index 673d9f7..9e8f619 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -3,7 +3,7 @@ # obj-$(CONFIG_CRYPTO) += crypto.o -crypto-objs := api.o cipher.o digest.o compress.o +crypto-objs := api.o cipher.o compress.o obj-$(CONFIG_CRYPTO_WORKQUEUE) += crypto_wq.o @@ -22,7 +22,6 @@ obj-$(CONFIG_CRYPTO_BLKCIPHER2) += chainiv.o obj-$(CONFIG_CRYPTO_BLKCIPHER2) += eseqiv.o obj-$(CONFIG_CRYPTO_SEQIV) += seqiv.o -crypto_hash-objs := hash.o crypto_hash-objs += ahash.o crypto_hash-objs += shash.o obj-$(CONFIG_CRYPTO_HASH2) += crypto_hash.o @@ -33,6 +32,7 @@ cryptomgr-objs := algboss.o testmgr.o obj-$(CONFIG_CRYPTO_MANAGER2) += cryptomgr.o obj-$(CONFIG_CRYPTO_HMAC) += hmac.o +obj-$(CONFIG_CRYPTO_VMAC) += vmac.o obj-$(CONFIG_CRYPTO_XCBC) += xcbc.o obj-$(CONFIG_CRYPTO_NULL) += crypto_null.o obj-$(CONFIG_CRYPTO_MD4) += md4.o @@ -83,6 +83,7 @@ obj-$(CONFIG_CRYPTO_RNG2) += rng.o obj-$(CONFIG_CRYPTO_RNG2) += krng.o obj-$(CONFIG_CRYPTO_ANSI_CPRNG) += ansi_cprng.o obj-$(CONFIG_CRYPTO_TEST) += tcrypt.o +obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o # # generic algorithms and the async_tx api diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c index e11ce37..f6f0833 100644 --- a/crypto/ablkcipher.c +++ b/crypto/ablkcipher.c @@ -14,6 +14,7 @@ */ #include <crypto/internal/skcipher.h> +#include <linux/cpumask.h> #include <linux/err.h> #include <linux/init.h> #include <linux/kernel.h> @@ -25,6 +26,8 @@ #include "internal.h" +static const char *skcipher_default_geniv __read_mostly; + static int setkey_unaligned(struct crypto_ablkcipher *tfm, const u8 *key, unsigned int keylen) { @@ -180,7 +183,14 @@ EXPORT_SYMBOL_GPL(crypto_givcipher_type); const char *crypto_default_geniv(const struct crypto_alg *alg) { - return alg->cra_flags & CRYPTO_ALG_ASYNC ? "eseqiv" : "chainiv"; + if (((alg->cra_flags & CRYPTO_ALG_TYPE_MASK) == + CRYPTO_ALG_TYPE_BLKCIPHER ? alg->cra_blkcipher.ivsize : + alg->cra_ablkcipher.ivsize) != + alg->cra_blocksize) + return "chainiv"; + + return alg->cra_flags & CRYPTO_ALG_ASYNC ? + "eseqiv" : skcipher_default_geniv; } static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask) @@ -201,8 +211,9 @@ static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask) int err; larval = crypto_larval_lookup(alg->cra_driver_name, + (type & ~CRYPTO_ALG_TYPE_MASK) | CRYPTO_ALG_TYPE_GIVCIPHER, - CRYPTO_ALG_TYPE_MASK); + mask | CRYPTO_ALG_TYPE_MASK); err = PTR_ERR(larval); if (IS_ERR(larval)) goto out; @@ -360,3 +371,17 @@ err: return ERR_PTR(err); } EXPORT_SYMBOL_GPL(crypto_alloc_ablkcipher); + +static int __init skcipher_module_init(void) +{ + skcipher_default_geniv = num_possible_cpus() > 1 ? + "eseqiv" : "chainiv"; + return 0; +} + +static void skcipher_module_exit(void) +{ +} + +module_init(skcipher_module_init); +module_exit(skcipher_module_exit); diff --git a/crypto/aes_generic.c b/crypto/aes_generic.c index b8b66ec..e78b7ee 100644 --- a/crypto/aes_generic.c +++ b/crypto/aes_generic.c @@ -1174,7 +1174,7 @@ EXPORT_SYMBOL_GPL(crypto_il_tab); ctx->key_enc[6 * i + 11] = t; \ } while (0) -#define loop8(i) do { \ +#define loop8tophalf(i) do { \ t = ror32(t, 8); \ t = ls_box(t) ^ rco_tab[i]; \ t ^= ctx->key_enc[8 * i]; \ @@ -1185,6 +1185,10 @@ EXPORT_SYMBOL_GPL(crypto_il_tab); ctx->key_enc[8 * i + 10] = t; \ t ^= ctx->key_enc[8 * i + 3]; \ ctx->key_enc[8 * i + 11] = t; \ +} while (0) + +#define loop8(i) do { \ + loop8tophalf(i); \ t = ctx->key_enc[8 * i + 4] ^ ls_box(t); \ ctx->key_enc[8 * i + 12] = t; \ t ^= ctx->key_enc[8 * i + 5]; \ @@ -1245,8 +1249,9 @@ int crypto_aes_expand_key(struct crypto_aes_ctx *ctx, const u8 *in_key, ctx->key_enc[5] = le32_to_cpu(key[5]); ctx->key_enc[6] = le32_to_cpu(key[6]); t = ctx->key_enc[7] = le32_to_cpu(key[7]); - for (i = 0; i < 7; ++i) + for (i = 0; i < 6; ++i) loop8(i); + loop8tophalf(i); break; } diff --git a/crypto/ahash.c b/crypto/ahash.c index f347637..33a4ff4 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -24,6 +24,19 @@ #include "internal.h" +struct ahash_request_priv { + crypto_completion_t complete; + void *data; + u8 *result; + void *ubuf[] CRYPTO_MINALIGN_ATTR; +}; + +static inline struct ahash_alg *crypto_ahash_alg(struct crypto_ahash *hash) +{ + return container_of(crypto_hash_alg_common(hash), struct ahash_alg, + halg); +} + static int hash_walk_next(struct crypto_hash_walk *walk) { unsigned int alignmask = walk->alignmask; @@ -132,36 +145,34 @@ int crypto_hash_walk_first_compat(struct hash_desc *hdesc, static int ahash_setkey_unaligned(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - struct ahash_alg *ahash = crypto_ahash_alg(tfm); unsigned long alignmask = crypto_ahash_alignmask(tfm); int ret; u8 *buffer, *alignbuffer; unsigned long absize; absize = keylen + alignmask; - buffer = kmalloc(absize, GFP_ATOMIC); + buffer = kmalloc(absize, GFP_KERNEL); if (!buffer) return -ENOMEM; alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); memcpy(alignbuffer, key, keylen); - ret = ahash->setkey(tfm, alignbuffer, keylen); - memset(alignbuffer, 0, keylen); - kfree(buffer); + ret = tfm->setkey(tfm, alignbuffer, keylen); + kzfree(buffer); return ret; } -static int ahash_setkey(struct crypto_ahash *tfm, const u8 *key, +int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) { - struct ahash_alg *ahash = crypto_ahash_alg(tfm); unsigned long alignmask = crypto_ahash_alignmask(tfm); if ((unsigned long)key & alignmask) return ahash_setkey_unaligned(tfm, key, keylen); - return ahash->setkey(tfm, key, keylen); + return tfm->setkey(tfm, key, keylen); } +EXPORT_SYMBOL_GPL(crypto_ahash_setkey); static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) @@ -169,44 +180,221 @@ static int ahash_nosetkey(struct crypto_ahash *tfm, const u8 *key, return -ENOSYS; } -int crypto_ahash_import(struct ahash_request *req, const u8 *in) +static inline unsigned int ahash_align_buffer_size(unsigned len, + unsigned long mask) +{ + return len + (mask & ~(crypto_tfm_ctx_alignment() - 1)); +} + +static void ahash_op_unaligned_finish(struct ahash_request *req, int err) +{ + struct ahash_request_priv *priv = req->priv; + + if (err == -EINPROGRESS) + return; + + if (!err) + memcpy(priv->result, req->result, + crypto_ahash_digestsize(crypto_ahash_reqtfm(req))); + + kzfree(priv); +} + +static void ahash_op_unaligned_done(struct crypto_async_request *req, int err) +{ + struct ahash_request *areq = req->data; + struct ahash_request_priv *priv = areq->priv; + crypto_completion_t complete = priv->complete; + void *data = priv->data; + + ahash_op_unaligned_finish(areq, err); + + complete(data, err); +} + +static int ahash_op_unaligned(struct ahash_request *req, + int (*op)(struct ahash_request *)) { struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); - struct ahash_alg *alg = crypto_ahash_alg(tfm); + unsigned long alignmask = crypto_ahash_alignmask(tfm); + unsigned int ds = crypto_ahash_digestsize(tfm); + struct ahash_request_priv *priv; + int err; + + priv = kmalloc(sizeof(*priv) + ahash_align_buffer_size(ds, alignmask), + (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC); + if (!priv) + return -ENOMEM; - memcpy(ahash_request_ctx(req), in, crypto_ahash_reqsize(tfm)); + priv->result = req->result; + priv->complete = req->base.complete; + priv->data = req->base.data; - if (alg->reinit) - alg->reinit(req); + req->result = PTR_ALIGN((u8 *)priv->ubuf, alignmask + 1); + req->base.complete = ahash_op_unaligned_done; + req->base.data = req; + req->priv = priv; - return 0; + err = op(req); + ahash_op_unaligned_finish(req, err); + + return err; } -EXPORT_SYMBOL_GPL(crypto_ahash_import); -static unsigned int crypto_ahash_ctxsize(struct crypto_alg *alg, u32 type, - u32 mask) +static int crypto_ahash_op(struct ahash_request *req, + int (*op)(struct ahash_request *)) { - return alg->cra_ctxsize; + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + unsigned long alignmask = crypto_ahash_alignmask(tfm); + + if ((unsigned long)req->result & alignmask) + return ahash_op_unaligned(req, op); + + return op(req); } -static int crypto_init_ahash_ops(struct crypto_tfm *tfm, u32 type, u32 mask) +int crypto_ahash_final(struct ahash_request *req) { - struct ahash_alg *alg = &tfm->__crt_alg->cra_ahash; - struct ahash_tfm *crt = &tfm->crt_ahash; + return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->final); +} +EXPORT_SYMBOL_GPL(crypto_ahash_final); - if (alg->digestsize > PAGE_SIZE / 8) - return -EINVAL; +int crypto_ahash_finup(struct ahash_request *req) +{ + return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->finup); +} +EXPORT_SYMBOL_GPL(crypto_ahash_finup); + +int crypto_ahash_digest(struct ahash_request *req) +{ + return crypto_ahash_op(req, crypto_ahash_reqtfm(req)->digest); +} +EXPORT_SYMBOL_GPL(crypto_ahash_digest); + +static void ahash_def_finup_finish2(struct ahash_request *req, int err) +{ + struct ahash_request_priv *priv = req->priv; + + if (err == -EINPROGRESS) + return; + + if (!err) + memcpy(priv->result, req->result, + crypto_ahash_digestsize(crypto_ahash_reqtfm(req))); - crt->init = alg->init; - crt->update = alg->update; - crt->final = alg->final; - crt->digest = alg->digest; - crt->setkey = alg->setkey ? ahash_setkey : ahash_nosetkey; - crt->digestsize = alg->digestsize; + kzfree(priv); +} + +static void ahash_def_finup_done2(struct crypto_async_request *req, int err) +{ + struct ahash_request *areq = req->data; + struct ahash_request_priv *priv = areq->priv; + crypto_completion_t complete = priv->complete; + void *data = priv->data; + + ahash_def_finup_finish2(areq, err); + + complete(data, err); +} + +static int ahash_def_finup_finish1(struct ahash_request *req, int err) +{ + if (err) + goto out; + + req->base.complete = ahash_def_finup_done2; + req->base.flags &= ~CRYPTO_TFM_REQ_MAY_SLEEP; + err = crypto_ahash_reqtfm(req)->final(req); + +out: + ahash_def_finup_finish2(req, err); + return err; +} + +static void ahash_def_finup_done1(struct crypto_async_request *req, int err) +{ + struct ahash_request *areq = req->data; + struct ahash_request_priv *priv = areq->priv; + crypto_completion_t complete = priv->complete; + void *data = priv->data; + + err = ahash_def_finup_finish1(areq, err); + + complete(data, err); +} + +static int ahash_def_finup(struct ahash_request *req) +{ + struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); + unsigned long alignmask = crypto_ahash_alignmask(tfm); + unsigned int ds = crypto_ahash_digestsize(tfm); + struct ahash_request_priv *priv; + + priv = kmalloc(sizeof(*priv) + ahash_align_buffer_size(ds, alignmask), + (req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP) ? + GFP_KERNEL : GFP_ATOMIC); + if (!priv) + return -ENOMEM; + + priv->result = req->result; + priv->complete = req->base.complete; + priv->data = req->base.data; + + req->result = PTR_ALIGN((u8 *)priv->ubuf, alignmask + 1); + req->base.complete = ahash_def_finup_done1; + req->base.data = req; + req->priv = priv; + + return ahash_def_finup_finish1(req, tfm->update(req)); +} + +static int ahash_no_export(struct ahash_request *req, void *out) +{ + return -ENOSYS; +} + +static int ahash_no_import(struct ahash_request *req, const void *in) +{ + return -ENOSYS; +} + +static int crypto_ahash_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_ahash *hash = __crypto_ahash_cast(tfm); + struct ahash_alg *alg = crypto_ahash_alg(hash); + + hash->setkey = ahash_nosetkey; + hash->export = ahash_no_export; + hash->import = ahash_no_import; + + if (tfm->__crt_alg->cra_type != &crypto_ahash_type) + return crypto_init_shash_ops_async(tfm); + + hash->init = alg->init; + hash->update = alg->update; + hash->final = alg->final; + hash->finup = alg->finup ?: ahash_def_finup; + hash->digest = alg->digest; + + if (alg->setkey) + hash->setkey = alg->setkey; + if (alg->export) + hash->export = alg->export; + if (alg->import) + hash->import = alg->import; return 0; } +static unsigned int crypto_ahash_extsize(struct crypto_alg *alg) +{ + if (alg->cra_type == &crypto_ahash_type) + return alg->cra_ctxsize; + + return sizeof(struct crypto_shash *); +} + static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) __attribute__ ((unused)); static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) @@ -215,17 +403,101 @@ static void crypto_ahash_show(struct seq_file *m, struct crypto_alg *alg) seq_printf(m, "async : %s\n", alg->cra_flags & CRYPTO_ALG_ASYNC ? "yes" : "no"); seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); - seq_printf(m, "digestsize : %u\n", alg->cra_ahash.digestsize); + seq_printf(m, "digestsize : %u\n", + __crypto_hash_alg_common(alg)->digestsize); } const struct crypto_type crypto_ahash_type = { - .ctxsize = crypto_ahash_ctxsize, - .init = crypto_init_ahash_ops, + .extsize = crypto_ahash_extsize, + .init_tfm = crypto_ahash_init_tfm, #ifdef CONFIG_PROC_FS .show = crypto_ahash_show, #endif + .maskclear = ~CRYPTO_ALG_TYPE_MASK, + .maskset = CRYPTO_ALG_TYPE_AHASH_MASK, + .type = CRYPTO_ALG_TYPE_AHASH, + .tfmsize = offsetof(struct crypto_ahash, base), }; EXPORT_SYMBOL_GPL(crypto_ahash_type); +struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type, + u32 mask) +{ + return crypto_alloc_tfm(alg_name, &crypto_ahash_type, type, mask); +} +EXPORT_SYMBOL_GPL(crypto_alloc_ahash); + +static int ahash_prepare_alg(struct ahash_alg *alg) +{ + struct crypto_alg *base = &alg->halg.base; + + if (alg->halg.digestsize > PAGE_SIZE / 8 || + alg->halg.statesize > PAGE_SIZE / 8) + return -EINVAL; + + base->cra_type = &crypto_ahash_type; + base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; + base->cra_flags |= CRYPTO_ALG_TYPE_AHASH; + + return 0; +} + +int crypto_register_ahash(struct ahash_alg *alg) +{ + struct crypto_alg *base = &alg->halg.base; + int err; + + err = ahash_prepare_alg(alg); + if (err) + return err; + + return crypto_register_alg(base); +} +EXPORT_SYMBOL_GPL(crypto_register_ahash); + +int crypto_unregister_ahash(struct ahash_alg *alg) +{ + return crypto_unregister_alg(&alg->halg.base); +} +EXPORT_SYMBOL_GPL(crypto_unregister_ahash); + +int ahash_register_instance(struct crypto_template *tmpl, + struct ahash_instance *inst) +{ + int err; + + err = ahash_prepare_alg(&inst->alg); + if (err) + return err; + + return crypto_register_instance(tmpl, ahash_crypto_instance(inst)); +} +EXPORT_SYMBOL_GPL(ahash_register_instance); + +void ahash_free_instance(struct crypto_instance *inst) +{ + crypto_drop_spawn(crypto_instance_ctx(inst)); + kfree(ahash_instance(inst)); +} +EXPORT_SYMBOL_GPL(ahash_free_instance); + +int crypto_init_ahash_spawn(struct crypto_ahash_spawn *spawn, + struct hash_alg_common *alg, + struct crypto_instance *inst) +{ + return crypto_init_spawn2(&spawn->base, &alg->base, inst, + &crypto_ahash_type); +} +EXPORT_SYMBOL_GPL(crypto_init_ahash_spawn); + +struct hash_alg_common *ahash_attr_alg(struct rtattr *rta, u32 type, u32 mask) +{ + struct crypto_alg *alg; + + alg = crypto_attr_alg2(rta, &crypto_ahash_type, type, mask); + return IS_ERR(alg) ? ERR_CAST(alg) : __crypto_hash_alg_common(alg); +} +EXPORT_SYMBOL_GPL(ahash_attr_alg); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Asynchronous cryptographic hash type"); diff --git a/crypto/algapi.c b/crypto/algapi.c index df0863d..f149b1c 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -81,16 +81,35 @@ static void crypto_destroy_instance(struct crypto_alg *alg) crypto_tmpl_put(tmpl); } +static struct list_head *crypto_more_spawns(struct crypto_alg *alg, + struct list_head *stack, + struct list_head *top, + struct list_head *secondary_spawns) +{ + struct crypto_spawn *spawn, *n; + + if (list_empty(stack)) + return NULL; + + spawn = list_first_entry(stack, struct crypto_spawn, list); + n = list_entry(spawn->list.next, struct crypto_spawn, list); + + if (spawn->alg && &n->list != stack && !n->alg) + n->alg = (n->list.next == stack) ? alg : + &list_entry(n->list.next, struct crypto_spawn, + list)->inst->alg; + + list_move(&spawn->list, secondary_spawns); + + return &n->list == stack ? top : &n->inst->alg.cra_users; +} + static void crypto_remove_spawn(struct crypto_spawn *spawn, - struct list_head *list, - struct list_head *secondary_spawns) + struct list_head *list) { struct crypto_instance *inst = spawn->inst; struct crypto_template *tmpl = inst->tmpl; - list_del_init(&spawn->list); - spawn->alg = NULL; - if (crypto_is_dead(&inst->alg)) return; @@ -106,25 +125,55 @@ static void crypto_remove_spawn(struct crypto_spawn *spawn, hlist_del(&inst->list); inst->alg.cra_destroy = crypto_destroy_instance; - list_splice(&inst->alg.cra_users, secondary_spawns); + BUG_ON(!list_empty(&inst->alg.cra_users)); } -static void crypto_remove_spawns(struct list_head *spawns, - struct list_head *list, u32 new_type) +static void crypto_remove_spawns(struct crypto_alg *alg, + struct list_head *list, + struct crypto_alg *nalg) { + u32 new_type = (nalg ?: alg)->cra_flags; struct crypto_spawn *spawn, *n; LIST_HEAD(secondary_spawns); + struct list_head *spawns; + LIST_HEAD(stack); + LIST_HEAD(top); + spawns = &alg->cra_users; list_for_each_entry_safe(spawn, n, spawns, list) { if ((spawn->alg->cra_flags ^ new_type) & spawn->mask) continue; - crypto_remove_spawn(spawn, list, &secondary_spawns); + list_move(&spawn->list, &top); } - while (!list_empty(&secondary_spawns)) { - list_for_each_entry_safe(spawn, n, &secondary_spawns, list) - crypto_remove_spawn(spawn, list, &secondary_spawns); + spawns = ⊤ + do { + while (!list_empty(spawns)) { + struct crypto_instance *inst; + + spawn = list_first_entry(spawns, struct crypto_spawn, + list); + inst = spawn->inst; + + BUG_ON(&inst->alg == alg); + + list_move(&spawn->list, &stack); + + if (&inst->alg == nalg) + break; + + spawn->alg = NULL; + spawns = &inst->alg.cra_users; + } + } while ((spawns = crypto_more_spawns(alg, &stack, &top, + &secondary_spawns))); + + list_for_each_entry_safe(spawn, n, &secondary_spawns, list) { + if (spawn->alg) + list_move(&spawn->list, &spawn->alg->cra_users); + else + crypto_remove_spawn(spawn, list); } } @@ -258,7 +307,7 @@ found: q->cra_priority > alg->cra_priority) continue; - crypto_remove_spawns(&q->cra_users, &list, alg->cra_flags); + crypto_remove_spawns(q, &list, alg); } complete: @@ -330,7 +379,7 @@ static int crypto_remove_alg(struct crypto_alg *alg, struct list_head *list) crypto_notify(CRYPTO_MSG_ALG_UNREGISTER, alg); list_del_init(&alg->cra_list); - crypto_remove_spawns(&alg->cra_users, list, alg->cra_flags); + crypto_remove_spawns(alg, list, NULL); return 0; } @@ -488,20 +537,38 @@ int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, } EXPORT_SYMBOL_GPL(crypto_init_spawn); +int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg, + struct crypto_instance *inst, + const struct crypto_type *frontend) +{ + int err = -EINVAL; + + if (frontend && (alg->cra_flags ^ frontend->type) & frontend->maskset) + goto out; + + spawn->frontend = frontend; + err = crypto_init_spawn(spawn, alg, inst, frontend->maskset); + +out: + return err; +} +EXPORT_SYMBOL_GPL(crypto_init_spawn2); + void crypto_drop_spawn(struct crypto_spawn *spawn) { + if (!spawn->alg) + return; + down_write(&crypto_alg_sem); list_del(&spawn->list); up_write(&crypto_alg_sem); } EXPORT_SYMBOL_GPL(crypto_drop_spawn); -struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, - u32 mask) +static struct crypto_alg *crypto_spawn_alg(struct crypto_spawn *spawn) { struct crypto_alg *alg; struct crypto_alg *alg2; - struct crypto_tfm *tfm; down_read(&crypto_alg_sem); alg = spawn->alg; @@ -516,6 +583,19 @@ struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, return ERR_PTR(-EAGAIN); } + return alg; +} + +struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, + u32 mask) +{ + struct crypto_alg *alg; + struct crypto_tfm *tfm; + + alg = crypto_spawn_alg(spawn); + if (IS_ERR(alg)) + return ERR_CAST(alg); + tfm = ERR_PTR(-EINVAL); if (unlikely((alg->cra_flags ^ type) & mask)) goto out_put_alg; @@ -532,6 +612,27 @@ out_put_alg: } EXPORT_SYMBOL_GPL(crypto_spawn_tfm); +void *crypto_spawn_tfm2(struct crypto_spawn *spawn) +{ + struct crypto_alg *alg; + struct crypto_tfm *tfm; + + alg = crypto_spawn_alg(spawn); + if (IS_ERR(alg)) + return ERR_CAST(alg); + + tfm = crypto_create_tfm(alg, spawn->frontend); + if (IS_ERR(tfm)) + goto out_put_alg; + + return tfm; + +out_put_alg: + crypto_mod_put(alg); + return tfm; +} +EXPORT_SYMBOL_GPL(crypto_spawn_tfm2); + int crypto_register_notifier(struct notifier_block *nb) { return blocking_notifier_chain_register(&crypto_chain, nb); @@ -595,7 +696,9 @@ const char *crypto_attr_alg_name(struct rtattr *rta) } EXPORT_SYMBOL_GPL(crypto_attr_alg_name); -struct crypto_alg *crypto_attr_alg(struct rtattr *rta, u32 type, u32 mask) +struct crypto_alg *crypto_attr_alg2(struct rtattr *rta, + const struct crypto_type *frontend, + u32 type, u32 mask) { const char *name; int err; @@ -605,9 +708,9 @@ struct crypto_alg *crypto_attr_alg(struct rtattr *rta, u32 type, u32 mask) if (IS_ERR(name)) return ERR_PTR(err); - return crypto_alg_mod_lookup(name, type, mask); + return crypto_find_alg(name, frontend, type, mask); } -EXPORT_SYMBOL_GPL(crypto_attr_alg); +EXPORT_SYMBOL_GPL(crypto_attr_alg2); int crypto_attr_u32(struct rtattr *rta, u32 *num) { @@ -627,17 +730,20 @@ int crypto_attr_u32(struct rtattr *rta, u32 *num) } EXPORT_SYMBOL_GPL(crypto_attr_u32); -struct crypto_instance *crypto_alloc_instance(const char *name, - struct crypto_alg *alg) +void *crypto_alloc_instance2(const char *name, struct crypto_alg *alg, + unsigned int head) { struct crypto_instance *inst; - struct crypto_spawn *spawn; + char *p; int err; - inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); - if (!inst) + p = kzalloc(head + sizeof(*inst) + sizeof(struct crypto_spawn), + GFP_KERNEL); + if (!p) return ERR_PTR(-ENOMEM); + inst = (void *)(p + head); + err = -ENAMETOOLONG; if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", name, alg->cra_name) >= CRYPTO_MAX_ALG_NAME) @@ -647,6 +753,25 @@ struct crypto_instance *crypto_alloc_instance(const char *name, name, alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_free_inst; + return p; + +err_free_inst: + kfree(p); + return ERR_PTR(err); +} +EXPORT_SYMBOL_GPL(crypto_alloc_instance2); + +struct crypto_instance *crypto_alloc_instance(const char *name, + struct crypto_alg *alg) +{ + struct crypto_instance *inst; + struct crypto_spawn *spawn; + int err; + + inst = crypto_alloc_instance2(name, alg, 0); + if (IS_ERR(inst)) + goto out; + spawn = crypto_instance_ctx(inst); err = crypto_init_spawn(spawn, alg, inst, CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); @@ -658,7 +783,10 @@ struct crypto_instance *crypto_alloc_instance(const char *name, err_free_inst: kfree(inst); - return ERR_PTR(err); + inst = ERR_PTR(err); + +out: + return inst; } EXPORT_SYMBOL_GPL(crypto_alloc_instance); diff --git a/crypto/algboss.c b/crypto/algboss.c index 9908dd8..412241c 100644 --- a/crypto/algboss.c +++ b/crypto/algboss.c @@ -68,6 +68,11 @@ static int cryptomgr_probe(void *data) goto err; do { + if (tmpl->create) { + err = tmpl->create(tmpl, param->tb); + continue; + } + inst = tmpl->alloc(param->tb); if (IS_ERR(inst)) err = PTR_ERR(inst); diff --git a/crypto/ansi_cprng.c b/crypto/ansi_cprng.c index d80ed4c..3aa6e38 100644 --- a/crypto/ansi_cprng.c +++ b/crypto/ansi_cprng.c @@ -187,7 +187,6 @@ static int _get_more_prng_bytes(struct prng_context *ctx) /* Our exported functions */ static int get_prng_bytes(char *buf, size_t nbytes, struct prng_context *ctx) { - unsigned long flags; unsigned char *ptr = buf; unsigned int byte_count = (unsigned int)nbytes; int err; @@ -196,7 +195,7 @@ static int get_prng_bytes(char *buf, size_t nbytes, struct prng_context *ctx) if (nbytes < 0) return -EINVAL; - spin_lock_irqsave(&ctx->prng_lock, flags); + spin_lock_bh(&ctx->prng_lock); err = -EINVAL; if (ctx->flags & PRNG_NEED_RESET) @@ -268,7 +267,7 @@ empty_rbuf: goto remainder; done: - spin_unlock_irqrestore(&ctx->prng_lock, flags); + spin_unlock_bh(&ctx->prng_lock); dbgprint(KERN_CRIT "returning %d from get_prng_bytes in context %p\n", err, ctx); return err; @@ -284,10 +283,9 @@ static int reset_prng_context(struct prng_context *ctx, unsigned char *V, unsigned char *DT) { int ret; - int rc = -EINVAL; unsigned char *prng_key; - spin_lock(&ctx->prng_lock); + spin_lock_bh(&ctx->prng_lock); ctx->flags |= PRNG_NEED_RESET; prng_key = (key != NULL) ? key : (unsigned char *)DEFAULT_PRNG_KEY; @@ -308,34 +306,20 @@ static int reset_prng_context(struct prng_context *ctx, memset(ctx->rand_data, 0, DEFAULT_BLK_SZ); memset(ctx->last_rand_data, 0, DEFAULT_BLK_SZ); - if (ctx->tfm) - crypto_free_cipher(ctx->tfm); - - ctx->tfm = crypto_alloc_cipher("aes", 0, 0); - if (IS_ERR(ctx->tfm)) { - dbgprint(KERN_CRIT "Failed to alloc tfm for context %p\n", - ctx); - ctx->tfm = NULL; - goto out; - } - ctx->rand_data_valid = DEFAULT_BLK_SZ; ret = crypto_cipher_setkey(ctx->tfm, prng_key, klen); if (ret) { dbgprint(KERN_CRIT "PRNG: setkey() failed flags=%x\n", crypto_cipher_get_flags(ctx->tfm)); - crypto_free_cipher(ctx->tfm); goto out; } - rc = 0; + ret = 0; ctx->flags &= ~PRNG_NEED_RESET; out: - spin_unlock(&ctx->prng_lock); - - return rc; - + spin_unlock_bh(&ctx->prng_lock); + return ret; } static int cprng_init(struct crypto_tfm *tfm) @@ -343,6 +327,12 @@ static int cprng_init(struct crypto_tfm *tfm) struct prng_context *ctx = crypto_tfm_ctx(tfm); spin_lock_init(&ctx->prng_lock); + ctx->tfm = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(ctx->tfm)) { + dbgprint(KERN_CRIT "Failed to alloc tfm for context %p\n", + ctx); + return PTR_ERR(ctx->tfm); + } if (reset_prng_context(ctx, NULL, DEFAULT_PRNG_KSZ, NULL, NULL) < 0) return -EINVAL; @@ -418,17 +408,10 @@ static struct crypto_alg rng_alg = { /* Module initalization */ static int __init prng_mod_init(void) { - int ret = 0; - if (fips_enabled) rng_alg.cra_priority += 200; - ret = crypto_register_alg(&rng_alg); - - if (ret) - goto out; -out: - return 0; + return crypto_register_alg(&rng_alg); } static void __exit prng_mod_fini(void) diff --git a/crypto/api.c b/crypto/api.c index d5944f9..798526d 100644 --- a/crypto/api.c +++ b/crypto/api.c @@ -285,13 +285,6 @@ static int crypto_init_ops(struct crypto_tfm *tfm, u32 type, u32 mask) switch (crypto_tfm_alg_type(tfm)) { case CRYPTO_ALG_TYPE_CIPHER: return crypto_init_cipher_ops(tfm); - - case CRYPTO_ALG_TYPE_DIGEST: - if ((mask & CRYPTO_ALG_TYPE_HASH_MASK) != - CRYPTO_ALG_TYPE_HASH_MASK) - return crypto_init_digest_ops_async(tfm); - else - return crypto_init_digest_ops(tfm); case CRYPTO_ALG_TYPE_COMPRESS: return crypto_init_compress_ops(tfm); @@ -318,11 +311,7 @@ static void crypto_exit_ops(struct crypto_tfm *tfm) case CRYPTO_ALG_TYPE_CIPHER: crypto_exit_cipher_ops(tfm); break; - - case CRYPTO_ALG_TYPE_DIGEST: - crypto_exit_digest_ops(tfm); - break; - + case CRYPTO_ALG_TYPE_COMPRESS: crypto_exit_compress_ops(tfm); break; @@ -349,11 +338,7 @@ static unsigned int crypto_ctxsize(struct crypto_alg *alg, u32 type, u32 mask) case CRYPTO_ALG_TYPE_CIPHER: len += crypto_cipher_ctxsize(alg); break; - - case CRYPTO_ALG_TYPE_DIGEST: - len += crypto_digest_ctxsize(alg); - break; - + case CRYPTO_ALG_TYPE_COMPRESS: len += crypto_compress_ctxsize(alg); break; @@ -472,7 +457,7 @@ void *crypto_create_tfm(struct crypto_alg *alg, int err = -ENOMEM; tfmsize = frontend->tfmsize; - total = tfmsize + sizeof(*tfm) + frontend->extsize(alg, frontend); + total = tfmsize + sizeof(*tfm) + frontend->extsize(alg); mem = kzalloc(total, GFP_KERNEL); if (mem == NULL) @@ -481,7 +466,7 @@ void *crypto_create_tfm(struct crypto_alg *alg, tfm = (struct crypto_tfm *)(mem + tfmsize); tfm->__crt_alg = alg; - err = frontend->init_tfm(tfm, frontend); + err = frontend->init_tfm(tfm); if (err) goto out_free_tfm; @@ -503,6 +488,27 @@ out: } EXPORT_SYMBOL_GPL(crypto_create_tfm); +struct crypto_alg *crypto_find_alg(const char *alg_name, + const struct crypto_type *frontend, + u32 type, u32 mask) +{ + struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask) = + crypto_alg_mod_lookup; + + if (frontend) { + type &= frontend->maskclear; + mask &= frontend->maskclear; + type |= frontend->type; + mask |= frontend->maskset; + + if (frontend->lookup) + lookup = frontend->lookup; + } + + return lookup(alg_name, type, mask); +} +EXPORT_SYMBOL_GPL(crypto_find_alg); + /* * crypto_alloc_tfm - Locate algorithm and allocate transform * @alg_name: Name of algorithm @@ -526,21 +532,13 @@ EXPORT_SYMBOL_GPL(crypto_create_tfm); void *crypto_alloc_tfm(const char *alg_name, const struct crypto_type *frontend, u32 type, u32 mask) { - struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask); void *tfm; int err; - type &= frontend->maskclear; - mask &= frontend->maskclear; - type |= frontend->type; - mask |= frontend->maskset; - - lookup = frontend->lookup ?: crypto_alg_mod_lookup; - for (;;) { struct crypto_alg *alg; - alg = lookup(alg_name, type, mask); + alg = crypto_find_alg(alg_name, frontend, type, mask); if (IS_ERR(alg)) { err = PTR_ERR(alg); goto err; diff --git a/crypto/authenc.c b/crypto/authenc.c index 5793b64..4d6f49a 100644 --- a/crypto/authenc.c +++ b/crypto/authenc.c @@ -23,24 +23,36 @@ #include <linux/slab.h> #include <linux/spinlock.h> +typedef u8 *(*authenc_ahash_t)(struct aead_request *req, unsigned int flags); + struct authenc_instance_ctx { - struct crypto_spawn auth; + struct crypto_ahash_spawn auth; struct crypto_skcipher_spawn enc; }; struct crypto_authenc_ctx { - spinlock_t auth_lock; - struct crypto_hash *auth; + unsigned int reqoff; + struct crypto_ahash *auth; struct crypto_ablkcipher *enc; }; +struct authenc_request_ctx { + unsigned int cryptlen; + struct scatterlist *sg; + struct scatterlist asg[2]; + struct scatterlist cipher[2]; + crypto_completion_t complete; + crypto_completion_t update_complete; + char tail[]; +}; + static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key, unsigned int keylen) { unsigned int authkeylen; unsigned int enckeylen; struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct crypto_hash *auth = ctx->auth; + struct crypto_ahash *auth = ctx->auth; struct crypto_ablkcipher *enc = ctx->enc; struct rtattr *rta = (void *)key; struct crypto_authenc_key_param *param; @@ -64,11 +76,11 @@ static int crypto_authenc_setkey(struct crypto_aead *authenc, const u8 *key, authkeylen = keylen - enckeylen; - crypto_hash_clear_flags(auth, CRYPTO_TFM_REQ_MASK); - crypto_hash_set_flags(auth, crypto_aead_get_flags(authenc) & + crypto_ahash_clear_flags(auth, CRYPTO_TFM_REQ_MASK); + crypto_ahash_set_flags(auth, crypto_aead_get_flags(authenc) & CRYPTO_TFM_REQ_MASK); - err = crypto_hash_setkey(auth, key, authkeylen); - crypto_aead_set_flags(authenc, crypto_hash_get_flags(auth) & + err = crypto_ahash_setkey(auth, key, authkeylen); + crypto_aead_set_flags(authenc, crypto_ahash_get_flags(auth) & CRYPTO_TFM_RES_MASK); if (err) @@ -103,40 +115,198 @@ static void authenc_chain(struct scatterlist *head, struct scatterlist *sg, sg_mark_end(head); } -static u8 *crypto_authenc_hash(struct aead_request *req, unsigned int flags, - struct scatterlist *cipher, - unsigned int cryptlen) +static void authenc_geniv_ahash_update_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + + if (err) + goto out; + + ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result, + areq_ctx->cryptlen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & + CRYPTO_TFM_REQ_MAY_SLEEP, + areq_ctx->complete, req); + + err = crypto_ahash_finup(ahreq); + if (err) + goto out; + + scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg, + areq_ctx->cryptlen, + crypto_aead_authsize(authenc), 1); + +out: + aead_request_complete(req, err); +} + +static void authenc_geniv_ahash_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + + if (err) + goto out; + + scatterwalk_map_and_copy(ahreq->result, areq_ctx->sg, + areq_ctx->cryptlen, + crypto_aead_authsize(authenc), 1); + +out: + aead_request_complete(req, err); +} + +static void authenc_verify_ahash_update_done(struct crypto_async_request *areq, + int err) { + u8 *ihash; + unsigned int authsize; + struct ablkcipher_request *abreq; + struct aead_request *req = areq->data; struct crypto_aead *authenc = crypto_aead_reqtfm(req); struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); - struct crypto_hash *auth = ctx->auth; - struct hash_desc desc = { - .tfm = auth, - .flags = aead_request_flags(req) & flags, - }; - u8 *hash = aead_request_ctx(req); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + + if (err) + goto out; + + ahash_request_set_crypt(ahreq, areq_ctx->sg, ahreq->result, + areq_ctx->cryptlen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & + CRYPTO_TFM_REQ_MAY_SLEEP, + areq_ctx->complete, req); + + err = crypto_ahash_finup(ahreq); + if (err) + goto out; + + authsize = crypto_aead_authsize(authenc); + ihash = ahreq->result + authsize; + scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, + authsize, 0); + + err = memcmp(ihash, ahreq->result, authsize) ? -EBADMSG: 0; + if (err) + goto out; + + abreq = aead_request_ctx(req); + ablkcipher_request_set_tfm(abreq, ctx->enc); + ablkcipher_request_set_callback(abreq, aead_request_flags(req), + req->base.complete, req->base.data); + ablkcipher_request_set_crypt(abreq, req->src, req->dst, + req->cryptlen, req->iv); + + err = crypto_ablkcipher_decrypt(abreq); + +out: + aead_request_complete(req, err); +} + +static void authenc_verify_ahash_done(struct crypto_async_request *areq, + int err) +{ + u8 *ihash; + unsigned int authsize; + struct ablkcipher_request *abreq; + struct aead_request *req = areq->data; + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + + if (err) + goto out; + + authsize = crypto_aead_authsize(authenc); + ihash = ahreq->result + authsize; + scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, + authsize, 0); + + err = memcmp(ihash, ahreq->result, authsize) ? -EBADMSG: 0; + if (err) + goto out; + + abreq = aead_request_ctx(req); + ablkcipher_request_set_tfm(abreq, ctx->enc); + ablkcipher_request_set_callback(abreq, aead_request_flags(req), + req->base.complete, req->base.data); + ablkcipher_request_set_crypt(abreq, req->src, req->dst, + req->cryptlen, req->iv); + + err = crypto_ablkcipher_decrypt(abreq); + +out: + aead_request_complete(req, err); +} + +static u8 *crypto_authenc_ahash_fb(struct aead_request *req, unsigned int flags) +{ + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct crypto_ahash *auth = ctx->auth; + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + u8 *hash = areq_ctx->tail; int err; - hash = (u8 *)ALIGN((unsigned long)hash + crypto_hash_alignmask(auth), - crypto_hash_alignmask(auth) + 1); + hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth), + crypto_ahash_alignmask(auth) + 1); + + ahash_request_set_tfm(ahreq, auth); - spin_lock_bh(&ctx->auth_lock); - err = crypto_hash_init(&desc); + err = crypto_ahash_init(ahreq); if (err) - goto auth_unlock; + return ERR_PTR(err); + + ahash_request_set_crypt(ahreq, req->assoc, hash, req->assoclen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, + areq_ctx->update_complete, req); - err = crypto_hash_update(&desc, req->assoc, req->assoclen); + err = crypto_ahash_update(ahreq); if (err) - goto auth_unlock; + return ERR_PTR(err); + + ahash_request_set_crypt(ahreq, areq_ctx->sg, hash, + areq_ctx->cryptlen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, + areq_ctx->complete, req); - err = crypto_hash_update(&desc, cipher, cryptlen); + err = crypto_ahash_finup(ahreq); if (err) - goto auth_unlock; + return ERR_PTR(err); - err = crypto_hash_final(&desc, hash); -auth_unlock: - spin_unlock_bh(&ctx->auth_lock); + return hash; +} + +static u8 *crypto_authenc_ahash(struct aead_request *req, unsigned int flags) +{ + struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct crypto_authenc_ctx *ctx = crypto_aead_ctx(authenc); + struct crypto_ahash *auth = ctx->auth; + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); + struct ahash_request *ahreq = (void *)(areq_ctx->tail + ctx->reqoff); + u8 *hash = areq_ctx->tail; + int err; + hash = (u8 *)ALIGN((unsigned long)hash + crypto_ahash_alignmask(auth), + crypto_ahash_alignmask(auth) + 1); + + ahash_request_set_tfm(ahreq, auth); + ahash_request_set_crypt(ahreq, areq_ctx->sg, hash, + areq_ctx->cryptlen); + ahash_request_set_callback(ahreq, aead_request_flags(req) & flags, + areq_ctx->complete, req); + + err = crypto_ahash_digest(ahreq); if (err) return ERR_PTR(err); @@ -147,11 +317,15 @@ static int crypto_authenc_genicv(struct aead_request *req, u8 *iv, unsigned int flags) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); struct scatterlist *dst = req->dst; - struct scatterlist cipher[2]; - struct page *dstp; + struct scatterlist *assoc = req->assoc; + struct scatterlist *cipher = areq_ctx->cipher; + struct scatterlist *asg = areq_ctx->asg; unsigned int ivsize = crypto_aead_ivsize(authenc); - unsigned int cryptlen; + unsigned int cryptlen = req->cryptlen; + authenc_ahash_t authenc_ahash_fn = crypto_authenc_ahash_fb; + struct page *dstp; u8 *vdst; u8 *hash; @@ -163,10 +337,25 @@ static int crypto_authenc_genicv(struct aead_request *req, u8 *iv, sg_set_buf(cipher, iv, ivsize); authenc_chain(cipher, dst, vdst == iv + ivsize); dst = cipher; + cryptlen += ivsize; } - cryptlen = req->cryptlen + ivsize; - hash = crypto_authenc_hash(req, flags, dst, cryptlen); + if (sg_is_last(assoc)) { + authenc_ahash_fn = crypto_authenc_ahash; + sg_init_table(asg, 2); + sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset); + authenc_chain(asg, dst, 0); + dst = asg; + cryptlen += req->assoclen; + } + + areq_ctx->cryptlen = cryptlen; + areq_ctx->sg = dst; + + areq_ctx->complete = authenc_geniv_ahash_done; + areq_ctx->update_complete = authenc_geniv_ahash_update_done; + + hash = authenc_ahash_fn(req, flags); if (IS_ERR(hash)) return PTR_ERR(hash); @@ -256,22 +445,25 @@ static int crypto_authenc_givencrypt(struct aead_givcrypt_request *req) } static int crypto_authenc_verify(struct aead_request *req, - struct scatterlist *cipher, - unsigned int cryptlen) + authenc_ahash_t authenc_ahash_fn) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); u8 *ohash; u8 *ihash; unsigned int authsize; - ohash = crypto_authenc_hash(req, CRYPTO_TFM_REQ_MAY_SLEEP, cipher, - cryptlen); + areq_ctx->complete = authenc_verify_ahash_done; + areq_ctx->complete = authenc_verify_ahash_update_done; + + ohash = authenc_ahash_fn(req, CRYPTO_TFM_REQ_MAY_SLEEP); if (IS_ERR(ohash)) return PTR_ERR(ohash); authsize = crypto_aead_authsize(authenc); ihash = ohash + authsize; - scatterwalk_map_and_copy(ihash, cipher, cryptlen, authsize, 0); + scatterwalk_map_and_copy(ihash, areq_ctx->sg, areq_ctx->cryptlen, + authsize, 0); return memcmp(ihash, ohash, authsize) ? -EBADMSG: 0; } @@ -279,10 +471,14 @@ static int crypto_authenc_iverify(struct aead_request *req, u8 *iv, unsigned int cryptlen) { struct crypto_aead *authenc = crypto_aead_reqtfm(req); + struct authenc_request_ctx *areq_ctx = aead_request_ctx(req); struct scatterlist *src = req->src; - struct scatterlist cipher[2]; - struct page *srcp; + struct scatterlist *assoc = req->assoc; + struct scatterlist *cipher = areq_ctx->cipher; + struct scatterlist *asg = areq_ctx->asg; unsigned int ivsize = crypto_aead_ivsize(authenc); + authenc_ahash_t authenc_ahash_fn = crypto_authenc_ahash_fb; + struct page *srcp; u8 *vsrc; srcp = sg_page(src); @@ -293,9 +489,22 @@ static int crypto_authenc_iverify(struct aead_request *req, u8 *iv, sg_set_buf(cipher, iv, ivsize); authenc_chain(cipher, src, vsrc == iv + ivsize); src = cipher; + cryptlen += ivsize; + } + + if (sg_is_last(assoc)) { + authenc_ahash_fn = crypto_authenc_ahash; + sg_init_table(asg, 2); + sg_set_page(asg, sg_page(assoc), assoc->length, assoc->offset); + authenc_chain(asg, src, 0); + src = asg; + cryptlen += req->assoclen; } - return crypto_authenc_verify(req, src, cryptlen + ivsize); + areq_ctx->cryptlen = cryptlen; + areq_ctx->sg = src; + + return crypto_authenc_verify(req, authenc_ahash_fn); } static int crypto_authenc_decrypt(struct aead_request *req) @@ -326,38 +535,41 @@ static int crypto_authenc_decrypt(struct aead_request *req) static int crypto_authenc_init_tfm(struct crypto_tfm *tfm) { - struct crypto_instance *inst = (void *)tfm->__crt_alg; + struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); struct authenc_instance_ctx *ictx = crypto_instance_ctx(inst); struct crypto_authenc_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_hash *auth; + struct crypto_ahash *auth; struct crypto_ablkcipher *enc; int err; - auth = crypto_spawn_hash(&ictx->auth); + auth = crypto_spawn_ahash(&ictx->auth); if (IS_ERR(auth)) return PTR_ERR(auth); + ctx->reqoff = ALIGN(2 * crypto_ahash_digestsize(auth) + + crypto_ahash_alignmask(auth), + crypto_ahash_alignmask(auth) + 1); + enc = crypto_spawn_skcipher(&ictx->enc); err = PTR_ERR(enc); if (IS_ERR(enc)) - goto err_free_hash; + goto err_free_ahash; ctx->auth = auth; ctx->enc = enc; + tfm->crt_aead.reqsize = max_t(unsigned int, - (crypto_hash_alignmask(auth) & - ~(crypto_tfm_ctx_alignment() - 1)) + - crypto_hash_digestsize(auth) * 2, - sizeof(struct skcipher_givcrypt_request) + - crypto_ablkcipher_reqsize(enc) + - crypto_ablkcipher_ivsize(enc)); - - spin_lock_init(&ctx->auth_lock); + crypto_ahash_reqsize(auth) + ctx->reqoff + + sizeof(struct authenc_request_ctx) + + sizeof(struct ahash_request), + sizeof(struct skcipher_givcrypt_request) + + crypto_ablkcipher_reqsize(enc) + + crypto_ablkcipher_ivsize(enc)); return 0; -err_free_hash: - crypto_free_hash(auth); +err_free_ahash: + crypto_free_ahash(auth); return err; } @@ -365,7 +577,7 @@ static void crypto_authenc_exit_tfm(struct crypto_tfm *tfm) { struct crypto_authenc_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_hash(ctx->auth); + crypto_free_ahash(ctx->auth); crypto_free_ablkcipher(ctx->enc); } @@ -373,7 +585,8 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) { struct crypto_attr_type *algt; struct crypto_instance *inst; - struct crypto_alg *auth; + struct hash_alg_common *auth; + struct crypto_alg *auth_base; struct crypto_alg *enc; struct authenc_instance_ctx *ctx; const char *enc_name; @@ -387,11 +600,13 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) return ERR_PTR(-EINVAL); - auth = crypto_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_HASH_MASK); + auth = ahash_attr_alg(tb[1], CRYPTO_ALG_TYPE_HASH, + CRYPTO_ALG_TYPE_AHASH_MASK); if (IS_ERR(auth)) return ERR_PTR(PTR_ERR(auth)); + auth_base = &auth->base; + enc_name = crypto_attr_alg_name(tb[2]); err = PTR_ERR(enc_name); if (IS_ERR(enc_name)) @@ -404,7 +619,7 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) ctx = crypto_instance_ctx(inst); - err = crypto_init_spawn(&ctx->auth, auth, inst, CRYPTO_ALG_TYPE_MASK); + err = crypto_init_ahash_spawn(&ctx->auth, auth, inst); if (err) goto err_free_inst; @@ -419,28 +634,25 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) err = -ENAMETOOLONG; if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME, - "authenc(%s,%s)", auth->cra_name, enc->cra_name) >= + "authenc(%s,%s)", auth_base->cra_name, enc->cra_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "authenc(%s,%s)", auth->cra_driver_name, + "authenc(%s,%s)", auth_base->cra_driver_name, enc->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto err_drop_enc; inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD; inst->alg.cra_flags |= enc->cra_flags & CRYPTO_ALG_ASYNC; - inst->alg.cra_priority = enc->cra_priority * 10 + auth->cra_priority; + inst->alg.cra_priority = enc->cra_priority * + 10 + auth_base->cra_priority; inst->alg.cra_blocksize = enc->cra_blocksize; - inst->alg.cra_alignmask = auth->cra_alignmask | enc->cra_alignmask; + inst->alg.cra_alignmask = auth_base->cra_alignmask | enc->cra_alignmask; inst->alg.cra_type = &crypto_aead_type; inst->alg.cra_aead.ivsize = enc->cra_ablkcipher.ivsize; - inst->alg.cra_aead.maxauthsize = auth->cra_type == &crypto_hash_type ? - auth->cra_hash.digestsize : - auth->cra_type ? - __crypto_shash_alg(auth)->digestsize : - auth->cra_digest.dia_digestsize; + inst->alg.cra_aead.maxauthsize = auth->digestsize; inst->alg.cra_ctxsize = sizeof(struct crypto_authenc_ctx); @@ -453,13 +665,13 @@ static struct crypto_instance *crypto_authenc_alloc(struct rtattr **tb) inst->alg.cra_aead.givencrypt = crypto_authenc_givencrypt; out: - crypto_mod_put(auth); + crypto_mod_put(auth_base); return inst; err_drop_enc: crypto_drop_skcipher(&ctx->enc); err_drop_auth: - crypto_drop_spawn(&ctx->auth); + crypto_drop_ahash(&ctx->auth); err_free_inst: kfree(inst); out_put_auth: @@ -472,7 +684,7 @@ static void crypto_authenc_free(struct crypto_instance *inst) struct authenc_instance_ctx *ctx = crypto_instance_ctx(inst); crypto_drop_skcipher(&ctx->enc); - crypto_drop_spawn(&ctx->auth); + crypto_drop_ahash(&ctx->auth); kfree(inst); } diff --git a/crypto/cryptd.c b/crypto/cryptd.c index ae5fa99..3533582 100644 --- a/crypto/cryptd.c +++ b/crypto/cryptd.c @@ -39,6 +39,11 @@ struct cryptd_instance_ctx { struct cryptd_queue *queue; }; +struct hashd_instance_ctx { + struct crypto_shash_spawn spawn; + struct cryptd_queue *queue; +}; + struct cryptd_blkcipher_ctx { struct crypto_blkcipher *child; }; @@ -48,11 +53,12 @@ struct cryptd_blkcipher_request_ctx { }; struct cryptd_hash_ctx { - struct crypto_hash *child; + struct crypto_shash *child; }; struct cryptd_hash_request_ctx { crypto_completion_t complete; + struct shash_desc desc; }; static void cryptd_queue_worker(struct work_struct *work); @@ -249,32 +255,24 @@ static void cryptd_blkcipher_exit_tfm(struct crypto_tfm *tfm) crypto_free_blkcipher(ctx->child); } -static struct crypto_instance *cryptd_alloc_instance(struct crypto_alg *alg, - struct cryptd_queue *queue) +static void *cryptd_alloc_instance(struct crypto_alg *alg, unsigned int head, + unsigned int tail) { + char *p; struct crypto_instance *inst; - struct cryptd_instance_ctx *ctx; int err; - inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); - if (!inst) { - inst = ERR_PTR(-ENOMEM); - goto out; - } + p = kzalloc(head + sizeof(*inst) + tail, GFP_KERNEL); + if (!p) + return ERR_PTR(-ENOMEM); + + inst = (void *)(p + head); err = -ENAMETOOLONG; if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, "cryptd(%s)", alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_free_inst; - ctx = crypto_instance_ctx(inst); - err = crypto_init_spawn(&ctx->spawn, alg, inst, - CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); - if (err) - goto out_free_inst; - - ctx->queue = queue; - memcpy(inst->alg.cra_name, alg->cra_name, CRYPTO_MAX_ALG_NAME); inst->alg.cra_priority = alg->cra_priority + 50; @@ -282,29 +280,41 @@ static struct crypto_instance *cryptd_alloc_instance(struct crypto_alg *alg, inst->alg.cra_alignmask = alg->cra_alignmask; out: - return inst; + return p; out_free_inst: - kfree(inst); - inst = ERR_PTR(err); + kfree(p); + p = ERR_PTR(err); goto out; } -static struct crypto_instance *cryptd_alloc_blkcipher( - struct rtattr **tb, struct cryptd_queue *queue) +static int cryptd_create_blkcipher(struct crypto_template *tmpl, + struct rtattr **tb, + struct cryptd_queue *queue) { + struct cryptd_instance_ctx *ctx; struct crypto_instance *inst; struct crypto_alg *alg; + int err; alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_BLKCIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) - return ERR_CAST(alg); + return PTR_ERR(alg); - inst = cryptd_alloc_instance(alg, queue); + inst = cryptd_alloc_instance(alg, 0, sizeof(*ctx)); + err = PTR_ERR(inst); if (IS_ERR(inst)) goto out_put_alg; + ctx = crypto_instance_ctx(inst); + ctx->queue = queue; + + err = crypto_init_spawn(&ctx->spawn, alg, inst, + CRYPTO_ALG_TYPE_MASK | CRYPTO_ALG_ASYNC); + if (err) + goto out_free_inst; + inst->alg.cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; inst->alg.cra_type = &crypto_ablkcipher_type; @@ -323,26 +333,34 @@ static struct crypto_instance *cryptd_alloc_blkcipher( inst->alg.cra_ablkcipher.encrypt = cryptd_blkcipher_encrypt_enqueue; inst->alg.cra_ablkcipher.decrypt = cryptd_blkcipher_decrypt_enqueue; + err = crypto_register_instance(tmpl, inst); + if (err) { + crypto_drop_spawn(&ctx->spawn); +out_free_inst: + kfree(inst); + } + out_put_alg: crypto_mod_put(alg); - return inst; + return err; } static int cryptd_hash_init_tfm(struct crypto_tfm *tfm) { struct crypto_instance *inst = crypto_tfm_alg_instance(tfm); - struct cryptd_instance_ctx *ictx = crypto_instance_ctx(inst); - struct crypto_spawn *spawn = &ictx->spawn; + struct hashd_instance_ctx *ictx = crypto_instance_ctx(inst); + struct crypto_shash_spawn *spawn = &ictx->spawn; struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); - struct crypto_hash *cipher; + struct crypto_shash *hash; - cipher = crypto_spawn_hash(spawn); - if (IS_ERR(cipher)) - return PTR_ERR(cipher); + hash = crypto_spawn_shash(spawn); + if (IS_ERR(hash)) + return PTR_ERR(hash); - ctx->child = cipher; - tfm->crt_ahash.reqsize = - sizeof(struct cryptd_hash_request_ctx); + ctx->child = hash; + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct cryptd_hash_request_ctx) + + crypto_shash_descsize(hash)); return 0; } @@ -350,22 +368,22 @@ static void cryptd_hash_exit_tfm(struct crypto_tfm *tfm) { struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_hash(ctx->child); + crypto_free_shash(ctx->child); } static int cryptd_hash_setkey(struct crypto_ahash *parent, const u8 *key, unsigned int keylen) { struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(parent); - struct crypto_hash *child = ctx->child; + struct crypto_shash *child = ctx->child; int err; - crypto_hash_clear_flags(child, CRYPTO_TFM_REQ_MASK); - crypto_hash_set_flags(child, crypto_ahash_get_flags(parent) & - CRYPTO_TFM_REQ_MASK); - err = crypto_hash_setkey(child, key, keylen); - crypto_ahash_set_flags(parent, crypto_hash_get_flags(child) & - CRYPTO_TFM_RES_MASK); + crypto_shash_clear_flags(child, CRYPTO_TFM_REQ_MASK); + crypto_shash_set_flags(child, crypto_ahash_get_flags(parent) & + CRYPTO_TFM_REQ_MASK); + err = crypto_shash_setkey(child, key, keylen); + crypto_ahash_set_flags(parent, crypto_shash_get_flags(child) & + CRYPTO_TFM_RES_MASK); return err; } @@ -385,21 +403,19 @@ static int cryptd_hash_enqueue(struct ahash_request *req, static void cryptd_hash_init(struct crypto_async_request *req_async, int err) { - struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_hash *child = ctx->child; - struct ahash_request *req = ahash_request_cast(req_async); - struct cryptd_hash_request_ctx *rctx; - struct hash_desc desc; - - rctx = ahash_request_ctx(req); + struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); + struct crypto_shash *child = ctx->child; + struct ahash_request *req = ahash_request_cast(req_async); + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + struct shash_desc *desc = &rctx->desc; if (unlikely(err == -EINPROGRESS)) goto out; - desc.tfm = child; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; + desc->tfm = child; + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - err = crypto_hash_crt(child)->init(&desc); + err = crypto_shash_init(desc); req->base.complete = rctx->complete; @@ -416,23 +432,15 @@ static int cryptd_hash_init_enqueue(struct ahash_request *req) static void cryptd_hash_update(struct crypto_async_request *req_async, int err) { - struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_hash *child = ctx->child; - struct ahash_request *req = ahash_request_cast(req_async); + struct ahash_request *req = ahash_request_cast(req_async); struct cryptd_hash_request_ctx *rctx; - struct hash_desc desc; rctx = ahash_request_ctx(req); if (unlikely(err == -EINPROGRESS)) goto out; - desc.tfm = child; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; - - err = crypto_hash_crt(child)->update(&desc, - req->src, - req->nbytes); + err = shash_ahash_update(req, &rctx->desc); req->base.complete = rctx->complete; @@ -449,21 +457,13 @@ static int cryptd_hash_update_enqueue(struct ahash_request *req) static void cryptd_hash_final(struct crypto_async_request *req_async, int err) { - struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_hash *child = ctx->child; - struct ahash_request *req = ahash_request_cast(req_async); - struct cryptd_hash_request_ctx *rctx; - struct hash_desc desc; - - rctx = ahash_request_ctx(req); + struct ahash_request *req = ahash_request_cast(req_async); + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); if (unlikely(err == -EINPROGRESS)) goto out; - desc.tfm = child; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; - - err = crypto_hash_crt(child)->final(&desc, req->result); + err = crypto_shash_final(&rctx->desc, req->result); req->base.complete = rctx->complete; @@ -478,26 +478,44 @@ static int cryptd_hash_final_enqueue(struct ahash_request *req) return cryptd_hash_enqueue(req, cryptd_hash_final); } -static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) +static void cryptd_hash_finup(struct crypto_async_request *req_async, int err) { - struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); - struct crypto_hash *child = ctx->child; - struct ahash_request *req = ahash_request_cast(req_async); - struct cryptd_hash_request_ctx *rctx; - struct hash_desc desc; + struct ahash_request *req = ahash_request_cast(req_async); + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); - rctx = ahash_request_ctx(req); + if (unlikely(err == -EINPROGRESS)) + goto out; + + err = shash_ahash_finup(req, &rctx->desc); + + req->base.complete = rctx->complete; + +out: + local_bh_disable(); + rctx->complete(&req->base, err); + local_bh_enable(); +} + +static int cryptd_hash_finup_enqueue(struct ahash_request *req) +{ + return cryptd_hash_enqueue(req, cryptd_hash_finup); +} + +static void cryptd_hash_digest(struct crypto_async_request *req_async, int err) +{ + struct cryptd_hash_ctx *ctx = crypto_tfm_ctx(req_async->tfm); + struct crypto_shash *child = ctx->child; + struct ahash_request *req = ahash_request_cast(req_async); + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + struct shash_desc *desc = &rctx->desc; if (unlikely(err == -EINPROGRESS)) goto out; - desc.tfm = child; - desc.flags = CRYPTO_TFM_REQ_MAY_SLEEP; + desc->tfm = child; + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - err = crypto_hash_crt(child)->digest(&desc, - req->src, - req->nbytes, - req->result); + err = shash_ahash_digest(req, desc); req->base.complete = rctx->complete; @@ -512,64 +530,108 @@ static int cryptd_hash_digest_enqueue(struct ahash_request *req) return cryptd_hash_enqueue(req, cryptd_hash_digest); } -static struct crypto_instance *cryptd_alloc_hash( - struct rtattr **tb, struct cryptd_queue *queue) +static int cryptd_hash_export(struct ahash_request *req, void *out) { - struct crypto_instance *inst; + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + + return crypto_shash_export(&rctx->desc, out); +} + +static int cryptd_hash_import(struct ahash_request *req, const void *in) +{ + struct cryptd_hash_request_ctx *rctx = ahash_request_ctx(req); + + return crypto_shash_import(&rctx->desc, in); +} + +static int cryptd_create_hash(struct crypto_template *tmpl, struct rtattr **tb, + struct cryptd_queue *queue) +{ + struct hashd_instance_ctx *ctx; + struct ahash_instance *inst; + struct shash_alg *salg; struct crypto_alg *alg; + int err; - alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_HASH_MASK); - if (IS_ERR(alg)) - return ERR_PTR(PTR_ERR(alg)); + salg = shash_attr_alg(tb[1], 0, 0); + if (IS_ERR(salg)) + return PTR_ERR(salg); - inst = cryptd_alloc_instance(alg, queue); + alg = &salg->base; + inst = cryptd_alloc_instance(alg, ahash_instance_headroom(), + sizeof(*ctx)); + err = PTR_ERR(inst); if (IS_ERR(inst)) goto out_put_alg; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC; - inst->alg.cra_type = &crypto_ahash_type; + ctx = ahash_instance_ctx(inst); + ctx->queue = queue; - inst->alg.cra_ahash.digestsize = alg->cra_hash.digestsize; - inst->alg.cra_ctxsize = sizeof(struct cryptd_hash_ctx); + err = crypto_init_shash_spawn(&ctx->spawn, salg, + ahash_crypto_instance(inst)); + if (err) + goto out_free_inst; - inst->alg.cra_init = cryptd_hash_init_tfm; - inst->alg.cra_exit = cryptd_hash_exit_tfm; + inst->alg.halg.base.cra_flags = CRYPTO_ALG_ASYNC; - inst->alg.cra_ahash.init = cryptd_hash_init_enqueue; - inst->alg.cra_ahash.update = cryptd_hash_update_enqueue; - inst->alg.cra_ahash.final = cryptd_hash_final_enqueue; - inst->alg.cra_ahash.setkey = cryptd_hash_setkey; - inst->alg.cra_ahash.digest = cryptd_hash_digest_enqueue; + inst->alg.halg.digestsize = salg->digestsize; + inst->alg.halg.base.cra_ctxsize = sizeof(struct cryptd_hash_ctx); + + inst->alg.halg.base.cra_init = cryptd_hash_init_tfm; + inst->alg.halg.base.cra_exit = cryptd_hash_exit_tfm; + + inst->alg.init = cryptd_hash_init_enqueue; + inst->alg.update = cryptd_hash_update_enqueue; + inst->alg.final = cryptd_hash_final_enqueue; + inst->alg.finup = cryptd_hash_finup_enqueue; + inst->alg.export = cryptd_hash_export; + inst->alg.import = cryptd_hash_import; + inst->alg.setkey = cryptd_hash_setkey; + inst->alg.digest = cryptd_hash_digest_enqueue; + + err = ahash_register_instance(tmpl, inst); + if (err) { + crypto_drop_shash(&ctx->spawn); +out_free_inst: + kfree(inst); + } out_put_alg: crypto_mod_put(alg); - return inst; + return err; } static struct cryptd_queue queue; -static struct crypto_instance *cryptd_alloc(struct rtattr **tb) +static int cryptd_create(struct crypto_template *tmpl, struct rtattr **tb) { struct crypto_attr_type *algt; algt = crypto_get_attr_type(tb); if (IS_ERR(algt)) - return ERR_CAST(algt); + return PTR_ERR(algt); switch (algt->type & algt->mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_BLKCIPHER: - return cryptd_alloc_blkcipher(tb, &queue); + return cryptd_create_blkcipher(tmpl, tb, &queue); case CRYPTO_ALG_TYPE_DIGEST: - return cryptd_alloc_hash(tb, &queue); + return cryptd_create_hash(tmpl, tb, &queue); } - return ERR_PTR(-EINVAL); + return -EINVAL; } static void cryptd_free(struct crypto_instance *inst) { struct cryptd_instance_ctx *ctx = crypto_instance_ctx(inst); + struct hashd_instance_ctx *hctx = crypto_instance_ctx(inst); + + switch (inst->alg.cra_flags & CRYPTO_ALG_TYPE_MASK) { + case CRYPTO_ALG_TYPE_AHASH: + crypto_drop_shash(&hctx->spawn); + kfree(ahash_instance(inst)); + return; + } crypto_drop_spawn(&ctx->spawn); kfree(inst); @@ -577,7 +639,7 @@ static void cryptd_free(struct crypto_instance *inst) static struct crypto_template cryptd_tmpl = { .name = "cryptd", - .alloc = cryptd_alloc, + .create = cryptd_create, .free = cryptd_free, .module = THIS_MODULE, }; @@ -620,6 +682,41 @@ void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm) } EXPORT_SYMBOL_GPL(cryptd_free_ablkcipher); +struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, + u32 type, u32 mask) +{ + char cryptd_alg_name[CRYPTO_MAX_ALG_NAME]; + struct crypto_ahash *tfm; + + if (snprintf(cryptd_alg_name, CRYPTO_MAX_ALG_NAME, + "cryptd(%s)", alg_name) >= CRYPTO_MAX_ALG_NAME) + return ERR_PTR(-EINVAL); + tfm = crypto_alloc_ahash(cryptd_alg_name, type, mask); + if (IS_ERR(tfm)) + return ERR_CAST(tfm); + if (tfm->base.__crt_alg->cra_module != THIS_MODULE) { + crypto_free_ahash(tfm); + return ERR_PTR(-EINVAL); + } + + return __cryptd_ahash_cast(tfm); +} +EXPORT_SYMBOL_GPL(cryptd_alloc_ahash); + +struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm) +{ + struct cryptd_hash_ctx *ctx = crypto_ahash_ctx(&tfm->base); + + return ctx->child; +} +EXPORT_SYMBOL_GPL(cryptd_ahash_child); + +void cryptd_free_ahash(struct cryptd_ahash *tfm) +{ + crypto_free_ahash(&tfm->base); +} +EXPORT_SYMBOL_GPL(cryptd_free_ahash); + static int __init cryptd_init(void) { int err; diff --git a/crypto/ctr.c b/crypto/ctr.c index 2d7425f..6c3bfab 100644 --- a/crypto/ctr.c +++ b/crypto/ctr.c @@ -219,6 +219,8 @@ static struct crypto_instance *crypto_ctr_alloc(struct rtattr **tb) inst->alg.cra_blkcipher.encrypt = crypto_ctr_crypt; inst->alg.cra_blkcipher.decrypt = crypto_ctr_crypt; + inst->alg.cra_blkcipher.geniv = "chainiv"; + out: crypto_mod_put(alg); return inst; diff --git a/crypto/gcm.c b/crypto/gcm.c index e70afd0..5fc3292 100644 --- a/crypto/gcm.c +++ b/crypto/gcm.c @@ -11,7 +11,10 @@ #include <crypto/gf128mul.h> #include <crypto/internal/aead.h> #include <crypto/internal/skcipher.h> +#include <crypto/internal/hash.h> #include <crypto/scatterwalk.h> +#include <crypto/hash.h> +#include "internal.h" #include <linux/completion.h> #include <linux/err.h> #include <linux/init.h> @@ -21,11 +24,12 @@ struct gcm_instance_ctx { struct crypto_skcipher_spawn ctr; + struct crypto_ahash_spawn ghash; }; struct crypto_gcm_ctx { struct crypto_ablkcipher *ctr; - struct gf128mul_4k *gf128; + struct crypto_ahash *ghash; }; struct crypto_rfc4106_ctx { @@ -34,10 +38,9 @@ struct crypto_rfc4106_ctx { }; struct crypto_gcm_ghash_ctx { - u32 bytes; - u32 flags; - struct gf128mul_4k *gf128; - u8 buffer[16]; + unsigned int cryptlen; + struct scatterlist *src; + crypto_completion_t complete; }; struct crypto_gcm_req_priv_ctx { @@ -45,8 +48,11 @@ struct crypto_gcm_req_priv_ctx { u8 iauth_tag[16]; struct scatterlist src[2]; struct scatterlist dst[2]; - struct crypto_gcm_ghash_ctx ghash; - struct ablkcipher_request abreq; + struct crypto_gcm_ghash_ctx ghash_ctx; + union { + struct ahash_request ahreq; + struct ablkcipher_request abreq; + } u; }; struct crypto_gcm_setkey_result { @@ -54,6 +60,8 @@ struct crypto_gcm_setkey_result { struct completion completion; }; +static void *gcm_zeroes; + static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( struct aead_request *req) { @@ -62,113 +70,6 @@ static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx( return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1); } -static void crypto_gcm_ghash_init(struct crypto_gcm_ghash_ctx *ctx, u32 flags, - struct gf128mul_4k *gf128) -{ - ctx->bytes = 0; - ctx->flags = flags; - ctx->gf128 = gf128; - memset(ctx->buffer, 0, 16); -} - -static void crypto_gcm_ghash_update(struct crypto_gcm_ghash_ctx *ctx, - const u8 *src, unsigned int srclen) -{ - u8 *dst = ctx->buffer; - - if (ctx->bytes) { - int n = min(srclen, ctx->bytes); - u8 *pos = dst + (16 - ctx->bytes); - - ctx->bytes -= n; - srclen -= n; - - while (n--) - *pos++ ^= *src++; - - if (!ctx->bytes) - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - } - - while (srclen >= 16) { - crypto_xor(dst, src, 16); - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - src += 16; - srclen -= 16; - } - - if (srclen) { - ctx->bytes = 16 - srclen; - while (srclen--) - *dst++ ^= *src++; - } -} - -static void crypto_gcm_ghash_update_sg(struct crypto_gcm_ghash_ctx *ctx, - struct scatterlist *sg, int len) -{ - struct scatter_walk walk; - u8 *src; - int n; - - if (!len) - return; - - scatterwalk_start(&walk, sg); - - while (len) { - n = scatterwalk_clamp(&walk, len); - - if (!n) { - scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg)); - n = scatterwalk_clamp(&walk, len); - } - - src = scatterwalk_map(&walk, 0); - - crypto_gcm_ghash_update(ctx, src, n); - len -= n; - - scatterwalk_unmap(src, 0); - scatterwalk_advance(&walk, n); - scatterwalk_done(&walk, 0, len); - if (len) - crypto_yield(ctx->flags); - } -} - -static void crypto_gcm_ghash_flush(struct crypto_gcm_ghash_ctx *ctx) -{ - u8 *dst = ctx->buffer; - - if (ctx->bytes) { - u8 *tmp = dst + (16 - ctx->bytes); - - while (ctx->bytes--) - *tmp++ ^= 0; - - gf128mul_4k_lle((be128 *)dst, ctx->gf128); - } - - ctx->bytes = 0; -} - -static void crypto_gcm_ghash_final_xor(struct crypto_gcm_ghash_ctx *ctx, - unsigned int authlen, - unsigned int cryptlen, u8 *dst) -{ - u8 *buf = ctx->buffer; - u128 lengths; - - lengths.a = cpu_to_be64(authlen * 8); - lengths.b = cpu_to_be64(cryptlen * 8); - - crypto_gcm_ghash_flush(ctx); - crypto_xor(buf, (u8 *)&lengths, 16); - gf128mul_4k_lle((be128 *)buf, ctx->gf128); - crypto_xor(dst, buf, 16); -} - static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err) { struct crypto_gcm_setkey_result *result = req->data; @@ -184,6 +85,7 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, unsigned int keylen) { struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); + struct crypto_ahash *ghash = ctx->ghash; struct crypto_ablkcipher *ctr = ctx->ctr; struct { be128 hash; @@ -233,13 +135,12 @@ static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (err) goto out; - if (ctx->gf128 != NULL) - gf128mul_free_4k(ctx->gf128); - - ctx->gf128 = gf128mul_init_4k_lle(&data->hash); - - if (ctx->gf128 == NULL) - err = -ENOMEM; + crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK); + crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) & + CRYPTO_TFM_REQ_MASK); + err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128)); + crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) & + CRYPTO_TFM_RES_MASK); out: kfree(data); @@ -272,8 +173,6 @@ static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - u32 flags = req->base.tfm->crt_flags; - struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; struct scatterlist *dst; __be32 counter = cpu_to_be32(1); @@ -296,108 +195,398 @@ static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req, ablkcipher_request_set_crypt(ablk_req, pctx->src, dst, cryptlen + sizeof(pctx->auth_tag), req->iv); +} + +static inline unsigned int gcm_remain(unsigned int len) +{ + len &= 0xfU; + return len ? 16 - len : 0; +} + +static void gcm_hash_len_done(struct crypto_async_request *areq, int err); +static void gcm_hash_final_done(struct crypto_async_request *areq, int err); - crypto_gcm_ghash_init(ghash, flags, ctx->gf128); +static int gcm_hash_update(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx, + crypto_completion_t complete, + struct scatterlist *src, + unsigned int len) +{ + struct ahash_request *ahreq = &pctx->u.ahreq; - crypto_gcm_ghash_update_sg(ghash, req->assoc, req->assoclen); - crypto_gcm_ghash_flush(ghash); + ahash_request_set_callback(ahreq, aead_request_flags(req), + complete, req); + ahash_request_set_crypt(ahreq, src, NULL, len); + + return crypto_ahash_update(ahreq); } -static int crypto_gcm_hash(struct aead_request *req) +static int gcm_hash_remain(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx, + unsigned int remain, + crypto_completion_t complete) { - struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct ahash_request *ahreq = &pctx->u.ahreq; + + ahash_request_set_callback(ahreq, aead_request_flags(req), + complete, req); + sg_init_one(pctx->src, gcm_zeroes, remain); + ahash_request_set_crypt(ahreq, pctx->src, NULL, remain); + + return crypto_ahash_update(ahreq); +} + +static int gcm_hash_len(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) +{ + struct ahash_request *ahreq = &pctx->u.ahreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + u128 lengths; + + lengths.a = cpu_to_be64(req->assoclen * 8); + lengths.b = cpu_to_be64(gctx->cryptlen * 8); + memcpy(pctx->iauth_tag, &lengths, 16); + sg_init_one(pctx->src, pctx->iauth_tag, 16); + ahash_request_set_callback(ahreq, aead_request_flags(req), + gcm_hash_len_done, req); + ahash_request_set_crypt(ahreq, pctx->src, + NULL, sizeof(lengths)); + + return crypto_ahash_update(ahreq); +} + +static int gcm_hash_final(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) +{ + struct ahash_request *ahreq = &pctx->u.ahreq; + + ahash_request_set_callback(ahreq, aead_request_flags(req), + gcm_hash_final_done, req); + ahash_request_set_crypt(ahreq, NULL, pctx->iauth_tag, 0); + + return crypto_ahash_final(ahreq); +} + +static void gcm_hash_final_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - u8 *auth_tag = pctx->auth_tag; - struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + + if (!err) + crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); - crypto_gcm_ghash_update_sg(ghash, req->dst, req->cryptlen); - crypto_gcm_ghash_final_xor(ghash, req->assoclen, req->cryptlen, - auth_tag); + gctx->complete(areq, err); +} + +static void gcm_hash_len_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + + if (!err) { + err = gcm_hash_final(req, pctx); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + gcm_hash_final_done(areq, err); +} + +static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + + if (!err) { + err = gcm_hash_len(req, pctx); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + gcm_hash_len_done(areq, err); +} + +static void gcm_hash_crypt_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + unsigned int remain; + + if (!err) { + remain = gcm_remain(gctx->cryptlen); + BUG_ON(!remain); + err = gcm_hash_remain(req, pctx, remain, + gcm_hash_crypt_remain_done); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + gcm_hash_crypt_remain_done(areq, err); +} + +static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + crypto_completion_t complete; + unsigned int remain = 0; + + if (!err && gctx->cryptlen) { + remain = gcm_remain(gctx->cryptlen); + complete = remain ? gcm_hash_crypt_done : + gcm_hash_crypt_remain_done; + err = gcm_hash_update(req, pctx, complete, + gctx->src, gctx->cryptlen); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + if (remain) + gcm_hash_crypt_done(areq, err); + else + gcm_hash_crypt_remain_done(areq, err); +} + +static void gcm_hash_assoc_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + unsigned int remain; + + if (!err) { + remain = gcm_remain(req->assoclen); + BUG_ON(!remain); + err = gcm_hash_remain(req, pctx, remain, + gcm_hash_assoc_remain_done); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + gcm_hash_assoc_remain_done(areq, err); +} + +static void gcm_hash_init_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + crypto_completion_t complete; + unsigned int remain = 0; + + if (!err && req->assoclen) { + remain = gcm_remain(req->assoclen); + complete = remain ? gcm_hash_assoc_done : + gcm_hash_assoc_remain_done; + err = gcm_hash_update(req, pctx, complete, + req->assoc, req->assoclen); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + if (remain) + gcm_hash_assoc_done(areq, err); + else + gcm_hash_assoc_remain_done(areq, err); +} + +static int gcm_hash(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) +{ + struct ahash_request *ahreq = &pctx->u.ahreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + unsigned int remain; + crypto_completion_t complete; + int err; + + ahash_request_set_tfm(ahreq, ctx->ghash); + + ahash_request_set_callback(ahreq, aead_request_flags(req), + gcm_hash_init_done, req); + err = crypto_ahash_init(ahreq); + if (err) + return err; + remain = gcm_remain(req->assoclen); + complete = remain ? gcm_hash_assoc_done : gcm_hash_assoc_remain_done; + err = gcm_hash_update(req, pctx, complete, req->assoc, req->assoclen); + if (err) + return err; + if (remain) { + err = gcm_hash_remain(req, pctx, remain, + gcm_hash_assoc_remain_done); + if (err) + return err; + } + remain = gcm_remain(gctx->cryptlen); + complete = remain ? gcm_hash_crypt_done : gcm_hash_crypt_remain_done; + err = gcm_hash_update(req, pctx, complete, gctx->src, gctx->cryptlen); + if (err) + return err; + if (remain) { + err = gcm_hash_remain(req, pctx, remain, + gcm_hash_crypt_remain_done); + if (err) + return err; + } + err = gcm_hash_len(req, pctx); + if (err) + return err; + err = gcm_hash_final(req, pctx); + if (err) + return err; + + return 0; +} + +static void gcm_enc_copy_hash(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + u8 *auth_tag = pctx->auth_tag; scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen, crypto_aead_authsize(aead), 1); - return 0; } -static void crypto_gcm_encrypt_done(struct crypto_async_request *areq, int err) +static void gcm_enc_hash_done(struct crypto_async_request *areq, + int err) { struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); if (!err) - err = crypto_gcm_hash(req); + gcm_enc_copy_hash(req, pctx); aead_request_complete(req, err); } +static void gcm_encrypt_done(struct crypto_async_request *areq, + int err) +{ + struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + + if (!err) { + err = gcm_hash(req, pctx); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + gcm_enc_hash_done(areq, err); +} + static int crypto_gcm_encrypt(struct aead_request *req) { struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ablkcipher_request *abreq = &pctx->abreq; + struct ablkcipher_request *abreq = &pctx->u.abreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; int err; crypto_gcm_init_crypt(abreq, req, req->cryptlen); ablkcipher_request_set_callback(abreq, aead_request_flags(req), - crypto_gcm_encrypt_done, req); + gcm_encrypt_done, req); + + gctx->src = req->dst; + gctx->cryptlen = req->cryptlen; + gctx->complete = gcm_enc_hash_done; err = crypto_ablkcipher_encrypt(abreq); if (err) return err; - return crypto_gcm_hash(req); + err = gcm_hash(req, pctx); + if (err) + return err; + + crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16); + gcm_enc_copy_hash(req, pctx); + + return 0; } -static int crypto_gcm_verify(struct aead_request *req) +static int crypto_gcm_verify(struct aead_request *req, + struct crypto_gcm_req_priv_ctx *pctx) { struct crypto_aead *aead = crypto_aead_reqtfm(req); - struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; u8 *auth_tag = pctx->auth_tag; u8 *iauth_tag = pctx->iauth_tag; unsigned int authsize = crypto_aead_authsize(aead); unsigned int cryptlen = req->cryptlen - authsize; - crypto_gcm_ghash_final_xor(ghash, req->assoclen, cryptlen, auth_tag); - - authsize = crypto_aead_authsize(aead); + crypto_xor(auth_tag, iauth_tag, 16); scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0); return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0; } -static void crypto_gcm_decrypt_done(struct crypto_async_request *areq, int err) +static void gcm_decrypt_done(struct crypto_async_request *areq, int err) { struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); if (!err) - err = crypto_gcm_verify(req); + err = crypto_gcm_verify(req, pctx); aead_request_complete(req, err); } +static void gcm_dec_hash_done(struct crypto_async_request *areq, int err) +{ + struct aead_request *req = areq->data; + struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); + struct ablkcipher_request *abreq = &pctx->u.abreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; + + if (!err) { + ablkcipher_request_set_callback(abreq, aead_request_flags(req), + gcm_decrypt_done, req); + crypto_gcm_init_crypt(abreq, req, gctx->cryptlen); + err = crypto_ablkcipher_decrypt(abreq); + if (err == -EINPROGRESS || err == -EBUSY) + return; + } + + gcm_decrypt_done(areq, err); +} + static int crypto_gcm_decrypt(struct aead_request *req) { struct crypto_aead *aead = crypto_aead_reqtfm(req); struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req); - struct ablkcipher_request *abreq = &pctx->abreq; - struct crypto_gcm_ghash_ctx *ghash = &pctx->ghash; - unsigned int cryptlen = req->cryptlen; + struct ablkcipher_request *abreq = &pctx->u.abreq; + struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx; unsigned int authsize = crypto_aead_authsize(aead); + unsigned int cryptlen = req->cryptlen; int err; if (cryptlen < authsize) return -EINVAL; cryptlen -= authsize; - crypto_gcm_init_crypt(abreq, req, cryptlen); - ablkcipher_request_set_callback(abreq, aead_request_flags(req), - crypto_gcm_decrypt_done, req); + gctx->src = req->src; + gctx->cryptlen = cryptlen; + gctx->complete = gcm_dec_hash_done; - crypto_gcm_ghash_update_sg(ghash, req->src, cryptlen); + err = gcm_hash(req, pctx); + if (err) + return err; + ablkcipher_request_set_callback(abreq, aead_request_flags(req), + gcm_decrypt_done, req); + crypto_gcm_init_crypt(abreq, req, cryptlen); err = crypto_ablkcipher_decrypt(abreq); if (err) return err; - return crypto_gcm_verify(req); + return crypto_gcm_verify(req, pctx); } static int crypto_gcm_init_tfm(struct crypto_tfm *tfm) @@ -406,43 +595,56 @@ static int crypto_gcm_init_tfm(struct crypto_tfm *tfm) struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst); struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); struct crypto_ablkcipher *ctr; + struct crypto_ahash *ghash; unsigned long align; int err; + ghash = crypto_spawn_ahash(&ictx->ghash); + if (IS_ERR(ghash)) + return PTR_ERR(ghash); + ctr = crypto_spawn_skcipher(&ictx->ctr); err = PTR_ERR(ctr); if (IS_ERR(ctr)) - return err; + goto err_free_hash; ctx->ctr = ctr; - ctx->gf128 = NULL; + ctx->ghash = ghash; align = crypto_tfm_alg_alignmask(tfm); align &= ~(crypto_tfm_ctx_alignment() - 1); tfm->crt_aead.reqsize = align + - sizeof(struct crypto_gcm_req_priv_ctx) + - crypto_ablkcipher_reqsize(ctr); + offsetof(struct crypto_gcm_req_priv_ctx, u) + + max(sizeof(struct ablkcipher_request) + + crypto_ablkcipher_reqsize(ctr), + sizeof(struct ahash_request) + + crypto_ahash_reqsize(ghash)); return 0; + +err_free_hash: + crypto_free_ahash(ghash); + return err; } static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm) { struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm); - if (ctx->gf128 != NULL) - gf128mul_free_4k(ctx->gf128); - + crypto_free_ahash(ctx->ghash); crypto_free_ablkcipher(ctx->ctr); } static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, const char *full_name, - const char *ctr_name) + const char *ctr_name, + const char *ghash_name) { struct crypto_attr_type *algt; struct crypto_instance *inst; struct crypto_alg *ctr; + struct crypto_alg *ghash_alg; + struct ahash_alg *ghash_ahash_alg; struct gcm_instance_ctx *ctx; int err; @@ -454,17 +656,31 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask) return ERR_PTR(-EINVAL); + ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type, + CRYPTO_ALG_TYPE_HASH, + CRYPTO_ALG_TYPE_AHASH_MASK); + err = PTR_ERR(ghash_alg); + if (IS_ERR(ghash_alg)) + return ERR_PTR(err); + + err = -ENOMEM; inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); if (!inst) - return ERR_PTR(-ENOMEM); + goto out_put_ghash; ctx = crypto_instance_ctx(inst); + ghash_ahash_alg = container_of(ghash_alg, struct ahash_alg, halg.base); + err = crypto_init_ahash_spawn(&ctx->ghash, &ghash_ahash_alg->halg, + inst); + if (err) + goto err_free_inst; + crypto_set_skcipher_spawn(&ctx->ctr, inst); err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0, crypto_requires_sync(algt->type, algt->mask)); if (err) - goto err_free_inst; + goto err_drop_ghash; ctr = crypto_skcipher_spawn_alg(&ctx->ctr); @@ -479,7 +695,8 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, err = -ENAMETOOLONG; if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME, - "gcm_base(%s)", ctr->cra_driver_name) >= + "gcm_base(%s,%s)", ctr->cra_driver_name, + ghash_alg->cra_driver_name) >= CRYPTO_MAX_ALG_NAME) goto out_put_ctr; @@ -502,12 +719,16 @@ static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb, inst->alg.cra_aead.decrypt = crypto_gcm_decrypt; out: + crypto_mod_put(ghash_alg); return inst; out_put_ctr: crypto_drop_skcipher(&ctx->ctr); +err_drop_ghash: + crypto_drop_ahash(&ctx->ghash); err_free_inst: kfree(inst); +out_put_ghash: inst = ERR_PTR(err); goto out; } @@ -532,7 +753,7 @@ static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb) CRYPTO_MAX_ALG_NAME) return ERR_PTR(-ENAMETOOLONG); - return crypto_gcm_alloc_common(tb, full_name, ctr_name); + return crypto_gcm_alloc_common(tb, full_name, ctr_name, "ghash"); } static void crypto_gcm_free(struct crypto_instance *inst) @@ -540,6 +761,7 @@ static void crypto_gcm_free(struct crypto_instance *inst) struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst); crypto_drop_skcipher(&ctx->ctr); + crypto_drop_ahash(&ctx->ghash); kfree(inst); } @@ -554,6 +776,7 @@ static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb) { int err; const char *ctr_name; + const char *ghash_name; char full_name[CRYPTO_MAX_ALG_NAME]; ctr_name = crypto_attr_alg_name(tb[1]); @@ -561,11 +784,16 @@ static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb) if (IS_ERR(ctr_name)) return ERR_PTR(err); - if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s)", - ctr_name) >= CRYPTO_MAX_ALG_NAME) + ghash_name = crypto_attr_alg_name(tb[2]); + err = PTR_ERR(ghash_name); + if (IS_ERR(ghash_name)) + return ERR_PTR(err); + + if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)", + ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME) return ERR_PTR(-ENAMETOOLONG); - return crypto_gcm_alloc_common(tb, full_name, ctr_name); + return crypto_gcm_alloc_common(tb, full_name, ctr_name, ghash_name); } static struct crypto_template crypto_gcm_base_tmpl = { @@ -784,6 +1012,10 @@ static int __init crypto_gcm_module_init(void) { int err; + gcm_zeroes = kzalloc(16, GFP_KERNEL); + if (!gcm_zeroes) + return -ENOMEM; + err = crypto_register_template(&crypto_gcm_base_tmpl); if (err) goto out; @@ -796,18 +1028,20 @@ static int __init crypto_gcm_module_init(void) if (err) goto out_undo_gcm; -out: - return err; + return 0; out_undo_gcm: crypto_unregister_template(&crypto_gcm_tmpl); out_undo_base: crypto_unregister_template(&crypto_gcm_base_tmpl); - goto out; +out: + kfree(gcm_zeroes); + return err; } static void __exit crypto_gcm_module_exit(void) { + kfree(gcm_zeroes); crypto_unregister_template(&crypto_rfc4106_tmpl); crypto_unregister_template(&crypto_gcm_tmpl); crypto_unregister_template(&crypto_gcm_base_tmpl); diff --git a/crypto/ghash-generic.c b/crypto/ghash-generic.c new file mode 100644 index 0000000..be44256 --- /dev/null +++ b/crypto/ghash-generic.c @@ -0,0 +1,170 @@ +/* + * GHASH: digest algorithm for GCM (Galois/Counter Mode). + * + * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi> + * Copyright (c) 2009 Intel Corp. + * Author: Huang Ying <ying.huang@intel.com> + * + * The algorithm implementation is copied from gcm.c. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + */ + +#include <crypto/algapi.h> +#include <crypto/gf128mul.h> +#include <crypto/internal/hash.h> +#include <linux/crypto.h> +#include <linux/init.h> +#include <linux/kernel.h> +#include <linux/module.h> + +#define GHASH_BLOCK_SIZE 16 +#define GHASH_DIGEST_SIZE 16 + +struct ghash_ctx { + struct gf128mul_4k *gf128; +}; + +struct ghash_desc_ctx { + u8 buffer[GHASH_BLOCK_SIZE]; + u32 bytes; +}; + +static int ghash_init(struct shash_desc *desc) +{ + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + + memset(dctx, 0, sizeof(*dctx)); + + return 0; +} + +static int ghash_setkey(struct crypto_shash *tfm, + const u8 *key, unsigned int keylen) +{ + struct ghash_ctx *ctx = crypto_shash_ctx(tfm); + + if (keylen != GHASH_BLOCK_SIZE) { + crypto_shash_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + if (ctx->gf128) + gf128mul_free_4k(ctx->gf128); + ctx->gf128 = gf128mul_init_4k_lle((be128 *)key); + if (!ctx->gf128) + return -ENOMEM; + + return 0; +} + +static int ghash_update(struct shash_desc *desc, + const u8 *src, unsigned int srclen) +{ + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + u8 *dst = dctx->buffer; + + if (dctx->bytes) { + int n = min(srclen, dctx->bytes); + u8 *pos = dst + (GHASH_BLOCK_SIZE - dctx->bytes); + + dctx->bytes -= n; + srclen -= n; + + while (n--) + *pos++ ^= *src++; + + if (!dctx->bytes) + gf128mul_4k_lle((be128 *)dst, ctx->gf128); + } + + while (srclen >= GHASH_BLOCK_SIZE) { + crypto_xor(dst, src, GHASH_BLOCK_SIZE); + gf128mul_4k_lle((be128 *)dst, ctx->gf128); + src += GHASH_BLOCK_SIZE; + srclen -= GHASH_BLOCK_SIZE; + } + + if (srclen) { + dctx->bytes = GHASH_BLOCK_SIZE - srclen; + while (srclen--) + *dst++ ^= *src++; + } + + return 0; +} + +static void ghash_flush(struct ghash_ctx *ctx, struct ghash_desc_ctx *dctx) +{ + u8 *dst = dctx->buffer; + + if (dctx->bytes) { + u8 *tmp = dst + (GHASH_BLOCK_SIZE - dctx->bytes); + + while (dctx->bytes--) + *tmp++ ^= 0; + + gf128mul_4k_lle((be128 *)dst, ctx->gf128); + } + + dctx->bytes = 0; +} + +static int ghash_final(struct shash_desc *desc, u8 *dst) +{ + struct ghash_desc_ctx *dctx = shash_desc_ctx(desc); + struct ghash_ctx *ctx = crypto_shash_ctx(desc->tfm); + u8 *buf = dctx->buffer; + + ghash_flush(ctx, dctx); + memcpy(dst, buf, GHASH_BLOCK_SIZE); + + return 0; +} + +static void ghash_exit_tfm(struct crypto_tfm *tfm) +{ + struct ghash_ctx *ctx = crypto_tfm_ctx(tfm); + if (ctx->gf128) + gf128mul_free_4k(ctx->gf128); +} + +static struct shash_alg ghash_alg = { + .digestsize = GHASH_DIGEST_SIZE, + .init = ghash_init, + .update = ghash_update, + .final = ghash_final, + .setkey = ghash_setkey, + .descsize = sizeof(struct ghash_desc_ctx), + .base = { + .cra_name = "ghash", + .cra_driver_name = "ghash-generic", + .cra_priority = 100, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = GHASH_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct ghash_ctx), + .cra_module = THIS_MODULE, + .cra_list = LIST_HEAD_INIT(ghash_alg.base.cra_list), + .cra_exit = ghash_exit_tfm, + }, +}; + +static int __init ghash_mod_init(void) +{ + return crypto_register_shash(&ghash_alg); +} + +static void __exit ghash_mod_exit(void) +{ + crypto_unregister_shash(&ghash_alg); +} + +module_init(ghash_mod_init); +module_exit(ghash_mod_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("GHASH Message Digest Algorithm"); +MODULE_ALIAS("ghash"); diff --git a/crypto/hmac.c b/crypto/hmac.c index 0ad39c3..15c2eb5 100644 --- a/crypto/hmac.c +++ b/crypto/hmac.c @@ -27,7 +27,7 @@ #include <linux/string.h> struct hmac_ctx { - struct crypto_hash *child; + struct crypto_shash *hash; }; static inline void *align_ptr(void *p, unsigned int align) @@ -35,65 +35,45 @@ static inline void *align_ptr(void *p, unsigned int align) return (void *)ALIGN((unsigned long)p, align); } -static inline struct hmac_ctx *hmac_ctx(struct crypto_hash *tfm) +static inline struct hmac_ctx *hmac_ctx(struct crypto_shash *tfm) { - return align_ptr(crypto_hash_ctx_aligned(tfm) + - crypto_hash_blocksize(tfm) * 2 + - crypto_hash_digestsize(tfm), sizeof(void *)); + return align_ptr(crypto_shash_ctx_aligned(tfm) + + crypto_shash_statesize(tfm) * 2, + crypto_tfm_ctx_alignment()); } -static int hmac_setkey(struct crypto_hash *parent, +static int hmac_setkey(struct crypto_shash *parent, const u8 *inkey, unsigned int keylen) { - int bs = crypto_hash_blocksize(parent); - int ds = crypto_hash_digestsize(parent); - char *ipad = crypto_hash_ctx_aligned(parent); - char *opad = ipad + bs; - char *digest = opad + bs; - struct hmac_ctx *ctx = align_ptr(digest + ds, sizeof(void *)); - struct crypto_hash *tfm = ctx->child; + int bs = crypto_shash_blocksize(parent); + int ds = crypto_shash_digestsize(parent); + int ss = crypto_shash_statesize(parent); + char *ipad = crypto_shash_ctx_aligned(parent); + char *opad = ipad + ss; + struct hmac_ctx *ctx = align_ptr(opad + ss, + crypto_tfm_ctx_alignment()); + struct crypto_shash *hash = ctx->hash; + struct { + struct shash_desc shash; + char ctx[crypto_shash_descsize(hash)]; + } desc; unsigned int i; + desc.shash.tfm = hash; + desc.shash.flags = crypto_shash_get_flags(parent) & + CRYPTO_TFM_REQ_MAY_SLEEP; + if (keylen > bs) { - struct hash_desc desc; - struct scatterlist tmp; - int tmplen; int err; - desc.tfm = tfm; - desc.flags = crypto_hash_get_flags(parent); - desc.flags &= CRYPTO_TFM_REQ_MAY_SLEEP; - - err = crypto_hash_init(&desc); + err = crypto_shash_digest(&desc.shash, inkey, keylen, ipad); if (err) return err; - tmplen = bs * 2 + ds; - sg_init_one(&tmp, ipad, tmplen); - - for (; keylen > tmplen; inkey += tmplen, keylen -= tmplen) { - memcpy(ipad, inkey, tmplen); - err = crypto_hash_update(&desc, &tmp, tmplen); - if (err) - return err; - } - - if (keylen) { - memcpy(ipad, inkey, keylen); - err = crypto_hash_update(&desc, &tmp, keylen); - if (err) - return err; - } - - err = crypto_hash_final(&desc, digest); - if (err) - return err; - - inkey = digest; keylen = ds; - } + } else + memcpy(ipad, inkey, keylen); - memcpy(ipad, inkey, keylen); memset(ipad + keylen, 0, bs - keylen); memcpy(opad, ipad, bs); @@ -102,184 +82,178 @@ static int hmac_setkey(struct crypto_hash *parent, opad[i] ^= 0x5c; } - return 0; + return crypto_shash_init(&desc.shash) ?: + crypto_shash_update(&desc.shash, ipad, bs) ?: + crypto_shash_export(&desc.shash, ipad) ?: + crypto_shash_init(&desc.shash) ?: + crypto_shash_update(&desc.shash, opad, bs) ?: + crypto_shash_export(&desc.shash, opad); } -static int hmac_init(struct hash_desc *pdesc) +static int hmac_export(struct shash_desc *pdesc, void *out) { - struct crypto_hash *parent = pdesc->tfm; - int bs = crypto_hash_blocksize(parent); - int ds = crypto_hash_digestsize(parent); - char *ipad = crypto_hash_ctx_aligned(parent); - struct hmac_ctx *ctx = align_ptr(ipad + bs * 2 + ds, sizeof(void *)); - struct hash_desc desc; - struct scatterlist tmp; - int err; + struct shash_desc *desc = shash_desc_ctx(pdesc); - desc.tfm = ctx->child; - desc.flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - sg_init_one(&tmp, ipad, bs); + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - err = crypto_hash_init(&desc); - if (unlikely(err)) - return err; - - return crypto_hash_update(&desc, &tmp, bs); + return crypto_shash_export(desc, out); } -static int hmac_update(struct hash_desc *pdesc, - struct scatterlist *sg, unsigned int nbytes) +static int hmac_import(struct shash_desc *pdesc, const void *in) { + struct shash_desc *desc = shash_desc_ctx(pdesc); struct hmac_ctx *ctx = hmac_ctx(pdesc->tfm); - struct hash_desc desc; - desc.tfm = ctx->child; - desc.flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + desc->tfm = ctx->hash; + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_hash_update(&desc, sg, nbytes); + return crypto_shash_import(desc, in); } -static int hmac_final(struct hash_desc *pdesc, u8 *out) +static int hmac_init(struct shash_desc *pdesc) { - struct crypto_hash *parent = pdesc->tfm; - int bs = crypto_hash_blocksize(parent); - int ds = crypto_hash_digestsize(parent); - char *opad = crypto_hash_ctx_aligned(parent) + bs; - char *digest = opad + bs; - struct hmac_ctx *ctx = align_ptr(digest + ds, sizeof(void *)); - struct hash_desc desc; - struct scatterlist tmp; - int err; + return hmac_import(pdesc, crypto_shash_ctx_aligned(pdesc->tfm)); +} - desc.tfm = ctx->child; - desc.flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - sg_init_one(&tmp, opad, bs + ds); +static int hmac_update(struct shash_desc *pdesc, + const u8 *data, unsigned int nbytes) +{ + struct shash_desc *desc = shash_desc_ctx(pdesc); - err = crypto_hash_final(&desc, digest); - if (unlikely(err)) - return err; + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - return crypto_hash_digest(&desc, &tmp, bs + ds, out); + return crypto_shash_update(desc, data, nbytes); } -static int hmac_digest(struct hash_desc *pdesc, struct scatterlist *sg, - unsigned int nbytes, u8 *out) +static int hmac_final(struct shash_desc *pdesc, u8 *out) { - struct crypto_hash *parent = pdesc->tfm; - int bs = crypto_hash_blocksize(parent); - int ds = crypto_hash_digestsize(parent); - char *ipad = crypto_hash_ctx_aligned(parent); - char *opad = ipad + bs; - char *digest = opad + bs; - struct hmac_ctx *ctx = align_ptr(digest + ds, sizeof(void *)); - struct hash_desc desc; - struct scatterlist sg1[2]; - struct scatterlist sg2[1]; - int err; + struct crypto_shash *parent = pdesc->tfm; + int ds = crypto_shash_digestsize(parent); + int ss = crypto_shash_statesize(parent); + char *opad = crypto_shash_ctx_aligned(parent) + ss; + struct shash_desc *desc = shash_desc_ctx(pdesc); - desc.tfm = ctx->child; - desc.flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; - sg_init_table(sg1, 2); - sg_set_buf(sg1, ipad, bs); - scatterwalk_sg_chain(sg1, 2, sg); + return crypto_shash_final(desc, out) ?: + crypto_shash_import(desc, opad) ?: + crypto_shash_finup(desc, out, ds, out); +} - sg_init_table(sg2, 1); - sg_set_buf(sg2, opad, bs + ds); +static int hmac_finup(struct shash_desc *pdesc, const u8 *data, + unsigned int nbytes, u8 *out) +{ - err = crypto_hash_digest(&desc, sg1, nbytes + bs, digest); - if (unlikely(err)) - return err; + struct crypto_shash *parent = pdesc->tfm; + int ds = crypto_shash_digestsize(parent); + int ss = crypto_shash_statesize(parent); + char *opad = crypto_shash_ctx_aligned(parent) + ss; + struct shash_desc *desc = shash_desc_ctx(pdesc); - return crypto_hash_digest(&desc, sg2, bs + ds, out); + desc->flags = pdesc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + + return crypto_shash_finup(desc, data, nbytes, out) ?: + crypto_shash_import(desc, opad) ?: + crypto_shash_finup(desc, out, ds, out); } static int hmac_init_tfm(struct crypto_tfm *tfm) { - struct crypto_hash *hash; + struct crypto_shash *parent = __crypto_shash_cast(tfm); + struct crypto_shash *hash; struct crypto_instance *inst = (void *)tfm->__crt_alg; - struct crypto_spawn *spawn = crypto_instance_ctx(inst); - struct hmac_ctx *ctx = hmac_ctx(__crypto_hash_cast(tfm)); + struct crypto_shash_spawn *spawn = crypto_instance_ctx(inst); + struct hmac_ctx *ctx = hmac_ctx(parent); - hash = crypto_spawn_hash(spawn); + hash = crypto_spawn_shash(spawn); if (IS_ERR(hash)) return PTR_ERR(hash); - ctx->child = hash; + parent->descsize = sizeof(struct shash_desc) + + crypto_shash_descsize(hash); + + ctx->hash = hash; return 0; } static void hmac_exit_tfm(struct crypto_tfm *tfm) { - struct hmac_ctx *ctx = hmac_ctx(__crypto_hash_cast(tfm)); - crypto_free_hash(ctx->child); + struct hmac_ctx *ctx = hmac_ctx(__crypto_shash_cast(tfm)); + crypto_free_shash(ctx->hash); } -static void hmac_free(struct crypto_instance *inst) +static int hmac_create(struct crypto_template *tmpl, struct rtattr **tb) { - crypto_drop_spawn(crypto_instance_ctx(inst)); - kfree(inst); -} - -static struct crypto_instance *hmac_alloc(struct rtattr **tb) -{ - struct crypto_instance *inst; + struct shash_instance *inst; struct crypto_alg *alg; + struct shash_alg *salg; int err; int ds; + int ss; - err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_HASH); + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH); if (err) - return ERR_PTR(err); - - alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_HASH, - CRYPTO_ALG_TYPE_HASH_MASK); - if (IS_ERR(alg)) - return ERR_CAST(alg); - - inst = ERR_PTR(-EINVAL); - ds = alg->cra_type == &crypto_hash_type ? - alg->cra_hash.digestsize : - alg->cra_type ? - __crypto_shash_alg(alg)->digestsize : - alg->cra_digest.dia_digestsize; - if (ds > alg->cra_blocksize) + return err; + + salg = shash_attr_alg(tb[1], 0, 0); + if (IS_ERR(salg)) + return PTR_ERR(salg); + + err = -EINVAL; + ds = salg->digestsize; + ss = salg->statesize; + alg = &salg->base; + if (ds > alg->cra_blocksize || + ss < alg->cra_blocksize) goto out_put_alg; - inst = crypto_alloc_instance("hmac", alg); + inst = shash_alloc_instance("hmac", alg); + err = PTR_ERR(inst); if (IS_ERR(inst)) goto out_put_alg; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_HASH; - inst->alg.cra_priority = alg->cra_priority; - inst->alg.cra_blocksize = alg->cra_blocksize; - inst->alg.cra_alignmask = alg->cra_alignmask; - inst->alg.cra_type = &crypto_hash_type; - - inst->alg.cra_hash.digestsize = ds; - - inst->alg.cra_ctxsize = sizeof(struct hmac_ctx) + - ALIGN(inst->alg.cra_blocksize * 2 + ds, - sizeof(void *)); - - inst->alg.cra_init = hmac_init_tfm; - inst->alg.cra_exit = hmac_exit_tfm; - - inst->alg.cra_hash.init = hmac_init; - inst->alg.cra_hash.update = hmac_update; - inst->alg.cra_hash.final = hmac_final; - inst->alg.cra_hash.digest = hmac_digest; - inst->alg.cra_hash.setkey = hmac_setkey; + err = crypto_init_shash_spawn(shash_instance_ctx(inst), salg, + shash_crypto_instance(inst)); + if (err) + goto out_free_inst; + + inst->alg.base.cra_priority = alg->cra_priority; + inst->alg.base.cra_blocksize = alg->cra_blocksize; + inst->alg.base.cra_alignmask = alg->cra_alignmask; + + ss = ALIGN(ss, alg->cra_alignmask + 1); + inst->alg.digestsize = ds; + inst->alg.statesize = ss; + + inst->alg.base.cra_ctxsize = sizeof(struct hmac_ctx) + + ALIGN(ss * 2, crypto_tfm_ctx_alignment()); + + inst->alg.base.cra_init = hmac_init_tfm; + inst->alg.base.cra_exit = hmac_exit_tfm; + + inst->alg.init = hmac_init; + inst->alg.update = hmac_update; + inst->alg.final = hmac_final; + inst->alg.finup = hmac_finup; + inst->alg.export = hmac_export; + inst->alg.import = hmac_import; + inst->alg.setkey = hmac_setkey; + + err = shash_register_instance(tmpl, inst); + if (err) { +out_free_inst: + shash_free_instance(shash_crypto_instance(inst)); + } out_put_alg: crypto_mod_put(alg); - return inst; + return err; } static struct crypto_template hmac_tmpl = { .name = "hmac", - .alloc = hmac_alloc, - .free = hmac_free, + .create = hmac_create, + .free = shash_free_instance, .module = THIS_MODULE, }; diff --git a/crypto/internal.h b/crypto/internal.h index 113579a..2d22636 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -25,12 +25,7 @@ #include <linux/notifier.h> #include <linux/rwsem.h> #include <linux/slab.h> - -#ifdef CONFIG_CRYPTO_FIPS -extern int fips_enabled; -#else -#define fips_enabled 0 -#endif +#include <linux/fips.h> /* Crypto notification events. */ enum { @@ -65,18 +60,6 @@ static inline void crypto_exit_proc(void) { } #endif -static inline unsigned int crypto_digest_ctxsize(struct crypto_alg *alg) -{ - unsigned int len = alg->cra_ctxsize; - - if (alg->cra_alignmask) { - len = ALIGN(len, (unsigned long)alg->cra_alignmask + 1); - len += alg->cra_digest.dia_digestsize; - } - - return len; -} - static inline unsigned int crypto_cipher_ctxsize(struct crypto_alg *alg) { return alg->cra_ctxsize; @@ -91,12 +74,9 @@ struct crypto_alg *crypto_mod_get(struct crypto_alg *alg); struct crypto_alg *crypto_alg_lookup(const char *name, u32 type, u32 mask); struct crypto_alg *crypto_alg_mod_lookup(const char *name, u32 type, u32 mask); -int crypto_init_digest_ops(struct crypto_tfm *tfm); -int crypto_init_digest_ops_async(struct crypto_tfm *tfm); int crypto_init_cipher_ops(struct crypto_tfm *tfm); int crypto_init_compress_ops(struct crypto_tfm *tfm); -void crypto_exit_digest_ops(struct crypto_tfm *tfm); void crypto_exit_cipher_ops(struct crypto_tfm *tfm); void crypto_exit_compress_ops(struct crypto_tfm *tfm); @@ -111,12 +91,12 @@ struct crypto_tfm *__crypto_alloc_tfm(struct crypto_alg *alg, u32 type, u32 mask); void *crypto_create_tfm(struct crypto_alg *alg, const struct crypto_type *frontend); +struct crypto_alg *crypto_find_alg(const char *alg_name, + const struct crypto_type *frontend, + u32 type, u32 mask); void *crypto_alloc_tfm(const char *alg_name, const struct crypto_type *frontend, u32 type, u32 mask); -int crypto_register_instance(struct crypto_template *tmpl, - struct crypto_instance *inst); - int crypto_register_notifier(struct notifier_block *nb); int crypto_unregister_notifier(struct notifier_block *nb); int crypto_probing_notify(unsigned long val, void *v); diff --git a/crypto/pcompress.c b/crypto/pcompress.c index bcadc03..f7c4a7d 100644 --- a/crypto/pcompress.c +++ b/crypto/pcompress.c @@ -36,14 +36,12 @@ static int crypto_pcomp_init(struct crypto_tfm *tfm, u32 type, u32 mask) return 0; } -static unsigned int crypto_pcomp_extsize(struct crypto_alg *alg, - const struct crypto_type *frontend) +static unsigned int crypto_pcomp_extsize(struct crypto_alg *alg) { return alg->cra_ctxsize; } -static int crypto_pcomp_init_tfm(struct crypto_tfm *tfm, - const struct crypto_type *frontend) +static int crypto_pcomp_init_tfm(struct crypto_tfm *tfm) { return 0; } diff --git a/crypto/rng.c b/crypto/rng.c index 6e94bc7..ba05e73 100644 --- a/crypto/rng.c +++ b/crypto/rng.c @@ -123,4 +123,4 @@ void crypto_put_default_rng(void) EXPORT_SYMBOL_GPL(crypto_put_default_rng); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("Random Number Genertor"); +MODULE_DESCRIPTION("Random Number Generator"); diff --git a/crypto/sha1_generic.c b/crypto/sha1_generic.c index 9efef20..0416091 100644 --- a/crypto/sha1_generic.c +++ b/crypto/sha1_generic.c @@ -25,31 +25,21 @@ #include <crypto/sha.h> #include <asm/byteorder.h> -struct sha1_ctx { - u64 count; - u32 state[5]; - u8 buffer[64]; -}; - static int sha1_init(struct shash_desc *desc) { - struct sha1_ctx *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); - static const struct sha1_ctx initstate = { - 0, - { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, - { 0, } + *sctx = (struct sha1_state){ + .state = { SHA1_H0, SHA1_H1, SHA1_H2, SHA1_H3, SHA1_H4 }, }; - *sctx = initstate; - return 0; } static int sha1_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha1_ctx *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); unsigned int partial, done; const u8 *src; @@ -85,7 +75,7 @@ static int sha1_update(struct shash_desc *desc, const u8 *data, /* Add padding and return the message digest. */ static int sha1_final(struct shash_desc *desc, u8 *out) { - struct sha1_ctx *sctx = shash_desc_ctx(desc); + struct sha1_state *sctx = shash_desc_ctx(desc); __be32 *dst = (__be32 *)out; u32 i, index, padlen; __be64 bits; @@ -111,12 +101,31 @@ static int sha1_final(struct shash_desc *desc, u8 *out) return 0; } +static int sha1_export(struct shash_desc *desc, void *out) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int sha1_import(struct shash_desc *desc, const void *in) +{ + struct sha1_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + static struct shash_alg alg = { .digestsize = SHA1_DIGEST_SIZE, .init = sha1_init, .update = sha1_update, .final = sha1_final, - .descsize = sizeof(struct sha1_ctx), + .export = sha1_export, + .import = sha1_import, + .descsize = sizeof(struct sha1_state), + .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name= "sha1-generic", diff --git a/crypto/sha256_generic.c b/crypto/sha256_generic.c index 6349d83..c48459e 100644 --- a/crypto/sha256_generic.c +++ b/crypto/sha256_generic.c @@ -25,12 +25,6 @@ #include <crypto/sha.h> #include <asm/byteorder.h> -struct sha256_ctx { - u32 count[2]; - u32 state[8]; - u8 buf[128]; -}; - static inline u32 Ch(u32 x, u32 y, u32 z) { return z ^ (x & (y ^ z)); @@ -222,7 +216,7 @@ static void sha256_transform(u32 *state, const u8 *input) static int sha224_init(struct shash_desc *desc) { - struct sha256_ctx *sctx = shash_desc_ctx(desc); + struct sha256_state *sctx = shash_desc_ctx(desc); sctx->state[0] = SHA224_H0; sctx->state[1] = SHA224_H1; sctx->state[2] = SHA224_H2; @@ -231,15 +225,14 @@ static int sha224_init(struct shash_desc *desc) sctx->state[5] = SHA224_H5; sctx->state[6] = SHA224_H6; sctx->state[7] = SHA224_H7; - sctx->count[0] = 0; - sctx->count[1] = 0; + sctx->count = 0; return 0; } static int sha256_init(struct shash_desc *desc) { - struct sha256_ctx *sctx = shash_desc_ctx(desc); + struct sha256_state *sctx = shash_desc_ctx(desc); sctx->state[0] = SHA256_H0; sctx->state[1] = SHA256_H1; sctx->state[2] = SHA256_H2; @@ -248,7 +241,7 @@ static int sha256_init(struct shash_desc *desc) sctx->state[5] = SHA256_H5; sctx->state[6] = SHA256_H6; sctx->state[7] = SHA256_H7; - sctx->count[0] = sctx->count[1] = 0; + sctx->count = 0; return 0; } @@ -256,58 +249,54 @@ static int sha256_init(struct shash_desc *desc) static int sha256_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha256_ctx *sctx = shash_desc_ctx(desc); - unsigned int i, index, part_len; - - /* Compute number of bytes mod 128 */ - index = (unsigned int)((sctx->count[0] >> 3) & 0x3f); - - /* Update number of bits */ - if ((sctx->count[0] += (len << 3)) < (len << 3)) { - sctx->count[1]++; - sctx->count[1] += (len >> 29); - } - - part_len = 64 - index; - - /* Transform as many times as possible. */ - if (len >= part_len) { - memcpy(&sctx->buf[index], data, part_len); - sha256_transform(sctx->state, sctx->buf); - - for (i = part_len; i + 63 < len; i += 64) - sha256_transform(sctx->state, &data[i]); - index = 0; - } else { - i = 0; + struct sha256_state *sctx = shash_desc_ctx(desc); + unsigned int partial, done; + const u8 *src; + + partial = sctx->count & 0x3f; + sctx->count += len; + done = 0; + src = data; + + if ((partial + len) > 63) { + if (partial) { + done = -partial; + memcpy(sctx->buf + partial, data, done + 64); + src = sctx->buf; + } + + do { + sha256_transform(sctx->state, src); + done += 64; + src = data + done; + } while (done + 63 < len); + + partial = 0; } - - /* Buffer remaining input */ - memcpy(&sctx->buf[index], &data[i], len-i); + memcpy(sctx->buf + partial, src, len - done); return 0; } static int sha256_final(struct shash_desc *desc, u8 *out) { - struct sha256_ctx *sctx = shash_desc_ctx(desc); + struct sha256_state *sctx = shash_desc_ctx(desc); __be32 *dst = (__be32 *)out; - __be32 bits[2]; + __be64 bits; unsigned int index, pad_len; int i; static const u8 padding[64] = { 0x80, }; /* Save number of bits */ - bits[1] = cpu_to_be32(sctx->count[0]); - bits[0] = cpu_to_be32(sctx->count[1]); + bits = cpu_to_be64(sctx->count << 3); /* Pad out to 56 mod 64. */ - index = (sctx->count[0] >> 3) & 0x3f; + index = sctx->count & 0x3f; pad_len = (index < 56) ? (56 - index) : ((64+56) - index); sha256_update(desc, padding, pad_len); /* Append length (before padding) */ - sha256_update(desc, (const u8 *)bits, sizeof(bits)); + sha256_update(desc, (const u8 *)&bits, sizeof(bits)); /* Store state in digest */ for (i = 0; i < 8; i++) @@ -331,12 +320,31 @@ static int sha224_final(struct shash_desc *desc, u8 *hash) return 0; } +static int sha256_export(struct shash_desc *desc, void *out) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(out, sctx, sizeof(*sctx)); + return 0; +} + +static int sha256_import(struct shash_desc *desc, const void *in) +{ + struct sha256_state *sctx = shash_desc_ctx(desc); + + memcpy(sctx, in, sizeof(*sctx)); + return 0; +} + static struct shash_alg sha256 = { .digestsize = SHA256_DIGEST_SIZE, .init = sha256_init, .update = sha256_update, .final = sha256_final, - .descsize = sizeof(struct sha256_ctx), + .export = sha256_export, + .import = sha256_import, + .descsize = sizeof(struct sha256_state), + .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name= "sha256-generic", @@ -351,7 +359,7 @@ static struct shash_alg sha224 = { .init = sha224_init, .update = sha256_update, .final = sha224_final, - .descsize = sizeof(struct sha256_ctx), + .descsize = sizeof(struct sha256_state), .base = { .cra_name = "sha224", .cra_driver_name= "sha224-generic", diff --git a/crypto/sha512_generic.c b/crypto/sha512_generic.c index 3bea38d..9ed9f60 100644 --- a/crypto/sha512_generic.c +++ b/crypto/sha512_generic.c @@ -21,12 +21,6 @@ #include <linux/percpu.h> #include <asm/byteorder.h> -struct sha512_ctx { - u64 state[8]; - u32 count[4]; - u8 buf[128]; -}; - static DEFINE_PER_CPU(u64[80], msg_schedule); static inline u64 Ch(u64 x, u64 y, u64 z) @@ -141,7 +135,7 @@ sha512_transform(u64 *state, const u8 *input) static int sha512_init(struct shash_desc *desc) { - struct sha512_ctx *sctx = shash_desc_ctx(desc); + struct sha512_state *sctx = shash_desc_ctx(desc); sctx->state[0] = SHA512_H0; sctx->state[1] = SHA512_H1; sctx->state[2] = SHA512_H2; @@ -150,7 +144,7 @@ sha512_init(struct shash_desc *desc) sctx->state[5] = SHA512_H5; sctx->state[6] = SHA512_H6; sctx->state[7] = SHA512_H7; - sctx->count[0] = sctx->count[1] = sctx->count[2] = sctx->count[3] = 0; + sctx->count[0] = sctx->count[1] = 0; return 0; } @@ -158,7 +152,7 @@ sha512_init(struct shash_desc *desc) static int sha384_init(struct shash_desc *desc) { - struct sha512_ctx *sctx = shash_desc_ctx(desc); + struct sha512_state *sctx = shash_desc_ctx(desc); sctx->state[0] = SHA384_H0; sctx->state[1] = SHA384_H1; sctx->state[2] = SHA384_H2; @@ -167,7 +161,7 @@ sha384_init(struct shash_desc *desc) sctx->state[5] = SHA384_H5; sctx->state[6] = SHA384_H6; sctx->state[7] = SHA384_H7; - sctx->count[0] = sctx->count[1] = sctx->count[2] = sctx->count[3] = 0; + sctx->count[0] = sctx->count[1] = 0; return 0; } @@ -175,20 +169,16 @@ sha384_init(struct shash_desc *desc) static int sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) { - struct sha512_ctx *sctx = shash_desc_ctx(desc); + struct sha512_state *sctx = shash_desc_ctx(desc); unsigned int i, index, part_len; /* Compute number of bytes mod 128 */ - index = (unsigned int)((sctx->count[0] >> 3) & 0x7F); - - /* Update number of bits */ - if ((sctx->count[0] += (len << 3)) < (len << 3)) { - if ((sctx->count[1] += 1) < 1) - if ((sctx->count[2] += 1) < 1) - sctx->count[3]++; - sctx->count[1] += (len >> 29); - } + index = sctx->count[0] & 0x7f; + + /* Update number of bytes */ + if (!(sctx->count[0] += len)) + sctx->count[1]++; part_len = 128 - index; @@ -214,21 +204,19 @@ sha512_update(struct shash_desc *desc, const u8 *data, unsigned int len) static int sha512_final(struct shash_desc *desc, u8 *hash) { - struct sha512_ctx *sctx = shash_desc_ctx(desc); + struct sha512_state *sctx = shash_desc_ctx(desc); static u8 padding[128] = { 0x80, }; __be64 *dst = (__be64 *)hash; - __be32 bits[4]; + __be64 bits[2]; unsigned int index, pad_len; int i; /* Save number of bits */ - bits[3] = cpu_to_be32(sctx->count[0]); - bits[2] = cpu_to_be32(sctx->count[1]); - bits[1] = cpu_to_be32(sctx->count[2]); - bits[0] = cpu_to_be32(sctx->count[3]); + bits[1] = cpu_to_be64(sctx->count[0] << 3); + bits[0] = cpu_to_be64(sctx->count[1] << 3 | sctx->count[0] >> 61); /* Pad out to 112 mod 128. */ - index = (sctx->count[0] >> 3) & 0x7f; + index = sctx->count[0] & 0x7f; pad_len = (index < 112) ? (112 - index) : ((128+112) - index); sha512_update(desc, padding, pad_len); @@ -240,7 +228,7 @@ sha512_final(struct shash_desc *desc, u8 *hash) dst[i] = cpu_to_be64(sctx->state[i]); /* Zeroize sensitive information. */ - memset(sctx, 0, sizeof(struct sha512_ctx)); + memset(sctx, 0, sizeof(struct sha512_state)); return 0; } @@ -262,7 +250,7 @@ static struct shash_alg sha512 = { .init = sha512_init, .update = sha512_update, .final = sha512_final, - .descsize = sizeof(struct sha512_ctx), + .descsize = sizeof(struct sha512_state), .base = { .cra_name = "sha512", .cra_flags = CRYPTO_ALG_TYPE_SHASH, @@ -276,7 +264,7 @@ static struct shash_alg sha384 = { .init = sha384_init, .update = sha512_update, .final = sha384_final, - .descsize = sizeof(struct sha512_ctx), + .descsize = sizeof(struct sha512_state), .base = { .cra_name = "sha384", .cra_flags = CRYPTO_ALG_TYPE_SHASH, diff --git a/crypto/shash.c b/crypto/shash.c index 2ccc8b0..91f7b9d 100644 --- a/crypto/shash.c +++ b/crypto/shash.c @@ -22,6 +22,12 @@ static const struct crypto_type crypto_shash_type; +static int shash_no_setkey(struct crypto_shash *tfm, const u8 *key, + unsigned int keylen) +{ + return -ENOSYS; +} + static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key, unsigned int keylen) { @@ -39,8 +45,7 @@ static int shash_setkey_unaligned(struct crypto_shash *tfm, const u8 *key, alignbuffer = (u8 *)ALIGN((unsigned long)buffer, alignmask + 1); memcpy(alignbuffer, key, keylen); err = shash->setkey(tfm, alignbuffer, keylen); - memset(alignbuffer, 0, keylen); - kfree(buffer); + kzfree(buffer); return err; } @@ -50,9 +55,6 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, struct shash_alg *shash = crypto_shash_alg(tfm); unsigned long alignmask = crypto_shash_alignmask(tfm); - if (!shash->setkey) - return -ENOSYS; - if ((unsigned long)key & alignmask) return shash_setkey_unaligned(tfm, key, keylen); @@ -74,15 +76,19 @@ static int shash_update_unaligned(struct shash_desc *desc, const u8 *data, unsigned long alignmask = crypto_shash_alignmask(tfm); unsigned int unaligned_len = alignmask + 1 - ((unsigned long)data & alignmask); - u8 buf[shash_align_buffer_size(unaligned_len, alignmask)] + u8 ubuf[shash_align_buffer_size(unaligned_len, alignmask)] __attribute__ ((aligned)); + u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1); + int err; if (unaligned_len > len) unaligned_len = len; memcpy(buf, data, unaligned_len); + err = shash->update(desc, buf, unaligned_len); + memset(buf, 0, unaligned_len); - return shash->update(desc, buf, unaligned_len) ?: + return err ?: shash->update(desc, data + unaligned_len, len - unaligned_len); } @@ -106,12 +112,19 @@ static int shash_final_unaligned(struct shash_desc *desc, u8 *out) unsigned long alignmask = crypto_shash_alignmask(tfm); struct shash_alg *shash = crypto_shash_alg(tfm); unsigned int ds = crypto_shash_digestsize(tfm); - u8 buf[shash_align_buffer_size(ds, alignmask)] + u8 ubuf[shash_align_buffer_size(ds, alignmask)] __attribute__ ((aligned)); + u8 *buf = PTR_ALIGN(&ubuf[0], alignmask + 1); int err; err = shash->final(desc, buf); + if (err) + goto out; + memcpy(out, buf, ds); + +out: + memset(buf, 0, ds); return err; } @@ -142,8 +155,7 @@ int crypto_shash_finup(struct shash_desc *desc, const u8 *data, struct shash_alg *shash = crypto_shash_alg(tfm); unsigned long alignmask = crypto_shash_alignmask(tfm); - if (((unsigned long)data | (unsigned long)out) & alignmask || - !shash->finup) + if (((unsigned long)data | (unsigned long)out) & alignmask) return shash_finup_unaligned(desc, data, len, out); return shash->finup(desc, data, len, out); @@ -154,8 +166,7 @@ static int shash_digest_unaligned(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { return crypto_shash_init(desc) ?: - crypto_shash_update(desc, data, len) ?: - crypto_shash_final(desc, out); + crypto_shash_finup(desc, data, len, out); } int crypto_shash_digest(struct shash_desc *desc, const u8 *data, @@ -165,27 +176,24 @@ int crypto_shash_digest(struct shash_desc *desc, const u8 *data, struct shash_alg *shash = crypto_shash_alg(tfm); unsigned long alignmask = crypto_shash_alignmask(tfm); - if (((unsigned long)data | (unsigned long)out) & alignmask || - !shash->digest) + if (((unsigned long)data | (unsigned long)out) & alignmask) return shash_digest_unaligned(desc, data, len, out); return shash->digest(desc, data, len, out); } EXPORT_SYMBOL_GPL(crypto_shash_digest); -int crypto_shash_import(struct shash_desc *desc, const u8 *in) +static int shash_default_export(struct shash_desc *desc, void *out) { - struct crypto_shash *tfm = desc->tfm; - struct shash_alg *alg = crypto_shash_alg(tfm); - - memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(tfm)); - - if (alg->reinit) - alg->reinit(desc); + memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(desc->tfm)); + return 0; +} +static int shash_default_import(struct shash_desc *desc, const void *in) +{ + memcpy(shash_desc_ctx(desc), in, crypto_shash_descsize(desc->tfm)); return 0; } -EXPORT_SYMBOL_GPL(crypto_shash_import); static int shash_async_setkey(struct crypto_ahash *tfm, const u8 *key, unsigned int keylen) @@ -206,9 +214,8 @@ static int shash_async_init(struct ahash_request *req) return crypto_shash_init(desc); } -static int shash_async_update(struct ahash_request *req) +int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc) { - struct shash_desc *desc = ahash_request_ctx(req); struct crypto_hash_walk walk; int nbytes; @@ -218,13 +225,51 @@ static int shash_async_update(struct ahash_request *req) return nbytes; } +EXPORT_SYMBOL_GPL(shash_ahash_update); + +static int shash_async_update(struct ahash_request *req) +{ + return shash_ahash_update(req, ahash_request_ctx(req)); +} static int shash_async_final(struct ahash_request *req) { return crypto_shash_final(ahash_request_ctx(req), req->result); } -static int shash_async_digest(struct ahash_request *req) +int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc) +{ + struct crypto_hash_walk walk; + int nbytes; + + nbytes = crypto_hash_walk_first(req, &walk); + if (!nbytes) + return crypto_shash_final(desc, req->result); + + do { + nbytes = crypto_hash_walk_last(&walk) ? + crypto_shash_finup(desc, walk.data, nbytes, + req->result) : + crypto_shash_update(desc, walk.data, nbytes); + nbytes = crypto_hash_walk_done(&walk, nbytes); + } while (nbytes > 0); + + return nbytes; +} +EXPORT_SYMBOL_GPL(shash_ahash_finup); + +static int shash_async_finup(struct ahash_request *req) +{ + struct crypto_shash **ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct shash_desc *desc = ahash_request_ctx(req); + + desc->tfm = *ctx; + desc->flags = req->base.flags; + + return shash_ahash_finup(req, desc); +} + +int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc) { struct scatterlist *sg = req->src; unsigned int offset = sg->offset; @@ -232,34 +277,40 @@ static int shash_async_digest(struct ahash_request *req) int err; if (nbytes < min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset)) { - struct crypto_shash **ctx = - crypto_ahash_ctx(crypto_ahash_reqtfm(req)); - struct shash_desc *desc = ahash_request_ctx(req); void *data; - desc->tfm = *ctx; - desc->flags = req->base.flags; - data = crypto_kmap(sg_page(sg), 0); err = crypto_shash_digest(desc, data + offset, nbytes, req->result); crypto_kunmap(data, 0); crypto_yield(desc->flags); - goto out; - } + } else + err = crypto_shash_init(desc) ?: + shash_ahash_finup(req, desc); - err = shash_async_init(req); - if (err) - goto out; + return err; +} +EXPORT_SYMBOL_GPL(shash_ahash_digest); - err = shash_async_update(req); - if (err) - goto out; +static int shash_async_digest(struct ahash_request *req) +{ + struct crypto_shash **ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); + struct shash_desc *desc = ahash_request_ctx(req); - err = shash_async_final(req); + desc->tfm = *ctx; + desc->flags = req->base.flags; -out: - return err; + return shash_ahash_digest(req, desc); +} + +static int shash_async_export(struct ahash_request *req, void *out) +{ + return crypto_shash_export(ahash_request_ctx(req), out); +} + +static int shash_async_import(struct ahash_request *req, const void *in) +{ + return crypto_shash_import(ahash_request_ctx(req), in); } static void crypto_exit_shash_ops_async(struct crypto_tfm *tfm) @@ -269,11 +320,11 @@ static void crypto_exit_shash_ops_async(struct crypto_tfm *tfm) crypto_free_shash(*ctx); } -static int crypto_init_shash_ops_async(struct crypto_tfm *tfm) +int crypto_init_shash_ops_async(struct crypto_tfm *tfm) { struct crypto_alg *calg = tfm->__crt_alg; struct shash_alg *alg = __crypto_shash_alg(calg); - struct ahash_tfm *crt = &tfm->crt_ahash; + struct crypto_ahash *crt = __crypto_ahash_cast(tfm); struct crypto_shash **ctx = crypto_tfm_ctx(tfm); struct crypto_shash *shash; @@ -291,11 +342,17 @@ static int crypto_init_shash_ops_async(struct crypto_tfm *tfm) crt->init = shash_async_init; crt->update = shash_async_update; - crt->final = shash_async_final; + crt->final = shash_async_final; + crt->finup = shash_async_finup; crt->digest = shash_async_digest; - crt->setkey = shash_async_setkey; - crt->digestsize = alg->digestsize; + if (alg->setkey) + crt->setkey = shash_async_setkey; + if (alg->export) + crt->export = shash_async_export; + if (alg->import) + crt->import = shash_async_import; + crt->reqsize = sizeof(struct shash_desc) + crypto_shash_descsize(shash); return 0; @@ -304,14 +361,16 @@ static int crypto_init_shash_ops_async(struct crypto_tfm *tfm) static int shash_compat_setkey(struct crypto_hash *tfm, const u8 *key, unsigned int keylen) { - struct shash_desc *desc = crypto_hash_ctx(tfm); + struct shash_desc **descp = crypto_hash_ctx(tfm); + struct shash_desc *desc = *descp; return crypto_shash_setkey(desc->tfm, key, keylen); } static int shash_compat_init(struct hash_desc *hdesc) { - struct shash_desc *desc = crypto_hash_ctx(hdesc->tfm); + struct shash_desc **descp = crypto_hash_ctx(hdesc->tfm); + struct shash_desc *desc = *descp; desc->flags = hdesc->flags; @@ -321,7 +380,8 @@ static int shash_compat_init(struct hash_desc *hdesc) static int shash_compat_update(struct hash_desc *hdesc, struct scatterlist *sg, unsigned int len) { - struct shash_desc *desc = crypto_hash_ctx(hdesc->tfm); + struct shash_desc **descp = crypto_hash_ctx(hdesc->tfm); + struct shash_desc *desc = *descp; struct crypto_hash_walk walk; int nbytes; @@ -334,7 +394,9 @@ static int shash_compat_update(struct hash_desc *hdesc, struct scatterlist *sg, static int shash_compat_final(struct hash_desc *hdesc, u8 *out) { - return crypto_shash_final(crypto_hash_ctx(hdesc->tfm), out); + struct shash_desc **descp = crypto_hash_ctx(hdesc->tfm); + + return crypto_shash_final(*descp, out); } static int shash_compat_digest(struct hash_desc *hdesc, struct scatterlist *sg, @@ -344,7 +406,8 @@ static int shash_compat_digest(struct hash_desc *hdesc, struct scatterlist *sg, int err; if (nbytes < min(sg->length, ((unsigned int)(PAGE_SIZE)) - offset)) { - struct shash_desc *desc = crypto_hash_ctx(hdesc->tfm); + struct shash_desc **descp = crypto_hash_ctx(hdesc->tfm); + struct shash_desc *desc = *descp; void *data; desc->flags = hdesc->flags; @@ -372,9 +435,11 @@ out: static void crypto_exit_shash_ops_compat(struct crypto_tfm *tfm) { - struct shash_desc *desc= crypto_tfm_ctx(tfm); + struct shash_desc **descp = crypto_tfm_ctx(tfm); + struct shash_desc *desc = *descp; crypto_free_shash(desc->tfm); + kzfree(desc); } static int crypto_init_shash_ops_compat(struct crypto_tfm *tfm) @@ -382,8 +447,9 @@ static int crypto_init_shash_ops_compat(struct crypto_tfm *tfm) struct hash_tfm *crt = &tfm->crt_hash; struct crypto_alg *calg = tfm->__crt_alg; struct shash_alg *alg = __crypto_shash_alg(calg); - struct shash_desc *desc = crypto_tfm_ctx(tfm); + struct shash_desc **descp = crypto_tfm_ctx(tfm); struct crypto_shash *shash; + struct shash_desc *desc; if (!crypto_mod_get(calg)) return -EAGAIN; @@ -394,6 +460,14 @@ static int crypto_init_shash_ops_compat(struct crypto_tfm *tfm) return PTR_ERR(shash); } + desc = kmalloc(sizeof(*desc) + crypto_shash_descsize(shash), + GFP_KERNEL); + if (!desc) { + crypto_free_shash(shash); + return -ENOMEM; + } + + *descp = desc; desc->tfm = shash; tfm->exit = crypto_exit_shash_ops_compat; @@ -413,8 +487,6 @@ static int crypto_init_shash_ops(struct crypto_tfm *tfm, u32 type, u32 mask) switch (mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_HASH_MASK: return crypto_init_shash_ops_compat(tfm); - case CRYPTO_ALG_TYPE_AHASH_MASK: - return crypto_init_shash_ops_async(tfm); } return -EINVAL; @@ -423,26 +495,23 @@ static int crypto_init_shash_ops(struct crypto_tfm *tfm, u32 type, u32 mask) static unsigned int crypto_shash_ctxsize(struct crypto_alg *alg, u32 type, u32 mask) { - struct shash_alg *salg = __crypto_shash_alg(alg); - switch (mask & CRYPTO_ALG_TYPE_MASK) { case CRYPTO_ALG_TYPE_HASH_MASK: - return sizeof(struct shash_desc) + salg->descsize; - case CRYPTO_ALG_TYPE_AHASH_MASK: - return sizeof(struct crypto_shash *); + return sizeof(struct shash_desc *); } return 0; } -static int crypto_shash_init_tfm(struct crypto_tfm *tfm, - const struct crypto_type *frontend) +static int crypto_shash_init_tfm(struct crypto_tfm *tfm) { + struct crypto_shash *hash = __crypto_shash_cast(tfm); + + hash->descsize = crypto_shash_alg(hash)->descsize; return 0; } -static unsigned int crypto_shash_extsize(struct crypto_alg *alg, - const struct crypto_type *frontend) +static unsigned int crypto_shash_extsize(struct crypto_alg *alg) { return alg->cra_ctxsize; } @@ -456,7 +525,6 @@ static void crypto_shash_show(struct seq_file *m, struct crypto_alg *alg) seq_printf(m, "type : shash\n"); seq_printf(m, "blocksize : %u\n", alg->cra_blocksize); seq_printf(m, "digestsize : %u\n", salg->digestsize); - seq_printf(m, "descsize : %u\n", salg->descsize); } static const struct crypto_type crypto_shash_type = { @@ -480,18 +548,43 @@ struct crypto_shash *crypto_alloc_shash(const char *alg_name, u32 type, } EXPORT_SYMBOL_GPL(crypto_alloc_shash); -int crypto_register_shash(struct shash_alg *alg) +static int shash_prepare_alg(struct shash_alg *alg) { struct crypto_alg *base = &alg->base; if (alg->digestsize > PAGE_SIZE / 8 || - alg->descsize > PAGE_SIZE / 8) + alg->descsize > PAGE_SIZE / 8 || + alg->statesize > PAGE_SIZE / 8) return -EINVAL; base->cra_type = &crypto_shash_type; base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; base->cra_flags |= CRYPTO_ALG_TYPE_SHASH; + if (!alg->finup) + alg->finup = shash_finup_unaligned; + if (!alg->digest) + alg->digest = shash_digest_unaligned; + if (!alg->export) { + alg->export = shash_default_export; + alg->import = shash_default_import; + alg->statesize = alg->descsize; + } + if (!alg->setkey) + alg->setkey = shash_no_setkey; + + return 0; +} + +int crypto_register_shash(struct shash_alg *alg) +{ + struct crypto_alg *base = &alg->base; + int err; + + err = shash_prepare_alg(alg); + if (err) + return err; + return crypto_register_alg(base); } EXPORT_SYMBOL_GPL(crypto_register_shash); @@ -502,5 +595,44 @@ int crypto_unregister_shash(struct shash_alg *alg) } EXPORT_SYMBOL_GPL(crypto_unregister_shash); +int shash_register_instance(struct crypto_template *tmpl, + struct shash_instance *inst) +{ + int err; + + err = shash_prepare_alg(&inst->alg); + if (err) + return err; + + return crypto_register_instance(tmpl, shash_crypto_instance(inst)); +} +EXPORT_SYMBOL_GPL(shash_register_instance); + +void shash_free_instance(struct crypto_instance *inst) +{ + crypto_drop_spawn(crypto_instance_ctx(inst)); + kfree(shash_instance(inst)); +} +EXPORT_SYMBOL_GPL(shash_free_instance); + +int crypto_init_shash_spawn(struct crypto_shash_spawn *spawn, + struct shash_alg *alg, + struct crypto_instance *inst) +{ + return crypto_init_spawn2(&spawn->base, &alg->base, inst, + &crypto_shash_type); +} +EXPORT_SYMBOL_GPL(crypto_init_shash_spawn); + +struct shash_alg *shash_attr_alg(struct rtattr *rta, u32 type, u32 mask) +{ + struct crypto_alg *alg; + + alg = crypto_attr_alg2(rta, &crypto_shash_type, type, mask); + return IS_ERR(alg) ? ERR_CAST(alg) : + container_of(alg, struct shash_alg, base); +} +EXPORT_SYMBOL_GPL(shash_attr_alg); + MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Synchronous cryptographic hash type"); diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index d59ba50..aa3f84c 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -45,6 +45,9 @@ */ static unsigned int sec; +static char *alg = NULL; +static u32 type; +static u32 mask; static int mode; static char *tvmem[TVMEMSIZE]; @@ -716,6 +719,10 @@ static int do_test(int m) ret += tcrypt_test("hmac(rmd160)"); break; + case 109: + ret += tcrypt_test("vmac(aes)"); + break; + case 150: ret += tcrypt_test("ansi_cprng"); break; @@ -885,6 +892,12 @@ static int do_test(int m) return ret; } +static int do_alg_test(const char *alg, u32 type, u32 mask) +{ + return crypto_has_alg(alg, type, mask ?: CRYPTO_ALG_TYPE_MASK) ? + 0 : -ENOENT; +} + static int __init tcrypt_mod_init(void) { int err = -ENOMEM; @@ -896,7 +909,11 @@ static int __init tcrypt_mod_init(void) goto err_free_tv; } - err = do_test(mode); + if (alg) + err = do_alg_test(alg, type, mask); + else + err = do_test(mode); + if (err) { printk(KERN_ERR "tcrypt: one or more tests failed!\n"); goto err_free_tv; @@ -928,6 +945,9 @@ static void __exit tcrypt_mod_fini(void) { } module_init(tcrypt_mod_init); module_exit(tcrypt_mod_fini); +module_param(alg, charp, 0); +module_param(type, uint, 0); +module_param(mask, uint, 0); module_param(mode, int, 0); module_param(sec, uint, 0); MODULE_PARM_DESC(sec, "Length in seconds of speed tests " diff --git a/crypto/testmgr.c b/crypto/testmgr.c index e9e9d84..6d5b746 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -190,10 +190,6 @@ static int test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, hash_buff = xbuf[0]; - ret = -EINVAL; - if (WARN_ON(template[i].psize > PAGE_SIZE)) - goto out; - memcpy(hash_buff, template[i].plaintext, template[i].psize); sg_init_one(&sg[0], hash_buff, template[i].psize); @@ -2252,6 +2248,15 @@ static const struct alg_test_desc alg_test_descs[] = { } } }, { + .alg = "vmac(aes)", + .test = alg_test_hash, + .suite = { + .hash = { + .vecs = aes_vmac128_tv_template, + .count = VMAC_AES_TEST_VECTORS + } + } + }, { .alg = "wp256", .test = alg_test_hash, .suite = { @@ -2348,6 +2353,7 @@ static int alg_find_test(const char *alg) int alg_test(const char *driver, const char *alg, u32 type, u32 mask) { int i; + int j; int rc; if ((type & CRYPTO_ALG_TYPE_MASK) == CRYPTO_ALG_TYPE_CIPHER) { @@ -2369,14 +2375,22 @@ int alg_test(const char *driver, const char *alg, u32 type, u32 mask) } i = alg_find_test(alg); - if (i < 0) + j = alg_find_test(driver); + if (i < 0 && j < 0) goto notest; - if (fips_enabled && !alg_test_descs[i].fips_allowed) + if (fips_enabled && ((i >= 0 && !alg_test_descs[i].fips_allowed) || + (j >= 0 && !alg_test_descs[j].fips_allowed))) goto non_fips_alg; - rc = alg_test_descs[i].test(alg_test_descs + i, driver, - type, mask); + rc = 0; + if (i >= 0) + rc |= alg_test_descs[i].test(alg_test_descs + i, driver, + type, mask); + if (j >= 0) + rc |= alg_test_descs[j].test(alg_test_descs + j, driver, + type, mask); + test_done: if (fips_enabled && rc) panic("%s: %s alg self test failed in fips mode!\n", driver, alg); diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 6931622..9963b18 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -1654,6 +1654,22 @@ static struct hash_testvec aes_xcbc128_tv_template[] = { } }; +#define VMAC_AES_TEST_VECTORS 1 +static char vmac_string[128] = {'\x01', '\x01', '\x01', '\x01', + '\x02', '\x03', '\x02', '\x02', + '\x02', '\x04', '\x01', '\x07', + '\x04', '\x01', '\x04', '\x03',}; +static struct hash_testvec aes_vmac128_tv_template[] = { + { + .key = "\x00\x01\x02\x03\x04\x05\x06\x07" + "\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", + .plaintext = vmac_string, + .digest = "\xcb\xd7\x8a\xfd\xb7\x33\x79\xe7", + .psize = 128, + .ksize = 16, + }, +}; + /* * SHA384 HMAC test vectors from RFC4231 */ diff --git a/crypto/vmac.c b/crypto/vmac.c new file mode 100644 index 0000000..0a9468e --- /dev/null +++ b/crypto/vmac.c @@ -0,0 +1,678 @@ +/* + * Modified to interface to the Linux kernel + * Copyright (c) 2009, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + */ + +/* -------------------------------------------------------------------------- + * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai. + * This implementation is herby placed in the public domain. + * The authors offers no warranty. Use at your own risk. + * Please send bug reports to the authors. + * Last modified: 17 APR 08, 1700 PDT + * ----------------------------------------------------------------------- */ + +#include <linux/init.h> +#include <linux/types.h> +#include <linux/crypto.h> +#include <linux/scatterlist.h> +#include <asm/byteorder.h> +#include <crypto/scatterwalk.h> +#include <crypto/vmac.h> +#include <crypto/internal/hash.h> + +/* + * Constants and masks + */ +#define UINT64_C(x) x##ULL +const u64 p64 = UINT64_C(0xfffffffffffffeff); /* 2^64 - 257 prime */ +const u64 m62 = UINT64_C(0x3fffffffffffffff); /* 62-bit mask */ +const u64 m63 = UINT64_C(0x7fffffffffffffff); /* 63-bit mask */ +const u64 m64 = UINT64_C(0xffffffffffffffff); /* 64-bit mask */ +const u64 mpoly = UINT64_C(0x1fffffff1fffffff); /* Poly key mask */ + +#ifdef __LITTLE_ENDIAN +#define INDEX_HIGH 1 +#define INDEX_LOW 0 +#else +#define INDEX_HIGH 0 +#define INDEX_LOW 1 +#endif + +/* + * The following routines are used in this implementation. They are + * written via macros to simulate zero-overhead call-by-reference. + * + * MUL64: 64x64->128-bit multiplication + * PMUL64: assumes top bits cleared on inputs + * ADD128: 128x128->128-bit addition + */ + +#define ADD128(rh, rl, ih, il) \ + do { \ + u64 _il = (il); \ + (rl) += (_il); \ + if ((rl) < (_il)) \ + (rh)++; \ + (rh) += (ih); \ + } while (0) + +#define MUL32(i1, i2) ((u64)(u32)(i1)*(u32)(i2)) + +#define PMUL64(rh, rl, i1, i2) /* Assumes m doesn't overflow */ \ + do { \ + u64 _i1 = (i1), _i2 = (i2); \ + u64 m = MUL32(_i1, _i2>>32) + MUL32(_i1>>32, _i2); \ + rh = MUL32(_i1>>32, _i2>>32); \ + rl = MUL32(_i1, _i2); \ + ADD128(rh, rl, (m >> 32), (m << 32)); \ + } while (0) + +#define MUL64(rh, rl, i1, i2) \ + do { \ + u64 _i1 = (i1), _i2 = (i2); \ + u64 m1 = MUL32(_i1, _i2>>32); \ + u64 m2 = MUL32(_i1>>32, _i2); \ + rh = MUL32(_i1>>32, _i2>>32); \ + rl = MUL32(_i1, _i2); \ + ADD128(rh, rl, (m1 >> 32), (m1 << 32)); \ + ADD128(rh, rl, (m2 >> 32), (m2 << 32)); \ + } while (0) + +/* + * For highest performance the L1 NH and L2 polynomial hashes should be + * carefully implemented to take advantage of one's target architechture. + * Here these two hash functions are defined multiple time; once for + * 64-bit architectures, once for 32-bit SSE2 architectures, and once + * for the rest (32-bit) architectures. + * For each, nh_16 *must* be defined (works on multiples of 16 bytes). + * Optionally, nh_vmac_nhbytes can be defined (for multiples of + * VMAC_NHBYTES), and nh_16_2 and nh_vmac_nhbytes_2 (versions that do two + * NH computations at once). + */ + +#ifdef CONFIG_64BIT + +#define nh_16(mp, kp, nw, rh, rl) \ + do { \ + int i; u64 th, tl; \ + rh = rl = 0; \ + for (i = 0; i < nw; i += 2) { \ + MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ + le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ + ADD128(rh, rl, th, tl); \ + } \ + } while (0) + +#define nh_16_2(mp, kp, nw, rh, rl, rh1, rl1) \ + do { \ + int i; u64 th, tl; \ + rh1 = rl1 = rh = rl = 0; \ + for (i = 0; i < nw; i += 2) { \ + MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ + le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i+2], \ + le64_to_cpup((mp)+i+1)+(kp)[i+3]); \ + ADD128(rh1, rl1, th, tl); \ + } \ + } while (0) + +#if (VMAC_NHBYTES >= 64) /* These versions do 64-bytes of message at a time */ +#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ + do { \ + int i; u64 th, tl; \ + rh = rl = 0; \ + for (i = 0; i < nw; i += 8) { \ + MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ + le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+2], \ + le64_to_cpup((mp)+i+3)+(kp)[i+3]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+4], \ + le64_to_cpup((mp)+i+5)+(kp)[i+5]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+6], \ + le64_to_cpup((mp)+i+7)+(kp)[i+7]); \ + ADD128(rh, rl, th, tl); \ + } \ + } while (0) + +#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh1, rl1) \ + do { \ + int i; u64 th, tl; \ + rh1 = rl1 = rh = rl = 0; \ + for (i = 0; i < nw; i += 8) { \ + MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i], \ + le64_to_cpup((mp)+i+1)+(kp)[i+1]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i)+(kp)[i+2], \ + le64_to_cpup((mp)+i+1)+(kp)[i+3]); \ + ADD128(rh1, rl1, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+2], \ + le64_to_cpup((mp)+i+3)+(kp)[i+3]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i+2)+(kp)[i+4], \ + le64_to_cpup((mp)+i+3)+(kp)[i+5]); \ + ADD128(rh1, rl1, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+4], \ + le64_to_cpup((mp)+i+5)+(kp)[i+5]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i+4)+(kp)[i+6], \ + le64_to_cpup((mp)+i+5)+(kp)[i+7]); \ + ADD128(rh1, rl1, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+6], \ + le64_to_cpup((mp)+i+7)+(kp)[i+7]); \ + ADD128(rh, rl, th, tl); \ + MUL64(th, tl, le64_to_cpup((mp)+i+6)+(kp)[i+8], \ + le64_to_cpup((mp)+i+7)+(kp)[i+9]); \ + ADD128(rh1, rl1, th, tl); \ + } \ + } while (0) +#endif + +#define poly_step(ah, al, kh, kl, mh, ml) \ + do { \ + u64 t1h, t1l, t2h, t2l, t3h, t3l, z = 0; \ + /* compute ab*cd, put bd into result registers */ \ + PMUL64(t3h, t3l, al, kh); \ + PMUL64(t2h, t2l, ah, kl); \ + PMUL64(t1h, t1l, ah, 2*kh); \ + PMUL64(ah, al, al, kl); \ + /* add 2 * ac to result */ \ + ADD128(ah, al, t1h, t1l); \ + /* add together ad + bc */ \ + ADD128(t2h, t2l, t3h, t3l); \ + /* now (ah,al), (t2l,2*t2h) need summing */ \ + /* first add the high registers, carrying into t2h */ \ + ADD128(t2h, ah, z, t2l); \ + /* double t2h and add top bit of ah */ \ + t2h = 2 * t2h + (ah >> 63); \ + ah &= m63; \ + /* now add the low registers */ \ + ADD128(ah, al, mh, ml); \ + ADD128(ah, al, z, t2h); \ + } while (0) + +#else /* ! CONFIG_64BIT */ + +#ifndef nh_16 +#define nh_16(mp, kp, nw, rh, rl) \ + do { \ + u64 t1, t2, m1, m2, t; \ + int i; \ + rh = rl = t = 0; \ + for (i = 0; i < nw; i += 2) { \ + t1 = le64_to_cpup(mp+i) + kp[i]; \ + t2 = le64_to_cpup(mp+i+1) + kp[i+1]; \ + m2 = MUL32(t1 >> 32, t2); \ + m1 = MUL32(t1, t2 >> 32); \ + ADD128(rh, rl, MUL32(t1 >> 32, t2 >> 32), \ + MUL32(t1, t2)); \ + rh += (u64)(u32)(m1 >> 32) \ + + (u32)(m2 >> 32); \ + t += (u64)(u32)m1 + (u32)m2; \ + } \ + ADD128(rh, rl, (t >> 32), (t << 32)); \ + } while (0) +#endif + +static void poly_step_func(u64 *ahi, u64 *alo, + const u64 *kh, const u64 *kl, + const u64 *mh, const u64 *ml) +{ +#define a0 (*(((u32 *)alo)+INDEX_LOW)) +#define a1 (*(((u32 *)alo)+INDEX_HIGH)) +#define a2 (*(((u32 *)ahi)+INDEX_LOW)) +#define a3 (*(((u32 *)ahi)+INDEX_HIGH)) +#define k0 (*(((u32 *)kl)+INDEX_LOW)) +#define k1 (*(((u32 *)kl)+INDEX_HIGH)) +#define k2 (*(((u32 *)kh)+INDEX_LOW)) +#define k3 (*(((u32 *)kh)+INDEX_HIGH)) + + u64 p, q, t; + u32 t2; + + p = MUL32(a3, k3); + p += p; + p += *(u64 *)mh; + p += MUL32(a0, k2); + p += MUL32(a1, k1); + p += MUL32(a2, k0); + t = (u32)(p); + p >>= 32; + p += MUL32(a0, k3); + p += MUL32(a1, k2); + p += MUL32(a2, k1); + p += MUL32(a3, k0); + t |= ((u64)((u32)p & 0x7fffffff)) << 32; + p >>= 31; + p += (u64)(((u32 *)ml)[INDEX_LOW]); + p += MUL32(a0, k0); + q = MUL32(a1, k3); + q += MUL32(a2, k2); + q += MUL32(a3, k1); + q += q; + p += q; + t2 = (u32)(p); + p >>= 32; + p += (u64)(((u32 *)ml)[INDEX_HIGH]); + p += MUL32(a0, k1); + p += MUL32(a1, k0); + q = MUL32(a2, k3); + q += MUL32(a3, k2); + q += q; + p += q; + *(u64 *)(alo) = (p << 32) | t2; + p >>= 32; + *(u64 *)(ahi) = p + t; + +#undef a0 +#undef a1 +#undef a2 +#undef a3 +#undef k0 +#undef k1 +#undef k2 +#undef k3 +} + +#define poly_step(ah, al, kh, kl, mh, ml) \ + poly_step_func(&(ah), &(al), &(kh), &(kl), &(mh), &(ml)) + +#endif /* end of specialized NH and poly definitions */ + +/* At least nh_16 is defined. Defined others as needed here */ +#ifndef nh_16_2 +#define nh_16_2(mp, kp, nw, rh, rl, rh2, rl2) \ + do { \ + nh_16(mp, kp, nw, rh, rl); \ + nh_16(mp, ((kp)+2), nw, rh2, rl2); \ + } while (0) +#endif +#ifndef nh_vmac_nhbytes +#define nh_vmac_nhbytes(mp, kp, nw, rh, rl) \ + nh_16(mp, kp, nw, rh, rl) +#endif +#ifndef nh_vmac_nhbytes_2 +#define nh_vmac_nhbytes_2(mp, kp, nw, rh, rl, rh2, rl2) \ + do { \ + nh_vmac_nhbytes(mp, kp, nw, rh, rl); \ + nh_vmac_nhbytes(mp, ((kp)+2), nw, rh2, rl2); \ + } while (0) +#endif + +static void vhash_abort(struct vmac_ctx *ctx) +{ + ctx->polytmp[0] = ctx->polykey[0] ; + ctx->polytmp[1] = ctx->polykey[1] ; + ctx->first_block_processed = 0; +} + +static u64 l3hash(u64 p1, u64 p2, + u64 k1, u64 k2, u64 len) +{ + u64 rh, rl, t, z = 0; + + /* fully reduce (p1,p2)+(len,0) mod p127 */ + t = p1 >> 63; + p1 &= m63; + ADD128(p1, p2, len, t); + /* At this point, (p1,p2) is at most 2^127+(len<<64) */ + t = (p1 > m63) + ((p1 == m63) && (p2 == m64)); + ADD128(p1, p2, z, t); + p1 &= m63; + + /* compute (p1,p2)/(2^64-2^32) and (p1,p2)%(2^64-2^32) */ + t = p1 + (p2 >> 32); + t += (t >> 32); + t += (u32)t > 0xfffffffeu; + p1 += (t >> 32); + p2 += (p1 << 32); + + /* compute (p1+k1)%p64 and (p2+k2)%p64 */ + p1 += k1; + p1 += (0 - (p1 < k1)) & 257; + p2 += k2; + p2 += (0 - (p2 < k2)) & 257; + + /* compute (p1+k1)*(p2+k2)%p64 */ + MUL64(rh, rl, p1, p2); + t = rh >> 56; + ADD128(t, rl, z, rh); + rh <<= 8; + ADD128(t, rl, z, rh); + t += t << 8; + rl += t; + rl += (0 - (rl < t)) & 257; + rl += (0 - (rl > p64-1)) & 257; + return rl; +} + +static void vhash_update(const unsigned char *m, + unsigned int mbytes, /* Pos multiple of VMAC_NHBYTES */ + struct vmac_ctx *ctx) +{ + u64 rh, rl, *mptr; + const u64 *kptr = (u64 *)ctx->nhkey; + int i; + u64 ch, cl; + u64 pkh = ctx->polykey[0]; + u64 pkl = ctx->polykey[1]; + + mptr = (u64 *)m; + i = mbytes / VMAC_NHBYTES; /* Must be non-zero */ + + ch = ctx->polytmp[0]; + cl = ctx->polytmp[1]; + + if (!ctx->first_block_processed) { + ctx->first_block_processed = 1; + nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); + rh &= m62; + ADD128(ch, cl, rh, rl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + i--; + } + + while (i--) { + nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); + rh &= m62; + poly_step(ch, cl, pkh, pkl, rh, rl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + } + + ctx->polytmp[0] = ch; + ctx->polytmp[1] = cl; +} + +static u64 vhash(unsigned char m[], unsigned int mbytes, + u64 *tagl, struct vmac_ctx *ctx) +{ + u64 rh, rl, *mptr; + const u64 *kptr = (u64 *)ctx->nhkey; + int i, remaining; + u64 ch, cl; + u64 pkh = ctx->polykey[0]; + u64 pkl = ctx->polykey[1]; + + mptr = (u64 *)m; + i = mbytes / VMAC_NHBYTES; + remaining = mbytes % VMAC_NHBYTES; + + if (ctx->first_block_processed) { + ch = ctx->polytmp[0]; + cl = ctx->polytmp[1]; + } else if (i) { + nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, ch, cl); + ch &= m62; + ADD128(ch, cl, pkh, pkl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + i--; + } else if (remaining) { + nh_16(mptr, kptr, 2*((remaining+15)/16), ch, cl); + ch &= m62; + ADD128(ch, cl, pkh, pkl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + goto do_l3; + } else {/* Empty String */ + ch = pkh; cl = pkl; + goto do_l3; + } + + while (i--) { + nh_vmac_nhbytes(mptr, kptr, VMAC_NHBYTES/8, rh, rl); + rh &= m62; + poly_step(ch, cl, pkh, pkl, rh, rl); + mptr += (VMAC_NHBYTES/sizeof(u64)); + } + if (remaining) { + nh_16(mptr, kptr, 2*((remaining+15)/16), rh, rl); + rh &= m62; + poly_step(ch, cl, pkh, pkl, rh, rl); + } + +do_l3: + vhash_abort(ctx); + remaining *= 8; + return l3hash(ch, cl, ctx->l3key[0], ctx->l3key[1], remaining); +} + +static u64 vmac(unsigned char m[], unsigned int mbytes, + unsigned char n[16], u64 *tagl, + struct vmac_ctx_t *ctx) +{ + u64 *in_n, *out_p; + u64 p, h; + int i; + + in_n = ctx->__vmac_ctx.cached_nonce; + out_p = ctx->__vmac_ctx.cached_aes; + + i = n[15] & 1; + if ((*(u64 *)(n+8) != in_n[1]) || (*(u64 *)(n) != in_n[0])) { + in_n[0] = *(u64 *)(n); + in_n[1] = *(u64 *)(n+8); + ((unsigned char *)in_n)[15] &= 0xFE; + crypto_cipher_encrypt_one(ctx->child, + (unsigned char *)out_p, (unsigned char *)in_n); + + ((unsigned char *)in_n)[15] |= (unsigned char)(1-i); + } + p = be64_to_cpup(out_p + i); + h = vhash(m, mbytes, (u64 *)0, &ctx->__vmac_ctx); + return p + h; +} + +static int vmac_set_key(unsigned char user_key[], struct vmac_ctx_t *ctx) +{ + u64 in[2] = {0}, out[2]; + unsigned i; + int err = 0; + + err = crypto_cipher_setkey(ctx->child, user_key, VMAC_KEY_LEN); + if (err) + return err; + + /* Fill nh key */ + ((unsigned char *)in)[0] = 0x80; + for (i = 0; i < sizeof(ctx->__vmac_ctx.nhkey)/8; i += 2) { + crypto_cipher_encrypt_one(ctx->child, + (unsigned char *)out, (unsigned char *)in); + ctx->__vmac_ctx.nhkey[i] = be64_to_cpup(out); + ctx->__vmac_ctx.nhkey[i+1] = be64_to_cpup(out+1); + ((unsigned char *)in)[15] += 1; + } + + /* Fill poly key */ + ((unsigned char *)in)[0] = 0xC0; + in[1] = 0; + for (i = 0; i < sizeof(ctx->__vmac_ctx.polykey)/8; i += 2) { + crypto_cipher_encrypt_one(ctx->child, + (unsigned char *)out, (unsigned char *)in); + ctx->__vmac_ctx.polytmp[i] = + ctx->__vmac_ctx.polykey[i] = + be64_to_cpup(out) & mpoly; + ctx->__vmac_ctx.polytmp[i+1] = + ctx->__vmac_ctx.polykey[i+1] = + be64_to_cpup(out+1) & mpoly; + ((unsigned char *)in)[15] += 1; + } + + /* Fill ip key */ + ((unsigned char *)in)[0] = 0xE0; + in[1] = 0; + for (i = 0; i < sizeof(ctx->__vmac_ctx.l3key)/8; i += 2) { + do { + crypto_cipher_encrypt_one(ctx->child, + (unsigned char *)out, (unsigned char *)in); + ctx->__vmac_ctx.l3key[i] = be64_to_cpup(out); + ctx->__vmac_ctx.l3key[i+1] = be64_to_cpup(out+1); + ((unsigned char *)in)[15] += 1; + } while (ctx->__vmac_ctx.l3key[i] >= p64 + || ctx->__vmac_ctx.l3key[i+1] >= p64); + } + + /* Invalidate nonce/aes cache and reset other elements */ + ctx->__vmac_ctx.cached_nonce[0] = (u64)-1; /* Ensure illegal nonce */ + ctx->__vmac_ctx.cached_nonce[1] = (u64)0; /* Ensure illegal nonce */ + ctx->__vmac_ctx.first_block_processed = 0; + + return err; +} + +static int vmac_setkey(struct crypto_shash *parent, + const u8 *key, unsigned int keylen) +{ + struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); + + if (keylen != VMAC_KEY_LEN) { + crypto_shash_set_flags(parent, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + + return vmac_set_key((u8 *)key, ctx); +} + +static int vmac_init(struct shash_desc *pdesc) +{ + struct crypto_shash *parent = pdesc->tfm; + struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); + + memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx)); + return 0; +} + +static int vmac_update(struct shash_desc *pdesc, const u8 *p, + unsigned int len) +{ + struct crypto_shash *parent = pdesc->tfm; + struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); + + vhash_update(p, len, &ctx->__vmac_ctx); + + return 0; +} + +static int vmac_final(struct shash_desc *pdesc, u8 *out) +{ + struct crypto_shash *parent = pdesc->tfm; + struct vmac_ctx_t *ctx = crypto_shash_ctx(parent); + vmac_t mac; + u8 nonce[16] = {}; + + mac = vmac(NULL, 0, nonce, NULL, ctx); + memcpy(out, &mac, sizeof(vmac_t)); + memset(&mac, 0, sizeof(vmac_t)); + memset(&ctx->__vmac_ctx, 0, sizeof(struct vmac_ctx)); + return 0; +} + +static int vmac_init_tfm(struct crypto_tfm *tfm) +{ + struct crypto_cipher *cipher; + struct crypto_instance *inst = (void *)tfm->__crt_alg; + struct crypto_spawn *spawn = crypto_instance_ctx(inst); + struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm); + + cipher = crypto_spawn_cipher(spawn); + if (IS_ERR(cipher)) + return PTR_ERR(cipher); + + ctx->child = cipher; + return 0; +} + +static void vmac_exit_tfm(struct crypto_tfm *tfm) +{ + struct vmac_ctx_t *ctx = crypto_tfm_ctx(tfm); + crypto_free_cipher(ctx->child); +} + +static int vmac_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct shash_instance *inst; + struct crypto_alg *alg; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH); + if (err) + return err; + + alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, + CRYPTO_ALG_TYPE_MASK); + if (IS_ERR(alg)) + return PTR_ERR(alg); + + inst = shash_alloc_instance("vmac", alg); + err = PTR_ERR(inst); + if (IS_ERR(inst)) + goto out_put_alg; + + err = crypto_init_spawn(shash_instance_ctx(inst), alg, + shash_crypto_instance(inst), + CRYPTO_ALG_TYPE_MASK); + if (err) + goto out_free_inst; + + inst->alg.base.cra_priority = alg->cra_priority; + inst->alg.base.cra_blocksize = alg->cra_blocksize; + inst->alg.base.cra_alignmask = alg->cra_alignmask; + + inst->alg.digestsize = sizeof(vmac_t); + inst->alg.base.cra_ctxsize = sizeof(struct vmac_ctx_t); + inst->alg.base.cra_init = vmac_init_tfm; + inst->alg.base.cra_exit = vmac_exit_tfm; + + inst->alg.init = vmac_init; + inst->alg.update = vmac_update; + inst->alg.final = vmac_final; + inst->alg.setkey = vmac_setkey; + + err = shash_register_instance(tmpl, inst); + if (err) { +out_free_inst: + shash_free_instance(shash_crypto_instance(inst)); + } + +out_put_alg: + crypto_mod_put(alg); + return err; +} + +static struct crypto_template vmac_tmpl = { + .name = "vmac", + .create = vmac_create, + .free = shash_free_instance, + .module = THIS_MODULE, +}; + +static int __init vmac_module_init(void) +{ + return crypto_register_template(&vmac_tmpl); +} + +static void __exit vmac_module_exit(void) +{ + crypto_unregister_template(&vmac_tmpl); +} + +module_init(vmac_module_init); +module_exit(vmac_module_exit); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("VMAC hash algorithm"); + diff --git a/crypto/xcbc.c b/crypto/xcbc.c index b63b633..bb7b67f 100644 --- a/crypto/xcbc.c +++ b/crypto/xcbc.c @@ -19,211 +19,142 @@ * Kazunori Miyazawa <miyazawa@linux-ipv6.org> */ -#include <crypto/scatterwalk.h> -#include <linux/crypto.h> +#include <crypto/internal/hash.h> #include <linux/err.h> -#include <linux/hardirq.h> #include <linux/kernel.h> -#include <linux/mm.h> -#include <linux/rtnetlink.h> -#include <linux/slab.h> -#include <linux/scatterlist.h> static u_int32_t ks[12] = {0x01010101, 0x01010101, 0x01010101, 0x01010101, 0x02020202, 0x02020202, 0x02020202, 0x02020202, 0x03030303, 0x03030303, 0x03030303, 0x03030303}; + /* * +------------------------ * | <parent tfm> * +------------------------ - * | crypto_xcbc_ctx + * | xcbc_tfm_ctx * +------------------------ - * | odds (block size) + * | consts (block size * 2) * +------------------------ - * | prev (block size) + */ +struct xcbc_tfm_ctx { + struct crypto_cipher *child; + u8 ctx[]; +}; + +/* * +------------------------ - * | key (block size) + * | <shash desc> * +------------------------ - * | consts (block size * 3) + * | xcbc_desc_ctx + * +------------------------ + * | odds (block size) + * +------------------------ + * | prev (block size) * +------------------------ */ -struct crypto_xcbc_ctx { - struct crypto_cipher *child; - u8 *odds; - u8 *prev; - u8 *key; - u8 *consts; - void (*xor)(u8 *a, const u8 *b, unsigned int bs); - unsigned int keylen; +struct xcbc_desc_ctx { unsigned int len; + u8 ctx[]; }; -static void xor_128(u8 *a, const u8 *b, unsigned int bs) -{ - ((u32 *)a)[0] ^= ((u32 *)b)[0]; - ((u32 *)a)[1] ^= ((u32 *)b)[1]; - ((u32 *)a)[2] ^= ((u32 *)b)[2]; - ((u32 *)a)[3] ^= ((u32 *)b)[3]; -} - -static int _crypto_xcbc_digest_setkey(struct crypto_hash *parent, - struct crypto_xcbc_ctx *ctx) +static int crypto_xcbc_digest_setkey(struct crypto_shash *parent, + const u8 *inkey, unsigned int keylen) { - int bs = crypto_hash_blocksize(parent); + unsigned long alignmask = crypto_shash_alignmask(parent); + struct xcbc_tfm_ctx *ctx = crypto_shash_ctx(parent); + int bs = crypto_shash_blocksize(parent); + u8 *consts = PTR_ALIGN(&ctx->ctx[0], alignmask + 1); int err = 0; u8 key1[bs]; - if ((err = crypto_cipher_setkey(ctx->child, ctx->key, ctx->keylen))) - return err; + if ((err = crypto_cipher_setkey(ctx->child, inkey, keylen))) + return err; - crypto_cipher_encrypt_one(ctx->child, key1, ctx->consts); + crypto_cipher_encrypt_one(ctx->child, consts, (u8 *)ks + bs); + crypto_cipher_encrypt_one(ctx->child, consts + bs, (u8 *)ks + bs * 2); + crypto_cipher_encrypt_one(ctx->child, key1, (u8 *)ks); return crypto_cipher_setkey(ctx->child, key1, bs); -} - -static int crypto_xcbc_digest_setkey(struct crypto_hash *parent, - const u8 *inkey, unsigned int keylen) -{ - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(parent); - - if (keylen != crypto_cipher_blocksize(ctx->child)) - return -EINVAL; - ctx->keylen = keylen; - memcpy(ctx->key, inkey, keylen); - ctx->consts = (u8*)ks; - - return _crypto_xcbc_digest_setkey(parent, ctx); } -static int crypto_xcbc_digest_init(struct hash_desc *pdesc) +static int crypto_xcbc_digest_init(struct shash_desc *pdesc) { - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(pdesc->tfm); - int bs = crypto_hash_blocksize(pdesc->tfm); + unsigned long alignmask = crypto_shash_alignmask(pdesc->tfm); + struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); + int bs = crypto_shash_blocksize(pdesc->tfm); + u8 *prev = PTR_ALIGN(&ctx->ctx[0], alignmask + 1) + bs; ctx->len = 0; - memset(ctx->odds, 0, bs); - memset(ctx->prev, 0, bs); + memset(prev, 0, bs); return 0; } -static int crypto_xcbc_digest_update2(struct hash_desc *pdesc, - struct scatterlist *sg, - unsigned int nbytes) +static int crypto_xcbc_digest_update(struct shash_desc *pdesc, const u8 *p, + unsigned int len) { - struct crypto_hash *parent = pdesc->tfm; - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(parent); - struct crypto_cipher *tfm = ctx->child; - int bs = crypto_hash_blocksize(parent); - - for (;;) { - struct page *pg = sg_page(sg); - unsigned int offset = sg->offset; - unsigned int slen = sg->length; - - if (unlikely(slen > nbytes)) - slen = nbytes; - - nbytes -= slen; - - while (slen > 0) { - unsigned int len = min(slen, ((unsigned int)(PAGE_SIZE)) - offset); - char *p = crypto_kmap(pg, 0) + offset; - - /* checking the data can fill the block */ - if ((ctx->len + len) <= bs) { - memcpy(ctx->odds + ctx->len, p, len); - ctx->len += len; - slen -= len; - - /* checking the rest of the page */ - if (len + offset >= PAGE_SIZE) { - offset = 0; - pg++; - } else - offset += len; - - crypto_kunmap(p, 0); - crypto_yield(pdesc->flags); - continue; - } - - /* filling odds with new data and encrypting it */ - memcpy(ctx->odds + ctx->len, p, bs - ctx->len); - len -= bs - ctx->len; - p += bs - ctx->len; - - ctx->xor(ctx->prev, ctx->odds, bs); - crypto_cipher_encrypt_one(tfm, ctx->prev, ctx->prev); - - /* clearing the length */ - ctx->len = 0; - - /* encrypting the rest of data */ - while (len > bs) { - ctx->xor(ctx->prev, p, bs); - crypto_cipher_encrypt_one(tfm, ctx->prev, - ctx->prev); - p += bs; - len -= bs; - } - - /* keeping the surplus of blocksize */ - if (len) { - memcpy(ctx->odds, p, len); - ctx->len = len; - } - crypto_kunmap(p, 0); - crypto_yield(pdesc->flags); - slen -= min(slen, ((unsigned int)(PAGE_SIZE)) - offset); - offset = 0; - pg++; - } - - if (!nbytes) - break; - sg = scatterwalk_sg_next(sg); + struct crypto_shash *parent = pdesc->tfm; + unsigned long alignmask = crypto_shash_alignmask(parent); + struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent); + struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); + struct crypto_cipher *tfm = tctx->child; + int bs = crypto_shash_blocksize(parent); + u8 *odds = PTR_ALIGN(&ctx->ctx[0], alignmask + 1); + u8 *prev = odds + bs; + + /* checking the data can fill the block */ + if ((ctx->len + len) <= bs) { + memcpy(odds + ctx->len, p, len); + ctx->len += len; + return 0; } - return 0; -} + /* filling odds with new data and encrypting it */ + memcpy(odds + ctx->len, p, bs - ctx->len); + len -= bs - ctx->len; + p += bs - ctx->len; -static int crypto_xcbc_digest_update(struct hash_desc *pdesc, - struct scatterlist *sg, - unsigned int nbytes) -{ - if (WARN_ON_ONCE(in_irq())) - return -EDEADLK; - return crypto_xcbc_digest_update2(pdesc, sg, nbytes); -} + crypto_xor(prev, odds, bs); + crypto_cipher_encrypt_one(tfm, prev, prev); -static int crypto_xcbc_digest_final(struct hash_desc *pdesc, u8 *out) -{ - struct crypto_hash *parent = pdesc->tfm; - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(parent); - struct crypto_cipher *tfm = ctx->child; - int bs = crypto_hash_blocksize(parent); - int err = 0; - - if (ctx->len == bs) { - u8 key2[bs]; + /* clearing the length */ + ctx->len = 0; - if ((err = crypto_cipher_setkey(tfm, ctx->key, ctx->keylen)) != 0) - return err; + /* encrypting the rest of data */ + while (len > bs) { + crypto_xor(prev, p, bs); + crypto_cipher_encrypt_one(tfm, prev, prev); + p += bs; + len -= bs; + } - crypto_cipher_encrypt_one(tfm, key2, - (u8 *)(ctx->consts + bs)); + /* keeping the surplus of blocksize */ + if (len) { + memcpy(odds, p, len); + ctx->len = len; + } - ctx->xor(ctx->prev, ctx->odds, bs); - ctx->xor(ctx->prev, key2, bs); - _crypto_xcbc_digest_setkey(parent, ctx); + return 0; +} - crypto_cipher_encrypt_one(tfm, out, ctx->prev); - } else { - u8 key3[bs]; +static int crypto_xcbc_digest_final(struct shash_desc *pdesc, u8 *out) +{ + struct crypto_shash *parent = pdesc->tfm; + unsigned long alignmask = crypto_shash_alignmask(parent); + struct xcbc_tfm_ctx *tctx = crypto_shash_ctx(parent); + struct xcbc_desc_ctx *ctx = shash_desc_ctx(pdesc); + struct crypto_cipher *tfm = tctx->child; + int bs = crypto_shash_blocksize(parent); + u8 *consts = PTR_ALIGN(&tctx->ctx[0], alignmask + 1); + u8 *odds = PTR_ALIGN(&ctx->ctx[0], alignmask + 1); + u8 *prev = odds + bs; + unsigned int offset = 0; + + if (ctx->len != bs) { unsigned int rlen; - u8 *p = ctx->odds + ctx->len; + u8 *p = odds + ctx->len; + *p = 0x80; p++; @@ -231,32 +162,15 @@ static int crypto_xcbc_digest_final(struct hash_desc *pdesc, u8 *out) if (rlen) memset(p, 0, rlen); - if ((err = crypto_cipher_setkey(tfm, ctx->key, ctx->keylen)) != 0) - return err; - - crypto_cipher_encrypt_one(tfm, key3, - (u8 *)(ctx->consts + bs * 2)); - - ctx->xor(ctx->prev, ctx->odds, bs); - ctx->xor(ctx->prev, key3, bs); - - _crypto_xcbc_digest_setkey(parent, ctx); - - crypto_cipher_encrypt_one(tfm, out, ctx->prev); + offset += bs; } - return 0; -} + crypto_xor(prev, odds, bs); + crypto_xor(prev, consts + offset, bs); -static int crypto_xcbc_digest(struct hash_desc *pdesc, - struct scatterlist *sg, unsigned int nbytes, u8 *out) -{ - if (WARN_ON_ONCE(in_irq())) - return -EDEADLK; + crypto_cipher_encrypt_one(tfm, out, prev); - crypto_xcbc_digest_init(pdesc); - crypto_xcbc_digest_update2(pdesc, sg, nbytes); - return crypto_xcbc_digest_final(pdesc, out); + return 0; } static int xcbc_init_tfm(struct crypto_tfm *tfm) @@ -264,95 +178,95 @@ static int xcbc_init_tfm(struct crypto_tfm *tfm) struct crypto_cipher *cipher; struct crypto_instance *inst = (void *)tfm->__crt_alg; struct crypto_spawn *spawn = crypto_instance_ctx(inst); - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(__crypto_hash_cast(tfm)); - int bs = crypto_hash_blocksize(__crypto_hash_cast(tfm)); + struct xcbc_tfm_ctx *ctx = crypto_tfm_ctx(tfm); cipher = crypto_spawn_cipher(spawn); if (IS_ERR(cipher)) return PTR_ERR(cipher); - switch(bs) { - case 16: - ctx->xor = xor_128; - break; - default: - return -EINVAL; - } - ctx->child = cipher; - ctx->odds = (u8*)(ctx+1); - ctx->prev = ctx->odds + bs; - ctx->key = ctx->prev + bs; return 0; }; static void xcbc_exit_tfm(struct crypto_tfm *tfm) { - struct crypto_xcbc_ctx *ctx = crypto_hash_ctx_aligned(__crypto_hash_cast(tfm)); + struct xcbc_tfm_ctx *ctx = crypto_tfm_ctx(tfm); crypto_free_cipher(ctx->child); } -static struct crypto_instance *xcbc_alloc(struct rtattr **tb) +static int xcbc_create(struct crypto_template *tmpl, struct rtattr **tb) { - struct crypto_instance *inst; + struct shash_instance *inst; struct crypto_alg *alg; + unsigned long alignmask; int err; - err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_HASH); + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SHASH); if (err) - return ERR_PTR(err); + return err; alg = crypto_get_attr_alg(tb, CRYPTO_ALG_TYPE_CIPHER, CRYPTO_ALG_TYPE_MASK); if (IS_ERR(alg)) - return ERR_CAST(alg); + return PTR_ERR(alg); switch(alg->cra_blocksize) { case 16: break; default: - inst = ERR_PTR(-EINVAL); goto out_put_alg; } - inst = crypto_alloc_instance("xcbc", alg); + inst = shash_alloc_instance("xcbc", alg); + err = PTR_ERR(inst); if (IS_ERR(inst)) goto out_put_alg; - inst->alg.cra_flags = CRYPTO_ALG_TYPE_HASH; - inst->alg.cra_priority = alg->cra_priority; - inst->alg.cra_blocksize = alg->cra_blocksize; - inst->alg.cra_alignmask = alg->cra_alignmask; - inst->alg.cra_type = &crypto_hash_type; - - inst->alg.cra_hash.digestsize = alg->cra_blocksize; - inst->alg.cra_ctxsize = sizeof(struct crypto_xcbc_ctx) + - ALIGN(inst->alg.cra_blocksize * 3, sizeof(void *)); - inst->alg.cra_init = xcbc_init_tfm; - inst->alg.cra_exit = xcbc_exit_tfm; - - inst->alg.cra_hash.init = crypto_xcbc_digest_init; - inst->alg.cra_hash.update = crypto_xcbc_digest_update; - inst->alg.cra_hash.final = crypto_xcbc_digest_final; - inst->alg.cra_hash.digest = crypto_xcbc_digest; - inst->alg.cra_hash.setkey = crypto_xcbc_digest_setkey; + err = crypto_init_spawn(shash_instance_ctx(inst), alg, + shash_crypto_instance(inst), + CRYPTO_ALG_TYPE_MASK); + if (err) + goto out_free_inst; + + alignmask = alg->cra_alignmask | 3; + inst->alg.base.cra_alignmask = alignmask; + inst->alg.base.cra_priority = alg->cra_priority; + inst->alg.base.cra_blocksize = alg->cra_blocksize; + + inst->alg.digestsize = alg->cra_blocksize; + inst->alg.descsize = ALIGN(sizeof(struct xcbc_desc_ctx), + crypto_tfm_ctx_alignment()) + + (alignmask & + ~(crypto_tfm_ctx_alignment() - 1)) + + alg->cra_blocksize * 2; + + inst->alg.base.cra_ctxsize = ALIGN(sizeof(struct xcbc_tfm_ctx), + alignmask + 1) + + alg->cra_blocksize * 2; + inst->alg.base.cra_init = xcbc_init_tfm; + inst->alg.base.cra_exit = xcbc_exit_tfm; + + inst->alg.init = crypto_xcbc_digest_init; + inst->alg.update = crypto_xcbc_digest_update; + inst->alg.final = crypto_xcbc_digest_final; + inst->alg.setkey = crypto_xcbc_digest_setkey; + + err = shash_register_instance(tmpl, inst); + if (err) { +out_free_inst: + shash_free_instance(shash_crypto_instance(inst)); + } out_put_alg: crypto_mod_put(alg); - return inst; -} - -static void xcbc_free(struct crypto_instance *inst) -{ - crypto_drop_spawn(crypto_instance_ctx(inst)); - kfree(inst); + return err; } static struct crypto_template crypto_xcbc_tmpl = { .name = "xcbc", - .alloc = xcbc_alloc, - .free = xcbc_free, + .create = xcbc_create, + .free = shash_free_instance, .module = THIS_MODULE, }; diff --git a/drivers/char/hw_random/amd-rng.c b/drivers/char/hw_random/amd-rng.c index cd0ba51..0d8c578 100644 --- a/drivers/char/hw_random/amd-rng.c +++ b/drivers/char/hw_random/amd-rng.c @@ -44,8 +44,8 @@ * want to register another driver on the same PCI id. */ static const struct pci_device_id pci_tbl[] = { - { 0x1022, 0x7443, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0, }, - { 0x1022, 0x746b, PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0, }, + { PCI_VDEVICE(AMD, 0x7443), 0, }, + { PCI_VDEVICE(AMD, 0x746b), 0, }, { 0, }, /* terminate list */ }; MODULE_DEVICE_TABLE(pci, pci_tbl); diff --git a/drivers/char/hw_random/geode-rng.c b/drivers/char/hw_random/geode-rng.c index 64d513f6..4c4d4e1 100644 --- a/drivers/char/hw_random/geode-rng.c +++ b/drivers/char/hw_random/geode-rng.c @@ -46,8 +46,7 @@ * want to register another driver on the same PCI id. */ static const struct pci_device_id pci_tbl[] = { - { PCI_VENDOR_ID_AMD, PCI_DEVICE_ID_AMD_LX_AES, - PCI_ANY_ID, PCI_ANY_ID, 0, 0, 0, }, + { PCI_VDEVICE(AMD, PCI_DEVICE_ID_AMD_LX_AES), 0, }, { 0, }, /* terminate list */ }; MODULE_DEVICE_TABLE(pci, pci_tbl); diff --git a/drivers/char/random.c b/drivers/char/random.c index 8c74448..d8a9255 100644 --- a/drivers/char/random.c +++ b/drivers/char/random.c @@ -240,6 +240,7 @@ #include <linux/spinlock.h> #include <linux/percpu.h> #include <linux/cryptohash.h> +#include <linux/fips.h> #ifdef CONFIG_GENERIC_HARDIRQS # include <linux/irq.h> @@ -413,6 +414,7 @@ struct entropy_store { unsigned add_ptr; int entropy_count; int input_rotate; + __u8 *last_data; }; static __u32 input_pool_data[INPUT_POOL_WORDS]; @@ -852,12 +854,21 @@ static ssize_t extract_entropy(struct entropy_store *r, void *buf, { ssize_t ret = 0, i; __u8 tmp[EXTRACT_SIZE]; + unsigned long flags; xfer_secondary_pool(r, nbytes); nbytes = account(r, nbytes, min, reserved); while (nbytes) { extract_buf(r, tmp); + + if (r->last_data) { + spin_lock_irqsave(&r->lock, flags); + if (!memcmp(tmp, r->last_data, EXTRACT_SIZE)) + panic("Hardware RNG duplicated output!\n"); + memcpy(r->last_data, tmp, EXTRACT_SIZE); + spin_unlock_irqrestore(&r->lock, flags); + } i = min_t(int, nbytes, EXTRACT_SIZE); memcpy(buf, tmp, i); nbytes -= i; @@ -940,6 +951,9 @@ static void init_std_data(struct entropy_store *r) now = ktime_get_real(); mix_pool_bytes(r, &now, sizeof(now)); mix_pool_bytes(r, utsname(), sizeof(*(utsname()))); + /* Enable continuous test in fips mode */ + if (fips_enabled) + r->last_data = kmalloc(EXTRACT_SIZE, GFP_KERNEL); } static int rand_initialize(void) diff --git a/drivers/crypto/Kconfig b/drivers/crypto/Kconfig index 5b27692..b08403d 100644 --- a/drivers/crypto/Kconfig +++ b/drivers/crypto/Kconfig @@ -13,7 +13,6 @@ if CRYPTO_HW config CRYPTO_DEV_PADLOCK tristate "Support for VIA PadLock ACE" depends on X86 && !UML - select CRYPTO_ALGAPI help Some VIA processors come with an integrated crypto engine (so called VIA PadLock ACE, Advanced Cryptography Engine) @@ -39,6 +38,7 @@ config CRYPTO_DEV_PADLOCK_AES config CRYPTO_DEV_PADLOCK_SHA tristate "PadLock driver for SHA1 and SHA256 algorithms" depends on CRYPTO_DEV_PADLOCK + select CRYPTO_HASH select CRYPTO_SHA1 select CRYPTO_SHA256 help @@ -157,6 +157,19 @@ config S390_PRNG ANSI X9.17 standard. The PRNG is usable via the char device /dev/prandom. +config CRYPTO_DEV_MV_CESA + tristate "Marvell's Cryptographic Engine" + depends on PLAT_ORION + select CRYPTO_ALGAPI + select CRYPTO_AES + select CRYPTO_BLKCIPHER2 + help + This driver allows you to utilize the Cryptographic Engines and + Security Accelerator (CESA) which can be found on the Marvell Orion + and Kirkwood SoCs, such as QNAP's TS-209. + + Currently the driver supports AES in ECB and CBC mode without DMA. + config CRYPTO_DEV_HIFN_795X tristate "Driver HIFN 795x crypto accelerator chips" select CRYPTO_DES diff --git a/drivers/crypto/Makefile b/drivers/crypto/Makefile index 9bf4a2b..6ffcb3f 100644 --- a/drivers/crypto/Makefile +++ b/drivers/crypto/Makefile @@ -2,6 +2,7 @@ obj-$(CONFIG_CRYPTO_DEV_PADLOCK_AES) += padlock-aes.o obj-$(CONFIG_CRYPTO_DEV_PADLOCK_SHA) += padlock-sha.o obj-$(CONFIG_CRYPTO_DEV_GEODE) += geode-aes.o obj-$(CONFIG_CRYPTO_DEV_HIFN_795X) += hifn_795x.o +obj-$(CONFIG_CRYPTO_DEV_MV_CESA) += mv_cesa.o obj-$(CONFIG_CRYPTO_DEV_TALITOS) += talitos.o obj-$(CONFIG_CRYPTO_DEV_IXP4XX) += ixp4xx_crypto.o obj-$(CONFIG_CRYPTO_DEV_PPC4XX) += amcc/ diff --git a/drivers/crypto/amcc/crypto4xx_alg.c b/drivers/crypto/amcc/crypto4xx_alg.c index 61b6e1b..a33243c 100644 --- a/drivers/crypto/amcc/crypto4xx_alg.c +++ b/drivers/crypto/amcc/crypto4xx_alg.c @@ -208,7 +208,8 @@ static int crypto4xx_hash_alg_init(struct crypto_tfm *tfm, } } - tfm->crt_ahash.reqsize = sizeof(struct crypto4xx_ctx); + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct crypto4xx_ctx)); sa = (struct dynamic_sa_ctl *) ctx->sa_in; set_dynamic_sa_command_0(sa, SA_SAVE_HASH, SA_NOT_SAVE_IV, SA_NOT_LOAD_HASH, SA_LOAD_IV_FROM_SA, diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 4c0dfb2..46e899a 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -31,8 +31,6 @@ #include <asm/dcr.h> #include <asm/dcr-regs.h> #include <asm/cacheflush.h> -#include <crypto/internal/hash.h> -#include <crypto/algapi.h> #include <crypto/aes.h> #include <crypto/sha.h> #include "crypto4xx_reg_def.h" @@ -998,10 +996,15 @@ static int crypto4xx_alg_init(struct crypto_tfm *tfm) ctx->sa_out_dma_addr = 0; ctx->sa_len = 0; - if (alg->cra_type == &crypto_ablkcipher_type) + switch (alg->cra_flags & CRYPTO_ALG_TYPE_MASK) { + default: tfm->crt_ablkcipher.reqsize = sizeof(struct crypto4xx_ctx); - else if (alg->cra_type == &crypto_ahash_type) - tfm->crt_ahash.reqsize = sizeof(struct crypto4xx_ctx); + break; + case CRYPTO_ALG_TYPE_AHASH: + crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), + sizeof(struct crypto4xx_ctx)); + break; + } return 0; } @@ -1015,7 +1018,8 @@ static void crypto4xx_alg_exit(struct crypto_tfm *tfm) } int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, - struct crypto_alg *crypto_alg, int array_size) + struct crypto4xx_alg_common *crypto_alg, + int array_size) { struct crypto4xx_alg *alg; int i; @@ -1027,13 +1031,18 @@ int crypto4xx_register_alg(struct crypto4xx_device *sec_dev, return -ENOMEM; alg->alg = crypto_alg[i]; - INIT_LIST_HEAD(&alg->alg.cra_list); - if (alg->alg.cra_init == NULL) - alg->alg.cra_init = crypto4xx_alg_init; - if (alg->alg.cra_exit == NULL) - alg->alg.cra_exit = crypto4xx_alg_exit; alg->dev = sec_dev; - rc = crypto_register_alg(&alg->alg); + + switch (alg->alg.type) { + case CRYPTO_ALG_TYPE_AHASH: + rc = crypto_register_ahash(&alg->alg.u.hash); + break; + + default: + rc = crypto_register_alg(&alg->alg.u.cipher); + break; + } + if (rc) { list_del(&alg->entry); kfree(alg); @@ -1051,7 +1060,14 @@ static void crypto4xx_unregister_alg(struct crypto4xx_device *sec_dev) list_for_each_entry_safe(alg, tmp, &sec_dev->alg_list, entry) { list_del(&alg->entry); - crypto_unregister_alg(&alg->alg); + switch (alg->alg.type) { + case CRYPTO_ALG_TYPE_AHASH: + crypto_unregister_ahash(&alg->alg.u.hash); + break; + + default: + crypto_unregister_alg(&alg->alg.u.cipher); + } kfree(alg); } } @@ -1104,17 +1120,18 @@ static irqreturn_t crypto4xx_ce_interrupt_handler(int irq, void *data) /** * Supported Crypto Algorithms */ -struct crypto_alg crypto4xx_alg[] = { +struct crypto4xx_alg_common crypto4xx_alg[] = { /* Crypto AES modes */ - { + { .type = CRYPTO_ALG_TYPE_ABLKCIPHER, .u.cipher = { .cra_name = "cbc(aes)", .cra_driver_name = "cbc-aes-ppc4xx", .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, .cra_ctxsize = sizeof(struct crypto4xx_ctx), - .cra_alignmask = 0, .cra_type = &crypto_ablkcipher_type, + .cra_init = crypto4xx_alg_init, + .cra_exit = crypto4xx_alg_exit, .cra_module = THIS_MODULE, .cra_u = { .ablkcipher = { @@ -1126,29 +1143,7 @@ struct crypto_alg crypto4xx_alg[] = { .decrypt = crypto4xx_decrypt, } } - }, - /* Hash SHA1 */ - { - .cra_name = "sha1", - .cra_driver_name = "sha1-ppc4xx", - .cra_priority = CRYPTO4XX_CRYPTO_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_AHASH | CRYPTO_ALG_ASYNC, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct crypto4xx_ctx), - .cra_alignmask = 0, - .cra_type = &crypto_ahash_type, - .cra_init = crypto4xx_sha1_alg_init, - .cra_module = THIS_MODULE, - .cra_u = { - .ahash = { - .digestsize = SHA1_DIGEST_SIZE, - .init = crypto4xx_hash_init, - .update = crypto4xx_hash_update, - .final = crypto4xx_hash_final, - .digest = crypto4xx_hash_digest, - } - } - }, + }}, }; /** diff --git a/drivers/crypto/amcc/crypto4xx_core.h b/drivers/crypto/amcc/crypto4xx_core.h index 1ef1034..da9cbe3 100644 --- a/drivers/crypto/amcc/crypto4xx_core.h +++ b/drivers/crypto/amcc/crypto4xx_core.h @@ -22,6 +22,8 @@ #ifndef __CRYPTO4XX_CORE_H__ #define __CRYPTO4XX_CORE_H__ +#include <crypto/internal/hash.h> + #define PPC460SX_SDR0_SRST 0x201 #define PPC405EX_SDR0_SRST 0x200 #define PPC460EX_SDR0_SRST 0x201 @@ -138,14 +140,31 @@ struct crypto4xx_req_ctx { u16 sa_len; }; +struct crypto4xx_alg_common { + u32 type; + union { + struct crypto_alg cipher; + struct ahash_alg hash; + } u; +}; + struct crypto4xx_alg { struct list_head entry; - struct crypto_alg alg; + struct crypto4xx_alg_common alg; struct crypto4xx_device *dev; }; -#define crypto_alg_to_crypto4xx_alg(x) \ - container_of(x, struct crypto4xx_alg, alg) +static inline struct crypto4xx_alg *crypto_alg_to_crypto4xx_alg( + struct crypto_alg *x) +{ + switch (x->cra_flags & CRYPTO_ALG_TYPE_MASK) { + case CRYPTO_ALG_TYPE_AHASH: + return container_of(__crypto_ahash_alg(x), + struct crypto4xx_alg, alg.u.hash); + } + + return container_of(x, struct crypto4xx_alg, alg.u.cipher); +} extern int crypto4xx_alloc_sa(struct crypto4xx_ctx *ctx, u32 size); extern void crypto4xx_free_sa(struct crypto4xx_ctx *ctx); diff --git a/drivers/crypto/mv_cesa.c b/drivers/crypto/mv_cesa.c new file mode 100644 index 0000000..b21ef63 --- /dev/null +++ b/drivers/crypto/mv_cesa.c @@ -0,0 +1,606 @@ +/* + * Support for Marvell's crypto engine which can be found on some Orion5X + * boards. + * + * Author: Sebastian Andrzej Siewior < sebastian at breakpoint dot cc > + * License: GPLv2 + * + */ +#include <crypto/aes.h> +#include <crypto/algapi.h> +#include <linux/crypto.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/kthread.h> +#include <linux/platform_device.h> +#include <linux/scatterlist.h> + +#include "mv_cesa.h" +/* + * STM: + * /---------------------------------------\ + * | | request complete + * \./ | + * IDLE -> new request -> BUSY -> done -> DEQUEUE + * /°\ | + * | | more scatter entries + * \________________/ + */ +enum engine_status { + ENGINE_IDLE, + ENGINE_BUSY, + ENGINE_W_DEQUEUE, +}; + +/** + * struct req_progress - used for every crypt request + * @src_sg_it: sg iterator for src + * @dst_sg_it: sg iterator for dst + * @sg_src_left: bytes left in src to process (scatter list) + * @src_start: offset to add to src start position (scatter list) + * @crypt_len: length of current crypt process + * @sg_dst_left: bytes left dst to process in this scatter list + * @dst_start: offset to add to dst start position (scatter list) + * @total_req_bytes: total number of bytes processed (request). + * + * sg helper are used to iterate over the scatterlist. Since the size of the + * SRAM may be less than the scatter size, this struct struct is used to keep + * track of progress within current scatterlist. + */ +struct req_progress { + struct sg_mapping_iter src_sg_it; + struct sg_mapping_iter dst_sg_it; + + /* src mostly */ + int sg_src_left; + int src_start; + int crypt_len; + /* dst mostly */ + int sg_dst_left; + int dst_start; + int total_req_bytes; +}; + +struct crypto_priv { + void __iomem *reg; + void __iomem *sram; + int irq; + struct task_struct *queue_th; + + /* the lock protects queue and eng_st */ + spinlock_t lock; + struct crypto_queue queue; + enum engine_status eng_st; + struct ablkcipher_request *cur_req; + struct req_progress p; + int max_req_size; + int sram_size; +}; + +static struct crypto_priv *cpg; + +struct mv_ctx { + u8 aes_enc_key[AES_KEY_LEN]; + u32 aes_dec_key[8]; + int key_len; + u32 need_calc_aes_dkey; +}; + +enum crypto_op { + COP_AES_ECB, + COP_AES_CBC, +}; + +struct mv_req_ctx { + enum crypto_op op; + int decrypt; +}; + +static void compute_aes_dec_key(struct mv_ctx *ctx) +{ + struct crypto_aes_ctx gen_aes_key; + int key_pos; + + if (!ctx->need_calc_aes_dkey) + return; + + crypto_aes_expand_key(&gen_aes_key, ctx->aes_enc_key, ctx->key_len); + + key_pos = ctx->key_len + 24; + memcpy(ctx->aes_dec_key, &gen_aes_key.key_enc[key_pos], 4 * 4); + switch (ctx->key_len) { + case AES_KEYSIZE_256: + key_pos -= 2; + /* fall */ + case AES_KEYSIZE_192: + key_pos -= 2; + memcpy(&ctx->aes_dec_key[4], &gen_aes_key.key_enc[key_pos], + 4 * 4); + break; + } + ctx->need_calc_aes_dkey = 0; +} + +static int mv_setkey_aes(struct crypto_ablkcipher *cipher, const u8 *key, + unsigned int len) +{ + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(cipher); + struct mv_ctx *ctx = crypto_tfm_ctx(tfm); + + switch (len) { + case AES_KEYSIZE_128: + case AES_KEYSIZE_192: + case AES_KEYSIZE_256: + break; + default: + crypto_ablkcipher_set_flags(cipher, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; + } + ctx->key_len = len; + ctx->need_calc_aes_dkey = 1; + + memcpy(ctx->aes_enc_key, key, AES_KEY_LEN); + return 0; +} + +static void setup_data_in(struct ablkcipher_request *req) +{ + int ret; + void *buf; + + if (!cpg->p.sg_src_left) { + ret = sg_miter_next(&cpg->p.src_sg_it); + BUG_ON(!ret); + cpg->p.sg_src_left = cpg->p.src_sg_it.length; + cpg->p.src_start = 0; + } + + cpg->p.crypt_len = min(cpg->p.sg_src_left, cpg->max_req_size); + + buf = cpg->p.src_sg_it.addr; + buf += cpg->p.src_start; + + memcpy(cpg->sram + SRAM_DATA_IN_START, buf, cpg->p.crypt_len); + + cpg->p.sg_src_left -= cpg->p.crypt_len; + cpg->p.src_start += cpg->p.crypt_len; +} + +static void mv_process_current_q(int first_block) +{ + struct ablkcipher_request *req = cpg->cur_req; + struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); + struct sec_accel_config op; + + switch (req_ctx->op) { + case COP_AES_ECB: + op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_ECB; + break; + case COP_AES_CBC: + op.config = CFG_OP_CRYPT_ONLY | CFG_ENCM_AES | CFG_ENC_MODE_CBC; + op.enc_iv = ENC_IV_POINT(SRAM_DATA_IV) | + ENC_IV_BUF_POINT(SRAM_DATA_IV_BUF); + if (first_block) + memcpy(cpg->sram + SRAM_DATA_IV, req->info, 16); + break; + } + if (req_ctx->decrypt) { + op.config |= CFG_DIR_DEC; + memcpy(cpg->sram + SRAM_DATA_KEY_P, ctx->aes_dec_key, + AES_KEY_LEN); + } else { + op.config |= CFG_DIR_ENC; + memcpy(cpg->sram + SRAM_DATA_KEY_P, ctx->aes_enc_key, + AES_KEY_LEN); + } + + switch (ctx->key_len) { + case AES_KEYSIZE_128: + op.config |= CFG_AES_LEN_128; + break; + case AES_KEYSIZE_192: + op.config |= CFG_AES_LEN_192; + break; + case AES_KEYSIZE_256: + op.config |= CFG_AES_LEN_256; + break; + } + op.enc_p = ENC_P_SRC(SRAM_DATA_IN_START) | + ENC_P_DST(SRAM_DATA_OUT_START); + op.enc_key_p = SRAM_DATA_KEY_P; + + setup_data_in(req); + op.enc_len = cpg->p.crypt_len; + memcpy(cpg->sram + SRAM_CONFIG, &op, + sizeof(struct sec_accel_config)); + + writel(SRAM_CONFIG, cpg->reg + SEC_ACCEL_DESC_P0); + /* GO */ + writel(SEC_CMD_EN_SEC_ACCL0, cpg->reg + SEC_ACCEL_CMD); + + /* + * XXX: add timer if the interrupt does not occur for some mystery + * reason + */ +} + +static void mv_crypto_algo_completion(void) +{ + struct ablkcipher_request *req = cpg->cur_req; + struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); + + if (req_ctx->op != COP_AES_CBC) + return ; + + memcpy(req->info, cpg->sram + SRAM_DATA_IV_BUF, 16); +} + +static void dequeue_complete_req(void) +{ + struct ablkcipher_request *req = cpg->cur_req; + void *buf; + int ret; + + cpg->p.total_req_bytes += cpg->p.crypt_len; + do { + int dst_copy; + + if (!cpg->p.sg_dst_left) { + ret = sg_miter_next(&cpg->p.dst_sg_it); + BUG_ON(!ret); + cpg->p.sg_dst_left = cpg->p.dst_sg_it.length; + cpg->p.dst_start = 0; + } + + buf = cpg->p.dst_sg_it.addr; + buf += cpg->p.dst_start; + + dst_copy = min(cpg->p.crypt_len, cpg->p.sg_dst_left); + + memcpy(buf, cpg->sram + SRAM_DATA_OUT_START, dst_copy); + + cpg->p.sg_dst_left -= dst_copy; + cpg->p.crypt_len -= dst_copy; + cpg->p.dst_start += dst_copy; + } while (cpg->p.crypt_len > 0); + + BUG_ON(cpg->eng_st != ENGINE_W_DEQUEUE); + if (cpg->p.total_req_bytes < req->nbytes) { + /* process next scatter list entry */ + cpg->eng_st = ENGINE_BUSY; + mv_process_current_q(0); + } else { + sg_miter_stop(&cpg->p.src_sg_it); + sg_miter_stop(&cpg->p.dst_sg_it); + mv_crypto_algo_completion(); + cpg->eng_st = ENGINE_IDLE; + req->base.complete(&req->base, 0); + } +} + +static int count_sgs(struct scatterlist *sl, unsigned int total_bytes) +{ + int i = 0; + + do { + total_bytes -= sl[i].length; + i++; + + } while (total_bytes > 0); + + return i; +} + +static void mv_enqueue_new_req(struct ablkcipher_request *req) +{ + int num_sgs; + + cpg->cur_req = req; + memset(&cpg->p, 0, sizeof(struct req_progress)); + + num_sgs = count_sgs(req->src, req->nbytes); + sg_miter_start(&cpg->p.src_sg_it, req->src, num_sgs, SG_MITER_FROM_SG); + + num_sgs = count_sgs(req->dst, req->nbytes); + sg_miter_start(&cpg->p.dst_sg_it, req->dst, num_sgs, SG_MITER_TO_SG); + mv_process_current_q(1); +} + +static int queue_manag(void *data) +{ + cpg->eng_st = ENGINE_IDLE; + do { + struct ablkcipher_request *req; + struct crypto_async_request *async_req = NULL; + struct crypto_async_request *backlog; + + __set_current_state(TASK_INTERRUPTIBLE); + + if (cpg->eng_st == ENGINE_W_DEQUEUE) + dequeue_complete_req(); + + spin_lock_irq(&cpg->lock); + if (cpg->eng_st == ENGINE_IDLE) { + backlog = crypto_get_backlog(&cpg->queue); + async_req = crypto_dequeue_request(&cpg->queue); + if (async_req) { + BUG_ON(cpg->eng_st != ENGINE_IDLE); + cpg->eng_st = ENGINE_BUSY; + } + } + spin_unlock_irq(&cpg->lock); + + if (backlog) { + backlog->complete(backlog, -EINPROGRESS); + backlog = NULL; + } + + if (async_req) { + req = container_of(async_req, + struct ablkcipher_request, base); + mv_enqueue_new_req(req); + async_req = NULL; + } + + schedule(); + + } while (!kthread_should_stop()); + return 0; +} + +static int mv_handle_req(struct ablkcipher_request *req) +{ + unsigned long flags; + int ret; + + spin_lock_irqsave(&cpg->lock, flags); + ret = ablkcipher_enqueue_request(&cpg->queue, req); + spin_unlock_irqrestore(&cpg->lock, flags); + wake_up_process(cpg->queue_th); + return ret; +} + +static int mv_enc_aes_ecb(struct ablkcipher_request *req) +{ + struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); + + req_ctx->op = COP_AES_ECB; + req_ctx->decrypt = 0; + + return mv_handle_req(req); +} + +static int mv_dec_aes_ecb(struct ablkcipher_request *req) +{ + struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); + + req_ctx->op = COP_AES_ECB; + req_ctx->decrypt = 1; + + compute_aes_dec_key(ctx); + return mv_handle_req(req); +} + +static int mv_enc_aes_cbc(struct ablkcipher_request *req) +{ + struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); + + req_ctx->op = COP_AES_CBC; + req_ctx->decrypt = 0; + + return mv_handle_req(req); +} + +static int mv_dec_aes_cbc(struct ablkcipher_request *req) +{ + struct mv_ctx *ctx = crypto_tfm_ctx(req->base.tfm); + struct mv_req_ctx *req_ctx = ablkcipher_request_ctx(req); + + req_ctx->op = COP_AES_CBC; + req_ctx->decrypt = 1; + + compute_aes_dec_key(ctx); + return mv_handle_req(req); +} + +static int mv_cra_init(struct crypto_tfm *tfm) +{ + tfm->crt_ablkcipher.reqsize = sizeof(struct mv_req_ctx); + return 0; +} + +irqreturn_t crypto_int(int irq, void *priv) +{ + u32 val; + + val = readl(cpg->reg + SEC_ACCEL_INT_STATUS); + if (!(val & SEC_INT_ACCEL0_DONE)) + return IRQ_NONE; + + val &= ~SEC_INT_ACCEL0_DONE; + writel(val, cpg->reg + FPGA_INT_STATUS); + writel(val, cpg->reg + SEC_ACCEL_INT_STATUS); + BUG_ON(cpg->eng_st != ENGINE_BUSY); + cpg->eng_st = ENGINE_W_DEQUEUE; + wake_up_process(cpg->queue_th); + return IRQ_HANDLED; +} + +struct crypto_alg mv_aes_alg_ecb = { + .cra_name = "ecb(aes)", + .cra_driver_name = "mv-ecb-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = 16, + .cra_ctxsize = sizeof(struct mv_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = mv_cra_init, + .cra_u = { + .ablkcipher = { + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = mv_setkey_aes, + .encrypt = mv_enc_aes_ecb, + .decrypt = mv_dec_aes_ecb, + }, + }, +}; + +struct crypto_alg mv_aes_alg_cbc = { + .cra_name = "cbc(aes)", + .cra_driver_name = "mv-cbc-aes", + .cra_priority = 300, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct mv_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = mv_cra_init, + .cra_u = { + .ablkcipher = { + .ivsize = AES_BLOCK_SIZE, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .setkey = mv_setkey_aes, + .encrypt = mv_enc_aes_cbc, + .decrypt = mv_dec_aes_cbc, + }, + }, +}; + +static int mv_probe(struct platform_device *pdev) +{ + struct crypto_priv *cp; + struct resource *res; + int irq; + int ret; + + if (cpg) { + printk(KERN_ERR "Second crypto dev?\n"); + return -EEXIST; + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "regs"); + if (!res) + return -ENXIO; + + cp = kzalloc(sizeof(*cp), GFP_KERNEL); + if (!cp) + return -ENOMEM; + + spin_lock_init(&cp->lock); + crypto_init_queue(&cp->queue, 50); + cp->reg = ioremap(res->start, res->end - res->start + 1); + if (!cp->reg) { + ret = -ENOMEM; + goto err; + } + + res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "sram"); + if (!res) { + ret = -ENXIO; + goto err_unmap_reg; + } + cp->sram_size = res->end - res->start + 1; + cp->max_req_size = cp->sram_size - SRAM_CFG_SPACE; + cp->sram = ioremap(res->start, cp->sram_size); + if (!cp->sram) { + ret = -ENOMEM; + goto err_unmap_reg; + } + + irq = platform_get_irq(pdev, 0); + if (irq < 0 || irq == NO_IRQ) { + ret = irq; + goto err_unmap_sram; + } + cp->irq = irq; + + platform_set_drvdata(pdev, cp); + cpg = cp; + + cp->queue_th = kthread_run(queue_manag, cp, "mv_crypto"); + if (IS_ERR(cp->queue_th)) { + ret = PTR_ERR(cp->queue_th); + goto err_thread; + } + + ret = request_irq(irq, crypto_int, IRQF_DISABLED, dev_name(&pdev->dev), + cp); + if (ret) + goto err_unmap_sram; + + writel(SEC_INT_ACCEL0_DONE, cpg->reg + SEC_ACCEL_INT_MASK); + writel(SEC_CFG_STOP_DIG_ERR, cpg->reg + SEC_ACCEL_CFG); + + ret = crypto_register_alg(&mv_aes_alg_ecb); + if (ret) + goto err_reg; + + ret = crypto_register_alg(&mv_aes_alg_cbc); + if (ret) + goto err_unreg_ecb; + return 0; +err_unreg_ecb: + crypto_unregister_alg(&mv_aes_alg_ecb); +err_thread: + free_irq(irq, cp); +err_reg: + kthread_stop(cp->queue_th); +err_unmap_sram: + iounmap(cp->sram); +err_unmap_reg: + iounmap(cp->reg); +err: + kfree(cp); + cpg = NULL; + platform_set_drvdata(pdev, NULL); + return ret; +} + +static int mv_remove(struct platform_device *pdev) +{ + struct crypto_priv *cp = platform_get_drvdata(pdev); + + crypto_unregister_alg(&mv_aes_alg_ecb); + crypto_unregister_alg(&mv_aes_alg_cbc); + kthread_stop(cp->queue_th); + free_irq(cp->irq, cp); + memset(cp->sram, 0, cp->sram_size); + iounmap(cp->sram); + iounmap(cp->reg); + kfree(cp); + cpg = NULL; + return 0; +} + +static struct platform_driver marvell_crypto = { + .probe = mv_probe, + .remove = mv_remove, + .driver = { + .owner = THIS_MODULE, + .name = "mv_crypto", + }, +}; +MODULE_ALIAS("platform:mv_crypto"); + +static int __init mv_crypto_init(void) +{ + return platform_driver_register(&marvell_crypto); +} +module_init(mv_crypto_init); + +static void __exit mv_crypto_exit(void) +{ + platform_driver_unregister(&marvell_crypto); +} +module_exit(mv_crypto_exit); + +MODULE_AUTHOR("Sebastian Andrzej Siewior <sebastian@breakpoint.cc>"); +MODULE_DESCRIPTION("Support for Marvell's cryptographic engine"); +MODULE_LICENSE("GPL"); diff --git a/drivers/crypto/mv_cesa.h b/drivers/crypto/mv_cesa.h new file mode 100644 index 0000000..c3e25d3 --- /dev/null +++ b/drivers/crypto/mv_cesa.h @@ -0,0 +1,119 @@ +#ifndef __MV_CRYPTO_H__ + +#define DIGEST_INITIAL_VAL_A 0xdd00 +#define DES_CMD_REG 0xdd58 + +#define SEC_ACCEL_CMD 0xde00 +#define SEC_CMD_EN_SEC_ACCL0 (1 << 0) +#define SEC_CMD_EN_SEC_ACCL1 (1 << 1) +#define SEC_CMD_DISABLE_SEC (1 << 2) + +#define SEC_ACCEL_DESC_P0 0xde04 +#define SEC_DESC_P0_PTR(x) (x) + +#define SEC_ACCEL_DESC_P1 0xde14 +#define SEC_DESC_P1_PTR(x) (x) + +#define SEC_ACCEL_CFG 0xde08 +#define SEC_CFG_STOP_DIG_ERR (1 << 0) +#define SEC_CFG_CH0_W_IDMA (1 << 7) +#define SEC_CFG_CH1_W_IDMA (1 << 8) +#define SEC_CFG_ACT_CH0_IDMA (1 << 9) +#define SEC_CFG_ACT_CH1_IDMA (1 << 10) + +#define SEC_ACCEL_STATUS 0xde0c +#define SEC_ST_ACT_0 (1 << 0) +#define SEC_ST_ACT_1 (1 << 1) + +/* + * FPGA_INT_STATUS looks like a FPGA leftover and is documented only in Errata + * 4.12. It looks like that it was part of an IRQ-controller in FPGA and + * someone forgot to remove it while switching to the core and moving to + * SEC_ACCEL_INT_STATUS. + */ +#define FPGA_INT_STATUS 0xdd68 +#define SEC_ACCEL_INT_STATUS 0xde20 +#define SEC_INT_AUTH_DONE (1 << 0) +#define SEC_INT_DES_E_DONE (1 << 1) +#define SEC_INT_AES_E_DONE (1 << 2) +#define SEC_INT_AES_D_DONE (1 << 3) +#define SEC_INT_ENC_DONE (1 << 4) +#define SEC_INT_ACCEL0_DONE (1 << 5) +#define SEC_INT_ACCEL1_DONE (1 << 6) +#define SEC_INT_ACC0_IDMA_DONE (1 << 7) +#define SEC_INT_ACC1_IDMA_DONE (1 << 8) + +#define SEC_ACCEL_INT_MASK 0xde24 + +#define AES_KEY_LEN (8 * 4) + +struct sec_accel_config { + + u32 config; +#define CFG_OP_MAC_ONLY 0 +#define CFG_OP_CRYPT_ONLY 1 +#define CFG_OP_MAC_CRYPT 2 +#define CFG_OP_CRYPT_MAC 3 +#define CFG_MACM_MD5 (4 << 4) +#define CFG_MACM_SHA1 (5 << 4) +#define CFG_MACM_HMAC_MD5 (6 << 4) +#define CFG_MACM_HMAC_SHA1 (7 << 4) +#define CFG_ENCM_DES (1 << 8) +#define CFG_ENCM_3DES (2 << 8) +#define CFG_ENCM_AES (3 << 8) +#define CFG_DIR_ENC (0 << 12) +#define CFG_DIR_DEC (1 << 12) +#define CFG_ENC_MODE_ECB (0 << 16) +#define CFG_ENC_MODE_CBC (1 << 16) +#define CFG_3DES_EEE (0 << 20) +#define CFG_3DES_EDE (1 << 20) +#define CFG_AES_LEN_128 (0 << 24) +#define CFG_AES_LEN_192 (1 << 24) +#define CFG_AES_LEN_256 (2 << 24) + + u32 enc_p; +#define ENC_P_SRC(x) (x) +#define ENC_P_DST(x) ((x) << 16) + + u32 enc_len; +#define ENC_LEN(x) (x) + + u32 enc_key_p; +#define ENC_KEY_P(x) (x) + + u32 enc_iv; +#define ENC_IV_POINT(x) ((x) << 0) +#define ENC_IV_BUF_POINT(x) ((x) << 16) + + u32 mac_src_p; +#define MAC_SRC_DATA_P(x) (x) +#define MAC_SRC_TOTAL_LEN(x) ((x) << 16) + + u32 mac_digest; + u32 mac_iv; +}__attribute__ ((packed)); + /* + * /-----------\ 0 + * | ACCEL CFG | 4 * 8 + * |-----------| 0x20 + * | CRYPT KEY | 8 * 4 + * |-----------| 0x40 + * | IV IN | 4 * 4 + * |-----------| 0x40 (inplace) + * | IV BUF | 4 * 4 + * |-----------| 0x50 + * | DATA IN | 16 * x (max ->max_req_size) + * |-----------| 0x50 (inplace operation) + * | DATA OUT | 16 * x (max ->max_req_size) + * \-----------/ SRAM size + */ +#define SRAM_CONFIG 0x00 +#define SRAM_DATA_KEY_P 0x20 +#define SRAM_DATA_IV 0x40 +#define SRAM_DATA_IV_BUF 0x40 +#define SRAM_DATA_IN_START 0x50 +#define SRAM_DATA_OUT_START 0x50 + +#define SRAM_CFG_SPACE 0x50 + +#endif diff --git a/drivers/crypto/padlock-sha.c b/drivers/crypto/padlock-sha.c index a2c8e85..76cb6b3 100644 --- a/drivers/crypto/padlock-sha.c +++ b/drivers/crypto/padlock-sha.c @@ -12,81 +12,43 @@ * */ -#include <crypto/algapi.h> +#include <crypto/internal/hash.h> #include <crypto/sha.h> #include <linux/err.h> #include <linux/module.h> #include <linux/init.h> #include <linux/errno.h> -#include <linux/cryptohash.h> #include <linux/interrupt.h> #include <linux/kernel.h> #include <linux/scatterlist.h> #include <asm/i387.h> #include "padlock.h" -#define SHA1_DEFAULT_FALLBACK "sha1-generic" -#define SHA256_DEFAULT_FALLBACK "sha256-generic" +struct padlock_sha_desc { + struct shash_desc fallback; +}; struct padlock_sha_ctx { - char *data; - size_t used; - int bypass; - void (*f_sha_padlock)(const char *in, char *out, int count); - struct hash_desc fallback; + struct crypto_shash *fallback; }; -static inline struct padlock_sha_ctx *ctx(struct crypto_tfm *tfm) -{ - return crypto_tfm_ctx(tfm); -} - -/* We'll need aligned address on the stack */ -#define NEAREST_ALIGNED(ptr) \ - ((void *)ALIGN((size_t)(ptr), PADLOCK_ALIGNMENT)) - -static struct crypto_alg sha1_alg, sha256_alg; - -static void padlock_sha_bypass(struct crypto_tfm *tfm) +static int padlock_sha_init(struct shash_desc *desc) { - if (ctx(tfm)->bypass) - return; + struct padlock_sha_desc *dctx = shash_desc_ctx(desc); + struct padlock_sha_ctx *ctx = crypto_shash_ctx(desc->tfm); - crypto_hash_init(&ctx(tfm)->fallback); - if (ctx(tfm)->data && ctx(tfm)->used) { - struct scatterlist sg; - - sg_init_one(&sg, ctx(tfm)->data, ctx(tfm)->used); - crypto_hash_update(&ctx(tfm)->fallback, &sg, sg.length); - } - - ctx(tfm)->used = 0; - ctx(tfm)->bypass = 1; -} - -static void padlock_sha_init(struct crypto_tfm *tfm) -{ - ctx(tfm)->used = 0; - ctx(tfm)->bypass = 0; + dctx->fallback.tfm = ctx->fallback; + dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + return crypto_shash_init(&dctx->fallback); } -static void padlock_sha_update(struct crypto_tfm *tfm, - const uint8_t *data, unsigned int length) +static int padlock_sha_update(struct shash_desc *desc, + const u8 *data, unsigned int length) { - /* Our buffer is always one page. */ - if (unlikely(!ctx(tfm)->bypass && - (ctx(tfm)->used + length > PAGE_SIZE))) - padlock_sha_bypass(tfm); - - if (unlikely(ctx(tfm)->bypass)) { - struct scatterlist sg; - sg_init_one(&sg, (uint8_t *)data, length); - crypto_hash_update(&ctx(tfm)->fallback, &sg, length); - return; - } + struct padlock_sha_desc *dctx = shash_desc_ctx(desc); - memcpy(ctx(tfm)->data + ctx(tfm)->used, data, length); - ctx(tfm)->used += length; + dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + return crypto_shash_update(&dctx->fallback, data, length); } static inline void padlock_output_block(uint32_t *src, @@ -96,165 +58,206 @@ static inline void padlock_output_block(uint32_t *src, *dst++ = swab32(*src++); } -static void padlock_do_sha1(const char *in, char *out, int count) +static int padlock_sha1_finup(struct shash_desc *desc, const u8 *in, + unsigned int count, u8 *out) { /* We can't store directly to *out as it may be unaligned. */ /* BTW Don't reduce the buffer size below 128 Bytes! * PadLock microcode needs it that big. */ - char buf[128+16]; - char *result = NEAREST_ALIGNED(buf); + char result[128] __attribute__ ((aligned(PADLOCK_ALIGNMENT))); + struct padlock_sha_desc *dctx = shash_desc_ctx(desc); + struct sha1_state state; + unsigned int space; + unsigned int leftover; int ts_state; + int err; + + dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + err = crypto_shash_export(&dctx->fallback, &state); + if (err) + goto out; + + if (state.count + count > ULONG_MAX) + return crypto_shash_finup(&dctx->fallback, in, count, out); + + leftover = ((state.count - 1) & (SHA1_BLOCK_SIZE - 1)) + 1; + space = SHA1_BLOCK_SIZE - leftover; + if (space) { + if (count > space) { + err = crypto_shash_update(&dctx->fallback, in, space) ?: + crypto_shash_export(&dctx->fallback, &state); + if (err) + goto out; + count -= space; + in += space; + } else { + memcpy(state.buffer + leftover, in, count); + in = state.buffer; + count += leftover; + state.count &= ~(SHA1_BLOCK_SIZE - 1); + } + } + + memcpy(result, &state.state, SHA1_DIGEST_SIZE); - ((uint32_t *)result)[0] = SHA1_H0; - ((uint32_t *)result)[1] = SHA1_H1; - ((uint32_t *)result)[2] = SHA1_H2; - ((uint32_t *)result)[3] = SHA1_H3; - ((uint32_t *)result)[4] = SHA1_H4; - /* prevent taking the spurious DNA fault with padlock. */ ts_state = irq_ts_save(); asm volatile (".byte 0xf3,0x0f,0xa6,0xc8" /* rep xsha1 */ - : "+S"(in), "+D"(result) - : "c"(count), "a"(0)); + : \ + : "c"((unsigned long)state.count + count), \ + "a"((unsigned long)state.count), \ + "S"(in), "D"(result)); irq_ts_restore(ts_state); padlock_output_block((uint32_t *)result, (uint32_t *)out, 5); + +out: + return err; } -static void padlock_do_sha256(const char *in, char *out, int count) +static int padlock_sha1_final(struct shash_desc *desc, u8 *out) +{ + u8 buf[4]; + + return padlock_sha1_finup(desc, buf, 0, out); +} + +static int padlock_sha256_finup(struct shash_desc *desc, const u8 *in, + unsigned int count, u8 *out) { /* We can't store directly to *out as it may be unaligned. */ /* BTW Don't reduce the buffer size below 128 Bytes! * PadLock microcode needs it that big. */ - char buf[128+16]; - char *result = NEAREST_ALIGNED(buf); + char result[128] __attribute__ ((aligned(PADLOCK_ALIGNMENT))); + struct padlock_sha_desc *dctx = shash_desc_ctx(desc); + struct sha256_state state; + unsigned int space; + unsigned int leftover; int ts_state; + int err; + + dctx->fallback.flags = desc->flags & CRYPTO_TFM_REQ_MAY_SLEEP; + err = crypto_shash_export(&dctx->fallback, &state); + if (err) + goto out; + + if (state.count + count > ULONG_MAX) + return crypto_shash_finup(&dctx->fallback, in, count, out); + + leftover = ((state.count - 1) & (SHA256_BLOCK_SIZE - 1)) + 1; + space = SHA256_BLOCK_SIZE - leftover; + if (space) { + if (count > space) { + err = crypto_shash_update(&dctx->fallback, in, space) ?: + crypto_shash_export(&dctx->fallback, &state); + if (err) + goto out; + count -= space; + in += space; + } else { + memcpy(state.buf + leftover, in, count); + in = state.buf; + count += leftover; + state.count &= ~(SHA1_BLOCK_SIZE - 1); + } + } - ((uint32_t *)result)[0] = SHA256_H0; - ((uint32_t *)result)[1] = SHA256_H1; - ((uint32_t *)result)[2] = SHA256_H2; - ((uint32_t *)result)[3] = SHA256_H3; - ((uint32_t *)result)[4] = SHA256_H4; - ((uint32_t *)result)[5] = SHA256_H5; - ((uint32_t *)result)[6] = SHA256_H6; - ((uint32_t *)result)[7] = SHA256_H7; + memcpy(result, &state.state, SHA256_DIGEST_SIZE); /* prevent taking the spurious DNA fault with padlock. */ ts_state = irq_ts_save(); asm volatile (".byte 0xf3,0x0f,0xa6,0xd0" /* rep xsha256 */ - : "+S"(in), "+D"(result) - : "c"(count), "a"(0)); + : \ + : "c"((unsigned long)state.count + count), \ + "a"((unsigned long)state.count), \ + "S"(in), "D"(result)); irq_ts_restore(ts_state); padlock_output_block((uint32_t *)result, (uint32_t *)out, 8); + +out: + return err; } -static void padlock_sha_final(struct crypto_tfm *tfm, uint8_t *out) +static int padlock_sha256_final(struct shash_desc *desc, u8 *out) { - if (unlikely(ctx(tfm)->bypass)) { - crypto_hash_final(&ctx(tfm)->fallback, out); - ctx(tfm)->bypass = 0; - return; - } + u8 buf[4]; - /* Pass the input buffer to PadLock microcode... */ - ctx(tfm)->f_sha_padlock(ctx(tfm)->data, out, ctx(tfm)->used); - - ctx(tfm)->used = 0; + return padlock_sha256_finup(desc, buf, 0, out); } static int padlock_cra_init(struct crypto_tfm *tfm) { + struct crypto_shash *hash = __crypto_shash_cast(tfm); const char *fallback_driver_name = tfm->__crt_alg->cra_name; - struct crypto_hash *fallback_tfm; - - /* For now we'll allocate one page. This - * could eventually be configurable one day. */ - ctx(tfm)->data = (char *)__get_free_page(GFP_KERNEL); - if (!ctx(tfm)->data) - return -ENOMEM; + struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm); + struct crypto_shash *fallback_tfm; + int err = -ENOMEM; /* Allocate a fallback and abort if it failed. */ - fallback_tfm = crypto_alloc_hash(fallback_driver_name, 0, - CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK); + fallback_tfm = crypto_alloc_shash(fallback_driver_name, 0, + CRYPTO_ALG_NEED_FALLBACK); if (IS_ERR(fallback_tfm)) { printk(KERN_WARNING PFX "Fallback driver '%s' could not be loaded!\n", fallback_driver_name); - free_page((unsigned long)(ctx(tfm)->data)); - return PTR_ERR(fallback_tfm); + err = PTR_ERR(fallback_tfm); + goto out; } - ctx(tfm)->fallback.tfm = fallback_tfm; + ctx->fallback = fallback_tfm; + hash->descsize += crypto_shash_descsize(fallback_tfm); return 0; -} - -static int padlock_sha1_cra_init(struct crypto_tfm *tfm) -{ - ctx(tfm)->f_sha_padlock = padlock_do_sha1; - return padlock_cra_init(tfm); -} - -static int padlock_sha256_cra_init(struct crypto_tfm *tfm) -{ - ctx(tfm)->f_sha_padlock = padlock_do_sha256; - - return padlock_cra_init(tfm); +out: + return err; } static void padlock_cra_exit(struct crypto_tfm *tfm) { - if (ctx(tfm)->data) { - free_page((unsigned long)(ctx(tfm)->data)); - ctx(tfm)->data = NULL; - } + struct padlock_sha_ctx *ctx = crypto_tfm_ctx(tfm); - crypto_free_hash(ctx(tfm)->fallback.tfm); - ctx(tfm)->fallback.tfm = NULL; + crypto_free_shash(ctx->fallback); } -static struct crypto_alg sha1_alg = { - .cra_name = "sha1", - .cra_driver_name = "sha1-padlock", - .cra_priority = PADLOCK_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_DIGEST | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA1_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct padlock_sha_ctx), - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(sha1_alg.cra_list), - .cra_init = padlock_sha1_cra_init, - .cra_exit = padlock_cra_exit, - .cra_u = { - .digest = { - .dia_digestsize = SHA1_DIGEST_SIZE, - .dia_init = padlock_sha_init, - .dia_update = padlock_sha_update, - .dia_final = padlock_sha_final, - } +static struct shash_alg sha1_alg = { + .digestsize = SHA1_DIGEST_SIZE, + .init = padlock_sha_init, + .update = padlock_sha_update, + .finup = padlock_sha1_finup, + .final = padlock_sha1_final, + .descsize = sizeof(struct padlock_sha_desc), + .base = { + .cra_name = "sha1", + .cra_driver_name = "sha1-padlock", + .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA1_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct padlock_sha_ctx), + .cra_module = THIS_MODULE, + .cra_init = padlock_cra_init, + .cra_exit = padlock_cra_exit, } }; -static struct crypto_alg sha256_alg = { - .cra_name = "sha256", - .cra_driver_name = "sha256-padlock", - .cra_priority = PADLOCK_CRA_PRIORITY, - .cra_flags = CRYPTO_ALG_TYPE_DIGEST | - CRYPTO_ALG_NEED_FALLBACK, - .cra_blocksize = SHA256_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct padlock_sha_ctx), - .cra_module = THIS_MODULE, - .cra_list = LIST_HEAD_INIT(sha256_alg.cra_list), - .cra_init = padlock_sha256_cra_init, - .cra_exit = padlock_cra_exit, - .cra_u = { - .digest = { - .dia_digestsize = SHA256_DIGEST_SIZE, - .dia_init = padlock_sha_init, - .dia_update = padlock_sha_update, - .dia_final = padlock_sha_final, - } +static struct shash_alg sha256_alg = { + .digestsize = SHA256_DIGEST_SIZE, + .init = padlock_sha_init, + .update = padlock_sha_update, + .finup = padlock_sha256_finup, + .final = padlock_sha256_final, + .descsize = sizeof(struct padlock_sha_desc), + .base = { + .cra_name = "sha256", + .cra_driver_name = "sha256-padlock", + .cra_priority = PADLOCK_CRA_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH | + CRYPTO_ALG_NEED_FALLBACK, + .cra_blocksize = SHA256_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct padlock_sha_ctx), + .cra_module = THIS_MODULE, + .cra_init = padlock_cra_init, + .cra_exit = padlock_cra_exit, } }; @@ -272,11 +275,11 @@ static int __init padlock_init(void) return -ENODEV; } - rc = crypto_register_alg(&sha1_alg); + rc = crypto_register_shash(&sha1_alg); if (rc) goto out; - rc = crypto_register_alg(&sha256_alg); + rc = crypto_register_shash(&sha256_alg); if (rc) goto out_unreg1; @@ -285,7 +288,7 @@ static int __init padlock_init(void) return 0; out_unreg1: - crypto_unregister_alg(&sha1_alg); + crypto_unregister_shash(&sha1_alg); out: printk(KERN_ERR PFX "VIA PadLock SHA1/SHA256 initialization failed.\n"); return rc; @@ -293,8 +296,8 @@ out: static void __exit padlock_fini(void) { - crypto_unregister_alg(&sha1_alg); - crypto_unregister_alg(&sha256_alg); + crypto_unregister_shash(&sha1_alg); + crypto_unregister_shash(&sha256_alg); } module_init(padlock_init); diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index c70775f..c47ffe8 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -86,6 +86,25 @@ struct talitos_request { void *context; }; +/* per-channel fifo management */ +struct talitos_channel { + /* request fifo */ + struct talitos_request *fifo; + + /* number of requests pending in channel h/w fifo */ + atomic_t submit_count ____cacheline_aligned; + + /* request submission (head) lock */ + spinlock_t head_lock ____cacheline_aligned; + /* index to next free descriptor request */ + int head; + + /* request release (tail) lock */ + spinlock_t tail_lock ____cacheline_aligned; + /* index to next in-progress/done descriptor request */ + int tail; +}; + struct talitos_private { struct device *dev; struct of_device *ofdev; @@ -101,15 +120,6 @@ struct talitos_private { /* SEC Compatibility info */ unsigned long features; - /* next channel to be assigned next incoming descriptor */ - atomic_t last_chan; - - /* per-channel number of requests pending in channel h/w fifo */ - atomic_t *submit_count; - - /* per-channel request fifo */ - struct talitos_request **fifo; - /* * length of the request fifo * fifo_len is chfifo_len rounded up to next power of 2 @@ -117,15 +127,10 @@ struct talitos_private { */ unsigned int fifo_len; - /* per-channel index to next free descriptor request */ - int *head; - - /* per-channel index to next in-progress/done descriptor request */ - int *tail; + struct talitos_channel *chan; - /* per-channel request submission (head) and release (tail) locks */ - spinlock_t *head_lock; - spinlock_t *tail_lock; + /* next channel to be assigned next incoming descriptor */ + atomic_t last_chan ____cacheline_aligned; /* request callback tasklet */ struct tasklet_struct done_task; @@ -141,6 +146,12 @@ struct talitos_private { #define TALITOS_FTR_SRC_LINK_TBL_LEN_INCLUDES_EXTENT 0x00000001 #define TALITOS_FTR_HW_AUTH_CHECK 0x00000002 +static void to_talitos_ptr(struct talitos_ptr *talitos_ptr, dma_addr_t dma_addr) +{ + talitos_ptr->ptr = cpu_to_be32(lower_32_bits(dma_addr)); + talitos_ptr->eptr = cpu_to_be32(upper_32_bits(dma_addr)); +} + /* * map virtual single (contiguous) pointer to h/w descriptor pointer */ @@ -150,8 +161,10 @@ static void map_single_talitos_ptr(struct device *dev, unsigned char extent, enum dma_data_direction dir) { + dma_addr_t dma_addr = dma_map_single(dev, data, len, dir); + talitos_ptr->len = cpu_to_be16(len); - talitos_ptr->ptr = cpu_to_be32(dma_map_single(dev, data, len, dir)); + to_talitos_ptr(talitos_ptr, dma_addr); talitos_ptr->j_extent = extent; } @@ -182,9 +195,9 @@ static int reset_channel(struct device *dev, int ch) return -EIO; } - /* set done writeback and IRQ */ - setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_CDWE | - TALITOS_CCCR_LO_CDIE); + /* set 36-bit addressing, done writeback enable and done IRQ enable */ + setbits32(priv->reg + TALITOS_CCCR_LO(ch), TALITOS_CCCR_LO_EAE | + TALITOS_CCCR_LO_CDWE | TALITOS_CCCR_LO_CDIE); /* and ICCR writeback, if available */ if (priv->features & TALITOS_FTR_HW_AUTH_CHECK) @@ -282,16 +295,16 @@ static int talitos_submit(struct device *dev, struct talitos_desc *desc, /* emulate SEC's round-robin channel fifo polling scheme */ ch = atomic_inc_return(&priv->last_chan) & (priv->num_channels - 1); - spin_lock_irqsave(&priv->head_lock[ch], flags); + spin_lock_irqsave(&priv->chan[ch].head_lock, flags); - if (!atomic_inc_not_zero(&priv->submit_count[ch])) { + if (!atomic_inc_not_zero(&priv->chan[ch].submit_count)) { /* h/w fifo is full */ - spin_unlock_irqrestore(&priv->head_lock[ch], flags); + spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); return -EAGAIN; } - head = priv->head[ch]; - request = &priv->fifo[ch][head]; + head = priv->chan[ch].head; + request = &priv->chan[ch].fifo[head]; /* map descriptor and save caller data */ request->dma_desc = dma_map_single(dev, desc, sizeof(*desc), @@ -300,16 +313,19 @@ static int talitos_submit(struct device *dev, struct talitos_desc *desc, request->context = context; /* increment fifo head */ - priv->head[ch] = (priv->head[ch] + 1) & (priv->fifo_len - 1); + priv->chan[ch].head = (priv->chan[ch].head + 1) & (priv->fifo_len - 1); smp_wmb(); request->desc = desc; /* GO! */ wmb(); - out_be32(priv->reg + TALITOS_FF_LO(ch), request->dma_desc); + out_be32(priv->reg + TALITOS_FF(ch), + cpu_to_be32(upper_32_bits(request->dma_desc))); + out_be32(priv->reg + TALITOS_FF_LO(ch), + cpu_to_be32(lower_32_bits(request->dma_desc))); - spin_unlock_irqrestore(&priv->head_lock[ch], flags); + spin_unlock_irqrestore(&priv->chan[ch].head_lock, flags); return -EINPROGRESS; } @@ -324,11 +340,11 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) unsigned long flags; int tail, status; - spin_lock_irqsave(&priv->tail_lock[ch], flags); + spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); - tail = priv->tail[ch]; - while (priv->fifo[ch][tail].desc) { - request = &priv->fifo[ch][tail]; + tail = priv->chan[ch].tail; + while (priv->chan[ch].fifo[tail].desc) { + request = &priv->chan[ch].fifo[tail]; /* descriptors with their done bits set don't get the error */ rmb(); @@ -354,22 +370,22 @@ static void flush_channel(struct device *dev, int ch, int error, int reset_ch) request->desc = NULL; /* increment fifo tail */ - priv->tail[ch] = (tail + 1) & (priv->fifo_len - 1); + priv->chan[ch].tail = (tail + 1) & (priv->fifo_len - 1); - spin_unlock_irqrestore(&priv->tail_lock[ch], flags); + spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); - atomic_dec(&priv->submit_count[ch]); + atomic_dec(&priv->chan[ch].submit_count); saved_req.callback(dev, saved_req.desc, saved_req.context, status); /* channel may resume processing in single desc error case */ if (error && !reset_ch && status == error) return; - spin_lock_irqsave(&priv->tail_lock[ch], flags); - tail = priv->tail[ch]; + spin_lock_irqsave(&priv->chan[ch].tail_lock, flags); + tail = priv->chan[ch].tail; } - spin_unlock_irqrestore(&priv->tail_lock[ch], flags); + spin_unlock_irqrestore(&priv->chan[ch].tail_lock, flags); } /* @@ -397,20 +413,20 @@ static void talitos_done(unsigned long data) static struct talitos_desc *current_desc(struct device *dev, int ch) { struct talitos_private *priv = dev_get_drvdata(dev); - int tail = priv->tail[ch]; + int tail = priv->chan[ch].tail; dma_addr_t cur_desc; cur_desc = in_be32(priv->reg + TALITOS_CDPR_LO(ch)); - while (priv->fifo[ch][tail].dma_desc != cur_desc) { + while (priv->chan[ch].fifo[tail].dma_desc != cur_desc) { tail = (tail + 1) & (priv->fifo_len - 1); - if (tail == priv->tail[ch]) { + if (tail == priv->chan[ch].tail) { dev_err(dev, "couldn't locate current descriptor\n"); return NULL; } } - return priv->fifo[ch][tail].desc; + return priv->chan[ch].fifo[tail].desc; } /* @@ -929,7 +945,7 @@ static int sg_to_link_tbl(struct scatterlist *sg, int sg_count, int n_sg = sg_count; while (n_sg--) { - link_tbl_ptr->ptr = cpu_to_be32(sg_dma_address(sg)); + to_talitos_ptr(link_tbl_ptr, sg_dma_address(sg)); link_tbl_ptr->len = cpu_to_be16(sg_dma_len(sg)); link_tbl_ptr->j_extent = 0; link_tbl_ptr++; @@ -970,7 +986,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, struct talitos_desc *desc = &edesc->desc; unsigned int cryptlen = areq->cryptlen; unsigned int authsize = ctx->authsize; - unsigned int ivsize; + unsigned int ivsize = crypto_aead_ivsize(aead); int sg_count, ret; int sg_link_tbl_len; @@ -978,11 +994,9 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, map_single_talitos_ptr(dev, &desc->ptr[0], ctx->authkeylen, &ctx->key, 0, DMA_TO_DEVICE); /* hmac data */ - map_single_talitos_ptr(dev, &desc->ptr[1], sg_virt(areq->src) - - sg_virt(areq->assoc), sg_virt(areq->assoc), 0, - DMA_TO_DEVICE); + map_single_talitos_ptr(dev, &desc->ptr[1], areq->assoclen + ivsize, + sg_virt(areq->assoc), 0, DMA_TO_DEVICE); /* cipher iv */ - ivsize = crypto_aead_ivsize(aead); map_single_talitos_ptr(dev, &desc->ptr[2], ivsize, giv ?: areq->iv, 0, DMA_TO_DEVICE); @@ -1006,7 +1020,7 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, edesc->src_is_chained); if (sg_count == 1) { - desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->src)); + to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->src)); } else { sg_link_tbl_len = cryptlen; @@ -1017,14 +1031,14 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, &edesc->link_tbl[0]); if (sg_count > 1) { desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; - desc->ptr[4].ptr = cpu_to_be32(edesc->dma_link_tbl); + to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl); dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); } else { /* Only one segment now, so no link tbl needed */ - desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq-> - src)); + to_talitos_ptr(&desc->ptr[4], + sg_dma_address(areq->src)); } } @@ -1039,14 +1053,14 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, edesc->dst_is_chained); if (sg_count == 1) { - desc->ptr[5].ptr = cpu_to_be32(sg_dma_address(areq->dst)); + to_talitos_ptr(&desc->ptr[5], sg_dma_address(areq->dst)); } else { struct talitos_ptr *link_tbl_ptr = &edesc->link_tbl[edesc->src_nents + 1]; - desc->ptr[5].ptr = cpu_to_be32((struct talitos_ptr *) - edesc->dma_link_tbl + - edesc->src_nents + 1); + to_talitos_ptr(&desc->ptr[5], edesc->dma_link_tbl + + (edesc->src_nents + 1) * + sizeof(struct talitos_ptr)); sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, link_tbl_ptr); @@ -1059,11 +1073,9 @@ static int ipsec_esp(struct talitos_edesc *edesc, struct aead_request *areq, link_tbl_ptr->len = cpu_to_be16(authsize); /* icv data follows link tables */ - link_tbl_ptr->ptr = cpu_to_be32((struct talitos_ptr *) - edesc->dma_link_tbl + - edesc->src_nents + - edesc->dst_nents + 2); - + to_talitos_ptr(link_tbl_ptr, edesc->dma_link_tbl + + (edesc->src_nents + edesc->dst_nents + 2) * + sizeof(struct talitos_ptr)); desc->ptr[5].j_extent |= DESC_PTR_LNKTBL_JUMP; dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); @@ -1338,7 +1350,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* first DWORD empty */ desc->ptr[0].len = 0; - desc->ptr[0].ptr = 0; + to_talitos_ptr(&desc->ptr[0], 0); desc->ptr[0].j_extent = 0; /* cipher iv */ @@ -1362,20 +1374,20 @@ static int common_nonsnoop(struct talitos_edesc *edesc, edesc->src_is_chained); if (sg_count == 1) { - desc->ptr[3].ptr = cpu_to_be32(sg_dma_address(areq->src)); + to_talitos_ptr(&desc->ptr[3], sg_dma_address(areq->src)); } else { sg_count = sg_to_link_tbl(areq->src, sg_count, cryptlen, &edesc->link_tbl[0]); if (sg_count > 1) { + to_talitos_ptr(&desc->ptr[3], edesc->dma_link_tbl); desc->ptr[3].j_extent |= DESC_PTR_LNKTBL_JUMP; - desc->ptr[3].ptr = cpu_to_be32(edesc->dma_link_tbl); dma_sync_single_for_device(dev, edesc->dma_link_tbl, edesc->dma_len, DMA_BIDIRECTIONAL); } else { /* Only one segment now, so no link tbl needed */ - desc->ptr[3].ptr = cpu_to_be32(sg_dma_address(areq-> - src)); + to_talitos_ptr(&desc->ptr[3], + sg_dma_address(areq->src)); } } @@ -1390,15 +1402,15 @@ static int common_nonsnoop(struct talitos_edesc *edesc, edesc->dst_is_chained); if (sg_count == 1) { - desc->ptr[4].ptr = cpu_to_be32(sg_dma_address(areq->dst)); + to_talitos_ptr(&desc->ptr[4], sg_dma_address(areq->dst)); } else { struct talitos_ptr *link_tbl_ptr = &edesc->link_tbl[edesc->src_nents + 1]; + to_talitos_ptr(&desc->ptr[4], edesc->dma_link_tbl + + (edesc->src_nents + 1) * + sizeof(struct talitos_ptr)); desc->ptr[4].j_extent |= DESC_PTR_LNKTBL_JUMP; - desc->ptr[4].ptr = cpu_to_be32((struct talitos_ptr *) - edesc->dma_link_tbl + - edesc->src_nents + 1); sg_count = sg_to_link_tbl(areq->dst, sg_count, cryptlen, link_tbl_ptr); dma_sync_single_for_device(ctx->dev, edesc->dma_link_tbl, @@ -1411,7 +1423,7 @@ static int common_nonsnoop(struct talitos_edesc *edesc, /* last DWORD empty */ desc->ptr[6].len = 0; - desc->ptr[6].ptr = 0; + to_talitos_ptr(&desc->ptr[6], 0); desc->ptr[6].j_extent = 0; ret = talitos_submit(dev, desc, callback, areq); @@ -1742,17 +1754,11 @@ static int talitos_remove(struct of_device *ofdev) if (hw_supports(dev, DESC_HDR_SEL0_RNG)) talitos_unregister_rng(dev); - kfree(priv->submit_count); - kfree(priv->tail); - kfree(priv->head); - - if (priv->fifo) - for (i = 0; i < priv->num_channels; i++) - kfree(priv->fifo[i]); + for (i = 0; i < priv->num_channels; i++) + if (priv->chan[i].fifo) + kfree(priv->chan[i].fifo); - kfree(priv->fifo); - kfree(priv->head_lock); - kfree(priv->tail_lock); + kfree(priv->chan); if (priv->irq != NO_IRQ) { free_irq(priv->irq, dev); @@ -1872,58 +1878,36 @@ static int talitos_probe(struct of_device *ofdev, if (of_device_is_compatible(np, "fsl,sec2.1")) priv->features |= TALITOS_FTR_HW_AUTH_CHECK; - priv->head_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels, - GFP_KERNEL); - priv->tail_lock = kmalloc(sizeof(spinlock_t) * priv->num_channels, - GFP_KERNEL); - if (!priv->head_lock || !priv->tail_lock) { - dev_err(dev, "failed to allocate fifo locks\n"); + priv->chan = kzalloc(sizeof(struct talitos_channel) * + priv->num_channels, GFP_KERNEL); + if (!priv->chan) { + dev_err(dev, "failed to allocate channel management space\n"); err = -ENOMEM; goto err_out; } for (i = 0; i < priv->num_channels; i++) { - spin_lock_init(&priv->head_lock[i]); - spin_lock_init(&priv->tail_lock[i]); - } - - priv->fifo = kmalloc(sizeof(struct talitos_request *) * - priv->num_channels, GFP_KERNEL); - if (!priv->fifo) { - dev_err(dev, "failed to allocate request fifo\n"); - err = -ENOMEM; - goto err_out; + spin_lock_init(&priv->chan[i].head_lock); + spin_lock_init(&priv->chan[i].tail_lock); } priv->fifo_len = roundup_pow_of_two(priv->chfifo_len); for (i = 0; i < priv->num_channels; i++) { - priv->fifo[i] = kzalloc(sizeof(struct talitos_request) * - priv->fifo_len, GFP_KERNEL); - if (!priv->fifo[i]) { + priv->chan[i].fifo = kzalloc(sizeof(struct talitos_request) * + priv->fifo_len, GFP_KERNEL); + if (!priv->chan[i].fifo) { dev_err(dev, "failed to allocate request fifo %d\n", i); err = -ENOMEM; goto err_out; } } - priv->submit_count = kmalloc(sizeof(atomic_t) * priv->num_channels, - GFP_KERNEL); - if (!priv->submit_count) { - dev_err(dev, "failed to allocate fifo submit count space\n"); - err = -ENOMEM; - goto err_out; - } for (i = 0; i < priv->num_channels; i++) - atomic_set(&priv->submit_count[i], -(priv->chfifo_len - 1)); + atomic_set(&priv->chan[i].submit_count, + -(priv->chfifo_len - 1)); - priv->head = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL); - priv->tail = kzalloc(sizeof(int) * priv->num_channels, GFP_KERNEL); - if (!priv->head || !priv->tail) { - dev_err(dev, "failed to allocate request index space\n"); - err = -ENOMEM; - goto err_out; - } + dma_set_mask(dev, DMA_BIT_MASK(36)); /* reset and initialize the h/w */ err = init_device(dev); diff --git a/drivers/crypto/talitos.h b/drivers/crypto/talitos.h index 575981f..ff5a145 100644 --- a/drivers/crypto/talitos.h +++ b/drivers/crypto/talitos.h @@ -57,6 +57,7 @@ #define TALITOS_CCCR_RESET 0x1 /* channel reset */ #define TALITOS_CCCR_LO(ch) (ch * TALITOS_CH_STRIDE + 0x110c) #define TALITOS_CCCR_LO_IWSE 0x80 /* chan. ICCR writeback enab. */ +#define TALITOS_CCCR_LO_EAE 0x20 /* extended address enable */ #define TALITOS_CCCR_LO_CDWE 0x10 /* chan. done writeback enab. */ #define TALITOS_CCCR_LO_NT 0x4 /* notification type */ #define TALITOS_CCCR_LO_CDIE 0x2 /* channel done IRQ enable */ diff --git a/include/crypto/algapi.h b/include/crypto/algapi.h index 5a2bd1c..1ffb53f 100644 --- a/include/crypto/algapi.h +++ b/include/crypto/algapi.h @@ -22,11 +22,9 @@ struct seq_file; struct crypto_type { unsigned int (*ctxsize)(struct crypto_alg *alg, u32 type, u32 mask); - unsigned int (*extsize)(struct crypto_alg *alg, - const struct crypto_type *frontend); + unsigned int (*extsize)(struct crypto_alg *alg); int (*init)(struct crypto_tfm *tfm, u32 type, u32 mask); - int (*init_tfm)(struct crypto_tfm *tfm, - const struct crypto_type *frontend); + int (*init_tfm)(struct crypto_tfm *tfm); void (*show)(struct seq_file *m, struct crypto_alg *alg); struct crypto_alg *(*lookup)(const char *name, u32 type, u32 mask); @@ -52,6 +50,7 @@ struct crypto_template { struct crypto_instance *(*alloc)(struct rtattr **tb); void (*free)(struct crypto_instance *inst); + int (*create)(struct crypto_template *tmpl, struct rtattr **tb); char name[CRYPTO_MAX_ALG_NAME]; }; @@ -60,6 +59,7 @@ struct crypto_spawn { struct list_head list; struct crypto_alg *alg; struct crypto_instance *inst; + const struct crypto_type *frontend; u32 mask; }; @@ -114,11 +114,19 @@ int crypto_register_template(struct crypto_template *tmpl); void crypto_unregister_template(struct crypto_template *tmpl); struct crypto_template *crypto_lookup_template(const char *name); +int crypto_register_instance(struct crypto_template *tmpl, + struct crypto_instance *inst); + int crypto_init_spawn(struct crypto_spawn *spawn, struct crypto_alg *alg, struct crypto_instance *inst, u32 mask); +int crypto_init_spawn2(struct crypto_spawn *spawn, struct crypto_alg *alg, + struct crypto_instance *inst, + const struct crypto_type *frontend); + void crypto_drop_spawn(struct crypto_spawn *spawn); struct crypto_tfm *crypto_spawn_tfm(struct crypto_spawn *spawn, u32 type, u32 mask); +void *crypto_spawn_tfm2(struct crypto_spawn *spawn); static inline void crypto_set_spawn(struct crypto_spawn *spawn, struct crypto_instance *inst) @@ -129,8 +137,19 @@ static inline void crypto_set_spawn(struct crypto_spawn *spawn, struct crypto_attr_type *crypto_get_attr_type(struct rtattr **tb); int crypto_check_attr_type(struct rtattr **tb, u32 type); const char *crypto_attr_alg_name(struct rtattr *rta); -struct crypto_alg *crypto_attr_alg(struct rtattr *rta, u32 type, u32 mask); +struct crypto_alg *crypto_attr_alg2(struct rtattr *rta, + const struct crypto_type *frontend, + u32 type, u32 mask); + +static inline struct crypto_alg *crypto_attr_alg(struct rtattr *rta, + u32 type, u32 mask) +{ + return crypto_attr_alg2(rta, NULL, type, mask); +} + int crypto_attr_u32(struct rtattr *rta, u32 *num); +void *crypto_alloc_instance2(const char *name, struct crypto_alg *alg, + unsigned int head); struct crypto_instance *crypto_alloc_instance(const char *name, struct crypto_alg *alg); @@ -157,12 +176,8 @@ int blkcipher_walk_virt_block(struct blkcipher_desc *desc, static inline void *crypto_tfm_ctx_aligned(struct crypto_tfm *tfm) { - unsigned long addr = (unsigned long)crypto_tfm_ctx(tfm); - unsigned long align = crypto_tfm_alg_alignmask(tfm); - - if (align <= crypto_tfm_ctx_alignment()) - align = 1; - return (void *)ALIGN(addr, align); + return PTR_ALIGN(crypto_tfm_ctx(tfm), + crypto_tfm_alg_alignmask(tfm) + 1); } static inline struct crypto_instance *crypto_tfm_alg_instance( diff --git a/include/crypto/cryptd.h b/include/crypto/cryptd.h index 55fa7bb..2f65a6e 100644 --- a/include/crypto/cryptd.h +++ b/include/crypto/cryptd.h @@ -7,6 +7,7 @@ #include <linux/crypto.h> #include <linux/kernel.h> +#include <crypto/hash.h> struct cryptd_ablkcipher { struct crypto_ablkcipher base; @@ -24,4 +25,20 @@ struct cryptd_ablkcipher *cryptd_alloc_ablkcipher(const char *alg_name, struct crypto_blkcipher *cryptd_ablkcipher_child(struct cryptd_ablkcipher *tfm); void cryptd_free_ablkcipher(struct cryptd_ablkcipher *tfm); +struct cryptd_ahash { + struct crypto_ahash base; +}; + +static inline struct cryptd_ahash *__cryptd_ahash_cast( + struct crypto_ahash *tfm) +{ + return (struct cryptd_ahash *)tfm; +} + +/* alg_name should be algorithm to be cryptd-ed */ +struct cryptd_ahash *cryptd_alloc_ahash(const char *alg_name, + u32 type, u32 mask); +struct crypto_shash *cryptd_ahash_child(struct cryptd_ahash *tfm); +void cryptd_free_ahash(struct cryptd_ahash *tfm); + #endif diff --git a/include/crypto/hash.h b/include/crypto/hash.h index d56bb71..26cb1eb 100644 --- a/include/crypto/hash.h +++ b/include/crypto/hash.h @@ -15,6 +15,42 @@ #include <linux/crypto.h> +struct crypto_ahash; + +struct hash_alg_common { + unsigned int digestsize; + unsigned int statesize; + + struct crypto_alg base; +}; + +struct ahash_request { + struct crypto_async_request base; + + unsigned int nbytes; + struct scatterlist *src; + u8 *result; + + /* This field may only be used by the ahash API code. */ + void *priv; + + void *__ctx[] CRYPTO_MINALIGN_ATTR; +}; + +struct ahash_alg { + int (*init)(struct ahash_request *req); + int (*update)(struct ahash_request *req); + int (*final)(struct ahash_request *req); + int (*finup)(struct ahash_request *req); + int (*digest)(struct ahash_request *req); + int (*export)(struct ahash_request *req, void *out); + int (*import)(struct ahash_request *req, const void *in); + int (*setkey)(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen); + + struct hash_alg_common halg; +}; + struct shash_desc { struct crypto_shash *tfm; u32 flags; @@ -24,7 +60,6 @@ struct shash_desc { struct shash_alg { int (*init)(struct shash_desc *desc); - int (*reinit)(struct shash_desc *desc); int (*update)(struct shash_desc *desc, const u8 *data, unsigned int len); int (*final)(struct shash_desc *desc, u8 *out); @@ -32,38 +67,48 @@ struct shash_alg { unsigned int len, u8 *out); int (*digest)(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out); + int (*export)(struct shash_desc *desc, void *out); + int (*import)(struct shash_desc *desc, const void *in); int (*setkey)(struct crypto_shash *tfm, const u8 *key, unsigned int keylen); unsigned int descsize; - unsigned int digestsize; + + /* These fields must match hash_alg_common. */ + unsigned int digestsize + __attribute__ ((aligned(__alignof__(struct hash_alg_common)))); + unsigned int statesize; struct crypto_alg base; }; struct crypto_ahash { + int (*init)(struct ahash_request *req); + int (*update)(struct ahash_request *req); + int (*final)(struct ahash_request *req); + int (*finup)(struct ahash_request *req); + int (*digest)(struct ahash_request *req); + int (*export)(struct ahash_request *req, void *out); + int (*import)(struct ahash_request *req, const void *in); + int (*setkey)(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen); + + unsigned int reqsize; struct crypto_tfm base; }; struct crypto_shash { + unsigned int descsize; struct crypto_tfm base; }; static inline struct crypto_ahash *__crypto_ahash_cast(struct crypto_tfm *tfm) { - return (struct crypto_ahash *)tfm; + return container_of(tfm, struct crypto_ahash, base); } -static inline struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, - u32 type, u32 mask) -{ - type &= ~CRYPTO_ALG_TYPE_MASK; - mask &= ~CRYPTO_ALG_TYPE_MASK; - type |= CRYPTO_ALG_TYPE_AHASH; - mask |= CRYPTO_ALG_TYPE_AHASH_MASK; - - return __crypto_ahash_cast(crypto_alloc_base(alg_name, type, mask)); -} +struct crypto_ahash *crypto_alloc_ahash(const char *alg_name, u32 type, + u32 mask); static inline struct crypto_tfm *crypto_ahash_tfm(struct crypto_ahash *tfm) { @@ -72,7 +117,7 @@ static inline struct crypto_tfm *crypto_ahash_tfm(struct crypto_ahash *tfm) static inline void crypto_free_ahash(struct crypto_ahash *tfm) { - crypto_free_tfm(crypto_ahash_tfm(tfm)); + crypto_destroy_tfm(tfm, crypto_ahash_tfm(tfm)); } static inline unsigned int crypto_ahash_alignmask( @@ -81,14 +126,26 @@ static inline unsigned int crypto_ahash_alignmask( return crypto_tfm_alg_alignmask(crypto_ahash_tfm(tfm)); } -static inline struct ahash_tfm *crypto_ahash_crt(struct crypto_ahash *tfm) +static inline struct hash_alg_common *__crypto_hash_alg_common( + struct crypto_alg *alg) +{ + return container_of(alg, struct hash_alg_common, base); +} + +static inline struct hash_alg_common *crypto_hash_alg_common( + struct crypto_ahash *tfm) { - return &crypto_ahash_tfm(tfm)->crt_ahash; + return __crypto_hash_alg_common(crypto_ahash_tfm(tfm)->__crt_alg); } static inline unsigned int crypto_ahash_digestsize(struct crypto_ahash *tfm) { - return crypto_ahash_crt(tfm)->digestsize; + return crypto_hash_alg_common(tfm)->digestsize; +} + +static inline unsigned int crypto_ahash_statesize(struct crypto_ahash *tfm) +{ + return crypto_hash_alg_common(tfm)->statesize; } static inline u32 crypto_ahash_get_flags(struct crypto_ahash *tfm) @@ -114,7 +171,7 @@ static inline struct crypto_ahash *crypto_ahash_reqtfm( static inline unsigned int crypto_ahash_reqsize(struct crypto_ahash *tfm) { - return crypto_ahash_crt(tfm)->reqsize; + return tfm->reqsize; } static inline void *ahash_request_ctx(struct ahash_request *req) @@ -122,44 +179,30 @@ static inline void *ahash_request_ctx(struct ahash_request *req) return req->__ctx; } -static inline int crypto_ahash_setkey(struct crypto_ahash *tfm, - const u8 *key, unsigned int keylen) -{ - struct ahash_tfm *crt = crypto_ahash_crt(tfm); - - return crt->setkey(tfm, key, keylen); -} +int crypto_ahash_setkey(struct crypto_ahash *tfm, const u8 *key, + unsigned int keylen); +int crypto_ahash_finup(struct ahash_request *req); +int crypto_ahash_final(struct ahash_request *req); +int crypto_ahash_digest(struct ahash_request *req); -static inline int crypto_ahash_digest(struct ahash_request *req) +static inline int crypto_ahash_export(struct ahash_request *req, void *out) { - struct ahash_tfm *crt = crypto_ahash_crt(crypto_ahash_reqtfm(req)); - return crt->digest(req); + return crypto_ahash_reqtfm(req)->export(req, out); } -static inline void crypto_ahash_export(struct ahash_request *req, u8 *out) +static inline int crypto_ahash_import(struct ahash_request *req, const void *in) { - memcpy(out, ahash_request_ctx(req), - crypto_ahash_reqsize(crypto_ahash_reqtfm(req))); + return crypto_ahash_reqtfm(req)->import(req, in); } -int crypto_ahash_import(struct ahash_request *req, const u8 *in); - static inline int crypto_ahash_init(struct ahash_request *req) { - struct ahash_tfm *crt = crypto_ahash_crt(crypto_ahash_reqtfm(req)); - return crt->init(req); + return crypto_ahash_reqtfm(req)->init(req); } static inline int crypto_ahash_update(struct ahash_request *req) { - struct ahash_tfm *crt = crypto_ahash_crt(crypto_ahash_reqtfm(req)); - return crt->update(req); -} - -static inline int crypto_ahash_final(struct ahash_request *req) -{ - struct ahash_tfm *crt = crypto_ahash_crt(crypto_ahash_reqtfm(req)); - return crt->final(req); + return crypto_ahash_reqtfm(req)->update(req); } static inline void ahash_request_set_tfm(struct ahash_request *req, @@ -184,7 +227,7 @@ static inline struct ahash_request *ahash_request_alloc( static inline void ahash_request_free(struct ahash_request *req) { - kfree(req); + kzfree(req); } static inline struct ahash_request *ahash_request_cast( @@ -251,6 +294,11 @@ static inline unsigned int crypto_shash_digestsize(struct crypto_shash *tfm) return crypto_shash_alg(tfm)->digestsize; } +static inline unsigned int crypto_shash_statesize(struct crypto_shash *tfm) +{ + return crypto_shash_alg(tfm)->statesize; +} + static inline u32 crypto_shash_get_flags(struct crypto_shash *tfm) { return crypto_tfm_get_flags(crypto_shash_tfm(tfm)); @@ -268,7 +316,7 @@ static inline void crypto_shash_clear_flags(struct crypto_shash *tfm, u32 flags) static inline unsigned int crypto_shash_descsize(struct crypto_shash *tfm) { - return crypto_shash_alg(tfm)->descsize; + return tfm->descsize; } static inline void *shash_desc_ctx(struct shash_desc *desc) @@ -281,12 +329,15 @@ int crypto_shash_setkey(struct crypto_shash *tfm, const u8 *key, int crypto_shash_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out); -static inline void crypto_shash_export(struct shash_desc *desc, u8 *out) +static inline int crypto_shash_export(struct shash_desc *desc, void *out) { - memcpy(out, shash_desc_ctx(desc), crypto_shash_descsize(desc->tfm)); + return crypto_shash_alg(desc->tfm)->export(desc, out); } -int crypto_shash_import(struct shash_desc *desc, const u8 *in); +static inline int crypto_shash_import(struct shash_desc *desc, const void *in) +{ + return crypto_shash_alg(desc->tfm)->import(desc, in); +} static inline int crypto_shash_init(struct shash_desc *desc) { diff --git a/include/crypto/internal/hash.h b/include/crypto/internal/hash.h index 82b7056..5bfad8c 100644 --- a/include/crypto/internal/hash.h +++ b/include/crypto/internal/hash.h @@ -34,6 +34,22 @@ struct crypto_hash_walk { unsigned int flags; }; +struct ahash_instance { + struct ahash_alg alg; +}; + +struct shash_instance { + struct shash_alg alg; +}; + +struct crypto_ahash_spawn { + struct crypto_spawn base; +}; + +struct crypto_shash_spawn { + struct crypto_spawn base; +}; + extern const struct crypto_type crypto_ahash_type; int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err); @@ -43,18 +59,100 @@ int crypto_hash_walk_first_compat(struct hash_desc *hdesc, struct crypto_hash_walk *walk, struct scatterlist *sg, unsigned int len); +static inline int crypto_hash_walk_last(struct crypto_hash_walk *walk) +{ + return !(walk->entrylen | walk->total); +} + +int crypto_register_ahash(struct ahash_alg *alg); +int crypto_unregister_ahash(struct ahash_alg *alg); +int ahash_register_instance(struct crypto_template *tmpl, + struct ahash_instance *inst); +void ahash_free_instance(struct crypto_instance *inst); + +int crypto_init_ahash_spawn(struct crypto_ahash_spawn *spawn, + struct hash_alg_common *alg, + struct crypto_instance *inst); + +static inline void crypto_drop_ahash(struct crypto_ahash_spawn *spawn) +{ + crypto_drop_spawn(&spawn->base); +} + +struct hash_alg_common *ahash_attr_alg(struct rtattr *rta, u32 type, u32 mask); + int crypto_register_shash(struct shash_alg *alg); int crypto_unregister_shash(struct shash_alg *alg); +int shash_register_instance(struct crypto_template *tmpl, + struct shash_instance *inst); +void shash_free_instance(struct crypto_instance *inst); + +int crypto_init_shash_spawn(struct crypto_shash_spawn *spawn, + struct shash_alg *alg, + struct crypto_instance *inst); + +static inline void crypto_drop_shash(struct crypto_shash_spawn *spawn) +{ + crypto_drop_spawn(&spawn->base); +} + +struct shash_alg *shash_attr_alg(struct rtattr *rta, u32 type, u32 mask); + +int shash_ahash_update(struct ahash_request *req, struct shash_desc *desc); +int shash_ahash_finup(struct ahash_request *req, struct shash_desc *desc); +int shash_ahash_digest(struct ahash_request *req, struct shash_desc *desc); + +int crypto_init_shash_ops_async(struct crypto_tfm *tfm); static inline void *crypto_ahash_ctx(struct crypto_ahash *tfm) { - return crypto_tfm_ctx(&tfm->base); + return crypto_tfm_ctx(crypto_ahash_tfm(tfm)); +} + +static inline struct ahash_alg *__crypto_ahash_alg(struct crypto_alg *alg) +{ + return container_of(__crypto_hash_alg_common(alg), struct ahash_alg, + halg); +} + +static inline void crypto_ahash_set_reqsize(struct crypto_ahash *tfm, + unsigned int reqsize) +{ + tfm->reqsize = reqsize; +} + +static inline struct crypto_instance *ahash_crypto_instance( + struct ahash_instance *inst) +{ + return container_of(&inst->alg.halg.base, struct crypto_instance, alg); } -static inline struct ahash_alg *crypto_ahash_alg( - struct crypto_ahash *tfm) +static inline struct ahash_instance *ahash_instance( + struct crypto_instance *inst) { - return &crypto_ahash_tfm(tfm)->__crt_alg->cra_ahash; + return container_of(&inst->alg, struct ahash_instance, alg.halg.base); +} + +static inline void *ahash_instance_ctx(struct ahash_instance *inst) +{ + return crypto_instance_ctx(ahash_crypto_instance(inst)); +} + +static inline unsigned int ahash_instance_headroom(void) +{ + return sizeof(struct ahash_alg) - sizeof(struct crypto_alg); +} + +static inline struct ahash_instance *ahash_alloc_instance( + const char *name, struct crypto_alg *alg) +{ + return crypto_alloc_instance2(name, alg, ahash_instance_headroom()); +} + +static inline struct crypto_ahash *crypto_spawn_ahash( + struct crypto_ahash_spawn *spawn) +{ + return crypto_spawn_tfm2(&spawn->base); } static inline int ahash_enqueue_request(struct crypto_queue *queue, @@ -80,5 +178,46 @@ static inline void *crypto_shash_ctx(struct crypto_shash *tfm) return crypto_tfm_ctx(&tfm->base); } +static inline struct crypto_instance *shash_crypto_instance( + struct shash_instance *inst) +{ + return container_of(&inst->alg.base, struct crypto_instance, alg); +} + +static inline struct shash_instance *shash_instance( + struct crypto_instance *inst) +{ + return container_of(__crypto_shash_alg(&inst->alg), + struct shash_instance, alg); +} + +static inline void *shash_instance_ctx(struct shash_instance *inst) +{ + return crypto_instance_ctx(shash_crypto_instance(inst)); +} + +static inline struct shash_instance *shash_alloc_instance( + const char *name, struct crypto_alg *alg) +{ + return crypto_alloc_instance2(name, alg, + sizeof(struct shash_alg) - sizeof(*alg)); +} + +static inline struct crypto_shash *crypto_spawn_shash( + struct crypto_shash_spawn *spawn) +{ + return crypto_spawn_tfm2(&spawn->base); +} + +static inline void *crypto_shash_ctx_aligned(struct crypto_shash *tfm) +{ + return crypto_tfm_ctx_aligned(&tfm->base); +} + +static inline struct crypto_shash *__crypto_shash_cast(struct crypto_tfm *tfm) +{ + return container_of(tfm, struct crypto_shash, base); +} + #endif /* _CRYPTO_INTERNAL_HASH_H */ diff --git a/include/crypto/sha.h b/include/crypto/sha.h index c0ccc2b..069e85b 100644 --- a/include/crypto/sha.h +++ b/include/crypto/sha.h @@ -5,6 +5,8 @@ #ifndef _CRYPTO_SHA_H #define _CRYPTO_SHA_H +#include <linux/types.h> + #define SHA1_DIGEST_SIZE 20 #define SHA1_BLOCK_SIZE 64 @@ -62,4 +64,22 @@ #define SHA512_H6 0x1f83d9abfb41bd6bULL #define SHA512_H7 0x5be0cd19137e2179ULL +struct sha1_state { + u64 count; + u32 state[SHA1_DIGEST_SIZE / 4]; + u8 buffer[SHA1_BLOCK_SIZE]; +}; + +struct sha256_state { + u64 count; + u32 state[SHA256_DIGEST_SIZE / 4]; + u8 buf[SHA256_BLOCK_SIZE]; +}; + +struct sha512_state { + u64 count[2]; + u64 state[SHA512_DIGEST_SIZE / 8]; + u8 buf[SHA512_BLOCK_SIZE]; +}; + #endif diff --git a/include/crypto/vmac.h b/include/crypto/vmac.h new file mode 100644 index 0000000..c4467c5 --- /dev/null +++ b/include/crypto/vmac.h @@ -0,0 +1,61 @@ +/* + * Modified to interface to the Linux kernel + * Copyright (c) 2009, Intel Corporation. + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program; if not, write to the Free Software Foundation, Inc., 59 Temple + * Place - Suite 330, Boston, MA 02111-1307 USA. + */ + +#ifndef __CRYPTO_VMAC_H +#define __CRYPTO_VMAC_H + +/* -------------------------------------------------------------------------- + * VMAC and VHASH Implementation by Ted Krovetz (tdk@acm.org) and Wei Dai. + * This implementation is herby placed in the public domain. + * The authors offers no warranty. Use at your own risk. + * Please send bug reports to the authors. + * Last modified: 17 APR 08, 1700 PDT + * ----------------------------------------------------------------------- */ + +/* + * User definable settings. + */ +#define VMAC_TAG_LEN 64 +#define VMAC_KEY_SIZE 128/* Must be 128, 192 or 256 */ +#define VMAC_KEY_LEN (VMAC_KEY_SIZE/8) +#define VMAC_NHBYTES 128/* Must 2^i for any 3 < i < 13 Standard = 128*/ + +/* + * This implementation uses u32 and u64 as names for unsigned 32- + * and 64-bit integer types. These are defined in C99 stdint.h. The + * following may need adaptation if you are not running a C99 or + * Microsoft C environment. + */ +struct vmac_ctx { + u64 nhkey[(VMAC_NHBYTES/8)+2*(VMAC_TAG_LEN/64-1)]; + u64 polykey[2*VMAC_TAG_LEN/64]; + u64 l3key[2*VMAC_TAG_LEN/64]; + u64 polytmp[2*VMAC_TAG_LEN/64]; + u64 cached_nonce[2]; + u64 cached_aes[2]; + int first_block_processed; +}; + +typedef u64 vmac_t; + +struct vmac_ctx_t { + struct crypto_cipher *child; + struct vmac_ctx __vmac_ctx; +}; + +#endif /* __CRYPTO_VMAC_H */ diff --git a/include/linux/crypto.h b/include/linux/crypto.h index ec29fa2..fd92988 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -115,7 +115,6 @@ struct crypto_async_request; struct crypto_aead; struct crypto_blkcipher; struct crypto_hash; -struct crypto_ahash; struct crypto_rng; struct crypto_tfm; struct crypto_type; @@ -146,16 +145,6 @@ struct ablkcipher_request { void *__ctx[] CRYPTO_MINALIGN_ATTR; }; -struct ahash_request { - struct crypto_async_request base; - - unsigned int nbytes; - struct scatterlist *src; - u8 *result; - - void *__ctx[] CRYPTO_MINALIGN_ATTR; -}; - /** * struct aead_request - AEAD request * @base: Common attributes for async crypto requests @@ -220,18 +209,6 @@ struct ablkcipher_alg { unsigned int ivsize; }; -struct ahash_alg { - int (*init)(struct ahash_request *req); - int (*reinit)(struct ahash_request *req); - int (*update)(struct ahash_request *req); - int (*final)(struct ahash_request *req); - int (*digest)(struct ahash_request *req); - int (*setkey)(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen); - - unsigned int digestsize; -}; - struct aead_alg { int (*setkey)(struct crypto_aead *tfm, const u8 *key, unsigned int keylen); @@ -318,7 +295,6 @@ struct rng_alg { #define cra_cipher cra_u.cipher #define cra_digest cra_u.digest #define cra_hash cra_u.hash -#define cra_ahash cra_u.ahash #define cra_compress cra_u.compress #define cra_rng cra_u.rng @@ -346,7 +322,6 @@ struct crypto_alg { struct cipher_alg cipher; struct digest_alg digest; struct hash_alg hash; - struct ahash_alg ahash; struct compress_alg compress; struct rng_alg rng; } cra_u; @@ -433,18 +408,6 @@ struct hash_tfm { unsigned int digestsize; }; -struct ahash_tfm { - int (*init)(struct ahash_request *req); - int (*update)(struct ahash_request *req); - int (*final)(struct ahash_request *req); - int (*digest)(struct ahash_request *req); - int (*setkey)(struct crypto_ahash *tfm, const u8 *key, - unsigned int keylen); - - unsigned int digestsize; - unsigned int reqsize; -}; - struct compress_tfm { int (*cot_compress)(struct crypto_tfm *tfm, const u8 *src, unsigned int slen, @@ -465,7 +428,6 @@ struct rng_tfm { #define crt_blkcipher crt_u.blkcipher #define crt_cipher crt_u.cipher #define crt_hash crt_u.hash -#define crt_ahash crt_u.ahash #define crt_compress crt_u.compress #define crt_rng crt_u.rng @@ -479,7 +441,6 @@ struct crypto_tfm { struct blkcipher_tfm blkcipher; struct cipher_tfm cipher; struct hash_tfm hash; - struct ahash_tfm ahash; struct compress_tfm compress; struct rng_tfm rng; } crt_u; @@ -770,7 +731,7 @@ static inline struct ablkcipher_request *ablkcipher_request_alloc( static inline void ablkcipher_request_free(struct ablkcipher_request *req) { - kfree(req); + kzfree(req); } static inline void ablkcipher_request_set_callback( @@ -901,7 +862,7 @@ static inline struct aead_request *aead_request_alloc(struct crypto_aead *tfm, static inline void aead_request_free(struct aead_request *req) { - kfree(req); + kzfree(req); } static inline void aead_request_set_callback(struct aead_request *req, diff --git a/include/linux/fips.h b/include/linux/fips.h new file mode 100644 index 0000000..f8fb07b --- /dev/null +++ b/include/linux/fips.h @@ -0,0 +1,10 @@ +#ifndef _FIPS_H +#define _FIPS_H + +#ifdef CONFIG_CRYPTO_FIPS +extern int fips_enabled; +#else +#define fips_enabled 0 +#endif + +#endif |
