From 9f5594c91e7cf5952ce39aff8b88c95a19a27e06 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 9 Oct 2015 20:43:38 +0100 Subject: crypto: marvell/cesa - fix stack smashing in marvell/hash.c Several of the algorithms in marvell/hash.c have a statesize of zero. When an AF_ALG accept() on an already-accepted file descriptor to calls into hash_accept(), this causes: char state[crypto_ahash_statesize(crypto_ahash_reqtfm(req))]; to be zero-sized, but we still pass this to: err = crypto_ahash_export(req, state); which proceeds to write to 'state' as if it was a "struct md5_state", "struct sha1_state" etc. Add the necessary initialisers for the .statesize member. Acked-by: Boris Brezillon Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index ae9272e..aa18324 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -872,6 +872,7 @@ struct ahash_alg mv_md5_alg = { .import = mv_cesa_md5_import, .halg = { .digestsize = MD5_DIGEST_SIZE, + .statesize = sizeof(struct md5_state), .base = { .cra_name = "md5", .cra_driver_name = "mv-md5", @@ -961,6 +962,7 @@ struct ahash_alg mv_sha1_alg = { .import = mv_cesa_sha1_import, .halg = { .digestsize = SHA1_DIGEST_SIZE, + .statesize = sizeof(struct sha1_state), .base = { .cra_name = "sha1", .cra_driver_name = "mv-sha1", @@ -1050,6 +1052,7 @@ struct ahash_alg mv_sha256_alg = { .import = mv_cesa_sha256_import, .halg = { .digestsize = SHA256_DIGEST_SIZE, + .statesize = sizeof(struct sha256_state), .base = { .cra_name = "sha256", .cra_driver_name = "mv-sha256", -- cgit v1.1 From e72f407ee793f1455051cfa834e758afca09e4f6 Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 9 Oct 2015 20:43:43 +0100 Subject: crypto: marvell/cesa - initialise struct mv_cesa_ahash_req When a AF_ALG fd is accepted a second time (hence hash_accept() is used), hash_accept_parent() allocates a new private context using sock_kmalloc(). This context is uninitialised. After use of the new fd, we eventually end up with the kernel complaining: marvell-cesa f1090000.crypto: dma_pool_free cesa_padding, c0627770/0 (bad dma) where c0627770 is a random address. Poisoning the memory allocated by the above sock_kmalloc() produces kernel oopses within the marvell hash code, particularly the interrupt handling. The following simplfied call sequence occurs: hash_accept() crypto_ahash_export() marvell hash export function af_alg_accept() hash_accept_parent() <== allocates uninitialised struct hash_ctx crypto_ahash_import() marvell hash import function hash_ctx contains the struct mv_cesa_ahash_req in its req.__ctx member, and, as the marvell hash import function only partially initialises this structure, we end up with a lot of members which are left with whatever data was in memory prior to sock_kmalloc(). Add zero-initialisation of this structure. Signed-off-by: Russell King Acked-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index aa18324..13bbf7c 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -833,6 +833,10 @@ static int mv_cesa_md5_import(struct ahash_request *req, const void *in) unsigned int cache_ptr; int ret; + ret = crypto_ahash_init(req); + if (ret) + return ret; + creq->len = in_state->byte_count; memcpy(creq->state, in_state->hash, digsize); creq->cache_ptr = 0; @@ -923,6 +927,10 @@ static int mv_cesa_sha1_import(struct ahash_request *req, const void *in) unsigned int cache_ptr; int ret; + ret = crypto_ahash_init(req); + if (ret) + return ret; + creq->len = in_state->count; memcpy(creq->state, in_state->state, digsize); creq->cache_ptr = 0; @@ -1024,6 +1032,10 @@ static int mv_cesa_sha256_import(struct ahash_request *req, const void *in) unsigned int cache_ptr; int ret; + ret = crypto_ahash_init(req); + if (ret) + return ret; + creq->len = in_state->count; memcpy(creq->state, in_state->state, digsize); creq->cache_ptr = 0; -- cgit v1.1 From c3bf02a22c5369b25090d12c202ea0807d01978a Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 9 Oct 2015 20:43:48 +0100 Subject: crypto: marvell/cesa - fix wrong hash results Attempting to use the sha1 digest for openssh via openssl reveals that the result from the hash is wrong: this happens when we export the state from one socket and import it into another via calling accept(). The reason for this is because the operation is reset to "initial block" state, whereas we may be past the first fragment of data to be hashed. Arrange for the operation code to avoid the initialisation of the state, thereby preserving the imported state. Signed-off-by: Russell King Acked-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 13bbf7c..4bc2896 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -837,6 +837,11 @@ static int mv_cesa_md5_import(struct ahash_request *req, const void *in) if (ret) return ret; + if (in_state->byte_count >= sizeof(in_state->block)) + mv_cesa_update_op_cfg(&creq->op_tmpl, + CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + creq->len = in_state->byte_count; memcpy(creq->state, in_state->hash, digsize); creq->cache_ptr = 0; @@ -931,6 +936,11 @@ static int mv_cesa_sha1_import(struct ahash_request *req, const void *in) if (ret) return ret; + if (in_state->count >= SHA1_BLOCK_SIZE) + mv_cesa_update_op_cfg(&creq->op_tmpl, + CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + creq->len = in_state->count; memcpy(creq->state, in_state->state, digsize); creq->cache_ptr = 0; @@ -1036,6 +1046,11 @@ static int mv_cesa_sha256_import(struct ahash_request *req, const void *in) if (ret) return ret; + if (in_state->count >= SHA256_BLOCK_SIZE) + mv_cesa_update_op_cfg(&creq->op_tmpl, + CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + creq->len = in_state->count; memcpy(creq->state, in_state->state, digsize); creq->cache_ptr = 0; -- cgit v1.1 From a6479ea487ca8ce11289e88a55a7e348f1ee39ac Mon Sep 17 00:00:00 2001 From: Russell King Date: Fri, 9 Oct 2015 21:14:22 +0100 Subject: crypto: marvell/cesa - factor out common import/export functions As all the import functions and export functions are virtually identical, factor out their common parts into a generic mv_cesa_ahash_import() and mv_cesa_ahash_export() respectively. This performs the actual import or export, and we pass the data pointers and length into these functions. We have to switch a % const operation to do_div() in the common import function to avoid provoking gcc to use the expensive 64-bit by 64-bit modulus operation. Signed-off-by: Russell King Acked-by: Boris Brezillon Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 157 ++++++++++++++---------------------------- 1 file changed, 53 insertions(+), 104 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 4bc2896..5e0e102 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -797,39 +797,32 @@ static int mv_cesa_ahash_finup(struct ahash_request *req) return ret; } -static int mv_cesa_md5_init(struct ahash_request *req) +static int mv_cesa_ahash_export(struct ahash_request *req, void *hash, + u64 *len, void *cache) { - struct mv_cesa_op_ctx tmpl; - - mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_MD5); - - mv_cesa_ahash_init(req, &tmpl); - - return 0; -} - -static int mv_cesa_md5_export(struct ahash_request *req, void *out) -{ - struct md5_state *out_state = out; struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); unsigned int digsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize; + + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); - out_state->byte_count = creq->len; - memcpy(out_state->hash, creq->state, digsize); - memset(out_state->block, 0, sizeof(out_state->block)); + *len = creq->len; + memcpy(hash, creq->state, digsize); + memset(cache, 0, blocksize); if (creq->cache) - memcpy(out_state->block, creq->cache, creq->cache_ptr); + memcpy(cache, creq->cache, creq->cache_ptr); return 0; } -static int mv_cesa_md5_import(struct ahash_request *req, const void *in) +static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash, + u64 len, const void *cache) { - const struct md5_state *in_state = in; struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); unsigned int digsize = crypto_ahash_digestsize(ahash); + unsigned int blocksize; unsigned int cache_ptr; int ret; @@ -837,16 +830,17 @@ static int mv_cesa_md5_import(struct ahash_request *req, const void *in) if (ret) return ret; - if (in_state->byte_count >= sizeof(in_state->block)) + blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); + if (len >= blocksize) mv_cesa_update_op_cfg(&creq->op_tmpl, CESA_SA_DESC_CFG_MID_FRAG, CESA_SA_DESC_CFG_FRAG_MSK); - creq->len = in_state->byte_count; - memcpy(creq->state, in_state->hash, digsize); + creq->len = len; + memcpy(creq->state, hash, digsize); creq->cache_ptr = 0; - cache_ptr = creq->len % sizeof(in_state->block); + cache_ptr = do_div(len, blocksize); if (!cache_ptr) return 0; @@ -854,12 +848,39 @@ static int mv_cesa_md5_import(struct ahash_request *req, const void *in) if (ret) return ret; - memcpy(creq->cache, in_state->block, cache_ptr); + memcpy(creq->cache, cache, cache_ptr); creq->cache_ptr = cache_ptr; return 0; } +static int mv_cesa_md5_init(struct ahash_request *req) +{ + struct mv_cesa_op_ctx tmpl; + + mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_MD5); + + mv_cesa_ahash_init(req, &tmpl); + + return 0; +} + +static int mv_cesa_md5_export(struct ahash_request *req, void *out) +{ + struct md5_state *out_state = out; + + return mv_cesa_ahash_export(req, out_state->hash, + &out_state->byte_count, out_state->block); +} + +static int mv_cesa_md5_import(struct ahash_request *req, const void *in) +{ + const struct md5_state *in_state = in; + + return mv_cesa_ahash_import(req, in_state->hash, in_state->byte_count, + in_state->block); +} + static int mv_cesa_md5_digest(struct ahash_request *req) { int ret; @@ -910,53 +931,17 @@ static int mv_cesa_sha1_init(struct ahash_request *req) static int mv_cesa_sha1_export(struct ahash_request *req, void *out) { struct sha1_state *out_state = out; - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - unsigned int digsize = crypto_ahash_digestsize(ahash); - - out_state->count = creq->len; - memcpy(out_state->state, creq->state, digsize); - memset(out_state->buffer, 0, sizeof(out_state->buffer)); - if (creq->cache) - memcpy(out_state->buffer, creq->cache, creq->cache_ptr); - return 0; + return mv_cesa_ahash_export(req, out_state->state, &out_state->count, + out_state->buffer); } static int mv_cesa_sha1_import(struct ahash_request *req, const void *in) { const struct sha1_state *in_state = in; - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - unsigned int digsize = crypto_ahash_digestsize(ahash); - unsigned int cache_ptr; - int ret; - - ret = crypto_ahash_init(req); - if (ret) - return ret; - - if (in_state->count >= SHA1_BLOCK_SIZE) - mv_cesa_update_op_cfg(&creq->op_tmpl, - CESA_SA_DESC_CFG_MID_FRAG, - CESA_SA_DESC_CFG_FRAG_MSK); - - creq->len = in_state->count; - memcpy(creq->state, in_state->state, digsize); - creq->cache_ptr = 0; - - cache_ptr = creq->len % SHA1_BLOCK_SIZE; - if (!cache_ptr) - return 0; - - ret = mv_cesa_ahash_alloc_cache(req); - if (ret) - return ret; - - memcpy(creq->cache, in_state->buffer, cache_ptr); - creq->cache_ptr = cache_ptr; - return 0; + return mv_cesa_ahash_import(req, in_state->state, in_state->count, + in_state->buffer); } static int mv_cesa_sha1_digest(struct ahash_request *req) @@ -1020,53 +1005,17 @@ static int mv_cesa_sha256_digest(struct ahash_request *req) static int mv_cesa_sha256_export(struct ahash_request *req, void *out) { struct sha256_state *out_state = out; - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - unsigned int ds = crypto_ahash_digestsize(ahash); - out_state->count = creq->len; - memcpy(out_state->state, creq->state, ds); - memset(out_state->buf, 0, sizeof(out_state->buf)); - if (creq->cache) - memcpy(out_state->buf, creq->cache, creq->cache_ptr); - - return 0; + return mv_cesa_ahash_export(req, out_state->state, &out_state->count, + out_state->buf); } static int mv_cesa_sha256_import(struct ahash_request *req, const void *in) { const struct sha256_state *in_state = in; - struct crypto_ahash *ahash = crypto_ahash_reqtfm(req); - struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - unsigned int digsize = crypto_ahash_digestsize(ahash); - unsigned int cache_ptr; - int ret; - - ret = crypto_ahash_init(req); - if (ret) - return ret; - - if (in_state->count >= SHA256_BLOCK_SIZE) - mv_cesa_update_op_cfg(&creq->op_tmpl, - CESA_SA_DESC_CFG_MID_FRAG, - CESA_SA_DESC_CFG_FRAG_MSK); - creq->len = in_state->count; - memcpy(creq->state, in_state->state, digsize); - creq->cache_ptr = 0; - - cache_ptr = creq->len % SHA256_BLOCK_SIZE; - if (!cache_ptr) - return 0; - - ret = mv_cesa_ahash_alloc_cache(req); - if (ret) - return ret; - - memcpy(creq->cache, in_state->buf, cache_ptr); - creq->cache_ptr = cache_ptr; - - return 0; + return mv_cesa_ahash_import(req, in_state->state, in_state->count, + in_state->buf); } struct ahash_alg mv_sha256_alg = { -- cgit v1.1 From 80754539ec936b0afe8a3c406b7d67612977cc71 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:23:30 +0100 Subject: crypto: marvell/cesa - easier way to get the transform There's an easier way to get at the hash transform - rather than using crypto_ahash_tfm(ahash), we can get it directly from req->base.tfm. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 5e0e102..40241fd 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -805,7 +805,7 @@ static int mv_cesa_ahash_export(struct ahash_request *req, void *hash, unsigned int digsize = crypto_ahash_digestsize(ahash); unsigned int blocksize; - blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); + blocksize = crypto_ahash_blocksize(ahash); *len = creq->len; memcpy(hash, creq->state, digsize); @@ -830,7 +830,7 @@ static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash, if (ret) return ret; - blocksize = crypto_tfm_alg_blocksize(crypto_ahash_tfm(ahash)); + blocksize = crypto_ahash_blocksize(ahash); if (len >= blocksize) mv_cesa_update_op_cfg(&creq->op_tmpl, CESA_SA_DESC_CFG_MID_FRAG, -- cgit v1.1 From 4c2b130c8ae79af734e5e57f3662e25ef4d20e8d Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:23:35 +0100 Subject: crypto: marvell/cesa - keep creq->state in CPU endian format at all times Currently, we read/write the state in CPU endian, but on the final request, we convert its endian according to the requested algorithm. (md5 is little endian, SHA are big endian.) Always keep creq->state in CPU native endian format, and perform the necessary conversion when copying the hash to the result. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.h | 2 +- drivers/crypto/marvell/hash.c | 25 ++++++++++++++----------- 2 files changed, 15 insertions(+), 12 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index b60698b..5a1fabd 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -612,7 +612,7 @@ struct mv_cesa_ahash_req { u64 len; int src_nents; bool last_req; - __be32 state[8]; + u32 state[8]; }; /* CESA functions */ diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 40241fd..7894126 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -347,18 +347,21 @@ static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) ahashreq->nbytes - creq->cache_ptr); if (creq->last_req) { - for (i = 0; i < digsize / 4; i++) { - /* - * Hardware provides MD5 digest in a different - * endianness than SHA-1 and SHA-256 ones. - */ - if (digsize == MD5_DIGEST_SIZE) - creq->state[i] = cpu_to_le32(creq->state[i]); - else - creq->state[i] = cpu_to_be32(creq->state[i]); - } + /* + * Hardware's MD5 digest is in little endian format, but + * SHA in big endian format + */ + if (digsize == MD5_DIGEST_SIZE) { + __le32 *result = (void *)ahashreq->result; + + for (i = 0; i < digsize / 4; i++) + result[i] = cpu_to_le32(creq->state[i]); + } else { + __be32 *result = (void *)ahashreq->result; - memcpy(ahashreq->result, creq->state, digsize); + for (i = 0; i < digsize / 4; i++) + result[i] = cpu_to_be32(creq->state[i]); + } } return ret; -- cgit v1.1 From a9eb678f8addc1bfb80bacc45f6df1108fd1c0d9 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:23:40 +0100 Subject: crypto: marvell/cesa - add flag to determine algorithm endianness Rather than determining whether we're using a MD5 hash by looking at the digest size, switch to a cleaner solution using a per-request flag initialised by the method type. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.h | 1 + drivers/crypto/marvell/hash.c | 17 +++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index 5a1fabd..e5f70ab 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -612,6 +612,7 @@ struct mv_cesa_ahash_req { u64 len; int src_nents; bool last_req; + bool algo_le; u32 state[8]; }; diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 7894126..f86594fb 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -351,7 +351,7 @@ static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) * Hardware's MD5 digest is in little endian format, but * SHA in big endian format */ - if (digsize == MD5_DIGEST_SIZE) { + if (creq->algo_le) { __le32 *result = (void *)ahashreq->result; for (i = 0; i < digsize / 4; i++) @@ -407,7 +407,7 @@ static const struct mv_cesa_req_ops mv_cesa_ahash_req_ops = { }; static int mv_cesa_ahash_init(struct ahash_request *req, - struct mv_cesa_op_ctx *tmpl) + struct mv_cesa_op_ctx *tmpl, bool algo_le) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); @@ -421,6 +421,7 @@ static int mv_cesa_ahash_init(struct ahash_request *req, mv_cesa_set_mac_op_frag_len(tmpl, 0); creq->op_tmpl = *tmpl; creq->len = 0; + creq->algo_le = algo_le; return 0; } @@ -863,7 +864,7 @@ static int mv_cesa_md5_init(struct ahash_request *req) mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_MD5); - mv_cesa_ahash_init(req, &tmpl); + mv_cesa_ahash_init(req, &tmpl, true); return 0; } @@ -926,7 +927,7 @@ static int mv_cesa_sha1_init(struct ahash_request *req) mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA1); - mv_cesa_ahash_init(req, &tmpl); + mv_cesa_ahash_init(req, &tmpl, false); return 0; } @@ -989,7 +990,7 @@ static int mv_cesa_sha256_init(struct ahash_request *req) mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA256); - mv_cesa_ahash_init(req, &tmpl); + mv_cesa_ahash_init(req, &tmpl, false); return 0; } @@ -1220,7 +1221,7 @@ static int mv_cesa_ahmac_md5_init(struct ahash_request *req) mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_MD5); memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); - mv_cesa_ahash_init(req, &tmpl); + mv_cesa_ahash_init(req, &tmpl, true); return 0; } @@ -1290,7 +1291,7 @@ static int mv_cesa_ahmac_sha1_init(struct ahash_request *req) mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA1); memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); - mv_cesa_ahash_init(req, &tmpl); + mv_cesa_ahash_init(req, &tmpl, false); return 0; } @@ -1380,7 +1381,7 @@ static int mv_cesa_ahmac_sha256_init(struct ahash_request *req) mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA256); memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); - mv_cesa_ahash_init(req, &tmpl); + mv_cesa_ahash_init(req, &tmpl, false); return 0; } -- cgit v1.1 From 51954a968bbedcabf2b42fec2ec51386f0dee0c4 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:23:46 +0100 Subject: crypto: marvell/cesa - fix the bit length endianness The endianness of the bit length used in the final stage depends on the endianness of the algorithm - md5 hashes need it to be in little endian format, whereas SHA hashes need it in big endian format. Use the previously added algorithm endianness flag to control this. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index f86594fb..060bddd 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -179,7 +179,6 @@ static int mv_cesa_ahash_pad_len(struct mv_cesa_ahash_req *creq) static int mv_cesa_ahash_pad_req(struct mv_cesa_ahash_req *creq, u8 *buf) { - __be64 bits = cpu_to_be64(creq->len << 3); unsigned int index, padlen; buf[0] = 0x80; @@ -187,7 +186,14 @@ static int mv_cesa_ahash_pad_req(struct mv_cesa_ahash_req *creq, u8 *buf) index = creq->len & CESA_HASH_BLOCK_SIZE_MSK; padlen = mv_cesa_ahash_pad_len(creq); memset(buf + 1, 0, padlen - 1); - memcpy(buf + padlen, &bits, sizeof(bits)); + + if (creq->algo_le) { + __le64 bits = cpu_to_le64(creq->len << 3); + memcpy(buf + padlen, &bits, sizeof(bits)); + } else { + __be64 bits = cpu_to_be64(creq->len << 3); + memcpy(buf + padlen, &bits, sizeof(bits)); + } return padlen + 8; } -- cgit v1.1 From d30cb2fa34829161cfa1f847049b3fcfcfeebc93 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:23:51 +0100 Subject: crypto: marvell/cesa - ensure template operation is initialised Ensure that the template operation is fully initialised, otherwise we end up loading data from the kernel stack into the engines, which can upset the hash results. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 060bddd..8c0aaa96 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -866,7 +866,7 @@ static int mv_cesa_ahash_import(struct ahash_request *req, const void *hash, static int mv_cesa_md5_init(struct ahash_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_MD5); @@ -929,7 +929,7 @@ struct ahash_alg mv_md5_alg = { static int mv_cesa_sha1_init(struct ahash_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA1); @@ -992,7 +992,7 @@ struct ahash_alg mv_sha1_alg = { static int mv_cesa_sha256_init(struct ahash_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_SHA256); @@ -1222,7 +1222,7 @@ static int mv_cesa_ahmac_cra_init(struct crypto_tfm *tfm) static int mv_cesa_ahmac_md5_init(struct ahash_request *req) { struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_MD5); memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); @@ -1292,7 +1292,7 @@ struct ahash_alg mv_ahmac_md5_alg = { static int mv_cesa_ahmac_sha1_init(struct ahash_request *req) { struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA1); memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); @@ -1382,7 +1382,7 @@ static int mv_cesa_ahmac_sha256_setkey(struct crypto_ahash *tfm, const u8 *key, static int mv_cesa_ahmac_sha256_init(struct ahash_request *req) { struct mv_cesa_hmac_ctx *ctx = crypto_tfm_ctx(req->base.tfm); - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_MACM_HMAC_SHA256); memcpy(tmpl.ctx.hash.iv, ctx->iv, sizeof(ctx->iv)); -- cgit v1.1 From c439e4eec73373c6d5b7129ece69d67978ada307 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:23:56 +0100 Subject: crypto: marvell/cesa - const-ify argument to mv_cesa_get_op_cfg() mv_cesa_get_op_cfg() does not write to its argument, it only reads. So, let's make it const. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index e5f70ab..14f8285 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -627,7 +627,7 @@ static inline void mv_cesa_update_op_cfg(struct mv_cesa_op_ctx *op, op->desc.config |= cpu_to_le32(cfg); } -static inline u32 mv_cesa_get_op_cfg(struct mv_cesa_op_ctx *op) +static inline u32 mv_cesa_get_op_cfg(const struct mv_cesa_op_ctx *op) { return le32_to_cpu(op->desc.config); } -- cgit v1.1 From 8651791e5403a0e0634b76dee91b102dc4f40385 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:01 +0100 Subject: crypto: marvell/cesa - factor out first fragment decisions to helper Multiple locations in the driver test the operation context fragment type, checking whether it is a first fragment or not. Introduce a mv_cesa_mac_op_is_first_frag() helper, which returns true if the fragment operation is for a first fragment. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.h | 6 ++++++ drivers/crypto/marvell/hash.c | 9 +++------ 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index 14f8285..1103405 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -686,6 +686,12 @@ static inline u32 mv_cesa_get_int_mask(struct mv_cesa_engine *engine) return engine->int_mask; } +static inline bool mv_cesa_mac_op_is_first_frag(const struct mv_cesa_op_ctx *op) +{ + return (mv_cesa_get_op_cfg(op) & CESA_SA_DESC_CFG_FRAG_MSK) == + CESA_SA_DESC_CFG_FIRST_FRAG; +} + int mv_cesa_queue_req(struct crypto_async_request *req); /* TDMA functions */ diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 8c0aaa96..5ae2061 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -524,8 +524,7 @@ mv_cesa_ahash_dma_add_data(struct mv_cesa_tdma_chain *chain, mv_cesa_set_mac_op_frag_len(op, dma_iter->base.op_len); - if ((mv_cesa_get_op_cfg(&creq->op_tmpl) & CESA_SA_DESC_CFG_FRAG_MSK) == - CESA_SA_DESC_CFG_FIRST_FRAG) + if (mv_cesa_mac_op_is_first_frag(&creq->op_tmpl)) mv_cesa_update_op_cfg(&creq->op_tmpl, CESA_SA_DESC_CFG_MID_FRAG, CESA_SA_DESC_CFG_FRAG_MSK); @@ -561,8 +560,7 @@ mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, if (op && creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) { u32 frag = CESA_SA_DESC_CFG_NOT_FRAG; - if ((mv_cesa_get_op_cfg(op) & CESA_SA_DESC_CFG_FRAG_MSK) != - CESA_SA_DESC_CFG_FIRST_FRAG) + if (!mv_cesa_mac_op_is_first_frag(op)) frag = CESA_SA_DESC_CFG_LAST_FRAG; mv_cesa_update_op_cfg(op, frag, CESA_SA_DESC_CFG_FRAG_MSK); @@ -600,8 +598,7 @@ mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, if (padoff >= trailerlen) return op; - if ((mv_cesa_get_op_cfg(&creq->op_tmpl) & CESA_SA_DESC_CFG_FRAG_MSK) != - CESA_SA_DESC_CFG_FIRST_FRAG) + if (!mv_cesa_mac_op_is_first_frag(&creq->op_tmpl)) mv_cesa_update_op_cfg(&creq->op_tmpl, CESA_SA_DESC_CFG_MID_FRAG, CESA_SA_DESC_CFG_FRAG_MSK); -- cgit v1.1 From 9621288673cb2a1805eadf29b92e5bcec6a0e2e5 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:06 +0100 Subject: crypto: marvell/cesa - factor out adding an operation and launching it Add a helper to add the fragment operation block followed by the DMA entry to launch the operation. Although at the moment this pattern only strictly appears at one site, two other sites can be factored as well by slightly changing the order in which the DMA operations are performed. This should be harmless as the only thing which matters is to have all the data loaded into SRAM prior to launching the operation. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 74 +++++++++++++++++++++---------------------- 1 file changed, 36 insertions(+), 38 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 5ae2061..e7991cb 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -472,6 +472,29 @@ static int mv_cesa_ahash_cache_req(struct ahash_request *req, bool *cached) } static struct mv_cesa_op_ctx * +mv_cesa_dma_add_frag(struct mv_cesa_tdma_chain *chain, + struct mv_cesa_op_ctx *tmpl, unsigned int frag_len, + gfp_t flags) +{ + struct mv_cesa_op_ctx *op; + int ret; + + op = mv_cesa_dma_add_op(chain, tmpl, false, flags); + if (IS_ERR(op)) + return op; + + /* Set the operation block fragment length. */ + mv_cesa_set_mac_op_frag_len(op, frag_len); + + /* Append dummy desc to launch operation */ + ret = mv_cesa_dma_add_dummy_launch(chain, flags); + if (ret) + return ERR_PTR(ret); + + return op; +} + +static struct mv_cesa_op_ctx * mv_cesa_ahash_dma_add_cache(struct mv_cesa_tdma_chain *chain, struct mv_cesa_ahash_dma_iter *dma_iter, struct mv_cesa_ahash_req *creq, @@ -493,18 +516,9 @@ mv_cesa_ahash_dma_add_cache(struct mv_cesa_tdma_chain *chain, if (ret) return ERR_PTR(ret); - if (!dma_iter->base.op_len) { - op = mv_cesa_dma_add_op(chain, &creq->op_tmpl, false, flags); - if (IS_ERR(op)) - return op; - - mv_cesa_set_mac_op_frag_len(op, creq->cache_ptr); - - /* Add dummy desc to launch crypto operation */ - ret = mv_cesa_dma_add_dummy_launch(chain, flags); - if (ret) - return ERR_PTR(ret); - } + if (!dma_iter->base.op_len) + op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, + creq->cache_ptr, flags); return op; } @@ -518,28 +532,22 @@ mv_cesa_ahash_dma_add_data(struct mv_cesa_tdma_chain *chain, struct mv_cesa_op_ctx *op; int ret; - op = mv_cesa_dma_add_op(chain, &creq->op_tmpl, false, flags); + /* Add input transfers */ + ret = mv_cesa_dma_add_op_transfers(chain, &dma_iter->base, + &dma_iter->src, flags); + if (ret) + return ERR_PTR(ret); + + op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, dma_iter->base.op_len, + flags); if (IS_ERR(op)) return op; - mv_cesa_set_mac_op_frag_len(op, dma_iter->base.op_len); - if (mv_cesa_mac_op_is_first_frag(&creq->op_tmpl)) mv_cesa_update_op_cfg(&creq->op_tmpl, CESA_SA_DESC_CFG_MID_FRAG, CESA_SA_DESC_CFG_FRAG_MSK); - /* Add input transfers */ - ret = mv_cesa_dma_add_op_transfers(chain, &dma_iter->base, - &dma_iter->src, flags); - if (ret) - return ERR_PTR(ret); - - /* Add dummy desc to launch crypto operation */ - ret = mv_cesa_dma_add_dummy_launch(chain, flags); - if (ret) - return ERR_PTR(ret); - return op; } @@ -603,12 +611,6 @@ mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, CESA_SA_DESC_CFG_MID_FRAG, CESA_SA_DESC_CFG_FRAG_MSK); - op = mv_cesa_dma_add_op(chain, &creq->op_tmpl, false, flags); - if (IS_ERR(op)) - return op; - - mv_cesa_set_mac_op_frag_len(op, trailerlen - padoff); - ret = mv_cesa_dma_add_data_transfer(chain, CESA_SA_DATA_SRAM_OFFSET, ahashdreq->padding_dma + @@ -619,12 +621,8 @@ mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, if (ret) return ERR_PTR(ret); - /* Add dummy desc to launch crypto operation */ - ret = mv_cesa_dma_add_dummy_launch(chain, flags); - if (ret) - return ERR_PTR(ret); - - return op; + return mv_cesa_dma_add_frag(chain, &creq->op_tmpl, trailerlen - padoff, + flags); } static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) -- cgit v1.1 From 2f396a91d1d01a423fd87b7c6ff71600d06b9c05 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:11 +0100 Subject: crypto: marvell/cesa - always ensure mid-fragments after first-fragment If we add a template first-fragment operation, always update the template to be a mid-fragment. This ensures that mid-fragments always follow on from a first fragment in every case. This means we can move the first to mid-fragment update code out of mv_cesa_ahash_dma_add_data(). Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index e7991cb..3811ec0 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -491,6 +491,11 @@ mv_cesa_dma_add_frag(struct mv_cesa_tdma_chain *chain, if (ret) return ERR_PTR(ret); + if (mv_cesa_mac_op_is_first_frag(tmpl)) + mv_cesa_update_op_cfg(tmpl, + CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + return op; } @@ -529,7 +534,6 @@ mv_cesa_ahash_dma_add_data(struct mv_cesa_tdma_chain *chain, struct mv_cesa_ahash_req *creq, gfp_t flags) { - struct mv_cesa_op_ctx *op; int ret; /* Add input transfers */ @@ -538,17 +542,8 @@ mv_cesa_ahash_dma_add_data(struct mv_cesa_tdma_chain *chain, if (ret) return ERR_PTR(ret); - op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, dma_iter->base.op_len, - flags); - if (IS_ERR(op)) - return op; - - if (mv_cesa_mac_op_is_first_frag(&creq->op_tmpl)) - mv_cesa_update_op_cfg(&creq->op_tmpl, - CESA_SA_DESC_CFG_MID_FRAG, - CESA_SA_DESC_CFG_FRAG_MSK); - - return op; + return mv_cesa_dma_add_frag(chain, &creq->op_tmpl, dma_iter->base.op_len, + flags); } static struct mv_cesa_op_ctx * -- cgit v1.1 From 0971d09a8566abd6640acdbe3fd304e67e257bd3 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:16 +0100 Subject: crypto: marvell/cesa - move mv_cesa_dma_add_frag() calls Move the calls to mv_cesa_dma_add_frag() into the parent function, mv_cesa_ahash_dma_req_init(). This is in preparation to changing when we generate the operation blocks, as we need to avoid generating a block for a partial hash block at the end of the user data. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 71 ++++++++++++++++++------------------------- 1 file changed, 29 insertions(+), 42 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 3811ec0..d2a947e 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -499,51 +499,23 @@ mv_cesa_dma_add_frag(struct mv_cesa_tdma_chain *chain, return op; } -static struct mv_cesa_op_ctx * +static int mv_cesa_ahash_dma_add_cache(struct mv_cesa_tdma_chain *chain, struct mv_cesa_ahash_dma_iter *dma_iter, struct mv_cesa_ahash_req *creq, gfp_t flags) { struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; - struct mv_cesa_op_ctx *op = NULL; - int ret; if (!creq->cache_ptr) - return NULL; - - ret = mv_cesa_dma_add_data_transfer(chain, - CESA_SA_DATA_SRAM_OFFSET, - ahashdreq->cache_dma, - creq->cache_ptr, - CESA_TDMA_DST_IN_SRAM, - flags); - if (ret) - return ERR_PTR(ret); - - if (!dma_iter->base.op_len) - op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, - creq->cache_ptr, flags); - - return op; -} - -static struct mv_cesa_op_ctx * -mv_cesa_ahash_dma_add_data(struct mv_cesa_tdma_chain *chain, - struct mv_cesa_ahash_dma_iter *dma_iter, - struct mv_cesa_ahash_req *creq, - gfp_t flags) -{ - int ret; - - /* Add input transfers */ - ret = mv_cesa_dma_add_op_transfers(chain, &dma_iter->base, - &dma_iter->src, flags); - if (ret) - return ERR_PTR(ret); + return 0; - return mv_cesa_dma_add_frag(chain, &creq->op_tmpl, dma_iter->base.op_len, - flags); + return mv_cesa_dma_add_data_transfer(chain, + CESA_SA_DATA_SRAM_OFFSET, + ahashdreq->cache_dma, + creq->cache_ptr, + CESA_TDMA_DST_IN_SRAM, + flags); } static struct mv_cesa_op_ctx * @@ -647,19 +619,34 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) mv_cesa_tdma_desc_iter_init(&chain); mv_cesa_ahash_req_iter_init(&iter, req); - op = mv_cesa_ahash_dma_add_cache(&chain, &iter, - creq, flags); - if (IS_ERR(op)) { - ret = PTR_ERR(op); + /* + * Add the cache (left-over data from a previous block) first. + * This will never overflow the SRAM size. + */ + ret = mv_cesa_ahash_dma_add_cache(&chain, &iter, creq, flags); + if (ret) goto err_free_tdma; + + if (creq->cache_ptr && !iter.base.op_len) { + op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, + creq->cache_ptr, flags); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto err_free_tdma; + } } do { if (!iter.base.op_len) break; - op = mv_cesa_ahash_dma_add_data(&chain, &iter, - creq, flags); + ret = mv_cesa_dma_add_op_transfers(&chain, &iter.base, + &iter.src, flags); + if (ret) + goto err_free_tdma; + + op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, + iter.base.op_len, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); goto err_free_tdma; -- cgit v1.1 From d9bba4c3ebbc65b9113624eb1690bed776fb6106 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:21 +0100 Subject: crypto: marvell/cesa - use presence of scatterlist to determine data load Use the presence of the scatterlist to determine whether we should load any new user data to the engine. The following shall always be true at this point: iter.base.op_len == 0 === iter.src.sg In doing so, we can: 1. eliminate the test for iter.base.op_len inside the loop, which makes the loop operation more obvious and understandable. 2. move the operation generation for the cache-only case. This prepares the code for the next step in its transformation, and also uncovers a bug that will be fixed in the next patch. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index d2a947e..3497d70 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -627,7 +627,27 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (ret) goto err_free_tdma; - if (creq->cache_ptr && !iter.base.op_len) { + if (iter.src.sg) { + /* + * Add all the new data, inserting an operation block and + * launch command between each full SRAM block-worth of + * data. + */ + do { + ret = mv_cesa_dma_add_op_transfers(&chain, &iter.base, + &iter.src, flags); + if (ret) + goto err_free_tdma; + + op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, + iter.base.op_len, flags); + if (IS_ERR(op)) { + ret = PTR_ERR(op); + goto err_free_tdma; + } + } while (mv_cesa_ahash_req_iter_next_op(&iter)); + } else if (creq->cache_ptr) { + /* Account for the data that was in the cache. */ op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, creq->cache_ptr, flags); if (IS_ERR(op)) { @@ -636,23 +656,6 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) } } - do { - if (!iter.base.op_len) - break; - - ret = mv_cesa_dma_add_op_transfers(&chain, &iter.base, - &iter.src, flags); - if (ret) - goto err_free_tdma; - - op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, - iter.base.op_len, flags); - if (IS_ERR(op)) { - ret = PTR_ERR(op); - goto err_free_tdma; - } - } while (mv_cesa_ahash_req_iter_next_op(&iter)); - op = mv_cesa_ahash_dma_last_req(&chain, &iter, creq, op, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); -- cgit v1.1 From bd274b10856b91f702fa767114aa7e6ac45b933d Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:26 +0100 Subject: crypto: marvell/cesa - ensure iter.base.op_len is the full op length When we process the last request of data, and the request contains user data, the loop in mv_cesa_ahash_dma_req_init() marks the first data size as being iter.base.op_len which does not include the size of the cache data. This means we end up hashing an insufficient amount of data. Fix this by always including the cache size in the first operation length of any request. This has the effect that for a request containing no user data, iter.base.op_len === iter.src.op_offset === creq->cache_ptr As a result, we include one further change to use iter.base.op_len in the cache-but-no-user-data case to make the next change clearer. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 3497d70..de1d854 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -27,10 +27,10 @@ mv_cesa_ahash_req_iter_init(struct mv_cesa_ahash_dma_iter *iter, struct ahash_request *req) { struct mv_cesa_ahash_req *creq = ahash_request_ctx(req); - unsigned int len = req->nbytes; + unsigned int len = req->nbytes + creq->cache_ptr; if (!creq->last_req) - len = (len + creq->cache_ptr) & ~CESA_HASH_BLOCK_SIZE_MSK; + len &= ~CESA_HASH_BLOCK_SIZE_MSK; mv_cesa_req_dma_iter_init(&iter->base, len); mv_cesa_sg_dma_iter_init(&iter->src, req->src, DMA_TO_DEVICE); @@ -646,10 +646,10 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) goto err_free_tdma; } } while (mv_cesa_ahash_req_iter_next_op(&iter)); - } else if (creq->cache_ptr) { + } else if (iter.base.op_len) { /* Account for the data that was in the cache. */ op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, - creq->cache_ptr, flags); + iter.base.op_len, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); goto err_free_tdma; -- cgit v1.1 From e41bbebddec44f7c06585f42686b2a529513fc6a Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:32 +0100 Subject: crypto: marvell/cesa - avoid adding final operation within loop Avoid adding the final operation within the loop, but instead add it outside. We combine this with the handling for the no-data case. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index de1d854..ce457d6 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -602,6 +602,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) struct mv_cesa_tdma_chain chain; struct mv_cesa_ahash_dma_iter iter; struct mv_cesa_op_ctx *op = NULL; + unsigned int frag_len; int ret; dreq->chain.first = NULL; @@ -631,25 +632,34 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) /* * Add all the new data, inserting an operation block and * launch command between each full SRAM block-worth of - * data. + * data. We intentionally do not add the final op block. */ - do { + while (true) { ret = mv_cesa_dma_add_op_transfers(&chain, &iter.base, &iter.src, flags); if (ret) goto err_free_tdma; + frag_len = iter.base.op_len; + + if (!mv_cesa_ahash_req_iter_next_op(&iter)) + break; + op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, - iter.base.op_len, flags); + frag_len, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); goto err_free_tdma; } - } while (mv_cesa_ahash_req_iter_next_op(&iter)); - } else if (iter.base.op_len) { + } + } else { /* Account for the data that was in the cache. */ - op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, - iter.base.op_len, flags); + frag_len = iter.base.op_len; + } + + if (frag_len) { + op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, frag_len, + flags); if (IS_ERR(op)) { ret = PTR_ERR(op); goto err_free_tdma; -- cgit v1.1 From 58953e15ef015ba9136b7234a7098cf827428999 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:37 +0100 Subject: crypto: marvell/cesa - rearrange last request handling Move the test for the last request out of mv_cesa_ahash_dma_last_req() to its caller, and move the mv_cesa_dma_add_frag() down into this function. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index ce457d6..71d29f0 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -522,15 +522,21 @@ static struct mv_cesa_op_ctx * mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, struct mv_cesa_ahash_dma_iter *dma_iter, struct mv_cesa_ahash_req *creq, - struct mv_cesa_op_ctx *op, - gfp_t flags) + unsigned int frag_len, gfp_t flags) { struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; unsigned int len, trailerlen, padoff = 0; + struct mv_cesa_op_ctx *op; int ret; - if (!creq->last_req) - return op; + if (frag_len) { + op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len, + flags); + if (IS_ERR(op)) + return op; + } else { + op = NULL; + } if (op && creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) { u32 frag = CESA_SA_DESC_CFG_NOT_FRAG; @@ -657,16 +663,18 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) frag_len = iter.base.op_len; } - if (frag_len) { + /* + * At this point, frag_len indicates whether we have any data + * outstanding which needs an operation. Queue up the final + * operation, which depends whether this is the final request. + */ + if (creq->last_req) + op = mv_cesa_ahash_dma_last_req(&chain, &iter, creq, frag_len, + flags); + else if (frag_len) op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, frag_len, flags); - if (IS_ERR(op)) { - ret = PTR_ERR(op); - goto err_free_tdma; - } - } - op = mv_cesa_ahash_dma_last_req(&chain, &iter, creq, op, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); goto err_free_tdma; -- cgit v1.1 From aee84a7e6ed515f0f217317ff261a15db4bff0bb Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:42 +0100 Subject: crypto: marvell/cesa - rearrange handling for hw finished hashes Rearrange the last request handling for hardware finished hashes by moving the generation of the fragment operation into this path. This results in a simplified sequence to handle this case, and allows us to move the software padded case further down into the function. Add comments describing these parts. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 71d29f0..d282716 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -529,32 +529,45 @@ mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, struct mv_cesa_op_ctx *op; int ret; - if (frag_len) { + /* + * If the transfer is smaller than our maximum length, and we have + * some data outstanding, we can ask the engine to finish the hash. + */ + if (creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX && frag_len) { op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len, flags); if (IS_ERR(op)) return op; - } else { - op = NULL; - } - - if (op && creq->len <= CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX) { - u32 frag = CESA_SA_DESC_CFG_NOT_FRAG; - - if (!mv_cesa_mac_op_is_first_frag(op)) - frag = CESA_SA_DESC_CFG_LAST_FRAG; - mv_cesa_update_op_cfg(op, frag, CESA_SA_DESC_CFG_FRAG_MSK); + mv_cesa_set_mac_op_total_len(op, creq->len); + mv_cesa_update_op_cfg(op, mv_cesa_mac_op_is_first_frag(op) ? + CESA_SA_DESC_CFG_NOT_FRAG : + CESA_SA_DESC_CFG_LAST_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); return op; } + /* + * The request is longer than the engine can handle, or we have + * no data outstanding. Manually generate the padding, adding it + * as a "mid" fragment. + */ ret = mv_cesa_ahash_dma_alloc_padding(ahashdreq, flags); if (ret) return ERR_PTR(ret); trailerlen = mv_cesa_ahash_pad_req(creq, ahashdreq->padding); + if (frag_len) { + op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len, + flags); + if (IS_ERR(op)) + return op; + } else { + op = NULL; + } + if (op) { len = min(CESA_SA_SRAM_PAYLOAD_SIZE - dma_iter->base.op_len, trailerlen); -- cgit v1.1 From ab270e7055e0aa9d4dc3bf419cdfdde340174978 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:47 +0100 Subject: crypto: marvell/cesa - rearrange handling for sw padded hashes Rearrange the last request handling for hashes which require software padding. We prepare the padding to be appended, and then append as much of the padding to any existing data that's already queued up, adding an operation block and launching the operation. Any remainder is then appended as a separate operation. This ensures that the hardware only ever sees multiples of the hash block size to be operated on for software padded hashes, thus ensuring that the engine always indicates that it has finished the calculation. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 44 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 26 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index d282716..f4d5a72 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -559,38 +559,30 @@ mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, trailerlen = mv_cesa_ahash_pad_req(creq, ahashdreq->padding); - if (frag_len) { - op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len, - flags); - if (IS_ERR(op)) - return op; - } else { - op = NULL; - } - - if (op) { - len = min(CESA_SA_SRAM_PAYLOAD_SIZE - dma_iter->base.op_len, - trailerlen); - if (len) { - ret = mv_cesa_dma_add_data_transfer(chain, + len = min(CESA_SA_SRAM_PAYLOAD_SIZE - frag_len, trailerlen); + if (len) { + ret = mv_cesa_dma_add_data_transfer(chain, CESA_SA_DATA_SRAM_OFFSET + - dma_iter->base.op_len, + frag_len, ahashdreq->padding_dma, len, CESA_TDMA_DST_IN_SRAM, flags); - if (ret) - return ERR_PTR(ret); + if (ret) + return ERR_PTR(ret); - mv_cesa_update_op_cfg(op, CESA_SA_DESC_CFG_MID_FRAG, - CESA_SA_DESC_CFG_FRAG_MSK); - mv_cesa_set_mac_op_frag_len(op, - dma_iter->base.op_len + len); - padoff += len; - } - } + op = mv_cesa_dma_add_frag(chain, &creq->op_tmpl, frag_len + len, + flags); + if (IS_ERR(op)) + return op; - if (padoff >= trailerlen) - return op; + mv_cesa_update_op_cfg(op, CESA_SA_DESC_CFG_MID_FRAG, + CESA_SA_DESC_CFG_FRAG_MSK); + + if (len == trailerlen) + return op; + + padoff += len; + } if (!mv_cesa_mac_op_is_first_frag(&creq->op_tmpl)) mv_cesa_update_op_cfg(&creq->op_tmpl, -- cgit v1.1 From 8efbc2c0f6b4e52e384ecb1714511f0194c4d56a Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 17:24:52 +0100 Subject: crypto: marvell/cesa - fix first-fragment handling in mv_cesa_ahash_dma_last_req() When adding the software padding, this must be done using the first/mid fragment mode, and any subsequent operation needs to be a mid-fragment. Fix this. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index f4d5a72..4c80126 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -575,20 +575,12 @@ mv_cesa_ahash_dma_last_req(struct mv_cesa_tdma_chain *chain, if (IS_ERR(op)) return op; - mv_cesa_update_op_cfg(op, CESA_SA_DESC_CFG_MID_FRAG, - CESA_SA_DESC_CFG_FRAG_MSK); - if (len == trailerlen) return op; padoff += len; } - if (!mv_cesa_mac_op_is_first_frag(&creq->op_tmpl)) - mv_cesa_update_op_cfg(&creq->op_tmpl, - CESA_SA_DESC_CFG_MID_FRAG, - CESA_SA_DESC_CFG_FRAG_MSK); - ret = mv_cesa_dma_add_data_transfer(chain, CESA_SA_DATA_SRAM_OFFSET, ahashdreq->padding_dma + -- cgit v1.1 From 8c07f3a8c487df241f5e809e8f6d5dd7c9d75b54 Mon Sep 17 00:00:00 2001 From: Boris Brezillon Date: Sun, 18 Oct 2015 17:24:57 +0100 Subject: crypto: marvell/cesa - fix memory leak To: Boris Brezillon ,Arnaud Ebalard ,Thomas Petazzoni ,Jason Cooper The local chain variable is not cleaned up if an error occurs in the middle of DMA chain creation. Fix that by dropping the local chain variable and using the dreq->chain field which will be cleaned up by mv_cesa_dma_cleanup() in case of errors. Signed-off-by: Boris Brezillon Reported-by: Thomas Petazzoni Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/hash.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index 4c80126..d813de6 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -602,7 +602,6 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) GFP_KERNEL : GFP_ATOMIC; struct mv_cesa_ahash_dma_req *ahashdreq = &creq->req.dma; struct mv_cesa_tdma_req *dreq = &ahashdreq->base; - struct mv_cesa_tdma_chain chain; struct mv_cesa_ahash_dma_iter iter; struct mv_cesa_op_ctx *op = NULL; unsigned int frag_len; @@ -620,14 +619,14 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) } } - mv_cesa_tdma_desc_iter_init(&chain); + mv_cesa_tdma_desc_iter_init(&dreq->chain); mv_cesa_ahash_req_iter_init(&iter, req); /* * Add the cache (left-over data from a previous block) first. * This will never overflow the SRAM size. */ - ret = mv_cesa_ahash_dma_add_cache(&chain, &iter, creq, flags); + ret = mv_cesa_ahash_dma_add_cache(&dreq->chain, &iter, creq, flags); if (ret) goto err_free_tdma; @@ -638,7 +637,8 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) * data. We intentionally do not add the final op block. */ while (true) { - ret = mv_cesa_dma_add_op_transfers(&chain, &iter.base, + ret = mv_cesa_dma_add_op_transfers(&dreq->chain, + &iter.base, &iter.src, flags); if (ret) goto err_free_tdma; @@ -648,7 +648,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (!mv_cesa_ahash_req_iter_next_op(&iter)) break; - op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, + op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, frag_len, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); @@ -666,11 +666,11 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) * operation, which depends whether this is the final request. */ if (creq->last_req) - op = mv_cesa_ahash_dma_last_req(&chain, &iter, creq, frag_len, - flags); + op = mv_cesa_ahash_dma_last_req(&dreq->chain, &iter, creq, + frag_len, flags); else if (frag_len) - op = mv_cesa_dma_add_frag(&chain, &creq->op_tmpl, frag_len, - flags); + op = mv_cesa_dma_add_frag(&dreq->chain, &creq->op_tmpl, + frag_len, flags); if (IS_ERR(op)) { ret = PTR_ERR(op); @@ -679,7 +679,7 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) if (op) { /* Add dummy desc to wait for crypto operation end */ - ret = mv_cesa_dma_add_dummy_end(&chain, flags); + ret = mv_cesa_dma_add_dummy_end(&dreq->chain, flags); if (ret) goto err_free_tdma; } @@ -690,8 +690,6 @@ static int mv_cesa_ahash_dma_req_init(struct ahash_request *req) else creq->cache_ptr = 0; - dreq->chain = chain; - return 0; err_free_tdma: -- cgit v1.1 From b150856152cc442050ed3041e912ff0258c50f87 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 18:31:00 +0100 Subject: crypto: marvell/cesa - use readl_relaxed()/writel_relaxed() Use relaxed IO accessors where appropriate. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.h | 2 +- drivers/crypto/marvell/cipher.c | 2 +- drivers/crypto/marvell/hash.c | 7 +++---- drivers/crypto/marvell/tdma.c | 20 ++++++++++---------- 4 files changed, 15 insertions(+), 16 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index 1103405..8b5c047 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -677,7 +677,7 @@ static inline void mv_cesa_set_int_mask(struct mv_cesa_engine *engine, if (int_mask == engine->int_mask) return; - writel(int_mask, engine->regs + CESA_SA_INT_MSK); + writel_relaxed(int_mask, engine->regs + CESA_SA_INT_MSK); engine->int_mask = int_mask; } diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index 0745cf3..06dee02 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -105,7 +105,7 @@ static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) } mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); - writel(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index d813de6..a9bea0b 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -281,7 +281,7 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) creq->cache_ptr = new_cache_ptr; mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); - writel(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); + writel_relaxed(CESA_SA_CFG_PARA_DIS, engine->regs + CESA_SA_CFG); writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } @@ -344,7 +344,7 @@ static int mv_cesa_ahash_process(struct crypto_async_request *req, u32 status) digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); for (i = 0; i < digsize / 4; i++) - creq->state[i] = readl(engine->regs + CESA_IVDIG(i)); + creq->state[i] = readl_relaxed(engine->regs + CESA_IVDIG(i)); if (creq->cache_ptr) sg_pcopy_to_buffer(ahashreq->src, creq->src_nents, @@ -390,8 +390,7 @@ static void mv_cesa_ahash_prepare(struct crypto_async_request *req, digsize = crypto_ahash_digestsize(crypto_ahash_reqtfm(ahashreq)); for (i = 0; i < digsize / 4; i++) - writel(creq->state[i], - engine->regs + CESA_IVDIG(i)); + writel_relaxed(creq->state[i], engine->regs + CESA_IVDIG(i)); } static void mv_cesa_ahash_req_cleanup(struct crypto_async_request *req) diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c index 64a366c..e8e8a7f 100644 --- a/drivers/crypto/marvell/tdma.c +++ b/drivers/crypto/marvell/tdma.c @@ -41,18 +41,18 @@ void mv_cesa_dma_step(struct mv_cesa_tdma_req *dreq) { struct mv_cesa_engine *engine = dreq->base.engine; - writel(0, engine->regs + CESA_SA_CFG); + writel_relaxed(0, engine->regs + CESA_SA_CFG); mv_cesa_set_int_mask(engine, CESA_SA_INT_ACC0_IDMA_DONE); - writel(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B | - CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN, - engine->regs + CESA_TDMA_CONTROL); - - writel(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT | - CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS, - engine->regs + CESA_SA_CFG); - writel(dreq->chain.first->cur_dma, - engine->regs + CESA_TDMA_NEXT_ADDR); + writel_relaxed(CESA_TDMA_DST_BURST_128B | CESA_TDMA_SRC_BURST_128B | + CESA_TDMA_NO_BYTE_SWAP | CESA_TDMA_EN, + engine->regs + CESA_TDMA_CONTROL); + + writel_relaxed(CESA_SA_CFG_ACT_CH0_IDMA | CESA_SA_CFG_MULTI_PKT | + CESA_SA_CFG_CH0_W_IDMA | CESA_SA_CFG_PARA_DIS, + engine->regs + CESA_SA_CFG); + writel_relaxed(dreq->chain.first->cur_dma, + engine->regs + CESA_TDMA_NEXT_ADDR); writel(CESA_SA_CMD_EN_CESA_SA_ACCL0, engine->regs + CESA_SA_CMD); } -- cgit v1.1 From 5d754137ab905aaaf69acf81c9c6c7005bcfe6e4 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 18:31:05 +0100 Subject: crypto: marvell/cesa - use dma_addr_t for cur_dma cur_dma is part of the software state, not read by the hardware. Storing it in LE32 format is wrong, use dma_addr_t for this. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.h | 4 +++- drivers/crypto/marvell/tdma.c | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index 8b5c047..d1edf01 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -297,7 +297,9 @@ struct mv_cesa_tdma_desc { u32 src; u32 dst; u32 next_dma; - u32 cur_dma; + + /* Software state */ + dma_addr_t cur_dma; struct mv_cesa_tdma_desc *next; union { struct mv_cesa_op_ctx *op; diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c index e8e8a7f..c8256f5 100644 --- a/drivers/crypto/marvell/tdma.c +++ b/drivers/crypto/marvell/tdma.c @@ -69,7 +69,7 @@ void mv_cesa_dma_cleanup(struct mv_cesa_tdma_req *dreq) tdma = tdma->next; dma_pool_free(cesa_dev->dma->tdma_desc_pool, old_tdma, - le32_to_cpu(old_tdma->cur_dma)); + old_tdma->cur_dma); } dreq->chain.first = NULL; @@ -105,9 +105,9 @@ mv_cesa_dma_add_desc(struct mv_cesa_tdma_chain *chain, gfp_t flags) return ERR_PTR(-ENOMEM); memset(new_tdma, 0, sizeof(*new_tdma)); - new_tdma->cur_dma = cpu_to_le32(dma_handle); + new_tdma->cur_dma = dma_handle; if (chain->last) { - chain->last->next_dma = new_tdma->cur_dma; + chain->last->next_dma = cpu_to_le32(dma_handle); chain->last->next = new_tdma; } else { chain->first = new_tdma; -- cgit v1.1 From 35622eae188761f30df916a6cae4fa8c942a6366 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 18:31:10 +0100 Subject: crypto: marvell/cesa - use gfp_t for gfp flags Use gfp_t not u32 for the GFP flags. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.h | 6 ++---- drivers/crypto/marvell/tdma.c | 5 ++--- 2 files changed, 4 insertions(+), 7 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index d1edf01..aa1949a 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -771,10 +771,8 @@ int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, dma_addr_t dst, dma_addr_t src, u32 size, u32 flags, gfp_t gfp_flags); -int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, - u32 flags); - -int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, u32 flags); +int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags); +int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags); int mv_cesa_dma_add_op_transfers(struct mv_cesa_tdma_chain *chain, struct mv_cesa_dma_iter *dma_iter, diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c index c8256f5..2738e24 100644 --- a/drivers/crypto/marvell/tdma.c +++ b/drivers/crypto/marvell/tdma.c @@ -166,8 +166,7 @@ int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, return 0; } -int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, - u32 flags) +int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, gfp_t flags) { struct mv_cesa_tdma_desc *tdma; @@ -178,7 +177,7 @@ int mv_cesa_dma_add_dummy_launch(struct mv_cesa_tdma_chain *chain, return 0; } -int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, u32 flags) +int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags) { struct mv_cesa_tdma_desc *tdma; -- cgit v1.1 From 0f3304dc18b1316bc6d32921f2a7837938f90c30 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 18:31:15 +0100 Subject: crypto: marvell/cesa - use memcpy_fromio()/memcpy_toio() Use the IO memcpy() functions when copying from/to MMIO memory. These locations were found via sparse. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cipher.c | 11 ++++++----- drivers/crypto/marvell/hash.c | 16 ++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/cipher.c b/drivers/crypto/marvell/cipher.c index 06dee02..2864d1e 100644 --- a/drivers/crypto/marvell/cipher.c +++ b/drivers/crypto/marvell/cipher.c @@ -98,10 +98,10 @@ static void mv_cesa_ablkcipher_std_step(struct ablkcipher_request *req) /* FIXME: only update enc_len field */ if (!sreq->skip_ctx) { - memcpy(engine->sram, &sreq->op, sizeof(sreq->op)); + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); sreq->skip_ctx = true; } else { - memcpy(engine->sram, &sreq->op, sizeof(sreq->op.desc)); + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op.desc)); } mv_cesa_set_int_mask(engine, CESA_SA_INT_ACCEL0_DONE); @@ -145,8 +145,9 @@ static int mv_cesa_ablkcipher_process(struct crypto_async_request *req, if (ret) return ret; - memcpy(ablkreq->info, engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, - crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq))); + memcpy_fromio(ablkreq->info, + engine->sram + CESA_SA_CRYPT_IV_SRAM_OFFSET, + crypto_ablkcipher_ivsize(crypto_ablkcipher_reqtfm(ablkreq))); return 0; } @@ -181,7 +182,7 @@ mv_cesa_ablkcipher_std_prepare(struct ablkcipher_request *req) sreq->size = 0; sreq->offset = 0; mv_cesa_adjust_op(engine, &sreq->op); - memcpy(engine->sram, &sreq->op, sizeof(sreq->op)); + memcpy_toio(engine->sram, &sreq->op, sizeof(sreq->op)); } static inline void mv_cesa_ablkcipher_prepare(struct crypto_async_request *req, diff --git a/drivers/crypto/marvell/hash.c b/drivers/crypto/marvell/hash.c index a9bea0b..a70193e 100644 --- a/drivers/crypto/marvell/hash.c +++ b/drivers/crypto/marvell/hash.c @@ -209,8 +209,8 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) size_t len; if (creq->cache_ptr) - memcpy(engine->sram + CESA_SA_DATA_SRAM_OFFSET, creq->cache, - creq->cache_ptr); + memcpy_toio(engine->sram + CESA_SA_DATA_SRAM_OFFSET, + creq->cache, creq->cache_ptr); len = min_t(size_t, req->nbytes + creq->cache_ptr - sreq->offset, CESA_SA_SRAM_PAYLOAD_SIZE); @@ -251,10 +251,10 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) if (len + trailerlen > CESA_SA_SRAM_PAYLOAD_SIZE) { len &= CESA_HASH_BLOCK_SIZE_MSK; new_cache_ptr = 64 - trailerlen; - memcpy(creq->cache, - engine->sram + - CESA_SA_DATA_SRAM_OFFSET + len, - new_cache_ptr); + memcpy_fromio(creq->cache, + engine->sram + + CESA_SA_DATA_SRAM_OFFSET + len, + new_cache_ptr); } else { len += mv_cesa_ahash_pad_req(creq, engine->sram + len + @@ -272,7 +272,7 @@ static void mv_cesa_ahash_std_step(struct ahash_request *req) mv_cesa_update_op_cfg(op, frag_mode, CESA_SA_DESC_CFG_FRAG_MSK); /* FIXME: only update enc_len field */ - memcpy(engine->sram, op, sizeof(*op)); + memcpy_toio(engine->sram, op, sizeof(*op)); if (frag_mode == CESA_SA_DESC_CFG_FIRST_FRAG) mv_cesa_update_op_cfg(op, CESA_SA_DESC_CFG_MID_FRAG, @@ -312,7 +312,7 @@ static void mv_cesa_ahash_std_prepare(struct ahash_request *req) sreq->offset = 0; mv_cesa_adjust_op(engine, &creq->op_tmpl); - memcpy(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); + memcpy_toio(engine->sram, &creq->op_tmpl, sizeof(creq->op_tmpl)); } static void mv_cesa_ahash_step(struct crypto_async_request *req) -- cgit v1.1 From ea1f662bde067563328ec3083884aaffb3d902d8 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 18:31:20 +0100 Subject: crypto: marvell/cesa - fix missing cpu_to_le32() in mv_cesa_dma_add_op() When tdma->src is freed in mv_cesa_dma_cleanup(), we convert the DMA address from a little-endian value prior to calling dma_pool_free(). However, mv_cesa_dma_add_op() assigns tdma->src without first converting the DMA address to little endian. Fix this. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/tdma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c index 2738e24..06bdeac 100644 --- a/drivers/crypto/marvell/tdma.c +++ b/drivers/crypto/marvell/tdma.c @@ -140,7 +140,7 @@ struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, tdma = chain->last; tdma->op = op; tdma->byte_cnt = (skip_ctx ? sizeof(op->desc) : sizeof(*op)) | BIT(31); - tdma->src = dma_handle; + tdma->src = cpu_to_le32(dma_handle); tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP; return op; -- cgit v1.1 From 6de59d453ae567c1b637ef9469c37fd8367cd1b4 Mon Sep 17 00:00:00 2001 From: Russell King Date: Sun, 18 Oct 2015 18:31:26 +0100 Subject: crypto: marvell/cesa - use __le32 for hardware descriptors Much of the driver uses cpu_to_le32() to convert values for descriptors to little endian before writing. Use __le32 to define the hardware- accessed parts of the descriptors, and ensure most places where it's reasonable to do so use cpu_to_le32() when assigning to these. Signed-off-by: Russell King Signed-off-by: Herbert Xu --- drivers/crypto/marvell/cesa.h | 32 ++++++++++++++++---------------- drivers/crypto/marvell/tdma.c | 9 ++++++--- 2 files changed, 22 insertions(+), 19 deletions(-) (limited to 'drivers/crypto/marvell') diff --git a/drivers/crypto/marvell/cesa.h b/drivers/crypto/marvell/cesa.h index aa1949a..186868e 100644 --- a/drivers/crypto/marvell/cesa.h +++ b/drivers/crypto/marvell/cesa.h @@ -174,19 +174,19 @@ #define CESA_SA_DESC_MAC_DATA(offset) \ cpu_to_le32(CESA_SA_DATA_SRAM_OFFSET + (offset)) -#define CESA_SA_DESC_MAC_DATA_MSK GENMASK(15, 0) +#define CESA_SA_DESC_MAC_DATA_MSK cpu_to_le32(GENMASK(15, 0)) #define CESA_SA_DESC_MAC_TOTAL_LEN(total_len) cpu_to_le32((total_len) << 16) -#define CESA_SA_DESC_MAC_TOTAL_LEN_MSK GENMASK(31, 16) +#define CESA_SA_DESC_MAC_TOTAL_LEN_MSK cpu_to_le32(GENMASK(31, 16)) #define CESA_SA_DESC_MAC_SRC_TOTAL_LEN_MAX 0xffff #define CESA_SA_DESC_MAC_DIGEST(offset) \ cpu_to_le32(CESA_SA_MAC_DIG_SRAM_OFFSET + (offset)) -#define CESA_SA_DESC_MAC_DIGEST_MSK GENMASK(15, 0) +#define CESA_SA_DESC_MAC_DIGEST_MSK cpu_to_le32(GENMASK(15, 0)) #define CESA_SA_DESC_MAC_FRAG_LEN(frag_len) cpu_to_le32((frag_len) << 16) -#define CESA_SA_DESC_MAC_FRAG_LEN_MSK GENMASK(31, 16) +#define CESA_SA_DESC_MAC_FRAG_LEN_MSK cpu_to_le32(GENMASK(31, 16)) #define CESA_SA_DESC_MAC_IV(offset) \ cpu_to_le32((CESA_SA_MAC_IIV_SRAM_OFFSET + (offset)) | \ @@ -219,14 +219,14 @@ * to be executed. */ struct mv_cesa_sec_accel_desc { - u32 config; - u32 enc_p; - u32 enc_len; - u32 enc_key_p; - u32 enc_iv; - u32 mac_src_p; - u32 mac_digest; - u32 mac_iv; + __le32 config; + __le32 enc_p; + __le32 enc_len; + __le32 enc_key_p; + __le32 enc_iv; + __le32 mac_src_p; + __le32 mac_digest; + __le32 mac_iv; }; /** @@ -293,10 +293,10 @@ struct mv_cesa_op_ctx { * operation. */ struct mv_cesa_tdma_desc { - u32 byte_cnt; - u32 src; - u32 dst; - u32 next_dma; + __le32 byte_cnt; + __le32 src; + __le32 dst; + __le32 next_dma; /* Software state */ dma_addr_t cur_dma; diff --git a/drivers/crypto/marvell/tdma.c b/drivers/crypto/marvell/tdma.c index 06bdeac..7642798 100644 --- a/drivers/crypto/marvell/tdma.c +++ b/drivers/crypto/marvell/tdma.c @@ -126,6 +126,7 @@ struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, struct mv_cesa_tdma_desc *tdma; struct mv_cesa_op_ctx *op; dma_addr_t dma_handle; + unsigned int size; tdma = mv_cesa_dma_add_desc(chain, flags); if (IS_ERR(tdma)) @@ -137,9 +138,11 @@ struct mv_cesa_op_ctx *mv_cesa_dma_add_op(struct mv_cesa_tdma_chain *chain, *op = *op_templ; + size = skip_ctx ? sizeof(op->desc) : sizeof(*op); + tdma = chain->last; tdma->op = op; - tdma->byte_cnt = (skip_ctx ? sizeof(op->desc) : sizeof(*op)) | BIT(31); + tdma->byte_cnt = cpu_to_le32(size | BIT(31)); tdma->src = cpu_to_le32(dma_handle); tdma->flags = CESA_TDMA_DST_IN_SRAM | CESA_TDMA_OP; @@ -156,7 +159,7 @@ int mv_cesa_dma_add_data_transfer(struct mv_cesa_tdma_chain *chain, if (IS_ERR(tdma)) return PTR_ERR(tdma); - tdma->byte_cnt = size | BIT(31); + tdma->byte_cnt = cpu_to_le32(size | BIT(31)); tdma->src = src; tdma->dst = dst; @@ -185,7 +188,7 @@ int mv_cesa_dma_add_dummy_end(struct mv_cesa_tdma_chain *chain, gfp_t flags) if (IS_ERR(tdma)) return PTR_ERR(tdma); - tdma->byte_cnt = BIT(31); + tdma->byte_cnt = cpu_to_le32(BIT(31)); return 0; } -- cgit v1.1