diff options
author | Linus Torvalds <torvalds@linux-foundation.org> | 2017-05-02 15:53:46 -0700 |
---|---|---|
committer | Linus Torvalds <torvalds@linux-foundation.org> | 2017-05-02 15:53:46 -0700 |
commit | 5a0387a8a8efb90ae7fea1e2e5c62de3efa74691 (patch) | |
tree | 9e5bbbafe7fea01c843d86c7c3d40f29f962c474 /drivers/crypto/mediatek | |
parent | 204f144c9fcac355843412b6ba1150086488a208 (diff) | |
parent | 929562b144783b9212625305eadcbbd800809643 (diff) | |
download | op-kernel-dev-5a0387a8a8efb90ae7fea1e2e5c62de3efa74691.zip op-kernel-dev-5a0387a8a8efb90ae7fea1e2e5c62de3efa74691.tar.gz |
Merge branch 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6
Pull crypto updates from Herbert Xu:
"Here is the crypto update for 4.12:
API:
- Add batch registration for acomp/scomp
- Change acomp testing to non-unique compressed result
- Extend algorithm name limit to 128 bytes
- Require setkey before accept(2) in algif_aead
Algorithms:
- Add support for deflate rfc1950 (zlib)
Drivers:
- Add accelerated crct10dif for powerpc
- Add crc32 in stm32
- Add sha384/sha512 in ccp
- Add 3des/gcm(aes) for v5 devices in ccp
- Add Queue Interface (QI) backend support in caam
- Add new Exynos RNG driver
- Add ThunderX ZIP driver
- Add driver for hardware random generator on MT7623 SoC"
* 'linus' of git://git.kernel.org/pub/scm/linux/kernel/git/herbert/crypto-2.6: (101 commits)
crypto: stm32 - Fix OF module alias information
crypto: algif_aead - Require setkey before accept(2)
crypto: scomp - add support for deflate rfc1950 (zlib)
crypto: scomp - allow registration of multiple scomps
crypto: ccp - Change ISR handler method for a v5 CCP
crypto: ccp - Change ISR handler method for a v3 CCP
crypto: crypto4xx - rename ce_ring_contol to ce_ring_control
crypto: testmgr - Allow ecb(cipher_null) in FIPS mode
Revert "crypto: arm64/sha - Add constant operand modifier to ASM_EXPORT"
crypto: ccp - Disable interrupts early on unload
crypto: ccp - Use only the relevant interrupt bits
hwrng: mtk - Add driver for hardware random generator on MT7623 SoC
dt-bindings: hwrng: Add Mediatek hardware random generator bindings
crypto: crct10dif-vpmsum - Fix missing preempt_disable()
crypto: testmgr - replace compression known answer test
crypto: acomp - allow registration of multiple acomps
hwrng: n2 - Use devm_kcalloc() in n2rng_probe()
crypto: chcr - Fix error handling related to 'chcr_alloc_shash'
padata: get_next is never NULL
crypto: exynos - Add new Exynos RNG driver
...
Diffstat (limited to 'drivers/crypto/mediatek')
-rw-r--r-- | drivers/crypto/mediatek/mtk-aes.c | 421 | ||||
-rw-r--r-- | drivers/crypto/mediatek/mtk-platform.c | 15 | ||||
-rw-r--r-- | drivers/crypto/mediatek/mtk-platform.h | 56 | ||||
-rw-r--r-- | drivers/crypto/mediatek/mtk-sha.c | 309 |
4 files changed, 369 insertions, 432 deletions
diff --git a/drivers/crypto/mediatek/mtk-aes.c b/drivers/crypto/mediatek/mtk-aes.c index 3a47cdb..9e845e8 100644 --- a/drivers/crypto/mediatek/mtk-aes.c +++ b/drivers/crypto/mediatek/mtk-aes.c @@ -19,13 +19,10 @@ #define AES_BUF_ORDER 2 #define AES_BUF_SIZE ((PAGE_SIZE << AES_BUF_ORDER) \ & ~(AES_BLOCK_SIZE - 1)) +#define AES_MAX_STATE_BUF_SIZE SIZE_IN_WORDS(AES_KEYSIZE_256 + \ + AES_BLOCK_SIZE * 2) +#define AES_MAX_CT_SIZE 6 -/* AES command token size */ -#define AES_CT_SIZE_ECB 2 -#define AES_CT_SIZE_CBC 3 -#define AES_CT_SIZE_CTR 3 -#define AES_CT_SIZE_GCM_OUT 5 -#define AES_CT_SIZE_GCM_IN 6 #define AES_CT_CTRL_HDR cpu_to_le32(0x00220000) /* AES-CBC/ECB/CTR command token */ @@ -50,6 +47,8 @@ #define AES_TFM_128BITS cpu_to_le32(0xb << 16) #define AES_TFM_192BITS cpu_to_le32(0xd << 16) #define AES_TFM_256BITS cpu_to_le32(0xf << 16) +#define AES_TFM_GHASH_DIGEST cpu_to_le32(0x2 << 21) +#define AES_TFM_GHASH cpu_to_le32(0x4 << 23) /* AES transform information word 1 fields */ #define AES_TFM_ECB cpu_to_le32(0x0 << 0) #define AES_TFM_CBC cpu_to_le32(0x1 << 0) @@ -59,10 +58,9 @@ #define AES_TFM_FULL_IV cpu_to_le32(0xf << 5) /* using IV 0-3 */ #define AES_TFM_IV_CTR_MODE cpu_to_le32(0x1 << 10) #define AES_TFM_ENC_HASH cpu_to_le32(0x1 << 17) -#define AES_TFM_GHASH_DIG cpu_to_le32(0x2 << 21) -#define AES_TFM_GHASH cpu_to_le32(0x4 << 23) /* AES flags */ +#define AES_FLAGS_CIPHER_MSK GENMASK(2, 0) #define AES_FLAGS_ECB BIT(0) #define AES_FLAGS_CBC BIT(1) #define AES_FLAGS_CTR BIT(2) @@ -70,19 +68,15 @@ #define AES_FLAGS_ENCRYPT BIT(4) #define AES_FLAGS_BUSY BIT(5) +#define AES_AUTH_TAG_ERR cpu_to_le32(BIT(26)) + /** - * Command token(CT) is a set of hardware instructions that - * are used to control engine's processing flow of AES. - * - * Transform information(TFM) is used to define AES state and - * contains all keys and initial vectors. - * - * The engine requires CT and TFM to do: - * - Commands decoding and control of the engine's data path. - * - Coordinating hardware data fetch and store operations. - * - Result token construction and output. + * mtk_aes_info - hardware information of AES + * @cmd: command token, hardware instruction + * @tfm: transform state of cipher algorithm. + * @state: contains keys and initial vectors. * - * Memory map of GCM's TFM: + * Memory layout of GCM buffer: * /-----------\ * | AES KEY | 128/196/256 bits * |-----------| @@ -90,14 +84,16 @@ * |-----------| * | IVs | 4 * 4 bytes * \-----------/ + * + * The engine requires all these info to do: + * - Commands decoding and control of the engine's data path. + * - Coordinating hardware data fetch and store operations. + * - Result token construction and output. */ -struct mtk_aes_ct { - __le32 cmd[AES_CT_SIZE_GCM_IN]; -}; - -struct mtk_aes_tfm { - __le32 ctrl[2]; - __le32 state[SIZE_IN_WORDS(AES_KEYSIZE_256 + AES_BLOCK_SIZE * 2)]; +struct mtk_aes_info { + __le32 cmd[AES_MAX_CT_SIZE]; + __le32 tfm[2]; + __le32 state[AES_MAX_STATE_BUF_SIZE]; }; struct mtk_aes_reqctx { @@ -107,11 +103,12 @@ struct mtk_aes_reqctx { struct mtk_aes_base_ctx { struct mtk_cryp *cryp; u32 keylen; + __le32 keymode; + mtk_aes_fn start; - struct mtk_aes_ct ct; + struct mtk_aes_info info; dma_addr_t ct_dma; - struct mtk_aes_tfm tfm; dma_addr_t tfm_dma; __le32 ct_hdr; @@ -248,6 +245,33 @@ static inline void mtk_aes_restore_sg(const struct mtk_aes_dma *dma) sg->length += dma->remainder; } +static inline void mtk_aes_write_state_le(__le32 *dst, const u32 *src, u32 size) +{ + int i; + + for (i = 0; i < SIZE_IN_WORDS(size); i++) + dst[i] = cpu_to_le32(src[i]); +} + +static inline void mtk_aes_write_state_be(__be32 *dst, const u32 *src, u32 size) +{ + int i; + + for (i = 0; i < SIZE_IN_WORDS(size); i++) + dst[i] = cpu_to_be32(src[i]); +} + +static inline int mtk_aes_complete(struct mtk_cryp *cryp, + struct mtk_aes_rec *aes, + int err) +{ + aes->flags &= ~AES_FLAGS_BUSY; + aes->areq->complete(aes->areq, err); + /* Handle new request */ + tasklet_schedule(&aes->queue_task); + return err; +} + /* * Write descriptors for processing. This will configure the engine, load * the transform information and then start the packet processing. @@ -262,7 +286,7 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) /* Write command descriptors */ for (nents = 0; nents < slen; ++nents, ssg = sg_next(ssg)) { - cmd = ring->cmd_base + ring->cmd_pos; + cmd = ring->cmd_next; cmd->hdr = MTK_DESC_BUF_LEN(ssg->length); cmd->buf = cpu_to_le32(sg_dma_address(ssg)); @@ -274,25 +298,30 @@ static int mtk_aes_xmit(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) cmd->tfm = cpu_to_le32(aes->ctx->tfm_dma); } - if (++ring->cmd_pos == MTK_DESC_NUM) - ring->cmd_pos = 0; + /* Shift ring buffer and check boundary */ + if (++ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) + ring->cmd_next = ring->cmd_base; } cmd->hdr |= MTK_DESC_LAST; /* Prepare result descriptors */ for (nents = 0; nents < dlen; ++nents, dsg = sg_next(dsg)) { - res = ring->res_base + ring->res_pos; + res = ring->res_next; res->hdr = MTK_DESC_BUF_LEN(dsg->length); res->buf = cpu_to_le32(sg_dma_address(dsg)); if (nents == 0) res->hdr |= MTK_DESC_FIRST; - if (++ring->res_pos == MTK_DESC_NUM) - ring->res_pos = 0; + /* Shift ring buffer and check boundary */ + if (++ring->res_next == ring->res_base + MTK_DESC_NUM) + ring->res_next = ring->res_base; } res->hdr |= MTK_DESC_LAST; + /* Pointer to current result descriptor */ + ring->res_prev = res; + /* Prepare enough space for authenticated tag */ if (aes->flags & AES_FLAGS_GCM) res->hdr += AES_BLOCK_SIZE; @@ -313,9 +342,7 @@ static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) { struct mtk_aes_base_ctx *ctx = aes->ctx; - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct), - DMA_TO_DEVICE); - dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm), + dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->info), DMA_TO_DEVICE); if (aes->src.sg == aes->dst.sg) { @@ -346,16 +373,14 @@ static void mtk_aes_unmap(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) { struct mtk_aes_base_ctx *ctx = aes->ctx; + struct mtk_aes_info *info = &ctx->info; - ctx->ct_dma = dma_map_single(cryp->dev, &ctx->ct, sizeof(ctx->ct), + ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), DMA_TO_DEVICE); if (unlikely(dma_mapping_error(cryp->dev, ctx->ct_dma))) - return -EINVAL; + goto exit; - ctx->tfm_dma = dma_map_single(cryp->dev, &ctx->tfm, sizeof(ctx->tfm), - DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(cryp->dev, ctx->tfm_dma))) - goto tfm_map_err; + ctx->tfm_dma = ctx->ct_dma + sizeof(info->cmd); if (aes->src.sg == aes->dst.sg) { aes->src.sg_len = dma_map_sg(cryp->dev, aes->src.sg, @@ -382,13 +407,9 @@ static int mtk_aes_map(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) return mtk_aes_xmit(cryp, aes); sg_map_err: - dma_unmap_single(cryp->dev, ctx->tfm_dma, sizeof(ctx->tfm), - DMA_TO_DEVICE); -tfm_map_err: - dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(ctx->ct), - DMA_TO_DEVICE); - - return -EINVAL; + dma_unmap_single(cryp->dev, ctx->ct_dma, sizeof(*info), DMA_TO_DEVICE); +exit: + return mtk_aes_complete(cryp, aes, -EINVAL); } /* Initialize transform information of CBC/ECB/CTR mode */ @@ -397,50 +418,43 @@ static void mtk_aes_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, { struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq); struct mtk_aes_base_ctx *ctx = aes->ctx; + struct mtk_aes_info *info = &ctx->info; + u32 cnt = 0; ctx->ct_hdr = AES_CT_CTRL_HDR | cpu_to_le32(len); - ctx->ct.cmd[0] = AES_CMD0 | cpu_to_le32(len); - ctx->ct.cmd[1] = AES_CMD1; + info->cmd[cnt++] = AES_CMD0 | cpu_to_le32(len); + info->cmd[cnt++] = AES_CMD1; + info->tfm[0] = AES_TFM_SIZE(ctx->keylen) | ctx->keymode; if (aes->flags & AES_FLAGS_ENCRYPT) - ctx->tfm.ctrl[0] = AES_TFM_BASIC_OUT; + info->tfm[0] |= AES_TFM_BASIC_OUT; else - ctx->tfm.ctrl[0] = AES_TFM_BASIC_IN; + info->tfm[0] |= AES_TFM_BASIC_IN; - if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128)) - ctx->tfm.ctrl[0] |= AES_TFM_128BITS; - else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256)) - ctx->tfm.ctrl[0] |= AES_TFM_256BITS; - else - ctx->tfm.ctrl[0] |= AES_TFM_192BITS; - - if (aes->flags & AES_FLAGS_CBC) { - const u32 *iv = (const u32 *)req->info; - u32 *iv_state = ctx->tfm.state + ctx->keylen; - int i; - - ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen + - SIZE_IN_WORDS(AES_BLOCK_SIZE)); - ctx->tfm.ctrl[1] = AES_TFM_CBC | AES_TFM_FULL_IV; - - for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++) - iv_state[i] = cpu_to_le32(iv[i]); - - ctx->ct.cmd[2] = AES_CMD2; - ctx->ct_size = AES_CT_SIZE_CBC; - } else if (aes->flags & AES_FLAGS_ECB) { - ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen); - ctx->tfm.ctrl[1] = AES_TFM_ECB; - - ctx->ct_size = AES_CT_SIZE_ECB; - } else if (aes->flags & AES_FLAGS_CTR) { - ctx->tfm.ctrl[0] |= AES_TFM_SIZE(ctx->keylen + - SIZE_IN_WORDS(AES_BLOCK_SIZE)); - ctx->tfm.ctrl[1] = AES_TFM_CTR_LOAD | AES_TFM_FULL_IV; - - ctx->ct.cmd[2] = AES_CMD2; - ctx->ct_size = AES_CT_SIZE_CTR; + switch (aes->flags & AES_FLAGS_CIPHER_MSK) { + case AES_FLAGS_CBC: + info->tfm[1] = AES_TFM_CBC; + break; + case AES_FLAGS_ECB: + info->tfm[1] = AES_TFM_ECB; + goto ecb; + case AES_FLAGS_CTR: + info->tfm[1] = AES_TFM_CTR_LOAD; + goto ctr; + + default: + /* Should not happen... */ + return; } + + mtk_aes_write_state_le(info->state + ctx->keylen, req->info, + AES_BLOCK_SIZE); +ctr: + info->tfm[0] += AES_TFM_SIZE(SIZE_IN_WORDS(AES_BLOCK_SIZE)); + info->tfm[1] |= AES_TFM_FULL_IV; + info->cmd[cnt++] = AES_CMD2; +ecb: + ctx->ct_size = cnt; } static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, @@ -465,7 +479,7 @@ static int mtk_aes_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, padlen = mtk_aes_padlen(len); if (len + padlen > AES_BUF_SIZE) - return -ENOMEM; + return mtk_aes_complete(cryp, aes, -ENOMEM); if (!src_aligned) { sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); @@ -525,13 +539,10 @@ static int mtk_aes_handle_queue(struct mtk_cryp *cryp, u8 id, return ctx->start(cryp, aes); } -static int mtk_aes_complete(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) +static int mtk_aes_transfer_complete(struct mtk_cryp *cryp, + struct mtk_aes_rec *aes) { - aes->flags &= ~AES_FLAGS_BUSY; - aes->areq->complete(aes->areq, 0); - - /* Handle new request */ - return mtk_aes_handle_queue(cryp, aes->id, NULL); + return mtk_aes_complete(cryp, aes, 0); } static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) @@ -540,7 +551,7 @@ static int mtk_aes_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) struct mtk_aes_reqctx *rctx = ablkcipher_request_ctx(req); mtk_aes_set_mode(aes, rctx); - aes->resume = mtk_aes_complete; + aes->resume = mtk_aes_transfer_complete; return mtk_aes_dma(cryp, aes, req->src, req->dst, req->nbytes); } @@ -557,15 +568,14 @@ static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) struct mtk_aes_ctr_ctx *cctx = mtk_aes_ctr_ctx_cast(ctx); struct ablkcipher_request *req = ablkcipher_request_cast(aes->areq); struct scatterlist *src, *dst; - int i; - u32 start, end, ctr, blocks, *iv_state; + u32 start, end, ctr, blocks; size_t datalen; bool fragmented = false; /* Check for transfer completion. */ cctx->offset += aes->total; if (cctx->offset >= req->nbytes) - return mtk_aes_complete(cryp, aes); + return mtk_aes_transfer_complete(cryp, aes); /* Compute data length. */ datalen = req->nbytes - cctx->offset; @@ -587,9 +597,8 @@ static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) scatterwalk_ffwd(cctx->dst, req->dst, cctx->offset)); /* Write IVs into transform state buffer. */ - iv_state = ctx->tfm.state + ctx->keylen; - for (i = 0; i < SIZE_IN_WORDS(AES_BLOCK_SIZE); i++) - iv_state[i] = cpu_to_le32(cctx->iv[i]); + mtk_aes_write_state_le(ctx->info.state + ctx->keylen, cctx->iv, + AES_BLOCK_SIZE); if (unlikely(fragmented)) { /* @@ -599,7 +608,6 @@ static int mtk_aes_ctr_transfer(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) cctx->iv[3] = cpu_to_be32(ctr); crypto_inc((u8 *)cctx->iv, AES_BLOCK_SIZE); } - aes->resume = mtk_aes_ctr_transfer; return mtk_aes_dma(cryp, aes, src, dst, datalen); } @@ -615,6 +623,7 @@ static int mtk_aes_ctr_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) memcpy(cctx->iv, req->info, AES_BLOCK_SIZE); cctx->offset = 0; aes->total = 0; + aes->resume = mtk_aes_ctr_transfer; return mtk_aes_ctr_transfer(cryp, aes); } @@ -624,21 +633,25 @@ static int mtk_aes_setkey(struct crypto_ablkcipher *tfm, const u8 *key, u32 keylen) { struct mtk_aes_base_ctx *ctx = crypto_ablkcipher_ctx(tfm); - const u32 *aes_key = (const u32 *)key; - u32 *key_state = ctx->tfm.state; - int i; - if (keylen != AES_KEYSIZE_128 && - keylen != AES_KEYSIZE_192 && - keylen != AES_KEYSIZE_256) { + switch (keylen) { + case AES_KEYSIZE_128: + ctx->keymode = AES_TFM_128BITS; + break; + case AES_KEYSIZE_192: + ctx->keymode = AES_TFM_192BITS; + break; + case AES_KEYSIZE_256: + ctx->keymode = AES_TFM_256BITS; + break; + + default: crypto_ablkcipher_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } ctx->keylen = SIZE_IN_WORDS(keylen); - - for (i = 0; i < ctx->keylen; i++) - key_state[i] = cpu_to_le32(aes_key[i]); + mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen); return 0; } @@ -789,6 +802,19 @@ mtk_aes_gcm_ctx_cast(struct mtk_aes_base_ctx *ctx) return container_of(ctx, struct mtk_aes_gcm_ctx, base); } +/* + * Engine will verify and compare tag automatically, so we just need + * to check returned status which stored in the result descriptor. + */ +static int mtk_aes_gcm_tag_verify(struct mtk_cryp *cryp, + struct mtk_aes_rec *aes) +{ + u32 status = cryp->ring[aes->id]->res_prev->ct; + + return mtk_aes_complete(cryp, aes, (status & AES_AUTH_TAG_ERR) ? + -EBADMSG : 0); +} + /* Initialize transform information of GCM mode */ static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, @@ -797,45 +823,35 @@ static void mtk_aes_gcm_info_init(struct mtk_cryp *cryp, struct aead_request *req = aead_request_cast(aes->areq); struct mtk_aes_base_ctx *ctx = aes->ctx; struct mtk_aes_gcm_ctx *gctx = mtk_aes_gcm_ctx_cast(ctx); - const u32 *iv = (const u32 *)req->iv; - u32 *iv_state = ctx->tfm.state + ctx->keylen + - SIZE_IN_WORDS(AES_BLOCK_SIZE); + struct mtk_aes_info *info = &ctx->info; u32 ivsize = crypto_aead_ivsize(crypto_aead_reqtfm(req)); - int i; + u32 cnt = 0; ctx->ct_hdr = AES_CT_CTRL_HDR | len; - ctx->ct.cmd[0] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen); - ctx->ct.cmd[1] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen); - ctx->ct.cmd[2] = AES_GCM_CMD2; - ctx->ct.cmd[3] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen); + info->cmd[cnt++] = AES_GCM_CMD0 | cpu_to_le32(req->assoclen); + info->cmd[cnt++] = AES_GCM_CMD1 | cpu_to_le32(req->assoclen); + info->cmd[cnt++] = AES_GCM_CMD2; + info->cmd[cnt++] = AES_GCM_CMD3 | cpu_to_le32(gctx->textlen); if (aes->flags & AES_FLAGS_ENCRYPT) { - ctx->ct.cmd[4] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize); - ctx->ct_size = AES_CT_SIZE_GCM_OUT; - ctx->tfm.ctrl[0] = AES_TFM_GCM_OUT; + info->cmd[cnt++] = AES_GCM_CMD4 | cpu_to_le32(gctx->authsize); + info->tfm[0] = AES_TFM_GCM_OUT; } else { - ctx->ct.cmd[4] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize); - ctx->ct.cmd[5] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize); - ctx->ct_size = AES_CT_SIZE_GCM_IN; - ctx->tfm.ctrl[0] = AES_TFM_GCM_IN; + info->cmd[cnt++] = AES_GCM_CMD5 | cpu_to_le32(gctx->authsize); + info->cmd[cnt++] = AES_GCM_CMD6 | cpu_to_le32(gctx->authsize); + info->tfm[0] = AES_TFM_GCM_IN; } + ctx->ct_size = cnt; - if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_128)) - ctx->tfm.ctrl[0] |= AES_TFM_128BITS; - else if (ctx->keylen == SIZE_IN_WORDS(AES_KEYSIZE_256)) - ctx->tfm.ctrl[0] |= AES_TFM_256BITS; - else - ctx->tfm.ctrl[0] |= AES_TFM_192BITS; + info->tfm[0] |= AES_TFM_GHASH_DIGEST | AES_TFM_GHASH | AES_TFM_SIZE( + ctx->keylen + SIZE_IN_WORDS(AES_BLOCK_SIZE + ivsize)) | + ctx->keymode; + info->tfm[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | AES_TFM_3IV | + AES_TFM_ENC_HASH; - ctx->tfm.ctrl[0] |= AES_TFM_GHASH_DIG | AES_TFM_GHASH | - AES_TFM_SIZE(ctx->keylen + SIZE_IN_WORDS( - AES_BLOCK_SIZE + ivsize)); - ctx->tfm.ctrl[1] = AES_TFM_CTR_INIT | AES_TFM_IV_CTR_MODE | - AES_TFM_3IV | AES_TFM_ENC_HASH; - - for (i = 0; i < SIZE_IN_WORDS(ivsize); i++) - iv_state[i] = cpu_to_le32(iv[i]); + mtk_aes_write_state_le(info->state + ctx->keylen + SIZE_IN_WORDS( + AES_BLOCK_SIZE), (const u32 *)req->iv, ivsize); } static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, @@ -856,7 +872,7 @@ static int mtk_aes_gcm_dma(struct mtk_cryp *cryp, struct mtk_aes_rec *aes, if (!src_aligned || !dst_aligned) { if (aes->total > AES_BUF_SIZE) - return -ENOMEM; + return mtk_aes_complete(cryp, aes, -ENOMEM); if (!src_aligned) { sg_copy_to_buffer(src, sg_nents(src), aes->buf, len); @@ -892,6 +908,8 @@ static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) if (aes->flags & AES_FLAGS_ENCRYPT) { u32 tag[4]; + + aes->resume = mtk_aes_transfer_complete; /* Compute total process length. */ aes->total = len + gctx->authsize; /* Compute text length. */ @@ -899,10 +917,10 @@ static int mtk_aes_gcm_start(struct mtk_cryp *cryp, struct mtk_aes_rec *aes) /* Hardware will append authenticated tag to output buffer */ scatterwalk_map_and_copy(tag, req->dst, len, gctx->authsize, 1); } else { + aes->resume = mtk_aes_gcm_tag_verify; aes->total = len; gctx->textlen = req->cryptlen - gctx->authsize; } - aes->resume = mtk_aes_complete; return mtk_aes_gcm_dma(cryp, aes, req->src, req->dst, len); } @@ -915,7 +933,7 @@ static int mtk_aes_gcm_crypt(struct aead_request *req, u64 mode) rctx->mode = AES_FLAGS_GCM | mode; return mtk_aes_handle_queue(ctx->cryp, !!(mode & AES_FLAGS_ENCRYPT), - &req->base); + &req->base); } static void mtk_gcm_setkey_done(struct crypto_async_request *req, int err) @@ -949,24 +967,26 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, struct scatterlist sg[1]; struct skcipher_request req; } *data; - const u32 *aes_key; - u32 *key_state, *hash_state; - int err, i; + int err; - if (keylen != AES_KEYSIZE_256 && - keylen != AES_KEYSIZE_192 && - keylen != AES_KEYSIZE_128) { + switch (keylen) { + case AES_KEYSIZE_128: + ctx->keymode = AES_TFM_128BITS; + break; + case AES_KEYSIZE_192: + ctx->keymode = AES_TFM_192BITS; + break; + case AES_KEYSIZE_256: + ctx->keymode = AES_TFM_256BITS; + break; + + default: crypto_aead_set_flags(aead, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } - key_state = ctx->tfm.state; - aes_key = (u32 *)key; ctx->keylen = SIZE_IN_WORDS(keylen); - for (i = 0; i < ctx->keylen; i++) - ctx->tfm.state[i] = cpu_to_le32(aes_key[i]); - /* Same as crypto_gcm_setkey() from crypto/gcm.c */ crypto_skcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK); crypto_skcipher_set_flags(ctr, crypto_aead_get_flags(aead) & @@ -1001,10 +1021,11 @@ static int mtk_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, if (err) goto out; - hash_state = key_state + ctx->keylen; - - for (i = 0; i < 4; i++) - hash_state[i] = cpu_to_be32(data->hash[i]); + /* Write key into state buffer */ + mtk_aes_write_state_le(ctx->info.state, (const u32 *)key, keylen); + /* Write key(H) into state buffer */ + mtk_aes_write_state_be(ctx->info.state + ctx->keylen, data->hash, + AES_BLOCK_SIZE); out: kzfree(data); return err; @@ -1092,58 +1113,36 @@ static struct aead_alg aes_gcm_alg = { }, }; -static void mtk_aes_enc_task(unsigned long data) +static void mtk_aes_queue_task(unsigned long data) { - struct mtk_cryp *cryp = (struct mtk_cryp *)data; - struct mtk_aes_rec *aes = cryp->aes[0]; + struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; - mtk_aes_unmap(cryp, aes); - aes->resume(cryp, aes); + mtk_aes_handle_queue(aes->cryp, aes->id, NULL); } -static void mtk_aes_dec_task(unsigned long data) +static void mtk_aes_done_task(unsigned long data) { - struct mtk_cryp *cryp = (struct mtk_cryp *)data; - struct mtk_aes_rec *aes = cryp->aes[1]; + struct mtk_aes_rec *aes = (struct mtk_aes_rec *)data; + struct mtk_cryp *cryp = aes->cryp; mtk_aes_unmap(cryp, aes); aes->resume(cryp, aes); } -static irqreturn_t mtk_aes_enc_irq(int irq, void *dev_id) +static irqreturn_t mtk_aes_irq(int irq, void *dev_id) { - struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id; - struct mtk_aes_rec *aes = cryp->aes[0]; - u32 val = mtk_aes_read(cryp, RDR_STAT(RING0)); + struct mtk_aes_rec *aes = (struct mtk_aes_rec *)dev_id; + struct mtk_cryp *cryp = aes->cryp; + u32 val = mtk_aes_read(cryp, RDR_STAT(aes->id)); - mtk_aes_write(cryp, RDR_STAT(RING0), val); + mtk_aes_write(cryp, RDR_STAT(aes->id), val); if (likely(AES_FLAGS_BUSY & aes->flags)) { - mtk_aes_write(cryp, RDR_PROC_COUNT(RING0), MTK_CNT_RST); - mtk_aes_write(cryp, RDR_THRESH(RING0), + mtk_aes_write(cryp, RDR_PROC_COUNT(aes->id), MTK_CNT_RST); + mtk_aes_write(cryp, RDR_THRESH(aes->id), MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - tasklet_schedule(&aes->task); - } else { - dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); - } - return IRQ_HANDLED; -} - -static irqreturn_t mtk_aes_dec_irq(int irq, void *dev_id) -{ - struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id; - struct mtk_aes_rec *aes = cryp->aes[1]; - u32 val = mtk_aes_read(cryp, RDR_STAT(RING1)); - - mtk_aes_write(cryp, RDR_STAT(RING1), val); - - if (likely(AES_FLAGS_BUSY & aes->flags)) { - mtk_aes_write(cryp, RDR_PROC_COUNT(RING1), MTK_CNT_RST); - mtk_aes_write(cryp, RDR_THRESH(RING1), - MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - - tasklet_schedule(&aes->task); + tasklet_schedule(&aes->done_task); } else { dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); } @@ -1171,14 +1170,20 @@ static int mtk_aes_record_init(struct mtk_cryp *cryp) if (!aes[i]->buf) goto err_cleanup; - aes[i]->id = i; + aes[i]->cryp = cryp; spin_lock_init(&aes[i]->lock); crypto_init_queue(&aes[i]->queue, AES_QUEUE_SIZE); + + tasklet_init(&aes[i]->queue_task, mtk_aes_queue_task, + (unsigned long)aes[i]); + tasklet_init(&aes[i]->done_task, mtk_aes_done_task, + (unsigned long)aes[i]); } - tasklet_init(&aes[0]->task, mtk_aes_enc_task, (unsigned long)cryp); - tasklet_init(&aes[1]->task, mtk_aes_dec_task, (unsigned long)cryp); + /* Link to ring0 and ring1 respectively */ + aes[0]->id = MTK_RING0; + aes[1]->id = MTK_RING1; return 0; @@ -1196,7 +1201,9 @@ static void mtk_aes_record_free(struct mtk_cryp *cryp) int i; for (i = 0; i < MTK_REC_NUM; i++) { - tasklet_kill(&cryp->aes[i]->task); + tasklet_kill(&cryp->aes[i]->done_task); + tasklet_kill(&cryp->aes[i]->queue_task); + free_page((unsigned long)cryp->aes[i]->buf); kfree(cryp->aes[i]); } @@ -1246,25 +1253,23 @@ int mtk_cipher_alg_register(struct mtk_cryp *cryp) if (ret) goto err_record; - /* Ring0 is use by encryption record */ - ret = devm_request_irq(cryp->dev, cryp->irq[RING0], mtk_aes_enc_irq, - IRQF_TRIGGER_LOW, "mtk-aes", cryp); + ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING0], mtk_aes_irq, + 0, "mtk-aes", cryp->aes[0]); if (ret) { - dev_err(cryp->dev, "unable to request AES encryption irq.\n"); + dev_err(cryp->dev, "unable to request AES irq.\n"); goto err_res; } - /* Ring1 is use by decryption record */ - ret = devm_request_irq(cryp->dev, cryp->irq[RING1], mtk_aes_dec_irq, - IRQF_TRIGGER_LOW, "mtk-aes", cryp); + ret = devm_request_irq(cryp->dev, cryp->irq[MTK_RING1], mtk_aes_irq, + 0, "mtk-aes", cryp->aes[1]); if (ret) { - dev_err(cryp->dev, "unable to request AES decryption irq.\n"); + dev_err(cryp->dev, "unable to request AES irq.\n"); goto err_res; } /* Enable ring0 and ring1 interrupt */ - mtk_aes_write(cryp, AIC_ENABLE_SET(RING0), MTK_IRQ_RDR0); - mtk_aes_write(cryp, AIC_ENABLE_SET(RING1), MTK_IRQ_RDR1); + mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING0), MTK_IRQ_RDR0); + mtk_aes_write(cryp, AIC_ENABLE_SET(MTK_RING1), MTK_IRQ_RDR1); spin_lock(&mtk_aes.lock); list_add_tail(&cryp->aes_list, &mtk_aes.dev_list); diff --git a/drivers/crypto/mediatek/mtk-platform.c b/drivers/crypto/mediatek/mtk-platform.c index a9c713d..b6ecc28 100644 --- a/drivers/crypto/mediatek/mtk-platform.c +++ b/drivers/crypto/mediatek/mtk-platform.c @@ -334,7 +334,7 @@ static int mtk_packet_engine_setup(struct mtk_cryp *cryp) /* Enable the 4 rings for the packet engines. */ mtk_desc_ring_link(cryp, 0xf); - for (i = 0; i < RING_MAX; i++) { + for (i = 0; i < MTK_RING_MAX; i++) { mtk_cmd_desc_ring_setup(cryp, i, &cap); mtk_res_desc_ring_setup(cryp, i, &cap); } @@ -359,7 +359,7 @@ static int mtk_aic_cap_check(struct mtk_cryp *cryp, int hw) { u32 val; - if (hw == RING_MAX) + if (hw == MTK_RING_MAX) val = readl(cryp->base + AIC_G_VERSION); else val = readl(cryp->base + AIC_VERSION(hw)); @@ -368,7 +368,7 @@ static int mtk_aic_cap_check(struct mtk_cryp *cryp, int hw) if (val != MTK_AIC_VER11 && val != MTK_AIC_VER12) return -ENXIO; - if (hw == RING_MAX) + if (hw == MTK_RING_MAX) val = readl(cryp->base + AIC_G_OPTIONS); else val = readl(cryp->base + AIC_OPTIONS(hw)); @@ -389,7 +389,7 @@ static int mtk_aic_init(struct mtk_cryp *cryp, int hw) return err; /* Disable all interrupts and set initial configuration */ - if (hw == RING_MAX) { + if (hw == MTK_RING_MAX) { writel(0, cryp->base + AIC_G_ENABLE_CTRL); writel(0, cryp->base + AIC_G_POL_CTRL); writel(0, cryp->base + AIC_G_TYPE_CTRL); @@ -431,7 +431,7 @@ static void mtk_desc_dma_free(struct mtk_cryp *cryp) { int i; - for (i = 0; i < RING_MAX; i++) { + for (i = 0; i < MTK_RING_MAX; i++) { dma_free_coherent(cryp->dev, MTK_DESC_RING_SZ, cryp->ring[i]->res_base, cryp->ring[i]->res_dma); @@ -447,7 +447,7 @@ static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) struct mtk_ring **ring = cryp->ring; int i, err = ENOMEM; - for (i = 0; i < RING_MAX; i++) { + for (i = 0; i < MTK_RING_MAX; i++) { ring[i] = kzalloc(sizeof(**ring), GFP_KERNEL); if (!ring[i]) goto err_cleanup; @@ -465,6 +465,9 @@ static int mtk_desc_ring_alloc(struct mtk_cryp *cryp) GFP_KERNEL); if (!ring[i]->res_base) goto err_cleanup; + + ring[i]->cmd_next = ring[i]->cmd_base; + ring[i]->res_next = ring[i]->res_base; } return 0; diff --git a/drivers/crypto/mediatek/mtk-platform.h b/drivers/crypto/mediatek/mtk-platform.h index ed6d871..303c152 100644 --- a/drivers/crypto/mediatek/mtk-platform.h +++ b/drivers/crypto/mediatek/mtk-platform.h @@ -38,14 +38,14 @@ * Ring 2/3 are used by SHA. */ enum { - RING0 = 0, - RING1, - RING2, - RING3, - RING_MAX, + MTK_RING0, + MTK_RING1, + MTK_RING2, + MTK_RING3, + MTK_RING_MAX }; -#define MTK_REC_NUM (RING_MAX / 2) +#define MTK_REC_NUM (MTK_RING_MAX / 2) #define MTK_IRQ_NUM 5 /** @@ -84,11 +84,12 @@ struct mtk_desc { /** * struct mtk_ring - Descriptor ring * @cmd_base: pointer to command descriptor ring base + * @cmd_next: pointer to the next command descriptor * @cmd_dma: DMA address of command descriptor ring - * @cmd_pos: current position in the command descriptor ring * @res_base: pointer to result descriptor ring base + * @res_next: pointer to the next result descriptor + * @res_prev: pointer to the previous result descriptor * @res_dma: DMA address of result descriptor ring - * @res_pos: current position in the result descriptor ring * * A descriptor ring is a circular buffer that is used to manage * one or more descriptors. There are two type of descriptor rings; @@ -96,11 +97,12 @@ struct mtk_desc { */ struct mtk_ring { struct mtk_desc *cmd_base; + struct mtk_desc *cmd_next; dma_addr_t cmd_dma; - u32 cmd_pos; struct mtk_desc *res_base; + struct mtk_desc *res_next; + struct mtk_desc *res_prev; dma_addr_t res_dma; - u32 res_pos; }; /** @@ -125,9 +127,11 @@ typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes); /** * struct mtk_aes_rec - AES operation record + * @cryp: pointer to Cryptographic device * @queue: crypto request queue * @areq: pointer to async request - * @task: the tasklet is use in AES interrupt + * @done_task: the tasklet is use in AES interrupt + * @queue_task: the tasklet is used to dequeue request * @ctx: pointer to current context * @src: the structure that holds source sg list info * @dst: the structure that holds destination sg list info @@ -136,16 +140,18 @@ typedef int (*mtk_aes_fn)(struct mtk_cryp *cryp, struct mtk_aes_rec *aes); * @resume: pointer to resume function * @total: request buffer length * @buf: pointer to page buffer - * @id: record identification + * @id: the current use of ring * @flags: it's describing AES operation state * @lock: the async queue lock * * Structure used to record AES execution state. */ struct mtk_aes_rec { + struct mtk_cryp *cryp; struct crypto_queue queue; struct crypto_async_request *areq; - struct tasklet_struct task; + struct tasklet_struct done_task; + struct tasklet_struct queue_task; struct mtk_aes_base_ctx *ctx; struct mtk_aes_dma src; struct mtk_aes_dma dst; @@ -166,19 +172,23 @@ struct mtk_aes_rec { /** * struct mtk_sha_rec - SHA operation record + * @cryp: pointer to Cryptographic device * @queue: crypto request queue * @req: pointer to ahash request - * @task: the tasklet is use in SHA interrupt - * @id: record identification + * @done_task: the tasklet is use in SHA interrupt + * @queue_task: the tasklet is used to dequeue request + * @id: the current use of ring * @flags: it's describing SHA operation state - * @lock: the ablkcipher queue lock + * @lock: the async queue lock * * Structure used to record SHA execution state. */ struct mtk_sha_rec { + struct mtk_cryp *cryp; struct crypto_queue queue; struct ahash_request *req; - struct tasklet_struct task; + struct tasklet_struct done_task; + struct tasklet_struct queue_task; u8 id; unsigned long flags; @@ -193,13 +203,11 @@ struct mtk_sha_rec { * @clk_ethif: pointer to ethif clock * @clk_cryp: pointer to crypto clock * @irq: global system and rings IRQ - * @ring: pointer to execution state of AES - * @aes: pointer to execution state of SHA - * @sha: each execution record map to a ring + * @ring: pointer to descriptor rings + * @aes: pointer to operation record of AES + * @sha: pointer to operation record of SHA * @aes_list: device list of AES * @sha_list: device list of SHA - * @tmp: pointer to temporary buffer for internal use - * @tmp_dma: DMA address of temporary buffer * @rec: it's used to select SHA record for tfm * * Structure storing cryptographic device information. @@ -211,15 +219,13 @@ struct mtk_cryp { struct clk *clk_cryp; int irq[MTK_IRQ_NUM]; - struct mtk_ring *ring[RING_MAX]; + struct mtk_ring *ring[MTK_RING_MAX]; struct mtk_aes_rec *aes[MTK_REC_NUM]; struct mtk_sha_rec *sha[MTK_REC_NUM]; struct list_head aes_list; struct list_head sha_list; - void *tmp; - dma_addr_t tmp_dma; bool rec; }; diff --git a/drivers/crypto/mediatek/mtk-sha.c b/drivers/crypto/mediatek/mtk-sha.c index 55e3805..2226f12 100644 --- a/drivers/crypto/mediatek/mtk-sha.c +++ b/drivers/crypto/mediatek/mtk-sha.c @@ -17,13 +17,13 @@ #define SHA_ALIGN_MSK (sizeof(u32) - 1) #define SHA_QUEUE_SIZE 512 -#define SHA_TMP_BUF_SIZE 512 #define SHA_BUF_SIZE ((u32)PAGE_SIZE) #define SHA_OP_UPDATE 1 #define SHA_OP_FINAL 2 #define SHA_DATA_LEN_MSK cpu_to_le32(GENMASK(16, 0)) +#define SHA_MAX_DIGEST_BUF_SIZE 32 /* SHA command token */ #define SHA_CT_SIZE 5 @@ -34,7 +34,6 @@ /* SHA transform information */ #define SHA_TFM_HASH cpu_to_le32(0x2 << 0) -#define SHA_TFM_INNER_DIG cpu_to_le32(0x1 << 21) #define SHA_TFM_SIZE(x) cpu_to_le32((x) << 8) #define SHA_TFM_START cpu_to_le32(0x1 << 4) #define SHA_TFM_CONTINUE cpu_to_le32(0x1 << 5) @@ -61,31 +60,17 @@ #define SHA_FLAGS_PAD BIT(10) /** - * mtk_sha_ct is a set of hardware instructions(command token) - * that are used to control engine's processing flow of SHA, - * and it contains the first two words of transform state. + * mtk_sha_info - hardware information of AES + * @cmd: command token, hardware instruction + * @tfm: transform state of cipher algorithm. + * @state: contains keys and initial vectors. + * */ -struct mtk_sha_ct { +struct mtk_sha_info { __le32 ctrl[2]; __le32 cmd[3]; -}; - -/** - * mtk_sha_tfm is used to define SHA transform state - * and store result digest that produced by engine. - */ -struct mtk_sha_tfm { - __le32 ctrl[2]; - __le32 digest[SIZE_IN_WORDS(SHA512_DIGEST_SIZE)]; -}; - -/** - * mtk_sha_info consists of command token and transform state - * of SHA, its role is similar to mtk_aes_info. - */ -struct mtk_sha_info { - struct mtk_sha_ct ct; - struct mtk_sha_tfm tfm; + __le32 tfm[2]; + __le32 digest[SHA_MAX_DIGEST_BUF_SIZE]; }; struct mtk_sha_reqctx { @@ -94,7 +79,6 @@ struct mtk_sha_reqctx { unsigned long op; u64 digcnt; - bool start; size_t bufcnt; dma_addr_t dma_addr; @@ -153,6 +137,21 @@ static inline void mtk_sha_write(struct mtk_cryp *cryp, writel_relaxed(value, cryp->base + offset); } +static inline void mtk_sha_ring_shift(struct mtk_ring *ring, + struct mtk_desc **cmd_curr, + struct mtk_desc **res_curr, + int *count) +{ + *cmd_curr = ring->cmd_next++; + *res_curr = ring->res_next++; + (*count)++; + + if (ring->cmd_next == ring->cmd_base + MTK_DESC_NUM) { + ring->cmd_next = ring->cmd_base; + ring->res_next = ring->res_base; + } +} + static struct mtk_cryp *mtk_sha_find_dev(struct mtk_sha_ctx *tctx) { struct mtk_cryp *cryp = NULL; @@ -251,7 +250,9 @@ static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) bits[1] = cpu_to_be64(size << 3); bits[0] = cpu_to_be64(size >> 61); - if (ctx->flags & (SHA_FLAGS_SHA384 | SHA_FLAGS_SHA512)) { + switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { + case SHA_FLAGS_SHA384: + case SHA_FLAGS_SHA512: index = ctx->bufcnt & 0x7f; padlen = (index < 112) ? (112 - index) : ((128 + 112) - index); *(ctx->buffer + ctx->bufcnt) = 0x80; @@ -259,7 +260,9 @@ static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) memcpy(ctx->buffer + ctx->bufcnt + padlen, bits, 16); ctx->bufcnt += padlen + 16; ctx->flags |= SHA_FLAGS_PAD; - } else { + break; + + default: index = ctx->bufcnt & 0x3f; padlen = (index < 56) ? (56 - index) : ((64 + 56) - index); *(ctx->buffer + ctx->bufcnt) = 0x80; @@ -267,36 +270,35 @@ static void mtk_sha_fill_padding(struct mtk_sha_reqctx *ctx, u32 len) memcpy(ctx->buffer + ctx->bufcnt + padlen, &bits[1], 8); ctx->bufcnt += padlen + 8; ctx->flags |= SHA_FLAGS_PAD; + break; } } /* Initialize basic transform information of SHA */ static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx) { - struct mtk_sha_ct *ct = &ctx->info.ct; - struct mtk_sha_tfm *tfm = &ctx->info.tfm; + struct mtk_sha_info *info = &ctx->info; ctx->ct_hdr = SHA_CT_CTRL_HDR; ctx->ct_size = SHA_CT_SIZE; - tfm->ctrl[0] = SHA_TFM_HASH | SHA_TFM_INNER_DIG | - SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds)); + info->tfm[0] = SHA_TFM_HASH | SHA_TFM_SIZE(SIZE_IN_WORDS(ctx->ds)); switch (ctx->flags & SHA_FLAGS_ALGO_MSK) { case SHA_FLAGS_SHA1: - tfm->ctrl[0] |= SHA_TFM_SHA1; + info->tfm[0] |= SHA_TFM_SHA1; break; case SHA_FLAGS_SHA224: - tfm->ctrl[0] |= SHA_TFM_SHA224; + info->tfm[0] |= SHA_TFM_SHA224; break; case SHA_FLAGS_SHA256: - tfm->ctrl[0] |= SHA_TFM_SHA256; + info->tfm[0] |= SHA_TFM_SHA256; break; case SHA_FLAGS_SHA384: - tfm->ctrl[0] |= SHA_TFM_SHA384; + info->tfm[0] |= SHA_TFM_SHA384; break; case SHA_FLAGS_SHA512: - tfm->ctrl[0] |= SHA_TFM_SHA512; + info->tfm[0] |= SHA_TFM_SHA512; break; default: @@ -304,13 +306,13 @@ static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx) return; } - tfm->ctrl[1] = SHA_TFM_HASH_STORE; - ct->ctrl[0] = tfm->ctrl[0] | SHA_TFM_CONTINUE | SHA_TFM_START; - ct->ctrl[1] = tfm->ctrl[1]; + info->tfm[1] = SHA_TFM_HASH_STORE; + info->ctrl[0] = info->tfm[0] | SHA_TFM_CONTINUE | SHA_TFM_START; + info->ctrl[1] = info->tfm[1]; - ct->cmd[0] = SHA_CMD0; - ct->cmd[1] = SHA_CMD1; - ct->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds)); + info->cmd[0] = SHA_CMD0; + info->cmd[1] = SHA_CMD1; + info->cmd[2] = SHA_CMD2 | SHA_TFM_DIGEST(SIZE_IN_WORDS(ctx->ds)); } /* @@ -319,23 +321,21 @@ static void mtk_sha_info_init(struct mtk_sha_reqctx *ctx) */ static int mtk_sha_info_update(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, - size_t len) + size_t len1, size_t len2) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); struct mtk_sha_info *info = &ctx->info; - struct mtk_sha_ct *ct = &info->ct; - - if (ctx->start) - ctx->start = false; - else - ct->ctrl[0] &= ~SHA_TFM_START; ctx->ct_hdr &= ~SHA_DATA_LEN_MSK; - ctx->ct_hdr |= cpu_to_le32(len); - ct->cmd[0] &= ~SHA_DATA_LEN_MSK; - ct->cmd[0] |= cpu_to_le32(len); + ctx->ct_hdr |= cpu_to_le32(len1 + len2); + info->cmd[0] &= ~SHA_DATA_LEN_MSK; + info->cmd[0] |= cpu_to_le32(len1 + len2); + + /* Setting SHA_TFM_START only for the first iteration */ + if (ctx->digcnt) + info->ctrl[0] &= ~SHA_TFM_START; - ctx->digcnt += len; + ctx->digcnt += len1; ctx->ct_dma = dma_map_single(cryp->dev, info, sizeof(*info), DMA_BIDIRECTIONAL); @@ -343,7 +343,8 @@ static int mtk_sha_info_update(struct mtk_cryp *cryp, dev_err(cryp->dev, "dma %zu bytes error\n", sizeof(*info)); return -EINVAL; } - ctx->tfm_dma = ctx->ct_dma + sizeof(*ct); + + ctx->tfm_dma = ctx->ct_dma + sizeof(info->ctrl) + sizeof(info->cmd); return 0; } @@ -408,7 +409,6 @@ static int mtk_sha_init(struct ahash_request *req) ctx->bufcnt = 0; ctx->digcnt = 0; ctx->buffer = tctx->buf; - ctx->start = true; if (tctx->flags & SHA_FLAGS_HMAC) { struct mtk_sha_hmac_ctx *bctx = tctx->base; @@ -422,89 +422,39 @@ static int mtk_sha_init(struct ahash_request *req) } static int mtk_sha_xmit(struct mtk_cryp *cryp, struct mtk_sha_rec *sha, - dma_addr_t addr, size_t len) + dma_addr_t addr1, size_t len1, + dma_addr_t addr2, size_t len2) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(sha->req); struct mtk_ring *ring = cryp->ring[sha->id]; - struct mtk_desc *cmd = ring->cmd_base + ring->cmd_pos; - struct mtk_desc *res = ring->res_base + ring->res_pos; - int err; - - err = mtk_sha_info_update(cryp, sha, len); - if (err) - return err; - - /* Fill in the command/result descriptors */ - res->hdr = MTK_DESC_FIRST | MTK_DESC_LAST | MTK_DESC_BUF_LEN(len); - res->buf = cpu_to_le32(cryp->tmp_dma); - - cmd->hdr = MTK_DESC_FIRST | MTK_DESC_LAST | MTK_DESC_BUF_LEN(len) | - MTK_DESC_CT_LEN(ctx->ct_size); - - cmd->buf = cpu_to_le32(addr); - cmd->ct = cpu_to_le32(ctx->ct_dma); - cmd->ct_hdr = ctx->ct_hdr; - cmd->tfm = cpu_to_le32(ctx->tfm_dma); - - if (++ring->cmd_pos == MTK_DESC_NUM) - ring->cmd_pos = 0; - - ring->res_pos = ring->cmd_pos; - /* - * Make sure that all changes to the DMA ring are done before we - * start engine. - */ - wmb(); - /* Start DMA transfer */ - mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(1)); - mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(1)); - - return -EINPROGRESS; -} - -static int mtk_sha_xmit2(struct mtk_cryp *cryp, - struct mtk_sha_rec *sha, - struct mtk_sha_reqctx *ctx, - size_t len1, size_t len2) -{ - struct mtk_ring *ring = cryp->ring[sha->id]; - struct mtk_desc *cmd = ring->cmd_base + ring->cmd_pos; - struct mtk_desc *res = ring->res_base + ring->res_pos; - int err; + struct mtk_desc *cmd, *res; + int err, count = 0; - err = mtk_sha_info_update(cryp, sha, len1 + len2); + err = mtk_sha_info_update(cryp, sha, len1, len2); if (err) return err; /* Fill in the command/result descriptors */ - res->hdr = MTK_DESC_BUF_LEN(len1) | MTK_DESC_FIRST; - res->buf = cpu_to_le32(cryp->tmp_dma); + mtk_sha_ring_shift(ring, &cmd, &res, &count); - cmd->hdr = MTK_DESC_BUF_LEN(len1) | MTK_DESC_FIRST | + res->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1); + cmd->hdr = MTK_DESC_FIRST | MTK_DESC_BUF_LEN(len1) | MTK_DESC_CT_LEN(ctx->ct_size); - cmd->buf = cpu_to_le32(sg_dma_address(ctx->sg)); + cmd->buf = cpu_to_le32(addr1); cmd->ct = cpu_to_le32(ctx->ct_dma); cmd->ct_hdr = ctx->ct_hdr; cmd->tfm = cpu_to_le32(ctx->tfm_dma); - if (++ring->cmd_pos == MTK_DESC_NUM) - ring->cmd_pos = 0; - - ring->res_pos = ring->cmd_pos; - - cmd = ring->cmd_base + ring->cmd_pos; - res = ring->res_base + ring->res_pos; - - res->hdr = MTK_DESC_BUF_LEN(len2) | MTK_DESC_LAST; - res->buf = cpu_to_le32(cryp->tmp_dma); + if (len2) { + mtk_sha_ring_shift(ring, &cmd, &res, &count); - cmd->hdr = MTK_DESC_BUF_LEN(len2) | MTK_DESC_LAST; - cmd->buf = cpu_to_le32(ctx->dma_addr); - - if (++ring->cmd_pos == MTK_DESC_NUM) - ring->cmd_pos = 0; + res->hdr = MTK_DESC_BUF_LEN(len2); + cmd->hdr = MTK_DESC_BUF_LEN(len2); + cmd->buf = cpu_to_le32(addr2); + } - ring->res_pos = ring->cmd_pos; + cmd->hdr |= MTK_DESC_LAST; + res->hdr |= MTK_DESC_LAST; /* * Make sure that all changes to the DMA ring are done before we @@ -512,8 +462,8 @@ static int mtk_sha_xmit2(struct mtk_cryp *cryp, */ wmb(); /* Start DMA transfer */ - mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(2)); - mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(2)); + mtk_sha_write(cryp, RDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); + mtk_sha_write(cryp, CDR_PREP_COUNT(sha->id), MTK_DESC_CNT(count)); return -EINPROGRESS; } @@ -532,7 +482,7 @@ static int mtk_sha_dma_map(struct mtk_cryp *cryp, ctx->flags &= ~SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count); + return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count, 0, 0); } static int mtk_sha_update_slow(struct mtk_cryp *cryp, @@ -625,7 +575,8 @@ static int mtk_sha_update_start(struct mtk_cryp *cryp, if (len == 0) { ctx->flags &= ~SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, ctx->dma_addr, count); + return mtk_sha_xmit(cryp, sha, ctx->dma_addr, + count, 0, 0); } else { ctx->sg = sg; @@ -635,7 +586,8 @@ static int mtk_sha_update_start(struct mtk_cryp *cryp, } ctx->flags |= SHA_FLAGS_SG; - return mtk_sha_xmit2(cryp, sha, ctx, len, count); + return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), + len, ctx->dma_addr, count); } } @@ -646,7 +598,8 @@ static int mtk_sha_update_start(struct mtk_cryp *cryp, ctx->flags |= SHA_FLAGS_SG; - return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), len); + return mtk_sha_xmit(cryp, sha, sg_dma_address(ctx->sg), + len, 0, 0); } static int mtk_sha_final_req(struct mtk_cryp *cryp, @@ -668,7 +621,7 @@ static int mtk_sha_final_req(struct mtk_cryp *cryp, static int mtk_sha_finish(struct ahash_request *req) { struct mtk_sha_reqctx *ctx = ahash_request_ctx(req); - u32 *digest = ctx->info.tfm.digest; + __le32 *digest = ctx->info.digest; u32 *result = (u32 *)req->result; int i; @@ -694,7 +647,7 @@ static void mtk_sha_finish_req(struct mtk_cryp *cryp, sha->req->base.complete(&sha->req->base, err); /* Handle new request */ - mtk_sha_handle_queue(cryp, sha->id - RING2, NULL); + tasklet_schedule(&sha->queue_task); } static int mtk_sha_handle_queue(struct mtk_cryp *cryp, u8 id, @@ -1216,60 +1169,38 @@ static struct ahash_alg algs_sha384_sha512[] = { }, }; -static void mtk_sha_task0(unsigned long data) +static void mtk_sha_queue_task(unsigned long data) { - struct mtk_cryp *cryp = (struct mtk_cryp *)data; - struct mtk_sha_rec *sha = cryp->sha[0]; + struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; - mtk_sha_unmap(cryp, sha); - mtk_sha_complete(cryp, sha); + mtk_sha_handle_queue(sha->cryp, sha->id - MTK_RING2, NULL); } -static void mtk_sha_task1(unsigned long data) +static void mtk_sha_done_task(unsigned long data) { - struct mtk_cryp *cryp = (struct mtk_cryp *)data; - struct mtk_sha_rec *sha = cryp->sha[1]; + struct mtk_sha_rec *sha = (struct mtk_sha_rec *)data; + struct mtk_cryp *cryp = sha->cryp; mtk_sha_unmap(cryp, sha); mtk_sha_complete(cryp, sha); } -static irqreturn_t mtk_sha_ring2_irq(int irq, void *dev_id) +static irqreturn_t mtk_sha_irq(int irq, void *dev_id) { - struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id; - struct mtk_sha_rec *sha = cryp->sha[0]; - u32 val = mtk_sha_read(cryp, RDR_STAT(RING2)); + struct mtk_sha_rec *sha = (struct mtk_sha_rec *)dev_id; + struct mtk_cryp *cryp = sha->cryp; + u32 val = mtk_sha_read(cryp, RDR_STAT(sha->id)); - mtk_sha_write(cryp, RDR_STAT(RING2), val); + mtk_sha_write(cryp, RDR_STAT(sha->id), val); if (likely((SHA_FLAGS_BUSY & sha->flags))) { - mtk_sha_write(cryp, RDR_PROC_COUNT(RING2), MTK_CNT_RST); - mtk_sha_write(cryp, RDR_THRESH(RING2), + mtk_sha_write(cryp, RDR_PROC_COUNT(sha->id), MTK_CNT_RST); + mtk_sha_write(cryp, RDR_THRESH(sha->id), MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - tasklet_schedule(&sha->task); + tasklet_schedule(&sha->done_task); } else { - dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); - } - return IRQ_HANDLED; -} - -static irqreturn_t mtk_sha_ring3_irq(int irq, void *dev_id) -{ - struct mtk_cryp *cryp = (struct mtk_cryp *)dev_id; - struct mtk_sha_rec *sha = cryp->sha[1]; - u32 val = mtk_sha_read(cryp, RDR_STAT(RING3)); - - mtk_sha_write(cryp, RDR_STAT(RING3), val); - - if (likely((SHA_FLAGS_BUSY & sha->flags))) { - mtk_sha_write(cryp, RDR_PROC_COUNT(RING3), MTK_CNT_RST); - mtk_sha_write(cryp, RDR_THRESH(RING3), - MTK_RDR_PROC_THRESH | MTK_RDR_PROC_MODE); - - tasklet_schedule(&sha->task); - } else { - dev_warn(cryp->dev, "AES interrupt when no active requests.\n"); + dev_warn(cryp->dev, "SHA interrupt when no active requests.\n"); } return IRQ_HANDLED; } @@ -1288,14 +1219,20 @@ static int mtk_sha_record_init(struct mtk_cryp *cryp) if (!sha[i]) goto err_cleanup; - sha[i]->id = i + RING2; + sha[i]->cryp = cryp; spin_lock_init(&sha[i]->lock); crypto_init_queue(&sha[i]->queue, SHA_QUEUE_SIZE); + + tasklet_init(&sha[i]->queue_task, mtk_sha_queue_task, + (unsigned long)sha[i]); + tasklet_init(&sha[i]->done_task, mtk_sha_done_task, + (unsigned long)sha[i]); } - tasklet_init(&sha[0]->task, mtk_sha_task0, (unsigned long)cryp); - tasklet_init(&sha[1]->task, mtk_sha_task1, (unsigned long)cryp); + /* Link to ring2 and ring3 respectively */ + sha[0]->id = MTK_RING2; + sha[1]->id = MTK_RING3; cryp->rec = 1; @@ -1312,7 +1249,9 @@ static void mtk_sha_record_free(struct mtk_cryp *cryp) int i; for (i = 0; i < MTK_REC_NUM; i++) { - tasklet_kill(&cryp->sha[i]->task); + tasklet_kill(&cryp->sha[i]->done_task); + tasklet_kill(&cryp->sha[i]->queue_task); + kfree(cryp->sha[i]); } } @@ -1368,35 +1307,23 @@ int mtk_hash_alg_register(struct mtk_cryp *cryp) if (err) goto err_record; - /* Ring2 is use by SHA record0 */ - err = devm_request_irq(cryp->dev, cryp->irq[RING2], - mtk_sha_ring2_irq, IRQF_TRIGGER_LOW, - "mtk-sha", cryp); + err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING2], mtk_sha_irq, + 0, "mtk-sha", cryp->sha[0]); if (err) { dev_err(cryp->dev, "unable to request sha irq0.\n"); goto err_res; } - /* Ring3 is use by SHA record1 */ - err = devm_request_irq(cryp->dev, cryp->irq[RING3], - mtk_sha_ring3_irq, IRQF_TRIGGER_LOW, - "mtk-sha", cryp); + err = devm_request_irq(cryp->dev, cryp->irq[MTK_RING3], mtk_sha_irq, + 0, "mtk-sha", cryp->sha[1]); if (err) { dev_err(cryp->dev, "unable to request sha irq1.\n"); goto err_res; } /* Enable ring2 and ring3 interrupt for hash */ - mtk_sha_write(cryp, AIC_ENABLE_SET(RING2), MTK_IRQ_RDR2); - mtk_sha_write(cryp, AIC_ENABLE_SET(RING3), MTK_IRQ_RDR3); - - cryp->tmp = dma_alloc_coherent(cryp->dev, SHA_TMP_BUF_SIZE, - &cryp->tmp_dma, GFP_KERNEL); - if (!cryp->tmp) { - dev_err(cryp->dev, "unable to allocate tmp buffer.\n"); - err = -EINVAL; - goto err_res; - } + mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING2), MTK_IRQ_RDR2); + mtk_sha_write(cryp, AIC_ENABLE_SET(MTK_RING3), MTK_IRQ_RDR3); spin_lock(&mtk_sha.lock); list_add_tail(&cryp->sha_list, &mtk_sha.dev_list); @@ -1412,8 +1339,6 @@ err_algs: spin_lock(&mtk_sha.lock); list_del(&cryp->sha_list); spin_unlock(&mtk_sha.lock); - dma_free_coherent(cryp->dev, SHA_TMP_BUF_SIZE, - cryp->tmp, cryp->tmp_dma); err_res: mtk_sha_record_free(cryp); err_record: @@ -1429,7 +1354,5 @@ void mtk_hash_alg_release(struct mtk_cryp *cryp) spin_unlock(&mtk_sha.lock); mtk_sha_unregister_algs(); - dma_free_coherent(cryp->dev, SHA_TMP_BUF_SIZE, - cryp->tmp, cryp->tmp_dma); mtk_sha_record_free(cryp); } |