diff options
70 files changed, 1785 insertions, 709 deletions
diff --git a/MAINTAINERS b/MAINTAINERS index cd8383e..3ac697a 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -4434,6 +4434,7 @@ F: include/linux/hwmon*.h HARDWARE RANDOM NUMBER GENERATOR CORE M: Matt Mackall <mpm@selenic.com> M: Herbert Xu <herbert@gondor.apana.org.au> +L: linux-crypto@vger.kernel.org S: Odd fixes F: Documentation/hw_random.txt F: drivers/char/hw_random/ diff --git a/arch/mips/cavium-octeon/Makefile b/arch/mips/cavium-octeon/Makefile index 42f5f1a..69a8a8d 100644 --- a/arch/mips/cavium-octeon/Makefile +++ b/arch/mips/cavium-octeon/Makefile @@ -16,6 +16,7 @@ obj-y := cpu.o setup.o octeon-platform.o octeon-irq.o csrc-octeon.o obj-y += dma-octeon.o obj-y += octeon-memcpy.o obj-y += executive/ +obj-y += crypto/ obj-$(CONFIG_MTD) += flash_setup.o obj-$(CONFIG_SMP) += smp.o diff --git a/arch/mips/cavium-octeon/crypto/Makefile b/arch/mips/cavium-octeon/crypto/Makefile new file mode 100644 index 0000000..a74f76d --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/Makefile @@ -0,0 +1,7 @@ +# +# OCTEON-specific crypto modules. +# + +obj-y += octeon-crypto.o + +obj-$(CONFIG_CRYPTO_MD5_OCTEON) += octeon-md5.o diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.c b/arch/mips/cavium-octeon/crypto/octeon-crypto.c new file mode 100644 index 0000000..7c82ff4 --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.c @@ -0,0 +1,66 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2004-2012 Cavium Networks + */ + +#include <asm/cop2.h> +#include <linux/module.h> +#include <linux/interrupt.h> + +#include "octeon-crypto.h" + +/** + * Enable access to Octeon's COP2 crypto hardware for kernel use. Wrap any + * crypto operations in calls to octeon_crypto_enable/disable in order to make + * sure the state of COP2 isn't corrupted if userspace is also performing + * hardware crypto operations. Allocate the state parameter on the stack. + * Preemption must be disabled to prevent context switches. + * + * @state: Pointer to state structure to store current COP2 state in. + * + * Returns: Flags to be passed to octeon_crypto_disable() + */ +unsigned long octeon_crypto_enable(struct octeon_cop2_state *state) +{ + int status; + unsigned long flags; + + local_irq_save(flags); + status = read_c0_status(); + write_c0_status(status | ST0_CU2); + if (KSTK_STATUS(current) & ST0_CU2) { + octeon_cop2_save(&(current->thread.cp2)); + KSTK_STATUS(current) &= ~ST0_CU2; + status &= ~ST0_CU2; + } else if (status & ST0_CU2) { + octeon_cop2_save(state); + } + local_irq_restore(flags); + return status & ST0_CU2; +} +EXPORT_SYMBOL_GPL(octeon_crypto_enable); + +/** + * Disable access to Octeon's COP2 crypto hardware in the kernel. This must be + * called after an octeon_crypto_enable() before any context switch or return to + * userspace. + * + * @state: Pointer to COP2 state to restore + * @flags: Return value from octeon_crypto_enable() + */ +void octeon_crypto_disable(struct octeon_cop2_state *state, + unsigned long crypto_flags) +{ + unsigned long flags; + + local_irq_save(flags); + if (crypto_flags & ST0_CU2) + octeon_cop2_restore(state); + else + write_c0_status(read_c0_status() & ~ST0_CU2); + local_irq_restore(flags); +} +EXPORT_SYMBOL_GPL(octeon_crypto_disable); diff --git a/arch/mips/cavium-octeon/crypto/octeon-crypto.h b/arch/mips/cavium-octeon/crypto/octeon-crypto.h new file mode 100644 index 0000000..e2a4aec --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/octeon-crypto.h @@ -0,0 +1,75 @@ +/* + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + * + * Copyright (C) 2012-2013 Cavium Inc., All Rights Reserved. + * + * MD5 instruction definitions added by Aaro Koskinen <aaro.koskinen@iki.fi>. + * + */ +#ifndef __LINUX_OCTEON_CRYPTO_H +#define __LINUX_OCTEON_CRYPTO_H + +#include <linux/sched.h> +#include <asm/mipsregs.h> + +#define OCTEON_CR_OPCODE_PRIORITY 300 + +extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state); +extern void octeon_crypto_disable(struct octeon_cop2_state *state, + unsigned long flags); + +/* + * Macros needed to implement MD5: + */ + +/* + * The index can be 0-1. + */ +#define write_octeon_64bit_hash_dword(value, index) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x0048+" STR(index) \ + : \ + : [rt] "d" (value)); \ +} while (0) + +/* + * The index can be 0-1. + */ +#define read_octeon_64bit_hash_dword(index) \ +({ \ + u64 __value; \ + \ + __asm__ __volatile__ ( \ + "dmfc2 %[rt],0x0048+" STR(index) \ + : [rt] "=d" (__value) \ + : ); \ + \ + __value; \ +}) + +/* + * The index can be 0-6. + */ +#define write_octeon_64bit_block_dword(value, index) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x0040+" STR(index) \ + : \ + : [rt] "d" (value)); \ +} while (0) + +/* + * The value is the final block dword (64-bit). + */ +#define octeon_md5_start(value) \ +do { \ + __asm__ __volatile__ ( \ + "dmtc2 %[rt],0x4047" \ + : \ + : [rt] "d" (value)); \ +} while (0) + +#endif /* __LINUX_OCTEON_CRYPTO_H */ diff --git a/arch/mips/cavium-octeon/crypto/octeon-md5.c b/arch/mips/cavium-octeon/crypto/octeon-md5.c new file mode 100644 index 0000000..b909881 --- /dev/null +++ b/arch/mips/cavium-octeon/crypto/octeon-md5.c @@ -0,0 +1,216 @@ +/* + * Cryptographic API. + * + * MD5 Message Digest Algorithm (RFC1321). + * + * Adapted for OCTEON by Aaro Koskinen <aaro.koskinen@iki.fi>. + * + * Based on crypto/md5.c, which is: + * + * Derived from cryptoapi implementation, originally based on the + * public domain implementation written by Colin Plumb in 1993. + * + * Copyright (c) Cryptoapi developers. + * Copyright (c) 2002 James Morris <jmorris@intercode.com.au> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the Free + * Software Foundation; either version 2 of the License, or (at your option) + * any later version. + */ + +#include <crypto/md5.h> +#include <linux/init.h> +#include <linux/types.h> +#include <linux/module.h> +#include <linux/string.h> +#include <asm/byteorder.h> +#include <linux/cryptohash.h> +#include <asm/octeon/octeon.h> +#include <crypto/internal/hash.h> + +#include "octeon-crypto.h" + +/* + * We pass everything as 64-bit. OCTEON can handle misaligned data. + */ + +static void octeon_md5_store_hash(struct md5_state *ctx) +{ + u64 *hash = (u64 *)ctx->hash; + + write_octeon_64bit_hash_dword(hash[0], 0); + write_octeon_64bit_hash_dword(hash[1], 1); +} + +static void octeon_md5_read_hash(struct md5_state *ctx) +{ + u64 *hash = (u64 *)ctx->hash; + + hash[0] = read_octeon_64bit_hash_dword(0); + hash[1] = read_octeon_64bit_hash_dword(1); +} + +static void octeon_md5_transform(const void *_block) +{ + const u64 *block = _block; + + write_octeon_64bit_block_dword(block[0], 0); + write_octeon_64bit_block_dword(block[1], 1); + write_octeon_64bit_block_dword(block[2], 2); + write_octeon_64bit_block_dword(block[3], 3); + write_octeon_64bit_block_dword(block[4], 4); + write_octeon_64bit_block_dword(block[5], 5); + write_octeon_64bit_block_dword(block[6], 6); + octeon_md5_start(block[7]); +} + +static int octeon_md5_init(struct shash_desc *desc) +{ + struct md5_state *mctx = shash_desc_ctx(desc); + + mctx->hash[0] = cpu_to_le32(0x67452301); + mctx->hash[1] = cpu_to_le32(0xefcdab89); + mctx->hash[2] = cpu_to_le32(0x98badcfe); + mctx->hash[3] = cpu_to_le32(0x10325476); + mctx->byte_count = 0; + + return 0; +} + +static int octeon_md5_update(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + struct md5_state *mctx = shash_desc_ctx(desc); + const u32 avail = sizeof(mctx->block) - (mctx->byte_count & 0x3f); + struct octeon_cop2_state state; + unsigned long flags; + + mctx->byte_count += len; + + if (avail > len) { + memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), + data, len); + return 0; + } + + memcpy((char *)mctx->block + (sizeof(mctx->block) - avail), data, + avail); + + local_bh_disable(); + preempt_disable(); + flags = octeon_crypto_enable(&state); + octeon_md5_store_hash(mctx); + + octeon_md5_transform(mctx->block); + data += avail; + len -= avail; + + while (len >= sizeof(mctx->block)) { + octeon_md5_transform(data); + data += sizeof(mctx->block); + len -= sizeof(mctx->block); + } + + octeon_md5_read_hash(mctx); + octeon_crypto_disable(&state, flags); + preempt_enable(); + local_bh_enable(); + + memcpy(mctx->block, data, len); + + return 0; +} + +static int octeon_md5_final(struct shash_desc *desc, u8 *out) +{ + struct md5_state *mctx = shash_desc_ctx(desc); + const unsigned int offset = mctx->byte_count & 0x3f; + char *p = (char *)mctx->block + offset; + int padding = 56 - (offset + 1); + struct octeon_cop2_state state; + unsigned long flags; + + *p++ = 0x80; + + local_bh_disable(); + preempt_disable(); + flags = octeon_crypto_enable(&state); + octeon_md5_store_hash(mctx); + + if (padding < 0) { + memset(p, 0x00, padding + sizeof(u64)); + octeon_md5_transform(mctx->block); + p = (char *)mctx->block; + padding = 56; + } + + memset(p, 0, padding); + mctx->block[14] = cpu_to_le32(mctx->byte_count << 3); + mctx->block[15] = cpu_to_le32(mctx->byte_count >> 29); + octeon_md5_transform(mctx->block); + + octeon_md5_read_hash(mctx); + octeon_crypto_disable(&state, flags); + preempt_enable(); + local_bh_enable(); + + memcpy(out, mctx->hash, sizeof(mctx->hash)); + memset(mctx, 0, sizeof(*mctx)); + + return 0; +} + +static int octeon_md5_export(struct shash_desc *desc, void *out) +{ + struct md5_state *ctx = shash_desc_ctx(desc); + + memcpy(out, ctx, sizeof(*ctx)); + return 0; +} + +static int octeon_md5_import(struct shash_desc *desc, const void *in) +{ + struct md5_state *ctx = shash_desc_ctx(desc); + + memcpy(ctx, in, sizeof(*ctx)); + return 0; +} + +static struct shash_alg alg = { + .digestsize = MD5_DIGEST_SIZE, + .init = octeon_md5_init, + .update = octeon_md5_update, + .final = octeon_md5_final, + .export = octeon_md5_export, + .import = octeon_md5_import, + .descsize = sizeof(struct md5_state), + .statesize = sizeof(struct md5_state), + .base = { + .cra_name = "md5", + .cra_driver_name= "octeon-md5", + .cra_priority = OCTEON_CR_OPCODE_PRIORITY, + .cra_flags = CRYPTO_ALG_TYPE_SHASH, + .cra_blocksize = MD5_HMAC_BLOCK_SIZE, + .cra_module = THIS_MODULE, + } +}; + +static int __init md5_mod_init(void) +{ + if (!octeon_has_crypto()) + return -ENOTSUPP; + return crypto_register_shash(&alg); +} + +static void __exit md5_mod_fini(void) +{ + crypto_unregister_shash(&alg); +} + +module_init(md5_mod_init); +module_exit(md5_mod_fini); + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("MD5 Message Digest Algorithm (OCTEON)"); +MODULE_AUTHOR("Aaro Koskinen <aaro.koskinen@iki.fi>"); diff --git a/arch/mips/cavium-octeon/executive/octeon-model.c b/arch/mips/cavium-octeon/executive/octeon-model.c index e15b049..b2104bd 100644 --- a/arch/mips/cavium-octeon/executive/octeon-model.c +++ b/arch/mips/cavium-octeon/executive/octeon-model.c @@ -27,6 +27,9 @@ #include <asm/octeon/octeon.h> +enum octeon_feature_bits __octeon_feature_bits __read_mostly; +EXPORT_SYMBOL_GPL(__octeon_feature_bits); + /** * Read a byte of fuse data * @byte_addr: address to read @@ -103,6 +106,9 @@ static const char *__init octeon_model_get_string_buffer(uint32_t chip_id, else suffix = "NSP"; + if (!fus_dat2.s.nocrypto) + __octeon_feature_bits |= OCTEON_HAS_CRYPTO; + /* * Assume pass number is encoded using <5:3><2:0>. Exceptions * will be fixed later. diff --git a/arch/mips/include/asm/octeon/octeon-feature.h b/arch/mips/include/asm/octeon/octeon-feature.h index c4fe81f..8ebd3f57 100644 --- a/arch/mips/include/asm/octeon/octeon-feature.h +++ b/arch/mips/include/asm/octeon/octeon-feature.h @@ -46,8 +46,6 @@ enum octeon_feature { OCTEON_FEATURE_SAAD, /* Does this Octeon support the ZIP offload engine? */ OCTEON_FEATURE_ZIP, - /* Does this Octeon support crypto acceleration using COP2? */ - OCTEON_FEATURE_CRYPTO, OCTEON_FEATURE_DORM_CRYPTO, /* Does this Octeon support PCI express? */ OCTEON_FEATURE_PCIE, @@ -86,6 +84,21 @@ enum octeon_feature { OCTEON_MAX_FEATURE }; +enum octeon_feature_bits { + OCTEON_HAS_CRYPTO = 0x0001, /* Crypto acceleration using COP2 */ +}; +extern enum octeon_feature_bits __octeon_feature_bits; + +/** + * octeon_has_crypto() - Check if this OCTEON has crypto acceleration support. + * + * Returns: Non-zero if the feature exists. Zero if the feature does not exist. + */ +static inline int octeon_has_crypto(void) +{ + return __octeon_feature_bits & OCTEON_HAS_CRYPTO; +} + /** * Determine if the current Octeon supports a specific feature. These * checks have been optimized to be fairly quick, but they should still diff --git a/arch/mips/include/asm/octeon/octeon.h b/arch/mips/include/asm/octeon/octeon.h index d781f9e..6dfefd2 100644 --- a/arch/mips/include/asm/octeon/octeon.h +++ b/arch/mips/include/asm/octeon/octeon.h @@ -44,11 +44,6 @@ extern int octeon_get_boot_num_arguments(void); extern const char *octeon_get_boot_argument(int arg); extern void octeon_hal_setup_reserved32(void); extern void octeon_user_io_init(void); -struct octeon_cop2_state; -extern unsigned long octeon_crypto_enable(struct octeon_cop2_state *state); -extern void octeon_crypto_disable(struct octeon_cop2_state *state, - unsigned long flags); -extern asmlinkage void octeon_cop2_restore(struct octeon_cop2_state *task); extern void octeon_init_cvmcount(void); extern void octeon_setup_delays(void); diff --git a/arch/sparc/crypto/aes_glue.c b/arch/sparc/crypto/aes_glue.c index 7054087..2e48eb8 100644 --- a/arch/sparc/crypto/aes_glue.c +++ b/arch/sparc/crypto/aes_glue.c @@ -497,7 +497,7 @@ module_init(aes_sparc64_mod_init); module_exit(aes_sparc64_mod_fini); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("AES Secure Hash Algorithm, sparc64 aes opcode accelerated"); +MODULE_DESCRIPTION("Rijndael (AES) Cipher Algorithm, sparc64 aes opcode accelerated"); MODULE_ALIAS_CRYPTO("aes"); diff --git a/arch/sparc/crypto/camellia_glue.c b/arch/sparc/crypto/camellia_glue.c index 641f55c..6bf2479 100644 --- a/arch/sparc/crypto/camellia_glue.c +++ b/arch/sparc/crypto/camellia_glue.c @@ -322,6 +322,6 @@ module_exit(camellia_sparc64_mod_fini); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Camellia Cipher Algorithm, sparc64 camellia opcode accelerated"); -MODULE_ALIAS_CRYPTO("aes"); +MODULE_ALIAS_CRYPTO("camellia"); #include "crop_devid.c" diff --git a/arch/sparc/crypto/des_glue.c b/arch/sparc/crypto/des_glue.c index d115009..dd6a34f 100644 --- a/arch/sparc/crypto/des_glue.c +++ b/arch/sparc/crypto/des_glue.c @@ -533,5 +533,6 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("DES & Triple DES EDE Cipher Algorithms, sparc64 des opcode accelerated"); MODULE_ALIAS_CRYPTO("des"); +MODULE_ALIAS_CRYPTO("des3_ede"); #include "crop_devid.c" diff --git a/arch/sparc/crypto/md5_glue.c b/arch/sparc/crypto/md5_glue.c index 64c7ff5..b688731 100644 --- a/arch/sparc/crypto/md5_glue.c +++ b/arch/sparc/crypto/md5_glue.c @@ -183,7 +183,7 @@ module_init(md5_sparc64_mod_init); module_exit(md5_sparc64_mod_fini); MODULE_LICENSE("GPL"); -MODULE_DESCRIPTION("MD5 Secure Hash Algorithm, sparc64 md5 opcode accelerated"); +MODULE_DESCRIPTION("MD5 Message Digest Algorithm, sparc64 md5 opcode accelerated"); MODULE_ALIAS_CRYPTO("md5"); diff --git a/arch/x86/crypto/aesni-intel_asm.S b/arch/x86/crypto/aesni-intel_asm.S index 477e9d7..6bd2c6c 100644 --- a/arch/x86/crypto/aesni-intel_asm.S +++ b/arch/x86/crypto/aesni-intel_asm.S @@ -32,12 +32,23 @@ #include <linux/linkage.h> #include <asm/inst.h> +/* + * The following macros are used to move an (un)aligned 16 byte value to/from + * an XMM register. This can done for either FP or integer values, for FP use + * movaps (move aligned packed single) or integer use movdqa (move double quad + * aligned). It doesn't make a performance difference which instruction is used + * since Nehalem (original Core i7) was released. However, the movaps is a byte + * shorter, so that is the one we'll use for now. (same for unaligned). + */ +#define MOVADQ movaps +#define MOVUDQ movups + #ifdef __x86_64__ + .data .align 16 .Lgf128mul_x_ble_mask: .octa 0x00000000000000010000000000000087 - POLY: .octa 0xC2000000000000000000000000000001 TWOONE: .octa 0x00000001000000000000000000000001 @@ -89,6 +100,7 @@ enc: .octa 0x2 #define arg8 STACK_OFFSET+16(%r14) #define arg9 STACK_OFFSET+24(%r14) #define arg10 STACK_OFFSET+32(%r14) +#define keysize 2*15*16(%arg1) #endif @@ -213,10 +225,12 @@ enc: .octa 0x2 .macro INITIAL_BLOCKS_DEC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \ XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation + MOVADQ SHUF_MASK(%rip), %xmm14 mov arg7, %r10 # %r10 = AAD mov arg8, %r12 # %r12 = aadLen mov %r12, %r11 pxor %xmm\i, %xmm\i + _get_AAD_loop\num_initial_blocks\operation: movd (%r10), \TMP1 pslldq $12, \TMP1 @@ -225,16 +239,18 @@ _get_AAD_loop\num_initial_blocks\operation: add $4, %r10 sub $4, %r12 jne _get_AAD_loop\num_initial_blocks\operation + cmp $16, %r11 je _get_AAD_loop2_done\num_initial_blocks\operation + mov $16, %r12 _get_AAD_loop2\num_initial_blocks\operation: psrldq $4, %xmm\i sub $4, %r12 cmp %r11, %r12 jne _get_AAD_loop2\num_initial_blocks\operation + _get_AAD_loop2_done\num_initial_blocks\operation: - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data xor %r11, %r11 # initialise the data pointer offset as zero @@ -243,59 +259,34 @@ _get_AAD_loop2_done\num_initial_blocks\operation: mov %arg5, %rax # %rax = *Y0 movdqu (%rax), \XMM0 # XMM0 = Y0 - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM0 .if (\i == 5) || (\i == 6) || (\i == 7) + MOVADQ ONE(%RIP),\TMP1 + MOVADQ (%arg1),\TMP2 .irpc index, \i_seq - paddd ONE(%rip), \XMM0 # INCR Y0 + paddd \TMP1, \XMM0 # INCR Y0 movdqa \XMM0, %xmm\index - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap - -.endr -.irpc index, \i_seq - pxor 16*0(%arg1), %xmm\index -.endr -.irpc index, \i_seq - movaps 0x10(%rdi), \TMP1 - AESENC \TMP1, %xmm\index # Round 1 -.endr -.irpc index, \i_seq - movaps 0x20(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x30(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x40(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x50(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x60(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 + pxor \TMP2, %xmm\index .endr -.irpc index, \i_seq - movaps 0x70(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x80(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x90(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 + lea 0x10(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + add $5,%eax # 128->9, 192->11, 256->13 + +aes_loop_initial_dec\num_initial_blocks: + MOVADQ (%r10),\TMP1 +.irpc index, \i_seq + AESENC \TMP1, %xmm\index .endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_initial_dec\num_initial_blocks + + MOVADQ (%r10), \TMP1 .irpc index, \i_seq - movaps 0xa0(%arg1), \TMP1 - AESENCLAST \TMP1, %xmm\index # Round 10 + AESENCLAST \TMP1, %xmm\index # Last Round .endr .irpc index, \i_seq movdqu (%arg3 , %r11, 1), \TMP1 @@ -305,10 +296,8 @@ _get_AAD_loop2_done\num_initial_blocks\operation: add $16, %r11 movdqa \TMP1, %xmm\index - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\index - - # prepare plaintext/ciphertext for GHASH computation + # prepare plaintext/ciphertext for GHASH computation .endr .endif GHASH_MUL %xmm\i, \TMP3, \TMP1, \TMP2, \TMP4, \TMP5, \XMM1 @@ -338,30 +327,28 @@ _get_AAD_loop2_done\num_initial_blocks\operation: * Precomputations for HashKey parallel with encryption of first 4 blocks. * Haskey_i_k holds XORed values of the low and high parts of the Haskey_i */ - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM1 - movdqa SHUF_MASK(%rip), %xmm14 + MOVADQ ONE(%rip), \TMP1 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM1 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM2 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM2 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM3 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM3 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM4 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM4 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap - pxor 16*0(%arg1), \XMM1 - pxor 16*0(%arg1), \XMM2 - pxor 16*0(%arg1), \XMM3 - pxor 16*0(%arg1), \XMM4 + MOVADQ 0(%arg1),\TMP1 + pxor \TMP1, \XMM1 + pxor \TMP1, \XMM2 + pxor \TMP1, \XMM3 + pxor \TMP1, \XMM4 movdqa \TMP3, \TMP5 pshufd $78, \TMP3, \TMP1 pxor \TMP3, \TMP1 @@ -399,7 +386,23 @@ _get_AAD_loop2_done\num_initial_blocks\operation: pshufd $78, \TMP5, \TMP1 pxor \TMP5, \TMP1 movdqa \TMP1, HashKey_4_k(%rsp) - movaps 0xa0(%arg1), \TMP2 + lea 0xa0(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + sub $4,%eax # 128->0, 192->2, 256->4 + jz aes_loop_pre_dec_done\num_initial_blocks + +aes_loop_pre_dec\num_initial_blocks: + MOVADQ (%r10),\TMP2 +.irpc index, 1234 + AESENC \TMP2, %xmm\index +.endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_pre_dec\num_initial_blocks + +aes_loop_pre_dec_done\num_initial_blocks: + MOVADQ (%r10), \TMP2 AESENCLAST \TMP2, \XMM1 AESENCLAST \TMP2, \XMM2 AESENCLAST \TMP2, \XMM3 @@ -421,15 +424,11 @@ _get_AAD_loop2_done\num_initial_blocks\operation: movdqu \XMM4, 16*3(%arg2 , %r11 , 1) movdqa \TMP1, \XMM4 add $64, %r11 - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap pxor \XMMDst, \XMM1 # combine GHASHed value with the corresponding ciphertext - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap _initial_blocks_done\num_initial_blocks\operation: @@ -451,6 +450,7 @@ _initial_blocks_done\num_initial_blocks\operation: .macro INITIAL_BLOCKS_ENC num_initial_blocks TMP1 TMP2 TMP3 TMP4 TMP5 XMM0 XMM1 \ XMM2 XMM3 XMM4 XMMDst TMP6 TMP7 i i_seq operation + MOVADQ SHUF_MASK(%rip), %xmm14 mov arg7, %r10 # %r10 = AAD mov arg8, %r12 # %r12 = aadLen mov %r12, %r11 @@ -472,7 +472,6 @@ _get_AAD_loop2\num_initial_blocks\operation: cmp %r11, %r12 jne _get_AAD_loop2\num_initial_blocks\operation _get_AAD_loop2_done\num_initial_blocks\operation: - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\i # byte-reflect the AAD data xor %r11, %r11 # initialise the data pointer offset as zero @@ -481,59 +480,35 @@ _get_AAD_loop2_done\num_initial_blocks\operation: mov %arg5, %rax # %rax = *Y0 movdqu (%rax), \XMM0 # XMM0 = Y0 - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM0 .if (\i == 5) || (\i == 6) || (\i == 7) -.irpc index, \i_seq - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, %xmm\index - movdqa SHUF_MASK(%rip), %xmm14 - PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap -.endr -.irpc index, \i_seq - pxor 16*0(%arg1), %xmm\index -.endr -.irpc index, \i_seq - movaps 0x10(%rdi), \TMP1 - AESENC \TMP1, %xmm\index # Round 1 -.endr -.irpc index, \i_seq - movaps 0x20(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr + MOVADQ ONE(%RIP),\TMP1 + MOVADQ 0(%arg1),\TMP2 .irpc index, \i_seq - movaps 0x30(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, %xmm\index + PSHUFB_XMM %xmm14, %xmm\index # perform a 16 byte swap + pxor \TMP2, %xmm\index .endr -.irpc index, \i_seq - movaps 0x40(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x50(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x60(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x70(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x80(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 -.endr -.irpc index, \i_seq - movaps 0x90(%arg1), \TMP1 - AESENC \TMP1, %xmm\index # Round 2 + lea 0x10(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + add $5,%eax # 128->9, 192->11, 256->13 + +aes_loop_initial_enc\num_initial_blocks: + MOVADQ (%r10),\TMP1 +.irpc index, \i_seq + AESENC \TMP1, %xmm\index .endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_initial_enc\num_initial_blocks + + MOVADQ (%r10), \TMP1 .irpc index, \i_seq - movaps 0xa0(%arg1), \TMP1 - AESENCLAST \TMP1, %xmm\index # Round 10 + AESENCLAST \TMP1, %xmm\index # Last Round .endr .irpc index, \i_seq movdqu (%arg3 , %r11, 1), \TMP1 @@ -541,8 +516,6 @@ _get_AAD_loop2_done\num_initial_blocks\operation: movdqu %xmm\index, (%arg2 , %r11, 1) # write back plaintext/ciphertext for num_initial_blocks add $16, %r11 - - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, %xmm\index # prepare plaintext/ciphertext for GHASH computation @@ -575,30 +548,28 @@ _get_AAD_loop2_done\num_initial_blocks\operation: * Precomputations for HashKey parallel with encryption of first 4 blocks. * Haskey_i_k holds XORed values of the low and high parts of the Haskey_i */ - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM1 - movdqa SHUF_MASK(%rip), %xmm14 + MOVADQ ONE(%RIP),\TMP1 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM1 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM2 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM2 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM3 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM3 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap - paddd ONE(%rip), \XMM0 # INCR Y0 - movdqa \XMM0, \XMM4 - movdqa SHUF_MASK(%rip), %xmm14 + paddd \TMP1, \XMM0 # INCR Y0 + MOVADQ \XMM0, \XMM4 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap - pxor 16*0(%arg1), \XMM1 - pxor 16*0(%arg1), \XMM2 - pxor 16*0(%arg1), \XMM3 - pxor 16*0(%arg1), \XMM4 + MOVADQ 0(%arg1),\TMP1 + pxor \TMP1, \XMM1 + pxor \TMP1, \XMM2 + pxor \TMP1, \XMM3 + pxor \TMP1, \XMM4 movdqa \TMP3, \TMP5 pshufd $78, \TMP3, \TMP1 pxor \TMP3, \TMP1 @@ -636,7 +607,23 @@ _get_AAD_loop2_done\num_initial_blocks\operation: pshufd $78, \TMP5, \TMP1 pxor \TMP5, \TMP1 movdqa \TMP1, HashKey_4_k(%rsp) - movaps 0xa0(%arg1), \TMP2 + lea 0xa0(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + sub $4,%eax # 128->0, 192->2, 256->4 + jz aes_loop_pre_enc_done\num_initial_blocks + +aes_loop_pre_enc\num_initial_blocks: + MOVADQ (%r10),\TMP2 +.irpc index, 1234 + AESENC \TMP2, %xmm\index +.endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_pre_enc\num_initial_blocks + +aes_loop_pre_enc_done\num_initial_blocks: + MOVADQ (%r10), \TMP2 AESENCLAST \TMP2, \XMM1 AESENCLAST \TMP2, \XMM2 AESENCLAST \TMP2, \XMM3 @@ -655,15 +642,11 @@ _get_AAD_loop2_done\num_initial_blocks\operation: movdqu \XMM4, 16*3(%arg2 , %r11 , 1) add $64, %r11 - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM1 # perform a 16 byte swap pxor \XMMDst, \XMM1 # combine GHASHed value with the corresponding ciphertext - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM2 # perform a 16 byte swap - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM3 # perform a 16 byte swap - movdqa SHUF_MASK(%rip), %xmm14 PSHUFB_XMM %xmm14, \XMM4 # perform a 16 byte swap _initial_blocks_done\num_initial_blocks\operation: @@ -794,7 +777,23 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation AESENC \TMP3, \XMM3 AESENC \TMP3, \XMM4 PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 - movaps 0xa0(%arg1), \TMP3 + lea 0xa0(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + sub $4,%eax # 128->0, 192->2, 256->4 + jz aes_loop_par_enc_done + +aes_loop_par_enc: + MOVADQ (%r10),\TMP3 +.irpc index, 1234 + AESENC \TMP3, %xmm\index +.endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_par_enc + +aes_loop_par_enc_done: + MOVADQ (%r10), \TMP3 AESENCLAST \TMP3, \XMM1 # Round 10 AESENCLAST \TMP3, \XMM2 AESENCLAST \TMP3, \XMM3 @@ -986,8 +985,24 @@ TMP6 XMM0 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8 operation AESENC \TMP3, \XMM3 AESENC \TMP3, \XMM4 PCLMULQDQ 0x00, \TMP5, \XMM8 # XMM8 = a0*b0 - movaps 0xa0(%arg1), \TMP3 - AESENCLAST \TMP3, \XMM1 # Round 10 + lea 0xa0(%arg1),%r10 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + sub $4,%eax # 128->0, 192->2, 256->4 + jz aes_loop_par_dec_done + +aes_loop_par_dec: + MOVADQ (%r10),\TMP3 +.irpc index, 1234 + AESENC \TMP3, %xmm\index +.endr + add $16,%r10 + sub $1,%eax + jnz aes_loop_par_dec + +aes_loop_par_dec_done: + MOVADQ (%r10), \TMP3 + AESENCLAST \TMP3, \XMM1 # last round AESENCLAST \TMP3, \XMM2 AESENCLAST \TMP3, \XMM3 AESENCLAST \TMP3, \XMM4 @@ -1155,33 +1170,29 @@ TMP7 XMM1 XMM2 XMM3 XMM4 XMMDst pxor \TMP6, \XMMDst # reduced result is in XMMDst .endm -/* Encryption of a single block done*/ -.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1 - pxor (%arg1), \XMM0 - movaps 16(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 32(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 48(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 64(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 80(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 96(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 112(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 128(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 144(%arg1), \TMP1 - AESENC \TMP1, \XMM0 - movaps 160(%arg1), \TMP1 - AESENCLAST \TMP1, \XMM0 -.endm +/* Encryption of a single block +* uses eax & r10 +*/ +.macro ENCRYPT_SINGLE_BLOCK XMM0 TMP1 + pxor (%arg1), \XMM0 + mov keysize,%eax + shr $2,%eax # 128->4, 192->6, 256->8 + add $5,%eax # 128->9, 192->11, 256->13 + lea 16(%arg1), %r10 # get first expanded key address + +_esb_loop_\@: + MOVADQ (%r10),\TMP1 + AESENC \TMP1,\XMM0 + add $16,%r10 + sub $1,%eax + jnz _esb_loop_\@ + + MOVADQ (%r10),\TMP1 + AESENCLAST \TMP1,\XMM0 +.endm /***************************************************************************** * void aesni_gcm_dec(void *aes_ctx, // AES Key schedule. Starts on a 16 byte boundary. * u8 *out, // Plaintext output. Encrypt in-place is allowed. diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index ae855f4..947c6bf 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -43,6 +43,7 @@ #include <asm/crypto/glue_helper.h> #endif + /* This data is stored at the end of the crypto_tfm struct. * It's a type of per "session" data storage location. * This needs to be 16 byte aligned. @@ -182,7 +183,8 @@ static void aesni_gcm_enc_avx(void *ctx, u8 *out, u8 *hash_subkey, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len) { - if (plaintext_len < AVX_GEN2_OPTSIZE) { + struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; + if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)){ aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, aad_len, auth_tag, auth_tag_len); } else { @@ -197,7 +199,8 @@ static void aesni_gcm_dec_avx(void *ctx, u8 *out, u8 *hash_subkey, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len) { - if (ciphertext_len < AVX_GEN2_OPTSIZE) { + struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; + if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad, aad_len, auth_tag, auth_tag_len); } else { @@ -231,7 +234,8 @@ static void aesni_gcm_enc_avx2(void *ctx, u8 *out, u8 *hash_subkey, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len) { - if (plaintext_len < AVX_GEN2_OPTSIZE) { + struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; + if ((plaintext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { aesni_gcm_enc(ctx, out, in, plaintext_len, iv, hash_subkey, aad, aad_len, auth_tag, auth_tag_len); } else if (plaintext_len < AVX_GEN4_OPTSIZE) { @@ -250,7 +254,8 @@ static void aesni_gcm_dec_avx2(void *ctx, u8 *out, u8 *hash_subkey, const u8 *aad, unsigned long aad_len, u8 *auth_tag, unsigned long auth_tag_len) { - if (ciphertext_len < AVX_GEN2_OPTSIZE) { + struct crypto_aes_ctx *aes_ctx = (struct crypto_aes_ctx*)ctx; + if ((ciphertext_len < AVX_GEN2_OPTSIZE) || (aes_ctx-> key_length != AES_KEYSIZE_128)) { aesni_gcm_dec(ctx, out, in, ciphertext_len, iv, hash_subkey, aad, aad_len, auth_tag, auth_tag_len); } else if (ciphertext_len < AVX_GEN4_OPTSIZE) { @@ -511,7 +516,7 @@ static int ctr_crypt(struct blkcipher_desc *desc, kernel_fpu_begin(); while ((nbytes = walk.nbytes) >= AES_BLOCK_SIZE) { aesni_ctr_enc_tfm(ctx, walk.dst.virt.addr, walk.src.virt.addr, - nbytes & AES_BLOCK_MASK, walk.iv); + nbytes & AES_BLOCK_MASK, walk.iv); nbytes &= AES_BLOCK_SIZE - 1; err = blkcipher_walk_done(desc, &walk, nbytes); } @@ -902,7 +907,8 @@ static int rfc4106_set_key(struct crypto_aead *parent, const u8 *key, } /*Account for 4 byte nonce at the end.*/ key_len -= 4; - if (key_len != AES_KEYSIZE_128) { + if (key_len != AES_KEYSIZE_128 && key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256) { crypto_tfm_set_flags(tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); return -EINVAL; } @@ -1013,6 +1019,7 @@ static int __driver_rfc4106_encrypt(struct aead_request *req) __be32 counter = cpu_to_be32(1); struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + u32 key_len = ctx->aes_key_expanded.key_length; void *aes_ctx = &(ctx->aes_key_expanded); unsigned long auth_tag_len = crypto_aead_authsize(tfm); u8 iv_tab[16+AESNI_ALIGN]; @@ -1027,6 +1034,13 @@ static int __driver_rfc4106_encrypt(struct aead_request *req) /* to 8 or 12 bytes */ if (unlikely(req->assoclen != 8 && req->assoclen != 12)) return -EINVAL; + if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16)) + return -EINVAL; + if (unlikely(key_len != AES_KEYSIZE_128 && + key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256)) + return -EINVAL; + /* IV below built */ for (i = 0; i < 4; i++) *(iv+i) = ctx->nonce[i]; @@ -1091,6 +1105,7 @@ static int __driver_rfc4106_decrypt(struct aead_request *req) int retval = 0; struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aesni_rfc4106_gcm_ctx *ctx = aesni_rfc4106_gcm_ctx_get(tfm); + u32 key_len = ctx->aes_key_expanded.key_length; void *aes_ctx = &(ctx->aes_key_expanded); unsigned long auth_tag_len = crypto_aead_authsize(tfm); u8 iv_and_authTag[32+AESNI_ALIGN]; @@ -1104,6 +1119,13 @@ static int __driver_rfc4106_decrypt(struct aead_request *req) if (unlikely((req->cryptlen < auth_tag_len) || (req->assoclen != 8 && req->assoclen != 12))) return -EINVAL; + if (unlikely(auth_tag_len != 8 && auth_tag_len != 12 && auth_tag_len != 16)) + return -EINVAL; + if (unlikely(key_len != AES_KEYSIZE_128 && + key_len != AES_KEYSIZE_192 && + key_len != AES_KEYSIZE_256)) + return -EINVAL; + /* Assuming we are supporting rfc4106 64-bit extended */ /* sequence numbers We need to have the AAD length */ /* equal to 8 or 12 bytes */ diff --git a/arch/x86/crypto/des3_ede_glue.c b/arch/x86/crypto/des3_ede_glue.c index 38a14f8..d6fc59a 100644 --- a/arch/x86/crypto/des3_ede_glue.c +++ b/arch/x86/crypto/des3_ede_glue.c @@ -504,6 +504,4 @@ MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Triple DES EDE Cipher Algorithm, asm optimized"); MODULE_ALIAS_CRYPTO("des3_ede"); MODULE_ALIAS_CRYPTO("des3_ede-asm"); -MODULE_ALIAS_CRYPTO("des"); -MODULE_ALIAS_CRYPTO("des-asm"); MODULE_AUTHOR("Jussi Kivilinna <jussi.kivilinna@iki.fi>"); diff --git a/crypto/Kconfig b/crypto/Kconfig index 87bbc9c..50f4da4 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -427,6 +427,15 @@ config CRYPTO_MD5 help MD5 message digest algorithm (RFC1321). +config CRYPTO_MD5_OCTEON + tristate "MD5 digest algorithm (OCTEON)" + depends on CPU_CAVIUM_OCTEON + select CRYPTO_MD5 + select CRYPTO_HASH + help + MD5 message digest algorithm (RFC1321) implemented + using OCTEON crypto instructions, when available. + config CRYPTO_MD5_SPARC64 tristate "MD5 digest algorithm (SPARC64)" depends on SPARC64 @@ -1505,6 +1514,15 @@ config CRYPTO_USER_API_SKCIPHER This option enables the user-spaces interface for symmetric key cipher algorithms. +config CRYPTO_USER_API_RNG + tristate "User-space interface for random number generator algorithms" + depends on NET + select CRYPTO_RNG + select CRYPTO_USER_API + help + This option enables the user-spaces interface for random + number generator algorithms. + config CRYPTO_HASH_INFO bool diff --git a/crypto/Makefile b/crypto/Makefile index 1445b91..ba19465 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -99,6 +99,7 @@ obj-$(CONFIG_CRYPTO_GHASH) += ghash-generic.o obj-$(CONFIG_CRYPTO_USER_API) += af_alg.o obj-$(CONFIG_CRYPTO_USER_API_HASH) += algif_hash.o obj-$(CONFIG_CRYPTO_USER_API_SKCIPHER) += algif_skcipher.o +obj-$(CONFIG_CRYPTO_USER_API_RNG) += algif_rng.o # # generic algorithms and the async_tx api diff --git a/crypto/ablkcipher.c b/crypto/ablkcipher.c index 40886c4..db201bca 100644 --- a/crypto/ablkcipher.c +++ b/crypto/ablkcipher.c @@ -69,6 +69,7 @@ static inline void ablkcipher_queue_write(struct ablkcipher_walk *walk, static inline u8 *ablkcipher_get_spot(u8 *start, unsigned int len) { u8 *end_page = (u8 *)(((unsigned long)(start + len - 1)) & PAGE_MASK); + return max(start, end_page); } @@ -86,7 +87,7 @@ static inline unsigned int ablkcipher_done_slow(struct ablkcipher_walk *walk, if (n == len_this_page) break; n -= len_this_page; - scatterwalk_start(&walk->out, scatterwalk_sg_next(walk->out.sg)); + scatterwalk_start(&walk->out, sg_next(walk->out.sg)); } return bsize; @@ -284,6 +285,7 @@ static int ablkcipher_walk_first(struct ablkcipher_request *req, walk->iv = req->info; if (unlikely(((unsigned long)walk->iv & alignmask))) { int err = ablkcipher_copy_iv(walk, tfm, alignmask); + if (err) return err; } @@ -589,7 +591,8 @@ static int crypto_givcipher_default(struct crypto_alg *alg, u32 type, u32 mask) if (IS_ERR(inst)) goto put_tmpl; - if ((err = crypto_register_instance(tmpl, inst))) { + err = crypto_register_instance(tmpl, inst); + if (err) { tmpl->free(inst); goto put_tmpl; } diff --git a/crypto/aead.c b/crypto/aead.c index 547491e..2222710 100644 --- a/crypto/aead.c +++ b/crypto/aead.c @@ -448,7 +448,8 @@ static int crypto_nivaead_default(struct crypto_alg *alg, u32 type, u32 mask) if (IS_ERR(inst)) goto put_tmpl; - if ((err = crypto_register_instance(tmpl, inst))) { + err = crypto_register_instance(tmpl, inst); + if (err) { tmpl->free(inst); goto put_tmpl; } diff --git a/crypto/af_alg.c b/crypto/af_alg.c index 3e80d8b..7f8b7edc 100644 --- a/crypto/af_alg.c +++ b/crypto/af_alg.c @@ -188,7 +188,7 @@ static int alg_setkey(struct sock *sk, char __user *ukey, err = type->setkey(ask->private, key, keylen); out: - sock_kfree_s(sk, key, keylen); + sock_kzfree_s(sk, key, keylen); return err; } @@ -215,6 +215,13 @@ static int alg_setsockopt(struct socket *sock, int level, int optname, goto unlock; err = alg_setkey(sk, optval, optlen); + break; + case ALG_SET_AEAD_AUTHSIZE: + if (sock->state == SS_CONNECTED) + goto unlock; + if (!type->setauthsize) + goto unlock; + err = type->setauthsize(ask->private, optlen); } unlock: @@ -387,7 +394,7 @@ int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con) if (cmsg->cmsg_level != SOL_ALG) continue; - switch(cmsg->cmsg_type) { + switch (cmsg->cmsg_type) { case ALG_SET_IV: if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv))) return -EINVAL; diff --git a/crypto/ahash.c b/crypto/ahash.c index f6a36a5..8acb88603 100644 --- a/crypto/ahash.c +++ b/crypto/ahash.c @@ -55,6 +55,7 @@ static int hash_walk_next(struct crypto_hash_walk *walk) if (offset & alignmask) { unsigned int unaligned = alignmask + 1 - (offset & alignmask); + if (nbytes > unaligned) nbytes = unaligned; } @@ -120,7 +121,7 @@ int crypto_hash_walk_done(struct crypto_hash_walk *walk, int err) if (!walk->total) return 0; - walk->sg = scatterwalk_sg_next(walk->sg); + walk->sg = sg_next(walk->sg); return hash_walk_new_entry(walk); } diff --git a/crypto/algapi.c b/crypto/algapi.c index 71a8143..83b04e0 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -473,6 +473,7 @@ void crypto_unregister_template(struct crypto_template *tmpl) list = &tmpl->instances; hlist_for_each_entry(inst, list, list) { int err = crypto_remove_alg(&inst->alg, &users); + BUG_ON(err); } diff --git a/crypto/algif_rng.c b/crypto/algif_rng.c new file mode 100644 index 0000000..67f612c --- /dev/null +++ b/crypto/algif_rng.c @@ -0,0 +1,192 @@ +/* + * algif_rng: User-space interface for random number generators + * + * This file provides the user-space API for random number generators. + * + * Copyright (C) 2014, Stephan Mueller <smueller@chronox.de> + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, and the entire permission notice in its entirety, + * including the disclaimer of warranties. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote + * products derived from this software without specific prior + * written permission. + * + * ALTERNATIVELY, this product may be distributed under the terms of + * the GNU General Public License, in which case the provisions of the GPL2 + * are required INSTEAD OF the above restrictions. (This clause is + * necessary due to a potential bad interaction between the GPL and + * the restrictions contained in a BSD-style copyright.) + * + * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED + * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, ALL OF + * WHICH ARE HEREBY DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT + * OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + * USE OF THIS SOFTWARE, EVEN IF NOT ADVISED OF THE POSSIBILITY OF SUCH + * DAMAGE. + */ + +#include <linux/module.h> +#include <crypto/rng.h> +#include <linux/random.h> +#include <crypto/if_alg.h> +#include <linux/net.h> +#include <net/sock.h> + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Stephan Mueller <smueller@chronox.de>"); +MODULE_DESCRIPTION("User-space interface for random number generators"); + +struct rng_ctx { +#define MAXSIZE 128 + unsigned int len; + struct crypto_rng *drng; +}; + +static int rng_recvmsg(struct kiocb *unused, struct socket *sock, + struct msghdr *msg, size_t len, int flags) +{ + struct sock *sk = sock->sk; + struct alg_sock *ask = alg_sk(sk); + struct rng_ctx *ctx = ask->private; + int err = -EFAULT; + int genlen = 0; + u8 result[MAXSIZE]; + + if (len == 0) + return 0; + if (len > MAXSIZE) + len = MAXSIZE; + + /* + * although not strictly needed, this is a precaution against coding + * errors + */ + memset(result, 0, len); + + /* + * The enforcement of a proper seeding of an RNG is done within an + * RNG implementation. Some RNGs (DRBG, krng) do not need specific + * seeding as they automatically seed. The X9.31 DRNG will return + * an error if it was not seeded properly. + */ + genlen = crypto_rng_get_bytes(ctx->drng, result, len); + if (genlen < 0) + return genlen; + + err = memcpy_to_msg(msg, result, len); + memzero_explicit(result, genlen); + + return err ? err : len; +} + +static struct proto_ops algif_rng_ops = { + .family = PF_ALG, + + .connect = sock_no_connect, + .socketpair = sock_no_socketpair, + .getname = sock_no_getname, + .ioctl = sock_no_ioctl, + .listen = sock_no_listen, + .shutdown = sock_no_shutdown, + .getsockopt = sock_no_getsockopt, + .mmap = sock_no_mmap, + .bind = sock_no_bind, + .accept = sock_no_accept, + .setsockopt = sock_no_setsockopt, + .poll = sock_no_poll, + .sendmsg = sock_no_sendmsg, + .sendpage = sock_no_sendpage, + + .release = af_alg_release, + .recvmsg = rng_recvmsg, +}; + +static void *rng_bind(const char *name, u32 type, u32 mask) +{ + return crypto_alloc_rng(name, type, mask); +} + +static void rng_release(void *private) +{ + crypto_free_rng(private); +} + +static void rng_sock_destruct(struct sock *sk) +{ + struct alg_sock *ask = alg_sk(sk); + struct rng_ctx *ctx = ask->private; + + sock_kfree_s(sk, ctx, ctx->len); + af_alg_release_parent(sk); +} + +static int rng_accept_parent(void *private, struct sock *sk) +{ + struct rng_ctx *ctx; + struct alg_sock *ask = alg_sk(sk); + unsigned int len = sizeof(*ctx); + + ctx = sock_kmalloc(sk, len, GFP_KERNEL); + if (!ctx) + return -ENOMEM; + + ctx->len = len; + + /* + * No seeding done at that point -- if multiple accepts are + * done on one RNG instance, each resulting FD points to the same + * state of the RNG. + */ + + ctx->drng = private; + ask->private = ctx; + sk->sk_destruct = rng_sock_destruct; + + return 0; +} + +static int rng_setkey(void *private, const u8 *seed, unsigned int seedlen) +{ + /* + * Check whether seedlen is of sufficient size is done in RNG + * implementations. + */ + return crypto_rng_reset(private, (u8 *)seed, seedlen); +} + +static const struct af_alg_type algif_type_rng = { + .bind = rng_bind, + .release = rng_release, + .accept = rng_accept_parent, + .setkey = rng_setkey, + .ops = &algif_rng_ops, + .name = "rng", + .owner = THIS_MODULE +}; + +static int __init rng_init(void) +{ + return af_alg_register_type(&algif_type_rng); +} + +static void __exit rng_exit(void) +{ + int err = af_alg_unregister_type(&algif_type_rng); + BUG_ON(err); +} + +module_init(rng_init); +module_exit(rng_exit); diff --git a/crypto/algif_skcipher.c b/crypto/algif_skcipher.c index 6fc12c3..0c8a1e5 100644 --- a/crypto/algif_skcipher.c +++ b/crypto/algif_skcipher.c @@ -330,6 +330,7 @@ static int skcipher_sendmsg(struct kiocb *unused, struct socket *sock, sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); sg = sgl->sg; + sg_unmark_end(sg + sgl->cur); do { i = sgl->cur; plen = min_t(int, len, PAGE_SIZE); @@ -355,6 +356,9 @@ static int skcipher_sendmsg(struct kiocb *unused, struct socket *sock, sgl->cur++; } while (len && sgl->cur < MAX_SGL_ENTS); + if (!size) + sg_mark_end(sg + sgl->cur - 1); + ctx->merge = plen & (PAGE_SIZE - 1); } @@ -401,6 +405,10 @@ static ssize_t skcipher_sendpage(struct socket *sock, struct page *page, ctx->merge = 0; sgl = list_entry(ctx->tsgl.prev, struct skcipher_sg_list, list); + if (sgl->cur) + sg_unmark_end(sgl->sg + sgl->cur - 1); + + sg_mark_end(sgl->sg + sgl->cur); get_page(page); sg_set_page(sgl->sg + sgl->cur, page, size, offset); sgl->cur++; diff --git a/crypto/cts.c b/crypto/cts.c index bd94058..e467ec0 100644 --- a/crypto/cts.c +++ b/crypto/cts.c @@ -290,6 +290,9 @@ static struct crypto_instance *crypto_cts_alloc(struct rtattr **tb) if (!is_power_of_2(alg->cra_blocksize)) goto out_put_alg; + if (strncmp(alg->cra_name, "cbc(", 4)) + goto out_put_alg; + inst = crypto_alloc_instance("cts", alg); if (IS_ERR(inst)) goto out_put_alg; @@ -307,8 +310,6 @@ static struct crypto_instance *crypto_cts_alloc(struct rtattr **tb) inst->alg.cra_blkcipher.min_keysize = alg->cra_blkcipher.min_keysize; inst->alg.cra_blkcipher.max_keysize = alg->cra_blkcipher.max_keysize; - inst->alg.cra_blkcipher.geniv = "seqiv"; - inst->alg.cra_ctxsize = sizeof(struct crypto_cts_ctx); inst->alg.cra_init = crypto_cts_init_tfm; diff --git a/crypto/drbg.c b/crypto/drbg.c index d748a1d..d8ff16e 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -98,7 +98,6 @@ */ #include <crypto/drbg.h> -#include <linux/string.h> /*************************************************************** * Backend cipher definitions available to DRBG @@ -223,15 +222,6 @@ static inline unsigned short drbg_sec_strength(drbg_flag_t flags) * function. Thus, the function implicitly knows the size of the * buffer. * - * The FIPS test can be called in an endless loop until it returns - * true. Although the code looks like a potential for a deadlock, it - * is not the case, because returning a false cannot mathematically - * occur (except once when a reseed took place and the updated state - * would is now set up such that the generation of new value returns - * an identical one -- this is most unlikely and would happen only once). - * Thus, if this function repeatedly returns false and thus would cause - * a deadlock, the integrity of the entire kernel is lost. - * * @drbg DRBG handle * @buf output buffer of random data to be checked * @@ -258,6 +248,8 @@ static bool drbg_fips_continuous_test(struct drbg_state *drbg, return false; } ret = memcmp(drbg->prev, buf, drbg_blocklen(drbg)); + if (!ret) + panic("DRBG continuous self test failed\n"); memcpy(drbg->prev, buf, drbg_blocklen(drbg)); /* the test shall pass when the two compared values are not equal */ return ret != 0; @@ -498,9 +490,9 @@ static int drbg_ctr_df(struct drbg_state *drbg, ret = 0; out: - memzero_explicit(iv, drbg_blocklen(drbg)); - memzero_explicit(temp, drbg_statelen(drbg)); - memzero_explicit(pad, drbg_blocklen(drbg)); + memset(iv, 0, drbg_blocklen(drbg)); + memset(temp, 0, drbg_statelen(drbg)); + memset(pad, 0, drbg_blocklen(drbg)); return ret; } @@ -574,9 +566,9 @@ static int drbg_ctr_update(struct drbg_state *drbg, struct list_head *seed, ret = 0; out: - memzero_explicit(temp, drbg_statelen(drbg) + drbg_blocklen(drbg)); + memset(temp, 0, drbg_statelen(drbg) + drbg_blocklen(drbg)); if (2 != reseed) - memzero_explicit(df_data, drbg_statelen(drbg)); + memset(df_data, 0, drbg_statelen(drbg)); return ret; } @@ -634,7 +626,7 @@ static int drbg_ctr_generate(struct drbg_state *drbg, len = ret; out: - memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg)); + memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); return len; } @@ -872,7 +864,7 @@ static int drbg_hash_df(struct drbg_state *drbg, } out: - memzero_explicit(tmp, drbg_blocklen(drbg)); + memset(tmp, 0, drbg_blocklen(drbg)); return ret; } @@ -916,7 +908,7 @@ static int drbg_hash_update(struct drbg_state *drbg, struct list_head *seed, ret = drbg_hash_df(drbg, drbg->C, drbg_statelen(drbg), &datalist2); out: - memzero_explicit(drbg->scratchpad, drbg_statelen(drbg)); + memset(drbg->scratchpad, 0, drbg_statelen(drbg)); return ret; } @@ -951,7 +943,7 @@ static int drbg_hash_process_addtl(struct drbg_state *drbg, drbg->scratchpad, drbg_blocklen(drbg)); out: - memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg)); + memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); return ret; } @@ -998,7 +990,7 @@ static int drbg_hash_hashgen(struct drbg_state *drbg, } out: - memzero_explicit(drbg->scratchpad, + memset(drbg->scratchpad, 0, (drbg_statelen(drbg) + drbg_blocklen(drbg))); return len; } @@ -1047,7 +1039,7 @@ static int drbg_hash_generate(struct drbg_state *drbg, drbg_add_buf(drbg->V, drbg_statelen(drbg), u.req, 8); out: - memzero_explicit(drbg->scratchpad, drbg_blocklen(drbg)); + memset(drbg->scratchpad, 0, drbg_blocklen(drbg)); return len; } diff --git a/crypto/scatterwalk.c b/crypto/scatterwalk.c index 79ca227..3bd749c 100644 --- a/crypto/scatterwalk.c +++ b/crypto/scatterwalk.c @@ -62,7 +62,7 @@ static void scatterwalk_pagedone(struct scatter_walk *walk, int out, walk->offset += PAGE_SIZE - 1; walk->offset &= PAGE_MASK; if (walk->offset >= walk->sg->offset + walk->sg->length) - scatterwalk_start(walk, scatterwalk_sg_next(walk->sg)); + scatterwalk_start(walk, sg_next(walk->sg)); } } @@ -116,7 +116,7 @@ void scatterwalk_map_and_copy(void *buf, struct scatterlist *sg, break; offset += sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } scatterwalk_advance(&walk, start - offset); @@ -136,7 +136,7 @@ int scatterwalk_bytes_sglen(struct scatterlist *sg, int num_bytes) do { offset += sg->length; n++; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); /* num_bytes is too large */ if (unlikely(!sg && (num_bytes < offset))) diff --git a/crypto/seqiv.c b/crypto/seqiv.c index 9daa854c..b7bb9a2 100644 --- a/crypto/seqiv.c +++ b/crypto/seqiv.c @@ -267,6 +267,12 @@ static struct crypto_instance *seqiv_ablkcipher_alloc(struct rtattr **tb) if (IS_ERR(inst)) goto out; + if (inst->alg.cra_ablkcipher.ivsize < sizeof(u64)) { + skcipher_geniv_free(inst); + inst = ERR_PTR(-EINVAL); + goto out; + } + inst->alg.cra_ablkcipher.givencrypt = seqiv_givencrypt_first; inst->alg.cra_init = seqiv_init; @@ -287,6 +293,12 @@ static struct crypto_instance *seqiv_aead_alloc(struct rtattr **tb) if (IS_ERR(inst)) goto out; + if (inst->alg.cra_aead.ivsize < sizeof(u64)) { + aead_geniv_free(inst); + inst = ERR_PTR(-EINVAL); + goto out; + } + inst->alg.cra_aead.givencrypt = seqiv_aead_givencrypt_first; inst->alg.cra_init = seqiv_aead_init; diff --git a/crypto/tcrypt.c b/crypto/tcrypt.c index 1d864e9..4b9e23f 100644 --- a/crypto/tcrypt.c +++ b/crypto/tcrypt.c @@ -250,19 +250,19 @@ static void sg_init_aead(struct scatterlist *sg, char *xbuf[XBUFSIZE], int np = (buflen + PAGE_SIZE - 1)/PAGE_SIZE; int k, rem; - np = (np > XBUFSIZE) ? XBUFSIZE : np; - rem = buflen % PAGE_SIZE; if (np > XBUFSIZE) { rem = PAGE_SIZE; np = XBUFSIZE; + } else { + rem = buflen % PAGE_SIZE; } + sg_init_table(sg, np); - for (k = 0; k < np; ++k) { - if (k == (np-1)) - sg_set_buf(&sg[k], xbuf[k], rem); - else - sg_set_buf(&sg[k], xbuf[k], PAGE_SIZE); - } + np--; + for (k = 0; k < np; k++) + sg_set_buf(&sg[k], xbuf[k], PAGE_SIZE); + + sg_set_buf(&sg[k], xbuf[k], rem); } static void test_aead_speed(const char *algo, int enc, unsigned int secs, @@ -280,16 +280,20 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, struct scatterlist *sgout; const char *e; void *assoc; - char iv[MAX_IVLEN]; + char *iv; char *xbuf[XBUFSIZE]; char *xoutbuf[XBUFSIZE]; char *axbuf[XBUFSIZE]; unsigned int *b_size; unsigned int iv_len; + iv = kzalloc(MAX_IVLEN, GFP_KERNEL); + if (!iv) + return; + if (aad_size >= PAGE_SIZE) { pr_err("associate data length (%u) too big\n", aad_size); - return; + goto out_noxbuf; } if (enc == ENCRYPT) @@ -355,7 +359,7 @@ static void test_aead_speed(const char *algo, int enc, unsigned int secs, iv_len = crypto_aead_ivsize(tfm); if (iv_len) - memset(&iv, 0xff, iv_len); + memset(iv, 0xff, iv_len); crypto_aead_clear_flags(tfm, ~0); printk(KERN_INFO "test %u (%d bit key, %d byte blocks): ", @@ -408,6 +412,7 @@ out_nooutbuf: out_noaxbuf: testmgr_free_buf(xbuf); out_noxbuf: + kfree(iv); return; } @@ -764,10 +769,9 @@ static inline int do_one_ahash_op(struct ahash_request *req, int ret) if (ret == -EINPROGRESS || ret == -EBUSY) { struct tcrypt_result *tr = req->base.data; - ret = wait_for_completion_interruptible(&tr->completion); - if (!ret) - ret = tr->err; + wait_for_completion(&tr->completion); reinit_completion(&tr->completion); + ret = tr->err; } return ret; } @@ -993,10 +997,9 @@ static inline int do_one_acipher_op(struct ablkcipher_request *req, int ret) if (ret == -EINPROGRESS || ret == -EBUSY) { struct tcrypt_result *tr = req->base.data; - ret = wait_for_completion_interruptible(&tr->completion); - if (!ret) - ret = tr->err; + wait_for_completion(&tr->completion); reinit_completion(&tr->completion); + ret = tr->err; } return ret; diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 037368d..f4ed6d4 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -181,10 +181,9 @@ static void testmgr_free_buf(char *buf[XBUFSIZE]) static int wait_async_op(struct tcrypt_result *tr, int ret) { if (ret == -EINPROGRESS || ret == -EBUSY) { - ret = wait_for_completion_interruptible(&tr->completion); - if (!ret) - ret = tr->err; + wait_for_completion(&tr->completion); reinit_completion(&tr->completion); + ret = tr->err; } return ret; } @@ -353,12 +352,11 @@ static int __test_hash(struct crypto_ahash *tfm, struct hash_testvec *template, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &tresult.completion); - if (!ret && !(ret = tresult.err)) { - reinit_completion(&tresult.completion); + wait_for_completion(&tresult.completion); + reinit_completion(&tresult.completion); + ret = tresult.err; + if (!ret) break; - } /* fall through */ default: printk(KERN_ERR "alg: hash: digest failed " @@ -431,7 +429,7 @@ static int __test_aead(struct crypto_aead *tfm, int enc, struct scatterlist *sgout; const char *e, *d; struct tcrypt_result result; - unsigned int authsize; + unsigned int authsize, iv_len; void *input; void *output; void *assoc; @@ -502,10 +500,11 @@ static int __test_aead(struct crypto_aead *tfm, int enc, memcpy(input, template[i].input, template[i].ilen); memcpy(assoc, template[i].assoc, template[i].alen); + iv_len = crypto_aead_ivsize(tfm); if (template[i].iv) - memcpy(iv, template[i].iv, MAX_IVLEN); + memcpy(iv, template[i].iv, iv_len); else - memset(iv, 0, MAX_IVLEN); + memset(iv, 0, iv_len); crypto_aead_clear_flags(tfm, ~0); if (template[i].wk) @@ -569,12 +568,11 @@ static int __test_aead(struct crypto_aead *tfm, int enc, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &result.completion); - if (!ret && !(ret = result.err)) { - reinit_completion(&result.completion); + wait_for_completion(&result.completion); + reinit_completion(&result.completion); + ret = result.err; + if (!ret) break; - } case -EBADMSG: if (template[i].novrfy) /* verification failure was expected */ @@ -720,12 +718,11 @@ static int __test_aead(struct crypto_aead *tfm, int enc, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &result.completion); - if (!ret && !(ret = result.err)) { - reinit_completion(&result.completion); + wait_for_completion(&result.completion); + reinit_completion(&result.completion); + ret = result.err; + if (!ret) break; - } case -EBADMSG: if (template[i].novrfy) /* verification failure was expected */ @@ -1002,12 +999,11 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &result.completion); - if (!ret && !((ret = result.err))) { - reinit_completion(&result.completion); + wait_for_completion(&result.completion); + reinit_completion(&result.completion); + ret = result.err; + if (!ret) break; - } /* fall through */ default: pr_err("alg: skcipher%s: %s failed on test %d for %s: ret=%d\n", @@ -1097,12 +1093,11 @@ static int __test_skcipher(struct crypto_ablkcipher *tfm, int enc, break; case -EINPROGRESS: case -EBUSY: - ret = wait_for_completion_interruptible( - &result.completion); - if (!ret && !((ret = result.err))) { - reinit_completion(&result.completion); + wait_for_completion(&result.completion); + reinit_completion(&result.completion); + ret = result.err; + if (!ret) break; - } /* fall through */ default: pr_err("alg: skcipher%s: %s failed on chunk test %d for %s: ret=%d\n", @@ -3299,6 +3294,7 @@ static const struct alg_test_desc alg_test_descs[] = { }, { .alg = "rfc4106(gcm(aes))", .test = alg_test_aead, + .fips_allowed = 1, .suite = { .aead = { .enc = { diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 1500cfd..32a8a86 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -42,6 +42,7 @@ #include <linux/delay.h> #include <linux/slab.h> #include <linux/random.h> +#include <linux/err.h> #include <asm/uaccess.h> @@ -53,7 +54,10 @@ static struct hwrng *current_rng; static struct task_struct *hwrng_fill; static LIST_HEAD(rng_list); +/* Protects rng_list and current_rng */ static DEFINE_MUTEX(rng_mutex); +/* Protects rng read functions, data_avail, rng_buffer and rng_fillbuf */ +static DEFINE_MUTEX(reading_mutex); static int data_avail; static u8 *rng_buffer, *rng_fillbuf; static unsigned short current_quality; @@ -66,6 +70,8 @@ module_param(default_quality, ushort, 0644); MODULE_PARM_DESC(default_quality, "default entropy content of hwrng per mill"); +static void drop_current_rng(void); +static int hwrng_init(struct hwrng *rng); static void start_khwrngd(void); static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, @@ -81,13 +87,83 @@ static void add_early_randomness(struct hwrng *rng) unsigned char bytes[16]; int bytes_read; + mutex_lock(&reading_mutex); bytes_read = rng_get_data(rng, bytes, sizeof(bytes), 1); + mutex_unlock(&reading_mutex); if (bytes_read > 0) add_device_randomness(bytes, bytes_read); } -static inline int hwrng_init(struct hwrng *rng) +static inline void cleanup_rng(struct kref *kref) { + struct hwrng *rng = container_of(kref, struct hwrng, ref); + + if (rng->cleanup) + rng->cleanup(rng); + + complete(&rng->cleanup_done); +} + +static int set_current_rng(struct hwrng *rng) +{ + int err; + + BUG_ON(!mutex_is_locked(&rng_mutex)); + + err = hwrng_init(rng); + if (err) + return err; + + drop_current_rng(); + current_rng = rng; + + return 0; +} + +static void drop_current_rng(void) +{ + BUG_ON(!mutex_is_locked(&rng_mutex)); + if (!current_rng) + return; + + /* decrease last reference for triggering the cleanup */ + kref_put(¤t_rng->ref, cleanup_rng); + current_rng = NULL; +} + +/* Returns ERR_PTR(), NULL or refcounted hwrng */ +static struct hwrng *get_current_rng(void) +{ + struct hwrng *rng; + + if (mutex_lock_interruptible(&rng_mutex)) + return ERR_PTR(-ERESTARTSYS); + + rng = current_rng; + if (rng) + kref_get(&rng->ref); + + mutex_unlock(&rng_mutex); + return rng; +} + +static void put_rng(struct hwrng *rng) +{ + /* + * Hold rng_mutex here so we serialize in case they set_current_rng + * on rng again immediately. + */ + mutex_lock(&rng_mutex); + if (rng) + kref_put(&rng->ref, cleanup_rng); + mutex_unlock(&rng_mutex); +} + +static int hwrng_init(struct hwrng *rng) +{ + if (kref_get_unless_zero(&rng->ref)) + goto skip_init; + if (rng->init) { int ret; @@ -95,6 +171,11 @@ static inline int hwrng_init(struct hwrng *rng) if (ret) return ret; } + + kref_init(&rng->ref); + reinit_completion(&rng->cleanup_done); + +skip_init: add_early_randomness(rng); current_quality = rng->quality ? : default_quality; @@ -108,12 +189,6 @@ static inline int hwrng_init(struct hwrng *rng) return 0; } -static inline void hwrng_cleanup(struct hwrng *rng) -{ - if (rng && rng->cleanup) - rng->cleanup(rng); -} - static int rng_dev_open(struct inode *inode, struct file *filp) { /* enforce read-only access to this chrdev */ @@ -128,6 +203,7 @@ static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, int wait) { int present; + BUG_ON(!mutex_is_locked(&reading_mutex)); if (rng->read) return rng->read(rng, (void *)buffer, size, wait); @@ -148,25 +224,27 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, ssize_t ret = 0; int err = 0; int bytes_read, len; + struct hwrng *rng; while (size) { - if (mutex_lock_interruptible(&rng_mutex)) { - err = -ERESTARTSYS; + rng = get_current_rng(); + if (IS_ERR(rng)) { + err = PTR_ERR(rng); goto out; } - - if (!current_rng) { + if (!rng) { err = -ENODEV; - goto out_unlock; + goto out; } + mutex_lock(&reading_mutex); if (!data_avail) { - bytes_read = rng_get_data(current_rng, rng_buffer, + bytes_read = rng_get_data(rng, rng_buffer, rng_buffer_size(), !(filp->f_flags & O_NONBLOCK)); if (bytes_read < 0) { err = bytes_read; - goto out_unlock; + goto out_unlock_reading; } data_avail = bytes_read; } @@ -174,7 +252,7 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, if (!data_avail) { if (filp->f_flags & O_NONBLOCK) { err = -EAGAIN; - goto out_unlock; + goto out_unlock_reading; } } else { len = data_avail; @@ -186,14 +264,15 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, if (copy_to_user(buf + ret, rng_buffer + data_avail, len)) { err = -EFAULT; - goto out_unlock; + goto out_unlock_reading; } size -= len; ret += len; } - mutex_unlock(&rng_mutex); + mutex_unlock(&reading_mutex); + put_rng(rng); if (need_resched()) schedule_timeout_interruptible(1); @@ -205,8 +284,10 @@ static ssize_t rng_dev_read(struct file *filp, char __user *buf, } out: return ret ? : err; -out_unlock: - mutex_unlock(&rng_mutex); + +out_unlock_reading: + mutex_unlock(&reading_mutex); + put_rng(rng); goto out; } @@ -239,16 +320,9 @@ static ssize_t hwrng_attr_current_store(struct device *dev, err = -ENODEV; list_for_each_entry(rng, &rng_list, list) { if (strcmp(rng->name, buf) == 0) { - if (rng == current_rng) { - err = 0; - break; - } - err = hwrng_init(rng); - if (err) - break; - hwrng_cleanup(current_rng); - current_rng = rng; err = 0; + if (rng != current_rng) + err = set_current_rng(rng); break; } } @@ -261,17 +335,15 @@ static ssize_t hwrng_attr_current_show(struct device *dev, struct device_attribute *attr, char *buf) { - int err; ssize_t ret; - const char *name = "none"; + struct hwrng *rng; - err = mutex_lock_interruptible(&rng_mutex); - if (err) - return -ERESTARTSYS; - if (current_rng) - name = current_rng->name; - ret = snprintf(buf, PAGE_SIZE, "%s\n", name); - mutex_unlock(&rng_mutex); + rng = get_current_rng(); + if (IS_ERR(rng)) + return PTR_ERR(rng); + + ret = snprintf(buf, PAGE_SIZE, "%s\n", rng ? rng->name : "none"); + put_rng(rng); return ret; } @@ -305,14 +377,14 @@ static DEVICE_ATTR(rng_available, S_IRUGO, NULL); -static void unregister_miscdev(void) +static void __exit unregister_miscdev(void) { device_remove_file(rng_miscdev.this_device, &dev_attr_rng_available); device_remove_file(rng_miscdev.this_device, &dev_attr_rng_current); misc_deregister(&rng_miscdev); } -static int register_miscdev(void) +static int __init register_miscdev(void) { int err; @@ -342,15 +414,22 @@ static int hwrng_fillfn(void *unused) long rc; while (!kthread_should_stop()) { - if (!current_rng) + struct hwrng *rng; + + rng = get_current_rng(); + if (IS_ERR(rng) || !rng) break; - rc = rng_get_data(current_rng, rng_fillbuf, + mutex_lock(&reading_mutex); + rc = rng_get_data(rng, rng_fillbuf, rng_buffer_size(), 1); + mutex_unlock(&reading_mutex); + put_rng(rng); if (rc <= 0) { pr_warn("hwrng: no data available\n"); msleep_interruptible(10000); continue; } + /* Outside lock, sure, but y'know: randomness. */ add_hwgenerator_randomness((void *)rng_fillbuf, rc, rc * current_quality * 8 >> 10); } @@ -400,23 +479,16 @@ int hwrng_register(struct hwrng *rng) goto out_unlock; } + init_completion(&rng->cleanup_done); + complete(&rng->cleanup_done); + old_rng = current_rng; - if (!old_rng) { - err = hwrng_init(rng); - if (err) - goto out_unlock; - current_rng = rng; - } err = 0; if (!old_rng) { - err = register_miscdev(); - if (err) { - hwrng_cleanup(rng); - current_rng = NULL; + err = set_current_rng(rng); + if (err) goto out_unlock; - } } - INIT_LIST_HEAD(&rng->list); list_add_tail(&rng->list, &rng_list); if (old_rng && !rng->init) { @@ -439,42 +511,49 @@ EXPORT_SYMBOL_GPL(hwrng_register); void hwrng_unregister(struct hwrng *rng) { - int err; - mutex_lock(&rng_mutex); list_del(&rng->list); if (current_rng == rng) { - hwrng_cleanup(rng); - if (list_empty(&rng_list)) { - current_rng = NULL; - } else { - current_rng = list_entry(rng_list.prev, struct hwrng, list); - err = hwrng_init(current_rng); - if (err) - current_rng = NULL; + drop_current_rng(); + if (!list_empty(&rng_list)) { + struct hwrng *tail; + + tail = list_entry(rng_list.prev, struct hwrng, list); + + set_current_rng(tail); } } + if (list_empty(&rng_list)) { - unregister_miscdev(); + mutex_unlock(&rng_mutex); if (hwrng_fill) kthread_stop(hwrng_fill); - } + } else + mutex_unlock(&rng_mutex); - mutex_unlock(&rng_mutex); + wait_for_completion(&rng->cleanup_done); } EXPORT_SYMBOL_GPL(hwrng_unregister); -static void __exit hwrng_exit(void) +static int __init hwrng_modinit(void) +{ + return register_miscdev(); +} + +static void __exit hwrng_modexit(void) { mutex_lock(&rng_mutex); BUG_ON(current_rng); kfree(rng_buffer); kfree(rng_fillbuf); mutex_unlock(&rng_mutex); + + unregister_miscdev(); } -module_exit(hwrng_exit); +module_init(hwrng_modinit); +module_exit(hwrng_modexit); MODULE_DESCRIPTION("H/W Random Number Generator (RNG) driver"); MODULE_LICENSE("GPL"); diff --git a/drivers/char/hw_random/virtio-rng.c b/drivers/char/hw_random/virtio-rng.c index 72295ea..3fa2f8a 100644 --- a/drivers/char/hw_random/virtio-rng.c +++ b/drivers/char/hw_random/virtio-rng.c @@ -39,7 +39,6 @@ struct virtrng_info { bool hwrng_removed; }; - static void random_recv_done(struct virtqueue *vq) { struct virtrng_info *vi = vq->vdev->priv; diff --git a/drivers/crypto/amcc/crypto4xx_sa.c b/drivers/crypto/amcc/crypto4xx_sa.c index de8a7a4..69182e2 100644 --- a/drivers/crypto/amcc/crypto4xx_sa.c +++ b/drivers/crypto/amcc/crypto4xx_sa.c @@ -34,29 +34,6 @@ #include "crypto4xx_sa.h" #include "crypto4xx_core.h" -u32 get_dynamic_sa_offset_iv_field(struct crypto4xx_ctx *ctx) -{ - u32 offset; - union dynamic_sa_contents cts; - - if (ctx->direction == DIR_INBOUND) - cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_in))->sa_contents; - else - cts.w = ((struct dynamic_sa_ctl *)(ctx->sa_out))->sa_contents; - offset = cts.bf.key_size - + cts.bf.inner_size - + cts.bf.outer_size - + cts.bf.spi - + cts.bf.seq_num0 - + cts.bf.seq_num1 - + cts.bf.seq_num_mask0 - + cts.bf.seq_num_mask1 - + cts.bf.seq_num_mask2 - + cts.bf.seq_num_mask3; - - return sizeof(struct dynamic_sa_ctl) + offset * 4; -} - u32 get_dynamic_sa_offset_state_ptr_field(struct crypto4xx_ctx *ctx) { u32 offset; diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 53d1c33..6597aac 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -673,9 +673,9 @@ err_map_out: dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); err_map_in: +err_alloc: free_page((unsigned long)dd->buf_out); free_page((unsigned long)dd->buf_in); -err_alloc: if (err) pr_err("error: %d\n", err); return err; diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index d94f07c..34db04a 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -102,10 +102,6 @@ struct atmel_sha_ctx { struct atmel_sha_dev *dd; unsigned long flags; - - /* fallback stuff */ - struct crypto_shash *fallback; - }; #define ATMEL_SHA_QUEUE_LENGTH 50 @@ -974,19 +970,8 @@ static int atmel_sha_digest(struct ahash_request *req) return atmel_sha_init(req) ?: atmel_sha_finup(req); } -static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) +static int atmel_sha_cra_init(struct crypto_tfm *tfm) { - struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm); - const char *alg_name = crypto_tfm_alg_name(tfm); - - /* Allocate a fallback and abort if it failed. */ - tctx->fallback = crypto_alloc_shash(alg_name, 0, - CRYPTO_ALG_NEED_FALLBACK); - if (IS_ERR(tctx->fallback)) { - pr_err("atmel-sha: fallback driver '%s' could not be loaded.\n", - alg_name); - return PTR_ERR(tctx->fallback); - } crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), sizeof(struct atmel_sha_reqctx) + SHA_BUFFER_LEN + SHA512_BLOCK_SIZE); @@ -994,19 +979,6 @@ static int atmel_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) return 0; } -static int atmel_sha_cra_init(struct crypto_tfm *tfm) -{ - return atmel_sha_cra_init_alg(tfm, NULL); -} - -static void atmel_sha_cra_exit(struct crypto_tfm *tfm) -{ - struct atmel_sha_ctx *tctx = crypto_tfm_ctx(tfm); - - crypto_free_shash(tctx->fallback); - tctx->fallback = NULL; -} - static struct ahash_alg sha_1_256_algs[] = { { .init = atmel_sha_init, @@ -1020,14 +992,12 @@ static struct ahash_alg sha_1_256_algs[] = { .cra_name = "sha1", .cra_driver_name = "atmel-sha1", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA1_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }, @@ -1043,14 +1013,12 @@ static struct ahash_alg sha_1_256_algs[] = { .cra_name = "sha256", .cra_driver_name = "atmel-sha256", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA256_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }, @@ -1068,14 +1036,12 @@ static struct ahash_alg sha_224_alg = { .cra_name = "sha224", .cra_driver_name = "atmel-sha224", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA224_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }; @@ -1093,14 +1059,12 @@ static struct ahash_alg sha_384_512_algs[] = { .cra_name = "sha384", .cra_driver_name = "atmel-sha384", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA384_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0x3, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }, @@ -1116,14 +1080,12 @@ static struct ahash_alg sha_384_512_algs[] = { .cra_name = "sha512", .cra_driver_name = "atmel-sha512", .cra_priority = 100, - .cra_flags = CRYPTO_ALG_ASYNC | - CRYPTO_ALG_NEED_FALLBACK, + .cra_flags = CRYPTO_ALG_ASYNC, .cra_blocksize = SHA512_BLOCK_SIZE, .cra_ctxsize = sizeof(struct atmel_sha_ctx), .cra_alignmask = 0x3, .cra_module = THIS_MODULE, .cra_init = atmel_sha_cra_init, - .cra_exit = atmel_sha_cra_exit, } } }, diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index 5e7c896..258772d 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -376,9 +376,9 @@ err_map_out: dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen, DMA_TO_DEVICE); err_map_in: +err_alloc: free_page((unsigned long)dd->buf_out); free_page((unsigned long)dd->buf_in); -err_alloc: if (err) pr_err("error: %d\n", err); return err; diff --git a/drivers/crypto/bfin_crc.c b/drivers/crypto/bfin_crc.c index 9ae149bd..d9af940 100644 --- a/drivers/crypto/bfin_crc.c +++ b/drivers/crypto/bfin_crc.c @@ -110,7 +110,7 @@ static int sg_count(struct scatterlist *sg_list) while (!sg_is_last(sg)) { sg_nents++; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return sg_nents; @@ -744,7 +744,7 @@ static int __init bfin_crypto_crc_mod_init(void) ret = platform_driver_register(&bfin_crypto_crc_driver); if (ret) { - pr_info(KERN_ERR "unable to register driver\n"); + pr_err("unable to register driver\n"); return ret; } diff --git a/drivers/crypto/caam/caamalg.c b/drivers/crypto/caam/caamalg.c index 3187400..29071a1 100644 --- a/drivers/crypto/caam/caamalg.c +++ b/drivers/crypto/caam/caamalg.c @@ -2532,7 +2532,7 @@ static void init_ablkcipher_job(u32 *sh_desc, dma_addr_t ptr, in_options = 0; } else { src_dma = edesc->sec4_sg_dma; - sec4_sg_index += (iv_contig ? 0 : 1) + edesc->src_nents; + sec4_sg_index += edesc->src_nents + 1; in_options = LDST_SGF; } append_seq_in_ptr(desc, src_dma, req->nbytes + ivsize, in_options); @@ -2714,10 +2714,10 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (!all_contig) { if (!is_gcm) { sg_to_sec4_sg(req->assoc, - (assoc_nents ? : 1), + assoc_nents, edesc->sec4_sg + sec4_sg_index, 0); - sec4_sg_index += assoc_nents ? : 1; + sec4_sg_index += assoc_nents; } dma_to_sec4_sg_one(edesc->sec4_sg + sec4_sg_index, @@ -2726,17 +2726,17 @@ static struct aead_edesc *aead_edesc_alloc(struct aead_request *req, if (is_gcm) { sg_to_sec4_sg(req->assoc, - (assoc_nents ? : 1), + assoc_nents, edesc->sec4_sg + sec4_sg_index, 0); - sec4_sg_index += assoc_nents ? : 1; + sec4_sg_index += assoc_nents; } sg_to_sec4_sg_last(req->src, - (src_nents ? : 1), + src_nents, edesc->sec4_sg + sec4_sg_index, 0); - sec4_sg_index += src_nents ? : 1; + sec4_sg_index += src_nents; } if (dst_nents) { sg_to_sec4_sg_last(req->dst, dst_nents, diff --git a/drivers/crypto/caam/ctrl.c b/drivers/crypto/caam/ctrl.c index 70f1e6f..efba4cc 100644 --- a/drivers/crypto/caam/ctrl.c +++ b/drivers/crypto/caam/ctrl.c @@ -175,13 +175,10 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, { struct caam_drv_private *ctrlpriv = dev_get_drvdata(ctrldev); struct caam_ctrl __iomem *ctrl; - struct rng4tst __iomem *r4tst; u32 *desc, status, rdsta_val; int ret = 0, sh_idx; ctrl = (struct caam_ctrl __iomem *)ctrlpriv->ctrl; - r4tst = &ctrl->r4tst[0]; - desc = kmalloc(CAAM_CMD_SZ * 7, GFP_KERNEL); if (!desc) return -ENOMEM; @@ -209,8 +206,7 @@ static int instantiate_rng(struct device *ctrldev, int state_handle_mask, * without any error (HW optimizations for later * CAAM eras), then try again. */ - rdsta_val = - rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK; + rdsta_val = rd_reg32(&ctrl->r4tst[0].rdsta) & RDSTA_IFMASK; if (status || !(rdsta_val & (1 << sh_idx))) ret = -EAGAIN; if (ret) diff --git a/drivers/crypto/caam/error.c b/drivers/crypto/caam/error.c index 66d73bf..33e41ea 100644 --- a/drivers/crypto/caam/error.c +++ b/drivers/crypto/caam/error.c @@ -151,10 +151,15 @@ static void report_ccb_status(struct device *jrdev, const u32 status, else snprintf(err_err_code, sizeof(err_err_code), "%02x", err_id); - dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n", - status, error, idx_str, idx, - cha_str, cha_err_code, - err_str, err_err_code); + /* + * CCB ICV check failures are part of normal operation life; + * we leave the upper layers to do what they want with them. + */ + if (err_id != JRSTA_CCBERR_ERRID_ICVCHK) + dev_err(jrdev, "%08x: %s: %s %d: %s%s: %s%s\n", + status, error, idx_str, idx, + cha_str, cha_err_code, + err_str, err_err_code); } static void report_jump_status(struct device *jrdev, const u32 status, diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 9b3ef1bc..b8b5d47 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -384,30 +384,28 @@ static int caam_jr_init(struct device *dev) if (error) { dev_err(dev, "can't connect JobR %d interrupt (%d)\n", jrp->ridx, jrp->irq); - irq_dispose_mapping(jrp->irq); - jrp->irq = 0; - return -EINVAL; + goto out_kill_deq; } error = caam_reset_hw_jr(dev); if (error) - return error; + goto out_free_irq; + error = -ENOMEM; jrp->inpring = dma_alloc_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, &inpbusaddr, GFP_KERNEL); + if (!jrp->inpring) + goto out_free_irq; jrp->outring = dma_alloc_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, &outbusaddr, GFP_KERNEL); + if (!jrp->outring) + goto out_free_inpring; jrp->entinfo = kzalloc(sizeof(struct caam_jrentry_info) * JOBR_DEPTH, GFP_KERNEL); - - if ((jrp->inpring == NULL) || (jrp->outring == NULL) || - (jrp->entinfo == NULL)) { - dev_err(dev, "can't allocate job rings for %d\n", - jrp->ridx); - return -ENOMEM; - } + if (!jrp->entinfo) + goto out_free_outring; for (i = 0; i < JOBR_DEPTH; i++) jrp->entinfo[i].desc_addr_dma = !0; @@ -434,6 +432,19 @@ static int caam_jr_init(struct device *dev) (JOBR_INTC_TIME_THLD << JRCFG_ICTT_SHIFT)); return 0; + +out_free_outring: + dma_free_coherent(dev, sizeof(struct jr_outentry) * JOBR_DEPTH, + jrp->outring, outbusaddr); +out_free_inpring: + dma_free_coherent(dev, sizeof(dma_addr_t) * JOBR_DEPTH, + jrp->inpring, inpbusaddr); + dev_err(dev, "can't allocate job rings for %d\n", jrp->ridx); +out_free_irq: + free_irq(jrp->irq, dev); +out_kill_deq: + tasklet_kill(&jrp->irqtask); + return error; } @@ -484,8 +495,10 @@ static int caam_jr_probe(struct platform_device *pdev) /* Now do the platform independent part */ error = caam_jr_init(jrdev); /* now turn on hardware */ - if (error) + if (error) { + irq_dispose_mapping(jrpriv->irq); return error; + } jrpriv->dev = jrdev; spin_lock(&driver_data.jr_alloc_lock); diff --git a/drivers/crypto/caam/sg_sw_sec4.h b/drivers/crypto/caam/sg_sw_sec4.h index ce28a56..3b91821 100644 --- a/drivers/crypto/caam/sg_sw_sec4.h +++ b/drivers/crypto/caam/sg_sw_sec4.h @@ -37,7 +37,7 @@ sg_to_sec4_sg(struct scatterlist *sg, int sg_count, dma_to_sec4_sg_one(sec4_sg_ptr, sg_dma_address(sg), sg_dma_len(sg), offset); sec4_sg_ptr++; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); sg_count--; } return sec4_sg_ptr - 1; @@ -67,7 +67,7 @@ static inline int __sg_count(struct scatterlist *sg_list, int nbytes, nbytes -= sg->length; if (!sg_is_last(sg) && (sg + 1)->length == 0) *chained = true; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return sg_nents; @@ -93,7 +93,7 @@ static int dma_map_sg_chained(struct device *dev, struct scatterlist *sg, int i; for (i = 0; i < nents; i++) { dma_map_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } } else { dma_map_sg(dev, sg, nents, dir); @@ -109,7 +109,7 @@ static int dma_unmap_sg_chained(struct device *dev, struct scatterlist *sg, int i; for (i = 0; i < nents; i++) { dma_unmap_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } } else { dma_unmap_sg(dev, sg, nents, dir); diff --git a/drivers/crypto/ccp/ccp-dev.c b/drivers/crypto/ccp/ccp-dev.c index c6e6171..ca29c12 100644 --- a/drivers/crypto/ccp/ccp-dev.c +++ b/drivers/crypto/ccp/ccp-dev.c @@ -583,6 +583,7 @@ bool ccp_queues_suspended(struct ccp_device *ccp) #ifdef CONFIG_X86 static const struct x86_cpu_id ccp_support[] = { { X86_VENDOR_AMD, 22, }, + { }, }; #endif diff --git a/drivers/crypto/ixp4xx_crypto.c b/drivers/crypto/ixp4xx_crypto.c index f757a0f..48f4535 100644 --- a/drivers/crypto/ixp4xx_crypto.c +++ b/drivers/crypto/ixp4xx_crypto.c @@ -784,7 +784,7 @@ static struct buffer_desc *chainup_buffers(struct device *dev, struct buffer_desc *buf, gfp_t flags, enum dma_data_direction dir) { - for (;nbytes > 0; sg = scatterwalk_sg_next(sg)) { + for (; nbytes > 0; sg = sg_next(sg)) { unsigned len = min(nbytes, sg->length); struct buffer_desc *next_buf; u32 next_buf_phys; @@ -982,7 +982,7 @@ static int hmac_inconsistent(struct scatterlist *sg, unsigned start, break; offset += sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return (start + nbytes > offset + sg->length); } diff --git a/drivers/crypto/nx/nx.c b/drivers/crypto/nx/nx.c index a392465..1da6dc5 100644 --- a/drivers/crypto/nx/nx.c +++ b/drivers/crypto/nx/nx.c @@ -177,7 +177,7 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst, break; offset += sg_src->length; - sg_src = scatterwalk_sg_next(sg_src); + sg_src = sg_next(sg_src); } /* start - offset is the number of bytes to advance in the scatterlist @@ -187,9 +187,9 @@ struct nx_sg *nx_walk_and_build(struct nx_sg *nx_dst, while (len && (nx_sg - nx_dst) < sglen) { n = scatterwalk_clamp(&walk, len); if (!n) { - /* In cases where we have scatterlist chain scatterwalk_sg_next + /* In cases where we have scatterlist chain sg_next * handles with it properly */ - scatterwalk_start(&walk, scatterwalk_sg_next(walk.sg)); + scatterwalk_start(&walk, sg_next(walk.sg)); n = scatterwalk_clamp(&walk, len); } dst = scatterwalk_map(&walk); diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index f79dd41..42f95a4 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -994,7 +994,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) scatterwalk_advance(&dd->in_walk, 4); if (dd->in_sg->length == _calc_walked(in)) { - dd->in_sg = scatterwalk_sg_next(dd->in_sg); + dd->in_sg = sg_next(dd->in_sg); if (dd->in_sg) { scatterwalk_start(&dd->in_walk, dd->in_sg); @@ -1026,7 +1026,7 @@ static irqreturn_t omap_aes_irq(int irq, void *dev_id) *dst = omap_aes_read(dd, AES_REG_DATA_N(dd, i)); scatterwalk_advance(&dd->out_walk, 4); if (dd->out_sg->length == _calc_walked(out)) { - dd->out_sg = scatterwalk_sg_next(dd->out_sg); + dd->out_sg = sg_next(dd->out_sg); if (dd->out_sg) { scatterwalk_start(&dd->out_walk, dd->out_sg); diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index e350f5b..4630709 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -921,7 +921,7 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) scatterwalk_advance(&dd->in_walk, 4); if (dd->in_sg->length == _calc_walked(in)) { - dd->in_sg = scatterwalk_sg_next(dd->in_sg); + dd->in_sg = sg_next(dd->in_sg); if (dd->in_sg) { scatterwalk_start(&dd->in_walk, dd->in_sg); @@ -953,7 +953,7 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) *dst = omap_des_read(dd, DES_REG_DATA_N(dd, i)); scatterwalk_advance(&dd->out_walk, 4); if (dd->out_sg->length == _calc_walked(out)) { - dd->out_sg = scatterwalk_sg_next(dd->out_sg); + dd->out_sg = sg_next(dd->out_sg); if (dd->out_sg) { scatterwalk_start(&dd->out_walk, dd->out_sg); @@ -965,9 +965,9 @@ static irqreturn_t omap_des_irq(int irq, void *dev_id) } } - dd->total -= DES_BLOCK_SIZE; + BUG_ON(dd->total < DES_BLOCK_SIZE); - BUG_ON(dd->total < 0); + dd->total -= DES_BLOCK_SIZE; /* Clear IRQ status */ status &= ~DES_REG_IRQ_DATA_OUT; diff --git a/drivers/crypto/qat/qat_common/adf_accel_devices.h b/drivers/crypto/qat/qat_common/adf_accel_devices.h index 2ed4256..19c0efa 100644 --- a/drivers/crypto/qat/qat_common/adf_accel_devices.h +++ b/drivers/crypto/qat/qat_common/adf_accel_devices.h @@ -47,7 +47,6 @@ #ifndef ADF_ACCEL_DEVICES_H_ #define ADF_ACCEL_DEVICES_H_ #include <linux/module.h> -#include <linux/atomic.h> #include <linux/list.h> #include <linux/proc_fs.h> #include <linux/io.h> @@ -148,6 +147,11 @@ struct adf_hw_device_data { int (*alloc_irq)(struct adf_accel_dev *accel_dev); void (*free_irq)(struct adf_accel_dev *accel_dev); void (*enable_error_correction)(struct adf_accel_dev *accel_dev); + int (*init_admin_comms)(struct adf_accel_dev *accel_dev); + void (*exit_admin_comms)(struct adf_accel_dev *accel_dev); + int (*init_arb)(struct adf_accel_dev *accel_dev); + void (*exit_arb)(struct adf_accel_dev *accel_dev); + void (*enable_ints)(struct adf_accel_dev *accel_dev); const char *fw_name; uint32_t pci_dev_id; uint32_t fuses; diff --git a/drivers/crypto/qat/qat_common/adf_aer.c b/drivers/crypto/qat/qat_common/adf_aer.c index 10ce4a2..fa1fef8 100644 --- a/drivers/crypto/qat/qat_common/adf_aer.c +++ b/drivers/crypto/qat/qat_common/adf_aer.c @@ -82,28 +82,15 @@ struct adf_reset_dev_data { struct work_struct reset_work; }; -#define PPDSTAT_OFFSET 0x7E static void adf_dev_restore(struct adf_accel_dev *accel_dev) { struct pci_dev *pdev = accel_to_pci_dev(accel_dev); struct pci_dev *parent = pdev->bus->self; - uint16_t ppdstat = 0, bridge_ctl = 0; - int pending = 0; + uint16_t bridge_ctl = 0; pr_info("QAT: Resetting device qat_dev%d\n", accel_dev->accel_id); - pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat); - pending = ppdstat & PCI_EXP_DEVSTA_TRPND; - if (pending) { - int ctr = 0; - - do { - msleep(100); - pci_read_config_word(pdev, PPDSTAT_OFFSET, &ppdstat); - pending = ppdstat & PCI_EXP_DEVSTA_TRPND; - } while (pending && ctr++ < 10); - } - if (pending) + if (!pci_wait_for_pending_transaction(pdev)) pr_info("QAT: Transaction still in progress. Proceeding\n"); pci_read_config_word(parent, PCI_BRIDGE_CONTROL, &bridge_ctl); @@ -125,8 +112,9 @@ static void adf_device_reset_worker(struct work_struct *work) adf_dev_restarting_notify(accel_dev); adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); adf_dev_restore(accel_dev); - if (adf_dev_start(accel_dev)) { + if (adf_dev_init(accel_dev) || adf_dev_start(accel_dev)) { /* The device hanged and we can't restart it so stop here */ dev_err(&GET_DEV(accel_dev), "Restart device failed\n"); kfree(reset_data); @@ -148,8 +136,8 @@ static int adf_dev_aer_schedule_reset(struct adf_accel_dev *accel_dev, { struct adf_reset_dev_data *reset_data; - if (adf_dev_started(accel_dev) && - !test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) + if (!adf_dev_started(accel_dev) || + test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) return 0; set_bit(ADF_STATUS_RESTARTING, &accel_dev->status); diff --git a/drivers/crypto/qat/qat_common/adf_cfg.c b/drivers/crypto/qat/qat_common/adf_cfg.c index aba7f1d..de16da9 100644 --- a/drivers/crypto/qat/qat_common/adf_cfg.c +++ b/drivers/crypto/qat/qat_common/adf_cfg.c @@ -50,6 +50,7 @@ #include <linux/seq_file.h> #include "adf_accel_devices.h" #include "adf_cfg.h" +#include "adf_common_drv.h" static DEFINE_MUTEX(qat_cfg_read_lock); @@ -159,6 +160,7 @@ void adf_cfg_del_all(struct adf_accel_dev *accel_dev) down_write(&dev_cfg_data->lock); adf_cfg_section_del_all(&dev_cfg_data->sec_list); up_write(&dev_cfg_data->lock); + clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); } /** diff --git a/drivers/crypto/qat/qat_common/adf_common_drv.h b/drivers/crypto/qat/qat_common/adf_common_drv.h index 5e8f9d4..a62e485 100644 --- a/drivers/crypto/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/qat/qat_common/adf_common_drv.h @@ -93,7 +93,7 @@ int adf_service_unregister(struct service_hndl *service); int adf_dev_init(struct adf_accel_dev *accel_dev); int adf_dev_start(struct adf_accel_dev *accel_dev); int adf_dev_stop(struct adf_accel_dev *accel_dev); -int adf_dev_shutdown(struct adf_accel_dev *accel_dev); +void adf_dev_shutdown(struct adf_accel_dev *accel_dev); int adf_ctl_dev_register(void); void adf_ctl_dev_unregister(void); diff --git a/drivers/crypto/qat/qat_common/adf_ctl_drv.c b/drivers/crypto/qat/qat_common/adf_ctl_drv.c index 7ee93f8..74207a6 100644 --- a/drivers/crypto/qat/qat_common/adf_ctl_drv.c +++ b/drivers/crypto/qat/qat_common/adf_ctl_drv.c @@ -282,6 +282,8 @@ static int adf_ctl_stop_devices(uint32_t id) if (adf_dev_stop(accel_dev)) { pr_err("QAT: Failed to stop qat_dev%d\n", id); ret = -EFAULT; + } else { + adf_dev_shutdown(accel_dev); } } } @@ -343,7 +345,9 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, if (!adf_dev_started(accel_dev)) { pr_info("QAT: Starting acceleration device qat_dev%d.\n", ctl_data->device_id); - ret = adf_dev_start(accel_dev); + ret = adf_dev_init(accel_dev); + if (!ret) + ret = adf_dev_start(accel_dev); } else { pr_info("QAT: Acceleration device qat_dev%d already started.\n", ctl_data->device_id); @@ -351,6 +355,7 @@ static int adf_ctl_ioctl_dev_start(struct file *fp, unsigned int cmd, if (ret) { pr_err("QAT: Failed to start qat_dev%d\n", ctl_data->device_id); adf_dev_stop(accel_dev); + adf_dev_shutdown(accel_dev); } out: kfree(ctl_data); diff --git a/drivers/crypto/qat/qat_common/adf_init.c b/drivers/crypto/qat/qat_common/adf_init.c index 5c0e47a..8f0ca49 100644 --- a/drivers/crypto/qat/qat_common/adf_init.c +++ b/drivers/crypto/qat/qat_common/adf_init.c @@ -108,26 +108,47 @@ int adf_service_unregister(struct service_hndl *service) EXPORT_SYMBOL_GPL(adf_service_unregister); /** - * adf_dev_start() - Start acceleration service for the given accel device - * @accel_dev: Pointer to acceleration device. + * adf_dev_init() - Init data structures and services for the given accel device + * @accel_dev: Pointer to acceleration device. * - * Function notifies all the registered services that the acceleration device - * is ready to be used. - * To be used by QAT device specific drivers. + * Initialize the ring data structures and the admin comms and arbitration + * services. * * Return: 0 on success, error code othewise. */ -int adf_dev_start(struct adf_accel_dev *accel_dev) +int adf_dev_init(struct adf_accel_dev *accel_dev) { struct service_hndl *service; struct list_head *list_itr; struct adf_hw_device_data *hw_data = accel_dev->hw_device; + if (!hw_data) { + dev_err(&GET_DEV(accel_dev), + "QAT: Failed to init device - hw_data not set\n"); + return -EFAULT; + } + if (!test_bit(ADF_STATUS_CONFIGURED, &accel_dev->status)) { pr_info("QAT: Device not configured\n"); return -EFAULT; } - set_bit(ADF_STATUS_STARTING, &accel_dev->status); + + if (adf_init_etr_data(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed initialize etr\n"); + return -EFAULT; + } + + if (hw_data->init_admin_comms && hw_data->init_admin_comms(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed initialize admin comms\n"); + return -EFAULT; + } + + if (hw_data->init_arb && hw_data->init_arb(accel_dev)) { + dev_err(&GET_DEV(accel_dev), "Failed initialize hw arbiter\n"); + return -EFAULT; + } + + hw_data->enable_ints(accel_dev); if (adf_ae_init(accel_dev)) { pr_err("QAT: Failed to initialise Acceleration Engine\n"); @@ -178,6 +199,27 @@ int adf_dev_start(struct adf_accel_dev *accel_dev) hw_data->enable_error_correction(accel_dev); + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_init); + +/** + * adf_dev_start() - Start acceleration service for the given accel device + * @accel_dev: Pointer to acceleration device. + * + * Function notifies all the registered services that the acceleration device + * is ready to be used. + * To be used by QAT device specific drivers. + * + * Return: 0 on success, error code othewise. + */ +int adf_dev_start(struct adf_accel_dev *accel_dev) +{ + struct service_hndl *service; + struct list_head *list_itr; + + set_bit(ADF_STATUS_STARTING, &accel_dev->status); + if (adf_ae_start(accel_dev)) { pr_err("QAT: AE Start Failed\n"); return -EFAULT; @@ -232,16 +274,15 @@ EXPORT_SYMBOL_GPL(adf_dev_start); */ int adf_dev_stop(struct adf_accel_dev *accel_dev) { - struct adf_hw_device_data *hw_data = accel_dev->hw_device; struct service_hndl *service; struct list_head *list_itr; - int ret, wait = 0; + bool wait = false; + int ret; if (!adf_dev_started(accel_dev) && !test_bit(ADF_STATUS_STARTING, &accel_dev->status)) { return 0; } - clear_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); clear_bit(ADF_STATUS_STARTING, &accel_dev->status); clear_bit(ADF_STATUS_STARTED, &accel_dev->status); @@ -258,7 +299,7 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) if (!ret) { clear_bit(accel_dev->accel_id, &service->start_status); } else if (ret == -EAGAIN) { - wait = 1; + wait = true; clear_bit(accel_dev->accel_id, &service->start_status); } } @@ -278,13 +319,36 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) if (wait) msleep(100); - if (adf_dev_started(accel_dev)) { + if (test_bit(ADF_STATUS_AE_STARTED, &accel_dev->status)) { if (adf_ae_stop(accel_dev)) pr_err("QAT: failed to stop AE\n"); else clear_bit(ADF_STATUS_AE_STARTED, &accel_dev->status); } + return 0; +} +EXPORT_SYMBOL_GPL(adf_dev_stop); + +/** + * adf_dev_shutdown() - shutdown acceleration services and data strucutures + * @accel_dev: Pointer to acceleration device + * + * Cleanup the ring data structures and the admin comms and arbitration + * services. + */ +void adf_dev_shutdown(struct adf_accel_dev *accel_dev) +{ + struct adf_hw_device_data *hw_data = accel_dev->hw_device; + struct service_hndl *service; + struct list_head *list_itr; + + if (!hw_data) { + dev_err(&GET_DEV(accel_dev), + "QAT: Failed to shutdown device - hw_data not set\n"); + return; + } + if (test_bit(ADF_STATUS_AE_UCODE_LOADED, &accel_dev->status)) { if (adf_ae_fw_release(accel_dev)) pr_err("QAT: Failed to release the ucode\n"); @@ -335,9 +399,15 @@ int adf_dev_stop(struct adf_accel_dev *accel_dev) if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) adf_cfg_del_all(accel_dev); - return 0; + if (hw_data->exit_arb) + hw_data->exit_arb(accel_dev); + + if (hw_data->exit_admin_comms) + hw_data->exit_admin_comms(accel_dev); + + adf_cleanup_etr_data(accel_dev); } -EXPORT_SYMBOL_GPL(adf_dev_stop); +EXPORT_SYMBOL_GPL(adf_dev_shutdown); int adf_dev_restarting_notify(struct adf_accel_dev *accel_dev) { diff --git a/drivers/crypto/qat/qat_common/adf_transport_internal.h b/drivers/crypto/qat/qat_common/adf_transport_internal.h index c405460..a486962 100644 --- a/drivers/crypto/qat/qat_common/adf_transport_internal.h +++ b/drivers/crypto/qat/qat_common/adf_transport_internal.h @@ -48,7 +48,6 @@ #define ADF_TRANSPORT_INTRN_H #include <linux/interrupt.h> -#include <linux/atomic.h> #include <linux/spinlock_types.h> #include "adf_transport.h" diff --git a/drivers/crypto/qat/qat_common/icp_qat_hw.h b/drivers/crypto/qat/qat_common/icp_qat_hw.h index 5031f8c..68f191b 100644 --- a/drivers/crypto/qat/qat_common/icp_qat_hw.h +++ b/drivers/crypto/qat/qat_common/icp_qat_hw.h @@ -301,5 +301,5 @@ struct icp_qat_hw_cipher_aes256_f8 { struct icp_qat_hw_cipher_algo_blk { struct icp_qat_hw_cipher_aes256_f8 aes; -}; +} __aligned(64); #endif diff --git a/drivers/crypto/qat/qat_common/qat_algs.c b/drivers/crypto/qat/qat_common/qat_algs.c index 19eea1c..1dc5b0a 100644 --- a/drivers/crypto/qat/qat_common/qat_algs.c +++ b/drivers/crypto/qat/qat_common/qat_algs.c @@ -63,15 +63,15 @@ #include "icp_qat_fw.h" #include "icp_qat_fw_la.h" -#define QAT_AES_HW_CONFIG_ENC(alg) \ +#define QAT_AES_HW_CONFIG_CBC_ENC(alg) \ ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ - ICP_QAT_HW_CIPHER_NO_CONVERT, \ - ICP_QAT_HW_CIPHER_ENCRYPT) + ICP_QAT_HW_CIPHER_NO_CONVERT, \ + ICP_QAT_HW_CIPHER_ENCRYPT) -#define QAT_AES_HW_CONFIG_DEC(alg) \ +#define QAT_AES_HW_CONFIG_CBC_DEC(alg) \ ICP_QAT_HW_CIPHER_CONFIG_BUILD(ICP_QAT_HW_CIPHER_CBC_MODE, alg, \ - ICP_QAT_HW_CIPHER_KEY_CONVERT, \ - ICP_QAT_HW_CIPHER_DECRYPT) + ICP_QAT_HW_CIPHER_KEY_CONVERT, \ + ICP_QAT_HW_CIPHER_DECRYPT) static atomic_t active_dev; @@ -102,25 +102,31 @@ struct qat_alg_cd { }; } __aligned(64); -#define MAX_AUTH_STATE_SIZE sizeof(struct icp_qat_hw_auth_algo_blk) - -struct qat_auth_state { - uint8_t data[MAX_AUTH_STATE_SIZE + 64]; -} __aligned(64); - -struct qat_alg_session_ctx { +struct qat_alg_aead_ctx { struct qat_alg_cd *enc_cd; - dma_addr_t enc_cd_paddr; struct qat_alg_cd *dec_cd; + dma_addr_t enc_cd_paddr; dma_addr_t dec_cd_paddr; - struct icp_qat_fw_la_bulk_req enc_fw_req_tmpl; - struct icp_qat_fw_la_bulk_req dec_fw_req_tmpl; - struct qat_crypto_instance *inst; - struct crypto_tfm *tfm; + struct icp_qat_fw_la_bulk_req enc_fw_req; + struct icp_qat_fw_la_bulk_req dec_fw_req; struct crypto_shash *hash_tfm; enum icp_qat_hw_auth_algo qat_hash_alg; + struct qat_crypto_instance *inst; + struct crypto_tfm *tfm; uint8_t salt[AES_BLOCK_SIZE]; - spinlock_t lock; /* protects qat_alg_session_ctx struct */ + spinlock_t lock; /* protects qat_alg_aead_ctx struct */ +}; + +struct qat_alg_ablkcipher_ctx { + struct icp_qat_hw_cipher_algo_blk *enc_cd; + struct icp_qat_hw_cipher_algo_blk *dec_cd; + dma_addr_t enc_cd_paddr; + dma_addr_t dec_cd_paddr; + struct icp_qat_fw_la_bulk_req enc_fw_req; + struct icp_qat_fw_la_bulk_req dec_fw_req; + struct qat_crypto_instance *inst; + struct crypto_tfm *tfm; + spinlock_t lock; /* protects qat_alg_ablkcipher_ctx struct */ }; static int get_current_node(void) @@ -144,43 +150,37 @@ static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg) } static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash, - struct qat_alg_session_ctx *ctx, + struct qat_alg_aead_ctx *ctx, const uint8_t *auth_key, unsigned int auth_keylen) { - struct qat_auth_state auth_state; SHASH_DESC_ON_STACK(shash, ctx->hash_tfm); struct sha1_state sha1; struct sha256_state sha256; struct sha512_state sha512; int block_size = crypto_shash_blocksize(ctx->hash_tfm); int digest_size = crypto_shash_digestsize(ctx->hash_tfm); - uint8_t *ipad = auth_state.data; - uint8_t *opad = ipad + block_size; + char ipad[block_size]; + char opad[block_size]; __be32 *hash_state_out; __be64 *hash512_state_out; int i, offset; - memzero_explicit(auth_state.data, MAX_AUTH_STATE_SIZE + 64); + memset(ipad, 0, block_size); + memset(opad, 0, block_size); shash->tfm = ctx->hash_tfm; shash->flags = 0x0; if (auth_keylen > block_size) { - char buff[SHA512_BLOCK_SIZE]; int ret = crypto_shash_digest(shash, auth_key, - auth_keylen, buff); + auth_keylen, ipad); if (ret) return ret; - memcpy(ipad, buff, digest_size); - memcpy(opad, buff, digest_size); - memzero_explicit(ipad + digest_size, block_size - digest_size); - memzero_explicit(opad + digest_size, block_size - digest_size); + memcpy(opad, ipad, digest_size); } else { memcpy(ipad, auth_key, auth_keylen); memcpy(opad, auth_key, auth_keylen); - memzero_explicit(ipad + auth_keylen, block_size - auth_keylen); - memzero_explicit(opad + auth_keylen, block_size - auth_keylen); } for (i = 0; i < block_size; i++) { @@ -267,8 +267,6 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) header->comn_req_flags = ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR, QAT_COMN_PTR_TYPE_SGL); - ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, - ICP_QAT_FW_LA_DIGEST_IN_BUFFER); ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags, ICP_QAT_FW_LA_PARTIAL_NONE); ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags, @@ -279,8 +277,9 @@ static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header) ICP_QAT_FW_LA_NO_UPDATE_STATE); } -static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, - int alg, struct crypto_authenc_keys *keys) +static int qat_alg_aead_init_enc_session(struct qat_alg_aead_ctx *ctx, + int alg, + struct crypto_authenc_keys *keys) { struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; @@ -289,7 +288,7 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, struct icp_qat_hw_auth_algo_blk *hash = (struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx + sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen); - struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req_tmpl; + struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; void *ptr = &req_tmpl->cd_ctrl; @@ -297,7 +296,7 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr; /* CD setup */ - cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg); + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg); memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); hash->sha.inner_setup.auth_config.config = ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, @@ -311,6 +310,8 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, /* Request setup */ qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH; + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_DIGEST_IN_BUFFER); ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_RET_AUTH_RES); ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, @@ -356,8 +357,9 @@ static int qat_alg_init_enc_session(struct qat_alg_session_ctx *ctx, return 0; } -static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, - int alg, struct crypto_authenc_keys *keys) +static int qat_alg_aead_init_dec_session(struct qat_alg_aead_ctx *ctx, + int alg, + struct crypto_authenc_keys *keys) { struct crypto_aead *aead_tfm = __crypto_aead_cast(ctx->tfm); unsigned int digestsize = crypto_aead_crt(aead_tfm)->authsize; @@ -367,7 +369,7 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, (struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx + sizeof(struct icp_qat_hw_auth_setup) + roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2); - struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req_tmpl; + struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req; struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars; struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr; void *ptr = &req_tmpl->cd_ctrl; @@ -379,7 +381,7 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, sizeof(struct icp_qat_fw_la_cipher_req_params)); /* CD setup */ - cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg); + cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg); memcpy(cipher->aes.key, keys->enckey, keys->enckeylen); hash->sha.inner_setup.auth_config.config = ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1, @@ -394,6 +396,8 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, /* Request setup */ qat_alg_init_common_hdr(header); header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER; + ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags, + ICP_QAT_FW_LA_DIGEST_IN_BUFFER); ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags, ICP_QAT_FW_LA_NO_RET_AUTH_RES); ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags, @@ -444,36 +448,91 @@ static int qat_alg_init_dec_session(struct qat_alg_session_ctx *ctx, return 0; } -static int qat_alg_init_sessions(struct qat_alg_session_ctx *ctx, - const uint8_t *key, unsigned int keylen) +static void qat_alg_ablkcipher_init_com(struct qat_alg_ablkcipher_ctx *ctx, + struct icp_qat_fw_la_bulk_req *req, + struct icp_qat_hw_cipher_algo_blk *cd, + const uint8_t *key, unsigned int keylen) { - struct crypto_authenc_keys keys; - int alg; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr; + struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl; - if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE)) - return -EFAULT; + memcpy(cd->aes.key, key, keylen); + qat_alg_init_common_hdr(header); + header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER; + cd_pars->u.s.content_desc_params_sz = + sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3; + /* Cipher CD config setup */ + cd_ctrl->cipher_key_sz = keylen >> 3; + cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3; + cd_ctrl->cipher_cfg_offset = 0; + ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER); + ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR); +} - if (crypto_authenc_extractkeys(&keys, key, keylen)) - goto bad_key; +static void qat_alg_ablkcipher_init_enc(struct qat_alg_ablkcipher_ctx *ctx, + int alg, const uint8_t *key, + unsigned int keylen) +{ + struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd; + struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + + qat_alg_ablkcipher_init_com(ctx, req, enc_cd, key, keylen); + cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr; + enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_ENC(alg); +} - switch (keys.enckeylen) { +static void qat_alg_ablkcipher_init_dec(struct qat_alg_ablkcipher_ctx *ctx, + int alg, const uint8_t *key, + unsigned int keylen) +{ + struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd; + struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req; + struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars; + + qat_alg_ablkcipher_init_com(ctx, req, dec_cd, key, keylen); + cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr; + dec_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_CBC_DEC(alg); +} + +static int qat_alg_validate_key(int key_len, int *alg) +{ + switch (key_len) { case AES_KEYSIZE_128: - alg = ICP_QAT_HW_CIPHER_ALGO_AES128; + *alg = ICP_QAT_HW_CIPHER_ALGO_AES128; break; case AES_KEYSIZE_192: - alg = ICP_QAT_HW_CIPHER_ALGO_AES192; + *alg = ICP_QAT_HW_CIPHER_ALGO_AES192; break; case AES_KEYSIZE_256: - alg = ICP_QAT_HW_CIPHER_ALGO_AES256; + *alg = ICP_QAT_HW_CIPHER_ALGO_AES256; break; default: - goto bad_key; + return -EINVAL; } + return 0; +} - if (qat_alg_init_enc_session(ctx, alg, &keys)) +static int qat_alg_aead_init_sessions(struct qat_alg_aead_ctx *ctx, + const uint8_t *key, unsigned int keylen) +{ + struct crypto_authenc_keys keys; + int alg; + + if (crypto_rng_get_bytes(crypto_default_rng, ctx->salt, AES_BLOCK_SIZE)) + return -EFAULT; + + if (crypto_authenc_extractkeys(&keys, key, keylen)) + goto bad_key; + + if (qat_alg_validate_key(keys.enckeylen, &alg)) + goto bad_key; + + if (qat_alg_aead_init_enc_session(ctx, alg, &keys)) goto error; - if (qat_alg_init_dec_session(ctx, alg, &keys)) + if (qat_alg_aead_init_dec_session(ctx, alg, &keys)) goto error; return 0; @@ -484,22 +543,37 @@ error: return -EFAULT; } -static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key, - unsigned int keylen) +static int qat_alg_ablkcipher_init_sessions(struct qat_alg_ablkcipher_ctx *ctx, + const uint8_t *key, + unsigned int keylen) +{ + int alg; + + if (qat_alg_validate_key(keylen, &alg)) + goto bad_key; + + qat_alg_ablkcipher_init_enc(ctx, alg, key, keylen); + qat_alg_ablkcipher_init_dec(ctx, alg, key, keylen); + return 0; +bad_key: + crypto_tfm_set_flags(ctx->tfm, CRYPTO_TFM_RES_BAD_KEY_LEN); + return -EINVAL; +} + +static int qat_alg_aead_setkey(struct crypto_aead *tfm, const uint8_t *key, + unsigned int keylen) { - struct qat_alg_session_ctx *ctx = crypto_aead_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm); struct device *dev; spin_lock(&ctx->lock); if (ctx->enc_cd) { /* rekeying */ dev = &GET_DEV(ctx->inst->accel_dev); - memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd)); - memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd)); - memzero_explicit(&ctx->enc_fw_req_tmpl, - sizeof(struct icp_qat_fw_la_bulk_req)); - memzero_explicit(&ctx->dec_fw_req_tmpl, - sizeof(struct icp_qat_fw_la_bulk_req)); + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); } else { /* new key */ int node = get_current_node(); @@ -512,16 +586,14 @@ static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key, dev = &GET_DEV(inst->accel_dev); ctx->inst = inst; - ctx->enc_cd = dma_zalloc_coherent(dev, - sizeof(struct qat_alg_cd), + ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd), &ctx->enc_cd_paddr, GFP_ATOMIC); if (!ctx->enc_cd) { spin_unlock(&ctx->lock); return -ENOMEM; } - ctx->dec_cd = dma_zalloc_coherent(dev, - sizeof(struct qat_alg_cd), + ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd), &ctx->dec_cd_paddr, GFP_ATOMIC); if (!ctx->dec_cd) { @@ -530,18 +602,18 @@ static int qat_alg_setkey(struct crypto_aead *tfm, const uint8_t *key, } } spin_unlock(&ctx->lock); - if (qat_alg_init_sessions(ctx, key, keylen)) + if (qat_alg_aead_init_sessions(ctx, key, keylen)) goto out_free_all; return 0; out_free_all: - memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd)); + memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->dec_cd, ctx->dec_cd_paddr); ctx->dec_cd = NULL; out_free_enc: - memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd)); + memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->enc_cd, ctx->enc_cd_paddr); ctx->enc_cd = NULL; @@ -557,7 +629,8 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, dma_addr_t blp = qat_req->buf.blp; dma_addr_t blpout = qat_req->buf.bloutp; size_t sz = qat_req->buf.sz; - int i, bufs = bl->num_bufs; + size_t sz_out = qat_req->buf.sz_out; + int i; for (i = 0; i < bl->num_bufs; i++) dma_unmap_single(dev, bl->bufers[i].addr, @@ -567,14 +640,14 @@ static void qat_alg_free_bufl(struct qat_crypto_instance *inst, kfree(bl); if (blp != blpout) { /* If out of place operation dma unmap only data */ - int bufless = bufs - blout->num_mapped_bufs; + int bufless = blout->num_bufs - blout->num_mapped_bufs; - for (i = bufless; i < bufs; i++) { + for (i = bufless; i < blout->num_bufs; i++) { dma_unmap_single(dev, blout->bufers[i].addr, blout->bufers[i].len, DMA_BIDIRECTIONAL); } - dma_unmap_single(dev, blpout, sz, DMA_TO_DEVICE); + dma_unmap_single(dev, blpout, sz_out, DMA_TO_DEVICE); kfree(blout); } } @@ -587,19 +660,20 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, struct qat_crypto_request *qat_req) { struct device *dev = &GET_DEV(inst->accel_dev); - int i, bufs = 0, n = sg_nents(sgl), assoc_n = sg_nents(assoc); + int i, bufs = 0, sg_nctr = 0; + int n = sg_nents(sgl), assoc_n = sg_nents(assoc); struct qat_alg_buf_list *bufl; struct qat_alg_buf_list *buflout = NULL; dma_addr_t blp; dma_addr_t bloutp = 0; struct scatterlist *sg; - size_t sz = sizeof(struct qat_alg_buf_list) + + size_t sz_out, sz = sizeof(struct qat_alg_buf_list) + ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); if (unlikely(!n)) return -EINVAL; - bufl = kmalloc_node(sz, GFP_ATOMIC, + bufl = kzalloc_node(sz, GFP_ATOMIC, dev_to_node(&GET_DEV(inst->accel_dev))); if (unlikely(!bufl)) return -ENOMEM; @@ -620,15 +694,20 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, goto err; bufs++; } - bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen, - DMA_BIDIRECTIONAL); - bufl->bufers[bufs].len = ivlen; - if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) - goto err; - bufs++; + if (ivlen) { + bufl->bufers[bufs].addr = dma_map_single(dev, iv, ivlen, + DMA_BIDIRECTIONAL); + bufl->bufers[bufs].len = ivlen; + if (unlikely(dma_mapping_error(dev, bufl->bufers[bufs].addr))) + goto err; + bufs++; + } for_each_sg(sgl, sg, n, i) { - int y = i + bufs; + int y = sg_nctr + bufs; + + if (!sg->length) + continue; bufl->bufers[y].addr = dma_map_single(dev, sg_virt(sg), sg->length, @@ -636,8 +715,9 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufl->bufers[y].len = sg->length; if (unlikely(dma_mapping_error(dev, bufl->bufers[y].addr))) goto err; + sg_nctr++; } - bufl->num_bufs = n + bufs; + bufl->num_bufs = sg_nctr + bufs; qat_req->buf.bl = bufl; qat_req->buf.blp = blp; qat_req->buf.sz = sz; @@ -645,11 +725,15 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, if (sgl != sglout) { struct qat_alg_buf *bufers; - buflout = kmalloc_node(sz, GFP_ATOMIC, + n = sg_nents(sglout); + sz_out = sizeof(struct qat_alg_buf_list) + + ((1 + n + assoc_n) * sizeof(struct qat_alg_buf)); + sg_nctr = 0; + buflout = kzalloc_node(sz_out, GFP_ATOMIC, dev_to_node(&GET_DEV(inst->accel_dev))); if (unlikely(!buflout)) goto err; - bloutp = dma_map_single(dev, buflout, sz, DMA_TO_DEVICE); + bloutp = dma_map_single(dev, buflout, sz_out, DMA_TO_DEVICE); if (unlikely(dma_mapping_error(dev, bloutp))) goto err; bufers = buflout->bufers; @@ -660,60 +744,62 @@ static int qat_alg_sgl_to_bufl(struct qat_crypto_instance *inst, bufers[i].addr = bufl->bufers[i].addr; } for_each_sg(sglout, sg, n, i) { - int y = i + bufs; + int y = sg_nctr + bufs; + + if (!sg->length) + continue; bufers[y].addr = dma_map_single(dev, sg_virt(sg), sg->length, DMA_BIDIRECTIONAL); - buflout->bufers[y].len = sg->length; if (unlikely(dma_mapping_error(dev, bufers[y].addr))) goto err; + bufers[y].len = sg->length; + sg_nctr++; } - buflout->num_bufs = n + bufs; - buflout->num_mapped_bufs = n; + buflout->num_bufs = sg_nctr + bufs; + buflout->num_mapped_bufs = sg_nctr; qat_req->buf.blout = buflout; qat_req->buf.bloutp = bloutp; + qat_req->buf.sz_out = sz_out; } else { /* Otherwise set the src and dst to the same address */ qat_req->buf.bloutp = qat_req->buf.blp; + qat_req->buf.sz_out = 0; } return 0; err: dev_err(dev, "Failed to map buf for dma\n"); - for_each_sg(sgl, sg, n + bufs, i) { - if (!dma_mapping_error(dev, bufl->bufers[i].addr)) { + sg_nctr = 0; + for (i = 0; i < n + bufs; i++) + if (!dma_mapping_error(dev, bufl->bufers[i].addr)) dma_unmap_single(dev, bufl->bufers[i].addr, bufl->bufers[i].len, DMA_BIDIRECTIONAL); - } - } + if (!dma_mapping_error(dev, blp)) dma_unmap_single(dev, blp, sz, DMA_TO_DEVICE); kfree(bufl); if (sgl != sglout && buflout) { - for_each_sg(sglout, sg, n, i) { - int y = i + bufs; - - if (!dma_mapping_error(dev, buflout->bufers[y].addr)) - dma_unmap_single(dev, buflout->bufers[y].addr, - buflout->bufers[y].len, + n = sg_nents(sglout); + for (i = bufs; i < n + bufs; i++) + if (!dma_mapping_error(dev, buflout->bufers[i].addr)) + dma_unmap_single(dev, buflout->bufers[i].addr, + buflout->bufers[i].len, DMA_BIDIRECTIONAL); - } if (!dma_mapping_error(dev, bloutp)) - dma_unmap_single(dev, bloutp, sz, DMA_TO_DEVICE); + dma_unmap_single(dev, bloutp, sz_out, DMA_TO_DEVICE); kfree(buflout); } return -ENOMEM; } -void qat_alg_callback(void *resp) +static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp, + struct qat_crypto_request *qat_req) { - struct icp_qat_fw_la_resp *qat_resp = resp; - struct qat_crypto_request *qat_req = - (void *)(__force long)qat_resp->opaque_data; - struct qat_alg_session_ctx *ctx = qat_req->ctx; + struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx; struct qat_crypto_instance *inst = ctx->inst; - struct aead_request *areq = qat_req->areq; + struct aead_request *areq = qat_req->aead_req; uint8_t stat_filed = qat_resp->comn_resp.comn_status; int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); @@ -723,11 +809,35 @@ void qat_alg_callback(void *resp) areq->base.complete(&areq->base, res); } -static int qat_alg_dec(struct aead_request *areq) +static void qat_ablkcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp, + struct qat_crypto_request *qat_req) +{ + struct qat_alg_ablkcipher_ctx *ctx = qat_req->ablkcipher_ctx; + struct qat_crypto_instance *inst = ctx->inst; + struct ablkcipher_request *areq = qat_req->ablkcipher_req; + uint8_t stat_filed = qat_resp->comn_resp.comn_status; + int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed); + + qat_alg_free_bufl(inst, qat_req); + if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK)) + res = -EINVAL; + areq->base.complete(&areq->base, res); +} + +void qat_alg_callback(void *resp) +{ + struct icp_qat_fw_la_resp *qat_resp = resp; + struct qat_crypto_request *qat_req = + (void *)(__force long)qat_resp->opaque_data; + + qat_req->cb(qat_resp, qat_req); +} + +static int qat_alg_aead_dec(struct aead_request *areq) { struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); struct qat_crypto_request *qat_req = aead_request_ctx(areq); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; @@ -741,9 +851,10 @@ static int qat_alg_dec(struct aead_request *areq) return ret; msg = &qat_req->req; - *msg = ctx->dec_fw_req_tmpl; - qat_req->ctx = ctx; - qat_req->areq = areq; + *msg = ctx->dec_fw_req; + qat_req->aead_ctx = ctx; + qat_req->aead_req = areq; + qat_req->cb = qat_aead_alg_callback; qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; @@ -766,12 +877,12 @@ static int qat_alg_dec(struct aead_request *areq) return -EINPROGRESS; } -static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv, - int enc_iv) +static int qat_alg_aead_enc_internal(struct aead_request *areq, uint8_t *iv, + int enc_iv) { struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq); struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); struct qat_crypto_request *qat_req = aead_request_ctx(areq); struct icp_qat_fw_la_cipher_req_params *cipher_param; struct icp_qat_fw_la_auth_req_params *auth_param; @@ -784,9 +895,10 @@ static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv, return ret; msg = &qat_req->req; - *msg = ctx->enc_fw_req_tmpl; - qat_req->ctx = ctx; - qat_req->areq = areq; + *msg = ctx->enc_fw_req; + qat_req->aead_ctx = ctx; + qat_req->aead_req = areq; + qat_req->cb = qat_aead_alg_callback; qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; @@ -815,31 +927,168 @@ static int qat_alg_enc_internal(struct aead_request *areq, uint8_t *iv, return -EINPROGRESS; } -static int qat_alg_enc(struct aead_request *areq) +static int qat_alg_aead_enc(struct aead_request *areq) { - return qat_alg_enc_internal(areq, areq->iv, 0); + return qat_alg_aead_enc_internal(areq, areq->iv, 0); } -static int qat_alg_genivenc(struct aead_givcrypt_request *req) +static int qat_alg_aead_genivenc(struct aead_givcrypt_request *req) { struct crypto_aead *aead_tfm = crypto_aead_reqtfm(&req->areq); struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm); - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); __be64 seq; memcpy(req->giv, ctx->salt, AES_BLOCK_SIZE); seq = cpu_to_be64(req->seq); memcpy(req->giv + AES_BLOCK_SIZE - sizeof(uint64_t), &seq, sizeof(uint64_t)); - return qat_alg_enc_internal(&req->areq, req->giv, 1); + return qat_alg_aead_enc_internal(&req->areq, req->giv, 1); } -static int qat_alg_init(struct crypto_tfm *tfm, - enum icp_qat_hw_auth_algo hash, const char *hash_name) +static int qat_alg_ablkcipher_setkey(struct crypto_ablkcipher *tfm, + const uint8_t *key, + unsigned int keylen) { - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_ablkcipher_ctx *ctx = crypto_ablkcipher_ctx(tfm); + struct device *dev; + + spin_lock(&ctx->lock); + if (ctx->enc_cd) { + /* rekeying */ + dev = &GET_DEV(ctx->inst->accel_dev); + memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd)); + memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd)); + memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req)); + memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req)); + } else { + /* new key */ + int node = get_current_node(); + struct qat_crypto_instance *inst = + qat_crypto_get_instance_node(node); + if (!inst) { + spin_unlock(&ctx->lock); + return -EINVAL; + } + + dev = &GET_DEV(inst->accel_dev); + ctx->inst = inst; + ctx->enc_cd = dma_zalloc_coherent(dev, sizeof(*ctx->enc_cd), + &ctx->enc_cd_paddr, + GFP_ATOMIC); + if (!ctx->enc_cd) { + spin_unlock(&ctx->lock); + return -ENOMEM; + } + ctx->dec_cd = dma_zalloc_coherent(dev, sizeof(*ctx->dec_cd), + &ctx->dec_cd_paddr, + GFP_ATOMIC); + if (!ctx->dec_cd) { + spin_unlock(&ctx->lock); + goto out_free_enc; + } + } + spin_unlock(&ctx->lock); + if (qat_alg_ablkcipher_init_sessions(ctx, key, keylen)) + goto out_free_all; + + return 0; + +out_free_all: + memset(ctx->dec_cd, 0, sizeof(*ctx->enc_cd)); + dma_free_coherent(dev, sizeof(*ctx->enc_cd), + ctx->dec_cd, ctx->dec_cd_paddr); + ctx->dec_cd = NULL; +out_free_enc: + memset(ctx->enc_cd, 0, sizeof(*ctx->dec_cd)); + dma_free_coherent(dev, sizeof(*ctx->dec_cd), + ctx->enc_cd, ctx->enc_cd_paddr); + ctx->enc_cd = NULL; + return -ENOMEM; +} + +static int qat_alg_ablkcipher_encrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm); + struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct icp_qat_fw_la_bulk_req *msg; + int ret, ctr = 0; + + ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst, + NULL, 0, qat_req); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->enc_fw_req; + qat_req->ablkcipher_ctx = ctx; + qat_req->ablkcipher_req = req; + qat_req->cb = qat_ablkcipher_alg_callback; + qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + cipher_param->cipher_length = req->nbytes; + cipher_param->cipher_offset = 0; + memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); + do { + ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); + } while (ret == -EAGAIN && ctr++ < 10); + + if (ret == -EAGAIN) { + qat_alg_free_bufl(ctx->inst, qat_req); + return -EBUSY; + } + return -EINPROGRESS; +} + +static int qat_alg_ablkcipher_decrypt(struct ablkcipher_request *req) +{ + struct crypto_ablkcipher *atfm = crypto_ablkcipher_reqtfm(req); + struct crypto_tfm *tfm = crypto_ablkcipher_tfm(atfm); + struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_request *qat_req = ablkcipher_request_ctx(req); + struct icp_qat_fw_la_cipher_req_params *cipher_param; + struct icp_qat_fw_la_bulk_req *msg; + int ret, ctr = 0; + + ret = qat_alg_sgl_to_bufl(ctx->inst, NULL, req->src, req->dst, + NULL, 0, qat_req); + if (unlikely(ret)) + return ret; + + msg = &qat_req->req; + *msg = ctx->dec_fw_req; + qat_req->ablkcipher_ctx = ctx; + qat_req->ablkcipher_req = req; + qat_req->cb = qat_ablkcipher_alg_callback; + qat_req->req.comn_mid.opaque_data = (uint64_t)(__force long)qat_req; + qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp; + qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp; + cipher_param = (void *)&qat_req->req.serv_specif_rqpars; + cipher_param->cipher_length = req->nbytes; + cipher_param->cipher_offset = 0; + memcpy(cipher_param->u.cipher_IV_array, req->info, AES_BLOCK_SIZE); + do { + ret = adf_send_message(ctx->inst->sym_tx, (uint32_t *)msg); + } while (ret == -EAGAIN && ctr++ < 10); + + if (ret == -EAGAIN) { + qat_alg_free_bufl(ctx->inst, qat_req); + return -EBUSY; + } + return -EINPROGRESS; +} + +static int qat_alg_aead_init(struct crypto_tfm *tfm, + enum icp_qat_hw_auth_algo hash, + const char *hash_name) +{ + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); - memzero_explicit(ctx, sizeof(*ctx)); ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0); if (IS_ERR(ctx->hash_tfm)) return -EFAULT; @@ -851,24 +1100,24 @@ static int qat_alg_init(struct crypto_tfm *tfm, return 0; } -static int qat_alg_sha1_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha1_init(struct crypto_tfm *tfm) { - return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1"); } -static int qat_alg_sha256_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha256_init(struct crypto_tfm *tfm) { - return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256"); } -static int qat_alg_sha512_init(struct crypto_tfm *tfm) +static int qat_alg_aead_sha512_init(struct crypto_tfm *tfm) { - return qat_alg_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); + return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512"); } -static void qat_alg_exit(struct crypto_tfm *tfm) +static void qat_alg_aead_exit(struct crypto_tfm *tfm) { - struct qat_alg_session_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm); struct qat_crypto_instance *inst = ctx->inst; struct device *dev; @@ -880,36 +1129,74 @@ static void qat_alg_exit(struct crypto_tfm *tfm) dev = &GET_DEV(inst->accel_dev); if (ctx->enc_cd) { - memzero_explicit(ctx->enc_cd, sizeof(struct qat_alg_cd)); + memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd)); dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->enc_cd, ctx->enc_cd_paddr); } if (ctx->dec_cd) { - memzero_explicit(ctx->dec_cd, sizeof(struct qat_alg_cd)); + memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd)); dma_free_coherent(dev, sizeof(struct qat_alg_cd), ctx->dec_cd, ctx->dec_cd_paddr); } qat_crypto_put_instance(inst); } +static int qat_alg_ablkcipher_init(struct crypto_tfm *tfm) +{ + struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); + + spin_lock_init(&ctx->lock); + tfm->crt_ablkcipher.reqsize = sizeof(struct ablkcipher_request) + + sizeof(struct qat_crypto_request); + ctx->tfm = tfm; + return 0; +} + +static void qat_alg_ablkcipher_exit(struct crypto_tfm *tfm) +{ + struct qat_alg_ablkcipher_ctx *ctx = crypto_tfm_ctx(tfm); + struct qat_crypto_instance *inst = ctx->inst; + struct device *dev; + + if (!inst) + return; + + dev = &GET_DEV(inst->accel_dev); + if (ctx->enc_cd) { + memset(ctx->enc_cd, 0, + sizeof(struct icp_qat_hw_cipher_algo_blk)); + dma_free_coherent(dev, + sizeof(struct icp_qat_hw_cipher_algo_blk), + ctx->enc_cd, ctx->enc_cd_paddr); + } + if (ctx->dec_cd) { + memset(ctx->dec_cd, 0, + sizeof(struct icp_qat_hw_cipher_algo_blk)); + dma_free_coherent(dev, + sizeof(struct icp_qat_hw_cipher_algo_blk), + ctx->dec_cd, ctx->dec_cd_paddr); + } + qat_crypto_put_instance(inst); +} + static struct crypto_alg qat_algs[] = { { .cra_name = "authenc(hmac(sha1),cbc(aes))", .cra_driver_name = "qat_aes_cbc_hmac_sha1", .cra_priority = 4001, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_session_ctx), + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), .cra_alignmask = 0, .cra_type = &crypto_aead_type, .cra_module = THIS_MODULE, - .cra_init = qat_alg_sha1_init, - .cra_exit = qat_alg_exit, + .cra_init = qat_alg_aead_sha1_init, + .cra_exit = qat_alg_aead_exit, .cra_u = { .aead = { - .setkey = qat_alg_setkey, - .decrypt = qat_alg_dec, - .encrypt = qat_alg_enc, - .givencrypt = qat_alg_genivenc, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .givencrypt = qat_alg_aead_genivenc, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA1_DIGEST_SIZE, }, @@ -920,18 +1207,18 @@ static struct crypto_alg qat_algs[] = { { .cra_priority = 4001, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_session_ctx), + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), .cra_alignmask = 0, .cra_type = &crypto_aead_type, .cra_module = THIS_MODULE, - .cra_init = qat_alg_sha256_init, - .cra_exit = qat_alg_exit, + .cra_init = qat_alg_aead_sha256_init, + .cra_exit = qat_alg_aead_exit, .cra_u = { .aead = { - .setkey = qat_alg_setkey, - .decrypt = qat_alg_dec, - .encrypt = qat_alg_enc, - .givencrypt = qat_alg_genivenc, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .givencrypt = qat_alg_aead_genivenc, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA256_DIGEST_SIZE, }, @@ -942,22 +1229,44 @@ static struct crypto_alg qat_algs[] = { { .cra_priority = 4001, .cra_flags = CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC, .cra_blocksize = AES_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct qat_alg_session_ctx), + .cra_ctxsize = sizeof(struct qat_alg_aead_ctx), .cra_alignmask = 0, .cra_type = &crypto_aead_type, .cra_module = THIS_MODULE, - .cra_init = qat_alg_sha512_init, - .cra_exit = qat_alg_exit, + .cra_init = qat_alg_aead_sha512_init, + .cra_exit = qat_alg_aead_exit, .cra_u = { .aead = { - .setkey = qat_alg_setkey, - .decrypt = qat_alg_dec, - .encrypt = qat_alg_enc, - .givencrypt = qat_alg_genivenc, + .setkey = qat_alg_aead_setkey, + .decrypt = qat_alg_aead_dec, + .encrypt = qat_alg_aead_enc, + .givencrypt = qat_alg_aead_genivenc, .ivsize = AES_BLOCK_SIZE, .maxauthsize = SHA512_DIGEST_SIZE, }, }, +}, { + .cra_name = "cbc(aes)", + .cra_driver_name = "qat_aes_cbc", + .cra_priority = 4001, + .cra_flags = CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC, + .cra_blocksize = AES_BLOCK_SIZE, + .cra_ctxsize = sizeof(struct qat_alg_ablkcipher_ctx), + .cra_alignmask = 0, + .cra_type = &crypto_ablkcipher_type, + .cra_module = THIS_MODULE, + .cra_init = qat_alg_ablkcipher_init, + .cra_exit = qat_alg_ablkcipher_exit, + .cra_u = { + .ablkcipher = { + .setkey = qat_alg_ablkcipher_setkey, + .decrypt = qat_alg_ablkcipher_decrypt, + .encrypt = qat_alg_ablkcipher_encrypt, + .min_keysize = AES_MIN_KEY_SIZE, + .max_keysize = AES_MAX_KEY_SIZE, + .ivsize = AES_BLOCK_SIZE, + }, + }, } }; int qat_algs_register(void) @@ -966,8 +1275,11 @@ int qat_algs_register(void) int i; for (i = 0; i < ARRAY_SIZE(qat_algs); i++) - qat_algs[i].cra_flags = CRYPTO_ALG_TYPE_AEAD | - CRYPTO_ALG_ASYNC; + qat_algs[i].cra_flags = + (qat_algs[i].cra_type == &crypto_aead_type) ? + CRYPTO_ALG_TYPE_AEAD | CRYPTO_ALG_ASYNC : + CRYPTO_ALG_TYPE_ABLKCIPHER | CRYPTO_ALG_ASYNC; + return crypto_register_algs(qat_algs, ARRAY_SIZE(qat_algs)); } return 0; diff --git a/drivers/crypto/qat/qat_common/qat_crypto.h b/drivers/crypto/qat/qat_common/qat_crypto.h index ab8468d..d503007 100644 --- a/drivers/crypto/qat/qat_common/qat_crypto.h +++ b/drivers/crypto/qat/qat_common/qat_crypto.h @@ -72,12 +72,24 @@ struct qat_crypto_request_buffs { struct qat_alg_buf_list *blout; dma_addr_t bloutp; size_t sz; + size_t sz_out; }; +struct qat_crypto_request; + struct qat_crypto_request { struct icp_qat_fw_la_bulk_req req; - struct qat_alg_session_ctx *ctx; - struct aead_request *areq; + union { + struct qat_alg_aead_ctx *aead_ctx; + struct qat_alg_ablkcipher_ctx *ablkcipher_ctx; + }; + union { + struct aead_request *aead_req; + struct ablkcipher_request *ablkcipher_req; + }; struct qat_crypto_request_buffs buf; + void (*cb)(struct icp_qat_fw_la_resp *resp, + struct qat_crypto_request *req); }; + #endif diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c index ef05825..6a735d5 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_dh895xcc_hw_data.c @@ -46,6 +46,7 @@ */ #include <adf_accel_devices.h> #include "adf_dh895xcc_hw_data.h" +#include "adf_common_drv.h" #include "adf_drv.h" /* Worker thread to service arbiter mappings based on dev SKUs */ @@ -182,6 +183,19 @@ static void adf_enable_error_correction(struct adf_accel_dev *accel_dev) } } +static void adf_enable_ints(struct adf_accel_dev *accel_dev) +{ + void __iomem *addr; + + addr = (&GET_BARS(accel_dev)[ADF_DH895XCC_PMISC_BAR])->virt_addr; + + /* Enable bundle and misc interrupts */ + ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, + ADF_DH895XCC_SMIA0_MASK); + ADF_CSR_WR(addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, + ADF_DH895XCC_SMIA1_MASK); +} + void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) { hw_data->dev_class = &dh895xcc_class; @@ -206,6 +220,11 @@ void adf_init_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) hw_data->get_misc_bar_id = get_misc_bar_id; hw_data->get_sku = get_sku; hw_data->fw_name = ADF_DH895XCC_FW; + hw_data->init_admin_comms = adf_init_admin_comms; + hw_data->exit_admin_comms = adf_exit_admin_comms; + hw_data->init_arb = adf_init_arb; + hw_data->exit_arb = adf_exit_arb; + hw_data->enable_ints = adf_enable_ints; } void adf_clean_hw_data_dh895xcc(struct adf_hw_device_data *hw_data) diff --git a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c index 948f66b..8ffdb95 100644 --- a/drivers/crypto/qat/qat_dh895xcc/adf_drv.c +++ b/drivers/crypto/qat/qat_dh895xcc/adf_drv.c @@ -90,9 +90,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) struct adf_accel_pci *accel_pci_dev = &accel_dev->accel_pci_dev; int i; - adf_exit_admin_comms(accel_dev); - adf_exit_arb(accel_dev); - adf_cleanup_etr_data(accel_dev); + adf_dev_shutdown(accel_dev); for (i = 0; i < ADF_PCI_MAX_BARS; i++) { struct adf_bar *bar = &accel_pci_dev->pci_bars[i]; @@ -119,7 +117,7 @@ static void adf_cleanup_accel(struct adf_accel_dev *accel_dev) kfree(accel_dev); } -static int qat_dev_start(struct adf_accel_dev *accel_dev) +static int adf_dev_configure(struct adf_accel_dev *accel_dev) { int cpus = num_online_cpus(); int banks = GET_MAX_BANKS(accel_dev); @@ -206,7 +204,7 @@ static int qat_dev_start(struct adf_accel_dev *accel_dev) goto err; set_bit(ADF_STATUS_CONFIGURED, &accel_dev->status); - return adf_dev_start(accel_dev); + return 0; err: dev_err(&GET_DEV(accel_dev), "Failed to start QAT accel dev\n"); return -EINVAL; @@ -217,7 +215,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) struct adf_accel_dev *accel_dev; struct adf_accel_pci *accel_pci_dev; struct adf_hw_device_data *hw_data; - void __iomem *pmisc_bar_addr = NULL; char name[ADF_DEVICE_NAME_LENGTH]; unsigned int i, bar_nr; int ret; @@ -347,8 +344,6 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) ret = -EFAULT; goto out_err; } - if (i == ADF_DH895XCC_PMISC_BAR) - pmisc_bar_addr = bar->virt_addr; } pci_set_master(pdev); @@ -358,36 +353,21 @@ static int adf_probe(struct pci_dev *pdev, const struct pci_device_id *ent) goto out_err; } - if (adf_init_etr_data(accel_dev)) { - dev_err(&pdev->dev, "Failed initialize etr\n"); - ret = -EFAULT; - goto out_err; - } - - if (adf_init_admin_comms(accel_dev)) { - dev_err(&pdev->dev, "Failed initialize admin comms\n"); - ret = -EFAULT; - goto out_err; - } - - if (adf_init_arb(accel_dev)) { - dev_err(&pdev->dev, "Failed initialize hw arbiter\n"); - ret = -EFAULT; - goto out_err; - } if (pci_save_state(pdev)) { dev_err(&pdev->dev, "Failed to save pci state\n"); ret = -ENOMEM; goto out_err; } - /* Enable bundle and misc interrupts */ - ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF0_MASK_OFFSET, - ADF_DH895XCC_SMIA0_MASK); - ADF_CSR_WR(pmisc_bar_addr, ADF_DH895XCC_SMIAPF1_MASK_OFFSET, - ADF_DH895XCC_SMIA1_MASK); + ret = adf_dev_configure(accel_dev); + if (ret) + goto out_err; + + ret = adf_dev_init(accel_dev); + if (ret) + goto out_err; - ret = qat_dev_start(accel_dev); + ret = adf_dev_start(accel_dev); if (ret) { adf_dev_stop(accel_dev); goto out_err; diff --git a/drivers/crypto/qce/dma.c b/drivers/crypto/qce/dma.c index 0fb21e1..378cb76 100644 --- a/drivers/crypto/qce/dma.c +++ b/drivers/crypto/qce/dma.c @@ -64,7 +64,7 @@ int qce_mapsg(struct device *dev, struct scatterlist *sg, int nents, err = dma_map_sg(dev, sg, 1, dir); if (!err) return -EFAULT; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } } else { err = dma_map_sg(dev, sg, nents, dir); @@ -81,7 +81,7 @@ void qce_unmapsg(struct device *dev, struct scatterlist *sg, int nents, if (chained) while (sg) { dma_unmap_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } else dma_unmap_sg(dev, sg, nents, dir); @@ -100,7 +100,7 @@ int qce_countsg(struct scatterlist *sglist, int nbytes, bool *chained) nbytes -= sg->length; if (!sg_is_last(sg) && (sg + 1)->length == 0 && chained) *chained = true; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return nents; diff --git a/drivers/crypto/qce/sha.c b/drivers/crypto/qce/sha.c index f338593..5c5df1d 100644 --- a/drivers/crypto/qce/sha.c +++ b/drivers/crypto/qce/sha.c @@ -285,7 +285,7 @@ static int qce_ahash_update(struct ahash_request *req) break; len += sg_dma_len(sg); sg_last = sg; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } if (!sg_last) diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 220b92f..290a7f0 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -940,7 +940,7 @@ static int sahara_walk_and_recalc(struct scatterlist *sg, unsigned int nbytes) break; } nbytes -= sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return nbytes; diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 067ec21..ebbae8d 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -743,7 +743,7 @@ static int talitos_map_sg(struct device *dev, struct scatterlist *sg, if (unlikely(chained)) while (sg) { dma_map_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } else dma_map_sg(dev, sg, nents, dir); @@ -755,7 +755,7 @@ static void talitos_unmap_sg_chain(struct device *dev, struct scatterlist *sg, { while (sg) { dma_unmap_sg(dev, sg, 1, dir); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } } @@ -915,7 +915,7 @@ static int sg_to_link_tbl(struct scatterlist *sg, int sg_count, link_tbl_ptr->j_extent = 0; link_tbl_ptr++; cryptlen -= sg_dma_len(sg); - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } /* adjust (decrease) last one (or two) entry's len to cryptlen */ @@ -1102,7 +1102,7 @@ static int sg_count(struct scatterlist *sg_list, int nbytes, bool *chained) nbytes -= sg->length; if (!sg_is_last(sg) && (sg + 1)->length == 0) *chained = true; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } return sg_nents; diff --git a/drivers/crypto/ux500/cryp/cryp_core.c b/drivers/crypto/ux500/cryp/cryp_core.c index f831bb9..d594ae9 100644 --- a/drivers/crypto/ux500/cryp/cryp_core.c +++ b/drivers/crypto/ux500/cryp/cryp_core.c @@ -479,13 +479,13 @@ static void cryp_dma_setup_channel(struct cryp_device_data *device_data, .dst_addr = device_data->phybase + CRYP_DMA_TX_FIFO, .dst_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES, .dst_maxburst = 4, - }; + }; struct dma_slave_config cryp2mem = { .direction = DMA_DEV_TO_MEM, .src_addr = device_data->phybase + CRYP_DMA_RX_FIFO, .src_addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES, .src_maxburst = 4, - }; + }; dma_cap_zero(device_data->dma.mask); dma_cap_set(DMA_SLAVE, device_data->dma.mask); @@ -814,7 +814,7 @@ static int get_nents(struct scatterlist *sg, int nbytes) while (nbytes > 0) { nbytes -= sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); nents++; } @@ -1774,8 +1774,8 @@ static int ux500_cryp_resume(struct device *dev) static SIMPLE_DEV_PM_OPS(ux500_cryp_pm, ux500_cryp_suspend, ux500_cryp_resume); static const struct of_device_id ux500_cryp_match[] = { - { .compatible = "stericsson,ux500-cryp" }, - { }, + { .compatible = "stericsson,ux500-cryp" }, + { }, }; static struct platform_driver cryp_driver = { diff --git a/include/crypto/if_alg.h b/include/crypto/if_alg.h index 88ea64e..178525e 100644 --- a/include/crypto/if_alg.h +++ b/include/crypto/if_alg.h @@ -50,6 +50,7 @@ struct af_alg_type { void (*release)(void *private); int (*setkey)(void *private, const u8 *key, unsigned int keylen); int (*accept)(void *private, struct sock *sk); + int (*setauthsize)(void *private, unsigned int authsize); struct proto_ops *ops; struct module *owner; diff --git a/include/crypto/scatterwalk.h b/include/crypto/scatterwalk.h index 7ef512f..20e4226 100644 --- a/include/crypto/scatterwalk.h +++ b/include/crypto/scatterwalk.h @@ -33,21 +33,13 @@ static inline void scatterwalk_sg_chain(struct scatterlist *sg1, int num, sg1[num - 1].page_link |= 0x01; } -static inline struct scatterlist *scatterwalk_sg_next(struct scatterlist *sg) -{ - if (sg_is_last(sg)) - return NULL; - - return (++sg)->length ? sg : sg_chain_ptr(sg); -} - static inline void scatterwalk_crypto_chain(struct scatterlist *head, struct scatterlist *sg, int chain, int num) { if (chain) { head->length += sg->length; - sg = scatterwalk_sg_next(sg); + sg = sg_next(sg); } if (sg) diff --git a/include/linux/crypto.h b/include/linux/crypto.h index 9c8776d..fb5ef16 100644 --- a/include/linux/crypto.h +++ b/include/linux/crypto.h @@ -1147,7 +1147,7 @@ static inline void ablkcipher_request_free(struct ablkcipher_request *req) * cipher operation completes. * * The callback function is registered with the ablkcipher_request handle and - * must comply with the following template: + * must comply with the following template * * void callback_function(struct crypto_async_request *req, int error) */ @@ -1174,7 +1174,7 @@ static inline void ablkcipher_request_set_callback( * * For encryption, the source is treated as the plaintext and the * destination is the ciphertext. For a decryption operation, the use is - * reversed: the source is the ciphertext and the destination is the plaintext. + * reversed - the source is the ciphertext and the destination is the plaintext. */ static inline void ablkcipher_request_set_crypt( struct ablkcipher_request *req, @@ -1412,6 +1412,9 @@ static inline int crypto_aead_encrypt(struct aead_request *req) */ static inline int crypto_aead_decrypt(struct aead_request *req) { + if (req->cryptlen < crypto_aead_authsize(crypto_aead_reqtfm(req))) + return -EINVAL; + return crypto_aead_crt(crypto_aead_reqtfm(req))->decrypt(req); } @@ -1506,7 +1509,7 @@ static inline void aead_request_free(struct aead_request *req) * completes * * The callback function is registered with the aead_request handle and - * must comply with the following template: + * must comply with the following template * * void callback_function(struct crypto_async_request *req, int error) */ @@ -1533,7 +1536,7 @@ static inline void aead_request_set_callback(struct aead_request *req, * * For encryption, the source is treated as the plaintext and the * destination is the ciphertext. For a decryption operation, the use is - * reversed: the source is the ciphertext and the destination is the plaintext. + * reversed - the source is the ciphertext and the destination is the plaintext. * * IMPORTANT NOTE AEAD requires an authentication tag (MAC). For decryption, * the caller must concatenate the ciphertext followed by the diff --git a/include/linux/hw_random.h b/include/linux/hw_random.h index 914bb08..eb7b414 100644 --- a/include/linux/hw_random.h +++ b/include/linux/hw_random.h @@ -12,8 +12,10 @@ #ifndef LINUX_HWRANDOM_H_ #define LINUX_HWRANDOM_H_ +#include <linux/completion.h> #include <linux/types.h> #include <linux/list.h> +#include <linux/kref.h> /** * struct hwrng - Hardware Random Number Generator driver @@ -44,6 +46,8 @@ struct hwrng { /* internal. */ struct list_head list; + struct kref ref; + struct completion cleanup_done; }; /** Register a new Hardware Random Number Generator driver. */ diff --git a/lib/string.c b/lib/string.c index cdd97f4..ce81aae 100644 --- a/lib/string.c +++ b/lib/string.c @@ -596,6 +596,11 @@ EXPORT_SYMBOL(memset); * @s: Pointer to the start of the area. * @count: The size of the area. * + * Note: usually using memset() is just fine (!), but in cases + * where clearing out _local_ data at the end of a scope is + * necessary, memzero_explicit() should be used instead in + * order to prevent the compiler from optimising away zeroing. + * * memzero_explicit() doesn't need an arch-specific version as * it just invokes the one of memset() implicitly. */ |
