From d01d35973aabd48fc2e219005c0b7d92d6411b41 Mon Sep 17 00:00:00 2001 From: pjd Date: Thu, 27 Oct 2011 14:07:57 +0000 Subject: Improve AES-NI performance for AES-XTS: - Operate on uint64_t types when doing XORing, etc. instead of uint8_t. - Don't bzero() temporary block for every AES block. Do it once for entire data block. - AES-NI is available only on little endian architectures. Simplify code that takes block number from IV. Benchmarks: Memory-backed md(4) device, software AES-XTS, 4kB sector: # dd if=/dev/md0.eli bs=1m 59.61MB/s Memory-backed md(4) device, old AES-NI AES-XTS, 4kB sector: # dd if=/dev/md0.eli bs=1m 97.29MB/s Memory-backed md(4) device, new AES-NI AES-XTS, 4kB sector: # dd if=/dev/md0.eli bs=1m 221.26MB/s 127% performance improvement between old and new code. Harddisk, raw speed: # dd if=/dev/ada0 bs=1m 137.63MB/s Harddisk, software AES-XTS, 4kB sector: # dd if=/dev/ada0.eli bs=1m 47.83MB/s (34% of raw disk speed) Harddisk, old AES-NI AES-XTS, 4kB sector: # dd if=/dev/ada0.eli bs=1m 68.33MB/s (49% of raw disk speed) Harddisk, new AES-NI AES-XTS, 4kB sector: # dd if=/dev/ada0.eli bs=1m 108.35MB/s (78% of raw disk speed) 58% performance improvement between old and new code. As a side-note, GELI with AES-NI using AES-CBC can achive native disk speed. MFC after: 3 days --- sys/crypto/aesni/aesni_wrap.c | 53 ++++++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 26 deletions(-) (limited to 'sys/crypto') diff --git a/sys/crypto/aesni/aesni_wrap.c b/sys/crypto/aesni/aesni_wrap.c index 0f3eeb1..89c896d 100644 --- a/sys/crypto/aesni/aesni_wrap.c +++ b/sys/crypto/aesni/aesni_wrap.c @@ -87,33 +87,33 @@ aesni_decrypt_ecb(int rounds, const void *key_schedule, size_t len, #define AES_XTS_ALPHA 0x87 /* GF(2^128) generator polynomial */ static void -aesni_crypt_xts_block(int rounds, const void *key_schedule, uint8_t *tweak, - const uint8_t *from, uint8_t *to, int do_encrypt) +aesni_crypt_xts_block(int rounds, const void *key_schedule, uint64_t *tweak, + const uint64_t *from, uint64_t *to, uint64_t *block, int do_encrypt) { - uint8_t block[AES_XTS_BLOCKSIZE]; - u_int i, carry_in, carry_out; + int carry; - for (i = 0; i < AES_XTS_BLOCKSIZE; i++) - block[i] = from[i] ^ tweak[i]; + block[0] = from[0] ^ tweak[0]; + block[1] = from[1] ^ tweak[1]; if (do_encrypt) - aesni_enc(rounds - 1, key_schedule, block, to, NULL); + aesni_enc(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL); else - aesni_dec(rounds - 1, key_schedule, block, to, NULL); + aesni_dec(rounds - 1, key_schedule, (uint8_t *)block, (uint8_t *)to, NULL); - for (i = 0; i < AES_XTS_BLOCKSIZE; i++) - to[i] ^= tweak[i]; + to[0] ^= tweak[0]; + to[1] ^= tweak[1]; /* Exponentiate tweak. */ - carry_in = 0; - for (i = 0; i < AES_XTS_BLOCKSIZE; i++) { - carry_out = tweak[i] & 0x80; - tweak[i] = (tweak[i] << 1) | (carry_in ? 1 : 0); - carry_in = carry_out; + carry = ((tweak[0] & 0x8000000000000000ULL) > 0); + tweak[0] <<= 1; + if (tweak[1] & 0x8000000000000000ULL) { + uint8_t *twk = (uint8_t *)tweak; + + twk[0] ^= AES_XTS_ALPHA; } - if (carry_in) - tweak[0] ^= AES_XTS_ALPHA; - bzero(block, sizeof(block)); + tweak[1] <<= 1; + if (carry) + tweak[1] |= 1; } static void @@ -121,32 +121,33 @@ aesni_crypt_xts(int rounds, const void *data_schedule, const void *tweak_schedule, size_t len, const uint8_t *from, uint8_t *to, const uint8_t iv[AES_BLOCK_LEN], int do_encrypt) { + uint64_t block[AES_XTS_BLOCKSIZE / 8]; uint8_t tweak[AES_XTS_BLOCKSIZE]; - uint64_t blocknum; size_t i; /* * Prepare tweak as E_k2(IV). IV is specified as LE representation * of a 64-bit block number which we allow to be passed in directly. */ - bcopy(iv, &blocknum, AES_XTS_IVSIZE); - for (i = 0; i < AES_XTS_IVSIZE; i++) { - tweak[i] = blocknum & 0xff; - blocknum >>= 8; - } +#if BYTE_ORDER == LITTLE_ENDIAN + bcopy(iv, tweak, AES_XTS_IVSIZE); /* Last 64 bits of IV are always zero. */ bzero(tweak + AES_XTS_IVSIZE, AES_XTS_IVSIZE); +#else +#error Only LITTLE_ENDIAN architectures are supported. +#endif aesni_enc(rounds - 1, tweak_schedule, tweak, tweak, NULL); len /= AES_XTS_BLOCKSIZE; for (i = 0; i < len; i++) { - aesni_crypt_xts_block(rounds, data_schedule, tweak, from, to, - do_encrypt); + aesni_crypt_xts_block(rounds, data_schedule, (uint64_t *)tweak, + (const uint64_t *)from, (uint64_t *)to, block, do_encrypt); from += AES_XTS_BLOCKSIZE; to += AES_XTS_BLOCKSIZE; } bzero(tweak, sizeof(tweak)); + bzero(block, sizeof(block)); } static void -- cgit v1.1