/* sha512-armv7-neon.S - ARM/NEON assembly implementation of SHA-512 transform * * Copyright © 2013-2014 Jussi Kivilinna * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License as published by the Free * Software Foundation; either version 2 of the License, or (at your option) * any later version. */ #include .syntax unified .code 32 .fpu neon .text /* structure of SHA512_CONTEXT */ #define hd_a 0 #define hd_b ((hd_a) + 8) #define hd_c ((hd_b) + 8) #define hd_d ((hd_c) + 8) #define hd_e ((hd_d) + 8) #define hd_f ((hd_e) + 8) #define hd_g ((hd_f) + 8) /* register macros */ #define RK %r2 #define RA d0 #define RB d1 #define RC d2 #define RD d3 #define RE d4 #define RF d5 #define RG d6 #define RH d7 #define RT0 d8 #define RT1 d9 #define RT2 d10 #define RT3 d11 #define RT4 d12 #define RT5 d13 #define RT6 d14 #define RT7 d15 #define RT01q q4 #define RT23q q5 #define RT45q q6 #define RT67q q7 #define RW0 d16 #define RW1 d17 #define RW2 d18 #define RW3 d19 #define RW4 d20 #define RW5 d21 #define RW6 d22 #define RW7 d23 #define RW8 d24 #define RW9 d25 #define RW10 d26 #define RW11 d27 #define RW12 d28 #define RW13 d29 #define RW14 d30 #define RW15 d31 #define RW01q q8 #define RW23q q9 #define RW45q q10 #define RW67q q11 #define RW89q q12 #define RW1011q q13 #define RW1213q q14 #define RW1415q q15 /*********************************************************************** * ARM assembly implementation of sha512 transform ***********************************************************************/ #define rounds2_0_63(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, rw01q, rw2, \ rw23q, rw1415q, rw9, rw10, interleave_op, arg1) \ /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ vshr.u64 RT2, re, #14; \ vshl.u64 RT3, re, #64 - 14; \ interleave_op(arg1); \ vshr.u64 RT4, re, #18; \ vshl.u64 RT5, re, #64 - 18; \ vld1.64 {RT0}, [RK]!; \ veor.64 RT23q, RT23q, RT45q; \ vshr.u64 RT4, re, #41; \ vshl.u64 RT5, re, #64 - 41; \ vadd.u64 RT0, RT0, rw0; \ veor.64 RT23q, RT23q, RT45q; \ vmov.64 RT7, re; \ veor.64 RT1, RT2, RT3; \ vbsl.64 RT7, rf, rg; \ \ vadd.u64 RT1, RT1, rh; \ vshr.u64 RT2, ra, #28; \ vshl.u64 RT3, ra, #64 - 28; \ vadd.u64 RT1, RT1, RT0; \ vshr.u64 RT4, ra, #34; \ vshl.u64 RT5, ra, #64 - 34; \ vadd.u64 RT1, RT1, RT7; \ \ /* h = Sum0 (a) + Maj (a, b, c); */ \ veor.64 RT23q, RT23q, RT45q; \ vshr.u64 RT4, ra, #39; \ vshl.u64 RT5, ra, #64 - 39; \ veor.64 RT0, ra, rb; \ veor.64 RT23q, RT23q, RT45q; \ vbsl.64 RT0, rc, rb; \ vadd.u64 rd, rd, RT1; /* d+=t1; */ \ veor.64 rh, RT2, RT3; \ \ /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ vshr.u64 RT2, rd, #14; \ vshl.u64 RT3, rd, #64 - 14; \ vadd.u64 rh, rh, RT0; \ vshr.u64 RT4, rd, #18; \ vshl.u64 RT5, rd, #64 - 18; \ vadd.u64 rh, rh, RT1; /* h+=t1; */ \ vld1.64 {RT0}, [RK]!; \ veor.64 RT23q, RT23q, RT45q; \ vshr.u64 RT4, rd, #41; \ vshl.u64 RT5, rd, #64 - 41; \ vadd.u64 RT0, RT0, rw1; \ veor.64 RT23q, RT23q, RT45q; \ vmov.64 RT7, rd; \ veor.64 RT1, RT2, RT3; \ vbsl.64 RT7, re, rf; \ \ vadd.u64 RT1, RT1, rg; \ vshr.u64 RT2, rh, #28; \ vshl.u64 RT3, rh, #64 - 28; \ vadd.u64 RT1, RT1, RT0; \ vshr.u64 RT4, rh, #34; \ vshl.u64 RT5, rh, #64 - 34; \ vadd.u64 RT1, RT1, RT7; \ \ /* g = Sum0 (h) + Maj (h, a, b); */ \ veor.64 RT23q, RT23q, RT45q; \ vshr.u64 RT4, rh, #39; \ vshl.u64 RT5, rh, #64 - 39; \ veor.64 RT0, rh, ra; \ veor.64 RT23q, RT23q, RT45q; \ vbsl.64 RT0, rb, ra; \ vadd.u64 rc, rc, RT1; /* c+=t1; */ \ veor.64 rg, RT2, RT3; \ \ /* w[0] += S1 (w[14]) + w[9] + S0 (w[1]); */ \ /* w[1] += S1 (w[15]) + w[10] + S0 (w[2]); */ \ \ /**** S0(w[1:2]) */ \ \ /* w[0:1] += w[9:10] */ \ /* RT23q = rw1:rw2 */ \ vext.u64 RT23q, rw01q, rw23q, #1; \ vadd.u64 rw0, rw9; \ vadd.u64 rg, rg, RT0; \ vadd.u64 rw1, rw10;\ vadd.u64 rg, rg, RT1; /* g+=t1; */ \ \ vshr.u64 RT45q, RT23q, #1; \ vshl.u64 RT67q, RT23q, #64 - 1; \ vshr.u64 RT01q, RT23q, #8; \ veor.u64 RT45q, RT45q, RT67q; \ vshl.u64 RT67q, RT23q, #64 - 8; \ veor.u64 RT45q, RT45q, RT01q; \ vshr.u64 RT01q, RT23q, #7; \ veor.u64 RT45q, RT45q, RT67q; \ \ /**** S1(w[14:15]) */ \ vshr.u64 RT23q, rw1415q, #6; \ veor.u64 RT01q, RT01q, RT45q; \ vshr.u64 RT45q, rw1415q, #19; \ vshl.u64 RT67q, rw1415q, #64 - 19; \ veor.u64 RT23q, RT23q, RT45q; \ vshr.u64 RT45q, rw1415q, #61; \ veor.u64 RT23q, RT23q, RT67q; \ vshl.u64 RT67q, rw1415q, #64 - 61; \ veor.u64 RT23q, RT23q, RT45q; \ vadd.u64 rw01q, RT01q; /* w[0:1] += S(w[1:2]) */ \ veor.u64 RT01q, RT23q, RT67q; #define vadd_RT01q(rw01q) \ /* w[0:1] += S(w[14:15]) */ \ vadd.u64 rw01q, RT01q; #define dummy(_) /*_*/ #define rounds2_64_79(ra, rb, rc, rd, re, rf, rg, rh, rw0, rw1, \ interleave_op1, arg1, interleave_op2, arg2) \ /* t1 = h + Sum1 (e) + Ch (e, f, g) + k[t] + w[t]; */ \ vshr.u64 RT2, re, #14; \ vshl.u64 RT3, re, #64 - 14; \ interleave_op1(arg1); \ vshr.u64 RT4, re, #18; \ vshl.u64 RT5, re, #64 - 18; \ interleave_op2(arg2); \ vld1.64 {RT0}, [RK]!; \ veor.64 RT23q, RT23q, RT45q; \ vshr.u64 RT4, re, #41; \ vshl.u64 RT5, re, #64 - 41; \ vadd.u64 RT0, RT0, rw0; \ veor.64 RT23q, RT23q, RT45q; \ vmov.64 RT7, re; \ veor.64 RT1, RT2, RT3; \ vbsl.64 RT7, rf, rg; \ \ vadd.u64 RT1, RT1, rh; \ vshr.u64 RT2, ra, #28; \ vshl.u64 RT3, ra, #64 - 28; \ vadd.u64 RT1, RT1, RT0; \ vshr.u64 RT4, ra, #34; \ vshl.u64 RT5, ra, #64 - 34; \ vadd.u64 RT1, RT1, RT7; \ \ /* h = Sum0 (a) + Maj (a, b, c); */ \ veor.64 RT23q, RT23q, RT45q; \ vshr.u64 RT4, ra, #39; \ vshl.u64 RT5, ra, #64 - 39; \ veor.64 RT0, ra, rb; \ veor.64 RT23q, RT23q, RT45q; \ vbsl.64 RT0, rc, rb; \ vadd.u64 rd, rd, RT1; /* d+=t1; */ \ veor.64 rh, RT2, RT3; \ \ /* t1 = g + Sum1 (d) + Ch (d, e, f) + k[t] + w[t]; */ \ vshr.u64 RT2, rd, #14; \ vshl.u64 RT3, rd, #64 - 14; \ vadd.u64 rh, rh, RT0; \ vshr.u64 RT4, rd, #18; \ vshl.u64 RT5, rd, #64 - 18; \ vadd.u64 rh, rh, RT1; /* h+=t1; */ \ vld1.64 {RT0}, [RK]!; \ veor.64 RT23q, RT23q, RT45q; \ vshr.u64 RT4, rd, #41; \ vshl.u64 RT5, rd, #64 - 41; \ vadd.u64 RT0, RT0, rw1; \ veor.64 RT23q, RT23q, RT45q; \ vmov.64 RT7, rd; \ veor.64 RT1, RT2, RT3; \ vbsl.64 RT7, re, rf; \ \ vadd.u64 RT1, RT1, rg; \ vshr.u64 RT2, rh, #28; \ vshl.u64 RT3, rh, #64 - 28; \ vadd.u64 RT1, RT1, RT0; \ vshr.u64 RT4, rh, #34; \ vshl.u64 RT5, rh, #64 - 34; \ vadd.u64 RT1, RT1, RT7; \ \ /* g = Sum0 (h) + Maj (h, a, b); */ \ veor.64 RT23q, RT23q, RT45q; \ vshr.u64 RT4, rh, #39; \ vshl.u64 RT5, rh, #64 - 39; \ veor.64 RT0, rh, ra; \ veor.64 RT23q, RT23q, RT45q; \ vbsl.64 RT0, rb, ra; \ vadd.u64 rc, rc, RT1; /* c+=t1; */ \ veor.64 rg, RT2, RT3; #define vadd_rg_RT0(rg) \ vadd.u64 rg, rg, RT0; #define vadd_rg_RT1(rg) \ vadd.u64 rg, rg, RT1; /* g+=t1; */ .align 3 ENTRY(sha512_transform_neon) /* Input: * %r0: SHA512_CONTEXT * %r1: data * %r2: u64 k[] constants * %r3: nblks */ push {%lr}; mov %lr, #0; /* Load context to d0-d7 */ vld1.64 {RA-RD}, [%r0]!; vld1.64 {RE-RH}, [%r0]; sub %r0, #(4*8); /* Load input to w[16], d16-d31 */ /* NOTE: Assumes that on ARMv7 unaligned accesses are always allowed. */ vld1.64 {RW0-RW3}, [%r1]!; vld1.64 {RW4-RW7}, [%r1]!; vld1.64 {RW8-RW11}, [%r1]!; vld1.64 {RW12-RW15}, [%r1]!; #ifdef __ARMEL__ /* byteswap */ vrev64.8 RW01q, RW01q; vrev64.8 RW23q, RW23q; vrev64.8 RW45q, RW45q; vrev64.8 RW67q, RW67q; vrev64.8 RW89q, RW89q; vrev64.8 RW1011q, RW1011q; vrev64.8 RW1213q, RW1213q; vrev64.8 RW1415q, RW1415q; #endif /* EABI says that d8-d15 must be preserved by callee. */ /*vpush {RT0-RT7};*/ .Loop: rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, RW23q, RW1415q, RW9, RW10, dummy, _); b .Lenter_rounds; .Loop_rounds: rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, RW01q, RW2, RW23q, RW1415q, RW9, RW10, vadd_RT01q, RW1415q); .Lenter_rounds: rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, RW23q, RW4, RW45q, RW01q, RW11, RW12, vadd_RT01q, RW01q); rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, RW45q, RW6, RW67q, RW23q, RW13, RW14, vadd_RT01q, RW23q); rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, RW67q, RW8, RW89q, RW45q, RW15, RW0, vadd_RT01q, RW45q); rounds2_0_63(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, RW89q, RW10, RW1011q, RW67q, RW1, RW2, vadd_RT01q, RW67q); rounds2_0_63(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, RW1011q, RW12, RW1213q, RW89q, RW3, RW4, vadd_RT01q, RW89q); add %lr, #16; rounds2_0_63(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, RW1213q, RW14, RW1415q, RW1011q, RW5, RW6, vadd_RT01q, RW1011q); cmp %lr, #64; rounds2_0_63(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, RW1415q, RW0, RW01q, RW1213q, RW7, RW8, vadd_RT01q, RW1213q); bne .Loop_rounds; subs %r3, #1; rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW0, RW1, vadd_RT01q, RW1415q, dummy, _); rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW2, RW3, vadd_rg_RT0, RG, vadd_rg_RT1, RG); beq .Lhandle_tail; vld1.64 {RW0-RW3}, [%r1]!; rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, vadd_rg_RT0, RE, vadd_rg_RT1, RE); rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, vadd_rg_RT0, RC, vadd_rg_RT1, RC); #ifdef __ARMEL__ vrev64.8 RW01q, RW01q; vrev64.8 RW23q, RW23q; #endif vld1.64 {RW4-RW7}, [%r1]!; rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, vadd_rg_RT0, RA, vadd_rg_RT1, RA); rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, vadd_rg_RT0, RG, vadd_rg_RT1, RG); #ifdef __ARMEL__ vrev64.8 RW45q, RW45q; vrev64.8 RW67q, RW67q; #endif vld1.64 {RW8-RW11}, [%r1]!; rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, vadd_rg_RT0, RE, vadd_rg_RT1, RE); rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, vadd_rg_RT0, RC, vadd_rg_RT1, RC); #ifdef __ARMEL__ vrev64.8 RW89q, RW89q; vrev64.8 RW1011q, RW1011q; #endif vld1.64 {RW12-RW15}, [%r1]!; vadd_rg_RT0(RA); vadd_rg_RT1(RA); /* Load context */ vld1.64 {RT0-RT3}, [%r0]!; vld1.64 {RT4-RT7}, [%r0]; sub %r0, #(4*8); #ifdef __ARMEL__ vrev64.8 RW1213q, RW1213q; vrev64.8 RW1415q, RW1415q; #endif vadd.u64 RA, RT0; vadd.u64 RB, RT1; vadd.u64 RC, RT2; vadd.u64 RD, RT3; vadd.u64 RE, RT4; vadd.u64 RF, RT5; vadd.u64 RG, RT6; vadd.u64 RH, RT7; /* Store the first half of context */ vst1.64 {RA-RD}, [%r0]!; sub RK, $(8*80); vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ mov %lr, #0; sub %r0, #(4*8); b .Loop; .Lhandle_tail: rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW4, RW5, vadd_rg_RT0, RE, vadd_rg_RT1, RE); rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW6, RW7, vadd_rg_RT0, RC, vadd_rg_RT1, RC); rounds2_64_79(RA, RB, RC, RD, RE, RF, RG, RH, RW8, RW9, vadd_rg_RT0, RA, vadd_rg_RT1, RA); rounds2_64_79(RG, RH, RA, RB, RC, RD, RE, RF, RW10, RW11, vadd_rg_RT0, RG, vadd_rg_RT1, RG); rounds2_64_79(RE, RF, RG, RH, RA, RB, RC, RD, RW12, RW13, vadd_rg_RT0, RE, vadd_rg_RT1, RE); rounds2_64_79(RC, RD, RE, RF, RG, RH, RA, RB, RW14, RW15, vadd_rg_RT0, RC, vadd_rg_RT1, RC); /* Load context to d16-d23 */ vld1.64 {RW0-RW3}, [%r0]!; vadd_rg_RT0(RA); vld1.64 {RW4-RW7}, [%r0]; vadd_rg_RT1(RA); sub %r0, #(4*8); vadd.u64 RA, RW0; vadd.u64 RB, RW1; vadd.u64 RC, RW2; vadd.u64 RD, RW3; vadd.u64 RE, RW4; vadd.u64 RF, RW5; vadd.u64 RG, RW6; vadd.u64 RH, RW7; /* Store the first half of context */ vst1.64 {RA-RD}, [%r0]!; /* Clear used registers */ /* d16-d31 */ veor.u64 RW01q, RW01q; veor.u64 RW23q, RW23q; veor.u64 RW45q, RW45q; veor.u64 RW67q, RW67q; vst1.64 {RE-RH}, [%r0]; /* Store the last half of context */ veor.u64 RW89q, RW89q; veor.u64 RW1011q, RW1011q; veor.u64 RW1213q, RW1213q; veor.u64 RW1415q, RW1415q; /* d8-d15 */ /*vpop {RT0-RT7};*/ /* d0-d7 (q0-q3) */ veor.u64 %q0, %q0; veor.u64 %q1, %q1; veor.u64 %q2, %q2; veor.u64 %q3, %q3; pop {%pc}; ENDPROC(sha512_transform_neon)