gcm-aarch64.c (3344B)
1 /* This Source Code Form is subject to the terms of the Mozilla Public 2 * License, v. 2.0. If a copy of the MPL was not distributed with this 3 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ 4 5 #ifdef FREEBL_NO_DEPEND 6 #include "stubs.h" 7 #endif 8 #include "gcm.h" 9 #include "secerr.h" 10 11 /* old gcc doesn't support some poly64x2_t intrinsic */ 12 #if defined(__aarch64__) && defined(IS_LITTLE_ENDIAN) && \ 13 (defined(__clang__) || defined(__GNUC__) && __GNUC__ > 6) 14 15 #include <arm_neon.h> 16 17 SECStatus 18 gcm_HashWrite_hw(gcmHashContext *ghash, unsigned char *outbuf) 19 { 20 uint8x16_t ci = vrbitq_u8(vreinterpretq_u8_u64(ghash->x)); 21 vst1q_u8(outbuf, ci); 22 return SECSuccess; 23 } 24 25 SECStatus 26 gcm_HashMult_hw(gcmHashContext *ghash, const unsigned char *buf, 27 unsigned int count) 28 { 29 const poly64x2_t p = vdupq_n_p64(0x87); 30 const uint8x16_t zero = vdupq_n_u8(0); 31 const uint64x2_t h = ghash->h; 32 uint64x2_t ci = ghash->x; 33 unsigned int i; 34 uint8x16_t z_low, z_high; 35 uint8x16_t t_low, t_high; 36 poly64x2_t t1; 37 uint8x16_t t2; 38 39 for (i = 0; i < count; i++, buf += 16) { 40 ci = vreinterpretq_u64_u8(veorq_u8(vreinterpretq_u8_u64(ci), 41 vrbitq_u8(vld1q_u8(buf)))); 42 43 /* Do binary mult ghash->X = Ci * ghash->H. */ 44 z_low = vreinterpretq_u8_p128( 45 vmull_p64((poly64_t)vget_low_p64(vreinterpretq_p64_u64(ci)), 46 (poly64_t)vget_low_p64(vreinterpretq_p64_u64(h)))); 47 z_high = vreinterpretq_u8_p128( 48 vmull_high_p64(vreinterpretq_p64_u64(ci), vreinterpretq_p64_u64(h))); 49 t1 = vreinterpretq_p64_u8( 50 vextq_u8(vreinterpretq_u8_u64(h), vreinterpretq_u8_u64(h), 8)); 51 t_low = vreinterpretq_u8_p128( 52 vmull_p64((poly64_t)vget_low_p64(vreinterpretq_p64_u64(ci)), 53 (poly64_t)vget_low_p64(t1))); 54 t_high = vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u64(ci), t1)); 55 t2 = veorq_u8(t_high, t_low); 56 z_low = veorq_u8(z_low, vextq_u8(zero, t2, 8)); 57 z_high = veorq_u8(z_high, vextq_u8(t2, zero, 8)); 58 59 /* polynomial reduction */ 60 t2 = vreinterpretq_u8_p128(vmull_high_p64(vreinterpretq_p64_u8(z_high), p)); 61 z_high = veorq_u8(z_high, vextq_u8(t2, zero, 8)); 62 z_low = veorq_u8(z_low, vextq_u8(zero, t2, 8)); 63 ci = veorq_u64(vreinterpretq_u64_u8(z_low), 64 vreinterpretq_u64_p128( 65 vmull_p64((poly64_t)vget_low_p64(vreinterpretq_p64_u8(z_high)), 66 (poly64_t)vget_low_p64(p)))); 67 } 68 69 ghash->x = ci; 70 return SECSuccess; 71 } 72 73 SECStatus 74 gcm_HashInit_hw(gcmHashContext *ghash) 75 { 76 /* Workaround of "used uninitialized in this function" error */ 77 uint64x2_t h = vdupq_n_u64(0); 78 79 ghash->ghash_mul = gcm_HashMult_hw; 80 ghash->x = vdupq_n_u64(0); 81 h = vsetq_lane_u64(__builtin_bswap64(ghash->h_low), h, 1); 82 h = vsetq_lane_u64(__builtin_bswap64(ghash->h_high), h, 0); 83 h = vreinterpretq_u64_u8(vrbitq_u8(vreinterpretq_u8_u64(h))); 84 ghash->h = h; 85 ghash->hw = PR_TRUE; 86 return SECSuccess; 87 } 88 89 SECStatus 90 gcm_HashZeroX_hw(gcmHashContext *ghash) 91 { 92 ghash->x = vdupq_n_u64(0); 93 return SECSuccess; 94 } 95 96 #endif /* defined(__clang__) || (defined(__GNUC__) && __GNUC__ > 6) */