From a4b75a3bfe885bbd92332e3f48f4075f68f24c2e Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Sat, 28 Oct 2023 16:26:14 +0000 Subject: [PATCH] Renamed files. --- src/cha1block.h | 48 +++++++++ src/cha4block.h | 160 ++++++++++++++++++++++++++++++ src/cha8block.h | 259 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 467 insertions(+) create mode 100644 src/cha1block.h create mode 100644 src/cha4block.h create mode 100644 src/cha8block.h diff --git a/src/cha1block.h b/src/cha1block.h new file mode 100644 index 0000000..b4459f8 --- /dev/null +++ b/src/cha1block.h @@ -0,0 +1,48 @@ +#include +#include +#include + +#define QUARTERSTEP(a, b, c, n) \ + a += b; \ + c ^= a; \ + c = (c << n) | (c >> (32 - n)) + +#define QUARTERROUND(a, b, c, d) \ + QUARTERSTEP(a, b, d, 16); \ + QUARTERSTEP(c, d, b, 12); \ + QUARTERSTEP(a, b, d, 8); \ + QUARTERSTEP(c, d, b, 7); + +static inline uint64_t +_cha_block(uint32_t* state, uint8_t* begin, uint8_t* end) { + uint64_t* counter = (uint64_t*)&state[12]; + uint8_t* c = begin; + while (c < end) { + uint32_t x[16]; + memcpy(x, state, sizeof x); + for (int i = 20; i > 0; i -= 2) { + QUARTERROUND(x[0], x[4], x[8], x[12]) + QUARTERROUND(x[1], x[5], x[9], x[13]) + QUARTERROUND(x[2], x[6], x[10], x[14]) + QUARTERROUND(x[3], x[7], x[11], x[15]) + QUARTERROUND(x[0], x[5], x[10], x[15]) + QUARTERROUND(x[1], x[6], x[11], x[12]) + QUARTERROUND(x[2], x[7], x[8], x[13]) + QUARTERROUND(x[3], x[4], x[9], x[14]) + } + for (int i = 0; i < 16; i++) + x[i] += state[i]; + + ++*counter; + + uint64_t bytes = end - c; + if (bytes < 64) { + memcpy(c, x, bytes); + c = end; + break; + } + memcpy(c, x, 64); + c += 64; + } + return c - begin; +} diff --git a/src/cha4block.h b/src/cha4block.h new file mode 100644 index 0000000..626bf10 --- /dev/null +++ b/src/cha4block.h @@ -0,0 +1,160 @@ + +#define VEC4_ROT(A, IMM) \ + _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) + +/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 & + * 16) (better) */ +#define VEC4_QUARTERROUND(A, B, C, D) \ + x_##A = _mm_add_epi32(x_##A, x_##B); \ + t_##A = _mm_xor_si128(x_##D, x_##A); \ + x_##D = _mm_shuffle_epi8(t_##A, rot16); \ + x_##C = _mm_add_epi32(x_##C, x_##D); \ + t_##C = _mm_xor_si128(x_##B, x_##C); \ + x_##B = VEC4_ROT(t_##C, 12); \ + x_##A = _mm_add_epi32(x_##A, x_##B); \ + t_##A = _mm_xor_si128(x_##D, x_##A); \ + x_##D = _mm_shuffle_epi8(t_##A, rot8); \ + x_##C = _mm_add_epi32(x_##C, x_##D); \ + t_##C = _mm_xor_si128(x_##B, x_##C); \ + x_##B = VEC4_ROT(t_##C, 7) + +#define ONEQUAD(A, B, C, D, CT) \ + { \ + /* Add original block */ \ + x_##A = _mm_add_epi32(x_##A, orig##A); \ + x_##B = _mm_add_epi32(x_##B, orig##B); \ + x_##C = _mm_add_epi32(x_##C, orig##C); \ + x_##D = _mm_add_epi32(x_##D, orig##D); \ + /* Transpose */ \ + t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \ + t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \ + t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \ + t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \ + x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \ + x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \ + x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \ + x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \ + \ + _mm_storeu_si128((__m128i*)(CT), x_##A); \ + _mm_storeu_si128((__m128i*)(CT + 64), x_##B); \ + _mm_storeu_si128((__m128i*)(CT + 128), x_##C); \ + _mm_storeu_si128((__m128i*)(CT + 192), x_##D); \ + } + +static inline uint64_t +_cha_4block(uint32_t* state, uint8_t* begin, uint8_t* end) { + if (end - begin < 256) + return 0; + uint8_t* c = begin; + uint32_t* x = state; + + /* constant for shuffling bytes (replacing multiple-of-8 rotates) */ + const __m128i rot16 = + _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + const __m128i rot8 = + _mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); + + // Load state to vectors, duplicate four times + __m128i x_0 = _mm_set1_epi32(x[0]); + __m128i x_1 = _mm_set1_epi32(x[1]); + __m128i x_2 = _mm_set1_epi32(x[2]); + __m128i x_3 = _mm_set1_epi32(x[3]); + __m128i x_4 = _mm_set1_epi32(x[4]); + __m128i x_5 = _mm_set1_epi32(x[5]); + __m128i x_6 = _mm_set1_epi32(x[6]); + __m128i x_7 = _mm_set1_epi32(x[7]); + __m128i x_8 = _mm_set1_epi32(x[8]); + __m128i x_9 = _mm_set1_epi32(x[9]); + __m128i x_10 = _mm_set1_epi32(x[10]); + __m128i x_11 = _mm_set1_epi32(x[11]); + __m128i x_12; + __m128i x_13; + __m128i x_14 = _mm_set1_epi32(x[14]); + __m128i x_15 = _mm_set1_epi32(x[15]); + __m128i orig0 = x_0; + __m128i orig1 = x_1; + __m128i orig2 = x_2; + __m128i orig3 = x_3; + __m128i orig4 = x_4; + __m128i orig5 = x_5; + __m128i orig6 = x_6; + __m128i orig7 = x_7; + __m128i orig8 = x_8; + __m128i orig9 = x_9; + __m128i orig10 = x_10; + __m128i orig11 = x_11; + __m128i orig12 = {}; + __m128i orig13 = {}; + __m128i orig14 = x_14; + __m128i orig15 = x_15; + __m128i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12, + t_13, t_14, t_15; + + const __m128i addv12 = _mm_set_epi64x(1, 0); + const __m128i addv13 = _mm_set_epi64x(3, 2); + + while (end - c >= 256) { + x_0 = orig0; + x_1 = orig1; + x_2 = orig2; + x_3 = orig3; + x_4 = orig4; + x_5 = orig5; + x_6 = orig6; + x_7 = orig7; + x_8 = orig8; + x_9 = orig9; + x_10 = orig10; + x_11 = orig11; + x_14 = orig14; + x_15 = orig15; + + // Calculate counter + 0..3 for adjacent blocks (x12 low and x13 + // high of each) + uint32_t in12 = state[12]; + uint32_t in13 = state[13]; + uint64_t in1213 = ((uint64_t)in12) | (((uint64_t)in13) << 32); + __m128i t12, t13; + t12 = _mm_set1_epi64x(in1213); + t13 = _mm_set1_epi64x(in1213); + x_12 = _mm_add_epi64(addv12, t12); + x_13 = _mm_add_epi64(addv13, t13); + t12 = _mm_unpacklo_epi32(x_12, x_13); + t13 = _mm_unpackhi_epi32(x_12, x_13); + x_12 = _mm_unpacklo_epi32(t12, t13); + x_13 = _mm_unpackhi_epi32(t12, t13); + orig12 = x_12; + orig13 = x_13; + in1213 += 4; + state[12] = in1213 & 0xFFFFFFFF; + state[13] = (in1213 >> 32) & 0xFFFFFFFF; + + for (int i = 0; i < 10; ++i) { + // Mix columns + VEC4_QUARTERROUND(0, 4, 8, 12); + VEC4_QUARTERROUND(1, 5, 9, 13); + VEC4_QUARTERROUND(2, 6, 10, 14); + VEC4_QUARTERROUND(3, 7, 11, 15); + // Mix diagonals + VEC4_QUARTERROUND(0, 5, 10, 15); + VEC4_QUARTERROUND(1, 6, 11, 12); + VEC4_QUARTERROUND(2, 7, 8, 13); + VEC4_QUARTERROUND(3, 4, 9, 14); + } + + ONEQUAD(0, 1, 2, 3, c); + ONEQUAD(4, 5, 6, 7, c + 16); + ONEQUAD(8, 9, 10, 11, c + 32); + ONEQUAD(12, 13, 14, 15, c + 48); + + // *counter += 4; + c += 256; + } + return c - begin; // Bytes written +} + +#undef ONEQUAD +#undef ONEQUAD_TRANSPOSE +#undef VEC4_ROT +#undef VEC4_QUARTERROUND +#undef VEC4_QUARTERROUND_SHUFFLE diff --git a/src/cha8block.h b/src/cha8block.h new file mode 100644 index 0000000..12d54ee --- /dev/null +++ b/src/cha8block.h @@ -0,0 +1,259 @@ + +#define VEC8_ROT(A, IMM) \ + _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) + +/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 & + * 16) (better) */ +#define VEC8_QUARTERROUND(A, B, C, D) \ + x[A] = _mm256_add_epi32(x[A], x[B]); \ + t[A] = _mm256_xor_si256(x[D], x[A]); \ + x[D] = _mm256_shuffle_epi8(t[A], rot16); \ + x[C] = _mm256_add_epi32(x[C], x[D]); \ + t[C] = _mm256_xor_si256(x[B], x[C]); \ + x[B] = VEC8_ROT(t[C], 12); \ + x[A] = _mm256_add_epi32(x[A], x[B]); \ + t[A] = _mm256_xor_si256(x[D], x[A]); \ + x[D] = _mm256_shuffle_epi8(t[A], rot8); \ + x[C] = _mm256_add_epi32(x[C], x[D]); \ + t[C] = _mm256_xor_si256(x[B], x[C]); \ + x[B] = VEC8_ROT(t[C], 7) + +#define VEC8_LINE1(A, B, C, D) \ + x[A] = _mm256_add_epi32(x[A], x[B]); \ + x[D] = _mm256_shuffle_epi8(_mm256_xor_si256(x[D], x[A]), rot16) +#define VEC8_LINE2(A, B, C, D) \ + x[C] = _mm256_add_epi32(x[C], x[D]); \ + x[B] = VEC8_ROT(_mm256_xor_si256(x[B], x[C]), 12) +#define VEC8_LINE3(A, B, C, D) \ + x[A] = _mm256_add_epi32(x[A], x[B]); \ + x[D] = _mm256_shuffle_epi8(_mm256_xor_si256(x[D], x[A]), rot8) +#define VEC8_LINE4(A, B, C, D) \ + x[C] = _mm256_add_epi32(x[C], x[D]); \ + x[B] = VEC8_ROT(_mm256_xor_si256(x[B], x[C]), 7) + +#define VEC8_ROUND_SEQ( \ + A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ +) \ + VEC8_LINE1(A1, B1, C1, D1); \ + VEC8_LINE1(A2, B2, C2, D2); \ + VEC8_LINE1(A3, B3, C3, D3); \ + VEC8_LINE1(A4, B4, C4, D4); \ + VEC8_LINE2(A1, B1, C1, D1); \ + VEC8_LINE2(A2, B2, C2, D2); \ + VEC8_LINE2(A3, B3, C3, D3); \ + VEC8_LINE2(A4, B4, C4, D4); \ + VEC8_LINE3(A1, B1, C1, D1); \ + VEC8_LINE3(A2, B2, C2, D2); \ + VEC8_LINE3(A3, B3, C3, D3); \ + VEC8_LINE3(A4, B4, C4, D4); \ + VEC8_LINE4(A1, B1, C1, D1); \ + VEC8_LINE4(A2, B2, C2, D2); \ + VEC8_LINE4(A3, B3, C3, D3); \ + VEC8_LINE4(A4, B4, C4, D4) + +#define VEC8_ROUND_HALF( \ + A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ +) \ + VEC8_LINE1(A1, B1, C1, D1); \ + VEC8_LINE1(A2, B2, C2, D2); \ + VEC8_LINE2(A1, B1, C1, D1); \ + VEC8_LINE2(A2, B2, C2, D2); \ + VEC8_LINE3(A1, B1, C1, D1); \ + VEC8_LINE3(A2, B2, C2, D2); \ + VEC8_LINE4(A1, B1, C1, D1); \ + VEC8_LINE4(A2, B2, C2, D2); \ + VEC8_LINE1(A3, B3, C3, D3); \ + VEC8_LINE1(A4, B4, C4, D4); \ + VEC8_LINE2(A3, B3, C3, D3); \ + VEC8_LINE2(A4, B4, C4, D4); \ + VEC8_LINE3(A3, B3, C3, D3); \ + VEC8_LINE3(A4, B4, C4, D4); \ + VEC8_LINE4(A3, B3, C3, D3); \ + VEC8_LINE4(A4, B4, C4, D4) + +#define VEC8_ROUND_HALFANDHALF( \ + A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ +) \ + VEC8_LINE1(A1, B1, C1, D1); \ + VEC8_LINE1(A2, B2, C2, D2); \ + VEC8_LINE2(A1, B1, C1, D1); \ + VEC8_LINE2(A2, B2, C2, D2); \ + VEC8_LINE1(A3, B3, C3, D3); \ + VEC8_LINE1(A4, B4, C4, D4); \ + VEC8_LINE2(A3, B3, C3, D3); \ + VEC8_LINE2(A4, B4, C4, D4); \ + VEC8_LINE3(A1, B1, C1, D1); \ + VEC8_LINE3(A2, B2, C2, D2); \ + VEC8_LINE4(A1, B1, C1, D1); \ + VEC8_LINE4(A2, B2, C2, D2); \ + VEC8_LINE3(A3, B3, C3, D3); \ + VEC8_LINE3(A4, B4, C4, D4); \ + VEC8_LINE4(A3, B3, C3, D3); \ + VEC8_LINE4(A4, B4, C4, D4) + +#define VEC8_ROUND( \ + A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ +) \ + VEC8_ROUND_SEQ( \ + A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ + ) + +#define ONEQUAD_TRANSPOSE(A, B, C, D) \ + { \ + __m128i t0, t1, t2, t3; \ + x[A] = _mm256_add_epi32(x[A], orig[A]); \ + x[B] = _mm256_add_epi32(x[B], orig[B]); \ + x[C] = _mm256_add_epi32(x[C], orig[C]); \ + x[D] = _mm256_add_epi32(x[D], orig[D]); \ + t[A] = _mm256_unpacklo_epi32(x[A], x[B]); \ + t[B] = _mm256_unpacklo_epi32(x[C], x[D]); \ + t[C] = _mm256_unpackhi_epi32(x[A], x[B]); \ + t[D] = _mm256_unpackhi_epi32(x[C], x[D]); \ + x[A] = _mm256_unpacklo_epi64(t[A], t[B]); \ + x[B] = _mm256_unpackhi_epi64(t[A], t[B]); \ + x[C] = _mm256_unpacklo_epi64(t[C], t[D]); \ + x[D] = _mm256_unpackhi_epi64(t[C], t[D]); \ + _mm_storeu_si128( \ + (__m128i*)(c + 0), _mm256_extracti128_si256(x[A], 0) \ + ); \ + _mm_storeu_si128( \ + (__m128i*)(c + 64), _mm256_extracti128_si256(x[B], 0) \ + ); \ + _mm_storeu_si128( \ + (__m128i*)(c + 128), _mm256_extracti128_si256(x[C], 0) \ + ); \ + _mm_storeu_si128( \ + (__m128i*)(c + 192), _mm256_extracti128_si256(x[D], 0) \ + ); \ + _mm_storeu_si128( \ + (__m128i*)(c + 256), _mm256_extracti128_si256(x[A], 1) \ + ); \ + _mm_storeu_si128( \ + (__m128i*)(c + 320), _mm256_extracti128_si256(x[B], 1) \ + ); \ + _mm_storeu_si128( \ + (__m128i*)(c + 384), _mm256_extracti128_si256(x[C], 1) \ + ); \ + _mm_storeu_si128( \ + (__m128i*)(c + 448), _mm256_extracti128_si256(x[D], 1) \ + ); \ + } + +#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) + +#define ONEQUAD_UNPCK(A, B, C, D) \ + { \ + x[A] = _mm256_add_epi32(x[A], orig[A]); \ + x[B] = _mm256_add_epi32(x[B], orig[B]); \ + x[C] = _mm256_add_epi32(x[C], orig[C]); \ + x[D] = _mm256_add_epi32(x[D], orig[D]); \ + t[A] = _mm256_unpacklo_epi32(x[A], x[B]); \ + t[B] = _mm256_unpacklo_epi32(x[C], x[D]); \ + t[C] = _mm256_unpackhi_epi32(x[A], x[B]); \ + t[D] = _mm256_unpackhi_epi32(x[C], x[D]); \ + x[A] = _mm256_unpacklo_epi64(t[A], t[B]); \ + x[B] = _mm256_unpackhi_epi64(t[A], t[B]); \ + x[C] = _mm256_unpacklo_epi64(t[C], t[D]); \ + x[D] = _mm256_unpackhi_epi64(t[C], t[D]); \ + } + +#define ONEOCTO(A, B, C, D, A2, B2, C2, D2, c) \ + { \ + ONEQUAD_UNPCK(A, B, C, D); \ + ONEQUAD_UNPCK(A2, B2, C2, D2); \ + t[A] = _mm256_permute2x128_si256(x[A], x[A2], 0x20); \ + t[A2] = _mm256_permute2x128_si256(x[A], x[A2], 0x31); \ + t[B] = _mm256_permute2x128_si256(x[B], x[B2], 0x20); \ + t[B2] = _mm256_permute2x128_si256(x[B], x[B2], 0x31); \ + t[C] = _mm256_permute2x128_si256(x[C], x[C2], 0x20); \ + t[C2] = _mm256_permute2x128_si256(x[C], x[C2], 0x31); \ + t[D] = _mm256_permute2x128_si256(x[D], x[D2], 0x20); \ + t[D2] = _mm256_permute2x128_si256(x[D], x[D2], 0x31); \ + _mm256_storeu_si256((__m256i*)(c), t[A]); \ + _mm256_storeu_si256((__m256i*)(c + 64), t[B]); \ + _mm256_storeu_si256((__m256i*)(c + 128), t[C]); \ + _mm256_storeu_si256((__m256i*)(c + 192), t[D]); \ + _mm256_storeu_si256((__m256i*)(c + 256), t[A2]); \ + _mm256_storeu_si256((__m256i*)(c + 320), t[B2]); \ + _mm256_storeu_si256((__m256i*)(c + 384), t[C2]); \ + _mm256_storeu_si256((__m256i*)(c + 448), t[D2]); \ + } + +static inline uint64_t +_cha_8block(uint32_t* state, uint8_t* begin, uint8_t* end) { + if (end - begin < 512) + return 0; + + uint8_t* c = begin; + uint64_t* counter = (uint64_t*)(state + 12); + /* constant for shuffling bytes (replacing multiple-of-8 rotates) */ + __m256i rot16 = _mm256_set_epi8( + 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, 13, 12, 15, 14, 9, + 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 + ); + __m256i rot8 = _mm256_set_epi8( + 14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3, 14, 13, 12, 15, 10, + 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3 + ); + + /* the naive way seems as fast (if not a bit faster) than the vector way */ + __m256i x[16], orig[16], t[16]; + for (int i = 0; i < 16; ++i) + orig[i] = _mm256_set1_epi32(state[i]); + + const __m256i addv12 = _mm256_set_epi64x(3, 2, 1, 0); + const __m256i addv13 = _mm256_set_epi64x(7, 6, 5, 4); + + while (end - c >= 512) { + for (int i = 0; i < 16; ++i) + if (i != 12 && i != 13) + x[i] = orig[i]; + + // Calculate the eight parallel counters on x_12 and x_13 + t[13] = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(*counter)); + t[12] = _mm256_add_epi64(addv12, t[13]); + t[13] = _mm256_add_epi64(addv13, t[13]); + x[12] = _mm256_unpacklo_epi32(t[12], t[13]); + x[13] = _mm256_unpackhi_epi32(t[12], t[13]); + t[12] = _mm256_unpacklo_epi32(x[12], x[13]); + t[13] = _mm256_unpackhi_epi32(x[12], x[13]); + + /* required because unpack* are intra-lane */ + const __m256i permute = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0); + x[12] = _mm256_permutevar8x32_epi32(t[12], permute); + x[13] = _mm256_permutevar8x32_epi32(t[13], permute); + + orig[12] = x[12]; + orig[13] = x[13]; + + for (int i = 0; i < 10; ++i) { + VEC8_ROUND(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + VEC8_ROUND(0, 5, 10, 15, 1, 6, 11, 12, 2, 7, 8, 13, 3, 4, 9, 14); + } + + ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7, c); + ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15, c + 32); + + *counter += 8; + c += 512; + } + return c - begin; +} + +#undef ONEQUAD +#undef ONEQUAD_TRANSPOSE +#undef ONEQUAD_UNPCK +#undef ONEOCTO +#undef VEC8_ROT +#undef VEC8_QUARTERROUND +#undef VEC8_QUARTERROUND_NAIVE +#undef VEC8_QUARTERROUND_SHUFFLE +#undef VEC8_QUARTERROUND_SHUFFLE2 +#undef VEC8_LINE1 +#undef VEC8_LINE2 +#undef VEC8_LINE3 +#undef VEC8_LINE4 +#undef VEC8_ROUND +#undef VEC8_ROUND_SEQ +#undef VEC8_ROUND_HALF +#undef VEC8_ROUND_HALFANDHALF