From 236f7eedf6c4536c7f72c138cb04a1aea9f938e5 Mon Sep 17 00:00:00 2001 From: Leo Vasanko Date: Tue, 28 Nov 2023 23:39:38 +0000 Subject: [PATCH] Rewrite, cleanup, configurable ChaCha rounds. --- benchmarks/charand.c | 14 ++- meson.build | 8 -- src/cha1block.h | 64 +++------- src/cha4block.h | 178 +++++++++----------------- src/cha8block.h | 290 ++++++++++++------------------------------- src/chacha20.h | 110 ++++++++++------ src/cli.c | 34 +++-- 7 files changed, 260 insertions(+), 438 deletions(-) diff --git a/benchmarks/charand.c b/benchmarks/charand.c index 19625c6..16c0833 100644 --- a/benchmarks/charand.c +++ b/benchmarks/charand.c @@ -1,16 +1,20 @@ -#include "chacha20.c" +#include "chacha20.h" #include #include -int main(void) -{ +int main(void) { uint64_t N = 1000000; uint8_t* buf = malloc(N); const uint8_t key[32] = {0}; const uint8_t nonce[16] = {0}; - for (uint64_t i = 0; i < 1000; ++i) - { + for (uint64_t i = 0; i < 1000; ++i) { cha_generate(buf, N, key, nonce); } + for (unsigned i = 0; i < 16; ++i) + printf("%02X ", buf[i]); + for (unsigned i = 0; i < 16; ++i) + printf(" %02X", buf[1024 + i]); + + puts(""); free(buf); } diff --git a/meson.build b/meson.build index cad8096..e5e8662 100644 --- a/meson.build +++ b/meson.build @@ -2,15 +2,7 @@ project('randquik', 'c') executable( 'randquik', 'src/cli.c', - 'src/chacha20.c', c_args: ['-Wall', '-O3', '-march=native'], install: true, ) dependency('threads') - -library( - 'randquik-chacha20', - 'src/chacha20.c', - build_by_default: true, - c_args: ['-Wall', '-O3', '-march=native'], -) diff --git a/src/cha1block.h b/src/cha1block.h index cd03e92..581ce62 100644 --- a/src/cha1block.h +++ b/src/cha1block.h @@ -1,50 +1,26 @@ #include -#include #include -#define QUARTERSTEP(a, b, c, n) \ - a += b; \ - c ^= a; \ - c = (c << n) | (c >> (32 - n)) +// clang-format off +#define QUARTERSTEP(a, b, c, n) a += b; c ^= a; c = (c << n) | (c >> (32 - n)) +#define QUARTERROUND(a, b, c, d) {\ + QUARTERSTEP(a, b, d, 16); QUARTERSTEP(c, d, b, 12); \ + QUARTERSTEP(a, b, d, 8); QUARTERSTEP(c, d, b, 7); } -#define QUARTERROUND(a, b, c, d) \ - QUARTERSTEP(a, b, d, 16); \ - QUARTERSTEP(c, d, b, 12); \ - QUARTERSTEP(a, b, d, 8); \ - QUARTERSTEP(c, d, b, 7); - -static inline uint64_t _cha_block(cha_ctx* ctx, uint8_t* begin, uint8_t* end) { - uint64_t* counter = (uint64_t*)&ctx->state[12]; - uint8_t* c = begin; - while (c < end) { - uint32_t x[16]; - memcpy(x, ctx->state, sizeof x); - for (int i = 20; i > 0; i -= 2) { - QUARTERROUND(x[0], x[4], x[8], x[12]) - QUARTERROUND(x[1], x[5], x[9], x[13]) - QUARTERROUND(x[2], x[6], x[10], x[14]) - QUARTERROUND(x[3], x[7], x[11], x[15]) - QUARTERROUND(x[0], x[5], x[10], x[15]) - QUARTERROUND(x[1], x[6], x[11], x[12]) - QUARTERROUND(x[2], x[7], x[8], x[13]) - QUARTERROUND(x[3], x[4], x[9], x[14]) - } - for (int i = 0; i < 16; i++) - x[i] += ctx->state[i]; - - ++*counter; - - uint64_t bytes = end - c; - if (bytes < 64) { - memcpy(c, x, bytes); - c = end; - // Leftover bytes are stored in ctx for next call - ctx->uncount = 64 - bytes; - memcpy(ctx->unconsumed, (uint8_t*)x + bytes, ctx->uncount); - break; - } - memcpy(c, x, 64); - c += 64; +static inline uint64_t _cha_block(uint8_t out[CHA_BLOCK_SIZE], uint32_t state[16], unsigned rounds) { + uint32_t x[16]; + for (unsigned i = 0; i < 16; ++i) x[i] = state[i]; // Faster than memcpy + for (unsigned i = rounds / 2; i-->0;) { + // Mix columns, then diagonals + for (unsigned j = 0; j < 4; ++j) QUARTERROUND(x[j], x[4 + j], x[8 + j], x[12 + j]); + for (unsigned j = 0; j < 4; ++j) QUARTERROUND(x[j], x[4 + (j+1)%4], x[8 + (j+2)%4], x[12 + (j+3)%4]); } - return c - begin; + uint32_t* buf = (uint32_t*)out; + for (unsigned i = 0; i < 16; ++i) buf[i] = x[i] + state[i]; + memset(x, 0, sizeof x); + ++*(uint64_t*)(state + 12); // Increment counter + return CHA_BLOCK_SIZE; } + +#undef QUARTERROUND +#undef QUARTERSTEP diff --git a/src/cha4block.h b/src/cha4block.h index a82d001..c7ff692 100644 --- a/src/cha4block.h +++ b/src/cha4block.h @@ -1,140 +1,73 @@ #if defined(__x86_64__) -#include // AVX2 -#include // SSSE3 +#include // SSE2 #elif defined(__aarch64__) #include "sse2neon.h" #endif +// clang-format off + #define VEC4_ROT(A, IMM) \ _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) /* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 & * 16) (better) */ #define VEC4_QUARTERROUND(A, B, C, D) \ - x_##A = _mm_add_epi32(x_##A, x_##B); \ - t_##A = _mm_xor_si128(x_##D, x_##A); \ - x_##D = _mm_shuffle_epi8(t_##A, rot16); \ - x_##C = _mm_add_epi32(x_##C, x_##D); \ - t_##C = _mm_xor_si128(x_##B, x_##C); \ - x_##B = VEC4_ROT(t_##C, 12); \ - x_##A = _mm_add_epi32(x_##A, x_##B); \ - t_##A = _mm_xor_si128(x_##D, x_##A); \ - x_##D = _mm_shuffle_epi8(t_##A, rot8); \ - x_##C = _mm_add_epi32(x_##C, x_##D); \ - t_##C = _mm_xor_si128(x_##B, x_##C); \ - x_##B = VEC4_ROT(t_##C, 7) + x[A] = _mm_add_epi32(x[A], x[B]); \ + x[D] = _mm_shuffle_epi8(_mm_xor_si128(x[D], x[A]), rot16); \ + x[C] = _mm_add_epi32(x[C], x[D]); \ + x[B] = VEC4_ROT(_mm_xor_si128(x[B], x[C]), 12); \ + x[A] = _mm_add_epi32(x[A], x[B]); \ + x[D] = _mm_shuffle_epi8(_mm_xor_si128(x[D], x[A]), rot8); \ + x[C] = _mm_add_epi32(x[C], x[D]); \ + x[B] = VEC4_ROT(_mm_xor_si128(x[B], x[C]), 7) #define ONEQUAD(A, B, C, D, CT) \ { \ /* Add original block */ \ - x_##A = _mm_add_epi32(x_##A, orig##A); \ - x_##B = _mm_add_epi32(x_##B, orig##B); \ - x_##C = _mm_add_epi32(x_##C, orig##C); \ - x_##D = _mm_add_epi32(x_##D, orig##D); \ + x[A] = _mm_add_epi32(x[A], orig[A]); \ + x[B] = _mm_add_epi32(x[B], orig[B]); \ + x[C] = _mm_add_epi32(x[C], orig[C]); \ + x[D] = _mm_add_epi32(x[D], orig[D]); \ /* Transpose */ \ - t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \ - t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \ - t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \ - t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \ - x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \ - x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \ - x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \ - x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \ + __m128i abl = _mm_unpacklo_epi32(x[A], x[B]); \ + __m128i cdl = _mm_unpacklo_epi32(x[C], x[D]); \ + __m128i abh = _mm_unpackhi_epi32(x[A], x[B]); \ + __m128i cdh = _mm_unpackhi_epi32(x[C], x[D]); \ + x[A] = _mm_unpacklo_epi64(abl, cdl); /* a0 b0 c0 d0 */ \ + x[B] = _mm_unpackhi_epi64(abl, cdl); /* a1 b1 c1 d1 */ \ + x[C] = _mm_unpacklo_epi64(abh, cdh); /* a2 b2 c2 d2 */ \ + x[D] = _mm_unpackhi_epi64(abh, cdh); /* a3 b3 c3 d3 */ \ \ - _mm_storeu_si128((__m128i*)(CT), x_##A); \ - _mm_storeu_si128((__m128i*)(CT + 64), x_##B); \ - _mm_storeu_si128((__m128i*)(CT + 128), x_##C); \ - _mm_storeu_si128((__m128i*)(CT + 192), x_##D); \ + _mm_storeu_si128((__m128i*)(CT), x[A]); \ + _mm_storeu_si128((__m128i*)(CT + 64), x[B]); \ + _mm_storeu_si128((__m128i*)(CT + 128), x[C]); \ + _mm_storeu_si128((__m128i*)(CT + 192), x[D]); \ } -static inline uint64_t _cha_4block(cha_ctx* ctx, uint8_t* begin, uint8_t* end) { - if (end - begin < 256) - return 0; - uint8_t* c = begin; - uint32_t* state = ctx->state; +#define COUNTER_INCREMENT(a, b, c, d) \ + { \ + __m128i addv = _mm_set_epi32(d, c, b, a); \ + orig[12] = _mm_add_epi32(orig[12], addv); \ + addv = _mm_srli_epi32(_mm_cmplt_epi32(orig[12], addv), 31); \ + orig[13] = _mm_add_epi32(orig[13], addv); \ + } +static inline uint64_t +_cha_4block(uint8_t* buf, size_t bufsize, uint32_t state[16], unsigned rounds) { /* constant for shuffling bytes (replacing multiple-of-8 rotates) */ const __m128i rot16 = _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); const __m128i rot8 = _mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); - - // Load state to vectors, duplicate four times - __m128i x_0 = _mm_set1_epi32(state[0]); - __m128i x_1 = _mm_set1_epi32(state[1]); - __m128i x_2 = _mm_set1_epi32(state[2]); - __m128i x_3 = _mm_set1_epi32(state[3]); - __m128i x_4 = _mm_set1_epi32(state[4]); - __m128i x_5 = _mm_set1_epi32(state[5]); - __m128i x_6 = _mm_set1_epi32(state[6]); - __m128i x_7 = _mm_set1_epi32(state[7]); - __m128i x_8 = _mm_set1_epi32(state[8]); - __m128i x_9 = _mm_set1_epi32(state[9]); - __m128i x_10 = _mm_set1_epi32(state[10]); - __m128i x_11 = _mm_set1_epi32(state[11]); - __m128i x_12; - __m128i x_13; - __m128i x_14 = _mm_set1_epi32(state[14]); - __m128i x_15 = _mm_set1_epi32(state[15]); - __m128i orig0 = x_0; - __m128i orig1 = x_1; - __m128i orig2 = x_2; - __m128i orig3 = x_3; - __m128i orig4 = x_4; - __m128i orig5 = x_5; - __m128i orig6 = x_6; - __m128i orig7 = x_7; - __m128i orig8 = x_8; - __m128i orig9 = x_9; - __m128i orig10 = x_10; - __m128i orig11 = x_11; - __m128i orig12 = {}; - __m128i orig13 = {}; - __m128i orig14 = x_14; - __m128i orig15 = x_15; - __m128i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12, - t_13, t_14, t_15; - - const __m128i addv12 = _mm_set_epi64x(1, 0); - const __m128i addv13 = _mm_set_epi64x(3, 2); - - while (end - c >= 256) { - x_0 = orig0; - x_1 = orig1; - x_2 = orig2; - x_3 = orig3; - x_4 = orig4; - x_5 = orig5; - x_6 = orig6; - x_7 = orig7; - x_8 = orig8; - x_9 = orig9; - x_10 = orig10; - x_11 = orig11; - x_14 = orig14; - x_15 = orig15; - - // Calculate counter + 0..3 for adjacent blocks (x12 low and x13 - // high of each) - uint32_t in12 = state[12]; - uint32_t in13 = state[13]; - uint64_t in1213 = ((uint64_t)in12) | (((uint64_t)in13) << 32); - __m128i t12, t13; - t12 = _mm_set1_epi64x(in1213); - t13 = _mm_set1_epi64x(in1213); - x_12 = _mm_add_epi64(addv12, t12); - x_13 = _mm_add_epi64(addv13, t13); - t12 = _mm_unpacklo_epi32(x_12, x_13); - t13 = _mm_unpackhi_epi32(x_12, x_13); - x_12 = _mm_unpacklo_epi32(t12, t13); - x_13 = _mm_unpackhi_epi32(t12, t13); - orig12 = x_12; - orig13 = x_13; - in1213 += 4; - state[12] = in1213 & 0xFFFFFFFF; - state[13] = (in1213 >> 32) & 0xFFFFFFFF; - - for (int i = 0; i < 10; ++i) { + // Load state to vectors, duplicate four times, only different counters + __m128i orig[16]; + for (unsigned i = 0; i < 16; ++i) orig[i] = _mm_set1_epi32(state[i]); + COUNTER_INCREMENT(0, 1, 2, 3); + const unsigned batches = bufsize / 256; + for (unsigned b = batches; b-->0;) { + __m128i x[16]; + for (unsigned i = 0; i < 16; ++i) x[i] = orig[i]; + for (unsigned r = rounds / 2; r-->0;) { // Mix columns VEC4_QUARTERROUND(0, 4, 8, 12); VEC4_QUARTERROUND(1, 5, 9, 13); @@ -146,18 +79,21 @@ static inline uint64_t _cha_4block(cha_ctx* ctx, uint8_t* begin, uint8_t* end) { VEC4_QUARTERROUND(2, 7, 8, 13); VEC4_QUARTERROUND(3, 4, 9, 14); } - - ONEQUAD(0, 1, 2, 3, c); - ONEQUAD(4, 5, 6, 7, c + 16); - ONEQUAD(8, 9, 10, 11, c + 32); - ONEQUAD(12, 13, 14, 15, c + 48); - - // *counter += 4; - c += 256; + // Add original block, unpack output + ONEQUAD(0, 1, 2, 3, buf); + ONEQUAD(4, 5, 6, 7, buf + 16); + ONEQUAD(8, 9, 10, 11, buf + 32); + ONEQUAD(12, 13, 14, 15, buf + 48); + COUNTER_INCREMENT(4, 4, 4, 4); + buf += 256; } - return c - begin; // Bytes written + // Update counter + state[12] = _mm_cvtsi128_si32(orig[12]); + state[13] = _mm_cvtsi128_si32(orig[13]); + return batches * 256; } +#undef COUNTER_INCREMENT #undef ONEQUAD #undef ONEQUAD_TRANSPOSE #undef VEC4_ROT diff --git a/src/cha8block.h b/src/cha8block.h index f493a5e..795fe0a 100644 --- a/src/cha8block.h +++ b/src/cha8block.h @@ -1,25 +1,11 @@ #include // AVX2 #include // SSSE3 +// clang-format off + #define VEC8_ROT(A, IMM) \ _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) -/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 & - * 16) (better) */ -#define VEC8_QUARTERROUND(A, B, C, D) \ - x[A] = _mm256_add_epi32(x[A], x[B]); \ - t[A] = _mm256_xor_si256(x[D], x[A]); \ - x[D] = _mm256_shuffle_epi8(t[A], rot16); \ - x[C] = _mm256_add_epi32(x[C], x[D]); \ - t[C] = _mm256_xor_si256(x[B], x[C]); \ - x[B] = VEC8_ROT(t[C], 12); \ - x[A] = _mm256_add_epi32(x[A], x[B]); \ - t[A] = _mm256_xor_si256(x[D], x[A]); \ - x[D] = _mm256_shuffle_epi8(t[A], rot8); \ - x[C] = _mm256_add_epi32(x[C], x[D]); \ - t[C] = _mm256_xor_si256(x[B], x[C]); \ - x[B] = VEC8_ROT(t[C], 7) - #define VEC8_LINE1(A, B, C, D) \ x[A] = _mm256_add_epi32(x[A], x[B]); \ x[D] = _mm256_shuffle_epi8(_mm256_xor_si256(x[D], x[A]), rot16) @@ -33,229 +19,111 @@ x[C] = _mm256_add_epi32(x[C], x[D]); \ x[B] = VEC8_ROT(_mm256_xor_si256(x[B], x[C]), 7) -#define VEC8_ROUND_SEQ( \ - A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ -) \ - VEC8_LINE1(A1, B1, C1, D1); \ - VEC8_LINE1(A2, B2, C2, D2); \ - VEC8_LINE1(A3, B3, C3, D3); \ - VEC8_LINE1(A4, B4, C4, D4); \ - VEC8_LINE2(A1, B1, C1, D1); \ - VEC8_LINE2(A2, B2, C2, D2); \ - VEC8_LINE2(A3, B3, C3, D3); \ - VEC8_LINE2(A4, B4, C4, D4); \ - VEC8_LINE3(A1, B1, C1, D1); \ - VEC8_LINE3(A2, B2, C2, D2); \ - VEC8_LINE3(A3, B3, C3, D3); \ - VEC8_LINE3(A4, B4, C4, D4); \ - VEC8_LINE4(A1, B1, C1, D1); \ - VEC8_LINE4(A2, B2, C2, D2); \ - VEC8_LINE4(A3, B3, C3, D3); \ - VEC8_LINE4(A4, B4, C4, D4) - -#define VEC8_ROUND_HALF( \ - A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ -) \ - VEC8_LINE1(A1, B1, C1, D1); \ - VEC8_LINE1(A2, B2, C2, D2); \ - VEC8_LINE2(A1, B1, C1, D1); \ - VEC8_LINE2(A2, B2, C2, D2); \ - VEC8_LINE3(A1, B1, C1, D1); \ - VEC8_LINE3(A2, B2, C2, D2); \ - VEC8_LINE4(A1, B1, C1, D1); \ - VEC8_LINE4(A2, B2, C2, D2); \ - VEC8_LINE1(A3, B3, C3, D3); \ - VEC8_LINE1(A4, B4, C4, D4); \ - VEC8_LINE2(A3, B3, C3, D3); \ - VEC8_LINE2(A4, B4, C4, D4); \ - VEC8_LINE3(A3, B3, C3, D3); \ - VEC8_LINE3(A4, B4, C4, D4); \ - VEC8_LINE4(A3, B3, C3, D3); \ - VEC8_LINE4(A4, B4, C4, D4) - -#define VEC8_ROUND_HALFANDHALF( \ - A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ -) \ - VEC8_LINE1(A1, B1, C1, D1); \ - VEC8_LINE1(A2, B2, C2, D2); \ - VEC8_LINE2(A1, B1, C1, D1); \ - VEC8_LINE2(A2, B2, C2, D2); \ - VEC8_LINE1(A3, B3, C3, D3); \ - VEC8_LINE1(A4, B4, C4, D4); \ - VEC8_LINE2(A3, B3, C3, D3); \ - VEC8_LINE2(A4, B4, C4, D4); \ - VEC8_LINE3(A1, B1, C1, D1); \ - VEC8_LINE3(A2, B2, C2, D2); \ - VEC8_LINE4(A1, B1, C1, D1); \ - VEC8_LINE4(A2, B2, C2, D2); \ - VEC8_LINE3(A3, B3, C3, D3); \ - VEC8_LINE3(A4, B4, C4, D4); \ - VEC8_LINE4(A3, B3, C3, D3); \ - VEC8_LINE4(A4, B4, C4, D4) - #define VEC8_ROUND( \ A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ ) \ - VEC8_ROUND_SEQ( \ - A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ - ) + VEC8_LINE1(A1, B1, C1, D1); \ + VEC8_LINE1(A2, B2, C2, D2); \ + VEC8_LINE1(A3, B3, C3, D3); \ + VEC8_LINE1(A4, B4, C4, D4); \ + VEC8_LINE2(A1, B1, C1, D1); \ + VEC8_LINE2(A2, B2, C2, D2); \ + VEC8_LINE2(A3, B3, C3, D3); \ + VEC8_LINE2(A4, B4, C4, D4); \ + VEC8_LINE3(A1, B1, C1, D1); \ + VEC8_LINE3(A2, B2, C2, D2); \ + VEC8_LINE3(A3, B3, C3, D3); \ + VEC8_LINE3(A4, B4, C4, D4); \ + VEC8_LINE4(A1, B1, C1, D1); \ + VEC8_LINE4(A2, B2, C2, D2); \ + VEC8_LINE4(A3, B3, C3, D3); \ + VEC8_LINE4(A4, B4, C4, D4) -#define ONEQUAD_TRANSPOSE(A, B, C, D) \ +#define TRANSPOSE(A, B, C, D) \ { \ - __m128i t0, t1, t2, t3; \ - x[A] = _mm256_add_epi32(x[A], orig[A]); \ - x[B] = _mm256_add_epi32(x[B], orig[B]); \ - x[C] = _mm256_add_epi32(x[C], orig[C]); \ - x[D] = _mm256_add_epi32(x[D], orig[D]); \ - t[A] = _mm256_unpacklo_epi32(x[A], x[B]); \ - t[B] = _mm256_unpacklo_epi32(x[C], x[D]); \ - t[C] = _mm256_unpackhi_epi32(x[A], x[B]); \ - t[D] = _mm256_unpackhi_epi32(x[C], x[D]); \ - x[A] = _mm256_unpacklo_epi64(t[A], t[B]); \ - x[B] = _mm256_unpackhi_epi64(t[A], t[B]); \ - x[C] = _mm256_unpacklo_epi64(t[C], t[D]); \ - x[D] = _mm256_unpackhi_epi64(t[C], t[D]); \ - _mm_storeu_si128( \ - (__m128i*)(c + 0), _mm256_extracti128_si256(x[A], 0) \ - ); \ - _mm_storeu_si128( \ - (__m128i*)(c + 64), _mm256_extracti128_si256(x[B], 0) \ - ); \ - _mm_storeu_si128( \ - (__m128i*)(c + 128), _mm256_extracti128_si256(x[C], 0) \ - ); \ - _mm_storeu_si128( \ - (__m128i*)(c + 192), _mm256_extracti128_si256(x[D], 0) \ - ); \ - _mm_storeu_si128( \ - (__m128i*)(c + 256), _mm256_extracti128_si256(x[A], 1) \ - ); \ - _mm_storeu_si128( \ - (__m128i*)(c + 320), _mm256_extracti128_si256(x[B], 1) \ - ); \ - _mm_storeu_si128( \ - (__m128i*)(c + 384), _mm256_extracti128_si256(x[C], 1) \ - ); \ - _mm_storeu_si128( \ - (__m128i*)(c + 448), _mm256_extracti128_si256(x[D], 1) \ - ); \ - } - -#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) - -#define ONEQUAD_UNPCK(A, B, C, D) \ - { \ - x[A] = _mm256_add_epi32(x[A], orig[A]); \ - x[B] = _mm256_add_epi32(x[B], orig[B]); \ - x[C] = _mm256_add_epi32(x[C], orig[C]); \ - x[D] = _mm256_add_epi32(x[D], orig[D]); \ - t[A] = _mm256_unpacklo_epi32(x[A], x[B]); \ - t[B] = _mm256_unpacklo_epi32(x[C], x[D]); \ - t[C] = _mm256_unpackhi_epi32(x[A], x[B]); \ - t[D] = _mm256_unpackhi_epi32(x[C], x[D]); \ - x[A] = _mm256_unpacklo_epi64(t[A], t[B]); \ - x[B] = _mm256_unpackhi_epi64(t[A], t[B]); \ - x[C] = _mm256_unpacklo_epi64(t[C], t[D]); \ - x[D] = _mm256_unpackhi_epi64(t[C], t[D]); \ + const __m256i t0 = _mm256_unpacklo_epi32(x[A], x[B]), \ + t1 = _mm256_unpacklo_epi32(x[C], x[D]), \ + t2 = _mm256_unpackhi_epi32(x[A], x[B]), \ + t3 = _mm256_unpackhi_epi32(x[C], x[D]); \ + x[A] = _mm256_unpacklo_epi64(t0, t1); \ + x[B] = _mm256_unpackhi_epi64(t0, t1); \ + x[C] = _mm256_unpacklo_epi64(t2, t3); \ + x[D] = _mm256_unpackhi_epi64(t2, t3); \ } #define ONEOCTO(A, B, C, D, A2, B2, C2, D2, c) \ { \ - ONEQUAD_UNPCK(A, B, C, D); \ - ONEQUAD_UNPCK(A2, B2, C2, D2); \ - t[A] = _mm256_permute2x128_si256(x[A], x[A2], 0x20); \ - t[A2] = _mm256_permute2x128_si256(x[A], x[A2], 0x31); \ - t[B] = _mm256_permute2x128_si256(x[B], x[B2], 0x20); \ - t[B2] = _mm256_permute2x128_si256(x[B], x[B2], 0x31); \ - t[C] = _mm256_permute2x128_si256(x[C], x[C2], 0x20); \ - t[C2] = _mm256_permute2x128_si256(x[C], x[C2], 0x31); \ - t[D] = _mm256_permute2x128_si256(x[D], x[D2], 0x20); \ - t[D2] = _mm256_permute2x128_si256(x[D], x[D2], 0x31); \ - _mm256_storeu_si256((__m256i*)(c), t[A]); \ - _mm256_storeu_si256((__m256i*)(c + 64), t[B]); \ - _mm256_storeu_si256((__m256i*)(c + 128), t[C]); \ - _mm256_storeu_si256((__m256i*)(c + 192), t[D]); \ - _mm256_storeu_si256((__m256i*)(c + 256), t[A2]); \ - _mm256_storeu_si256((__m256i*)(c + 320), t[B2]); \ - _mm256_storeu_si256((__m256i*)(c + 384), t[C2]); \ - _mm256_storeu_si256((__m256i*)(c + 448), t[D2]); \ + TRANSPOSE(A, B, C, D); \ + TRANSPOSE(A2, B2, C2, D2); \ + _mm256_storeu_si256((__m256i*)(c), _mm256_permute2x128_si256(x[A], x[A2], 0x20)); \ + _mm256_storeu_si256((__m256i*)(c + 64), _mm256_permute2x128_si256(x[B], x[B2], 0x20)); \ + _mm256_storeu_si256((__m256i*)(c + 128), _mm256_permute2x128_si256(x[C], x[C2], 0x20)); \ + _mm256_storeu_si256((__m256i*)(c + 192), _mm256_permute2x128_si256(x[D], x[D2], 0x20)); \ + _mm256_storeu_si256((__m256i*)(c + 256), _mm256_permute2x128_si256(x[A], x[A2], 0x31)); \ + _mm256_storeu_si256((__m256i*)(c + 320), _mm256_permute2x128_si256(x[B], x[B2], 0x31)); \ + _mm256_storeu_si256((__m256i*)(c + 384), _mm256_permute2x128_si256(x[C], x[C2], 0x31)); \ + _mm256_storeu_si256((__m256i*)(c + 448), _mm256_permute2x128_si256(x[D], x[D2], 0x31)); \ } -static inline uint64_t _cha_8block(cha_ctx* ctx, uint8_t* begin, uint8_t* end) { - if (end - begin < 512) - return 0; +#define COUNTER_INCREMENT(addv) \ + { \ + orig[12] = _mm256_add_epi32(orig[12], addv); \ + orig[13] = _mm256_add_epi32(orig[13], _mm256_srli_epi32(_mm256_cmpgt_epi32(addv, orig[12]), 31)); \ + } - uint8_t* c = begin; - uint32_t* state = ctx->state; - uint64_t* counter = (uint64_t*)(state + 12); +static inline uint64_t +_cha_8block(uint8_t* buf, size_t bufsize, uint32_t state[16], unsigned rounds) { + unsigned batches = bufsize / 512; /* constant for shuffling bytes (replacing multiple-of-8 rotates) */ - __m256i rot16 = _mm256_set_epi8( - 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, 13, 12, 15, 14, 9, - 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 + const __m256i rot16 = _mm256_set_epi8( + 13, 12, 15, 14, + 9, 8, 11, 10, + 5, 4, 7, 6, + 1, 0, 3, 2, + 13, 12, 15, 14, + 9, 8, 11, 10, + 5, 4, 7, 6, + 1, 0, 3, 2 ); - __m256i rot8 = _mm256_set_epi8( - 14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3, 14, 13, 12, 15, 10, - 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3 + const __m256i rot8 = _mm256_set_epi8( + 14, 13, 12, 15, + 10, 9, 8, 11, + 6, 5, 4, 7, + 2, 1, 0, 3, + 14, 13, 12, 15, + 10, 9, 8, 11, + 6, 5, 4, 7, + 2, 1, 0, 3 ); - - /* the naive way seems as fast (if not a bit faster) than the vector way */ - __m256i x[16], orig[16], t[16]; + __m256i orig[16]; for (int i = 0; i < 16; ++i) orig[i] = _mm256_set1_epi32(state[i]); + COUNTER_INCREMENT(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0)); - const __m256i addv12 = _mm256_set_epi64x(3, 2, 1, 0); - const __m256i addv13 = _mm256_set_epi64x(7, 6, 5, 4); - - while (end - c >= 512) { - for (int i = 0; i < 16; ++i) - if (i != 12 && i != 13) - x[i] = orig[i]; - - // Calculate the eight parallel counters on x_12 and x_13 - t[13] = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(*counter)); - t[12] = _mm256_add_epi64(addv12, t[13]); - t[13] = _mm256_add_epi64(addv13, t[13]); - x[12] = _mm256_unpacklo_epi32(t[12], t[13]); - x[13] = _mm256_unpackhi_epi32(t[12], t[13]); - t[12] = _mm256_unpacklo_epi32(x[12], x[13]); - t[13] = _mm256_unpackhi_epi32(x[12], x[13]); - - /* required because unpack* are intra-lane */ - const __m256i permute = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0); - x[12] = _mm256_permutevar8x32_epi32(t[12], permute); - x[13] = _mm256_permutevar8x32_epi32(t[13], permute); - - orig[12] = x[12]; - orig[13] = x[13]; - - for (int i = 0; i < 10; ++i) { + for (unsigned b = batches; b-->0;) { + __m256i x[16]; + for (int i = 0; i < 16; ++i) x[i] = orig[i]; + for (unsigned r = rounds / 2; r-->0;) { VEC8_ROUND(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); VEC8_ROUND(0, 5, 10, 15, 1, 6, 11, 12, 2, 7, 8, 13, 3, 4, 9, 14); } - - ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7, c); - ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15, c + 32); - - *counter += 8; - c += 512; + for (unsigned i = 0; i < 16; ++i) x[i] = _mm256_add_epi32(x[i], orig[i]); + ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7, buf); + ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15, buf + 32); + COUNTER_INCREMENT(_mm256_set1_epi32(8)); + buf += 512; } - return c - begin; + state[12] = _mm256_extract_epi32(orig[12], 0); + state[13] = _mm256_extract_epi32(orig[13], 0); + return batches * 512; } -#undef ONEQUAD -#undef ONEQUAD_TRANSPOSE -#undef ONEQUAD_UNPCK +#undef COUNTER_INCREMENT #undef ONEOCTO +#undef TRANSPOSE #undef VEC8_ROT -#undef VEC8_QUARTERROUND -#undef VEC8_QUARTERROUND_NAIVE -#undef VEC8_QUARTERROUND_SHUFFLE -#undef VEC8_QUARTERROUND_SHUFFLE2 #undef VEC8_LINE1 #undef VEC8_LINE2 #undef VEC8_LINE3 #undef VEC8_LINE4 #undef VEC8_ROUND -#undef VEC8_ROUND_SEQ -#undef VEC8_ROUND_HALF -#undef VEC8_ROUND_HALFANDHALF diff --git a/src/chacha20.h b/src/chacha20.h index 2f84dd0..3a6af50 100644 --- a/src/chacha20.h +++ b/src/chacha20.h @@ -2,12 +2,8 @@ #include #define CHA_BLOCK_SIZE 64 - -typedef struct cha_ctx { - uint32_t state[16]; - uint8_t unconsumed[CHA_BLOCK_SIZE]; - uint8_t uncount; -} cha_ctx; +#define BATCH_BLOCKS 8 +#define BATCH_SIZE (BATCH_BLOCKS * CHA_BLOCK_SIZE) #if defined(__x86_64__) #ifdef __GNUC__ @@ -16,6 +12,7 @@ typedef struct cha_ctx { #pragma GCC target("avx2") #endif #include "cha8block.h" +#else #endif #include "cha1block.h" @@ -31,12 +28,21 @@ typedef struct cha_ctx { #include #include +typedef struct cha_ctx { + uint32_t state[16]; + uint8_t unconsumed[BATCH_SIZE]; + uint32_t offset, end; + unsigned rounds; +} cha_ctx; + /// @brief Initialize cha_ctx /// @param ctx holds ChaCha20 state /// @param key 32 byte key -/// @param iv 16 bytes, where normally initial 4-8 bytes are zeroes and the rest -/// nonce -void cha_init(cha_ctx* ctx, const uint8_t* key, const uint8_t* iv) { +/// @param iv 16 bytes, usually the first 4-8 bytes are zeroes, the rest nonce +/// @param rounds ChaCha iteration count: 8=fast, 12=balanced, 20=secure +void cha_init( + cha_ctx* ctx, const uint8_t* key, const uint8_t* iv, unsigned rounds +) { ctx->state[0] = 0x61707865; ctx->state[1] = 0x3320646e; ctx->state[2] = 0x79622d32; @@ -44,61 +50,85 @@ void cha_init(cha_ctx* ctx, const uint8_t* key, const uint8_t* iv) { memcpy(ctx->state + 4, key, 32); memcpy(ctx->state + 12, iv, 16); memset(ctx->unconsumed, 0, sizeof ctx->unconsumed); - ctx->uncount = 0; + ctx->offset = ctx->end = 0; + ctx->rounds = rounds; } /// Dispose of sensitive data within the context -void cha_wipe(cha_ctx* ctx); - void cha_wipe(cha_ctx* ctx) { memset(ctx, 0, sizeof(cha_ctx)); } +/// @brief Advance or rewind the stream to any arbitrary location +/// @param ctx ChaCha context +/// @param offset Offset in blocks of 64 bytes (counter change) +void cha_seek_blocks(cha_ctx* ctx, int64_t offset) { + *(uint64_t*)(ctx->state + 12) += offset; + ctx->offset = ctx->end = 0; +} + +uint64_t cha_generate_batch( + uint8_t* out, size_t outsize, uint32_t* state, unsigned rounds +) { +#if defined(__x86_64__) + if (__builtin_cpu_supports("ssse3")) { + if (__builtin_cpu_supports("avx2")) { + return _cha_8block(out, outsize, state, rounds); + } + return _cha_4block(out, outsize, state, rounds); + } +#elif defined(__aarch64__) + return _cha_4block(out, outsize, state); +#endif + unsigned count = 0; + unsigned n = _cha_block(out, state, rounds); + count += n; + out += n; + return count; +} + /// @brief Incremental generation, keeps state between calls -/// @param ctx ChaCha20 context +/// @param ctx ChaCha context /// @param out output buffer /// @param outlen output buffer length void cha_update(cha_ctx* ctx, uint8_t* out, uint64_t outlen) { // The included header will mess with these variables - uint8_t* c = out; uint8_t* end = out + outlen; - if (ctx->uncount) { + if (ctx->offset) { + // Need to generate stored buffer? + if (ctx->end == 0) + ctx->end = cha_generate_batch( + ctx->unconsumed, BATCH_SIZE, ctx->state, ctx->rounds + ); // Deliver stored bytes first - uint64_t N = ctx->uncount >= outlen ? outlen : ctx->uncount; - memcpy(c, ctx->unconsumed, N); - ctx->uncount -= N; - c += N; - if (ctx->uncount) { - memmove(ctx->unconsumed, ctx->unconsumed + N, ctx->uncount); - } - if (c == out + outlen) + uint64_t N = ctx->end - ctx->offset; + if (N > outlen) + N = outlen; + memcpy(out, ctx->unconsumed + ctx->offset, N); + ctx->offset += N; + out += N; + if (out == end) return; } -#if defined(__x86_64__) - // TODO: Handle resume if we are not at block boundary - if (__builtin_cpu_supports("ssse3")) { - if (__builtin_cpu_supports("avx2")) { - c += _cha_8block(ctx, c, end); - assert(end - c < 512); - } - c += _cha_4block(ctx, c, end); - assert(end - c < 256); + out += cha_generate_batch(out, end - out, ctx->state, ctx->rounds); + const uint32_t N = end - out; + if (N) { + ctx->end = cha_generate_batch( + ctx->unconsumed, BATCH_SIZE, ctx->state, ctx->rounds + ); + memcpy(out, ctx->unconsumed, N); + ctx->offset = N; } -#elif defined(__aarch64__) - c += _cha_4block(ctx, c, end); -#endif - c += _cha_block(ctx, c, end); - assert(c == end); } -/// @brief Produce a requested number of random bytes of the stream, one shot. +/// @brief Produce a requested number of random bytes, single shot. /// @param out output buffer /// @param outlen output buffer length /// @param key 32 byte key -/// @param iv 16 bytes, where normally initial 4-8 bytes are zeroes (counter) +/// @param iv 16 bytes, where normally initial 4-8 bytes are 0 (counter) void cha_generate( uint8_t* out, uint64_t outlen, const uint8_t key[32], const uint8_t iv[16] ) { cha_ctx ctx; - cha_init(&ctx, key, iv); + cha_init(&ctx, key, iv, 20); cha_update(&ctx, out, outlen); cha_wipe(&ctx); } diff --git a/src/cli.c b/src/cli.c index afbc0ac..758eddc 100644 --- a/src/cli.c +++ b/src/cli.c @@ -28,6 +28,7 @@ typedef struct thread_args { unsigned char* buf; unsigned char key[32]; unsigned workers; + unsigned rounds; pthread_mutex_t lock; pthread_cond_t cond; pthread_t thread; @@ -35,20 +36,22 @@ typedef struct thread_args { void* producer_thread(void* a) { thread_args* args = (thread_args*)a; - const uint64_t ivstep = args->workers * BLOCK_SIZE / 64; + const uint64_t ivstep = args->workers * BATCH_BLOCKS; + cha_ctx ctx; + cha_init(&ctx, args->key, default_iv, args->rounds); + cha_seek_blocks(&ctx, args->index * BLOCK_SIZE / 64); while (!quit) { pthread_mutex_lock(&args->lock); while (args->done) { pthread_cond_wait(&args->cond, &args->lock); } - unsigned char iv[16]; - memcpy(iv, default_iv, 16); - *(uint64_t*)iv += args->index * ivstep; // Counter increment - cha_generate(args->buf, BLOCK_SIZE, args->key, default_iv); + cha_update(&ctx, args->buf, BLOCK_SIZE); + cha_seek_blocks(&ctx, ivstep); args->done = 1; pthread_cond_signal(&args->cond); pthread_mutex_unlock(&args->lock); } + cha_wipe(&ctx); return NULL; } @@ -79,7 +82,7 @@ void print_status( int fast( FILE* f, unsigned workers, uint64_t max_bytes, unsigned char const key[32], - unsigned char const iv[16] + unsigned char const iv[16], unsigned rounds ) { thread_args args[workers]; memset(args, 0, sizeof args); @@ -87,6 +90,7 @@ int fast( args[i].index = i; args[i].buf = malloc(BLOCK_SIZE); args[i].workers = workers; + args[i].rounds = rounds; memcpy(args[i].key, key, 32); pthread_mutex_init(&args[i].lock, NULL); pthread_cond_init(&args[i].cond, NULL); @@ -160,7 +164,8 @@ void print_hex(unsigned char* buf, size_t len) { void help(char** argv) { fprintf( stderr, - "Usage: %s [-t #threads] [-s hexseed] [-b #bytes] [-o outputfile]\n\n", + "Usage: %s [-t #threads] [-s hexseed] [-b #bytes] [-c #rounds] [-o " + "outputfile]\n\n", argv[0] ); } @@ -169,10 +174,11 @@ int main(int argc, char** argv) { unsigned char key[32] = {}; unsigned char iv[16] = {}; unsigned int workers = 8; + unsigned int rounds = 20; char* output = NULL; uint64_t max_bytes = 0; bool seeded = false; - for (char opt; (opt = getopt(argc, argv, "bost")) != -1;) { + for (char opt; (opt = getopt(argc, argv, "bostc")) != -1;) { if (opt == 't') { if (optind >= argc || sscanf(argv[optind++], "%u", &workers) != 1) { fprintf( @@ -182,6 +188,16 @@ int main(int argc, char** argv) { } continue; } + if (opt == 'c') { + if (optind >= argc || sscanf(argv[optind++], "%u", &rounds) != 1) { + fprintf( + stderr, + "Expected the number ChaCha rounds (8, 12 or 20) after -c\n" + ); + return 1; + } + continue; + } if (opt == 's') { if (optind >= argc || !parse_hex(argv[optind++], key, 32)) { fprintf(stderr, "Expected a hex seed string after -s\n"); @@ -250,7 +266,7 @@ int main(int argc, char** argv) { } signal(SIGINT, signal_handler); signal(SIGTERM, signal_handler); - int ret = fast(f, workers, max_bytes, key, iv); + int ret = fast(f, workers, max_bytes, key, iv, rounds); fclose(f); return ret; }