LeoVasanko/RandQuik
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rewrite, cleanup, configurable ChaCha rounds.

Browse Source
This commit is contained in:
LeoVasanko
2023-11-28 23:39:38 +00:00
parent 998560f5a0
commit 236f7eedf6
7 changed files with 260 additions and 438 deletions
Download Patch File Download Diff File Expand all files Collapse all files
benchmarks/charand.c
+9 -5
View File
@@ -1,16 +1,20 @@
#include "chacha20.c" #include "chacha20.h"
#include <stdio.h> #include <stdio.h>
#include <stdlib.h> #include <stdlib.h>
int main(void) int main(void) {
{
uint64_t N = 1000000; uint64_t N = 1000000;
uint8_t* buf = malloc(N); uint8_t* buf = malloc(N);
const uint8_t key[32] = {0}; const uint8_t key[32] = {0};
const uint8_t nonce[16] = {0}; const uint8_t nonce[16] = {0};
for (uint64_t i = 0; i < 1000; ++i) for (uint64_t i = 0; i < 1000; ++i) {
{
cha_generate(buf, N, key, nonce); cha_generate(buf, N, key, nonce);
} }
for (unsigned i = 0; i < 16; ++i)
printf("%02X ", buf[i]);
for (unsigned i = 0; i < 16; ++i)
printf(" %02X", buf[1024 + i]);
puts("");
free(buf); free(buf);
} }
meson.build
-8
View File
@@ -2,15 +2,7 @@ project('randquik', 'c')
executable( executable(
'randquik', 'randquik',
'src/cli.c', 'src/cli.c',
'src/chacha20.c',
c_args: ['-Wall', '-O3', '-march=native'], c_args: ['-Wall', '-O3', '-march=native'],
install: true, install: true,
) )
dependency('threads') dependency('threads')
library(
'randquik-chacha20',
'src/chacha20.c',
build_by_default: true,
c_args: ['-Wall', '-O3', '-march=native'],
)
src/cha1block.h
+20 -44
View File
@@ -1,50 +1,26 @@
#include <stdint.h> #include <stdint.h>
#include <stdlib.h>
#include <string.h> #include <string.h>
#define QUARTERSTEP(a, b, c, n) \ // clang-format off
a += b; \ #define QUARTERSTEP(a, b, c, n) a += b; c ^= a; c = (c << n) | (c >> (32 - n))
c ^= a; \ #define QUARTERROUND(a, b, c, d) {\
c = (c << n) | (c >> (32 - n)) QUARTERSTEP(a, b, d, 16); QUARTERSTEP(c, d, b, 12); \
QUARTERSTEP(a, b, d, 8); QUARTERSTEP(c, d, b, 7); }
#define QUARTERROUND(a, b, c, d) \ static inline uint64_t _cha_block(uint8_t out[CHA_BLOCK_SIZE], uint32_t state[16], unsigned rounds) {
QUARTERSTEP(a, b, d, 16); \ uint32_t x[16];
QUARTERSTEP(c, d, b, 12); \ for (unsigned i = 0; i < 16; ++i) x[i] = state[i]; // Faster than memcpy
QUARTERSTEP(a, b, d, 8); \ for (unsigned i = rounds / 2; i-->0;) {
QUARTERSTEP(c, d, b, 7); // Mix columns, then diagonals
for (unsigned j = 0; j < 4; ++j) QUARTERROUND(x[j], x[4 + j], x[8 + j], x[12 + j]);
static inline uint64_t _cha_block(cha_ctx* ctx, uint8_t* begin, uint8_t* end) { for (unsigned j = 0; j < 4; ++j) QUARTERROUND(x[j], x[4 + (j+1)%4], x[8 + (j+2)%4], x[12 + (j+3)%4]);
uint64_t* counter = (uint64_t*)&ctx->state[12];
uint8_t* c = begin;
while (c < end) {
uint32_t x[16];
memcpy(x, ctx->state, sizeof x);
for (int i = 20; i > 0; i -= 2) {
QUARTERROUND(x[0], x[4], x[8], x[12])
QUARTERROUND(x[1], x[5], x[9], x[13])
QUARTERROUND(x[2], x[6], x[10], x[14])
QUARTERROUND(x[3], x[7], x[11], x[15])
QUARTERROUND(x[0], x[5], x[10], x[15])
QUARTERROUND(x[1], x[6], x[11], x[12])
QUARTERROUND(x[2], x[7], x[8], x[13])
QUARTERROUND(x[3], x[4], x[9], x[14])
}
for (int i = 0; i < 16; i++)
x[i] += ctx->state[i];
++*counter;
uint64_t bytes = end - c;
if (bytes < 64) {
memcpy(c, x, bytes);
c = end;
// Leftover bytes are stored in ctx for next call
ctx->uncount = 64 - bytes;
memcpy(ctx->unconsumed, (uint8_t*)x + bytes, ctx->uncount);
break;
}
memcpy(c, x, 64);
c += 64;
} }
return c - begin; uint32_t* buf = (uint32_t*)out;
for (unsigned i = 0; i < 16; ++i) buf[i] = x[i] + state[i];
memset(x, 0, sizeof x);
++*(uint64_t*)(state + 12); // Increment counter
return CHA_BLOCK_SIZE;
} }
#undef QUARTERROUND
#undef QUARTERSTEP
src/cha4block.h
+57 -121
View File
@@ -1,140 +1,73 @@
#if defined(__x86_64__) #if defined(__x86_64__)
#include <immintrin.h> // AVX2 #include <emmintrin.h> // SSE2
#include <tmmintrin.h> // SSSE3
#elif defined(__aarch64__) #elif defined(__aarch64__)
#include "sse2neon.h" #include "sse2neon.h"
#endif #endif
// clang-format off
#define VEC4_ROT(A, IMM) \ #define VEC4_ROT(A, IMM) \
_mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM)))
/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 & /* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 &
* 16) (better) */ * 16) (better) */
#define VEC4_QUARTERROUND(A, B, C, D) \ #define VEC4_QUARTERROUND(A, B, C, D) \
x_##A = _mm_add_epi32(x_##A, x_##B); \ x[A] = _mm_add_epi32(x[A], x[B]); \
t_##A = _mm_xor_si128(x_##D, x_##A); \ x[D] = _mm_shuffle_epi8(_mm_xor_si128(x[D], x[A]), rot16); \
x_##D = _mm_shuffle_epi8(t_##A, rot16); \ x[C] = _mm_add_epi32(x[C], x[D]); \
x_##C = _mm_add_epi32(x_##C, x_##D); \ x[B] = VEC4_ROT(_mm_xor_si128(x[B], x[C]), 12); \
t_##C = _mm_xor_si128(x_##B, x_##C); \ x[A] = _mm_add_epi32(x[A], x[B]); \
x_##B = VEC4_ROT(t_##C, 12); \ x[D] = _mm_shuffle_epi8(_mm_xor_si128(x[D], x[A]), rot8); \
x_##A = _mm_add_epi32(x_##A, x_##B); \ x[C] = _mm_add_epi32(x[C], x[D]); \
t_##A = _mm_xor_si128(x_##D, x_##A); \ x[B] = VEC4_ROT(_mm_xor_si128(x[B], x[C]), 7)
x_##D = _mm_shuffle_epi8(t_##A, rot8); \
x_##C = _mm_add_epi32(x_##C, x_##D); \
t_##C = _mm_xor_si128(x_##B, x_##C); \
x_##B = VEC4_ROT(t_##C, 7)
#define ONEQUAD(A, B, C, D, CT) \ #define ONEQUAD(A, B, C, D, CT) \
{ \ { \
/* Add original block */ \ /* Add original block */ \
x_##A = _mm_add_epi32(x_##A, orig##A); \ x[A] = _mm_add_epi32(x[A], orig[A]); \
x_##B = _mm_add_epi32(x_##B, orig##B); \ x[B] = _mm_add_epi32(x[B], orig[B]); \
x_##C = _mm_add_epi32(x_##C, orig##C); \ x[C] = _mm_add_epi32(x[C], orig[C]); \
x_##D = _mm_add_epi32(x_##D, orig##D); \ x[D] = _mm_add_epi32(x[D], orig[D]); \
/* Transpose */ \ /* Transpose */ \
t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \ __m128i abl = _mm_unpacklo_epi32(x[A], x[B]); \
t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \ __m128i cdl = _mm_unpacklo_epi32(x[C], x[D]); \
t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \ __m128i abh = _mm_unpackhi_epi32(x[A], x[B]); \
t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \ __m128i cdh = _mm_unpackhi_epi32(x[C], x[D]); \
x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \ x[A] = _mm_unpacklo_epi64(abl, cdl); /* a0 b0 c0 d0 */ \
x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \ x[B] = _mm_unpackhi_epi64(abl, cdl); /* a1 b1 c1 d1 */ \
x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \ x[C] = _mm_unpacklo_epi64(abh, cdh); /* a2 b2 c2 d2 */ \
x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \ x[D] = _mm_unpackhi_epi64(abh, cdh); /* a3 b3 c3 d3 */ \
\ \
_mm_storeu_si128((__m128i*)(CT), x_##A); \ _mm_storeu_si128((__m128i*)(CT), x[A]); \
_mm_storeu_si128((__m128i*)(CT + 64), x_##B); \ _mm_storeu_si128((__m128i*)(CT + 64), x[B]); \
_mm_storeu_si128((__m128i*)(CT + 128), x_##C); \ _mm_storeu_si128((__m128i*)(CT + 128), x[C]); \
_mm_storeu_si128((__m128i*)(CT + 192), x_##D); \ _mm_storeu_si128((__m128i*)(CT + 192), x[D]); \
} }
static inline uint64_t _cha_4block(cha_ctx* ctx, uint8_t* begin, uint8_t* end) { #define COUNTER_INCREMENT(a, b, c, d) \
if (end - begin < 256) { \
return 0; __m128i addv = _mm_set_epi32(d, c, b, a); \
uint8_t* c = begin; orig[12] = _mm_add_epi32(orig[12], addv); \
uint32_t* state = ctx->state; addv = _mm_srli_epi32(_mm_cmplt_epi32(orig[12], addv), 31); \
orig[13] = _mm_add_epi32(orig[13], addv); \
}
static inline uint64_t
_cha_4block(uint8_t* buf, size_t bufsize, uint32_t state[16], unsigned rounds) {
/* constant for shuffling bytes (replacing multiple-of-8 rotates) */ /* constant for shuffling bytes (replacing multiple-of-8 rotates) */
const __m128i rot16 = const __m128i rot16 =
_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2);
const __m128i rot8 = const __m128i rot8 =
_mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); _mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3);
// Load state to vectors, duplicate four times, only different counters
// Load state to vectors, duplicate four times __m128i orig[16];
__m128i x_0 = _mm_set1_epi32(state[0]); for (unsigned i = 0; i < 16; ++i) orig[i] = _mm_set1_epi32(state[i]);
__m128i x_1 = _mm_set1_epi32(state[1]); COUNTER_INCREMENT(0, 1, 2, 3);
__m128i x_2 = _mm_set1_epi32(state[2]); const unsigned batches = bufsize / 256;
__m128i x_3 = _mm_set1_epi32(state[3]); for (unsigned b = batches; b-->0;) {
__m128i x_4 = _mm_set1_epi32(state[4]); __m128i x[16];
__m128i x_5 = _mm_set1_epi32(state[5]); for (unsigned i = 0; i < 16; ++i) x[i] = orig[i];
__m128i x_6 = _mm_set1_epi32(state[6]); for (unsigned r = rounds / 2; r-->0;) {
__m128i x_7 = _mm_set1_epi32(state[7]);
__m128i x_8 = _mm_set1_epi32(state[8]);
__m128i x_9 = _mm_set1_epi32(state[9]);
__m128i x_10 = _mm_set1_epi32(state[10]);
__m128i x_11 = _mm_set1_epi32(state[11]);
__m128i x_12;
__m128i x_13;
__m128i x_14 = _mm_set1_epi32(state[14]);
__m128i x_15 = _mm_set1_epi32(state[15]);
__m128i orig0 = x_0;
__m128i orig1 = x_1;
__m128i orig2 = x_2;
__m128i orig3 = x_3;
__m128i orig4 = x_4;
__m128i orig5 = x_5;
__m128i orig6 = x_6;
__m128i orig7 = x_7;
__m128i orig8 = x_8;
__m128i orig9 = x_9;
__m128i orig10 = x_10;
__m128i orig11 = x_11;
__m128i orig12 = {};
__m128i orig13 = {};
__m128i orig14 = x_14;
__m128i orig15 = x_15;
__m128i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12,
t_13, t_14, t_15;
const __m128i addv12 = _mm_set_epi64x(1, 0);
const __m128i addv13 = _mm_set_epi64x(3, 2);
while (end - c >= 256) {
x_0 = orig0;
x_1 = orig1;
x_2 = orig2;
x_3 = orig3;
x_4 = orig4;
x_5 = orig5;
x_6 = orig6;
x_7 = orig7;
x_8 = orig8;
x_9 = orig9;
x_10 = orig10;
x_11 = orig11;
x_14 = orig14;
x_15 = orig15;
// Calculate counter + 0..3 for adjacent blocks (x12 low and x13
// high of each)
uint32_t in12 = state[12];
uint32_t in13 = state[13];
uint64_t in1213 = ((uint64_t)in12) | (((uint64_t)in13) << 32);
__m128i t12, t13;
t12 = _mm_set1_epi64x(in1213);
t13 = _mm_set1_epi64x(in1213);
x_12 = _mm_add_epi64(addv12, t12);
x_13 = _mm_add_epi64(addv13, t13);
t12 = _mm_unpacklo_epi32(x_12, x_13);
t13 = _mm_unpackhi_epi32(x_12, x_13);
x_12 = _mm_unpacklo_epi32(t12, t13);
x_13 = _mm_unpackhi_epi32(t12, t13);
orig12 = x_12;
orig13 = x_13;
in1213 += 4;
state[12] = in1213 & 0xFFFFFFFF;
state[13] = (in1213 >> 32) & 0xFFFFFFFF;
for (int i = 0; i < 10; ++i) {
// Mix columns // Mix columns
VEC4_QUARTERROUND(0, 4, 8, 12); VEC4_QUARTERROUND(0, 4, 8, 12);
VEC4_QUARTERROUND(1, 5, 9, 13); VEC4_QUARTERROUND(1, 5, 9, 13);
@@ -146,18 +79,21 @@ static inline uint64_t _cha_4block(cha_ctx* ctx, uint8_t* begin, uint8_t* end) {
VEC4_QUARTERROUND(2, 7, 8, 13); VEC4_QUARTERROUND(2, 7, 8, 13);
VEC4_QUARTERROUND(3, 4, 9, 14); VEC4_QUARTERROUND(3, 4, 9, 14);
} }
// Add original block, unpack output
ONEQUAD(0, 1, 2, 3, c); ONEQUAD(0, 1, 2, 3, buf);
ONEQUAD(4, 5, 6, 7, c + 16); ONEQUAD(4, 5, 6, 7, buf + 16);
ONEQUAD(8, 9, 10, 11, c + 32); ONEQUAD(8, 9, 10, 11, buf + 32);
ONEQUAD(12, 13, 14, 15, c + 48); ONEQUAD(12, 13, 14, 15, buf + 48);
COUNTER_INCREMENT(4, 4, 4, 4);
// *counter += 4; buf += 256;
c += 256;
} }
return c - begin; // Bytes written // Update counter
state[12] = _mm_cvtsi128_si32(orig[12]);
state[13] = _mm_cvtsi128_si32(orig[13]);
return batches * 256;
} }
#undef COUNTER_INCREMENT
#undef ONEQUAD #undef ONEQUAD
#undef ONEQUAD_TRANSPOSE #undef ONEQUAD_TRANSPOSE
#undef VEC4_ROT #undef VEC4_ROT
src/cha8block.h
+79 -211
View File
@@ -1,25 +1,11 @@
#include <immintrin.h> // AVX2 #include <immintrin.h> // AVX2
#include <tmmintrin.h> // SSSE3 #include <tmmintrin.h> // SSSE3
// clang-format off
#define VEC8_ROT(A, IMM) \ #define VEC8_ROT(A, IMM) \
_mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM)))
/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 &
* 16) (better) */
#define VEC8_QUARTERROUND(A, B, C, D) \
x[A] = _mm256_add_epi32(x[A], x[B]); \
t[A] = _mm256_xor_si256(x[D], x[A]); \
x[D] = _mm256_shuffle_epi8(t[A], rot16); \
x[C] = _mm256_add_epi32(x[C], x[D]); \
t[C] = _mm256_xor_si256(x[B], x[C]); \
x[B] = VEC8_ROT(t[C], 12); \
x[A] = _mm256_add_epi32(x[A], x[B]); \
t[A] = _mm256_xor_si256(x[D], x[A]); \
x[D] = _mm256_shuffle_epi8(t[A], rot8); \
x[C] = _mm256_add_epi32(x[C], x[D]); \
t[C] = _mm256_xor_si256(x[B], x[C]); \
x[B] = VEC8_ROT(t[C], 7)
#define VEC8_LINE1(A, B, C, D) \ #define VEC8_LINE1(A, B, C, D) \
x[A] = _mm256_add_epi32(x[A], x[B]); \ x[A] = _mm256_add_epi32(x[A], x[B]); \
x[D] = _mm256_shuffle_epi8(_mm256_xor_si256(x[D], x[A]), rot16) x[D] = _mm256_shuffle_epi8(_mm256_xor_si256(x[D], x[A]), rot16)
@@ -33,229 +19,111 @@
x[C] = _mm256_add_epi32(x[C], x[D]); \ x[C] = _mm256_add_epi32(x[C], x[D]); \
x[B] = VEC8_ROT(_mm256_xor_si256(x[B], x[C]), 7) x[B] = VEC8_ROT(_mm256_xor_si256(x[B], x[C]), 7)
#define VEC8_ROUND_SEQ( \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \
) \
VEC8_LINE1(A1, B1, C1, D1); \
VEC8_LINE1(A2, B2, C2, D2); \
VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define VEC8_ROUND_HALF( \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \
) \
VEC8_LINE1(A1, B1, C1, D1); \
VEC8_LINE1(A2, B2, C2, D2); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define VEC8_ROUND_HALFANDHALF( \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \
) \
VEC8_LINE1(A1, B1, C1, D1); \
VEC8_LINE1(A2, B2, C2, D2); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define VEC8_ROUND( \ #define VEC8_ROUND( \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \
) \ ) \
VEC8_ROUND_SEQ( \ VEC8_LINE1(A1, B1, C1, D1); \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \ VEC8_LINE1(A2, B2, C2, D2); \
) VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define ONEQUAD_TRANSPOSE(A, B, C, D) \ #define TRANSPOSE(A, B, C, D) \
{ \ { \
__m128i t0, t1, t2, t3; \ const __m256i t0 = _mm256_unpacklo_epi32(x[A], x[B]), \
x[A] = _mm256_add_epi32(x[A], orig[A]); \ t1 = _mm256_unpacklo_epi32(x[C], x[D]), \
x[B] = _mm256_add_epi32(x[B], orig[B]); \ t2 = _mm256_unpackhi_epi32(x[A], x[B]), \
x[C] = _mm256_add_epi32(x[C], orig[C]); \ t3 = _mm256_unpackhi_epi32(x[C], x[D]); \
x[D] = _mm256_add_epi32(x[D], orig[D]); \ x[A] = _mm256_unpacklo_epi64(t0, t1); \
t[A] = _mm256_unpacklo_epi32(x[A], x[B]); \ x[B] = _mm256_unpackhi_epi64(t0, t1); \
t[B] = _mm256_unpacklo_epi32(x[C], x[D]); \ x[C] = _mm256_unpacklo_epi64(t2, t3); \
t[C] = _mm256_unpackhi_epi32(x[A], x[B]); \ x[D] = _mm256_unpackhi_epi64(t2, t3); \
t[D] = _mm256_unpackhi_epi32(x[C], x[D]); \
x[A] = _mm256_unpacklo_epi64(t[A], t[B]); \
x[B] = _mm256_unpackhi_epi64(t[A], t[B]); \
x[C] = _mm256_unpacklo_epi64(t[C], t[D]); \
x[D] = _mm256_unpackhi_epi64(t[C], t[D]); \
_mm_storeu_si128( \
(__m128i*)(c + 0), _mm256_extracti128_si256(x[A], 0) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 64), _mm256_extracti128_si256(x[B], 0) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 128), _mm256_extracti128_si256(x[C], 0) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 192), _mm256_extracti128_si256(x[D], 0) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 256), _mm256_extracti128_si256(x[A], 1) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 320), _mm256_extracti128_si256(x[B], 1) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 384), _mm256_extracti128_si256(x[C], 1) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 448), _mm256_extracti128_si256(x[D], 1) \
); \
}
#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D)
#define ONEQUAD_UNPCK(A, B, C, D) \
{ \
x[A] = _mm256_add_epi32(x[A], orig[A]); \
x[B] = _mm256_add_epi32(x[B], orig[B]); \
x[C] = _mm256_add_epi32(x[C], orig[C]); \
x[D] = _mm256_add_epi32(x[D], orig[D]); \
t[A] = _mm256_unpacklo_epi32(x[A], x[B]); \
t[B] = _mm256_unpacklo_epi32(x[C], x[D]); \
t[C] = _mm256_unpackhi_epi32(x[A], x[B]); \
t[D] = _mm256_unpackhi_epi32(x[C], x[D]); \
x[A] = _mm256_unpacklo_epi64(t[A], t[B]); \
x[B] = _mm256_unpackhi_epi64(t[A], t[B]); \
x[C] = _mm256_unpacklo_epi64(t[C], t[D]); \
x[D] = _mm256_unpackhi_epi64(t[C], t[D]); \
} }
#define ONEOCTO(A, B, C, D, A2, B2, C2, D2, c) \ #define ONEOCTO(A, B, C, D, A2, B2, C2, D2, c) \
{ \ { \
ONEQUAD_UNPCK(A, B, C, D); \ TRANSPOSE(A, B, C, D); \
ONEQUAD_UNPCK(A2, B2, C2, D2); \ TRANSPOSE(A2, B2, C2, D2); \
t[A] = _mm256_permute2x128_si256(x[A], x[A2], 0x20); \ _mm256_storeu_si256((__m256i*)(c), _mm256_permute2x128_si256(x[A], x[A2], 0x20)); \
t[A2] = _mm256_permute2x128_si256(x[A], x[A2], 0x31); \ _mm256_storeu_si256((__m256i*)(c + 64), _mm256_permute2x128_si256(x[B], x[B2], 0x20)); \
t[B] = _mm256_permute2x128_si256(x[B], x[B2], 0x20); \ _mm256_storeu_si256((__m256i*)(c + 128), _mm256_permute2x128_si256(x[C], x[C2], 0x20)); \
t[B2] = _mm256_permute2x128_si256(x[B], x[B2], 0x31); \ _mm256_storeu_si256((__m256i*)(c + 192), _mm256_permute2x128_si256(x[D], x[D2], 0x20)); \
t[C] = _mm256_permute2x128_si256(x[C], x[C2], 0x20); \ _mm256_storeu_si256((__m256i*)(c + 256), _mm256_permute2x128_si256(x[A], x[A2], 0x31)); \
t[C2] = _mm256_permute2x128_si256(x[C], x[C2], 0x31); \ _mm256_storeu_si256((__m256i*)(c + 320), _mm256_permute2x128_si256(x[B], x[B2], 0x31)); \
t[D] = _mm256_permute2x128_si256(x[D], x[D2], 0x20); \ _mm256_storeu_si256((__m256i*)(c + 384), _mm256_permute2x128_si256(x[C], x[C2], 0x31)); \
t[D2] = _mm256_permute2x128_si256(x[D], x[D2], 0x31); \ _mm256_storeu_si256((__m256i*)(c + 448), _mm256_permute2x128_si256(x[D], x[D2], 0x31)); \
_mm256_storeu_si256((__m256i*)(c), t[A]); \
_mm256_storeu_si256((__m256i*)(c + 64), t[B]); \
_mm256_storeu_si256((__m256i*)(c + 128), t[C]); \
_mm256_storeu_si256((__m256i*)(c + 192), t[D]); \
_mm256_storeu_si256((__m256i*)(c + 256), t[A2]); \
_mm256_storeu_si256((__m256i*)(c + 320), t[B2]); \
_mm256_storeu_si256((__m256i*)(c + 384), t[C2]); \
_mm256_storeu_si256((__m256i*)(c + 448), t[D2]); \
} }
static inline uint64_t _cha_8block(cha_ctx* ctx, uint8_t* begin, uint8_t* end) { #define COUNTER_INCREMENT(addv) \
if (end - begin < 512) { \
return 0; orig[12] = _mm256_add_epi32(orig[12], addv); \
orig[13] = _mm256_add_epi32(orig[13], _mm256_srli_epi32(_mm256_cmpgt_epi32(addv, orig[12]), 31)); \
}
uint8_t* c = begin; static inline uint64_t
uint32_t* state = ctx->state; _cha_8block(uint8_t* buf, size_t bufsize, uint32_t state[16], unsigned rounds) {
uint64_t* counter = (uint64_t*)(state + 12); unsigned batches = bufsize / 512;
/* constant for shuffling bytes (replacing multiple-of-8 rotates) */ /* constant for shuffling bytes (replacing multiple-of-8 rotates) */
__m256i rot16 = _mm256_set_epi8( const __m256i rot16 = _mm256_set_epi8(
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, 13, 12, 15, 14, 9, 13, 12, 15, 14,
8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2 9, 8, 11, 10,
5, 4, 7, 6,
1, 0, 3, 2,
13, 12, 15, 14,
9, 8, 11, 10,
5, 4, 7, 6,
1, 0, 3, 2
); );
__m256i rot8 = _mm256_set_epi8( const __m256i rot8 = _mm256_set_epi8(
14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3, 14, 13, 12, 15, 10, 14, 13, 12, 15,
9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3 10, 9, 8, 11,
6, 5, 4, 7,
2, 1, 0, 3,
14, 13, 12, 15,
10, 9, 8, 11,
6, 5, 4, 7,
2, 1, 0, 3
); );
__m256i orig[16];
/* the naive way seems as fast (if not a bit faster) than the vector way */
__m256i x[16], orig[16], t[16];
for (int i = 0; i < 16; ++i) for (int i = 0; i < 16; ++i)
orig[i] = _mm256_set1_epi32(state[i]); orig[i] = _mm256_set1_epi32(state[i]);
COUNTER_INCREMENT(_mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0));
const __m256i addv12 = _mm256_set_epi64x(3, 2, 1, 0); for (unsigned b = batches; b-->0;) {
const __m256i addv13 = _mm256_set_epi64x(7, 6, 5, 4); __m256i x[16];
for (int i = 0; i < 16; ++i) x[i] = orig[i];
while (end - c >= 512) { for (unsigned r = rounds / 2; r-->0;) {
for (int i = 0; i < 16; ++i)
if (i != 12 && i != 13)
x[i] = orig[i];
// Calculate the eight parallel counters on x_12 and x_13
t[13] = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(*counter));
t[12] = _mm256_add_epi64(addv12, t[13]);
t[13] = _mm256_add_epi64(addv13, t[13]);
x[12] = _mm256_unpacklo_epi32(t[12], t[13]);
x[13] = _mm256_unpackhi_epi32(t[12], t[13]);
t[12] = _mm256_unpacklo_epi32(x[12], x[13]);
t[13] = _mm256_unpackhi_epi32(x[12], x[13]);
/* required because unpack* are intra-lane */
const __m256i permute = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0);
x[12] = _mm256_permutevar8x32_epi32(t[12], permute);
x[13] = _mm256_permutevar8x32_epi32(t[13], permute);
orig[12] = x[12];
orig[13] = x[13];
for (int i = 0; i < 10; ++i) {
VEC8_ROUND(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); VEC8_ROUND(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
VEC8_ROUND(0, 5, 10, 15, 1, 6, 11, 12, 2, 7, 8, 13, 3, 4, 9, 14); VEC8_ROUND(0, 5, 10, 15, 1, 6, 11, 12, 2, 7, 8, 13, 3, 4, 9, 14);
} }
for (unsigned i = 0; i < 16; ++i) x[i] = _mm256_add_epi32(x[i], orig[i]);
ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7, c); ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7, buf);
ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15, c + 32); ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15, buf + 32);
COUNTER_INCREMENT(_mm256_set1_epi32(8));
*counter += 8; buf += 512;
c += 512;
} }
return c - begin; state[12] = _mm256_extract_epi32(orig[12], 0);
state[13] = _mm256_extract_epi32(orig[13], 0);
return batches * 512;
} }
#undef ONEQUAD #undef COUNTER_INCREMENT
#undef ONEQUAD_TRANSPOSE
#undef ONEQUAD_UNPCK
#undef ONEOCTO #undef ONEOCTO
#undef TRANSPOSE
#undef VEC8_ROT #undef VEC8_ROT
#undef VEC8_QUARTERROUND
#undef VEC8_QUARTERROUND_NAIVE
#undef VEC8_QUARTERROUND_SHUFFLE
#undef VEC8_QUARTERROUND_SHUFFLE2
#undef VEC8_LINE1 #undef VEC8_LINE1
#undef VEC8_LINE2 #undef VEC8_LINE2
#undef VEC8_LINE3 #undef VEC8_LINE3
#undef VEC8_LINE4 #undef VEC8_LINE4
#undef VEC8_ROUND #undef VEC8_ROUND
#undef VEC8_ROUND_SEQ
#undef VEC8_ROUND_HALF
#undef VEC8_ROUND_HALFANDHALF
src/chacha20.h
+70 -40
View File
@@ -2,12 +2,8 @@
#include <stdint.h> #include <stdint.h>
#define CHA_BLOCK_SIZE 64 #define CHA_BLOCK_SIZE 64
#define BATCH_BLOCKS 8
typedef struct cha_ctx { #define BATCH_SIZE (BATCH_BLOCKS * CHA_BLOCK_SIZE)
uint32_t state[16];
uint8_t unconsumed[CHA_BLOCK_SIZE];
uint8_t uncount;
} cha_ctx;
#if defined(__x86_64__) #if defined(__x86_64__)
#ifdef __GNUC__ #ifdef __GNUC__
@@ -16,6 +12,7 @@ typedef struct cha_ctx {
#pragma GCC target("avx2") #pragma GCC target("avx2")
#endif #endif
#include "cha8block.h" #include "cha8block.h"
#else
#endif #endif
#include "cha1block.h" #include "cha1block.h"
@@ -31,12 +28,21 @@ typedef struct cha_ctx {
#include <time.h> #include <time.h>
#include <unistd.h> #include <unistd.h>
typedef struct cha_ctx {
uint32_t state[16];
uint8_t unconsumed[BATCH_SIZE];
uint32_t offset, end;
unsigned rounds;
} cha_ctx;
/// @brief Initialize cha_ctx /// @brief Initialize cha_ctx
/// @param ctx holds ChaCha20 state /// @param ctx holds ChaCha20 state
/// @param key 32 byte key /// @param key 32 byte key
/// @param iv 16 bytes, where normally initial 4-8 bytes are zeroes and the rest /// @param iv 16 bytes, usually the first 4-8 bytes are zeroes, the rest nonce
/// nonce /// @param rounds ChaCha iteration count: 8=fast, 12=balanced, 20=secure
void cha_init(cha_ctx* ctx, const uint8_t* key, const uint8_t* iv) { void cha_init(
cha_ctx* ctx, const uint8_t* key, const uint8_t* iv, unsigned rounds
) {
ctx->state[0] = 0x61707865; ctx->state[0] = 0x61707865;
ctx->state[1] = 0x3320646e; ctx->state[1] = 0x3320646e;
ctx->state[2] = 0x79622d32; ctx->state[2] = 0x79622d32;
@@ -44,61 +50,85 @@ void cha_init(cha_ctx* ctx, const uint8_t* key, const uint8_t* iv) {
memcpy(ctx->state + 4, key, 32); memcpy(ctx->state + 4, key, 32);
memcpy(ctx->state + 12, iv, 16); memcpy(ctx->state + 12, iv, 16);
memset(ctx->unconsumed, 0, sizeof ctx->unconsumed); memset(ctx->unconsumed, 0, sizeof ctx->unconsumed);
ctx->uncount = 0; ctx->offset = ctx->end = 0;
ctx->rounds = rounds;
} }
/// Dispose of sensitive data within the context /// Dispose of sensitive data within the context
void cha_wipe(cha_ctx* ctx);
void cha_wipe(cha_ctx* ctx) { memset(ctx, 0, sizeof(cha_ctx)); } void cha_wipe(cha_ctx* ctx) { memset(ctx, 0, sizeof(cha_ctx)); }
/// @brief Advance or rewind the stream to any arbitrary location
/// @param ctx ChaCha context
/// @param offset Offset in blocks of 64 bytes (counter change)
void cha_seek_blocks(cha_ctx* ctx, int64_t offset) {
*(uint64_t*)(ctx->state + 12) += offset;
ctx->offset = ctx->end = 0;
}
uint64_t cha_generate_batch(
uint8_t* out, size_t outsize, uint32_t* state, unsigned rounds
) {
#if defined(__x86_64__)
if (__builtin_cpu_supports("ssse3")) {
if (__builtin_cpu_supports("avx2")) {
return _cha_8block(out, outsize, state, rounds);
}
return _cha_4block(out, outsize, state, rounds);
}
#elif defined(__aarch64__)
return _cha_4block(out, outsize, state);
#endif
unsigned count = 0;
unsigned n = _cha_block(out, state, rounds);
count += n;
out += n;
return count;
}
/// @brief Incremental generation, keeps state between calls /// @brief Incremental generation, keeps state between calls
/// @param ctx ChaCha20 context /// @param ctx ChaCha context
/// @param out output buffer /// @param out output buffer
/// @param outlen output buffer length /// @param outlen output buffer length
void cha_update(cha_ctx* ctx, uint8_t* out, uint64_t outlen) { void cha_update(cha_ctx* ctx, uint8_t* out, uint64_t outlen) {
// The included header will mess with these variables // The included header will mess with these variables
uint8_t* c = out;
uint8_t* end = out + outlen; uint8_t* end = out + outlen;
if (ctx->uncount) { if (ctx->offset) {
// Need to generate stored buffer?
if (ctx->end == 0)
ctx->end = cha_generate_batch(
ctx->unconsumed, BATCH_SIZE, ctx->state, ctx->rounds
);
// Deliver stored bytes first // Deliver stored bytes first
uint64_t N = ctx->uncount >= outlen ? outlen : ctx->uncount; uint64_t N = ctx->end - ctx->offset;
memcpy(c, ctx->unconsumed, N); if (N > outlen)
ctx->uncount -= N; N = outlen;
c += N; memcpy(out, ctx->unconsumed + ctx->offset, N);
if (ctx->uncount) { ctx->offset += N;
memmove(ctx->unconsumed, ctx->unconsumed + N, ctx->uncount); out += N;
} if (out == end)
if (c == out + outlen)
return; return;
} }
#if defined(__x86_64__) out += cha_generate_batch(out, end - out, ctx->state, ctx->rounds);
// TODO: Handle resume if we are not at block boundary const uint32_t N = end - out;
if (__builtin_cpu_supports("ssse3")) { if (N) {
if (__builtin_cpu_supports("avx2")) { ctx->end = cha_generate_batch(
c += _cha_8block(ctx, c, end); ctx->unconsumed, BATCH_SIZE, ctx->state, ctx->rounds
assert(end - c < 512); );
} memcpy(out, ctx->unconsumed, N);
c += _cha_4block(ctx, c, end); ctx->offset = N;
assert(end - c < 256);
} }
#elif defined(__aarch64__)
c += _cha_4block(ctx, c, end);
#endif
c += _cha_block(ctx, c, end);
assert(c == end);
} }
/// @brief Produce a requested number of random bytes of the stream, one shot. /// @brief Produce a requested number of random bytes, single shot.
/// @param out output buffer /// @param out output buffer
/// @param outlen output buffer length /// @param outlen output buffer length
/// @param key 32 byte key /// @param key 32 byte key
/// @param iv 16 bytes, where normally initial 4-8 bytes are zeroes (counter) /// @param iv 16 bytes, where normally initial 4-8 bytes are 0 (counter)
void cha_generate( void cha_generate(
uint8_t* out, uint64_t outlen, const uint8_t key[32], const uint8_t iv[16] uint8_t* out, uint64_t outlen, const uint8_t key[32], const uint8_t iv[16]
) { ) {
cha_ctx ctx; cha_ctx ctx;
cha_init(&ctx, key, iv); cha_init(&ctx, key, iv, 20);
cha_update(&ctx, out, outlen); cha_update(&ctx, out, outlen);
cha_wipe(&ctx); cha_wipe(&ctx);
} }
src/cli.c
+25 -9
View File
@@ -28,6 +28,7 @@ typedef struct thread_args {
unsigned char* buf; unsigned char* buf;
unsigned char key[32]; unsigned char key[32];
unsigned workers; unsigned workers;
unsigned rounds;
pthread_mutex_t lock; pthread_mutex_t lock;
pthread_cond_t cond; pthread_cond_t cond;
pthread_t thread; pthread_t thread;
@@ -35,20 +36,22 @@ typedef struct thread_args {
void* producer_thread(void* a) { void* producer_thread(void* a) {
thread_args* args = (thread_args*)a; thread_args* args = (thread_args*)a;
const uint64_t ivstep = args->workers * BLOCK_SIZE / 64; const uint64_t ivstep = args->workers * BATCH_BLOCKS;
cha_ctx ctx;
cha_init(&ctx, args->key, default_iv, args->rounds);
cha_seek_blocks(&ctx, args->index * BLOCK_SIZE / 64);
while (!quit) { while (!quit) {
pthread_mutex_lock(&args->lock); pthread_mutex_lock(&args->lock);
while (args->done) { while (args->done) {
pthread_cond_wait(&args->cond, &args->lock); pthread_cond_wait(&args->cond, &args->lock);
} }
unsigned char iv[16]; cha_update(&ctx, args->buf, BLOCK_SIZE);
memcpy(iv, default_iv, 16); cha_seek_blocks(&ctx, ivstep);
*(uint64_t*)iv += args->index * ivstep; // Counter increment
cha_generate(args->buf, BLOCK_SIZE, args->key, default_iv);
args->done = 1; args->done = 1;
pthread_cond_signal(&args->cond); pthread_cond_signal(&args->cond);
pthread_mutex_unlock(&args->lock); pthread_mutex_unlock(&args->lock);
} }
cha_wipe(&ctx);
return NULL; return NULL;
} }
@@ -79,7 +82,7 @@ void print_status(
int fast( int fast(
FILE* f, unsigned workers, uint64_t max_bytes, unsigned char const key[32], FILE* f, unsigned workers, uint64_t max_bytes, unsigned char const key[32],
unsigned char const iv[16] unsigned char const iv[16], unsigned rounds
) { ) {
thread_args args[workers]; thread_args args[workers];
memset(args, 0, sizeof args); memset(args, 0, sizeof args);
@@ -87,6 +90,7 @@ int fast(
args[i].index = i; args[i].index = i;
args[i].buf = malloc(BLOCK_SIZE); args[i].buf = malloc(BLOCK_SIZE);
args[i].workers = workers; args[i].workers = workers;
args[i].rounds = rounds;
memcpy(args[i].key, key, 32); memcpy(args[i].key, key, 32);
pthread_mutex_init(&args[i].lock, NULL); pthread_mutex_init(&args[i].lock, NULL);
pthread_cond_init(&args[i].cond, NULL); pthread_cond_init(&args[i].cond, NULL);
@@ -160,7 +164,8 @@ void print_hex(unsigned char* buf, size_t len) {
void help(char** argv) { void help(char** argv) {
fprintf( fprintf(
stderr, stderr,
"Usage: %s [-t #threads] [-s hexseed] [-b #bytes] [-o outputfile]\n\n", "Usage: %s [-t #threads] [-s hexseed] [-b #bytes] [-c #rounds] [-o "
"outputfile]\n\n",
argv[0] argv[0]
); );
} }
@@ -169,10 +174,11 @@ int main(int argc, char** argv) {
unsigned char key[32] = {}; unsigned char key[32] = {};
unsigned char iv[16] = {}; unsigned char iv[16] = {};
unsigned int workers = 8; unsigned int workers = 8;
unsigned int rounds = 20;
char* output = NULL; char* output = NULL;
uint64_t max_bytes = 0; uint64_t max_bytes = 0;
bool seeded = false; bool seeded = false;
for (char opt; (opt = getopt(argc, argv, "bost")) != -1;) { for (char opt; (opt = getopt(argc, argv, "bostc")) != -1;) {
if (opt == 't') { if (opt == 't') {
if (optind >= argc || sscanf(argv[optind++], "%u", &workers) != 1) { if (optind >= argc || sscanf(argv[optind++], "%u", &workers) != 1) {
fprintf( fprintf(
@@ -182,6 +188,16 @@ int main(int argc, char** argv) {
} }
continue; continue;
} }
if (opt == 'c') {
if (optind >= argc || sscanf(argv[optind++], "%u", &rounds) != 1) {
fprintf(
stderr,
"Expected the number ChaCha rounds (8, 12 or 20) after -c\n"
);
return 1;
}
continue;
}
if (opt == 's') { if (opt == 's') {
if (optind >= argc || !parse_hex(argv[optind++], key, 32)) { if (optind >= argc || !parse_hex(argv[optind++], key, 32)) {
fprintf(stderr, "Expected a hex seed string after -s\n"); fprintf(stderr, "Expected a hex seed string after -s\n");
@@ -250,7 +266,7 @@ int main(int argc, char** argv) {
} }
signal(SIGINT, signal_handler); signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler); signal(SIGTERM, signal_handler);
int ret = fast(f, workers, max_bytes, key, iv); int ret = fast(f, workers, max_bytes, key, iv, rounds);
fclose(f); fclose(f);
return ret; return ret;
} }