LeoVasanko/RandQuik
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor

Browse Source
This commit is contained in:
LeoVasanko
2023-10-28 16:24:18 +00:00
parent ab67daf050
commit fe80552d6b
4 changed files with 3 additions and 476 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/c-stream.h
-48
View File
@@ -1,48 +0,0 @@
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#define QUARTERSTEP(a, b, c, n) \
a += b; \
c ^= a; \
c = (c << n) | (c >> (32 - n))
#define QUARTERROUND(a, b, c, d) \
QUARTERSTEP(a, b, d, 16); \
QUARTERSTEP(c, d, b, 12); \
QUARTERSTEP(a, b, d, 8); \
QUARTERSTEP(c, d, b, 7);
static inline uint64_t
_cha_block(uint32_t* state, uint8_t* begin, uint8_t* end) {
uint64_t* counter = (uint64_t*)&state[12];
uint8_t* c = begin;
while (c < end) {
uint32_t x[16];
memcpy(x, state, sizeof x);
for (int i = 20; i > 0; i -= 2) {
QUARTERROUND(x[0], x[4], x[8], x[12])
QUARTERROUND(x[1], x[5], x[9], x[13])
QUARTERROUND(x[2], x[6], x[10], x[14])
QUARTERROUND(x[3], x[7], x[11], x[15])
QUARTERROUND(x[0], x[5], x[10], x[15])
QUARTERROUND(x[1], x[6], x[11], x[12])
QUARTERROUND(x[2], x[7], x[8], x[13])
QUARTERROUND(x[3], x[4], x[9], x[14])
}
for (int i = 0; i < 16; i++)
x[i] += state[i];
++*counter;
uint64_t bytes = end - c;
if (bytes < 64) {
memcpy(c, x, bytes);
c = end;
break;
}
memcpy(c, x, 64);
c += 64;
}
return c - begin;
}
src/chacha20.c
+3 -3
View File
@@ -19,9 +19,9 @@
#include <tmmintrin.h> #include <tmmintrin.h>
#include <unistd.h> #include <unistd.h>
#include "c-stream.h" #include "cha1block.h"
#include "u4-stream.h" #include "cha4block.h"
#include "u8-stream.h" #include "cha8block.h"
void cha_init(cha_ctx* ctx, const uint8_t* key, const uint8_t* iv) { void cha_init(cha_ctx* ctx, const uint8_t* key, const uint8_t* iv) {
ctx->input[0] = 0x61707865; ctx->input[0] = 0x61707865;
src/u4-stream.h
-166
View File
@@ -1,166 +0,0 @@
#define VEC4_ROT(A, IMM) \
_mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM)))
/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 &
* 16) (better) */
#define VEC4_QUARTERROUND(A, B, C, D) \
x_##A = _mm_add_epi32(x_##A, x_##B); \
t_##A = _mm_xor_si128(x_##D, x_##A); \
x_##D = _mm_shuffle_epi8(t_##A, rot16); \
x_##C = _mm_add_epi32(x_##C, x_##D); \
t_##C = _mm_xor_si128(x_##B, x_##C); \
x_##B = VEC4_ROT(t_##C, 12); \
x_##A = _mm_add_epi32(x_##A, x_##B); \
t_##A = _mm_xor_si128(x_##D, x_##A); \
x_##D = _mm_shuffle_epi8(t_##A, rot8); \
x_##C = _mm_add_epi32(x_##C, x_##D); \
t_##C = _mm_xor_si128(x_##B, x_##C); \
x_##B = VEC4_ROT(t_##C, 7)
#define ONEQUAD(A, B, C, D, CT) \
{ \
/* Add original block */ \
x_##A = _mm_add_epi32(x_##A, orig##A); \
x_##B = _mm_add_epi32(x_##B, orig##B); \
x_##C = _mm_add_epi32(x_##C, orig##C); \
x_##D = _mm_add_epi32(x_##D, orig##D); \
/* Transpose */ \
t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \
t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \
t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \
t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \
x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \
x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \
x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \
x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \
\
_mm_storeu_si128((__m128i*)(CT), x_##A); \
_mm_storeu_si128((__m128i*)(CT + 64), x_##B); \
_mm_storeu_si128((__m128i*)(CT + 128), x_##C); \
_mm_storeu_si128((__m128i*)(CT + 192), x_##D); \
}
static inline uint64_t
_cha_4block(uint32_t* state, uint8_t* begin, uint8_t* end) {
if (end - begin < 256)
return 0;
uint8_t* c = begin;
uint32_t* x = state;
uint64_t* counter = (uint64_t*)&state[12]; // low u32 in 12, high u32 in 13
const __m256i vec_increment =
_mm256_set_epi64x(3, 2, 1, 0); // 0, 1, 2, 3 for the increments
const __m256i interleave =
_mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); // Indices for counters
/* constant for shuffling bytes (replacing multiple-of-8 rotates) */
const __m128i rot16 =
_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2);
const __m128i rot8 =
_mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3);
// Load state to vectors, duplicate four times
__m128i x_0 = _mm_set1_epi32(x[0]);
__m128i x_1 = _mm_set1_epi32(x[1]);
__m128i x_2 = _mm_set1_epi32(x[2]);
__m128i x_3 = _mm_set1_epi32(x[3]);
__m128i x_4 = _mm_set1_epi32(x[4]);
__m128i x_5 = _mm_set1_epi32(x[5]);
__m128i x_6 = _mm_set1_epi32(x[6]);
__m128i x_7 = _mm_set1_epi32(x[7]);
__m128i x_8 = _mm_set1_epi32(x[8]);
__m128i x_9 = _mm_set1_epi32(x[9]);
__m128i x_10 = _mm_set1_epi32(x[10]);
__m128i x_11 = _mm_set1_epi32(x[11]);
__m128i x_12;
__m128i x_13;
__m128i x_14 = _mm_set1_epi32(x[14]);
__m128i x_15 = _mm_set1_epi32(x[15]);
__m128i orig0 = x_0;
__m128i orig1 = x_1;
__m128i orig2 = x_2;
__m128i orig3 = x_3;
__m128i orig4 = x_4;
__m128i orig5 = x_5;
__m128i orig6 = x_6;
__m128i orig7 = x_7;
__m128i orig8 = x_8;
__m128i orig9 = x_9;
__m128i orig10 = x_10;
__m128i orig11 = x_11;
__m128i orig12 = {};
__m128i orig13 = {};
__m128i orig14 = x_14;
__m128i orig15 = x_15;
__m128i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12,
t_13, t_14, t_15;
const __m128i addv12 = _mm_set_epi64x(1, 0);
const __m128i addv13 = _mm_set_epi64x(3, 2);
while (end - c >= 256) {
x_0 = orig0;
x_1 = orig1;
x_2 = orig2;
x_3 = orig3;
x_4 = orig4;
x_5 = orig5;
x_6 = orig6;
x_7 = orig7;
x_8 = orig8;
x_9 = orig9;
x_10 = orig10;
x_11 = orig11;
x_14 = orig14;
x_15 = orig15;
// Calculate counter + 0..3 for adjacent blocks (x12 low and x13
// high of each)
uint32_t in12 = state[12];
uint32_t in13 = state[13];
uint64_t in1213 = ((uint64_t)in12) | (((uint64_t)in13) << 32);
__m128i t12, t13;
t12 = _mm_set1_epi64x(in1213);
t13 = _mm_set1_epi64x(in1213);
x_12 = _mm_add_epi64(addv12, t12);
x_13 = _mm_add_epi64(addv13, t13);
t12 = _mm_unpacklo_epi32(x_12, x_13);
t13 = _mm_unpackhi_epi32(x_12, x_13);
x_12 = _mm_unpacklo_epi32(t12, t13);
x_13 = _mm_unpackhi_epi32(t12, t13);
orig12 = x_12;
orig13 = x_13;
in1213 += 4;
state[12] = in1213 & 0xFFFFFFFF;
state[13] = (in1213 >> 32) & 0xFFFFFFFF;
for (int i = 0; i < 10; ++i) {
// Mix columns
VEC4_QUARTERROUND(0, 4, 8, 12);
VEC4_QUARTERROUND(1, 5, 9, 13);
VEC4_QUARTERROUND(2, 6, 10, 14);
VEC4_QUARTERROUND(3, 7, 11, 15);
// Mix diagonals
VEC4_QUARTERROUND(0, 5, 10, 15);
VEC4_QUARTERROUND(1, 6, 11, 12);
VEC4_QUARTERROUND(2, 7, 8, 13);
VEC4_QUARTERROUND(3, 4, 9, 14);
}
ONEQUAD(0, 1, 2, 3, c);
ONEQUAD(4, 5, 6, 7, c + 16);
ONEQUAD(8, 9, 10, 11, c + 32);
ONEQUAD(12, 13, 14, 15, c + 48);
// *counter += 4;
c += 256;
}
return c - begin; // Bytes written
}
#undef ONEQUAD
#undef ONEQUAD_TRANSPOSE
#undef VEC4_ROT
#undef VEC4_QUARTERROUND
#undef VEC4_QUARTERROUND_SHUFFLE
src/u8-stream.h
-259
View File
@@ -1,259 +0,0 @@
#define VEC8_ROT(A, IMM) \
_mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM)))
/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 &
* 16) (better) */
#define VEC8_QUARTERROUND(A, B, C, D) \
x[A] = _mm256_add_epi32(x[A], x[B]); \
t[A] = _mm256_xor_si256(x[D], x[A]); \
x[D] = _mm256_shuffle_epi8(t[A], rot16); \
x[C] = _mm256_add_epi32(x[C], x[D]); \
t[C] = _mm256_xor_si256(x[B], x[C]); \
x[B] = VEC8_ROT(t[C], 12); \
x[A] = _mm256_add_epi32(x[A], x[B]); \
t[A] = _mm256_xor_si256(x[D], x[A]); \
x[D] = _mm256_shuffle_epi8(t[A], rot8); \
x[C] = _mm256_add_epi32(x[C], x[D]); \
t[C] = _mm256_xor_si256(x[B], x[C]); \
x[B] = VEC8_ROT(t[C], 7)
#define VEC8_LINE1(A, B, C, D) \
x[A] = _mm256_add_epi32(x[A], x[B]); \
x[D] = _mm256_shuffle_epi8(_mm256_xor_si256(x[D], x[A]), rot16)
#define VEC8_LINE2(A, B, C, D) \
x[C] = _mm256_add_epi32(x[C], x[D]); \
x[B] = VEC8_ROT(_mm256_xor_si256(x[B], x[C]), 12)
#define VEC8_LINE3(A, B, C, D) \
x[A] = _mm256_add_epi32(x[A], x[B]); \
x[D] = _mm256_shuffle_epi8(_mm256_xor_si256(x[D], x[A]), rot8)
#define VEC8_LINE4(A, B, C, D) \
x[C] = _mm256_add_epi32(x[C], x[D]); \
x[B] = VEC8_ROT(_mm256_xor_si256(x[B], x[C]), 7)
#define VEC8_ROUND_SEQ( \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \
) \
VEC8_LINE1(A1, B1, C1, D1); \
VEC8_LINE1(A2, B2, C2, D2); \
VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define VEC8_ROUND_HALF( \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \
) \
VEC8_LINE1(A1, B1, C1, D1); \
VEC8_LINE1(A2, B2, C2, D2); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define VEC8_ROUND_HALFANDHALF( \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \
) \
VEC8_LINE1(A1, B1, C1, D1); \
VEC8_LINE1(A2, B2, C2, D2); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define VEC8_ROUND( \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \
) \
VEC8_ROUND_SEQ( \
A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, D4 \
)
#define ONEQUAD_TRANSPOSE(A, B, C, D) \
{ \
__m128i t0, t1, t2, t3; \
x[A] = _mm256_add_epi32(x[A], orig[A]); \
x[B] = _mm256_add_epi32(x[B], orig[B]); \
x[C] = _mm256_add_epi32(x[C], orig[C]); \
x[D] = _mm256_add_epi32(x[D], orig[D]); \
t[A] = _mm256_unpacklo_epi32(x[A], x[B]); \
t[B] = _mm256_unpacklo_epi32(x[C], x[D]); \
t[C] = _mm256_unpackhi_epi32(x[A], x[B]); \
t[D] = _mm256_unpackhi_epi32(x[C], x[D]); \
x[A] = _mm256_unpacklo_epi64(t[A], t[B]); \
x[B] = _mm256_unpackhi_epi64(t[A], t[B]); \
x[C] = _mm256_unpacklo_epi64(t[C], t[D]); \
x[D] = _mm256_unpackhi_epi64(t[C], t[D]); \
_mm_storeu_si128( \
(__m128i*)(c + 0), _mm256_extracti128_si256(x[A], 0) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 64), _mm256_extracti128_si256(x[B], 0) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 128), _mm256_extracti128_si256(x[C], 0) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 192), _mm256_extracti128_si256(x[D], 0) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 256), _mm256_extracti128_si256(x[A], 1) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 320), _mm256_extracti128_si256(x[B], 1) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 384), _mm256_extracti128_si256(x[C], 1) \
); \
_mm_storeu_si128( \
(__m128i*)(c + 448), _mm256_extracti128_si256(x[D], 1) \
); \
}
#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D)
#define ONEQUAD_UNPCK(A, B, C, D) \
{ \
x[A] = _mm256_add_epi32(x[A], orig[A]); \
x[B] = _mm256_add_epi32(x[B], orig[B]); \
x[C] = _mm256_add_epi32(x[C], orig[C]); \
x[D] = _mm256_add_epi32(x[D], orig[D]); \
t[A] = _mm256_unpacklo_epi32(x[A], x[B]); \
t[B] = _mm256_unpacklo_epi32(x[C], x[D]); \
t[C] = _mm256_unpackhi_epi32(x[A], x[B]); \
t[D] = _mm256_unpackhi_epi32(x[C], x[D]); \
x[A] = _mm256_unpacklo_epi64(t[A], t[B]); \
x[B] = _mm256_unpackhi_epi64(t[A], t[B]); \
x[C] = _mm256_unpacklo_epi64(t[C], t[D]); \
x[D] = _mm256_unpackhi_epi64(t[C], t[D]); \
}
#define ONEOCTO(A, B, C, D, A2, B2, C2, D2, c) \
{ \
ONEQUAD_UNPCK(A, B, C, D); \
ONEQUAD_UNPCK(A2, B2, C2, D2); \
t[A] = _mm256_permute2x128_si256(x[A], x[A2], 0x20); \
t[A2] = _mm256_permute2x128_si256(x[A], x[A2], 0x31); \
t[B] = _mm256_permute2x128_si256(x[B], x[B2], 0x20); \
t[B2] = _mm256_permute2x128_si256(x[B], x[B2], 0x31); \
t[C] = _mm256_permute2x128_si256(x[C], x[C2], 0x20); \
t[C2] = _mm256_permute2x128_si256(x[C], x[C2], 0x31); \
t[D] = _mm256_permute2x128_si256(x[D], x[D2], 0x20); \
t[D2] = _mm256_permute2x128_si256(x[D], x[D2], 0x31); \
_mm256_storeu_si256((__m256i*)(c), t[A]); \
_mm256_storeu_si256((__m256i*)(c + 64), t[B]); \
_mm256_storeu_si256((__m256i*)(c + 128), t[C]); \
_mm256_storeu_si256((__m256i*)(c + 192), t[D]); \
_mm256_storeu_si256((__m256i*)(c + 256), t[A2]); \
_mm256_storeu_si256((__m256i*)(c + 320), t[B2]); \
_mm256_storeu_si256((__m256i*)(c + 384), t[C2]); \
_mm256_storeu_si256((__m256i*)(c + 448), t[D2]); \
}
static inline uint64_t
_cha_8block(uint32_t* state, uint8_t* begin, uint8_t* end) {
if (end - begin < 512)
return 0;
uint8_t* c = begin;
uint64_t* counter = (uint64_t*)(state + 12);
/* constant for shuffling bytes (replacing multiple-of-8 rotates) */
__m256i rot16 = _mm256_set_epi8(
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, 13, 12, 15, 14, 9,
8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2
);
__m256i rot8 = _mm256_set_epi8(
14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3, 14, 13, 12, 15, 10,
9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3
);
/* the naive way seems as fast (if not a bit faster) than the vector way */
__m256i x[16], orig[16], t[16];
for (int i = 0; i < 16; ++i)
orig[i] = _mm256_set1_epi32(state[i]);
const __m256i addv12 = _mm256_set_epi64x(3, 2, 1, 0);
const __m256i addv13 = _mm256_set_epi64x(7, 6, 5, 4);
while (end - c >= 512) {
for (int i = 0; i < 16; ++i)
if (i != 12 && i != 13)
x[i] = orig[i];
// Calculate the eight parallel counters on x_12 and x_13
t[13] = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(*counter));
t[12] = _mm256_add_epi64(addv12, t[13]);
t[13] = _mm256_add_epi64(addv13, t[13]);
x[12] = _mm256_unpacklo_epi32(t[12], t[13]);
x[13] = _mm256_unpackhi_epi32(t[12], t[13]);
t[12] = _mm256_unpacklo_epi32(x[12], x[13]);
t[13] = _mm256_unpackhi_epi32(x[12], x[13]);
/* required because unpack* are intra-lane */
const __m256i permute = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0);
x[12] = _mm256_permutevar8x32_epi32(t[12], permute);
x[13] = _mm256_permutevar8x32_epi32(t[13], permute);
orig[12] = x[12];
orig[13] = x[13];
for (int i = 0; i < 10; ++i) {
VEC8_ROUND(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
VEC8_ROUND(0, 5, 10, 15, 1, 6, 11, 12, 2, 7, 8, 13, 3, 4, 9, 14);
}
ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7, c);
ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15, c + 32);
*counter += 8;
c += 512;
}
return c - begin;
}
#undef ONEQUAD
#undef ONEQUAD_TRANSPOSE
#undef ONEQUAD_UNPCK
#undef ONEOCTO
#undef VEC8_ROT
#undef VEC8_QUARTERROUND
#undef VEC8_QUARTERROUND_NAIVE
#undef VEC8_QUARTERROUND_SHUFFLE
#undef VEC8_QUARTERROUND_SHUFFLE2
#undef VEC8_LINE1
#undef VEC8_LINE2
#undef VEC8_LINE3
#undef VEC8_LINE4
#undef VEC8_ROUND
#undef VEC8_ROUND_SEQ
#undef VEC8_ROUND_HALF
#undef VEC8_ROUND_HALFANDHALF