LeoVasanko/RandQuik
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial commit

Browse Source
This commit is contained in:
LeoVasanko
2023-10-28 10:51:13 +00:00
commit 3deea0c962
10 changed files with 883 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/c-stream.h
+51
View File
@@ -0,0 +1,51 @@
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#define QUARTERSTEP(a, b, c, n) \
a += b; \
c ^= a; \
c = (c << n) | (c >> (32 - n))
#define QUARTERROUND(a, b, c, d) \
QUARTERSTEP(a, b, d, 16); \
QUARTERSTEP(c, d, b, 12); \
QUARTERSTEP(a, b, d, 8); \
QUARTERSTEP(c, d, b, 7);
{
// Change variables x and orig...
uint32_t const *orig = x;
while (bytes > 0)
{
uint32_t x[16];
memcpy(x, orig, sizeof x);
for (int i = 20; i > 0; i -= 2)
{
QUARTERROUND(x[0], x[4], x[8], x[12])
QUARTERROUND(x[1], x[5], x[9], x[13])
QUARTERROUND(x[2], x[6], x[10], x[14])
QUARTERROUND(x[3], x[7], x[11], x[15])
QUARTERROUND(x[0], x[5], x[10], x[15])
QUARTERROUND(x[1], x[6], x[11], x[12])
QUARTERROUND(x[2], x[7], x[8], x[13])
QUARTERROUND(x[3], x[4], x[9], x[14])
}
for (int i = 0; i < 16; i++)
x[i] += orig[i];
uint64_t *counter = (uint64_t *)&orig[12];
++*counter;
if (bytes < 64)
{
memcpy(c, x, bytes);
c += bytes;
bytes = 0;
break;
}
memcpy(c, x, 64);
bytes -= 64;
c += 64;
}
}
src/randquik.c
+312
View File
@@ -0,0 +1,312 @@
#ifdef __GNUC__
#pragma GCC target("sse2")
#pragma GCC target("ssse3")
#pragma GCC target("avx2")
#endif
#include "randquik.h"
#include <errno.h>
#include <signal.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <emmintrin.h>
#include <immintrin.h>
#include <tmmintrin.h>
#include <pthread.h>
#include <time.h>
#include <unistd.h>
#define BLOCK_SIZE (1 << 21) // 2 MiB seems optimal for speed
static volatile bool quit = false;
void signal_handler(int sig)
{
quit = true;
signal(SIGINT, SIG_DFL);
signal(SIGTERM, SIG_DFL);
}
static const unsigned char default_iv[16] = "\0\0\0\0\0\0\0\0RandQuik";
typedef struct thread_args
{
int index;
int done;
unsigned char *buf;
unsigned char key[32];
unsigned workers;
pthread_mutex_t lock;
pthread_cond_t cond;
pthread_t thread;
} thread_args;
void *producer_thread(void *a)
{
thread_args *args = (thread_args *)a;
const unsigned long long ivstep = args->workers * BLOCK_SIZE / 64;
while (!quit)
{
pthread_mutex_lock(&args->lock);
while (args->done)
{
pthread_cond_wait(&args->cond, &args->lock);
}
unsigned char iv[16];
memcpy(iv, default_iv, 16);
*(uint64_t *)iv += args->index * ivstep; // Counter increment
chacha20_stream(args->buf, BLOCK_SIZE, args->key, default_iv);
args->done = 1;
pthread_cond_signal(&args->cond);
pthread_mutex_unlock(&args->lock);
}
return NULL;
}
void print_status(unsigned long long bytes, unsigned long long max_bytes, struct timespec start_time)
{
struct timespec end_time;
clock_gettime(CLOCK_MONOTONIC, &end_time);
double t = (end_time.tv_sec - start_time.tv_sec) + 1e-9 * (end_time.tv_nsec - start_time.tv_nsec);
char buf[64] = {};
double speed = bytes / t;
char const *unit = "MB";
double m = 1e-6;
if (speed > 0.5e9)
{
unit = "GB";
m = 1e-9;
}
if (max_bytes)
{
snprintf(buf, sizeof buf - 1, " of %'.0lf", m * max_bytes);
}
fprintf(stderr, "\r%5.0lf%s %s written, %.2lf %s/s.\e[K", m * bytes, buf, unit, m * speed, unit);
}
int fast(FILE *f, unsigned workers, unsigned long long max_bytes, unsigned char const key[32], unsigned char const iv[16])
{
thread_args args[workers];
memset(args, 0, sizeof args);
for (int i = 0; i < workers; ++i)
{
args[i].index = i;
args[i].buf = malloc(BLOCK_SIZE);
args[i].workers = workers;
memcpy(args[i].key, key, 32);
pthread_mutex_init(&args[i].lock, NULL);
pthread_cond_init(&args[i].cond, NULL);
pthread_create(&args[i].thread, NULL, producer_thread, &args[i]);
}
struct timespec start_time;
clock_gettime(CLOCK_MONOTONIC, &start_time);
int i = -1;
unsigned long long bytes = 0;
while (!quit)
{
i = (i + 1) % workers;
pthread_mutex_lock(&args[i].lock);
while (!args[i].done)
{
pthread_cond_wait(&args[i].cond, &args[i].lock);
}
if (bytes % (1 << 30) == 0 || bytes + BLOCK_SIZE >= max_bytes)
{
print_status(bytes, max_bytes, start_time);
}
unsigned long long sz = BLOCK_SIZE;
if (max_bytes && bytes + sz >= max_bytes)
{
fprintf(stderr, "\r\e[KMax reached\n");
sz = max_bytes - bytes;
quit = true;
}
if (fwrite(args[i].buf, sz, 1, f) != 1)
{
quit = true;
fprintf(stderr, "\r\e[KWrite failed: %s\n", strerror(errno));
}
bytes += sz;
args[i].done = 0;
pthread_cond_signal(&args[i].cond);
pthread_mutex_unlock(&args[i].lock);
}
print_status(bytes, max_bytes, start_time);
for (int i = 0; i < workers; ++i)
{
args[i].done = 0;
pthread_cancel(args[i].thread);
pthread_join(args[i].thread, NULL);
pthread_mutex_destroy(&args[i].lock);
pthread_cond_destroy(&args[i].cond);
free(args[i].buf);
}
fprintf(stderr, "\nRandQuik wrote %llu bytes!\n\n", bytes);
return 0;
}
bool parse_hex(char *str, unsigned char *buf, size_t len)
{
for (size_t i = 0; i < len; ++i)
{
int sz = 0;
if (sscanf(str, "%2hhx%n", buf + i, &sz) != 1)
{
if (*str)
{
fprintf(stderr, "Unable to read seed at `%s`\n\n", str);
return false;
}
return true; // Shorter than key length is OK
}
str += sz;
}
return true;
}
void print_hex(unsigned char *buf, size_t len)
{
for (size_t i = 0; i < len; ++i)
{
fprintf(stderr, "%02hhx", buf[i]);
}
}
void help(char **argv)
{
fprintf(stderr, "Usage: %s [-t #threads] [-s hexseed] [-b #bytes] [-o outputfile]\n\n", argv[0]);
}
int main(int argc, char **argv)
{
unsigned char key[32] = {};
unsigned char iv[16] = {};
unsigned int workers = 8;
char *output = NULL;
unsigned long long max_bytes = 0;
bool seeded = false;
for (char opt; (opt = getopt(argc, argv, "bost")) != -1;)
{
if (opt == 't')
{
if (optind >= argc || sscanf(argv[optind++], "%u", &workers) != 1)
{
fprintf(stderr, "Expected the number of worker threads after -t\n");
exit(EXIT_FAILURE);
}
continue;
}
if (opt == 's')
{
if (optind >= argc || !parse_hex(argv[optind++], key, 32))
{
fprintf(stderr, "Expected a hex seed string after -s\n");
exit(EXIT_FAILURE);
}
seeded = true;
continue;
}
if (opt == 'o')
{
if (optind >= argc)
{
fprintf(stderr, "Expected output filename after -s\n");
return 1;
}
if (strcmp(argv[optind], "-") != 0)
{
output = argv[optind++];
}
continue;
}
if (opt == 'b')
{
if (optind >= argc || sscanf(argv[optind++], "%llu", &max_bytes) != 1)
{
fprintf(stderr, "Expected a maximum number of bytes to read after -b\n");
exit(EXIT_FAILURE);
}
continue;
}
help(argv);
return 1;
}
FILE *f = stdout;
if (output)
{
f = fopen(output, "wb");
if (!f)
{
fprintf(stderr, "Failed to open %s for writing.\n", output);
return 1;
}
}
else if (isatty(1))
{
fprintf(stderr, "Won't print random on console. Pipe me to another program or file instead.\n\n");
help(argv);
return 1;
}
if (!seeded)
{
FILE *urand = fopen("/dev/urandom", "rb");
if (!urand || fread(key, 32, 1, urand) != 1)
{
fprintf(stderr, "Failed to seed from /dev/urandom. Use -s hexstring for manual seeding.\n");
fclose(urand);
return 1;
}
fclose(urand);
fprintf(stderr, "Random seed generated. This sequence may be repeated by:\n%s -s ", argv[0]);
print_hex(key, 32);
fprintf(stderr, "\n\n");
}
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
int ret = fast(f, workers, max_bytes, key, iv);
fclose(f);
return ret;
}
typedef struct chacha_ctx
{
uint32_t input[16];
} chacha_ctx;
static void
chacha_init(chacha_ctx *ctx, const uint8_t *k, const uint8_t *iv)
{
ctx->input[0] = 0x61707865;
ctx->input[1] = 0x3320646e;
ctx->input[2] = 0x79622d32;
ctx->input[3] = 0x6b206574;
memcpy(ctx->input + 4, k, 32);
memcpy(ctx->input + 12, iv, 16);
}
int chacha20_stream(unsigned char *out, unsigned long long outlen,
const unsigned char key[32], const unsigned char iv[16])
{
chacha_ctx ctx;
chacha_init(&ctx, key, iv);
{
// The included header will mess with these variables
unsigned long long bytes = outlen;
uint32_t *x = ctx.input;
unsigned char *c = out;
if (__builtin_cpu_supports("avx2"))
{
#include "u8-stream.h"
#include "u4-stream.h"
}
#include "c-stream.h"
}
memset(&ctx, 0, sizeof ctx);
return 0;
}
src/randquik.h
+2
View File
@@ -0,0 +1,2 @@
int chacha20_stream(unsigned char *out, unsigned long long outlen,
const unsigned char key[32], const unsigned char iv[16]);
src/u4-stream.h
+144
View File
@@ -0,0 +1,144 @@
#define VEC4_ROT(A, IMM) \
_mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM)))
/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 &
* 16) (better) */
#define VEC4_QUARTERROUND(A, B, C, D) \
x_##A = _mm_add_epi32(x_##A, x_##B); \
t_##A = _mm_xor_si128(x_##D, x_##A); \
x_##D = _mm_shuffle_epi8(t_##A, rot16); \
x_##C = _mm_add_epi32(x_##C, x_##D); \
t_##C = _mm_xor_si128(x_##B, x_##C); \
x_##B = VEC4_ROT(t_##C, 12); \
x_##A = _mm_add_epi32(x_##A, x_##B); \
t_##A = _mm_xor_si128(x_##D, x_##A); \
x_##D = _mm_shuffle_epi8(t_##A, rot8); \
x_##C = _mm_add_epi32(x_##C, x_##D); \
t_##C = _mm_xor_si128(x_##B, x_##C); \
x_##B = VEC4_ROT(t_##C, 7)
#define ONEQUAD(A, B, C, D, CT) \
{ \
/* Add original block */ \
x_##A = _mm_add_epi32(x_##A, orig##A); \
x_##B = _mm_add_epi32(x_##B, orig##B); \
x_##C = _mm_add_epi32(x_##C, orig##C); \
x_##D = _mm_add_epi32(x_##D, orig##D); \
/* Transpose */ \
t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \
t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \
t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \
t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \
x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \
x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \
x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \
x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \
\
_mm_storeu_si128((__m128i *)(CT), x_##A); \
_mm_storeu_si128((__m128i *)(CT + 64), x_##B); \
_mm_storeu_si128((__m128i *)(CT + 128), x_##C); \
_mm_storeu_si128((__m128i *)(CT + 192), x_##D); \
}
if (bytes >= 256)
{
const __m256i vec_increment = _mm256_set_epi64x(3, 2, 1, 0); // 0, 1, 2, 3 for the increments
const __m256i interleave = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); // Indices for counters
/* constant for shuffling bytes (replacing multiple-of-8 rotates) */
const __m128i rot16 =
_mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2);
const __m128i rot8 =
_mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3);
// Load state to vectors, duplicate four times
__m128i x_0 = _mm_set1_epi32(x[0]);
__m128i x_1 = _mm_set1_epi32(x[1]);
__m128i x_2 = _mm_set1_epi32(x[2]);
__m128i x_3 = _mm_set1_epi32(x[3]);
__m128i x_4 = _mm_set1_epi32(x[4]);
__m128i x_5 = _mm_set1_epi32(x[5]);
__m128i x_6 = _mm_set1_epi32(x[6]);
__m128i x_7 = _mm_set1_epi32(x[7]);
__m128i x_8 = _mm_set1_epi32(x[8]);
__m128i x_9 = _mm_set1_epi32(x[9]);
__m128i x_10 = _mm_set1_epi32(x[10]);
__m128i x_11 = _mm_set1_epi32(x[11]);
__m128i x_12;
__m128i x_13;
__m128i x_14 = _mm_set1_epi32(x[14]);
__m128i x_15 = _mm_set1_epi32(x[15]);
__m128i orig0 = x_0;
__m128i orig1 = x_1;
__m128i orig2 = x_2;
__m128i orig3 = x_3;
__m128i orig4 = x_4;
__m128i orig5 = x_5;
__m128i orig6 = x_6;
__m128i orig7 = x_7;
__m128i orig8 = x_8;
__m128i orig9 = x_9;
__m128i orig10 = x_10;
__m128i orig11 = x_11;
__m128i orig12 = {};
__m128i orig13 = {};
__m128i orig14 = x_14;
__m128i orig15 = x_15;
__m128i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12,
t_13, t_14, t_15;
while (bytes >= 256)
{
x_0 = orig0;
x_1 = orig1;
x_2 = orig2;
x_3 = orig3;
x_4 = orig4;
x_5 = orig5;
x_6 = orig6;
x_7 = orig7;
x_8 = orig8;
x_9 = orig9;
x_10 = orig10;
x_11 = orig11;
x_14 = orig14;
x_15 = orig15;
// Calculate counter + 0..3 for adjacent blocks (x12 low and x13 high of each)
uint64_t *counter = (uint64_t *)&x[12];
__m256i counters = _mm256_add_epi64(_mm256_set1_epi64x(*counter), vec_increment);
counters = _mm256_permutevar8x32_epi32(counters, interleave);
x_12 = _mm256_extracti128_si256(counters, 0);
x_13 = _mm256_extracti128_si256(counters, 1);
for (int i = 0; i < 10; ++i)
{
// Mix columns
VEC4_QUARTERROUND(0, 4, 8, 12);
VEC4_QUARTERROUND(1, 5, 9, 13);
VEC4_QUARTERROUND(2, 6, 10, 14);
VEC4_QUARTERROUND(3, 7, 11, 15);
// Mix diagonals
VEC4_QUARTERROUND(0, 5, 10, 15);
VEC4_QUARTERROUND(1, 6, 11, 12);
VEC4_QUARTERROUND(2, 7, 8, 13);
VEC4_QUARTERROUND(3, 4, 9, 14);
}
ONEQUAD(0, 1, 2, 3, c);
ONEQUAD(4, 5, 6, 7, c + 16);
ONEQUAD(8, 9, 10, 11, c + 32);
ONEQUAD(12, 13, 14, 15, c + 48);
*counter += 4;
bytes -= 256;
c += 256;
}
}
#undef ONEQUAD
#undef ONEQUAD_TRANSPOSE
#undef VEC4_ROT
#undef VEC4_QUARTERROUND
#undef VEC4_QUARTERROUND_SHUFFLE
src/u8-stream.h
+278
View File
@@ -0,0 +1,278 @@
#define VEC8_ROT(A, IMM) \
_mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM)))
/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 &
* 16) (better) */
#define VEC8_QUARTERROUND(A, B, C, D) \
x_##A = _mm256_add_epi32(x_##A, x_##B); \
t_##A = _mm256_xor_si256(x_##D, x_##A); \
x_##D = _mm256_shuffle_epi8(t_##A, rot16); \
x_##C = _mm256_add_epi32(x_##C, x_##D); \
t_##C = _mm256_xor_si256(x_##B, x_##C); \
x_##B = VEC8_ROT(t_##C, 12); \
x_##A = _mm256_add_epi32(x_##A, x_##B); \
t_##A = _mm256_xor_si256(x_##D, x_##A); \
x_##D = _mm256_shuffle_epi8(t_##A, rot8); \
x_##C = _mm256_add_epi32(x_##C, x_##D); \
t_##C = _mm256_xor_si256(x_##B, x_##C); \
x_##B = VEC8_ROT(t_##C, 7)
#define VEC8_LINE1(A, B, C, D) \
x_##A = _mm256_add_epi32(x_##A, x_##B); \
x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16)
#define VEC8_LINE2(A, B, C, D) \
x_##C = _mm256_add_epi32(x_##C, x_##D); \
x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12)
#define VEC8_LINE3(A, B, C, D) \
x_##A = _mm256_add_epi32(x_##A, x_##B); \
x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8)
#define VEC8_LINE4(A, B, C, D) \
x_##C = _mm256_add_epi32(x_##C, x_##D); \
x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7)
#define VEC8_ROUND_SEQ(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, \
C4, D4) \
VEC8_LINE1(A1, B1, C1, D1); \
VEC8_LINE1(A2, B2, C2, D2); \
VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define VEC8_ROUND_HALF(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, \
B4, C4, D4) \
VEC8_LINE1(A1, B1, C1, D1); \
VEC8_LINE1(A2, B2, C2, D2); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define VEC8_ROUND_HALFANDHALF(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, \
A4, B4, C4, D4) \
VEC8_LINE1(A1, B1, C1, D1); \
VEC8_LINE1(A2, B2, C2, D2); \
VEC8_LINE2(A1, B1, C1, D1); \
VEC8_LINE2(A2, B2, C2, D2); \
VEC8_LINE1(A3, B3, C3, D3); \
VEC8_LINE1(A4, B4, C4, D4); \
VEC8_LINE2(A3, B3, C3, D3); \
VEC8_LINE2(A4, B4, C4, D4); \
VEC8_LINE3(A1, B1, C1, D1); \
VEC8_LINE3(A2, B2, C2, D2); \
VEC8_LINE4(A1, B1, C1, D1); \
VEC8_LINE4(A2, B2, C2, D2); \
VEC8_LINE3(A3, B3, C3, D3); \
VEC8_LINE3(A4, B4, C4, D4); \
VEC8_LINE4(A3, B3, C3, D3); \
VEC8_LINE4(A4, B4, C4, D4)
#define VEC8_ROUND(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, \
D4) \
VEC8_ROUND_SEQ(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, \
D4)
if (bytes >= 512)
{
/* constant for shuffling bytes (replacing multiple-of-8 rotates) */
__m256i rot16 =
_mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2,
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2);
__m256i rot8 =
_mm256_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3,
14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3);
/* the naive way seems as fast (if not a bit faster) than the vector way */
__m256i x_0 = _mm256_set1_epi32(x[0]);
__m256i x_1 = _mm256_set1_epi32(x[1]);
__m256i x_2 = _mm256_set1_epi32(x[2]);
__m256i x_3 = _mm256_set1_epi32(x[3]);
__m256i x_4 = _mm256_set1_epi32(x[4]);
__m256i x_5 = _mm256_set1_epi32(x[5]);
__m256i x_6 = _mm256_set1_epi32(x[6]);
__m256i x_7 = _mm256_set1_epi32(x[7]);
__m256i x_8 = _mm256_set1_epi32(x[8]);
__m256i x_9 = _mm256_set1_epi32(x[9]);
__m256i x_10 = _mm256_set1_epi32(x[10]);
__m256i x_11 = _mm256_set1_epi32(x[11]);
__m256i x_12;
__m256i x_13;
__m256i x_14 = _mm256_set1_epi32(x[14]);
__m256i x_15 = _mm256_set1_epi32(x[15]);
__m256i orig0 = x_0;
__m256i orig1 = x_1;
__m256i orig2 = x_2;
__m256i orig3 = x_3;
__m256i orig4 = x_4;
__m256i orig5 = x_5;
__m256i orig6 = x_6;
__m256i orig7 = x_7;
__m256i orig8 = x_8;
__m256i orig9 = x_9;
__m256i orig10 = x_10;
__m256i orig11 = x_11;
__m256i orig12;
__m256i orig13;
__m256i orig14 = x_14;
__m256i orig15 = x_15;
__m256i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12,
t_13, t_14, t_15;
const __m256i addv12 = _mm256_set_epi64x(3, 2, 1, 0);
const __m256i addv13 = _mm256_set_epi64x(7, 6, 5, 4);
while (bytes >= 512)
{
__m256i t12, t13;
x_0 = orig0;
x_1 = orig1;
x_2 = orig2;
x_3 = orig3;
x_4 = orig4;
x_5 = orig5;
x_6 = orig6;
x_7 = orig7;
x_8 = orig8;
x_9 = orig9;
x_10 = orig10;
x_11 = orig11;
x_14 = orig14;
x_15 = orig15;
// Calculate the eight parallel counters on x_12 and x_13
uint64_t *counter = (uint64_t *)(x + 12);
t13 = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(*counter));
t12 = _mm256_add_epi64(addv12, t13);
t13 = _mm256_add_epi64(addv13, t13);
x_12 = _mm256_unpacklo_epi32(t12, t13);
x_13 = _mm256_unpackhi_epi32(t12, t13);
t12 = _mm256_unpacklo_epi32(x_12, x_13);
t13 = _mm256_unpackhi_epi32(x_12, x_13);
/* required because unpack* are intra-lane */
const __m256i permute = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0);
x_12 = _mm256_permutevar8x32_epi32(t12, permute);
x_13 = _mm256_permutevar8x32_epi32(t13, permute);
orig12 = x_12;
orig13 = x_13;
for (int i = 0; i < 10; ++i)
{
VEC8_ROUND(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15);
VEC8_ROUND(0, 5, 10, 15, 1, 6, 11, 12, 2, 7, 8, 13, 3, 4, 9, 14);
}
#define ONEQUAD_TRANSPOSE(A, B, C, D) \
{ \
__m128i t0, t1, t2, t3; \
x_##A = _mm256_add_epi32(x_##A, orig##A); \
x_##B = _mm256_add_epi32(x_##B, orig##B); \
x_##C = _mm256_add_epi32(x_##C, orig##C); \
x_##D = _mm256_add_epi32(x_##D, orig##D); \
t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \
t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \
t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \
t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \
x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \
x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \
x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \
x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \
_mm_storeu_si128((__m128i *)(c + 0), _mm256_extracti128_si256(x_##A, 0)); \
_mm_storeu_si128((__m128i *)(c + 64), _mm256_extracti128_si256(x_##B, 0)); \
_mm_storeu_si128((__m128i *)(c + 128), _mm256_extracti128_si256(x_##C, 0)); \
_mm_storeu_si128((__m128i *)(c + 192), _mm256_extracti128_si256(x_##D, 0)); \
_mm_storeu_si128((__m128i *)(c + 256), _mm256_extracti128_si256(x_##A, 1)); \
_mm_storeu_si128((__m128i *)(c + 320), _mm256_extracti128_si256(x_##B, 1)); \
_mm_storeu_si128((__m128i *)(c + 384), _mm256_extracti128_si256(x_##C, 1)); \
_mm_storeu_si128((__m128i *)(c + 448), _mm256_extracti128_si256(x_##D, 1)); \
}
#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D)
#define ONEQUAD_UNPCK(A, B, C, D) \
{ \
x_##A = _mm256_add_epi32(x_##A, orig##A); \
x_##B = _mm256_add_epi32(x_##B, orig##B); \
x_##C = _mm256_add_epi32(x_##C, orig##C); \
x_##D = _mm256_add_epi32(x_##D, orig##D); \
t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \
t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \
t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \
t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \
x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \
x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \
x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \
x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \
}
#define ONEOCTO(A, B, C, D, A2, B2, C2, D2, c) \
{ \
ONEQUAD_UNPCK(A, B, C, D); \
ONEQUAD_UNPCK(A2, B2, C2, D2); \
t_##A = _mm256_permute2x128_si256(x_##A, x_##A2, 0x20); \
t_##A2 = _mm256_permute2x128_si256(x_##A, x_##A2, 0x31); \
t_##B = _mm256_permute2x128_si256(x_##B, x_##B2, 0x20); \
t_##B2 = _mm256_permute2x128_si256(x_##B, x_##B2, 0x31); \
t_##C = _mm256_permute2x128_si256(x_##C, x_##C2, 0x20); \
t_##C2 = _mm256_permute2x128_si256(x_##C, x_##C2, 0x31); \
t_##D = _mm256_permute2x128_si256(x_##D, x_##D2, 0x20); \
t_##D2 = _mm256_permute2x128_si256(x_##D, x_##D2, 0x31); \
_mm256_storeu_si256((__m256i *)(c + 0), t_##A); \
_mm256_storeu_si256((__m256i *)(c + 64), t_##B); \
_mm256_storeu_si256((__m256i *)(c + 128), t_##C); \
_mm256_storeu_si256((__m256i *)(c + 192), t_##D); \
_mm256_storeu_si256((__m256i *)(c + 256), t_##A2); \
_mm256_storeu_si256((__m256i *)(c + 320), t_##B2); \
_mm256_storeu_si256((__m256i *)(c + 384), t_##C2); \
_mm256_storeu_si256((__m256i *)(c + 448), t_##D2); \
}
ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7, c);
ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15, c + 32);
*counter += 8;
bytes -= 512;
c += 512;
}
}
#undef ONEQUAD
#undef ONEQUAD_TRANSPOSE
#undef ONEQUAD_UNPCK
#undef ONEOCTO
#undef VEC8_ROT
#undef VEC8_QUARTERROUND
#undef VEC8_QUARTERROUND_NAIVE
#undef VEC8_QUARTERROUND_SHUFFLE
#undef VEC8_QUARTERROUND_SHUFFLE2
#undef VEC8_LINE1
#undef VEC8_LINE2
#undef VEC8_LINE3
#undef VEC8_LINE4
#undef VEC8_ROUND
#undef VEC8_ROUND_SEQ
#undef VEC8_ROUND_HALF
#undef VEC8_ROUND_HALFANDHALF