commit 3deea0c9621799780a87f085810f681a1c5efc3c Author: Leo Vasanko Date: Sat Oct 28 10:51:13 2023 +0000 Initial commit diff --git a/README.md b/README.md new file mode 100644 index 0000000..9a3e74f --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# Extremely fast Cryptographically Secure RNG + +I was disappointed with the sad state of random number generators. Many languages don't ship anything useful and some are stuck with whatever the OS provides. All existing implementations are slow, typically maxing out at a few hundred megabytes per second. + +To give some perspective, my tool reaches 37.8 GB/s, writing /dev/null. Several gigabytes per second also on actual SSDs. + +Secondly, flaws have been found in non-cryptographic algorithms, especially on Mersenne Twister. The cryptographic options are simply better, being free of such issues as sequences repeating, and additionally being, well, cryptographically secure. Surprisingly, they appear even to be faster now, so there really ought to be no reason to stick with the old. diff --git a/benchmarks/crand.c b/benchmarks/crand.c new file mode 100644 index 0000000..bb9d142 --- /dev/null +++ b/benchmarks/crand.c @@ -0,0 +1,11 @@ +#include +#include + +int main(void) +{ + printf("RAND_MAX = %d\n", RAND_MAX); + for (unsigned long long i = 0; i < 1000000000; ++i) + { + rand(); + } +} diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..e9c9299 --- /dev/null +++ b/meson.build @@ -0,0 +1,8 @@ +project('randquik', 'c') +executable( + 'randquik', + 'src/randquik.c', + c_args: ['-Wall', '-O3', '-march=native'], + install: true, +) +dependency('threads') diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..31d6801 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,67 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "RandQuik" +version = "0.1.0" +description = "Extremely fast and cryptographically secure random number generator." +readme = "README.md" +license = "" +authors = [{ name = "Vasanko" }] +classifiers = [] +dependencies = [] +requires-python = ">=3.10" +keywords = [ + "random", + "generator", + "fast", + "secure", + "cryptographic", + "randomness", +] + +[project.urls] +Homepage = "" + +[project.optional-dependencies] +dev = ["pytest"] + +[tool.hatchling] + +[tool.hatch.version] +source = "hatchling.version:Version" + +[tool.hatch.build] +hooks.vcs.version-file = "randquik/_version.py" +hooks.vcs.template = """ +# This file is automatically generated by hatch build. +__version__ = {version!r} +""" + +targets.sdist.include = ["/randquik"] + +[tool.ruff] +extend-select = ["I", "W", "UP", "C4", "ISC", "S"] +# Worth selecting but still too broken: ASYNC, B, DTZ, FA +ignore = [ + "D100", + "D101", + "D102", + "D103", + "E402", + "E741", + "F811", + "F821", + # ruff format complains about these: + "ISC001", + "S101", + "S102", + "S104", + "S311", + "S603", + "S607", + "W191", +] +show-source = true +show-fixes = true diff --git a/randquik/__init__.py b/randquik/__init__.py new file mode 100644 index 0000000..0e240c9 --- /dev/null +++ b/randquik/__init__.py @@ -0,0 +1,3 @@ +from _version import __version__ + +__version__ # Public API diff --git a/src/c-stream.h b/src/c-stream.h new file mode 100644 index 0000000..6bc6239 --- /dev/null +++ b/src/c-stream.h @@ -0,0 +1,51 @@ +#include +#include +#include + +#define QUARTERSTEP(a, b, c, n) \ + a += b; \ + c ^= a; \ + c = (c << n) | (c >> (32 - n)) + +#define QUARTERROUND(a, b, c, d) \ + QUARTERSTEP(a, b, d, 16); \ + QUARTERSTEP(c, d, b, 12); \ + QUARTERSTEP(a, b, d, 8); \ + QUARTERSTEP(c, d, b, 7); + +{ + // Change variables x and orig... + uint32_t const *orig = x; + while (bytes > 0) + { + uint32_t x[16]; + memcpy(x, orig, sizeof x); + for (int i = 20; i > 0; i -= 2) + { + QUARTERROUND(x[0], x[4], x[8], x[12]) + QUARTERROUND(x[1], x[5], x[9], x[13]) + QUARTERROUND(x[2], x[6], x[10], x[14]) + QUARTERROUND(x[3], x[7], x[11], x[15]) + QUARTERROUND(x[0], x[5], x[10], x[15]) + QUARTERROUND(x[1], x[6], x[11], x[12]) + QUARTERROUND(x[2], x[7], x[8], x[13]) + QUARTERROUND(x[3], x[4], x[9], x[14]) + } + for (int i = 0; i < 16; i++) + x[i] += orig[i]; + + uint64_t *counter = (uint64_t *)&orig[12]; + ++*counter; + + if (bytes < 64) + { + memcpy(c, x, bytes); + c += bytes; + bytes = 0; + break; + } + memcpy(c, x, 64); + bytes -= 64; + c += 64; + } +} diff --git a/src/randquik.c b/src/randquik.c new file mode 100644 index 0000000..4601fd3 --- /dev/null +++ b/src/randquik.c @@ -0,0 +1,312 @@ +#ifdef __GNUC__ +#pragma GCC target("sse2") +#pragma GCC target("ssse3") +#pragma GCC target("avx2") +#endif + +#include "randquik.h" + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#define BLOCK_SIZE (1 << 21) // 2 MiB seems optimal for speed + +static volatile bool quit = false; + +void signal_handler(int sig) +{ + quit = true; + signal(SIGINT, SIG_DFL); + signal(SIGTERM, SIG_DFL); +} + +static const unsigned char default_iv[16] = "\0\0\0\0\0\0\0\0RandQuik"; +typedef struct thread_args +{ + int index; + int done; + unsigned char *buf; + unsigned char key[32]; + unsigned workers; + pthread_mutex_t lock; + pthread_cond_t cond; + pthread_t thread; +} thread_args; + +void *producer_thread(void *a) +{ + thread_args *args = (thread_args *)a; + const unsigned long long ivstep = args->workers * BLOCK_SIZE / 64; + while (!quit) + { + pthread_mutex_lock(&args->lock); + while (args->done) + { + pthread_cond_wait(&args->cond, &args->lock); + } + unsigned char iv[16]; + memcpy(iv, default_iv, 16); + *(uint64_t *)iv += args->index * ivstep; // Counter increment + chacha20_stream(args->buf, BLOCK_SIZE, args->key, default_iv); + args->done = 1; + pthread_cond_signal(&args->cond); + pthread_mutex_unlock(&args->lock); + } + return NULL; +} + +void print_status(unsigned long long bytes, unsigned long long max_bytes, struct timespec start_time) +{ + struct timespec end_time; + clock_gettime(CLOCK_MONOTONIC, &end_time); + double t = (end_time.tv_sec - start_time.tv_sec) + 1e-9 * (end_time.tv_nsec - start_time.tv_nsec); + char buf[64] = {}; + double speed = bytes / t; + char const *unit = "MB"; + double m = 1e-6; + if (speed > 0.5e9) + { + unit = "GB"; + m = 1e-9; + } + if (max_bytes) + { + snprintf(buf, sizeof buf - 1, " of %'.0lf", m * max_bytes); + } + + fprintf(stderr, "\r%5.0lf%s %s written, %.2lf %s/s.\e[K", m * bytes, buf, unit, m * speed, unit); +} +int fast(FILE *f, unsigned workers, unsigned long long max_bytes, unsigned char const key[32], unsigned char const iv[16]) +{ + thread_args args[workers]; + memset(args, 0, sizeof args); + for (int i = 0; i < workers; ++i) + { + args[i].index = i; + args[i].buf = malloc(BLOCK_SIZE); + args[i].workers = workers; + memcpy(args[i].key, key, 32); + pthread_mutex_init(&args[i].lock, NULL); + pthread_cond_init(&args[i].cond, NULL); + pthread_create(&args[i].thread, NULL, producer_thread, &args[i]); + } + + struct timespec start_time; + clock_gettime(CLOCK_MONOTONIC, &start_time); + + int i = -1; + unsigned long long bytes = 0; + while (!quit) + { + i = (i + 1) % workers; + pthread_mutex_lock(&args[i].lock); + while (!args[i].done) + { + pthread_cond_wait(&args[i].cond, &args[i].lock); + } + if (bytes % (1 << 30) == 0 || bytes + BLOCK_SIZE >= max_bytes) + { + print_status(bytes, max_bytes, start_time); + } + unsigned long long sz = BLOCK_SIZE; + if (max_bytes && bytes + sz >= max_bytes) + { + fprintf(stderr, "\r\e[KMax reached\n"); + sz = max_bytes - bytes; + quit = true; + } + if (fwrite(args[i].buf, sz, 1, f) != 1) + { + quit = true; + fprintf(stderr, "\r\e[KWrite failed: %s\n", strerror(errno)); + } + bytes += sz; + args[i].done = 0; + pthread_cond_signal(&args[i].cond); + pthread_mutex_unlock(&args[i].lock); + } + + print_status(bytes, max_bytes, start_time); + for (int i = 0; i < workers; ++i) + { + args[i].done = 0; + pthread_cancel(args[i].thread); + pthread_join(args[i].thread, NULL); + pthread_mutex_destroy(&args[i].lock); + pthread_cond_destroy(&args[i].cond); + free(args[i].buf); + } + fprintf(stderr, "\nRandQuik wrote %llu bytes!\n\n", bytes); + return 0; +} + +bool parse_hex(char *str, unsigned char *buf, size_t len) +{ + for (size_t i = 0; i < len; ++i) + { + int sz = 0; + if (sscanf(str, "%2hhx%n", buf + i, &sz) != 1) + { + if (*str) + { + fprintf(stderr, "Unable to read seed at `%s`\n\n", str); + return false; + } + return true; // Shorter than key length is OK + } + str += sz; + } + return true; +} + +void print_hex(unsigned char *buf, size_t len) +{ + for (size_t i = 0; i < len; ++i) + { + fprintf(stderr, "%02hhx", buf[i]); + } +} + +void help(char **argv) +{ + fprintf(stderr, "Usage: %s [-t #threads] [-s hexseed] [-b #bytes] [-o outputfile]\n\n", argv[0]); +} + +int main(int argc, char **argv) +{ + unsigned char key[32] = {}; + unsigned char iv[16] = {}; + unsigned int workers = 8; + char *output = NULL; + unsigned long long max_bytes = 0; + bool seeded = false; + for (char opt; (opt = getopt(argc, argv, "bost")) != -1;) + { + if (opt == 't') + { + if (optind >= argc || sscanf(argv[optind++], "%u", &workers) != 1) + { + fprintf(stderr, "Expected the number of worker threads after -t\n"); + exit(EXIT_FAILURE); + } + continue; + } + if (opt == 's') + { + if (optind >= argc || !parse_hex(argv[optind++], key, 32)) + { + fprintf(stderr, "Expected a hex seed string after -s\n"); + exit(EXIT_FAILURE); + } + seeded = true; + continue; + } + if (opt == 'o') + { + if (optind >= argc) + { + fprintf(stderr, "Expected output filename after -s\n"); + return 1; + } + if (strcmp(argv[optind], "-") != 0) + { + output = argv[optind++]; + } + continue; + } + if (opt == 'b') + { + if (optind >= argc || sscanf(argv[optind++], "%llu", &max_bytes) != 1) + { + fprintf(stderr, "Expected a maximum number of bytes to read after -b\n"); + exit(EXIT_FAILURE); + } + continue; + } + help(argv); + return 1; + } + FILE *f = stdout; + if (output) + { + f = fopen(output, "wb"); + if (!f) + { + fprintf(stderr, "Failed to open %s for writing.\n", output); + return 1; + } + } + else if (isatty(1)) + { + fprintf(stderr, "Won't print random on console. Pipe me to another program or file instead.\n\n"); + help(argv); + return 1; + } + if (!seeded) + { + FILE *urand = fopen("/dev/urandom", "rb"); + if (!urand || fread(key, 32, 1, urand) != 1) + { + fprintf(stderr, "Failed to seed from /dev/urandom. Use -s hexstring for manual seeding.\n"); + fclose(urand); + return 1; + } + fclose(urand); + fprintf(stderr, "Random seed generated. This sequence may be repeated by:\n%s -s ", argv[0]); + print_hex(key, 32); + fprintf(stderr, "\n\n"); + } + signal(SIGINT, signal_handler); + signal(SIGTERM, signal_handler); + int ret = fast(f, workers, max_bytes, key, iv); + fclose(f); + return ret; +} + +typedef struct chacha_ctx +{ + uint32_t input[16]; +} chacha_ctx; + +static void +chacha_init(chacha_ctx *ctx, const uint8_t *k, const uint8_t *iv) +{ + ctx->input[0] = 0x61707865; + ctx->input[1] = 0x3320646e; + ctx->input[2] = 0x79622d32; + ctx->input[3] = 0x6b206574; + memcpy(ctx->input + 4, k, 32); + memcpy(ctx->input + 12, iv, 16); +} + +int chacha20_stream(unsigned char *out, unsigned long long outlen, + const unsigned char key[32], const unsigned char iv[16]) +{ + chacha_ctx ctx; + chacha_init(&ctx, key, iv); + { + // The included header will mess with these variables + unsigned long long bytes = outlen; + uint32_t *x = ctx.input; + unsigned char *c = out; + if (__builtin_cpu_supports("avx2")) + { +#include "u8-stream.h" +#include "u4-stream.h" + } +#include "c-stream.h" + } + memset(&ctx, 0, sizeof ctx); + return 0; +} diff --git a/src/randquik.h b/src/randquik.h new file mode 100644 index 0000000..6e95306 --- /dev/null +++ b/src/randquik.h @@ -0,0 +1,2 @@ +int chacha20_stream(unsigned char *out, unsigned long long outlen, + const unsigned char key[32], const unsigned char iv[16]); diff --git a/src/u4-stream.h b/src/u4-stream.h new file mode 100644 index 0000000..2eebda5 --- /dev/null +++ b/src/u4-stream.h @@ -0,0 +1,144 @@ + +#define VEC4_ROT(A, IMM) \ + _mm_or_si128(_mm_slli_epi32(A, IMM), _mm_srli_epi32(A, (32 - IMM))) + +/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 & + * 16) (better) */ +#define VEC4_QUARTERROUND(A, B, C, D) \ + x_##A = _mm_add_epi32(x_##A, x_##B); \ + t_##A = _mm_xor_si128(x_##D, x_##A); \ + x_##D = _mm_shuffle_epi8(t_##A, rot16); \ + x_##C = _mm_add_epi32(x_##C, x_##D); \ + t_##C = _mm_xor_si128(x_##B, x_##C); \ + x_##B = VEC4_ROT(t_##C, 12); \ + x_##A = _mm_add_epi32(x_##A, x_##B); \ + t_##A = _mm_xor_si128(x_##D, x_##A); \ + x_##D = _mm_shuffle_epi8(t_##A, rot8); \ + x_##C = _mm_add_epi32(x_##C, x_##D); \ + t_##C = _mm_xor_si128(x_##B, x_##C); \ + x_##B = VEC4_ROT(t_##C, 7) + +#define ONEQUAD(A, B, C, D, CT) \ + { \ + /* Add original block */ \ + x_##A = _mm_add_epi32(x_##A, orig##A); \ + x_##B = _mm_add_epi32(x_##B, orig##B); \ + x_##C = _mm_add_epi32(x_##C, orig##C); \ + x_##D = _mm_add_epi32(x_##D, orig##D); \ + /* Transpose */ \ + t_##A = _mm_unpacklo_epi32(x_##A, x_##B); \ + t_##B = _mm_unpacklo_epi32(x_##C, x_##D); \ + t_##C = _mm_unpackhi_epi32(x_##A, x_##B); \ + t_##D = _mm_unpackhi_epi32(x_##C, x_##D); \ + x_##A = _mm_unpacklo_epi64(t_##A, t_##B); \ + x_##B = _mm_unpackhi_epi64(t_##A, t_##B); \ + x_##C = _mm_unpacklo_epi64(t_##C, t_##D); \ + x_##D = _mm_unpackhi_epi64(t_##C, t_##D); \ + \ + _mm_storeu_si128((__m128i *)(CT), x_##A); \ + _mm_storeu_si128((__m128i *)(CT + 64), x_##B); \ + _mm_storeu_si128((__m128i *)(CT + 128), x_##C); \ + _mm_storeu_si128((__m128i *)(CT + 192), x_##D); \ + } + +if (bytes >= 256) +{ + const __m256i vec_increment = _mm256_set_epi64x(3, 2, 1, 0); // 0, 1, 2, 3 for the increments + const __m256i interleave = _mm256_set_epi32(7, 5, 3, 1, 6, 4, 2, 0); // Indices for counters + + /* constant for shuffling bytes (replacing multiple-of-8 rotates) */ + const __m128i rot16 = + _mm_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + const __m128i rot8 = + _mm_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); + + // Load state to vectors, duplicate four times + __m128i x_0 = _mm_set1_epi32(x[0]); + __m128i x_1 = _mm_set1_epi32(x[1]); + __m128i x_2 = _mm_set1_epi32(x[2]); + __m128i x_3 = _mm_set1_epi32(x[3]); + __m128i x_4 = _mm_set1_epi32(x[4]); + __m128i x_5 = _mm_set1_epi32(x[5]); + __m128i x_6 = _mm_set1_epi32(x[6]); + __m128i x_7 = _mm_set1_epi32(x[7]); + __m128i x_8 = _mm_set1_epi32(x[8]); + __m128i x_9 = _mm_set1_epi32(x[9]); + __m128i x_10 = _mm_set1_epi32(x[10]); + __m128i x_11 = _mm_set1_epi32(x[11]); + __m128i x_12; + __m128i x_13; + __m128i x_14 = _mm_set1_epi32(x[14]); + __m128i x_15 = _mm_set1_epi32(x[15]); + __m128i orig0 = x_0; + __m128i orig1 = x_1; + __m128i orig2 = x_2; + __m128i orig3 = x_3; + __m128i orig4 = x_4; + __m128i orig5 = x_5; + __m128i orig6 = x_6; + __m128i orig7 = x_7; + __m128i orig8 = x_8; + __m128i orig9 = x_9; + __m128i orig10 = x_10; + __m128i orig11 = x_11; + __m128i orig12 = {}; + __m128i orig13 = {}; + __m128i orig14 = x_14; + __m128i orig15 = x_15; + __m128i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12, + t_13, t_14, t_15; + + while (bytes >= 256) + { + x_0 = orig0; + x_1 = orig1; + x_2 = orig2; + x_3 = orig3; + x_4 = orig4; + x_5 = orig5; + x_6 = orig6; + x_7 = orig7; + x_8 = orig8; + x_9 = orig9; + x_10 = orig10; + x_11 = orig11; + x_14 = orig14; + x_15 = orig15; + + // Calculate counter + 0..3 for adjacent blocks (x12 low and x13 high of each) + uint64_t *counter = (uint64_t *)&x[12]; + __m256i counters = _mm256_add_epi64(_mm256_set1_epi64x(*counter), vec_increment); + counters = _mm256_permutevar8x32_epi32(counters, interleave); + x_12 = _mm256_extracti128_si256(counters, 0); + x_13 = _mm256_extracti128_si256(counters, 1); + + for (int i = 0; i < 10; ++i) + { + // Mix columns + VEC4_QUARTERROUND(0, 4, 8, 12); + VEC4_QUARTERROUND(1, 5, 9, 13); + VEC4_QUARTERROUND(2, 6, 10, 14); + VEC4_QUARTERROUND(3, 7, 11, 15); + // Mix diagonals + VEC4_QUARTERROUND(0, 5, 10, 15); + VEC4_QUARTERROUND(1, 6, 11, 12); + VEC4_QUARTERROUND(2, 7, 8, 13); + VEC4_QUARTERROUND(3, 4, 9, 14); + } + + ONEQUAD(0, 1, 2, 3, c); + ONEQUAD(4, 5, 6, 7, c + 16); + ONEQUAD(8, 9, 10, 11, c + 32); + ONEQUAD(12, 13, 14, 15, c + 48); + + *counter += 4; + bytes -= 256; + c += 256; + } +} + +#undef ONEQUAD +#undef ONEQUAD_TRANSPOSE +#undef VEC4_ROT +#undef VEC4_QUARTERROUND +#undef VEC4_QUARTERROUND_SHUFFLE diff --git a/src/u8-stream.h b/src/u8-stream.h new file mode 100644 index 0000000..e1dc73f --- /dev/null +++ b/src/u8-stream.h @@ -0,0 +1,278 @@ + +#define VEC8_ROT(A, IMM) \ + _mm256_or_si256(_mm256_slli_epi32(A, IMM), _mm256_srli_epi32(A, (32 - IMM))) + +/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 & + * 16) (better) */ +#define VEC8_QUARTERROUND(A, B, C, D) \ + x_##A = _mm256_add_epi32(x_##A, x_##B); \ + t_##A = _mm256_xor_si256(x_##D, x_##A); \ + x_##D = _mm256_shuffle_epi8(t_##A, rot16); \ + x_##C = _mm256_add_epi32(x_##C, x_##D); \ + t_##C = _mm256_xor_si256(x_##B, x_##C); \ + x_##B = VEC8_ROT(t_##C, 12); \ + x_##A = _mm256_add_epi32(x_##A, x_##B); \ + t_##A = _mm256_xor_si256(x_##D, x_##A); \ + x_##D = _mm256_shuffle_epi8(t_##A, rot8); \ + x_##C = _mm256_add_epi32(x_##C, x_##D); \ + t_##C = _mm256_xor_si256(x_##B, x_##C); \ + x_##B = VEC8_ROT(t_##C, 7) + +#define VEC8_LINE1(A, B, C, D) \ + x_##A = _mm256_add_epi32(x_##A, x_##B); \ + x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot16) +#define VEC8_LINE2(A, B, C, D) \ + x_##C = _mm256_add_epi32(x_##C, x_##D); \ + x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 12) +#define VEC8_LINE3(A, B, C, D) \ + x_##A = _mm256_add_epi32(x_##A, x_##B); \ + x_##D = _mm256_shuffle_epi8(_mm256_xor_si256(x_##D, x_##A), rot8) +#define VEC8_LINE4(A, B, C, D) \ + x_##C = _mm256_add_epi32(x_##C, x_##D); \ + x_##B = VEC8_ROT(_mm256_xor_si256(x_##B, x_##C), 7) + +#define VEC8_ROUND_SEQ(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, \ + C4, D4) \ + VEC8_LINE1(A1, B1, C1, D1); \ + VEC8_LINE1(A2, B2, C2, D2); \ + VEC8_LINE1(A3, B3, C3, D3); \ + VEC8_LINE1(A4, B4, C4, D4); \ + VEC8_LINE2(A1, B1, C1, D1); \ + VEC8_LINE2(A2, B2, C2, D2); \ + VEC8_LINE2(A3, B3, C3, D3); \ + VEC8_LINE2(A4, B4, C4, D4); \ + VEC8_LINE3(A1, B1, C1, D1); \ + VEC8_LINE3(A2, B2, C2, D2); \ + VEC8_LINE3(A3, B3, C3, D3); \ + VEC8_LINE3(A4, B4, C4, D4); \ + VEC8_LINE4(A1, B1, C1, D1); \ + VEC8_LINE4(A2, B2, C2, D2); \ + VEC8_LINE4(A3, B3, C3, D3); \ + VEC8_LINE4(A4, B4, C4, D4) + +#define VEC8_ROUND_HALF(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, \ + B4, C4, D4) \ + VEC8_LINE1(A1, B1, C1, D1); \ + VEC8_LINE1(A2, B2, C2, D2); \ + VEC8_LINE2(A1, B1, C1, D1); \ + VEC8_LINE2(A2, B2, C2, D2); \ + VEC8_LINE3(A1, B1, C1, D1); \ + VEC8_LINE3(A2, B2, C2, D2); \ + VEC8_LINE4(A1, B1, C1, D1); \ + VEC8_LINE4(A2, B2, C2, D2); \ + VEC8_LINE1(A3, B3, C3, D3); \ + VEC8_LINE1(A4, B4, C4, D4); \ + VEC8_LINE2(A3, B3, C3, D3); \ + VEC8_LINE2(A4, B4, C4, D4); \ + VEC8_LINE3(A3, B3, C3, D3); \ + VEC8_LINE3(A4, B4, C4, D4); \ + VEC8_LINE4(A3, B3, C3, D3); \ + VEC8_LINE4(A4, B4, C4, D4) + +#define VEC8_ROUND_HALFANDHALF(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, \ + A4, B4, C4, D4) \ + VEC8_LINE1(A1, B1, C1, D1); \ + VEC8_LINE1(A2, B2, C2, D2); \ + VEC8_LINE2(A1, B1, C1, D1); \ + VEC8_LINE2(A2, B2, C2, D2); \ + VEC8_LINE1(A3, B3, C3, D3); \ + VEC8_LINE1(A4, B4, C4, D4); \ + VEC8_LINE2(A3, B3, C3, D3); \ + VEC8_LINE2(A4, B4, C4, D4); \ + VEC8_LINE3(A1, B1, C1, D1); \ + VEC8_LINE3(A2, B2, C2, D2); \ + VEC8_LINE4(A1, B1, C1, D1); \ + VEC8_LINE4(A2, B2, C2, D2); \ + VEC8_LINE3(A3, B3, C3, D3); \ + VEC8_LINE3(A4, B4, C4, D4); \ + VEC8_LINE4(A3, B3, C3, D3); \ + VEC8_LINE4(A4, B4, C4, D4) + +#define VEC8_ROUND(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, \ + D4) \ + VEC8_ROUND_SEQ(A1, B1, C1, D1, A2, B2, C2, D2, A3, B3, C3, D3, A4, B4, C4, \ + D4) + +if (bytes >= 512) +{ + /* constant for shuffling bytes (replacing multiple-of-8 rotates) */ + __m256i rot16 = + _mm256_set_epi8(13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, + 13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2); + __m256i rot8 = + _mm256_set_epi8(14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3, + 14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3); + + /* the naive way seems as fast (if not a bit faster) than the vector way */ + __m256i x_0 = _mm256_set1_epi32(x[0]); + __m256i x_1 = _mm256_set1_epi32(x[1]); + __m256i x_2 = _mm256_set1_epi32(x[2]); + __m256i x_3 = _mm256_set1_epi32(x[3]); + __m256i x_4 = _mm256_set1_epi32(x[4]); + __m256i x_5 = _mm256_set1_epi32(x[5]); + __m256i x_6 = _mm256_set1_epi32(x[6]); + __m256i x_7 = _mm256_set1_epi32(x[7]); + __m256i x_8 = _mm256_set1_epi32(x[8]); + __m256i x_9 = _mm256_set1_epi32(x[9]); + __m256i x_10 = _mm256_set1_epi32(x[10]); + __m256i x_11 = _mm256_set1_epi32(x[11]); + __m256i x_12; + __m256i x_13; + __m256i x_14 = _mm256_set1_epi32(x[14]); + __m256i x_15 = _mm256_set1_epi32(x[15]); + + __m256i orig0 = x_0; + __m256i orig1 = x_1; + __m256i orig2 = x_2; + __m256i orig3 = x_3; + __m256i orig4 = x_4; + __m256i orig5 = x_5; + __m256i orig6 = x_6; + __m256i orig7 = x_7; + __m256i orig8 = x_8; + __m256i orig9 = x_9; + __m256i orig10 = x_10; + __m256i orig11 = x_11; + __m256i orig12; + __m256i orig13; + __m256i orig14 = x_14; + __m256i orig15 = x_15; + __m256i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12, + t_13, t_14, t_15; + const __m256i addv12 = _mm256_set_epi64x(3, 2, 1, 0); + const __m256i addv13 = _mm256_set_epi64x(7, 6, 5, 4); + + while (bytes >= 512) + { + __m256i t12, t13; + + x_0 = orig0; + x_1 = orig1; + x_2 = orig2; + x_3 = orig3; + x_4 = orig4; + x_5 = orig5; + x_6 = orig6; + x_7 = orig7; + x_8 = orig8; + x_9 = orig9; + x_10 = orig10; + x_11 = orig11; + x_14 = orig14; + x_15 = orig15; + + // Calculate the eight parallel counters on x_12 and x_13 + uint64_t *counter = (uint64_t *)(x + 12); + t13 = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(*counter)); + t12 = _mm256_add_epi64(addv12, t13); + t13 = _mm256_add_epi64(addv13, t13); + x_12 = _mm256_unpacklo_epi32(t12, t13); + x_13 = _mm256_unpackhi_epi32(t12, t13); + t12 = _mm256_unpacklo_epi32(x_12, x_13); + t13 = _mm256_unpackhi_epi32(x_12, x_13); + + /* required because unpack* are intra-lane */ + const __m256i permute = _mm256_set_epi32(7, 6, 3, 2, 5, 4, 1, 0); + x_12 = _mm256_permutevar8x32_epi32(t12, permute); + x_13 = _mm256_permutevar8x32_epi32(t13, permute); + + orig12 = x_12; + orig13 = x_13; + + for (int i = 0; i < 10; ++i) + { + VEC8_ROUND(0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15); + VEC8_ROUND(0, 5, 10, 15, 1, 6, 11, 12, 2, 7, 8, 13, 3, 4, 9, 14); + } + +#define ONEQUAD_TRANSPOSE(A, B, C, D) \ + { \ + __m128i t0, t1, t2, t3; \ + x_##A = _mm256_add_epi32(x_##A, orig##A); \ + x_##B = _mm256_add_epi32(x_##B, orig##B); \ + x_##C = _mm256_add_epi32(x_##C, orig##C); \ + x_##D = _mm256_add_epi32(x_##D, orig##D); \ + t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \ + t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \ + t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \ + t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \ + x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \ + x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \ + x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \ + x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \ + _mm_storeu_si128((__m128i *)(c + 0), _mm256_extracti128_si256(x_##A, 0)); \ + _mm_storeu_si128((__m128i *)(c + 64), _mm256_extracti128_si256(x_##B, 0)); \ + _mm_storeu_si128((__m128i *)(c + 128), _mm256_extracti128_si256(x_##C, 0)); \ + _mm_storeu_si128((__m128i *)(c + 192), _mm256_extracti128_si256(x_##D, 0)); \ + _mm_storeu_si128((__m128i *)(c + 256), _mm256_extracti128_si256(x_##A, 1)); \ + _mm_storeu_si128((__m128i *)(c + 320), _mm256_extracti128_si256(x_##B, 1)); \ + _mm_storeu_si128((__m128i *)(c + 384), _mm256_extracti128_si256(x_##C, 1)); \ + _mm_storeu_si128((__m128i *)(c + 448), _mm256_extracti128_si256(x_##D, 1)); \ + } + +#define ONEQUAD(A, B, C, D) ONEQUAD_TRANSPOSE(A, B, C, D) + +#define ONEQUAD_UNPCK(A, B, C, D) \ + { \ + x_##A = _mm256_add_epi32(x_##A, orig##A); \ + x_##B = _mm256_add_epi32(x_##B, orig##B); \ + x_##C = _mm256_add_epi32(x_##C, orig##C); \ + x_##D = _mm256_add_epi32(x_##D, orig##D); \ + t_##A = _mm256_unpacklo_epi32(x_##A, x_##B); \ + t_##B = _mm256_unpacklo_epi32(x_##C, x_##D); \ + t_##C = _mm256_unpackhi_epi32(x_##A, x_##B); \ + t_##D = _mm256_unpackhi_epi32(x_##C, x_##D); \ + x_##A = _mm256_unpacklo_epi64(t_##A, t_##B); \ + x_##B = _mm256_unpackhi_epi64(t_##A, t_##B); \ + x_##C = _mm256_unpacklo_epi64(t_##C, t_##D); \ + x_##D = _mm256_unpackhi_epi64(t_##C, t_##D); \ + } + +#define ONEOCTO(A, B, C, D, A2, B2, C2, D2, c) \ + { \ + ONEQUAD_UNPCK(A, B, C, D); \ + ONEQUAD_UNPCK(A2, B2, C2, D2); \ + t_##A = _mm256_permute2x128_si256(x_##A, x_##A2, 0x20); \ + t_##A2 = _mm256_permute2x128_si256(x_##A, x_##A2, 0x31); \ + t_##B = _mm256_permute2x128_si256(x_##B, x_##B2, 0x20); \ + t_##B2 = _mm256_permute2x128_si256(x_##B, x_##B2, 0x31); \ + t_##C = _mm256_permute2x128_si256(x_##C, x_##C2, 0x20); \ + t_##C2 = _mm256_permute2x128_si256(x_##C, x_##C2, 0x31); \ + t_##D = _mm256_permute2x128_si256(x_##D, x_##D2, 0x20); \ + t_##D2 = _mm256_permute2x128_si256(x_##D, x_##D2, 0x31); \ + _mm256_storeu_si256((__m256i *)(c + 0), t_##A); \ + _mm256_storeu_si256((__m256i *)(c + 64), t_##B); \ + _mm256_storeu_si256((__m256i *)(c + 128), t_##C); \ + _mm256_storeu_si256((__m256i *)(c + 192), t_##D); \ + _mm256_storeu_si256((__m256i *)(c + 256), t_##A2); \ + _mm256_storeu_si256((__m256i *)(c + 320), t_##B2); \ + _mm256_storeu_si256((__m256i *)(c + 384), t_##C2); \ + _mm256_storeu_si256((__m256i *)(c + 448), t_##D2); \ + } + + ONEOCTO(0, 1, 2, 3, 4, 5, 6, 7, c); + ONEOCTO(8, 9, 10, 11, 12, 13, 14, 15, c + 32); + + *counter += 8; + bytes -= 512; + c += 512; + } +} + +#undef ONEQUAD +#undef ONEQUAD_TRANSPOSE +#undef ONEQUAD_UNPCK +#undef ONEOCTO +#undef VEC8_ROT +#undef VEC8_QUARTERROUND +#undef VEC8_QUARTERROUND_NAIVE +#undef VEC8_QUARTERROUND_SHUFFLE +#undef VEC8_QUARTERROUND_SHUFFLE2 +#undef VEC8_LINE1 +#undef VEC8_LINE2 +#undef VEC8_LINE3 +#undef VEC8_LINE4 +#undef VEC8_ROUND +#undef VEC8_ROUND_SEQ +#undef VEC8_ROUND_HALF +#undef VEC8_ROUND_HALFANDHALF