LeoVasanko/RandQuik
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed cha vectorized algorithms

Browse Source
This commit is contained in:
LeoVasanko
2023-10-28 16:22:51 +00:00
parent a6daac4ce6
commit ab67daf050
6 changed files with 34 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/c-stream.h
+1 -1
View File
@@ -44,5 +44,5 @@ _cha_block(uint32_t* state, uint8_t* begin, uint8_t* end) {
memcpy(c, x, 64);
c += 64;
}
return end - c;
return c - begin;
}
src/chacha20.c
+5 -3
View File
@@ -39,9 +39,11 @@ int cha_update(cha_ctx* ctx, uint8_t* out, uint64_t outlen) {
uint8_t* c = out;
uint8_t* end = out + outlen;
// TODO: Handle resume if we are not at block boundary
if (__builtin_cpu_supports("avx2")) {
c += _cha_8block(ctx->input, c, end);
assert(end - c < 512);
if (__builtin_cpu_supports("ssse3")) {
if (__builtin_cpu_supports("avx2")) {
c += _cha_8block(ctx->input, c, end);
assert(end - c < 512);
}
c += _cha_4block(ctx->input, c, end);
assert(end - c < 256);
}
src/randquik.c
-312
View File
@@ -1,312 +0,0 @@
#ifdef __GNUC__
#pragma GCC target("sse2")
#pragma GCC target("ssse3")
#pragma GCC target("avx2")
#endif
#include "randquik.h"
#include <errno.h>
#include <signal.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include <emmintrin.h>
#include <immintrin.h>
#include <tmmintrin.h>
#include <pthread.h>
#include <time.h>
#include <unistd.h>
#define BLOCK_SIZE (1 << 21) // 2 MiB seems optimal for speed
static volatile bool quit = false;
void signal_handler(int sig)
{
quit = true;
signal(SIGINT, SIG_DFL);
signal(SIGTERM, SIG_DFL);
}
static const unsigned char default_iv[16] = "\0\0\0\0\0\0\0\0RandQuik";
typedef struct thread_args
{
int index;
int done;
unsigned char *buf;
unsigned char key[32];
unsigned workers;
pthread_mutex_t lock;
pthread_cond_t cond;
pthread_t thread;
} thread_args;
void *producer_thread(void *a)
{
thread_args *args = (thread_args *)a;
const unsigned long long ivstep = args->workers * BLOCK_SIZE / 64;
while (!quit)
{
pthread_mutex_lock(&args->lock);
while (args->done)
{
pthread_cond_wait(&args->cond, &args->lock);
}
unsigned char iv[16];
memcpy(iv, default_iv, 16);
*(uint64_t *)iv += args->index * ivstep; // Counter increment
chacha20_stream(args->buf, BLOCK_SIZE, args->key, default_iv);
args->done = 1;
pthread_cond_signal(&args->cond);
pthread_mutex_unlock(&args->lock);
}
return NULL;
}
void print_status(unsigned long long bytes, unsigned long long max_bytes, struct timespec start_time)
{
struct timespec end_time;
clock_gettime(CLOCK_MONOTONIC, &end_time);
double t = (end_time.tv_sec - start_time.tv_sec) + 1e-9 * (end_time.tv_nsec - start_time.tv_nsec);
char buf[64] = {};
double speed = bytes / t;
char const *unit = "MB";
double m = 1e-6;
if (speed > 0.5e9)
{
unit = "GB";
m = 1e-9;
}
if (max_bytes)
{
snprintf(buf, sizeof buf - 1, " of %'.0lf", m * max_bytes);
}
fprintf(stderr, "\r%5.0lf%s %s written, %.2lf %s/s.\e[K", m * bytes, buf, unit, m * speed, unit);
}
int fast(FILE *f, unsigned workers, unsigned long long max_bytes, unsigned char const key[32], unsigned char const iv[16])
{
thread_args args[workers];
memset(args, 0, sizeof args);
for (int i = 0; i < workers; ++i)
{
args[i].index = i;
args[i].buf = malloc(BLOCK_SIZE);
args[i].workers = workers;
memcpy(args[i].key, key, 32);
pthread_mutex_init(&args[i].lock, NULL);
pthread_cond_init(&args[i].cond, NULL);
pthread_create(&args[i].thread, NULL, producer_thread, &args[i]);
}
struct timespec start_time;
clock_gettime(CLOCK_MONOTONIC, &start_time);
int i = -1;
unsigned long long bytes = 0;
while (!quit)
{
i = (i + 1) % workers;
pthread_mutex_lock(&args[i].lock);
while (!args[i].done)
{
pthread_cond_wait(&args[i].cond, &args[i].lock);
}
if (bytes % (1 << 30) == 0 || bytes + BLOCK_SIZE >= max_bytes)
{
print_status(bytes, max_bytes, start_time);
}
unsigned long long sz = BLOCK_SIZE;
if (max_bytes && bytes + sz >= max_bytes)
{
fprintf(stderr, "\r\e[KMax reached\n");
sz = max_bytes - bytes;
quit = true;
}
if (fwrite(args[i].buf, sz, 1, f) != 1)
{
quit = true;
fprintf(stderr, "\r\e[KWrite failed: %s\n", strerror(errno));
}
bytes += sz;
args[i].done = 0;
pthread_cond_signal(&args[i].cond);
pthread_mutex_unlock(&args[i].lock);
}
print_status(bytes, max_bytes, start_time);
for (int i = 0; i < workers; ++i)
{
args[i].done = 0;
pthread_cancel(args[i].thread);
pthread_join(args[i].thread, NULL);
pthread_mutex_destroy(&args[i].lock);
pthread_cond_destroy(&args[i].cond);
free(args[i].buf);
}
fprintf(stderr, "\nRandQuik wrote %llu bytes!\n\n", bytes);
return 0;
}
bool parse_hex(char *str, unsigned char *buf, size_t len)
{
for (size_t i = 0; i < len; ++i)
{
int sz = 0;
if (sscanf(str, "%2hhx%n", buf + i, &sz) != 1)
{
if (*str)
{
fprintf(stderr, "Unable to read seed at `%s`\n\n", str);
return false;
}
return true; // Shorter than key length is OK
}
str += sz;
}
return true;
}
void print_hex(unsigned char *buf, size_t len)
{
for (size_t i = 0; i < len; ++i)
{
fprintf(stderr, "%02hhx", buf[i]);
}
}
void help(char **argv)
{
fprintf(stderr, "Usage: %s [-t #threads] [-s hexseed] [-b #bytes] [-o outputfile]\n\n", argv[0]);
}
int main(int argc, char **argv)
{
unsigned char key[32] = {};
unsigned char iv[16] = {};
unsigned int workers = 8;
char *output = NULL;
unsigned long long max_bytes = 0;
bool seeded = false;
for (char opt; (opt = getopt(argc, argv, "bost")) != -1;)
{
if (opt == 't')
{
if (optind >= argc || sscanf(argv[optind++], "%u", &workers) != 1)
{
fprintf(stderr, "Expected the number of worker threads after -t\n");
exit(EXIT_FAILURE);
}
continue;
}
if (opt == 's')
{
if (optind >= argc || !parse_hex(argv[optind++], key, 32))
{
fprintf(stderr, "Expected a hex seed string after -s\n");
exit(EXIT_FAILURE);
}
seeded = true;
continue;
}
if (opt == 'o')
{
if (optind >= argc)
{
fprintf(stderr, "Expected output filename after -s\n");
return 1;
}
if (strcmp(argv[optind], "-") != 0)
{
output = argv[optind++];
}
continue;
}
if (opt == 'b')
{
if (optind >= argc || sscanf(argv[optind++], "%llu", &max_bytes) != 1)
{
fprintf(stderr, "Expected a maximum number of bytes to read after -b\n");
exit(EXIT_FAILURE);
}
continue;
}
help(argv);
return 1;
}
FILE *f = stdout;
if (output)
{
f = fopen(output, "wb");
if (!f)
{
fprintf(stderr, "Failed to open %s for writing.\n", output);
return 1;
}
}
else if (isatty(1))
{
fprintf(stderr, "Won't print random on console. Pipe me to another program or file instead.\n\n");
help(argv);
return 1;
}
if (!seeded)
{
FILE *urand = fopen("/dev/urandom", "rb");
if (!urand || fread(key, 32, 1, urand) != 1)
{
fprintf(stderr, "Failed to seed from /dev/urandom. Use -s hexstring for manual seeding.\n");
fclose(urand);
return 1;
}
fclose(urand);
fprintf(stderr, "Random seed generated. This sequence may be repeated by:\n%s -s ", argv[0]);
print_hex(key, 32);
fprintf(stderr, "\n\n");
}
signal(SIGINT, signal_handler);
signal(SIGTERM, signal_handler);
int ret = fast(f, workers, max_bytes, key, iv);
fclose(f);
return ret;
}
typedef struct chacha_ctx
{
uint32_t input[16];
} chacha_ctx;
static void
chacha_init(chacha_ctx *ctx, const uint8_t *k, const uint8_t *iv)
{
ctx->input[0] = 0x61707865;
ctx->input[1] = 0x3320646e;
ctx->input[2] = 0x79622d32;
ctx->input[3] = 0x6b206574;
memcpy(ctx->input + 4, k, 32);
memcpy(ctx->input + 12, iv, 16);
}
int chacha20_stream(unsigned char *out, unsigned long long outlen,
const unsigned char key[32], const unsigned char iv[16])
{
chacha_ctx ctx;
chacha_init(&ctx, key, iv);
{
// The included header will mess with these variables
unsigned long long bytes = outlen;
uint32_t *x = ctx.input;
unsigned char *c = out;
if (__builtin_cpu_supports("avx2"))
{
#include "u8-stream.h"
#include "u4-stream.h"
}
#include "c-stream.h"
}
memset(&ctx, 0, sizeof ctx);
return 0;
}
src/randquik.h
-2
View File
@@ -1,2 +0,0 @@
int chacha20_stream(unsigned char *out, unsigned long long outlen,
const unsigned char key[32], const unsigned char iv[16]);
src/u4-stream.h
+24 -8
View File
@@ -45,7 +45,10 @@ static inline uint64_t
_cha_4block(uint32_t* state, uint8_t* begin, uint8_t* end) {
if (end - begin < 256)
return 0;
uint8_t* c = begin;
uint32_t* x = state;
uint64_t* counter = (uint64_t*)&state[12]; // low u32 in 12, high u32 in 13
const __m256i vec_increment =
_mm256_set_epi64x(3, 2, 1, 0); // 0, 1, 2, 3 for the increments
const __m256i interleave =
@@ -93,8 +96,9 @@ _cha_4block(uint32_t* state, uint8_t* begin, uint8_t* end) {
__m128i t_0, t_1, t_2, t_3, t_4, t_5, t_6, t_7, t_8, t_9, t_10, t_11, t_12,
t_13, t_14, t_15;
uint8_t* c = begin;
uint64_t* counter = (uint64_t*)&x[12]; // low u32 in 12, high u32 in 13
const __m128i addv12 = _mm_set_epi64x(1, 0);
const __m128i addv13 = _mm_set_epi64x(3, 2);
while (end - c >= 256) {
x_0 = orig0;
x_1 = orig1;
@@ -113,11 +117,23 @@ _cha_4block(uint32_t* state, uint8_t* begin, uint8_t* end) {
// Calculate counter + 0..3 for adjacent blocks (x12 low and x13
// high of each)
__m256i counters =
_mm256_add_epi64(_mm256_set1_epi64x(*counter), vec_increment);
counters = _mm256_permutevar8x32_epi32(counters, interleave);
x_12 = _mm256_extracti128_si256(counters, 0);
x_13 = _mm256_extracti128_si256(counters, 1);
uint32_t in12 = state[12];
uint32_t in13 = state[13];
uint64_t in1213 = ((uint64_t)in12) | (((uint64_t)in13) << 32);
__m128i t12, t13;
t12 = _mm_set1_epi64x(in1213);
t13 = _mm_set1_epi64x(in1213);
x_12 = _mm_add_epi64(addv12, t12);
x_13 = _mm_add_epi64(addv13, t13);
t12 = _mm_unpacklo_epi32(x_12, x_13);
t13 = _mm_unpackhi_epi32(x_12, x_13);
x_12 = _mm_unpacklo_epi32(t12, t13);
x_13 = _mm_unpackhi_epi32(t12, t13);
orig12 = x_12;
orig13 = x_13;
in1213 += 4;
state[12] = in1213 & 0xFFFFFFFF;
state[13] = (in1213 >> 32) & 0xFFFFFFFF;
for (int i = 0; i < 10; ++i) {
// Mix columns
@@ -137,7 +153,7 @@ _cha_4block(uint32_t* state, uint8_t* begin, uint8_t* end) {
ONEQUAD(8, 9, 10, 11, c + 32);
ONEQUAD(12, 13, 14, 15, c + 48);
*counter += 4;
// *counter += 4;
c += 256;
}
return c - begin; // Bytes written
src/u8-stream.h
+4 -3
View File
@@ -169,7 +169,7 @@
t[C2] = _mm256_permute2x128_si256(x[C], x[C2], 0x31); \
t[D] = _mm256_permute2x128_si256(x[D], x[D2], 0x20); \
t[D2] = _mm256_permute2x128_si256(x[D], x[D2], 0x31); \
_mm256_storeu_si256((__m256i*)(c + 0), t[A]); \
_mm256_storeu_si256((__m256i*)(c), t[A]); \
_mm256_storeu_si256((__m256i*)(c + 64), t[B]); \
_mm256_storeu_si256((__m256i*)(c + 128), t[C]); \
_mm256_storeu_si256((__m256i*)(c + 192), t[D]); \
@@ -185,6 +185,7 @@ _cha_8block(uint32_t* state, uint8_t* begin, uint8_t* end) {
return 0;
uint8_t* c = begin;
uint64_t* counter = (uint64_t*)(state + 12);
/* constant for shuffling bytes (replacing multiple-of-8 rotates) */
__m256i rot16 = _mm256_set_epi8(
13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2, 13, 12, 15, 14, 9,
@@ -205,10 +206,10 @@ _cha_8block(uint32_t* state, uint8_t* begin, uint8_t* end) {
while (end - c >= 512) {
for (int i = 0; i < 16; ++i)
x[i] = orig[i];
if (i != 12 && i != 13)
x[i] = orig[i];
// Calculate the eight parallel counters on x_12 and x_13
uint64_t* counter = (uint64_t*)(x + 12);
t[13] = _mm256_broadcastq_epi64(_mm_cvtsi64_si128(*counter));
t[12] = _mm256_add_epi64(addv12, t[13]);
t[13] = _mm256_add_epi64(addv13, t[13]);