Native Apple Silicon SIMD code, cleanup.

src/cha4neon.h

@@ -0,0 +1,101 @@
+#include <arm_neon.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+// clang-format off
+
+#define VEC4_ROT(A, IMM)                                                       \
+    vreinterpretq_u32_u8(vorrq_u8(vshlq_n_u32(A, IMM), vshrq_n_u32(A, 32 - IMM)))
+
+
+/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 &
+ * 16) (better) */
+#define VEC4_QUARTERROUND(A, B, C, D)                                          \
+    x[A] = vaddq_u32(x[A], x[B]);                                          \
+    x[D] = vqtbl1q_u8(veorq_u32(x[D], x[A]), rot16);                 \
+    x[C] = vaddq_u32(x[C], x[D]);                                          \
+    x[B] = VEC4_ROT(veorq_u32(x[B], x[C]), 12);                            \
+    x[A] = vaddq_u32(x[A], x[B]);                                          \
+    x[D] = vqtbl1q_u8(veorq_u32(x[D], x[A]), rot8);                  \
+    x[C] = vaddq_u32(x[C], x[D]);                                          \
+    x[B] = VEC4_ROT(veorq_u32(x[B], x[C]), 7)
+
+#define ONEQUAD(A, B, C, D, OUT)                                                \
+    {                                                                          \
+        /* Add original block */                                               \
+        x[A] = vaddq_u32(x[A], orig[A]);                                   \
+        x[B] = vaddq_u32(x[B], orig[B]);                                   \
+        x[C] = vaddq_u32(x[C], orig[C]);                                   \
+        x[D] = vaddq_u32(x[D], orig[D]);                                   \
+        /* Transpose */                                                        \
+        uint32x4x2_t ab = vtrnq_u32(x[A], x[B]); \
+        uint32x4x2_t cd = vtrnq_u32(x[C], x[D]); \
+        x[A] = vcombine_u32(vget_low_u32(ab.val[0]), vget_low_u32(cd.val[0])); \
+        x[B] = vcombine_u32(vget_high_u32(ab.val[0]), vget_high_u32(cd.val[0])); \
+        x[C] = vcombine_u32(vget_low_u32(ab.val[1]), vget_low_u32(cd.val[1])); \
+        x[D] = vcombine_u32(vget_high_u32(ab.val[1]), vget_high_u32(cd.val[1])); \
+        /* Write out 1/4 of each block */                                      \
+        vst1q_u32((uint32_t*)(OUT), x[A]);                               \
+        vst1q_u32((uint32_t*)(OUT + 64), x[B]);                          \
+        vst1q_u32((uint32_t*)(OUT + 128), x[C]);                         \
+        vst1q_u32((uint32_t*)(OUT + 192), x[D]);                         \
+    }
+
+#define COUNTER_INCREMENT(addv)                                          \
+    {                                                                     \
+        orig[12] = vaddq_u32(orig[12], addv);                              \
+        orig[13] = vaddq_u32(orig[13], vshrq_n_u32(vcltq_u32(orig[12], addv), 31)); \
+    }
+
+static inline uint64_t
+_cha_4block(uint8_t* buf, size_t bufsize, uint32_t state[16], unsigned rounds) {
+    /* constant for shuffling bytes (replacing multiple-of-8 rotates) */
+    static const uint8_t _rot16a[] = {13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2};
+    static const uint8_t _rot8a[] = {14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3};
+    const uint8x16_t rot16 = vld1q_u8(_rot16a);
+    const uint8x16_t rot8 = vld1q_u8(_rot8a);
+    // Load state to vectors, duplicate four times, only different counters
+    uint32x4_t orig[16];
+    for (unsigned i = 0; i < 16; ++i) orig[i] = vdupq_n_u32(state[i]);
+    uint32x4_t addv = vdupq_n_u32(0);
+    addv = vsetq_lane_u32(1, addv, 1);
+    addv = vsetq_lane_u32(2, addv, 2);
+    addv = vsetq_lane_u32(3, addv, 3);
+    COUNTER_INCREMENT(addv);
+    addv = vdupq_n_u32(4);
+    const unsigned batches = bufsize / 256;
+    for (unsigned b = batches; b-->0;) {
+        uint32x4_t x[16];
+        for (unsigned i = 0; i < 16; ++i) x[i] = orig[i];
+        for (unsigned r = rounds / 2; r-->0;) {
+            // Mix columns
+            VEC4_QUARTERROUND(0, 4, 8, 12);
+            VEC4_QUARTERROUND(1, 5, 9, 13);
+            VEC4_QUARTERROUND(2, 6, 10, 14);
+            VEC4_QUARTERROUND(3, 7, 11, 15);
+            // Mix diagonals
+            VEC4_QUARTERROUND(0, 5, 10, 15);
+            VEC4_QUARTERROUND(1, 6, 11, 12);
+            VEC4_QUARTERROUND(2, 7, 8, 13);
+            VEC4_QUARTERROUND(3, 4, 9, 14);
+        }
+        // Add original block, unpack output
+        ONEQUAD(0, 1, 2, 3, buf);
+        ONEQUAD(4, 5, 6, 7, buf + 16);
+        ONEQUAD(8, 9, 10, 11, buf + 32);
+        ONEQUAD(12, 13, 14, 15, buf + 48);
+        COUNTER_INCREMENT(addv);
+        buf += 256;
+    }
+    // Store counter
+    state[12] = vgetq_lane_u32(orig[12], 0);
+    state[13] = vgetq_lane_u32(orig[13], 0);
+    return batches * 256;
+}
+
+#undef COUNTER_INCREMENT
+#undef ONEQUAD
+#undef ONEQUAD_TRANSPOSE
+#undef VEC4_ROT
+#undef VEC4_QUARTERROUND
+#undef VEC4_QUARTERROUND_SHUFFLE

src/chacha20.h

@@ -11,13 +11,13 @@
 #pragma GCC target("ssse3")
 #pragma GCC target("avx2")
 #endif
-#include "cha4block.h"
+#include "cha4ssse3.h"
-#include "cha8block.h"
+#include "cha8avx2.h"
 #elif defined(__aarch64__)
-#include "cha4block.h"
+#include "cha4neon.h"
 #endif
 
-#include "cha1block.h"
+#include "cha1c.h"
 
 #include <assert.h>
 #include <stdbool.h>

src/cli.c

@@ -136,7 +136,7 @@ int fast(
         pthread_cond_destroy(&args[i].cond);
         free(args[i].buf);
     }
-    fprintf(stderr, "\nRandQuik wrote %lu bytes!\n\n", bytes);
+    fprintf(stderr, "\nRandQuik wrote %llu bytes!\n\n", bytes);
     return 0;
 }
 
@@ -164,7 +164,7 @@ void print_hex(unsigned char* buf, size_t len) {
 void help(char** argv) {
     fprintf(
       stderr,
-      "Usage: %s [-t #threads] [-s hexseed] [-b #bytes] [-c #rounds] [-o "
+      "Usage: %s [-t #threads] [-s hexseed] [-b #bytes] [-r #rounds] [-o "
       "outputfile]\n\n",
       argv[0]
     );
@@ -178,7 +178,7 @@ int main(int argc, char** argv) {
     char* output = NULL;
     uint64_t max_bytes = 0;
     bool seeded = false;
-    for (char opt; (opt = getopt(argc, argv, "bostc")) != -1;) {
+    for (char opt; (opt = getopt(argc, argv, "bostr")) != -1;) {
         if (opt == 't') {
             if (optind >= argc || sscanf(argv[optind++], "%u", &workers) != 1) {
                 fprintf(
@@ -188,11 +188,11 @@ int main(int argc, char** argv) {
             }
             continue;
         }
-        if (opt == 'c') {
+        if (opt == 'r') {
             if (optind >= argc || sscanf(argv[optind++], "%u", &rounds) != 1) {
                 fprintf(
                   stderr,
-                  "Expected the number ChaCha rounds (8, 12 or 20) after -c\n"
+                  "Expected the number ChaCha rounds (8, 12 or 20) after -r\n"
                 );
                 return 1;
             }
@@ -217,7 +217,7 @@ int main(int argc, char** argv) {
             continue;
         }
         if (opt == 'b') {
-            if (optind >= argc || sscanf(argv[optind++], "%lu", &max_bytes) != 1) {
+            if (optind >= argc || sscanf(argv[optind++], "%llu", &max_bytes) != 1) {
                 fprintf(
                   stderr,
                   "Expected a maximum number of bytes to read after -b\n"

src/npbitgen.h

@@ -7,7 +7,7 @@ static uint64_t cha_uint64(void* st) {
         ctx->end =
           ctx->gen(ctx->unconsumed, BATCH_SIZE, ctx->state, ctx->rounds);
     }
-    register uint64_t ret = *(uint64_t*)(ctx->unconsumed + ctx->offset);
+    uint64_t ret = *(uint64_t*)(ctx->unconsumed + ctx->offset);
     ctx->offset += sizeof(uint64_t);
     return ret;
 }