Native Apple Silicon SIMD code, cleanup.

2023-11-29 19:07:44 +00:00
parent e081d19b2f
commit ed5ba3379f
8 changed files with 112 additions and 9235 deletions
@@ -0,0 +1,101 @@
+#include <arm_neon.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+// clang-format off
+
+#define VEC4_ROT(A, IMM)                                                       \
+    vreinterpretq_u32_u8(vorrq_u8(vshlq_n_u32(A, IMM), vshrq_n_u32(A, 32 - IMM)))
+
+
+/* same, but replace 2 of the shift/shift/or "rotation" by byte shuffles (8 &
+ * 16) (better) */
+#define VEC4_QUARTERROUND(A, B, C, D)                                          \
+    x[A] = vaddq_u32(x[A], x[B]);                                          \
+    x[D] = vqtbl1q_u8(veorq_u32(x[D], x[A]), rot16);                 \
+    x[C] = vaddq_u32(x[C], x[D]);                                          \
+    x[B] = VEC4_ROT(veorq_u32(x[B], x[C]), 12);                            \
+    x[A] = vaddq_u32(x[A], x[B]);                                          \
+    x[D] = vqtbl1q_u8(veorq_u32(x[D], x[A]), rot8);                  \
+    x[C] = vaddq_u32(x[C], x[D]);                                          \
+    x[B] = VEC4_ROT(veorq_u32(x[B], x[C]), 7)
+
+#define ONEQUAD(A, B, C, D, OUT)                                                \
+    {                                                                          \
+        /* Add original block */                                               \
+        x[A] = vaddq_u32(x[A], orig[A]);                                   \
+        x[B] = vaddq_u32(x[B], orig[B]);                                   \
+        x[C] = vaddq_u32(x[C], orig[C]);                                   \
+        x[D] = vaddq_u32(x[D], orig[D]);                                   \
+        /* Transpose */                                                        \
+        uint32x4x2_t ab = vtrnq_u32(x[A], x[B]); \
+        uint32x4x2_t cd = vtrnq_u32(x[C], x[D]); \
+        x[A] = vcombine_u32(vget_low_u32(ab.val[0]), vget_low_u32(cd.val[0])); \
+        x[B] = vcombine_u32(vget_high_u32(ab.val[0]), vget_high_u32(cd.val[0])); \
+        x[C] = vcombine_u32(vget_low_u32(ab.val[1]), vget_low_u32(cd.val[1])); \
+        x[D] = vcombine_u32(vget_high_u32(ab.val[1]), vget_high_u32(cd.val[1])); \
+        /* Write out 1/4 of each block */                                      \
+        vst1q_u32((uint32_t*)(OUT), x[A]);                               \
+        vst1q_u32((uint32_t*)(OUT + 64), x[B]);                          \
+        vst1q_u32((uint32_t*)(OUT + 128), x[C]);                         \
+        vst1q_u32((uint32_t*)(OUT + 192), x[D]);                         \
+    }
+
+#define COUNTER_INCREMENT(addv)                                          \
+    {                                                                     \
+        orig[12] = vaddq_u32(orig[12], addv);                              \
+        orig[13] = vaddq_u32(orig[13], vshrq_n_u32(vcltq_u32(orig[12], addv), 31)); \
+    }
+
+static inline uint64_t
+_cha_4block(uint8_t* buf, size_t bufsize, uint32_t state[16], unsigned rounds) {
+    /* constant for shuffling bytes (replacing multiple-of-8 rotates) */
+    static const uint8_t _rot16a[] = {13, 12, 15, 14, 9, 8, 11, 10, 5, 4, 7, 6, 1, 0, 3, 2};
+    static const uint8_t _rot8a[] = {14, 13, 12, 15, 10, 9, 8, 11, 6, 5, 4, 7, 2, 1, 0, 3};
+    const uint8x16_t rot16 = vld1q_u8(_rot16a);
+    const uint8x16_t rot8 = vld1q_u8(_rot8a);
+    // Load state to vectors, duplicate four times, only different counters
+    uint32x4_t orig[16];
+    for (unsigned i = 0; i < 16; ++i) orig[i] = vdupq_n_u32(state[i]);
+    uint32x4_t addv = vdupq_n_u32(0);
+    addv = vsetq_lane_u32(1, addv, 1);
+    addv = vsetq_lane_u32(2, addv, 2);
+    addv = vsetq_lane_u32(3, addv, 3);
+    COUNTER_INCREMENT(addv);
+    addv = vdupq_n_u32(4);
+    const unsigned batches = bufsize / 256;
+    for (unsigned b = batches; b-->0;) {
+        uint32x4_t x[16];
+        for (unsigned i = 0; i < 16; ++i) x[i] = orig[i];
+        for (unsigned r = rounds / 2; r-->0;) {
+            // Mix columns
+            VEC4_QUARTERROUND(0, 4, 8, 12);
+            VEC4_QUARTERROUND(1, 5, 9, 13);
+            VEC4_QUARTERROUND(2, 6, 10, 14);
+            VEC4_QUARTERROUND(3, 7, 11, 15);
+            // Mix diagonals
+            VEC4_QUARTERROUND(0, 5, 10, 15);
+            VEC4_QUARTERROUND(1, 6, 11, 12);
+            VEC4_QUARTERROUND(2, 7, 8, 13);
+            VEC4_QUARTERROUND(3, 4, 9, 14);
+        }
+        // Add original block, unpack output
+        ONEQUAD(0, 1, 2, 3, buf);
+        ONEQUAD(4, 5, 6, 7, buf + 16);
+        ONEQUAD(8, 9, 10, 11, buf + 32);
+        ONEQUAD(12, 13, 14, 15, buf + 48);
+        COUNTER_INCREMENT(addv);
+        buf += 256;
+    }
+    // Store counter
+    state[12] = vgetq_lane_u32(orig[12], 0);
+    state[13] = vgetq_lane_u32(orig[13], 0);
+    return batches * 256;
+}
+
+#undef COUNTER_INCREMENT
+#undef ONEQUAD
+#undef ONEQUAD_TRANSPOSE
+#undef VEC4_ROT
+#undef VEC4_QUARTERROUND
+#undef VEC4_QUARTERROUND_SHUFFLE