1#include <stddef.h>
 2#include <arm_sve.h>
 3#include "library.h"
 4#include "rpo_hash.h"
 5
 6// The STATE_WIDTH of RPO hash is 12x u64 elements.
 7// The current generation of SVE-enabled processors - Neoverse V1
 8// (e.g. AWS Graviton3) have 256-bit vector registers (4x u64)
 9// This allows us to split the state into 3 vectors of 4 elements
10// and process all 3 independent of each other.
11
12// We see the biggest performance gains by leveraging both
13// vector and scalar operations on parts of the state array.
14// Due to high latency of vector operations, the processor is able
15// to reorder and pipeline scalar instructions while we wait for
16// vector results. This effectively gives us some 'free' scalar
17// operations and masks vector latency.
18//
19// This also means that we can fully saturate all four arithmetic
20// units of the processor (2x scalar, 2x SIMD)
21//
22// THIS ANALYSIS NEEDS TO BE PERFORMED AGAIN ONCE PROCESSORS
23// GAIN WIDER REGISTERS. It's quite possible that with 8x u64
24// vectors processing 2 partially filled vectors might
25// be easier and faster than dealing with scalar operations
26// on the remainder of the array.
27//
28// FOR NOW THIS IS ONLY ENABLED ON 4x u64 VECTORS! It falls back
29// to the regular, already highly-optimized scalar version
30// if the conditions are not met.
31
32bool add_constants_and_apply_sbox(uint64_t state[STATE_WIDTH], uint64_t constants[STATE_WIDTH]) {
33    const uint64_t vl = svcntd();   // number of u64 numbers in one SVE vector
34
35    if (vl != 4) {
36        return false;
37    }
38
39    svbool_t ptrue = svptrue_b64();
40
41    svuint64_t state1 = svld1(ptrue, state + 0*vl);
42    svuint64_t state2 = svld1(ptrue, state + 1*vl);
43
44    svuint64_t const1 = svld1(ptrue, constants + 0*vl);
45    svuint64_t const2 = svld1(ptrue, constants + 1*vl);
46
47    add_constants(ptrue, &state1, &const1, &state2, &const2, state+8, constants+8);
48    apply_sbox(ptrue, &state1, &state2, state+8);
49
50    svst1(ptrue, state + 0*vl, state1);
51    svst1(ptrue, state + 1*vl, state2);
52
53    return true;
54}
55
56bool add_constants_and_apply_inv_sbox(uint64_t state[STATE_WIDTH], uint64_t constants[STATE_WIDTH]) {
57    const uint64_t vl = svcntd();   // number of u64 numbers in one SVE vector
58
59    if (vl != 4) {
60        return false;
61    }
62
63    svbool_t ptrue = svptrue_b64();
64
65    svuint64_t state1 = svld1(ptrue, state + 0 * vl);
66    svuint64_t state2 = svld1(ptrue, state + 1 * vl);
67
68    svuint64_t const1 = svld1(ptrue, constants + 0 * vl);
69    svuint64_t const2 = svld1(ptrue, constants + 1 * vl);
70
71    add_constants(ptrue, &state1, &const1, &state2, &const2, state + 8, constants + 8);
72    apply_inv_sbox(ptrue, &state1, &state2, state + 8);
73
74    svst1(ptrue, state + 0 * vl, state1);
75    svst1(ptrue, state + 1 * vl, state2);
76
77    return true;
78}