1#ifndef RPO_SVE_RPO_HASH_H
  2#define RPO_SVE_RPO_HASH_H
  3
  4#include <arm_sve.h>
  5#include <stddef.h>
  6#include <stdint.h>
  7#include <string.h>
  8
  9#define COPY(NAME, VIN1, VIN2, SIN3)                \
 10    svuint64_t NAME ## _1 = VIN1;                   \
 11    svuint64_t NAME ## _2 = VIN2;                   \
 12    uint64_t NAME ## _3[4];                         \
 13    memcpy(NAME ## _3, SIN3, 4 * sizeof(uint64_t))
 14
 15#define MULTIPLY(PRED, DEST, OP)                    \
 16    mul(PRED, &DEST ## _1, &OP ## _1, &DEST ## _2, &OP ## _2, DEST ## _3, OP ## _3)
 17
 18#define SQUARE(PRED, NAME)                          \
 19    sq(PRED, &NAME ## _1, &NAME ## _2, NAME ## _3)
 20
 21#define SQUARE_DEST(PRED, DEST, SRC)                \
 22    COPY(DEST, SRC ## _1, SRC ## _2, SRC ## _3);    \
 23    SQUARE(PRED, DEST);
 24
 25#define POW_ACC(PRED, NAME, CNT, TAIL)              \
 26    for (size_t i = 0; i < CNT; i++) {              \
 27        SQUARE(PRED, NAME);                         \
 28    }                                               \
 29    MULTIPLY(PRED, NAME, TAIL);
 30
 31#define POW_ACC_DEST(PRED, DEST, CNT, HEAD, TAIL)   \
 32    COPY(DEST, HEAD ## _1, HEAD ## _2, HEAD ## _3); \
 33    POW_ACC(PRED, DEST, CNT, TAIL)
 34
 35extern inline void add_constants(
 36    svbool_t pg,
 37    svuint64_t *state1,
 38    svuint64_t *const1,
 39    svuint64_t *state2,
 40    svuint64_t *const2,
 41    uint64_t *state3,
 42    uint64_t *const3
 43) {
 44    uint64_t Ms = 0xFFFFFFFF00000001ull;
 45    svuint64_t Mv = svindex_u64(Ms, 0);
 46
 47    uint64_t p_1 = Ms - const3[0];
 48    uint64_t p_2 = Ms - const3[1];
 49    uint64_t p_3 = Ms - const3[2];
 50    uint64_t p_4 = Ms - const3[3];
 51
 52    uint64_t x_1, x_2, x_3, x_4;
 53    uint32_t adj_1 = -__builtin_sub_overflow(state3[0], p_1, &x_1);
 54    uint32_t adj_2 = -__builtin_sub_overflow(state3[1], p_2, &x_2);
 55    uint32_t adj_3 = -__builtin_sub_overflow(state3[2], p_3, &x_3);
 56    uint32_t adj_4 = -__builtin_sub_overflow(state3[3], p_4, &x_4);
 57
 58    state3[0] = x_1 - (uint64_t)adj_1;
 59    state3[1] = x_2 - (uint64_t)adj_2;
 60    state3[2] = x_3 - (uint64_t)adj_3;
 61    state3[3] = x_4 - (uint64_t)adj_4;
 62
 63    svuint64_t p1 = svsub_x(pg, Mv, *const1);
 64    svuint64_t p2 = svsub_x(pg, Mv, *const2);
 65
 66    svuint64_t x1 = svsub_x(pg, *state1, p1);
 67    svuint64_t x2 = svsub_x(pg, *state2, p2);
 68
 69    svbool_t pt1 = svcmplt_u64(pg, *state1, p1);
 70    svbool_t pt2 = svcmplt_u64(pg, *state2, p2);
 71
 72    *state1 = svsub_m(pt1, x1, (uint32_t)-1);
 73    *state2 = svsub_m(pt2, x2, (uint32_t)-1);
 74}
 75
 76extern inline void mul(
 77    svbool_t pg,
 78    svuint64_t *r1,
 79    const svuint64_t *op1,
 80    svuint64_t *r2,
 81    const svuint64_t *op2,
 82    uint64_t *r3,
 83    const uint64_t *op3
 84) {
 85    __uint128_t x_1 = r3[0];
 86    __uint128_t x_2 = r3[1];
 87    __uint128_t x_3 = r3[2];
 88    __uint128_t x_4 = r3[3];
 89
 90    x_1 *= (__uint128_t) op3[0];
 91    x_2 *= (__uint128_t) op3[1];
 92    x_3 *= (__uint128_t) op3[2];
 93    x_4 *= (__uint128_t) op3[3];
 94
 95    uint64_t x0_1 = x_1;
 96    uint64_t x0_2 = x_2;
 97    uint64_t x0_3 = x_3;
 98    uint64_t x0_4 = x_4;
 99
100    svuint64_t l1 = svmul_x(pg, *r1, *op1);
101    svuint64_t l2 = svmul_x(pg, *r2, *op2);
102
103    uint64_t x1_1 = (x_1 >> 64);
104    uint64_t x1_2 = (x_2 >> 64);
105    uint64_t x1_3 = (x_3 >> 64);
106    uint64_t x1_4 = (x_4 >> 64);
107
108    uint64_t a_1, a_2, a_3, a_4;
109    uint64_t e_1 = __builtin_add_overflow(x0_1, (x0_1 << 32), &a_1);
110    uint64_t e_2 = __builtin_add_overflow(x0_2, (x0_2 << 32), &a_2);
111    uint64_t e_3 = __builtin_add_overflow(x0_3, (x0_3 << 32), &a_3);
112    uint64_t e_4 = __builtin_add_overflow(x0_4, (x0_4 << 32), &a_4);
113
114    svuint64_t ls1 = svlsl_x(pg, l1, 32);
115    svuint64_t ls2 = svlsl_x(pg, l2, 32);
116
117    svuint64_t a1 = svadd_x(pg, l1, ls1);
118    svuint64_t a2 = svadd_x(pg, l2, ls2);
119
120    svbool_t e1 = svcmplt(pg, a1, l1);
121    svbool_t e2 = svcmplt(pg, a2, l2);
122
123    svuint64_t as1 = svlsr_x(pg, a1, 32);
124    svuint64_t as2 = svlsr_x(pg, a2, 32);
125
126    svuint64_t b1 = svsub_x(pg, a1, as1);
127    svuint64_t b2 = svsub_x(pg, a2, as2);
128
129    b1 = svsub_m(e1, b1, 1);
130    b2 = svsub_m(e2, b2, 1);
131
132    uint64_t b_1 = a_1 - (a_1 >> 32) - e_1;
133    uint64_t b_2 = a_2 - (a_2 >> 32) - e_2;
134    uint64_t b_3 = a_3 - (a_3 >> 32) - e_3;
135    uint64_t b_4 = a_4 - (a_4 >> 32) - e_4;
136
137    uint64_t r_1, r_2, r_3, r_4;
138    uint32_t c_1 = __builtin_sub_overflow(x1_1, b_1, &r_1);
139    uint32_t c_2 = __builtin_sub_overflow(x1_2, b_2, &r_2);
140    uint32_t c_3 = __builtin_sub_overflow(x1_3, b_3, &r_3);
141    uint32_t c_4 = __builtin_sub_overflow(x1_4, b_4, &r_4);
142
143    svuint64_t h1 = svmulh_x(pg, *r1, *op1);
144    svuint64_t h2 = svmulh_x(pg, *r2, *op2);
145
146    svuint64_t tr1 = svsub_x(pg, h1, b1);
147    svuint64_t tr2 = svsub_x(pg, h2, b2);
148
149    svbool_t c1 = svcmplt_u64(pg, h1, b1);
150    svbool_t c2 = svcmplt_u64(pg, h2, b2);
151
152    *r1 = svsub_m(c1, tr1, (uint32_t) -1);
153    *r2 = svsub_m(c2, tr2, (uint32_t) -1);
154
155    uint32_t minus1_1 = 0 - c_1;
156    uint32_t minus1_2 = 0 - c_2;
157    uint32_t minus1_3 = 0 - c_3;
158    uint32_t minus1_4 = 0 - c_4;
159
160    r3[0] = r_1 - (uint64_t)minus1_1;
161    r3[1] = r_2 - (uint64_t)minus1_2;
162    r3[2] = r_3 - (uint64_t)minus1_3;
163    r3[3] = r_4 - (uint64_t)minus1_4;
164}
165
166extern inline void sq(svbool_t pg, svuint64_t *a, svuint64_t *b, uint64_t *c) {
167    mul(pg, a, a, b, b, c, c);
168}
169
170extern inline void apply_sbox(
171    svbool_t pg,
172    svuint64_t *state1,
173    svuint64_t *state2,
174    uint64_t *state3
175) {
176    COPY(x, *state1, *state2, state3);                // copy input to x
177    SQUARE(pg, x);                                    // x contains input^2
178    mul(pg, state1, &x_1, state2, &x_2, state3, x_3); // state contains input^3
179    SQUARE(pg, x);                                    // x contains input^4
180    mul(pg, state1, &x_1, state2, &x_2, state3, x_3); // state contains input^7
181}
182
183extern inline void apply_inv_sbox(
184    svbool_t pg,
185    svuint64_t *state_1,
186    svuint64_t *state_2,
187    uint64_t *state_3
188) {
189    // base^10
190    COPY(t1, *state_1, *state_2, state_3);
191    SQUARE(pg, t1);
192
193    // base^100
194    SQUARE_DEST(pg, t2, t1);
195
196    // base^100100
197    POW_ACC_DEST(pg, t3, 3, t2, t2);
198
199    // base^100100100100
200    POW_ACC_DEST(pg, t4, 6, t3, t3);
201
202    // compute base^100100100100100100100100
203    POW_ACC_DEST(pg, t5, 12, t4, t4);
204
205    // compute base^100100100100100100100100100100
206    POW_ACC_DEST(pg, t6, 6, t5, t3);
207
208    // compute base^1001001001001001001001001001000100100100100100100100100100100
209    POW_ACC_DEST(pg, t7, 31, t6, t6);
210
211    // compute base^1001001001001001001001001001000110110110110110110110110110110111
212    SQUARE(pg, t7);
213    MULTIPLY(pg, t7, t6);
214    SQUARE(pg, t7);
215    SQUARE(pg, t7);
216    MULTIPLY(pg, t7, t1);
217    MULTIPLY(pg, t7, t2);
218    mul(pg, state_1, &t7_1, state_2, &t7_2, state_3, t7_3);
219}
220
221#endif //RPO_SVE_RPO_HASH_H