miden-crypto/arch/arm64-sve/rpo/rpo_hash_128bit.h
arch/arm64-sve/rpo/rpo_hash_128bit.hRaw
1#ifndef RPO_SVE_RPO_HASH_128_H
2#define RPO_SVE_RPO_HASH_128_H
3
4#include <arm_sve.h>
5#include <stddef.h>
6#include <stdint.h>
7#include <string.h>
8
9#define STATE_WIDTH 12
10
11#define COPY_128(NAME, VIN1, VIN2, VIN3, VIN4, SIN) \
12 svuint64_t NAME ## _1 = VIN1; \
13 svuint64_t NAME ## _2 = VIN2; \
14 svuint64_t NAME ## _3 = VIN3; \
15 svuint64_t NAME ## _4 = VIN4; \
16 uint64_t NAME ## _tail[4]; \
17 memcpy(NAME ## _tail, SIN, 4 * sizeof(uint64_t))
18
19#define MULTIPLY_128(PRED, DEST, OP) \
20 mul_128(PRED, &DEST ## _1, &OP ## _1, &DEST ## _2, &OP ## _2, &DEST ## _3, &OP ## _3, &DEST ## _4, &OP ## _4, DEST ## _tail, OP ## _tail)
21
22#define SQUARE_128(PRED, NAME) \
23 sq_128(PRED, &NAME ## _1, &NAME ## _2, &NAME ## _3, &NAME ## _4, NAME ## _tail)
24
25#define SQUARE_DEST_128(PRED, DEST, SRC) \
26 COPY_128(DEST, SRC ## _1, SRC ## _2, SRC ## _3, SRC ## _4, SRC ## _tail); \
27 SQUARE_128(PRED, DEST);
28
29#define POW_ACC_128(PRED, NAME, CNT, TAIL) \
30 for (size_t i = 0; i < CNT; i++) { \
31 SQUARE_128(PRED, NAME); \
32 } \
33 MULTIPLY_128(PRED, NAME, TAIL);
34
35#define POW_ACC_DEST(PRED, DEST, CNT, HEAD, TAIL) \
36 COPY_128(DEST, HEAD ## _1, HEAD ## _2, HEAD ## _3, HEAD ## _4, HEAD ## _tail); \
37 POW_ACC_128(PRED, DEST, CNT, TAIL)
38
39extern inline void add_constants_128(
40 svbool_t pg,
41 svuint64_t *state1,
42 svuint64_t *const1,
43 svuint64_t *state2,
44 svuint64_t *const2,
45 svuint64_t *state3,
46 svuint64_t *const3,
47 svuint64_t *state4,
48 svuint64_t *const4,
49
50 uint64_t *state_tail,
51 uint64_t *const_tail
52) {
53 uint64_t Ms = 0xFFFFFFFF00000001ull;
54 svuint64_t Mv = svindex_u64(Ms, 0);
55
56 uint64_t p_1 = Ms - const_tail[0];
57 uint64_t p_2 = Ms - const_tail[1];
58 uint64_t p_3 = Ms - const_tail[2];
59 uint64_t p_4 = Ms - const_tail[3];
60
61 uint64_t x_1, x_2, x_3, x_4;
62 uint32_t adj_1 = -__builtin_sub_overflow(state_tail[0], p_1, &x_1);
63 uint32_t adj_2 = -__builtin_sub_overflow(state_tail[1], p_2, &x_2);
64 uint32_t adj_3 = -__builtin_sub_overflow(state_tail[2], p_3, &x_3);
65 uint32_t adj_4 = -__builtin_sub_overflow(state_tail[3], p_4, &x_4);
66
67 state_tail[0] = x_1 - (uint64_t)adj_1;
68 state_tail[1] = x_2 - (uint64_t)adj_2;
69 state_tail[2] = x_3 - (uint64_t)adj_3;
70 state_tail[3] = x_4 - (uint64_t)adj_4;
71
72 svuint64_t p1 = svsub_x(pg, Mv, *const1);
73 svuint64_t p2 = svsub_x(pg, Mv, *const2);
74 svuint64_t p3 = svsub_x(pg, Mv, *const3);
75 svuint64_t p4 = svsub_x(pg, Mv, *const4);
76
77 svuint64_t x1 = svsub_x(pg, *state1, p1);
78 svuint64_t x2 = svsub_x(pg, *state2, p2);
79 svuint64_t x3 = svsub_x(pg, *state3, p3);
80 svuint64_t x4 = svsub_x(pg, *state4, p4);
81
82 svbool_t pt1 = svcmplt_u64(pg, *state1, p1);
83 svbool_t pt2 = svcmplt_u64(pg, *state2, p2);
84 svbool_t pt3 = svcmplt_u64(pg, *state3, p3);
85 svbool_t pt4 = svcmplt_u64(pg, *state4, p4);
86
87 *state1 = svsub_m(pt1, x1, (uint32_t)-1);
88 *state2 = svsub_m(pt2, x2, (uint32_t)-1);
89 *state3 = svsub_m(pt3, x3, (uint32_t)-1);
90 *state4 = svsub_m(pt4, x4, (uint32_t)-1);
91}
92
93extern inline void mul_128(
94 svbool_t pg,
95 svuint64_t *r1,
96 const svuint64_t *op1,
97 svuint64_t *r2,
98 const svuint64_t *op2,
99 svuint64_t *r3,
100 const svuint64_t *op3,
101 svuint64_t *r4,
102 const svuint64_t *op4,
103 uint64_t *r_tail,
104 const uint64_t *op_tail
105) {
106 __uint128_t x_1 = r_tail[0];
107 __uint128_t x_2 = r_tail[1];
108 __uint128_t x_3 = r_tail[2];
109 __uint128_t x_4 = r_tail[3];
110
111 x_1 *= (__uint128_t) op_tail[0];
112 x_2 *= (__uint128_t) op_tail[1];
113 x_3 *= (__uint128_t) op_tail[2];
114 x_4 *= (__uint128_t) op_tail[3];
115
116 uint64_t x0_1 = x_1;
117 uint64_t x0_2 = x_2;
118 uint64_t x0_3 = x_3;
119 uint64_t x0_4 = x_4;
120
121 svuint64_t l1 = svmul_x(pg, *r1, *op1);
122 svuint64_t l2 = svmul_x(pg, *r2, *op2);
123 svuint64_t l3 = svmul_x(pg, *r3, *op3);
124 svuint64_t l4 = svmul_x(pg, *r4, *op4);
125
126 uint64_t x1_1 = (x_1 >> 64);
127 uint64_t x1_2 = (x_2 >> 64);
128 uint64_t x1_3 = (x_3 >> 64);
129 uint64_t x1_4 = (x_4 >> 64);
130
131 uint64_t a_1, a_2, a_3, a_4;
132 uint64_t e_1 = __builtin_add_overflow(x0_1, (x0_1 << 32), &a_1);
133 uint64_t e_2 = __builtin_add_overflow(x0_2, (x0_2 << 32), &a_2);
134 uint64_t e_3 = __builtin_add_overflow(x0_3, (x0_3 << 32), &a_3);
135 uint64_t e_4 = __builtin_add_overflow(x0_4, (x0_4 << 32), &a_4);
136
137 svuint64_t ls1 = svlsl_x(pg, l1, 32);
138 svuint64_t ls2 = svlsl_x(pg, l2, 32);
139 svuint64_t ls3 = svlsl_x(pg, l3, 32);
140 svuint64_t ls4 = svlsl_x(pg, l4, 32);
141
142 svuint64_t a1 = svadd_x(pg, l1, ls1);
143 svuint64_t a2 = svadd_x(pg, l2, ls2);
144 svuint64_t a3 = svadd_x(pg, l3, ls3);
145 svuint64_t a4 = svadd_x(pg, l4, ls4);
146
147 svbool_t e1 = svcmplt(pg, a1, l1);
148 svbool_t e2 = svcmplt(pg, a2, l2);
149 svbool_t e3 = svcmplt(pg, a3, l3);
150 svbool_t e4 = svcmplt(pg, a4, l4);
151
152 svuint64_t as1 = svlsr_x(pg, a1, 32);
153 svuint64_t as2 = svlsr_x(pg, a2, 32);
154 svuint64_t as3 = svlsr_x(pg, a3, 32);
155 svuint64_t as4 = svlsr_x(pg, a4, 32);
156
157 svuint64_t b1 = svsub_x(pg, a1, as1);
158 svuint64_t b2 = svsub_x(pg, a2, as2);
159 svuint64_t b3 = svsub_x(pg, a3, as3);
160 svuint64_t b4 = svsub_x(pg, a4, as4);
161
162 b1 = svsub_m(e1, b1, 1);
163 b2 = svsub_m(e2, b2, 1);
164 b3 = svsub_m(e3, b3, 1);
165 b4 = svsub_m(e4, b4, 1);
166
167 uint64_t b_1 = a_1 - (a_1 >> 32) - e_1;
168 uint64_t b_2 = a_2 - (a_2 >> 32) - e_2;
169 uint64_t b_3 = a_3 - (a_3 >> 32) - e_3;
170 uint64_t b_4 = a_4 - (a_4 >> 32) - e_4;
171
172 uint64_t r_1, r_2, r_3, r_4;
173 uint32_t c_1 = __builtin_sub_overflow(x1_1, b_1, &r_1);
174 uint32_t c_2 = __builtin_sub_overflow(x1_2, b_2, &r_2);
175 uint32_t c_3 = __builtin_sub_overflow(x1_3, b_3, &r_3);
176 uint32_t c_4 = __builtin_sub_overflow(x1_4, b_4, &r_4);
177
178 svuint64_t h1 = svmulh_x(pg, *r1, *op1);
179 svuint64_t h2 = svmulh_x(pg, *r2, *op2);
180 svuint64_t h3 = svmulh_x(pg, *r3, *op3);
181 svuint64_t h4 = svmulh_x(pg, *r4, *op4);
182
183 svuint64_t tr1 = svsub_x(pg, h1, b1);
184 svuint64_t tr2 = svsub_x(pg, h2, b2);
185 svuint64_t tr3 = svsub_x(pg, h3, b3);
186 svuint64_t tr4 = svsub_x(pg, h4, b4);
187
188 svbool_t c1 = svcmplt_u64(pg, h1, b1);
189 svbool_t c2 = svcmplt_u64(pg, h2, b2);
190 svbool_t c3 = svcmplt_u64(pg, h3, b3);
191 svbool_t c4 = svcmplt_u64(pg, h4, b4);
192
193 *r1 = svsub_m(c1, tr1, (uint32_t) -1);
194 *r2 = svsub_m(c2, tr2, (uint32_t) -1);
195 *r3 = svsub_m(c3, tr3, (uint32_t) -1);
196 *r4 = svsub_m(c4, tr4, (uint32_t) -1);
197
198 uint32_t minus1_1 = 0 - c_1;
199 uint32_t minus1_2 = 0 - c_2;
200 uint32_t minus1_3 = 0 - c_3;
201 uint32_t minus1_4 = 0 - c_4;
202
203 r_tail[0] = r_1 - (uint64_t)minus1_1;
204 r_tail[1] = r_2 - (uint64_t)minus1_2;
205 r_tail[2] = r_3 - (uint64_t)minus1_3;
206 r_tail[3] = r_4 - (uint64_t)minus1_4;
207}
208
209extern inline void sq_128(svbool_t pg, svuint64_t *a, svuint64_t *b, svuint64_t *c, svuint64_t *d, uint64_t *e) {
210 mul_128(pg, a, a, b, b, c, c, d, d, e, e);
211}
212
213extern inline void apply_sbox_128(
214 svbool_t pg,
215 svuint64_t *state1,
216 svuint64_t *state2,
217 svuint64_t *state3,
218 svuint64_t *state4,
219 uint64_t *state_tail
220) {
221 COPY_128(x, *state1, *state2, *state3, *state4, state_tail); // copy input to x
222 SQUARE_128(pg, x); // x contains input^2
223 mul_128(pg, state1, &x_1, state2, &x_2, state3, &x_3, state4, &x_4, state_tail, x_tail); // state contains input^3
224 SQUARE_128(pg, x); // x contains input^4
225 mul_128(pg, state1, &x_1, state2, &x_2, state3, &x_3, state4, &x_4, state_tail, x_tail); // state contains input^7
226}
227
228extern inline void apply_inv_sbox_128(
229 svbool_t pg,
230 svuint64_t *state1,
231 svuint64_t *state2,
232 svuint64_t *state3,
233 svuint64_t *state4,
234 uint64_t *state_tail
235) {
236 // base^10
237 COPY_128(t1, *state1, *state2, *state3, *state4, state_tail);
238 SQUARE_128(pg, t1);
239
240 // base^100
241 SQUARE_DEST_128(pg, t2, t1);
242
243 // base^100100
244 POW_ACC_DEST(pg, t3, 3, t2, t2);
245
246 // base^100100100100
247 POW_ACC_DEST(pg, t4, 6, t3, t3);
248
249 // compute base^100100100100100100100100
250 POW_ACC_DEST(pg, t5, 12, t4, t4);
251
252 // compute base^100100100100100100100100100100
253 POW_ACC_DEST(pg, t6, 6, t5, t3);
254
255 // compute base^1001001001001001001001001001000100100100100100100100100100100
256 POW_ACC_DEST(pg, t7, 31, t6, t6);
257
258 // compute base^1001001001001001001001001001000110110110110110110110110110110111
259 SQUARE_128(pg, t7);
260 MULTIPLY_128(pg, t7, t6);
261 SQUARE_128(pg, t7);
262 SQUARE_128(pg, t7);
263 MULTIPLY_128(pg, t7, t1);
264 MULTIPLY_128(pg, t7, t2);
265 mul_128(pg, state1, &t7_1, state2, &t7_2, state3, &t7_3, state4, &t7_4, state_tail, t7_tail);
266}
267
268bool add_constants_and_apply_sbox_128(uint64_t state[STATE_WIDTH], uint64_t constants[STATE_WIDTH]) {
269 const uint64_t vl = 2; // number of u64 numbers in one 128 bit SVE vector
270 svbool_t ptrue = svptrue_b64();
271
272 svuint64_t state1 = svld1(ptrue, state + 0 * vl);
273 svuint64_t state2 = svld1(ptrue, state + 1 * vl);
274 svuint64_t state3 = svld1(ptrue, state + 2 * vl);
275 svuint64_t state4 = svld1(ptrue, state + 3 * vl);
276
277 svuint64_t const1 = svld1(ptrue, constants + 0 * vl);
278 svuint64_t const2 = svld1(ptrue, constants + 1 * vl);
279 svuint64_t const3 = svld1(ptrue, constants + 2 * vl);
280 svuint64_t const4 = svld1(ptrue, constants + 3 * vl);
281
282 add_constants_128(ptrue, &state1, &const1, &state2, &const2, &state3, &const3, &state4, &const4, state + 8, constants + 8);
283 apply_sbox_128(ptrue, &state1, &state2, &state3, &state4, state + 8);
284
285 svst1(ptrue, state + 0 * vl, state1);
286 svst1(ptrue, state + 1 * vl, state2);
287 svst1(ptrue, state + 2 * vl, state3);
288 svst1(ptrue, state + 3 * vl, state4);
289
290 return true;
291}
292
293bool add_constants_and_apply_inv_sbox_128(uint64_t state[STATE_WIDTH], uint64_t constants[STATE_WIDTH]) {
294 const uint64_t vl = 2; // number of u64 numbers in one 128 bit SVE vector
295 svbool_t ptrue = svptrue_b64();
296
297 svuint64_t state1 = svld1(ptrue, state + 0 * vl);
298 svuint64_t state2 = svld1(ptrue, state + 1 * vl);
299 svuint64_t state3 = svld1(ptrue, state + 2 * vl);
300 svuint64_t state4 = svld1(ptrue, state + 3 * vl);
301
302 svuint64_t const1 = svld1(ptrue, constants + 0 * vl);
303 svuint64_t const2 = svld1(ptrue, constants + 1 * vl);
304 svuint64_t const3 = svld1(ptrue, constants + 2 * vl);
305 svuint64_t const4 = svld1(ptrue, constants + 3 * vl);
306
307 add_constants_128(ptrue, &state1, &const1, &state2, &const2, &state3, &const3, &state4, &const4, state + 8, constants + 8);
308 apply_inv_sbox_128(ptrue, &state1, &state2, &state3, &state4, state + 8);
309
310 svst1(ptrue, state + 0 * vl, state1);
311 svst1(ptrue, state + 1 * vl, state2);
312 svst1(ptrue, state + 2 * vl, state3);
313 svst1(ptrue, state + 3 * vl, state4);
314
315 return true;
316}
317
318#endif //RPO_SVE_RPO_HASH_128_H