poulpy/poulpy-backend/src/cpu_spqlios/spqlios-arithmetic/spqlios/ext/neon_accel/macrofx4.h
poulpy-backend/src/cpu_spqlios/spqlios-arithmetic/spqlios/ext/neon_accel/macrofx4.hRaw
1/*
2 * This file is extracted from the implementation of the FFT on Arm64/Neon
3 * available in https://github.com/cothan/Falcon-Arm (neon/macrof.h).
4 * =============================================================================
5 * Copyright (c) 2022 by Cryptographic Engineering Research Group (CERG)
6 * ECE Department, George Mason University
7 * Fairfax, VA, U.S.A.
8 * @author: Duc Tri Nguyen dnguye69@gmu.edu, cothannguyen@gmail.com
9 * Licensed under the Apache License, Version 2.0 (the "License");
10 * =============================================================================
11 *
12 * This 64-bit Floating point NEON macro x4 has not been modified and is provided as is.
13 */
14
15#ifndef MACROFX4_H
16#define MACROFX4_H
17
18#include <arm_neon.h>
19
20#include "macrof.h"
21
22#define vloadx4(c, addr) c = vld1q_f64_x4(addr);
23
24#define vstorex4(addr, c) vst1q_f64_x4(addr, c);
25
26#define vfdupx4(c, constant) \
27 c.val[0] = vdupq_n_f64(constant); \
28 c.val[1] = vdupq_n_f64(constant); \
29 c.val[2] = vdupq_n_f64(constant); \
30 c.val[3] = vdupq_n_f64(constant);
31
32#define vfnegx4(c, a) \
33 c.val[0] = vnegq_f64(a.val[0]); \
34 c.val[1] = vnegq_f64(a.val[1]); \
35 c.val[2] = vnegq_f64(a.val[2]); \
36 c.val[3] = vnegq_f64(a.val[3]);
37
38#define vfmulnx4(c, a, n) \
39 c.val[0] = vmulq_n_f64(a.val[0], n); \
40 c.val[1] = vmulq_n_f64(a.val[1], n); \
41 c.val[2] = vmulq_n_f64(a.val[2], n); \
42 c.val[3] = vmulq_n_f64(a.val[3], n);
43
44// c = a - b
45#define vfsubx4(c, a, b) \
46 c.val[0] = vsubq_f64(a.val[0], b.val[0]); \
47 c.val[1] = vsubq_f64(a.val[1], b.val[1]); \
48 c.val[2] = vsubq_f64(a.val[2], b.val[2]); \
49 c.val[3] = vsubq_f64(a.val[3], b.val[3]);
50
51// c = a + b
52#define vfaddx4(c, a, b) \
53 c.val[0] = vaddq_f64(a.val[0], b.val[0]); \
54 c.val[1] = vaddq_f64(a.val[1], b.val[1]); \
55 c.val[2] = vaddq_f64(a.val[2], b.val[2]); \
56 c.val[3] = vaddq_f64(a.val[3], b.val[3]);
57
58#define vfmulx4(c, a, b) \
59 c.val[0] = vmulq_f64(a.val[0], b.val[0]); \
60 c.val[1] = vmulq_f64(a.val[1], b.val[1]); \
61 c.val[2] = vmulq_f64(a.val[2], b.val[2]); \
62 c.val[3] = vmulq_f64(a.val[3], b.val[3]);
63
64#define vfmulx4_i(c, a, b) \
65 c.val[0] = vmulq_f64(a.val[0], b); \
66 c.val[1] = vmulq_f64(a.val[1], b); \
67 c.val[2] = vmulq_f64(a.val[2], b); \
68 c.val[3] = vmulq_f64(a.val[3], b);
69
70#define vfinvx4(c, a) \
71 c.val[0] = vdivq_f64(vdupq_n_f64(1.0), a.val[0]); \
72 c.val[1] = vdivq_f64(vdupq_n_f64(1.0), a.val[1]); \
73 c.val[2] = vdivq_f64(vdupq_n_f64(1.0), a.val[2]); \
74 c.val[3] = vdivq_f64(vdupq_n_f64(1.0), a.val[3]);
75
76#define vfcvtx4(c, a) \
77 c.val[0] = vcvtq_f64_s64(a.val[0]); \
78 c.val[1] = vcvtq_f64_s64(a.val[1]); \
79 c.val[2] = vcvtq_f64_s64(a.val[2]); \
80 c.val[3] = vcvtq_f64_s64(a.val[3]);
81
82#define vfmlax4(d, c, a, b) \
83 vfmla(d.val[0], c.val[0], a.val[0], b.val[0]); \
84 vfmla(d.val[1], c.val[1], a.val[1], b.val[1]); \
85 vfmla(d.val[2], c.val[2], a.val[2], b.val[2]); \
86 vfmla(d.val[3], c.val[3], a.val[3], b.val[3]);
87
88#define vfmlsx4(d, c, a, b) \
89 vfmls(d.val[0], c.val[0], a.val[0], b.val[0]); \
90 vfmls(d.val[1], c.val[1], a.val[1], b.val[1]); \
91 vfmls(d.val[2], c.val[2], a.val[2], b.val[2]); \
92 vfmls(d.val[3], c.val[3], a.val[3], b.val[3]);
93
94#define vfrintx4(c, a) \
95 c.val[0] = vcvtnq_s64_f64(a.val[0]); \
96 c.val[1] = vcvtnq_s64_f64(a.val[1]); \
97 c.val[2] = vcvtnq_s64_f64(a.val[2]); \
98 c.val[3] = vcvtnq_s64_f64(a.val[3]);
99
100/*
101 * Wrapper for FFT, split/merge and poly_float.c
102 */
103
104#define FPC_MUL(d_re, d_im, a_re, a_im, b_re, b_im) \
105 vfmul(d_re, a_re, b_re); \
106 vfmls(d_re, d_re, a_im, b_im); \
107 vfmul(d_im, a_re, b_im); \
108 vfmla(d_im, d_im, a_im, b_re);
109
110#define FPC_MULx2(d_re, d_im, a_re, a_im, b_re, b_im) \
111 vfmul(d_re.val[0], a_re.val[0], b_re.val[0]); \
112 vfmls(d_re.val[0], d_re.val[0], a_im.val[0], b_im.val[0]); \
113 vfmul(d_re.val[1], a_re.val[1], b_re.val[1]); \
114 vfmls(d_re.val[1], d_re.val[1], a_im.val[1], b_im.val[1]); \
115 vfmul(d_im.val[0], a_re.val[0], b_im.val[0]); \
116 vfmla(d_im.val[0], d_im.val[0], a_im.val[0], b_re.val[0]); \
117 vfmul(d_im.val[1], a_re.val[1], b_im.val[1]); \
118 vfmla(d_im.val[1], d_im.val[1], a_im.val[1], b_re.val[1]);
119
120#define FPC_MULx4(d_re, d_im, a_re, a_im, b_re, b_im) \
121 vfmul(d_re.val[0], a_re.val[0], b_re.val[0]); \
122 vfmls(d_re.val[0], d_re.val[0], a_im.val[0], b_im.val[0]); \
123 vfmul(d_re.val[1], a_re.val[1], b_re.val[1]); \
124 vfmls(d_re.val[1], d_re.val[1], a_im.val[1], b_im.val[1]); \
125 vfmul(d_re.val[2], a_re.val[2], b_re.val[2]); \
126 vfmls(d_re.val[2], d_re.val[2], a_im.val[2], b_im.val[2]); \
127 vfmul(d_re.val[3], a_re.val[3], b_re.val[3]); \
128 vfmls(d_re.val[3], d_re.val[3], a_im.val[3], b_im.val[3]); \
129 vfmul(d_im.val[0], a_re.val[0], b_im.val[0]); \
130 vfmla(d_im.val[0], d_im.val[0], a_im.val[0], b_re.val[0]); \
131 vfmul(d_im.val[1], a_re.val[1], b_im.val[1]); \
132 vfmla(d_im.val[1], d_im.val[1], a_im.val[1], b_re.val[1]); \
133 vfmul(d_im.val[2], a_re.val[2], b_im.val[2]); \
134 vfmla(d_im.val[2], d_im.val[2], a_im.val[2], b_re.val[2]); \
135 vfmul(d_im.val[3], a_re.val[3], b_im.val[3]); \
136 vfmla(d_im.val[3], d_im.val[3], a_im.val[3], b_re.val[3]);
137
138#define FPC_MLA(d_re, d_im, a_re, a_im, b_re, b_im) \
139 vfmla(d_re, d_re, a_re, b_re); \
140 vfmls(d_re, d_re, a_im, b_im); \
141 vfmla(d_im, d_im, a_re, b_im); \
142 vfmla(d_im, d_im, a_im, b_re);
143
144#define FPC_MLAx2(d_re, d_im, a_re, a_im, b_re, b_im) \
145 vfmla(d_re.val[0], d_re.val[0], a_re.val[0], b_re.val[0]); \
146 vfmls(d_re.val[0], d_re.val[0], a_im.val[0], b_im.val[0]); \
147 vfmla(d_re.val[1], d_re.val[1], a_re.val[1], b_re.val[1]); \
148 vfmls(d_re.val[1], d_re.val[1], a_im.val[1], b_im.val[1]); \
149 vfmla(d_im.val[0], d_im.val[0], a_re.val[0], b_im.val[0]); \
150 vfmla(d_im.val[0], d_im.val[0], a_im.val[0], b_re.val[0]); \
151 vfmla(d_im.val[1], d_im.val[1], a_re.val[1], b_im.val[1]); \
152 vfmla(d_im.val[1], d_im.val[1], a_im.val[1], b_re.val[1]);
153
154#define FPC_MLAx4(d_re, d_im, a_re, a_im, b_re, b_im) \
155 vfmla(d_re.val[0], d_re.val[0], a_re.val[0], b_re.val[0]); \
156 vfmls(d_re.val[0], d_re.val[0], a_im.val[0], b_im.val[0]); \
157 vfmla(d_re.val[1], d_re.val[1], a_re.val[1], b_re.val[1]); \
158 vfmls(d_re.val[1], d_re.val[1], a_im.val[1], b_im.val[1]); \
159 vfmla(d_re.val[2], d_re.val[2], a_re.val[2], b_re.val[2]); \
160 vfmls(d_re.val[2], d_re.val[2], a_im.val[2], b_im.val[2]); \
161 vfmla(d_re.val[3], d_re.val[3], a_re.val[3], b_re.val[3]); \
162 vfmls(d_re.val[3], d_re.val[3], a_im.val[3], b_im.val[3]); \
163 vfmla(d_im.val[0], d_im.val[0], a_re.val[0], b_im.val[0]); \
164 vfmla(d_im.val[0], d_im.val[0], a_im.val[0], b_re.val[0]); \
165 vfmla(d_im.val[1], d_im.val[1], a_re.val[1], b_im.val[1]); \
166 vfmla(d_im.val[1], d_im.val[1], a_im.val[1], b_re.val[1]); \
167 vfmla(d_im.val[2], d_im.val[2], a_re.val[2], b_im.val[2]); \
168 vfmla(d_im.val[2], d_im.val[2], a_im.val[2], b_re.val[2]); \
169 vfmla(d_im.val[3], d_im.val[3], a_re.val[3], b_im.val[3]); \
170 vfmla(d_im.val[3], d_im.val[3], a_im.val[3], b_re.val[3]);
171
172#define FPC_MUL_CONJx4(d_re, d_im, a_re, a_im, b_re, b_im) \
173 vfmul(d_re.val[0], b_im.val[0], a_im.val[0]); \
174 vfmla(d_re.val[0], d_re.val[0], a_re.val[0], b_re.val[0]); \
175 vfmul(d_re.val[1], b_im.val[1], a_im.val[1]); \
176 vfmla(d_re.val[1], d_re.val[1], a_re.val[1], b_re.val[1]); \
177 vfmul(d_re.val[2], b_im.val[2], a_im.val[2]); \
178 vfmla(d_re.val[2], d_re.val[2], a_re.val[2], b_re.val[2]); \
179 vfmul(d_re.val[3], b_im.val[3], a_im.val[3]); \
180 vfmla(d_re.val[3], d_re.val[3], a_re.val[3], b_re.val[3]); \
181 vfmul(d_im.val[0], b_re.val[0], a_im.val[0]); \
182 vfmls(d_im.val[0], d_im.val[0], a_re.val[0], b_im.val[0]); \
183 vfmul(d_im.val[1], b_re.val[1], a_im.val[1]); \
184 vfmls(d_im.val[1], d_im.val[1], a_re.val[1], b_im.val[1]); \
185 vfmul(d_im.val[2], b_re.val[2], a_im.val[2]); \
186 vfmls(d_im.val[2], d_im.val[2], a_re.val[2], b_im.val[2]); \
187 vfmul(d_im.val[3], b_re.val[3], a_im.val[3]); \
188 vfmls(d_im.val[3], d_im.val[3], a_re.val[3], b_im.val[3]);
189
190#define FPC_MLA_CONJx4(d_re, d_im, a_re, a_im, b_re, b_im) \
191 vfmla(d_re.val[0], d_re.val[0], b_im.val[0], a_im.val[0]); \
192 vfmla(d_re.val[0], d_re.val[0], a_re.val[0], b_re.val[0]); \
193 vfmla(d_re.val[1], d_re.val[1], b_im.val[1], a_im.val[1]); \
194 vfmla(d_re.val[1], d_re.val[1], a_re.val[1], b_re.val[1]); \
195 vfmla(d_re.val[2], d_re.val[2], b_im.val[2], a_im.val[2]); \
196 vfmla(d_re.val[2], d_re.val[2], a_re.val[2], b_re.val[2]); \
197 vfmla(d_re.val[3], d_re.val[3], b_im.val[3], a_im.val[3]); \
198 vfmla(d_re.val[3], d_re.val[3], a_re.val[3], b_re.val[3]); \
199 vfmla(d_im.val[0], d_im.val[0], b_re.val[0], a_im.val[0]); \
200 vfmls(d_im.val[0], d_im.val[0], a_re.val[0], b_im.val[0]); \
201 vfmla(d_im.val[1], d_im.val[1], b_re.val[1], a_im.val[1]); \
202 vfmls(d_im.val[1], d_im.val[1], a_re.val[1], b_im.val[1]); \
203 vfmla(d_im.val[2], d_im.val[2], b_re.val[2], a_im.val[2]); \
204 vfmls(d_im.val[2], d_im.val[2], a_re.val[2], b_im.val[2]); \
205 vfmla(d_im.val[3], d_im.val[3], b_re.val[3], a_im.val[3]); \
206 vfmls(d_im.val[3], d_im.val[3], a_re.val[3], b_im.val[3]);
207
208#define FPC_MUL_LANE(d_re, d_im, a_re, a_im, b_re_im) \
209 vfmul_lane(d_re, a_re, b_re_im, 0); \
210 vfmls_lane(d_re, d_re, a_im, b_re_im, 1); \
211 vfmul_lane(d_im, a_re, b_re_im, 1); \
212 vfmla_lane(d_im, d_im, a_im, b_re_im, 0);
213
214#define FPC_MUL_LANEx4(d_re, d_im, a_re, a_im, b_re_im) \
215 vfmul_lane(d_re.val[0], a_re.val[0], b_re_im, 0); \
216 vfmls_lane(d_re.val[0], d_re.val[0], a_im.val[0], b_re_im, 1); \
217 vfmul_lane(d_re.val[1], a_re.val[1], b_re_im, 0); \
218 vfmls_lane(d_re.val[1], d_re.val[1], a_im.val[1], b_re_im, 1); \
219 vfmul_lane(d_re.val[2], a_re.val[2], b_re_im, 0); \
220 vfmls_lane(d_re.val[2], d_re.val[2], a_im.val[2], b_re_im, 1); \
221 vfmul_lane(d_re.val[3], a_re.val[3], b_re_im, 0); \
222 vfmls_lane(d_re.val[3], d_re.val[3], a_im.val[3], b_re_im, 1); \
223 vfmul_lane(d_im.val[0], a_re.val[0], b_re_im, 1); \
224 vfmla_lane(d_im.val[0], d_im.val[0], a_im.val[0], b_re_im, 0); \
225 vfmul_lane(d_im.val[1], a_re.val[1], b_re_im, 1); \
226 vfmla_lane(d_im.val[1], d_im.val[1], a_im.val[1], b_re_im, 0); \
227 vfmul_lane(d_im.val[2], a_re.val[2], b_re_im, 1); \
228 vfmla_lane(d_im.val[2], d_im.val[2], a_im.val[2], b_re_im, 0); \
229 vfmul_lane(d_im.val[3], a_re.val[3], b_re_im, 1); \
230 vfmla_lane(d_im.val[3], d_im.val[3], a_im.val[3], b_re_im, 0);
231
232#define FWD_TOP(t_re, t_im, b_re, b_im, zeta_re, zeta_im) FPC_MUL(t_re, t_im, b_re, b_im, zeta_re, zeta_im);
233
234#define FWD_TOP_LANE(t_re, t_im, b_re, b_im, zeta) FPC_MUL_LANE(t_re, t_im, b_re, b_im, zeta);
235
236#define FWD_TOP_LANEx4(t_re, t_im, b_re, b_im, zeta) FPC_MUL_LANEx4(t_re, t_im, b_re, b_im, zeta);
237
238/*
239 * FPC
240 */
241
242#define FPC_SUB(d_re, d_im, a_re, a_im, b_re, b_im) \
243 d_re = vsubq_f64(a_re, b_re); \
244 d_im = vsubq_f64(a_im, b_im);
245
246#define FPC_SUBx4(d_re, d_im, a_re, a_im, b_re, b_im) \
247 d_re.val[0] = vsubq_f64(a_re.val[0], b_re.val[0]); \
248 d_im.val[0] = vsubq_f64(a_im.val[0], b_im.val[0]); \
249 d_re.val[1] = vsubq_f64(a_re.val[1], b_re.val[1]); \
250 d_im.val[1] = vsubq_f64(a_im.val[1], b_im.val[1]); \
251 d_re.val[2] = vsubq_f64(a_re.val[2], b_re.val[2]); \
252 d_im.val[2] = vsubq_f64(a_im.val[2], b_im.val[2]); \
253 d_re.val[3] = vsubq_f64(a_re.val[3], b_re.val[3]); \
254 d_im.val[3] = vsubq_f64(a_im.val[3], b_im.val[3]);
255
256#define FPC_ADD(d_re, d_im, a_re, a_im, b_re, b_im) \
257 d_re = vaddq_f64(a_re, b_re); \
258 d_im = vaddq_f64(a_im, b_im);
259
260#define FPC_ADDx4(d_re, d_im, a_re, a_im, b_re, b_im) \
261 d_re.val[0] = vaddq_f64(a_re.val[0], b_re.val[0]); \
262 d_im.val[0] = vaddq_f64(a_im.val[0], b_im.val[0]); \
263 d_re.val[1] = vaddq_f64(a_re.val[1], b_re.val[1]); \
264 d_im.val[1] = vaddq_f64(a_im.val[1], b_im.val[1]); \
265 d_re.val[2] = vaddq_f64(a_re.val[2], b_re.val[2]); \
266 d_im.val[2] = vaddq_f64(a_im.val[2], b_im.val[2]); \
267 d_re.val[3] = vaddq_f64(a_re.val[3], b_re.val[3]); \
268 d_im.val[3] = vaddq_f64(a_im.val[3], b_im.val[3]);
269
270#define FWD_BOT(a_re, a_im, b_re, b_im, t_re, t_im) \
271 FPC_SUB(b_re, b_im, a_re, a_im, t_re, t_im); \
272 FPC_ADD(a_re, a_im, a_re, a_im, t_re, t_im);
273
274#define FWD_BOTx4(a_re, a_im, b_re, b_im, t_re, t_im) \
275 FPC_SUBx4(b_re, b_im, a_re, a_im, t_re, t_im); \
276 FPC_ADDx4(a_re, a_im, a_re, a_im, t_re, t_im);
277
278/*
279 * FPC_J
280 */
281
282#define FPC_ADDJ(d_re, d_im, a_re, a_im, b_re, b_im) \
283 d_re = vsubq_f64(a_re, b_im); \
284 d_im = vaddq_f64(a_im, b_re);
285
286#define FPC_ADDJx4(d_re, d_im, a_re, a_im, b_re, b_im) \
287 d_re.val[0] = vsubq_f64(a_re.val[0], b_im.val[0]); \
288 d_im.val[0] = vaddq_f64(a_im.val[0], b_re.val[0]); \
289 d_re.val[1] = vsubq_f64(a_re.val[1], b_im.val[1]); \
290 d_im.val[1] = vaddq_f64(a_im.val[1], b_re.val[1]); \
291 d_re.val[2] = vsubq_f64(a_re.val[2], b_im.val[2]); \
292 d_im.val[2] = vaddq_f64(a_im.val[2], b_re.val[2]); \
293 d_re.val[3] = vsubq_f64(a_re.val[3], b_im.val[3]); \
294 d_im.val[3] = vaddq_f64(a_im.val[3], b_re.val[3]);
295
296#define FPC_SUBJ(d_re, d_im, a_re, a_im, b_re, b_im) \
297 d_re = vaddq_f64(a_re, b_im); \
298 d_im = vsubq_f64(a_im, b_re);
299
300#define FPC_SUBJx4(d_re, d_im, a_re, a_im, b_re, b_im) \
301 d_re.val[0] = vaddq_f64(a_re.val[0], b_im.val[0]); \
302 d_im.val[0] = vsubq_f64(a_im.val[0], b_re.val[0]); \
303 d_re.val[1] = vaddq_f64(a_re.val[1], b_im.val[1]); \
304 d_im.val[1] = vsubq_f64(a_im.val[1], b_re.val[1]); \
305 d_re.val[2] = vaddq_f64(a_re.val[2], b_im.val[2]); \
306 d_im.val[2] = vsubq_f64(a_im.val[2], b_re.val[2]); \
307 d_re.val[3] = vaddq_f64(a_re.val[3], b_im.val[3]); \
308 d_im.val[3] = vsubq_f64(a_im.val[3], b_re.val[3]);
309
310#define FWD_BOTJ(a_re, a_im, b_re, b_im, t_re, t_im) \
311 FPC_SUBJ(b_re, b_im, a_re, a_im, t_re, t_im); \
312 FPC_ADDJ(a_re, a_im, a_re, a_im, t_re, t_im);
313
314#define FWD_BOTJx4(a_re, a_im, b_re, b_im, t_re, t_im) \
315 FPC_SUBJx4(b_re, b_im, a_re, a_im, t_re, t_im); \
316 FPC_ADDJx4(a_re, a_im, a_re, a_im, t_re, t_im);
317
318//============== Inverse FFT
319/*
320 * FPC_J
321 * a * conj(b)
322 * Original (without swap):
323 * d_re = b_im * a_im + a_re * b_re;
324 * d_im = b_re * a_im - a_re * b_im;
325 */
326#define FPC_MUL_BOTJ_LANE(d_re, d_im, a_re, a_im, b_re_im) \
327 vfmul_lane(d_re, a_re, b_re_im, 0); \
328 vfmla_lane(d_re, d_re, a_im, b_re_im, 1); \
329 vfmul_lane(d_im, a_im, b_re_im, 0); \
330 vfmls_lane(d_im, d_im, a_re, b_re_im, 1);
331
332#define FPC_MUL_BOTJ_LANEx4(d_re, d_im, a_re, a_im, b_re_im) \
333 vfmul_lane(d_re.val[0], a_re.val[0], b_re_im, 0); \
334 vfmla_lane(d_re.val[0], d_re.val[0], a_im.val[0], b_re_im, 1); \
335 vfmul_lane(d_im.val[0], a_im.val[0], b_re_im, 0); \
336 vfmls_lane(d_im.val[0], d_im.val[0], a_re.val[0], b_re_im, 1); \
337 vfmul_lane(d_re.val[1], a_re.val[1], b_re_im, 0); \
338 vfmla_lane(d_re.val[1], d_re.val[1], a_im.val[1], b_re_im, 1); \
339 vfmul_lane(d_im.val[1], a_im.val[1], b_re_im, 0); \
340 vfmls_lane(d_im.val[1], d_im.val[1], a_re.val[1], b_re_im, 1); \
341 vfmul_lane(d_re.val[2], a_re.val[2], b_re_im, 0); \
342 vfmla_lane(d_re.val[2], d_re.val[2], a_im.val[2], b_re_im, 1); \
343 vfmul_lane(d_im.val[2], a_im.val[2], b_re_im, 0); \
344 vfmls_lane(d_im.val[2], d_im.val[2], a_re.val[2], b_re_im, 1); \
345 vfmul_lane(d_re.val[3], a_re.val[3], b_re_im, 0); \
346 vfmla_lane(d_re.val[3], d_re.val[3], a_im.val[3], b_re_im, 1); \
347 vfmul_lane(d_im.val[3], a_im.val[3], b_re_im, 0); \
348 vfmls_lane(d_im.val[3], d_im.val[3], a_re.val[3], b_re_im, 1);
349
350#define FPC_MUL_BOTJ(d_re, d_im, a_re, a_im, b_re, b_im) \
351 vfmul(d_re, b_im, a_im); \
352 vfmla(d_re, d_re, a_re, b_re); \
353 vfmul(d_im, b_re, a_im); \
354 vfmls(d_im, d_im, a_re, b_im);
355
356#define INV_TOPJ(t_re, t_im, a_re, a_im, b_re, b_im) \
357 FPC_SUB(t_re, t_im, a_re, a_im, b_re, b_im); \
358 FPC_ADD(a_re, a_im, a_re, a_im, b_re, b_im);
359
360#define INV_TOPJx4(t_re, t_im, a_re, a_im, b_re, b_im) \
361 FPC_SUBx4(t_re, t_im, a_re, a_im, b_re, b_im); \
362 FPC_ADDx4(a_re, a_im, a_re, a_im, b_re, b_im);
363
364#define INV_BOTJ(b_re, b_im, t_re, t_im, zeta_re, zeta_im) FPC_MUL_BOTJ(b_re, b_im, t_re, t_im, zeta_re, zeta_im);
365
366#define INV_BOTJ_LANE(b_re, b_im, t_re, t_im, zeta) FPC_MUL_BOTJ_LANE(b_re, b_im, t_re, t_im, zeta);
367
368#define INV_BOTJ_LANEx4(b_re, b_im, t_re, t_im, zeta) FPC_MUL_BOTJ_LANEx4(b_re, b_im, t_re, t_im, zeta);
369
370/*
371 * FPC_Jm
372 * a * -conj(b)
373 * d_re = a_re * b_im - a_im * b_re;
374 * d_im = a_im * b_im + a_re * b_re;
375 */
376#define FPC_MUL_BOTJm_LANE(d_re, d_im, a_re, a_im, b_re_im) \
377 vfmul_lane(d_re, a_re, b_re_im, 1); \
378 vfmls_lane(d_re, d_re, a_im, b_re_im, 0); \
379 vfmul_lane(d_im, a_re, b_re_im, 0); \
380 vfmla_lane(d_im, d_im, a_im, b_re_im, 1);
381
382#define FPC_MUL_BOTJm_LANEx4(d_re, d_im, a_re, a_im, b_re_im) \
383 vfmul_lane(d_re.val[0], a_re.val[0], b_re_im, 1); \
384 vfmls_lane(d_re.val[0], d_re.val[0], a_im.val[0], b_re_im, 0); \
385 vfmul_lane(d_im.val[0], a_re.val[0], b_re_im, 0); \
386 vfmla_lane(d_im.val[0], d_im.val[0], a_im.val[0], b_re_im, 1); \
387 vfmul_lane(d_re.val[1], a_re.val[1], b_re_im, 1); \
388 vfmls_lane(d_re.val[1], d_re.val[1], a_im.val[1], b_re_im, 0); \
389 vfmul_lane(d_im.val[1], a_re.val[1], b_re_im, 0); \
390 vfmla_lane(d_im.val[1], d_im.val[1], a_im.val[1], b_re_im, 1); \
391 vfmul_lane(d_re.val[2], a_re.val[2], b_re_im, 1); \
392 vfmls_lane(d_re.val[2], d_re.val[2], a_im.val[2], b_re_im, 0); \
393 vfmul_lane(d_im.val[2], a_re.val[2], b_re_im, 0); \
394 vfmla_lane(d_im.val[2], d_im.val[2], a_im.val[2], b_re_im, 1); \
395 vfmul_lane(d_re.val[3], a_re.val[3], b_re_im, 1); \
396 vfmls_lane(d_re.val[3], d_re.val[3], a_im.val[3], b_re_im, 0); \
397 vfmul_lane(d_im.val[3], a_re.val[3], b_re_im, 0); \
398 vfmla_lane(d_im.val[3], d_im.val[3], a_im.val[3], b_re_im, 1);
399
400#define FPC_MUL_BOTJm(d_re, d_im, a_re, a_im, b_re, b_im) \
401 vfmul(d_re, a_re, b_im); \
402 vfmls(d_re, d_re, a_im, b_re); \
403 vfmul(d_im, a_im, b_im); \
404 vfmla(d_im, d_im, a_re, b_re);
405
406#define INV_TOPJm(t_re, t_im, a_re, a_im, b_re, b_im) \
407 FPC_SUB(t_re, t_im, b_re, b_im, a_re, a_im); \
408 FPC_ADD(a_re, a_im, a_re, a_im, b_re, b_im);
409
410#define INV_TOPJmx4(t_re, t_im, a_re, a_im, b_re, b_im) \
411 FPC_SUBx4(t_re, t_im, b_re, b_im, a_re, a_im); \
412 FPC_ADDx4(a_re, a_im, a_re, a_im, b_re, b_im);
413
414#define INV_BOTJm(b_re, b_im, t_re, t_im, zeta_re, zeta_im) FPC_MUL_BOTJm(b_re, b_im, t_re, t_im, zeta_re, zeta_im);
415
416#define INV_BOTJm_LANE(b_re, b_im, t_re, t_im, zeta) FPC_MUL_BOTJm_LANE(b_re, b_im, t_re, t_im, zeta);
417
418#define INV_BOTJm_LANEx4(b_re, b_im, t_re, t_im, zeta) FPC_MUL_BOTJm_LANEx4(b_re, b_im, t_re, t_im, zeta);
419
420#endif