poulpy/spqlios/lib/spqlios/ext/neon_accel/macrofx4.h
spqlios/lib/spqlios/ext/neon_accel/macrofx4.hRaw
1/*
2 * This file is extracted from the implementation of the FFT on Arm64/Neon
3 * available in https://github.com/cothan/Falcon-Arm (neon/macrof.h).
4 * =============================================================================
5 * Copyright (c) 2022 by Cryptographic Engineering Research Group (CERG)
6 * ECE Department, George Mason University
7 * Fairfax, VA, U.S.A.
8 * @author: Duc Tri Nguyen dnguye69@gmu.edu, cothannguyen@gmail.com
9 * Licensed under the Apache License, Version 2.0 (the "License");
10 * =============================================================================
11 *
12 * This 64-bit Floating point NEON macro x4 has not been modified and is provided as is.
13 */
14
15#ifndef MACROFX4_H
16#define MACROFX4_H
17
18#include <arm_neon.h>
19#include "macrof.h"
20
21#define vloadx4(c, addr) c = vld1q_f64_x4(addr);
22
23#define vstorex4(addr, c) vst1q_f64_x4(addr, c);
24
25#define vfdupx4(c, constant) \
26 c.val[0] = vdupq_n_f64(constant); \
27 c.val[1] = vdupq_n_f64(constant); \
28 c.val[2] = vdupq_n_f64(constant); \
29 c.val[3] = vdupq_n_f64(constant);
30
31#define vfnegx4(c, a) \
32 c.val[0] = vnegq_f64(a.val[0]); \
33 c.val[1] = vnegq_f64(a.val[1]); \
34 c.val[2] = vnegq_f64(a.val[2]); \
35 c.val[3] = vnegq_f64(a.val[3]);
36
37#define vfmulnx4(c, a, n) \
38 c.val[0] = vmulq_n_f64(a.val[0], n); \
39 c.val[1] = vmulq_n_f64(a.val[1], n); \
40 c.val[2] = vmulq_n_f64(a.val[2], n); \
41 c.val[3] = vmulq_n_f64(a.val[3], n);
42
43// c = a - b
44#define vfsubx4(c, a, b) \
45 c.val[0] = vsubq_f64(a.val[0], b.val[0]); \
46 c.val[1] = vsubq_f64(a.val[1], b.val[1]); \
47 c.val[2] = vsubq_f64(a.val[2], b.val[2]); \
48 c.val[3] = vsubq_f64(a.val[3], b.val[3]);
49
50// c = a + b
51#define vfaddx4(c, a, b) \
52 c.val[0] = vaddq_f64(a.val[0], b.val[0]); \
53 c.val[1] = vaddq_f64(a.val[1], b.val[1]); \
54 c.val[2] = vaddq_f64(a.val[2], b.val[2]); \
55 c.val[3] = vaddq_f64(a.val[3], b.val[3]);
56
57#define vfmulx4(c, a, b) \
58 c.val[0] = vmulq_f64(a.val[0], b.val[0]); \
59 c.val[1] = vmulq_f64(a.val[1], b.val[1]); \
60 c.val[2] = vmulq_f64(a.val[2], b.val[2]); \
61 c.val[3] = vmulq_f64(a.val[3], b.val[3]);
62
63#define vfmulx4_i(c, a, b) \
64 c.val[0] = vmulq_f64(a.val[0], b); \
65 c.val[1] = vmulq_f64(a.val[1], b); \
66 c.val[2] = vmulq_f64(a.val[2], b); \
67 c.val[3] = vmulq_f64(a.val[3], b);
68
69#define vfinvx4(c, a) \
70 c.val[0] = vdivq_f64(vdupq_n_f64(1.0), a.val[0]); \
71 c.val[1] = vdivq_f64(vdupq_n_f64(1.0), a.val[1]); \
72 c.val[2] = vdivq_f64(vdupq_n_f64(1.0), a.val[2]); \
73 c.val[3] = vdivq_f64(vdupq_n_f64(1.0), a.val[3]);
74
75#define vfcvtx4(c, a) \
76 c.val[0] = vcvtq_f64_s64(a.val[0]); \
77 c.val[1] = vcvtq_f64_s64(a.val[1]); \
78 c.val[2] = vcvtq_f64_s64(a.val[2]); \
79 c.val[3] = vcvtq_f64_s64(a.val[3]);
80
81#define vfmlax4(d, c, a, b) \
82 vfmla(d.val[0], c.val[0], a.val[0], b.val[0]); \
83 vfmla(d.val[1], c.val[1], a.val[1], b.val[1]); \
84 vfmla(d.val[2], c.val[2], a.val[2], b.val[2]); \
85 vfmla(d.val[3], c.val[3], a.val[3], b.val[3]);
86
87#define vfmlsx4(d, c, a, b) \
88 vfmls(d.val[0], c.val[0], a.val[0], b.val[0]); \
89 vfmls(d.val[1], c.val[1], a.val[1], b.val[1]); \
90 vfmls(d.val[2], c.val[2], a.val[2], b.val[2]); \
91 vfmls(d.val[3], c.val[3], a.val[3], b.val[3]);
92
93#define vfrintx4(c, a) \
94 c.val[0] = vcvtnq_s64_f64(a.val[0]); \
95 c.val[1] = vcvtnq_s64_f64(a.val[1]); \
96 c.val[2] = vcvtnq_s64_f64(a.val[2]); \
97 c.val[3] = vcvtnq_s64_f64(a.val[3]);
98
99/*
100 * Wrapper for FFT, split/merge and poly_float.c
101 */
102
103#define FPC_MUL(d_re, d_im, a_re, a_im, b_re, b_im) \
104 vfmul(d_re, a_re, b_re); \
105 vfmls(d_re, d_re, a_im, b_im); \
106 vfmul(d_im, a_re, b_im); \
107 vfmla(d_im, d_im, a_im, b_re);
108
109#define FPC_MULx2(d_re, d_im, a_re, a_im, b_re, b_im) \
110 vfmul(d_re.val[0], a_re.val[0], b_re.val[0]); \
111 vfmls(d_re.val[0], d_re.val[0], a_im.val[0], b_im.val[0]); \
112 vfmul(d_re.val[1], a_re.val[1], b_re.val[1]); \
113 vfmls(d_re.val[1], d_re.val[1], a_im.val[1], b_im.val[1]); \
114 vfmul(d_im.val[0], a_re.val[0], b_im.val[0]); \
115 vfmla(d_im.val[0], d_im.val[0], a_im.val[0], b_re.val[0]); \
116 vfmul(d_im.val[1], a_re.val[1], b_im.val[1]); \
117 vfmla(d_im.val[1], d_im.val[1], a_im.val[1], b_re.val[1]);
118
119#define FPC_MULx4(d_re, d_im, a_re, a_im, b_re, b_im) \
120 vfmul(d_re.val[0], a_re.val[0], b_re.val[0]); \
121 vfmls(d_re.val[0], d_re.val[0], a_im.val[0], b_im.val[0]); \
122 vfmul(d_re.val[1], a_re.val[1], b_re.val[1]); \
123 vfmls(d_re.val[1], d_re.val[1], a_im.val[1], b_im.val[1]); \
124 vfmul(d_re.val[2], a_re.val[2], b_re.val[2]); \
125 vfmls(d_re.val[2], d_re.val[2], a_im.val[2], b_im.val[2]); \
126 vfmul(d_re.val[3], a_re.val[3], b_re.val[3]); \
127 vfmls(d_re.val[3], d_re.val[3], a_im.val[3], b_im.val[3]); \
128 vfmul(d_im.val[0], a_re.val[0], b_im.val[0]); \
129 vfmla(d_im.val[0], d_im.val[0], a_im.val[0], b_re.val[0]); \
130 vfmul(d_im.val[1], a_re.val[1], b_im.val[1]); \
131 vfmla(d_im.val[1], d_im.val[1], a_im.val[1], b_re.val[1]); \
132 vfmul(d_im.val[2], a_re.val[2], b_im.val[2]); \
133 vfmla(d_im.val[2], d_im.val[2], a_im.val[2], b_re.val[2]); \
134 vfmul(d_im.val[3], a_re.val[3], b_im.val[3]); \
135 vfmla(d_im.val[3], d_im.val[3], a_im.val[3], b_re.val[3]);
136
137#define FPC_MLA(d_re, d_im, a_re, a_im, b_re, b_im) \
138 vfmla(d_re, d_re, a_re, b_re); \
139 vfmls(d_re, d_re, a_im, b_im); \
140 vfmla(d_im, d_im, a_re, b_im); \
141 vfmla(d_im, d_im, a_im, b_re);
142
143#define FPC_MLAx2(d_re, d_im, a_re, a_im, b_re, b_im) \
144 vfmla(d_re.val[0], d_re.val[0], a_re.val[0], b_re.val[0]); \
145 vfmls(d_re.val[0], d_re.val[0], a_im.val[0], b_im.val[0]); \
146 vfmla(d_re.val[1], d_re.val[1], a_re.val[1], b_re.val[1]); \
147 vfmls(d_re.val[1], d_re.val[1], a_im.val[1], b_im.val[1]); \
148 vfmla(d_im.val[0], d_im.val[0], a_re.val[0], b_im.val[0]); \
149 vfmla(d_im.val[0], d_im.val[0], a_im.val[0], b_re.val[0]); \
150 vfmla(d_im.val[1], d_im.val[1], a_re.val[1], b_im.val[1]); \
151 vfmla(d_im.val[1], d_im.val[1], a_im.val[1], b_re.val[1]);
152
153#define FPC_MLAx4(d_re, d_im, a_re, a_im, b_re, b_im) \
154 vfmla(d_re.val[0], d_re.val[0], a_re.val[0], b_re.val[0]); \
155 vfmls(d_re.val[0], d_re.val[0], a_im.val[0], b_im.val[0]); \
156 vfmla(d_re.val[1], d_re.val[1], a_re.val[1], b_re.val[1]); \
157 vfmls(d_re.val[1], d_re.val[1], a_im.val[1], b_im.val[1]); \
158 vfmla(d_re.val[2], d_re.val[2], a_re.val[2], b_re.val[2]); \
159 vfmls(d_re.val[2], d_re.val[2], a_im.val[2], b_im.val[2]); \
160 vfmla(d_re.val[3], d_re.val[3], a_re.val[3], b_re.val[3]); \
161 vfmls(d_re.val[3], d_re.val[3], a_im.val[3], b_im.val[3]); \
162 vfmla(d_im.val[0], d_im.val[0], a_re.val[0], b_im.val[0]); \
163 vfmla(d_im.val[0], d_im.val[0], a_im.val[0], b_re.val[0]); \
164 vfmla(d_im.val[1], d_im.val[1], a_re.val[1], b_im.val[1]); \
165 vfmla(d_im.val[1], d_im.val[1], a_im.val[1], b_re.val[1]); \
166 vfmla(d_im.val[2], d_im.val[2], a_re.val[2], b_im.val[2]); \
167 vfmla(d_im.val[2], d_im.val[2], a_im.val[2], b_re.val[2]); \
168 vfmla(d_im.val[3], d_im.val[3], a_re.val[3], b_im.val[3]); \
169 vfmla(d_im.val[3], d_im.val[3], a_im.val[3], b_re.val[3]);
170
171#define FPC_MUL_CONJx4(d_re, d_im, a_re, a_im, b_re, b_im) \
172 vfmul(d_re.val[0], b_im.val[0], a_im.val[0]); \
173 vfmla(d_re.val[0], d_re.val[0], a_re.val[0], b_re.val[0]); \
174 vfmul(d_re.val[1], b_im.val[1], a_im.val[1]); \
175 vfmla(d_re.val[1], d_re.val[1], a_re.val[1], b_re.val[1]); \
176 vfmul(d_re.val[2], b_im.val[2], a_im.val[2]); \
177 vfmla(d_re.val[2], d_re.val[2], a_re.val[2], b_re.val[2]); \
178 vfmul(d_re.val[3], b_im.val[3], a_im.val[3]); \
179 vfmla(d_re.val[3], d_re.val[3], a_re.val[3], b_re.val[3]); \
180 vfmul(d_im.val[0], b_re.val[0], a_im.val[0]); \
181 vfmls(d_im.val[0], d_im.val[0], a_re.val[0], b_im.val[0]); \
182 vfmul(d_im.val[1], b_re.val[1], a_im.val[1]); \
183 vfmls(d_im.val[1], d_im.val[1], a_re.val[1], b_im.val[1]); \
184 vfmul(d_im.val[2], b_re.val[2], a_im.val[2]); \
185 vfmls(d_im.val[2], d_im.val[2], a_re.val[2], b_im.val[2]); \
186 vfmul(d_im.val[3], b_re.val[3], a_im.val[3]); \
187 vfmls(d_im.val[3], d_im.val[3], a_re.val[3], b_im.val[3]);
188
189#define FPC_MLA_CONJx4(d_re, d_im, a_re, a_im, b_re, b_im) \
190 vfmla(d_re.val[0], d_re.val[0], b_im.val[0], a_im.val[0]); \
191 vfmla(d_re.val[0], d_re.val[0], a_re.val[0], b_re.val[0]); \
192 vfmla(d_re.val[1], d_re.val[1], b_im.val[1], a_im.val[1]); \
193 vfmla(d_re.val[1], d_re.val[1], a_re.val[1], b_re.val[1]); \
194 vfmla(d_re.val[2], d_re.val[2], b_im.val[2], a_im.val[2]); \
195 vfmla(d_re.val[2], d_re.val[2], a_re.val[2], b_re.val[2]); \
196 vfmla(d_re.val[3], d_re.val[3], b_im.val[3], a_im.val[3]); \
197 vfmla(d_re.val[3], d_re.val[3], a_re.val[3], b_re.val[3]); \
198 vfmla(d_im.val[0], d_im.val[0], b_re.val[0], a_im.val[0]); \
199 vfmls(d_im.val[0], d_im.val[0], a_re.val[0], b_im.val[0]); \
200 vfmla(d_im.val[1], d_im.val[1], b_re.val[1], a_im.val[1]); \
201 vfmls(d_im.val[1], d_im.val[1], a_re.val[1], b_im.val[1]); \
202 vfmla(d_im.val[2], d_im.val[2], b_re.val[2], a_im.val[2]); \
203 vfmls(d_im.val[2], d_im.val[2], a_re.val[2], b_im.val[2]); \
204 vfmla(d_im.val[3], d_im.val[3], b_re.val[3], a_im.val[3]); \
205 vfmls(d_im.val[3], d_im.val[3], a_re.val[3], b_im.val[3]);
206
207#define FPC_MUL_LANE(d_re, d_im, a_re, a_im, b_re_im) \
208 vfmul_lane(d_re, a_re, b_re_im, 0); \
209 vfmls_lane(d_re, d_re, a_im, b_re_im, 1); \
210 vfmul_lane(d_im, a_re, b_re_im, 1); \
211 vfmla_lane(d_im, d_im, a_im, b_re_im, 0);
212
213#define FPC_MUL_LANEx4(d_re, d_im, a_re, a_im, b_re_im) \
214 vfmul_lane(d_re.val[0], a_re.val[0], b_re_im, 0); \
215 vfmls_lane(d_re.val[0], d_re.val[0], a_im.val[0], b_re_im, 1); \
216 vfmul_lane(d_re.val[1], a_re.val[1], b_re_im, 0); \
217 vfmls_lane(d_re.val[1], d_re.val[1], a_im.val[1], b_re_im, 1); \
218 vfmul_lane(d_re.val[2], a_re.val[2], b_re_im, 0); \
219 vfmls_lane(d_re.val[2], d_re.val[2], a_im.val[2], b_re_im, 1); \
220 vfmul_lane(d_re.val[3], a_re.val[3], b_re_im, 0); \
221 vfmls_lane(d_re.val[3], d_re.val[3], a_im.val[3], b_re_im, 1); \
222 vfmul_lane(d_im.val[0], a_re.val[0], b_re_im, 1); \
223 vfmla_lane(d_im.val[0], d_im.val[0], a_im.val[0], b_re_im, 0); \
224 vfmul_lane(d_im.val[1], a_re.val[1], b_re_im, 1); \
225 vfmla_lane(d_im.val[1], d_im.val[1], a_im.val[1], b_re_im, 0); \
226 vfmul_lane(d_im.val[2], a_re.val[2], b_re_im, 1); \
227 vfmla_lane(d_im.val[2], d_im.val[2], a_im.val[2], b_re_im, 0); \
228 vfmul_lane(d_im.val[3], a_re.val[3], b_re_im, 1); \
229 vfmla_lane(d_im.val[3], d_im.val[3], a_im.val[3], b_re_im, 0);
230
231#define FWD_TOP(t_re, t_im, b_re, b_im, zeta_re, zeta_im) \
232 FPC_MUL(t_re, t_im, b_re, b_im, zeta_re, zeta_im);
233
234#define FWD_TOP_LANE(t_re, t_im, b_re, b_im, zeta) \
235 FPC_MUL_LANE(t_re, t_im, b_re, b_im, zeta);
236
237#define FWD_TOP_LANEx4(t_re, t_im, b_re, b_im, zeta) \
238 FPC_MUL_LANEx4(t_re, t_im, b_re, b_im, zeta);
239
240/*
241 * FPC
242 */
243
244#define FPC_SUB(d_re, d_im, a_re, a_im, b_re, b_im) \
245 d_re = vsubq_f64(a_re, b_re); \
246 d_im = vsubq_f64(a_im, b_im);
247
248#define FPC_SUBx4(d_re, d_im, a_re, a_im, b_re, b_im) \
249 d_re.val[0] = vsubq_f64(a_re.val[0], b_re.val[0]); \
250 d_im.val[0] = vsubq_f64(a_im.val[0], b_im.val[0]); \
251 d_re.val[1] = vsubq_f64(a_re.val[1], b_re.val[1]); \
252 d_im.val[1] = vsubq_f64(a_im.val[1], b_im.val[1]); \
253 d_re.val[2] = vsubq_f64(a_re.val[2], b_re.val[2]); \
254 d_im.val[2] = vsubq_f64(a_im.val[2], b_im.val[2]); \
255 d_re.val[3] = vsubq_f64(a_re.val[3], b_re.val[3]); \
256 d_im.val[3] = vsubq_f64(a_im.val[3], b_im.val[3]);
257
258#define FPC_ADD(d_re, d_im, a_re, a_im, b_re, b_im) \
259 d_re = vaddq_f64(a_re, b_re); \
260 d_im = vaddq_f64(a_im, b_im);
261
262#define FPC_ADDx4(d_re, d_im, a_re, a_im, b_re, b_im) \
263 d_re.val[0] = vaddq_f64(a_re.val[0], b_re.val[0]); \
264 d_im.val[0] = vaddq_f64(a_im.val[0], b_im.val[0]); \
265 d_re.val[1] = vaddq_f64(a_re.val[1], b_re.val[1]); \
266 d_im.val[1] = vaddq_f64(a_im.val[1], b_im.val[1]); \
267 d_re.val[2] = vaddq_f64(a_re.val[2], b_re.val[2]); \
268 d_im.val[2] = vaddq_f64(a_im.val[2], b_im.val[2]); \
269 d_re.val[3] = vaddq_f64(a_re.val[3], b_re.val[3]); \
270 d_im.val[3] = vaddq_f64(a_im.val[3], b_im.val[3]);
271
272#define FWD_BOT(a_re, a_im, b_re, b_im, t_re, t_im) \
273 FPC_SUB(b_re, b_im, a_re, a_im, t_re, t_im); \
274 FPC_ADD(a_re, a_im, a_re, a_im, t_re, t_im);
275
276#define FWD_BOTx4(a_re, a_im, b_re, b_im, t_re, t_im) \
277 FPC_SUBx4(b_re, b_im, a_re, a_im, t_re, t_im); \
278 FPC_ADDx4(a_re, a_im, a_re, a_im, t_re, t_im);
279
280/*
281 * FPC_J
282 */
283
284#define FPC_ADDJ(d_re, d_im, a_re, a_im, b_re, b_im) \
285 d_re = vsubq_f64(a_re, b_im); \
286 d_im = vaddq_f64(a_im, b_re);
287
288#define FPC_ADDJx4(d_re, d_im, a_re, a_im, b_re, b_im) \
289 d_re.val[0] = vsubq_f64(a_re.val[0], b_im.val[0]); \
290 d_im.val[0] = vaddq_f64(a_im.val[0], b_re.val[0]); \
291 d_re.val[1] = vsubq_f64(a_re.val[1], b_im.val[1]); \
292 d_im.val[1] = vaddq_f64(a_im.val[1], b_re.val[1]); \
293 d_re.val[2] = vsubq_f64(a_re.val[2], b_im.val[2]); \
294 d_im.val[2] = vaddq_f64(a_im.val[2], b_re.val[2]); \
295 d_re.val[3] = vsubq_f64(a_re.val[3], b_im.val[3]); \
296 d_im.val[3] = vaddq_f64(a_im.val[3], b_re.val[3]);
297
298#define FPC_SUBJ(d_re, d_im, a_re, a_im, b_re, b_im) \
299 d_re = vaddq_f64(a_re, b_im); \
300 d_im = vsubq_f64(a_im, b_re);
301
302#define FPC_SUBJx4(d_re, d_im, a_re, a_im, b_re, b_im) \
303 d_re.val[0] = vaddq_f64(a_re.val[0], b_im.val[0]); \
304 d_im.val[0] = vsubq_f64(a_im.val[0], b_re.val[0]); \
305 d_re.val[1] = vaddq_f64(a_re.val[1], b_im.val[1]); \
306 d_im.val[1] = vsubq_f64(a_im.val[1], b_re.val[1]); \
307 d_re.val[2] = vaddq_f64(a_re.val[2], b_im.val[2]); \
308 d_im.val[2] = vsubq_f64(a_im.val[2], b_re.val[2]); \
309 d_re.val[3] = vaddq_f64(a_re.val[3], b_im.val[3]); \
310 d_im.val[3] = vsubq_f64(a_im.val[3], b_re.val[3]);
311
312#define FWD_BOTJ(a_re, a_im, b_re, b_im, t_re, t_im) \
313 FPC_SUBJ(b_re, b_im, a_re, a_im, t_re, t_im); \
314 FPC_ADDJ(a_re, a_im, a_re, a_im, t_re, t_im);
315
316#define FWD_BOTJx4(a_re, a_im, b_re, b_im, t_re, t_im) \
317 FPC_SUBJx4(b_re, b_im, a_re, a_im, t_re, t_im); \
318 FPC_ADDJx4(a_re, a_im, a_re, a_im, t_re, t_im);
319
320//============== Inverse FFT
321/*
322 * FPC_J
323 * a * conj(b)
324 * Original (without swap):
325 * d_re = b_im * a_im + a_re * b_re;
326 * d_im = b_re * a_im - a_re * b_im;
327 */
328#define FPC_MUL_BOTJ_LANE(d_re, d_im, a_re, a_im, b_re_im) \
329 vfmul_lane(d_re, a_re, b_re_im, 0); \
330 vfmla_lane(d_re, d_re, a_im, b_re_im, 1); \
331 vfmul_lane(d_im, a_im, b_re_im, 0); \
332 vfmls_lane(d_im, d_im, a_re, b_re_im, 1);
333
334#define FPC_MUL_BOTJ_LANEx4(d_re, d_im, a_re, a_im, b_re_im) \
335 vfmul_lane(d_re.val[0], a_re.val[0], b_re_im, 0); \
336 vfmla_lane(d_re.val[0], d_re.val[0], a_im.val[0], b_re_im, 1); \
337 vfmul_lane(d_im.val[0], a_im.val[0], b_re_im, 0); \
338 vfmls_lane(d_im.val[0], d_im.val[0], a_re.val[0], b_re_im, 1); \
339 vfmul_lane(d_re.val[1], a_re.val[1], b_re_im, 0); \
340 vfmla_lane(d_re.val[1], d_re.val[1], a_im.val[1], b_re_im, 1); \
341 vfmul_lane(d_im.val[1], a_im.val[1], b_re_im, 0); \
342 vfmls_lane(d_im.val[1], d_im.val[1], a_re.val[1], b_re_im, 1); \
343 vfmul_lane(d_re.val[2], a_re.val[2], b_re_im, 0); \
344 vfmla_lane(d_re.val[2], d_re.val[2], a_im.val[2], b_re_im, 1); \
345 vfmul_lane(d_im.val[2], a_im.val[2], b_re_im, 0); \
346 vfmls_lane(d_im.val[2], d_im.val[2], a_re.val[2], b_re_im, 1); \
347 vfmul_lane(d_re.val[3], a_re.val[3], b_re_im, 0); \
348 vfmla_lane(d_re.val[3], d_re.val[3], a_im.val[3], b_re_im, 1); \
349 vfmul_lane(d_im.val[3], a_im.val[3], b_re_im, 0); \
350 vfmls_lane(d_im.val[3], d_im.val[3], a_re.val[3], b_re_im, 1);
351
352#define FPC_MUL_BOTJ(d_re, d_im, a_re, a_im, b_re, b_im) \
353 vfmul(d_re, b_im, a_im); \
354 vfmla(d_re, d_re, a_re, b_re); \
355 vfmul(d_im, b_re, a_im); \
356 vfmls(d_im, d_im, a_re, b_im);
357
358#define INV_TOPJ(t_re, t_im, a_re, a_im, b_re, b_im) \
359 FPC_SUB(t_re, t_im, a_re, a_im, b_re, b_im); \
360 FPC_ADD(a_re, a_im, a_re, a_im, b_re, b_im);
361
362#define INV_TOPJx4(t_re, t_im, a_re, a_im, b_re, b_im) \
363 FPC_SUBx4(t_re, t_im, a_re, a_im, b_re, b_im); \
364 FPC_ADDx4(a_re, a_im, a_re, a_im, b_re, b_im);
365
366#define INV_BOTJ(b_re, b_im, t_re, t_im, zeta_re, zeta_im) \
367 FPC_MUL_BOTJ(b_re, b_im, t_re, t_im, zeta_re, zeta_im);
368
369#define INV_BOTJ_LANE(b_re, b_im, t_re, t_im, zeta) \
370 FPC_MUL_BOTJ_LANE(b_re, b_im, t_re, t_im, zeta);
371
372#define INV_BOTJ_LANEx4(b_re, b_im, t_re, t_im, zeta) \
373 FPC_MUL_BOTJ_LANEx4(b_re, b_im, t_re, t_im, zeta);
374
375/*
376 * FPC_Jm
377 * a * -conj(b)
378 * d_re = a_re * b_im - a_im * b_re;
379 * d_im = a_im * b_im + a_re * b_re;
380 */
381#define FPC_MUL_BOTJm_LANE(d_re, d_im, a_re, a_im, b_re_im) \
382 vfmul_lane(d_re, a_re, b_re_im, 1); \
383 vfmls_lane(d_re, d_re, a_im, b_re_im, 0); \
384 vfmul_lane(d_im, a_re, b_re_im, 0); \
385 vfmla_lane(d_im, d_im, a_im, b_re_im, 1);
386
387#define FPC_MUL_BOTJm_LANEx4(d_re, d_im, a_re, a_im, b_re_im) \
388 vfmul_lane(d_re.val[0], a_re.val[0], b_re_im, 1); \
389 vfmls_lane(d_re.val[0], d_re.val[0], a_im.val[0], b_re_im, 0); \
390 vfmul_lane(d_im.val[0], a_re.val[0], b_re_im, 0); \
391 vfmla_lane(d_im.val[0], d_im.val[0], a_im.val[0], b_re_im, 1); \
392 vfmul_lane(d_re.val[1], a_re.val[1], b_re_im, 1); \
393 vfmls_lane(d_re.val[1], d_re.val[1], a_im.val[1], b_re_im, 0); \
394 vfmul_lane(d_im.val[1], a_re.val[1], b_re_im, 0); \
395 vfmla_lane(d_im.val[1], d_im.val[1], a_im.val[1], b_re_im, 1); \
396 vfmul_lane(d_re.val[2], a_re.val[2], b_re_im, 1); \
397 vfmls_lane(d_re.val[2], d_re.val[2], a_im.val[2], b_re_im, 0); \
398 vfmul_lane(d_im.val[2], a_re.val[2], b_re_im, 0); \
399 vfmla_lane(d_im.val[2], d_im.val[2], a_im.val[2], b_re_im, 1); \
400 vfmul_lane(d_re.val[3], a_re.val[3], b_re_im, 1); \
401 vfmls_lane(d_re.val[3], d_re.val[3], a_im.val[3], b_re_im, 0); \
402 vfmul_lane(d_im.val[3], a_re.val[3], b_re_im, 0); \
403 vfmla_lane(d_im.val[3], d_im.val[3], a_im.val[3], b_re_im, 1);
404
405#define FPC_MUL_BOTJm(d_re, d_im, a_re, a_im, b_re, b_im) \
406 vfmul(d_re, a_re, b_im); \
407 vfmls(d_re, d_re, a_im, b_re); \
408 vfmul(d_im, a_im, b_im); \
409 vfmla(d_im, d_im, a_re, b_re);
410
411#define INV_TOPJm(t_re, t_im, a_re, a_im, b_re, b_im) \
412 FPC_SUB(t_re, t_im, b_re, b_im, a_re, a_im); \
413 FPC_ADD(a_re, a_im, a_re, a_im, b_re, b_im);
414
415#define INV_TOPJmx4(t_re, t_im, a_re, a_im, b_re, b_im) \
416 FPC_SUBx4(t_re, t_im, b_re, b_im, a_re, a_im); \
417 FPC_ADDx4(a_re, a_im, a_re, a_im, b_re, b_im);
418
419#define INV_BOTJm(b_re, b_im, t_re, t_im, zeta_re, zeta_im) \
420 FPC_MUL_BOTJm(b_re, b_im, t_re, t_im, zeta_re, zeta_im);
421
422#define INV_BOTJm_LANE(b_re, b_im, t_re, t_im, zeta) \
423 FPC_MUL_BOTJm_LANE(b_re, b_im, t_re, t_im, zeta);
424
425#define INV_BOTJm_LANEx4(b_re, b_im, t_re, t_im, zeta) \
426 FPC_MUL_BOTJm_LANEx4(b_re, b_im, t_re, t_im, zeta);
427
428#endif