poulpy/poulpy-cpu-avx/src/vec_znx.rs
poulpy-cpu-avx/src/vec_znx.rsRaw
1use poulpy_hal::{
2 api::{
3 TakeSlice, VecZnxAutomorphismInplaceTmpBytes, VecZnxLshTmpBytes, VecZnxMergeRingsTmpBytes,
4 VecZnxMulXpMinusOneInplaceTmpBytes, VecZnxNormalizeTmpBytes, VecZnxRotateInplaceTmpBytes, VecZnxRshTmpBytes,
5 VecZnxSplitRingTmpBytes,
6 },
7 layouts::{Module, ScalarZnxToRef, Scratch, VecZnxToMut, VecZnxToRef},
8 oep::{
9 TakeSliceImpl, VecZnxAddImpl, VecZnxAddInplaceImpl, VecZnxAddNormalImpl, VecZnxAddScalarImpl, VecZnxAddScalarInplaceImpl,
10 VecZnxAutomorphismImpl, VecZnxAutomorphismInplaceImpl, VecZnxAutomorphismInplaceTmpBytesImpl, VecZnxCopyImpl,
11 VecZnxFillNormalImpl, VecZnxFillUniformImpl, VecZnxLshImpl, VecZnxLshInplaceImpl, VecZnxLshTmpBytesImpl,
12 VecZnxMergeRingsImpl, VecZnxMergeRingsTmpBytesImpl, VecZnxMulXpMinusOneImpl, VecZnxMulXpMinusOneInplaceImpl,
13 VecZnxMulXpMinusOneInplaceTmpBytesImpl, VecZnxNegateImpl, VecZnxNegateInplaceImpl, VecZnxNormalizeImpl,
14 VecZnxNormalizeInplaceImpl, VecZnxNormalizeTmpBytesImpl, VecZnxRotateImpl, VecZnxRotateInplaceImpl,
15 VecZnxRotateInplaceTmpBytesImpl, VecZnxRshImpl, VecZnxRshInplaceImpl, VecZnxRshTmpBytesImpl, VecZnxSplitRingImpl,
16 VecZnxSplitRingTmpBytesImpl, VecZnxSubImpl, VecZnxSubInplaceImpl, VecZnxSubNegateInplaceImpl, VecZnxSubScalarImpl,
17 VecZnxSubScalarInplaceImpl, VecZnxSwitchRingImpl, VecZnxZeroImpl,
18 },
19 reference::vec_znx::{
20 vec_znx_add, vec_znx_add_inplace, vec_znx_add_normal_ref, vec_znx_add_scalar, vec_znx_add_scalar_inplace,
21 vec_znx_automorphism, vec_znx_automorphism_inplace, vec_znx_automorphism_inplace_tmp_bytes, vec_znx_copy,
22 vec_znx_fill_normal_ref, vec_znx_fill_uniform_ref, vec_znx_lsh, vec_znx_lsh_inplace, vec_znx_lsh_tmp_bytes,
23 vec_znx_merge_rings, vec_znx_merge_rings_tmp_bytes, vec_znx_mul_xp_minus_one, vec_znx_mul_xp_minus_one_inplace,
24 vec_znx_mul_xp_minus_one_inplace_tmp_bytes, vec_znx_negate, vec_znx_negate_inplace, vec_znx_normalize,
25 vec_znx_normalize_inplace, vec_znx_normalize_tmp_bytes, vec_znx_rotate, vec_znx_rotate_inplace,
26 vec_znx_rotate_inplace_tmp_bytes, vec_znx_rsh, vec_znx_rsh_inplace, vec_znx_rsh_tmp_bytes, vec_znx_split_ring,
27 vec_znx_split_ring_tmp_bytes, vec_znx_sub, vec_znx_sub_inplace, vec_znx_sub_negate_inplace, vec_znx_sub_scalar,
28 vec_znx_sub_scalar_inplace, vec_znx_switch_ring, vec_znx_zero,
29 },
30 source::Source,
31};
32
33use crate::FFT64Avx;
34
35unsafe impl VecZnxZeroImpl<Self> for FFT64Avx {
36 fn vec_znx_zero_impl<R>(_module: &Module<Self>, res: &mut R, res_col: usize)
37 where
38 R: VecZnxToMut,
39 {
40 vec_znx_zero::<_, FFT64Avx>(res, res_col);
41 }
42}
43
44unsafe impl VecZnxNormalizeTmpBytesImpl<Self> for FFT64Avx {
45 fn vec_znx_normalize_tmp_bytes_impl(module: &Module<Self>) -> usize {
46 vec_znx_normalize_tmp_bytes(module.n())
47 }
48}
49
50unsafe impl VecZnxNormalizeImpl<Self> for FFT64Avx
51where
52 Self: TakeSliceImpl<Self> + VecZnxNormalizeTmpBytesImpl<Self>,
53{
54 fn vec_znx_normalize_impl<R, A>(
55 module: &Module<Self>,
56 res: &mut R,
57 res_base2k: usize,
58 res_offset: i64,
59 res_col: usize,
60 a: &A,
61 a_base2k: usize,
62 a_col: usize,
63 scratch: &mut Scratch<Self>,
64 ) where
65 R: VecZnxToMut,
66 A: VecZnxToRef,
67 {
68 let (carry, _) = scratch.take_slice(module.vec_znx_normalize_tmp_bytes() / size_of::<i64>());
69 vec_znx_normalize::<R, A, Self>(res, res_base2k, res_offset, res_col, a, a_base2k, a_col, carry);
70 }
71}
72
73unsafe impl VecZnxNormalizeInplaceImpl<Self> for FFT64Avx
74where
75 Self: TakeSliceImpl<Self> + VecZnxNormalizeTmpBytesImpl<Self>,
76{
77 fn vec_znx_normalize_inplace_impl<R>(
78 module: &Module<Self>,
79 base2k: usize,
80 res: &mut R,
81 res_col: usize,
82 scratch: &mut Scratch<Self>,
83 ) where
84 R: VecZnxToMut,
85 {
86 let (carry, _) = scratch.take_slice(module.vec_znx_normalize_tmp_bytes() / size_of::<i64>());
87 vec_znx_normalize_inplace::<R, Self>(base2k, res, res_col, carry);
88 }
89}
90
91unsafe impl VecZnxAddImpl<Self> for FFT64Avx {
92 fn vec_znx_add_impl<R, A, B>(_module: &Module<Self>, res: &mut R, res_col: usize, a: &A, a_col: usize, b: &B, b_col: usize)
93 where
94 R: VecZnxToMut,
95 A: VecZnxToRef,
96 B: VecZnxToRef,
97 {
98 vec_znx_add::<R, A, B, Self>(res, res_col, a, a_col, b, b_col);
99 }
100}
101
102unsafe impl VecZnxAddInplaceImpl<Self> for FFT64Avx {
103 fn vec_znx_add_inplace_impl<R, A>(_module: &Module<Self>, res: &mut R, res_col: usize, a: &A, a_col: usize)
104 where
105 R: VecZnxToMut,
106 A: VecZnxToRef,
107 {
108 vec_znx_add_inplace::<R, A, Self>(res, res_col, a, a_col);
109 }
110}
111
112unsafe impl VecZnxAddScalarInplaceImpl<Self> for FFT64Avx {
113 fn vec_znx_add_scalar_inplace_impl<R, A>(
114 _module: &Module<Self>,
115 res: &mut R,
116 res_col: usize,
117 res_limb: usize,
118 a: &A,
119 a_col: usize,
120 ) where
121 R: VecZnxToMut,
122 A: ScalarZnxToRef,
123 {
124 vec_znx_add_scalar_inplace::<R, A, Self>(res, res_col, res_limb, a, a_col);
125 }
126}
127
128unsafe impl VecZnxAddScalarImpl<Self> for FFT64Avx {
129 fn vec_znx_add_scalar_impl<R, A, B>(
130 _module: &Module<Self>,
131 res: &mut R,
132 res_col: usize,
133 a: &A,
134 a_col: usize,
135 b: &B,
136 b_col: usize,
137 b_limb: usize,
138 ) where
139 R: VecZnxToMut,
140 A: ScalarZnxToRef,
141 B: VecZnxToRef,
142 {
143 vec_znx_add_scalar::<R, A, B, Self>(res, res_col, a, a_col, b, b_col, b_limb);
144 }
145}
146
147unsafe impl VecZnxSubImpl<Self> for FFT64Avx {
148 fn vec_znx_sub_impl<R, A, B>(_module: &Module<Self>, res: &mut R, res_col: usize, a: &A, a_col: usize, b: &B, b_col: usize)
149 where
150 R: VecZnxToMut,
151 A: VecZnxToRef,
152 B: VecZnxToRef,
153 {
154 vec_znx_sub::<R, A, B, Self>(res, res_col, a, a_col, b, b_col);
155 }
156}
157
158unsafe impl VecZnxSubInplaceImpl<Self> for FFT64Avx {
159 fn vec_znx_sub_inplace_impl<R, A>(_module: &Module<Self>, res: &mut R, res_col: usize, a: &A, a_col: usize)
160 where
161 R: VecZnxToMut,
162 A: VecZnxToRef,
163 {
164 vec_znx_sub_inplace::<R, A, Self>(res, res_col, a, a_col);
165 }
166}
167
168unsafe impl VecZnxSubNegateInplaceImpl<Self> for FFT64Avx {
169 fn vec_znx_sub_negate_inplace_impl<R, A>(_module: &Module<Self>, res: &mut R, res_col: usize, a: &A, a_col: usize)
170 where
171 R: VecZnxToMut,
172 A: VecZnxToRef,
173 {
174 vec_znx_sub_negate_inplace::<R, A, Self>(res, res_col, a, a_col);
175 }
176}
177
178unsafe impl VecZnxSubScalarImpl<Self> for FFT64Avx {
179 fn vec_znx_sub_scalar_impl<R, A, B>(
180 _module: &Module<Self>,
181 res: &mut R,
182 res_col: usize,
183 a: &A,
184 a_col: usize,
185 b: &B,
186 b_col: usize,
187 b_limb: usize,
188 ) where
189 R: VecZnxToMut,
190 A: ScalarZnxToRef,
191 B: VecZnxToRef,
192 {
193 vec_znx_sub_scalar::<R, A, B, Self>(res, res_col, a, a_col, b, b_col, b_limb);
194 }
195}
196
197unsafe impl VecZnxSubScalarInplaceImpl<Self> for FFT64Avx {
198 fn vec_znx_sub_scalar_inplace_impl<R, A>(
199 _module: &Module<Self>,
200 res: &mut R,
201 res_col: usize,
202 res_limb: usize,
203 a: &A,
204 a_col: usize,
205 ) where
206 R: VecZnxToMut,
207 A: ScalarZnxToRef,
208 {
209 vec_znx_sub_scalar_inplace::<R, A, Self>(res, res_col, res_limb, a, a_col);
210 }
211}
212
213unsafe impl VecZnxNegateImpl<Self> for FFT64Avx {
214 fn vec_znx_negate_impl<R, A>(_module: &Module<Self>, res: &mut R, res_col: usize, a: &A, a_col: usize)
215 where
216 R: VecZnxToMut,
217 A: VecZnxToRef,
218 {
219 vec_znx_negate::<R, A, Self>(res, res_col, a, a_col);
220 }
221}
222
223unsafe impl VecZnxNegateInplaceImpl<Self> for FFT64Avx {
224 fn vec_znx_negate_inplace_impl<R>(_module: &Module<Self>, res: &mut R, res_col: usize)
225 where
226 R: VecZnxToMut,
227 {
228 vec_znx_negate_inplace::<R, Self>(res, res_col);
229 }
230}
231
232unsafe impl VecZnxLshTmpBytesImpl<Self> for FFT64Avx {
233 fn vec_znx_lsh_tmp_bytes_impl(module: &Module<Self>) -> usize {
234 vec_znx_lsh_tmp_bytes(module.n())
235 }
236}
237
238unsafe impl VecZnxRshTmpBytesImpl<Self> for FFT64Avx {
239 fn vec_znx_rsh_tmp_bytes_impl(module: &Module<Self>) -> usize {
240 vec_znx_rsh_tmp_bytes(module.n())
241 }
242}
243
244unsafe impl VecZnxLshImpl<Self> for FFT64Avx
245where
246 Module<Self>: VecZnxNormalizeTmpBytes,
247 Scratch<Self>: TakeSlice,
248{
249 fn vec_znx_lsh_impl<R, A>(
250 module: &Module<Self>,
251 base2k: usize,
252 k: usize,
253 res: &mut R,
254 res_col: usize,
255 a: &A,
256 a_col: usize,
257 scratch: &mut Scratch<Self>,
258 ) where
259 R: VecZnxToMut,
260 A: VecZnxToRef,
261 {
262 let (carry, _) = scratch.take_slice(module.vec_znx_lsh_tmp_bytes() / size_of::<i64>());
263 vec_znx_lsh::<_, _, Self>(base2k, k, res, res_col, a, a_col, carry);
264 }
265}
266
267unsafe impl VecZnxLshInplaceImpl<Self> for FFT64Avx
268where
269 Module<Self>: VecZnxNormalizeTmpBytes,
270 Scratch<Self>: TakeSlice,
271{
272 fn vec_znx_lsh_inplace_impl<A>(
273 module: &Module<Self>,
274 base2k: usize,
275 k: usize,
276 a: &mut A,
277 a_col: usize,
278 scratch: &mut Scratch<Self>,
279 ) where
280 A: VecZnxToMut,
281 {
282 let (carry, _) = scratch.take_slice(module.vec_znx_lsh_tmp_bytes() / size_of::<i64>());
283 vec_znx_lsh_inplace::<_, Self>(base2k, k, a, a_col, carry);
284 }
285}
286
287unsafe impl VecZnxRshImpl<Self> for FFT64Avx
288where
289 Module<Self>: VecZnxNormalizeTmpBytes,
290 Scratch<Self>: TakeSlice,
291{
292 fn vec_znx_rsh_impl<R, A>(
293 module: &Module<Self>,
294 base2k: usize,
295 k: usize,
296 res: &mut R,
297 res_col: usize,
298 a: &A,
299 a_col: usize,
300 scratch: &mut Scratch<Self>,
301 ) where
302 R: VecZnxToMut,
303 A: VecZnxToRef,
304 {
305 let (carry, _) = scratch.take_slice(module.vec_znx_rsh_tmp_bytes() / size_of::<i64>());
306 vec_znx_rsh::<_, _, Self>(base2k, k, res, res_col, a, a_col, carry);
307 }
308}
309
310unsafe impl VecZnxRshInplaceImpl<Self> for FFT64Avx
311where
312 Module<Self>: VecZnxNormalizeTmpBytes,
313 Scratch<Self>: TakeSlice,
314{
315 fn vec_znx_rsh_inplace_impl<A>(
316 module: &Module<Self>,
317 base2k: usize,
318 k: usize,
319 a: &mut A,
320 a_col: usize,
321 scratch: &mut Scratch<Self>,
322 ) where
323 A: VecZnxToMut,
324 {
325 let (carry, _) = scratch.take_slice(module.vec_znx_rsh_tmp_bytes() / size_of::<i64>());
326 vec_znx_rsh_inplace::<_, Self>(base2k, k, a, a_col, carry);
327 }
328}
329
330unsafe impl VecZnxRotateImpl<Self> for FFT64Avx {
331 fn vec_znx_rotate_impl<R, A>(_module: &Module<Self>, p: i64, res: &mut R, res_col: usize, a: &A, a_col: usize)
332 where
333 R: VecZnxToMut,
334 A: VecZnxToRef,
335 {
336 vec_znx_rotate::<R, A, Self>(p, res, res_col, a, a_col);
337 }
338}
339
340unsafe impl VecZnxRotateInplaceTmpBytesImpl<Self> for FFT64Avx
341where
342 Scratch<Self>: TakeSlice,
343{
344 fn vec_znx_rotate_inplace_tmp_bytes_impl(module: &Module<Self>) -> usize {
345 vec_znx_rotate_inplace_tmp_bytes(module.n())
346 }
347}
348
349unsafe impl VecZnxRotateInplaceImpl<Self> for FFT64Avx
350where
351 Scratch<Self>: TakeSlice,
352 Self: VecZnxRotateInplaceTmpBytesImpl<Self>,
353{
354 fn vec_znx_rotate_inplace_impl<R>(module: &Module<Self>, p: i64, res: &mut R, res_col: usize, scratch: &mut Scratch<Self>)
355 where
356 R: VecZnxToMut,
357 {
358 let (tmp, _) = scratch.take_slice(module.vec_znx_rotate_inplace_tmp_bytes() / size_of::<i64>());
359 vec_znx_rotate_inplace::<R, Self>(p, res, res_col, tmp);
360 }
361}
362
363unsafe impl VecZnxAutomorphismImpl<Self> for FFT64Avx {
364 fn vec_znx_automorphism_impl<R, A>(_module: &Module<Self>, p: i64, res: &mut R, res_col: usize, a: &A, a_col: usize)
365 where
366 R: VecZnxToMut,
367 A: VecZnxToRef,
368 {
369 vec_znx_automorphism::<R, A, Self>(p, res, res_col, a, a_col);
370 }
371}
372
373unsafe impl VecZnxAutomorphismInplaceTmpBytesImpl<Self> for FFT64Avx {
374 fn vec_znx_automorphism_inplace_tmp_bytes_impl(module: &Module<Self>) -> usize {
375 vec_znx_automorphism_inplace_tmp_bytes(module.n())
376 }
377}
378
379unsafe impl VecZnxAutomorphismInplaceImpl<Self> for FFT64Avx
380where
381 Scratch<Self>: TakeSlice,
382 Self: VecZnxAutomorphismInplaceTmpBytesImpl<Self>,
383{
384 fn vec_znx_automorphism_inplace_impl<R>(
385 module: &Module<Self>,
386 p: i64,
387 res: &mut R,
388 res_col: usize,
389 scratch: &mut Scratch<Self>,
390 ) where
391 R: VecZnxToMut,
392 {
393 let (tmp, _) = scratch.take_slice(module.vec_znx_automorphism_inplace_tmp_bytes() / size_of::<i64>());
394 vec_znx_automorphism_inplace::<R, Self>(p, res, res_col, tmp);
395 }
396}
397
398unsafe impl VecZnxMulXpMinusOneImpl<Self> for FFT64Avx {
399 fn vec_znx_mul_xp_minus_one_impl<R, A>(_module: &Module<Self>, p: i64, res: &mut R, res_col: usize, a: &A, a_col: usize)
400 where
401 R: VecZnxToMut,
402 A: VecZnxToRef,
403 {
404 vec_znx_mul_xp_minus_one::<R, A, Self>(p, res, res_col, a, a_col);
405 }
406}
407
408unsafe impl VecZnxMulXpMinusOneInplaceTmpBytesImpl<Self> for FFT64Avx
409where
410 Scratch<Self>: TakeSlice,
411 Self: VecZnxMulXpMinusOneImpl<Self>,
412{
413 fn vec_znx_mul_xp_minus_one_inplace_tmp_bytes_impl(module: &Module<Self>) -> usize {
414 vec_znx_mul_xp_minus_one_inplace_tmp_bytes(module.n())
415 }
416}
417
418unsafe impl VecZnxMulXpMinusOneInplaceImpl<Self> for FFT64Avx {
419 fn vec_znx_mul_xp_minus_one_inplace_impl<R>(
420 module: &Module<Self>,
421 p: i64,
422 res: &mut R,
423 res_col: usize,
424 scratch: &mut Scratch<Self>,
425 ) where
426 R: VecZnxToMut,
427 {
428 let (tmp, _) = scratch.take_slice(module.vec_znx_mul_xp_minus_one_inplace_tmp_bytes() / size_of::<i64>());
429 vec_znx_mul_xp_minus_one_inplace::<R, Self>(p, res, res_col, tmp);
430 }
431}
432
433unsafe impl VecZnxSplitRingTmpBytesImpl<Self> for FFT64Avx {
434 fn vec_znx_split_ring_tmp_bytes_impl(module: &Module<Self>) -> usize {
435 vec_znx_split_ring_tmp_bytes(module.n())
436 }
437}
438
439unsafe impl VecZnxSplitRingImpl<Self> for FFT64Avx
440where
441 Module<Self>: VecZnxSplitRingTmpBytes,
442 Scratch<Self>: TakeSlice,
443{
444 fn vec_znx_split_ring_impl<R, A>(
445 module: &Module<Self>,
446 res: &mut [R],
447 res_col: usize,
448 a: &A,
449 a_col: usize,
450 scratch: &mut Scratch<Self>,
451 ) where
452 R: VecZnxToMut,
453 A: VecZnxToRef,
454 {
455 let (tmp, _) = scratch.take_slice(module.vec_znx_split_ring_tmp_bytes() / size_of::<i64>());
456 vec_znx_split_ring::<R, A, Self>(res, res_col, a, a_col, tmp);
457 }
458}
459
460unsafe impl VecZnxMergeRingsTmpBytesImpl<Self> for FFT64Avx {
461 fn vec_znx_merge_rings_tmp_bytes_impl(module: &Module<Self>) -> usize {
462 vec_znx_merge_rings_tmp_bytes(module.n())
463 }
464}
465
466unsafe impl VecZnxMergeRingsImpl<Self> for FFT64Avx
467where
468 Module<Self>: VecZnxMergeRingsTmpBytes,
469{
470 fn vec_znx_merge_rings_impl<R, A>(
471 module: &Module<Self>,
472 res: &mut R,
473 res_col: usize,
474 a: &[A],
475 a_col: usize,
476 scratch: &mut Scratch<Self>,
477 ) where
478 R: VecZnxToMut,
479 A: VecZnxToRef,
480 {
481 let (tmp, _) = scratch.take_slice(module.vec_znx_merge_rings_tmp_bytes() / size_of::<i64>());
482 vec_znx_merge_rings::<R, A, Self>(res, res_col, a, a_col, tmp);
483 }
484}
485
486unsafe impl VecZnxSwitchRingImpl<Self> for FFT64Avx
487where
488 Self: VecZnxCopyImpl<Self>,
489{
490 fn vec_znx_switch_ring_impl<R, A>(_module: &Module<Self>, res: &mut R, res_col: usize, a: &A, a_col: usize)
491 where
492 R: VecZnxToMut,
493 A: VecZnxToRef,
494 {
495 vec_znx_switch_ring::<R, A, Self>(res, res_col, a, a_col);
496 }
497}
498
499unsafe impl VecZnxCopyImpl<Self> for FFT64Avx {
500 fn vec_znx_copy_impl<R, A>(_module: &Module<Self>, res: &mut R, res_col: usize, a: &A, a_col: usize)
501 where
502 R: VecZnxToMut,
503 A: VecZnxToRef,
504 {
505 vec_znx_copy::<R, A, Self>(res, res_col, a, a_col)
506 }
507}
508
509unsafe impl VecZnxFillUniformImpl<Self> for FFT64Avx {
510 fn vec_znx_fill_uniform_impl<R>(_module: &Module<Self>, base2k: usize, res: &mut R, res_col: usize, source: &mut Source)
511 where
512 R: VecZnxToMut,
513 {
514 vec_znx_fill_uniform_ref(base2k, res, res_col, source)
515 }
516}
517
518unsafe impl VecZnxFillNormalImpl<Self> for FFT64Avx {
519 fn vec_znx_fill_normal_impl<R>(
520 _module: &Module<Self>,
521 base2k: usize,
522 res: &mut R,
523 res_col: usize,
524 k: usize,
525 source: &mut Source,
526 sigma: f64,
527 bound: f64,
528 ) where
529 R: VecZnxToMut,
530 {
531 vec_znx_fill_normal_ref(base2k, res, res_col, k, sigma, bound, source);
532 }
533}
534
535unsafe impl VecZnxAddNormalImpl<Self> for FFT64Avx {
536 fn vec_znx_add_normal_impl<R>(
537 _module: &Module<Self>,
538 base2k: usize,
539 res: &mut R,
540 res_col: usize,
541 k: usize,
542 source: &mut Source,
543 sigma: f64,
544 bound: f64,
545 ) where
546 R: VecZnxToMut,
547 {
548 vec_znx_add_normal_ref(base2k, res, res_col, k, sigma, bound, source);
549 }
550}