poulpy/spqlios/lib/test/testlib/fft64_layouts.cpp
spqlios/lib/test/testlib/fft64_layouts.cppRaw
1#include "fft64_layouts.h"
2#ifdef VALGRIND_MEM_TESTS
3#include "valgrind/memcheck.h"
4#endif
5
6void* alloc64(uint64_t size) {
7 static uint64_t _msk64 = -64;
8 if (size == 0) return nullptr;
9 uint64_t rsize = (size + 63) & _msk64;
10 uint8_t* reps = (uint8_t*)spqlios_alloc(rsize);
11 REQUIRE_DRAMATICALLY(reps != 0, "Out of memory");
12#ifdef VALGRIND_MEM_TESTS
13 VALGRIND_MAKE_MEM_NOACCESS(reps + size, rsize - size);
14#endif
15 return reps;
16}
17
18fft64_vec_znx_dft_layout::fft64_vec_znx_dft_layout(uint64_t n, uint64_t size)
19 : nn(n), //
20 size(size), //
21 data((VEC_ZNX_DFT*)alloc64(n * size * 8)), //
22 view(n / 2, size, (double*)data) {}
23
24fft64_vec_znx_dft_layout::~fft64_vec_znx_dft_layout() { spqlios_free(data); }
25
26double* fft64_vec_znx_dft_layout::get_addr(uint64_t idx) {
27 REQUIRE_DRAMATICALLY(idx < size, "index overflow " << idx << " / " << size);
28 return ((double*)data) + idx * nn;
29}
30const double* fft64_vec_znx_dft_layout::get_addr(uint64_t idx) const {
31 REQUIRE_DRAMATICALLY(idx < size, "index overflow " << idx << " / " << size);
32 return ((double*)data) + idx * nn;
33}
34reim_fft64vec fft64_vec_znx_dft_layout::get_copy_zext(uint64_t idx) const {
35 if (idx < size) {
36 return reim_fft64vec(nn, get_addr(idx));
37 } else {
38 return reim_fft64vec::zero(nn);
39 }
40}
41void fft64_vec_znx_dft_layout::fill_dft_random_log2bound(uint64_t bits) {
42 for (uint64_t i = 0; i < size; ++i) {
43 set(i, simple_fft64(znx_i64::random_log2bound(nn, bits)));
44 }
45}
46void fft64_vec_znx_dft_layout::set(uint64_t idx, const reim_fft64vec& value) {
47 REQUIRE_DRAMATICALLY(value.nn() == nn, "ring dimension mismatch");
48 value.save_as(get_addr(idx));
49}
50thash fft64_vec_znx_dft_layout::content_hash() const { return test_hash(data, size * nn * sizeof(double)); }
51
52reim4_elem fft64_vec_znx_dft_layout::get(uint64_t idx, uint64_t blk) const {
53 REQUIRE_DRAMATICALLY(idx < size, "index overflow: " << idx << " / " << size);
54 REQUIRE_DRAMATICALLY(blk < nn / 8, "blk overflow: " << blk << " / " << nn / 8);
55 double* reim = ((double*)data) + idx * nn;
56 return reim4_elem(reim + blk * 4, reim + nn / 2 + blk * 4);
57}
58reim4_elem fft64_vec_znx_dft_layout::get_zext(uint64_t idx, uint64_t blk) const {
59 REQUIRE_DRAMATICALLY(blk < nn / 8, "blk overflow: " << blk << " / " << nn / 8);
60 if (idx < size) {
61 return get(idx, blk);
62 } else {
63 return reim4_elem::zero();
64 }
65}
66void fft64_vec_znx_dft_layout::set(uint64_t idx, uint64_t blk, const reim4_elem& value) {
67 REQUIRE_DRAMATICALLY(idx < size, "index overflow: " << idx << " / " << size);
68 REQUIRE_DRAMATICALLY(blk < nn / 8, "blk overflow: " << blk << " / " << nn / 8);
69 double* reim = ((double*)data) + idx * nn;
70 value.save_re_im(reim + blk * 4, reim + nn / 2 + blk * 4);
71}
72void fft64_vec_znx_dft_layout::fill_random(double log2bound) {
73 for (uint64_t i = 0; i < size; ++i) {
74 set(i, reim_fft64vec::random(nn, log2bound));
75 }
76}
77void fft64_vec_znx_dft_layout::fill_dft_random(uint64_t log2bound) {
78 for (uint64_t i = 0; i < size; ++i) {
79 set(i, reim_fft64vec::dft_random(nn, log2bound));
80 }
81}
82
83fft64_vec_znx_big_layout::fft64_vec_znx_big_layout(uint64_t n, uint64_t size)
84 : nn(n), //
85 size(size), //
86 data((VEC_ZNX_BIG*)alloc64(n * size * 8)) {}
87
88znx_i64 fft64_vec_znx_big_layout::get_copy(uint64_t index) const {
89 REQUIRE_DRAMATICALLY(index < size, "index overflow: " << index << " / " << size);
90 return znx_i64(nn, ((int64_t*)data) + index * nn);
91}
92znx_i64 fft64_vec_znx_big_layout::get_copy_zext(uint64_t index) const {
93 if (index < size) {
94 return znx_i64(nn, ((int64_t*)data) + index * nn);
95 } else {
96 return znx_i64::zero(nn);
97 }
98}
99void fft64_vec_znx_big_layout::set(uint64_t index, const znx_i64& value) {
100 REQUIRE_DRAMATICALLY(index < size, "index overflow: " << index << " / " << size);
101 value.save_as(((int64_t*)data) + index * nn);
102}
103void fft64_vec_znx_big_layout::fill_random() {
104 for (uint64_t i = 0; i < size; ++i) {
105 set(i, znx_i64::random_log2bound(nn, 1));
106 }
107}
108fft64_vec_znx_big_layout::~fft64_vec_znx_big_layout() { spqlios_free(data); }
109
110fft64_vmp_pmat_layout::fft64_vmp_pmat_layout(uint64_t n, uint64_t nrows, uint64_t ncols)
111 : nn(n),
112 nrows(nrows),
113 ncols(ncols), //
114 data((VMP_PMAT*)alloc64(nrows * ncols * nn * 8)) {}
115
116double* fft64_vmp_pmat_layout::get_addr(uint64_t row, uint64_t col, uint64_t blk) const {
117 REQUIRE_DRAMATICALLY(row < nrows, "row overflow: " << row << " / " << nrows);
118 REQUIRE_DRAMATICALLY(col < ncols, "col overflow: " << col << " / " << ncols);
119 REQUIRE_DRAMATICALLY(blk < nn / 8, "block overflow: " << blk << " / " << (nn / 8));
120 double* d = (double*)data;
121 if (col == (ncols - 1) && (ncols % 2 == 1)) {
122 // special case: last column out of an odd column number
123 return d + blk * nrows * ncols * 8 // major: blk
124 + col * nrows * 8 // col == ncols-1
125 + row * 8;
126 } else {
127 // general case: columns go by pair
128 return d + blk * nrows * ncols * 8 // major: blk
129 + (col / 2) * (2 * nrows) * 8 // second: col pair index
130 + row * 2 * 8 // third: row index
131 + (col % 2) * 8; // minor: col in colpair
132 }
133}
134
135reim4_elem fft64_vmp_pmat_layout::get(uint64_t row, uint64_t col, uint64_t blk) const {
136 return reim4_elem(get_addr(row, col, blk));
137}
138reim4_elem fft64_vmp_pmat_layout::get_zext(uint64_t row, uint64_t col, uint64_t blk) const {
139 REQUIRE_DRAMATICALLY(blk < nn / 8, "block overflow: " << blk << " / " << (nn / 8));
140 if (row < nrows && col < ncols) {
141 return reim4_elem(get_addr(row, col, blk));
142 } else {
143 return reim4_elem::zero();
144 }
145}
146void fft64_vmp_pmat_layout::set(uint64_t row, uint64_t col, uint64_t blk, const reim4_elem& value) const {
147 value.save_as(get_addr(row, col, blk));
148}
149
150fft64_vmp_pmat_layout::~fft64_vmp_pmat_layout() { spqlios_free(data); }
151
152reim_fft64vec fft64_vmp_pmat_layout::get_zext(uint64_t row, uint64_t col) const {
153 if (row >= nrows || col >= ncols) {
154 return reim_fft64vec::zero(nn);
155 }
156 if (nn < 8) {
157 // the pmat is just col major
158 double* addr = (double*)data + (row + col * nrows) * nn;
159 return reim_fft64vec(nn, addr);
160 }
161 // otherwise, reconstruct it block by block
162 reim_fft64vec res(nn);
163 for (uint64_t blk = 0; blk < nn / 8; ++blk) {
164 reim4_elem v = get(row, col, blk);
165 res.set_blk(blk, v);
166 }
167 return res;
168}
169void fft64_vmp_pmat_layout::set(uint64_t row, uint64_t col, const reim_fft64vec& value) {
170 REQUIRE_DRAMATICALLY(row < nrows, "row overflow: " << row << " / " << nrows);
171 REQUIRE_DRAMATICALLY(col < ncols, "row overflow: " << col << " / " << ncols);
172 if (nn < 8) {
173 // the pmat is just col major
174 double* addr = (double*)data + (row + col * nrows) * nn;
175 value.save_as(addr);
176 return;
177 }
178 // otherwise, reconstruct it block by block
179 for (uint64_t blk = 0; blk < nn / 8; ++blk) {
180 reim4_elem v = value.get_blk(blk);
181 set(row, col, blk, v);
182 }
183}
184void fft64_vmp_pmat_layout::fill_random(double log2bound) {
185 for (uint64_t row = 0; row < nrows; ++row) {
186 for (uint64_t col = 0; col < ncols; ++col) {
187 set(row, col, reim_fft64vec::random(nn, log2bound));
188 }
189 }
190}
191void fft64_vmp_pmat_layout::fill_dft_random(uint64_t log2bound) {
192 for (uint64_t row = 0; row < nrows; ++row) {
193 for (uint64_t col = 0; col < ncols; ++col) {
194 set(row, col, reim_fft64vec::dft_random(nn, log2bound));
195 }
196 }
197}
198
199fft64_svp_ppol_layout::fft64_svp_ppol_layout(uint64_t n)
200 : nn(n), //
201 data((SVP_PPOL*)alloc64(nn * 8)) {}
202
203reim_fft64vec fft64_svp_ppol_layout::get_copy() const { return reim_fft64vec(nn, (double*)data); }
204
205void fft64_svp_ppol_layout::set(const reim_fft64vec& value) { value.save_as((double*)data); }
206
207void fft64_svp_ppol_layout::fill_dft_random(uint64_t log2bound) { set(reim_fft64vec::dft_random(nn, log2bound)); }
208
209void fft64_svp_ppol_layout::fill_random(double log2bound) { set(reim_fft64vec::random(nn, log2bound)); }
210
211fft64_svp_ppol_layout::~fft64_svp_ppol_layout() { spqlios_free(data); }
212thash fft64_svp_ppol_layout::content_hash() const { return test_hash(data, nn * sizeof(double)); }
213
214fft64_cnv_left_layout::fft64_cnv_left_layout(uint64_t n, uint64_t size)
215 : nn(n), //
216 size(size),
217 data((CNV_PVEC_L*)alloc64(size * nn * 8)) {}
218
219reim4_elem fft64_cnv_left_layout::get(uint64_t idx, uint64_t blk) {
220 REQUIRE_DRAMATICALLY(idx < size, "idx overflow: " << idx << " / " << size);
221 REQUIRE_DRAMATICALLY(blk < nn / 8, "block overflow: " << blk << " / " << (nn / 8));
222 return reim4_elem(((double*)data) + blk * size + idx);
223}
224
225fft64_cnv_left_layout::~fft64_cnv_left_layout() { spqlios_free(data); }
226
227fft64_cnv_right_layout::fft64_cnv_right_layout(uint64_t n, uint64_t size)
228 : nn(n), //
229 size(size),
230 data((CNV_PVEC_R*)alloc64(size * nn * 8)) {}
231
232reim4_elem fft64_cnv_right_layout::get(uint64_t idx, uint64_t blk) {
233 REQUIRE_DRAMATICALLY(idx < size, "idx overflow: " << idx << " / " << size);
234 REQUIRE_DRAMATICALLY(blk < nn / 8, "block overflow: " << blk << " / " << (nn / 8));
235 return reim4_elem(((double*)data) + blk * size + idx);
236}
237
238fft64_cnv_right_layout::~fft64_cnv_right_layout() { spqlios_free(data); }