1#include "vec_rnx_layout.h"
  2
  3#include <cstring>
  4
  5#include "../../spqlios/arithmetic/vec_rnx_arithmetic.h"
  6
  7#ifdef VALGRIND_MEM_TESTS
  8#include "valgrind/memcheck.h"
  9#endif
 10
 11#define CANARY_PADDING (1024)
 12#define GARBAGE_VALUE (242)
 13
 14rnx_vec_f64_layout::rnx_vec_f64_layout(uint64_t n, uint64_t size, uint64_t slice) : n(n), size(size), slice(slice) {
 15  REQUIRE_DRAMATICALLY(is_pow2(n), "not a power of 2" << n);
 16  REQUIRE_DRAMATICALLY(slice >= n, "slice too small" << slice << " < " << n);
 17  this->region = (uint8_t*)malloc(size * slice * sizeof(int64_t) + 2 * CANARY_PADDING);
 18  this->data_start = (double*)(region + CANARY_PADDING);
 19  // ensure that any invalid value is kind-of garbage
 20  memset(region, GARBAGE_VALUE, size * slice * sizeof(int64_t) + 2 * CANARY_PADDING);
 21  // mark inter-slice memory as not accessible
 22#ifdef VALGRIND_MEM_TESTS
 23  VALGRIND_MAKE_MEM_NOACCESS(region, CANARY_PADDING);
 24  VALGRIND_MAKE_MEM_NOACCESS(region + size * slice * sizeof(int64_t) + CANARY_PADDING, CANARY_PADDING);
 25  for (uint64_t i = 0; i < size; ++i) {
 26    VALGRIND_MAKE_MEM_UNDEFINED(data_start + i * slice, n * sizeof(int64_t));
 27  }
 28  if (size != slice) {
 29    for (uint64_t i = 0; i < size; ++i) {
 30      VALGRIND_MAKE_MEM_NOACCESS(data_start + i * slice + n, (slice - n) * sizeof(int64_t));
 31    }
 32  }
 33#endif
 34}
 35
 36rnx_vec_f64_layout::~rnx_vec_f64_layout() { free(region); }
 37
 38rnx_f64 rnx_vec_f64_layout::get_copy_zext(uint64_t index) const {
 39  if (index < size) {
 40    return rnx_f64(n, data_start + index * slice);
 41  } else {
 42    return rnx_f64::zero(n);
 43  }
 44}
 45
 46rnx_f64 rnx_vec_f64_layout::get_copy(uint64_t index) const {
 47  REQUIRE_DRAMATICALLY(index < size, "index overflow: " << index << " / " << size);
 48  return rnx_f64(n, data_start + index * slice);
 49}
 50
 51reim_fft64vec rnx_vec_f64_layout::get_dft_copy_zext(uint64_t index) const {
 52  if (index < size) {
 53    return reim_fft64vec(n, data_start + index * slice);
 54  } else {
 55    return reim_fft64vec::zero(n);
 56  }
 57}
 58
 59reim_fft64vec rnx_vec_f64_layout::get_dft_copy(uint64_t index) const {
 60  REQUIRE_DRAMATICALLY(index < size, "index overflow: " << index << " / " << size);
 61  return reim_fft64vec(n, data_start + index * slice);
 62}
 63
 64void rnx_vec_f64_layout::set(uint64_t index, const rnx_f64& elem) {
 65  REQUIRE_DRAMATICALLY(index < size, "index overflow: " << index << " / " << size);
 66  REQUIRE_DRAMATICALLY(elem.nn() == n, "incompatible ring dimensions: " << elem.nn() << " / " << n);
 67  elem.save_as(data_start + index * slice);
 68}
 69
 70double* rnx_vec_f64_layout::data() { return data_start; }
 71const double* rnx_vec_f64_layout::data() const { return data_start; }
 72
 73void rnx_vec_f64_layout::fill_random(double log2bound) {
 74  for (uint64_t i = 0; i < size; ++i) {
 75    set(i, rnx_f64::random_log2bound(n, log2bound));
 76  }
 77}
 78
 79thash rnx_vec_f64_layout::content_hash() const {
 80  test_hasher hasher;
 81  for (uint64_t i = 0; i < size; ++i) {
 82    hasher.update(data() + i * slice, n * sizeof(int64_t));
 83  }
 84  return hasher.hash();
 85}
 86
 87fft64_rnx_vmp_pmat_layout::fft64_rnx_vmp_pmat_layout(uint64_t n, uint64_t nrows, uint64_t ncols)
 88    : nn(n),
 89      nrows(nrows),
 90      ncols(ncols),  //
 91      data((RNX_VMP_PMAT*)alloc64(nrows * ncols * nn * 8)) {}
 92
 93double* fft64_rnx_vmp_pmat_layout::get_addr(uint64_t row, uint64_t col, uint64_t blk) const {
 94  REQUIRE_DRAMATICALLY(row < nrows, "row overflow: " << row << " / " << nrows);
 95  REQUIRE_DRAMATICALLY(col < ncols, "col overflow: " << col << " / " << ncols);
 96  REQUIRE_DRAMATICALLY(blk < nn / 8, "block overflow: " << blk << " / " << (nn / 8));
 97  double* d = (double*)data;
 98  if (col == (ncols - 1) && (ncols % 2 == 1)) {
 99    // special case: last column out of an odd column number
100    return d + blk * nrows * ncols * 8  // major: blk
101           + col * nrows * 8            // col == ncols-1
102           + row * 8;
103  } else {
104    // general case: columns go by pair
105    return d + blk * nrows * ncols * 8    // major: blk
106           + (col / 2) * (2 * nrows) * 8  // second: col pair index
107           + row * 2 * 8                  // third: row index
108           + (col % 2) * 8;               // minor: col in colpair
109  }
110}
111
112reim4_elem fft64_rnx_vmp_pmat_layout::get(uint64_t row, uint64_t col, uint64_t blk) const {
113  return reim4_elem(get_addr(row, col, blk));
114}
115reim4_elem fft64_rnx_vmp_pmat_layout::get_zext(uint64_t row, uint64_t col, uint64_t blk) const {
116  REQUIRE_DRAMATICALLY(blk < nn / 8, "block overflow: " << blk << " / " << (nn / 8));
117  if (row < nrows && col < ncols) {
118    return reim4_elem(get_addr(row, col, blk));
119  } else {
120    return reim4_elem::zero();
121  }
122}
123void fft64_rnx_vmp_pmat_layout::set(uint64_t row, uint64_t col, uint64_t blk, const reim4_elem& value) const {
124  value.save_as(get_addr(row, col, blk));
125}
126
127fft64_rnx_vmp_pmat_layout::~fft64_rnx_vmp_pmat_layout() { spqlios_free(data); }
128
129reim_fft64vec fft64_rnx_vmp_pmat_layout::get_zext(uint64_t row, uint64_t col) const {
130  if (row >= nrows || col >= ncols) {
131    return reim_fft64vec::zero(nn);
132  }
133  if (nn < 8) {
134    // the pmat is just col major
135    double* addr = (double*)data + (row + col * nrows) * nn;
136    return reim_fft64vec(nn, addr);
137  }
138  // otherwise, reconstruct it block by block
139  reim_fft64vec res(nn);
140  for (uint64_t blk = 0; blk < nn / 8; ++blk) {
141    reim4_elem v = get(row, col, blk);
142    res.set_blk(blk, v);
143  }
144  return res;
145}
146void fft64_rnx_vmp_pmat_layout::set(uint64_t row, uint64_t col, const reim_fft64vec& value) {
147  REQUIRE_DRAMATICALLY(row < nrows, "row overflow: " << row << " / " << nrows);
148  REQUIRE_DRAMATICALLY(col < ncols, "row overflow: " << col << " / " << ncols);
149  if (nn < 8) {
150    // the pmat is just col major
151    double* addr = (double*)data + (row + col * nrows) * nn;
152    value.save_as(addr);
153    return;
154  }
155  // otherwise, reconstruct it block by block
156  for (uint64_t blk = 0; blk < nn / 8; ++blk) {
157    reim4_elem v = value.get_blk(blk);
158    set(row, col, blk, v);
159  }
160}
161void fft64_rnx_vmp_pmat_layout::fill_random(double log2bound) {
162  for (uint64_t row = 0; row < nrows; ++row) {
163    for (uint64_t col = 0; col < ncols; ++col) {
164      set(row, col, reim_fft64vec::random(nn, log2bound));
165    }
166  }
167}
168
169fft64_rnx_svp_ppol_layout::fft64_rnx_svp_ppol_layout(uint64_t n)
170    : nn(n),  //
171      data((RNX_SVP_PPOL*)alloc64(nn * 8)) {}
172
173reim_fft64vec fft64_rnx_svp_ppol_layout::get_copy() const { return reim_fft64vec(nn, (double*)data); }
174
175void fft64_rnx_svp_ppol_layout::set(const reim_fft64vec& value) { value.save_as((double*)data); }
176
177void fft64_rnx_svp_ppol_layout::fill_dft_random(uint64_t log2bound) { set(reim_fft64vec::dft_random(nn, log2bound)); }
178
179void fft64_rnx_svp_ppol_layout::fill_random(double log2bound) { set(reim_fft64vec::random(nn, log2bound)); }
180
181fft64_rnx_svp_ppol_layout::~fft64_rnx_svp_ppol_layout() { spqlios_free(data); }
182thash fft64_rnx_svp_ppol_layout::content_hash() const { return test_hash(data, nn * sizeof(double)); }