Ref. + AVX code & generic tests + benches (#85)

This commit is contained in:
Jean-Philippe Bossuat
2025-09-15 16:16:11 +02:00
committed by GitHub
parent 99b9e3e10e
commit 56dbd29c59
286 changed files with 27797 additions and 7270 deletions

View File

@@ -1,8 +1,9 @@
use itertools::izip;
use rug::{Assign, Float};
use crate::layouts::{
DataMut, DataRef, VecZnx, VecZnxToMut, VecZnxToRef, Zn, ZnToMut, ZnToRef, ZnxInfos, ZnxView, ZnxViewMut, ZnxZero,
use crate::{
layouts::{DataMut, DataRef, VecZnx, VecZnxToMut, VecZnxToRef, Zn, ZnToMut, ZnToRef, ZnxInfos, ZnxView, ZnxViewMut},
reference::znx::znx_zero_ref,
};
impl<D: DataMut> VecZnx<D> {
@@ -28,7 +29,7 @@ impl<D: DataMut> VecZnx<D> {
// Zeroes coefficients of the i-th column
(0..a.size()).for_each(|i| {
a.zero_at(col, i);
znx_zero_ref(a.at_mut(col, i));
});
// If 2^{basek} * 2^{k_rem} < 2^{63}-1, then we can simply copy
@@ -183,7 +184,7 @@ impl<D: DataRef> VecZnx<D> {
let prec: u32 = (basek * size) as u32;
// 2^{basek}
let base = Float::with_val(prec, (1 << basek) as f64);
let base = Float::with_val(prec, (1u64 << basek) as f64);
// y[i] = sum x[j][i] * 2^{-basek*j}
(0..size).for_each(|i| {

View File

@@ -1,17 +1,21 @@
use crate::{
alloc_aligned,
layouts::{
Data, DataMut, DataRef, DataView, DataViewMut, FillUniform, ReaderFrom, Reset, ToOwnedDeep, VecZnx, WriterTo, ZnxInfos,
ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
Data, DataMut, DataRef, DataView, DataViewMut, DigestU64, FillUniform, ReaderFrom, Reset, ToOwnedDeep, VecZnx, WriterTo,
ZnxInfos, ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
},
source::Source,
};
use std::fmt;
use std::{
fmt,
hash::{DefaultHasher, Hasher},
};
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use rand::RngCore;
#[derive(PartialEq, Eq, Clone)]
#[repr(C)]
#[derive(PartialEq, Eq, Clone, Hash)]
pub struct MatZnx<D: Data> {
data: D,
n: usize,
@@ -21,6 +25,19 @@ pub struct MatZnx<D: Data> {
cols_out: usize,
}
impl<D: DataRef> DigestU64 for MatZnx<D> {
fn digest_u64(&self) -> u64 {
let mut h: DefaultHasher = DefaultHasher::new();
h.write(self.data.as_ref());
h.write_usize(self.n);
h.write_usize(self.size);
h.write_usize(self.rows);
h.write_usize(self.cols_in);
h.write_usize(self.cols_out);
h.finish()
}
}
impl<D: DataRef> ToOwnedDeep for MatZnx<D> {
type Owned = MatZnx<Vec<u8>>;
fn to_owned_deep(&self) -> Self::Owned {
@@ -57,6 +74,10 @@ impl<D: Data> ZnxInfos for MatZnx<D> {
fn size(&self) -> usize {
self.size
}
fn poly_count(&self) -> usize {
self.rows() * self.cols_in() * self.cols_out() * self.size()
}
}
impl<D: Data> ZnxSliceSize for MatZnx<D> {
@@ -175,8 +196,18 @@ impl<D: DataMut> MatZnx<D> {
}
impl<D: DataMut> FillUniform for MatZnx<D> {
fn fill_uniform(&mut self, source: &mut Source) {
source.fill_bytes(self.data.as_mut());
fn fill_uniform(&mut self, log_bound: usize, source: &mut Source) {
match log_bound {
64 => source.fill_bytes(self.data.as_mut()),
0 => panic!("invalid log_bound, cannot be zero"),
_ => {
let mask: u64 = (1u64 << log_bound) - 1;
for x in self.raw_mut().iter_mut() {
let r = source.next_u64() & mask;
*x = ((r << (64 - log_bound)) as i64) >> (64 - log_bound);
}
}
}
}
}

View File

@@ -34,3 +34,7 @@ pub trait ToOwnedDeep {
type Owned;
fn to_owned_deep(&self) -> Self::Owned;
}
pub trait DigestU64 {
fn digest_u64(&self) -> u64;
}

View File

@@ -20,7 +20,30 @@ pub struct Module<B: Backend> {
_marker: PhantomData<B>,
}
unsafe impl<B: Backend> Sync for Module<B> {}
unsafe impl<B: Backend> Send for Module<B> {}
impl<B: Backend> Module<B> {
#[allow(clippy::missing_safety_doc)]
#[inline]
pub fn new_marker(n: u64) -> Self {
Self {
ptr: NonNull::dangling(),
n,
_marker: PhantomData,
}
}
#[allow(clippy::missing_safety_doc)]
#[inline]
pub unsafe fn from_nonnull(ptr: NonNull<B::Handle>, n: u64) -> Self {
Self {
ptr,
n,
_marker: PhantomData,
}
}
/// Construct from a raw pointer managed elsewhere.
/// SAFETY: `ptr` must be non-null and remain valid for the lifetime of this Module.
#[inline]

View File

@@ -1,3 +1,5 @@
use std::hash::{DefaultHasher, Hasher};
use rand::seq::SliceRandom;
use rand_core::RngCore;
use rand_distr::{Distribution, weighted::WeightedIndex};
@@ -5,19 +7,30 @@ use rand_distr::{Distribution, weighted::WeightedIndex};
use crate::{
alloc_aligned,
layouts::{
Data, DataMut, DataRef, DataView, DataViewMut, FillUniform, ReaderFrom, Reset, ToOwnedDeep, VecZnx, WriterTo, ZnxInfos,
ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
Data, DataMut, DataRef, DataView, DataViewMut, DigestU64, FillUniform, ReaderFrom, Reset, ToOwnedDeep, VecZnx, WriterTo,
ZnxInfos, ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
},
source::Source,
};
#[derive(PartialEq, Eq, Debug, Clone)]
#[repr(C)]
#[derive(PartialEq, Eq, Debug, Clone, Hash)]
pub struct ScalarZnx<D: Data> {
pub data: D,
pub n: usize,
pub cols: usize,
}
impl<D: DataRef> DigestU64 for ScalarZnx<D> {
fn digest_u64(&self) -> u64 {
let mut h: DefaultHasher = DefaultHasher::new();
h.write(self.data.as_ref());
h.write_usize(self.n);
h.write_usize(self.cols);
h.finish()
}
}
impl<D: DataRef> ToOwnedDeep for ScalarZnx<D> {
type Owned = ScalarZnx<Vec<u8>>;
fn to_owned_deep(&self) -> Self::Owned {
@@ -145,8 +158,18 @@ impl<D: DataMut> ZnxZero for ScalarZnx<D> {
}
impl<D: DataMut> FillUniform for ScalarZnx<D> {
fn fill_uniform(&mut self, source: &mut Source) {
source.fill_bytes(self.data.as_mut());
fn fill_uniform(&mut self, log_bound: usize, source: &mut Source) {
match log_bound {
64 => source.fill_bytes(self.data.as_mut()),
0 => panic!("invalid log_bound, cannot be zero"),
_ => {
let mask: u64 = (1u64 << log_bound) - 1;
for x in self.raw_mut().iter_mut() {
let r = source.next_u64() & mask;
*x = ((r << (64 - log_bound)) as i64) >> (64 - log_bound);
}
}
}
}
}

View File

@@ -2,11 +2,13 @@ use std::marker::PhantomData;
use crate::layouts::Backend;
#[repr(C)]
pub struct ScratchOwned<B: Backend> {
pub data: Vec<u8>,
pub _phantom: PhantomData<B>,
}
#[repr(C)]
pub struct Scratch<B: Backend> {
pub _phantom: PhantomData<B>,
pub data: [u8],

View File

@@ -4,7 +4,7 @@ use rug::{
ops::{AddAssignRound, DivAssignRound, SubAssignRound},
};
use crate::layouts::{DataRef, VecZnx, ZnxInfos};
use crate::layouts::{Backend, DataRef, VecZnx, VecZnxBig, VecZnxBigToRef, ZnxInfos};
impl<D: DataRef> VecZnx<D> {
pub fn std(&self, basek: usize, col: usize) -> f64 {
@@ -27,3 +27,17 @@ impl<D: DataRef> VecZnx<D> {
std.to_f64()
}
}
impl<D: DataRef, B: Backend + Backend<ScalarBig = i64>> VecZnxBig<D, B> {
pub fn std(&self, basek: usize, col: usize) -> f64 {
let self_ref: VecZnxBig<&[u8], B> = self.to_ref();
let znx: VecZnx<&[u8]> = VecZnx {
data: self_ref.data,
n: self_ref.n,
cols: self_ref.cols,
size: self_ref.size,
max_size: self_ref.max_size,
};
znx.std(basek, col)
}
}

View File

@@ -1,12 +1,19 @@
use std::marker::PhantomData;
use std::{
fmt,
hash::{DefaultHasher, Hasher},
marker::PhantomData,
};
use crate::{
alloc_aligned,
layouts::{Backend, Data, DataMut, DataRef, DataView, DataViewMut, ReaderFrom, WriterTo, ZnxInfos, ZnxSliceSize, ZnxView},
layouts::{
Backend, Data, DataMut, DataRef, DataView, DataViewMut, DigestU64, ReaderFrom, WriterTo, ZnxInfos, ZnxSliceSize, ZnxView,
},
oep::SvpPPolAllocBytesImpl,
};
#[derive(PartialEq, Eq)]
#[repr(C)]
#[derive(PartialEq, Eq, Hash)]
pub struct SvpPPol<D: Data, B: Backend> {
pub data: D,
pub n: usize,
@@ -14,6 +21,16 @@ pub struct SvpPPol<D: Data, B: Backend> {
pub _phantom: PhantomData<B>,
}
impl<D: DataRef, B: Backend> DigestU64 for SvpPPol<D, B> {
fn digest_u64(&self) -> u64 {
let mut h: DefaultHasher = DefaultHasher::new();
h.write(self.data.as_ref());
h.write_usize(self.n);
h.write_usize(self.cols);
h.finish()
}
}
impl<D: Data, B: Backend> ZnxSliceSize for SvpPPol<D, B> {
fn sl(&self) -> usize {
B::layout_prep_word_count() * self.n()
@@ -153,3 +170,32 @@ impl<D: DataRef, B: Backend> WriterTo for SvpPPol<D, B> {
Ok(())
}
}
impl<D: DataRef, B: Backend> fmt::Display for SvpPPol<D, B> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
writeln!(f, "SvpPPol(n={}, cols={})", self.n, self.cols)?;
for col in 0..self.cols {
writeln!(f, "Column {}:", col)?;
let coeffs = self.at(col, 0);
write!(f, "[")?;
let max_show = 100;
let show_count = coeffs.len().min(max_show);
for (i, &coeff) in coeffs.iter().take(show_count).enumerate() {
if i > 0 {
write!(f, ", ")?;
}
write!(f, "{}", coeff)?;
}
if coeffs.len() > max_show {
write!(f, ", ... ({} more)", coeffs.len() - max_show)?;
}
writeln!(f, "]")?;
}
Ok(())
}
}

View File

@@ -1,10 +1,13 @@
use std::fmt;
use std::{
fmt,
hash::{DefaultHasher, Hasher},
};
use crate::{
alloc_aligned,
layouts::{
Data, DataMut, DataRef, DataView, DataViewMut, FillUniform, ReaderFrom, Reset, ToOwnedDeep, WriterTo, ZnxInfos,
ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
Data, DataMut, DataRef, DataView, DataViewMut, DigestU64, FillUniform, ReaderFrom, Reset, ToOwnedDeep, WriterTo,
ZnxInfos, ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
},
source::Source,
};
@@ -12,7 +15,8 @@ use crate::{
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use rand::RngCore;
#[derive(PartialEq, Eq, Clone, Copy)]
#[repr(C)]
#[derive(PartialEq, Eq, Clone, Copy, Hash)]
pub struct VecZnx<D: Data> {
pub data: D,
pub n: usize,
@@ -21,6 +25,18 @@ pub struct VecZnx<D: Data> {
pub max_size: usize,
}
impl<D: DataRef> DigestU64 for VecZnx<D> {
fn digest_u64(&self) -> u64 {
let mut h: DefaultHasher = DefaultHasher::new();
h.write(self.data.as_ref());
h.write_usize(self.n);
h.write_usize(self.cols);
h.write_usize(self.size);
h.write_usize(self.max_size);
h.finish()
}
}
impl<D: DataRef> ToOwnedDeep for VecZnx<D> {
type Owned = VecZnx<Vec<u8>>;
fn to_owned_deep(&self) -> Self::Owned {
@@ -173,8 +189,18 @@ impl<D: DataRef> fmt::Display for VecZnx<D> {
}
impl<D: DataMut> FillUniform for VecZnx<D> {
fn fill_uniform(&mut self, source: &mut Source) {
source.fill_bytes(self.data.as_mut());
fn fill_uniform(&mut self, log_bound: usize, source: &mut Source) {
match log_bound {
64 => source.fill_bytes(self.data.as_mut()),
0 => panic!("invalid log_bound, cannot be zero"),
_ => {
let mask: u64 = (1u64 << log_bound) - 1;
for x in self.raw_mut().iter_mut() {
let r = source.next_u64() & mask;
*x = ((r << (64 - log_bound)) as i64) >> (64 - log_bound);
}
}
}
}
}

View File

@@ -1,15 +1,21 @@
use std::marker::PhantomData;
use std::{
hash::{DefaultHasher, Hasher},
marker::PhantomData,
};
use rand_distr::num_traits::Zero;
use std::fmt;
use crate::{
alloc_aligned,
layouts::{Backend, Data, DataMut, DataRef, DataView, DataViewMut, ZnxInfos, ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero},
layouts::{
Backend, Data, DataMut, DataRef, DataView, DataViewMut, DigestU64, ZnxInfos, ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
},
oep::VecZnxBigAllocBytesImpl,
};
#[derive(PartialEq, Eq)]
#[repr(C)]
#[derive(PartialEq, Eq, Hash)]
pub struct VecZnxBig<D: Data, B: Backend> {
pub data: D,
pub n: usize,
@@ -19,6 +25,18 @@ pub struct VecZnxBig<D: Data, B: Backend> {
pub _phantom: PhantomData<B>,
}
impl<D: DataRef, B: Backend> DigestU64 for VecZnxBig<D, B> {
fn digest_u64(&self) -> u64 {
let mut h: DefaultHasher = DefaultHasher::new();
h.write(self.data.as_ref());
h.write_usize(self.n);
h.write_usize(self.cols);
h.write_usize(self.size);
h.write_usize(self.max_size);
h.finish()
}
}
impl<D: Data, B: Backend> ZnxSliceSize for VecZnxBig<D, B> {
fn sl(&self) -> usize {
B::layout_big_word_count() * self.n() * self.cols()

View File

@@ -1,14 +1,21 @@
use std::{fmt, marker::PhantomData};
use std::{
fmt,
hash::{DefaultHasher, Hasher},
marker::PhantomData,
};
use rand_distr::num_traits::Zero;
use crate::{
alloc_aligned,
layouts::{
Backend, Data, DataMut, DataRef, DataView, DataViewMut, VecZnxBig, ZnxInfos, ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
Backend, Data, DataMut, DataRef, DataView, DataViewMut, DigestU64, VecZnxBig, ZnxInfos, ZnxSliceSize, ZnxView,
ZnxViewMut, ZnxZero,
},
oep::VecZnxBigAllocBytesImpl,
oep::VecZnxDftAllocBytesImpl,
};
#[repr(C)]
#[derive(PartialEq, Eq)]
pub struct VecZnxDft<D: Data, B: Backend> {
pub data: D,
@@ -19,6 +26,18 @@ pub struct VecZnxDft<D: Data, B: Backend> {
pub _phantom: PhantomData<B>,
}
impl<D: DataRef, B: Backend> DigestU64 for VecZnxDft<D, B> {
fn digest_u64(&self) -> u64 {
let mut h: DefaultHasher = DefaultHasher::new();
h.write(self.data.as_ref());
h.write_usize(self.n);
h.write_usize(self.cols);
h.write_usize(self.size);
h.write_usize(self.max_size);
h.finish()
}
}
impl<D: Data, B: Backend> ZnxSliceSize for VecZnxDft<D, B> {
fn sl(&self) -> usize {
B::layout_prep_word_count() * self.n() * self.cols()
@@ -94,10 +113,10 @@ where
impl<D: DataRef + From<Vec<u8>>, B: Backend> VecZnxDft<D, B>
where
B: VecZnxBigAllocBytesImpl<B>,
B: VecZnxDftAllocBytesImpl<B>,
{
pub fn alloc(n: usize, cols: usize, size: usize) -> Self {
let data: Vec<u8> = alloc_aligned::<u8>(B::vec_znx_big_alloc_bytes_impl(n, cols, size));
let data: Vec<u8> = alloc_aligned::<u8>(B::vec_znx_dft_alloc_bytes_impl(n, cols, size));
Self {
data: data.into(),
n,
@@ -110,7 +129,7 @@ where
pub fn from_bytes(n: usize, cols: usize, size: usize, bytes: impl Into<Vec<u8>>) -> Self {
let data: Vec<u8> = bytes.into();
assert!(data.len() == B::vec_znx_big_alloc_bytes_impl(n, cols, size));
assert!(data.len() == B::vec_znx_dft_alloc_bytes_impl(n, cols, size));
Self {
data: data.into(),
n,

View File

@@ -1,12 +1,16 @@
use std::marker::PhantomData;
use std::{
hash::{DefaultHasher, Hasher},
marker::PhantomData,
};
use crate::{
alloc_aligned,
layouts::{Backend, Data, DataMut, DataRef, DataView, DataViewMut, ZnxInfos, ZnxView},
layouts::{Backend, Data, DataMut, DataRef, DataView, DataViewMut, DigestU64, ZnxInfos, ZnxView},
oep::VmpPMatAllocBytesImpl,
};
#[derive(PartialEq, Eq)]
#[repr(C)]
#[derive(PartialEq, Eq, Hash)]
pub struct VmpPMat<D: Data, B: Backend> {
data: D,
n: usize,
@@ -17,6 +21,19 @@ pub struct VmpPMat<D: Data, B: Backend> {
_phantom: PhantomData<B>,
}
impl<D: DataRef, B: Backend> DigestU64 for VmpPMat<D, B> {
fn digest_u64(&self) -> u64 {
let mut h: DefaultHasher = DefaultHasher::new();
h.write(self.data.as_ref());
h.write_usize(self.n);
h.write_usize(self.size);
h.write_usize(self.rows);
h.write_usize(self.cols_in);
h.write_usize(self.cols_out);
h.finish()
}
}
impl<D: DataRef, B: Backend> ZnxView for VmpPMat<D, B> {
type Scalar = B::ScalarPrep;
}
@@ -37,6 +54,10 @@ impl<D: Data, B: Backend> ZnxInfos for VmpPMat<D, B> {
fn size(&self) -> usize {
self.size
}
fn poly_count(&self) -> usize {
self.rows() * self.cols_in() * self.size() * self.cols_out()
}
}
impl<D: Data, B: Backend> DataView for VmpPMat<D, B> {

View File

@@ -1,10 +1,13 @@
use std::fmt;
use std::{
fmt,
hash::{DefaultHasher, Hasher},
};
use crate::{
alloc_aligned,
layouts::{
Data, DataMut, DataRef, DataView, DataViewMut, FillUniform, ReaderFrom, Reset, ToOwnedDeep, WriterTo, ZnxInfos,
ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
Data, DataMut, DataRef, DataView, DataViewMut, DigestU64, FillUniform, ReaderFrom, Reset, ToOwnedDeep, WriterTo,
ZnxInfos, ZnxSliceSize, ZnxView, ZnxViewMut, ZnxZero,
},
source::Source,
};
@@ -12,7 +15,8 @@ use crate::{
use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
use rand::RngCore;
#[derive(PartialEq, Eq, Clone, Copy)]
#[repr(C)]
#[derive(PartialEq, Eq, Clone, Copy, Hash)]
pub struct Zn<D: Data> {
pub data: D,
pub n: usize,
@@ -21,6 +25,18 @@ pub struct Zn<D: Data> {
pub max_size: usize,
}
impl<D: DataRef> DigestU64 for Zn<D> {
fn digest_u64(&self) -> u64 {
let mut h: DefaultHasher = DefaultHasher::new();
h.write(self.data.as_ref());
h.write_usize(self.n);
h.write_usize(self.cols);
h.write_usize(self.size);
h.write_usize(self.max_size);
h.finish()
}
}
impl<D: DataRef> ToOwnedDeep for Zn<D> {
type Owned = Zn<Vec<u8>>;
fn to_owned_deep(&self) -> Self::Owned {
@@ -173,8 +189,18 @@ impl<D: DataRef> fmt::Display for Zn<D> {
}
impl<D: DataMut> FillUniform for Zn<D> {
fn fill_uniform(&mut self, source: &mut Source) {
source.fill_bytes(self.data.as_mut());
fn fill_uniform(&mut self, log_bound: usize, source: &mut Source) {
match log_bound {
64 => source.fill_bytes(self.data.as_mut()),
0 => panic!("invalid log_bound, cannot be zero"),
_ => {
let mask: u64 = (1u64 << log_bound) - 1;
for x in self.raw_mut().iter_mut() {
let r = source.next_u64() & mask;
*x = ((r << (64 - log_bound)) as i64) >> (64 - log_bound);
}
}
}
}
}

View File

@@ -117,7 +117,7 @@ where
}
pub trait FillUniform {
fn fill_uniform(&mut self, source: &mut Source);
fn fill_uniform(&mut self, log_bound: usize, source: &mut Source);
}
pub trait Reset {