poulpy/core/src/glwe/keyswitch.rs
core/src/glwe/keyswitch.rsRaw
1use backend::hal::{
2 api::{
3 DataViewMut, ScratchAvailable, TakeVecZnxDft, VecZnxBigAddSmallInplace, VecZnxBigNormalize, VecZnxBigNormalizeTmpBytes,
4 VecZnxDftAllocBytes, VecZnxDftFromVecZnx, VecZnxDftToVecZnxBigConsume, VmpApply, VmpApplyAdd, VmpApplyTmpBytes, ZnxInfos,
5 },
6 layouts::{Backend, DataMut, DataRef, Module, Scratch, VecZnx, VecZnxBig, VecZnxDft, VmpPMat},
7};
8
9use crate::{GLWECiphertext, GLWESwitchingKeyExec, Infos};
10
11pub trait GLWEKeyswitchFamily<B: Backend> = VecZnxDftAllocBytes
12 + VmpApplyTmpBytes
13 + VecZnxBigNormalizeTmpBytes
14 + VmpApplyTmpBytes
15 + VmpApply<B>
16 + VmpApplyAdd<B>
17 + VecZnxDftFromVecZnx<B>
18 + VecZnxDftToVecZnxBigConsume<B>
19 + VecZnxBigAddSmallInplace<B>
20 + VecZnxBigNormalize<B>;
21
22impl GLWECiphertext<Vec<u8>> {
23 pub fn keyswitch_scratch_space<B: Backend>(
24 module: &Module<B>,
25 basek: usize,
26 k_out: usize,
27 k_in: usize,
28 k_ksk: usize,
29 digits: usize,
30 rank_in: usize,
31 rank_out: usize,
32 ) -> usize
33 where
34 Module<B>: GLWEKeyswitchFamily<B>,
35 {
36 let in_size: usize = k_in.div_ceil(basek).div_ceil(digits);
37 let out_size: usize = k_out.div_ceil(basek);
38 let ksk_size: usize = k_ksk.div_ceil(basek);
39 let res_dft: usize = module.vec_znx_dft_alloc_bytes(rank_out + 1, ksk_size); // TODO OPTIMIZE
40 let ai_dft: usize = module.vec_znx_dft_alloc_bytes(rank_in, in_size);
41 let vmp: usize = module.vmp_apply_tmp_bytes(out_size, in_size, in_size, rank_in, rank_out + 1, ksk_size)
42 + module.vec_znx_dft_alloc_bytes(rank_in, in_size);
43 let normalize: usize = module.vec_znx_big_normalize_tmp_bytes(module.n());
44 return res_dft + ((ai_dft + vmp) | normalize);
45 }
46
47 pub fn keyswitch_from_fourier_scratch_space<B: Backend>(
48 module: &Module<B>,
49 basek: usize,
50 k_out: usize,
51 k_in: usize,
52 k_ksk: usize,
53 digits: usize,
54 rank_in: usize,
55 rank_out: usize,
56 ) -> usize
57 where
58 Module<B>: GLWEKeyswitchFamily<B>,
59 {
60 Self::keyswitch_scratch_space(module, basek, k_out, k_in, k_ksk, digits, rank_in, rank_out)
61 }
62
63 pub fn keyswitch_inplace_scratch_space<B: Backend>(
64 module: &Module<B>,
65 basek: usize,
66 k_out: usize,
67 k_ksk: usize,
68 digits: usize,
69 rank: usize,
70 ) -> usize
71 where
72 Module<B>: GLWEKeyswitchFamily<B>,
73 {
74 Self::keyswitch_scratch_space(module, basek, k_out, k_out, k_ksk, digits, rank, rank)
75 }
76}
77
78impl<DataSelf: DataRef> GLWECiphertext<DataSelf> {
79 pub(crate) fn assert_keyswitch<B: Backend, DataLhs, DataRhs>(
80 &self,
81 module: &Module<B>,
82 lhs: &GLWECiphertext<DataLhs>,
83 rhs: &GLWESwitchingKeyExec<DataRhs, B>,
84 scratch: &Scratch<B>,
85 ) where
86 DataLhs: DataRef,
87 DataRhs: DataRef,
88 Module<B>: GLWEKeyswitchFamily<B>,
89 Scratch<B>: ScratchAvailable,
90 {
91 let basek: usize = self.basek();
92 assert_eq!(
93 lhs.rank(),
94 rhs.rank_in(),
95 "lhs.rank(): {} != rhs.rank_in(): {}",
96 lhs.rank(),
97 rhs.rank_in()
98 );
99 assert_eq!(
100 self.rank(),
101 rhs.rank_out(),
102 "self.rank(): {} != rhs.rank_out(): {}",
103 self.rank(),
104 rhs.rank_out()
105 );
106 assert_eq!(self.basek(), basek);
107 assert_eq!(lhs.basek(), basek);
108 assert_eq!(rhs.n(), module.n());
109 assert_eq!(self.n(), module.n());
110 assert_eq!(lhs.n(), module.n());
111 assert!(
112 scratch.available()
113 >= GLWECiphertext::keyswitch_scratch_space(
114 module,
115 self.basek(),
116 self.k(),
117 lhs.k(),
118 rhs.k(),
119 rhs.digits(),
120 rhs.rank_in(),
121 rhs.rank_out(),
122 )
123 );
124 }
125}
126
127impl<DataSelf: DataMut> GLWECiphertext<DataSelf> {
128 pub fn keyswitch<DataLhs: DataRef, DataRhs: DataRef, B: Backend>(
129 &mut self,
130 module: &Module<B>,
131 lhs: &GLWECiphertext<DataLhs>,
132 rhs: &GLWESwitchingKeyExec<DataRhs, B>,
133 scratch: &mut Scratch<B>,
134 ) where
135 Module<B>: GLWEKeyswitchFamily<B>,
136 Scratch<B>: TakeVecZnxDft<B> + ScratchAvailable,
137 {
138 #[cfg(debug_assertions)]
139 {
140 self.assert_keyswitch(module, lhs, rhs, scratch);
141 }
142 let (res_dft, scratch1) = scratch.take_vec_znx_dft(module, self.cols(), rhs.size()); // Todo optimise
143 let res_big: VecZnxBig<_, B> = keyswitch(module, res_dft, lhs, rhs, scratch1);
144 (0..self.cols()).for_each(|i| {
145 module.vec_znx_big_normalize(self.basek(), &mut self.data, i, &res_big, i, scratch1);
146 })
147 }
148
149 pub fn keyswitch_inplace<DataRhs: DataRef, B: Backend>(
150 &mut self,
151 module: &Module<B>,
152 rhs: &GLWESwitchingKeyExec<DataRhs, B>,
153 scratch: &mut Scratch<B>,
154 ) where
155 Module<B>: GLWEKeyswitchFamily<B>,
156 Scratch<B>: TakeVecZnxDft<B> + ScratchAvailable,
157 {
158 unsafe {
159 let self_ptr: *mut GLWECiphertext<DataSelf> = self as *mut GLWECiphertext<DataSelf>;
160 self.keyswitch(&module, &*self_ptr, rhs, scratch);
161 }
162 }
163}
164
165pub(crate) fn keyswitch<B: Backend, DataRes, DataIn, DataKey>(
166 module: &Module<B>,
167 res_dft: VecZnxDft<DataRes, B>,
168 lhs: &GLWECiphertext<DataIn>,
169 rhs: &GLWESwitchingKeyExec<DataKey, B>,
170 scratch: &mut Scratch<B>,
171) -> VecZnxBig<DataRes, B>
172where
173 DataRes: DataMut,
174 DataIn: DataRef,
175 DataKey: DataRef,
176 Module<B>: GLWEKeyswitchFamily<B>,
177 Scratch<B>: TakeVecZnxDft<B>,
178{
179 if rhs.digits() == 1 {
180 return keyswitch_vmp_one_digit(module, res_dft, &lhs.data, &rhs.key.data, scratch);
181 }
182
183 keyswitch_vmp_multiple_digits(
184 module,
185 res_dft,
186 &lhs.data,
187 &rhs.key.data,
188 rhs.digits(),
189 scratch,
190 )
191}
192
193fn keyswitch_vmp_one_digit<B: Backend, DataRes, DataIn, DataVmp>(
194 module: &Module<B>,
195 mut res_dft: VecZnxDft<DataRes, B>,
196 a: &VecZnx<DataIn>,
197 mat: &VmpPMat<DataVmp, B>,
198 scratch: &mut Scratch<B>,
199) -> VecZnxBig<DataRes, B>
200where
201 DataRes: DataMut,
202 DataIn: DataRef,
203 DataVmp: DataRef,
204 Module<B>:
205 VecZnxDftAllocBytes + VecZnxDftFromVecZnx<B> + VmpApply<B> + VecZnxDftToVecZnxBigConsume<B> + VecZnxBigAddSmallInplace<B>,
206 Scratch<B>: TakeVecZnxDft<B>,
207{
208 let cols: usize = a.cols();
209 let (mut ai_dft, scratch1) = scratch.take_vec_znx_dft(module, cols - 1, a.size());
210 (0..cols - 1).for_each(|col_i| {
211 module.vec_znx_dft_from_vec_znx(1, 0, &mut ai_dft, col_i, a, col_i + 1);
212 });
213 module.vmp_apply(&mut res_dft, &ai_dft, mat, scratch1);
214 let mut res_big: VecZnxBig<DataRes, B> = module.vec_znx_dft_to_vec_znx_big_consume(res_dft);
215 module.vec_znx_big_add_small_inplace(&mut res_big, 0, a, 0);
216 res_big
217}
218
219fn keyswitch_vmp_multiple_digits<B: Backend, DataRes, DataIn, DataVmp>(
220 module: &Module<B>,
221 mut res_dft: VecZnxDft<DataRes, B>,
222 a: &VecZnx<DataIn>,
223 mat: &VmpPMat<DataVmp, B>,
224 digits: usize,
225 scratch: &mut Scratch<B>,
226) -> VecZnxBig<DataRes, B>
227where
228 DataRes: DataMut,
229 DataIn: DataRef,
230 DataVmp: DataRef,
231 Module<B>: VecZnxDftAllocBytes
232 + VecZnxDftFromVecZnx<B>
233 + VmpApply<B>
234 + VmpApplyAdd<B>
235 + VecZnxDftToVecZnxBigConsume<B>
236 + VecZnxBigAddSmallInplace<B>,
237 Scratch<B>: TakeVecZnxDft<B>,
238{
239 let cols: usize = a.cols();
240 let size: usize = a.size();
241 let (mut ai_dft, scratch1) = scratch.take_vec_znx_dft(module, cols - 1, size.div_ceil(digits));
242
243 ai_dft.data_mut().fill(0);
244
245 (0..digits).for_each(|di| {
246 ai_dft.set_size((size + di) / digits);
247
248 // Small optimization for digits > 2
249 // VMP produce some error e, and since we aggregate vmp * 2^{di * B}, then
250 // we also aggregate ei * 2^{di * B}, with the largest error being ei * 2^{(digits-1) * B}.
251 // As such we can ignore the last digits-2 limbs safely of the sum of vmp products.
252 // It is possible to further ignore the last digits-1 limbs, but this introduce
253 // ~0.5 to 1 bit of additional noise, and thus not chosen here to ensure that the same
254 // noise is kept with respect to the ideal functionality.
255 res_dft.set_size(mat.size() - ((digits - di) as isize - 2).max(0) as usize);
256
257 (0..cols - 1).for_each(|col_i| {
258 module.vec_znx_dft_from_vec_znx(digits, digits - di - 1, &mut ai_dft, col_i, a, col_i + 1);
259 });
260
261 if di == 0 {
262 module.vmp_apply(&mut res_dft, &ai_dft, mat, scratch1);
263 } else {
264 module.vmp_apply_add(&mut res_dft, &ai_dft, mat, di, scratch1);
265 }
266 });
267
268 res_dft.set_size(res_dft.max_size());
269 let mut res_big: VecZnxBig<DataRes, B> = module.vec_znx_dft_to_vec_znx_big_consume(res_dft);
270 module.vec_znx_big_add_small_inplace(&mut res_big, 0, a, 0);
271 res_big
272}