poulpy/poulpy-schemes/src/tfhe/bdd_arithmetic/eval.rs
poulpy-schemes/src/tfhe/bdd_arithmetic/eval.rsRaw
1use core::panic;
2use std::thread;
3
4use itertools::Itertools;
5use poulpy_core::{
6 GLWECopy, GLWEExternalProductInternal, GLWENormalize, GLWESub, ScratchTakeCore,
7 layouts::{GGSWInfos, GGSWPrepared, GLWE, GLWEInfos, GLWEToMut, GLWEToRef, LWEInfos, prepared::GGSWPreparedToRef},
8};
9use poulpy_hal::{
10 api::{
11 ScratchAvailable, ScratchTakeBasic, VecZnxBigAddSmallInplace, VecZnxBigNormalize, VecZnxBigNormalizeTmpBytes,
12 VecZnxDftBytesOf,
13 },
14 layouts::{Backend, DataMut, Module, Scratch, VecZnxBig, ZnxZero},
15};
16
17use crate::tfhe::bdd_arithmetic::GetGGSWBit;
18
19pub trait BitCircuitInfo: Sync {
20 fn info(&self) -> (&[Node], usize);
21}
22
23pub trait GetBitCircuitInfo: Sync {
24 fn input_size(&self) -> usize;
25 fn output_size(&self) -> usize;
26 fn get_circuit(&self, bit: usize) -> (&[Node], usize);
27}
28
29pub struct BitCircuit<const N: usize> {
30 pub nodes: [Node; N],
31 pub max_inter_state: usize,
32}
33
34pub trait BitCircuitFamily {
35 const INPUT_BITS: usize;
36 const OUTPUT_BITS: usize;
37}
38
39pub struct Circuit<C: BitCircuitInfo, const N: usize>(pub [C; N]);
40
41impl<C, const N: usize> GetBitCircuitInfo for Circuit<C, N>
42where
43 C: BitCircuitInfo + BitCircuitFamily,
44{
45 fn input_size(&self) -> usize {
46 C::INPUT_BITS
47 }
48 fn output_size(&self) -> usize {
49 C::OUTPUT_BITS
50 }
51 fn get_circuit(&self, bit: usize) -> (&[Node], usize) {
52 self.0[bit].info()
53 }
54}
55
56pub trait ExecuteBDDCircuit<BE: Backend> {
57 fn execute_bdd_circuit_tmp_bytes<R, G>(&self, res_infos: &R, state_size: usize, ggsw_infos: &G) -> usize
58 where
59 R: GLWEInfos,
60 G: GGSWInfos;
61
62 fn execute_bdd_circuit<C, G, O>(&self, out: &mut [GLWE<O>], inputs: &G, circuit: &C, scratch: &mut Scratch<BE>)
63 where
64 G: GetGGSWBit<BE> + BitSize,
65 C: GetBitCircuitInfo,
66 O: DataMut,
67 {
68 self.execute_bdd_circuit_multi_thread(1, out, inputs, circuit, scratch);
69 }
70
71 fn execute_bdd_circuit_multi_thread<C, G, O>(
72 &self,
73 threads: usize,
74 out: &mut [GLWE<O>],
75 inputs: &G,
76 circuit: &C,
77 scratch: &mut Scratch<BE>,
78 ) where
79 G: GetGGSWBit<BE> + BitSize,
80 C: GetBitCircuitInfo,
81 O: DataMut;
82}
83
84pub trait BitSize {
85 fn bit_size(&self) -> usize;
86}
87
88impl<BE: Backend> ExecuteBDDCircuit<BE> for Module<BE>
89where
90 Self: Cmux<BE> + GLWECopy,
91 Scratch<BE>: ScratchTakeCore<BE>,
92{
93 fn execute_bdd_circuit_tmp_bytes<R, G>(&self, res_infos: &R, state_size: usize, ggsw_infos: &G) -> usize
94 where
95 R: GLWEInfos,
96 G: GGSWInfos,
97 {
98 2 * state_size * GLWE::bytes_of_from_infos(res_infos) + self.cmux_tmp_bytes(res_infos, res_infos, ggsw_infos)
99 }
100
101 fn execute_bdd_circuit_multi_thread<C, G, O>(
102 &self,
103 threads: usize,
104 out: &mut [GLWE<O>],
105 inputs: &G,
106 circuit: &C,
107 scratch: &mut Scratch<BE>,
108 ) where
109 G: GetGGSWBit<BE> + BitSize,
110 C: GetBitCircuitInfo,
111 O: DataMut,
112 {
113 #[cfg(debug_assertions)]
114 {
115 assert!(
116 inputs.bit_size() >= circuit.input_size(),
117 "inputs.bit_size(): {} < circuit.input_size():{}",
118 inputs.bit_size(),
119 circuit.input_size()
120 );
121 assert!(
122 out.len() >= circuit.output_size(),
123 "out.len(): {} < circuit.output_size(): {}",
124 out.len(),
125 circuit.output_size()
126 );
127 }
128
129 let mut max_state_size = 0;
130 for i in 0..circuit.output_size() {
131 let (_, state_size) = circuit.get_circuit(i);
132 max_state_size = max_state_size.max(state_size)
133 }
134
135 let scratch_thread_size: usize = self.execute_bdd_circuit_tmp_bytes(&out[0], max_state_size, &inputs.get_bit(0));
136
137 assert!(
138 scratch.available() >= threads * scratch_thread_size,
139 "scratch.available(): {} < threads:{threads} * scratch_thread_size: {scratch_thread_size}",
140 scratch.available()
141 );
142
143 let (mut scratches, _) = scratch.split_mut(threads, scratch_thread_size);
144
145 let chunk_size: usize = circuit.output_size().div_ceil(threads);
146
147 thread::scope(|scope| {
148 for (scratch_thread, out_chunk) in scratches
149 .iter_mut()
150 .zip(out[..circuit.output_size()].chunks_mut(chunk_size))
151 {
152 // Capture chunk + thread scratch by move
153 scope.spawn(move || {
154 for (idx, out_i) in out_chunk.iter_mut().enumerate() {
155 let (nodes, state_size) = circuit.get_circuit(idx);
156
157 if state_size == 0 {
158 out_i.data_mut().zero();
159 } else {
160 eval_level(self, out_i, inputs, nodes, state_size, *scratch_thread);
161 }
162 }
163 });
164 }
165 });
166
167 for out_i in out.iter_mut().skip(circuit.output_size()) {
168 out_i.data_mut().zero();
169 }
170 }
171}
172
173fn eval_level<M, R, G, BE: Backend>(
174 module: &M,
175 res: &mut R,
176 inputs: &G,
177 nodes: &[Node],
178 state_size: usize,
179 scratch: &mut Scratch<BE>,
180) where
181 M: Cmux<BE> + GLWECopy,
182 R: GLWEToMut,
183 G: GetGGSWBit<BE> + BitSize,
184 Scratch<BE>: ScratchTakeCore<BE>,
185{
186 assert!(nodes.len().is_multiple_of(state_size));
187 let res: &mut GLWE<&mut [u8]> = &mut res.to_mut();
188
189 let (mut level, scratch_1) = scratch.take_glwe_slice(state_size * 2, res);
190
191 level.iter_mut().for_each(|ct| ct.data_mut().zero());
192
193 // TODO: implement API on GLWE
194 level[1]
195 .data_mut()
196 .encode_coeff_i64(res.base2k().into(), 0, 2, 0, 1);
197
198 let mut level_ref: Vec<&mut GLWE<&mut [u8]>> = level.iter_mut().collect_vec();
199 let (mut prev_level, mut next_level) = level_ref.split_at_mut(state_size);
200
201 let (all_but_last, last) = nodes.split_at(nodes.len() - state_size);
202
203 for nodes_lvl in all_but_last.chunks_exact(state_size) {
204 for (j, node) in nodes_lvl.iter().enumerate() {
205 match node {
206 Node::Cmux(in_idx, hi_idx, lo_idx) => {
207 module.cmux(
208 next_level[j],
209 prev_level[*hi_idx],
210 prev_level[*lo_idx],
211 &inputs.get_bit(*in_idx),
212 scratch_1,
213 );
214 }
215 Node::Copy => module.glwe_copy(next_level[j], prev_level[j]), /* Update BDD circuits to order Cmux -> Copy -> None so that mem swap can be used */
216 Node::None => {}
217 }
218 }
219
220 (prev_level, next_level) = (next_level, prev_level);
221 }
222
223 // Last chunck of max_inter_state Nodes is always structured as
224 // [CMUX, NONE, NONE, ..., NONE]
225 match &last[0] {
226 Node::Cmux(in_idx, hi_idx, lo_idx) => {
227 module.cmux(
228 res,
229 prev_level[*hi_idx],
230 prev_level[*lo_idx],
231 &inputs.get_bit(*in_idx),
232 scratch_1,
233 );
234 }
235 _ => {
236 panic!("invalid last node, should be CMUX")
237 }
238 }
239}
240
241impl<const N: usize> BitCircuit<N> {
242 pub const fn new(nodes: [Node; N], max_inter_state: usize) -> Self {
243 Self {
244 nodes,
245 max_inter_state,
246 }
247 }
248}
249impl<const N: usize> BitCircuitInfo for BitCircuit<N> {
250 fn info(&self) -> (&[Node], usize) {
251 (self.nodes.as_ref(), self.max_inter_state)
252 }
253}
254
255#[derive(Debug)]
256pub enum Node {
257 Cmux(usize, usize, usize),
258 Copy,
259 None,
260}
261
262pub trait Cmux<BE: Backend>
263where
264 Self: Sized
265 + GLWEExternalProductInternal<BE>
266 + GLWESub
267 + VecZnxBigAddSmallInplace<BE>
268 + GLWENormalize<BE>
269 + VecZnxDftBytesOf
270 + VecZnxBigNormalize<BE>
271 + VecZnxBigNormalizeTmpBytes,
272{
273 fn cmux_tmp_bytes<R, A, B>(&self, res_infos: &R, a_infos: &A, b_infos: &B) -> usize
274 where
275 R: GLWEInfos,
276 A: GLWEInfos,
277 B: GGSWInfos,
278 {
279 let res_dft: usize = self.bytes_of_vec_znx_dft((b_infos.rank() + 1).into(), b_infos.size());
280 res_dft
281 + self
282 .glwe_external_product_internal_tmp_bytes(res_infos, a_infos, b_infos)
283 .max(self.vec_znx_big_normalize_tmp_bytes())
284 }
285
286 fn cmux<R, T, F, S>(&self, res: &mut R, t: &T, f: &F, s: &S, scratch: &mut Scratch<BE>)
287 where
288 R: GLWEToMut,
289 T: GLWEToRef,
290 F: GLWEToRef,
291 S: GGSWPreparedToRef<BE> + GGSWInfos,
292 Scratch<BE>: ScratchTakeCore<BE>,
293 {
294 let res: &mut GLWE<&mut [u8]> = &mut res.to_mut();
295 let s: &GGSWPrepared<&[u8], BE> = &s.to_ref();
296 let f: GLWE<&[u8]> = f.to_ref();
297
298 let res_base2k: usize = res.base2k().into();
299 let ggsw_base2k: usize = s.base2k().into();
300
301 self.glwe_sub(res, t, &f);
302 let (res_dft, scratch_1) = scratch.take_vec_znx_dft(self, (res.rank() + 1).into(), s.size()); // Todo optimise
303 let mut res_big: VecZnxBig<&mut [u8], BE> = self.glwe_external_product_internal(res_dft, res, s, scratch_1);
304 for j in 0..(res.rank() + 1).into() {
305 self.vec_znx_big_add_small_inplace(&mut res_big, j, f.data(), j);
306 self.vec_znx_big_normalize(
307 res_base2k,
308 res.data_mut(),
309 j,
310 ggsw_base2k,
311 &res_big,
312 j,
313 scratch_1,
314 );
315 }
316 }
317
318 fn cmux_inplace<R, A, S>(&self, res: &mut R, a: &A, s: &S, scratch: &mut Scratch<BE>)
319 where
320 R: GLWEToMut,
321 A: GLWEToRef,
322 S: GGSWPreparedToRef<BE> + GGSWInfos,
323 Scratch<BE>: ScratchTakeCore<BE>,
324 {
325 let res: &mut GLWE<&mut [u8]> = &mut res.to_mut();
326 let s: &GGSWPrepared<&[u8], BE> = &s.to_ref();
327 let a: GLWE<&[u8]> = a.to_ref();
328 let res_base2k: usize = res.base2k().into();
329 let ggsw_base2k: usize = s.base2k().into();
330 self.glwe_sub_inplace(res, &a);
331 let (res_dft, scratch_1) = scratch.take_vec_znx_dft(self, (res.rank() + 1).into(), s.size()); // Todo optimise
332 let mut res_big: VecZnxBig<&mut [u8], BE> = self.glwe_external_product_internal(res_dft, res, s, scratch_1);
333 for j in 0..(res.rank() + 1).into() {
334 self.vec_znx_big_add_small_inplace(&mut res_big, j, a.data(), j);
335 self.vec_znx_big_normalize(
336 res_base2k,
337 res.data_mut(),
338 j,
339 ggsw_base2k,
340 &res_big,
341 j,
342 scratch_1,
343 );
344 }
345 }
346}
347
348impl<BE: Backend> Cmux<BE> for Module<BE>
349where
350 Self: Sized
351 + GLWEExternalProductInternal<BE>
352 + GLWESub
353 + VecZnxBigAddSmallInplace<BE>
354 + GLWENormalize<BE>
355 + VecZnxDftBytesOf
356 + VecZnxBigNormalize<BE>
357 + VecZnxBigNormalizeTmpBytes,
358 Scratch<BE>: ScratchTakeCore<BE>,
359{
360}