1# ----------------------------------------------------------------------
  2# This kernel is a direct port of the IFFT16 routine from spqlios-arithmetic
  3# (https://github.com/tfhe/spqlios-arithmetic)
  4# ----------------------------------------------------------------------
  5#
  6
  7.text
  8.globl  ifft16_avx2_fma_asm
  9.hidden ifft16_avx2_fma_asm
 10.p2align 4, 0x90
 11.type   ifft16_avx2_fma_asm,@function
 12ifft16_avx2_fma_asm:
 13.att_syntax prefix
 14
 15vmovupd     (%rdi),%ymm0       # ra0
 16vmovupd     0x20(%rdi),%ymm1   # ra4
 17vmovupd     0x40(%rdi),%ymm2   # ra8
 18vmovupd     0x60(%rdi),%ymm3   # ra12
 19vmovupd     (%rsi),%ymm4       # ia0
 20vmovupd     0x20(%rsi),%ymm5   # ia4
 21vmovupd     0x40(%rsi),%ymm6   # ia8
 22vmovupd     0x60(%rsi),%ymm7   # ia12
 23
 241:
 25vmovupd     0x00(%rdx),%ymm12
 26vmovupd     0x20(%rdx),%ymm13
 27
 28vperm2f128 $0x31,%ymm2,%ymm0,%ymm8   # ymm8 contains re to mul (tw)
 29vperm2f128 $0x31,%ymm3,%ymm1,%ymm9   # ymm9 contains re to mul (itw)
 30vperm2f128 $0x31,%ymm6,%ymm4,%ymm10   # ymm10 contains im to mul (tw)
 31vperm2f128 $0x31,%ymm7,%ymm5,%ymm11   # ymm11 contains im to mul (itw)
 32vperm2f128 $0x20,%ymm2,%ymm0,%ymm0   # ymm0 contains re to add (tw)
 33vperm2f128 $0x20,%ymm3,%ymm1,%ymm1   # ymm1 contains re to add (itw)
 34vperm2f128 $0x20,%ymm6,%ymm4,%ymm2   # ymm2 contains im to add (tw)
 35vperm2f128 $0x20,%ymm7,%ymm5,%ymm3   # ymm3 contains im to add (itw)
 36
 37vunpckhpd %ymm1,%ymm0,%ymm4   # (0,1) -> (0,4)
 38vunpckhpd %ymm3,%ymm2,%ymm6   # (2,3) -> (2,6)
 39vunpckhpd %ymm9,%ymm8,%ymm5   # (8,9) -> (1,5)
 40vunpckhpd %ymm11,%ymm10,%ymm7 # (10,11) -> (3,7)
 41vunpcklpd %ymm1,%ymm0,%ymm0
 42vunpcklpd %ymm3,%ymm2,%ymm2
 43vunpcklpd %ymm9,%ymm8,%ymm1
 44vunpcklpd %ymm11,%ymm10,%ymm3
 45
 46# invctwiddle Re:(ymm0,ymm4) and Im:(ymm2,ymm6) with omega=(ymm12,ymm13)
 47# invcitwiddle Re:(ymm1,ymm5) and Im:(ymm3,ymm7) with omega=(ymm12,ymm13)
 48vsubpd      %ymm4,%ymm0,%ymm8  # retw
 49vsubpd      %ymm5,%ymm1,%ymm9  # reitw
 50vsubpd      %ymm6,%ymm2,%ymm10 # imtw
 51vsubpd      %ymm7,%ymm3,%ymm11 # imitw
 52vaddpd      %ymm4,%ymm0,%ymm0
 53vaddpd      %ymm5,%ymm1,%ymm1
 54vaddpd      %ymm6,%ymm2,%ymm2
 55vaddpd      %ymm7,%ymm3,%ymm3
 56# multiply 8,9,10,11 by 12,13, result to: 4,5,6,7
 57# twiddles use reom=ymm12, imom=ymm13
 58# invtwiddles use reom=ymm13, imom=-ymm12
 59vmulpd      %ymm10,%ymm13,%ymm4 # imtw.omai  (tw)
 60vmulpd      %ymm11,%ymm12,%ymm5 # imitw.omar (itw)
 61vmulpd      %ymm8,%ymm13,%ymm6  # retw.omai  (tw)
 62vmulpd      %ymm9,%ymm12,%ymm7  # reitw.omar (itw)
 63vfmsub231pd %ymm8,%ymm12,%ymm4  # rprod0 (tw)
 64vfmadd231pd %ymm9,%ymm13,%ymm5  # rprod4 (itw)
 65vfmadd231pd %ymm10,%ymm12,%ymm6 # iprod0 (tw)
 66vfmsub231pd %ymm11,%ymm13,%ymm7 # iprod4 (itw)
 67
 68vunpckhpd %ymm7,%ymm3,%ymm11   # (0,4) -> (0,1)
 69vunpckhpd %ymm5,%ymm1,%ymm9    # (2,6) -> (2,3)
 70vunpcklpd %ymm7,%ymm3,%ymm10
 71vunpcklpd %ymm5,%ymm1,%ymm8
 72vunpckhpd %ymm6,%ymm2,%ymm3    # (1,5) -> (8,9)
 73vunpckhpd %ymm4,%ymm0,%ymm1    # (3,7) -> (10,11)
 74vunpcklpd %ymm6,%ymm2,%ymm2
 75vunpcklpd %ymm4,%ymm0,%ymm0
 76
 772:
 78vmovupd     0x40(%rdx),%ymm12
 79vshufpd     $15, %ymm12, %ymm12, %ymm13  # ymm13: omaiii'i'
 80vshufpd     $0,  %ymm12, %ymm12, %ymm12  # ymm12: omarrr'r'
 81
 82# invctwiddle Re:(ymm0,ymm8) and Im:(ymm2,ymm10) with omega=(ymm12,ymm13)
 83# invcitwiddle Re:(ymm1,ymm9) and Im:(ymm3,ymm11) with omega=(ymm12,ymm13)
 84vsubpd      %ymm8,%ymm0,%ymm4  # retw
 85vsubpd      %ymm9,%ymm1,%ymm5  # reitw
 86vsubpd      %ymm10,%ymm2,%ymm6 # imtw
 87vsubpd      %ymm11,%ymm3,%ymm7 # imitw
 88vaddpd      %ymm8,%ymm0,%ymm0
 89vaddpd      %ymm9,%ymm1,%ymm1
 90vaddpd      %ymm10,%ymm2,%ymm2
 91vaddpd      %ymm11,%ymm3,%ymm3
 92# multiply 4,5,6,7 by 12,13, result to 8,9,10,11
 93# twiddles use reom=ymm12, imom=ymm13
 94# invtwiddles use reom=ymm13, imom=-ymm12
 95vmulpd      %ymm6,%ymm13,%ymm8  # imtw.omai (tw)
 96vmulpd      %ymm7,%ymm12,%ymm9  # imitw.omar (itw)
 97vmulpd      %ymm4,%ymm13,%ymm10 # retw.omai (tw)
 98vmulpd      %ymm5,%ymm12,%ymm11 # reitw.omar (itw)
 99vfmsub231pd %ymm4,%ymm12,%ymm8 # rprod0 (tw)
100vfmadd231pd %ymm5,%ymm13,%ymm9 # rprod4 (itw)
101vfmadd231pd %ymm6,%ymm12,%ymm10 # iprod0 (tw)
102vfmsub231pd %ymm7,%ymm13,%ymm11 # iprod4 (itw)
103
104vperm2f128 $0x31,%ymm10,%ymm2,%ymm6
105vperm2f128 $0x31,%ymm11,%ymm3,%ymm7
106vperm2f128 $0x20,%ymm10,%ymm2,%ymm4
107vperm2f128 $0x20,%ymm11,%ymm3,%ymm5
108vperm2f128 $0x31,%ymm8,%ymm0,%ymm2
109vperm2f128 $0x31,%ymm9,%ymm1,%ymm3
110vperm2f128 $0x20,%ymm8,%ymm0,%ymm0
111vperm2f128 $0x20,%ymm9,%ymm1,%ymm1
112
1133:
114vmovupd     0x60(%rdx),%xmm12
115vinsertf128 $1, %xmm12, %ymm12, %ymm12   # omriri
116vshufpd     $15, %ymm12, %ymm12, %ymm13  # ymm13: omai
117vshufpd     $0,  %ymm12, %ymm12, %ymm12  # ymm12: omar
118
119# invctwiddle Re:(ymm0,ymm1) and Im:(ymm4,ymm5) with omega=(ymm12,ymm13)
120# invcitwiddle Re:(ymm2,ymm3) and Im:(ymm6,ymm7) with omega=(ymm12,ymm13)
121vsubpd      %ymm1,%ymm0,%ymm8   # retw
122vsubpd      %ymm3,%ymm2,%ymm9   # reitw
123vsubpd      %ymm5,%ymm4,%ymm10  # imtw
124vsubpd      %ymm7,%ymm6,%ymm11  # imitw
125vaddpd      %ymm1,%ymm0,%ymm0
126vaddpd      %ymm3,%ymm2,%ymm2
127vaddpd      %ymm5,%ymm4,%ymm4
128vaddpd      %ymm7,%ymm6,%ymm6
129# multiply 8,9,10,11 by 12,13, result to 1,3,5,7
130# twiddles use reom=ymm12, imom=ymm13
131# invtwiddles use reom=ymm13, imom=-ymm12
132vmulpd      %ymm10,%ymm13,%ymm1 # imtw.omai (tw)
133vmulpd      %ymm11,%ymm12,%ymm3 # imitw.omar (itw)
134vmulpd      %ymm8,%ymm13,%ymm5  # retw.omai (tw)
135vmulpd      %ymm9,%ymm12,%ymm7  # reitw.omar (itw)
136vfmsub231pd %ymm8,%ymm12,%ymm1 # rprod0 (tw)
137vfmadd231pd %ymm9,%ymm13,%ymm3 # rprod4 (itw)
138vfmadd231pd %ymm10,%ymm12,%ymm5 # iprod0 (tw)
139vfmsub231pd %ymm11,%ymm13,%ymm7 # iprod4 (itw)
140
1414:
142vmovupd     0x70(%rdx),%xmm12
143vinsertf128 $1, %xmm12, %ymm12, %ymm12   # omriri
144vshufpd     $15, %ymm12, %ymm12, %ymm13  # ymm13: omai
145vshufpd     $0,  %ymm12, %ymm12, %ymm12  # ymm12: omar
146
147# invctwiddle Re:(ymm0,ymm2) and Im:(ymm4,ymm6) with omega=(ymm12,ymm13)
148# invctwiddle Re:(ymm1,ymm3) and Im:(ymm5,ymm7) with omega=(ymm12,ymm13)
149vsubpd      %ymm2,%ymm0,%ymm8  # retw1
150vsubpd      %ymm3,%ymm1,%ymm9  # retw2
151vsubpd      %ymm6,%ymm4,%ymm10 # imtw1
152vsubpd      %ymm7,%ymm5,%ymm11 # imtw2
153vaddpd      %ymm2,%ymm0,%ymm0
154vaddpd      %ymm3,%ymm1,%ymm1
155vaddpd      %ymm6,%ymm4,%ymm4
156vaddpd      %ymm7,%ymm5,%ymm5
157# multiply 8,9,10,11 by 12,13, result to 2,3,6,7
158# twiddles use reom=ymm12, imom=ymm13
159vmulpd      %ymm10,%ymm13,%ymm2 # imtw1.omai
160vmulpd      %ymm11,%ymm13,%ymm3 # imtw2.omai
161vmulpd      %ymm8,%ymm13,%ymm6  # retw1.omai
162vmulpd      %ymm9,%ymm13,%ymm7  # retw2.omai
163vfmsub231pd %ymm8,%ymm12,%ymm2 # rprod0
164vfmsub231pd %ymm9,%ymm12,%ymm3 # rprod4
165vfmadd231pd %ymm10,%ymm12,%ymm6 # iprod0
166vfmadd231pd %ymm11,%ymm12,%ymm7 # iprod4
167
1685:
169vmovupd     %ymm0,(%rdi)       # ra0
170vmovupd     %ymm1,0x20(%rdi)   # ra4
171vmovupd     %ymm2,0x40(%rdi)   # ra8
172vmovupd     %ymm3,0x60(%rdi)   # ra12
173vmovupd     %ymm4,(%rsi)       # ia0
174vmovupd     %ymm5,0x20(%rsi)   # ia4
175vmovupd     %ymm6,0x40(%rsi)   # ia8
176vmovupd     %ymm7,0x60(%rsi)   # ia12
177vzeroupper
178ret
179
180.size	ifft16_avx_fma, .-ifft16_avx_fma
181.section .note.GNU-stack,"",@progbits