1# ----------------------------------------------------------------------
  2# This kernel is a direct port of the FFT16 routine from spqlios-arithmetic
  3# (https://github.com/tfhe/spqlios-arithmetic)
  4# ----------------------------------------------------------------------
  5#
  6
  7.text
  8.globl  fft16_avx2_fma_asm
  9.hidden fft16_avx2_fma_asm
 10.p2align 4, 0x90
 11.type   fft16_avx2_fma_asm,@function
 12fft16_avx2_fma_asm:
 13.att_syntax prefix
 14
 15# SysV args: %rdi = re*, %rsi = im*, %rdx = omg*
 16# stage 0: load inputs
 17vmovupd     (%rdi),%ymm0       # ra0
 18vmovupd     0x20(%rdi),%ymm1   # ra4
 19vmovupd     0x40(%rdi),%ymm2   # ra8
 20vmovupd     0x60(%rdi),%ymm3   # ra12
 21vmovupd     (%rsi),%ymm4       # ia0
 22vmovupd     0x20(%rsi),%ymm5   # ia4
 23vmovupd     0x40(%rsi),%ymm6   # ia8
 24vmovupd     0x60(%rsi),%ymm7   # ia12
 25
 26# stage 1
 27vmovupd     (%rdx),%xmm12
 28vinsertf128 $1, %xmm12, %ymm12, %ymm12   # omriri
 29vshufpd     $15, %ymm12, %ymm12, %ymm13  # omai
 30vshufpd     $0,  %ymm12, %ymm12, %ymm12  # omar
 31vmulpd      %ymm6,%ymm13,%ymm8
 32vmulpd      %ymm7,%ymm13,%ymm9
 33vmulpd      %ymm2,%ymm13,%ymm10
 34vmulpd      %ymm3,%ymm13,%ymm11
 35vfmsub231pd %ymm2,%ymm12,%ymm8
 36vfmsub231pd %ymm3,%ymm12,%ymm9
 37vfmadd231pd %ymm6,%ymm12,%ymm10
 38vfmadd231pd %ymm7,%ymm12,%ymm11
 39vsubpd      %ymm8,%ymm0,%ymm2
 40vsubpd      %ymm9,%ymm1,%ymm3
 41vsubpd      %ymm10,%ymm4,%ymm6
 42vsubpd      %ymm11,%ymm5,%ymm7
 43vaddpd      %ymm8,%ymm0,%ymm0
 44vaddpd      %ymm9,%ymm1,%ymm1
 45vaddpd      %ymm10,%ymm4,%ymm4
 46vaddpd      %ymm11,%ymm5,%ymm5
 47
 48# stage 2
 49vmovupd     16(%rdx),%xmm12
 50vinsertf128 $1, %xmm12, %ymm12, %ymm12   # omriri
 51vshufpd     $15, %ymm12, %ymm12, %ymm13  # omai
 52vshufpd     $0,  %ymm12, %ymm12, %ymm12  # omar
 53vmulpd      %ymm5,%ymm13,%ymm8
 54vmulpd      %ymm7,%ymm12,%ymm9
 55vmulpd      %ymm1,%ymm13,%ymm10
 56vmulpd      %ymm3,%ymm12,%ymm11
 57vfmsub231pd %ymm1,%ymm12,%ymm8
 58vfmadd231pd %ymm3,%ymm13,%ymm9
 59vfmadd231pd %ymm5,%ymm12,%ymm10
 60vfmsub231pd %ymm7,%ymm13,%ymm11
 61vsubpd      %ymm8,%ymm0,%ymm1
 62vaddpd      %ymm9,%ymm2,%ymm3
 63vsubpd      %ymm10,%ymm4,%ymm5
 64vaddpd      %ymm11,%ymm6,%ymm7
 65vaddpd      %ymm8,%ymm0,%ymm0
 66vsubpd      %ymm9,%ymm2,%ymm2
 67vaddpd      %ymm10,%ymm4,%ymm4
 68vsubpd      %ymm11,%ymm6,%ymm6
 69
 70# stage 3
 71vmovupd     0x20(%rdx),%ymm12
 72vshufpd     $15, %ymm12, %ymm12, %ymm13  # omai
 73vshufpd     $0,  %ymm12, %ymm12, %ymm12  # omar
 74
 75vperm2f128  $0x31,%ymm2,%ymm0,%ymm8
 76vperm2f128  $0x31,%ymm3,%ymm1,%ymm9
 77vperm2f128  $0x31,%ymm6,%ymm4,%ymm10
 78vperm2f128  $0x31,%ymm7,%ymm5,%ymm11
 79vperm2f128  $0x20,%ymm2,%ymm0,%ymm0
 80vperm2f128  $0x20,%ymm3,%ymm1,%ymm1
 81vperm2f128  $0x20,%ymm6,%ymm4,%ymm2
 82vperm2f128  $0x20,%ymm7,%ymm5,%ymm3
 83
 84vmulpd      %ymm10,%ymm13,%ymm4
 85vmulpd      %ymm11,%ymm12,%ymm5
 86vmulpd      %ymm8,%ymm13,%ymm6
 87vmulpd      %ymm9,%ymm12,%ymm7
 88vfmsub231pd %ymm8,%ymm12,%ymm4
 89vfmadd231pd %ymm9,%ymm13,%ymm5
 90vfmadd231pd %ymm10,%ymm12,%ymm6
 91vfmsub231pd %ymm11,%ymm13,%ymm7
 92vsubpd      %ymm4,%ymm0,%ymm8
 93vaddpd      %ymm5,%ymm1,%ymm9
 94vsubpd      %ymm6,%ymm2,%ymm10
 95vaddpd      %ymm7,%ymm3,%ymm11
 96vaddpd      %ymm4,%ymm0,%ymm0
 97vsubpd      %ymm5,%ymm1,%ymm1
 98vaddpd      %ymm6,%ymm2,%ymm2
 99vsubpd      %ymm7,%ymm3,%ymm3
100
101# stage 4
102vmovupd     0x40(%rdx),%ymm12
103vmovupd     0x60(%rdx),%ymm13
104
105vunpckhpd   %ymm1,%ymm0,%ymm4
106vunpckhpd   %ymm3,%ymm2,%ymm6
107vunpckhpd   %ymm9,%ymm8,%ymm5
108vunpckhpd   %ymm11,%ymm10,%ymm7
109vunpcklpd   %ymm1,%ymm0,%ymm0
110vunpcklpd   %ymm3,%ymm2,%ymm2
111vunpcklpd   %ymm9,%ymm8,%ymm1
112vunpcklpd   %ymm11,%ymm10,%ymm3
113
114vmulpd      %ymm6,%ymm13,%ymm8
115vmulpd      %ymm7,%ymm12,%ymm9
116vmulpd      %ymm4,%ymm13,%ymm10
117vmulpd      %ymm5,%ymm12,%ymm11
118vfmsub231pd %ymm4,%ymm12,%ymm8
119vfmadd231pd %ymm5,%ymm13,%ymm9
120vfmadd231pd %ymm6,%ymm12,%ymm10
121vfmsub231pd %ymm7,%ymm13,%ymm11
122vsubpd      %ymm8,%ymm0,%ymm4
123vaddpd      %ymm9,%ymm1,%ymm5
124vsubpd      %ymm10,%ymm2,%ymm6
125vaddpd      %ymm11,%ymm3,%ymm7
126vaddpd      %ymm8,%ymm0,%ymm0
127vsubpd      %ymm9,%ymm1,%ymm1
128vaddpd      %ymm10,%ymm2,%ymm2
129vsubpd      %ymm11,%ymm3,%ymm3
130
131vunpckhpd   %ymm7,%ymm3,%ymm11
132vunpckhpd   %ymm5,%ymm1,%ymm9
133vunpcklpd   %ymm7,%ymm3,%ymm10
134vunpcklpd   %ymm5,%ymm1,%ymm8
135vunpckhpd   %ymm6,%ymm2,%ymm3
136vunpckhpd   %ymm4,%ymm0,%ymm1
137vunpcklpd   %ymm6,%ymm2,%ymm2
138vunpcklpd   %ymm4,%ymm0,%ymm0
139
140vperm2f128  $0x31,%ymm10,%ymm2,%ymm6
141vperm2f128  $0x31,%ymm11,%ymm3,%ymm7
142vperm2f128  $0x20,%ymm10,%ymm2,%ymm4
143vperm2f128  $0x20,%ymm11,%ymm3,%ymm5
144vperm2f128  $0x31,%ymm8,%ymm0,%ymm2
145vperm2f128  $0x31,%ymm9,%ymm1,%ymm3
146vperm2f128  $0x20,%ymm8,%ymm0,%ymm0
147vperm2f128  $0x20,%ymm9,%ymm1,%ymm1
148
149# stores
150vmovupd     %ymm0,(%rdi)       # ra0
151vmovupd     %ymm1,0x20(%rdi)   # ra4
152vmovupd     %ymm2,0x40(%rdi)   # ra8
153vmovupd     %ymm3,0x60(%rdi)   # ra12
154vmovupd     %ymm4,(%rsi)       # ia0
155vmovupd     %ymm5,0x20(%rsi)   # ia4
156vmovupd     %ymm6,0x40(%rsi)   # ia8
157vmovupd     %ymm7,0x60(%rsi)   # ia12
158vzeroupper
159ret
160
161.size   fft16_avx2_fma_asm, .-fft16_avx2_fma_asm
162.section .note.GNU-stack,"",@progbits