1#rdi datare ptr
  2#rsi dataim ptr
  3#rdx om ptr
  4.globl reim_ifft16_avx_fma
  5reim_ifft16_avx_fma:
  6vmovupd     (%rdi),%ymm0       # ra0
  7vmovupd     0x20(%rdi),%ymm1   # ra4
  8vmovupd     0x40(%rdi),%ymm2   # ra8
  9vmovupd     0x60(%rdi),%ymm3   # ra12
 10vmovupd     (%rsi),%ymm4       # ia0
 11vmovupd     0x20(%rsi),%ymm5   # ia4
 12vmovupd     0x40(%rsi),%ymm6   # ia8
 13vmovupd     0x60(%rsi),%ymm7   # ia12
 14
 151:
 16vmovupd     0x00(%rdx),%ymm12
 17vmovupd     0x20(%rdx),%ymm13
 18
 19vperm2f128 $0x31,%ymm2,%ymm0,%ymm8   # ymm8 contains re to mul (tw)
 20vperm2f128 $0x31,%ymm3,%ymm1,%ymm9   # ymm9 contains re to mul (itw)
 21vperm2f128 $0x31,%ymm6,%ymm4,%ymm10   # ymm10 contains im to mul (tw)
 22vperm2f128 $0x31,%ymm7,%ymm5,%ymm11   # ymm11 contains im to mul (itw)
 23vperm2f128 $0x20,%ymm2,%ymm0,%ymm0   # ymm0 contains re to add (tw)
 24vperm2f128 $0x20,%ymm3,%ymm1,%ymm1   # ymm1 contains re to add (itw)
 25vperm2f128 $0x20,%ymm6,%ymm4,%ymm2   # ymm2 contains im to add (tw)
 26vperm2f128 $0x20,%ymm7,%ymm5,%ymm3   # ymm3 contains im to add (itw)
 27
 28vunpckhpd %ymm1,%ymm0,%ymm4   # (0,1) -> (0,4)
 29vunpckhpd %ymm3,%ymm2,%ymm6   # (2,3) -> (2,6)
 30vunpckhpd %ymm9,%ymm8,%ymm5   # (8,9) -> (1,5)
 31vunpckhpd %ymm11,%ymm10,%ymm7 # (10,11) -> (3,7)
 32vunpcklpd %ymm1,%ymm0,%ymm0
 33vunpcklpd %ymm3,%ymm2,%ymm2
 34vunpcklpd %ymm9,%ymm8,%ymm1
 35vunpcklpd %ymm11,%ymm10,%ymm3
 36
 37# invctwiddle Re:(ymm0,ymm4) and Im:(ymm2,ymm6) with omega=(ymm12,ymm13)
 38# invcitwiddle Re:(ymm1,ymm5) and Im:(ymm3,ymm7) with omega=(ymm12,ymm13)
 39vsubpd      %ymm4,%ymm0,%ymm8  # retw
 40vsubpd      %ymm5,%ymm1,%ymm9  # reitw
 41vsubpd      %ymm6,%ymm2,%ymm10 # imtw
 42vsubpd      %ymm7,%ymm3,%ymm11 # imitw
 43vaddpd      %ymm4,%ymm0,%ymm0
 44vaddpd      %ymm5,%ymm1,%ymm1
 45vaddpd      %ymm6,%ymm2,%ymm2
 46vaddpd      %ymm7,%ymm3,%ymm3
 47# multiply 8,9,10,11 by 12,13, result to: 4,5,6,7
 48# twiddles use reom=ymm12, imom=ymm13
 49# invtwiddles use reom=ymm13, imom=-ymm12
 50vmulpd      %ymm10,%ymm13,%ymm4 # imtw.omai  (tw)
 51vmulpd      %ymm11,%ymm12,%ymm5 # imitw.omar (itw)
 52vmulpd      %ymm8,%ymm13,%ymm6  # retw.omai  (tw)
 53vmulpd      %ymm9,%ymm12,%ymm7  # reitw.omar (itw)
 54vfmsub231pd %ymm8,%ymm12,%ymm4  # rprod0 (tw)
 55vfmadd231pd %ymm9,%ymm13,%ymm5  # rprod4 (itw)
 56vfmadd231pd %ymm10,%ymm12,%ymm6 # iprod0 (tw)
 57vfmsub231pd %ymm11,%ymm13,%ymm7 # iprod4 (itw)
 58
 59vunpckhpd %ymm7,%ymm3,%ymm11   # (0,4) -> (0,1)
 60vunpckhpd %ymm5,%ymm1,%ymm9    # (2,6) -> (2,3)
 61vunpcklpd %ymm7,%ymm3,%ymm10
 62vunpcklpd %ymm5,%ymm1,%ymm8
 63vunpckhpd %ymm6,%ymm2,%ymm3    # (1,5) -> (8,9)
 64vunpckhpd %ymm4,%ymm0,%ymm1    # (3,7) -> (10,11)
 65vunpcklpd %ymm6,%ymm2,%ymm2
 66vunpcklpd %ymm4,%ymm0,%ymm0
 67
 68/*
 69vperm2f128 $0x31,%ymm10,%ymm2,%ymm6
 70vperm2f128 $0x31,%ymm11,%ymm3,%ymm7
 71vperm2f128 $0x20,%ymm10,%ymm2,%ymm4
 72vperm2f128 $0x20,%ymm11,%ymm3,%ymm5
 73vperm2f128 $0x31,%ymm8,%ymm0,%ymm2
 74vperm2f128 $0x31,%ymm9,%ymm1,%ymm3
 75vperm2f128 $0x20,%ymm8,%ymm0,%ymm0
 76vperm2f128 $0x20,%ymm9,%ymm1,%ymm1
 77*/
 782:
 79vmovupd     0x40(%rdx),%ymm12
 80vshufpd     $15, %ymm12, %ymm12, %ymm13  # ymm13: omaiii'i'
 81vshufpd     $0,  %ymm12, %ymm12, %ymm12  # ymm12: omarrr'r'
 82
 83/*
 84vperm2f128 $0x31,%ymm2,%ymm0,%ymm8   # ymm8 contains re to mul (tw)
 85vperm2f128 $0x31,%ymm3,%ymm1,%ymm9   # ymm9 contains re to mul (itw)
 86vperm2f128 $0x31,%ymm6,%ymm4,%ymm10   # ymm10 contains im to mul (tw)
 87vperm2f128 $0x31,%ymm7,%ymm5,%ymm11   # ymm11 contains im to mul (itw)
 88vperm2f128 $0x20,%ymm2,%ymm0,%ymm0   # ymm0 contains re to add (tw)
 89vperm2f128 $0x20,%ymm3,%ymm1,%ymm1   # ymm1 contains re to add (itw)
 90vperm2f128 $0x20,%ymm6,%ymm4,%ymm2   # ymm2 contains im to add (tw)
 91vperm2f128 $0x20,%ymm7,%ymm5,%ymm3   # ymm3 contains im to add (itw)
 92*/
 93
 94# invctwiddle Re:(ymm0,ymm8) and Im:(ymm2,ymm10) with omega=(ymm12,ymm13)
 95# invcitwiddle Re:(ymm1,ymm9) and Im:(ymm3,ymm11) with omega=(ymm12,ymm13)
 96vsubpd      %ymm8,%ymm0,%ymm4  # retw
 97vsubpd      %ymm9,%ymm1,%ymm5  # reitw
 98vsubpd      %ymm10,%ymm2,%ymm6 # imtw
 99vsubpd      %ymm11,%ymm3,%ymm7 # imitw
100vaddpd      %ymm8,%ymm0,%ymm0
101vaddpd      %ymm9,%ymm1,%ymm1
102vaddpd      %ymm10,%ymm2,%ymm2
103vaddpd      %ymm11,%ymm3,%ymm3
104# multiply 4,5,6,7 by 12,13, result to 8,9,10,11
105# twiddles use reom=ymm12, imom=ymm13
106# invtwiddles use reom=ymm13, imom=-ymm12
107vmulpd      %ymm6,%ymm13,%ymm8  # imtw.omai (tw)
108vmulpd      %ymm7,%ymm12,%ymm9  # imitw.omar (itw)
109vmulpd      %ymm4,%ymm13,%ymm10 # retw.omai (tw)
110vmulpd      %ymm5,%ymm12,%ymm11 # reitw.omar (itw)
111vfmsub231pd %ymm4,%ymm12,%ymm8 # rprod0 (tw)
112vfmadd231pd %ymm5,%ymm13,%ymm9 # rprod4 (itw)
113vfmadd231pd %ymm6,%ymm12,%ymm10 # iprod0 (tw)
114vfmsub231pd %ymm7,%ymm13,%ymm11 # iprod4 (itw)
115
116vperm2f128 $0x31,%ymm10,%ymm2,%ymm6
117vperm2f128 $0x31,%ymm11,%ymm3,%ymm7
118vperm2f128 $0x20,%ymm10,%ymm2,%ymm4
119vperm2f128 $0x20,%ymm11,%ymm3,%ymm5
120vperm2f128 $0x31,%ymm8,%ymm0,%ymm2
121vperm2f128 $0x31,%ymm9,%ymm1,%ymm3
122vperm2f128 $0x20,%ymm8,%ymm0,%ymm0
123vperm2f128 $0x20,%ymm9,%ymm1,%ymm1
124
1253:
126vmovupd     0x60(%rdx),%xmm12
127vinsertf128 $1, %xmm12, %ymm12, %ymm12   # omriri
128vshufpd     $15, %ymm12, %ymm12, %ymm13  # ymm13: omai
129vshufpd     $0,  %ymm12, %ymm12, %ymm12  # ymm12: omar
130
131# invctwiddle Re:(ymm0,ymm1) and Im:(ymm4,ymm5) with omega=(ymm12,ymm13)
132# invcitwiddle Re:(ymm2,ymm3) and Im:(ymm6,ymm7) with omega=(ymm12,ymm13)
133vsubpd      %ymm1,%ymm0,%ymm8   # retw
134vsubpd      %ymm3,%ymm2,%ymm9   # reitw
135vsubpd      %ymm5,%ymm4,%ymm10  # imtw
136vsubpd      %ymm7,%ymm6,%ymm11  # imitw
137vaddpd      %ymm1,%ymm0,%ymm0
138vaddpd      %ymm3,%ymm2,%ymm2
139vaddpd      %ymm5,%ymm4,%ymm4
140vaddpd      %ymm7,%ymm6,%ymm6
141# multiply 8,9,10,11 by 12,13, result to 1,3,5,7
142# twiddles use reom=ymm12, imom=ymm13
143# invtwiddles use reom=ymm13, imom=-ymm12
144vmulpd      %ymm10,%ymm13,%ymm1 # imtw.omai (tw)
145vmulpd      %ymm11,%ymm12,%ymm3 # imitw.omar (itw)
146vmulpd      %ymm8,%ymm13,%ymm5  # retw.omai (tw)
147vmulpd      %ymm9,%ymm12,%ymm7  # reitw.omar (itw)
148vfmsub231pd %ymm8,%ymm12,%ymm1 # rprod0 (tw)
149vfmadd231pd %ymm9,%ymm13,%ymm3 # rprod4 (itw)
150vfmadd231pd %ymm10,%ymm12,%ymm5 # iprod0 (tw)
151vfmsub231pd %ymm11,%ymm13,%ymm7 # iprod4 (itw)
152
1534:
154vmovupd     0x70(%rdx),%xmm12
155vinsertf128 $1, %xmm12, %ymm12, %ymm12   # omriri
156vshufpd     $15, %ymm12, %ymm12, %ymm13  # ymm13: omai
157vshufpd     $0,  %ymm12, %ymm12, %ymm12  # ymm12: omar
158
159# invctwiddle Re:(ymm0,ymm2) and Im:(ymm4,ymm6) with omega=(ymm12,ymm13)
160# invctwiddle Re:(ymm1,ymm3) and Im:(ymm5,ymm7) with omega=(ymm12,ymm13)
161vsubpd      %ymm2,%ymm0,%ymm8  # retw1
162vsubpd      %ymm3,%ymm1,%ymm9  # retw2
163vsubpd      %ymm6,%ymm4,%ymm10 # imtw1
164vsubpd      %ymm7,%ymm5,%ymm11 # imtw2
165vaddpd      %ymm2,%ymm0,%ymm0
166vaddpd      %ymm3,%ymm1,%ymm1
167vaddpd      %ymm6,%ymm4,%ymm4
168vaddpd      %ymm7,%ymm5,%ymm5
169# multiply 8,9,10,11 by 12,13, result to 2,3,6,7
170# twiddles use reom=ymm12, imom=ymm13
171vmulpd      %ymm10,%ymm13,%ymm2 # imtw1.omai
172vmulpd      %ymm11,%ymm13,%ymm3 # imtw2.omai
173vmulpd      %ymm8,%ymm13,%ymm6  # retw1.omai
174vmulpd      %ymm9,%ymm13,%ymm7  # retw2.omai
175vfmsub231pd %ymm8,%ymm12,%ymm2 # rprod0
176vfmsub231pd %ymm9,%ymm12,%ymm3 # rprod4
177vfmadd231pd %ymm10,%ymm12,%ymm6 # iprod0
178vfmadd231pd %ymm11,%ymm12,%ymm7 # iprod4
179
1805:
181vmovupd     %ymm0,(%rdi)       # ra0
182vmovupd     %ymm1,0x20(%rdi)   # ra4
183vmovupd     %ymm2,0x40(%rdi)   # ra8
184vmovupd     %ymm3,0x60(%rdi)   # ra12
185vmovupd     %ymm4,(%rsi)       # ia0
186vmovupd     %ymm5,0x20(%rsi)   # ia4
187vmovupd     %ymm6,0x40(%rsi)   # ia8
188vmovupd     %ymm7,0x60(%rsi)   # ia12
189vzeroupper
190ret
191.size	reim_ifft16_avx_fma, .-reim_ifft16_avx_fma
192.section .note.GNU-stack,"",@progbits