1#rdi datare ptr
  2#rsi dataim ptr
  3#rdx om ptr
  4.globl reim_fft16_avx_fma
  5reim_fft16_avx_fma:
  6vmovupd     (%rdi),%ymm0       # ra0
  7vmovupd     0x20(%rdi),%ymm1   # ra4
  8vmovupd     0x40(%rdi),%ymm2   # ra8
  9vmovupd     0x60(%rdi),%ymm3   # ra12
 10vmovupd     (%rsi),%ymm4       # ia0
 11vmovupd     0x20(%rsi),%ymm5   # ia4
 12vmovupd     0x40(%rsi),%ymm6   # ia8
 13vmovupd     0x60(%rsi),%ymm7   # ia12
 14
 15vmovupd     (%rdx),%xmm12
 16vinsertf128 $1, %xmm12, %ymm12, %ymm12   # omriri
 17vshufpd     $15, %ymm12, %ymm12, %ymm13  # ymm13: omai
 18vshufpd     $0,  %ymm12, %ymm12, %ymm12  # ymm12: omar
 19vmulpd      %ymm6,%ymm13,%ymm8 # ia0.omai
 20vmulpd      %ymm7,%ymm13,%ymm9 # ia4.omai
 21vmulpd      %ymm2,%ymm13,%ymm10 # ra0.omai
 22vmulpd      %ymm3,%ymm13,%ymm11 # ra4.omai
 23vfmsub231pd %ymm2,%ymm12,%ymm8 # rprod0
 24vfmsub231pd %ymm3,%ymm12,%ymm9 # rprod4
 25vfmadd231pd %ymm6,%ymm12,%ymm10 # iprod0
 26vfmadd231pd %ymm7,%ymm12,%ymm11 # iprod4
 27vsubpd      %ymm8,%ymm0,%ymm2
 28vsubpd      %ymm9,%ymm1,%ymm3
 29vsubpd      %ymm10,%ymm4,%ymm6
 30vsubpd      %ymm11,%ymm5,%ymm7
 31vaddpd      %ymm8,%ymm0,%ymm0
 32vaddpd      %ymm9,%ymm1,%ymm1
 33vaddpd      %ymm10,%ymm4,%ymm4
 34vaddpd      %ymm11,%ymm5,%ymm5
 35
 361:
 37vmovupd     16(%rdx),%xmm12
 38vinsertf128 $1, %xmm12, %ymm12, %ymm12   # omriri
 39vshufpd     $15, %ymm12, %ymm12, %ymm13  # ymm13: omai
 40vshufpd     $0,  %ymm12, %ymm12, %ymm12  # ymm12: omar
 41vmulpd      %ymm5,%ymm13,%ymm8 # ia0.omai (tw)
 42vmulpd      %ymm7,%ymm12,%ymm9 # ia4.omar (itw)
 43vmulpd      %ymm1,%ymm13,%ymm10 # ra0.omai (tw)
 44vmulpd      %ymm3,%ymm12,%ymm11 # ra4.omar (itw)
 45vfmsub231pd %ymm1,%ymm12,%ymm8 # rprod0 (tw)
 46vfmadd231pd %ymm3,%ymm13,%ymm9 # rprod4 (itw)
 47vfmadd231pd %ymm5,%ymm12,%ymm10 # iprod0 (tw)
 48vfmsub231pd %ymm7,%ymm13,%ymm11 # iprod4 (itw)
 49vsubpd      %ymm8,%ymm0,%ymm1
 50vaddpd      %ymm9,%ymm2,%ymm3
 51vsubpd      %ymm10,%ymm4,%ymm5
 52vaddpd      %ymm11,%ymm6,%ymm7
 53vaddpd      %ymm8,%ymm0,%ymm0
 54vsubpd      %ymm9,%ymm2,%ymm2
 55vaddpd      %ymm10,%ymm4,%ymm4
 56vsubpd      %ymm11,%ymm6,%ymm6
 57
 582:
 59vmovupd     0x20(%rdx),%ymm12
 60vshufpd     $15, %ymm12, %ymm12, %ymm13  # ymm13: omaiii'i'
 61vshufpd     $0,  %ymm12, %ymm12, %ymm12  # ymm12: omarrr'r'
 62
 63vperm2f128 $0x31,%ymm2,%ymm0,%ymm8   # ymm8 contains re to mul (tw)
 64vperm2f128 $0x31,%ymm3,%ymm1,%ymm9   # ymm9 contains re to mul (itw)
 65vperm2f128 $0x31,%ymm6,%ymm4,%ymm10   # ymm10 contains im to mul (tw)
 66vperm2f128 $0x31,%ymm7,%ymm5,%ymm11   # ymm11 contains im to mul (itw)
 67vperm2f128 $0x20,%ymm2,%ymm0,%ymm0   # ymm0 contains re to add (tw)
 68vperm2f128 $0x20,%ymm3,%ymm1,%ymm1   # ymm1 contains re to add (itw)
 69vperm2f128 $0x20,%ymm6,%ymm4,%ymm2   # ymm2 contains im to add (tw)
 70vperm2f128 $0x20,%ymm7,%ymm5,%ymm3   # ymm3 contains im to add (itw)
 71
 72vmulpd      %ymm10,%ymm13,%ymm4 # ia0.omai (tw)
 73vmulpd      %ymm11,%ymm12,%ymm5 # ia4.omar (itw)
 74vmulpd      %ymm8,%ymm13,%ymm6 # ra0.omai (tw)
 75vmulpd      %ymm9,%ymm12,%ymm7 # ra4.omar (itw)
 76vfmsub231pd %ymm8,%ymm12,%ymm4 # rprod0 (tw)
 77vfmadd231pd %ymm9,%ymm13,%ymm5 # rprod4 (itw)
 78vfmadd231pd %ymm10,%ymm12,%ymm6 # iprod0 (tw)
 79vfmsub231pd %ymm11,%ymm13,%ymm7 # iprod4 (itw)
 80vsubpd      %ymm4,%ymm0,%ymm8
 81vaddpd      %ymm5,%ymm1,%ymm9
 82vsubpd      %ymm6,%ymm2,%ymm10
 83vaddpd      %ymm7,%ymm3,%ymm11
 84vaddpd      %ymm4,%ymm0,%ymm0
 85vsubpd      %ymm5,%ymm1,%ymm1
 86vaddpd      %ymm6,%ymm2,%ymm2
 87vsubpd      %ymm7,%ymm3,%ymm3
 88
 89#vperm2f128 $0x31,%ymm10,%ymm2,%ymm6
 90#vperm2f128 $0x31,%ymm11,%ymm3,%ymm7
 91#vperm2f128 $0x20,%ymm10,%ymm2,%ymm4
 92#vperm2f128 $0x20,%ymm11,%ymm3,%ymm5
 93#vperm2f128 $0x31,%ymm8,%ymm0,%ymm2
 94#vperm2f128 $0x31,%ymm9,%ymm1,%ymm3
 95#vperm2f128 $0x20,%ymm8,%ymm0,%ymm0
 96#vperm2f128 $0x20,%ymm9,%ymm1,%ymm1
 97
 983:
 99vmovupd     0x40(%rdx),%ymm12
100vmovupd     0x60(%rdx),%ymm13
101
102#vperm2f128 $0x31,%ymm2,%ymm0,%ymm8   # ymm8 contains re to mul (tw)
103#vperm2f128 $0x31,%ymm3,%ymm1,%ymm9   # ymm9 contains re to mul (itw)
104#vperm2f128 $0x31,%ymm6,%ymm4,%ymm10   # ymm10 contains im to mul (tw)
105#vperm2f128 $0x31,%ymm7,%ymm5,%ymm11   # ymm11 contains im to mul (itw)
106#vperm2f128 $0x20,%ymm2,%ymm0,%ymm0   # ymm0 contains re to add (tw)
107#vperm2f128 $0x20,%ymm3,%ymm1,%ymm1   # ymm1 contains re to add (itw)
108#vperm2f128 $0x20,%ymm6,%ymm4,%ymm2   # ymm2 contains im to add (tw)
109#vperm2f128 $0x20,%ymm7,%ymm5,%ymm3   # ymm3 contains im to add (itw)
110
111vunpckhpd %ymm1,%ymm0,%ymm4   # (0,1) -> (0,4)
112vunpckhpd %ymm3,%ymm2,%ymm6   # (2,3) -> (2,6)
113vunpckhpd %ymm9,%ymm8,%ymm5   # (8,9) -> (1,5)
114vunpckhpd %ymm11,%ymm10,%ymm7 # (10,11) -> (3,7)
115vunpcklpd %ymm1,%ymm0,%ymm0
116vunpcklpd %ymm3,%ymm2,%ymm2
117vunpcklpd %ymm9,%ymm8,%ymm1
118vunpcklpd %ymm11,%ymm10,%ymm3
119
120vmulpd      %ymm6,%ymm13,%ymm8 # ia0.omai (tw)
121vmulpd      %ymm7,%ymm12,%ymm9 # ia4.omar (itw)
122vmulpd      %ymm4,%ymm13,%ymm10 # ra0.omai (tw)
123vmulpd      %ymm5,%ymm12,%ymm11 # ra4.omar (itw)
124vfmsub231pd %ymm4,%ymm12,%ymm8 # rprod0 (tw)
125vfmadd231pd %ymm5,%ymm13,%ymm9 # rprod4 (itw)
126vfmadd231pd %ymm6,%ymm12,%ymm10 # iprod0 (tw)
127vfmsub231pd %ymm7,%ymm13,%ymm11 # iprod4 (itw)
128vsubpd      %ymm8,%ymm0,%ymm4
129vaddpd      %ymm9,%ymm1,%ymm5
130vsubpd      %ymm10,%ymm2,%ymm6
131vaddpd      %ymm11,%ymm3,%ymm7
132vaddpd      %ymm8,%ymm0,%ymm0
133vsubpd      %ymm9,%ymm1,%ymm1
134vaddpd      %ymm10,%ymm2,%ymm2
135vsubpd      %ymm11,%ymm3,%ymm3
136
137vunpckhpd %ymm7,%ymm3,%ymm11   # (0,4) -> (0,1)
138vunpckhpd %ymm5,%ymm1,%ymm9    # (2,6) -> (2,3)
139vunpcklpd %ymm7,%ymm3,%ymm10
140vunpcklpd %ymm5,%ymm1,%ymm8
141vunpckhpd %ymm6,%ymm2,%ymm3    # (1,5) -> (8,9)
142vunpckhpd %ymm4,%ymm0,%ymm1    # (3,7) -> (10,11)
143vunpcklpd %ymm6,%ymm2,%ymm2
144vunpcklpd %ymm4,%ymm0,%ymm0
145
146vperm2f128 $0x31,%ymm10,%ymm2,%ymm6
147vperm2f128 $0x31,%ymm11,%ymm3,%ymm7
148vperm2f128 $0x20,%ymm10,%ymm2,%ymm4
149vperm2f128 $0x20,%ymm11,%ymm3,%ymm5
150vperm2f128 $0x31,%ymm8,%ymm0,%ymm2
151vperm2f128 $0x31,%ymm9,%ymm1,%ymm3
152vperm2f128 $0x20,%ymm8,%ymm0,%ymm0
153vperm2f128 $0x20,%ymm9,%ymm1,%ymm1
154
1554:
156vmovupd     %ymm0,(%rdi)       # ra0
157vmovupd     %ymm1,0x20(%rdi)   # ra4
158vmovupd     %ymm2,0x40(%rdi)   # ra8
159vmovupd     %ymm3,0x60(%rdi)   # ra12
160vmovupd     %ymm4,(%rsi)       # ia0
161vmovupd     %ymm5,0x20(%rsi)   # ia4
162vmovupd     %ymm6,0x40(%rsi)   # ia8
163vmovupd     %ymm7,0x60(%rsi)   # ia12
164vzeroupper
165ret
166.size	reim_fft16_avx_fma, .-reim_fft16_avx_fma
167.section .note.GNU-stack,"",@progbits