1# shifted FFT over X^16-i
  2# 1st argument (rdi) contains 16 complexes
  3# 2nd argument (rsi) contains: 8 complexes
  4#     omega,alpha,beta,j.beta,gamma,j.gamma,k.gamma,kj.gamma
  5#     alpha = sqrt(omega), beta = sqrt(alpha), gamma = sqrt(beta)
  6#     j = sqrt(i), k=sqrt(j)
  7.globl cplx_ifft16_avx_fma
  8cplx_ifft16_avx_fma:
  9vmovupd (%rdi),%ymm8        # load data into registers %ymm8 -> %ymm15
 10vmovupd 0x20(%rdi),%ymm9
 11vmovupd 0x40(%rdi),%ymm10
 12vmovupd 0x60(%rdi),%ymm11
 13vmovupd 0x80(%rdi),%ymm12
 14vmovupd 0xa0(%rdi),%ymm13
 15vmovupd 0xc0(%rdi),%ymm14
 16vmovupd 0xe0(%rdi),%ymm15
 17
 18.fourth_pass:
 19vmovupd 0(%rsi),%ymm0                 /* gamma   */
 20vmovupd 32(%rsi),%ymm2                /* delta   */
 21vshufpd $15, %ymm0, %ymm0, %ymm1      /* ymm1: gama.iiii */
 22vshufpd $15, %ymm2, %ymm2, %ymm3      /* ymm3: delta.iiii */
 23vshufpd $0,  %ymm0, %ymm0, %ymm0      /* ymm0: gama.rrrr */
 24vshufpd $0,  %ymm2, %ymm2, %ymm2      /* ymm2: delta.rrrr */
 25vperm2f128 $0x31,%ymm10,%ymm8,%ymm4   # ymm4 contains c1,c5
 26vperm2f128 $0x31,%ymm11,%ymm9,%ymm5   # ymm5 contains c3,c7
 27vperm2f128 $0x31,%ymm14,%ymm12,%ymm6  # ymm6 contains c9,c13
 28vperm2f128 $0x31,%ymm15,%ymm13,%ymm7  # ymm7 contains c11,c15
 29vperm2f128 $0x20,%ymm10,%ymm8,%ymm8   # ymm8 contains c0,c4
 30vperm2f128 $0x20,%ymm11,%ymm9,%ymm9   # ymm9 contains c2,c6
 31vperm2f128 $0x20,%ymm14,%ymm12,%ymm10 # ymm10 contains c8,c12
 32vperm2f128 $0x20,%ymm15,%ymm13,%ymm11 # ymm11 contains c10,c14
 33vsubpd %ymm4,%ymm8,%ymm12  # tw:  to mul by gamma
 34vsubpd %ymm5,%ymm9,%ymm13  # itw: to mul by i.gamma
 35vsubpd %ymm6,%ymm10,%ymm14 # tw:  to mul by delta
 36vsubpd %ymm7,%ymm11,%ymm15 # itw: to mul by i.delta
 37vaddpd %ymm4,%ymm8,%ymm8
 38vaddpd %ymm5,%ymm9,%ymm9
 39vaddpd %ymm6,%ymm10,%ymm10
 40vaddpd %ymm7,%ymm11,%ymm11
 41vshufpd $5, %ymm12, %ymm12, %ymm4
 42vshufpd $5, %ymm13, %ymm13, %ymm5
 43vshufpd $5, %ymm14, %ymm14, %ymm6
 44vshufpd $5, %ymm15, %ymm15, %ymm7
 45vmulpd %ymm4,%ymm1,%ymm4
 46vmulpd %ymm5,%ymm0,%ymm5
 47vmulpd %ymm6,%ymm3,%ymm6
 48vmulpd %ymm7,%ymm2,%ymm7
 49vfmaddsub231pd  %ymm12, %ymm0, %ymm4     # ymm4 = (ymm0 * ymm12) +/- ymm4
 50vfmsubadd231pd  %ymm13, %ymm1, %ymm5
 51vfmaddsub231pd  %ymm14, %ymm2, %ymm6
 52vfmsubadd231pd  %ymm15, %ymm3, %ymm7
 53
 54vperm2f128 $0x20,%ymm6,%ymm10,%ymm12  # ymm4 contains c1,c5 -- x gamma
 55vperm2f128 $0x20,%ymm7,%ymm11,%ymm13  # ymm5 contains c3,c7 -- x igamma
 56vperm2f128 $0x31,%ymm6,%ymm10,%ymm14  # ymm6 contains c9,c13 -- x delta
 57vperm2f128 $0x31,%ymm7,%ymm11,%ymm15  # ymm7 contains c11,c15 -- x idelta
 58vperm2f128 $0x31,%ymm4,%ymm8,%ymm10   # ymm10 contains c8,c12
 59vperm2f128 $0x31,%ymm5,%ymm9,%ymm11   # ymm11 contains c10,c14
 60vperm2f128 $0x20,%ymm4,%ymm8,%ymm8    # ymm8 contains c0,c4
 61vperm2f128 $0x20,%ymm5,%ymm9,%ymm9    # ymm9 contains c2,c6
 62
 63
 64.third_pass:
 65vmovupd 64(%rsi),%xmm0                /* gamma   */
 66vmovupd 80(%rsi),%xmm2                /* delta   */
 67vinsertf128 $1, %xmm0, %ymm0, %ymm0
 68vinsertf128 $1, %xmm2, %ymm2, %ymm2
 69vshufpd $15, %ymm0, %ymm0, %ymm1      /* ymm1: gama.iiii */
 70vshufpd $15, %ymm2, %ymm2, %ymm3      /* ymm3: delta.iiii */
 71vshufpd $0,  %ymm0, %ymm0, %ymm0      /* ymm0: gama.rrrr */
 72vshufpd $0,  %ymm2, %ymm2, %ymm2      /* ymm2: delta.rrrr */
 73vsubpd %ymm9,%ymm8,%ymm4
 74vsubpd %ymm11,%ymm10,%ymm5
 75vsubpd %ymm13,%ymm12,%ymm6
 76vsubpd %ymm15,%ymm14,%ymm7
 77vaddpd %ymm9,%ymm8,%ymm8
 78vaddpd %ymm11,%ymm10,%ymm10
 79vaddpd %ymm13,%ymm12,%ymm12
 80vaddpd %ymm15,%ymm14,%ymm14
 81vshufpd $5, %ymm4, %ymm4, %ymm9
 82vshufpd $5, %ymm5, %ymm5, %ymm11
 83vshufpd $5, %ymm6, %ymm6, %ymm13
 84vshufpd $5, %ymm7, %ymm7, %ymm15
 85vmulpd %ymm9,%ymm1,%ymm9
 86vmulpd %ymm11,%ymm0,%ymm11
 87vmulpd %ymm13,%ymm3,%ymm13
 88vmulpd %ymm15,%ymm2,%ymm15
 89vfmaddsub231pd  %ymm4, %ymm0, %ymm9     # ymm9 = (ymm0 * ymm4) +/- ymm9
 90vfmsubadd231pd  %ymm5, %ymm1, %ymm11
 91vfmaddsub231pd  %ymm6, %ymm2, %ymm13
 92vfmsubadd231pd  %ymm7, %ymm3, %ymm15
 93
 94.second_pass:
 95vmovupd 96(%rsi),%xmm0                /* omri   */
 96vinsertf128 $1, %xmm0, %ymm0, %ymm0   /* omriri */
 97vshufpd $15, %ymm0, %ymm0, %ymm1      /* ymm1: omiiii */
 98vshufpd $0,  %ymm0, %ymm0, %ymm0      /* ymm0: omrrrr */
 99vsubpd %ymm10,%ymm8,%ymm4
100vsubpd %ymm11,%ymm9,%ymm5
101vsubpd %ymm14,%ymm12,%ymm6
102vsubpd %ymm15,%ymm13,%ymm7
103vaddpd %ymm10,%ymm8,%ymm8
104vaddpd %ymm11,%ymm9,%ymm9
105vaddpd %ymm14,%ymm12,%ymm12
106vaddpd %ymm15,%ymm13,%ymm13
107vshufpd $5, %ymm4, %ymm4, %ymm10
108vshufpd $5, %ymm5, %ymm5, %ymm11
109vshufpd $5, %ymm6, %ymm6, %ymm14
110vshufpd $5, %ymm7, %ymm7, %ymm15
111vmulpd %ymm10,%ymm1,%ymm10
112vmulpd %ymm11,%ymm1,%ymm11
113vmulpd %ymm14,%ymm0,%ymm14
114vmulpd %ymm15,%ymm0,%ymm15
115vfmaddsub231pd  %ymm4, %ymm0, %ymm10     # ymm10 = (ymm0 * ymm4) +/- ymm10
116vfmaddsub231pd  %ymm5, %ymm0, %ymm11
117vfmsubadd231pd  %ymm6, %ymm1, %ymm14
118vfmsubadd231pd  %ymm7, %ymm1, %ymm15
119
120.first_pass:
121vmovupd 112(%rsi),%xmm0               /* omri   */
122vinsertf128 $1, %xmm0, %ymm0, %ymm0   /* omriri */
123vshufpd $15, %ymm0, %ymm0, %ymm1      /* ymm1: omiiii */
124vshufpd $0,  %ymm0, %ymm0, %ymm0      /* ymm0: omrrrr */
125vsubpd %ymm12,%ymm8,%ymm4
126vsubpd %ymm13,%ymm9,%ymm5
127vsubpd %ymm14,%ymm10,%ymm6
128vsubpd %ymm15,%ymm11,%ymm7
129vaddpd %ymm12,%ymm8,%ymm8
130vaddpd %ymm13,%ymm9,%ymm9
131vaddpd %ymm14,%ymm10,%ymm10
132vaddpd %ymm15,%ymm11,%ymm11
133vshufpd $5, %ymm4, %ymm4, %ymm12
134vshufpd $5, %ymm5, %ymm5, %ymm13
135vshufpd $5, %ymm6, %ymm6, %ymm14
136vshufpd $5, %ymm7, %ymm7, %ymm15
137vmulpd %ymm12,%ymm1,%ymm12
138vmulpd %ymm13,%ymm1,%ymm13
139vmulpd %ymm14,%ymm1,%ymm14
140vmulpd %ymm15,%ymm1,%ymm15
141vfmaddsub231pd  %ymm4, %ymm0, %ymm12     # ymm12 = (ymm0 * ymm4) +/- ymm12
142vfmaddsub231pd  %ymm5, %ymm0, %ymm13
143vfmaddsub231pd  %ymm6, %ymm0, %ymm14
144vfmaddsub231pd  %ymm7, %ymm0, %ymm15
145
146.save_and_return:
147vmovupd %ymm8,(%rdi)
148vmovupd %ymm9,0x20(%rdi)
149vmovupd %ymm10,0x40(%rdi)
150vmovupd %ymm11,0x60(%rdi)
151vmovupd %ymm12,0x80(%rdi)
152vmovupd %ymm13,0xa0(%rdi)
153vmovupd %ymm14,0xc0(%rdi)
154vmovupd %ymm15,0xe0(%rdi)
155ret
156.size	cplx_ifft16_avx_fma, .-cplx_ifft16_avx_fma
157.section .note.GNU-stack,"",@progbits