1# shifted FFT over X^16-i
  2# 1st argument (rdi) contains 16 complexes
  3# 2nd argument (rsi) contains: 8 complexes
  4#     omega,alpha,beta,j.beta,gamma,j.gamma,k.gamma,kj.gamma
  5#     alpha = sqrt(omega), beta = sqrt(alpha), gamma = sqrt(beta)
  6#     j = sqrt(i), k=sqrt(j)
  7.globl cplx_fft16_avx_fma
  8cplx_fft16_avx_fma:
  9vmovupd (%rdi),%ymm8
 10vmovupd 0x20(%rdi),%ymm9
 11vmovupd 0x40(%rdi),%ymm10
 12vmovupd 0x60(%rdi),%ymm11
 13vmovupd 0x80(%rdi),%ymm12
 14vmovupd 0xa0(%rdi),%ymm13
 15vmovupd 0xc0(%rdi),%ymm14
 16vmovupd 0xe0(%rdi),%ymm15
 17
 18.first_pass:
 19vmovupd (%rsi),%xmm0                  /* omri   */
 20vinsertf128 $1, %xmm0, %ymm0, %ymm0   /* omriri */
 21vshufpd $15, %ymm0, %ymm0, %ymm1      /* ymm1: omiiii */
 22vshufpd $0,  %ymm0, %ymm0, %ymm0      /* ymm0: omrrrr */
 23vshufpd $5, %ymm12, %ymm12, %ymm4
 24vshufpd $5, %ymm13, %ymm13, %ymm5
 25vshufpd $5, %ymm14, %ymm14, %ymm6
 26vshufpd $5, %ymm15, %ymm15, %ymm7
 27vmulpd %ymm4,%ymm1,%ymm4
 28vmulpd %ymm5,%ymm1,%ymm5
 29vmulpd %ymm6,%ymm1,%ymm6
 30vmulpd %ymm7,%ymm1,%ymm7
 31vfmaddsub231pd  %ymm12, %ymm0, %ymm4     # ymm4 = (ymm0 * ymm12) +/- ymm4
 32vfmaddsub231pd  %ymm13, %ymm0, %ymm5
 33vfmaddsub231pd  %ymm14, %ymm0, %ymm6
 34vfmaddsub231pd  %ymm15, %ymm0, %ymm7
 35vsubpd %ymm4,%ymm8,%ymm12
 36vsubpd %ymm5,%ymm9,%ymm13
 37vsubpd %ymm6,%ymm10,%ymm14
 38vsubpd %ymm7,%ymm11,%ymm15
 39vaddpd %ymm4,%ymm8,%ymm8
 40vaddpd %ymm5,%ymm9,%ymm9
 41vaddpd %ymm6,%ymm10,%ymm10
 42vaddpd %ymm7,%ymm11,%ymm11
 43
 44.second_pass:
 45vmovupd 16(%rsi),%xmm0                /* omri   */
 46vinsertf128 $1, %xmm0, %ymm0, %ymm0   /* omriri */
 47vshufpd $15, %ymm0, %ymm0, %ymm1      /* ymm1: omiiii */
 48vshufpd $0,  %ymm0, %ymm0, %ymm0      /* ymm0: omrrrr */
 49vshufpd $5, %ymm10, %ymm10, %ymm4
 50vshufpd $5, %ymm11, %ymm11, %ymm5
 51vshufpd $5, %ymm14, %ymm14, %ymm6
 52vshufpd $5, %ymm15, %ymm15, %ymm7
 53vmulpd %ymm4,%ymm1,%ymm4
 54vmulpd %ymm5,%ymm1,%ymm5
 55vmulpd %ymm6,%ymm0,%ymm6
 56vmulpd %ymm7,%ymm0,%ymm7
 57vfmaddsub231pd  %ymm10, %ymm0, %ymm4     # ymm4 = (ymm0 * ymm10) +/- ymm4
 58vfmaddsub231pd  %ymm11, %ymm0, %ymm5
 59vfmsubadd231pd  %ymm14, %ymm1, %ymm6
 60vfmsubadd231pd  %ymm15, %ymm1, %ymm7
 61vsubpd %ymm4,%ymm8,%ymm10
 62vsubpd %ymm5,%ymm9,%ymm11
 63vaddpd %ymm6,%ymm12,%ymm14
 64vaddpd %ymm7,%ymm13,%ymm15
 65vaddpd %ymm4,%ymm8,%ymm8
 66vaddpd %ymm5,%ymm9,%ymm9
 67vsubpd %ymm6,%ymm12,%ymm12
 68vsubpd %ymm7,%ymm13,%ymm13
 69
 70.third_pass:
 71vmovupd 32(%rsi),%xmm0                /* gamma   */
 72vmovupd 48(%rsi),%xmm2                /* delta   */
 73vinsertf128 $1, %xmm0, %ymm0, %ymm0
 74vinsertf128 $1, %xmm2, %ymm2, %ymm2
 75vshufpd $15, %ymm0, %ymm0, %ymm1      /* ymm1: gama.iiii */
 76vshufpd $15, %ymm2, %ymm2, %ymm3      /* ymm3: delta.iiii */
 77vshufpd $0,  %ymm0, %ymm0, %ymm0      /* ymm0: gama.rrrr */
 78vshufpd $0,  %ymm2, %ymm2, %ymm2      /* ymm2: delta.rrrr */
 79vshufpd $5, %ymm9, %ymm9, %ymm4
 80vshufpd $5, %ymm11, %ymm11, %ymm5
 81vshufpd $5, %ymm13, %ymm13, %ymm6
 82vshufpd $5, %ymm15, %ymm15, %ymm7
 83vmulpd %ymm4,%ymm1,%ymm4
 84vmulpd %ymm5,%ymm0,%ymm5
 85vmulpd %ymm6,%ymm3,%ymm6
 86vmulpd %ymm7,%ymm2,%ymm7
 87vfmaddsub231pd  %ymm9, %ymm0, %ymm4     # ymm4 = (ymm0 * ymm10) +/- ymm4
 88vfmsubadd231pd  %ymm11, %ymm1, %ymm5
 89vfmaddsub231pd  %ymm13, %ymm2, %ymm6
 90vfmsubadd231pd  %ymm15, %ymm3, %ymm7
 91vsubpd %ymm4,%ymm8,%ymm9
 92vaddpd %ymm5,%ymm10,%ymm11
 93vsubpd %ymm6,%ymm12,%ymm13
 94vaddpd %ymm7,%ymm14,%ymm15
 95vaddpd %ymm4,%ymm8,%ymm8
 96vsubpd %ymm5,%ymm10,%ymm10
 97vaddpd %ymm6,%ymm12,%ymm12
 98vsubpd %ymm7,%ymm14,%ymm14
 99
100.fourth_pass:
101vmovupd 64(%rsi),%ymm0                /* gamma   */
102vmovupd 96(%rsi),%ymm2                /* delta   */
103vshufpd $15, %ymm0, %ymm0, %ymm1      /* ymm1: gama.iiii */
104vshufpd $15, %ymm2, %ymm2, %ymm3      /* ymm3: delta.iiii */
105vshufpd $0,  %ymm0, %ymm0, %ymm0      /* ymm0: gama.rrrr */
106vshufpd $0,  %ymm2, %ymm2, %ymm2      /* ymm2: delta.rrrr */
107vperm2f128 $0x31,%ymm10,%ymm8,%ymm4   # ymm4 contains c1,c5 -- x gamma
108vperm2f128 $0x31,%ymm11,%ymm9,%ymm5   # ymm5 contains c3,c7 -- x igamma
109vperm2f128 $0x31,%ymm14,%ymm12,%ymm6  # ymm6 contains c9,c13 -- x delta
110vperm2f128 $0x31,%ymm15,%ymm13,%ymm7  # ymm7 contains c11,c15 -- x idelta
111vperm2f128 $0x20,%ymm10,%ymm8,%ymm8   # ymm8 contains c0,c4
112vperm2f128 $0x20,%ymm11,%ymm9,%ymm9   # ymm9 contains c2,c6
113vperm2f128 $0x20,%ymm14,%ymm12,%ymm10 # ymm10 contains c8,c12
114vperm2f128 $0x20,%ymm15,%ymm13,%ymm11 # ymm11 contains c10,c14
115vshufpd $5, %ymm4, %ymm4, %ymm12
116vshufpd $5, %ymm5, %ymm5, %ymm13
117vshufpd $5, %ymm6, %ymm6, %ymm14
118vshufpd $5, %ymm7, %ymm7, %ymm15
119vmulpd %ymm12,%ymm1,%ymm12
120vmulpd %ymm13,%ymm0,%ymm13
121vmulpd %ymm14,%ymm3,%ymm14
122vmulpd %ymm15,%ymm2,%ymm15
123vfmaddsub231pd  %ymm4, %ymm0, %ymm12     # ymm12 = (ymm0 * ymm4) +/- ymm12
124vfmsubadd231pd  %ymm5, %ymm1, %ymm13
125vfmaddsub231pd  %ymm6, %ymm2, %ymm14
126vfmsubadd231pd  %ymm7, %ymm3, %ymm15
127vsubpd %ymm12,%ymm8,%ymm4
128vaddpd %ymm13,%ymm9,%ymm5
129vsubpd %ymm14,%ymm10,%ymm6
130vaddpd %ymm15,%ymm11,%ymm7
131vaddpd %ymm12,%ymm8,%ymm8
132vsubpd %ymm13,%ymm9,%ymm9
133vaddpd %ymm14,%ymm10,%ymm10
134vsubpd %ymm15,%ymm11,%ymm11
135
136vperm2f128 $0x20,%ymm6,%ymm10,%ymm12  # ymm4 contains c1,c5 -- x gamma
137vperm2f128 $0x20,%ymm7,%ymm11,%ymm13  # ymm5 contains c3,c7 -- x igamma
138vperm2f128 $0x31,%ymm6,%ymm10,%ymm14  # ymm6 contains c9,c13 -- x delta
139vperm2f128 $0x31,%ymm7,%ymm11,%ymm15  # ymm7 contains c11,c15 -- x idelta
140vperm2f128 $0x31,%ymm4,%ymm8,%ymm10   # ymm10 contains c8,c12
141vperm2f128 $0x31,%ymm5,%ymm9,%ymm11   # ymm11 contains c10,c14
142vperm2f128 $0x20,%ymm4,%ymm8,%ymm8    # ymm8 contains c0,c4
143vperm2f128 $0x20,%ymm5,%ymm9,%ymm9    # ymm9 contains c2,c6
144
145.save_and_return:
146vmovupd %ymm8,(%rdi)
147vmovupd %ymm9,0x20(%rdi)
148vmovupd %ymm10,0x40(%rdi)
149vmovupd %ymm11,0x60(%rdi)
150vmovupd %ymm12,0x80(%rdi)
151vmovupd %ymm13,0xa0(%rdi)
152vmovupd %ymm14,0xc0(%rdi)
153vmovupd %ymm15,0xe0(%rdi)
154ret
155.size	cplx_fft16_avx_fma, .-cplx_fft16_avx_fma
156.section .note.GNU-stack,"",@progbits