Refactor FFT boilerplate

src/algorithm/bluesteins_algorithm.rs

@@ -3,8 +3,6 @@ use std::sync::Arc;
 use num_complex::Complex;
 use num_traits::Zero;
 
-use crate::array_utils;
-use crate::common::{fft_error_inplace, fft_error_outofplace};
 use crate::{common::FftNum, twiddles, FftDirection};
 use crate::{Direction, Fft, Length};
 

src/algorithm/butterflies.rs

@@ -2,8 +2,7 @@ use num_complex::Complex;
 
 use crate::{common::FftNum, FftDirection};
 
-use crate::array_utils::{self, DoubleBuf, LoadStore};
-use crate::common::{fft_error_immut, fft_error_inplace, fft_error_outofplace};
+use crate::array_utils::{DoubleBuf, LoadStore};
 use crate::twiddles;
 use crate::{Direction, Fft, Length};
 
@@ -24,80 +23,48 @@ macro_rules! boilerplate_fft_butterfly {
                 output: &mut [Complex<T>],
                 _scratch: &mut [Complex<T>],
             ) {
-                if input.len() < self.len() || output.len() != input.len() {
-                    // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
-                    fft_error_immut(self.len(), input.len(), output.len(), 0, 0);
-                    return; // Unreachable, because fft_error_immut asserts, but it helps codegen to put it here
-                }
-
-                let result = array_utils::iter_chunks_zipped(
+                crate::fft_helper::fft_helper_immut(
                     input,
                     output,
+                    &mut [],
                     self.len(),
-                    |in_chunk, out_chunk| {
-                        unsafe {
-                            self.perform_fft_butterfly(DoubleBuf {
-                                input: in_chunk,
-                                output: out_chunk,
-                            })
-                        };
+                    0,
+                    |in_chunk, out_chunk, _| unsafe {
+                        self.perform_fft_butterfly(DoubleBuf {
+                            input: in_chunk,
+                            output: out_chunk,
+                        })
                     },
                 );
-
-                if result.is_err() {
-                    // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
-                    // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
-                    fft_error_immut(self.len(), input.len(), output.len(), 0, 0);
-                }
             }
             fn process_outofplace_with_scratch(
                 &self,
                 input: &mut [Complex<T>],
                 output: &mut [Complex<T>],
                 _scratch: &mut [Complex<T>],
             ) {
-                if input.len() < self.len() || output.len() != input.len() {
-                    // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
-                    fft_error_outofplace(self.len(), input.len(), output.len(), 0, 0);
-                    return; // Unreachable, because fft_error_immut asserts, but it helps codegen to put it here
-                }
-
-                let result = array_utils::iter_chunks_zipped(
+                crate::fft_helper::fft_helper_outofplace(
                     input,
                     output,
+                    &mut [],
                     self.len(),
-                    |in_chunk, out_chunk| {
-                        unsafe {
-                            self.perform_fft_butterfly(DoubleBuf {
-                                input: in_chunk,
-                                output: out_chunk,
-                            })
-                        };
+                    0,
+                    |in_chunk, out_chunk, _| unsafe {
+                        self.perform_fft_butterfly(DoubleBuf {
+                            input: in_chunk,
+                            output: out_chunk,
+                        })
                     },
                 );
-
-                if result.is_err() {
-                    // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
-                    // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
-                    fft_error_outofplace(self.len(), input.len(), output.len(), 0, 0);
-                }
             }
             fn process_with_scratch(&self, buffer: &mut [Complex<T>], _scratch: &mut [Complex<T>]) {
-                if buffer.len() < self.len() {
-                    // We want to trigger a panic, but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
-                    fft_error_inplace(self.len(), buffer.len(), 0, 0);
-                    return; // Unreachable, because fft_error_inplace asserts, but it helps codegen to put it here
-                }
-
-                let result = array_utils::iter_chunks_mut(buffer, self.len(), |chunk| unsafe {
-                    self.perform_fft_butterfly(chunk)
-                });
-
-                if result.is_err() {
-                    // We want to trigger a panic, because the buffer sizes weren't cleanly divisible by the FFT size,
-                    // but we want to avoid doing it in this function to reduce code size, so call a function marked cold and inline(never) that will do it for us
-                    fft_error_inplace(self.len(), buffer.len(), 0, 0);
-                }
+                crate::fft_helper::fft_helper_inplace(
+                    buffer,
+                    &mut [],
+                    self.len(),
+                    0,
+                    |chunk, _| unsafe { self.perform_fft_butterfly(chunk) },
+                );
             }
             #[inline(always)]
             fn get_inplace_scratch_len(&self) -> usize {

src/algorithm/dft.rs

@@ -1,8 +1,6 @@
 use num_complex::Complex;
 use num_traits::Zero;
 
-use crate::array_utils;
-use crate::common::{fft_error_immut, fft_error_inplace, fft_error_outofplace};
 use crate::{twiddles, FftDirection};
 use crate::{Direction, Fft, FftNum, Length};
 
@@ -44,6 +42,9 @@ impl<T: FftNum> Dft<T> {
     fn outofplace_scratch_len(&self) -> usize {
         0
     }
+    fn immut_scratch_len(&self) -> usize {
+        0
+    }
 
     fn perform_fft_immut(
         &self,
@@ -78,7 +79,7 @@ impl<T: FftNum> Dft<T> {
         self.perform_fft_immut(signal, spectrum, _scratch);
     }
 }
-boilerplate_fft_oop!(Dft, |this: &Dft<_>| this.twiddles.len(), |_: &Dft<_>| 0);
+boilerplate_fft_oop!(Dft, |this: &Dft<_>| this.twiddles.len());
 
 #[cfg(test)]
 mod unit_tests {

src/algorithm/good_thomas_algorithm.rs

@@ -7,7 +7,6 @@ use strength_reduce::StrengthReducedUsize;
 use transpose;
 
 use crate::array_utils;
-use crate::common::{fft_error_inplace, fft_error_outofplace};
 use crate::{common::FftNum, FftDirection};
 use crate::{Direction, Fft, Length};
 

src/algorithm/mixed_radix.rs

@@ -6,7 +6,6 @@ use num_traits::Zero;
 use transpose;
 
 use crate::array_utils;
-use crate::common::{fft_error_inplace, fft_error_outofplace};
 use crate::{common::FftNum, twiddles, FftDirection};
 use crate::{Direction, Fft, Length};
 

src/algorithm/raders_algorithm.rs

@@ -6,8 +6,6 @@ use num_traits::Zero;
 use primal_check::miller_rabin;
 use strength_reduce::StrengthReducedUsize;
 
-use crate::array_utils;
-use crate::common::{fft_error_inplace, fft_error_outofplace};
 use crate::math_utils;
 use crate::{common::FftNum, twiddles, FftDirection};
 use crate::{Direction, Fft, Length};

src/algorithm/radix3.rs

@@ -4,8 +4,7 @@ use num_complex::Complex;
 
 use crate::algorithm::butterflies::{Butterfly1, Butterfly27, Butterfly3, Butterfly9};
 use crate::algorithm::radixn::butterfly_3;
-use crate::array_utils::{self, bitreversed_transpose, compute_logarithm};
-use crate::common::{fft_error_immut, fft_error_inplace, fft_error_outofplace};
+use crate::array_utils::{bitreversed_transpose, compute_logarithm};
 use crate::{common::FftNum, twiddles, FftDirection};
 use crate::{Direction, Fft, Length};
 
@@ -120,6 +119,9 @@ impl<T: FftNum> Radix3<T> {
     fn outofplace_scratch_len(&self) -> usize {
         self.outofplace_scratch_len
     }
+    fn immut_scratch_len(&self) -> usize {
+        self.immut_scratch_len
+    }
 
     fn perform_fft_immut(
         &self,
@@ -193,8 +195,7 @@ impl<T: FftNum> Radix3<T> {
         }
     }
 }
-boilerplate_fft_oop!(Radix3, |this: &Radix3<_>| this.len, |this: &Radix3<_>| this
-    .immut_scratch_len);
+boilerplate_fft_oop!(Radix3, |this: &Radix3<_>| this.len);
 
 #[cfg(test)]
 mod unit_tests {

src/algorithm/radix4.rs

@@ -6,8 +6,7 @@ use crate::algorithm::butterflies::{
     Butterfly1, Butterfly16, Butterfly2, Butterfly32, Butterfly4, Butterfly8,
 };
 use crate::algorithm::radixn::butterfly_4;
-use crate::array_utils::{self, bitreversed_transpose};
-use crate::common::{fft_error_immut, fft_error_inplace, fft_error_outofplace};
+use crate::array_utils::bitreversed_transpose;
 use crate::{common::FftNum, twiddles, FftDirection};
 use crate::{Direction, Fft, Length};
 
@@ -125,8 +124,10 @@ impl<T: FftNum> Radix4<T> {
     fn outofplace_scratch_len(&self) -> usize {
         self.outofplace_scratch_len
     }
+    fn immut_scratch_len(&self) -> usize {
+        self.immut_scratch_len
+    }
 
-    #[inline]
     fn perform_fft_immut(
         &self,
         input: &[Complex<T>],
@@ -201,8 +202,7 @@ impl<T: FftNum> Radix4<T> {
         }
     }
 }
-boilerplate_fft_oop!(Radix4, |this: &Radix4<_>| this.len, |this: &Radix4<_>| this
-    .immut_scratch_len);
+boilerplate_fft_oop!(Radix4, |this: &Radix4<_>| this.len);
 
 #[cfg(test)]
 mod unit_tests {

src/algorithm/radixn.rs

@@ -2,8 +2,8 @@ use std::sync::Arc;
 
 use num_complex::Complex;
 
-use crate::array_utils::{self, factor_transpose, Load, LoadStore, TransposeFactor};
-use crate::common::{fft_error_immut, fft_error_inplace, fft_error_outofplace, RadixFactor};
+use crate::array_utils::{factor_transpose, Load, LoadStore, TransposeFactor};
+use crate::common::RadixFactor;
 use crate::{common::FftNum, twiddles, FftDirection};
 use crate::{Direction, Fft, Length};
 
@@ -160,6 +160,9 @@ impl<T: FftNum> RadixN<T> {
     fn outofplace_scratch_len(&self) -> usize {
         self.outofplace_scratch_len
     }
+    fn immut_scratch_len(&self) -> usize {
+        self.immut_scratch_len
+    }
 
     fn perform_fft_immut(
         &self,
@@ -329,8 +332,7 @@ impl<T: FftNum> RadixN<T> {
         }
     }
 }
-boilerplate_fft_oop!(RadixN, |this: &RadixN<_>| this.len, |this: &RadixN<_>| this
-    .immut_scratch_len);
+boilerplate_fft_oop!(RadixN, |this: &RadixN<_>| this.len);
 
 #[inline(never)]
 pub(crate) unsafe fn butterfly_2<T: FftNum>(