Fix AVX512 H-resampler coeff preparation for fixed-type coeff advances, regardless of filter_size_real.

Author: pinterf

avs_core/filters/intel/resample_avx512.h

@@ -133,7 +133,11 @@ void resizer_h_avx512_generic_float_pix16_sub4_ks_4_8_16(BYTE * dst8, const BYTE
 
 // transpose and hi/lo unpack of resampling program for permute-based H-resizers
 // allocate and fill pixel_coefficient_AVX512_H coeffs buffer of the ResamplingProgram
-void resize_prepare_coeffs_AVX512_H(ResamplingProgram* p, IScriptEnvironment* env, int iSamplesInTheGroup, int iGroupsCount);
+// fixed_kernel_size: number of taps the target kernel's fixed unrolled loop always reads
+// per x-group (now 4, 8 or 16), so the coefficient table stride matches the kernel's fixed
+// index advance amount, regardless of filter_size_real.
+// For variable-loop kernels (ks48/ks64) 0 must be passed, which read exactly filter_size_real taps.
+void resize_prepare_coeffs_AVX512_H(ResamplingProgram* p, IScriptEnvironment* env, int iSamplesInTheGroup, int iGroupsCount, int fixed_kernel_size);
 // allocate and fill pixel_coefficient_AVX512_float_H for float permutex-based H-resizers (ks16 variants)
 void resize_prepare_coeffs_AVX512_float_H(ResamplingProgram* p, IScriptEnvironment* env);
 

avs_core/filters/intel/resample_avx512b.cpp

@@ -3458,7 +3458,8 @@ template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks16_pretranspos
 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks48_pretransposed_coeffs_base<false>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
 template void resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks48_pretransposed_coeffs_base<true>(BYTE* dst8, const BYTE* src8, int dst_pitch, int src_pitch, ResamplingProgram* program, int width, int height, int bits_per_pixel);
 
-void resize_prepare_coeffs_AVX512_H(ResamplingProgram* p, IScriptEnvironment* env, int iSamplesInTheGroup, int iGroupsCount) {
+// fixed_kernel_size == 0 marks: "not fixed".
+void resize_prepare_coeffs_AVX512_H(ResamplingProgram* p, IScriptEnvironment* env, int iSamplesInTheGroup, int iGroupsCount, int fixed_kernel_size) {
   // note: filter_size_real was the max(kernel_sizes[])
   int filter_size_aligned = AlignNumber(p->filter_size_real, p->filter_size_alignment);
   // FIXME: really this needs to be dynamic based on SIMD used in resizer
@@ -3517,12 +3518,16 @@ void resize_prepare_coeffs_AVX512_H(ResamplingProgram* p, IScriptEnvironment* en
       return *(current_coeff + filter_size_padded * std::min(j, avail - 1) + ki);
     };
     // process by 2 rows because madd/dp can only make FMA from 2 unpacked uint16 pairs
-    // ks16 (iSamplesAtATime==32): filter always reads 16 vectors per x-group (advance by 16),
-    // so we must store all filter_size_aligned/2 pairs even when filter_size_real < 16.
+    // Specific unrolled kernels (ks4/ks8/ks16) read a fixed number of taps per group of x.
+    // They advance current_coeff_SIMD by an appropriate fixed amount, regardless of
+    // filter_size_real. The coefficient table must be padded to that same fixed width
+    // (fixed_kernel_size) or else discrapancy happens after the first x group.
     // Extra iterations access zero-padded taps and produce zero coefficient pairs — harmless.
-    // ks64 (iSamplesAtATime==64): filter advances by filter_size_real*2 (variable), so it uses
-    // filter_size_to_process to store exactly that many pairs — must NOT use filter_size_aligned.
-    const int i_limit = (iSamplesAtATime == 32) ? filter_size_aligned : filter_size_to_process;
+    // For variable loop kernels like ks48/ks64 (signalled as fixed_kernel_size==0) advance
+    // current_coeff_SIMD by filter_size_real*2 (pairs), so they must store exactly filter_size_to_process
+    // pairs, contrary to the previous case, padding to a fixed width would cause coeff desynchronization
+    // after the first x group.
+    const int i_limit = (fixed_kernel_size > 0) ? fixed_kernel_size : filter_size_to_process;
     for (int i = 0; i < i_limit; i += 2)
     {
       if (iSamplesAtATime == 64) // 2 groups of 32 coeffs for columns 0..31 and 32..63

avs_core/filters/resample.cpp

@@ -1732,7 +1732,7 @@ ResamplerH FilteredResizeH::GetResampler(int CPU, int pixelsize, int bits_per_pi
 
             Winners: (Fast) resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks4_vnni (Base) resize_h_planar_uint8_avx512_permutex_vstripe_ks4_base
           */
-          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/);
+          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/, 4/*fixed_kernel_size*/);
           if (has_AVX512_fast)
             return resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks4_pretransposed_coeffs_vnni;
           else
@@ -1763,14 +1763,14 @@ ResamplerH FilteredResizeH::GetResampler(int CPU, int pixelsize, int bits_per_pi
 
           Winners: (Fast) resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks8_vnni (Base) resize_h_planar_uint8_avx512_permutex_vstripe_ks8_base
           */
-          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/);
+          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/, 8/*fixed_kernel_size*/);
           if (has_AVX512_fast)
             return resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks8_pretransposed_coeffs_vnni;
           else
             return resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks8_pretransposed_coeffs_base;
         }
         if (!program->resize_h_planar_gather_permutex_vstripe_check(32/*iSamplesInTheGroup*/, 128/*permutex_index_diff_limit*/, 8/*kernel_size*/)) { // slower ks8 but more downsample ratio for /2
-          resize_prepare_coeffs_AVX512_H(program, env, 32/*iSamplesInTheGroup*/, 2/*iGroupsCount*/);
+          resize_prepare_coeffs_AVX512_H(program, env, 32/*iSamplesInTheGroup*/, 2/*iGroupsCount*/, 8/*fixed_kernel_size*/);
           if (has_AVX512_fast)
             return resize_h_planar_uint8_avx512_permutex_vstripe_2s32_ks8_pretransposed_coeffs_vnni;
           else
@@ -1794,7 +1794,7 @@ ResamplerH FilteredResizeH::GetResampler(int CPU, int pixelsize, int bits_per_pi
 
           Winners: (Fast) resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks16_vnni (Base) resize_h_planar_uint8_avx512_permutex_vstripe_ks16_base
           */
-          resize_prepare_coeffs_AVX512_H(program, env, 32/*iSamplesInTheGroup*/, 1/*iGroupsCount*/);
+          resize_prepare_coeffs_AVX512_H(program, env, 32/*iSamplesInTheGroup*/, 1/*iGroupsCount*/, 16/*fixed_kernel_size*/);
           if (has_AVX512_fast)
             return resize_h_planar_uint8_avx512_permutex_vstripe_mpz_ks16_pretransposed_coeffs_vnni;
           else
@@ -1803,7 +1803,7 @@ ResamplerH FilteredResizeH::GetResampler(int CPU, int pixelsize, int bits_per_pi
       }
       if (!program->resize_h_planar_gather_permutex_vstripe_check(32/*iSamplesInTheGroup*/, 128/*permutex_index_diff_limit*/, program->filter_size_real/*kernel_size*/))
       {
-        resize_prepare_coeffs_AVX512_H(program, env, 32/*iSamplesInTheGroup*/, 2/*iGroupsCount*/);
+        resize_prepare_coeffs_AVX512_H(program, env, 32/*iSamplesInTheGroup*/, 2/*iGroupsCount*/, 0/*fixed_kernel_size: variable-loop kernel*/);
         if (has_AVX512_fast)
           return resize_h_planar_uint8_avx512_permutex_vstripe_mpz_2s32_ks64_pretransposed_coeffs_vnni;
         else
@@ -1854,7 +1854,7 @@ ResamplerH FilteredResizeH::GetResampler(int CPU, int pixelsize, int bits_per_pi
             Fazit: The MP versions' difference is only two VNNI instructions between BASE/FAST, in benchmarks zero visible speed benefit is seen.
             Winners: (Both mp) (Fast) resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks4_vnni (Base) resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks4_base
           */
-          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/);
+          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/, 4/*fixed_kernel_size*/);
           if (bits_per_pixel < 16) {
             if (has_AVX512_fast)
               return resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks4_pretransposed_coeffs_vnni<true>;
@@ -1880,7 +1880,7 @@ ResamplerH FilteredResizeH::GetResampler(int CPU, int pixelsize, int bits_per_pi
             Fazit: The MP versions' difference is only two VNNI instructions between BASE/FAST, in benchmarks 1-2% visible speed benefit is seen.
             Winners: (Both mp) (Fast) resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks8_vnni (Base) resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks8_base
           */
-          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/);
+          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/, 8/*fixed_kernel_size*/);
           if (bits_per_pixel < 16) {
             if (has_AVX512_fast)
               return resize_h_planar_uint16_avx512_permutex_vstripe_mp_2s32_ks8_pretransposed_coeffs_vnni<true>;
@@ -1903,7 +1903,7 @@ ResamplerH FilteredResizeH::GetResampler(int CPU, int pixelsize, int bits_per_pi
         // Case E LanczosResize(int(width*0.5 + 0.5), height, taps=2) kernel size 8
         // Case R: LanczosResize(int(width*0.5 + 0.5), height, taps=2) kernel size 8 
         if (!program->resize_h_planar_gather_permutex_vstripe_check(16/*iSamplesInTheGroup*/, 64/*permutex_index_diff_limit*/, 8/*kernel_size*/)) {
-          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/);
+          resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/, 8/*fixed_kernel_size*/);
           if (bits_per_pixel < 16) {
             if (has_AVX512_fast)
               return resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks8_pretransposed_coeffs_vnni<true>;
@@ -1937,7 +1937,7 @@ ResamplerH FilteredResizeH::GetResampler(int CPU, int pixelsize, int bits_per_pi
             resizer_h_avx2_generic_uint16_t (fallback)                  1156 1085 1292
             Winners: (Both mp) (Fast) resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks8_vnni (Base) resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks8_base
           */
-          resize_prepare_coeffs_AVX512_H(program, env, 32/*iSamplesInTheGroup*/, 1/*iGroupsCount*/);
+          resize_prepare_coeffs_AVX512_H(program, env, 32/*iSamplesInTheGroup*/, 1/*iGroupsCount*/, 16/*fixed_kernel_size*/);
           if (bits_per_pixel < 16) {
             if (has_AVX512_fast)
               return resize_h_planar_uint16_avx512_permutex_vstripe_mp_ks16_pretransposed_coeffs_vnni<true>;
@@ -1962,7 +1962,7 @@ ResamplerH FilteredResizeH::GetResampler(int CPU, int pixelsize, int bits_per_pi
       // The function itself has no hard 48 limit; it loops over filter_size_real directly.
       if (!program->resize_h_planar_gather_permutex_vstripe_check(16/*iSamplesInTheGroup*/, 64/*permutex_index_diff_limit*/, program->filter_size_real/*kernel_size*/))
       {
-        resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/);
+        resize_prepare_coeffs_AVX512_H(program, env, 64/*iSamplesInTheGroup*/, 1/*iGroupsCount*/, 0/*fixed_kernel_size: variable-loop kernel*/);
         if (bits_per_pixel < 16) {
           if (has_AVX512_fast)
             return resize_h_planar_uint16_avx512_permutex_vstripe_mp_4s16_ks48_pretransposed_coeffs_vnni<true>;