CI update

Author: awxkee

src/fixed_point_vertical.rs

@@ -32,7 +32,7 @@ use crate::saturate_narrow::SaturateNarrow;
 use num_traits::AsPrimitive;
 use std::ops::{AddAssign, Mul};
 
-#[inline(always)]
+#[inline(never)]
 /// # Generics
 /// `T` - template buffer type
 /// `J` - accumulator type
@@ -86,11 +86,8 @@ pub(crate) fn convolve_column_handler_fixed_point_direct_buffer<
     }
 }
 
-#[inline(always)]
-/// # Generics
-/// `T` - template buffer type
-/// `J` - accumulator type
-pub(crate) fn convolve_column_handler_fixed_point_direct_buffer_double<
+#[inline(never)]
+pub(crate) fn convolve_column_handler_fixed_point_direct_buffer16<
     T: Copy + 'static + AsPrimitive<J> + Default,
     J: Copy + 'static + AsPrimitive<T> + Mul<Output = J> + AddAssign + SaturateNarrow<T> + Default,
     const BUFFER_SIZE: usize,
@@ -109,49 +106,181 @@ pub(crate) fn convolve_column_handler_fixed_point_direct_buffer_double<
     if filter.is_empty() {
         return;
     }
-    let mut direct_store0: [J; BUFFER_SIZE] = [ROUNDING_CONST.as_(); BUFFER_SIZE];
-    let mut direct_store1: [J; BUFFER_SIZE] = [ROUNDING_CONST.as_(); BUFFER_SIZE];
 
-    let v_start_px = x;
+    let rc: J = ROUNDING_CONST.as_();
+    let mut store0: [J; 4] = [rc; 4];
+    let mut store1: [J; 4] = [rc; 4];
+    let mut store2: [J; 4] = [rc; 4];
+    let mut store3: [J; 4] = [rc; 4];
+
+    let base = src_stride * bounds.start + x;
+    let quarter = BUFFER_SIZE / 4;
+
+    for j in 0..bounds.size {
+        let w: J = filter[j].as_();
+        let off = base + src_stride * j;
+
+        // Four contiguous non-overlapping chunks of the same row.
+        // Each chunk has a statically known length (BUFFER_SIZE/4),
+        // giving LLVM exactly one XMM register worth of i32 accumulators.
+        // The lack of data dependency between chunks enables the vectorizer
+        // to treat them as 4 independent pmovzxbd + pmulld chains.
+        let (chunk0, rest) = src[off..off + BUFFER_SIZE].split_at(quarter);
+        let (chunk1, rest) = rest.split_at(quarter);
+        let (chunk2, chunk3) = rest.split_at(quarter);
+
+        for (acc, &s) in store0.iter_mut().zip(chunk0) {
+            *acc += s.as_() * w;
+        }
+        for (acc, &s) in store1.iter_mut().zip(chunk1) {
+            *acc += s.as_() * w;
+        }
+        for (acc, &s) in store2.iter_mut().zip(chunk2) {
+            *acc += s.as_() * w;
+        }
+        for (acc, &s) in store3.iter_mut().zip(chunk3) {
+            *acc += s.as_() * w;
+        }
+    }
 
-    let py = bounds.start;
-    let weight = filter[0].as_();
-    let offset = src_stride * py + v_start_px;
-    let src_ptr0 = &src[offset..(offset + BUFFER_SIZE)];
-    let src_ptr1 = &src[(offset + BUFFER_SIZE)..(offset + BUFFER_SIZE * 2)];
+    // Writeback: four independent store chains
+    let v_dst = &mut dst[x..x + BUFFER_SIZE];
+    let (out0, rest) = v_dst.split_at_mut(quarter);
+    let (out1, rest) = rest.split_at_mut(quarter);
+    let (out2, out3) = rest.split_at_mut(quarter);
 
-    for (dst, src) in direct_store0.iter_mut().zip(src_ptr0) {
-        *dst += src.as_() * weight;
+    for (d, s) in out0.iter_mut().zip(store0) {
+        *d = s.saturate_narrow(bit_depth);
+    }
+    for (d, s) in out1.iter_mut().zip(store1) {
+        *d = s.saturate_narrow(bit_depth);
+    }
+    for (d, s) in out2.iter_mut().zip(store2) {
+        *d = s.saturate_narrow(bit_depth);
     }
+    for (d, s) in out3.iter_mut().zip(store3) {
+        *d = s.saturate_narrow(bit_depth);
+    }
+}
 
-    for (dst, src) in direct_store1.iter_mut().zip(src_ptr1) {
-        *dst += src.as_() * weight;
+#[inline(never)]
+/// # Generics
+/// `T` - template buffer type
+/// `J` - accumulator type
+pub(crate) fn convolve_column_handler_fixed_point_direct_buffer_double<
+    T: Copy + 'static + AsPrimitive<J> + Default,
+    J: Copy + 'static + AsPrimitive<T> + Mul<Output = J> + AddAssign + SaturateNarrow<T> + Default,
+    const BUFFER_SIZE: usize,
+>(
+    src: &[T],
+    src_stride: usize,
+    dst: &mut [T],
+    filter: &[i16],
+    bounds: &FilterBounds,
+    bit_depth: u32,
+    x: usize,
+) where
+    i32: AsPrimitive<J>,
+    i16: AsPrimitive<J>,
+{
+    if filter.is_empty() {
+        return;
     }
 
-    for (j, &k_weight) in filter.iter().take(bounds.size).skip(1).enumerate() {
-        // Adding 1 is necessary because skip do not incrementing value on values that skipped
-        let py = bounds.start + j + 1;
-        let weight = k_weight.as_();
-        let offset = src_stride * py + v_start_px;
-        let src_ptr0 = &src[offset..(offset + BUFFER_SIZE)];
-        let src_ptr1 = &src[(offset + BUFFER_SIZE)..(offset + BUFFER_SIZE * 2)];
+    let rc: J = ROUNDING_CONST.as_();
+    let quarter = BUFFER_SIZE / 4;
+
+    // 8 independent accumulator chains → 8 XMM registers, no spill
+    let mut s00: [J; 4] = [rc; 4];
+    let mut s01: [J; 4] = [rc; 4];
+    let mut s02: [J; 4] = [rc; 4];
+    let mut s03: [J; 4] = [rc; 4];
+    // second 16-pixel block
+    let mut s10: [J; 4] = [rc; 4];
+    let mut s11: [J; 4] = [rc; 4];
+    let mut s12: [J; 4] = [rc; 4];
+    let mut s13: [J; 4] = [rc; 4];
+
+    let base = src_stride * bounds.start + x;
+
+    for j in 0..bounds.size {
+        let w: J = filter[j].as_();
+        let off = base + src_stride * j;
+
+        // First 16-pixel block split into 4 independent chains
+        let (c00, rest) = src[off..off + BUFFER_SIZE].split_at(quarter);
+        let (c01, rest) = rest.split_at(quarter);
+        let (c02, c03) = rest.split_at(quarter);
+
+        // Second 16-pixel block split into 4 independent chains
+        let off1 = off + BUFFER_SIZE;
+        let (c10, rest) = src[off1..off1 + BUFFER_SIZE].split_at(quarter);
+        let (c11, rest) = rest.split_at(quarter);
+        let (c12, c13) = rest.split_at(quarter);
+
+        for (acc, &s) in s00.iter_mut().zip(c00) {
+            *acc += s.as_() * w;
+        }
+        for (acc, &s) in s01.iter_mut().zip(c01) {
+            *acc += s.as_() * w;
+        }
+        for (acc, &s) in s02.iter_mut().zip(c02) {
+            *acc += s.as_() * w;
+        }
+        for (acc, &s) in s03.iter_mut().zip(c03) {
+            *acc += s.as_() * w;
+        }
 
-        for (dst, src) in direct_store0.iter_mut().zip(src_ptr0.iter()) {
-            *dst += src.as_() * weight;
+        for (acc, &s) in s10.iter_mut().zip(c10) {
+            *acc += s.as_() * w;
         }
-        for (dst, src) in direct_store1.iter_mut().zip(src_ptr1.iter()) {
-            *dst += src.as_() * weight;
+        for (acc, &s) in s11.iter_mut().zip(c11) {
+            *acc += s.as_() * w;
+        }
+        for (acc, &s) in s12.iter_mut().zip(c12) {
+            *acc += s.as_() * w;
+        }
+        for (acc, &s) in s13.iter_mut().zip(c13) {
+            *acc += s.as_() * w;
         }
     }
 
-    let v_dst0 = &mut dst[v_start_px..(v_start_px + BUFFER_SIZE)];
-    for (dst, src) in v_dst0.iter_mut().zip(direct_store0) {
-        *dst = src.saturate_narrow(bit_depth);
+    // Writeback block 0
+    let v_dst0 = &mut dst[x..x + BUFFER_SIZE];
+    let (o00, rest) = v_dst0.split_at_mut(quarter);
+    let (o01, rest) = rest.split_at_mut(quarter);
+    let (o02, o03) = rest.split_at_mut(quarter);
+
+    for (d, s) in o00.iter_mut().zip(s00) {
+        *d = s.saturate_narrow(bit_depth);
+    }
+    for (d, s) in o01.iter_mut().zip(s01) {
+        *d = s.saturate_narrow(bit_depth);
+    }
+    for (d, s) in o02.iter_mut().zip(s02) {
+        *d = s.saturate_narrow(bit_depth);
+    }
+    for (d, s) in o03.iter_mut().zip(s03) {
+        *d = s.saturate_narrow(bit_depth);
     }
 
-    let v_dst1 = &mut dst[(v_start_px + BUFFER_SIZE)..(v_start_px + BUFFER_SIZE * 2)];
-    for (dst, src) in v_dst1.iter_mut().zip(direct_store1) {
-        *dst = src.saturate_narrow(bit_depth);
+    // Writeback block 1
+    let v_dst1 = &mut dst[x + BUFFER_SIZE..x + BUFFER_SIZE * 2];
+    let (o10, rest) = v_dst1.split_at_mut(quarter);
+    let (o11, rest) = rest.split_at_mut(quarter);
+    let (o12, o13) = rest.split_at_mut(quarter);
+
+    for (d, s) in o10.iter_mut().zip(s10) {
+        *d = s.saturate_narrow(bit_depth);
+    }
+    for (d, s) in o11.iter_mut().zip(s11) {
+        *d = s.saturate_narrow(bit_depth);
+    }
+    for (d, s) in o12.iter_mut().zip(s12) {
+        *d = s.saturate_narrow(bit_depth);
+    }
+    for (d, s) in o13.iter_mut().zip(s13) {
+        *d = s.saturate_narrow(bit_depth);
     }
 }
 
@@ -294,7 +423,7 @@ pub(crate) fn column_handler_fixed_point<
     }
 
     while cx + 16 < total_width {
-        convolve_column_handler_fixed_point_direct_buffer::<T, J, 16>(
+        convolve_column_handler_fixed_point_direct_buffer16::<T, J, 16>(
             src, src_stride, dst, weight, bounds, bit_depth, cx,
         );