Skip to content

Revert "Reimplement _mm_mulhi_epi16, ... without LLVM intrinsics" #2219

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
RalfJung wants to merge 4 commits into
base: main
Choose a base branch
from
+55 −110
Open

Revert "Reimplement _mm_mulhi_epi16, ... without LLVM intrinsics" #2219

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d6ff136
Revert "Reimplement _mm_mulhi_epi16, _mm_mulhi_epu16, _mm256_mulhi_ep…
RalfJung Sep 5, 2026
fd305a8
de-constify dependent intrinsics and tests
RalfJung Sep 5, 2026
3276291
fmt
RalfJung Sep 5, 2026
6b64344
also adjust AVX512 intrinsics
RalfJung Sep 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 10 additions & 18 deletions crates/core_arch/src/x86/avx2.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2231,14 +2231,8 @@ pub const fn _mm256_mul_epu32(a: __m256i, b: __m256i) -> __m256i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpmulhw))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_mulhi_epi16(a: __m256i, b: __m256i) -> __m256i {
unsafe {
let a = simd_cast::<_, i32x16>(a.as_i16x16());
let b = simd_cast::<_, i32x16>(b.as_i16x16());
let r = simd_shr(simd_mul(a, b), i32x16::splat(16));
transmute(simd_cast::<i32x16, i16x16>(r))
}
pub fn _mm256_mulhi_epi16(a: __m256i, b: __m256i) -> __m256i {
unsafe { transmute(pmulhw(a.as_i16x16(), b.as_i16x16())) }
}

/// Multiplies the packed unsigned 16-bit integers in `a` and `b`, producing
Expand All @@ -2250,14 +2244,8 @@ pub const fn _mm256_mulhi_epi16(a: __m256i, b: __m256i) -> __m256i {
#[target_feature(enable = "avx2")]
#[cfg_attr(test, assert_instr(vpmulhuw))]
#[stable(feature = "simd_x86", since = "1.27.0")]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_mulhi_epu16(a: __m256i, b: __m256i) -> __m256i {
unsafe {
let a = simd_cast::<_, u32x16>(a.as_u16x16());
let b = simd_cast::<_, u32x16>(b.as_u16x16());
let r = simd_shr(simd_mul(a, b), u32x16::splat(16));
transmute(simd_cast::<u32x16, u16x16>(r))
}
pub fn _mm256_mulhi_epu16(a: __m256i, b: __m256i) -> __m256i {
unsafe { transmute(pmulhuw(a.as_u16x16(), b.as_u16x16())) }
}

/// Multiplies the packed 16-bit integers in `a` and `b`, producing
Expand Down Expand Up @@ -3825,6 +3813,10 @@ unsafe extern "unadjusted" {
fn pmaddubsw(a: u8x32, b: i8x32) -> i16x16;
#[link_name = "llvm.x86.avx2.mpsadbw"]
fn mpsadbw(a: u8x32, b: u8x32, imm8: i8) -> u16x16;
#[link_name = "llvm.x86.avx2.pmulhu.w"]
fn pmulhuw(a: u16x16, b: u16x16) -> u16x16;
#[link_name = "llvm.x86.avx2.pmulh.w"]
fn pmulhw(a: i16x16, b: i16x16) -> i16x16;
#[link_name = "llvm.x86.avx2.pmul.hr.sw"]
fn pmulhrsw(a: i16x16, b: i16x16) -> i16x16;
#[link_name = "llvm.x86.avx2.packsswb"]
Expand Down Expand Up @@ -4935,7 +4927,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm256_mulhi_epi16() {
fn test_mm256_mulhi_epi16() {
let a = _mm256_set1_epi16(6535);
let b = _mm256_set1_epi16(6535);
let r = _mm256_mulhi_epi16(a, b);
Expand All @@ -4944,7 +4936,7 @@ mod tests {
}

#[simd_test(enable = "avx2")]
const fn test_mm256_mulhi_epu16() {
fn test_mm256_mulhi_epu16() {
let a = _mm256_set1_epi16(6535);
let b = _mm256_set1_epi16(6535);
let r = _mm256_mulhi_epu16(a, b);
Expand Down
109 changes: 35 additions & 74 deletions crates/core_arch/src/x86/avx512bw.rs
View file
Open in desktop

@RalfJung RalfJung Sep 5, 2026 •
edited by rustbot
Loading

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FWIW there are also _mm512 variants of this... those should then probably also use the LLVM intrinsic?

View changes since the review

Original file line number Diff line number Diff line change
Expand Up @@ -1450,14 +1450,8 @@ pub const fn _mm_maskz_subs_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128
#[target_feature(enable = "avx512bw")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhuw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm512_mulhi_epu16(a: __m512i, b: __m512i) -> __m512i {
unsafe {
let a = simd_cast::<_, u32x32>(a.as_u16x32());
let b = simd_cast::<_, u32x32>(b.as_u16x32());
let r = simd_shr(simd_mul(a, b), u32x32::splat(16));
transmute(simd_cast::<u32x32, u16x32>(r))
}
pub fn _mm512_mulhi_epu16(a: __m512i, b: __m512i) -> __m512i {
unsafe { transmute(vpmulhuw(a.as_u16x32(), b.as_u16x32())) }
}

/// Multiply the packed unsigned 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).
Expand All @@ -1467,13 +1461,7 @@ pub const fn _mm512_mulhi_epu16(a: __m512i, b: __m512i) -> __m512i {
#[target_feature(enable = "avx512bw")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhuw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm512_mask_mulhi_epu16(
src: __m512i,
k: __mmask32,
a: __m512i,
b: __m512i,
) -> __m512i {
pub fn _mm512_mask_mulhi_epu16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i {
unsafe {
let mul = _mm512_mulhi_epu16(a, b).as_u16x32();
transmute(simd_select_bitmask(k, mul, src.as_u16x32()))
Expand All @@ -1487,8 +1475,7 @@ pub const fn _mm512_mask_mulhi_epu16(
#[target_feature(enable = "avx512bw")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhuw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm512_maskz_mulhi_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i {
pub fn _mm512_maskz_mulhi_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i {
unsafe {
let mul = _mm512_mulhi_epu16(a, b).as_u16x32();
transmute(simd_select_bitmask(k, mul, u16x32::ZERO))
Expand All @@ -1502,13 +1489,7 @@ pub const fn _mm512_maskz_mulhi_epu16(k: __mmask32, a: __m512i, b: __m512i) -> _
#[target_feature(enable = "avx512bw,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhuw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_mask_mulhi_epu16(
src: __m256i,
k: __mmask16,
a: __m256i,
b: __m256i,
) -> __m256i {
pub fn _mm256_mask_mulhi_epu16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i {
unsafe {
let mul = _mm256_mulhi_epu16(a, b).as_u16x16();
transmute(simd_select_bitmask(k, mul, src.as_u16x16()))
Expand All @@ -1522,8 +1503,7 @@ pub const fn _mm256_mask_mulhi_epu16(
#[target_feature(enable = "avx512bw,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhuw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_maskz_mulhi_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i {
pub fn _mm256_maskz_mulhi_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i {
unsafe {
let mul = _mm256_mulhi_epu16(a, b).as_u16x16();
transmute(simd_select_bitmask(k, mul, u16x16::ZERO))
Expand All @@ -1537,8 +1517,7 @@ pub const fn _mm256_maskz_mulhi_epu16(k: __mmask16, a: __m256i, b: __m256i) -> _
#[target_feature(enable = "avx512bw,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhuw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm_mask_mulhi_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i {
pub fn _mm_mask_mulhi_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i {
unsafe {
let mul = _mm_mulhi_epu16(a, b).as_u16x8();
transmute(simd_select_bitmask(k, mul, src.as_u16x8()))
Expand All @@ -1552,8 +1531,7 @@ pub const fn _mm_mask_mulhi_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m1
#[target_feature(enable = "avx512bw,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhuw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm_maskz_mulhi_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i {
pub fn _mm_maskz_mulhi_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i {
unsafe {
let mul = _mm_mulhi_epu16(a, b).as_u16x8();
transmute(simd_select_bitmask(k, mul, u16x8::ZERO))
Expand All @@ -1567,14 +1545,8 @@ pub const fn _mm_maskz_mulhi_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m12
#[target_feature(enable = "avx512bw")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm512_mulhi_epi16(a: __m512i, b: __m512i) -> __m512i {
unsafe {
let a = simd_cast::<_, i32x32>(a.as_i16x32());
let b = simd_cast::<_, i32x32>(b.as_i16x32());
let r = simd_shr(simd_mul(a, b), i32x32::splat(16));
transmute(simd_cast::<i32x32, i16x32>(r))
}
pub fn _mm512_mulhi_epi16(a: __m512i, b: __m512i) -> __m512i {
unsafe { transmute(vpmulhw(a.as_i16x32(), b.as_i16x32())) }
}

/// Multiply the packed signed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set).
Expand All @@ -1584,13 +1556,7 @@ pub const fn _mm512_mulhi_epi16(a: __m512i, b: __m512i) -> __m512i {
#[target_feature(enable = "avx512bw")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm512_mask_mulhi_epi16(
src: __m512i,
k: __mmask32,
a: __m512i,
b: __m512i,
) -> __m512i {
pub fn _mm512_mask_mulhi_epi16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i {
unsafe {
let mul = _mm512_mulhi_epi16(a, b).as_i16x32();
transmute(simd_select_bitmask(k, mul, src.as_i16x32()))
Expand All @@ -1604,8 +1570,7 @@ pub const fn _mm512_mask_mulhi_epi16(
#[target_feature(enable = "avx512bw")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm512_maskz_mulhi_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i {
pub fn _mm512_maskz_mulhi_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i {
unsafe {
let mul = _mm512_mulhi_epi16(a, b).as_i16x32();
transmute(simd_select_bitmask(k, mul, i16x32::ZERO))
Expand All @@ -1619,13 +1584,7 @@ pub const fn _mm512_maskz_mulhi_epi16(k: __mmask32, a: __m512i, b: __m512i) -> _
#[target_feature(enable = "avx512bw,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_mask_mulhi_epi16(
src: __m256i,
k: __mmask16,
a: __m256i,
b: __m256i,
) -> __m256i {
pub fn _mm256_mask_mulhi_epi16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i {
unsafe {
let mul = _mm256_mulhi_epi16(a, b).as_i16x16();
transmute(simd_select_bitmask(k, mul, src.as_i16x16()))
Expand All @@ -1639,8 +1598,7 @@ pub const fn _mm256_mask_mulhi_epi16(
#[target_feature(enable = "avx512bw,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm256_maskz_mulhi_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i {
pub fn _mm256_maskz_mulhi_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i {
unsafe {
let mul = _mm256_mulhi_epi16(a, b).as_i16x16();
transmute(simd_select_bitmask(k, mul, i16x16::ZERO))
Expand All @@ -1654,8 +1612,7 @@ pub const fn _mm256_maskz_mulhi_epi16(k: __mmask16, a: __m256i, b: __m256i) -> _
#[target_feature(enable = "avx512bw,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm_mask_mulhi_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i {
pub fn _mm_mask_mulhi_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i {
unsafe {
let mul = _mm_mulhi_epi16(a, b).as_i16x8();
transmute(simd_select_bitmask(k, mul, src.as_i16x8()))
Expand All @@ -1669,8 +1626,7 @@ pub const fn _mm_mask_mulhi_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m1
#[target_feature(enable = "avx512bw,avx512vl")]
#[stable(feature = "stdarch_x86_avx512", since = "1.89")]
#[cfg_attr(test, assert_instr(vpmulhw))]
#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")]
pub const fn _mm_maskz_mulhi_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i {
pub fn _mm_maskz_mulhi_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i {
unsafe {
let mul = _mm_mulhi_epi16(a, b).as_i16x8();
transmute(simd_select_bitmask(k, mul, i16x8::ZERO))
Expand Down Expand Up @@ -12601,6 +12557,11 @@ unsafe extern "unadjusted" {
#[link_name = "llvm.x86.avx512.pmul.hr.sw.512"]
fn vpmulhrsw(a: i16x32, b: i16x32) -> i16x32;

#[link_name = "llvm.x86.avx512.pmulhu.w.512"]
fn vpmulhuw(a: u16x32, b: u16x32) -> u16x32;
#[link_name = "llvm.x86.avx512.pmulh.w.512"]
fn vpmulhw(a: i16x32, b: i16x32) -> i16x32;

#[link_name = "llvm.x86.avx512.pmaddw.d.512"]
fn vpmaddwd(a: i16x32, b: i16x32) -> i32x16;
#[link_name = "llvm.x86.avx512.pmaddubs.w.512"]
Expand Down Expand Up @@ -13899,7 +13860,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw")]
const fn test_mm512_mulhi_epu16() {
fn test_mm512_mulhi_epu16() {
let a = _mm512_set1_epi16(1);
let b = _mm512_set1_epi16(1);
let r = _mm512_mulhi_epu16(a, b);
Expand All @@ -13908,7 +13869,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw")]
const fn test_mm512_mask_mulhi_epu16() {
fn test_mm512_mask_mulhi_epu16() {
let a = _mm512_set1_epi16(1);
let b = _mm512_set1_epi16(1);
let r = _mm512_mask_mulhi_epu16(a, 0, a, b);
Expand All @@ -13921,7 +13882,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw")]
const fn test_mm512_maskz_mulhi_epu16() {
fn test_mm512_maskz_mulhi_epu16() {
let a = _mm512_set1_epi16(1);
let b = _mm512_set1_epi16(1);
let r = _mm512_maskz_mulhi_epu16(0, a, b);
Expand All @@ -13934,7 +13895,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw,avx512vl")]
const fn test_mm256_mask_mulhi_epu16() {
fn test_mm256_mask_mulhi_epu16() {
let a = _mm256_set1_epi16(1);
let b = _mm256_set1_epi16(1);
let r = _mm256_mask_mulhi_epu16(a, 0, a, b);
Expand All @@ -13945,7 +13906,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw,avx512vl")]
const fn test_mm256_maskz_mulhi_epu16() {
fn test_mm256_maskz_mulhi_epu16() {
let a = _mm256_set1_epi16(1);
let b = _mm256_set1_epi16(1);
let r = _mm256_maskz_mulhi_epu16(0, a, b);
Expand All @@ -13956,7 +13917,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw,avx512vl")]
const fn test_mm_mask_mulhi_epu16() {
fn test_mm_mask_mulhi_epu16() {
let a = _mm_set1_epi16(1);
let b = _mm_set1_epi16(1);
let r = _mm_mask_mulhi_epu16(a, 0, a, b);
Expand All @@ -13967,7 +13928,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw,avx512vl")]
const fn test_mm_maskz_mulhi_epu16() {
fn test_mm_maskz_mulhi_epu16() {
let a = _mm_set1_epi16(1);
let b = _mm_set1_epi16(1);
let r = _mm_maskz_mulhi_epu16(0, a, b);
Expand All @@ -13978,7 +13939,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw")]
const fn test_mm512_mulhi_epi16() {
fn test_mm512_mulhi_epi16() {
let a = _mm512_set1_epi16(1);
let b = _mm512_set1_epi16(1);
let r = _mm512_mulhi_epi16(a, b);
Expand All @@ -13987,7 +13948,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw")]
const fn test_mm512_mask_mulhi_epi16() {
fn test_mm512_mask_mulhi_epi16() {
let a = _mm512_set1_epi16(1);
let b = _mm512_set1_epi16(1);
let r = _mm512_mask_mulhi_epi16(a, 0, a, b);
Expand All @@ -14000,7 +13961,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw")]
const fn test_mm512_maskz_mulhi_epi16() {
fn test_mm512_maskz_mulhi_epi16() {
let a = _mm512_set1_epi16(1);
let b = _mm512_set1_epi16(1);
let r = _mm512_maskz_mulhi_epi16(0, a, b);
Expand All @@ -14013,7 +13974,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw,avx512vl")]
const fn test_mm256_mask_mulhi_epi16() {
fn test_mm256_mask_mulhi_epi16() {
let a = _mm256_set1_epi16(1);
let b = _mm256_set1_epi16(1);
let r = _mm256_mask_mulhi_epi16(a, 0, a, b);
Expand All @@ -14024,7 +13985,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw,avx512vl")]
const fn test_mm256_maskz_mulhi_epi16() {
fn test_mm256_maskz_mulhi_epi16() {
let a = _mm256_set1_epi16(1);
let b = _mm256_set1_epi16(1);
let r = _mm256_maskz_mulhi_epi16(0, a, b);
Expand All @@ -14035,7 +13996,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw,avx512vl")]
const fn test_mm_mask_mulhi_epi16() {
fn test_mm_mask_mulhi_epi16() {
let a = _mm_set1_epi16(1);
let b = _mm_set1_epi16(1);
let r = _mm_mask_mulhi_epi16(a, 0, a, b);
Expand All @@ -14046,7 +14007,7 @@ mod tests {
}

#[simd_test(enable = "avx512bw,avx512vl")]
const fn test_mm_maskz_mulhi_epi16() {
fn test_mm_maskz_mulhi_epi16() {
let a = _mm_set1_epi16(1);
let b = _mm_set1_epi16(1);
let r = _mm_maskz_mulhi_epi16(0, a, b);
Expand Down
Loading
Loading