From 8c87e3ec630ee48629fa83153e124c44c70566bd Mon Sep 17 00:00:00 2001 From: April Ivy Date: Sun, 26 Jul 2026 00:05:51 +0100 Subject: [PATCH 1/2] 8389111: C2: Use BMI2 BZHI for variable int low-bit masks on x86 --- src/hotspot/cpu/x86/x86.ad | 20 ++++++++ .../compiler/intrinsics/bmi/TestBzhiI2L.java | 47 ++++++++++++++++++- .../bmi/verifycode/BzhiTestI2L.java | 45 ++++++++++++++++++ 3 files changed, 111 insertions(+), 1 deletion(-) diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index b3e5a42926ccf..fb94f71ebeb98 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -12692,6 +12692,26 @@ instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src, immI_Pow2M1 mask, rR ins_pipe(ialu_reg_reg); %} +// Generate a low-bits mask with BZHI. The shift count must be masked explicitly +// because Java shift distances wrap modulo 32 while BZHI treats counts >= 32 as +// selecting the entire source operand. +instruct bzhiI_rReg_rReg(rRegI dst, rRegI src, immI_1 one, immI_M1 minus_one, + rFlagsReg cr) +%{ + predicate(VM_Version::supports_bmi2()); + match(Set dst (AndI src (AddI (LShiftI one dst) minus_one))); + effect(KILL cr); + + ins_cost(200); + format %{ "andl $dst, 31\n\t" + "bzhil $dst, $src, $dst\t# low bits mask" %} + ins_encode %{ + __ andl($dst$$Register, 31); + __ bzhil($dst$$Register, $src$$Register, $dst$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + // And Register with Immediate instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr) %{ diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiI2L.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiI2L.java index 18d822cecbc34..80cde4f5b06d9 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiI2L.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiI2L.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -23,6 +23,7 @@ /** * @test + * @bug 8389111 * @key randomness * @summary Verify that results of computations are the same w/ * and w/o usage of BZHI instruction @@ -58,6 +59,12 @@ public static void main(String args[]) throws Throwable { BMITestRunner.runTests(BzhiI2LCommutativeExpr.class, args, "-XX:+IgnoreUnrecognizedVMOptions", "-XX:+UseBMI2Instructions"); + BMITestRunner.runTests(BzhiIExpr.class, args, + "-XX:+IgnoreUnrecognizedVMOptions", + "-XX:+UseBMI2Instructions"); + BMITestRunner.runTests(BzhiICommutativeExpr.class, args, + "-XX:+IgnoreUnrecognizedVMOptions", + "-XX:+UseBMI2Instructions"); } public static class BzhiI2LExpr extends Expr.BMIUnaryIntToLongExpr { @@ -93,4 +100,42 @@ public long intToLongExpr(int src1) { return returnValue; } } + + public static class BzhiIExpr extends Expr.BMIBinaryIntExpr { + + public int intExpr(int value, int bits) { + return value & ((1 << bits) - 1); + } + + public int intExpr(int value, Expr.MemI bits) { + return value & ((1 << bits.value) - 1); + } + + public int intExpr(Expr.MemI value, int bits) { + return value.value & ((1 << bits) - 1); + } + + public int intExpr(Expr.MemI value, Expr.MemI bits) { + return value.value & ((1 << bits.value) - 1); + } + } + + public static class BzhiICommutativeExpr extends Expr.BMIBinaryIntExpr { + + public int intExpr(int value, int bits) { + return ((1 << bits) - 1) & value; + } + + public int intExpr(int value, Expr.MemI bits) { + return ((1 << bits.value) - 1) & value; + } + + public int intExpr(Expr.MemI value, int bits) { + return ((1 << bits) - 1) & value.value; + } + + public int intExpr(Expr.MemI value, Expr.MemI bits) { + return ((1 << bits.value) - 1) & value.value; + } + } } diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java index 8185dda4db1e4..f5ce87f2add66 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java @@ -23,6 +23,7 @@ /* * @test + * @bug 8389111 * @requires vm.simpleArch == "x64" & vm.flavor == "server" * @library /test/lib / * @modules java.base/jdk.internal.misc @@ -44,6 +45,46 @@ public class BzhiTestI2L extends BmiIntrinsicBase.BmiTestCase_x64 { + private static class BzhiTestI extends BmiIntrinsicBase.BmiTestCase { + + protected BzhiTestI(Method method) { + super(method); + + cpuFlag = "bmi2"; + vmFlag = "UseBMI2Instructions"; + + // From the Intel manual: VEX.LZ.0F38.W0 F5 /r. + instrMask = new byte[] { + (byte) 0xFF, + (byte) 0x1F, + (byte) 0x80, + (byte) 0xFF + }; + instrPattern = new byte[] { + (byte) 0xC4, + (byte) 0x02, + (byte) 0x00, + (byte) 0xF5 + }; + + // From the Intel APX specification: EVEX.128.NP.0F38.W0 F5 /r. + instrMaskAPX = new byte[] { + (byte) 0xFF, + (byte) 0x07, + (byte) 0x00, + (byte) 0x00, + (byte) 0xFF + }; + instrPatternAPX = new byte[] { + (byte) 0x62, + (byte) 0x02, + (byte) 0x00, + (byte) 0x00, + (byte) 0xF5 + }; + } + } + protected BzhiTestI2L(Method method) { super(method); @@ -93,5 +134,9 @@ protected BzhiTestI2L(Method method) { public static void main(String[] args) throws Exception { BmiIntrinsicBase.verifyTestCase(BzhiTestI2L::new, TestBzhiI2L.BzhiI2LExpr.class.getDeclaredMethods()); BmiIntrinsicBase.verifyTestCase(BzhiTestI2L::new, TestBzhiI2L.BzhiI2LCommutativeExpr.class.getDeclaredMethods()); + BmiIntrinsicBase.verifyTestCase(BzhiTestI::new, + TestBzhiI2L.BzhiIExpr.class.getDeclaredMethod("intExpr", int.class, int.class)); + BmiIntrinsicBase.verifyTestCase(BzhiTestI::new, + TestBzhiI2L.BzhiICommutativeExpr.class.getDeclaredMethod("intExpr", int.class, int.class)); } } From dcd66a97f0e0960eace39c1451909fb706a657f4 Mon Sep 17 00:00:00 2001 From: April Ivy Date: Mon, 27 Jul 2026 18:26:51 +0100 Subject: [PATCH 2/2] 8389111: Avoid redundant count masking for BZHI --- src/hotspot/cpu/x86/x86.ad | 58 ++++++++++++++-- .../compiler/intrinsics/bmi/TestBzhiI2L.java | 22 ++++++ .../compiler/intrinsics/bmi/TestBzhiIR.java | 68 +++++++++++++++++++ .../bmi/verifycode/BzhiTestI2L.java | 2 + 4 files changed, 146 insertions(+), 4 deletions(-) create mode 100644 test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiIR.java diff --git a/src/hotspot/cpu/x86/x86.ad b/src/hotspot/cpu/x86/x86.ad index fb94f71ebeb98..c4b63bae8c1ee 100644 --- a/src/hotspot/cpu/x86/x86.ad +++ b/src/hotspot/cpu/x86/x86.ad @@ -1457,6 +1457,7 @@ source_hpp %{ #include "peephole_x86_64.hpp" bool castLL_is_imm32(const Node* n); +bool is_bzhi_shift_count_bounded(const Node* n); %} @@ -1468,6 +1469,36 @@ bool castLL_is_imm32(const Node* n) { return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi)); } +static const Node* bzhi_shift_count(const Node* n) { + if (n == nullptr || n->Opcode() != Op_AndI) { + return nullptr; + } + for (uint i = 1; i < n->req(); i++) { + const Node* add = n->in(i); + if (add == nullptr || add->Opcode() != Op_AddI) { + continue; + } + for (uint j = 1; j < add->req(); j++) { + const Node* shift = add->in(j); + if (shift != nullptr && shift->Opcode() == Op_LShiftI && + shift->req() > 2) { + return shift->in(2); + } + } + } + return nullptr; +} + +bool is_bzhi_shift_count_bounded(const Node* n) { + const Node* count = bzhi_shift_count(n); + if (count == nullptr) { + return false; + } + const Type* type = (*Compile::current()->types())[count->_idx]; + const TypeInt* t = type == nullptr ? count->find_int_type() : type->isa_int(); + return t != nullptr && t->_lo >= 0 && t->_hi < BitsPerInt; +} + %} // Register masks @@ -12692,13 +12723,32 @@ instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src, immI_Pow2M1 mask, rR ins_pipe(ialu_reg_reg); %} -// Generate a low-bits mask with BZHI. The shift count must be masked explicitly -// because Java shift distances wrap modulo 32 while BZHI treats counts >= 32 as -// selecting the entire source operand. +// Generate a low-bits mask with BZHI when the shift count is known to be in +// [0, 31]. +instruct bzhiI_rReg_rReg_bounded(rRegI dst, rRegI src, immI_1 one, + immI_M1 minus_one, rFlagsReg cr) +%{ + predicate(VM_Version::supports_bmi2() && + is_bzhi_shift_count_bounded(n)); + match(Set dst (AndI src (AddI (LShiftI one dst) minus_one))); + effect(KILL cr); + + ins_cost(100); + format %{ "bzhil $dst, $src, $dst\t# low bits mask" %} + ins_encode %{ + __ bzhil($dst$$Register, $src$$Register, $dst$$Register); + %} + ins_pipe(ialu_reg_reg); +%} + +// For unrestricted shift counts, mask the count explicitly because Java shift +// distances wrap modulo 32 while BZHI treats counts >= 32 as selecting the +// entire source operand. instruct bzhiI_rReg_rReg(rRegI dst, rRegI src, immI_1 one, immI_M1 minus_one, rFlagsReg cr) %{ - predicate(VM_Version::supports_bmi2()); + predicate(VM_Version::supports_bmi2() && + !is_bzhi_shift_count_bounded(n)); match(Set dst (AndI src (AddI (LShiftI one dst) minus_one))); effect(KILL cr); diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiI2L.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiI2L.java index 80cde4f5b06d9..0ecd8adef9323 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiI2L.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiI2L.java @@ -65,6 +65,9 @@ public static void main(String args[]) throws Throwable { BMITestRunner.runTests(BzhiICommutativeExpr.class, args, "-XX:+IgnoreUnrecognizedVMOptions", "-XX:+UseBMI2Instructions"); + BMITestRunner.runTests(BzhiIBoundedExpr.class, args, + "-XX:+IgnoreUnrecognizedVMOptions", + "-XX:+UseBMI2Instructions"); } public static class BzhiI2LExpr extends Expr.BMIUnaryIntToLongExpr { @@ -138,4 +141,23 @@ public int intExpr(Expr.MemI value, Expr.MemI bits) { return ((1 << bits.value) - 1) & value.value; } } + + public static class BzhiIBoundedExpr extends Expr.BMIBinaryIntExpr { + + public int intExpr(int value, int bits) { + return value & ((1 << (bits & 31)) - 1); + } + + public int intExpr(int value, Expr.MemI bits) { + return value & ((1 << (bits.value & 31)) - 1); + } + + public int intExpr(Expr.MemI value, int bits) { + return value.value & ((1 << (bits & 31)) - 1); + } + + public int intExpr(Expr.MemI value, Expr.MemI bits) { + return value.value & ((1 << (bits.value & 31)) - 1); + } + } } diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiIR.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiIR.java new file mode 100644 index 0000000000000..dfa1458ce42dd --- /dev/null +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiIR.java @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +/** + * @test + * @bug 8389111 + * @summary Verify BZHI matching for bounded and unrestricted int shift counts + * @requires vm.simpleArch == "x64" & vm.compiler2.enabled + * @library /test/lib / + * @run driver compiler.intrinsics.bmi.TestBzhiIR + */ + +package compiler.intrinsics.bmi; + +import compiler.lib.ir_framework.Argument; +import compiler.lib.ir_framework.Arguments; +import compiler.lib.ir_framework.CompilePhase; +import compiler.lib.ir_framework.IR; +import compiler.lib.ir_framework.Test; +import compiler.lib.ir_framework.TestFramework; + +public class TestBzhiIR { + + public static void main(String[] args) { + TestFramework.runWithFlags("-XX:+IgnoreUnrecognizedVMOptions", + "-XX:+UseBMI2Instructions"); + } + + @Test + @Arguments(values = {Argument.RANDOM_EACH, Argument.RANDOM_EACH}) + @IR(counts = {"bzhiI_rReg_rReg\\b", "1"}, + failOn = {"bzhiI_rReg_rReg_bounded"}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"bmi2", "true"}) + public static int testUnrestricted(int value, int bits) { + return value & ((1 << bits) - 1); + } + + @Test + @Arguments(values = {Argument.RANDOM_EACH, Argument.RANDOM_EACH}) + @IR(counts = {"bzhiI_rReg_rReg_bounded", "1"}, + failOn = {"bzhiI_rReg_rReg\\b"}, + phase = CompilePhase.MATCHING, + applyIfCPUFeature = {"bmi2", "true"}) + public static int testBounded(int value, int bits) { + return value & ((1 << (bits & 31)) - 1); + } +} diff --git a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java index f5ce87f2add66..2b0d44aba7c8e 100644 --- a/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java +++ b/test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java @@ -138,5 +138,7 @@ public static void main(String[] args) throws Exception { TestBzhiI2L.BzhiIExpr.class.getDeclaredMethod("intExpr", int.class, int.class)); BmiIntrinsicBase.verifyTestCase(BzhiTestI::new, TestBzhiI2L.BzhiICommutativeExpr.class.getDeclaredMethod("intExpr", int.class, int.class)); + BmiIntrinsicBase.verifyTestCase(BzhiTestI::new, + TestBzhiI2L.BzhiIBoundedExpr.class.getDeclaredMethod("intExpr", int.class, int.class)); } }