8389111: C2: Use BMI2 BZHI for variable int low-bit masks on x86

src/hotspot/cpu/x86/x86.ad

@@ -1457,6 +1457,7 @@ source_hpp %{
 #include "peephole_x86_64.hpp"
 
 bool castLL_is_imm32(const Node* n);
+bool is_bzhi_shift_count_bounded(const Node* n);
 
 %}
 
@@ -1468,6 +1469,36 @@ bool castLL_is_imm32(const Node* n) {
   return (t->_lo == min_jlong || Assembler::is_simm32(t->_lo)) && (t->_hi == max_jlong || Assembler::is_simm32(t->_hi));
 }
 
+static const Node* bzhi_shift_count(const Node* n) {
+  if (n == nullptr || n->Opcode() != Op_AndI) {
+    return nullptr;
+  }
+  for (uint i = 1; i < n->req(); i++) {
+    const Node* add = n->in(i);
+    if (add == nullptr || add->Opcode() != Op_AddI) {
+      continue;
+    }
+    for (uint j = 1; j < add->req(); j++) {
+      const Node* shift = add->in(j);
+      if (shift != nullptr && shift->Opcode() == Op_LShiftI &&
+          shift->req() > 2) {
+        return shift->in(2);
+      }
+    }
+  }
+  return nullptr;
+}
+
+bool is_bzhi_shift_count_bounded(const Node* n) {
+  const Node* count = bzhi_shift_count(n);
+  if (count == nullptr) {
+    return false;
+  }
+  const Type* type = (*Compile::current()->types())[count->_idx];
+  const TypeInt* t = type == nullptr ? count->find_int_type() : type->isa_int();
+  return t != nullptr && t->_lo >= 0 && t->_hi < BitsPerInt;
+}
+
 %}
 
 // Register masks
@@ -12692,6 +12723,45 @@ instruct convI2LAndI_reg_immIbitmask(rRegL dst, rRegI src,  immI_Pow2M1 mask, rR
   ins_pipe(ialu_reg_reg);
 %}
 
+// Generate a low-bits mask with BZHI when the shift count is known to be in
+// [0, 31].
+instruct bzhiI_rReg_rReg_bounded(rRegI dst, rRegI src, immI_1 one,
+                                 immI_M1 minus_one, rFlagsReg cr)
+%{
+  predicate(VM_Version::supports_bmi2() &&
+            is_bzhi_shift_count_bounded(n));
+  match(Set dst (AndI src (AddI (LShiftI one dst) minus_one)));
+  effect(KILL cr);
+
+  ins_cost(100);
+  format %{ "bzhil   $dst, $src, $dst\t# low bits mask" %}
+  ins_encode %{
+    __ bzhil($dst$$Register, $src$$Register, $dst$$Register);
+  %}
+  ins_pipe(ialu_reg_reg);
+%}
+
+// For unrestricted shift counts, mask the count explicitly because Java shift
+// distances wrap modulo 32 while BZHI treats counts >= 32 as selecting the
+// entire source operand.
+instruct bzhiI_rReg_rReg(rRegI dst, rRegI src, immI_1 one, immI_M1 minus_one,
+                         rFlagsReg cr)
+%{
+  predicate(VM_Version::supports_bmi2() &&
+            !is_bzhi_shift_count_bounded(n));
+  match(Set dst (AndI src (AddI (LShiftI one dst) minus_one)));
+  effect(KILL cr);
+
+  ins_cost(200);
+  format %{ "andl    $dst, 31\n\t"
+            "bzhil   $dst, $src, $dst\t# low bits mask" %}
+  ins_encode %{
+    __ andl($dst$$Register, 31);
+    __ bzhil($dst$$Register, $src$$Register, $dst$$Register);
+  %}
+  ins_pipe(ialu_reg_reg);
+%}
+
 // And Register with Immediate
 instruct andI_rReg_imm(rRegI dst, immI src, rFlagsReg cr)
 %{

test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiI2L.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2021, 2026, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
 
 /**
  * @test
+ * @bug 8389111
  * @key randomness
  * @summary Verify that results of computations are the same w/
  *          and w/o usage of BZHI instruction
@@ -58,6 +59,15 @@ public static void main(String args[]) throws Throwable {
         BMITestRunner.runTests(BzhiI2LCommutativeExpr.class, args,
                                "-XX:+IgnoreUnrecognizedVMOptions",
                                "-XX:+UseBMI2Instructions");
+        BMITestRunner.runTests(BzhiIExpr.class, args,
+                               "-XX:+IgnoreUnrecognizedVMOptions",
+                               "-XX:+UseBMI2Instructions");
+        BMITestRunner.runTests(BzhiICommutativeExpr.class, args,
+                               "-XX:+IgnoreUnrecognizedVMOptions",
+                               "-XX:+UseBMI2Instructions");
+        BMITestRunner.runTests(BzhiIBoundedExpr.class, args,
+                               "-XX:+IgnoreUnrecognizedVMOptions",
+                               "-XX:+UseBMI2Instructions");
     }
 
     public static class BzhiI2LExpr extends Expr.BMIUnaryIntToLongExpr {
@@ -93,4 +103,61 @@ public long intToLongExpr(int src1) {
             return returnValue;
         }
     }
+
+    public static class BzhiIExpr extends Expr.BMIBinaryIntExpr {
+
+        public int intExpr(int value, int bits) {
+            return value & ((1 << bits) - 1);
+        }
+
+        public int intExpr(int value, Expr.MemI bits) {
+            return value & ((1 << bits.value) - 1);
+        }
+
+        public int intExpr(Expr.MemI value, int bits) {
+            return value.value & ((1 << bits) - 1);
+        }
+
+        public int intExpr(Expr.MemI value, Expr.MemI bits) {
+            return value.value & ((1 << bits.value) - 1);
+        }
+    }
+
+    public static class BzhiICommutativeExpr extends Expr.BMIBinaryIntExpr {
+
+        public int intExpr(int value, int bits) {
+            return ((1 << bits) - 1) & value;
+        }
+
+        public int intExpr(int value, Expr.MemI bits) {
+            return ((1 << bits.value) - 1) & value;
+        }
+
+        public int intExpr(Expr.MemI value, int bits) {
+            return ((1 << bits) - 1) & value.value;
+        }
+
+        public int intExpr(Expr.MemI value, Expr.MemI bits) {
+            return ((1 << bits.value) - 1) & value.value;
+        }
+    }
+
+    public static class BzhiIBoundedExpr extends Expr.BMIBinaryIntExpr {
+
+        public int intExpr(int value, int bits) {
+            return value & ((1 << (bits & 31)) - 1);
+        }
+
+        public int intExpr(int value, Expr.MemI bits) {
+            return value & ((1 << (bits.value & 31)) - 1);
+        }
+
+        public int intExpr(Expr.MemI value, int bits) {
+            return value.value & ((1 << (bits & 31)) - 1);
+        }
+
+        public int intExpr(Expr.MemI value, Expr.MemI bits) {
+            return value.value & ((1 << (bits.value & 31)) - 1);
+        }
+    }
 }

test/hotspot/jtreg/compiler/intrinsics/bmi/TestBzhiIR.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+/**
+ * @test
+ * @bug 8389111
+ * @summary Verify BZHI matching for bounded and unrestricted int shift counts
+ * @requires vm.simpleArch == "x64" & vm.compiler2.enabled
+ * @library /test/lib /
+ * @run driver compiler.intrinsics.bmi.TestBzhiIR
+ */
+
+package compiler.intrinsics.bmi;
+
+import compiler.lib.ir_framework.Argument;
+import compiler.lib.ir_framework.Arguments;
+import compiler.lib.ir_framework.CompilePhase;
+import compiler.lib.ir_framework.IR;
+import compiler.lib.ir_framework.Test;
+import compiler.lib.ir_framework.TestFramework;
+
+public class TestBzhiIR {
+
+    public static void main(String[] args) {
+        TestFramework.runWithFlags("-XX:+IgnoreUnrecognizedVMOptions",
+                                   "-XX:+UseBMI2Instructions");
+    }
+
+    @Test
+    @Arguments(values = {Argument.RANDOM_EACH, Argument.RANDOM_EACH})
+    @IR(counts = {"bzhiI_rReg_rReg\\b", "1"},
+        failOn = {"bzhiI_rReg_rReg_bounded"},
+        phase = CompilePhase.MATCHING,
+        applyIfCPUFeature = {"bmi2", "true"})
+    public static int testUnrestricted(int value, int bits) {
+        return value & ((1 << bits) - 1);
+    }
+
+    @Test
+    @Arguments(values = {Argument.RANDOM_EACH, Argument.RANDOM_EACH})
+    @IR(counts = {"bzhiI_rReg_rReg_bounded", "1"},
+        failOn = {"bzhiI_rReg_rReg\\b"},
+        phase = CompilePhase.MATCHING,
+        applyIfCPUFeature = {"bmi2", "true"})
+    public static int testBounded(int value, int bits) {
+        return value & ((1 << (bits & 31)) - 1);
+    }
+}

test/hotspot/jtreg/compiler/intrinsics/bmi/verifycode/BzhiTestI2L.java

@@ -23,6 +23,7 @@
 
 /*
  * @test
+ * @bug 8389111
  * @requires vm.simpleArch == "x64" & vm.flavor == "server"
  * @library /test/lib /
  * @modules java.base/jdk.internal.misc
@@ -44,6 +45,46 @@
 
 public class BzhiTestI2L extends BmiIntrinsicBase.BmiTestCase_x64 {
 
+    private static class BzhiTestI extends BmiIntrinsicBase.BmiTestCase {
+
+        protected BzhiTestI(Method method) {
+            super(method);
+
+            cpuFlag = "bmi2";
+            vmFlag = "UseBMI2Instructions";
+
+            // From the Intel manual: VEX.LZ.0F38.W0 F5 /r.
+            instrMask = new byte[] {
+                    (byte) 0xFF,
+                    (byte) 0x1F,
+                    (byte) 0x80,
+                    (byte) 0xFF
+            };
+            instrPattern = new byte[] {
+                    (byte) 0xC4,
+                    (byte) 0x02,
+                    (byte) 0x00,
+                    (byte) 0xF5
+            };
+
+            // From the Intel APX specification: EVEX.128.NP.0F38.W0 F5 /r.
+            instrMaskAPX = new byte[] {
+                    (byte) 0xFF,
+                    (byte) 0x07,
+                    (byte) 0x00,
+                    (byte) 0x00,
+                    (byte) 0xFF
+            };
+            instrPatternAPX = new byte[] {
+                    (byte) 0x62,
+                    (byte) 0x02,
+                    (byte) 0x00,
+                    (byte) 0x00,
+                    (byte) 0xF5
+            };
+        }
+    }
+
     protected BzhiTestI2L(Method method) {
         super(method);
 
@@ -93,5 +134,11 @@ protected BzhiTestI2L(Method method) {
     public static void main(String[] args) throws Exception {
         BmiIntrinsicBase.verifyTestCase(BzhiTestI2L::new, TestBzhiI2L.BzhiI2LExpr.class.getDeclaredMethods());
         BmiIntrinsicBase.verifyTestCase(BzhiTestI2L::new, TestBzhiI2L.BzhiI2LCommutativeExpr.class.getDeclaredMethods());
+        BmiIntrinsicBase.verifyTestCase(BzhiTestI::new,
+                TestBzhiI2L.BzhiIExpr.class.getDeclaredMethod("intExpr", int.class, int.class));
+        BmiIntrinsicBase.verifyTestCase(BzhiTestI::new,
+                TestBzhiI2L.BzhiICommutativeExpr.class.getDeclaredMethod("intExpr", int.class, int.class));
+        BmiIntrinsicBase.verifyTestCase(BzhiTestI::new,
+                TestBzhiI2L.BzhiIBoundedExpr.class.getDeclaredMethod("intExpr", int.class, int.class));
     }
 }