JIT: fix profile likelihood and overflow UB in optimizebools

Fix two issues in optOptimizeBoolsUpdateTrees and GetIntersection:

1. The !sameTarget branch likelihood formula was computing
   (1-p1) + p1*p2_false = 1 - p1*p2_true, but the correct
   probability of reaching B2's true target is
   (1-p1) * p2_true (B1 falls through, then B2 jumps).
   Also fixed the misleading comment.

2. The normalize lambda in GetIntersection could overflow
   when cns == SSIZE_T_MAX and cmp == GT_GT (signed overflow
   is UB in C++). Added an explicit guard and made the lambda
   return bool to signal failure.

3. Updated a stale comment referencing the nonexistent
   GTF_BOOLEAN flag in optIsBoolComp.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: 2 people

src/coreclr/jit/optimizebools.cpp

@@ -448,23 +448,30 @@ static bool GetIntersection(var_types  type,
     }
 
     // Convert to a canonical form with GT_GE or GT_LE (inclusive).
-    auto normalize = [](genTreeOps* cmp, ssize_t* cns) {
+    auto normalize = [](genTreeOps* cmp, ssize_t* cns) -> bool {
         if (*cmp == GT_GT)
         {
             // "X > cns" -> "X >= cns + 1"
+            if (*cns == SSIZE_T_MAX)
+            {
+                return false;
+            }
             *cns = *cns + 1;
             *cmp = GT_GE;
         }
         if (*cmp == GT_LT)
         {
             // "X < cns" -> "X <= cns - 1"
+            // cns is non-negative (checked above), so cns - 1 >= -1 and cannot underflow.
             *cns = *cns - 1;
             *cmp = GT_LE;
         }
-        // whether these overflow or not is checked below.
+        return true;
     };
-    normalize(&cmp1, &cns1);
-    normalize(&cmp2, &cns2);
+    if (!normalize(&cmp1, &cns1) || !normalize(&cmp2, &cns2))
+    {
+        return false;
+    }
 
     if (cmp1 == cmp2)
     {
@@ -1234,8 +1241,8 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees()
         }
         else
         {
-            // We originally reached B2's true target via
-            // B1 false OR B1 true B2 false.
+            // We originally reached B2's true target only via
+            // B1 false, then B2 true.
             //
             // We will now reach via B1 true.
             // Modify flow for true side of B1
@@ -1244,7 +1251,7 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees()
             origB1TrueEdge->setHeuristicBased(origB2TrueEdge->isHeuristicBased());
 
             newB1TrueLikelihood =
-                (1.0 - origB1TrueLikelihood) + origB1TrueLikelihood * origB2FalseEdge->getLikelihood();
+                (1.0 - origB1TrueLikelihood) * origB2TrueEdge->getLikelihood();
         }
 
         // Fix B1 true edge likelihood
@@ -1412,9 +1419,9 @@ GenTree* OptBoolsDsc::optIsBoolComp(OptTestInfo* pOptTest)
     ssize_t ival2 = opr2->AsIntCon()->gtIconVal;
 
     // Is the value a boolean?
-    // We can either have a boolean expression (marked GTF_BOOLEAN) or a constant 0/1.
+    // Currently we only recognize constant 0/1.
 
-    if (opr1->OperIs(GT_CNS_INT) && (opr1->IsIntegralConst(0) || opr1->IsIntegralConst(1)))
+    if (opr1->IsIntegralConst(0) || opr1->IsIntegralConst(1))
     {
         pOptTest->isBool = true;
     }