JIT: fix profile likelihood and overflow UB in optimizebools

src/coreclr/jit/optimizebools.cpp

@@ -448,23 +448,30 @@ static bool GetIntersection(var_types  type,
     }
 
     // Convert to a canonical form with GT_GE or GT_LE (inclusive).
-    auto normalize = [](genTreeOps* cmp, ssize_t* cns) {
+    auto normalize = [](genTreeOps* cmp, ssize_t* cns) -> bool {
         if (*cmp == GT_GT)
         {
             // "X > cns" -> "X >= cns + 1"
+            if (*cns == SSIZE_T_MAX)
+            {
+                return false;
+            }
             *cns = *cns + 1;
             *cmp = GT_GE;
         }
         if (*cmp == GT_LT)
         {
             // "X < cns" -> "X <= cns - 1"
+            // cns is non-negative (checked above), so cns - 1 >= -1 and cannot underflow.
             *cns = *cns - 1;
             *cmp = GT_LE;
         }
-        // whether these overflow or not is checked below.
+        return true;
     };
-    normalize(&cmp1, &cns1);
-    normalize(&cmp2, &cns2);
+    if (!normalize(&cmp1, &cns1) || !normalize(&cmp2, &cns2))
+    {
+        return false;
+    }
 
     if (cmp1 == cmp2)
     {
@@ -1234,17 +1241,16 @@ void OptBoolsDsc::optOptimizeBoolsUpdateTrees()
         }
         else
         {
-            // We originally reached B2's true target via
-            // B1 false OR B1 true B2 false.
+            // We originally reached B2's true target only via
+            // B1 false, then B2 true.
             //
             // We will now reach via B1 true.
             // Modify flow for true side of B1
             //
             m_compiler->fgRedirectEdge(m_b1->TrueEdgeRef(), m_b2->GetTrueTarget());
             origB1TrueEdge->setHeuristicBased(origB2TrueEdge->isHeuristicBased());
 
-            newB1TrueLikelihood =
-                (1.0 - origB1TrueLikelihood) + origB1TrueLikelihood * origB2FalseEdge->getLikelihood();
+            newB1TrueLikelihood = (1.0 - origB1TrueLikelihood) * origB2TrueEdge->getLikelihood();
         }
 
         // Fix B1 true edge likelihood
@@ -1412,9 +1418,9 @@ GenTree* OptBoolsDsc::optIsBoolComp(OptTestInfo* pOptTest)
     ssize_t ival2 = opr2->AsIntCon()->gtIconVal;
 
     // Is the value a boolean?
-    // We can either have a boolean expression (marked GTF_BOOLEAN) or a constant 0/1.
+    // Currently we only recognize constant 0/1.
 
-    if (opr1->OperIs(GT_CNS_INT) && (opr1->IsIntegralConst(0) || opr1->IsIntegralConst(1)))
+    if (opr1->IsIntegralConst(0) || opr1->IsIntegralConst(1))
     {
         pOptTest->isBool = true;
     }