OpenSSL CVE-2022-0778 Fix possible infinite loop in BN_mod_sqrt() (#423)

OpenSSL CVE-2022-0778 Fix possible infinite loop in BN_mod_sqrt()

This commit fixes the issue released as OpenSSL CVE-2022-0778 that
affects AWS-LC as well. A bug in BN_mod_sqrt() can cause the function
to enter an infinite loop. The issue is now fixed and two test
cases are added to verify that the function returns a failure instead
of hanging.

Co-authored-by: Dusan Kostic <[email protected]>

Author: dkostic

crypto/fipsmodule/bn/bn_tests.txt

@@ -11149,7 +11149,8 @@ P = 9df9d6cc20b8540411af4e5357ef2b0353cb1f2ab5ffc3e246b41c32f71e951f
 
 # NotModSquare tests.
 #
-# These test vectors are such that NotModSquare is not a square modulo P.
+# These test vectors are such that NotModSquare is not a square modulo P or
+# P is not a prime number so BN_mod_sqrt function can't compute correctly.
 
 NotModSquare = 03
 P = 07
@@ -11163,6 +11164,11 @@ P = 07
 NotModSquare = 9df9d6cc20b8540411af4e5357ef2b0353cb1f2ab5ffc3e246b41c32f71e951e
 P = 9df9d6cc20b8540411af4e5357ef2b0353cb1f2ab5ffc3e246b41c32f71e951f
 
+NotModSquare = 20a7ee
+P = 460201
+
+NotModSquare = 65bebdb00a96fc814ec44b81f98b59fba3c30203928fa5214c51e0a97091645280c947b005847f239758482b9bfc45b066fde340d1fe32fc9c1bf02e1b2d0ed
+P = 9df9d6cc20b8540411af4e5357ef2b0353cb1f2ab5ffc3e246b41c32f71e951f
 
 # ModInv tests.
 #

crypto/fipsmodule/bn/sqrt.c

@@ -62,7 +62,8 @@
 BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) {
   // Compute a square root of |a| mod |p| using the Tonelli/Shanks algorithm
   // (cf. Henri Cohen, "A Course in Algebraic Computational Number Theory",
-  // algorithm 1.5.1). |p| is assumed to be a prime.
+  // algorithm 1.5.1). |p| must be prime, otherwise an error or
+  // an incorrect "result" will be returned.
 
   BIGNUM *ret = in;
   int err = 1;
@@ -359,23 +360,26 @@ BIGNUM *BN_mod_sqrt(BIGNUM *in, const BIGNUM *a, const BIGNUM *p, BN_CTX *ctx) {
       goto vrfy;
     }
 
-
-    // find smallest  i  such that  b^(2^i) = 1
-    i = 1;
-    if (!BN_mod_sqr(t, b, p, ctx)) {
-      goto end;
-    }
-    while (!BN_is_one(t)) {
-      i++;
-      if (i == e) {
-        OPENSSL_PUT_ERROR(BN, BN_R_NOT_A_SQUARE);
-        goto end;
+    // Find the smallest i, 0 < i < e, such that b^(2^i) = 1.
+    for (i = 1; i < e; i++) {
+      if (i == 1) {
+        if (!BN_mod_sqr(t, b, p, ctx)) {
+          goto end;
+        }
+      } else {
+        if (!BN_mod_mul(t, t, t, p, ctx)) {
+          goto end;
+        }
       }
-      if (!BN_mod_mul(t, t, t, p, ctx)) {
-        goto end;
+      if (BN_is_one(t)) {
+        break;
       }
     }
-
+    // If not found, a is not a square or p is not prime.
+    if (i >= e) {
+      OPENSSL_PUT_ERROR(BN, BN_R_NOT_A_SQUARE);
+      goto end;
+    }
 
     // t := y^2^(e - i - 1)
     if (!BN_copy(t, y)) {