ec/suite_b: Thread cpu through scalar inverse.

Author: briansmith

src/ec/suite_b/ops/p256.rs

@@ -178,7 +178,7 @@ pub static PRIVATE_SCALAR_OPS: PrivateScalarOps = PrivateScalarOps {
 };
 
 #[allow(clippy::just_underscores_and_digits)]
-fn p256_scalar_inv_to_mont(a: Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
+fn p256_scalar_inv_to_mont(a: Scalar<R>, cpu: cpu::Features) -> Scalar<R> {
     // Calculate the modular inverse of scalar |a| using Fermat's Little
     // Theorem:
     //
@@ -189,27 +189,34 @@ fn p256_scalar_inv_to_mont(a: Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
     //    0xffffffff00000000ffffffffffffffffbce6faada7179e84f3b9cac2fc63254f
 
     #[inline]
-    fn mul(a: &Scalar<R>, b: &Scalar<R>) -> Scalar<R> {
+    fn mul(a: &Scalar<R>, b: &Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
         binary_op(p256_scalar_mul_mont, a, b)
     }
 
     #[inline]
-    fn sqr(a: &Scalar<R>) -> Scalar<R> {
+    fn sqr(a: &Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
         let mut tmp = Scalar::zero();
         unsafe { p256_scalar_sqr_rep_mont(tmp.limbs.as_mut_ptr(), a.limbs.as_ptr(), 1) }
         tmp
     }
 
     // Returns (`a` squared `squarings` times) * `b`.
-    fn sqr_mul(a: &Scalar<R>, squarings: LeakyWord, b: &Scalar<R>) -> Scalar<R> {
+    #[inline]
+    fn sqr_mul(
+        a: &Scalar<R>,
+        squarings: LeakyWord,
+        b: &Scalar<R>,
+        cpu: cpu::Features,
+    ) -> Scalar<R> {
         debug_assert!(squarings >= 1);
         let mut tmp = Scalar::zero();
         unsafe { p256_scalar_sqr_rep_mont(tmp.limbs.as_mut_ptr(), a.limbs.as_ptr(), squarings) }
-        mul(&tmp, b)
+        mul(&tmp, b, cpu)
     }
 
     // Sets `acc` = (`acc` squared `squarings` times) * `b`.
-    fn sqr_mul_acc(acc: &mut Scalar<R>, squarings: LeakyWord, b: &Scalar<R>) {
+    #[inline]
+    fn sqr_mul_acc(acc: &mut Scalar<R>, squarings: LeakyWord, b: &Scalar<R>, _cpu: cpu::Features) {
         debug_assert!(squarings >= 1);
         {
             let acc = acc.limbs.as_mut_ptr();
@@ -220,60 +227,60 @@ fn p256_scalar_inv_to_mont(a: Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
 
     let _1 = &a;
 
-    let _10 = sqr(_1); // 2
-    let _100 = sqr(&_10); // 4
-    let _101 = mul(&_100, _1); // 5
-    let _111 = mul(&_101, &_10); // 7
-
-    let _1000 = sqr(&_100); // 8
-    let _10000 = sqr(&_1000); // 16
-    let _100000 = sqr(&_10000); // 32
-
-    let _100111 = mul(&_111, &_100000); // 39 = 7 + 32
-    let _101011 = mul(&_100, &_100111); // 43 = 4 + 39
-    let _101111 = mul(&_100, &_101011); // 47 = 4 + 39
-    let _1001111 = mul(&_100000, &_101111); // 79 = 32 + 47
-    let _86 = sqr(&_101011); // 86 = 43 * 2
-    let _1011011 = mul(&_101, &_86); // 91 = 5 + 86
-    let _92 = mul(_1, &_1011011); // 92 = 1 + 91
-    let _1100011 = mul(&_111, &_92); // 99 = 7 + 92
-    let _10111111 = mul(&_92, &_1100011); // 191 = 92 + 99
-    let _11011111 = mul(&_100000, &_10111111); // 223 = 32 + 191
-
-    let ff = mul(&_100000, &_11011111); // 255 = 32 + 223
-    let ffff = sqr_mul(&ff, 0 + 8, &ff);
-    let ffffffff = sqr_mul(&ffff, 0 + 16, &ffff);
+    let _10 = sqr(_1, cpu); // 2
+    let _100 = sqr(&_10, cpu); // 4
+    let _101 = mul(&_100, _1, cpu); // 5
+    let _111 = mul(&_101, &_10, cpu); // 7
+
+    let _1000 = sqr(&_100, cpu); // 8
+    let _10000 = sqr(&_1000, cpu); // 16
+    let _100000 = sqr(&_10000, cpu); // 32
+
+    let _100111 = mul(&_111, &_100000, cpu); // 39 = 7 + 32
+    let _101011 = mul(&_100, &_100111, cpu); // 43 = 4 + 39
+    let _101111 = mul(&_100, &_101011, cpu); // 47 = 4 + 39
+    let _1001111 = mul(&_100000, &_101111, cpu); // 79 = 32 + 47
+    let _86 = sqr(&_101011, cpu); // 86 = 43 * 2
+    let _1011011 = mul(&_101, &_86, cpu); // 91 = 5 + 86
+    let _92 = mul(_1, &_1011011, cpu); // 92 = 1 + 91
+    let _1100011 = mul(&_111, &_92, cpu); // 99 = 7 + 92
+    let _10111111 = mul(&_92, &_1100011, cpu); // 191 = 92 + 99
+    let _11011111 = mul(&_100000, &_10111111, cpu); // 223 = 32 + 191
+
+    let ff = mul(&_100000, &_11011111, cpu); // 255 = 32 + 223
+    let ffff = sqr_mul(&ff, 0 + 8, &ff, cpu);
+    let ffffffff = sqr_mul(&ffff, 0 + 16, &ffff, cpu);
 
     // ffffffff00000000ffffffff
-    let mut acc = sqr_mul(&ffffffff, 32 + 32, &ffffffff);
+    let mut acc = sqr_mul(&ffffffff, 32 + 32, &ffffffff, cpu);
 
     // ffffffff00000000ffffffffffffffff
-    sqr_mul_acc(&mut acc, 0 + 32, &ffffffff);
+    sqr_mul_acc(&mut acc, 0 + 32, &ffffffff, cpu);
 
     // The rest of the exponent, in binary, is:
     //
     //    1011110011100110111110101010110110100111000101111001111010000100
     //    1111001110111001110010101100001011111100011000110010010101001111
 
-    sqr_mul_acc(&mut acc, 6, &_101111);
-    sqr_mul_acc(&mut acc, 2 + 3, &_111);
-    sqr_mul_acc(&mut acc, 2 + 8, &_11011111);
-    sqr_mul_acc(&mut acc, 1 + 3, &_101);
-    sqr_mul_acc(&mut acc, 1 + 7, &_1011011);
-    sqr_mul_acc(&mut acc, 1 + 6, &_100111);
-    sqr_mul_acc(&mut acc, 3 + 6, &_101111);
-    sqr_mul_acc(&mut acc, 2 + 3, &_111);
-    sqr_mul_acc(&mut acc, 3, &_101);
-    sqr_mul_acc(&mut acc, 4 + 7, &_1001111);
-    sqr_mul_acc(&mut acc, 2 + 3, &_111);
-    sqr_mul_acc(&mut acc, 1 + 3, &_111);
-    sqr_mul_acc(&mut acc, 2 + 3, &_111);
-    sqr_mul_acc(&mut acc, 2 + 6, &_101011);
-    sqr_mul_acc(&mut acc, 4 + 8, &_10111111);
-    sqr_mul_acc(&mut acc, 3 + 7, &_1100011);
-    sqr_mul_acc(&mut acc, 2 + 1, _1);
-    sqr_mul_acc(&mut acc, 2 + 3, &_101);
-    sqr_mul_acc(&mut acc, 1 + 7, &_1001111);
+    sqr_mul_acc(&mut acc, 6, &_101111, cpu);
+    sqr_mul_acc(&mut acc, 2 + 3, &_111, cpu);
+    sqr_mul_acc(&mut acc, 2 + 8, &_11011111, cpu);
+    sqr_mul_acc(&mut acc, 1 + 3, &_101, cpu);
+    sqr_mul_acc(&mut acc, 1 + 7, &_1011011, cpu);
+    sqr_mul_acc(&mut acc, 1 + 6, &_100111, cpu);
+    sqr_mul_acc(&mut acc, 3 + 6, &_101111, cpu);
+    sqr_mul_acc(&mut acc, 2 + 3, &_111, cpu);
+    sqr_mul_acc(&mut acc, 3, &_101, cpu);
+    sqr_mul_acc(&mut acc, 4 + 7, &_1001111, cpu);
+    sqr_mul_acc(&mut acc, 2 + 3, &_111, cpu);
+    sqr_mul_acc(&mut acc, 1 + 3, &_111, cpu);
+    sqr_mul_acc(&mut acc, 2 + 3, &_111, cpu);
+    sqr_mul_acc(&mut acc, 2 + 6, &_101011, cpu);
+    sqr_mul_acc(&mut acc, 4 + 8, &_10111111, cpu);
+    sqr_mul_acc(&mut acc, 3 + 7, &_1100011, cpu);
+    sqr_mul_acc(&mut acc, 2 + 1, _1, cpu);
+    sqr_mul_acc(&mut acc, 2 + 3, &_101, cpu);
+    sqr_mul_acc(&mut acc, 1 + 7, &_1001111, cpu);
 
     acc
 }

src/ec/suite_b/ops/p384.rs

@@ -140,7 +140,7 @@ pub static PRIVATE_SCALAR_OPS: PrivateScalarOps = PrivateScalarOps {
     scalar_inv_to_mont: p384_scalar_inv_to_mont,
 };
 
-fn p384_scalar_inv_to_mont(a: Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
+fn p384_scalar_inv_to_mont(a: Scalar<R>, cpu: cpu::Features) -> Scalar<R> {
     // Calculate the modular inverse of scalar |a| using Fermat's Little
     // Theorem:
     //
@@ -151,34 +151,40 @@ fn p384_scalar_inv_to_mont(a: Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
     //    0xffffffffffffffffffffffffffffffffffffffffffffffffc7634d81f4372ddf\
     //      581a0db248b0a77aecec196accc52971
 
-    fn mul(a: &Scalar<R>, b: &Scalar<R>) -> Scalar<R> {
+    fn mul(a: &Scalar<R>, b: &Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
         binary_op(p384_scalar_mul_mont, a, b)
     }
 
-    fn sqr(a: &Scalar<R>) -> Scalar<R> {
+    fn sqr(a: &Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
         binary_op(p384_scalar_mul_mont, a, a)
     }
 
-    fn sqr_mut(a: &mut Scalar<R>) {
+    fn sqr_mut(a: &mut Scalar<R>, _cpu: cpu::Features) {
         unary_op_from_binary_op_assign(p384_scalar_mul_mont, a);
     }
 
     // Returns (`a` squared `squarings` times) * `b`.
-    fn sqr_mul(a: &Scalar<R>, squarings: LeakyWord, b: &Scalar<R>) -> Scalar<R> {
+    fn sqr_mul(
+        a: &Scalar<R>,
+        squarings: LeakyWord,
+        b: &Scalar<R>,
+        cpu: cpu::Features,
+    ) -> Scalar<R> {
         debug_assert!(squarings >= 1);
-        let mut tmp = sqr(a);
+        let mut tmp = sqr(a, cpu);
         for _ in 1..squarings {
-            sqr_mut(&mut tmp);
+            sqr_mut(&mut tmp, cpu);
         }
-        mul(&tmp, b)
+        mul(&tmp, b, cpu)
     }
 
     // Sets `acc` = (`acc` squared `squarings` times) * `b`.
-    fn sqr_mul_acc(acc: &mut Scalar<R>, squarings: LeakyWord, b: &Scalar<R>) {
+    fn sqr_mul_acc(acc: &mut Scalar<R>, squarings: LeakyWord, b: &Scalar<R>, cpu: cpu::Features) {
         debug_assert!(squarings >= 1);
         for _ in 0..squarings {
-            sqr_mut(acc);
+            sqr_mut(acc, cpu);
         }
+        let _: cpu::Features = cpu;
         binary_op_assign(p384_scalar_mul_mont, acc, b)
     }
 
@@ -195,21 +201,26 @@ fn p384_scalar_inv_to_mont(a: Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
 
     let mut d = [Scalar::zero(); DIGIT_COUNT];
     d[B_1] = a;
-    let b_10 = sqr(&d[B_1]);
+    let b_10 = sqr(&d[B_1], cpu);
     for i in B_11..DIGIT_COUNT {
-        d[i] = mul(&d[i - 1], &b_10);
+        d[i] = mul(&d[i - 1], &b_10, cpu);
     }
 
-    let ff = sqr_mul(&d[B_1111], 0 + 4, &d[B_1111]);
-    let ffff = sqr_mul(&ff, 0 + 8, &ff);
-    let ffffffff = sqr_mul(&ffff, 0 + 16, &ffff);
+    let ff = sqr_mul(&d[B_1111], 0 + 4, &d[B_1111], cpu);
+    let ffff = sqr_mul(&ff, 0 + 8, &ff, cpu);
+    let ffffffff = sqr_mul(&ffff, 0 + 16, &ffff, cpu);
 
-    let ffffffffffffffff = sqr_mul(&ffffffff, 0 + 32, &ffffffff);
+    let ffffffffffffffff = sqr_mul(&ffffffff, 0 + 32, &ffffffff, cpu);
 
-    let ffffffffffffffffffffffff = sqr_mul(&ffffffffffffffff, 0 + 32, &ffffffff);
+    let ffffffffffffffffffffffff = sqr_mul(&ffffffffffffffff, 0 + 32, &ffffffff, cpu);
 
     // ffffffffffffffffffffffffffffffffffffffffffffffff
-    let mut acc = sqr_mul(&ffffffffffffffffffffffff, 0 + 96, &ffffffffffffffffffffffff);
+    let mut acc = sqr_mul(
+        &ffffffffffffffffffffffff,
+        0 + 96,
+        &ffffffffffffffffffffffff,
+        cpu,
+    );
 
     // The rest of the exponent, in binary, is:
     //
@@ -261,7 +272,12 @@ fn p384_scalar_inv_to_mont(a: Scalar<R>, _cpu: cpu::Features) -> Scalar<R> {
     ];
 
     for &(squarings, digit) in &REMAINING_WINDOWS[..] {
-        sqr_mul_acc(&mut acc, LeakyWord::from(squarings), &d[usize::from(digit)]);
+        sqr_mul_acc(
+            &mut acc,
+            LeakyWord::from(squarings),
+            &d[usize::from(digit)],
+            cpu,
+        );
     }
 
     acc