Skip to content

Commit ac77858

Browse files
Yudis-bitYudis-bit
committed
scalar: Use branchless 258-bit to 256-bit reduction in 4x64 reduce_512
Replace the 258-to-256-bit reduction and conditional overflow reduction with the branchless sequence from PR 453. By pre-incrementing p4, an extra SECP256K1_N_C is folded into the scalar. Because X < 2N for any 258-bit intermediate in reduce_512, the resulting carry out of limb 3 is strictly bounded to {0, 1} and acts as a canonicality selector. Decrementing p4 converts carry 0 into UINT64_MAX (adding N in the recovery pass) and carry 1 into 0 (no correction needed). The recovery pass and limb assignments are shared between the x86_64 assembly and portable C paths using secp256k1_uint128, eliminating inline assembly Phase 3 and secp256k1_scalar_reduce entirely.
1 parent 06ef94a commit ac77858

1 file changed

Lines changed: 52 additions & 73 deletions

File tree

‎src/scalar_4x64_impl.h‎

Lines changed: 52 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -330,11 +330,11 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) {
330330
}
331331

332332
SECP256K1_FORCE_INLINE static void secp256k1_scalar_reduce_512(secp256k1_scalar *r, const uint64_t *l) {
333+
secp256k1_uint128 acc;
334+
uint64_t p0, p1, p2, p3, p4;
333335
#ifdef USE_ASM_X86_64
334336
/* Reduce 512 bits into 385. */
335337
uint64_t m0, m1, m2, m3, m4, m5, m6;
336-
uint64_t p0, p1, p2, p3, p4;
337-
uint64_t c;
338338

339339
__asm__ __volatile__(
340340
/* Preload. */
@@ -534,64 +534,10 @@ SECP256K1_FORCE_INLINE static void secp256k1_scalar_reduce_512(secp256k1_scalar
534534
SECP256K1_CHECKMEM_MSAN_DEFINE(&p3, sizeof(p3));
535535
SECP256K1_CHECKMEM_MSAN_DEFINE(&p4, sizeof(p4));
536536

537-
/* Reduce 258 bits into 256. */
538-
__asm__ __volatile__(
539-
/* Preload */
540-
"movq %q5, %%r10\n"
541-
/* (rax,rdx) = p4 * c0 */
542-
"movq %7, %%rax\n"
543-
"mulq %%r10\n"
544-
/* (rax,rdx) += p0 */
545-
"addq %q1, %%rax\n"
546-
"adcq $0, %%rdx\n"
547-
/* extract r0 */
548-
"movq %%rax, 0(%q6)\n"
549-
/* Move to (r8,r9) */
550-
"movq %%rdx, %%r8\n"
551-
"xorq %%r9, %%r9\n"
552-
/* (r8,r9) += p1 */
553-
"addq %q2, %%r8\n"
554-
"adcq $0, %%r9\n"
555-
/* (r8,r9) += p4 * c1 */
556-
"movq %8, %%rax\n"
557-
"mulq %%r10\n"
558-
"addq %%rax, %%r8\n"
559-
"adcq %%rdx, %%r9\n"
560-
/* Extract r1 */
561-
"movq %%r8, 8(%q6)\n"
562-
"xorq %%r8, %%r8\n"
563-
/* (r9,r8) += p4 */
564-
"addq %%r10, %%r9\n"
565-
"adcq $0, %%r8\n"
566-
/* (r9,r8) += p2 */
567-
"addq %q3, %%r9\n"
568-
"adcq $0, %%r8\n"
569-
/* Extract r2 */
570-
"movq %%r9, 16(%q6)\n"
571-
"xorq %%r9, %%r9\n"
572-
/* (r8,r9) += p3 */
573-
"addq %q4, %%r8\n"
574-
"adcq $0, %%r9\n"
575-
/* Extract r3 */
576-
"movq %%r8, 24(%q6)\n"
577-
/* Extract c */
578-
"movq %%r9, %q0\n"
579-
: "=g"(c)
580-
: "g"(p0), "g"(p1), "g"(p2), "g"(p3), "g"(p4), "D"(r), "i"(SECP256K1_N_C_0), "i"(SECP256K1_N_C_1)
581-
: "rax", "rdx", "r8", "r9", "r10", "cc", "memory");
582-
583-
SECP256K1_CHECKMEM_MSAN_DEFINE(r, sizeof(*r));
584-
SECP256K1_CHECKMEM_MSAN_DEFINE(&c, sizeof(c));
585-
586537
#else
587-
secp256k1_uint128 c128;
588-
uint64_t c;
589-
secp256k1_uint128 acc;
590538
uint64_t n0 = l[4], n1 = l[5], n2 = l[6], n3 = l[7];
591539
uint64_t m0, m1, m2, m3, m4, m5;
592540
uint32_t m6;
593-
uint64_t p0, p1, p2, p3;
594-
uint32_t p4;
595541

596542
/* Reduce 512 bits into 385. */
597543
/* m[0..6] = l[0..3] + n[0..3] * SECP256K1_N_C. */
@@ -642,25 +588,58 @@ SECP256K1_FORCE_INLINE static void secp256k1_scalar_reduce_512(secp256k1_scalar
642588
extract_fast(p4);
643589
p4 += m6;
644590
VERIFY_CHECK(p4 <= 2);
645-
646-
/* Reduce 258 bits into 256. */
647-
/* r[0..3] = p[0..3] + p[4] * SECP256K1_N_C. */
648-
secp256k1_u128_from_u64(&c128, p0);
649-
secp256k1_u128_accum_mul(&c128, SECP256K1_N_C_0, p4);
650-
r->d[0] = secp256k1_u128_to_u64(&c128); secp256k1_u128_rshift(&c128, 64);
651-
secp256k1_u128_accum_u64(&c128, p1);
652-
secp256k1_u128_accum_mul(&c128, SECP256K1_N_C_1, p4);
653-
r->d[1] = secp256k1_u128_to_u64(&c128); secp256k1_u128_rshift(&c128, 64);
654-
secp256k1_u128_accum_u64(&c128, p2);
655-
secp256k1_u128_accum_u64(&c128, p4);
656-
r->d[2] = secp256k1_u128_to_u64(&c128); secp256k1_u128_rshift(&c128, 64);
657-
secp256k1_u128_accum_u64(&c128, p3);
658-
r->d[3] = secp256k1_u128_to_u64(&c128);
659-
c = secp256k1_u128_hi_u64(&c128);
660591
#endif
661592

662-
/* Final reduction of r. */
663-
secp256k1_scalar_reduce(r, c + secp256k1_scalar_check_overflow(r));
593+
/* R = p[0..3] + p4*N_C is congruent to X = p[0..3] + p4*2^256 mod N,
594+
* and p4 <= 2 gives R < 2N. Dropping p4*2^256 and adding (p4+1)*N_C
595+
* computes R + N_C, whose carry out of limb 3 is 1 iff R >= N, leaving
596+
* R - N; otherwise adding N back leaves R. */
597+
++p4;
598+
599+
/* Reduce 258 bits into 256 using the same 128-bit accumulator. */
600+
secp256k1_u128_mul(&acc, p4, SECP256K1_N_C_0);
601+
secp256k1_u128_accum_u64(&acc, p0);
602+
p0 = secp256k1_u128_to_u64(&acc);
603+
secp256k1_u128_rshift(&acc, 64);
604+
secp256k1_u128_accum_mul(&acc, p4, SECP256K1_N_C_1);
605+
secp256k1_u128_accum_u64(&acc, p1);
606+
p1 = secp256k1_u128_to_u64(&acc);
607+
secp256k1_u128_rshift(&acc, 64);
608+
secp256k1_u128_accum_u64(&acc, p4);
609+
secp256k1_u128_accum_u64(&acc, p2);
610+
p2 = secp256k1_u128_to_u64(&acc);
611+
secp256k1_u128_rshift(&acc, 64);
612+
secp256k1_u128_accum_u64(&acc, p3);
613+
p3 = secp256k1_u128_to_u64(&acc);
614+
secp256k1_u128_rshift(&acc, 64);
615+
p4 = secp256k1_u128_to_u64(&acc);
616+
VERIFY_CHECK(p4 <= 1);
617+
618+
/* uint64_t underflow gives UINT64_MAX for carry 0, and 0 for carry 1. */
619+
--p4;
620+
secp256k1_u128_from_u64(&acc, p0);
621+
secp256k1_u128_accum_u64(&acc, p4 & SECP256K1_N_0);
622+
p0 = secp256k1_u128_to_u64(&acc);
623+
secp256k1_u128_rshift(&acc, 64);
624+
secp256k1_u128_accum_u64(&acc, p1);
625+
secp256k1_u128_accum_u64(&acc, p4 & SECP256K1_N_1);
626+
p1 = secp256k1_u128_to_u64(&acc);
627+
secp256k1_u128_rshift(&acc, 64);
628+
secp256k1_u128_accum_u64(&acc, p2);
629+
secp256k1_u128_accum_u64(&acc, p4 & SECP256K1_N_2);
630+
p2 = secp256k1_u128_to_u64(&acc);
631+
secp256k1_u128_rshift(&acc, 64);
632+
secp256k1_u128_accum_u64(&acc, p3);
633+
secp256k1_u128_accum_u64(&acc, p4 & SECP256K1_N_3);
634+
p3 = secp256k1_u128_to_u64(&acc);
635+
secp256k1_u128_rshift(&acc, 64);
636+
VERIFY_CHECK(secp256k1_u128_to_u64(&acc) + p4 == 0);
637+
638+
r->d[0] = p0;
639+
r->d[1] = p1;
640+
r->d[2] = p2;
641+
r->d[3] = p3;
642+
SECP256K1_SCALAR_VERIFY(r);
664643
}
665644

666645
SECP256K1_FORCE_INLINE static void secp256k1_scalar_mul_512(uint64_t *l8, const secp256k1_scalar *a, const secp256k1_scalar *b) {

0 commit comments

Comments
 (0)