Skip to content
This repository was archived by the owner on Dec 27, 2019. It is now read-only.

Commit b590279

Browse files
zx2c4zx2c4
committed
poly1305: account for simd being toggled off midway
This is a very rare occurance, but we should account for it, so that the calculations aren't wrong. Here we convert from base 2^26 back to base 2^64.
1 parent 972a910 commit b590279

3 files changed

Lines changed: 132 additions & 26 deletions

File tree

‎src/crypto/zinc/poly1305/poly1305-arm-glue.h‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,60 @@ static void __init poly1305_fpu_init(void)
2929
#endif
3030
}
3131

32+
#if defined(CONFIG_ARM64)
33+
struct poly1305_arch_internal {
34+
union {
35+
u32 h[5];
36+
struct {
37+
u64 h0, h1, h2;
38+
};
39+
};
40+
u32 is_base2_26;
41+
u64 r[2];
42+
};
43+
#elif defined(CONFIG_ARM)
44+
struct poly1305_arch_internal {
45+
union {
46+
u32 h[5];
47+
struct {
48+
u64 h0, h1;
49+
u32 h2;
50+
};
51+
};
52+
/* u32 r0 -- in the padding here */
53+
u32 r[3];
54+
u32 is_base2_26;
55+
};
56+
#endif
57+
58+
59+
#if defined(ARM_USE_NEON)
60+
static void convert_to_base2_64(void *ctx)
61+
{
62+
struct poly1305_arch_internal *state = ctx;
63+
u32 cy;
64+
65+
if (likely(!state->is_base2_26))
66+
return;
67+
68+
cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
69+
cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
70+
cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
71+
cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
72+
state->h0 = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
73+
state->h1 = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
74+
state->h2 = state->h[4] >> 24;
75+
#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
76+
cy = (state->h2 >> 2) + (state->h2 & ~3ULL);
77+
state->h2 &= 3;
78+
state->h0 += cy;
79+
state->h1 += (cy = ULT(state->h0, cy));
80+
state->h2 += ULT(state->h1, cy);
81+
#undef ULT
82+
state->is_base2_26 = 0;
83+
}
84+
#endif
85+
3286
static inline bool poly1305_init_arch(void *ctx,
3387
const u8 key[POLY1305_KEY_SIZE])
3488
{
@@ -45,7 +99,9 @@ static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
4599
poly1305_blocks_neon(ctx, inp, len, padbit);
46100
return true;
47101
}
102+
convert_to_base2_64(ctx);
48103
#endif
104+
49105
poly1305_blocks_arm(ctx, inp, len, padbit);
50106
return true;
51107
}
@@ -59,7 +115,9 @@ static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
59115
poly1305_emit_neon(ctx, mac, nonce);
60116
return true;
61117
}
118+
convert_to_base2_64(ctx);
62119
#endif
120+
63121
poly1305_emit_arm(ctx, mac, nonce);
64122
return true;
65123
}

‎src/crypto/zinc/poly1305/poly1305-x86_64-glue.h‎

Lines changed: 57 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -61,13 +61,42 @@ static inline bool poly1305_init_arch(void *ctx,
6161
}
6262

6363
struct poly1305_arch_internal {
64-
u32 h[5];
65-
u32 is_base2_26;
64+
union {
65+
struct {
66+
u32 h[5];
67+
u32 is_base2_26;
68+
};
69+
u64 hs[3];
70+
};
6671
u64 r[2];
6772
u64 pad;
6873
struct { u32 r2, r1, r4, r3; } rn[9];
6974
};
7075

76+
static void convert_to_base2_64(void *ctx)
77+
{
78+
struct poly1305_arch_internal *state = ctx;
79+
u32 cy;
80+
81+
if (likely(!state->is_base2_26))
82+
return;
83+
84+
cy = state->h[0] >> 26; state->h[0] &= 0x3ffffff; state->h[1] += cy;
85+
cy = state->h[1] >> 26; state->h[1] &= 0x3ffffff; state->h[2] += cy;
86+
cy = state->h[2] >> 26; state->h[2] &= 0x3ffffff; state->h[3] += cy;
87+
cy = state->h[3] >> 26; state->h[3] &= 0x3ffffff; state->h[4] += cy;
88+
state->hs[0] = ((u64)state->h[2] << 52) | ((u64)state->h[1] << 26) | state->h[0];
89+
state->hs[1] = ((u64)state->h[4] << 40) | ((u64)state->h[3] << 14) | (state->h[2] >> 12);
90+
state->hs[2] = state->h[4] >> 24;
91+
#define ULT(a, b) ((a ^ ((a ^ b) | ((a - b) ^ b))) >> (sizeof(a) * 8 - 1))
92+
cy = (state->hs[2] >> 2) + (state->hs[2] & ~3ULL);
93+
state->hs[2] &= 3;
94+
state->hs[0] += cy;
95+
state->hs[1] += (cy = ULT(state->hs[0], cy));
96+
state->hs[2] += ULT(state->hs[1], cy);
97+
#undef ULT
98+
}
99+
71100
static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
72101
const size_t len, const u32 padbit,
73102
simd_context_t *simd_context)
@@ -77,24 +106,32 @@ static inline bool poly1305_blocks_arch(void *ctx, const u8 *inp,
77106
if (!poly1305_use_avx ||
78107
(len < (POLY1305_BLOCK_SIZE * 18) && !state->is_base2_26) ||
79108
!simd_use(simd_context))
80-
poly1305_blocks_x86_64(ctx, inp, len, padbit);
81-
else
109+
goto scalar;
110+
82111
#ifdef CONFIG_AS_AVX512
83-
if (poly1305_use_avx512)
112+
if (poly1305_use_avx512) {
84113
poly1305_blocks_avx512(ctx, inp, len, padbit);
85-
else
114+
return true;
115+
}
86116
#endif
117+
87118
#ifdef CONFIG_AS_AVX2
88-
if (poly1305_use_avx2)
119+
if (poly1305_use_avx2) {
89120
poly1305_blocks_avx2(ctx, inp, len, padbit);
90-
else
121+
return true;
122+
}
91123
#endif
124+
92125
#ifdef CONFIG_AS_AVX
93-
if (poly1305_use_avx)
126+
if (poly1305_use_avx) {
94127
poly1305_blocks_avx(ctx, inp, len, padbit);
95-
else
128+
return true;
129+
}
96130
#endif
97-
poly1305_blocks_x86_64(ctx, inp, len, padbit);
131+
132+
scalar:
133+
convert_to_base2_64(ctx);
134+
poly1305_blocks_x86_64(ctx, inp, len, padbit);
98135
return true;
99136
}
100137

@@ -105,23 +142,17 @@ static inline bool poly1305_emit_arch(void *ctx, u8 mac[POLY1305_MAC_SIZE],
105142
struct poly1305_arch_internal *state = ctx;
106143

107144
if (!poly1305_use_avx || !state->is_base2_26 ||!simd_use(simd_context))
108-
poly1305_emit_x86_64(ctx, mac, nonce);
109-
else
110-
#ifdef CONFIG_AS_AVX512
111-
if (poly1305_use_avx512)
112-
poly1305_emit_avx(ctx, mac, nonce);
113-
else
114-
#endif
115-
#ifdef CONFIG_AS_AVX2
116-
if (poly1305_use_avx2)
117-
poly1305_emit_avx(ctx, mac, nonce);
118-
else
119-
#endif
145+
goto scalar;
146+
120147
#ifdef CONFIG_AS_AVX
121-
if (poly1305_use_avx)
148+
if (poly1305_use_avx || poly1305_use_avx2 || poly1305_use_avx512) {
122149
poly1305_emit_avx(ctx, mac, nonce);
123-
else
150+
return true;
151+
}
124152
#endif
125-
poly1305_emit_x86_64(ctx, mac, nonce);
153+
154+
scalar:
155+
convert_to_base2_64(ctx);
156+
poly1305_emit_x86_64(ctx, mac, nonce);
126157
return true;
127158
}

‎src/crypto/zinc/selftest/poly1305.h‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -862,6 +862,23 @@ static bool __init poly1305_selftest(void)
862862
i + 1, j);
863863
success = false;
864864
}
865+
866+
memset(out, 0, sizeof(out));
867+
memset(&poly1305, 0, sizeof(poly1305));
868+
poly1305_init(&poly1305, poly1305_testvecs[i].key);
869+
poly1305_update(&poly1305, poly1305_testvecs[i].input,
870+
j, &simd_context);
871+
poly1305_update(&poly1305,
872+
poly1305_testvecs[i].input + j,
873+
poly1305_testvecs[i].ilen - j,
874+
(simd_context_t []){ HAVE_NO_SIMD });
875+
poly1305_final(&poly1305, out, &simd_context);
876+
if (memcmp(out, poly1305_testvecs[i].output,
877+
POLY1305_MAC_SIZE)) {
878+
pr_info("poly1305 self-test %zu (split %zu, mixed simd): FAIL\n",
879+
i + 1, j);
880+
success = false;
881+
}
865882
simd_relax(&simd_context);
866883
}
867884
}

0 commit comments

Comments
 (0)