Skip to content

Commit 29e92b8

Browse files
Nicoshevmeta-codesync[bot]
Nicoshev
authored and
meta-codesync[bot]
committed
Improve crc32 on aarch64
Summary: Switch aarch64 CRC32 implementation to one better suited for server-class CPUs We observe 14% to 25% higher throughput Before: crc32_2048 77.69ns 12.87M crc32_4096 114.12ns 8.76M crc32_8192 180.04ns 5.55M crc32_16384 303.57ns 3.29M crc32_32768 543.78ns 1.84M crc32_65536 1.02us 976.60K crc32_131072 1.98us 506.27K crc32_262144 3.85us 259.69K crc32_524288 7.75us 128.98K After: crc32_2048 63.37ns 15.78M crc32_4096 91.09ns 10.98M crc32_8192 152.72ns 6.55M crc32_16384 258.29ns 3.87M crc32_32768 471.42ns 2.12M crc32_65536 888.03ns 1.13M crc32_131072 1.72us 581.28K crc32_262144 3.37us 296.31K crc32_524288 6.68us 149.69K Reviewed By: yfeldblum Differential Revision: D95461291 fbshipit-source-id: 2f1f42bed7b9b2b6384cb0b33a998d7b39154110
1 parent 3192c54 commit 29e92b8

6 files changed

Lines changed: 48 additions & 66 deletions

File tree

‎third-party/folly/src/folly/external/fast-crc32/CMakeLists.txt‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,11 +77,11 @@ folly_add_library(
7777
)
7878

7979
folly_add_library(
80-
NAME neon_eor3_crc32_v9s3x2e_s3
80+
NAME neon_eor3_crc32_v8s2x4e_s1x2
8181
SRCS
82-
neon_eor3_crc32_v9s3x2e_s3.cpp
82+
neon_eor3_crc32_v8s2x4e_s1x2.cpp
8383
HEADERS
84-
neon_eor3_crc32_v9s3x2e_s3.h
84+
neon_eor3_crc32_v8s2x4e_s1x2.h
8585
DEPS
8686
folly_portability
8787
folly_system_aux_vector

third-party/folly/src/folly/external/fast-crc32/neon_eor3_crc32_v9s3x2e_s3.cpp renamed to third-party/folly/src/folly/external/fast-crc32/neon_eor3_crc32_v8s2x4e_s1x2.cpp

Lines changed: 32 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
/* ./generate -i neon_eor3 -p crc32 -a v9s3x2e_s3 */
33
/* MIT licensed */
44

5-
#include "folly/external/fast-crc32/neon_eor3_crc32_v9s3x2e_s3.h"
5+
#include "folly/external/fast-crc32/neon_eor3_crc32_v8s2x4e_s1x2.h"
66
#include <folly/system/AuxVector.h>
77
#include <folly/Portability.h>
88

@@ -21,11 +21,11 @@ namespace folly::detail {
2121
#if !(FOLLY_AARCH64 && FOLLY_NEON && FOLLY_ARM_FEATURE_CRYPTO && FOLLY_ARM_FEATURE_CRC32 && FOLLY_ARM_FEATURE_SHA3)
2222
#include <stdlib.h>
2323
namespace folly::detail {
24-
CRC_EXPORT uint32_t neon_eor3_crc32_v9s3x2e_s3(const uint8_t*, size_t, uint32_t) {
24+
CRC_EXPORT uint32_t neon_eor3_crc32_v8s2x4e_s1x2(const uint8_t*, size_t, uint32_t) {
2525
abort(); // not implemented on this platform
2626
}
2727

28-
CRC_EXPORT bool has_neon_eor3_crc32_v9s3x2e_s3() {
28+
CRC_EXPORT bool has_neon_eor3_crc32_v8s2x4e_s1x2() {
2929
return false;
3030
}
3131
}
@@ -78,14 +78,14 @@ CRC_AINLINE uint64x2_t crc_shift(uint32_t crc, size_t nbytes) {
7878
}
7979

8080
FOLLY_TARGET_ATTRIBUTE("+crc")
81-
CRC_EXPORT bool has_neon_eor3_crc32_v9s3x2e_s3() {
81+
CRC_EXPORT bool has_neon_eor3_crc32_v8s2x4e_s1x2() {
8282
auto caps = hwcaps();
8383

8484
return caps.aarch64_fp() && caps.aarch64_asimd() && caps.aarch64_pmull() &&
8585
caps.aarch64_crc32() && caps.aarch64_sha3();
8686
}
8787

88-
CRC_EXPORT uint32_t neon_eor3_crc32_v9s3x2e_s3(const uint8_t* buf, size_t len, uint32_t crc0) {
88+
CRC_EXPORT uint32_t neon_eor3_crc32_v8s2x4e_s1x2(const uint8_t* buf, size_t len, uint32_t crc0) {
8989
for (; len && ((uintptr_t)buf & 7); --len) {
9090
crc0 = __crc32b(crc0, *buf++);
9191
}
@@ -97,14 +97,12 @@ CRC_EXPORT uint32_t neon_eor3_crc32_v9s3x2e_s3(const uint8_t* buf, size_t len, u
9797
if (len >= 192) {
9898
const uint8_t* end = buf + len;
9999
size_t blk = (len - 0) / 192;
100-
size_t klen = blk * 16;
101-
const uint8_t* buf2 = buf + klen * 3;
102-
const uint8_t* limit = buf + klen - 32;
100+
size_t klen = blk * 32;
101+
const uint8_t* buf2 = buf + klen * 2;
102+
const uint8_t* limit = buf + klen - 64;
103103
uint32_t crc1 = 0;
104-
uint32_t crc2 = 0;
105104
uint64x2_t vc0;
106105
uint64x2_t vc1;
107-
uint64x2_t vc2;
108106
uint64_t vc;
109107
/* First vector chunk. */
110108
uint64x2_t x0 = vld1q_u64((const uint64_t*)buf2), y0;
@@ -115,10 +113,9 @@ CRC_EXPORT uint32_t neon_eor3_crc32_v9s3x2e_s3(const uint8_t* buf, size_t len, u
115113
uint64x2_t x5 = vld1q_u64((const uint64_t*)(buf2 + 80)), y5;
116114
uint64x2_t x6 = vld1q_u64((const uint64_t*)(buf2 + 96)), y6;
117115
uint64x2_t x7 = vld1q_u64((const uint64_t*)(buf2 + 112)), y7;
118-
uint64x2_t x8 = vld1q_u64((const uint64_t*)(buf2 + 128)), y8;
119116
uint64x2_t k;
120-
{ static const uint64_t CRC_ALIGN(16) k_[] = {0x26b70c3d, 0x3f41287a}; k = vld1q_u64(k_); }
121-
buf2 += 144;
117+
{ static const uint64_t CRC_ALIGN(16) k_[] = {0x33fff533, 0x910eeec1}; k = vld1q_u64(k_); }
118+
buf2 += 128;
122119
/* Main loop. */
123120
while (buf <= limit) {
124121
y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
@@ -129,7 +126,6 @@ CRC_EXPORT uint32_t neon_eor3_crc32_v9s3x2e_s3(const uint8_t* buf, size_t len, u
129126
y5 = clmul_lo(x5, k), x5 = clmul_hi(x5, k);
130127
y6 = clmul_lo(x6, k), x6 = clmul_hi(x6, k);
131128
y7 = clmul_lo(x7, k), x7 = clmul_hi(x7, k);
132-
y8 = clmul_lo(x8, k), x8 = clmul_hi(x8, k);
133129
x0 = veor3q_u64(x0, y0, vld1q_u64((const uint64_t*)buf2));
134130
x1 = veor3q_u64(x1, y1, vld1q_u64((const uint64_t*)(buf2 + 16)));
135131
x2 = veor3q_u64(x2, y2, vld1q_u64((const uint64_t*)(buf2 + 32)));
@@ -138,22 +134,20 @@ CRC_EXPORT uint32_t neon_eor3_crc32_v9s3x2e_s3(const uint8_t* buf, size_t len, u
138134
x5 = veor3q_u64(x5, y5, vld1q_u64((const uint64_t*)(buf2 + 80)));
139135
x6 = veor3q_u64(x6, y6, vld1q_u64((const uint64_t*)(buf2 + 96)));
140136
x7 = veor3q_u64(x7, y7, vld1q_u64((const uint64_t*)(buf2 + 112)));
141-
x8 = veor3q_u64(x8, y8, vld1q_u64((const uint64_t*)(buf2 + 128)));
142137
crc0 = __crc32d(crc0, *(const uint64_t*)buf);
143138
crc1 = __crc32d(crc1, *(const uint64_t*)(buf + klen));
144-
crc2 = __crc32d(crc2, *(const uint64_t*)(buf + klen * 2));
145139
crc0 = __crc32d(crc0, *(const uint64_t*)(buf + 8));
146140
crc1 = __crc32d(crc1, *(const uint64_t*)(buf + klen + 8));
147-
crc2 = __crc32d(crc2, *(const uint64_t*)(buf + klen * 2 + 8));
148-
buf += 16;
149-
buf2 += 144;
141+
crc0 = __crc32d(crc0, *(const uint64_t*)(buf + 16));
142+
crc1 = __crc32d(crc1, *(const uint64_t*)(buf + klen + 16));
143+
crc0 = __crc32d(crc0, *(const uint64_t*)(buf + 24));
144+
crc1 = __crc32d(crc1, *(const uint64_t*)(buf + klen + 24));
145+
buf += 32;
146+
buf2 += 128;
150147
}
151-
/* Reduce x0 ... x8 to just x0. */
148+
/* Reduce x0 ... x7 to just x0. */
152149
{ static const uint64_t CRC_ALIGN(16) k_[] = {0xae689191, 0xccaa009e}; k = vld1q_u64(k_); }
153150
y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
154-
x0 = veor3q_u64(x0, y0, x1);
155-
x1 = x2, x2 = x3, x3 = x4, x4 = x5, x5 = x6, x6 = x7, x7 = x8;
156-
y0 = clmul_lo(x0, k), x0 = clmul_hi(x0, k);
157151
y2 = clmul_lo(x2, k), x2 = clmul_hi(x2, k);
158152
y4 = clmul_lo(x4, k), x4 = clmul_hi(x4, k);
159153
y6 = clmul_lo(x6, k), x6 = clmul_hi(x6, k);
@@ -172,46 +166,34 @@ CRC_EXPORT uint32_t neon_eor3_crc32_v9s3x2e_s3(const uint8_t* buf, size_t len, u
172166
/* Final scalar chunk. */
173167
crc0 = __crc32d(crc0, *(const uint64_t*)buf);
174168
crc1 = __crc32d(crc1, *(const uint64_t*)(buf + klen));
175-
crc2 = __crc32d(crc2, *(const uint64_t*)(buf + klen * 2));
176169
crc0 = __crc32d(crc0, *(const uint64_t*)(buf + 8));
177170
crc1 = __crc32d(crc1, *(const uint64_t*)(buf + klen + 8));
178-
crc2 = __crc32d(crc2, *(const uint64_t*)(buf + klen * 2 + 8));
179-
vc0 = crc_shift(crc0, klen * 2 + blk * 144);
180-
vc1 = crc_shift(crc1, klen + blk * 144);
181-
vc2 = crc_shift(crc2, 0 + blk * 144);
182-
vc = vgetq_lane_u64(veor3q_u64(vc0, vc1, vc2), 0);
171+
crc0 = __crc32d(crc0, *(const uint64_t*)(buf + 16));
172+
crc1 = __crc32d(crc1, *(const uint64_t*)(buf + klen + 16));
173+
crc0 = __crc32d(crc0, *(const uint64_t*)(buf + 24));
174+
crc1 = __crc32d(crc1, *(const uint64_t*)(buf + klen + 24));
175+
vc0 = crc_shift(crc0, klen + blk * 128);
176+
vc1 = crc_shift(crc1, 0 + blk * 128);
177+
vc = vgetq_lane_u64(veorq_u64(vc0, vc1), 0);
183178
/* Reduce 128 bits to 32 bits, and multiply by x^32. */
184179
crc0 = __crc32d(0, vgetq_lane_u64(x0, 0));
185180
crc0 = __crc32d(crc0, vc ^ vgetq_lane_u64(x0, 1));
186181
buf = buf2;
187182
len = end - buf;
188183
}
189-
if (len >= 32) {
190-
size_t klen = ((len - 8) / 24) * 8;
191-
uint32_t crc1 = 0;
192-
uint32_t crc2 = 0;
193-
uint64x2_t vc0;
194-
uint64x2_t vc1;
195-
uint64_t vc;
184+
if (len >= 16) {
196185
/* Main loop. */
197186
do {
198187
crc0 = __crc32d(crc0, *(const uint64_t*)buf);
199-
crc1 = __crc32d(crc1, *(const uint64_t*)(buf + klen));
200-
crc2 = __crc32d(crc2, *(const uint64_t*)(buf + klen * 2));
201-
buf += 8;
202-
len -= 24;
203-
} while (len >= 32);
204-
vc0 = crc_shift(crc0, klen * 2 + 8);
205-
vc1 = crc_shift(crc1, klen + 8);
206-
vc = vgetq_lane_u64(veorq_u64(vc0, vc1), 0);
207-
/* Final 8 bytes. */
208-
buf += klen * 2;
209-
crc0 = crc2;
210-
crc0 = __crc32d(crc0, *(const uint64_t*)buf ^ vc), buf += 8;
211-
len -= 8;
188+
crc0 = __crc32d(crc0, *(const uint64_t*)(buf + 8));
189+
buf += 16;
190+
len -= 16;
191+
} while (len >= 16);
212192
}
213-
for (; len >= 8; buf += 8, len -= 8) {
193+
if (len >= 8) {
214194
crc0 = __crc32d(crc0, *(const uint64_t*)buf);
195+
len -= 8;
196+
buf += 8;
215197
}
216198
for (; len; --len) {
217199
crc0 = __crc32b(crc0, *buf++);

‎third-party/folly/src/folly/external/fast-crc32/neon_eor3_crc32_v8s2x4e_s1x2.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#pragma once
2+
#include <cstddef>
3+
#include <cstdint>
4+
5+
namespace folly::detail {
6+
uint32_t neon_eor3_crc32_v8s2x4e_s1x2(const uint8_t* buf, size_t len, uint32_t crc0);
7+
bool has_neon_eor3_crc32_v8s2x4e_s1x2();
8+
}

‎third-party/folly/src/folly/external/fast-crc32/neon_eor3_crc32_v9s3x2e_s3.h‎

Lines changed: 0 additions & 8 deletions
This file was deleted.

‎third-party/folly/src/folly/hash/CMakeLists.txt‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ folly_add_library(
2626
folly_detail_traponavx512
2727
folly_external_fast-crc32_avx512_crc32c_v8s3x4
2828
folly_external_fast-crc32_neon_crc32c_v3s4x2e_v2
29-
folly_external_fast-crc32_neon_eor3_crc32_v9s3x2e_s3
29+
folly_external_fast-crc32_neon_eor3_crc32_v8s2x4e_s1x2
3030
folly_external_fast-crc32_neon_eor3_crc32c_v8s2x4_s3
3131
folly_external_fast-crc32_sse_crc32c_v8s3x3
3232
folly_hash_detail_checksum_detail
@@ -37,7 +37,7 @@ target_link_libraries(folly_hash_checksum_obj PRIVATE
3737
folly_external_fast-crc32_avx512_crc32c_v8s3x4_obj
3838
folly_external_fast-crc32_sse_crc32c_v8s3x3_obj
3939
folly_external_fast-crc32_neon_crc32c_v3s4x2e_v2_obj
40-
folly_external_fast-crc32_neon_eor3_crc32_v9s3x2e_s3_obj
40+
folly_external_fast-crc32_neon_eor3_crc32_v8s2x4e_s1x2_obj
4141
folly_external_fast-crc32_neon_eor3_crc32c_v8s2x4_s3_obj)
4242

4343
folly_add_library(

‎third-party/folly/src/folly/hash/Checksum.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
#include <folly/detail/TrapOnAvx512.h>
2626
#include <folly/external/fast-crc32/avx512_crc32c_v8s3x4.h> // @manual
2727
#include <folly/external/fast-crc32/neon_crc32c_v3s4x2e_v2.h> // @manual
28-
#include <folly/external/fast-crc32/neon_eor3_crc32_v9s3x2e_s3.h> // @manual
28+
#include <folly/external/fast-crc32/neon_eor3_crc32_v8s2x4e_s1x2.h> // @manual
2929
#include <folly/external/fast-crc32/neon_eor3_crc32c_v8s2x4_s3.h> // @manual
3030
#include <folly/external/fast-crc32/sse_crc32c_v8s3x3.h> // @manual
3131
#include <folly/hash/detail/ChecksumDetail.h>
@@ -126,7 +126,7 @@ bool crc32c_hw_supported_neon() {
126126
}
127127

128128
bool crc32_hw_supported_neon_eor3_sha3() {
129-
static bool has_neon_eor3 = has_neon_eor3_crc32_v9s3x2e_s3();
129+
static bool has_neon_eor3 = has_neon_eor3_crc32_v8s2x4e_s1x2();
130130
return has_neon_eor3;
131131
}
132132

@@ -245,7 +245,7 @@ uint32_t crc32c(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
245245
uint32_t crc32(const uint8_t* data, size_t nbytes, uint32_t startingChecksum) {
246246
#if FOLLY_AARCH64
247247
if (nbytes >= 2048 && detail::crc32_hw_supported_neon_eor3_sha3()) {
248-
return detail::neon_eor3_crc32_v9s3x2e_s3(data, nbytes, startingChecksum);
248+
return detail::neon_eor3_crc32_v8s2x4e_s1x2(data, nbytes, startingChecksum);
249249
}
250250
#endif
251251

0 commit comments

Comments
 (0)