Skip to content

Commit c080314

Browse files
sleepybishopsleepybishop
committed
update oblas
1 parent 04e9280 commit c080314

2 files changed

Lines changed: 122 additions & 37 deletions

File tree

‎deps/obl/oblas_lite.c‎

Lines changed: 113 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -40,14 +40,6 @@ static void obl_axpy_ref(u8 *a, u8 *b, u8 u, unsigned k)
4040
*ap ^= gf2_8_mul(u, *bp);
4141
}
4242

43-
static void obl_scal_ref(u8 *a, u8 *b, u8 u, unsigned k)
44-
{
45-
(void)b;
46-
register u8 *ap = a, *ae = &a[k];
47-
for (; ap != ae; ap++)
48-
*ap = gf2_8_mul(u, *ap);
49-
}
50-
5143
static void obl_axiy_ref(u8 *a, u8 *b, u8 u, unsigned k)
5244
{
5345
register u8 *ap = a, *ae = &a[k], *bp = b;
@@ -56,23 +48,16 @@ static void obl_axiy_ref(u8 *a, u8 *b, u8 u, unsigned k)
5648
}
5749

5850
#else
59-
static void obl_axpy_ref(u8 *a, u8 *b, u8 u, unsigned k)
51+
static void obl_axpy_ref(u8 *restrict a, u8 *restrict b, u8 u, unsigned k)
6052
{
6153
register const u8 *u_row = &GF2_8_MUL[u << 8];
62-
register u8 *ap = a, *ae = &a[k], *bp = b;
63-
for (; ap != ae; ap++, bp++)
54+
register u8 *restrict ap = a;
55+
register u8 *ae = &a[k];
56+
register const u8 *restrict bp = b;
57+
for (; ap < ae; ap++, bp++)
6458
*ap ^= u_row[*bp];
6559
}
6660

67-
static void obl_scal_ref(u8 *a, u8 *b, u8 u, unsigned k)
68-
{
69-
(void)b;
70-
register const u8 *u_row = &GF2_8_MUL[u << 8];
71-
register u8 *ap = a, *ae = &a[k];
72-
for (; ap != ae; ap++)
73-
*ap = u_row[*ap];
74-
}
75-
7661
static void obl_axiy_ref(u8 *a, u8 *b, u8 u, unsigned k)
7762
{
7863
register const u8 *u_row = &GF2_8_MUL[u << 8];
@@ -137,25 +122,25 @@ static void obl_axpyb32_ref(u8 *a, u32 *b, u8 u, unsigned k)
137122
} while (0)
138123

139124
#define GENERATE_IMPL(suffix, attr, VEC_TYPE, VEC_LOAD, VEC_STORE, VEC_INIT, VEC_CORE, VEC_XOR) \
140-
attr static void obl_axpy_##suffix(u8 *a, u8 *b, u8 u, unsigned k) \
125+
attr static void obl_axpy_##suffix(u8 *restrict a, u8 *restrict b, u8 u, unsigned k) \
141126
{ \
142127
if (u == 1) { \
143-
u8 *ap = a; \
128+
u8 *restrict ap = a; \
144129
u8 *ae = a + k; \
145-
const u8 *bp = b; \
130+
const u8 *restrict bp = b; \
146131
for (; ap < ae; ap++, bp++) \
147132
*ap ^= *bp; \
148133
} else { \
149134
OBL_SHUF_TEMPLATE(obl_axpy, a, b, VEC_XOR, VEC_TYPE, VEC_LOAD, VEC_STORE, VEC_INIT, VEC_CORE, VEC_XOR); \
150135
} \
151136
} \
152-
attr static void obl_scal_##suffix(u8 *a, u8 u, unsigned k) \
153-
{ \
154-
OBL_SHUF_TEMPLATE(obl_scal, a, a, OBL_NOOP, VEC_TYPE, VEC_LOAD, VEC_STORE, VEC_INIT, VEC_CORE, VEC_XOR); \
155-
} \
156137
attr static void obl_axiy_##suffix(u8 *a, u8 *b, u8 u, unsigned k) \
157138
{ \
158139
OBL_SHUF_TEMPLATE(obl_axiy, a, b, OBL_NOOP, VEC_TYPE, VEC_LOAD, VEC_STORE, VEC_INIT, VEC_CORE, VEC_XOR); \
140+
} \
141+
attr static void obl_scal_##suffix(u8 *a, u8 u, unsigned k) \
142+
{ \
143+
obl_axiy_##suffix(a, a, u, k); \
159144
}
160145

161146
#if defined(OBLAS_ARCH_X86)
@@ -193,8 +178,8 @@ GENERATE_IMPL(avx2_gfni, __attribute__((target("avx2,gfni"))), __m256i, _mm256_l
193178
const u8 *u_lo = GF2_8_SHUF_LO + u * 16; \
194179
const u8 *u_hi = GF2_8_SHUF_HI + u * 16; \
195180
const __m256i mask = _mm256_set1_epi8(0x0f); \
196-
const __m256i urow_lo = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)u_lo)); \
197-
const __m256i urow_hi = _mm256_broadcastsi128_si256(_mm_loadu_si128((const __m128i *)u_hi))
181+
const __m256i urow_lo = _mm256_loadu2_m128i((const __m128i *)u_lo, (const __m128i *)u_lo); \
182+
const __m256i urow_hi = _mm256_loadu2_m128i((const __m128i *)u_hi, (const __m128i *)u_hi)
198183
#define VEC_CORE_avx2_std(bx, res) \
199184
__m256i lo_##res = _mm256_and_si256(bx, mask); \
200185
__m256i hi_##res = _mm256_and_si256(_mm256_srli_epi64(bx, 4), mask); \
@@ -231,15 +216,16 @@ __attribute__((target("avx512f,avx512bw,avx512dq,avx512vl"))) static void obl_ax
231216
__m512i *ap = (__m512i *)a;
232217
__m512i *ae = (__m512i *)(a + (k & ~63));
233218
__m512i scatter =
234-
_mm512_set_epi32(0x07070707, 0x07070707, 0x06060606, 0x06060606, 0x05050505, 0x05050505, 0x04040404, 0x04040404,
235-
0x03030303, 0x03030303, 0x02020202, 0x02020202, 0x01010101, 0x01010101, 0x00000000, 0x00000000);
219+
_mm512_set_epi32(0x07070707, 0x07070707, 0x06060606, 0x06060606, 0x05050505, 0x05050505, 0x04040404, 0x04040404, 0x03030303,
220+
0x03030303, 0x02020202, 0x02020202, 0x01010101, 0x01010101, 0x00000000, 0x00000000);
236221
__m512i cmpmask =
237222
_mm512_set_epi32(0x80402010, 0x08040201, 0x80402010, 0x08040201, 0x80402010, 0x08040201, 0x80402010, 0x08040201, 0x80402010,
238223
0x08040201, 0x80402010, 0x08040201, 0x80402010, 0x08040201, 0x80402010, 0x08040201);
239224
__m512i up = _mm512_set1_epi8(u);
240225
unsigned p = 0;
241226
for (; ap < ae; p += 2, ap++) {
242-
__m512i bcast = _mm512_set1_epi64(*(const long long *)(b + p));
227+
__m128i low = _mm_loadl_epi64((const __m128i *)(b + p));
228+
__m512i bcast = _mm512_broadcastq_epi64(low);
243229
__m512i ret = _mm512_shuffle_epi8(bcast, scatter);
244230
ret = _mm512_andnot_si512(ret, cmpmask);
245231
__mmask64 tmp = _mm512_cmpeq_epi8_mask(ret, _mm512_setzero_si512());
@@ -310,7 +296,9 @@ __attribute__((target("ssse3"))) static void obl_axpyb32_ssse3(u8 *a, u32 *b, u8
310296
lo_##res = vqtbl1q_u8(urow_lo, lo_##res); \
311297
hi_##res = vqtbl1q_u8(urow_hi, hi_##res); \
312298
uint8x16_t res = veorq_u8(lo_##res, hi_##res)
313-
GENERATE_IMPL(neon, , uint8x16_t, vld1q_u8, vst1q_u8, VEC_INIT_neon, VEC_CORE_neon, veorq_u8)
299+
#define VEC_LOAD_neon(ptr) vld1q_u8((const uint8_t *)(ptr))
300+
#define VEC_STORE_neon(ptr, val) vst1q_u8((uint8_t *)(ptr), val)
301+
GENERATE_IMPL(neon, , uint8x16_t, VEC_LOAD_neon, VEC_STORE_neon, VEC_INIT_neon, VEC_CORE_neon, veorq_u8)
314302

315303
static void obl_axpyb32_neon(u8 *a, u32 *b, u8 u, unsigned k)
316304
{
@@ -323,8 +311,8 @@ static void obl_axpyb32_neon(u8 *a, u32 *b, u8 u, unsigned k)
323311
unsigned p = 0;
324312
for (; ap < ae; p++) {
325313
uint8x16_t bcast = vreinterpretq_u8_u32(vdupq_n_u32(b[p]));
326-
uint8x16_t ret_lo = vceqzq_u8(vbicq_u8(cmpmask, vqtbl1q_u8(bcast, scatter_lo)));
327-
uint8x16_t ret_hi = vceqzq_u8(vbicq_u8(cmpmask, vqtbl1q_u8(bcast, scatter_hi)));
314+
uint8x16_t ret_lo = vceqq_u8(vbicq_u8(cmpmask, vqtbl1q_u8(bcast, scatter_lo)), vdupq_n_u8(0));
315+
uint8x16_t ret_hi = vceqq_u8(vbicq_u8(cmpmask, vqtbl1q_u8(bcast, scatter_hi)), vdupq_n_u8(0));
328316
ret_lo = vandq_u8(ret_lo, up);
329317
ret_hi = vandq_u8(ret_hi, up);
330318
vst1q_u8(ap, veorq_u8(vld1q_u8(ap), ret_lo));
@@ -335,9 +323,92 @@ static void obl_axpyb32_neon(u8 *a, u32 *b, u8 u, unsigned k)
335323
}
336324
#endif
337325

326+
#if defined(OBLAS_ARCH_RISCV) && defined(__riscv_vector)
327+
#include <riscv_vector.h>
328+
329+
static void obl_axpy_rvv(u8 *a, u8 *b, u8 u, unsigned k)
330+
{
331+
if (u == 0) {
332+
return;
333+
}
334+
if (u == 1) {
335+
size_t vl;
336+
for (unsigned i = 0; i < k; i += vl) {
337+
vl = __riscv_vsetvl_e8m1(k - i);
338+
vuint8m1_t va = __riscv_vle8_v_u8m1(a + i, vl);
339+
vuint8m1_t vb = __riscv_vle8_v_u8m1(b + i, vl);
340+
vuint8m1_t res = __riscv_vxor_vv_u8m1(va, vb, vl);
341+
__riscv_vse8_v_u8m1(a + i, res, vl);
342+
}
343+
return;
344+
}
345+
const u8 *u_lo = GF2_8_SHUF_LO + u * 16;
346+
const u8 *u_hi = GF2_8_SHUF_HI + u * 16;
347+
vuint8m1_t urow_lo = __riscv_vle8_v_u8m1(u_lo, 16);
348+
vuint8m1_t urow_hi = __riscv_vle8_v_u8m1(u_hi, 16);
349+
size_t vl;
350+
for (unsigned i = 0; i < k; i += vl) {
351+
vl = __riscv_vsetvl_e8m1(k - i);
352+
vuint8m1_t va = __riscv_vle8_v_u8m1(a + i, vl);
353+
vuint8m1_t vb = __riscv_vle8_v_u8m1(b + i, vl);
354+
vuint8m1_t indices_lo = __riscv_vand_vx_u8m1(vb, 0x0F, vl);
355+
vuint8m1_t indices_hi = __riscv_vsrl_vx_u8m1(vb, 4, vl);
356+
vuint8m1_t prod_lo = __riscv_vrgather_vv_u8m1(urow_lo, indices_lo, vl);
357+
vuint8m1_t prod_hi = __riscv_vrgather_vv_u8m1(urow_hi, indices_hi, vl);
358+
vuint8m1_t prod = __riscv_vxor_vv_u8m1(prod_lo, prod_hi, vl);
359+
vuint8m1_t res = __riscv_vxor_vv_u8m1(va, prod, vl);
360+
__riscv_vse8_v_u8m1(a + i, res, vl);
361+
}
362+
}
363+
364+
static void obl_axiy_rvv(u8 *a, u8 *b, u8 u, unsigned k)
365+
{
366+
if (u == 0) {
367+
size_t vl;
368+
for (unsigned i = 0; i < k; i += vl) {
369+
vl = __riscv_vsetvl_e8m1(k - i);
370+
vuint8m1_t zero = __riscv_vmv_v_x_u8m1(0, vl);
371+
__riscv_vse8_v_u8m1(a + i, zero, vl);
372+
}
373+
return;
374+
}
375+
if (u == 1) {
376+
if (a != b) {
377+
size_t vl;
378+
for (unsigned i = 0; i < k; i += vl) {
379+
vl = __riscv_vsetvl_e8m1(k - i);
380+
vuint8m1_t vb = __riscv_vle8_v_u8m1(b + i, vl);
381+
__riscv_vse8_v_u8m1(a + i, vb, vl);
382+
}
383+
}
384+
return;
385+
}
386+
const u8 *u_lo = GF2_8_SHUF_LO + u * 16;
387+
const u8 *u_hi = GF2_8_SHUF_HI + u * 16;
388+
vuint8m1_t urow_lo = __riscv_vle8_v_u8m1(u_lo, 16);
389+
vuint8m1_t urow_hi = __riscv_vle8_v_u8m1(u_hi, 16);
390+
size_t vl;
391+
for (unsigned i = 0; i < k; i += vl) {
392+
vl = __riscv_vsetvl_e8m1(k - i);
393+
vuint8m1_t vb = __riscv_vle8_v_u8m1(b + i, vl);
394+
vuint8m1_t indices_lo = __riscv_vand_vx_u8m1(vb, 0x0F, vl);
395+
vuint8m1_t indices_hi = __riscv_vsrl_vx_u8m1(vb, 4, vl);
396+
vuint8m1_t prod_lo = __riscv_vrgather_vv_u8m1(urow_lo, indices_lo, vl);
397+
vuint8m1_t prod_hi = __riscv_vrgather_vv_u8m1(urow_hi, indices_hi, vl);
398+
vuint8m1_t prod = __riscv_vxor_vv_u8m1(prod_lo, prod_hi, vl);
399+
__riscv_vse8_v_u8m1(a + i, prod, vl);
400+
}
401+
}
402+
403+
static void obl_scal_rvv(u8 *a, u8 u, unsigned k)
404+
{
405+
obl_axiy_rvv(a, a, u, k);
406+
}
407+
#endif
408+
338409
static void obl_scal_ref_wrapper(u8 *a, u8 u, unsigned k)
339410
{
340-
obl_scal_ref(a, NULL, u, k);
411+
obl_axiy_ref(a, a, u, k);
341412
}
342413

343414
void oblas_get_impl(struct oblas_impl *impl)
@@ -393,6 +464,12 @@ void oblas_get_impl(struct oblas_impl *impl)
393464
impl->axiy = obl_axiy_neon;
394465
impl->axpyb32 = obl_axpyb32_neon;
395466
impl->align_size = 16;
467+
#elif defined(OBLAS_ARCH_RISCV) && defined(__riscv_vector)
468+
impl->axpy = obl_axpy_rvv;
469+
impl->scal = obl_scal_rvv;
470+
impl->axiy = obl_axiy_rvv;
471+
impl->axpyb32 = obl_axpyb32_ref;
472+
impl->align_size = 16;
396473
#endif
397474
}
398475

‎deps/obl/oblas_lite.h‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,14 @@
11
#ifndef OBLAS_LITE_H
22
#define OBLAS_LITE_H
3-
43
#include <stdint.h>
54
#include <stddef.h>
65

76
#include "gf2_8_tables.h"
87

8+
#ifdef __cplusplus
9+
extern "C" {
10+
#endif
11+
912
struct oblas_impl {
1013
void (*axpy)(uint8_t *a, uint8_t *b, uint8_t u, unsigned k);
1114
void (*scal)(uint8_t *a, uint8_t u, unsigned k);
@@ -17,6 +20,7 @@ struct oblas_impl {
1720
void oblas_get_impl(struct oblas_impl *impl);
1821

1922
typedef uint8_t u8;
23+
typedef uint16_t u16;
2024
typedef uint32_t u32;
2125

2226
void obl_swap(u8 *a, u8 *b, unsigned k);
@@ -26,4 +30,8 @@ void *obl_alloc(size_t num_rows, size_t row_size, size_t alignment);
2630
void obl_free(void *ptr);
2731
#endif
2832

33+
#ifdef __cplusplus
34+
}
35+
#endif
36+
2937
#endif /* OBLAS_LITE_H */

0 commit comments

Comments
 (0)