@@ -2731,6 +2731,185 @@ void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len,
27312731 andn (crc, tmp5, crc);
27322732}
27332733
2734+ // Advance buf to the next 8-byte boundary, folding each byte into crc via the
2735+ // byte lookup table. tmp1 holds the alignment count, tmp2 the loaded byte.
2736+ void MacroAssembler::kernel_crc32c_clmul_align (Register crc, Register buf, Register len,
2737+ Register table, Register tmp1, Register tmp2) {
2738+ assert_different_registers (crc, buf, len, table, tmp1, tmp2);
2739+ Label L_adjust, L_loop, L_done;
2740+
2741+ // bytes to next 8-byte boundary: (-buf) & 7 = 8 - (buf & 7)
2742+ neg (tmp1, buf);
2743+ andi (tmp1, tmp1, 7 );
2744+
2745+ // tmp1 = min(tmp1, len)
2746+ bleu (tmp1, len, L_adjust);
2747+ mv (tmp1, len);
2748+ bind (L_adjust);
2749+ subw (len, len, tmp1);
2750+ beqz (tmp1, L_done);
2751+
2752+ bind (L_loop);
2753+ lbu (tmp2, Address (buf, 0 ));
2754+ addi (buf, buf, 1 );
2755+ update_byte_crc32 (crc, tmp2, table);
2756+ addi (tmp1, tmp1, -1 );
2757+ bnez (tmp1, L_loop);
2758+ bind (L_done);
2759+ }
2760+
2761+ // Fold 128 bits of input into the 128-bit accumulator (accum_hi:accum_lo).
2762+ // With the next 16 input bytes as (data_hi:data_lo) and * = carry-less mul:
2763+ // accum_lo' = (accum_lo*k1)[63:0] XOR (accum_hi*k2)[63:0] XOR data_lo
2764+ // accum_hi' = (accum_lo*k1)[127:64] XOR (accum_hi*k2)[127:64] XOR data_hi
2765+ void MacroAssembler::kernel_crc32c_clmul_fold_128 (Register accum_lo, Register accum_hi,
2766+ Register k1, Register k2, Register buf, Register tmp1, Register tmp2) {
2767+ assert_different_registers (accum_lo, accum_hi, k1, k2, buf, tmp1, tmp2);
2768+
2769+ clmul (tmp1, accum_lo, k1);
2770+ clmulh (tmp2, accum_lo, k1);
2771+
2772+ clmul (accum_lo, accum_hi, k2);
2773+ clmulh (accum_hi, accum_hi, k2);
2774+
2775+ xorr (accum_lo, accum_lo, tmp1);
2776+ ld (tmp1, Address (buf, 0 )); // tmp1 = data_lo
2777+ xorr (accum_lo, accum_lo, tmp1);
2778+
2779+ xorr (accum_hi, accum_hi, tmp2);
2780+ ld (tmp1, Address (buf, 8 )); // tmp1 = data_hi
2781+ xorr (accum_hi, accum_hi, tmp1);
2782+ }
2783+
2784+ // Reduce the 128-bit accumulator (accum_hi:accum_lo) to 64 bits using the
2785+ // ISA-L two-constant method. With rk1 at OFF_REDUCE_HI and rk2 at OFF_REDUCE_LO:
2786+ // x = accum_lo
2787+ // y0 = accum_hi XOR clmul(x, rk1)[63:0]
2788+ // y1 = clmulh(x, rk1)[95:64]
2789+ // accum_lo' = clmul(y0[31:0], rk2)[63:0] XOR (y1 : y0[63:32])
2790+ void MacroAssembler::kernel_crc32c_clmul_reduce_128_to_64 (Register accum_lo, Register accum_hi,
2791+ Register clmul_table, Register k, Register tmp1, Register tmp2) {
2792+ assert_different_registers (accum_lo, accum_hi, clmul_table, k, tmp1, tmp2);
2793+
2794+ const int OFF_REDUCE_LO = 16 ;
2795+ const int OFF_REDUCE_HI = 24 ;
2796+
2797+ ld (k, Address (clmul_table, OFF_REDUCE_HI ));
2798+ clmul (tmp1, accum_lo, k); // tmp1 = clmul(x, rk1)[63:0]
2799+ clmulh (tmp2, accum_lo, k); // tmp2 = clmulh(x, rk1) = clmul(x, rk1)[127:64]
2800+
2801+ xorr (accum_hi, accum_hi, tmp1); // y0 = accum_hi XOR clmul(x, rk1)[63:0]
2802+
2803+ zext (tmp1, accum_hi, 32 ); // tmp1 = y0[31:0]
2804+ srli (accum_hi, accum_hi, 32 ); // accum_hi = y0[63:32]
2805+
2806+ ld (k, Address (clmul_table, OFF_REDUCE_LO ));
2807+ clmul (accum_lo, tmp1, k); // accum_lo' = clmul(y0[31:0], rk2)[63:0]
2808+
2809+ slli (tmp2, tmp2, 32 ); // tmp2 = clmul(x, rk1)[95:64] << 32 = y1 : 0
2810+ xorr (tmp2, tmp2, accum_hi); // tmp2 = y1 : y0[63:32]
2811+ xorr (accum_lo, accum_lo, tmp2); // accum_lo' XOR= (y1 : y0[63:32])
2812+ }
2813+
2814+ // Reduce the 64-bit accumulator accum_lo to a residue mod P (the 33-bit
2815+ // CRC32C polynomial) using Barrett reduction, with mu = floor(x^64/P) at
2816+ // OFF_BARRETT_MU and poly = P at OFF_BARRETT_POLY:
2817+ // q = (accum_lo[31:0] * mu)[31:0] // quotient estimate
2818+ // accum_lo XOR= (q * poly)[63:0] // subtract q*P
2819+ // The high 32 bits of accum_lo then hold the final CRC.
2820+ void MacroAssembler::kernel_crc32c_clmul_barrett_64_to_32 (Register accum_lo, Register clmul_table,
2821+ Register k, Register tmp) {
2822+ assert_different_registers (accum_lo, clmul_table, k, tmp);
2823+
2824+ const int OFF_BARRETT_MU = 32 ;
2825+ const int OFF_BARRETT_POLY = 40 ;
2826+
2827+ zext (tmp, accum_lo, 32 ); // tmp = accum_lo[31:0]
2828+
2829+ ld (k, Address (clmul_table, OFF_BARRETT_MU ));
2830+ clmul (tmp, tmp, k);
2831+ zext (tmp, tmp, 32 ); // tmp = q = (accum_lo[31:0] * mu)[31:0]
2832+
2833+ ld (k, Address (clmul_table, OFF_BARRETT_POLY ));
2834+ clmul (tmp, tmp, k);
2835+
2836+ xorr (accum_lo, accum_lo, tmp);
2837+ }
2838+
2839+ void MacroAssembler::kernel_crc32c_clmul_fold (Register crc, Register buf, Register len,
2840+ Register byte_table, Register clmul_table,
2841+ Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register tmp6) {
2842+ assert_different_registers (crc, buf, len, byte_table, clmul_table, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
2843+ Label L_fold_loop, L_tail, L_tail_loop, L_exit;
2844+
2845+ const int FOLD_THRESHOLD = 64 ; // fold only for inputs >= 4 FOLD_STEPs
2846+ const int FOLD_STEP = 16 ;
2847+ const int OFF_FOLD_K1 = 0 ;
2848+ const int OFF_FOLD_K2 = 8 ;
2849+
2850+ mv (tmp4, FOLD_THRESHOLD );
2851+ blt (len, tmp4, L_tail);
2852+
2853+ kernel_crc32c_clmul_align (crc, buf, len, byte_table, tmp4, tmp5);
2854+ beqz (len, L_exit);
2855+
2856+ // fold_end = buf + len - (len mod 16); len = tail
2857+ const Register fold_end = tmp6;
2858+ add (fold_end, buf, len);
2859+ andi (len, len, FOLD_STEP - 1 );
2860+ sub (fold_end, fold_end, len);
2861+
2862+ // accum = (buf[0:8] XOR crc) : buf[8:16]
2863+ const Register accum_lo = crc;
2864+ const Register accum_hi = tmp1;
2865+ ld (tmp4, Address (buf, 0 ));
2866+ xorr (accum_lo, tmp4, crc);
2867+ ld (accum_hi, Address (buf, 8 ));
2868+ addi (buf, buf, FOLD_STEP );
2869+
2870+ ld (tmp2, Address (clmul_table, OFF_FOLD_K1 ));
2871+ ld (tmp3, Address (clmul_table, OFF_FOLD_K2 ));
2872+
2873+ bind (L_fold_loop);
2874+ kernel_crc32c_clmul_fold_128 (accum_lo, accum_hi, tmp2, tmp3, buf, tmp4, tmp5);
2875+ addi (buf, buf, 16 );
2876+ blt (buf, fold_end, L_fold_loop);
2877+
2878+ // reduce 128->64 (ISA-L) then 64->32 (Barrett); tmp2 = k; tmp4..tmp5 reused as scratch
2879+ kernel_crc32c_clmul_reduce_128_to_64 (accum_lo, accum_hi, clmul_table, tmp2, tmp4, tmp5);
2880+ kernel_crc32c_clmul_barrett_64_to_32 (accum_lo, clmul_table, tmp2, tmp4);
2881+ srli (crc, accum_lo, 32 );
2882+
2883+ bind (L_tail);
2884+ add (tmp5, buf, len); // tmp5 = tail_end
2885+ bind (L_tail_loop);
2886+ bgeu (buf, tmp5, L_exit);
2887+ lbu (tmp4, Address (buf, 0 ));
2888+ addi (buf, buf, 1 );
2889+ update_byte_crc32 (crc, tmp4, byte_table);
2890+ j (L_tail_loop);
2891+
2892+ bind (L_exit);
2893+ }
2894+
2895+ void MacroAssembler::kernel_crc32c (Register crc, Register buf, Register len,
2896+ Register byte_table, Register clmul_table,
2897+ Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register tmp6) {
2898+ assert_different_registers (crc, buf, len, byte_table, clmul_table, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
2899+ assert (UseZbc, " CRC32C intrinsic requires Zbc" );
2900+
2901+ zext (crc, crc, 32 );
2902+
2903+ // _crc32c_table is followed by the clmul constants
2904+ const int64_t single_table_size = 256 ;
2905+ const ExternalAddress table_addr = StubRoutines::crc32c_table_addr ();
2906+ la (byte_table, table_addr);
2907+ add (clmul_table, byte_table, 1 * single_table_size * sizeof (juint), tmp4);
2908+
2909+ kernel_crc32c_clmul_fold (crc, buf, len, byte_table, clmul_table,
2910+ tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
2911+ }
2912+
27342913#ifdef COMPILER2
27352914// Push vector registers in the bitset supplied.
27362915// Return the number of words pushed
0 commit comments