Merge pull request #10938 from NelsonVides/perf/json_swar

Use SWAR to optimize JSON string scanning (~5% to ~50% improvements)

OTP-20072

Author: dgud

lib/stdlib/src/json.erl

@@ -361,15 +361,15 @@ escape_binary(Bin) -> escape_binary_ascii(Bin, [$"], Bin, 0, 0).
 
 escape_binary_ascii(Binary, Acc, Orig, Skip, Len) ->
     case Binary of
-        <<B1, B2, B3, B4, B5, B6, B7, B8, Rest/binary>> when ?are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8) ->
-            escape_binary_ascii(Rest, Acc, Orig, Skip, Len + 8);
+        <<W:56, Rest/binary>> when ?are_all_ascii_plain_swar(W) ->
+            escape_binary_ascii(Rest, Acc, Orig, Skip, Len + 7);
         Other ->
             escape_binary(Other, Acc, Orig, Skip, Len)
     end.
 
 escape_binary(<<Byte, Rest/binary>>, Acc, Orig, Skip, Len) when ?is_ascii_plain(Byte) ->
-    %% we got here because there were either less than 8 bytes left
-    %% or we have an escape in the next 8 bytes,
+    %% we got here because there were either less than 7 bytes left
+    %% or we have an escape in the next 7 bytes,
     %% escape_binary_ascii would fail and dispatch here anyway
     escape_binary(Rest, Acc, Orig, Skip, Len + 1);
 escape_binary(<<Byte, Rest/binary>>, Acc, Orig, Skip0, Len) when ?is_ascii_escape(Byte) ->
@@ -410,8 +410,8 @@ escape_all(Bin) -> escape_all_ascii(Bin, [$"], Bin, 0, 0).
 
 escape_all_ascii(Binary, Acc, Orig, Skip, Len) ->
     case Binary of
-        <<B1, B2, B3, B4, B5, B6, B7, B8, Rest/binary>> when ?are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8) ->
-            escape_all_ascii(Rest, Acc, Orig, Skip, Len + 8);
+        <<W:56, Rest/binary>> when ?are_all_ascii_plain_swar(W) ->
+            escape_all_ascii(Rest, Acc, Orig, Skip, Len + 7);
         Other ->
             escape_all(Other, Acc, Orig, Skip, Len)
     end.
@@ -1175,8 +1175,8 @@ string(Binary, Original, Skip, Acc, Stack, Decode) ->
 
 string_ascii(Binary, Original, Skip, Acc, Stack, Decode, Len) ->
     case Binary of
-        <<B1, B2, B3, B4, B5, B6, B7, B8, Rest/binary>> when ?are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8) ->
-            string_ascii(Rest, Original, Skip, Acc, Stack, Decode, Len + 8);
+        <<W:56, Rest/binary>> when ?are_all_ascii_plain_swar(W) ->
+            string_ascii(Rest, Original, Skip, Acc, Stack, Decode, Len + 7);
         Other ->
             string(Other, Original, Skip, Acc, Stack, Decode, Len)
     end.
@@ -1218,8 +1218,8 @@ string_utf8(_, Orig, Skip, Acc, Stack, Decode, Len, _State0) ->
 
 string_ascii(Binary, Original, Skip, Acc, Stack, Decode, Start, Len, SAcc) ->
     case Binary of
-        <<B1, B2, B3, B4, B5, B6, B7, B8, Rest/binary>> when ?are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8) ->
-            string_ascii(Rest, Original, Skip, Acc, Stack, Decode, Start, Len + 8, SAcc);
+        <<W:56, Rest/binary>> when ?are_all_ascii_plain_swar(W) ->
+            string_ascii(Rest, Original, Skip, Acc, Stack, Decode, Start, Len + 7, SAcc);
         Other ->
             string(Other, Original, Skip, Acc, Stack, Decode, Start, Len, SAcc)
     end.

lib/stdlib/src/json.hrl

@@ -179,13 +179,63 @@
     Byte =:= 127
 ).
 
--define(are_all_ascii_plain(B1, B2, B3, B4, B5, B6, B7, B8),
-    (?is_ascii_plain(B1)) andalso
-    (?is_ascii_plain(B2)) andalso
-    (?is_ascii_plain(B3)) andalso
-    (?is_ascii_plain(B4)) andalso
-    (?is_ascii_plain(B5)) andalso
-    (?is_ascii_plain(B6)) andalso
-    (?is_ascii_plain(B7)) andalso
-    (?is_ascii_plain(B8))
+%% SWAR (SIMD Within A Register) check for 7 bytes of plain ASCII at once.
+%%
+%% Instead of matching 8 individual bytes and checking each against
+%% is_ascii_plain/1 (which the JIT compiles to 8 jump table lookups
+%% with indirect branches), we match a single 56-bit integer and use
+%% bitwise arithmetic to validate all 7 bytes in parallel.
+%%
+%% We use 56 bits (7 bytes) because it is the largest value that fits
+%% in a BEAM small integer (59-bit on 64-bit). This ensures all
+%% bitwise and arithmetic guard operations (band, bor, bxor, +, -)
+%% compile to single native instructions with no type checks or
+%% bignum fallback calls. Benchmarks showed 4/6/7-byte variants all
+%% outperform the original, with 7 bytes winning on string-heavy
+%% inputs (up to 55% faster) due to its larger stride.
+%%
+%% The byte-by-byte is_ascii_plain fallback path handles any remaining
+%% bytes (< 7) and is always entered when the SWAR check fails, so
+%% correctness does not depend on the SWAR path.
+
+-define(SWAR_MASK80, 16#80808080808080).
+-define(SWAR_MASK01, 16#01010101010101).
+
+%% Detect if any byte in a 56-bit word is zero (Mycroft's trick).
+%%
+%% This is a simplified variant that omits the standard (bnot V) term.
+%% The full formula is: ((V - 0x01..01) band (bnot V) band 0x80..80).
+%% The (bnot V) term filters out false positives from bytes >= 0x80,
+%% where subtracting 0x01 does not clear the high bit. This term is
+%% unnecessary here: check 1 in are_all_ascii_plain_swar/1 proves all
+%% bytes of W are < 128 before no_zero_byte is reached (thanks to
+%% andalso short-circuit evaluation), and XOR of two 7-bit values is
+%% still 7-bit, so no byte in V can have bit 7 set.
+%%
+%% We also avoid bnot because the JIT lacks an always_small fast path
+%% for it, emitting runtime type checks and bignum fallback calls even
+%% when the result provably fits in a small.
+%%
+%% Borrow propagation between bytes may cause rare false positives
+%% (a non-zero byte adjacent to a zero byte detected as zero), but
+%% these are harmless: we simply fall through to the byte-by-byte
+%% path which is always correct.
+-define(no_zero_byte(V),
+    ((V) - ?SWAR_MASK01) band ?SWAR_MASK80 =:= 0
+).
+
+%% SWAR check: all 7 bytes (in one 56-bit word) are "plain ASCII"
+%% i.e., in [32, 127] and not $" (0x22) or $\\ (0x5C).
+%%
+%% Four checks, each operating on all 7 bytes simultaneously:
+%%   1. band with 0x80..80 detects bytes >= 128
+%%   2. add 0x60..60 then band 0x80..80 detects bytes < 32
+%%      (byte + 0x60 sets the high bit iff byte >= 0x20, given byte < 0x80)
+%%   3. XOR with 0x22..22 maps $" to 0, then no_zero_byte detects it
+%%   4. XOR with 0x5C..5C maps $\\ to 0, then no_zero_byte detects it
+-define(are_all_ascii_plain_swar(W),
+    (W) band ?SWAR_MASK80 =:= 0 andalso
+    ((W) + 16#60606060606060) band ?SWAR_MASK80 =:= ?SWAR_MASK80 andalso
+    ?no_zero_byte((W) bxor 16#22222222222222) andalso
+    ?no_zero_byte((W) bxor 16#5C5C5C5C5C5C5C)
 ).