Compile literal binary patterns in the loader

Author: josevalim

erts/emulator/beam/beam_transform_helpers.c

@@ -25,7 +25,9 @@
 #endif
 
 #include "sys.h"
+#include "global.h"
 #include "beam_load.h"
+#include "erl_bif_unique.h"
 #include "erl_map.h"
 #include "beam_transform_helpers.h"
 
@@ -138,6 +140,29 @@ beam_load_sort_select_vals(BeamOpArg* base, size_t n)
           (int (*)(const void *, const void *)) oparg_compare);
 }
 
+int
+beam_load_compile_binary_pattern(LoaderState *stp, BeamOpArg pattern)
+{
+    Eterm heap[ERTS_MAGIC_REF_THING_SIZE + 3];
+    Binary *bin;
+    Eterm compiled, magic_ref, source, tag;
+    Eterm *hp = heap;
+    ErlOffHeap off_heap;
+
+    source = beamfile_get_literal(&stp->beam, pattern.val);
+    if (!erts_binary_compile_pattern(source, &tag, &bin)) {
+        return 0;
+    }
+
+    ERTS_INIT_OFF_HEAP(&off_heap);
+    magic_ref = erts_mk_magic_ref(&hp, &off_heap, bin);
+    compiled = TUPLE2(hp, tag, magic_ref);
+
+    stp->loaded_binary_pattern = beamfile_add_literal(&stp->beam, compiled, 0);
+    erts_cleanup_offheap(&off_heap);
+    return 1;
+}
+
 static int
 oparg_compare(BeamOpArg* a, BeamOpArg* b)
 {

erts/emulator/beam/beam_transform_helpers.h

@@ -27,4 +27,5 @@ int beam_load_safe_mul(UWord a, UWord b, UWord* resp);
 int beam_load_map_key_sort(LoaderState* stp, BeamOpArg Size, BeamOpArg* Rest);
 Eterm beam_load_get_term(LoaderState* stp, BeamOpArg Key);
 void beam_load_sort_select_vals(BeamOpArg* base, size_t n);
+int beam_load_compile_binary_pattern(LoaderState *stp, BeamOpArg pattern);
 #endif

erts/emulator/beam/emu/load.h

@@ -147,6 +147,9 @@ struct LoaderState_ {
     /* Translates lambda indexes to their canonical literal, if any. */
     SWord *lambda_literals;
 
+    /* Dynamic literal produced by the current binary pattern transform. */
+    SWord loaded_binary_pattern;
+
     int otp_20_or_higher;
 
     Uint last_func_start;

erts/emulator/beam/emu/ops.tab

@@ -904,6 +904,23 @@ call_ext_last u==0 u$func:os:perf_counter/0 D =>
 call_ext_only u==0 u$func:os:perf_counter/0 =>
     i_perf_counter | return
 
+# Compile literal binary search patterns while loading. Invalid patterns decline
+# these transforms and retain their original runtime error behavior.
+#
+
+move Pattern=q X0=x==0 |
+  call_ext _Ar=u _Func=u$bif:binary:compile_pattern/1 |
+  load_binary_pattern(Pattern) =>
+    loaded_binary_pattern(X0)
+move Pattern=q X0=x==0 |
+  call_ext_last _Ar=u _Func=u$bif:binary:compile_pattern/1 D |
+  load_binary_pattern(Pattern) =>
+    loaded_binary_pattern_last(X0, D)
+move Pattern=q X0=x==0 |
+  call_ext_only _Ar=u _Func=u$bif:binary:compile_pattern/1 |
+  load_binary_pattern(Pattern) =>
+    loaded_binary_pattern_only(X0)
+
 #
 # BIFs like process_info/1,2 require up-to-date information about the current
 # emulator state, which the ordinary call_light_bif instruction doesn't save.

erts/emulator/beam/erl_bif_binary.c

@@ -1100,18 +1100,23 @@ static int do_binary_match_compile(Eterm argument, Eterm *tag, Binary **binp)
     return -1;
 }
 
+int erts_binary_compile_pattern(Eterm argument, Eterm *tag, Binary **binp)
+{
+    return do_binary_match_compile(argument, tag, binp) == 0;
+}
+
 BIF_RETTYPE binary_compile_pattern_1(BIF_ALIST_1)
 {
     Binary *bin;
-    Eterm tag, ret;
+    Eterm magic_ref, ret, tag;
     Eterm *hp;
 
-    if (do_binary_match_compile(BIF_ARG_1,&tag,&bin)) {
-	BIF_ERROR(BIF_P,BADARG);
+    if (!erts_binary_compile_pattern(BIF_ARG_1, &tag, &bin)) {
+        BIF_ERROR(BIF_P, BADARG);
     }
-    hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE+3);
-    ret = erts_mk_magic_ref(&hp, &MSO(BIF_P), bin);
-    ret = TUPLE2(hp, tag, ret);
+    hp = HAlloc(BIF_P, ERTS_MAGIC_REF_THING_SIZE + 3);
+    magic_ref = erts_mk_magic_ref(&hp, &MSO(BIF_P), bin);
+    ret = TUPLE2(hp, tag, magic_ref);
     BIF_RET(ret);
 }
 

erts/emulator/beam/generators.tab

@@ -70,6 +70,47 @@ gen.get_float2(Fail, Ms, Live, Size, Unit, Flags, Dst) {
     return op;
 }
 
+LoadedBinaryPatternMove(Op, Dst) {
+    $BeamOpNameArity($Op, move, 2);
+    ($Op)->a[0].type = TAG_q;
+    ($Op)->a[0].val = S->loaded_binary_pattern;
+    ($Op)->a[1] = $Dst;
+}
+
+gen.loaded_binary_pattern(Dst) {
+    BeamOp *move;
+    $NewBeamOp(S, move);
+
+    $LoadedBinaryPatternMove(move, Dst);
+    return move;
+}
+
+gen.loaded_binary_pattern_last(Dst, Deallocate) {
+    BeamOp *move, *deallocate, *ret;
+    $NewBeamOp(S, move);
+    $NewBeamOp(S, deallocate);
+    $NewBeamOp(S, ret);
+
+    $LoadedBinaryPatternMove(move, Dst);
+    $BeamOpNameArity(deallocate, deallocate, 1);
+    deallocate->a[0] = Deallocate;
+    $BeamOpNameArity(ret, return, 0);
+    move->next = deallocate;
+    deallocate->next = ret;
+    return move;
+}
+
+gen.loaded_binary_pattern_only(Dst) {
+    BeamOp *move, *ret;
+    $NewBeamOp(S, move);
+    $NewBeamOp(S, ret);
+
+    $LoadedBinaryPatternMove(move, Dst);
+    $BeamOpNameArity(ret, return, 0);
+    move->next = ret;
+    return move;
+}
+
 gen.get_utf16(Fail, Ms, Flags, Dst) {
     BeamOp* op;
     $NewBeamOp(S, op);

erts/emulator/beam/global.h

@@ -1450,6 +1450,7 @@ Sint erts_re_set_loop_limit(Sint limit);
 /* erl_bif_binary.c */
 void erts_init_bif_binary(void);
 Sint erts_binary_set_loop_limit(Sint limit);
+int erts_binary_compile_pattern(Eterm argument, Eterm *tag, Binary **binp);
 
 /* erl_bif_persistent.c */
 Eterm erts_persistent_term_get(Eterm key);

erts/emulator/beam/jit/arm/ops.tab

@@ -82,6 +82,10 @@ move S X0=x==0 | line _Loc | call_last Ar Func D =>
 move S X0=x==0 | line _Loc | call_only Ar Func =>
     move S X0 | call_only Ar Func
 
+# Move literal call arguments past line instructions so they can participate
+# in load-time call transformations.
+move S=c X0=x==0 | line Loc => line Loc | move S X0
+
 # The line number in int_func_start/5 can be NIL.
 func_line n => empty_func_line
 
@@ -395,6 +399,23 @@ move_two_trim y d y d t
 
 move_trim s d t
 
+# Compile literal binary search patterns while loading. Invalid patterns decline
+# these transforms and retain their original runtime error behavior.
+#
+
+move Pattern=q X0=x==0 |
+  call_ext _Ar=u _Func=u$bif:binary:compile_pattern/1 |
+  load_binary_pattern(Pattern) =>
+    loaded_binary_pattern(X0)
+move Pattern=q X0=x==0 |
+  call_ext_last _Ar=u _Func=u$bif:binary:compile_pattern/1 D |
+  load_binary_pattern(Pattern) =>
+    loaded_binary_pattern_last(X0, D)
+move Pattern=q X0=x==0 |
+  call_ext_only _Ar=u _Func=u$bif:binary:compile_pattern/1 |
+  load_binary_pattern(Pattern) =>
+    loaded_binary_pattern_only(X0)
+
 move Src Dst => i_move Src Dst
 
 i_move s d

erts/emulator/beam/jit/load.h

@@ -95,6 +95,9 @@ struct LoaderState_ {
     /* Translates lambda indexes to their canonical literal, if any. */
     SWord *lambda_literals;
 
+    /* Dynamic literal produced by the current binary pattern transform. */
+    SWord loaded_binary_pattern;
+
     void *ba; /* Assembler used to create x86/AArch64 assembly */
 
     const void *executable_region; /* Native module after codegen */

erts/emulator/beam/jit/x86/ops.tab

@@ -82,6 +82,10 @@ move S X0=x==0 | line _Loc | call_last Ar Func D =>
 move S X0=x==0 | line _Loc | call_only Ar Func =>
      move S X0 | call_only Ar Func
 
+# Move literal call arguments past line instructions so they can participate
+# in load-time call transformations.
+move S=c X0=x==0 | line Loc => line Loc | move S X0
+
 # The line number in int_func_start/5 can be NIL.
 func_line n => empty_func_line
 
@@ -363,6 +367,23 @@ move S1=d D1=d | move S2=d D2=d | consecutive_words(S2, D1, S1, D2) =>
 move S1=d D1=d | move S2=d D2=d | consecutive_words(S2, D2, S1, D1) =>
     move_two_words S2 D2 S1 D1
 
+# Compile literal binary search patterns while loading. Invalid patterns decline
+# these transforms and retain their original runtime error behavior.
+#
+
+move Pattern=q X0=x==0 |
+  call_ext _Ar=u _Func=u$bif:binary:compile_pattern/1 |
+  load_binary_pattern(Pattern) =>
+    loaded_binary_pattern(X0)
+move Pattern=q X0=x==0 |
+  call_ext_last _Ar=u _Func=u$bif:binary:compile_pattern/1 D |
+  load_binary_pattern(Pattern) =>
+    loaded_binary_pattern_last(X0, D)
+move Pattern=q X0=x==0 |
+  call_ext_only _Ar=u _Func=u$bif:binary:compile_pattern/1 |
+  load_binary_pattern(Pattern) =>
+    loaded_binary_pattern_only(X0)
+
 move Src Dst => i_move Src Dst
 
 #