compiler: Fix bug in beam_types:subtract/2 for bitstrings

Fix #11494.
The bug caused a mismatch between type pass and validator.
Valid code would be rejected by the validator.

Author: lucioleKi

lib/compiler/src/beam_types.erl

@@ -399,11 +399,10 @@ subtract(#t_atom{elements=[_|_]=Set0}, #t_atom{elements=[_|_]=Set1}) ->
     end;
 subtract(#t_bitstring{size_unit=UnitA}=T, #t_bs_matchable{tail_unit=UnitB}) ->
     subtract_matchable(T, UnitA, UnitB);
-subtract(#t_bitstring{appendable=App,size_unit=UnitA}=T,
-         #t_bitstring{appendable=App,size_unit=UnitB}) ->
-    subtract_matchable(T, UnitA, UnitB);
-subtract(#t_bitstring{}=T, #t_bitstring{}) ->
+subtract(#t_bitstring{appendable=false}=T, #t_bitstring{appendable=true}) ->
     T;
+subtract(#t_bitstring{size_unit=UnitA}=T, #t_bitstring{size_unit=UnitB}) ->
+    subtract_matchable(T, UnitA, UnitB);
 subtract(#t_bs_context{tail_unit=UnitA}=T, #t_bs_matchable{tail_unit=UnitB}) ->
     subtract_matchable(T, UnitA, UnitB);
 subtract(#t_bs_context{tail_unit=UnitA}=T, #t_bs_context{tail_unit=UnitB}) ->

lib/compiler/test/beam_ssa_SUITE.erl

@@ -27,7 +27,7 @@
          mapfoldl/0,mapfoldl/1,
          grab_bag/1,redundant_br/1,
          coverage/1,normalize/1,
-         trycatch/1,gh_6599/1]).
+         trycatch/1,gh_6599/1,gh_11494/1]).
 
 suite() -> [{ct_hooks,[ts_install_cth]}].
 
@@ -51,7 +51,8 @@ groups() ->
        coverage,
        normalize,
        trycatch,
-       gh_6599
+       gh_6599,
+       gh_11494
       ]}].
 
 init_per_suite(Config) ->
@@ -1555,6 +1556,17 @@ gh_6599_7(X, Y) ->
         ok
     end.
 
+gh_11494(_Config) ->
+    inspect([100]),
+    inspect(<<>>),
+    ok.
+
+inspect(Value) ->
+    case Value of
+        Res when erlang:is_integer(Res);
+                 erlang:is_bitstring(Res) -> Res;
+        [Num] -> inspect(Num)
+    end.
 
 %% The identity function.
 id(I) -> I.