Use string_slice in ascii_string and binary_string (#155)

Author: dkuku

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog for NimbleParsec
 
+## Unreleased
+
+### Enhancements
+
+  * Compile `ascii_string/3` and `utf8_string/3` with `:min`/`:max` over a character
+    class that cannot match a newline into a scan plus a single slice of the input,
+    rather than accumulating a codepoint list and joining it. Between 1.1x and 7.6x
+    faster depending on how much is matched. Note the resulting string may be a
+    sub-binary of the input, so see the memory usage notes in the documentation if
+    you retain long strings from a much larger input
+
+### Bug fixes
+
+  * `ascii_string/3` and `utf8_string/3` nested in `choice/2`, `lookahead/2` or
+    `lookahead_not/2` now produce an error message naming the character class once,
+    instead of repeating it `:min` plus one times
+
 ## v1.4.2 (2025-01-21)
 
 ### Enhancements

lib/nimble_parsec.ex

@@ -231,6 +231,7 @@ defmodule NimbleParsec do
   #   1. Update the combinator type below
   #   2. Update the compiler with combinator
   #   3. Update the compiler with label step
+  #   4. Update the generator with combinator
   #
   @typep combinator :: bound_combinator | maybe_bound_combinator | unbound_combinator
 
@@ -250,6 +251,8 @@ defmodule NimbleParsec do
            | {:lookahead, t, :positive | :negative}
            | {:parsec, atom | {module, atom}}
            | {:repeat, t, mfargs, gen_times}
+           | {:string_slice, [inclusive_range], [exclusive_range], bin_modifier, non_neg_integer,
+              pos_integer | nil}
            | {:times, t, pos_integer}
 
   @doc ~S"""
@@ -377,6 +380,12 @@ defmodule NimbleParsec do
     generate(parsecs, mod, gen_times(t, int_random(gen), mod, acc))
   end
 
+  defp generate([{:string_slice, inclusive, exclusive, modifier, min, max} | parsecs], mod, acc) do
+    extra = if max, do: Enum.random(0..(max - min)), else: int_random(nil)
+    segment = [{:bin_segment, inclusive, exclusive, modifier}]
+    generate(parsecs, mod, gen_times(segment, min + extra, mod, acc))
+  end
+
   defp generate([{:times, t, max} | parsecs], mod, acc) do
     generate(parsecs, mod, gen_times(t, Enum.random(0..max), mod, acc))
   end
@@ -2004,6 +2013,13 @@ defmodule NimbleParsec do
        when is_list(opts) do
     {min, max} = validate_min_and_max!(opts)
 
+    case NimbleParsec.Compiler.string_slice(to_repeat, runtime, min, max) do
+      nil -> min_max_runtime_chars(combinator, to_repeat, min, max, compile, runtime, args)
+      slice -> [slice | combinator]
+    end
+  end
+
+  defp min_max_runtime_chars(combinator, to_repeat, min, max, compile, runtime, args) do
     chars =
       if min > 0 do
         min_max_compile_runtime_chars(empty(), to_repeat, min, compile, runtime, args)

lib/nimble_parsec/compiler.ex

@@ -231,6 +231,64 @@ defmodule NimbleParsec.Compiler do
     compile_unbound_traverse(combinators, kind, current, step, config, fun)
   end
 
+  defp compile_unbound_combinator(
+         {:string_slice, inclusive, exclusive, modifier, min, max},
+         current,
+         step,
+         config
+       ) do
+    {scan, step} = build_next(step, config)
+    {next, step} = build_next(step, config)
+
+    {var, _} = build_var(0)
+    input = apply_bin_modifier(var, modifier)
+
+    {count, init, incr, done} =
+      if max do
+        {[quote(do: count)], [0], [quote(do: count + 1)], [quote(do: _count)]}
+      else
+        {[], [], [], []}
+      end
+
+    args = fn rest, count ->
+      quote(do: [unquote(rest), orig, unquote_splicing(count), acc, stack, context, line, offset])
+    end
+
+    guards = compile_bin_ranges(var, inclusive, exclusive)
+    guards = if max, do: guards ++ [quote(do: count < unquote(max - min))], else: guards
+
+    recur_def =
+      {scan, args.(quote(do: <<unquote(input), rest::binary>>), count),
+       guards_list_to_quoted(guards), {scan, [], args.(quote(do: rest), incr)}}
+
+    slice =
+      if config.replace, do: quote(do: acc), else: quote(do: [binary_part(orig, 0, len) | acc])
+
+    done_def =
+      {scan, args.(quote(do: rest), done), true,
+       quote do
+         len = byte_size(orig) - byte_size(rest)
+         unquote(next)(rest, unquote(slice), stack, context, line, offset + len)
+       end}
+
+    {orig, guards, rest, failure_defs, inline, step} =
+      if min > 0 do
+        segments = List.duplicate({:bin_segment, inclusive, exclusive, modifier}, min)
+        {[], inputs, guards, _, _, metadata} = take_bound_combinators(segments)
+        {prefix, _} = compile_bound_bin_pattern(inputs, metadata, quote(do: rest))
+        failure_defs = [build_catch_all(:positive, current, segments, config)]
+
+        {quote(do: unquote(prefix) = orig), guards, quote(do: rest), failure_defs, [], step}
+      else
+        {quote(do: orig), [], quote(do: orig), [], [{current, @arity}], step}
+      end
+
+    head = quote(do: [unquote(orig), acc, stack, context, line, offset])
+    entry_def = {current, head, guards_list_to_quoted(guards), {scan, [], args.(rest, init)}}
+
+    {[entry_def | failure_defs] ++ [recur_def, done_def], inline, next, step, :catch_none}
+  end
+
   defp compile_unbound_combinator({:times, combinators, count}, current, step, config) do
     if all_no_context_combinators?(combinators) do
       compile_bound_times(combinators, count, current, step, config)
@@ -887,7 +945,7 @@ defmodule NimbleParsec.Compiler do
       end
 
     line =
-      if newline_allowed?(inclusive) and not newline_forbidden?(exclusive) do
+      if newline_possible?(inclusive, exclusive) do
         add_line(line, offset, var)
       else
         line
@@ -959,6 +1017,26 @@ defmodule NimbleParsec.Compiler do
     {:+, [], [var, extra]}
   end
 
+  @doc """
+  Lowers a repeated character class into a `:string_slice` combinator, if possible.
+
+  A class that may match a newline is left alone, as line tracking happens per
+  codepoint on the accumulator path.
+  """
+  def string_slice([{:bin_segment, ors, ands, modifier}], :__runtime_string__, min, max) do
+    if not newline_possible?(ors, ands) do
+      {:string_slice, ors, ands, modifier, min, max}
+    end
+  end
+
+  def string_slice(_to_repeat, _runtime, _min, _max), do: nil
+
+  # Whether a character class can match a newline, and therefore whether the
+  # combinator consuming it has to track lines at all.
+  defp newline_possible?(inclusive, exclusive) do
+    newline_allowed?(inclusive) and not newline_forbidden?(exclusive)
+  end
+
   defp newline_allowed?([]), do: true
 
   defp newline_allowed?(ors) do
@@ -1024,6 +1102,9 @@ defmodule NimbleParsec.Compiler do
     prefix <> Enum.join([Enum.join(inclusive, " or") | exclusive], ", and not")
   end
 
+  defp label({:string_slice, inclusive, exclusive, modifier, _min, _max}),
+    do: label({:bin_segment, inclusive, exclusive, modifier})
+
   defp label(:eos) do
     "end of string"
   end

test/nimble_parsec_test.exs

@@ -220,6 +220,7 @@ defmodule NimbleParsecTest do
     defparsecp :min_ascii_string, ascii_string([?0..?9], min: 2)
     defparsecp :max_ascii_string, ascii_string([?0..?9], max: 3)
     defparsecp :min_max_ascii_string, ascii_string([?0..?9], min: 2, max: 3)
+    defparsecp :min_zero_ascii_string, ascii_string([?0..?9], min: 0)
 
     @error "expected ASCII character in the range \"0\" to \"9\", followed by ASCII character in the range \"0\" to \"9\""
 
@@ -229,6 +230,11 @@ defmodule NimbleParsecTest do
       assert min_ascii_string("123o") == {:ok, ["123"], "o", %{}, {1, 0}, 3}
       assert min_ascii_string("1234") == {:ok, ["1234"], "", %{}, {1, 0}, 4}
       assert min_ascii_string("1") == {:error, @error, "1", %{}, {1, 0}, 0}
+
+      assert min_zero_ascii_string("12") == {:ok, ["12"], "", %{}, {1, 0}, 2}
+      assert min_zero_ascii_string("123o") == {:ok, ["123"], "o", %{}, {1, 0}, 3}
+      assert min_zero_ascii_string("o") == {:ok, [""], "o", %{}, {1, 0}, 0}
+      assert min_zero_ascii_string("") == {:ok, [""], "", %{}, {1, 0}, 0}
     end
 
     test "returns ok/error with max" do
@@ -247,6 +253,10 @@ defmodule NimbleParsecTest do
       assert min_max_ascii_string("12o") == {:ok, ["12"], "o", %{}, {1, 0}, 2}
     end
 
+    test "treats a max without a min as min: 0" do
+      assert ascii_string([?0..?9], max: 3) == ascii_string([?0..?9], min: 0, max: 3)
+    end
+
     test "is not bound" do
       assert not_bound?(ascii_string([?0..?9], min: 3))
       assert not_bound?(ascii_string([?0..?9], max: 3))
@@ -309,6 +319,10 @@ defmodule NimbleParsecTest do
       assert min_max_utf8_string("áé\xFF") == {:ok, ["áé"], "\xFF", %{}, {1, 0}, 4}
     end
 
+    test "treats a max without a min as min: 0" do
+      assert utf8_string([], max: 3) == utf8_string([], min: 0, max: 3)
+    end
+
     test "is not bound" do
       assert not_bound?(utf8_string([], min: 3))
       assert not_bound?(utf8_string([], max: 3))
@@ -320,6 +334,7 @@ defmodule NimbleParsecTest do
     defparsecp :min_sliced_utf8_string, utf8_string([?a..?z, ?á..?é], min: 2)
     defparsecp :max_sliced_utf8_string, utf8_string([?a..?z, ?á..?é], max: 3)
     defparsecp :min_max_sliced_utf8_string, utf8_string([?a..?z, ?á..?é], min: 2, max: 3)
+    defparsecp :min_zero_sliced_utf8_string, utf8_string([?a..?z, ?á..?é], min: 0)
     defparsecp :not_newline_utf8_string, utf8_string([not: ?\n], min: 1)
 
     @error "expected utf8 codepoint in the range \"a\" to \"z\" or in the range \"á\" to \"é\", " <>
@@ -331,6 +346,11 @@ defmodule NimbleParsecTest do
       assert min_sliced_utf8_string("áé1") == {:ok, ["áé"], "1", %{}, {1, 0}, 4}
       assert min_sliced_utf8_string("á") == {:error, @error, "á", %{}, {1, 0}, 0}
       assert min_sliced_utf8_string("1á") == {:error, @error, "1á", %{}, {1, 0}, 0}
+
+      assert min_zero_sliced_utf8_string("áé") == {:ok, ["áé"], "", %{}, {1, 0}, 4}
+      assert min_zero_sliced_utf8_string("aébc") == {:ok, ["aébc"], "", %{}, {1, 0}, 5}
+      assert min_zero_sliced_utf8_string("1á") == {:ok, [""], "1á", %{}, {1, 0}, 0}
+      assert min_zero_sliced_utf8_string("") == {:ok, [""], "", %{}, {1, 0}, 0}
     end
 
     test "returns ok/error with max" do
@@ -351,6 +371,46 @@ defmodule NimbleParsecTest do
     end
   end
 
+  describe "ascii_string/3 over a newline-free range nested in other combinators" do
+    defparsecp :lookahead_sliced,
+               lookahead(ascii_string([?a..?z], min: 2)) |> concat(string("ab"))
+
+    defparsecp :lookahead_not_sliced,
+               lookahead_not(ascii_string([?a..?z], min: 2)) |> concat(string("12"))
+
+    defparsecp :choice_sliced,
+               choice([ascii_string([?a..?z], min: 2), ascii_string([?0..?9], min: 2)])
+
+    # These combinators build their error message from the labels of what they
+    # wrap. A repeated character class names the class once, regardless of `min`.
+    @lower "ASCII character in the range \"a\" to \"z\""
+    @digit "ASCII character in the range \"0\" to \"9\""
+
+    test "lookahead names the character class once" do
+      assert lookahead_sliced("abc") == {:ok, ["ab"], "c", %{}, {1, 0}, 2}
+
+      assert lookahead_sliced("1ab") ==
+               {:error, "expected " <> @lower, "1ab", %{}, {1, 0}, 0}
+
+      assert lookahead_sliced("a") == {:error, "expected " <> @lower, "a", %{}, {1, 0}, 0}
+    end
+
+    test "lookahead_not names the character class once" do
+      assert lookahead_not_sliced("12") == {:ok, ["12"], "", %{}, {1, 0}, 2}
+
+      assert lookahead_not_sliced("ab12") ==
+               {:error, "did not expect " <> @lower, "ab12", %{}, {1, 0}, 0}
+    end
+
+    test "choice names each character class once" do
+      assert choice_sliced("abc") == {:ok, ["abc"], "", %{}, {1, 0}, 3}
+      assert choice_sliced("12x") == {:ok, ["12"], "x", %{}, {1, 0}, 2}
+
+      assert choice_sliced("!") ==
+               {:error, "expected #{@lower} or #{@digit}", "!", %{}, {1, 0}, 0}
+    end
+  end
+
   describe "ignore/2 combinator at compile time" do
     defparsecp :compile_ignore, ignore(string("TO"))
     defparsecp :compile_ignore_with_newline, ignore(string("T\nO"))