Add :recursion_limit option to Protobuf.JSON decoding

Closes #434.

Also knocks out a few conformance tests.

Author: whatyouhide

conformance/exemptions.txt

@@ -6,9 +6,6 @@ Recommended.Proto2.ProtobufInput.ValidMessageSetEncoding.SubmessageEncoding.NotU
 Recommended.Proto3.JsonInput.FieldNameDuplicate
 Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing1
 Recommended.Proto3.JsonInput.FieldNameDuplicateDifferentCasing2
-Recommended.Proto3.JsonInput.ListValueDeepNesting200
-Recommended.Proto3.JsonInput.StructDeepNesting200
-Recommended.Proto3.JsonInput.ValueDeepNesting200
 Required.Proto2.ProtobufInput.BadTag_FieldNumberSlightlyTooHigh
 Required.Proto2.ProtobufInput.BadTag_FieldNumberTooHigh
 Required.Proto2.ProtobufInput.BadTag_OverlongVarint

lib/protobuf/json.ex

@@ -110,7 +110,9 @@ defmodule Protobuf.JSON do
   A decoding option.
   """
   @typedoc since: "0.17.0"
-  @type decode_opt() :: {:ignore_unknown_fields, boolean()}
+  @type decode_opt() ::
+          {:ignore_unknown_fields, boolean()}
+          | {:recursion_limit, pos_integer()}
 
   @type json_data() :: %{optional(binary) => any}
 
@@ -289,7 +291,14 @@ defmodule Protobuf.JSON do
     * `:ignore_unknown_fields` (boolean): when `true`, unknown enum string values are
       treated as if the field was unset (and skipped from repeated fields and
       map values) instead of raising. Unknown JSON object keys are always
-      ignored regardless of this option. Defaults to `false`.
+      ignored regardless of this option. Defaults to `false`. *Available
+      since v0.17.0*.
+
+    * `:recursion_limit` (positive integer): the maximum nesting depth allowed
+      when decoding the dynamically-typed `Google.Protobuf.Value`,
+      `Google.Protobuf.ListValue`, and `Google.Protobuf.Struct` wrappers.
+      Exceeding it raises a `Protobuf.JSON.DecodeError`. Defaults to `100`. *Available
+      since v0.17.0*.
 
   ## Examples
 
@@ -376,23 +385,29 @@ defmodule Protobuf.JSON do
   @spec from_decoded(json_data(), module(), [decode_opt]) ::
           {:ok, struct()} | {:error, DecodeError.t()}
   def from_decoded(json_data, module, options \\ []) when is_atom(module) and is_list(options) do
-    decode_opts =
-      Enum.reduce(options, %{ignore_unknown_fields: false}, fn
-        {:ignore_unknown_fields, value}, acc when is_boolean(value) ->
-          Map.put(acc, :ignore_unknown_fields, value)
+    Enum.each(options, fn
+      {:ignore_unknown_fields, value} when is_boolean(value) ->
+        :ok
+
+      {:ignore_unknown_fields, value} ->
+        raise ArgumentError,
+              "option :ignore_unknown_fields must be a boolean, got: #{inspect(value)}"
+
+      {:recursion_limit, value} when is_integer(value) and value > 0 ->
+        :ok
 
-        {:ignore_unknown_fields, value}, _acc ->
-          raise ArgumentError,
-                "option :ignore_unknown_fields must be a boolean, got: #{inspect(value)}"
+      {:recursion_limit, value} ->
+        raise ArgumentError,
+              "option :recursion_limit must be a positive integer, got: #{inspect(value)}"
 
-        {key, _value}, _acc ->
-          raise ArgumentError, "unknown option: #{inspect(key)}"
+      {key, _value} ->
+        raise ArgumentError, "unknown option: #{inspect(key)}"
 
-        other, _acc ->
-          raise ArgumentError, "invalid element in options list: #{inspect(other)}"
-      end)
+      other ->
+        raise ArgumentError, "invalid element in options list: #{inspect(other)}"
+    end)
 
-    {:ok, Decode.from_json_data(json_data, module, decode_opts)}
+    {:ok, Decode.from_json_data(json_data, module, options)}
   catch
     error -> {:error, DecodeError.new(error)}
   end

lib/protobuf/json/decode.ex

@@ -50,17 +50,30 @@ defmodule Protobuf.JSON.Decode do
 
   @duration_seconds_range -315_576_000_000..315_576_000_000
 
-  @type opts() :: %{ignore_unknown_fields: boolean()}
+  # Default recursion limit, matching upstream Protobuf
+  # (ParseOptions::recursion_depth in C++, .recursion_limit in Java/Go).
+  # Applies to the dynamically-typed wrappers (Value/ListValue/Struct) where
+  # nesting is bounded only by the input, not the schema.
+  @default_recursion_limit 100
 
-  @spec from_json_data(term(), module(), opts()) :: struct()
-  def from_json_data(term, module, opts \\ %{ignore_unknown_fields: false})
+  @spec from_json_data(term(), module(), keyword()) :: struct()
+  def from_json_data(term, module, opts) when is_list(opts) and is_atom(module) do
+    state = %{
+      ignore_unknown_fields: Keyword.get(opts, :ignore_unknown_fields, false),
+      recursion_limit: Keyword.get(opts, :recursion_limit, @default_recursion_limit),
+      depth: 0
+    }
+
+    internal_from_json_data(term, module, state)
+  end
 
   # We start with all the Google built-in types that have specially-defined JSON decoding rules.
   # These rules are listed here: https://developers.google.com/protocol-buffers/docs/proto3#json
   # Note that we always have to keep the module names for the built-in types dynamic because
   # these built-in types **do not ship with our library**.
 
-  def from_json_data(string, Google.Protobuf.Duration = mod, _opts) when is_binary(string) do
+  defp internal_from_json_data(string, Google.Protobuf.Duration = mod, _state)
+       when is_binary(string) do
     # We need to check the sign from the raw string itself and can't rely on Integer.parse/1. This
     # is because if seconds is 0, then we couldn't determine whether it was "0" or "-0". For
     # example, "-0.5s".
@@ -81,63 +94,70 @@ defmodule Protobuf.JSON.Decode do
     end
   end
 
-  def from_json_data(string, Google.Protobuf.Timestamp = mod, _opts) when is_binary(string) do
+  defp internal_from_json_data(string, Google.Protobuf.Timestamp = mod, _state)
+       when is_binary(string) do
     case Protobuf.JSON.RFC3339.decode(string) do
       {:ok, seconds, nanos} -> struct!(mod, seconds: seconds, nanos: nanos)
       {:error, reason} -> throw({:bad_timestamp, string, reason})
     end
   end
 
-  def from_json_data(map, Google.Protobuf.Empty = mod, _opts) when map == %{} do
+  defp internal_from_json_data(map, Google.Protobuf.Empty = mod, _state) when map == %{} do
     struct!(mod)
   end
 
-  def from_json_data(int, Google.Protobuf.Int32Value = mod, _opts),
+  defp internal_from_json_data(int, Google.Protobuf.Int32Value = mod, _state),
     do: struct!(mod, value: decode_scalar(:int32, :unknown_name, int))
 
-  def from_json_data(int, Google.Protobuf.UInt32Value = mod, _opts),
+  defp internal_from_json_data(int, Google.Protobuf.UInt32Value = mod, _state),
     do: struct!(mod, value: decode_scalar(:uint32, :unknown_name, int))
 
-  def from_json_data(int, Google.Protobuf.UInt64Value = mod, _opts),
+  defp internal_from_json_data(int, Google.Protobuf.UInt64Value = mod, _state),
     do: struct!(mod, value: decode_scalar(:uint64, :unknown_name, int))
 
-  def from_json_data(int, Google.Protobuf.Int64Value = mod, _opts),
+  defp internal_from_json_data(int, Google.Protobuf.Int64Value = mod, _state),
     do: struct!(mod, value: decode_scalar(:int64, :unknown_name, int))
 
-  def from_json_data(number, mod, _opts)
-      when mod in [
-             Google.Protobuf.FloatValue,
-             Google.Protobuf.DoubleValue
-           ] and (is_float(number) or is_integer(number)) do
+  defp internal_from_json_data(number, mod, _state)
+       when mod in [
+              Google.Protobuf.FloatValue,
+              Google.Protobuf.DoubleValue
+            ] and (is_float(number) or is_integer(number)) do
     struct!(mod, value: number * 1.0)
   end
 
-  def from_json_data(bool, Google.Protobuf.BoolValue = mod, _opts) when is_boolean(bool) do
+  defp internal_from_json_data(bool, Google.Protobuf.BoolValue = mod, _state)
+       when is_boolean(bool) do
     struct!(mod, value: decode_scalar(:bool, :unknown_field, bool))
   end
 
-  def from_json_data(string, Google.Protobuf.StringValue = mod, _opts) when is_binary(string) do
+  defp internal_from_json_data(string, Google.Protobuf.StringValue = mod, _state)
+       when is_binary(string) do
     struct!(mod, value: decode_scalar(:string, :unknown_field, string))
   end
 
-  def from_json_data(bytes, Google.Protobuf.BytesValue = mod, _opts) when is_binary(bytes) do
+  defp internal_from_json_data(bytes, Google.Protobuf.BytesValue = mod, _state)
+       when is_binary(bytes) do
     struct!(mod, value: decode_scalar(:bytes, :unknown_field, bytes))
   end
 
-  def from_json_data(list, Google.Protobuf.ListValue = mod, opts) when is_list(list) do
-    struct!(mod, values: Enum.map(list, &from_json_data(&1, Google.Protobuf.Value, opts)))
+  defp internal_from_json_data(list, Google.Protobuf.ListValue = mod, state) when is_list(list) do
+    state = increase_depth_and_maybe_throw(state)
+    struct!(mod, values: Enum.map(list, &internal_from_json_data(&1, Google.Protobuf.Value, state)))
   end
 
-  def from_json_data(struct, Google.Protobuf.Struct = mod, opts) when is_map(struct) do
+  defp internal_from_json_data(struct, Google.Protobuf.Struct = mod, state) when is_map(struct) do
+    state = increase_depth_and_maybe_throw(state)
+
     fields =
       Map.new(struct, fn {key, val} ->
-        {key, from_json_data(val, Google.Protobuf.Value, opts)}
+        {key, internal_from_json_data(val, Google.Protobuf.Value, state)}
       end)
 
     struct!(mod, fields: fields)
   end
 
-  def from_json_data(term, Google.Protobuf.Value = mod, opts) do
+  defp internal_from_json_data(term, Google.Protobuf.Value = mod, state) do
     cond do
       is_nil(term) ->
         struct!(mod, kind: {:null_value, :NULL_VALUE})
@@ -155,17 +175,22 @@ defmodule Protobuf.JSON.Decode do
         struct!(mod, kind: {:bool_value, term})
 
       is_list(term) ->
-        struct!(mod, kind: {:list_value, from_json_data(term, Google.Protobuf.ListValue, opts)})
+        struct!(mod,
+          kind: {:list_value, internal_from_json_data(term, Google.Protobuf.ListValue, state)}
+        )
 
       is_map(term) ->
-        struct!(mod, kind: {:struct_value, from_json_data(term, Google.Protobuf.Struct, opts)})
+        struct!(mod,
+          kind: {:struct_value, internal_from_json_data(term, Google.Protobuf.Struct, state)}
+        )
 
       true ->
         throw({:bad_message, term, mod})
     end
   end
 
-  def from_json_data(data, Google.Protobuf.FieldMask = mod, _opts) when is_binary(data) do
+  defp internal_from_json_data(data, Google.Protobuf.FieldMask = mod, _state)
+       when is_binary(data) do
     paths = String.split(data, ",")
 
     cond do
@@ -175,7 +200,7 @@ defmodule Protobuf.JSON.Decode do
     end
   end
 
-  def from_json_data(%{"@type" => type_url} = data, Google.Protobuf.Any = mod, opts) do
+  defp internal_from_json_data(%{"@type" => type_url} = data, Google.Protobuf.Any = mod, state) do
     data = Map.delete(data, "@type")
     message_mod = Protobuf.Any.type_url_to_module(type_url)
 
@@ -186,35 +211,41 @@ defmodule Protobuf.JSON.Decode do
         # See: https://developers.google.com/protocol-buffers/docs/proto3#json
         {:ok, value} ->
           value
-          |> from_json_data(message_mod, opts)
+          |> internal_from_json_data(message_mod, state)
           |> message_mod.encode()
 
         # When a message doesn't have a built-in JSON representation (like
         # google.protobuf.Timestamp), then it's encoded as a JSON object and then a @type field is
         # added with the type_url for that message.
         :error ->
           data
-          |> from_json_data(message_mod, opts)
+          |> internal_from_json_data(message_mod, state)
           |> message_mod.encode()
       end
 
     struct!(mod, type_url: type_url, value: encoded)
   end
 
-  def from_json_data(data, module, opts) when is_map(data) and is_atom(module) do
+  defp internal_from_json_data(data, module, state) when is_map(data) and is_atom(module) do
     message_props = module.__message_props__()
-    regular = decode_regular_fields(data, message_props, opts)
-    oneofs = decode_oneof_fields(data, message_props, opts)
+    regular = decode_regular_fields(data, message_props, state)
+    oneofs = decode_oneof_fields(data, message_props, state)
 
     module
     |> struct(regular)
     |> struct(oneofs)
     |> transform_module(module)
   end
 
-  def from_json_data(data, module, _opts) when is_atom(module),
+  defp internal_from_json_data(data, module, _state) when is_atom(module),
     do: throw({:bad_message, data, module})
 
+  defp increase_depth_and_maybe_throw(%{depth: depth, recursion_limit: limit} = opts) do
+    depth = depth + 1
+    if depth > limit, do: throw({:recursion_limit_exceeded, limit})
+    %{opts | depth: depth}
+  end
+
   defp convert_field_mask_to_underscore(mask) do
     if mask =~ ~r/^[a-zA-Z0-9\.]+$/ do
       String.split(mask, ".")
@@ -353,8 +384,8 @@ defmodule Protobuf.JSON.Decode do
     end)
   end
 
-  defp decode_singular(%{type: module, embedded?: true}, value, opts) do
-    from_json_data(value, module, opts)
+  defp decode_singular(%{type: module, embedded?: true}, value, state) do
+    internal_from_json_data(value, module, state)
   end
 
   defp decode_scalar(:string, name, value) do

lib/protobuf/json/decode_error.ex

@@ -70,6 +70,10 @@ defmodule Protobuf.JSON.DecodeError do
     %__MODULE__{message: "Repeated field '#{field}' expected a list, got #{inspect(value)}"}
   end
 
+  def new({:recursion_limit_exceeded, limit}) do
+    %__MODULE__{message: "JSON value exceeds the recursion limit of #{limit}"}
+  end
+
   def new({:unexpected_end, position}) do
     %__MODULE__{message: "Unexpected end at position #{inspect(position)}"}
   end

test/protobuf/json/decode_test.exs

@@ -998,4 +998,52 @@ defmodule Protobuf.JSON.DecodeTest do
                }
     end
   end
+
+  describe ":recursion_limit option" do
+    @default_limit 100
+    defp depth_error(limit), do: error("JSON value exceeds the recursion limit of #{limit}")
+
+    test "nested Google.Protobuf.ListValue beyond the default limit is rejected" do
+      nested = Enum.reduce(1..(@default_limit + 1), 1, fn _, acc -> [acc] end)
+      assert decode(nested, Google.Protobuf.ListValue) == depth_error(@default_limit)
+    end
+
+    test "nested Google.Protobuf.Struct beyond the default limit is rejected" do
+      nested = Enum.reduce(1..(@default_limit + 1), 1, fn _, acc -> %{"k" => acc} end)
+      assert decode(nested, Google.Protobuf.Struct) == depth_error(@default_limit)
+    end
+
+    test "nested Google.Protobuf.Value beyond the default limit is rejected" do
+      # Value itself doesn't add a level; nesting comes from the wrapped lists.
+      nested = Enum.reduce(1..(@default_limit + 1), 1, fn _, acc -> [acc] end)
+      assert decode(nested, Google.Protobuf.Value) == depth_error(@default_limit)
+    end
+
+    test "nesting up to the default limit is accepted" do
+      nested = Enum.reduce(1..@default_limit, 1, fn _, acc -> [acc] end)
+      assert {:ok, %Google.Protobuf.ListValue{}} = decode(nested, Google.Protobuf.ListValue)
+    end
+
+    test "custom :recursion_limit caps nesting at the requested depth" do
+      nested = Enum.reduce(1..6, 1, fn _, acc -> [acc] end)
+
+      assert Protobuf.JSON.from_decoded(nested, Google.Protobuf.ListValue, recursion_limit: 5) ==
+               depth_error(5)
+
+      assert {:ok, %Google.Protobuf.ListValue{}} =
+               Protobuf.JSON.from_decoded(nested, Google.Protobuf.ListValue, recursion_limit: 6)
+    end
+
+    test "invalid :recursion_limit raises ArgumentError" do
+      for bad <- [0, -1, 1.0, "100", nil] do
+        assert_raise ArgumentError,
+                     "option :recursion_limit must be a positive integer, got: #{inspect(bad)}",
+                     fn ->
+                       Protobuf.JSON.from_decoded(%{}, Google.Protobuf.Struct,
+                         recursion_limit: bad
+                       )
+                     end
+      end
+    end
+  end
 end