Fix edge cases and add property tests

Author: lpgauth

README.md

@@ -189,6 +189,11 @@ Run benchmarks locally:
 MIX_ENV=bench mix run bench/torque_bench.exs
 ```
 
+## Limitations
+
+- **Nesting depth**: JSON documents nested deeper than 512 levels return `{:error, :nesting_too_deep}` from `decode/1`, `get/2`, and `encode/1` rather than crashing the VM. Real-world documents are never this deep; the limit exists to prevent stack overflow in the NIF.
+- **Numeric string keys**: Object keys that are pure integers (e.g. `"0"`, `"42"`) cannot be addressed via JSON Pointer because the pointer walker treats numeric path segments as array indices. Use `decode/1` if you need to access such keys.
+
 ## License
 
 MIT

lib/torque.ex

@@ -54,7 +54,8 @@ defmodule Torque do
       {:ok, "example.com"} = Torque.get(doc, "/site/domain")
       {:error, :no_such_field} = Torque.get(doc, "/missing")
   """
-  @spec get(reference(), binary()) :: {:ok, term()} | {:error, :no_such_field}
+  @spec get(reference(), binary()) ::
+          {:ok, term()} | {:error, :no_such_field | :nesting_too_deep}
   def get(doc, path) when is_reference(doc) and is_binary(path) do
     Torque.Native.get(doc, path)
   end
@@ -75,6 +76,7 @@ defmodule Torque do
     case Torque.Native.get(doc, path) do
       {:ok, value} -> value
       {:error, :no_such_field} -> default
+      {:error, reason} -> raise ArgumentError, "get error: #{reason}"
     end
   end
 
@@ -93,7 +95,8 @@ defmodule Torque do
       [{:ok, 1}, {:ok, 2}, {:error, :no_such_field}] =
         Torque.get_many(doc, ["/a", "/b", "/c"])
   """
-  @spec get_many(reference(), [binary()]) :: [{:ok, term()} | {:error, :no_such_field}]
+  @spec get_many(reference(), [binary()]) ::
+          [{:ok, term()} | {:error, :no_such_field | :nesting_too_deep}]
   def get_many(doc, paths) when is_reference(doc) and is_list(paths) do
     Torque.Native.get_many(doc, paths)
   end
@@ -141,7 +144,7 @@ defmodule Torque do
 
   Automatically uses a dirty CPU scheduler for inputs larger than 10 KB.
   """
-  @spec decode(binary()) :: {:ok, term()} | {:error, binary()}
+  @spec decode(binary()) :: {:ok, term()} | {:error, binary() | :nesting_too_deep}
   def decode(json) when is_binary(json) and byte_size(json) > @timeslice_bytes do
     Torque.Native.decode_dirty(json)
   end
@@ -176,7 +179,7 @@ defmodule Torque do
     * Other atoms (encoded as JSON strings)
     * `{keyword_list}` tuples (jiffy-style proplist objects)
   """
-  @spec encode(term()) :: {:ok, binary()} | {:error, binary()}
+  @spec encode(term()) :: {:ok, binary()} | {:error, binary() | :nesting_too_deep}
   def encode(term) do
     Torque.Native.encode(term)
   end

mix.exs

@@ -29,6 +29,7 @@ defmodule Torque.MixProject do
       {:ex_doc, "~> 0.35", only: :dev, runtime: false},
       {:dialyxir, "~> 1.4", only: :dev, runtime: false},
       {:jason, "~> 1.4"},
+      {:stream_data, "~> 1.1", only: :test},
       {:simdjsone, "~> 0.5.0", only: :bench},
       {:jiffy, "~> 1.1", only: :bench},
       {:benchee, "~> 1.3", only: :bench}

mix.lock

@@ -16,5 +16,6 @@
   "rustler_precompiled": {:hex, :rustler_precompiled, "0.8.4", "700a878312acfac79fb6c572bb8b57f5aae05fe1cf70d34b5974850bbf2c05bf", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:rustler, "~> 0.23", [hex: :rustler, repo: "hexpm", optional: true]}], "hexpm", "3b33d99b540b15f142ba47944f7a163a25069f6d608783c321029bc1ffb09514"},
   "simdjsone": {:hex, :simdjsone, "0.5.0", "31f0d90f14005c6d984b479ccdbd3cd871536f6e1e61b05760a9682b511e4dd2", [:rebar3], [], "hexpm", "5d4addaa8095f6ace8ca2a4ecf76965e63bc5b25b863dc4b0a15c7de6c3f6856"},
   "statistex": {:hex, :statistex, "1.1.0", "7fec1eb2f580a0d2c1a05ed27396a084ab064a40cfc84246dbfb0c72a5c761e5", [:mix], [], "hexpm", "f5950ea26ad43246ba2cce54324ac394a4e7408fdcf98b8e230f503a0cba9cf5"},
+  "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"},
   "toml": {:hex, :toml, "0.7.0", "fbcd773caa937d0c7a02c301a1feea25612720ac3fa1ccb8bfd9d30d822911de", [:mix], [], "hexpm", "0690246a2478c1defd100b0c9b89b4ea280a22be9a7b313a8a058a2408a2fa70"},
 }

native/torque_nif/src/atoms.rs

@@ -2,6 +2,7 @@ rustler::atoms! {
     ok,
     error,
     no_such_field,
+    nesting_too_deep,
     nil,
     // atoms for fast identity comparison in encoder
     r#true = "true",

native/torque_nif/src/decoder.rs

@@ -1,13 +1,24 @@
 use crate::atoms;
 use crate::nif_util::make_tuple2;
-use crate::types::value_to_term;
+use crate::types::{value_to_term, MAX_DEPTH};
 use crate::ParsedDocument;
 use rustler::sys::{enif_make_list_from_array, ERL_NIF_TERM};
 use rustler::{Binary, Encoder, Env, ListIterator, ResourceArc, Term};
 use sonic_rs::{JsonContainerTrait, JsonValueTrait};
 
 const GET_MANY_STACK: usize = 64;
 
+/// Returns the last value for `key` in an object, matching the last-value-wins
+/// behaviour of `value_to_term` / `build_map_dedup` for duplicate keys.
+#[inline]
+fn object_get_last<'v>(value: &'v sonic_rs::Value, key: &str) -> Option<&'v sonic_rs::Value> {
+    value
+        .as_object()?
+        .iter()
+        .rfind(|(k, _)| *k == key)
+        .map(|(_, v)| v)
+}
+
 #[inline]
 fn pointer_lookup<'v>(value: &'v sonic_rs::Value, path: &str) -> Option<&'v sonic_rs::Value> {
     let bytes = path.as_bytes();
@@ -32,9 +43,9 @@ fn pointer_lookup<'v>(value: &'v sonic_rs::Value, path: &str) -> Option<&'v soni
         }
         if segment.contains('~') {
             let unescaped = segment.replace("~1", "/").replace("~0", "~");
-            current = current.get(&unescaped as &str)?;
+            current = object_get_last(current, &unescaped)?;
         } else {
-            current = current.get(segment)?;
+            current = object_get_last(current, segment)?;
         }
     }
     Some(current)
@@ -76,8 +87,12 @@ fn get<'a>(env: Env<'a>, doc: ResourceArc<ParsedDocument>, path: &str) -> Term<'
     let ok_raw = atoms::ok().as_c_arg();
     let err_raw = atoms::error().as_c_arg();
     let nsf_raw = atoms::no_such_field().as_c_arg();
+    let ntd_raw = atoms::nesting_too_deep().as_c_arg();
     match pointer_lookup(&doc.value, path) {
-        Some(value) => make_tuple2(env, ok_raw, value_to_term(env, value).as_c_arg()),
+        Some(value) => match value_to_term(env, value, MAX_DEPTH) {
+            Some(term) => make_tuple2(env, ok_raw, term.as_c_arg()),
+            None => make_tuple2(env, err_raw, ntd_raw),
+        },
         None => make_tuple2(env, err_raw, nsf_raw),
     }
 }
@@ -90,9 +105,13 @@ fn get_one_result(
     ok_raw: ERL_NIF_TERM,
     err_raw: ERL_NIF_TERM,
     nsf_raw: ERL_NIF_TERM,
+    ntd_raw: ERL_NIF_TERM,
 ) -> ERL_NIF_TERM {
     match pointer_lookup(&doc.value, path) {
-        Some(value) => make_tuple2(env, ok_raw, value_to_term(env, value).as_c_arg()).as_c_arg(),
+        Some(value) => match value_to_term(env, value, MAX_DEPTH) {
+            Some(term) => make_tuple2(env, ok_raw, term.as_c_arg()).as_c_arg(),
+            None => make_tuple2(env, err_raw, ntd_raw).as_c_arg(),
+        },
         None => make_tuple2(env, err_raw, nsf_raw).as_c_arg(),
     }
 }
@@ -106,6 +125,7 @@ fn get_many<'a>(
     let ok_raw = atoms::ok().as_c_arg();
     let err_raw = atoms::error().as_c_arg();
     let nsf_raw = atoms::no_such_field().as_c_arg();
+    let ntd_raw = atoms::nesting_too_deep().as_c_arg();
 
     // Collect into stack array when possible
     let mut stack: [ERL_NIF_TERM; GET_MANY_STACK] = [0; GET_MANY_STACK];
@@ -132,7 +152,7 @@ fn get_many<'a>(
             }
         };
 
-        let r = get_one_result(env, &doc, path, ok_raw, err_raw, nsf_raw);
+        let r = get_one_result(env, &doc, path, ok_raw, err_raw, nsf_raw, ntd_raw);
         if count < GET_MANY_STACK && heap.is_none() {
             stack[count] = r;
         } else {
@@ -177,11 +197,14 @@ fn array_length<'a>(env: Env<'a>, doc: ResourceArc<ParsedDocument>, path: &str)
 
 fn do_decode<'a>(env: Env<'a>, bytes: &[u8]) -> Term<'a> {
     match sonic_rs::from_slice::<sonic_rs::Value>(bytes) {
-        Ok(value) => make_tuple2(
-            env,
-            atoms::ok().as_c_arg(),
-            value_to_term(env, &value).as_c_arg(),
-        ),
+        Ok(value) => match value_to_term(env, &value, MAX_DEPTH) {
+            Some(term) => make_tuple2(env, atoms::ok().as_c_arg(), term.as_c_arg()),
+            None => make_tuple2(
+                env,
+                atoms::error().as_c_arg(),
+                atoms::nesting_too_deep().as_c_arg(),
+            ),
+        },
         Err(e) => make_tuple2(
             env,
             atoms::error().as_c_arg(),
@@ -232,7 +255,10 @@ fn get_many_nil<'a>(
         };
 
         let r = match pointer_lookup(&doc.value, path) {
-            Some(value) => value_to_term(env, value).as_c_arg(),
+            Some(value) => match value_to_term(env, value, MAX_DEPTH) {
+                Some(term) => term.as_c_arg(),
+                None => nil_raw,
+            },
             None => nil_raw,
         };
         if count < GET_MANY_STACK && heap.is_none() {

native/torque_nif/src/encoder.rs

@@ -1,16 +1,18 @@
 use crate::atoms;
 use crate::nif_util::make_tuple2;
+use crate::types::MAX_DEPTH;
 use rustler::sys::{
     c_int, c_uint, enif_get_atom, enif_get_atom_length, enif_get_double, enif_get_int64,
-    enif_get_list_cell, enif_get_tuple, enif_inspect_binary, ErlNifBinary, ErlNifCharEncoding,
-    ErlNifEnv, ERL_NIF_TERM,
+    enif_get_list_cell, enif_get_tuple, enif_get_uint64, enif_inspect_binary, ErlNifBinary,
+    ErlNifCharEncoding, ErlNifEnv, ERL_NIF_TERM,
 };
 use rustler::{Encoder, Env, MapIterator, NewBinary, Term, TermType};
 use std::mem::MaybeUninit;
 
 #[derive(Debug)]
 enum EncodeError {
     BadArg,
+    DepthExceeded,
 }
 
 /// Read an atom's name into a stack buffer without heap allocation.
@@ -48,14 +50,19 @@ unsafe fn atom_to_stack_buf(
 fn encode<'a>(env: Env<'a>, term: Term<'a>) -> Term<'a> {
     let mut buf: Vec<u8> = Vec::with_capacity(2048);
     let env_raw = env.as_c_arg();
-    match encode_term(env, env_raw, term, &mut buf) {
+    match encode_term(env, env_raw, term, &mut buf, MAX_DEPTH) {
         Ok(()) => {
             let mut binary = NewBinary::new(env, buf.len());
             binary.as_mut_slice().copy_from_slice(&buf);
             let bin_term: Term = binary.into();
             make_tuple2(env, atoms::ok().as_c_arg(), bin_term.as_c_arg())
         }
-        Err(_) => make_tuple2(
+        Err(EncodeError::DepthExceeded) => make_tuple2(
+            env,
+            atoms::error().as_c_arg(),
+            atoms::nesting_too_deep().as_c_arg(),
+        ),
+        Err(EncodeError::BadArg) => make_tuple2(
             env,
             atoms::error().as_c_arg(),
             "encode error".encode(env).as_c_arg(),
@@ -69,14 +76,17 @@ fn encode<'a>(env: Env<'a>, term: Term<'a>) -> Term<'a> {
 fn encode_iodata<'a>(env: Env<'a>, term: Term<'a>) -> Term<'a> {
     let mut buf: Vec<u8> = Vec::with_capacity(2048);
     let env_raw = env.as_c_arg();
-    match encode_term(env, env_raw, term, &mut buf) {
+    match encode_term(env, env_raw, term, &mut buf, MAX_DEPTH) {
         Ok(()) => {
             let mut binary = NewBinary::new(env, buf.len());
             binary.as_mut_slice().copy_from_slice(&buf);
             binary.into()
         }
-        Err(_) => unsafe {
-            let reason = "encode error".encode(env).as_c_arg();
+        Err(e) => unsafe {
+            let reason = match e {
+                EncodeError::DepthExceeded => atoms::nesting_too_deep().as_c_arg(),
+                EncodeError::BadArg => "encode error".encode(env).as_c_arg(),
+            };
             Term::new(env, rustler::sys::enif_raise_exception(env_raw, reason))
         },
     }
@@ -88,15 +98,16 @@ fn encode_term(
     env_raw: *mut ErlNifEnv,
     term: Term,
     buf: &mut Vec<u8>,
+    depth: u32,
 ) -> Result<(), EncodeError> {
     match term.get_type() {
-        TermType::Map => encode_map(env, env_raw, term, buf),
-        TermType::List => encode_list(env, env_raw, term, buf),
+        TermType::Map => encode_map(env, env_raw, term, buf, depth),
+        TermType::List => encode_list(env, env_raw, term, buf, depth),
         TermType::Binary => encode_binary(env_raw, term, buf),
         TermType::Integer => encode_integer(env_raw, term, buf),
         TermType::Float => encode_float(env_raw, term, buf),
         TermType::Atom => encode_atom(env_raw, term, buf),
-        TermType::Tuple => encode_tuple(env, env_raw, term, buf),
+        TermType::Tuple => encode_tuple(env, env_raw, term, buf, depth),
         _ => Err(EncodeError::BadArg),
     }
 }
@@ -106,7 +117,11 @@ fn encode_map(
     env_raw: *mut ErlNifEnv,
     term: Term,
     buf: &mut Vec<u8>,
+    depth: u32,
 ) -> Result<(), EncodeError> {
+    if depth == 0 {
+        return Err(EncodeError::DepthExceeded);
+    }
     let iter = MapIterator::new(term).ok_or(EncodeError::BadArg)?;
     buf.push(b'{');
     let mut first = true;
@@ -117,7 +132,7 @@ fn encode_map(
         first = false;
         encode_map_key(env_raw, key, buf)?;
         buf.push(b':');
-        encode_term(env, env_raw, value, buf)?;
+        encode_term(env, env_raw, value, buf, depth - 1)?;
     }
     buf.push(b'}');
     Ok(())
@@ -158,7 +173,11 @@ fn encode_list(
     env_raw: *mut ErlNifEnv,
     term: Term,
     buf: &mut Vec<u8>,
+    depth: u32,
 ) -> Result<(), EncodeError> {
+    if depth == 0 {
+        return Err(EncodeError::DepthExceeded);
+    }
     buf.push(b'[');
     let mut first = true;
     let mut current = term.as_c_arg();
@@ -170,7 +189,7 @@ fn encode_list(
         }
         first = false;
         let item = unsafe { Term::new(env, head) };
-        encode_term(env, env_raw, item, buf)?;
+        encode_term(env, env_raw, item, buf, depth - 1)?;
         current = tail;
     }
     buf.push(b']');
@@ -204,12 +223,19 @@ fn encode_integer(
     buf: &mut Vec<u8>,
 ) -> Result<(), EncodeError> {
     let mut n: i64 = 0;
-    if unsafe { enif_get_int64(env_raw, term.as_c_arg(), &mut n) } == 0 {
-        return Err(EncodeError::BadArg);
+    if unsafe { enif_get_int64(env_raw, term.as_c_arg(), &mut n) } != 0 {
+        let mut itoa_buf = itoa::Buffer::new();
+        buf.extend_from_slice(itoa_buf.format(n).as_bytes());
+        return Ok(());
     }
-    let mut itoa_buf = itoa::Buffer::new();
-    buf.extend_from_slice(itoa_buf.format(n).as_bytes());
-    Ok(())
+    // Fallback for u64 range (i64::MAX + 1 ..= u64::MAX)
+    let mut u: u64 = 0;
+    if unsafe { enif_get_uint64(env_raw, term.as_c_arg(), &mut u) } != 0 {
+        let mut itoa_buf = itoa::Buffer::new();
+        buf.extend_from_slice(itoa_buf.format(u).as_bytes());
+        return Ok(());
+    }
+    Err(EncodeError::BadArg)
 }
 
 #[inline]
@@ -218,6 +244,10 @@ fn encode_float(env_raw: *mut ErlNifEnv, term: Term, buf: &mut Vec<u8>) -> Resul
     if unsafe { enif_get_double(env_raw, term.as_c_arg(), &mut n) } == 0 {
         return Err(EncodeError::BadArg);
     }
+    // ryu panics on non-finite floats; JSON has no representation for them
+    if !n.is_finite() {
+        return Err(EncodeError::BadArg);
+    }
     let mut ryu_buf = ryu::Buffer::new();
     buf.extend_from_slice(ryu_buf.format(n).as_bytes());
     Ok(())
@@ -264,12 +294,13 @@ fn encode_tuple(
     env_raw: *mut ErlNifEnv,
     term: Term,
     buf: &mut Vec<u8>,
+    depth: u32,
 ) -> Result<(), EncodeError> {
     let elements = unsafe { get_tuple_raw(env_raw, term)? };
     if elements.len() == 1 {
         let inner = unsafe { Term::new(env, elements[0]) };
         if inner.get_type() == TermType::List {
-            return encode_proplist(env, env_raw, inner, buf);
+            return encode_proplist(env, env_raw, inner, buf, depth);
         }
     }
     Err(EncodeError::BadArg)
@@ -280,7 +311,11 @@ fn encode_proplist(
     env_raw: *mut ErlNifEnv,
     term: Term,
     buf: &mut Vec<u8>,
+    depth: u32,
 ) -> Result<(), EncodeError> {
+    if depth == 0 {
+        return Err(EncodeError::DepthExceeded);
+    }
     buf.push(b'{');
     let mut first = true;
     let mut current = term.as_c_arg();
@@ -302,7 +337,7 @@ fn encode_proplist(
         let val = unsafe { Term::new(env, pair[1]) };
         encode_map_key(env_raw, key, buf)?;
         buf.push(b':');
-        encode_term(env, env_raw, val, buf)?;
+        encode_term(env, env_raw, val, buf, depth - 1)?;
         current = tail;
     }
     buf.push(b'}');