Merge pull request #8 from pkgodara/fix-invalid-utf8-binaries

Added utf-8 validation and error messages

Author: lpgauth

CLAUDE.md

@@ -8,7 +8,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co
 TORQUE_BUILD=true mix deps.get     # fetch deps + force local Rust build
 TORQUE_BUILD=true mix compile      # build (includes Rust NIF compilation)
 TORQUE_BUILD=true mix test         # run all tests
-mix test test/torque_test.exs:42   # run single test by line number
+mix test test/pointer_test.exs:42  # run single test by line number
 mix compile --warnings-as-errors   # build with strict warnings
 mix format                         # format Elixir code
 mix format --check-formatted       # check Elixir formatting
@@ -67,4 +67,4 @@ Inputs larger than 10 KB are automatically dispatched to dirty CPU schedulers to
 - `native/torque_nif/src/decoder.rs` — parse, get, get_many, decode NIFs
 - `native/torque_nif/src/encoder.rs` — direct term-walking JSON encoder
 - `native/torque_nif/src/types.rs` — sonic_rs Value → Erlang term conversion
-- `native/torque_nif/src/atoms.rs` — cached atoms (ok, error, no_such_field, nil)
+- `native/torque_nif/src/atoms.rs` — cached atoms (ok, error, nil, no_such_field, nesting_too_deep, unsupported_type, non_finite_float, invalid_key, malformed_proplist, invalid_utf8)

README.md

@@ -116,6 +116,29 @@ For objects with duplicate keys, the last value wins.
 | atom | string |
 | `{keyword_list}` | object |
 
+## Errors
+
+Functions return `{:error, reason}` tuples (or raise `ArgumentError` for bang/iodata variants). Possible `reason` atoms:
+
+### Decode / Parse
+
+| Atom | Returned by | Meaning |
+|------|-------------|---------|
+| `:nesting_too_deep` | `decode/1`, `parse/1`, `get/2,3` | Document exceeds 512 nesting levels |
+
+`parse/1` and `decode/1` also return `{:error, binary}` with a message from sonic-rs for malformed JSON.
+
+### Encode
+
+| Atom | Returned by | Meaning |
+|------|-------------|---------|
+| `:unsupported_type` | `encode/1` | Term has no JSON representation (PID, reference, port, …) |
+| `:invalid_utf8` | `encode/1` | Binary string or map key is not valid UTF-8 |
+| `:invalid_key` | `encode/1` | Map key is not an atom or binary (e.g. integer key) |
+| `:malformed_proplist` | `encode/1` | `{proplist}` contains a non-`{key, value}` element |
+| `:non_finite_float` | `encode/1` | Float is infinity or NaN (unreachable from normal BEAM code) |
+| `:nesting_too_deep` | `encode/1` | Term exceeds 512 nesting levels |
+
 ## Benchmarks
 
 Apple M2 Pro, OTP 28, Elixir 1.19:

native/torque_nif/src/atoms.rs

@@ -12,4 +12,5 @@ rustler::atoms! {
     non_finite_float,
     invalid_key,
     malformed_proplist,
+    invalid_utf8,
 }

native/torque_nif/src/encoder.rs

@@ -16,6 +16,7 @@ enum EncodeError {
     InvalidKey,
     MalformedProplist,
     DepthExceeded,
+    InvalidUtf8,
 }
 
 /// Read an atom's name into a stack buffer without heap allocation.
@@ -67,6 +68,7 @@ fn encode<'a>(env: Env<'a>, term: Term<'a>) -> Term<'a> {
                 EncodeError::NonFiniteFloat => atoms::non_finite_float().as_c_arg(),
                 EncodeError::InvalidKey => atoms::invalid_key().as_c_arg(),
                 EncodeError::MalformedProplist => atoms::malformed_proplist().as_c_arg(),
+                EncodeError::InvalidUtf8 => atoms::invalid_utf8().as_c_arg(),
             };
             make_tuple2(env, atoms::error().as_c_arg(), reason)
         }
@@ -92,6 +94,7 @@ fn encode_iodata<'a>(env: Env<'a>, term: Term<'a>) -> Term<'a> {
                 EncodeError::NonFiniteFloat => atoms::non_finite_float().as_c_arg(),
                 EncodeError::InvalidKey => atoms::invalid_key().as_c_arg(),
                 EncodeError::MalformedProplist => atoms::malformed_proplist().as_c_arg(),
+                EncodeError::InvalidUtf8 => atoms::invalid_utf8().as_c_arg(),
             };
             Term::new(env, rustler::sys::enif_raise_exception(env_raw, reason))
         },
@@ -165,6 +168,9 @@ fn encode_map_key(
                 }
                 let bin = bin.assume_init();
                 let slice = std::slice::from_raw_parts(bin.data, bin.size);
+                if std::str::from_utf8(slice).is_err() {
+                    return Err(EncodeError::InvalidUtf8);
+                }
                 escape_bytes(slice, buf);
             }
         }
@@ -215,6 +221,9 @@ fn encode_binary(
         }
         let bin = bin.assume_init();
         let slice = std::slice::from_raw_parts(bin.data, bin.size);
+        if std::str::from_utf8(slice).is_err() {
+            return Err(EncodeError::InvalidUtf8);
+        }
         buf.push(b'"');
         escape_bytes(slice, buf);
         buf.push(b'"');

test/encode_test.exs

@@ -96,10 +96,18 @@ defmodule Torque.EncodeTest do
       assert [%{"id" => 1}, %{"id" => 2}] = Jason.decode!(json)
     end
 
-    test "atom values encoded as strings" do
+    test "atom map values encoded as strings" do
       assert {:ok, json} = Torque.encode(%{status: :active})
       assert %{"status" => "active"} = Jason.decode!(json)
     end
+
+    test "invalid UTF-8 binary returns error" do
+      assert {:error, :invalid_utf8} = Torque.encode(<<0x80>>)
+    end
+
+    test "invalid UTF-8 binary map key returns error" do
+      assert {:error, :invalid_utf8} = Torque.encode(%{<<0x80>> => "value"})
+    end
   end
 
   describe "encode!/1" do
@@ -108,10 +116,16 @@ defmodule Torque.EncodeTest do
     end
 
     test "unsupported term raises" do
-      assert_raise ArgumentError, fn ->
+      assert_raise ArgumentError, ~r/unsupported_type/, fn ->
         Torque.encode!(self())
       end
     end
+
+    test "invalid UTF-8 binary raises" do
+      assert_raise ArgumentError, ~r/invalid_utf8/, fn ->
+        Torque.encode!(<<0x80>>)
+      end
+    end
   end
 
   describe "encode_to_iodata/1" do
@@ -126,9 +140,15 @@ defmodule Torque.EncodeTest do
     end
 
     test "unsupported term raises ArgumentError" do
-      assert_raise ArgumentError, fn ->
+      assert_raise ArgumentError, ~r/unsupported_type/, fn ->
         Torque.encode_to_iodata(self())
       end
     end
+
+    test "invalid UTF-8 binary raises ArgumentError" do
+      assert_raise ArgumentError, ~r/invalid_utf8/, fn ->
+        Torque.encode_to_iodata(<<0x80>>)
+      end
+    end
   end
 end

test/torque_test.exs test/pointer_test.exstest/torque_test.exs renamed to test/pointer_test.exs

@@ -475,7 +475,7 @@ defmodule Torque.PropertyTest do
 
     test "encode_to_iodata raises at depth 513" do
       term = Enum.reduce(1..513, "leaf", fn _, acc -> %{"x" => acc} end)
-      assert_raise ArgumentError, fn -> Torque.encode_to_iodata(term) end
+      assert_raise ArgumentError, ~r/nesting_too_deep/, fn -> Torque.encode_to_iodata(term) end
     end
   end
 
@@ -541,6 +541,13 @@ defmodule Torque.PropertyTest do
       assert {:error, :malformed_proplist} = Torque.encode({[{:a, :b, :c}]})
     end
 
+    # BEAM rejects non-finite floats (both arithmetic overflow and binary_to_term
+    # validation), so this NIF code path is not reachable from Elixir.
+    @tag :skip
+    test "non-finite float returns non_finite_float" do
+      assert {:error, :non_finite_float} = Torque.encode(:infinity)
+    end
+
     test "encode_to_iodata raises with unsupported_type message" do
       assert_raise ArgumentError, ~r/unsupported_type/, fn ->
         Torque.encode_to_iodata(self())
@@ -589,17 +596,6 @@ defmodule Torque.PropertyTest do
       end
     end
 
-    property "numeric-looking string keys work alongside integer array indexes" do
-      check all(n <- integer(0..100)) do
-        # Object key that looks like an integer — should be treated as a string key
-        json = ~s({"#{n}": "string_val"})
-        {:ok, doc} = Torque.parse(json)
-        # Accessing "/N" on an object should still find the string key "N"
-        result = Torque.get(doc, "/#{n}")
-        assert match?({:ok, _}, result) or match?({:error, :no_such_field}, result)
-      end
-    end
-
     test "path without leading slash returns no_such_field" do
       {:ok, doc} = Torque.parse(~s({"a":1}))
       assert {:error, :no_such_field} = Torque.get(doc, "a")