Also emit canonical Arrow JSON extension metadata keys

Arrow defines a canonical extension type for JSON
(https://arrow.apache.org/docs/format/CanonicalExtensions.html#json):

  ARROW:extension:name     = arrow.json
  ARROW:extension:metadata = {}

Emit those alongside the existing is_json=true key so JSON-bearing string
fields are recognized by the broader Arrow ecosystem (arrow-rs's Json
extension type, pyarrow, DuckDB, Polars, etc.) while remaining
back-compatible with consumers keyed on is_json.

The change is contained to json_field_metadata(); all four production
write sites already route through that helper. Detection via
is_json_union remains purely structural and is unaffected. Tests are
extended to assert all three keys, and the helper is re-exported at the
crate root so test fixtures can use it instead of duplicating literals.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: adriangb

src/common_union.rs

@@ -13,8 +13,18 @@ use datafusion::common::ScalarValue;
 ///
 /// Attach this to any Arrow `Field` whose values are JSON-encoded strings so
 /// downstream consumers can recognize them as JSON rather than opaque text.
+///
+/// Emits both the legacy `is_json` key (for back-compat with existing
+/// consumers of this crate) and Arrow's canonical JSON extension type keys
+/// (`ARROW:extension:name` = `arrow.json`, `ARROW:extension:metadata` = `{}`),
+/// see <https://arrow.apache.org/docs/format/CanonicalExtensions.html#json>.
+#[must_use]
 pub fn json_field_metadata() -> HashMap<String, String> {
-    HashMap::from([("is_json".to_string(), "true".to_string())])
+    HashMap::from([
+        ("is_json".to_string(), "true".to_string()),
+        ("ARROW:extension:name".to_string(), "arrow.json".to_string()),
+        ("ARROW:extension:metadata".to_string(), "{}".to_string()),
+    ])
 }
 
 pub fn is_json_union(data_type: &DataType) -> bool {

src/lib.rs

@@ -22,7 +22,7 @@ mod json_length;
 mod json_object_keys;
 mod rewrite;
 
-pub use common_union::{JsonUnionEncoder, JsonUnionValue, JSON_UNION_DATA_TYPE};
+pub use common_union::{json_field_metadata, JsonUnionEncoder, JsonUnionValue, JSON_UNION_DATA_TYPE};
 
 pub mod functions {
     pub use crate::json_as_text::json_as_text;

tests/main.rs

@@ -9,6 +9,7 @@ use datafusion::common::ScalarValue;
 use datafusion::config::ConfigOptions;
 use datafusion::logical_expr::{ColumnarValue, ScalarFunctionArgs};
 use datafusion::prelude::SessionContext;
+use datafusion_functions_json::json_field_metadata;
 use datafusion_functions_json::udfs::json_get_str_udf;
 use utils::{create_context, display_val, logical_plan, run_query, run_query_params};
 
@@ -162,15 +163,15 @@ async fn test_json_get_array_with_path() {
 }
 
 #[tokio::test]
-async fn test_json_get_array_inner_field_is_json_metadata() {
+async fn test_json_get_array_inner_field_json_metadata() {
     let sql = r#"select json_get_array('[{"a": 1}, {"b": 2}]') as v"#;
     let batches = run_query(sql).await.unwrap();
     let schema = batches[0].schema();
     let field = schema.field(0);
     let DataType::List(inner_field) = field.data_type() else {
         panic!("expected List, got {:?}", field.data_type());
     };
-    assert_eq!(inner_field.metadata().get("is_json").map(String::as_str), Some("true"));
+    assert_json_field_metadata(inner_field.metadata());
 
     let array_field = batches[0]
         .column(0)
@@ -180,9 +181,18 @@ async fn test_json_get_array_inner_field_is_json_metadata() {
     let DataType::List(produced_inner) = array_field.data_type() else {
         panic!("expected List in produced array");
     };
+    assert_json_field_metadata(produced_inner.metadata());
+}
+
+fn assert_json_field_metadata(metadata: &HashMap<String, String>) {
+    assert_eq!(metadata.get("is_json").map(String::as_str), Some("true"));
+    assert_eq!(
+        metadata.get("ARROW:extension:name").map(String::as_str),
+        Some("arrow.json")
+    );
     assert_eq!(
-        produced_inner.metadata().get("is_json").map(String::as_str),
-        Some("true")
+        metadata.get("ARROW:extension:metadata").map(String::as_str),
+        Some("{}")
     );
 }
 
@@ -437,12 +447,12 @@ async fn test_json_get_json_float() {
 }
 
 #[tokio::test]
-async fn test_json_get_json_is_json_metadata() {
+async fn test_json_get_json_json_metadata() {
     let sql = r#"select json_get_json('{"x": [1, 2]}', 'x') as v"#;
     let batches = run_query(sql).await.unwrap();
     let schema = batches[0].schema();
     let field = schema.field(0);
-    assert_eq!(field.metadata().get("is_json").map(String::as_str), Some("true"));
+    assert_json_field_metadata(field.metadata());
 }
 
 #[tokio::test]
@@ -632,8 +642,7 @@ fn test_json_get_utf8() {
             ],
             number_rows: 1,
             return_field: Arc::new(
-                Field::new("ret_field", DataType::Utf8, false)
-                    .with_metadata(HashMap::from_iter(vec![("is_json".to_string(), "true".to_string())])),
+                Field::new("ret_field", DataType::Utf8, false).with_metadata(json_field_metadata()),
             ),
             config_options: Arc::new(ConfigOptions::default()),
         })
@@ -666,8 +675,7 @@ fn test_json_get_large_utf8() {
             ],
             number_rows: 1,
             return_field: Arc::new(
-                Field::new("ret_field", DataType::Utf8, false)
-                    .with_metadata(HashMap::from_iter(vec![("is_json".to_string(), "true".to_string())])),
+                Field::new("ret_field", DataType::Utf8, false).with_metadata(json_field_metadata()),
             ),
             config_options: Arc::new(ConfigOptions::default()),
         })