fix nullable json inserts

Author: joe-clickhouse

clickhouse_connect/datatypes/dynamic.py

@@ -14,6 +14,8 @@
 
 SHARED_DATA_TYPE: ClickHouseType
 STRING_DATA_TYPE: ClickHouseType
+_JSON_NULL = b'null'
+_JSON_NULL_STR = 'null'
 
 json_serialization_format = 0x1
 
@@ -129,13 +131,22 @@ def json_sample_size(_, sample: Collection) -> int:
 
 
 def write_json(ch_type: ClickHouseType, column: Sequence, dest: bytearray, ctx: InsertContext):
+    if ch_type.nullable:
+        dest += bytearray(1 if v is None else 0 for v in column)
+
     first = first_value(column, ch_type.nullable)
     write_col = column
     encoding = ctx.encoding or ch_type.encoding
     if not isinstance(first, str) and ch_type.write_format(ctx) != 'string':
         to_json = any_to_json
-        write_col = [to_json(v) for v in column]
+        if ch_type.nullable:
+            write_col = [_JSON_NULL if v is None else to_json(v) for v in column]
+        else:
+            write_col = [to_json(v) for v in column]
         encoding = None
+    else:
+        write_col = [_JSON_NULL_STR if v is None else v for v in column]
+
     handle_error(data_conv.write_str_col(write_col, ch_type.nullable, encoding, dest), ctx)
 
 

tests/integration_tests/test_dynamic.py

@@ -2,6 +2,7 @@
 from ipaddress import IPv4Address
 from typing import Callable
 from uuid import UUID
+import json
 
 import pytest
 
@@ -150,6 +151,26 @@ def test_typed_json(test_client: Client, table_context: Callable):
         assert json1['a']['b'] == datetime.datetime(2020, 10, 15, 10, 15, 44, 877000)
 
 
+def test_nullable_json(test_client: Client, table_context: Callable):
+    type_available(test_client, "json")
+    with table_context("nullable_json", [
+        "key Int32",
+        "value_1 Nullable(JSON)",
+        "value_2 Nullable(JSON)",
+        "value_3 Nullable(JSON)"
+    ]):
+        v1 = {"item_a": 5, "item_b": 10}
+
+        test_client.insert("nullable_json", [[1, v1, json.dumps(v1), None], [2, v1, None, None]])
+        result = test_client.query('SELECT * FROM nullable_json ORDER BY key')
+        assert result.result_set[0][1] == v1
+        assert result.result_set[1][1] == v1
+        assert result.result_set[0][2] == v1
+        assert result.result_set[1][2] is None
+        assert result.result_set[0][3] is None
+        assert result.result_set[1][3] is None
+
+
 def test_complex_json(test_client: Client, table_context: Callable):
     type_available(test_client, 'json')
     if not test_client.min_version('24.10'):

tests/integration_tests/test_pandas.py

@@ -285,3 +285,50 @@ def test_pandas_small_blocks(test_config: TestConfig, test_client: Client):
     res = test_client.query_df('SELECT number, randomString(512) FROM numbers(1000000)',
                                settings={'max_block_size': 250})
     assert len(res) == 1000000
+
+
+def test_pandas_string_to_df_insert(test_client: Client, table_context: Callable):
+    with table_context(
+        "test_pandas_string_to_df_insert",
+        [
+            "id UInt32",
+            "timestamp Nullable(DateTime)",
+            "json_data Nullable(JSON)",
+        ],
+    ):
+
+        df = pd.DataFrame(
+            [[1, "simple"], [2, "with spaces"], [3, "特殊字符"], [4, ""]],
+            columns=["id", "s"],
+        )
+
+        json_data_dict = {"vm": "", "App Name": "MKT"}
+        json_data_dict2 = {"Room": "Leo"}
+
+        data = [
+            {
+                "id": 1,
+                "timestamp": datetime(year=2025, month=7, day=5, hour=12),
+                "json_data": json_data_dict,
+            },
+            {
+                "id": 2,
+                "timestamp": datetime(year=2025, month=7, day=6, hour=12),
+                "json_data": json_data_dict2,
+            },
+            {
+                "id": 3,
+                "timestamp": datetime(year=2025, month=7, day=7, hour=12),
+                "json_data": None,
+            },
+        ]
+
+        df = pd.DataFrame(data)
+        test_client.insert_df("test_pandas_string_to_df_insert", df)
+        result_df = test_client.query_df(
+            "SELECT * FROM test_pandas_string_to_df_insert ORDER BY id"
+        )
+
+        assert result_df.iloc[0]["json_data"] == json_data_dict
+        assert result_df.iloc[1]["json_data"] == json_data_dict2
+        assert result_df.iloc[2]["json_data"] is None