always set query_id

Author: joe-clickhouse

CHANGELOG.md

@@ -24,6 +24,7 @@ The supported method of passing ClickHouse server settings is to prefix such arg
 ## UNRELEASED
 ### Bug Fixes
 ### Improvements
+- Always generate query_id from the client side as a UUID4 if it not explicitly set. Closes [#596](https://github.com/ClickHouse/clickhouse-connect/issues/596)
 
 ## 0.10.0, 2025-11-14
 

clickhouse_connect/common.py

@@ -69,6 +69,7 @@ def _init_common(name: str, options: Sequence[Any], default: Any) -> None:
 
 
 _init_common('autogenerate_session_id', (True, False), True)
+_init_common('autogenerate_query_id', (True, False), True)
 _init_common('dict_parameter_format', ('json', 'map'), 'json')
 _init_common('invalid_setting_action', ('send', 'drop', 'error'), 'error')
 _init_common('max_connection_age', (), 10 * 60)  # Max time in seconds to keep reusing a database TCP connection

clickhouse_connect/driver/asyncclient.py

@@ -518,7 +518,9 @@ def create_query_context(self,
         Creates or updates a reusable QueryContext object
         :param query: Query statement/format string
         :param parameters: Optional dictionary used to format the query
-        :param settings: Optional dictionary of ClickHouse settings (key/string values)
+        :param settings: Optional dictionary of ClickHouse settings (key/string values). This includes both
+          server settings (e.g., max_threads, max_memory_usage) and HTTP interface parameters (e.g., query_id,
+          session_id, database). All settings are sent as URL query parameters.
         :param query_formats: See QueryContext __init__ docstring
         :param column_formats: See QueryContext __init__ docstring
         :param encoding: See QueryContext __init__ docstring
@@ -543,7 +545,9 @@ def create_query_context(self,
         :param use_extended_dtypes:  Only relevant to Pandas Dataframe queries.  Use Pandas "missing types", such as
           pandas.NA and pandas.NaT for ClickHouse NULL values, as well as extended Pandas dtypes such as IntegerArray
           and StringArray.  Defaulted to True for query_df methods
-        :param transport_settings: Optional dictionary of transport level settings (HTTP headers, etc.)
+        :param transport_settings: Optional dictionary of transport level settings sent as HTTP headers. Use this for
+          custom headers (e.g., X-Workload) for load balancers or proxies. ClickHouse parameters like query_id should
+          go in the settings dict, not here.
         :return: Reusable QueryContext
         """
 

clickhouse_connect/driver/client.py

@@ -473,7 +473,9 @@ def create_query_context(self,
         Creates or updates a reusable QueryContext object
         :param query: Query statement/format string
         :param parameters: Optional dictionary used to format the query
-        :param settings: Optional dictionary of ClickHouse settings (key/string values)
+        :param settings: Optional dictionary of ClickHouse settings (key/string values). This includes both
+          server settings (e.g., max_threads, max_memory_usage) and HTTP interface parameters (e.g., query_id,
+          session_id, database). All settings are sent as URL query parameters.
         :param query_formats: See QueryContext __init__ docstring
         :param column_formats: See QueryContext __init__ docstring
         :param encoding: See QueryContext __init__ docstring
@@ -500,7 +502,9 @@ def create_query_context(self,
         :param use_extended_dtypes:  Only relevant to Pandas Dataframe queries.  Use Pandas "missing types", such as
           pandas.NA and pandas.NaT for ClickHouse NULL values, as well as extended Pandas dtypes such as IntegerArray
           and StringArray.  Defaulted to True for query_df methods
-        :param transport_settings: Optional dictionary of transport level settings (HTTP headers, etc.)
+        :param transport_settings: Optional dictionary of transport level settings sent as HTTP headers. Use this for
+          custom headers (e.g., X-Workload) for load balancers or proxies. ClickHouse parameters like query_id should
+          go in the settings dict, not here.
         :return: Reusable QueryContext
         """
         resolved_utc_tz_aware = self.utc_tz_aware if utc_tz_aware is None else utc_tz_aware

clickhouse_connect/driver/httpclient.py

@@ -79,6 +79,7 @@ def __init__(self,
                  utc_tz_aware: Optional[bool] = None,
                  show_clickhouse_errors: Optional[bool] = None,
                  autogenerate_session_id: Optional[bool] = None,
+                 autogenerate_query_id: Optional[bool] = None,
                  tls_mode: Optional[str] = None,
                  proxy_path: str = '',
                  form_encode_query_params: bool = False,
@@ -161,6 +162,11 @@ def __init__(self,
         elif 'session_id' not in ch_settings and _autogenerate_session_id:
             ch_settings['session_id'] = str(uuid.uuid4())
 
+        # allow to override the global autogenerate_query_id setting via the constructor params
+        self._autogenerate_query_id = common.get_setting('autogenerate_query_id') \
+            if autogenerate_query_id is None \
+            else autogenerate_query_id
+
         if coerce_bool(compress):
             compression = ','.join(available_compression)
             self.write_compression = available_compression[0]
@@ -191,6 +197,18 @@ def __init__(self,
                 self._setting_status('http_headers_progress_interval_ms').is_writable:
             self._progress_interval = str(min(120000, max(10000, (send_receive_timeout - 5) * 1000)))
 
+    def _ensure_query_id(self, settings: Optional[dict]) -> Optional[dict]:
+        if not self._autogenerate_query_id:
+            return settings
+
+        if settings is None:
+            settings = {}
+
+        if "query_id" not in settings:
+            settings["query_id"] = str(uuid.uuid4())
+
+        return settings
+
     def set_client_setting(self, key, value):
         str_value = self._validate_setting(key, value, common.get_setting('invalid_setting_action'))
         if str_value is not None:
@@ -215,6 +233,8 @@ def _prep_query(self, context: QueryContext):
         return final_query + fmt
 
     def _query_with_context(self, context: QueryContext) -> QueryResult:
+        context.settings = self._ensure_query_id(context.settings)
+
         headers = {}
         params = {}
         if self.database:
@@ -307,6 +327,8 @@ def data_insert(self, context: InsertContext) -> QuerySummary:
             logger.debug('No data included in insert, skipping')
             return QuerySummary()
 
+        context.settings = self._ensure_query_id(context.settings)
+
         def error_handler(resp: HTTPResponse):
             # If we actually had a local exception when building the insert, throw that instead
             if context.insert_exception:
@@ -342,6 +364,8 @@ def raw_insert(self, table: str = None,
         """
         See BaseClient doc_string for this method
         """
+        settings = self._ensure_query_id(settings)
+
         params = {}
         headers = {'Content-Type': 'application/octet-stream'}
         if compression:
@@ -385,6 +409,8 @@ def command(self,
         """
         See BaseClient doc_string for this method
         """
+        settings = self._ensure_query_id(settings)
+
         cmd, params = bind_query(cmd, parameters, self.server_tz)
         headers = {}
         payload = None

tests/integration_tests/test_client.py

@@ -1,6 +1,7 @@
 from pathlib import Path
 from time import sleep
 from typing import Callable
+import uuid
 
 import pytest
 
@@ -382,3 +383,43 @@ def test_role_setting_works(test_client: Client, test_config: TestConfig):
     )
     res = role_client.query('SELECT currentRoles()')
     assert res.result_rows == [([role_limited],)]
+
+
+# pylint: disable=protected-access
+def test_autogenerate_query_id(test_client: Client, test_table_engine: str, test_config: TestConfig):
+    def _is_valid_uuid_v4_string(id_string: str) -> bool:
+        parsed_uuid = uuid.UUID(id_string)
+        return parsed_uuid.version == 4
+
+    result = test_client.query("SELECT 1")
+    assert _is_valid_uuid_v4_string(result.query_id)
+
+    manual_query_id = "test_manual_query_id_12345"
+    result = test_client.query("SELECT 2", settings={"query_id": manual_query_id})
+    assert result.query_id == manual_query_id
+
+    summary = test_client.command("DROP TABLE IF EXISTS does_not_exist")
+    assert _is_valid_uuid_v4_string(summary.query_id())
+
+    test_client.command("DROP TABLE IF EXISTS test_autogen_query_id")
+    test_client.command(f"CREATE TABLE test_autogen_query_id (id UInt32) ENGINE {test_table_engine} ORDER BY id")
+    summary = test_client.insert("test_autogen_query_id", [[1], [2], [3]], column_names=["id"])
+    assert _is_valid_uuid_v4_string(summary.query_id())
+    test_client.command("DROP TABLE test_autogen_query_id")
+
+    # Create client with autogenerate_query_id disabled
+    client_no_autogen = create_client(
+        host=test_config.host,
+        port=test_config.port,
+        username=test_config.username,
+        password=test_config.password,
+        autogenerate_query_id=False,
+    )
+    result = client_no_autogen.query("SELECT 4")
+    # Even with autogen disabled the server generates a query_id
+    #  so we still expect a query_id in the result, it's just generated by the server...
+    #  not sure how to verify that specifically though.
+    assert _is_valid_uuid_v4_string(result.query_id)
+
+    # We can verify that the setting on the client is False though
+    assert client_no_autogen._autogenerate_query_id is False