Release 0 8 16 (#485)

* Check for optional libraries in client methods

* Log unexpected http next chunk unexpected

* Log unexpected http next chunk unexpected

* Updates for 0.8.16 release

Author: genzgd

CHANGELOG.md

@@ -21,6 +21,23 @@ release (0.9.0), unrecognized arguments/keywords for these methods of creating a
 instead of being passed as ClickHouse server settings. This is in conjunction with some refactoring in Client construction.
 The supported method of passing ClickHouse server settings is to prefix such arguments/query parameters with`ch_`.
 
+## 0.8.16, 2025-03-28
+### Bug Fixes
+- Don't send a setting value if the setting is already correct according to the `system.settings` table. 
+Closes https://github.com/ClickHouse/clickhouse-connect/issues/469
+- Ensure that the http `user_agent` header is in ascii.  Note this could lead to an incorrectly encoded `os_user` if the
+os_user is not an Ascii string.  Closes https://github.com/ClickHouse/clickhouse-connect/issues/484
+- Fix "cannot access local variable" exception where the http client encounters an unexpected streaming error.  Also
+log that unexpected streaming error to assist debugging.  Closes https://github.com/ClickHouse/clickhouse-connect/issues/483
+- Check that arrow/pandas is installed when calling `query_df` and `query_arrow` and raise a more meaningful exception
+if the required library is absent.  Closes https://github.com/ClickHouse/clickhouse-connect/issues/477
+
+### Improvements
+- Some typing hints have been corrected.  Thanks to [Avery Fischer](https://github.com/biggerfisch) for the PR!
+- The docker based tests have been fixed to work with security improvements in recent ClickHouse releases
+- Query string cleanup is now (in theory) microseconds faster.  Thanks to [Sviatoslav Bobryshev](https://github.com/sbobryshev)
+for the optimization
+
 ## 0.8.15, 2025-01-25
 ### Bug Fix
 - The async client was not shutting down its associated executor thread pool, result in a memory leak if multiple

clickhouse_connect/__version__.py

@@ -1 +1 @@
-version = '0.8.15'
+version = '0.8.16'

clickhouse_connect/common.py

@@ -41,8 +41,9 @@ def build_client_name(client_name: str):
             os_user = f'; os_user:{getpass.getuser()}'
         except Exception:  # pylint: disable=broad-except
             pass
-    return (f'{client_name}{product_name}clickhouse-connect/{version()}' +
+    full_name = (f'{client_name}{product_name}clickhouse-connect/{version()}' +
             f' (lv:py/{py_version}; mode:sync; os:{sys.platform}{os_user})')
+    return full_name.encode('ascii', 'ignore').decode()
 
 
 def get_setting(name: str):

clickhouse_connect/driver/client.py

@@ -19,6 +19,7 @@
 from clickhouse_connect.driver.exceptions import ProgrammingError, OperationalError
 from clickhouse_connect.driver.external import ExternalData
 from clickhouse_connect.driver.insert import InsertContext
+from clickhouse_connect.driver.options import check_arrow, check_pandas, check_numpy
 from clickhouse_connect.driver.summary import QuerySummary
 from clickhouse_connect.driver.models import ColumnDef, SettingDef, SettingStatus
 from clickhouse_connect.driver.query import QueryResult, to_arrow, to_arrow_batches, QueryContext, arrow_buffer
@@ -68,7 +69,7 @@ def __init__(self,
         self.uri = uri
         self._init_common_settings(apply_server_timezone)
 
-    def _init_common_settings(self, apply_server_timezone:Optional[Union[str, bool]] ):
+    def _init_common_settings(self, apply_server_timezone: Optional[Union[str, bool]]):
         self.server_tz, dst_safe = pytz.UTC, True
         self.server_version, server_tz = \
             tuple(self.command('SELECT version(), timezone()', use_database=False))
@@ -122,14 +123,16 @@ def _validate_settings(self, settings: Optional[Dict[str, Any]]) -> Dict[str, st
         return validated
 
     def _validate_setting(self, key: str, value: Any, invalid_action: str) -> Optional[str]:
-        new_value = str(value)
+        str_value = str(value)
         if value is True:
-            new_value = '1'
+            str_value = '1'
         elif value is False:
-            new_value = '0'
+            str_value = '0'
         if key not in self.valid_transport_settings:
             setting_def = self.server_settings.get(key)
-            if setting_def is None or (setting_def.readonly and setting_def.value != new_value):
+            if setting_def and setting_def.value == str_value:
+                return None  # don't send settings that are already the expected value
+            if setting_def is None or setting_def.readonly:
                 if key in self.optional_transport_settings:
                     return None
                 if invalid_action == 'send':
@@ -139,7 +142,7 @@ def _validate_setting(self, key: str, value: Any, invalid_action: str) -> Option
                     return None
                 else:
                     raise ProgrammingError(f'Setting {key} is unknown or readonly') from None
-        return new_value
+        return str_value
 
     def _setting_status(self, key: str) -> SettingStatus:
         comp_setting = self.server_settings.get(key)
@@ -342,6 +345,7 @@ def query_np(self,
         create_query_context method
         :return: Numpy array representing the result set
         """
+        check_numpy()
         return self._context_query(locals(), use_numpy=True).np_result
 
     # pylint: disable=duplicate-code,too-many-arguments,unused-argument
@@ -361,6 +365,7 @@ def query_np_stream(self,
         create_query_context method
         :return: Generator that yield a numpy array per block representing the result set
         """
+        check_numpy()
         return self._context_query(locals(), use_numpy=True, streaming=True).np_stream
 
     # pylint: disable=duplicate-code,unused-argument
@@ -384,6 +389,7 @@ def query_df(self,
         create_query_context method
         :return: Pandas dataframe representing the result set
         """
+        check_pandas()
         return self._context_query(locals(), use_numpy=True, as_pandas=True).df_result
 
     # pylint: disable=duplicate-code,unused-argument
@@ -407,6 +413,7 @@ def query_df_stream(self,
         create_query_context method
         :return: Generator that yields a Pandas dataframe per block representing the result set
         """
+        check_pandas()
         return self._context_query(locals(), use_numpy=True,
                                    as_pandas=True,
                                    streaming=True).df_stream
@@ -519,6 +526,7 @@ def query_arrow(self,
         :param external_data ClickHouse "external data" to send with query
         :return: PyArrow.Table
         """
+        check_arrow()
         settings = self._update_arrow_settings(settings, use_strings)
         return to_arrow(self.raw_query(query,
                                        parameters,
@@ -541,6 +549,7 @@ def query_arrow_stream(self,
         :param external_data ClickHouse "external data" to send with query
         :return: Generator that yields a PyArrow.Table for per block representing the result set
         """
+        check_arrow()
         settings = self._update_arrow_settings(settings, use_strings)
         return to_arrow_batches(self.raw_stream(query,
                                                 parameters,
@@ -661,6 +670,7 @@ def insert_df(self, table: str = None,
             different data batches
         :return: QuerySummary with summary information, throws exception if insert fails
         """
+        check_pandas()
         if context is None:
             if column_names is None:
                 column_names = df.columns
@@ -686,6 +696,7 @@ def insert_arrow(self, table: str,
         :param settings: Optional dictionary of ClickHouse settings (key/string values)
         :return: QuerySummary with summary information, throws exception if insert fails
         """
+        check_arrow()
         full_table = table if '.' in table or not database else f'{database}.{table}'
         compression = self.write_compression if self.write_compression in ('zstd', 'lz4') else None
         column_names, insert_block = arrow_buffer(arrow_table, compression)

clickhouse_connect/driver/httpclient.py

@@ -527,7 +527,7 @@ def ping(self):
         See BaseClient doc_string for this method
         """
         try:
-            response = self.http.request('GET', f'{self.url}/ping', timeout=3)
+            response = self.http.request('GET', f'{self.url}/ping', timeout=3, preload_content=True)
             return 200 <= response.status < 300
         except HTTPError:
             logger.debug('ping failed', exc_info=True)

clickhouse_connect/driver/httputil.py

@@ -228,12 +228,13 @@ def buffered():
             read_gen = response.stream(chunk_size, decompress is None)
             while True:
                 while not done:
+                    chunk = None
                     try:
                         chunk = next(read_gen, None) # Always try to read at least one chunk if there are any left
                     except Exception: # pylint: disable=broad-except
                         # By swallowing an unexpected exception reading the stream, we will let consumers decide how to
                         # handle the unexpected end of stream
-                        pass
+                        logger.warning('unexpected failure to read next chunk', exc_info=True)
                     if not chunk:
                         done = True
                         break