DuckDBType error running examples #809
Description
Getting error: TypeError: unhashable type: '_duckdb.typing.DuckDBPyType' running any example from this repo.
Versions:
Server: Ubuntu 24
Python version: 3.12 / 3.11
Airbyte version: 0.31.5
Steps to reproduce:
On a fresh and clean Ubuntu 24 server:
git clone https://github.com/airbytehq/PyAirbyte.gitcd airbyteuv venvsource .venv/bin/activateuv pip install airbytepython3 examples/run_faker.py
Error with the stacktrace:
Traceback (most recent call last):
File "/opt/airbyte/examples/run_faker.py", line 19, in <module>
cache = ab.get_default_cache()
^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/airbyte/caches/util.py", line 37, in get_default_cache
return DuckDBCache(
^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/airbyte/caches/base.py", line 83, in __init__
temp_processor = self._sql_processor_class(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/airbyte/shared/sql_processor.py", line 212, in __init__
self._ensure_schema_exists()
File "/opt/airbyte/.venv/lib/python3.12/site-packages/airbyte/shared/sql_processor.py", line 516, in _ensure_schema_exists
schemas_list = self.normalizer.normalize_list(self._get_schemas_list())
^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/airbyte/shared/sql_processor.py", line 603, in _get_schemas_list
inspector: Inspector = sqlalchemy.inspect(self.get_sql_engine())
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/inspection.py", line 140, in inspect
ret = reg(subject)
^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/reflection.py", line 313, in _engine_insp
return Inspector._construct(Inspector._init_engine, bind)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/reflection.py", line 246, in _construct
init(self, bind)
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/reflection.py", line 257, in _init_engine
engine.connect().close()
^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 3277, in connect
return self._connection_cls(self)
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 143, in __init__
self._dbapi_connection = engine.raw_connection()
^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 3301, in raw_connection
return self.pool.connect()
^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/pool/base.py", line 447, in connect
return _ConnectionFairy._checkout(self)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/pool/base.py", line 1264, in _checkout
fairy = _ConnectionRecord.checkout(pool)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/pool/base.py", line 711, in checkout
rec = pool._do_get()
^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/pool/impl.py", line 177, in _do_get
with util.safe_reraise():
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/util/langhelpers.py", line 224, in __exit__
raise exc_value.with_traceback(exc_tb)
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/pool/impl.py", line 175, in _do_get
return self._create_connection()
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/pool/base.py", line 388, in _create_connection
return _ConnectionRecord(self)
^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/pool/base.py", line 673, in __init__
self.__connect()
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/pool/base.py", line 913, in __connect
)._exec_w_sync_on_first_run(self.dbapi_connection, self)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/event/attr.py", line 483, in _exec_w_sync_on_first_run
self(*args, **kw)
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/event/attr.py", line 497, in __call__
fn(*args, **kw)
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/util/langhelpers.py", line 1997, in go
return once_fn(*arg, **kw)
^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/create.py", line 767, in first_connect
dialect.initialize(c)
File "/opt/airbyte/.venv/lib/python3.12/site-packages/duckdb_engine/__init__.py", line 494, in initialize
DefaultDialect.initialize(self, connection)
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/default.py", line 533, in initialize
self.default_schema_name = self._get_default_schema_name(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/dialects/postgresql/base.py", line 3437, in _get_default_schema_name
return connection.exec_driver_sql("select current_schema()").scalar()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 1779, in exec_driver_sql
ret = self._execute_context(
^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 1846, in _execute_context
return self._exec_single_context(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 1986, in _exec_single_context
self._handle_dbapi_exception(
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 2358, in _handle_dbapi_exception
raise exc_info[1].with_traceback(exc_info[2])
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/base.py", line 1983, in _exec_single_context
result = context._setup_result_proxy()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/default.py", line 1853, in _setup_result_proxy
result = self._setup_dml_or_text_result()
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/default.py", line 1967, in _setup_dml_or_text_result
result: _cursor.CursorResult[Any] = _cursor.CursorResult(
^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/cursor.py", line 1447, in __init__
metadata = self._init_metadata(context, cursor_description)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/cursor.py", line 1528, in _init_metadata
self._metadata = metadata = CursorResultMetaData(
^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/cursor.py", line 376, in __init__
raw = self._merge_cursor_description(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/cursor.py", line 633, in _merge_cursor_description
context.get_result_processor(
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/engine/default.py", line 1804, in get_result_processor
return type_._cached_result_processor(self.dialect, coltype)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/opt/airbyte/.venv/lib/python3.12/site-packages/sqlalchemy/sql/type_api.py", line 951, in _cached_result_processor
d["result"][coltype] = rp
~~~~~~~~~~~^^^^^^^^^
TypeError: unhashable type: '_duckdb.typing.DuckDBPyType'
Same error TypeError: unhashable type: '_duckdb.typing.DuckDBPyType' is returned on other example scripts, except run_faker_samples.py, where DuckDB cache is not used.