From ce3f0207b97424ffde7ea6f5a81f6e31c6a1cdf2 Mon Sep 17 00:00:00 2001 From: shizoqua Date: Thu, 16 Jul 2026 18:11:45 +0100 Subject: [PATCH 1/3] Add SparkJDBCDataset credentials URL support Signed-off-by: shizoqua --- kedro-datasets/RELEASE.md | 1 + .../spark/spark_jdbc_dataset.py | 34 ++++++-- .../tests/spark/test_spark_jdbc_dataset.py | 83 ++++++++++++++++++- 3 files changed, 110 insertions(+), 8 deletions(-) diff --git a/kedro-datasets/RELEASE.md b/kedro-datasets/RELEASE.md index 7f7eefbc3..eff6c552d 100755 --- a/kedro-datasets/RELEASE.md +++ b/kedro-datasets/RELEASE.md @@ -14,6 +14,7 @@ - Fixed `MLRunModel` so user-supplied `load_args` are now passed to `joblib.load()` (previously silently dropped). Added a deserialization warning to the docstring. - Hardened `TensorFlowModelDataset`: `safe_mode=True` is now the default for `load_model()` to prevent arbitrary code execution from untrusted model files. Fixed a bug where `tf_device` was lost from `load_args` after the first load call. - Added deserialization risk warnings to docstrings of datasets that can execute arbitrary code when loading untrusted files. +- Added support for supplying `SparkJDBCDataset` JDBC URLs through credentials. ## Community contributions - [samiat4911](https://github.com/samiat4911) diff --git a/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py b/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py index aa27be9fb..31482ecca 100644 --- a/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py +++ b/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py @@ -23,7 +23,6 @@ class SparkJDBCDataset(AbstractDataset[DataFrame, DataFrame]): weather: type: spark.SparkJDBCDataset table: weather_table - url: jdbc:postgresql://localhost/test credentials: db_credentials load_args: properties: @@ -31,6 +30,12 @@ class SparkJDBCDataset(AbstractDataset[DataFrame, DataFrame]): save_args: properties: driver: org.postgresql.Driver + + # credentials.yml + db_credentials: + url: jdbc:postgresql://localhost/test + user: scott + password: tiger ``` Using the [Python API](https://docs.kedro.org/en/stable/catalog-data/advanced_data_catalog_usage/): @@ -67,8 +72,8 @@ class SparkJDBCDataset(AbstractDataset[DataFrame, DataFrame]): def __init__( # noqa: PLR0913 self, *, - url: str, table: str, + url: str | None = None, credentials: dict[str, Any] | None = None, load_args: dict[str, Any] | None = None, save_args: dict[str, Any] | None = None, @@ -77,7 +82,8 @@ def __init__( # noqa: PLR0913 """Creates a new ``SparkJDBCDataset``. Args: - url: A JDBC URL of the form ``jdbc:subprotocol:subname``. + url: A JDBC URL of the form ``jdbc:subprotocol:subname``. When not + provided, the URL can be supplied as ``url`` in ``credentials``. table: The name of the table to load or save data to. credentials: A dictionary of JDBC database connection arguments. Normally at least properties ``user`` and ``password`` with @@ -100,6 +106,9 @@ def __init__( # noqa: PLR0913 when a property is provided with a None value. """ + credentials = credentials or {} + url = url or credentials.get("url") + if not url: raise DatasetError( "'url' argument cannot be empty. Please " @@ -124,9 +133,14 @@ def __init__( # noqa: PLR0913 self._save_args = {**self.DEFAULT_SAVE_ARGS, **(save_args or {})} # Update properties in load_args and save_args with credentials. - if credentials is not None: + credentials_properties = { + cred_key: cred_value + for cred_key, cred_value in credentials.items() + if cred_key != "url" + } + if credentials_properties: # Check credentials for bad inputs. - for cred_key, cred_value in credentials.items(): + for cred_key, cred_value in credentials_properties.items(): if cred_value is None: raise DatasetError( f"Credential property '{cred_key}' cannot be None. " @@ -135,8 +149,14 @@ def __init__( # noqa: PLR0913 load_properties = self._load_args.get("properties", {}) save_properties = self._save_args.get("properties", {}) - self._load_args["properties"] = {**load_properties, **credentials} - self._save_args["properties"] = {**save_properties, **credentials} + self._load_args["properties"] = { + **load_properties, + **credentials_properties, + } + self._save_args["properties"] = { + **save_properties, + **credentials_properties, + } def _describe(self) -> dict[str, Any]: load_args = self._load_args diff --git a/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py b/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py index af9be9cac..bc61b9732 100644 --- a/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py +++ b/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py @@ -39,7 +39,19 @@ def test_missing_url(): " URL of the form 'jdbc:subprotocol:subname'." ) with pytest.raises(DatasetError, match=error_message): - SparkJDBCDataset(url=None, table="dummy_table") + SparkJDBCDataset(table="dummy_table") + + +def test_missing_url_when_credentials_do_not_contain_url(): + error_message = ( + "'url' argument cannot be empty. Please provide a JDBC" + " URL of the form 'jdbc:subprotocol:subname'." + ) + with pytest.raises(DatasetError, match=error_message): + SparkJDBCDataset( + table="dummy_table", + credentials={"user": "dummy_user", "password": "dummy_pw"}, + ) def test_missing_table(): @@ -69,6 +81,50 @@ def test_save_credentials(mocker, spark_jdbc_args_credentials): ) +def test_save_credentials_url(mocker): + mock_data = mocker.Mock() + credentials = { + "url": "credentials_url", + "user": "dummy_user", + "password": "dummy_pw", + } + dataset = SparkJDBCDataset(table="dummy_table", credentials=credentials) + + dataset.save(mock_data) + + mock_data.write.jdbc.assert_called_with( + "credentials_url", + "dummy_table", + properties={"user": "dummy_user", "password": "dummy_pw"}, + ) + assert credentials == { + "url": "credentials_url", + "user": "dummy_user", + "password": "dummy_pw", + } + + +def test_save_explicit_url_takes_precedence_over_credentials_url(mocker): + mock_data = mocker.Mock() + dataset = SparkJDBCDataset( + url="dummy_url", + table="dummy_table", + credentials={ + "url": "credentials_url", + "user": "dummy_user", + "password": "dummy_pw", + }, + ) + + dataset.save(mock_data) + + mock_data.write.jdbc.assert_called_with( + "dummy_url", + "dummy_table", + properties={"user": "dummy_user", "password": "dummy_pw"}, + ) + + def test_save_args(mocker, spark_jdbc_args_save_load): mock_data = mocker.Mock() dataset = SparkJDBCDataset(**spark_jdbc_args_save_load) @@ -108,6 +164,31 @@ def test_load_credentials(mocker, spark_jdbc_args_credentials): ) +def test_load_credentials_url(mocker): + spark = mocker.patch( + "kedro_datasets.spark.spark_jdbc_dataset.get_spark" + ).return_value + credentials = { + "url": "credentials_url", + "user": "dummy_user", + "password": "dummy_pw", + } + dataset = SparkJDBCDataset(table="dummy_table", credentials=credentials) + + dataset.load() + + spark.read.jdbc.assert_called_with( + "credentials_url", + "dummy_table", + properties={"user": "dummy_user", "password": "dummy_pw"}, + ) + assert credentials == { + "url": "credentials_url", + "user": "dummy_user", + "password": "dummy_pw", + } + + def test_load_args(mocker, spark_jdbc_args_save_load): spark = mocker.patch( "kedro_datasets.spark.spark_jdbc_dataset.get_spark" From 7d341e01b3470aced187d5c84ea9de0ecc40d1f9 Mon Sep 17 00:00:00 2001 From: shizoqua Date: Sat, 25 Jul 2026 09:14:57 +0100 Subject: [PATCH 2/3] Address review: update secrets baseline, add contributor, load-precedence test, docstring note Signed-off-by: shizoqua --- .secrets.baseline | 2 +- kedro-datasets/RELEASE.md | 1 + .../spark/spark_jdbc_dataset.py | 5 +++- .../tests/spark/test_spark_jdbc_dataset.py | 23 +++++++++++++++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index af11f3d5c..9a3666d9b 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -167,7 +167,7 @@ "filename": "kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py", "hashed_secret": "46e3d772a1888eadff26c7ada47fd7502d796e07", "is_verified": false, - "line_number": 53 + "line_number": 58 } ], "kedro-datasets/kedro_datasets_experimental/tests/netcdf/test_netcdf_dataset.py": [ diff --git a/kedro-datasets/RELEASE.md b/kedro-datasets/RELEASE.md index eff6c552d..795ce356d 100755 --- a/kedro-datasets/RELEASE.md +++ b/kedro-datasets/RELEASE.md @@ -18,6 +18,7 @@ ## Community contributions - [samiat4911](https://github.com/samiat4911) +- [Shizoqua](https://github.com/Shizoqua) # Release 9.5.0 diff --git a/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py b/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py index 31482ecca..5ed42a475 100644 --- a/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py +++ b/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py @@ -83,7 +83,10 @@ def __init__( # noqa: PLR0913 Args: url: A JDBC URL of the form ``jdbc:subprotocol:subname``. When not - provided, the URL can be supplied as ``url`` in ``credentials``. + provided, the URL can be supplied as ``url`` in ``credentials`` + (mirrors the ``pandas.SQLTableDataset`` ``credentials.con`` + convention for keeping connection endpoints out of + ``catalog.yml``). table: The name of the table to load or save data to. credentials: A dictionary of JDBC database connection arguments. Normally at least properties ``user`` and ``password`` with diff --git a/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py b/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py index bc61b9732..ada2ad82b 100644 --- a/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py +++ b/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py @@ -189,6 +189,29 @@ def test_load_credentials_url(mocker): } +def test_load_explicit_url_takes_precedence_over_credentials_url(mocker): + spark = mocker.patch( + "kedro_datasets.spark.spark_jdbc_dataset.get_spark" + ).return_value + dataset = SparkJDBCDataset( + url="dummy_url", + table="dummy_table", + credentials={ + "url": "credentials_url", + "user": "dummy_user", + "password": "dummy_pw", + }, + ) + + dataset.load() + + spark.read.jdbc.assert_called_with( + "dummy_url", + "dummy_table", + properties={"user": "dummy_user", "password": "dummy_pw"}, + ) + + def test_load_args(mocker, spark_jdbc_args_save_load): spark = mocker.patch( "kedro_datasets.spark.spark_jdbc_dataset.get_spark" From cf5958ab0d0752ddad1a7b3154a0b4afd28690dd Mon Sep 17 00:00:00 2001 From: shizoqua Date: Sat, 1 Aug 2026 14:31:14 +0100 Subject: [PATCH 3/3] Hide credentials URL from Spark JDBC dataset description Signed-off-by: shizoqua --- .../kedro_datasets/spark/spark_jdbc_dataset.py | 1 - .../tests/spark/test_spark_jdbc_dataset.py | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py b/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py index 5ed42a475..0d9830b3e 100644 --- a/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py +++ b/kedro-datasets/kedro_datasets/spark/spark_jdbc_dataset.py @@ -178,7 +178,6 @@ def _describe(self) -> dict[str, Any]: save_args = {**save_args, "properties": save_properties} return { - "url": self._url, "table": self._table, "load_args": load_args, "save_args": save_args, diff --git a/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py b/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py index ada2ad82b..08134e5b5 100644 --- a/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py +++ b/kedro-datasets/tests/spark/test_spark_jdbc_dataset.py @@ -212,6 +212,21 @@ def test_load_explicit_url_takes_precedence_over_credentials_url(mocker): ) +def test_describe_does_not_include_url_from_credentials(): + dataset = SparkJDBCDataset( + table="dummy_table", + credentials={ + "url": "credentials_url", + "user": "dummy_user", + "password": "dummy_pw", + }, + ) + + described = dataset._describe() + + assert "url" not in described + + def test_load_args(mocker, spark_jdbc_args_save_load): spark = mocker.patch( "kedro_datasets.spark.spark_jdbc_dataset.get_spark"