From 8dcc5ad5fcfaa2e309585f67de2e96827e57d2e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Tue, 30 Jun 2026 17:22:25 -0500 Subject: [PATCH 1/4] feat(aws): add S3 endpoint and user-agent support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Gonzalo Peña-Castellanos --- .../_create_a_data_source/_s3/_s3.md | 2 +- great_expectations/compatibility/aws.py | 37 +++++++++++++++++++ .../datasource/fluent/pandas_s3_datasource.py | 2 +- .../datasource/fluent/spark_s3_datasource.py | 2 +- .../pandas_execution_engine.py | 2 +- 5 files changed, 41 insertions(+), 4 deletions(-) diff --git a/docs/docusaurus/docs/core/connect_to_data/filesystem_data/_create_a_data_source/_s3/_s3.md b/docs/docusaurus/docs/core/connect_to_data/filesystem_data/_create_a_data_source/_s3/_s3.md index 90ac34eab697..a5535fc3f456 100644 --- a/docs/docusaurus/docs/core/connect_to_data/filesystem_data/_create_a_data_source/_s3/_s3.md +++ b/docs/docusaurus/docs/core/connect_to_data/filesystem_data/_create_a_data_source/_s3/_s3.md @@ -43,7 +43,7 @@ import PrereqDataContext from '../../../../_core_components/prerequisites/_preco The parameter `boto3_options` allows you to pass the following information: - `region_name`: Your AWS region name. - - `endpoint_url`: specifies an S3 endpoint. You can provide an environment variable reference such as `"${S3_ENDPOINT}"` to securely include this in your code. The string `"${S3_ENDPOINT}"` will be replaced with the value of the environment variable `S3_ENDPOINT`. + - `endpoint_url`: specifies an S3 endpoint. Set this to connect to an S3-compatible object store (Amazon S3, or a compatible provider such as Backblaze B2, Cloudflare R2, or MinIO); leave it unset for Amazon S3. You can provide an environment variable reference such as `"${S3_ENDPOINT}"` to securely include this in your code. The string `"${S3_ENDPOINT}"` will be replaced with the value of the environment variable `S3_ENDPOINT`. For more information on secure storage and retrieval of credentials in GX see [Configure credentials](/core/connect_to_data/sql_data/sql_data.md#configure-credentials). diff --git a/great_expectations/compatibility/aws.py b/great_expectations/compatibility/aws.py index 11f826fe33f9..dd976fccd31e 100644 --- a/great_expectations/compatibility/aws.py +++ b/great_expectations/compatibility/aws.py @@ -1,5 +1,8 @@ from __future__ import annotations +from importlib import metadata +from typing import Any, Dict + from great_expectations.compatibility.not_imported import NotImported BOTO_NOT_IMPORTED = NotImported( @@ -32,6 +35,40 @@ except ImportError: exceptions = BOTO_NOT_IMPORTED + +def _get_distribution_version() -> str: + try: + return metadata.version("great_expectations") + except metadata.PackageNotFoundError: + return "dev" + + +def get_s3_boto3_options(boto3_options: Dict[str, Any]) -> Dict[str, Any]: + """Return boto3 client options with a ``great-expectations`` agent suffix. + + Works with Amazon S3 and any S3-compatible object store (for example + Backblaze B2, Cloudflare R2, or MinIO). A caller-supplied ``config`` and + ``endpoint_url`` are preserved; the suffix is appended to an existing agent + string rather than replacing it. + """ + suffix = f"great-expectations/{_get_distribution_version()}" + options = dict(boto3_options) + + if not Config: + return options + + config = options.get("config") + existing = getattr(config, "user_agent_extra", None) if config else None + user_agent_extra = f"{existing} {suffix}" if existing else suffix + + if config: + options["config"] = config.merge(Config(user_agent_extra=user_agent_extra)) + else: + options["config"] = Config(user_agent_extra=user_agent_extra) + + return options + + try: import sqlalchemy_redshift except ImportError: diff --git a/great_expectations/datasource/fluent/pandas_s3_datasource.py b/great_expectations/datasource/fluent/pandas_s3_datasource.py index 53886b655a89..c62f7f55b12d 100644 --- a/great_expectations/datasource/fluent/pandas_s3_datasource.py +++ b/great_expectations/datasource/fluent/pandas_s3_datasource.py @@ -68,7 +68,7 @@ def _get_s3_client(self) -> BaseClient: "boto3_options", {} ) try: - s3_client = aws.boto3.client("s3", **boto3_options) + s3_client = aws.boto3.client("s3", **aws.get_s3_boto3_options(boto3_options)) except Exception as e: # Failure to create "s3_client" is most likely due invalid "boto3_options" dictionary. # noqa: E501 # FIXME CoP raise PandasS3DatasourceError( # noqa: TRY003 # FIXME CoP diff --git a/great_expectations/datasource/fluent/spark_s3_datasource.py b/great_expectations/datasource/fluent/spark_s3_datasource.py index 30531171fc2b..63e1a934f62f 100644 --- a/great_expectations/datasource/fluent/spark_s3_datasource.py +++ b/great_expectations/datasource/fluent/spark_s3_datasource.py @@ -70,7 +70,7 @@ def _get_s3_client(self) -> BaseClient: "boto3_options", {} ) try: - s3_client = aws.boto3.client("s3", **boto3_options) + s3_client = aws.boto3.client("s3", **aws.get_s3_boto3_options(boto3_options)) except Exception as e: # Failure to create "s3_client" is most likely due invalid "boto3_options" dictionary. # noqa: E501 # FIXME CoP raise SparkS3DatasourceError( # noqa: TRY003 # FIXME CoP diff --git a/great_expectations/execution_engine/pandas_execution_engine.py b/great_expectations/execution_engine/pandas_execution_engine.py index f5a5646c85f2..d46abb77b325 100644 --- a/great_expectations/execution_engine/pandas_execution_engine.py +++ b/great_expectations/execution_engine/pandas_execution_engine.py @@ -167,7 +167,7 @@ def _instantiate_azure_client(self) -> None: def _instantiate_s3_client(self) -> None: # If s3_client was passed in (from data source) use it, otherwise create our own self._s3 = self._config.get("s3_client") or aws.boto3.client( - "s3", **self.config.get("boto3_options", {}) + "s3", **aws.get_s3_boto3_options(self.config.get("boto3_options", {})) ) def _instantiate_gcs_client(self) -> None: From 54ce927fee35e76bff1a92347f1946bab8871602 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Wed, 8 Jul 2026 18:57:18 -0500 Subject: [PATCH 2/4] Fix S3 boto options review comments --- great_expectations/compatibility/aws.py | 6 ++-- tests/compatibility/test_aws.py | 43 +++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 tests/compatibility/test_aws.py diff --git a/great_expectations/compatibility/aws.py b/great_expectations/compatibility/aws.py index dd976fccd31e..5aef3bdd6c08 100644 --- a/great_expectations/compatibility/aws.py +++ b/great_expectations/compatibility/aws.py @@ -54,14 +54,14 @@ def get_s3_boto3_options(boto3_options: Dict[str, Any]) -> Dict[str, Any]: suffix = f"great-expectations/{_get_distribution_version()}" options = dict(boto3_options) - if not Config: + if isinstance(Config, NotImported): return options config = options.get("config") - existing = getattr(config, "user_agent_extra", None) if config else None + existing = getattr(config, "user_agent_extra", None) if config is not None else None user_agent_extra = f"{existing} {suffix}" if existing else suffix - if config: + if config is not None: options["config"] = config.merge(Config(user_agent_extra=user_agent_extra)) else: options["config"] = Config(user_agent_extra=user_agent_extra) diff --git a/tests/compatibility/test_aws.py b/tests/compatibility/test_aws.py new file mode 100644 index 000000000000..4ecd6f34e4d6 --- /dev/null +++ b/tests/compatibility/test_aws.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Any, Dict + +import pytest + +from great_expectations.compatibility import aws + +botocore_client = pytest.importorskip("botocore.client") + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def distribution_version(monkeypatch: pytest.MonkeyPatch) -> str: + version = "1.2.3" + monkeypatch.setattr(aws, "_get_distribution_version", lambda: version) + return version + + +def test_get_s3_boto3_options_adds_user_agent_suffix(distribution_version: str) -> None: + options = aws.get_s3_boto3_options({}) + + assert options["config"].user_agent_extra == f"great-expectations/{distribution_version}" + + +def test_get_s3_boto3_options_appends_user_agent_and_preserves_options( + distribution_version: str, +) -> None: + config = botocore_client.Config(user_agent_extra="my-app/1.0") + boto3_options: Dict[str, Any] = { + "config": config, + "endpoint_url": "https://s3.example.com", + } + + options = aws.get_s3_boto3_options(boto3_options) + + assert ( + options["config"].user_agent_extra + == f"my-app/1.0 great-expectations/{distribution_version}" + ) + assert options["endpoint_url"] == "https://s3.example.com" + assert boto3_options["config"] is config From f53d5028e31e9b6edd32c1253444c82ca8b6583e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Wed, 8 Jul 2026 19:03:31 -0500 Subject: [PATCH 3/4] Clarify S3 endpoint_url docs --- .../filesystem_data/_create_a_data_source/_s3/_s3.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/docusaurus/docs/core/connect_to_data/filesystem_data/_create_a_data_source/_s3/_s3.md b/docs/docusaurus/docs/core/connect_to_data/filesystem_data/_create_a_data_source/_s3/_s3.md index a5535fc3f456..96ae32ef69de 100644 --- a/docs/docusaurus/docs/core/connect_to_data/filesystem_data/_create_a_data_source/_s3/_s3.md +++ b/docs/docusaurus/docs/core/connect_to_data/filesystem_data/_create_a_data_source/_s3/_s3.md @@ -43,7 +43,7 @@ import PrereqDataContext from '../../../../_core_components/prerequisites/_preco The parameter `boto3_options` allows you to pass the following information: - `region_name`: Your AWS region name. - - `endpoint_url`: specifies an S3 endpoint. Set this to connect to an S3-compatible object store (Amazon S3, or a compatible provider such as Backblaze B2, Cloudflare R2, or MinIO); leave it unset for Amazon S3. You can provide an environment variable reference such as `"${S3_ENDPOINT}"` to securely include this in your code. The string `"${S3_ENDPOINT}"` will be replaced with the value of the environment variable `S3_ENDPOINT`. + - `endpoint_url`: specifies an S3 endpoint. Set this to connect to a non-AWS S3-compatible object store, such as Backblaze B2, Cloudflare R2, or MinIO; leave it unset for Amazon S3. You can provide an environment variable reference such as `"${S3_ENDPOINT}"` to securely include this in your code. The string `"${S3_ENDPOINT}"` will be replaced with the value of the environment variable `S3_ENDPOINT`. For more information on secure storage and retrieval of credentials in GX see [Configure credentials](/core/connect_to_data/sql_data/sql_data.md#configure-credentials). @@ -98,4 +98,3 @@ import PrereqDataContext from '../../../../_core_components/prerequisites/_preco - From 6ead077d23b6de8770a885039bc09a742f1f21b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gonzalo=20Pe=C3=B1a-Castellanos?= Date: Wed, 8 Jul 2026 19:10:50 -0500 Subject: [PATCH 4/4] Handle invalid S3 config options --- great_expectations/compatibility/aws.py | 3 +++ .../datasource/fluent/pandas_s3_datasource.py | 5 ++++- .../datasource/fluent/spark_s3_datasource.py | 5 ++++- .../execution_engine/pandas_execution_engine.py | 3 ++- tests/compatibility/test_aws.py | 11 +++++++++++ 5 files changed, 24 insertions(+), 3 deletions(-) diff --git a/great_expectations/compatibility/aws.py b/great_expectations/compatibility/aws.py index 5aef3bdd6c08..4f2da5467612 100644 --- a/great_expectations/compatibility/aws.py +++ b/great_expectations/compatibility/aws.py @@ -58,6 +58,9 @@ def get_s3_boto3_options(boto3_options: Dict[str, Any]) -> Dict[str, Any]: return options config = options.get("config") + if config is not None and not isinstance(config, Config): + return options + existing = getattr(config, "user_agent_extra", None) if config is not None else None user_agent_extra = f"{existing} {suffix}" if existing else suffix diff --git a/great_expectations/datasource/fluent/pandas_s3_datasource.py b/great_expectations/datasource/fluent/pandas_s3_datasource.py index c62f7f55b12d..2bf7c9c5b3ec 100644 --- a/great_expectations/datasource/fluent/pandas_s3_datasource.py +++ b/great_expectations/datasource/fluent/pandas_s3_datasource.py @@ -68,7 +68,10 @@ def _get_s3_client(self) -> BaseClient: "boto3_options", {} ) try: - s3_client = aws.boto3.client("s3", **aws.get_s3_boto3_options(boto3_options)) + s3_client = aws.boto3.client( + "s3", + **aws.get_s3_boto3_options(boto3_options), + ) except Exception as e: # Failure to create "s3_client" is most likely due invalid "boto3_options" dictionary. # noqa: E501 # FIXME CoP raise PandasS3DatasourceError( # noqa: TRY003 # FIXME CoP diff --git a/great_expectations/datasource/fluent/spark_s3_datasource.py b/great_expectations/datasource/fluent/spark_s3_datasource.py index 63e1a934f62f..8d26f97d3a04 100644 --- a/great_expectations/datasource/fluent/spark_s3_datasource.py +++ b/great_expectations/datasource/fluent/spark_s3_datasource.py @@ -70,7 +70,10 @@ def _get_s3_client(self) -> BaseClient: "boto3_options", {} ) try: - s3_client = aws.boto3.client("s3", **aws.get_s3_boto3_options(boto3_options)) + s3_client = aws.boto3.client( + "s3", + **aws.get_s3_boto3_options(boto3_options), + ) except Exception as e: # Failure to create "s3_client" is most likely due invalid "boto3_options" dictionary. # noqa: E501 # FIXME CoP raise SparkS3DatasourceError( # noqa: TRY003 # FIXME CoP diff --git a/great_expectations/execution_engine/pandas_execution_engine.py b/great_expectations/execution_engine/pandas_execution_engine.py index d46abb77b325..8720d7968711 100644 --- a/great_expectations/execution_engine/pandas_execution_engine.py +++ b/great_expectations/execution_engine/pandas_execution_engine.py @@ -167,7 +167,8 @@ def _instantiate_azure_client(self) -> None: def _instantiate_s3_client(self) -> None: # If s3_client was passed in (from data source) use it, otherwise create our own self._s3 = self._config.get("s3_client") or aws.boto3.client( - "s3", **aws.get_s3_boto3_options(self.config.get("boto3_options", {})) + "s3", + **aws.get_s3_boto3_options(self.config.get("boto3_options", {})), ) def _instantiate_gcs_client(self) -> None: diff --git a/tests/compatibility/test_aws.py b/tests/compatibility/test_aws.py index 4ecd6f34e4d6..162bbc0b1491 100644 --- a/tests/compatibility/test_aws.py +++ b/tests/compatibility/test_aws.py @@ -41,3 +41,14 @@ def test_get_s3_boto3_options_appends_user_agent_and_preserves_options( ) assert options["endpoint_url"] == "https://s3.example.com" assert boto3_options["config"] is config + + +def test_get_s3_boto3_options_preserves_invalid_config() -> None: + boto3_options: Dict[str, Any] = { + "config": {"user_agent_extra": "invalid"}, + "endpoint_url": "https://s3.example.com", + } + + options = aws.get_s3_boto3_options(boto3_options) + + assert options == boto3_options