Handle invalid S3 config options

Author: goanpeca

great_expectations/compatibility/aws.py

@@ -58,6 +58,9 @@ def get_s3_boto3_options(boto3_options: Dict[str, Any]) -> Dict[str, Any]:
         return options
 
     config = options.get("config")
+    if config is not None and not isinstance(config, Config):
+        return options
+
     existing = getattr(config, "user_agent_extra", None) if config is not None else None
     user_agent_extra = f"{existing} {suffix}" if existing else suffix
 

great_expectations/datasource/fluent/pandas_s3_datasource.py

@@ -68,7 +68,10 @@ def _get_s3_client(self) -> BaseClient:
                     "boto3_options", {}
                 )
                 try:
-                    s3_client = aws.boto3.client("s3", **aws.get_s3_boto3_options(boto3_options))
+                    s3_client = aws.boto3.client(
+                        "s3",
+                        **aws.get_s3_boto3_options(boto3_options),
+                    )
                 except Exception as e:
                     # Failure to create "s3_client" is most likely due invalid "boto3_options" dictionary.  # noqa: E501 # FIXME CoP
                     raise PandasS3DatasourceError(  # noqa: TRY003 # FIXME CoP

great_expectations/datasource/fluent/spark_s3_datasource.py

@@ -70,7 +70,10 @@ def _get_s3_client(self) -> BaseClient:
                     "boto3_options", {}
                 )
                 try:
-                    s3_client = aws.boto3.client("s3", **aws.get_s3_boto3_options(boto3_options))
+                    s3_client = aws.boto3.client(
+                        "s3",
+                        **aws.get_s3_boto3_options(boto3_options),
+                    )
                 except Exception as e:
                     # Failure to create "s3_client" is most likely due invalid "boto3_options" dictionary.  # noqa: E501 # FIXME CoP
                     raise SparkS3DatasourceError(  # noqa: TRY003 # FIXME CoP

great_expectations/execution_engine/pandas_execution_engine.py

@@ -167,7 +167,8 @@ def _instantiate_azure_client(self) -> None:
     def _instantiate_s3_client(self) -> None:
         # If s3_client was passed in (from data source) use it, otherwise create our own
         self._s3 = self._config.get("s3_client") or aws.boto3.client(
-            "s3", **aws.get_s3_boto3_options(self.config.get("boto3_options", {}))
+            "s3",
+            **aws.get_s3_boto3_options(self.config.get("boto3_options", {})),
         )
 
     def _instantiate_gcs_client(self) -> None:

tests/compatibility/test_aws.py

@@ -41,3 +41,14 @@ def test_get_s3_boto3_options_appends_user_agent_and_preserves_options(
     )
     assert options["endpoint_url"] == "https://s3.example.com"
     assert boto3_options["config"] is config
+
+
+def test_get_s3_boto3_options_preserves_invalid_config() -> None:
+    boto3_options: Dict[str, Any] = {
+        "config": {"user_agent_extra": "invalid"},
+        "endpoint_url": "https://s3.example.com",
+    }
+
+    options = aws.get_s3_boto3_options(boto3_options)
+
+    assert options == boto3_options