adding in global setting of aws access/secret keys

Author: sarahdasko

dbt/adapters/clickhouse/impl.py

@@ -251,6 +251,7 @@ def s3source_clause(
     ) -> str:
         s3config = self.config.vars.vars.get(config_name, {})
         s3config.update(s3_model_config)
+
         structure = structure or s3config.get('structure', '')
         struct = ''
         if structure:
@@ -261,7 +262,9 @@ def s3source_clause(
                 struct = f", '{','.join(structure)}'"
             else:
                 struct = f",'{structure}'"
+
         fmt = fmt or s3config.get('fmt')
+
         bucket = bucket or s3config.get('bucket', '')
         path = path or s3config.get('path', '')
         url = bucket.replace('https://', '')
@@ -270,22 +273,29 @@ def s3source_clause(
                 path = f'/{path}'
             url = f'{url}{path}'.replace('//', '/')
         url = f'https://{url}'
+
         access = ''
+        aws_access_key_id = aws_access_key_id or s3config.get('aws_access_key_id')
+        aws_secret_access_key = aws_secret_access_key or s3config.get('aws_secret_access_key')
         if aws_access_key_id and not aws_secret_access_key:
             raise DbtRuntimeError('S3 aws_access_key_id specified without aws_secret_access_key')
         if aws_secret_access_key and not aws_access_key_id:
             raise DbtRuntimeError('S3 aws_secret_access_key specified without aws_access_key_id')
         if aws_access_key_id:
             access = f", '{aws_access_key_id}', '{aws_secret_access_key}'"
+
         comp = compression or s3config.get('compression', '')
         if comp:
             comp = f"', {comp}'"
+
         extra_credentials = ''
         role_arn = role_arn or s3config.get('role_arn')
         if role_arn:
             extra_credentials = f", extra_credentials(role_arn='{role_arn}')"
+
         return f"s3('{url}'{access}, '{fmt}'{struct}{comp}{extra_credentials})"
 
+
     def check_schema_exists(self, database, schema):
         results = self.execute_macro(LIST_SCHEMAS_MACRO_NAME, kwargs={'database': database})
         return schema in (row[0] for row in results)

tests/integration/adapter/clickhouse/test_clickhouse_s3.py

@@ -115,6 +115,38 @@ def test_read(self, project):
         assert result[0] == 1000
 
 
+class TestS3AwsAccessGlobal:
+    @pytest.fixture(scope="class")
+    def project_config_update(self):
+        return {
+            'vars': {
+                'taxi_s3': {
+                    'bucket': 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/nyc-taxi/',
+                    'fmt': 'TabSeparatedWithNames',
+                    'aws_access_key_id': 'ABCDEFGHIJKLMNOPQRSTUVWXYZ',
+                    'aws_secret_access_key': '1234567890123456789012345678901234567890',
+                }
+            }
+        }
+
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "s3_taxis_source.sql": s3_taxis_full_source,
+            "schema.yml": schema_yaml,
+        }
+    
+    def test_role_arn_in_compiled_sql(self, project):
+        # Only compile, don't run
+        result = run_dbt(["compile", "--select", "s3_taxis_source.sql"], expect_pass=True)
+
+        # Assert the SQL contains the expected role_arn function call
+        assert (
+            ", 'ABCDEFGHIJKLMNOPQRSTUVWXYZ', '1234567890123456789012345678901234567890'"
+            in result.results[0].node.compiled_code
+        )
+
+
 class TestS3RoleArnGlobal:
     @pytest.fixture(scope="class")
     def project_config_update(self):
@@ -123,7 +155,7 @@ def project_config_update(self):
                 'taxi_s3': {
                     'bucket': 'https://datasets-documentation.s3.eu-west-3.amazonaws.com/nyc-taxi/',
                     'fmt': 'TabSeparatedWithNames',
-                    'role_arn': 'arn:aws:iam::123456789012:role/my-role'
+                    'role_arn': 'arn:aws:iam::123456789012:role/my-role',
                 }
             }
         }
@@ -140,4 +172,7 @@ def test_role_arn_in_compiled_sql(self, project):
         result = run_dbt(["compile", "--select", "s3_taxis_source.sql"], expect_pass=True)
 
         # Assert the SQL contains the expected role_arn function call
-        assert "extra_credentials(role_arn='arn:aws:iam::123456789012:role/my-role')" in result.results[0].node.compiled_code
+        assert (
+            "extra_credentials(role_arn='arn:aws:iam::123456789012:role/my-role')"
+            in result.results[0].node.compiled_code
+        )