From b6204c46b6dc71a43680e5fb60bf35f4504f4eeb Mon Sep 17 00:00:00 2001 From: Willy Raedy Date: Mon, 22 Jan 2024 17:06:16 -0600 Subject: [PATCH 1/3] fixes copy_s3 by using generic GCS function --- parsons/google/google_bigquery.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index 2ae3aa8337..0a72dc6560 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -687,12 +687,13 @@ def copy_s3( # copy from S3 to GCS tmp_gcs_bucket = check_env.check("GCS_TEMP_BUCKET", tmp_gcs_bucket) gcs_client = gcs_client or GoogleCloudStorage() - temp_blob_uri = gcs_client.copy_s3_to_gcs( - aws_source_bucket=bucket, + gcs_client.copy_bucket_to_gcs( + source="s3", + source_bucket=bucket, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, gcs_sink_bucket=tmp_gcs_bucket, - aws_s3_key=key, + source_path=key, ) temp_blob_name = key temp_blob_uri = gcs_client.format_uri( From aff65dc7f7d431fa95570e8c757ba9a841ea31f2 Mon Sep 17 00:00:00 2001 From: Wil T Date: Thu, 4 Apr 2024 16:35:10 -0400 Subject: [PATCH 2/3] fix failing test in #976 (#1019) * fix AssertionError: Expected 'copy_s3_to_gcs' to be called once. Called 0 times. the test was checking for the method to have been called, but the previous commit changed the call to another method. * Bump Multiple Docs Deps Around Sphinx (#975) * Bump multiple docs deps * Downgrade myst-parser --------- Co-authored-by: Soren Spicknall --- docs/requirements.txt | 6 +++--- test/test_databases/test_bigquery.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index e18e81c1ef..5274c9bd1c 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,5 +1,5 @@ -r ../requirements.txt -Sphinx==4.3.2 -sphinx-rtd-theme==1.0.0 -myst-parser==0.16.1 +Sphinx==5.3.0 +sphinx-rtd-theme==2.0.0 +myst-parser==1.0.0 sphinx-multiversion diff --git a/test/test_databases/test_bigquery.py b/test/test_databases/test_bigquery.py index e77baee8b9..f38d16ab25 100644 --- a/test/test_databases/test_bigquery.py +++ b/test/test_databases/test_bigquery.py @@ -316,12 +316,12 @@ def test_copy_s3(self): ) # check that the method did the right things - gcs_client.copy_s3_to_gcs.assert_called_once_with( + gcs_client.copy_bucket_to_gcs.assert_called_once_with( aws_source_bucket=bucket, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, gcs_sink_bucket=tmp_gcs_bucket, - aws_s3_key=key, + source_path=key, ) bq.copy_from_gcs.assert_called_once() gcs_client.delete_blob.assert_called_once() From 4c2d42d50811cf62edc9086120f6401028359e0b Mon Sep 17 00:00:00 2001 From: Wil T Date: Thu, 23 May 2024 16:08:57 -0400 Subject: [PATCH 3/3] assert correct arguments to copy_bucket_to_gcs (#1032) --- test/test_databases/test_bigquery.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_databases/test_bigquery.py b/test/test_databases/test_bigquery.py index f38d16ab25..c82b127e9e 100644 --- a/test/test_databases/test_bigquery.py +++ b/test/test_databases/test_bigquery.py @@ -292,6 +292,7 @@ def test_copy_large_compressed_file_from_gcs( def test_copy_s3(self): # setup dependencies / inputs + source = "s3" table_name = "table_name" bucket = "aws_bucket" key = "file.gzip" @@ -317,7 +318,8 @@ def test_copy_s3(self): # check that the method did the right things gcs_client.copy_bucket_to_gcs.assert_called_once_with( - aws_source_bucket=bucket, + source=source, + source_bucket=bucket, aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, gcs_sink_bucket=tmp_gcs_bucket,