Skip to content

Commit caf0ee2

Browse files
authored
Remove data type support value error in copy from gcs function (#1555)
* remove error * remove param * add override_data_type_check * ruff * add accepted data types param * remove mutable defualt
1 parent 985e292 commit caf0ee2

File tree

1 file changed

+25
-6
lines changed

1 file changed

+25
-6
lines changed

parsons/google/google_bigquery.py

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,19 @@ def copy_from_gcs(
456456
Other arguments to pass to the underlying load_table_from_uri
457457
call on the BigQuery client.
458458
"""
459-
self._validate_copy_inputs(if_exists=if_exists, data_type=data_type)
459+
self._validate_copy_inputs(
460+
if_exists=if_exists,
461+
data_type=data_type,
462+
accepted_data_types=[
463+
"csv",
464+
"json",
465+
"parquet",
466+
"datastore_backup",
467+
"newline_delimited_json",
468+
"avro",
469+
"orc",
470+
],
471+
)
460472

461473
job_config = self._process_job_config(
462474
job_config=job_config,
@@ -624,7 +636,11 @@ def copy_large_compressed_file_from_gcs(
624636
client.
625637
"""
626638

627-
self._validate_copy_inputs(if_exists=if_exists, data_type=data_type)
639+
self._validate_copy_inputs(
640+
if_exists=if_exists,
641+
data_type=data_type,
642+
accepted_data_types=["csv", "newline_delimited_json"],
643+
)
628644

629645
job_config = self._process_job_config(
630646
job_config=job_config,
@@ -1005,7 +1021,9 @@ def _prepare_local_upload_job(
10051021
):
10061022
data_type = "csv"
10071023

1008-
self._validate_copy_inputs(if_exists=if_exists, data_type=data_type)
1024+
self._validate_copy_inputs(
1025+
if_exists=if_exists, data_type=data_type, accepted_data_types=["csv"]
1026+
)
10091027

10101028
# If our source table is loaded from CSV with no transformations
10111029
# The original source file will be directly loaded to GCS
@@ -1559,14 +1577,15 @@ def _fetch_query_results(self, cursor) -> Table:
15591577
ptable = petl.frompickle(temp_filename)
15601578
return Table(ptable)
15611579

1562-
def _validate_copy_inputs(self, if_exists: str, data_type: str):
1580+
def _validate_copy_inputs(self, if_exists: str, data_type: str, accepted_data_types: list[str]):
15631581
if if_exists not in ["fail", "truncate", "append", "drop"]:
15641582
raise ValueError(
15651583
f"Unexpected value for if_exists: {if_exists}, must be one of "
15661584
'"append", "drop", "truncate", or "fail"'
15671585
)
1568-
if data_type not in ["csv", "json"]:
1569-
raise ValueError(f"Only supports csv or json files [data_type = {data_type}]")
1586+
1587+
if data_type not in accepted_data_types:
1588+
raise ValueError(f"Only supports {accepted_data_types} files [data_type = {data_type}]")
15701589

15711590
def _load_table_from_uri(
15721591
self, source_uris, destination, job_config, max_timeout, **load_kwargs

0 commit comments

Comments
 (0)