Skip to content

Commit 83533ce

Browse files
committed
add params
1 parent 743f860 commit 83533ce

File tree

1 file changed

+17
-4
lines changed

1 file changed

+17
-4
lines changed

parsons/google/google_bigquery.py

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,7 @@ def copy_from_gcs(
389389
new_file_extension: str = "csv",
390390
template_table: Optional[str] = None,
391391
max_timeout: int = 21600,
392+
source_column_match: Optional[str] = None,
392393
**load_kwargs,
393394
):
394395
"""
@@ -471,6 +472,8 @@ def copy_from_gcs(
471472
quote=quote,
472473
custom_schema=schema,
473474
template_table=template_table,
475+
source_column_match=source_column_match,
476+
474477
)
475478

476479
# load CSV from Cloud Storage into BigQuery
@@ -1452,6 +1455,7 @@ def _process_job_config(
14521455
custom_schema: Optional[list] = None,
14531456
template_table: Optional[str] = None,
14541457
parsons_table: Optional[Table] = None,
1458+
source_column_match: Optional[str] = None,
14551459
) -> LoadJobConfig:
14561460
"""
14571461
Internal function to neatly process a user-supplied job configuration object.
@@ -1490,12 +1494,21 @@ def _process_job_config(
14901494
job_config.skip_leading_rows = ignoreheader
14911495

14921496
if not job_config.source_format:
1497+
data_type_mappings = {
1498+
"csv": bigquery.SourceFormat.CSV,
1499+
"parquet": bigquery.SourceFormat.PARQUET,
1500+
"datastore_backup": bigquery.SourceFormat.DATASTORE_BACKUP,
1501+
"newline_delimited_json": bigquery.SourceFormat.NEWLINE_DELIMITED_JSON,
1502+
"avro": bigquery.SourceFormat.AVRO,
1503+
"orc": bigquery.SourceFormat.ORC,
1504+
}
14931505
job_config.source_format = (
1494-
bigquery.SourceFormat.CSV
1495-
if data_type == "csv"
1496-
else bigquery.SourceFormat.NEWLINE_DELIMITED_JSON
1506+
data_type_mappings[data_type]
14971507
)
1498-
1508+
1509+
if not job_config.source_column_match:
1510+
job_config.source_column_match = source_column_match
1511+
14991512
if not job_config.field_delimiter:
15001513
if data_type == "csv":
15011514
job_config.field_delimiter = csv_delimiter

0 commit comments

Comments
 (0)