From f54acf3754268f9883ff4e5636797171dc3b3afb Mon Sep 17 00:00:00 2001 From: Maximiliano Felice Date: Wed, 25 Nov 2020 22:49:12 -0300 Subject: [PATCH] Adding configurable BigQuery ProjectID on contrib jobs --- luigi/contrib/bigquery.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/luigi/contrib/bigquery.py b/luigi/contrib/bigquery.py index 55fc3b72c8..20a7dd0cd2 100644 --- a/luigi/contrib/bigquery.py +++ b/luigi/contrib/bigquery.py @@ -527,6 +527,12 @@ def allow_quoted_new_lines(self): """ Indicates if BigQuery should allow quoted data sections that contain newline characters in a CSV file. The default value is false.""" return False + @property + def project_id(self): + """Project ID on which to run the BigQuery Job + """ + return self.output().table.project_id + def run(self): output = self.output() assert isinstance(output, BigQueryTarget), 'Output must be a BigQueryTarget, not %s' % (output) @@ -565,7 +571,7 @@ def run(self): else: job['configuration']['load']['autodetect'] = True - bq_client.run_job(output.table.project_id, job, dataset=output.table.dataset) + bq_client.run_job(self.project_id, job, dataset=output.table.dataset) class BigQueryRunQueryTask(MixinBigQueryBulkComplete, luigi.Task): @@ -610,6 +616,12 @@ def use_legacy_sql(self): """ return True + @property + def project_id(self): + """Project ID on which to run the BigQuery Job + """ + return self.output().table.project_id + def run(self): output = self.output() assert isinstance(output, BigQueryTarget), 'Output must be a BigQueryTarget, not %s' % (output) @@ -643,7 +655,7 @@ def run(self): } } - bq_client.run_job(output.table.project_id, job, dataset=output.table.dataset) + bq_client.run_job(self.project_id, job, dataset=output.table.dataset) class BigQueryCreateViewTask(luigi.Task):