|
| 1 | +from __future__ import annotations |
| 2 | + |
1 | 3 | import inspect |
2 | 4 | import json |
3 | 5 | import re |
4 | 6 | import time |
5 | | -from typing import Any, Dict, Optional, Union |
| 7 | +from typing import Any, Dict, Optional, TYPE_CHECKING, Union |
6 | 8 | import uuid |
7 | 9 |
|
| 10 | +# These imports are heavy and only needed for BigFrames Python models. |
| 11 | +# They are lazy-loaded inside BigFramesHelper methods to avoid slowing |
| 12 | +# down every `dbt parse` invocation. See: https://github.com/dbt-labs/dbt-adapters/issues/1604 |
| 13 | +if TYPE_CHECKING: |
| 14 | + from google.cloud import aiplatform_v1 |
| 15 | + from google.cloud.aiplatform import gapic as aiplatform_gapic # noqa: F401 |
| 16 | + |
8 | 17 | from dbt.adapters.base import PythonJobHelper, PythonSubmissionResult |
9 | 18 | from dbt.adapters.bigquery import BigQueryCredentials |
10 | 19 | from dbt.adapters.bigquery.clients import ( |
|
24 | 33 |
|
25 | 34 | from google.api_core.operation import Operation |
26 | 35 | from google.auth.transport.requests import Request |
27 | | -from google.cloud import aiplatform_v1 |
28 | | -from google.cloud.aiplatform import gapic as aiplatform_gapic |
29 | 36 | from google.cloud.dataproc_v1 import CreateBatchRequest, Job, RuntimeConfig |
30 | 37 | from google.cloud.dataproc_v1.types.batches import Batch |
31 | 38 | from google.protobuf.json_format import ParseDict |
32 | | -import nbformat |
33 | 39 |
|
34 | 40 | _logger = AdapterLogger("BigQuery") |
35 | 41 |
|
@@ -231,13 +237,17 @@ def _get_token(self) -> str: |
231 | 237 | return creds.token |
232 | 238 |
|
233 | 239 | def _py_to_ipynb(self, compiled_code: str) -> str: |
| 240 | + import nbformat |
| 241 | + |
234 | 242 | notebook = nbformat.v4.new_notebook() |
235 | 243 | # Put all codes in one cell. |
236 | 244 | notebook.cells.append(nbformat.v4.new_code_cell(compiled_code)) |
237 | 245 |
|
238 | 246 | return nbformat.writes(notebook, nbformat.NO_CONVERT) |
239 | 247 |
|
240 | 248 | def _get_notebook_template_id(self) -> str: |
| 249 | + from google.cloud import aiplatform_v1 |
| 250 | + |
241 | 251 | # If user specifies a runtime template id, use it. |
242 | 252 | if self._notebook_template_id: |
243 | 253 | return self._notebook_template_id |
@@ -266,6 +276,8 @@ def _get_notebook_template_id(self) -> str: |
266 | 276 | return self._create_notebook_template() |
267 | 277 |
|
268 | 278 | def _create_notebook_template(self) -> str: |
| 279 | + from google.cloud import aiplatform_v1 |
| 280 | + |
269 | 281 | # Construct the full network and subnetwork resource names. |
270 | 282 | network_full_name = f"projects/{self._project}/global/networks/{_NETWORK_NAME}" |
271 | 283 | subnetwork_full_name = ( |
@@ -308,6 +320,8 @@ def _extract_template_id(self, template_name: str) -> str: |
308 | 320 | def _config_notebook_job( |
309 | 321 | self, notebook_template_id: str |
310 | 322 | ) -> aiplatform_v1.NotebookExecutionJob: |
| 323 | + from google.cloud import aiplatform_v1 |
| 324 | + |
311 | 325 | notebook_execution_job = aiplatform_v1.NotebookExecutionJob() |
312 | 326 | notebook_execution_job.notebook_runtime_template_resource_name = ( |
313 | 327 | f"projects/{self._project}/locations/{self._region}/" |
@@ -471,6 +485,8 @@ def submit(self, compiled_code: str) -> PythonSubmissionResult: |
471 | 485 |
|
472 | 486 | def _track_notebook_job_status(self, job_name: str) -> aiplatform_v1.NotebookExecutionJob: |
473 | 487 | """Tracks the notebook job until it completes or times out.""" |
| 488 | + from google.cloud.aiplatform import gapic as aiplatform_gapic # noqa: F811 |
| 489 | + |
474 | 490 | max_wait_time = self._polling_retry.timeout |
475 | 491 | elapsed = 0 |
476 | 492 |
|
@@ -505,6 +521,8 @@ def _track_notebook_job_status(self, job_name: str) -> aiplatform_v1.NotebookExe |
505 | 521 | def _submit_bigframes_job( |
506 | 522 | self, notebook_template_id: str |
507 | 523 | ) -> aiplatform_v1.NotebookExecutionJob: |
| 524 | + from google.cloud import aiplatform_v1 # noqa: F811 |
| 525 | + from google.cloud.aiplatform import gapic as aiplatform_gapic # noqa: F811 |
508 | 526 |
|
509 | 527 | notebook_execution_job = self._config_notebook_job(notebook_template_id) |
510 | 528 |
|
|
0 commit comments