From 0f5d5ece7eb4bcce7b82e9c48034505101b4f622 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Wed, 15 Oct 2025 19:58:27 +0000 Subject: [PATCH 1/8] Migrate gsutil usage to gcloud storage --- ...ith_dataflow_flex_template_component.ipynb | 22 +++++++------------ ...den_proprietary_image_classification.ipynb | 6 ++--- .../model_garden_pytorch_imagebind.ipynb | 15 +++++-------- ..._garden_weather_prediction_on_vertex.ipynb | 13 +++++------ ...mage_classification_batch_prediction.ipynb | 22 ++++++------------- ...e_object_detection_online_prediction.ipynb | 20 ++++++----------- ..._with_custom_model_evaluation_import.ipynb | 6 ++--- ...lm_evaluation_for_summarization_task.ipynb | 9 +++----- ...tom_training_with_prebuilt_container.ipynb | 11 ++++------ ...tom_training_with_prebuilt_container.ipynb | 12 ++++------ 10 files changed, 48 insertions(+), 88 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb index 6381c4101..5e3685a94 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb @@ -456,8 +456,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -476,8 +475,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -546,10 +544,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer ] }, { "cell_type": "markdown", @@ -678,8 +675,7 @@ "outputs": [], "source": [ "GCS_AVRO_SCHEMA = BUCKET_URI + \"/gaming_schema.avsc\"\n", - "! gsutil cp gaming_schema.avsc $GCS_AVRO_SCHEMA\n", - "\n", + "! gcloud storage cp gaming_schema.avsc $GCS_AVRO_SCHEMA\n", "\n", "GCS_FLEX_TEMPLATE_PATH = \"gs://dataflow-templates/latest/flex/File_Format_Conversion\"\n", "GCS_CONVERT_IN = \"gs://dataflow-samples/game/5000_gaming_data.csv\"\n", "GCS_CONVERT_OUT = BUCKET_URI + \"/parquet_out/\"\n", @@ -757,8 +753,7 @@ "\n", "pipeline.run()\n", "\n", - "! gsutil ls $GCS_CONVERT_OUT\n", - "\n", + "! gcloud storage ls $GCS_CONVERT_OUT\n", "\n", "! rm -f dataflow_file_conversion.yaml gaming_schema.avsc" ] }, @@ -813,8 +808,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb b/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb index 267d0b504..3b2dfd11b 100644 --- a/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb +++ b/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb @@ -309,12 +309,10 @@ }, "outputs": [], "source": [ - "count = ! gsutil cat $DATASET_FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $DATASET_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $DATASET_FILE | head" - ] + "! gcloud storage cat $DATASET_FILE | head" ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb b/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb index 2ccc9d71e..abef3764b 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb @@ -173,11 +173,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -201,8 +199,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -300,8 +297,7 @@ "%cd ImageBind/.assets\n", "! git reset --hard 95d27c7fd5a8362f3527e176c3a80ae5a4d880c0\n", "\n", - "! gsutil cp -r . $DATA_BUCKET\n", - "\n", + "! gcloud storage cp --recursive . $DATA_BUCKET\n", "\n", "%cd ../.." ] }, @@ -571,8 +567,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb index 69f7f34dc..dc3c02fda 100644 --- a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb @@ -148,11 +148,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -174,7 +173,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -583,8 +583,7 @@ "\n", "delete_bucket = True # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI\n", - "\n" + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n" ] } ], diff --git a/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb b/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb index 038a6d42a..be63f846a 100644 --- a/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb +++ b/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb @@ -287,8 +287,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -394,12 +393,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -580,8 +577,7 @@ }, "outputs": [], "source": [ - "test_items = !gsutil cat $IMPORT_FILE | head -n2\n", - "if len(str(test_items[0]).split(\",\")) == 3:\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(str(test_items[0]).split(\",\")) == 3:\n", " _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n", " _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n", "else:\n", @@ -614,9 +610,7 @@ "file_1 = test_item_1.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n", "\n", - "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n", - "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n", - "\n", + "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", "\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_2 = BUCKET_URI + \"/\" + file_2" ] @@ -659,8 +653,7 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gsutil cat $gcs_input_uri" - ] + "! gcloud storage cat $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -803,8 +796,7 @@ "# Delete the cloud storage bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb b/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb index ad6032e8f..b54f1308a 100644 --- a/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb +++ b/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb @@ -297,8 +297,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -386,7 +385,7 @@ "source": [ "#### Copying data between Google Cloud Storage Buckets\n", "\n", - "In this step, you prevent access issues for the images in your original dataset. The code below extracts folder paths from image paths, constructs destination paths for Cloud Storage, copies images using gsutil commands, updates image paths in the DataFrame, and finally saves the modified DataFrame back to Cloud Storage as a CSV file." + "In this step, you prevent access issues for the images in your original dataset. The code below extracts folder paths from image paths, constructs destination paths for Cloud Storage, copies images using gcloud storage commands, updates image paths in the DataFrame, and finally saves the modified DataFrame back to Cloud Storage as a CSV file." ] }, { @@ -415,8 +414,7 @@ "\n", "# Copy images using gsutil commands directly\n", "for src, dest in zip(df.iloc[:, 0], df[\"destination_path\"]):\n", - " ! gsutil -m cp {src} {dest}\n", - "\n", + " ! gcloud storage cp {src} {dest}\n", "\n", "print(f\"Files copied to {BUCKET_URI}\")" ] }, @@ -496,12 +494,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -706,8 +702,7 @@ }, "outputs": [], "source": [ - "test_items = !gsutil cat $IMPORT_FILE | head -n1\n", - "cols = str(test_items[0]).split(\",\")\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n", "cols = str(test_items[0]).split(\",\")\n", "if len(cols) == 11:\n", " test_item = str(cols[1])\n", " test_label = str(cols[2])\n", @@ -834,8 +829,7 @@ "dag.delete()\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb b/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb index 27a2ff1e1..f735e4b49 100644 --- a/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb +++ b/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb @@ -304,8 +304,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create {BUCKET_URI} --location={LOCATION} --project={PROJECT_ID}" ] }, { "cell_type": "markdown", @@ -778,8 +777,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/model_evaluation/model_based_llm_evaluation/autosxs_llm_evaluation_for_summarization_task.ipynb b/notebooks/official/model_evaluation/model_based_llm_evaluation/autosxs_llm_evaluation_for_summarization_task.ipynb index 3cc960db1..fbe43d9cd 100644 --- a/notebooks/official/model_evaluation/model_based_llm_evaluation/autosxs_llm_evaluation_for_summarization_task.ipynb +++ b/notebooks/official/model_evaluation/model_based_llm_evaluation/autosxs_llm_evaluation_for_summarization_task.ipynb @@ -355,8 +355,7 @@ "):\n", " BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + UUID\n", "\n", - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -519,8 +518,7 @@ "outputs": [], "source": [ "examples.to_json(\"evaluation_dataset.json\", orient=\"records\", lines=True)\n", - "! gsutil cp evaluation_dataset.json $BUCKET_URI/input/evaluation_dataset.json\n", - "DATASET = f\"{BUCKET_URI}/input/evaluation_dataset.json\"" + "! gcloud storage cp evaluation_dataset.json $BUCKET_URI/input/evaluation_dataset.json\n", "DATASET = f\"{BUCKET_URI}/input/evaluation_dataset.json\"" ] }, { @@ -765,8 +763,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb b/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb index 2f5ce0ce7..4edb31ddb 100644 --- a/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb +++ b/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb @@ -316,8 +316,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -480,7 +479,7 @@ "outputs": [], "source": [ "# Download the sample code\n", - "! gsutil cp gs://cloud-samples-data/ai-platform/hello-custom/hello-custom-sample-v1.tar.gz - | tar -xzv\n", + "! gcloud storage cp gs://cloud-samples-data/ai-platform/hello-custom/hello-custom-sample-v1.tar.gz - | tar -xzv\n", "%cd hello-custom-sample/" ] }, @@ -664,8 +663,7 @@ "outputs": [], "source": [ "GCS_BUCKET_TRAINING = f\"{BUCKET_URI}/data/\"\n", - "! gsutil cp dist/hello-custom-training-3.0.tar.gz {GCS_BUCKET_TRAINING}" - ] + "! gcloud storage cp dist/hello-custom-training-3.0.tar.gz {GCS_BUCKET_TRAINING}" ] }, { "cell_type": "markdown", @@ -787,8 +785,7 @@ "# Delete GCS bucket.\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "!rm -rf ../hello-custom-sample/" ] } diff --git a/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb b/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb index 165bbd3af..15ff3587e 100644 --- a/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb +++ b/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb @@ -356,8 +356,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -698,10 +697,8 @@ "source": [ "!cd {PYTHON_PACKAGE_APPLICATION_DIR} && python3 setup.py sdist --formats=gztar\n", "\n", - "!gsutil cp {source_package_file_name} {python_package_gcs_uri}\n", - "\n", - "!gsutil ls -l {python_package_gcs_uri}" - ] + "!gcloud storage cp {source_package_file_name} {python_package_gcs_uri}\n", "\n", + "!gcloud storage ls --long {python_package_gcs_uri}" ] }, { "cell_type": "markdown", @@ -833,8 +830,7 @@ "# delete the bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From ca51487de3ba6b5d93042dcf16c03a2bfdd9b8ca Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 11 Dec 2025 11:59:22 +0000 Subject: [PATCH 2/8] changes for 4299 --- ...ith_dataflow_flex_template_component.ipynb | 3 +- .../model_garden_pytorch_imagebind.ipynb | 15 ++- ..._garden_weather_prediction_on_vertex.ipynb | 123 +++++++++--------- ...tom_training_with_prebuilt_container.ipynb | 14 +- 4 files changed, 82 insertions(+), 73 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb index 5e3685a94..5d0882e8d 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb @@ -545,8 +545,7 @@ "outputs": [], "source": [ "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. - ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb b/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb index abef3764b..8bd3a89f0 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb @@ -173,9 +173,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -199,7 +201,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -297,7 +300,8 @@ "%cd ImageBind/.assets\n", "! git reset --hard 95d27c7fd5a8362f3527e176c3a80ae5a4d880c0\n", "\n", - "! gcloud storage cp --recursive . $DATA_BUCKET\n", "\n", + "! gcloud storage cp --recursive . $DATA_BUCKET\n", + "\n", "%cd ../.." ] }, @@ -567,7 +571,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb index dc3c02fda..7943a8076 100644 --- a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb @@ -31,18 +31,18 @@ }, "source": [ "# Vertex AI Model Garden - WeatherNext Forecasting\n", - "\u003ctable\u003e\u003ctbody\u003e\u003ctr\u003e\n", - " \u003ctd style=\"text-align: center\"\u003e\n", - " \u003ca href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_weather_prediction_on_vertex.ipynb\"\u003e\n", - " \u003cimg alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"\u003e\u003cbr\u003e Run in Colab Enterprise\n", - " \u003c/a\u003e\n", - " \u003c/td\u003e\n", - " \u003ctd style=\"text-align: center\"\u003e\n", - " \u003ca href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb\"\u003e\n", - " \u003cimg alt=\"GitHub logo\" src=\"https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png\" width=\"32px\"\u003e\u003cbr\u003e View on GitHub\n", - " \u003c/a\u003e\n", - " \u003c/td\u003e\n", - "\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e" + "\n", + " \n", + " \n", + "
\n", + " \n", + " \"Google
Run in Colab Enterprise\n", + " \n", + " 		\n", + " \n", + " \"GitHub
View on GitHub\n", + " \n", + "
" ] }, { @@ -81,15 +81,15 @@ "\n", "### Request For TPU Quota\n", "\n", - "By default, the quota for TPU training [Custom model training TPU v5e cores per region](https://console.cloud.google.com/iam-admin/quotas?location=us-central1\u0026metric=aiplatform.googleapis.com%2Fcustom_model_training_tpu_v5e) is 0. TPU quota is only available in `us-west1`, `us-west4`, `us-central1`. You can request for higher TPU quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota). It is suggested to request at least 4 v5e to run this notebook." + "By default, the quota for TPU training [Custom model training TPU v5e cores per region](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_training_tpu_v5e) is 0. TPU quota is only available in `us-west1`, `us-west4`, `us-central1`. You can request for higher TPU quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota). It is suggested to request at least 4 v5e to run this notebook." ] }, { "cell_type": "code", "execution_count": null, "metadata": { - "id": "2707b02ef5df", - "cellView": "form" + "cellView": "form", + "id": "2707b02ef5df" }, "outputs": [], "source": [ @@ -117,7 +117,7 @@ "from google.cloud import aiplatform, storage\n", "\n", "# Upgrade Vertex AI SDK.\n", - "! pip3 install --upgrade --quiet 'google-cloud-aiplatform\u003e=1.64.0'\n", + "! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.64.0'\n", "if not os.path.exists(\"./vertex-ai-samples\"):\n", " ! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n", "! pip3 uninstall --quiet -y xarray\n", @@ -148,10 +148,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -173,7 +174,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -181,7 +181,7 @@ "\n", "\n", "# Utility functions for vertex jobs.\n", - "def get_job_name_with_datetime(prefix: str) -\u003e str:\n", + "def get_job_name_with_datetime(prefix: str) -> str:\n", " \"\"\"Gets the job name with date time when triggering training or deployment\n", " jobs in Vertex AI.\n", " \"\"\"\n", @@ -232,7 +232,7 @@ "def get_existing_demo_step(\n", " num_forecast_steps: int,\n", " model_type: str = \"gen_small\",\n", - " ) -\u003e str:\n", + " ) -> str:\n", " # The demo data only supports some steps, and we can only run the predictions\n", " # if num_forecast_steps is smaller than or equal to the maximum supported setps.\n", " if model_type == \"gen_small\":\n", @@ -250,18 +250,18 @@ " # Find the proper demo data for forecasting.\n", " found_supported_step = supported_demo_steps[-1]\n", " for i, supported_step in enumerate(supported_demo_steps):\n", - " if num_forecast_steps \u003c= supported_step:\n", + " if num_forecast_steps <= supported_step:\n", " found_supported_step = supported_step\n", " break\n", - " if num_forecast_steps \u003e found_supported_step:\n", + " if num_forecast_steps > found_supported_step:\n", " raise ValueError(f\"Supported demo steps for {model_type} in gs://dm_graphcast are {supported_demo_steps}. {num_forecast_steps} is too large, and could not find proper demo data.\")\n", " return found_supported_step\n", "\n", "def get_suggested_machines(\n", " num_forecast_steps: int,\n", - " model_type: str = \"gen_small\",) -\u003e Tuple[str, str, int]:\n", + " model_type: str = \"gen_small\",) -> Tuple[str, str, int]:\n", " if model_type == \"gen_small\":\n", - " if num_forecast_steps \u003c= 16:\n", + " if num_forecast_steps <= 16:\n", " machine_type = \"ct5lp-hightpu-4t\"\n", " tpu_topology = \"2x2\"\n", " accelerator_count = 4\n", @@ -270,7 +270,7 @@ " tpu_topology = \"2x4\"\n", " accelerator_count = 8\n", " else:\n", - " if num_forecast_steps \u003c= 16:\n", + " if num_forecast_steps <= 16:\n", " machine_type = \"ct5lp-hightpu-1t\"\n", " tpu_topology = \"1x1\"\n", " accelerator_count = 1\n", @@ -285,11 +285,11 @@ " variable: str,\n", " level: Optional[int] = None,\n", " max_steps: Optional[int] = None\n", - " ) -\u003e xarray.Dataset:\n", + " ) -> xarray.Dataset:\n", " data = data[variable]\n", " if \"batch\" in data.dims:\n", " data = data.isel(batch=0)\n", - " if max_steps is not None and \"time\" in data.sizes and max_steps \u003c data.sizes[\"time\"]:\n", + " if max_steps is not None and \"time\" in data.sizes and max_steps < data.sizes[\"time\"]:\n", " data = data.isel(time=range(0, max_steps))\n", " if level is not None and \"level\" in data.coords:\n", " data = data.sel(level=level)\n", @@ -299,7 +299,7 @@ " data: xarray.Dataset,\n", " center: Optional[float] = None,\n", " robust: bool = False,\n", - " ) -\u003e tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", + " ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", " vmin = np.nanpercentile(data, (2 if robust else 0))\n", " vmax = np.nanpercentile(data, (98 if robust else 100))\n", " if center is not None:\n", @@ -315,7 +315,7 @@ " plot_size: float = 5,\n", " robust: bool = False,\n", " cols: int = 4\n", - " ) -\u003e tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", + " ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", "\n", " first_data = next(iter(data.values()))[0]\n", " max_steps = first_data.sizes.get(\"time\", 1)\n", @@ -366,15 +366,21 @@ }, { "cell_type": "markdown", - "source": [ - "## Forecasts and Visualizations" - ], "metadata": { "id": "qQIfscWiCC6l" - } + }, + "source": [ + "## Forecasts and Visualizations" + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "Hf_7MhTIMSJp" + }, + "outputs": [], "source": [ "# @title Configure Models\n", "# @markdown You can config WeatherNext models with *data_type*, *model_type* and *num_forecast_steps*.\n", @@ -429,16 +435,16 @@ "print(f\"tpu_topology is {tpu_topology}.\")\n", "print(f\"SCIENCE_DOCKER_URI is {SCIENCE_DOCKER_URI}.\")\n", "print(f\"The prediction instances: {instances}\")\n" - ], - "metadata": { - "id": "Hf_7MhTIMSJp", - "cellView": "form" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "LSejOqyVUvdF" + }, + "outputs": [], "source": [ "# @title Run Forecasts\n", "# @markdown This section will create vertex jobs to run forecasts.\n", @@ -510,16 +516,16 @@ " tpu_topology=tpu_topology,\n", " service_account=SERVICE_ACCOUNT,\n", ")" - ], - "metadata": { - "id": "LSejOqyVUvdF", - "cellView": "form" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "code", + "execution_count": null, + "metadata": { + "cellView": "form", + "id": "zP3b2vGDgvMN" + }, + "outputs": [], "source": [ "# @title Visualize Forecasts\n", "# @markdown For simplicity, we only pick the one forecast result for visualization.\n", @@ -554,22 +560,16 @@ " fig_title += f\" at {level} hPa\"\n", "\n", "plot_data(data, fig_title, plot_size, robust=True)" - ], - "metadata": { - "id": "zP3b2vGDgvMN", - "cellView": "form" - }, - "execution_count": null, - "outputs": [] + ] }, { "cell_type": "markdown", - "source": [ - "## Clean Up Resources" - ], "metadata": { "id": "fgVnRY2tCzjt" - } + }, + "source": [ + "## Clean Up Resources" + ] }, { "cell_type": "code", @@ -583,7 +583,8 @@ "\n", "delete_bucket = True # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n" + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n" ] } ], diff --git a/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb b/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb index 15ff3587e..8ff66b2d2 100644 --- a/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb +++ b/notebooks/official/tensorboard/tensorboard_profiler_custom_training_with_prebuilt_container.ipynb @@ -356,7 +356,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -695,10 +696,12 @@ }, "outputs": [], "source": [ - "!cd {PYTHON_PACKAGE_APPLICATION_DIR} && python3 setup.py sdist --formats=gztar\n", + "! cd {PYTHON_PACKAGE_APPLICATION_DIR} && python3 setup.py sdist --formats=gztar\n", + "\n", + "! gcloud storage cp {source_package_file_name} {python_package_gcs_uri}\n", "\n", - "!gcloud storage cp {source_package_file_name} {python_package_gcs_uri}\n", "\n", - "!gcloud storage ls --long {python_package_gcs_uri}" ] + "! gcloud storage ls --long {python_package_gcs_uri}" + ] }, { "cell_type": "markdown", @@ -830,7 +833,8 @@ "# delete the bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From 4b426b144edf4e5a029b39a9ffe7486b364fc554 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 16 Dec 2025 09:43:19 +0000 Subject: [PATCH 3/8] Apply automated linter fixes --- ...ith_dataflow_flex_template_component.ipynb | 21 +- ...den_proprietary_image_classification.ipynb | 6 +- .../model_garden_pytorch_imagebind.ipynb | 4 +- ..._garden_weather_prediction_on_vertex.ipynb | 322 ++++++++++-------- ...mage_classification_batch_prediction.ipynb | 22 +- ...e_object_detection_online_prediction.ipynb | 18 +- ..._with_custom_model_evaluation_import.ipynb | 6 +- ...lm_evaluation_for_summarization_task.ipynb | 9 +- ...tom_training_with_prebuilt_container.ipynb | 9 +- 9 files changed, 241 insertions(+), 176 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb index 5d0882e8d..89ca2271f 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_flex_template_component.ipynb @@ -456,7 +456,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -475,7 +476,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -544,8 +546,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -674,7 +678,8 @@ "outputs": [], "source": [ "GCS_AVRO_SCHEMA = BUCKET_URI + \"/gaming_schema.avsc\"\n", - "! gcloud storage cp gaming_schema.avsc $GCS_AVRO_SCHEMA\n", "\n", + "! gcloud storage cp gaming_schema.avsc $GCS_AVRO_SCHEMA\n", + "\n", "GCS_FLEX_TEMPLATE_PATH = \"gs://dataflow-templates/latest/flex/File_Format_Conversion\"\n", "GCS_CONVERT_IN = \"gs://dataflow-samples/game/5000_gaming_data.csv\"\n", "GCS_CONVERT_OUT = BUCKET_URI + \"/parquet_out/\"\n", @@ -752,7 +757,8 @@ "\n", "pipeline.run()\n", "\n", - "! gcloud storage ls $GCS_CONVERT_OUT\n", "\n", + "! gcloud storage ls $GCS_CONVERT_OUT\n", + "\n", "! rm -f dataflow_file_conversion.yaml gaming_schema.avsc" ] }, @@ -807,7 +813,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb b/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb index 3b2dfd11b..f3c886f2e 100644 --- a/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb +++ b/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb @@ -309,10 +309,12 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $DATASET_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $DATASET_FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $DATASET_FILE | head" ] + "! gcloud storage cat $DATASET_FILE | head" + ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb b/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb index 8bd3a89f0..2c7c5d8d1 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb @@ -225,7 +225,9 @@ "# @title Set the model variants\n", "\n", "# @markdown Select the accelerator type.\n", - "accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_TESLA_T4\"]\n", + "accelerator_type = (\n", + " \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_TESLA_T4\"]\n", + ")\n", "\n", "if accelerator_type == \"NVIDIA_L4\":\n", " machine_type = \"g2-standard-8\"\n", diff --git a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb index 7943a8076..5f2aa11cf 100644 --- a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb @@ -105,21 +105,22 @@ "# @markdown 3. **[Optional]** TPU is only available in `us-west1`, `us-west4`, `us-central1`.\n", "\n", "# REGION = \"\" # @param {type:\"string\"}\n", - "REGION = \"us-central1\" # @param [\"us-central1\", \"us-west1\", \"us-west4\"]\n", + "REGION = \"us-central1\" # @param [\"us-central1\", \"us-west1\", \"us-west4\"]\n", "\n", "# Import the necessary packages\n", "import datetime\n", + "import glob\n", "import importlib\n", "import os\n", "import uuid\n", - "from typing import Tuple, List\n", - "import glob\n", + "from typing import List, Tuple\n", + "\n", "from google.cloud import aiplatform, storage\n", "\n", "# Upgrade Vertex AI SDK.\n", "! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.64.0'\n", "if not os.path.exists(\"./vertex-ai-samples\"):\n", - " ! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n", + " ! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n", "! pip3 uninstall --quiet -y xarray\n", "! pip3 install --quiet xarray[complete]\n", "\n", @@ -187,11 +188,13 @@ " \"\"\"\n", " return prefix + datetime.datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n", "\n", + "\n", "def get_bucket_and_blob_name(filepath):\n", " # The gcs path is of the form gs:///\n", " gs_suffix = filepath.split(\"gs://\", 1)[1]\n", " return tuple(gs_suffix.split(\"/\", 1))\n", "\n", + "\n", "def upload_local_dir_to_gcs(local_dir_path, gcs_dir_path):\n", " \"\"\"Uploads files in a local directory to a GCS directory.\"\"\"\n", " client = storage.Client()\n", @@ -207,6 +210,7 @@ " blob.upload_from_filename(local_file)\n", " print(\"Copied {} to {}.\".format(local_file, gcs_file_path))\n", "\n", + "\n", "def download_gcs_blob_as_json(gcs_file_path):\n", " \"\"\"Download GCS blob and convert it to json.\"\"\"\n", " client = storage.Client()\n", @@ -216,152 +220,175 @@ "\n", " return json.loads(blob.download_as_bytes())\n", "\n", + "\n", + "import json\n", + "import math\n", + "from typing import Optional\n", + "\n", "# Utility functions for prediction visualization.\n", "import matplotlib\n", - "import xarray\n", - "from typing import Optional\n", - "import matplotlib.pyplot as plt\n", "import matplotlib.animation as animation\n", - "import math\n", - "from IPython.display import HTML\n", - "import json\n", + "import matplotlib.pyplot as plt\n", "import numpy as np\n", + "import xarray\n", + "from IPython.display import HTML\n", "\n", "print(xarray.backends.list_engines())\n", "\n", + "\n", "def get_existing_demo_step(\n", " num_forecast_steps: int,\n", " model_type: str = \"gen_small\",\n", - " ) -> str:\n", - " # The demo data only supports some steps, and we can only run the predictions\n", - " # if num_forecast_steps is smaller than or equal to the maximum supported setps.\n", - " if model_type == \"gen_small\":\n", - " # The max supported steps are obtained from gs://dm_graphcast/gencast/dataset.\n", - " supported_demo_steps = [1, 4, 12, 20, 30]\n", - " elif model_type == \"graph_small\":\n", - " # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n", - " supported_demo_steps = [1, 4, 12, 20, 40]\n", - " elif model_type == \"graph_operational\":\n", - " # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n", - " supported_demo_steps = [1, 4, 12]\n", - " else:\n", - " raise ValueError(\"Invalid model_type.\")\n", - "\n", - " # Find the proper demo data for forecasting.\n", - " found_supported_step = supported_demo_steps[-1]\n", - " for i, supported_step in enumerate(supported_demo_steps):\n", - " if num_forecast_steps <= supported_step:\n", - " found_supported_step = supported_step\n", - " break\n", - " if num_forecast_steps > found_supported_step:\n", - " raise ValueError(f\"Supported demo steps for {model_type} in gs://dm_graphcast are {supported_demo_steps}. {num_forecast_steps} is too large, and could not find proper demo data.\")\n", - " return found_supported_step\n", + ") -> str:\n", + " # The demo data only supports some steps, and we can only run the predictions\n", + " # if num_forecast_steps is smaller than or equal to the maximum supported setps.\n", + " if model_type == \"gen_small\":\n", + " # The max supported steps are obtained from gs://dm_graphcast/gencast/dataset.\n", + " supported_demo_steps = [1, 4, 12, 20, 30]\n", + " elif model_type == \"graph_small\":\n", + " # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n", + " supported_demo_steps = [1, 4, 12, 20, 40]\n", + " elif model_type == \"graph_operational\":\n", + " # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n", + " supported_demo_steps = [1, 4, 12]\n", + " else:\n", + " raise ValueError(\"Invalid model_type.\")\n", + "\n", + " # Find the proper demo data for forecasting.\n", + " found_supported_step = supported_demo_steps[-1]\n", + " for i, supported_step in enumerate(supported_demo_steps):\n", + " if num_forecast_steps <= supported_step:\n", + " found_supported_step = supported_step\n", + " break\n", + " if num_forecast_steps > found_supported_step:\n", + " raise ValueError(\n", + " f\"Supported demo steps for {model_type} in gs://dm_graphcast are {supported_demo_steps}. {num_forecast_steps} is too large, and could not find proper demo data.\"\n", + " )\n", + " return found_supported_step\n", + "\n", "\n", "def get_suggested_machines(\n", " num_forecast_steps: int,\n", - " model_type: str = \"gen_small\",) -> Tuple[str, str, int]:\n", + " model_type: str = \"gen_small\",\n", + ") -> Tuple[str, str, int]:\n", " if model_type == \"gen_small\":\n", - " if num_forecast_steps <= 16:\n", - " machine_type = \"ct5lp-hightpu-4t\"\n", - " tpu_topology = \"2x2\"\n", - " accelerator_count = 4\n", - " else:\n", - " machine_type = \"ct5lp-hightpu-8t\"\n", - " tpu_topology = \"2x4\"\n", - " accelerator_count = 8\n", + " if num_forecast_steps <= 16:\n", + " machine_type = \"ct5lp-hightpu-4t\"\n", + " tpu_topology = \"2x2\"\n", + " accelerator_count = 4\n", + " else:\n", + " machine_type = \"ct5lp-hightpu-8t\"\n", + " tpu_topology = \"2x4\"\n", + " accelerator_count = 8\n", " else:\n", - " if num_forecast_steps <= 16:\n", - " machine_type = \"ct5lp-hightpu-1t\"\n", - " tpu_topology = \"1x1\"\n", - " accelerator_count = 1\n", - " else:\n", - " machine_type = \"ct5lp-hightpu-4t\"\n", - " tpu_topology = \"2x2\"\n", - " accelerator_count = 4\n", + " if num_forecast_steps <= 16:\n", + " machine_type = \"ct5lp-hightpu-1t\"\n", + " tpu_topology = \"1x1\"\n", + " accelerator_count = 1\n", + " else:\n", + " machine_type = \"ct5lp-hightpu-4t\"\n", + " tpu_topology = \"2x2\"\n", + " accelerator_count = 4\n", " return machine_type, tpu_topology, accelerator_count\n", "\n", + "\n", "def select(\n", " data: xarray.Dataset,\n", " variable: str,\n", " level: Optional[int] = None,\n", - " max_steps: Optional[int] = None\n", - " ) -> xarray.Dataset:\n", - " data = data[variable]\n", - " if \"batch\" in data.dims:\n", - " data = data.isel(batch=0)\n", - " if max_steps is not None and \"time\" in data.sizes and max_steps < data.sizes[\"time\"]:\n", - " data = data.isel(time=range(0, max_steps))\n", - " if level is not None and \"level\" in data.coords:\n", - " data = data.sel(level=level)\n", - " return data\n", + " max_steps: Optional[int] = None,\n", + ") -> xarray.Dataset:\n", + " data = data[variable]\n", + " if \"batch\" in data.dims:\n", + " data = data.isel(batch=0)\n", + " if (\n", + " max_steps is not None\n", + " and \"time\" in data.sizes\n", + " and max_steps < data.sizes[\"time\"]\n", + " ):\n", + " data = data.isel(time=range(0, max_steps))\n", + " if level is not None and \"level\" in data.coords:\n", + " data = data.sel(level=level)\n", + " return data\n", + "\n", "\n", "def scale(\n", " data: xarray.Dataset,\n", " center: Optional[float] = None,\n", " robust: bool = False,\n", - " ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", - " vmin = np.nanpercentile(data, (2 if robust else 0))\n", - " vmax = np.nanpercentile(data, (98 if robust else 100))\n", - " if center is not None:\n", - " diff = max(vmax - center, center - vmin)\n", - " vmin = center - diff\n", - " vmax = center + diff\n", - " return (data, matplotlib.colors.Normalize(vmin, vmax),\n", - " (\"RdBu_r\" if center is not None else \"viridis\"))\n", + ") -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", + " vmin = np.nanpercentile(data, (2 if robust else 0))\n", + " vmax = np.nanpercentile(data, (98 if robust else 100))\n", + " if center is not None:\n", + " diff = max(vmax - center, center - vmin)\n", + " vmin = center - diff\n", + " vmax = center + diff\n", + " return (\n", + " data,\n", + " matplotlib.colors.Normalize(vmin, vmax),\n", + " (\"RdBu_r\" if center is not None else \"viridis\"),\n", + " )\n", + "\n", "\n", "def plot_data(\n", " data: dict[str, xarray.Dataset],\n", " fig_title: str,\n", " plot_size: float = 5,\n", " robust: bool = False,\n", - " cols: int = 4\n", - " ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", - "\n", - " first_data = next(iter(data.values()))[0]\n", - " max_steps = first_data.sizes.get(\"time\", 1)\n", - " assert all(max_steps == d.sizes.get(\"time\", 1) for d, _, _ in data.values())\n", - "\n", - " cols = min(cols, len(data))\n", - " rows = math.ceil(len(data) / cols)\n", - " figure = plt.figure(figsize=(plot_size * 2 * cols,\n", - " plot_size * rows))\n", - " figure.suptitle(fig_title, fontsize=16)\n", - " figure.subplots_adjust(wspace=0, hspace=0)\n", - " figure.tight_layout()\n", - "\n", - " images = []\n", - " for i, (title, (plot_data, norm, cmap)) in enumerate(data.items()):\n", - " ax = figure.add_subplot(rows, cols, i+1)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " ax.set_title(title)\n", - " im = ax.imshow(\n", - " plot_data.isel(time=0, missing_dims=\"ignore\"), norm=norm,\n", - " origin=\"lower\", cmap=cmap)\n", - " plt.colorbar(\n", - " mappable=im,\n", - " ax=ax,\n", - " orientation=\"vertical\",\n", - " pad=0.02,\n", - " aspect=16,\n", - " shrink=0.75,\n", - " cmap=cmap,\n", - " extend=(\"both\" if robust else \"neither\"))\n", - " images.append(im)\n", - "\n", - " def update(frame):\n", - " if \"time\" in first_data.dims:\n", - " td = datetime.timedelta(microseconds=first_data[\"time\"][frame].item() / 1000)\n", - " figure.suptitle(f\"{fig_title}, {td}\", fontsize=16)\n", - " else:\n", - " figure.suptitle(fig_title, fontsize=16)\n", - " for im, (plot_data, norm, cmap) in zip(images, data.values()):\n", - " im.set_data(plot_data.isel(time=frame, missing_dims=\"ignore\"))\n", - "\n", - " ani = animation.FuncAnimation(\n", - " fig=figure, func=update, frames=max_steps, interval=250)\n", - " plt.close(figure.number)\n", - " return HTML(ani.to_jshtml())\n" + " cols: int = 4,\n", + ") -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", + "\n", + " first_data = next(iter(data.values()))[0]\n", + " max_steps = first_data.sizes.get(\"time\", 1)\n", + " assert all(max_steps == d.sizes.get(\"time\", 1) for d, _, _ in data.values())\n", + "\n", + " cols = min(cols, len(data))\n", + " rows = math.ceil(len(data) / cols)\n", + " figure = plt.figure(figsize=(plot_size * 2 * cols, plot_size * rows))\n", + " figure.suptitle(fig_title, fontsize=16)\n", + " figure.subplots_adjust(wspace=0, hspace=0)\n", + " figure.tight_layout()\n", + "\n", + " images = []\n", + " for i, (title, (plot_data, norm, cmap)) in enumerate(data.items()):\n", + " ax = figure.add_subplot(rows, cols, i + 1)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " ax.set_title(title)\n", + " im = ax.imshow(\n", + " plot_data.isel(time=0, missing_dims=\"ignore\"),\n", + " norm=norm,\n", + " origin=\"lower\",\n", + " cmap=cmap,\n", + " )\n", + " plt.colorbar(\n", + " mappable=im,\n", + " ax=ax,\n", + " orientation=\"vertical\",\n", + " pad=0.02,\n", + " aspect=16,\n", + " shrink=0.75,\n", + " cmap=cmap,\n", + " extend=(\"both\" if robust else \"neither\"),\n", + " )\n", + " images.append(im)\n", + "\n", + " def update(frame):\n", + " if \"time\" in first_data.dims:\n", + " td = datetime.timedelta(\n", + " microseconds=first_data[\"time\"][frame].item() / 1000\n", + " )\n", + " figure.suptitle(f\"{fig_title}, {td}\", fontsize=16)\n", + " else:\n", + " figure.suptitle(fig_title, fontsize=16)\n", + " for im, (plot_data, norm, cmap) in zip(images, data.values()):\n", + " im.set_data(plot_data.isel(time=frame, missing_dims=\"ignore\"))\n", + "\n", + " ani = animation.FuncAnimation(\n", + " fig=figure, func=update, frames=max_steps, interval=250\n", + " )\n", + " plt.close(figure.number)\n", + " return HTML(ani.to_jshtml())" ] }, { @@ -385,6 +412,7 @@ "# @title Configure Models\n", "# @markdown You can config WeatherNext models with *data_type*, *model_type* and *num_forecast_steps*.\n", "\n", + "\n", "output_dir = f\"{BUCKET_URI}/science\"\n", "accelerator_type = \"TPU_V5e\"\n", "\n", @@ -394,32 +422,39 @@ "# @markdown The demo data for graph_small is from the date 2022-01-01 with resolution 1.0.\n", "# @markdown The demo data for graph_operational is from the date 2022-01-01 with resolution 0.25.\n", "\n", - "model_type = \"graph_operational\" # @param [\"gen_small\", \"graph_small\", \"graph_operational\"]\n", + "model_type = (\n", + " \"graph_operational\" # @param [\"gen_small\", \"graph_small\", \"graph_operational\"]\n", + ")\n", "\n", "\n", - "num_forecast_steps = 10 # @param {type:\"integer\"}\n", + "num_forecast_steps = 10 # @param {type:\"integer\"}\n", "# @markdown *num_forecast_steps* will specific the number of forecast steps, which will indicate the forcasting time combined with model leading time.\n", "# @markdown Assuming num_forecast_steps=4, and the leading time is 6 hours, then the results will contain forecasts with 6 hours, 12 hours, 18 hours and 24 hours.\n", "# @markdown WeatherNext Gen and Graph models support leading time as 12 hours and 6 hours separately.\n", "# @markdown num_forecast_steps will be truncated to the maximum of allowed values if it is beyond. Maximum of num_forecast_steps for Weather Gen and Graph models are 30 and 40 separately.\n", - "machine_type, tpu_topology, accelerator_count = get_suggested_machines(num_forecast_steps, model_type)\n", + "machine_type, tpu_topology, accelerator_count = get_suggested_machines(\n", + " num_forecast_steps, model_type\n", + ")\n", "\n", "SCIENCE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/science-serve.tpu.0-1.debian12.py310:20250331_0715_RC03\"\n", "data_storage_dir = \"gs://dm_graphcast\"\n", "existing_demo_step = get_existing_demo_step(num_forecast_steps, model_type)\n", "if model_type == \"gen_small\":\n", - " input_file = f\"{data_storage_dir}/gencast/dataset/source-era5_date-2019-03-29_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n", - " # num_ensemble_samples (WeatherNext Gen models only) specified the number of ensembling samples per step.\n", - " num_ensemble_samples = 8\n", - " parameters = {\"num_forecast_steps\": num_forecast_steps, \"num_ensemble_samples\": num_ensemble_samples}\n", + " input_file = f\"{data_storage_dir}/gencast/dataset/source-era5_date-2019-03-29_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n", + " # num_ensemble_samples (WeatherNext Gen models only) specified the number of ensembling samples per step.\n", + " num_ensemble_samples = 8\n", + " parameters = {\n", + " \"num_forecast_steps\": num_forecast_steps,\n", + " \"num_ensemble_samples\": num_ensemble_samples,\n", + " }\n", "elif model_type == \"graph_small\":\n", - " input_file = f\"{data_storage_dir}/graphcast/dataset/source-era5_date-2022-01-01_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n", - " parameters = {\"num_forecast_steps\": num_forecast_steps}\n", + " input_file = f\"{data_storage_dir}/graphcast/dataset/source-era5_date-2022-01-01_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n", + " parameters = {\"num_forecast_steps\": num_forecast_steps}\n", "elif model_type == \"graph_operational\":\n", - " input_file = f\"{data_storage_dir}/graphcast/dataset/source-hres_date-2022-01-01_res-0.25_levels-13_steps-{existing_demo_step:02d}.nc\"\n", - " parameters = {\"num_forecast_steps\": num_forecast_steps}\n", + " input_file = f\"{data_storage_dir}/graphcast/dataset/source-hres_date-2022-01-01_res-0.25_levels-13_steps-{existing_demo_step:02d}.nc\"\n", + " parameters = {\"num_forecast_steps\": num_forecast_steps}\n", "else:\n", - " raise ValueError(\"Invalid example_type.\")\n", + " raise ValueError(\"Invalid example_type.\")\n", "\n", "instances = [\n", " {\n", @@ -434,7 +469,7 @@ "print(f\"machine_type is {machine_type}.\")\n", "print(f\"tpu_topology is {tpu_topology}.\")\n", "print(f\"SCIENCE_DOCKER_URI is {SCIENCE_DOCKER_URI}.\")\n", - "print(f\"The prediction instances: {instances}\")\n" + "print(f\"The prediction instances: {instances}\")" ] }, { @@ -473,9 +508,7 @@ " outfile.write(json_str)\n", " outfile.write(\"\\n\")\n", "\n", - "upload_local_dir_to_gcs(\n", - " \"bath_prediction_input\", output_dir\n", - ")\n", + "upload_local_dir_to_gcs(\"bath_prediction_input\", output_dir)\n", "\n", "JOB_NAME = get_job_name_with_datetime(prefix=f\"jax_{model_type}\")\n", "\n", @@ -487,7 +520,7 @@ " \"./gdm_science/batch_prediction.py\",\n", " f\"--model_type={model_type}\",\n", " f\"--input_jsonl={input_jsonl}\",\n", - " f\"--output_jsonl={output_jsonl}\"\n", + " f\"--output_jsonl={output_jsonl}\",\n", "]\n", "\n", "print(f\"The input json file will be {input_jsonl}.\")\n", @@ -531,10 +564,10 @@ "# @markdown For simplicity, we only pick the one forecast result for visualization.\n", "# @markdown You can also visualize previous forecasts by setting `prediction_data_path`.\n", "prediction_json = download_gcs_blob_as_json(output_jsonl)\n", - "prediction_data_path = json.loads(prediction_json[0])[\"predictions\"] # @param\n", + "prediction_data_path = json.loads(prediction_json[0])[\"predictions\"] # @param\n", "\n", "# @markdown (Optional) The sample to visualize if there are multiple samples.\n", - "sample = 0 # @param\n", + "sample = 0 # @param\n", "\n", "print(prediction_data_path)\n", "predictions = xarray.open_zarr(prediction_data_path)\n", @@ -545,11 +578,11 @@ "steps = predictions.dims[\"time\"]\n", "print(\"steps=\", steps)\n", "if \"sample\" in predictions:\n", - " print(\"sample=\", len(predictions[\"sample\"]))\n", - " # Visualize one sample if there are many.\n", - " visualized_data = predictions.isel(sample=sample)\n", + " print(\"sample=\", len(predictions[\"sample\"]))\n", + " # Visualize one sample if there are many.\n", + " visualized_data = predictions.isel(sample=sample)\n", "else:\n", - " visualized_data = predictions\n", + " visualized_data = predictions\n", "\n", "data = {\n", " \" \": scale(select(visualized_data, variable, level, steps), robust=True),\n", @@ -557,7 +590,7 @@ "\n", "fig_title = variable\n", "if \"level\" in predictions[variable].coords:\n", - " fig_title += f\" at {level} hPa\"\n", + " fig_title += f\" at {level} hPa\"\n", "\n", "plot_data(data, fig_title, plot_size, robust=True)" ] @@ -583,15 +616,14 @@ "\n", "delete_bucket = True # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", - "\n" + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { "colab": { "name": "model_garden_weather_prediction_on_vertex.ipynb", - "provenance": [] + "toc_visible": true }, "kernelspec": { "display_name": "Python 3", diff --git a/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb b/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb index be63f846a..fdd000472 100644 --- a/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb +++ b/notebooks/official/automl/automl_image_classification_batch_prediction.ipynb @@ -287,7 +287,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -393,10 +394,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -577,7 +580,8 @@ }, "outputs": [], "source": [ - "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(str(test_items[0]).split(\",\")) == 3:\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", + "if len(str(test_items[0]).split(\",\")) == 3:\n", " _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n", " _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n", "else:\n", @@ -610,7 +614,9 @@ "file_1 = test_item_1.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n", "\n", - "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", "\n", + "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", + "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", + "\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_2 = BUCKET_URI + \"/\" + file_2" ] @@ -653,7 +659,8 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gcloud storage cat $gcs_input_uri" ] + "! gcloud storage cat $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -796,7 +803,8 @@ "# Delete the cloud storage bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb b/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb index b54f1308a..dfd8bcd29 100644 --- a/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb +++ b/notebooks/official/automl/automl_image_object_detection_online_prediction.ipynb @@ -297,7 +297,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -414,7 +415,8 @@ "\n", "# Copy images using gsutil commands directly\n", "for src, dest in zip(df.iloc[:, 0], df[\"destination_path\"]):\n", - " ! gcloud storage cp {src} {dest}\n", "\n", + " ! gcloud storage cp {src} {dest}\n", + "\n", "print(f\"Files copied to {BUCKET_URI}\")" ] }, @@ -494,10 +496,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -702,7 +706,8 @@ }, "outputs": [], "source": [ - "test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n", "cols = str(test_items[0]).split(\",\")\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n", + "cols = str(test_items[0]).split(\",\")\n", "if len(cols) == 11:\n", " test_item = str(cols[1])\n", " test_label = str(cols[2])\n", @@ -829,7 +834,8 @@ "dag.delete()\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb b/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb index f735e4b49..9909442f8 100644 --- a/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb +++ b/notebooks/official/model_evaluation/get_started_with_custom_model_evaluation_import.ipynb @@ -304,7 +304,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create {BUCKET_URI} --location={LOCATION} --project={PROJECT_ID}" ] + "! gcloud storage buckets create {BUCKET_URI} --location={LOCATION} --project={PROJECT_ID}" + ] }, { "cell_type": "markdown", @@ -777,7 +778,8 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/model_evaluation/model_based_llm_evaluation/autosxs_llm_evaluation_for_summarization_task.ipynb b/notebooks/official/model_evaluation/model_based_llm_evaluation/autosxs_llm_evaluation_for_summarization_task.ipynb index fbe43d9cd..897683f5a 100644 --- a/notebooks/official/model_evaluation/model_based_llm_evaluation/autosxs_llm_evaluation_for_summarization_task.ipynb +++ b/notebooks/official/model_evaluation/model_based_llm_evaluation/autosxs_llm_evaluation_for_summarization_task.ipynb @@ -355,7 +355,8 @@ "):\n", " BUCKET_URI = \"gs://\" + PROJECT_ID + \"-aip-\" + UUID\n", "\n", - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -518,7 +519,8 @@ "outputs": [], "source": [ "examples.to_json(\"evaluation_dataset.json\", orient=\"records\", lines=True)\n", - "! gcloud storage cp evaluation_dataset.json $BUCKET_URI/input/evaluation_dataset.json\n", "DATASET = f\"{BUCKET_URI}/input/evaluation_dataset.json\"" + "! gcloud storage cp evaluation_dataset.json $BUCKET_URI/input/evaluation_dataset.json\n", + "DATASET = f\"{BUCKET_URI}/input/evaluation_dataset.json\"" ] }, { @@ -763,7 +765,8 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb b/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb index 4edb31ddb..2a46735b6 100644 --- a/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb +++ b/notebooks/official/tensorboard/tensorboard_custom_training_with_prebuilt_container.ipynb @@ -316,7 +316,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -663,7 +664,8 @@ "outputs": [], "source": [ "GCS_BUCKET_TRAINING = f\"{BUCKET_URI}/data/\"\n", - "! gcloud storage cp dist/hello-custom-training-3.0.tar.gz {GCS_BUCKET_TRAINING}" ] + "! gcloud storage cp dist/hello-custom-training-3.0.tar.gz {GCS_BUCKET_TRAINING}" + ] }, { "cell_type": "markdown", @@ -785,7 +787,8 @@ "# Delete GCS bucket.\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "!rm -rf ../hello-custom-sample/" ] } From 0960f34f078e55c68490be82b2b4fccdff36bde1 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 16 Dec 2025 09:46:51 +0000 Subject: [PATCH 4/8] update --- .../model_garden_weather_prediction_on_vertex.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb index 5f2aa11cf..aa5cecde4 100644 --- a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb @@ -113,7 +113,7 @@ "import importlib\n", "import os\n", "import uuid\n", - "from typing import List, Tuple\n", + "from typing import Tuple\n", "\n", "from google.cloud import aiplatform, storage\n", "\n", From fe15792df0aa3c63440cf259664ab3a098d1bf1a Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 05:23:02 +0000 Subject: [PATCH 5/8] remove model_garden changes --- ...den_proprietary_image_classification.ipynb | 4 +- .../model_garden_pytorch_imagebind.ipynb | 14 +- ..._garden_weather_prediction_on_vertex.ipynb | 392 ++++++++---------- 3 files changed, 188 insertions(+), 222 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb b/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb index f3c886f2e..267d0b504 100644 --- a/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb +++ b/notebooks/community/model_garden/model_garden_proprietary_image_classification.ipynb @@ -309,11 +309,11 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $DATASET_FILE | wc -l\n", + "count = ! gsutil cat $DATASET_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $DATASET_FILE | head" + "! gsutil cat $DATASET_FILE | head" ] }, { diff --git a/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb b/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb index 2c7c5d8d1..2ccc9d71e 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_imagebind.ipynb @@ -173,10 +173,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -201,7 +201,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -225,9 +225,7 @@ "# @title Set the model variants\n", "\n", "# @markdown Select the accelerator type.\n", - "accelerator_type = (\n", - " \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_TESLA_T4\"]\n", - ")\n", + "accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_TESLA_T4\"]\n", "\n", "if accelerator_type == \"NVIDIA_L4\":\n", " machine_type = \"g2-standard-8\"\n", @@ -302,7 +300,7 @@ "%cd ImageBind/.assets\n", "! git reset --hard 95d27c7fd5a8362f3527e176c3a80ae5a4d880c0\n", "\n", - "! gcloud storage cp --recursive . $DATA_BUCKET\n", + "! gsutil cp -r . $DATA_BUCKET\n", "\n", "%cd ../.." ] @@ -573,7 +571,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb index aa5cecde4..ff1043ea7 100644 --- a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb @@ -88,8 +88,8 @@ "cell_type": "code", "execution_count": null, "metadata": { - "cellView": "form", - "id": "2707b02ef5df" + "id": "2707b02ef5df", + "cellView": "form" }, "outputs": [], "source": [ @@ -105,22 +105,21 @@ "# @markdown 3. **[Optional]** TPU is only available in `us-west1`, `us-west4`, `us-central1`.\n", "\n", "# REGION = \"\" # @param {type:\"string\"}\n", - "REGION = \"us-central1\" # @param [\"us-central1\", \"us-west1\", \"us-west4\"]\n", + "REGION = \"us-central1\" # @param [\"us-central1\", \"us-west1\", \"us-west4\"]\n", "\n", "# Import the necessary packages\n", "import datetime\n", - "import glob\n", "import importlib\n", "import os\n", "import uuid\n", - "from typing import Tuple\n", - "\n", + "from typing import Tuple, List\n", + "import glob\n", "from google.cloud import aiplatform, storage\n", "\n", "# Upgrade Vertex AI SDK.\n", "! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.64.0'\n", "if not os.path.exists(\"./vertex-ai-samples\"):\n", - " ! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n", + " ! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n", "! pip3 uninstall --quiet -y xarray\n", "! pip3 install --quiet xarray[complete]\n", "\n", @@ -149,10 +148,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -175,7 +174,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -188,13 +187,11 @@ " \"\"\"\n", " return prefix + datetime.datetime.now().strftime(\"_%Y%m%d_%H%M%S\")\n", "\n", - "\n", "def get_bucket_and_blob_name(filepath):\n", " # The gcs path is of the form gs:///\n", " gs_suffix = filepath.split(\"gs://\", 1)[1]\n", " return tuple(gs_suffix.split(\"/\", 1))\n", "\n", - "\n", "def upload_local_dir_to_gcs(local_dir_path, gcs_dir_path):\n", " \"\"\"Uploads files in a local directory to a GCS directory.\"\"\"\n", " client = storage.Client()\n", @@ -210,7 +207,6 @@ " blob.upload_from_filename(local_file)\n", " print(\"Copied {} to {}.\".format(local_file, gcs_file_path))\n", "\n", - "\n", "def download_gcs_blob_as_json(gcs_file_path):\n", " \"\"\"Download GCS blob and convert it to json.\"\"\"\n", " client = storage.Client()\n", @@ -220,199 +216,169 @@ "\n", " return json.loads(blob.download_as_bytes())\n", "\n", - "\n", - "import json\n", - "import math\n", - "from typing import Optional\n", - "\n", "# Utility functions for prediction visualization.\n", "import matplotlib\n", - "import matplotlib.animation as animation\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", "import xarray\n", + "from typing import Optional\n", + "import matplotlib.pyplot as plt\n", + "import matplotlib.animation as animation\n", + "import math\n", "from IPython.display import HTML\n", + "import json\n", + "import numpy as np\n", "\n", "print(xarray.backends.list_engines())\n", "\n", - "\n", "def get_existing_demo_step(\n", " num_forecast_steps: int,\n", " model_type: str = \"gen_small\",\n", - ") -> str:\n", - " # The demo data only supports some steps, and we can only run the predictions\n", - " # if num_forecast_steps is smaller than or equal to the maximum supported setps.\n", - " if model_type == \"gen_small\":\n", - " # The max supported steps are obtained from gs://dm_graphcast/gencast/dataset.\n", - " supported_demo_steps = [1, 4, 12, 20, 30]\n", - " elif model_type == \"graph_small\":\n", - " # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n", - " supported_demo_steps = [1, 4, 12, 20, 40]\n", - " elif model_type == \"graph_operational\":\n", - " # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n", - " supported_demo_steps = [1, 4, 12]\n", - " else:\n", - " raise ValueError(\"Invalid model_type.\")\n", - "\n", - " # Find the proper demo data for forecasting.\n", - " found_supported_step = supported_demo_steps[-1]\n", - " for i, supported_step in enumerate(supported_demo_steps):\n", - " if num_forecast_steps <= supported_step:\n", - " found_supported_step = supported_step\n", - " break\n", - " if num_forecast_steps > found_supported_step:\n", - " raise ValueError(\n", - " f\"Supported demo steps for {model_type} in gs://dm_graphcast are {supported_demo_steps}. {num_forecast_steps} is too large, and could not find proper demo data.\"\n", - " )\n", - " return found_supported_step\n", - "\n", + " ) -> str:\n", + " # The demo data only supports some steps, and we can only run the predictions\n", + " # if num_forecast_steps is smaller than or equal to the maximum supported setps.\n", + " if model_type == \"gen_small\":\n", + " # The max supported steps are obtained from gs://dm_graphcast/gencast/dataset.\n", + " supported_demo_steps = [1, 4, 12, 20, 30]\n", + " elif model_type == \"graph_small\":\n", + " # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n", + " supported_demo_steps = [1, 4, 12, 20, 40]\n", + " elif model_type == \"graph_operational\":\n", + " # The max supported steps are obtained from gs://dm_graphcast/graphcast/dataset.\n", + " supported_demo_steps = [1, 4, 12]\n", + " else:\n", + " raise ValueError(\"Invalid model_type.\")\n", + "\n", + " # Find the proper demo data for forecasting.\n", + " found_supported_step = supported_demo_steps[-1]\n", + " for i, supported_step in enumerate(supported_demo_steps):\n", + " if num_forecast_steps <= supported_step:\n", + " found_supported_step = supported_step\n", + " break\n", + " if num_forecast_steps > found_supported_step:\n", + " raise ValueError(f\"Supported demo steps for {model_type} in gs://dm_graphcast are {supported_demo_steps}. {num_forecast_steps} is too large, and could not find proper demo data.\")\n", + " return found_supported_step\n", "\n", "def get_suggested_machines(\n", " num_forecast_steps: int,\n", - " model_type: str = \"gen_small\",\n", - ") -> Tuple[str, str, int]:\n", + " model_type: str = \"gen_small\",) -> Tuple[str, str, int]:\n", " if model_type == \"gen_small\":\n", - " if num_forecast_steps <= 16:\n", - " machine_type = \"ct5lp-hightpu-4t\"\n", - " tpu_topology = \"2x2\"\n", - " accelerator_count = 4\n", - " else:\n", - " machine_type = \"ct5lp-hightpu-8t\"\n", - " tpu_topology = \"2x4\"\n", - " accelerator_count = 8\n", + " if num_forecast_steps <= 16:\n", + " machine_type = \"ct5lp-hightpu-4t\"\n", + " tpu_topology = \"2x2\"\n", + " accelerator_count = 4\n", + " else:\n", + " machine_type = \"ct5lp-hightpu-8t\"\n", + " tpu_topology = \"2x4\"\n", + " accelerator_count = 8\n", " else:\n", - " if num_forecast_steps <= 16:\n", - " machine_type = \"ct5lp-hightpu-1t\"\n", - " tpu_topology = \"1x1\"\n", - " accelerator_count = 1\n", - " else:\n", - " machine_type = \"ct5lp-hightpu-4t\"\n", - " tpu_topology = \"2x2\"\n", - " accelerator_count = 4\n", + " if num_forecast_steps <= 16:\n", + " machine_type = \"ct5lp-hightpu-1t\"\n", + " tpu_topology = \"1x1\"\n", + " accelerator_count = 1\n", + " else:\n", + " machine_type = \"ct5lp-hightpu-4t\"\n", + " tpu_topology = \"2x2\"\n", + " accelerator_count = 4\n", " return machine_type, tpu_topology, accelerator_count\n", "\n", - "\n", "def select(\n", " data: xarray.Dataset,\n", " variable: str,\n", " level: Optional[int] = None,\n", - " max_steps: Optional[int] = None,\n", - ") -> xarray.Dataset:\n", - " data = data[variable]\n", - " if \"batch\" in data.dims:\n", - " data = data.isel(batch=0)\n", - " if (\n", - " max_steps is not None\n", - " and \"time\" in data.sizes\n", - " and max_steps < data.sizes[\"time\"]\n", - " ):\n", - " data = data.isel(time=range(0, max_steps))\n", - " if level is not None and \"level\" in data.coords:\n", - " data = data.sel(level=level)\n", - " return data\n", - "\n", + " max_steps: Optional[int] = None\n", + " ) -> xarray.Dataset:\n", + " data = data[variable]\n", + " if \"batch\" in data.dims:\n", + " data = data.isel(batch=0)\n", + " if max_steps is not None and \"time\" in data.sizes and max_steps < data.sizes[\"time\"]:\n", + " data = data.isel(time=range(0, max_steps))\n", + " if level is not None and \"level\" in data.coords:\n", + " data = data.sel(level=level)\n", + " return data\n", "\n", "def scale(\n", " data: xarray.Dataset,\n", " center: Optional[float] = None,\n", " robust: bool = False,\n", - ") -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", - " vmin = np.nanpercentile(data, (2 if robust else 0))\n", - " vmax = np.nanpercentile(data, (98 if robust else 100))\n", - " if center is not None:\n", - " diff = max(vmax - center, center - vmin)\n", - " vmin = center - diff\n", - " vmax = center + diff\n", - " return (\n", - " data,\n", - " matplotlib.colors.Normalize(vmin, vmax),\n", - " (\"RdBu_r\" if center is not None else \"viridis\"),\n", - " )\n", - "\n", + " ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", + " vmin = np.nanpercentile(data, (2 if robust else 0))\n", + " vmax = np.nanpercentile(data, (98 if robust else 100))\n", + " if center is not None:\n", + " diff = max(vmax - center, center - vmin)\n", + " vmin = center - diff\n", + " vmax = center + diff\n", + " return (data, matplotlib.colors.Normalize(vmin, vmax),\n", + " (\"RdBu_r\" if center is not None else \"viridis\"))\n", "\n", "def plot_data(\n", " data: dict[str, xarray.Dataset],\n", " fig_title: str,\n", " plot_size: float = 5,\n", " robust: bool = False,\n", - " cols: int = 4,\n", - ") -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", - "\n", - " first_data = next(iter(data.values()))[0]\n", - " max_steps = first_data.sizes.get(\"time\", 1)\n", - " assert all(max_steps == d.sizes.get(\"time\", 1) for d, _, _ in data.values())\n", - "\n", - " cols = min(cols, len(data))\n", - " rows = math.ceil(len(data) / cols)\n", - " figure = plt.figure(figsize=(plot_size * 2 * cols, plot_size * rows))\n", - " figure.suptitle(fig_title, fontsize=16)\n", - " figure.subplots_adjust(wspace=0, hspace=0)\n", - " figure.tight_layout()\n", - "\n", - " images = []\n", - " for i, (title, (plot_data, norm, cmap)) in enumerate(data.items()):\n", - " ax = figure.add_subplot(rows, cols, i + 1)\n", - " ax.set_xticks([])\n", - " ax.set_yticks([])\n", - " ax.set_title(title)\n", - " im = ax.imshow(\n", - " plot_data.isel(time=0, missing_dims=\"ignore\"),\n", - " norm=norm,\n", - " origin=\"lower\",\n", - " cmap=cmap,\n", - " )\n", - " plt.colorbar(\n", - " mappable=im,\n", - " ax=ax,\n", - " orientation=\"vertical\",\n", - " pad=0.02,\n", - " aspect=16,\n", - " shrink=0.75,\n", - " cmap=cmap,\n", - " extend=(\"both\" if robust else \"neither\"),\n", - " )\n", - " images.append(im)\n", - "\n", - " def update(frame):\n", - " if \"time\" in first_data.dims:\n", - " td = datetime.timedelta(\n", - " microseconds=first_data[\"time\"][frame].item() / 1000\n", - " )\n", - " figure.suptitle(f\"{fig_title}, {td}\", fontsize=16)\n", - " else:\n", - " figure.suptitle(fig_title, fontsize=16)\n", - " for im, (plot_data, norm, cmap) in zip(images, data.values()):\n", - " im.set_data(plot_data.isel(time=frame, missing_dims=\"ignore\"))\n", - "\n", - " ani = animation.FuncAnimation(\n", - " fig=figure, func=update, frames=max_steps, interval=250\n", - " )\n", - " plt.close(figure.number)\n", - " return HTML(ani.to_jshtml())" + " cols: int = 4\n", + " ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", + "\n", + " first_data = next(iter(data.values()))[0]\n", + " max_steps = first_data.sizes.get(\"time\", 1)\n", + " assert all(max_steps == d.sizes.get(\"time\", 1) for d, _, _ in data.values())\n", + "\n", + " cols = min(cols, len(data))\n", + " rows = math.ceil(len(data) / cols)\n", + " figure = plt.figure(figsize=(plot_size * 2 * cols,\n", + " plot_size * rows))\n", + " figure.suptitle(fig_title, fontsize=16)\n", + " figure.subplots_adjust(wspace=0, hspace=0)\n", + " figure.tight_layout()\n", + "\n", + " images = []\n", + " for i, (title, (plot_data, norm, cmap)) in enumerate(data.items()):\n", + " ax = figure.add_subplot(rows, cols, i+1)\n", + " ax.set_xticks([])\n", + " ax.set_yticks([])\n", + " ax.set_title(title)\n", + " im = ax.imshow(\n", + " plot_data.isel(time=0, missing_dims=\"ignore\"), norm=norm,\n", + " origin=\"lower\", cmap=cmap)\n", + " plt.colorbar(\n", + " mappable=im,\n", + " ax=ax,\n", + " orientation=\"vertical\",\n", + " pad=0.02,\n", + " aspect=16,\n", + " shrink=0.75,\n", + " cmap=cmap,\n", + " extend=(\"both\" if robust else \"neither\"))\n", + " images.append(im)\n", + "\n", + " def update(frame):\n", + " if \"time\" in first_data.dims:\n", + " td = datetime.timedelta(microseconds=first_data[\"time\"][frame].item() / 1000)\n", + " figure.suptitle(f\"{fig_title}, {td}\", fontsize=16)\n", + " else:\n", + " figure.suptitle(fig_title, fontsize=16)\n", + " for im, (plot_data, norm, cmap) in zip(images, data.values()):\n", + " im.set_data(plot_data.isel(time=frame, missing_dims=\"ignore\"))\n", + "\n", + " ani = animation.FuncAnimation(\n", + " fig=figure, func=update, frames=max_steps, interval=250)\n", + " plt.close(figure.number)\n", + " return HTML(ani.to_jshtml())\n" ] }, { "cell_type": "markdown", - "metadata": { - "id": "qQIfscWiCC6l" - }, "source": [ "## Forecasts and Visualizations" - ] + ], + "metadata": { + "id": "qQIfscWiCC6l" + } }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "cellView": "form", - "id": "Hf_7MhTIMSJp" - }, - "outputs": [], "source": [ "# @title Configure Models\n", "# @markdown You can config WeatherNext models with *data_type*, *model_type* and *num_forecast_steps*.\n", "\n", - "\n", "output_dir = f\"{BUCKET_URI}/science\"\n", "accelerator_type = \"TPU_V5e\"\n", "\n", @@ -422,39 +388,32 @@ "# @markdown The demo data for graph_small is from the date 2022-01-01 with resolution 1.0.\n", "# @markdown The demo data for graph_operational is from the date 2022-01-01 with resolution 0.25.\n", "\n", - "model_type = (\n", - " \"graph_operational\" # @param [\"gen_small\", \"graph_small\", \"graph_operational\"]\n", - ")\n", + "model_type = \"graph_operational\" # @param [\"gen_small\", \"graph_small\", \"graph_operational\"]\n", "\n", "\n", - "num_forecast_steps = 10 # @param {type:\"integer\"}\n", + "num_forecast_steps = 10 # @param {type:\"integer\"}\n", "# @markdown *num_forecast_steps* will specific the number of forecast steps, which will indicate the forcasting time combined with model leading time.\n", "# @markdown Assuming num_forecast_steps=4, and the leading time is 6 hours, then the results will contain forecasts with 6 hours, 12 hours, 18 hours and 24 hours.\n", "# @markdown WeatherNext Gen and Graph models support leading time as 12 hours and 6 hours separately.\n", "# @markdown num_forecast_steps will be truncated to the maximum of allowed values if it is beyond. Maximum of num_forecast_steps for Weather Gen and Graph models are 30 and 40 separately.\n", - "machine_type, tpu_topology, accelerator_count = get_suggested_machines(\n", - " num_forecast_steps, model_type\n", - ")\n", + "machine_type, tpu_topology, accelerator_count = get_suggested_machines(num_forecast_steps, model_type)\n", "\n", "SCIENCE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai-restricted/vertex-vision-model-garden-dockers/science-serve.tpu.0-1.debian12.py310:20250331_0715_RC03\"\n", "data_storage_dir = \"gs://dm_graphcast\"\n", "existing_demo_step = get_existing_demo_step(num_forecast_steps, model_type)\n", "if model_type == \"gen_small\":\n", - " input_file = f\"{data_storage_dir}/gencast/dataset/source-era5_date-2019-03-29_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n", - " # num_ensemble_samples (WeatherNext Gen models only) specified the number of ensembling samples per step.\n", - " num_ensemble_samples = 8\n", - " parameters = {\n", - " \"num_forecast_steps\": num_forecast_steps,\n", - " \"num_ensemble_samples\": num_ensemble_samples,\n", - " }\n", + " input_file = f\"{data_storage_dir}/gencast/dataset/source-era5_date-2019-03-29_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n", + " # num_ensemble_samples (WeatherNext Gen models only) specified the number of ensembling samples per step.\n", + " num_ensemble_samples = 8\n", + " parameters = {\"num_forecast_steps\": num_forecast_steps, \"num_ensemble_samples\": num_ensemble_samples}\n", "elif model_type == \"graph_small\":\n", - " input_file = f\"{data_storage_dir}/graphcast/dataset/source-era5_date-2022-01-01_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n", - " parameters = {\"num_forecast_steps\": num_forecast_steps}\n", + " input_file = f\"{data_storage_dir}/graphcast/dataset/source-era5_date-2022-01-01_res-1.0_levels-13_steps-{existing_demo_step:02d}.nc\"\n", + " parameters = {\"num_forecast_steps\": num_forecast_steps}\n", "elif model_type == \"graph_operational\":\n", - " input_file = f\"{data_storage_dir}/graphcast/dataset/source-hres_date-2022-01-01_res-0.25_levels-13_steps-{existing_demo_step:02d}.nc\"\n", - " parameters = {\"num_forecast_steps\": num_forecast_steps}\n", + " input_file = f\"{data_storage_dir}/graphcast/dataset/source-hres_date-2022-01-01_res-0.25_levels-13_steps-{existing_demo_step:02d}.nc\"\n", + " parameters = {\"num_forecast_steps\": num_forecast_steps}\n", "else:\n", - " raise ValueError(\"Invalid example_type.\")\n", + " raise ValueError(\"Invalid example_type.\")\n", "\n", "instances = [\n", " {\n", @@ -469,17 +428,17 @@ "print(f\"machine_type is {machine_type}.\")\n", "print(f\"tpu_topology is {tpu_topology}.\")\n", "print(f\"SCIENCE_DOCKER_URI is {SCIENCE_DOCKER_URI}.\")\n", - "print(f\"The prediction instances: {instances}\")" - ] + "print(f\"The prediction instances: {instances}\")\n" + ], + "metadata": { + "id": "Hf_7MhTIMSJp", + "cellView": "form" + }, + "execution_count": null, + "outputs": [] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "cellView": "form", - "id": "LSejOqyVUvdF" - }, - "outputs": [], "source": [ "# @title Run Forecasts\n", "# @markdown This section will create vertex jobs to run forecasts.\n", @@ -508,7 +467,9 @@ " outfile.write(json_str)\n", " outfile.write(\"\\n\")\n", "\n", - "upload_local_dir_to_gcs(\"bath_prediction_input\", output_dir)\n", + "upload_local_dir_to_gcs(\n", + " \"bath_prediction_input\", output_dir\n", + ")\n", "\n", "JOB_NAME = get_job_name_with_datetime(prefix=f\"jax_{model_type}\")\n", "\n", @@ -520,7 +481,7 @@ " \"./gdm_science/batch_prediction.py\",\n", " f\"--model_type={model_type}\",\n", " f\"--input_jsonl={input_jsonl}\",\n", - " f\"--output_jsonl={output_jsonl}\",\n", + " f\"--output_jsonl={output_jsonl}\"\n", "]\n", "\n", "print(f\"The input json file will be {input_jsonl}.\")\n", @@ -549,25 +510,25 @@ " tpu_topology=tpu_topology,\n", " service_account=SERVICE_ACCOUNT,\n", ")" - ] + ], + "metadata": { + "id": "LSejOqyVUvdF", + "cellView": "form" + }, + "execution_count": null, + "outputs": [] }, { "cell_type": "code", - "execution_count": null, - "metadata": { - "cellView": "form", - "id": "zP3b2vGDgvMN" - }, - "outputs": [], "source": [ "# @title Visualize Forecasts\n", "# @markdown For simplicity, we only pick the one forecast result for visualization.\n", "# @markdown You can also visualize previous forecasts by setting `prediction_data_path`.\n", "prediction_json = download_gcs_blob_as_json(output_jsonl)\n", - "prediction_data_path = json.loads(prediction_json[0])[\"predictions\"] # @param\n", + "prediction_data_path = json.loads(prediction_json[0])[\"predictions\"] # @param\n", "\n", "# @markdown (Optional) The sample to visualize if there are multiple samples.\n", - "sample = 0 # @param\n", + "sample = 0 # @param\n", "\n", "print(prediction_data_path)\n", "predictions = xarray.open_zarr(prediction_data_path)\n", @@ -578,11 +539,11 @@ "steps = predictions.dims[\"time\"]\n", "print(\"steps=\", steps)\n", "if \"sample\" in predictions:\n", - " print(\"sample=\", len(predictions[\"sample\"]))\n", - " # Visualize one sample if there are many.\n", - " visualized_data = predictions.isel(sample=sample)\n", + " print(\"sample=\", len(predictions[\"sample\"]))\n", + " # Visualize one sample if there are many.\n", + " visualized_data = predictions.isel(sample=sample)\n", "else:\n", - " visualized_data = predictions\n", + " visualized_data = predictions\n", "\n", "data = {\n", " \" \": scale(select(visualized_data, variable, level, steps), robust=True),\n", @@ -590,19 +551,25 @@ "\n", "fig_title = variable\n", "if \"level\" in predictions[variable].coords:\n", - " fig_title += f\" at {level} hPa\"\n", + " fig_title += f\" at {level} hPa\"\n", "\n", "plot_data(data, fig_title, plot_size, robust=True)" - ] + ], + "metadata": { + "id": "zP3b2vGDgvMN", + "cellView": "form" + }, + "execution_count": null, + "outputs": [] }, { "cell_type": "markdown", - "metadata": { - "id": "fgVnRY2tCzjt" - }, "source": [ "## Clean Up Resources" - ] + ], + "metadata": { + "id": "fgVnRY2tCzjt" + } }, { "cell_type": "code", @@ -616,14 +583,15 @@ "\n", "delete_bucket = True # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" + " ! gsutil -m rm -r $BUCKET_URI\n", + "\n" ] } ], "metadata": { "colab": { "name": "model_garden_weather_prediction_on_vertex.ipynb", - "toc_visible": true + "provenance": [] }, "kernelspec": { "display_name": "Python 3", @@ -632,4 +600,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} +} \ No newline at end of file From 9f5cf1f857034b9da3a91f249aa27224d8e094c8 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 05:32:30 +0000 Subject: [PATCH 6/8] revert to main --- .../model_garden_weather_prediction_on_vertex.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb index ff1043ea7..c7079bb7d 100644 --- a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb @@ -600,4 +600,4 @@ }, "nbformat": 4, "nbformat_minor": 0 -} \ No newline at end of file +} From af5347510dfcbac51cbe474192f23a5884f2c74c Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Wed, 31 Dec 2025 05:12:15 +0000 Subject: [PATCH 7/8] revert model garden file --- ..._garden_weather_prediction_on_vertex.ipynb | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb index ff1043ea7..39574b30a 100644 --- a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb @@ -31,18 +31,18 @@ }, "source": [ "# Vertex AI Model Garden - WeatherNext Forecasting\n", - "\n", - " \n", - " \n", - "
\n", - " \n", - " \"Google
Run in Colab Enterprise\n", - " \n", - " 		\n", - " \n", - " \"GitHub
View on GitHub\n", - " \n", - "
" + "\u003ctable\u003e\u003ctbody\u003e\u003ctr\u003e\n", + " \u003ctd style=\"text-align: center\"\u003e\n", + " \u003ca href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_weather_prediction_on_vertex.ipynb\"\u003e\n", + " \u003cimg alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"\u003e\u003cbr\u003e Run in Colab Enterprise\n", + " \u003c/a\u003e\n", + " \u003c/td\u003e\n", + " \u003ctd style=\"text-align: center\"\u003e\n", + " \u003ca href=\"https://github.com/GoogleCloudPlatform/vertex-ai-samples/blob/main/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb\"\u003e\n", + " \u003cimg alt=\"GitHub logo\" src=\"https://github.githubassets.com/assets/GitHub-Mark-ea2971cee799.png\" width=\"32px\"\u003e\u003cbr\u003e View on GitHub\n", + " \u003c/a\u003e\n", + " \u003c/td\u003e\n", + "\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e" ] }, { @@ -81,7 +81,7 @@ "\n", "### Request For TPU Quota\n", "\n", - "By default, the quota for TPU training [Custom model training TPU v5e cores per region](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_training_tpu_v5e) is 0. TPU quota is only available in `us-west1`, `us-west4`, `us-central1`. You can request for higher TPU quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota). It is suggested to request at least 4 v5e to run this notebook." + "By default, the quota for TPU training [Custom model training TPU v5e cores per region](https://console.cloud.google.com/iam-admin/quotas?location=us-central1\u0026metric=aiplatform.googleapis.com%2Fcustom_model_training_tpu_v5e) is 0. TPU quota is only available in `us-west1`, `us-west4`, `us-central1`. You can request for higher TPU quota following the instructions at [\"Request a higher quota\"](https://cloud.google.com/docs/quota/view-manage#requesting_higher_quota). It is suggested to request at least 4 v5e to run this notebook." ] }, { @@ -117,7 +117,7 @@ "from google.cloud import aiplatform, storage\n", "\n", "# Upgrade Vertex AI SDK.\n", - "! pip3 install --upgrade --quiet 'google-cloud-aiplatform>=1.64.0'\n", + "! pip3 install --upgrade --quiet 'google-cloud-aiplatform\u003e=1.64.0'\n", "if not os.path.exists(\"./vertex-ai-samples\"):\n", " ! git clone https://github.com/GoogleCloudPlatform/vertex-ai-samples.git\n", "! pip3 uninstall --quiet -y xarray\n", @@ -181,7 +181,7 @@ "\n", "\n", "# Utility functions for vertex jobs.\n", - "def get_job_name_with_datetime(prefix: str) -> str:\n", + "def get_job_name_with_datetime(prefix: str) -\u003e str:\n", " \"\"\"Gets the job name with date time when triggering training or deployment\n", " jobs in Vertex AI.\n", " \"\"\"\n", @@ -232,7 +232,7 @@ "def get_existing_demo_step(\n", " num_forecast_steps: int,\n", " model_type: str = \"gen_small\",\n", - " ) -> str:\n", + " ) -\u003e str:\n", " # The demo data only supports some steps, and we can only run the predictions\n", " # if num_forecast_steps is smaller than or equal to the maximum supported setps.\n", " if model_type == \"gen_small\":\n", @@ -250,18 +250,18 @@ " # Find the proper demo data for forecasting.\n", " found_supported_step = supported_demo_steps[-1]\n", " for i, supported_step in enumerate(supported_demo_steps):\n", - " if num_forecast_steps <= supported_step:\n", + " if num_forecast_steps \u003c= supported_step:\n", " found_supported_step = supported_step\n", " break\n", - " if num_forecast_steps > found_supported_step:\n", + " if num_forecast_steps \u003e found_supported_step:\n", " raise ValueError(f\"Supported demo steps for {model_type} in gs://dm_graphcast are {supported_demo_steps}. {num_forecast_steps} is too large, and could not find proper demo data.\")\n", " return found_supported_step\n", "\n", "def get_suggested_machines(\n", " num_forecast_steps: int,\n", - " model_type: str = \"gen_small\",) -> Tuple[str, str, int]:\n", + " model_type: str = \"gen_small\",) -\u003e Tuple[str, str, int]:\n", " if model_type == \"gen_small\":\n", - " if num_forecast_steps <= 16:\n", + " if num_forecast_steps \u003c= 16:\n", " machine_type = \"ct5lp-hightpu-4t\"\n", " tpu_topology = \"2x2\"\n", " accelerator_count = 4\n", @@ -270,7 +270,7 @@ " tpu_topology = \"2x4\"\n", " accelerator_count = 8\n", " else:\n", - " if num_forecast_steps <= 16:\n", + " if num_forecast_steps \u003c= 16:\n", " machine_type = \"ct5lp-hightpu-1t\"\n", " tpu_topology = \"1x1\"\n", " accelerator_count = 1\n", @@ -285,11 +285,11 @@ " variable: str,\n", " level: Optional[int] = None,\n", " max_steps: Optional[int] = None\n", - " ) -> xarray.Dataset:\n", + " -\u003e xarray.Dataset:\n", " data = data[variable]\n", " if \"batch\" in data.dims:\n", " data = data.isel(batch=0)\n", - " if max_steps is not None and \"time\" in data.sizes and max_steps < data.sizes[\"time\"]:\n", + " if max_steps is not None and \"time\" in data.sizes and max_steps \u003c data.sizes[\"time\"]:\n", " data = data.isel(time=range(0, max_steps))\n", " if level is not None and \"level\" in data.coords:\n", " data = data.sel(level=level)\n", @@ -299,7 +299,7 @@ " data: xarray.Dataset,\n", " center: Optional[float] = None,\n", " robust: bool = False,\n", - " ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", + " ) -\u003e tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", " vmin = np.nanpercentile(data, (2 if robust else 0))\n", " vmax = np.nanpercentile(data, (98 if robust else 100))\n", " if center is not None:\n", @@ -315,7 +315,7 @@ " plot_size: float = 5,\n", " robust: bool = False,\n", " cols: int = 4\n", - " ) -> tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", + " ) -\u003e tuple[xarray.Dataset, matplotlib.colors.Normalize, str]:\n", "\n", " first_data = next(iter(data.values()))[0]\n", " max_steps = first_data.sizes.get(\"time\", 1)\n", From 919212df4fb2b83ec4a8100330649a5c691bab05 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Wed, 31 Dec 2025 10:54:42 +0530 Subject: [PATCH 8/8] Update model_garden_weather_prediction_on_vertex.ipynb --- .../model_garden_weather_prediction_on_vertex.ipynb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb index ed57bfbd8..69f7f34dc 100644 --- a/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_weather_prediction_on_vertex.ipynb @@ -31,7 +31,7 @@ }, "source": [ "# Vertex AI Model Garden - WeatherNext Forecasting\n", - "\u003ctable\u003e\u003ctbody\u003e\u003ctr\u003e\n", + "\u003ctable\u003e\u003ctbody\u003e\u003ctr\u003e\n", " \u003ctd style=\"text-align: center\"\u003e\n", " \u003ca href=\"https://console.cloud.google.com/vertex-ai/colab/import/https:%2F%2Fraw.githubusercontent.com%2FGoogleCloudPlatform%2Fvertex-ai-samples%2Fmain%2Fnotebooks%2Fcommunity%2Fmodel_garden%2Fmodel_garden_weather_prediction_on_vertex.ipynb\"\u003e\n", " \u003cimg alt=\"Google Cloud Colab Enterprise logo\" src=\"https://lh3.googleusercontent.com/JmcxdQi-qOpctIvWKgPtrzZdJJK-J3sWE1RsfjZNwshCFgE_9fULcNpuXYTilIR2hjwN\" width=\"32px\"\u003e\u003cbr\u003e Run in Colab Enterprise\n", @@ -259,7 +259,7 @@ "\n", "def get_suggested_machines(\n", " num_forecast_steps: int,\n", - " model_type: str = \"gen_small\",) -\u003e Tuple[str, str, int]:\n", + " model_type: str = \"gen_small\",) -\u003e Tuple[str, str, int]:\n", " if model_type == \"gen_small\":\n", " if num_forecast_steps \u003c= 16:\n", " machine_type = \"ct5lp-hightpu-4t\"\n", @@ -285,7 +285,7 @@ " variable: str,\n", " level: Optional[int] = None,\n", " max_steps: Optional[int] = None\n", - " -\u003e xarray.Dataset:\n", + " ) -\u003e xarray.Dataset:\n", " data = data[variable]\n", " if \"batch\" in data.dims:\n", " data = data.isel(batch=0)\n",