From 9c9fee5f3064c374bb509c5477261fa4946882e3 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Wed, 15 Oct 2025 18:59:03 +0000 Subject: [PATCH 1/9] Migrate gsutil usage to gcloud storage --- .../model_garden_keras_stable_diffusion.ipynb | 3 +-- ...del_garden_pytorch_instant_id_gradio.ipynb | 3 +-- ...del_garden_pytorch_sd_2_1_deployment.ipynb | 12 ++++------ .../vision/video_warehouse_sdk.ipynb | 3 +-- .../comparing_local_trained_models.ipynb | 6 ++--- ...om_training_autologging_local_script.ipynb | 17 ++++++-------- ..._binary_classification_batch_explain.ipynb | 22 ++++++------------- ...components_model_train_upload_deploy.ipynb | 14 +++++------- ...ion_with_prebuilt_serving_containers.ipynb | 11 +++------- .../distributed_hyperparameter_tuning.ipynb | 6 ++--- 10 files changed, 34 insertions(+), 63 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb index b65c47566..feafc0e4a 100644 --- a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb @@ -318,8 +318,7 @@ " ! mkdir -p $filename_without_ext\n", " ! tar -xvf $filename_with_ext -C .\n", "\n", - " ! gsutil -m cp -r $filename_without_ext $gcs_bucket/\n", - " gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n", + " ! gcloud storage cp --recursive $filename_without_ext $gcs_bucket/\n", " gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n", " print(\"Upload files to: \", gcs_path)\n", " return gcs_path\n", "\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb index b058e051d..e4bc5372a 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb @@ -562,8 +562,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb b/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb index 3b344a4bf..5cba1ee76 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb @@ -151,11 +151,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -179,8 +177,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -529,8 +526,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/vision/video_warehouse_sdk.ipynb b/notebooks/community/vision/video_warehouse_sdk.ipynb index 456bedd7a..038a07d46 100644 --- a/notebooks/community/vision/video_warehouse_sdk.ipynb +++ b/notebooks/community/vision/video_warehouse_sdk.ipynb @@ -153,8 +153,7 @@ }, "outputs": [], "source": [ - "!gsutil cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n", - "!pip install visionai-0.0.6-py3-none-any.whl --force-reinstall" + "!gcloud storage cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n", "!pip install visionai-0.0.6-py3-none-any.whl --force-reinstall" ] }, { diff --git a/notebooks/official/experiments/comparing_local_trained_models.ipynb b/notebooks/official/experiments/comparing_local_trained_models.ipynb index cb177fdad..cb8d416a8 100644 --- a/notebooks/official/experiments/comparing_local_trained_models.ipynb +++ b/notebooks/official/experiments/comparing_local_trained_models.ipynb @@ -286,8 +286,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -770,8 +769,7 @@ "delete_bucket = False # Set True for deletion\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb index 411970608..b49f4851e 100644 --- a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb +++ b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb @@ -300,8 +300,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -430,10 +429,10 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -482,8 +481,7 @@ "SOURCE_DATA_URL = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/uci_glass_preprocessed/glass.csv\"\n", "DESTINATION_DATA_URL = f\"{BUCKET_URI}/data/glass.csv\"\n", "\n", - "! gsutil cp $SOURCE_DATA_URL $DESTINATION_DATA_URL" - ] + "! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL" ] }, { "cell_type": "markdown", @@ -807,8 +805,7 @@ "\n", "# Delete Cloud Storage objects that were created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb b/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb index 24d086369..55778eb56 100644 --- a/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb @@ -292,8 +292,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -371,14 +370,11 @@ }, "outputs": [], "source": [ - "count = ! gsutil cat $IMPORT_FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $IMPORT_FILE | head\n", - "\n", - "heading = ! gsutil cat $IMPORT_FILE | head -n1\n", - "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -580,15 +576,12 @@ }, "outputs": [], "source": [ - "! gsutil cat $IMPORT_FILE | head -n 1 > tmp.csv\n", - "! gsutil cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n", - "\n", + "! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n", "! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n", "\n", "! cut -d, -f1-16 tmp.csv > batch.csv\n", "\n", "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n", "\n", - "! gsutil cp batch.csv $gcs_input_uri" - ] + "! gcloud storage cp batch.csv $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -729,8 +722,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "# Delete the locally generated files\n", "! rm batch.csv tmp.csv" ] diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb index 5846a8cd9..a5e2b07d3 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb @@ -332,8 +332,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -405,10 +404,10 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -732,8 +731,7 @@ "source": [ "delete_bucket = False\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb b/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb index a87751768..85a9d72b0 100644 --- a/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb +++ b/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb @@ -303,8 +303,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -545,10 +544,7 @@ "source": [ "MODEL_URI = f\"{BUCKET_URI}/{model_name}\"\n", "\n", - "!gsutil rm -r $MODEL_URI\n", - "!gsutil cp -r $model_path $MODEL_URI\n", - "!gsutil ls -al $MODEL_URI" - ] + "!gcloud storage rm --recursive $MODEL_URI\n", "!gcloud storage cp --recursive $model_path $MODEL_URI\n", "!gcloud storage ls --all-versions --long $MODEL_URI" ] }, { "cell_type": "markdown", @@ -823,8 +819,7 @@ "\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb index 4fd3ea62a..da0f46719 100644 --- a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb +++ b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb @@ -274,8 +274,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -700,8 +699,7 @@ "!gcloud container images delete $IMAGE_URI --force-delete-tags --quiet\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From 4d4722611a19abd22aa063ba0fc8ba63cb1c16b6 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Wed, 10 Dec 2025 15:36:54 +0000 Subject: [PATCH 2/9] update --- .../model_garden_pytorch_sd_2_1_deployment.ipynb | 12 ++++++++---- notebooks/community/vision/video_warehouse_sdk.ipynb | 3 ++- ...th_custom_training_autologging_local_script.ipynb | 2 -- ...peline_components_model_train_upload_deploy.ipynb | 2 -- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb b/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb index 5cba1ee76..1c1cca739 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb @@ -151,9 +151,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -177,7 +179,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -526,7 +529,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/vision/video_warehouse_sdk.ipynb b/notebooks/community/vision/video_warehouse_sdk.ipynb index 038a07d46..404f99ee9 100644 --- a/notebooks/community/vision/video_warehouse_sdk.ipynb +++ b/notebooks/community/vision/video_warehouse_sdk.ipynb @@ -153,7 +153,8 @@ }, "outputs": [], "source": [ - "!gcloud storage cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n", "!pip install visionai-0.0.6-py3-none-any.whl --force-reinstall" + "! gcloud storage cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n", + "! pip install visionai-0.0.6-py3-none-any.whl --force-reinstall" ] }, { diff --git a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb index b49f4851e..5067dfab8 100644 --- a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb +++ b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb @@ -429,9 +429,7 @@ }, "outputs": [], "source": [ - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb index a5e2b07d3..387f6656c 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb @@ -404,9 +404,7 @@ }, "outputs": [], "source": [ - "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { From c7a00a971087e7864e24f6c3da5777f1f8d09036 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 12 Dec 2025 05:18:05 +0000 Subject: [PATCH 3/9] linter changes for 4298 --- .../model_garden_keras_stable_diffusion.ipynb | 9 ++++---- ...del_garden_pytorch_instant_id_gradio.ipynb | 9 ++++---- .../comparing_local_trained_models.ipynb | 6 +++-- ...om_training_autologging_local_script.ipynb | 15 ++++++++----- ..._binary_classification_batch_explain.ipynb | 22 +++++++++++++------ ...components_model_train_upload_deploy.ipynb | 16 +++++++++----- ...ion_with_prebuilt_serving_containers.ipynb | 11 +++++++--- .../distributed_hyperparameter_tuning.ipynb | 6 +++-- 8 files changed, 60 insertions(+), 34 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb index feafc0e4a..4d1a055b2 100644 --- a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb @@ -318,7 +318,8 @@ " ! mkdir -p $filename_without_ext\n", " ! tar -xvf $filename_with_ext -C .\n", "\n", - " ! gcloud storage cp --recursive $filename_without_ext $gcs_bucket/\n", " gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n", + " ! gcloud storage cp --recursive $filename_without_ext $gcs_bucket/\n", + " gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n", " print(\"Upload files to: \", gcs_path)\n", " return gcs_path\n", "\n", @@ -359,7 +360,7 @@ " serving_container_predict_route=\"/predict\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\"\n", + " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\",\n", " )\n", " model.deploy(\n", " endpoint=endpoint,\n", @@ -370,9 +371,7 @@ " max_replica_count=1,\n", " deploy_request_timeout=1800,\n", " service_account=service_account,\n", - " system_labels={\n", - " \"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"\n", - " },\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"},\n", " )\n", " return model, endpoint\n", "\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb index e4bc5372a..b96b6538f 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb @@ -312,7 +312,7 @@ " serving_container_predict_route=\"/predictions/diffusers_serving\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/instantx/models/instant-id\"\n", + " model_garden_source_model_name=\"publishers/instantx/models/instant-id\",\n", " )\n", " machine_type = \"g2-standard-8\"\n", " accelerator_type = \"NVIDIA_L4\"\n", @@ -334,9 +334,7 @@ " deploy_request_timeout=1800,\n", " service_account=SERVICE_ACCOUNT,\n", " sync=False,\n", - " system_labels={\n", - " \"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"\n", - " },\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"},\n", " )\n", "\n", " gr.Info(\n", @@ -562,7 +560,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/official/experiments/comparing_local_trained_models.ipynb b/notebooks/official/experiments/comparing_local_trained_models.ipynb index cb8d416a8..49c5df791 100644 --- a/notebooks/official/experiments/comparing_local_trained_models.ipynb +++ b/notebooks/official/experiments/comparing_local_trained_models.ipynb @@ -286,7 +286,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -769,7 +770,8 @@ "delete_bucket = False # Set True for deletion\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb index 5067dfab8..93962b33f 100644 --- a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb +++ b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb @@ -300,7 +300,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -429,8 +430,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -479,7 +482,8 @@ "SOURCE_DATA_URL = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/uci_glass_preprocessed/glass.csv\"\n", "DESTINATION_DATA_URL = f\"{BUCKET_URI}/data/glass.csv\"\n", "\n", - "! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL" ] + "! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL" + ] }, { "cell_type": "markdown", @@ -803,7 +807,8 @@ "\n", "# Delete Cloud Storage objects that were created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb b/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb index 55778eb56..ee745c5dc 100644 --- a/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb @@ -292,7 +292,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -370,11 +371,14 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $IMPORT_FILE | head\n", "\n", - "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", + "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -576,12 +580,15 @@ }, "outputs": [], "source": [ - "! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n", "! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n", "\n", + "! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n", + "! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n", + "\n", "! cut -d, -f1-16 tmp.csv > batch.csv\n", "\n", "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n", "\n", - "! gcloud storage cp batch.csv $gcs_input_uri" ] + "! gcloud storage cp batch.csv $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -722,7 +729,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "# Delete the locally generated files\n", "! rm batch.csv tmp.csv" ] diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb index 387f6656c..e0964e192 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb @@ -332,7 +332,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -404,8 +405,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -503,7 +506,9 @@ }, "outputs": [], "source": [ - "hp_dict: str = '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n", + "hp_dict: str = (\n", + " '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n", + ")\n", "data_dir: str = (\n", " \"gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/bikes_weather/\"\n", ")\n", @@ -729,7 +734,8 @@ "source": [ "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb b/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb index 85a9d72b0..388fe4f77 100644 --- a/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb +++ b/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb @@ -303,7 +303,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -544,7 +545,10 @@ "source": [ "MODEL_URI = f\"{BUCKET_URI}/{model_name}\"\n", "\n", - "!gcloud storage rm --recursive $MODEL_URI\n", "!gcloud storage cp --recursive $model_path $MODEL_URI\n", "!gcloud storage ls --all-versions --long $MODEL_URI" ] + "!gcloud storage rm --recursive $MODEL_URI\n", + "!gcloud storage cp --recursive $model_path $MODEL_URI\n", + "!gcloud storage ls --all-versions --long $MODEL_URI" + ] }, { "cell_type": "markdown", @@ -819,7 +823,8 @@ "\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb index da0f46719..89efd8a4a 100644 --- a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb +++ b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb @@ -274,7 +274,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -699,7 +700,8 @@ "!gcloud container images delete $IMAGE_URI --force-delete-tags --quiet\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From 00100ff85e4166dcffcffdbc5ef8ecc8bff4ea95 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 12 Dec 2025 09:04:28 +0000 Subject: [PATCH 4/9] Revert "linter changes for 4298" This reverts commit c7a00a971087e7864e24f6c3da5777f1f8d09036. --- .../model_garden_keras_stable_diffusion.ipynb | 9 ++++---- ...del_garden_pytorch_instant_id_gradio.ipynb | 9 ++++---- .../comparing_local_trained_models.ipynb | 6 ++--- ...om_training_autologging_local_script.ipynb | 15 +++++-------- ..._binary_classification_batch_explain.ipynb | 22 ++++++------------- ...components_model_train_upload_deploy.ipynb | 16 +++++--------- ...ion_with_prebuilt_serving_containers.ipynb | 11 +++------- .../distributed_hyperparameter_tuning.ipynb | 6 ++--- 8 files changed, 34 insertions(+), 60 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb index 4d1a055b2..feafc0e4a 100644 --- a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb @@ -318,8 +318,7 @@ " ! mkdir -p $filename_without_ext\n", " ! tar -xvf $filename_with_ext -C .\n", "\n", - " ! gcloud storage cp --recursive $filename_without_ext $gcs_bucket/\n", - " gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n", + " ! gcloud storage cp --recursive $filename_without_ext $gcs_bucket/\n", " gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n", " print(\"Upload files to: \", gcs_path)\n", " return gcs_path\n", "\n", @@ -360,7 +359,7 @@ " serving_container_predict_route=\"/predict\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\",\n", + " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\"\n", " )\n", " model.deploy(\n", " endpoint=endpoint,\n", @@ -371,7 +370,9 @@ " max_replica_count=1,\n", " deploy_request_timeout=1800,\n", " service_account=service_account,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"},\n", + " system_labels={\n", + " \"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"\n", + " },\n", " )\n", " return model, endpoint\n", "\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb index b96b6538f..e4bc5372a 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb @@ -312,7 +312,7 @@ " serving_container_predict_route=\"/predictions/diffusers_serving\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/instantx/models/instant-id\",\n", + " model_garden_source_model_name=\"publishers/instantx/models/instant-id\"\n", " )\n", " machine_type = \"g2-standard-8\"\n", " accelerator_type = \"NVIDIA_L4\"\n", @@ -334,7 +334,9 @@ " deploy_request_timeout=1800,\n", " service_account=SERVICE_ACCOUNT,\n", " sync=False,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"},\n", + " system_labels={\n", + " \"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"\n", + " },\n", " )\n", "\n", " gr.Info(\n", @@ -560,8 +562,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/official/experiments/comparing_local_trained_models.ipynb b/notebooks/official/experiments/comparing_local_trained_models.ipynb index 49c5df791..cb8d416a8 100644 --- a/notebooks/official/experiments/comparing_local_trained_models.ipynb +++ b/notebooks/official/experiments/comparing_local_trained_models.ipynb @@ -286,8 +286,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -770,8 +769,7 @@ "delete_bucket = False # Set True for deletion\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb index 93962b33f..5067dfab8 100644 --- a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb +++ b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb @@ -300,8 +300,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -430,10 +429,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", - "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" - ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -482,8 +479,7 @@ "SOURCE_DATA_URL = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/uci_glass_preprocessed/glass.csv\"\n", "DESTINATION_DATA_URL = f\"{BUCKET_URI}/data/glass.csv\"\n", "\n", - "! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL" - ] + "! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL" ] }, { "cell_type": "markdown", @@ -807,8 +803,7 @@ "\n", "# Delete Cloud Storage objects that were created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb b/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb index ee745c5dc..55778eb56 100644 --- a/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb @@ -292,8 +292,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -371,14 +370,11 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $IMPORT_FILE | head\n", - "\n", - "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", - "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -580,15 +576,12 @@ }, "outputs": [], "source": [ - "! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n", - "! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n", - "\n", + "! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n", "! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n", "\n", "! cut -d, -f1-16 tmp.csv > batch.csv\n", "\n", "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n", "\n", - "! gcloud storage cp batch.csv $gcs_input_uri" - ] + "! gcloud storage cp batch.csv $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -729,8 +722,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "# Delete the locally generated files\n", "! rm batch.csv tmp.csv" ] diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb index e0964e192..387f6656c 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb @@ -332,8 +332,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -405,10 +404,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", - "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" - ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -506,9 +503,7 @@ }, "outputs": [], "source": [ - "hp_dict: str = (\n", - " '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n", - ")\n", + "hp_dict: str = '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n", "data_dir: str = (\n", " \"gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/bikes_weather/\"\n", ")\n", @@ -734,8 +729,7 @@ "source": [ "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb b/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb index 388fe4f77..85a9d72b0 100644 --- a/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb +++ b/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb @@ -303,8 +303,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -545,10 +544,7 @@ "source": [ "MODEL_URI = f\"{BUCKET_URI}/{model_name}\"\n", "\n", - "!gcloud storage rm --recursive $MODEL_URI\n", - "!gcloud storage cp --recursive $model_path $MODEL_URI\n", - "!gcloud storage ls --all-versions --long $MODEL_URI" - ] + "!gcloud storage rm --recursive $MODEL_URI\n", "!gcloud storage cp --recursive $model_path $MODEL_URI\n", "!gcloud storage ls --all-versions --long $MODEL_URI" ] }, { "cell_type": "markdown", @@ -823,8 +819,7 @@ "\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb index 89efd8a4a..da0f46719 100644 --- a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb +++ b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb @@ -274,8 +274,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -700,8 +699,7 @@ "!gcloud container images delete $IMAGE_URI --force-delete-tags --quiet\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From 7cbdc7ed3da6998a39b74699bba07c506ca35765 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Fri, 12 Dec 2025 18:22:42 +0000 Subject: [PATCH 5/9] Linter fixes --- .../model_garden_keras_stable_diffusion.ipynb | 9 ++++---- ...del_garden_pytorch_instant_id_gradio.ipynb | 9 ++++---- ...del_garden_pytorch_sd_2_1_deployment.ipynb | 12 ++++++---- .../vision/video_warehouse_sdk.ipynb | 3 ++- .../comparing_local_trained_models.ipynb | 6 +++-- ...om_training_autologging_local_script.ipynb | 17 ++++++++------ ..._binary_classification_batch_explain.ipynb | 22 +++++++++++++------ ...components_model_train_upload_deploy.ipynb | 18 +++++++++------ ...ion_with_prebuilt_serving_containers.ipynb | 11 +++++++--- .../distributed_hyperparameter_tuning.ipynb | 6 +++-- 10 files changed, 70 insertions(+), 43 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb index feafc0e4a..4d1a055b2 100644 --- a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb @@ -318,7 +318,8 @@ " ! mkdir -p $filename_without_ext\n", " ! tar -xvf $filename_with_ext -C .\n", "\n", - " ! gcloud storage cp --recursive $filename_without_ext $gcs_bucket/\n", " gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n", + " ! gcloud storage cp --recursive $filename_without_ext $gcs_bucket/\n", + " gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n", " print(\"Upload files to: \", gcs_path)\n", " return gcs_path\n", "\n", @@ -359,7 +360,7 @@ " serving_container_predict_route=\"/predict\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\"\n", + " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\",\n", " )\n", " model.deploy(\n", " endpoint=endpoint,\n", @@ -370,9 +371,7 @@ " max_replica_count=1,\n", " deploy_request_timeout=1800,\n", " service_account=service_account,\n", - " system_labels={\n", - " \"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"\n", - " },\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"},\n", " )\n", " return model, endpoint\n", "\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb index e4bc5372a..b96b6538f 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb @@ -312,7 +312,7 @@ " serving_container_predict_route=\"/predictions/diffusers_serving\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/instantx/models/instant-id\"\n", + " model_garden_source_model_name=\"publishers/instantx/models/instant-id\",\n", " )\n", " machine_type = \"g2-standard-8\"\n", " accelerator_type = \"NVIDIA_L4\"\n", @@ -334,9 +334,7 @@ " deploy_request_timeout=1800,\n", " service_account=SERVICE_ACCOUNT,\n", " sync=False,\n", - " system_labels={\n", - " \"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"\n", - " },\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"},\n", " )\n", "\n", " gr.Info(\n", @@ -562,7 +560,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb b/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb index 5cba1ee76..9cfb608e1 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb @@ -151,9 +151,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -177,7 +179,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -526,7 +529,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/vision/video_warehouse_sdk.ipynb b/notebooks/community/vision/video_warehouse_sdk.ipynb index 038a07d46..7997d0d14 100644 --- a/notebooks/community/vision/video_warehouse_sdk.ipynb +++ b/notebooks/community/vision/video_warehouse_sdk.ipynb @@ -153,7 +153,8 @@ }, "outputs": [], "source": [ - "!gcloud storage cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n", "!pip install visionai-0.0.6-py3-none-any.whl --force-reinstall" + "!gcloud storage cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n", + "!pip install visionai-0.0.6-py3-none-any.whl --force-reinstall" ] }, { diff --git a/notebooks/official/experiments/comparing_local_trained_models.ipynb b/notebooks/official/experiments/comparing_local_trained_models.ipynb index cb8d416a8..49c5df791 100644 --- a/notebooks/official/experiments/comparing_local_trained_models.ipynb +++ b/notebooks/official/experiments/comparing_local_trained_models.ipynb @@ -286,7 +286,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -769,7 +770,8 @@ "delete_bucket = False # Set True for deletion\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb index b49f4851e..93962b33f 100644 --- a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb +++ b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb @@ -300,7 +300,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -429,10 +430,10 @@ }, "outputs": [], "source": [ - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -481,7 +482,8 @@ "SOURCE_DATA_URL = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/uci_glass_preprocessed/glass.csv\"\n", "DESTINATION_DATA_URL = f\"{BUCKET_URI}/data/glass.csv\"\n", "\n", - "! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL" ] + "! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL" + ] }, { "cell_type": "markdown", @@ -805,7 +807,8 @@ "\n", "# Delete Cloud Storage objects that were created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb b/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb index 55778eb56..ee745c5dc 100644 --- a/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb @@ -292,7 +292,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -370,11 +371,14 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $IMPORT_FILE | head\n", "\n", - "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", + "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -576,12 +580,15 @@ }, "outputs": [], "source": [ - "! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n", "! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n", "\n", + "! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n", + "! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n", + "\n", "! cut -d, -f1-16 tmp.csv > batch.csv\n", "\n", "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n", "\n", - "! gcloud storage cp batch.csv $gcs_input_uri" ] + "! gcloud storage cp batch.csv $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -722,7 +729,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "# Delete the locally generated files\n", "! rm batch.csv tmp.csv" ] diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb index a5e2b07d3..e0964e192 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb @@ -332,7 +332,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -404,10 +405,10 @@ }, "outputs": [], "source": [ - "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -505,7 +506,9 @@ }, "outputs": [], "source": [ - "hp_dict: str = '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n", + "hp_dict: str = (\n", + " '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n", + ")\n", "data_dir: str = (\n", " \"gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/bikes_weather/\"\n", ")\n", @@ -731,7 +734,8 @@ "source": [ "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb b/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb index 85a9d72b0..388fe4f77 100644 --- a/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb +++ b/notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb @@ -303,7 +303,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -544,7 +545,10 @@ "source": [ "MODEL_URI = f\"{BUCKET_URI}/{model_name}\"\n", "\n", - "!gcloud storage rm --recursive $MODEL_URI\n", "!gcloud storage cp --recursive $model_path $MODEL_URI\n", "!gcloud storage ls --all-versions --long $MODEL_URI" ] + "!gcloud storage rm --recursive $MODEL_URI\n", + "!gcloud storage cp --recursive $model_path $MODEL_URI\n", + "!gcloud storage ls --all-versions --long $MODEL_URI" + ] }, { "cell_type": "markdown", @@ -819,7 +823,8 @@ "\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb index da0f46719..89efd8a4a 100644 --- a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb +++ b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb @@ -274,7 +274,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -699,7 +700,8 @@ "!gcloud container images delete $IMAGE_URI --force-delete-tags --quiet\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From 80438f972d22eb15f90252aef07bc82c719c6971 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 15 Dec 2025 05:11:28 +0000 Subject: [PATCH 6/9] update --- .../model_garden/model_garden_keras_stable_diffusion.ipynb | 4 ++-- .../model_garden/model_garden_pytorch_instant_id_gradio.ipynb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb index 4d1a055b2..3619cb2cf 100644 --- a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb @@ -360,7 +360,7 @@ " serving_container_predict_route=\"/predict\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\",\n", + " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\"\n", " )\n", " model.deploy(\n", " endpoint=endpoint,\n", @@ -371,7 +371,7 @@ " max_replica_count=1,\n", " deploy_request_timeout=1800,\n", " service_account=service_account,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"},\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"}\n", " )\n", " return model, endpoint\n", "\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb index b96b6538f..476ade9f7 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb @@ -312,7 +312,7 @@ " serving_container_predict_route=\"/predictions/diffusers_serving\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/instantx/models/instant-id\",\n", + " model_garden_source_model_name=\"publishers/instantx/models/instant-id\"\n", " )\n", " machine_type = \"g2-standard-8\"\n", " accelerator_type = \"NVIDIA_L4\"\n", @@ -334,7 +334,7 @@ " deploy_request_timeout=1800,\n", " service_account=SERVICE_ACCOUNT,\n", " sync=False,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"},\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"}\n", " )\n", "\n", " gr.Info(\n", From d572c98d6013a146fa0723d98997107445864845 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 15 Dec 2025 09:11:59 +0000 Subject: [PATCH 7/9] update --- .../model_garden/model_garden_keras_stable_diffusion.ipynb | 4 ++-- .../model_garden_pytorch_instant_id_gradio.ipynb | 4 ++-- ...arted_with_custom_training_autologging_local_script.ipynb | 5 +++-- ...cloud_pipeline_components_model_train_upload_deploy.ipynb | 5 +++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb index 3619cb2cf..4d1a055b2 100644 --- a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb @@ -360,7 +360,7 @@ " serving_container_predict_route=\"/predict\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\"\n", + " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\",\n", " )\n", " model.deploy(\n", " endpoint=endpoint,\n", @@ -371,7 +371,7 @@ " max_replica_count=1,\n", " deploy_request_timeout=1800,\n", " service_account=service_account,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"}\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"},\n", " )\n", " return model, endpoint\n", "\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb index 476ade9f7..b96b6538f 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb @@ -312,7 +312,7 @@ " serving_container_predict_route=\"/predictions/diffusers_serving\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/instantx/models/instant-id\"\n", + " model_garden_source_model_name=\"publishers/instantx/models/instant-id\",\n", " )\n", " machine_type = \"g2-standard-8\"\n", " accelerator_type = \"NVIDIA_L4\"\n", @@ -334,7 +334,7 @@ " deploy_request_timeout=1800,\n", " service_account=SERVICE_ACCOUNT,\n", " sync=False,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"}\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"},\n", " )\n", "\n", " gr.Info(\n", diff --git a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb index 6f3579a3b..93962b33f 100644 --- a/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb +++ b/notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb @@ -430,8 +430,9 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb index 5e8bd147e..e0964e192 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb @@ -405,8 +405,9 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { From 4f98b23aa6257d85e4948d1d015e7edaa22f55ee Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 18 Dec 2025 12:57:15 +0530 Subject: [PATCH 8/9] Update distributed_hyperparameter_tuning.ipynb --- .../official/training/distributed_hyperparameter_tuning.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb index 89efd8a4a..2de174875 100644 --- a/notebooks/official/training/distributed_hyperparameter_tuning.ipynb +++ b/notebooks/official/training/distributed_hyperparameter_tuning.ipynb @@ -692,7 +692,7 @@ }, "outputs": [], "source": [ - "# Set this to true only if you'd like to delete your bucket\n", + "# Set this to true only if you'd like to delete your bucket.\n", "delete_bucket = False\n", "\n", "hp_job.delete()\n", From 354b79103d92328de24b3777401fc789dc88518c Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 19 Dec 2025 16:47:09 +0000 Subject: [PATCH 9/9] removed changes in model_garden folder --- .../model_garden_keras_stable_diffusion.ipynb | 8 +++++--- .../model_garden_pytorch_instant_id_gradio.ipynb | 8 +++++--- .../model_garden_pytorch_sd_2_1_deployment.ipynb | 8 ++++---- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb index 4d1a055b2..b65c47566 100644 --- a/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_stable_diffusion.ipynb @@ -318,7 +318,7 @@ " ! mkdir -p $filename_without_ext\n", " ! tar -xvf $filename_with_ext -C .\n", "\n", - " ! gcloud storage cp --recursive $filename_without_ext $gcs_bucket/\n", + " ! gsutil -m cp -r $filename_without_ext $gcs_bucket/\n", " gcs_path = os.path.join(gcs_bucket, filename_without_ext)\n", " print(\"Upload files to: \", gcs_path)\n", " return gcs_path\n", @@ -360,7 +360,7 @@ " serving_container_predict_route=\"/predict\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\",\n", + " model_garden_source_model_name=\"publishers/google/models/stable-diffusion-v1-4\"\n", " )\n", " model.deploy(\n", " endpoint=endpoint,\n", @@ -371,7 +371,9 @@ " max_replica_count=1,\n", " deploy_request_timeout=1800,\n", " service_account=service_account,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"},\n", + " system_labels={\n", + " \"NOTEBOOK_NAME\": \"model_garden_keras_stable_diffusion.ipynb\"\n", + " },\n", " )\n", " return model, endpoint\n", "\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb index b96b6538f..b058e051d 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_instant_id_gradio.ipynb @@ -312,7 +312,7 @@ " serving_container_predict_route=\"/predictions/diffusers_serving\",\n", " serving_container_health_route=\"/ping\",\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/instantx/models/instant-id\",\n", + " model_garden_source_model_name=\"publishers/instantx/models/instant-id\"\n", " )\n", " machine_type = \"g2-standard-8\"\n", " accelerator_type = \"NVIDIA_L4\"\n", @@ -334,7 +334,9 @@ " deploy_request_timeout=1800,\n", " service_account=SERVICE_ACCOUNT,\n", " sync=False,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"},\n", + " system_labels={\n", + " \"NOTEBOOK_NAME\": \"model_garden_pytorch_instant_id_gradio.ipynb\"\n", + " },\n", " )\n", "\n", " gr.Info(\n", @@ -560,7 +562,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb b/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb index 1c1cca739..3b344a4bf 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_sd_2_1_deployment.ipynb @@ -151,10 +151,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -179,7 +179,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -529,7 +529,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ],