From 102d0a2f01c42a88e97d380295183392d0b86613 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Thu, 16 Oct 2025 14:39:18 +0000 Subject: [PATCH 1/4] Migrate gsutil usage to gcloud storage --- ...xt_classification_online_exported_ds.ipynb | 18 +++++-------- ...ural Language Text Entity Extraction.ipynb | 19 +++++-------- .../get_started_vertex_tensorboard.ipynb | 12 +++------ .../get_started_with_tfhub_models.ipynb | 12 +++------ .../stage5/get_started_with_autoscaling.ipynb | 9 +++---- ...et_started_with_xgboost_model_online.ipynb | 27 +++++++------------ ...el_garden_gemma_deployment_on_vertex.ipynb | 16 +++++------ .../model_garden_integration_with_agent.ipynb | 10 +++---- ...d_with_vertex_endpoint_and_shared_vm.ipynb | 8 +++--- .../get_started_with_pytorch_rov.ipynb | 15 ++++------- 10 files changed, 50 insertions(+), 96 deletions(-) diff --git a/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb b/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb index f6fa945d8..12758924e 100644 --- a/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb +++ b/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb @@ -426,8 +426,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -446,8 +445,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -991,12 +989,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -1153,8 +1149,7 @@ "source": [ "jsonl_index = result.exported_files[0]\n", "\n", - "! gsutil cat $jsonl_index | head" - ] + "! gcloud storage cat $jsonl_index | head" ] }, { "cell_type": "markdown", @@ -1286,8 +1281,7 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/migration/UJ7 legacy AutoML Natural Language Text Entity Extraction.ipynb b/notebooks/community/migration/UJ7 legacy AutoML Natural Language Text Entity Extraction.ipynb index e9cd0186e..8e2938c06 100644 --- a/notebooks/community/migration/UJ7 legacy AutoML Natural Language Text Entity Extraction.ipynb +++ b/notebooks/community/migration/UJ7 legacy AutoML Natural Language Text Entity Extraction.ipynb @@ -326,8 +326,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION gs://$BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -346,8 +345,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al gs://$BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -476,8 +474,7 @@ }, "outputs": [], "source": [ - "! gsutil cat $IMPORT_FILE | head -n 10" - ] + "! gcloud storage cat $IMPORT_FILE | head -n 10" ] }, { "cell_type": "markdown", @@ -1132,8 +1129,7 @@ " data = {\"id\": 0, \"text_snippet\": {\"content\": test_item}}\n", " f.write(json.dumps(data) + \"\\n\")\n", "\n", - "! gsutil cat $gcs_input_uri" - ] + "! gcloud storage cat $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -1280,9 +1276,7 @@ "source": [ "destination_uri = output_config[\"gcs_destination\"][\"output_uri_prefix\"][:-1]\n", "\n", - "! gsutil ls $destination_uri/*\n", - "! gsutil cat $destination_uri/prediction*/*.jsonl" - ] + "! gcloud storage ls $destination_uri/*\n", "! gcloud storage cat $destination_uri/prediction*/*.jsonl" ] }, { "cell_type": "markdown", @@ -1614,8 +1608,7 @@ "\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r gs://$BUCKET_NAME" - ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb b/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb index 77c513537..4e54cd4fb 100644 --- a/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_vertex_tensorboard.ipynb @@ -511,8 +511,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -531,8 +530,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1147,8 +1145,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_example.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_example.tar.gz" ] }, { "cell_type": "markdown", @@ -1357,8 +1354,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb b/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb index 9fa6b43a0..536306268 100644 --- a/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_with_tfhub_models.ipynb @@ -448,8 +448,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -468,8 +467,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1132,8 +1130,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz" ] }, { "cell_type": "markdown", @@ -1381,8 +1378,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket and \"BUCKET_URI\" in globals():\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage5/get_started_with_autoscaling.ipynb b/notebooks/community/ml_ops/stage5/get_started_with_autoscaling.ipynb index 517a92f64..a4b2d1ace 100644 --- a/notebooks/community/ml_ops/stage5/get_started_with_autoscaling.ipynb +++ b/notebooks/community/ml_ops/stage5/get_started_with_autoscaling.ipynb @@ -437,8 +437,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -457,8 +456,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1357,8 +1355,7 @@ "delete_bucket = True\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_xgboost_model_online.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_xgboost_model_online.ipynb index 79fa17264..61b04cd55 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_xgboost_model_online.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_xgboost_model_online.ipynb @@ -484,8 +484,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -504,8 +503,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -717,8 +715,7 @@ "with open(\"instance.yaml\", \"w\") as f:\n", " f.write(yaml)\n", "\n", - "! gsutil cp instance.yaml {BUCKET_URI}/instance.yaml" - ] + "! gcloud storage cp instance.yaml {BUCKET_URI}/instance.yaml" ] }, { "cell_type": "markdown", @@ -1274,14 +1271,11 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gsutil cat $EXECUTE_OUTPUT\n", - " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gsutil cat $GCP_RESOURCES\n", - " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gsutil cat $EVAL_METRICS\n", - " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1289,15 +1283,13 @@ "print(\"get-vertex-model\")\n", "artifacts = print_pipeline_output(pipeline, \"get-vertex-model\")\n", "print(\"\\n\\n\")\n", - "output = !gsutil cat $artifacts\n", - "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n", "print(\"\\n\\n\")\n", "print(\"endpoint-create\")\n", "artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n", "print(\"\\n\\n\")\n", - "output = !gsutil cat $artifacts\n", - "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n", " \"resourceName\"\n", "]\n", @@ -1404,8 +1396,7 @@ "delete_bucket = True\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb index 4c04a6733..f59141b53 100644 --- a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb @@ -152,11 +152,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -180,8 +178,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -229,8 +227,7 @@ "\n", " ! mkdir -p ./gemma\n", " ! curl -X GET \"{signed_url}\" | tar -xzvf - -C ./gemma/\n", - " ! gsutil -m cp -R ./gemma/* {MODEL_BUCKET}\n", - "\n", + " ! gcloud storage cp --recursive ./gemma/* {MODEL_BUCKET}\n", "\n", " model_path_prefix = MODEL_BUCKET\n", " HF_TOKEN = \"\"\n", "else:\n", @@ -1007,8 +1004,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb index 80548a4f5..0a8f6daa9 100644 --- a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb +++ b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb @@ -177,11 +177,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -202,8 +201,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", diff --git a/notebooks/official/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb b/notebooks/official/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb index 0a1f43248..72581959a 100644 --- a/notebooks/official/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb +++ b/notebooks/official/custom/get_started_with_vertex_endpoint_and_shared_vm.ipynb @@ -295,8 +295,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $LOCATION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1203,7 +1202,7 @@ }, "outputs": [], "source": [ - "! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg\n", + "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg\n", "\n", "import base64\n", "\n", @@ -1311,8 +1310,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "# Undeploy the models\n", "# When you're done doing predictions, you undeploy the model from the `Endpoint` resouce. \n", "# This deprovisions all compute resources and ends billing for the deployed model.\n", diff --git a/notebooks/official/ray_on_vertex_ai/get_started_with_pytorch_rov.ipynb b/notebooks/official/ray_on_vertex_ai/get_started_with_pytorch_rov.ipynb index c1fc277bc..485d198de 100644 --- a/notebooks/official/ray_on_vertex_ai/get_started_with_pytorch_rov.ipynb +++ b/notebooks/official/ray_on_vertex_ai/get_started_with_pytorch_rov.ipynb @@ -259,8 +259,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -1025,8 +1024,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -l {TRAINING_URI}" - ] + "! gcloud storage ls --long {TRAINING_URI}" ] }, { "cell_type": "markdown", @@ -1064,8 +1062,7 @@ }, "outputs": [], "source": [ - "! gsutil -q cp -r {TRAINING_URI} {DELIVERABLES_PATH}" - ] + "! gcloud storage cp --recursive {TRAINING_URI} {DELIVERABLES_PATH}" ] }, { "cell_type": "markdown", @@ -1348,8 +1345,7 @@ }, "outputs": [], "source": [ - "!gsutil cp {str(build_path)}/cifar10.mar {BUILD_URI}/model.mar" - ] + "!gcloud storage cp {str(build_path)}/cifar10.mar {BUILD_URI}/model.mar" ] }, { "cell_type": "markdown", @@ -1523,8 +1519,7 @@ "\n", "# Delete Cloud Storage objects that were created\n", "if delete_bucket:\n", - " ! gsutil -q -m rm -r $BUCKET_URI" - ] + " ! gcloud -q storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From a332732746d4bc81df549fd45c5abb43894b5e5f Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Mon, 15 Dec 2025 06:34:33 +0000 Subject: [PATCH 2/4] Changes for 4321 --- .../model_garden/model_garden_gemma_deployment_on_vertex.ipynb | 3 +-- .../model_garden/model_garden_integration_with_agent.ipynb | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb index f59141b53..0b004d7fd 100644 --- a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb @@ -154,7 +154,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -178,7 +178,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", diff --git a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb index 0a8f6daa9..b19339e87 100644 --- a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb +++ b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb @@ -179,8 +179,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", From ab13c61c9ee31ce53032d98668fb1de200ede317 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Wed, 17 Dec 2025 13:25:07 +0000 Subject: [PATCH 3/4] fix the linter issue for br 4321 --- ...wcase_custom_text_classification_online_exported_ds.ipynb | 3 --- ...gacy AutoML Natural Language Text Entity Extraction.ipynb | 5 +---- 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb b/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb index 12758924e..5d6ae1f5d 100644 --- a/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb +++ b/notebooks/community/gapic/custom/showcase_custom_text_classification_online_exported_ds.ipynb @@ -481,9 +481,6 @@ "import time\n", "\n", "from google.cloud.aiplatform import gapic as aip\n", - "from google.protobuf import json_format\n", - "from google.protobuf.json_format import MessageToJson, ParseDict\n", - "from google.protobuf.struct_pb2 import Struct, Value" ] }, { diff --git a/notebooks/community/migration/UJ7 legacy AutoML Natural Language Text Entity Extraction.ipynb b/notebooks/community/migration/UJ7 legacy AutoML Natural Language Text Entity Extraction.ipynb index 8e2938c06..964aaafc5 100644 --- a/notebooks/community/migration/UJ7 legacy AutoML Natural Language Text Entity Extraction.ipynb +++ b/notebooks/community/migration/UJ7 legacy AutoML Natural Language Text Entity Extraction.ipynb @@ -378,14 +378,11 @@ }, "outputs": [], "source": [ - "import json\n", "import os\n", "import sys\n", - "import time\n", "\n", "from google.cloud import automl\n", - "from google.protobuf.json_format import MessageToJson\n", - "from google.protobuf.struct_pb2 import Value" + "from google.protobuf.json_format import MessageToJson\n" ] }, { From 2bafa6585dffe6d9d074acbaf0806d9c8d06fff4 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Mon, 22 Dec 2025 19:54:18 +0000 Subject: [PATCH 4/4] removed changes for model garden --- .../model_garden_gemma_deployment_on_vertex.ipynb | 15 ++++++++++----- .../model_garden_integration_with_agent.ipynb | 9 ++++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb index 0b004d7fd..4c04a6733 100644 --- a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb @@ -152,9 +152,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -178,7 +180,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -226,7 +229,8 @@ "\n", " ! mkdir -p ./gemma\n", " ! curl -X GET \"{signed_url}\" | tar -xzvf - -C ./gemma/\n", - " ! gcloud storage cp --recursive ./gemma/* {MODEL_BUCKET}\n", "\n", + " ! gsutil -m cp -R ./gemma/* {MODEL_BUCKET}\n", + "\n", " model_path_prefix = MODEL_BUCKET\n", " HF_TOKEN = \"\"\n", "else:\n", @@ -1003,7 +1007,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gsutil -m rm -r $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb index b19339e87..80548a4f5 100644 --- a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb +++ b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb @@ -177,9 +177,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -200,7 +202,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n",