From 9f311988142ed9a6e00539d35a27004eadbd5ec3 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Wed, 15 Oct 2025 16:56:40 +0000 Subject: [PATCH 1/7] Migrate gsutil usage to gcloud storage --- .cloud-build/execute_changed_notebooks_helper.py | 2 +- ..._pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb | 15 +++++++-------- ..._pytorch_stable_diffusion_deployment_1_5.ipynb | 14 ++++++-------- ...able_diffusion_finetuning_dreambooth_1_5.ipynb | 15 ++++++--------- ...diffusion_finetuning_dreambooth_lora_1_5.ipynb | 3 +-- .../sdk_automl_tabular_regression_online_bq.ipynb | 6 ++---- .../official/generative_ai/distillation.ipynb | 10 +++------- ...etric-parameter-tracking-for-custom-jobs.ipynb | 9 +++------ .../get_started_with_psc_private_endpoint.ipynb | 12 ++++-------- .../prediction/llm_streaming_prediction.ipynb | 11 ++++------- 10 files changed, 37 insertions(+), 60 deletions(-) diff --git a/.cloud-build/execute_changed_notebooks_helper.py b/.cloud-build/execute_changed_notebooks_helper.py index 26289f89a..b38af683d 100755 --- a/.cloud-build/execute_changed_notebooks_helper.py +++ b/.cloud-build/execute_changed_notebooks_helper.py @@ -365,7 +365,7 @@ def process_and_execute_notebook( # Use gcloud to get tail try: result.error_message = subprocess.check_output( - ["gsutil", "cat", "-r", "-1000", log_file_uri], encoding="UTF-8" + ["gcloud", "storage", "cat", "--range", "-1000", log_file_uri], encoding="UTF-8" ) except Exception as error: result.error_message = str(error) diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb index d7c2d43a6..2c17f2514 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb @@ -159,11 +159,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + "# Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -187,8 +186,9 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "# Note: gsutil iam ch does not support modifying IAM policies that contain conditions. gcloud storage commands do support conditions.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -497,8 +497,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb index 87cf9c3b5..e1d09cd9f 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb @@ -138,10 +138,9 @@ " # Create a unique GCS bucket for this notebook, if not specified by the user\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -158,8 +157,8 @@ "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "# The pre-built serving docker image. It contains serving scripts and models.\n", "TEXT_TO_IMAGE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/pytorch-inference.cu125.0-4.ubuntu2204.py310\"\n", "IMAGE_TO_IMAGE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-diffusers-serve-opt:20240605_1400_RC00\"\n", @@ -474,8 +473,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb index 3154effe8..2b1844b07 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb @@ -135,11 +135,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -160,9 +159,8 @@ "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT_CC}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT_CC} --role=roles/storage.admin\n", "\n", "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)\n", "\n", "if \"google.colab\" in sys.modules:\n", @@ -489,8 +487,7 @@ "# Delete bucket.\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb index aaa37e825..8f0e9cf86 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb @@ -589,8 +589,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb b/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb index 2c89ff220..debef7751 100644 --- a/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb +++ b/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb @@ -267,8 +267,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -683,8 +682,7 @@ "delete_bucket = False # set True to delete bucket\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/generative_ai/distillation.ipynb b/notebooks/official/generative_ai/distillation.ipynb index 81671af50..3dcf63add 100644 --- a/notebooks/official/generative_ai/distillation.ipynb +++ b/notebooks/official/generative_ai/distillation.ipynb @@ -323,8 +323,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -347,9 +346,7 @@ }, "outputs": [], "source": [ - "! gsutil cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n", - "! gsutil cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl" - ] + "! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n", "! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl" ] }, { "cell_type": "markdown", @@ -554,8 +551,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = True\n", "if delete_bucket:\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb b/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb index 1c5cf42b2..4bf7d8b1a 100644 --- a/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb +++ b/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb @@ -274,8 +274,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -390,8 +389,7 @@ "outputs": [], "source": [ "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n", - "!gsutil cp abalone_train.csv {BUCKET_URI}/data/\n", - "\n", + "!gcloud storage cp abalone_train.csv {BUCKET_URI}/data/\n", "\n", "gcs_csv_path = f\"{BUCKET_URI}/data/abalone_train.csv\"" ] }, @@ -784,8 +782,7 @@ "\n", "\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb b/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb index 6262b17db..a4c81af1c 100644 --- a/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb +++ b/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb @@ -201,8 +201,7 @@ "source": [ "# Create GCS Bucket\n", "BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n", - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "code", @@ -237,8 +236,7 @@ "outputs": [], "source": [ "# Copy Models to the Bucket\n", - "! gsutil cp -r \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/*\" {BUCKET_URI}" - ] + "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/*\" {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -508,8 +506,7 @@ "outputs": [], "source": [ "# Download the requests files:\n", - "! gsutil cp {BUCKET_URI}/requests/* ./" - ] + "! gcloud storage cp {BUCKET_URI}/requests/* ./" ] }, { "cell_type": "code", @@ -716,8 +713,7 @@ }, "outputs": [], "source": [ - "! gsutil rm -r {BUCKET_URI}" - ] + "! gcloud storage rm --recursive {BUCKET_URI}" ] }, { "cell_type": "markdown", diff --git a/notebooks/official/prediction/llm_streaming_prediction.ipynb b/notebooks/official/prediction/llm_streaming_prediction.ipynb index 7216a0c21..de2ea15f5 100644 --- a/notebooks/official/prediction/llm_streaming_prediction.ipynb +++ b/notebooks/official/prediction/llm_streaming_prediction.ipynb @@ -286,8 +286,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -500,8 +499,7 @@ "source": [ "dataset = \"gs://cloud-samples-data/vertex-ai/prediction/llm/test_table.jsonl\"\n", "destination_uri_prefix = f\"{BUCKET_URI}/text-bison@001_\"\n", - "! gsutil cp -r gs://cloud-samples-data/vertex-ai/prediction/llm/text-bison@001_/ {destination_uri_prefix}\n", - "\n", + "! gcloud storage cp --recursive gs://cloud-samples-data/vertex-ai/prediction/llm/text-bison@001_/ {destination_uri_prefix}\n", "\n", "\n", "from vertexai.language_models import TextGenerationModel\n", "\n", @@ -580,7 +578,7 @@ "outputs": [], "source": [ "# Tuning model\n", - "! gsutil cp gs://cloud-samples-data/vertex-ai/prediction/llm/q_a_train_with_context.jsonl {BUCKET_URI}/q_a_train_with_context.jsonl\n", + "! gcloud storage cp gs://cloud-samples-data/vertex-ai/prediction/llm/q_a_train_with_context.jsonl {BUCKET_URI}/q_a_train_with_context.jsonl\n", "\n", "tuning_job = model3.tune_model(\n", " training_data=f\"{BUCKET_URI}/q_a_train_with_context.jsonl\",\n", @@ -740,8 +738,7 @@ "batch_job_2.delete()\n", "\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From 9a33ec5002a3a99c3b2dd6b13dfaeee342858333 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Wed, 10 Dec 2025 09:56:24 +0000 Subject: [PATCH 2/7] updates --- ..._pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb | 13 ++++++++----- ..._pytorch_stable_diffusion_deployment_1_5.ipynb | 13 ++++++++----- ...able_diffusion_finetuning_dreambooth_1_5.ipynb | 15 +++++++++------ ...diffusion_finetuning_dreambooth_lora_1_5.ipynb | 3 ++- .../sdk_automl_tabular_regression_online_bq.ipynb | 6 ++++-- .../official/generative_ai/distillation.ipynb | 10 +++++++--- ...etric-parameter-tracking-for-custom-jobs.ipynb | 9 ++++++--- .../get_started_with_psc_private_endpoint.ipynb | 12 ++++++++---- .../prediction/llm_streaming_prediction.ipynb | 9 ++++++--- 9 files changed, 58 insertions(+), 32 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb index 2c17f2514..859bf457a 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb @@ -159,10 +159,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - "# Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -188,7 +189,8 @@ "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "# Note: gsutil iam ch does not support modifying IAM policies that contain conditions. gcloud storage commands do support conditions.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -497,7 +499,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb index e1d09cd9f..d7f9fe452 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb @@ -138,9 +138,10 @@ " # Create a unique GCS bucket for this notebook, if not specified by the user\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", - " # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -158,7 +159,8 @@ "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "# The pre-built serving docker image. It contains serving scripts and models.\n", "TEXT_TO_IMAGE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/pytorch-inference.cu125.0-4.ubuntu2204.py310\"\n", "IMAGE_TO_IMAGE_DOCKER_URI = \"us-docker.pkg.dev/vertex-ai/vertex-vision-model-garden-dockers/pytorch-diffusers-serve-opt:20240605_1400_RC00\"\n", @@ -473,7 +475,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb index 2b1844b07..ca3baa51f 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb @@ -135,10 +135,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -159,8 +160,9 @@ "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT_CC} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT_CC} --role=roles/storage.admin\n", + "\n", "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)\n", "\n", "if \"google.colab\" in sys.modules:\n", @@ -487,7 +489,8 @@ "# Delete bucket.\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb index 8f0e9cf86..5f8c2d8de 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb @@ -589,7 +589,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb b/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb index debef7751..75124f4a8 100644 --- a/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb +++ b/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb @@ -267,7 +267,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -682,7 +683,8 @@ "delete_bucket = False # set True to delete bucket\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/generative_ai/distillation.ipynb b/notebooks/official/generative_ai/distillation.ipynb index 3dcf63add..04db0c9ba 100644 --- a/notebooks/official/generative_ai/distillation.ipynb +++ b/notebooks/official/generative_ai/distillation.ipynb @@ -323,7 +323,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -346,7 +347,9 @@ }, "outputs": [], "source": [ - "! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n", "! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl" ] + "! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n", + "! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl" + ] }, { "cell_type": "markdown", @@ -551,7 +554,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = True\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb b/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb index 4bf7d8b1a..c5aa6326e 100644 --- a/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb +++ b/notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb @@ -274,7 +274,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -389,7 +390,8 @@ "outputs": [], "source": [ "!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n", - "!gcloud storage cp abalone_train.csv {BUCKET_URI}/data/\n", "\n", + "!gcloud storage cp abalone_train.csv {BUCKET_URI}/data/\n", + "\n", "gcs_csv_path = f\"{BUCKET_URI}/data/abalone_train.csv\"" ] }, @@ -782,7 +784,8 @@ "\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb b/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb index a4c81af1c..ea44a3fee 100644 --- a/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb +++ b/notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb @@ -201,7 +201,8 @@ "source": [ "# Create GCS Bucket\n", "BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n", - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "code", @@ -236,7 +237,8 @@ "outputs": [], "source": [ "# Copy Models to the Bucket\n", - "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/*\" {BUCKET_URI}" ] + "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/*\" {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -506,7 +508,8 @@ "outputs": [], "source": [ "# Download the requests files:\n", - "! gcloud storage cp {BUCKET_URI}/requests/* ./" ] + "! gcloud storage cp {BUCKET_URI}/requests/* ./" + ] }, { "cell_type": "code", @@ -713,7 +716,8 @@ }, "outputs": [], "source": [ - "! gcloud storage rm --recursive {BUCKET_URI}" ] + "! gcloud storage rm --recursive {BUCKET_URI}" + ] }, { "cell_type": "markdown", diff --git a/notebooks/official/prediction/llm_streaming_prediction.ipynb b/notebooks/official/prediction/llm_streaming_prediction.ipynb index de2ea15f5..5a58f1cf8 100644 --- a/notebooks/official/prediction/llm_streaming_prediction.ipynb +++ b/notebooks/official/prediction/llm_streaming_prediction.ipynb @@ -286,7 +286,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -499,7 +500,8 @@ "source": [ "dataset = \"gs://cloud-samples-data/vertex-ai/prediction/llm/test_table.jsonl\"\n", "destination_uri_prefix = f\"{BUCKET_URI}/text-bison@001_\"\n", - "! gcloud storage cp --recursive gs://cloud-samples-data/vertex-ai/prediction/llm/text-bison@001_/ {destination_uri_prefix}\n", "\n", + "! gcloud storage cp --recursive gs://cloud-samples-data/vertex-ai/prediction/llm/text-bison@001_/ {destination_uri_prefix}\n", + "\n", "\n", "from vertexai.language_models import TextGenerationModel\n", "\n", @@ -738,7 +740,8 @@ "batch_job_2.delete()\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From cd8b895967c83f211e5e432435d6e39baff5235a Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Wed, 10 Dec 2025 12:47:28 +0000 Subject: [PATCH 3/7] update --- .../model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb | 2 -- 1 file changed, 2 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb index 859bf457a..d0fcf52ca 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb @@ -187,8 +187,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "# Note: gsutil iam ch does not support modifying IAM policies that contain conditions. gcloud storage commands do support conditions.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", From 01ccce747583f9b2206c3cd579784bb303ac3f26 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 18 Dec 2025 12:16:09 +0530 Subject: [PATCH 4/7] Update llm_streaming_prediction.ipynb --- notebooks/official/prediction/llm_streaming_prediction.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/official/prediction/llm_streaming_prediction.ipynb b/notebooks/official/prediction/llm_streaming_prediction.ipynb index 5a58f1cf8..24869cc9e 100644 --- a/notebooks/official/prediction/llm_streaming_prediction.ipynb +++ b/notebooks/official/prediction/llm_streaming_prediction.ipynb @@ -735,7 +735,7 @@ "source": [ "delete_bucket = False\n", "\n", - "# Delete job\n", + "# Delete jobs\n", "batch_job_1.delete()\n", "batch_job_2.delete()\n", "\n", From 539df5a176736f8cc4b42f181672502a76b9a94d Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 18 Dec 2025 12:18:38 +0530 Subject: [PATCH 5/7] revert change --- notebooks/official/prediction/llm_streaming_prediction.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/official/prediction/llm_streaming_prediction.ipynb b/notebooks/official/prediction/llm_streaming_prediction.ipynb index 24869cc9e..5a58f1cf8 100644 --- a/notebooks/official/prediction/llm_streaming_prediction.ipynb +++ b/notebooks/official/prediction/llm_streaming_prediction.ipynb @@ -735,7 +735,7 @@ "source": [ "delete_bucket = False\n", "\n", - "# Delete jobs\n", + "# Delete job\n", "batch_job_1.delete()\n", "batch_job_2.delete()\n", "\n", From e9ce8645b35123500f305292ce9aed45551e1d31 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 19 Dec 2025 15:29:58 +0000 Subject: [PATCH 6/7] removed model_garden folder changes --- ...arden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb | 8 ++++---- ...arden_pytorch_stable_diffusion_deployment_1_5.ipynb | 9 ++++----- ...ch_stable_diffusion_finetuning_dreambooth_1_5.ipynb | 10 +++++----- .../sdk_automl_tabular_regression_online_bq.ipynb | 4 ++-- 4 files changed, 15 insertions(+), 16 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb index d0fcf52ca..d7c2d43a6 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_qwen2_5_deployment_tpu.ipynb @@ -159,10 +159,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -187,7 +187,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -497,7 +497,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb index d7f9fe452..87cf9c3b5 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_deployment_1_5.ipynb @@ -138,9 +138,9 @@ " # Create a unique GCS bucket for this notebook, if not specified by the user\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -158,8 +158,7 @@ "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "# The pre-built serving docker image. It contains serving scripts and models.\n", "TEXT_TO_IMAGE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/vertex-model-garden/pytorch-inference.cu125.0-4.ubuntu2204.py310\"\n", @@ -475,7 +474,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb index ca3baa51f..3154effe8 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_1_5.ipynb @@ -135,10 +135,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -160,8 +160,8 @@ "print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT_CC} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT_CC}:roles/storage.admin $BUCKET_NAME\n", "\n", "aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)\n", "\n", @@ -489,7 +489,7 @@ "# Delete bucket.\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], diff --git a/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb b/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb index 75124f4a8..2c89ff220 100644 --- a/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb +++ b/notebooks/official/automl/sdk_automl_tabular_regression_online_bq.ipynb @@ -267,7 +267,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" + "! gsutil mb -l $LOCATION $BUCKET_URI" ] }, { @@ -683,7 +683,7 @@ "delete_bucket = False # set True to delete bucket\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" + " ! gsutil rm -r $BUCKET_URI" ] } ], From ac9c8f3b994ab01697eebdb708968f5aab0d046d Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 19 Dec 2025 15:32:15 +0000 Subject: [PATCH 7/7] removed model_garden folder changes --- ...ytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb index 5f8c2d8de..aaa37e825 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_finetuning_dreambooth_lora_1_5.ipynb @@ -589,7 +589,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ],