From 2a8dfd27919b0440bbb792e39bf62676d680cf45 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Thu, 16 Oct 2025 07:53:15 +0000 Subject: [PATCH 1/6] Migrate gsutil usage to gcloud storage --- .../get_started_with_data_labeling.ipynb | 15 ++++------- .../get_started_with_cmek_training.ipynb | 12 +++------ ...odel_garden_jax_paligemma_finetuning.ipynb | 26 +++++++------------ ...k_automl_image_classification_online.ipynb | 18 +++++-------- ...tabular_binary_classification_online.ipynb | 18 +++++-------- .../find_ideal_machine_type.ipynb | 12 +++------ .../custom-tabular-bq-managed-dataset.ipynb | 6 ++--- ...raining_container_and_model_registry.ipynb | 6 ++--- .../get_started_with_model_registry.ipynb | 6 ++--- ...istributed_training_reduction_server.ipynb | 11 +++----- 10 files changed, 45 insertions(+), 85 deletions(-) diff --git a/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb b/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb index 06c681bc3..23082b520 100644 --- a/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb +++ b/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb @@ -491,8 +491,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -511,8 +510,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -715,8 +713,7 @@ "print(IMPORT_FILE)\n", "\n", "# printing content of uploaded file\n", - "! gsutil cat $IMPORT_FILE" - ] + "! gcloud storage cat $IMPORT_FILE" ] }, { "cell_type": "markdown", @@ -838,8 +835,7 @@ "outputs": [], "source": [ "# create placeholder file for instructions for data labeling\n", - "! echo \"this is instruction\" >> instruction.txt | gsutil cp instruction.txt $BUCKET_URI" - ] +"! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" ] }, { "cell_type": "code", @@ -995,8 +991,7 @@ "\n", "# Delete the bucket created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb b/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb index fe85b853e..0dd3f7d3d 100644 --- a/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb @@ -449,8 +449,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -469,8 +468,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -926,8 +924,7 @@ }, "outputs": [], "source": [ - "test_item = !gsutil cat $IMPORT_FILE | head -n1\n", - "test_item, test_label = str(test_item[0]).split(\",\")\n", + "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item, test_label = str(test_item[0]).split(\",\")\n", "\n", "print(test_item, test_label)" ] @@ -1015,8 +1012,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb index d8c169917..2be492d01 100644 --- a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb @@ -163,11 +163,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -191,8 +189,9 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "# Note: gsutil iam ch does not support modifying IAM policies that contain conditions. gcloud storage commands do support conditions.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -237,11 +236,9 @@ " MODEL_BUCKET,\n", ")\n", "\n", - "! gsutil -m cp -R $VERTEX_AI_MODEL_GARDEN_PALIGEMMA/* $MODEL_BUCKET\n", - "\n", + "! gcloud storage cp --recursive $VERTEX_AI_MODEL_GARDEN_PALIGEMMA/* $MODEL_BUCKET\n", "\n", "assert (\n", - " os.system(f\"gsutil ls {MODEL_BUCKET}\") == 0\n", - "), f\"MODEL_BUCKET does not exist: {MODEL_BUCKET}.\"\n", + " os.system(f\"gcloud storage ls {MODEL_BUCKET}\") == 0\n", "), f\"MODEL_BUCKET does not exist: {MODEL_BUCKET}.\"\n", "model_path_prefix = MODEL_BUCKET" ] }, @@ -461,8 +458,7 @@ "temp_dir = tempfile.TemporaryDirectory()\n", "local_metrics_path = os.path.join(temp_dir.name, metrics_file_name)\n", "\n", - "! gsutil cp $metrics_path $local_metrics_path\n", - "\n", + "! gcloud storage cp $metrics_path $local_metrics_path\n", "\n", "steps = []\n", "training_losses = []\n", "with open(local_metrics_path, \"r\") as f:\n", @@ -507,8 +503,7 @@ "\n", "local_last_checkpoint_path = os.path.join(temp_dir.name, last_checkpoint_file_name)\n", "\n", - "! gsutil cp $last_checkpoint_path $local_last_checkpoint_path\n", - "\n", + "! gcloud storage cp $last_checkpoint_path $local_last_checkpoint_path\n", "\n", "with open(local_last_checkpoint_path, \"r\") as f:\n", " final_checkpoint_name = \"checkpoint.bv-\" + f.read()\n", " checkpoint_path = os.path.join(finetune_output_dir, final_checkpoint_name)\n", @@ -693,8 +688,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_automl_image_classification_online.ipynb b/notebooks/community/sdk/sdk_automl_image_classification_online.ipynb index 5cf707a0f..ae48383db 100644 --- a/notebooks/community/sdk/sdk_automl_image_classification_online.ipynb +++ b/notebooks/community/sdk/sdk_automl_image_classification_online.ipynb @@ -473,8 +473,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -493,8 +492,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -602,12 +600,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -821,8 +817,7 @@ }, "outputs": [], "source": [ - "test_item = !gsutil cat $IMPORT_FILE | head -n1\n", - "if len(str(test_item[0]).split(\",\")) == 3:\n", + "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n", " _, test_item, test_label = str(test_item[0]).split(\",\")\n", "else:\n", " test_item, test_label = str(test_item[0]).split(\",\")\n", @@ -990,8 +985,7 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online.ipynb b/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online.ipynb index 158509aae..7ac46b5d2 100644 --- a/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online.ipynb +++ b/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online.ipynb @@ -473,8 +473,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -493,8 +492,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -597,14 +595,11 @@ }, "outputs": [], "source": [ - "count = ! gsutil cat $IMPORT_FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $IMPORT_FILE | head\n", - "\n", - "heading = ! gsutil cat $IMPORT_FILE | head -n1\n", - "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -996,8 +991,7 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type.ipynb b/notebooks/community/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type.ipynb index 30e080f64..f67485593 100644 --- a/notebooks/community/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type.ipynb +++ b/notebooks/community/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type.ipynb @@ -355,8 +355,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -p $PROJECT_ID -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --project=$PROJECT_ID --location=$REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -375,8 +374,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -596,8 +594,7 @@ }, "outputs": [], "source": [ - "!sudo gsutil cp -r ./bert_sentence_embedding/00001/* $GCS_URI/1/" - ] + "!sudo gcloud storage cp --recursive ./bert_sentence_embedding/00001/* $GCS_URI/1/" ] }, { "cell_type": "markdown", @@ -1045,8 +1042,7 @@ }, "outputs": [], "source": [ - "!gsutil rm -r $GCS_URI/*" - ] + "!gcloud storage rm --recursive $GCS_URI/*" ] }, { "cell_type": "markdown", diff --git a/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb b/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb index 1122bdfcd..bbd9ddd9c 100644 --- a/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb +++ b/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb @@ -291,8 +291,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -911,8 +910,7 @@ "delete_bucket = True\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/custom/custom_training_container_and_model_registry.ipynb b/notebooks/official/custom/custom_training_container_and_model_registry.ipynb index 6f9c44176..516ffa0cf 100644 --- a/notebooks/official/custom/custom_training_container_and_model_registry.ipynb +++ b/notebooks/official/custom/custom_training_container_and_model_registry.ipynb @@ -283,8 +283,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -932,8 +931,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = True\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "# Delete application directory\n", "!rm -rf $APPLICATION_DIR" ] diff --git a/notebooks/official/model_registry/get_started_with_model_registry.ipynb b/notebooks/official/model_registry/get_started_with_model_registry.ipynb index 06e3ad3d8..e2062c0f0 100644 --- a/notebooks/official/model_registry/get_started_with_model_registry.ipynb +++ b/notebooks/official/model_registry/get_started_with_model_registry.ipynb @@ -303,8 +303,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -996,8 +995,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -rf {BUCKET_URI}\n", - "\n", + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n", "# Undeploy the models and delete the endpoint resources\n", "try:\n", " endpoint.undeploy_all()\n", diff --git a/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb b/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb index 7fe5ee5d0..b0c2def28 100644 --- a/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb +++ b/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb @@ -291,8 +291,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -720,9 +719,8 @@ "source": [ "! cd {PYTHON_PACKAGE_APPLICATION_DIR} && python3 setup.py sdist --formats=gztar\n", "\n", - "! gsutil cp {source_package_file_name} {python_package_gcs_uri}\n", - "\n", - "! gsutil ls -l {python_package_gcs_uri}" + "! gcloud storage cp {source_package_file_name} {python_package_gcs_uri}\n", "\n", + "! gcloud storage ls --long {python_package_gcs_uri}" ] }, { @@ -933,8 +931,7 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From 0fb7624d1a68acf17301fe685690725361e43899 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 12 Dec 2025 09:48:15 +0000 Subject: [PATCH 2/6] changes for 4308 --- ...odel_garden_jax_paligemma_finetuning.ipynb | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb index 2be492d01..2d3013541 100644 --- a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb @@ -28,7 +28,6 @@ { "cell_type": "markdown", "id": "VJWDivOv3OWy", - "language": "markdown", "metadata": { "id": "VJWDivOv3OWy" }, @@ -163,9 +162,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -189,9 +190,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "# Note: gsutil iam ch does not support modifying IAM policies that contain conditions. gcloud storage commands do support conditions.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -236,9 +236,11 @@ " MODEL_BUCKET,\n", ")\n", "\n", - "! gcloud storage cp --recursive $VERTEX_AI_MODEL_GARDEN_PALIGEMMA/* $MODEL_BUCKET\n", "\n", + "! gcloud storage cp --recursive $VERTEX_AI_MODEL_GARDEN_PALIGEMMA/* $MODEL_BUCKET\n", + "\n", "assert (\n", - " os.system(f\"gcloud storage ls {MODEL_BUCKET}\") == 0\n", "), f\"MODEL_BUCKET does not exist: {MODEL_BUCKET}.\"\n", + " os.system(f\"gcloud storage ls {MODEL_BUCKET}\") == 0\n", + "), f\"MODEL_BUCKET does not exist: {MODEL_BUCKET}.\"\n", "model_path_prefix = MODEL_BUCKET" ] }, @@ -458,7 +460,8 @@ "temp_dir = tempfile.TemporaryDirectory()\n", "local_metrics_path = os.path.join(temp_dir.name, metrics_file_name)\n", "\n", - "! gcloud storage cp $metrics_path $local_metrics_path\n", "\n", + "! gcloud storage cp $metrics_path $local_metrics_path\n", + "\n", "steps = []\n", "training_losses = []\n", "with open(local_metrics_path, \"r\") as f:\n", @@ -503,7 +506,8 @@ "\n", "local_last_checkpoint_path = os.path.join(temp_dir.name, last_checkpoint_file_name)\n", "\n", - "! gcloud storage cp $last_checkpoint_path $local_last_checkpoint_path\n", "\n", + "! gcloud storage cp $last_checkpoint_path $local_last_checkpoint_path\n", + "\n", "with open(local_last_checkpoint_path, \"r\") as f:\n", " final_checkpoint_name = \"checkpoint.bv-\" + f.read()\n", " checkpoint_path = os.path.join(finetune_output_dir, final_checkpoint_name)\n", @@ -688,7 +692,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { From e2a83abe6b6732a12ed005a328d36669bb72f27b Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 12 Dec 2025 10:07:02 +0000 Subject: [PATCH 3/6] added = in command --- ..._tabular_binary_classification_online.ipynb | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online.ipynb b/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online.ipynb index 7ac46b5d2..e1b0dc37f 100644 --- a/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online.ipynb +++ b/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online.ipynb @@ -473,7 +473,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -492,7 +493,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -595,11 +597,14 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $IMPORT_FILE | head\n", "\n", - "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", + "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -991,7 +996,8 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { From 3a2c468c5694eb67805fc3817f46dc67cf76fabc Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 16 Dec 2025 10:36:46 +0000 Subject: [PATCH 4/6] Apply automated linter fixes --- .../get_started_with_data_labeling.ipynb | 15 ++++++++++----- .../get_started_with_cmek_training.ipynb | 12 ++++++++---- ...model_garden_jax_paligemma_finetuning.ipynb | 8 +++++--- ...dk_automl_image_classification_online.ipynb | 18 ++++++++++++------ .../find_ideal_machine_type.ipynb | 12 ++++++++---- .../custom-tabular-bq-managed-dataset.ipynb | 6 ++++-- ...training_container_and_model_registry.ipynb | 6 ++++-- .../get_started_with_model_registry.ipynb | 6 ++++-- ...distributed_training_reduction_server.ipynb | 9 ++++++--- 9 files changed, 61 insertions(+), 31 deletions(-) diff --git a/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb b/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb index 23082b520..82cb0aaff 100644 --- a/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb +++ b/notebooks/community/ml_ops/stage1/get_started_with_data_labeling.ipynb @@ -491,7 +491,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -510,7 +511,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -713,7 +715,8 @@ "print(IMPORT_FILE)\n", "\n", "# printing content of uploaded file\n", - "! gcloud storage cat $IMPORT_FILE" ] + "! gcloud storage cat $IMPORT_FILE" + ] }, { "cell_type": "markdown", @@ -835,7 +838,8 @@ "outputs": [], "source": [ "# create placeholder file for instructions for data labeling\n", -"! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" ] + "! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" + ] }, { "cell_type": "code", @@ -991,7 +995,8 @@ "\n", "# Delete the bucket created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb b/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb index 0dd3f7d3d..46e730f63 100644 --- a/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_with_cmek_training.ipynb @@ -449,7 +449,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -468,7 +469,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -924,7 +926,8 @@ }, "outputs": [], "source": [ - "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item, test_label = str(test_item[0]).split(\",\")\n", + "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", + "test_item, test_label = str(test_item[0]).split(\",\")\n", "\n", "print(test_item, test_label)" ] @@ -1012,7 +1015,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb index 2d3013541..4df507866 100644 --- a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb @@ -464,7 +464,7 @@ "\n", "steps = []\n", "training_losses = []\n", - "with open(local_metrics_path, \"r\") as f:\n", + "with open(local_metrics_path) as f:\n", " for line in f:\n", " metric = json.loads(line)\n", " steps.append(metric[\"step\"])\n", @@ -508,7 +508,7 @@ "\n", "! gcloud storage cp $last_checkpoint_path $local_last_checkpoint_path\n", "\n", - "with open(local_last_checkpoint_path, \"r\") as f:\n", + "with open(local_last_checkpoint_path) as f:\n", " final_checkpoint_name = \"checkpoint.bv-\" + f.read()\n", " checkpoint_path = os.path.join(finetune_output_dir, final_checkpoint_name)\n", "\n", @@ -632,7 +632,9 @@ "\n", "# @markdown \n", "\n", - "image_url = \"https://storage.googleapis.com/longcap100/91.jpeg\" # @param {type:\"string\"}\n", + "image_url = (\n", + " \"https://storage.googleapis.com/longcap100/91.jpeg\" # @param {type:\"string\"}\n", + ")\n", "\n", "language_code = \"en\" # @param {type: \"string\"}\n", "\n", diff --git a/notebooks/community/sdk/sdk_automl_image_classification_online.ipynb b/notebooks/community/sdk/sdk_automl_image_classification_online.ipynb index ae48383db..b2f956db9 100644 --- a/notebooks/community/sdk/sdk_automl_image_classification_online.ipynb +++ b/notebooks/community/sdk/sdk_automl_image_classification_online.ipynb @@ -473,7 +473,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -492,7 +493,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -600,10 +602,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -817,7 +821,8 @@ }, "outputs": [], "source": [ - "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n", + "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", + "if len(str(test_item[0]).split(\",\")) == 3:\n", " _, test_item, test_label = str(test_item[0]).split(\",\")\n", "else:\n", " test_item, test_label = str(test_item[0]).split(\",\")\n", @@ -985,7 +990,8 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type.ipynb b/notebooks/community/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type.ipynb index f67485593..ca95aa23f 100644 --- a/notebooks/community/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type.ipynb +++ b/notebooks/community/vertex_endpoints/find_ideal_machine_type/find_ideal_machine_type.ipynb @@ -355,7 +355,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --project=$PROJECT_ID --location=$REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --project=$PROJECT_ID --location=$REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -374,7 +375,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -594,7 +596,8 @@ }, "outputs": [], "source": [ - "!sudo gcloud storage cp --recursive ./bert_sentence_embedding/00001/* $GCS_URI/1/" ] + "!sudo gcloud storage cp --recursive ./bert_sentence_embedding/00001/* $GCS_URI/1/" + ] }, { "cell_type": "markdown", @@ -1042,7 +1045,8 @@ }, "outputs": [], "source": [ - "!gcloud storage rm --recursive $GCS_URI/*" ] + "!gcloud storage rm --recursive $GCS_URI/*" + ] }, { "cell_type": "markdown", diff --git a/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb b/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb index bbd9ddd9c..83ff7253f 100644 --- a/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb +++ b/notebooks/official/custom/custom-tabular-bq-managed-dataset.ipynb @@ -291,7 +291,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -910,7 +911,8 @@ "delete_bucket = True\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/custom/custom_training_container_and_model_registry.ipynb b/notebooks/official/custom/custom_training_container_and_model_registry.ipynb index 516ffa0cf..633938829 100644 --- a/notebooks/official/custom/custom_training_container_and_model_registry.ipynb +++ b/notebooks/official/custom/custom_training_container_and_model_registry.ipynb @@ -283,7 +283,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -931,7 +932,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = True\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "# Delete application directory\n", "!rm -rf $APPLICATION_DIR" ] diff --git a/notebooks/official/model_registry/get_started_with_model_registry.ipynb b/notebooks/official/model_registry/get_started_with_model_registry.ipynb index e2062c0f0..8aefc922f 100644 --- a/notebooks/official/model_registry/get_started_with_model_registry.ipynb +++ b/notebooks/official/model_registry/get_started_with_model_registry.ipynb @@ -303,7 +303,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -995,7 +996,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n", + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", + "\n", "# Undeploy the models and delete the endpoint resources\n", "try:\n", " endpoint.undeploy_all()\n", diff --git a/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb b/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb index b0c2def28..836fab426 100644 --- a/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb +++ b/notebooks/official/reduction_server/pytorch_distributed_training_reduction_server.ipynb @@ -291,7 +291,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -719,7 +720,8 @@ "source": [ "! cd {PYTHON_PACKAGE_APPLICATION_DIR} && python3 setup.py sdist --formats=gztar\n", "\n", - "! gcloud storage cp {source_package_file_name} {python_package_gcs_uri}\n", "\n", + "! gcloud storage cp {source_package_file_name} {python_package_gcs_uri}\n", + "\n", "! gcloud storage ls --long {python_package_gcs_uri}" ] }, @@ -931,7 +933,8 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From 1d77a0f9d5615454eb35203357d0e4ace2ea2499 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 10:39:52 +0000 Subject: [PATCH 5/6] removed model_garden folder changes --- .../model_garden_jax_paligemma_finetuning.ipynb | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb index 4df507866..a7bb8097f 100644 --- a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb @@ -28,6 +28,7 @@ { "cell_type": "markdown", "id": "VJWDivOv3OWy", + "language": "markdown", "metadata": { "id": "VJWDivOv3OWy" }, @@ -162,10 +163,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -190,7 +191,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -236,10 +237,10 @@ " MODEL_BUCKET,\n", ")\n", "\n", - "! gcloud storage cp --recursive $VERTEX_AI_MODEL_GARDEN_PALIGEMMA/* $MODEL_BUCKET\n", + "! gsutil -m cp -R $VERTEX_AI_MODEL_GARDEN_PALIGEMMA/* $MODEL_BUCKET\n", "\n", "assert (\n", - " os.system(f\"gcloud storage ls {MODEL_BUCKET}\") == 0\n", + " os.system(f\"gsutil ls {MODEL_BUCKET}\") == 0\n", "), f\"MODEL_BUCKET does not exist: {MODEL_BUCKET}.\"\n", "model_path_prefix = MODEL_BUCKET" ] From 750f261570a01c2f10d0dc10ba3e045237e9114e Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 10:44:00 +0000 Subject: [PATCH 6/6] removed model_garden folder changes --- .../model_garden_jax_paligemma_finetuning.ipynb | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb index a7bb8097f..d8c169917 100644 --- a/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_paligemma_finetuning.ipynb @@ -461,11 +461,11 @@ "temp_dir = tempfile.TemporaryDirectory()\n", "local_metrics_path = os.path.join(temp_dir.name, metrics_file_name)\n", "\n", - "! gcloud storage cp $metrics_path $local_metrics_path\n", + "! gsutil cp $metrics_path $local_metrics_path\n", "\n", "steps = []\n", "training_losses = []\n", - "with open(local_metrics_path) as f:\n", + "with open(local_metrics_path, \"r\") as f:\n", " for line in f:\n", " metric = json.loads(line)\n", " steps.append(metric[\"step\"])\n", @@ -507,9 +507,9 @@ "\n", "local_last_checkpoint_path = os.path.join(temp_dir.name, last_checkpoint_file_name)\n", "\n", - "! gcloud storage cp $last_checkpoint_path $local_last_checkpoint_path\n", + "! gsutil cp $last_checkpoint_path $local_last_checkpoint_path\n", "\n", - "with open(local_last_checkpoint_path) as f:\n", + "with open(local_last_checkpoint_path, \"r\") as f:\n", " final_checkpoint_name = \"checkpoint.bv-\" + f.read()\n", " checkpoint_path = os.path.join(finetune_output_dir, final_checkpoint_name)\n", "\n", @@ -633,9 +633,7 @@ "\n", "# @markdown \n", "\n", - "image_url = (\n", - " \"https://storage.googleapis.com/longcap100/91.jpeg\" # @param {type:\"string\"}\n", - ")\n", + "image_url = \"https://storage.googleapis.com/longcap100/91.jpeg\" # @param {type:\"string\"}\n", "\n", "language_code = \"en\" # @param {type: \"string\"}\n", "\n", @@ -695,7 +693,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ],