From d38ac1cec8fe9a2b32bd8f52febb8a22a39e54f9 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Fri, 17 Oct 2025 12:46:38 +0000 Subject: [PATCH 01/10] Migrate gsutil usage to gcloud storage --- .../stage2/get_started_with_tabnet.ipynb | 12 +++---- ...started_with_hpt_pipeline_components.ipynb | 34 ++++++++----------- ...d_with_vertex_endpoint_and_shared_vm.ipynb | 11 +++--- ...tarted_with_custom_image_model_batch.ipynb | 12 +++---- ...l_garden_gemma2_finetuning_on_vertex.ipynb | 16 +++------ .../model_garden_pytorch_detectron2.ipynb | 14 ++++---- ...l_garden_pytorch_llama3_3_finetuning.ipynb | 18 ++++------ .../SDK_Custom_Predict_SDK_Integration.ipynb | 13 +++---- ...prediction_feature_store_integration.ipynb | 18 ++++------ .../official/workbench/spark/spark_ml.ipynb | 9 ++--- 10 files changed, 57 insertions(+), 100 deletions(-) diff --git a/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb b/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb index 4742266e0..98f188ee2 100644 --- a/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb @@ -467,8 +467,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -487,8 +486,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -711,8 +709,7 @@ "IMPORT_FILE = \"petfinder-tabular-classification-tabnet-with-header.csv\"\n", "TRAINING_DATA_PATH = f\"{BUCKET_URI}/data/petfinder/train.csv\"\n", "\n", - "! gsutil cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}" - ] + "! gcloud storage cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}" ] }, { "cell_type": "markdown", @@ -1422,8 +1419,7 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb index 34df1ad4e..739ac35f6 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb @@ -441,8 +441,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create $BUCKET_URI --location=$REGION" ] }, { "cell_type": "markdown", @@ -461,8 +460,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -531,10 +529,12 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + # Note: Conditions: gsutil iam ch does not support modifying IAM policies that contain conditions. gcloud storage commands do support conditions. + # Note: File Argument Order: In gsutil iam set, the policy file comes before the URLs. In gcloud storage bucket set-iam-policy, the policy file comes after the URL. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -1006,8 +1006,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz" ] }, { "cell_type": "markdown", @@ -1433,14 +1432,11 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gsutil cat $EXECUTE_OUTPUT\n", - " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gsutil cat $GCP_RESOURCES\n", - " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gsutil cat $EVAL_METRICS\n", - " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1454,8 +1450,7 @@ "print(\"getbesttrialop\")\n", "artifacts = print_pipeline_output(pipeline, \"getbesttrialop\")\n", "print(\"\\n\\n\")\n", - "output = !gsutil cat $artifacts\n", - "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "best_trial = json.loads(output[\"parameters\"][\"Output\"][\"stringValue\"])\n", "model_id = best_trial[\"id\"]\n", "print(\"BEST MODEL\", model_id)\n", @@ -1529,8 +1524,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb b/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb index b2677c56d..badfa31c5 100644 --- a/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb +++ b/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb @@ -449,8 +449,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -469,8 +468,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1374,7 +1372,7 @@ }, "outputs": [], "source": [ - "! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg\n", + "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg\n", "\n", "import base64\n", "\n", @@ -1574,8 +1572,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "!rm -f test.jpg" ] } diff --git a/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb index f4e4b849e..4c1e0ff95 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb @@ -455,8 +455,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -475,8 +474,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -927,8 +925,7 @@ " f.write(json.dumps(x.tolist()) + \"\\n\")\n", "\n", "# Upload to Cloud Storage bucket\n", - "! gsutil cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", - "\n", + "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", "\n", "print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)" ] }, @@ -1445,8 +1442,7 @@ " batch_prediction_job.delete()\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb index b0a878ce1..f3c5c851e 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb @@ -193,11 +193,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -221,8 +219,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -358,9 +355,7 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", - " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -1064,8 +1059,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb index 75b7dff31..c72cae4ff 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb @@ -201,11 +201,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + "# Note: -L (Full Listing) Output: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -229,8 +228,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -1215,8 +1214,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb index cc81473dc..4767f6883 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb @@ -185,11 +185,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -213,8 +212,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -395,9 +394,7 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", - " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -1084,8 +1081,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb index baee931d6..6433ec335 100644 --- a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb +++ b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb @@ -496,8 +496,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -516,8 +515,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -687,9 +685,7 @@ "outputs": [], "source": [ "%cd ..\n", - "!gsutil cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", - "!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" - ] + "!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] }, { "cell_type": "markdown", @@ -1792,8 +1788,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb b/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb index 54e6135fc..ced7eb02a 100644 --- a/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb +++ b/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb @@ -451,8 +451,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "code", @@ -462,8 +461,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -493,8 +491,7 @@ }, "outputs": [], "source": [ - "! gsutil cp -r gs://featurestore_integration/model/* $ARTIFACT_URI" - ] + "! gcloud storage cp --recursive gs://featurestore_integration/model/* $ARTIFACT_URI" ] }, { "cell_type": "markdown", @@ -855,8 +852,7 @@ }, "outputs": [], "source": [ - "! gsutil cp gs://featurestore_integration/data/mobile_gaming_dataset.csv $SOURCE_URI" - ] + "! gcloud storage cp gs://featurestore_integration/data/mobile_gaming_dataset.csv $SOURCE_URI" ] }, { "cell_type": "code", @@ -1198,8 +1194,7 @@ }, "outputs": [], "source": [ - "!gsutil cp prediction_featurestore_fetch_config.yaml $ARTIFACT_URI" - ] + "!gcloud storage cp prediction_featurestore_fetch_config.yaml $ARTIFACT_URI" ] }, { "cell_type": "markdown", @@ -1731,8 +1726,7 @@ "outputs": [], "source": [ "# Delete bucket\n", - "!gsutil -m rm -r $BUCKET_URI" - ] + "!gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/workbench/spark/spark_ml.ipynb b/notebooks/official/workbench/spark/spark_ml.ipynb index de300857b..4af188288 100644 --- a/notebooks/official/workbench/spark/spark_ml.ipynb +++ b/notebooks/official/workbench/spark/spark_ml.ipynb @@ -383,8 +383,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -1424,8 +1423,7 @@ " ! rm -rf $BUCKET_FOLDER\n", " ! mkdir $BUCKET_FOLDER\n", " bike_gbt_model.write().overwrite().save(f\"{BUCKET_FOLDER}\")\n", - " ! gsutil cp -r $BUCKET_FOLDER $BUCKET_URI\n", - "else:\n", + " ! gcloud storage cp --recursive $BUCKET_FOLDER $BUCKET_URI\n", "else:\n", " bike_gbt_model.write().overwrite().save(f\"{BUCKET_URI}/\")" ] }, @@ -1460,8 +1458,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] }, { "cell_type": "markdown", From d8571f88d4f0d5114fbb089e6e79c2d1115810d9 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 12 Dec 2025 08:10:39 +0000 Subject: [PATCH 02/10] Manual Changes --- .../stage3/get_started_with_hpt_pipeline_components.ipynb | 5 +---- .../model_garden/model_garden_pytorch_detectron2.ipynb | 1 - .../model_garden_pytorch_llama3_3_finetuning.ipynb | 2 -- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb index 739ac35f6..3e28b3b40 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb @@ -529,11 +529,8 @@ }, "outputs": [], "source": [ - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. - # Note: Conditions: gsutil iam ch does not support modifying IAM policies that contain conditions. gcloud storage commands do support conditions. - # Note: File Argument Order: In gsutil iam set, the policy file comes before the URLs. In gcloud storage bucket set-iam-policy, the policy file comes after the URL. "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { diff --git a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb index c72cae4ff..69bb5d25e 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb @@ -203,7 +203,6 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - "# Note: -L (Full Listing) Output: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is \"None\".\n", " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb index 4767f6883..039907984 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb @@ -187,7 +187,6 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -212,7 +211,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", From 8ca9b56c5b7d84ebb8510dd1fdc1205a034184d3 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 12 Dec 2025 13:05:24 +0000 Subject: [PATCH 03/10] Manual Changes --- .../stage3/get_started_with_hpt_pipeline_components.ipynb | 1 - .../model_garden_gemma2_finetuning_on_vertex.ipynb | 2 +- .../model_garden_gemma_deployment_on_vertex.ipynb | 2 +- .../model_garden/model_garden_integration_with_agent.ipynb | 7 +++---- .../model_garden/model_garden_pytorch_detectron2.ipynb | 3 +-- .../model_garden_pytorch_llama3_3_finetuning.ipynb | 2 +- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb index 3e28b3b40..6f3001cfe 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb @@ -529,7 +529,6 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, diff --git a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb index f3c5c851e..258b4c894 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb @@ -195,7 +195,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", diff --git a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb index 4c04a6733..28fa29701 100644 --- a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb @@ -180,7 +180,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", diff --git a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb index 80548a4f5..014c7e6c1 100644 --- a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb +++ b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb @@ -177,10 +177,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -202,8 +202,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb index 69bb5d25e..85ab31993 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb @@ -203,7 +203,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -227,7 +227,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb index 039907984..c00abfd2f 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb @@ -187,7 +187,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", From a1ed9c6f59a543c4010579c3134dc35614092c3b Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Wed, 17 Dec 2025 12:51:02 +0000 Subject: [PATCH 04/10] Linter fiex the issues for 4329 --- .../stage2/get_started_with_tabnet.ipynb | 12 +++++--- ...started_with_hpt_pipeline_components.ipynb | 30 ++++++++++++------- ...d_with_vertex_endpoint_and_shared_vm.ipynb | 17 +++++++---- ...tarted_with_custom_image_model_batch.ipynb | 12 +++++--- ...l_garden_gemma2_finetuning_on_vertex.ipynb | 20 +++++++++---- ...el_garden_gemma_deployment_on_vertex.ipynb | 6 ++-- .../model_garden_integration_with_agent.ipynb | 8 +++-- .../model_garden_pytorch_detectron2.ipynb | 13 +++++--- ...l_garden_pytorch_llama3_3_finetuning.ipynb | 24 ++++++++++----- .../SDK_Custom_Predict_SDK_Integration.ipynb | 13 +++++--- ...prediction_feature_store_integration.ipynb | 18 +++++++---- .../official/workbench/spark/spark_ml.ipynb | 9 ++++-- 12 files changed, 124 insertions(+), 58 deletions(-) diff --git a/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb b/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb index 98f188ee2..34e75f27d 100644 --- a/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb @@ -467,7 +467,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -486,7 +487,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -709,7 +711,8 @@ "IMPORT_FILE = \"petfinder-tabular-classification-tabnet-with-header.csv\"\n", "TRAINING_DATA_PATH = f\"{BUCKET_URI}/data/petfinder/train.csv\"\n", "\n", - "! gcloud storage cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}" ] + "! gcloud storage cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}" + ] }, { "cell_type": "markdown", @@ -1419,7 +1422,8 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb index 6f3001cfe..1e054045d 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb @@ -441,7 +441,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create $BUCKET_URI --location=$REGION" ] + "! gcloud storage buckets create $BUCKET_URI --location=$REGION" + ] }, { "cell_type": "markdown", @@ -460,7 +461,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -529,8 +531,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -1002,7 +1006,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz" ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz" + ] }, { "cell_type": "markdown", @@ -1428,11 +1433,14 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", + " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", + " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", + " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1446,7 +1454,8 @@ "print(\"getbesttrialop\")\n", "artifacts = print_pipeline_output(pipeline, \"getbesttrialop\")\n", "print(\"\\n\\n\")\n", - "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", + "output = json.loads(output[0])\n", "best_trial = json.loads(output[\"parameters\"][\"Output\"][\"stringValue\"])\n", "model_id = best_trial[\"id\"]\n", "print(\"BEST MODEL\", model_id)\n", @@ -1520,7 +1529,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb b/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb index badfa31c5..5716e02d1 100644 --- a/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb +++ b/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb @@ -449,7 +449,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -468,7 +469,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -1027,8 +1029,12 @@ "deployment_resource_pool.dedicated_resources.max_replica_count = MAX_NODES\n", "deployment_resource_pool.dedicated_resources.machine_spec.machine_type = DEPLOY_COMPUTE\n", "if DEPLOY_NGPU:\n", - " deployment_resource_pool.dedicated_resources.machine_spec.accelerator_type = DEPLOY_GPU\n", - " deployment_resource_pool.dedicated_resources.machine_spec.accelerator_count = DEPLOY_NGPU\n", + " deployment_resource_pool.dedicated_resources.machine_spec.accelerator_type = (\n", + " DEPLOY_GPU\n", + " )\n", + " deployment_resource_pool.dedicated_resources.machine_spec.accelerator_count = (\n", + " DEPLOY_NGPU\n", + " )\n", "\n", "request = aip_beta.CreateDeploymentResourcePoolRequest(\n", " parent=f\"projects/{PROJECT_ID}/locations/{REGION}\",\n", @@ -1572,7 +1578,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "!rm -f test.jpg" ] } diff --git a/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb index 4c1e0ff95..8fedb12d4 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb @@ -455,7 +455,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -474,7 +475,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -925,7 +927,8 @@ " f.write(json.dumps(x.tolist()) + \"\\n\")\n", "\n", "# Upload to Cloud Storage bucket\n", - "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", "\n", + "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", + "\n", "print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)" ] }, @@ -1442,7 +1445,8 @@ " batch_prediction_job.delete()\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb index 258b4c894..724c56a92 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb @@ -193,9 +193,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -219,7 +221,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -355,7 +358,9 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -407,7 +412,9 @@ "# @markdown 1. If `max_steps > 0`, it takes precedence over `epochs`. One can set a small `max_steps` value to quickly check the pipeline.\n", "\n", "# @markdown Accelerator type to use for training.\n", - "training_accelerator_type = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", + "training_accelerator_type = (\n", + " \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", + ")\n", "\n", "# The pre-built training docker image.\n", "if training_accelerator_type == \"NVIDIA_A100_80GB\":\n", @@ -1059,7 +1066,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb index 28fa29701..f91022ecd 100644 --- a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb @@ -180,7 +180,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -194,7 +194,9 @@ "# @markdown Alternatively, you can also load the original Gemma models for serving from Vertex AI after accepting the agreement.\n", "\n", "# @markdown **Select and fill one of the two following sections.**\n", - "LOAD_MODEL_FROM = \"Hugging Face\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + "LOAD_MODEL_FROM = (\n", + " \"Hugging Face\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + ")\n", "\n", "# @markdown ---\n", "\n", diff --git a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb index 014c7e6c1..fb8839b4b 100644 --- a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb +++ b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb @@ -177,10 +177,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -202,7 +203,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb index 85ab31993..06649bc8b 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb @@ -201,9 +201,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -227,7 +229,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -796,6 +799,7 @@ "\n", "# @markdown In case you are using your own dataset, kindly convert it to [COCO format](https://opencv.org/introduction-to-the-coco-dataset/).\n", "\n", + "\n", "!wget https://github.com/matterport/Mask_RCNN/releases/download/v2.1/balloon_dataset.zip\n", "!unzip balloon_dataset.zip > /dev/null\n", "\n", @@ -1212,7 +1216,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb index c00abfd2f..bfb5aeed7 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb @@ -185,9 +185,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -211,7 +213,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -235,7 +238,9 @@ "# @markdown Alternatively, you can also load the original Llama 3.3 models for finetuning and serving from Vertex AI after accepting the agreement.\n", "\n", "# @markdown **Only select and fill one of the following sections.**\n", - "LOAD_MODEL_FROM = \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + "LOAD_MODEL_FROM = (\n", + " \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + ")\n", "\n", "# @markdown ---\n", "\n", @@ -392,7 +397,9 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -444,7 +451,9 @@ "# @markdown 1. If `max_steps>0`, it takes precedence over `epochs`. One can set a small `max_steps` value to quickly check the pipeline.\n", "\n", "# @markdown Acceletor type to use for training.\n", - "training_accelerator_type = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", + "training_accelerator_type = (\n", + " \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", + ")\n", "\n", "# The pre-built training docker image.\n", "if training_accelerator_type == \"NVIDIA_A100_80GB\":\n", @@ -1079,7 +1088,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb index 6433ec335..63801fe48 100644 --- a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb +++ b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb @@ -496,7 +496,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -515,7 +516,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -685,7 +687,9 @@ "outputs": [], "source": [ "%cd ..\n", - "!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] + "!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", + "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" + ] }, { "cell_type": "markdown", @@ -1788,7 +1792,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb b/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb index ced7eb02a..fb658ff5e 100644 --- a/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb +++ b/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb @@ -451,7 +451,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "code", @@ -461,7 +462,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -491,7 +493,8 @@ }, "outputs": [], "source": [ - "! gcloud storage cp --recursive gs://featurestore_integration/model/* $ARTIFACT_URI" ] + "! gcloud storage cp --recursive gs://featurestore_integration/model/* $ARTIFACT_URI" + ] }, { "cell_type": "markdown", @@ -852,7 +855,8 @@ }, "outputs": [], "source": [ - "! gcloud storage cp gs://featurestore_integration/data/mobile_gaming_dataset.csv $SOURCE_URI" ] + "! gcloud storage cp gs://featurestore_integration/data/mobile_gaming_dataset.csv $SOURCE_URI" + ] }, { "cell_type": "code", @@ -1194,7 +1198,8 @@ }, "outputs": [], "source": [ - "!gcloud storage cp prediction_featurestore_fetch_config.yaml $ARTIFACT_URI" ] + "!gcloud storage cp prediction_featurestore_fetch_config.yaml $ARTIFACT_URI" + ] }, { "cell_type": "markdown", @@ -1726,7 +1731,8 @@ "outputs": [], "source": [ "# Delete bucket\n", - "!gcloud storage rm --recursive $BUCKET_URI" ] + "!gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/workbench/spark/spark_ml.ipynb b/notebooks/official/workbench/spark/spark_ml.ipynb index 4af188288..695b3334c 100644 --- a/notebooks/official/workbench/spark/spark_ml.ipynb +++ b/notebooks/official/workbench/spark/spark_ml.ipynb @@ -383,7 +383,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -1423,7 +1424,8 @@ " ! rm -rf $BUCKET_FOLDER\n", " ! mkdir $BUCKET_FOLDER\n", " bike_gbt_model.write().overwrite().save(f\"{BUCKET_FOLDER}\")\n", - " ! gcloud storage cp --recursive $BUCKET_FOLDER $BUCKET_URI\n", "else:\n", + " ! gcloud storage cp --recursive $BUCKET_FOLDER $BUCKET_URI\n", + "else:\n", " bike_gbt_model.write().overwrite().save(f\"{BUCKET_URI}/\")" ] }, @@ -1458,7 +1460,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] }, { "cell_type": "markdown", From a7ab35008129db39b2e8aa2c60c0488087f3f252 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Wed, 17 Dec 2025 14:34:04 +0000 Subject: [PATCH 05/10] Revert "Linter fiex the issues for 4329" This reverts commit a1ed9c6f59a543c4010579c3134dc35614092c3b. --- .../stage2/get_started_with_tabnet.ipynb | 12 +++----- ...started_with_hpt_pipeline_components.ipynb | 30 +++++++------------ ...d_with_vertex_endpoint_and_shared_vm.ipynb | 17 ++++------- ...tarted_with_custom_image_model_batch.ipynb | 12 +++----- ...l_garden_gemma2_finetuning_on_vertex.ipynb | 20 ++++--------- ...el_garden_gemma_deployment_on_vertex.ipynb | 6 ++-- .../model_garden_integration_with_agent.ipynb | 8 ++--- .../model_garden_pytorch_detectron2.ipynb | 13 +++----- ...l_garden_pytorch_llama3_3_finetuning.ipynb | 24 +++++---------- .../SDK_Custom_Predict_SDK_Integration.ipynb | 13 +++----- ...prediction_feature_store_integration.ipynb | 18 ++++------- .../official/workbench/spark/spark_ml.ipynb | 9 ++---- 12 files changed, 58 insertions(+), 124 deletions(-) diff --git a/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb b/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb index 34e75f27d..98f188ee2 100644 --- a/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_with_tabnet.ipynb @@ -467,8 +467,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -487,8 +486,7 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -711,8 +709,7 @@ "IMPORT_FILE = \"petfinder-tabular-classification-tabnet-with-header.csv\"\n", "TRAINING_DATA_PATH = f\"{BUCKET_URI}/data/petfinder/train.csv\"\n", "\n", - "! gcloud storage cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}" - ] + "! gcloud storage cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}" ] }, { "cell_type": "markdown", @@ -1422,8 +1419,7 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb index 1e054045d..6f3001cfe 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb @@ -441,8 +441,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create $BUCKET_URI --location=$REGION" - ] + "! gcloud storage buckets create $BUCKET_URI --location=$REGION" ] }, { "cell_type": "markdown", @@ -461,8 +460,7 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -531,10 +529,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", - "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" - ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -1006,8 +1002,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz" ] }, { "cell_type": "markdown", @@ -1433,14 +1428,11 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gcloud storage cat $EXECUTE_OUTPUT\n", - " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gcloud storage cat $GCP_RESOURCES\n", - " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gcloud storage cat $EVAL_METRICS\n", - " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1454,8 +1446,7 @@ "print(\"getbesttrialop\")\n", "artifacts = print_pipeline_output(pipeline, \"getbesttrialop\")\n", "print(\"\\n\\n\")\n", - "output = !gcloud storage cat $artifacts\n", - "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "best_trial = json.loads(output[\"parameters\"][\"Output\"][\"stringValue\"])\n", "model_id = best_trial[\"id\"]\n", "print(\"BEST MODEL\", model_id)\n", @@ -1529,8 +1520,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb b/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb index 5716e02d1..badfa31c5 100644 --- a/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb +++ b/notebooks/community/ml_ops/stage5/get_started_with_vertex_endpoint_and_shared_vm.ipynb @@ -449,8 +449,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -469,8 +468,7 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1029,12 +1027,8 @@ "deployment_resource_pool.dedicated_resources.max_replica_count = MAX_NODES\n", "deployment_resource_pool.dedicated_resources.machine_spec.machine_type = DEPLOY_COMPUTE\n", "if DEPLOY_NGPU:\n", - " deployment_resource_pool.dedicated_resources.machine_spec.accelerator_type = (\n", - " DEPLOY_GPU\n", - " )\n", - " deployment_resource_pool.dedicated_resources.machine_spec.accelerator_count = (\n", - " DEPLOY_NGPU\n", - " )\n", + " deployment_resource_pool.dedicated_resources.machine_spec.accelerator_type = DEPLOY_GPU\n", + " deployment_resource_pool.dedicated_resources.machine_spec.accelerator_count = DEPLOY_NGPU\n", "\n", "request = aip_beta.CreateDeploymentResourcePoolRequest(\n", " parent=f\"projects/{PROJECT_ID}/locations/{REGION}\",\n", @@ -1578,8 +1572,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "!rm -f test.jpg" ] } diff --git a/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb index 8fedb12d4..4c1e0ff95 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_custom_image_model_batch.ipynb @@ -455,8 +455,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -475,8 +474,7 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -927,8 +925,7 @@ " f.write(json.dumps(x.tolist()) + \"\\n\")\n", "\n", "# Upload to Cloud Storage bucket\n", - "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", - "\n", + "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", "\n", "print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)" ] }, @@ -1445,8 +1442,7 @@ " batch_prediction_job.delete()\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb index 724c56a92..258b4c894 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb @@ -193,11 +193,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -221,8 +219,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -358,9 +355,7 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", - " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -412,9 +407,7 @@ "# @markdown 1. If `max_steps > 0`, it takes precedence over `epochs`. One can set a small `max_steps` value to quickly check the pipeline.\n", "\n", "# @markdown Accelerator type to use for training.\n", - "training_accelerator_type = (\n", - " \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", - ")\n", + "training_accelerator_type = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", "\n", "# The pre-built training docker image.\n", "if training_accelerator_type == \"NVIDIA_A100_80GB\":\n", @@ -1066,8 +1059,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb index f91022ecd..28fa29701 100644 --- a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb @@ -180,7 +180,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -194,9 +194,7 @@ "# @markdown Alternatively, you can also load the original Gemma models for serving from Vertex AI after accepting the agreement.\n", "\n", "# @markdown **Select and fill one of the two following sections.**\n", - "LOAD_MODEL_FROM = (\n", - " \"Hugging Face\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", - ")\n", + "LOAD_MODEL_FROM = \"Hugging Face\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", "\n", "# @markdown ---\n", "\n", diff --git a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb index fb8839b4b..014c7e6c1 100644 --- a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb +++ b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb @@ -177,11 +177,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -203,8 +202,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb index 06649bc8b..85ab31993 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb @@ -201,11 +201,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -229,8 +227,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -799,7 +796,6 @@ "\n", "# @markdown In case you are using your own dataset, kindly convert it to [COCO format](https://opencv.org/introduction-to-the-coco-dataset/).\n", "\n", - "\n", "!wget https://github.com/matterport/Mask_RCNN/releases/download/v2.1/balloon_dataset.zip\n", "!unzip balloon_dataset.zip > /dev/null\n", "\n", @@ -1216,8 +1212,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb index bfb5aeed7..c00abfd2f 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb @@ -185,11 +185,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -213,8 +211,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -238,9 +235,7 @@ "# @markdown Alternatively, you can also load the original Llama 3.3 models for finetuning and serving from Vertex AI after accepting the agreement.\n", "\n", "# @markdown **Only select and fill one of the following sections.**\n", - "LOAD_MODEL_FROM = (\n", - " \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", - ")\n", + "LOAD_MODEL_FROM = \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", "\n", "# @markdown ---\n", "\n", @@ -397,9 +392,7 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", - " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -451,9 +444,7 @@ "# @markdown 1. If `max_steps>0`, it takes precedence over `epochs`. One can set a small `max_steps` value to quickly check the pipeline.\n", "\n", "# @markdown Acceletor type to use for training.\n", - "training_accelerator_type = (\n", - " \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", - ")\n", + "training_accelerator_type = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", "\n", "# The pre-built training docker image.\n", "if training_accelerator_type == \"NVIDIA_A100_80GB\":\n", @@ -1088,8 +1079,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb index 63801fe48..6433ec335 100644 --- a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb +++ b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_SDK_Integration.ipynb @@ -496,8 +496,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -516,8 +515,7 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -687,9 +685,7 @@ "outputs": [], "source": [ "%cd ..\n", - "!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", - "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" - ] + "!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] }, { "cell_type": "markdown", @@ -1792,8 +1788,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb b/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb index fb658ff5e..ced7eb02a 100644 --- a/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb +++ b/notebooks/community/prediction/feature_store_integration/prediction_feature_store_integration.ipynb @@ -451,8 +451,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "code", @@ -462,8 +461,7 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -493,8 +491,7 @@ }, "outputs": [], "source": [ - "! gcloud storage cp --recursive gs://featurestore_integration/model/* $ARTIFACT_URI" - ] + "! gcloud storage cp --recursive gs://featurestore_integration/model/* $ARTIFACT_URI" ] }, { "cell_type": "markdown", @@ -855,8 +852,7 @@ }, "outputs": [], "source": [ - "! gcloud storage cp gs://featurestore_integration/data/mobile_gaming_dataset.csv $SOURCE_URI" - ] + "! gcloud storage cp gs://featurestore_integration/data/mobile_gaming_dataset.csv $SOURCE_URI" ] }, { "cell_type": "code", @@ -1198,8 +1194,7 @@ }, "outputs": [], "source": [ - "!gcloud storage cp prediction_featurestore_fetch_config.yaml $ARTIFACT_URI" - ] + "!gcloud storage cp prediction_featurestore_fetch_config.yaml $ARTIFACT_URI" ] }, { "cell_type": "markdown", @@ -1731,8 +1726,7 @@ "outputs": [], "source": [ "# Delete bucket\n", - "!gcloud storage rm --recursive $BUCKET_URI" - ] + "!gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/workbench/spark/spark_ml.ipynb b/notebooks/official/workbench/spark/spark_ml.ipynb index 695b3334c..4af188288 100644 --- a/notebooks/official/workbench/spark/spark_ml.ipynb +++ b/notebooks/official/workbench/spark/spark_ml.ipynb @@ -383,8 +383,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -1424,8 +1423,7 @@ " ! rm -rf $BUCKET_FOLDER\n", " ! mkdir $BUCKET_FOLDER\n", " bike_gbt_model.write().overwrite().save(f\"{BUCKET_FOLDER}\")\n", - " ! gcloud storage cp --recursive $BUCKET_FOLDER $BUCKET_URI\n", - "else:\n", + " ! gcloud storage cp --recursive $BUCKET_FOLDER $BUCKET_URI\n", "else:\n", " bike_gbt_model.write().overwrite().save(f\"{BUCKET_URI}/\")" ] }, @@ -1460,8 +1458,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] }, { "cell_type": "markdown", From 25c37809125921c1d4b02daf63fa760c2d58d224 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Mon, 22 Dec 2025 09:59:34 +0000 Subject: [PATCH 06/10] Revert "Manual Changes" This reverts commit 8ca9b56c5b7d84ebb8510dd1fdc1205a034184d3. --- .../stage3/get_started_with_hpt_pipeline_components.ipynb | 1 + .../model_garden_gemma2_finetuning_on_vertex.ipynb | 2 +- .../model_garden_gemma_deployment_on_vertex.ipynb | 2 +- .../model_garden/model_garden_integration_with_agent.ipynb | 7 ++++--- .../model_garden/model_garden_pytorch_detectron2.ipynb | 3 ++- .../model_garden_pytorch_llama3_3_finetuning.ipynb | 2 +- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb index 6f3001cfe..3e28b3b40 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_hpt_pipeline_components.ipynb @@ -529,6 +529,7 @@ }, "outputs": [], "source": [ + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, diff --git a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb index 258b4c894..f3c5c851e 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb @@ -195,7 +195,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", diff --git a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb index 28fa29701..4c04a6733 100644 --- a/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_deployment_on_vertex.ipynb @@ -180,7 +180,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", diff --git a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb index 014c7e6c1..80548a4f5 100644 --- a/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb +++ b/notebooks/community/model_garden/model_garden_integration_with_agent.ipynb @@ -177,10 +177,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -202,7 +202,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb index 85ab31993..69bb5d25e 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb @@ -203,7 +203,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -227,6 +227,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb index c00abfd2f..039907984 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb @@ -187,7 +187,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", From 41dad18a892907a5a2e82c87273b901f820224f1 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Mon, 22 Dec 2025 18:39:30 +0000 Subject: [PATCH 07/10] changes removed model garden --- ...del_garden_gemma2_finetuning_on_vertex.ipynb | 16 +++++++++++----- .../model_garden_pytorch_detectron2.ipynb | 13 ++++++++----- ...del_garden_pytorch_llama3_3_finetuning.ipynb | 17 +++++++++++------ 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb index f3c5c851e..8c8b30e9b 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb @@ -193,9 +193,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gsutil ls -L {BUCKET_URI} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -219,7 +221,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -355,7 +358,9 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", + " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", + " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -1059,7 +1064,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gsutil -m rm -r $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb index 69bb5d25e..75b7dff31 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb @@ -201,9 +201,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -227,8 +229,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -1213,7 +1215,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gsutil -m rm -r $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb index 039907984..c9914a381 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb @@ -124,7 +124,7 @@ "\n", "# @markdown 3. For serving, **[click here](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_l4_gpus)** to check if your project already has the required 1 L4 GPU in the us-central1 region. If yes, then run this notebook in the us-central1 region. If you need more L4 GPUs for your project, then you can follow [these instructions](https://cloud.google.com/docs/quotas/view-manage#viewing_your_quota_console) to request more. Alternatively, if you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus).\n", "\n", - "# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n", + "# @markdown > | Machin188e Type | Accelerator Type | Recommended Regions |\n", "# @markdown | ----------- | ----------- | ----------- |\n", "# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n", "# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n", @@ -185,9 +185,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gsutil ls -b {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -211,7 +213,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n", + "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -392,7 +395,9 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", + " ! gsutil -m cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gsutil -m cp {pretrained_model_id}/config.json ./tokenizer\n", + " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -1079,7 +1084,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gsutil rm -r $BUCKET_NAME" ] } ], "metadata": { From a42f0a05ad6d51d55d064bfb9c49e7fd9af68bb7 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 23 Dec 2025 11:19:06 +0530 Subject: [PATCH 08/10] Update model_garden_gemma2_finetuning_on_vertex.ipynb --- .../model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb index 8c8b30e9b..b0a878ce1 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb @@ -196,7 +196,7 @@ " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -L {BUCKET_URI} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", From 46e35492e99c7ecabcee996be528d49de612c4ee Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 23 Dec 2025 11:22:06 +0530 Subject: [PATCH 09/10] Update model_garden_pytorch_llama3_3_finetuning.ipynb --- .../model_garden_pytorch_llama3_3_finetuning.ipynb | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb index c9914a381..62c1ef2dd 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb @@ -124,7 +124,7 @@ "\n", "# @markdown 3. For serving, **[click here](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_l4_gpus)** to check if your project already has the required 1 L4 GPU in the us-central1 region. If yes, then run this notebook in the us-central1 region. If you need more L4 GPUs for your project, then you can follow [these instructions](https://cloud.google.com/docs/quotas/view-manage#viewing_your_quota_console) to request more. Alternatively, if you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus).\n", "\n", - "# @markdown > | Machin188e Type | Accelerator Type | Recommended Regions |\n", + "# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n", "# @markdown | ----------- | ----------- | ----------- |\n", "# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n", "# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n", @@ -188,7 +188,7 @@ " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -b {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -213,7 +213,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -395,8 +395,8 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gsutil -m cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gsutil -m cp {pretrained_model_id}/config.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", @@ -1084,7 +1084,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_NAME" ] + " ! gsutil -m rm -r $BUCKET_NAME" + ] } ], "metadata": { From 149694155b9337d029d2524c2d11ed5ee2f6636a Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 23 Dec 2025 11:22:49 +0530 Subject: [PATCH 10/10] Update model_garden_pytorch_llama3_3_finetuning.ipynb --- .../model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb index 62c1ef2dd..cc81473dc 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb @@ -1084,7 +1084,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ],