From b2961a28ad3f8092c7236dad6431e74e4218942a Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Thu, 16 Oct 2025 13:37:54 +0000 Subject: [PATCH 1/7] Migrate gsutil usage to gcloud storage --- ...Natural Language Text Classification.ipynb | 26 +++++--------- .../stage2/get_started_bqml_training.ipynb | 12 +++---- ...ed_with_dataflow_pipeline_components.ipynb | 36 +++++++------------ .../get_started_with_vertex_ml_metadata.ipynb | 13 +++---- ...l_garden_pytorch_mistral_peft_tuning.ipynb | 18 ++++------ ...l_garden_pytorch_mixtral_peft_tuning.ipynb | 17 ++++----- ...ml-image-classification-batch-online.ipynb | 25 +++++-------- .../model_monitoring/model_monitoring.ipynb | 9 ++--- .../tabnet_on_vertex_pipelines.ipynb | 13 +++---- .../inventory_prediction.ipynb | 6 ++-- 10 files changed, 62 insertions(+), 113 deletions(-) diff --git a/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb b/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb index c5008ce38..d3c34180d 100644 --- a/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb +++ b/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb @@ -336,8 +336,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION gs://$BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -356,8 +355,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al gs://$BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -483,8 +481,7 @@ }, "outputs": [], "source": [ - "! gsutil cat $IMPORT_FILE | head -n 10" - ] + "! gcloud storage cat $IMPORT_FILE | head -n 10" ] }, { "cell_type": "markdown", @@ -1140,8 +1137,7 @@ }, "outputs": [], "source": [ - "test_item = ! gsutil cat $IMPORT_FILE | head -n1\n", - "test_item, test_label = str(test_item[0]).split(\",\")\n", + "test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item, test_label = str(test_item[0]).split(\",\")\n", "\n", "print(test_item, test_label)" ] @@ -1175,9 +1171,7 @@ }, "outputs": [], "source": [ - "! gsutil cat $gcs_input_uri\n", - "! gsutil cat $test_item_uri" - ] + "! gcloud storage cat $gcs_input_uri\n", "! gcloud storage cat $test_item_uri" ] }, { "cell_type": "markdown", @@ -1325,9 +1319,7 @@ "source": [ "destination_uri = output_config[\"gcs_destination\"][\"output_uri_prefix\"][:-1]\n", "\n", - "! gsutil ls $destination_uri/*\n", - "! gsutil cat $destination_uri/prediction*/*.jsonl" - ] + "! gcloud storage ls $destination_uri/*\n", "! gcloud storage cat $destination_uri/prediction*/*.jsonl" ] }, { "cell_type": "markdown", @@ -1441,8 +1433,7 @@ }, "outputs": [], "source": [ - "test_item = ! gsutil cat $IMPORT_FILE | head -n1\n", - "test_item, test_label = str(test_item[0]).split(\",\")" + "test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item, test_label = str(test_item[0]).split(\",\")" ] }, { @@ -1635,8 +1626,7 @@ "\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r gs://$BUCKET_NAME" - ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb b/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb index de87018c9..3f3bbee0f 100644 --- a/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb @@ -435,8 +435,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -455,8 +454,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -850,8 +848,7 @@ "! bq extract -m $param\n", "\n", "MODEL_DIR = f\"{BUCKET_URI}/{BQ_DATASET_NAME}\"\n", - "! gsutil ls $MODEL_DIR" - ] + "! gcloud storage ls $MODEL_DIR" ] }, { "cell_type": "markdown", @@ -1398,8 +1395,7 @@ "delete_storage = False\n", "if delete_storage or os.getenv(\"IS_TESTING\"):\n", " # Delete the created GCS bucket\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb index 30b23e420..104486f64 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb @@ -462,8 +462,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -482,8 +481,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -552,10 +550,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -801,10 +798,8 @@ "outputs": [], "source": [ "GCS_WC_PY = BUCKET_URI + \"/wc.py\"\n", - "! gsutil cp wc.py $GCS_WC_PY\n", - "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", - "! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n", - "\n", + "! gcloud storage cp wc.py $GCS_WC_PY\n", "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", + "! gcloud storage cp requirements.txt $GCS_REQUIREMENTS_TXT\n", "\n", "GCS_WC_OUT = BUCKET_URI + \"/wc_out.txt\"" ] }, @@ -872,8 +867,7 @@ "\n", "pipeline.run()\n", "\n", - "! gsutil cat {GCS_WC_OUT}* | head -n10\n", - "\n", + "! gcloud storage cat {GCS_WC_OUT}* | head -n10\n", "\n", "! rm -f dataflow_wc.json wc.py requirements.txt" ] }, @@ -1097,12 +1091,10 @@ "outputs": [], "source": [ "GCS_SPLIT_PY = BUCKET_URI + \"/split.py\"\n", - "! gsutil cp split.py $GCS_SPLIT_PY\n", - "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", - "! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n", + "! gcloud storage cp split.py $GCS_SPLIT_PY\n", "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", + "! gcloud storage cp requirements.txt $GCS_REQUIREMENTS_TXT\n", "GCS_SETUP_PY = BUCKET_URI + \"/setup.py\"\n", - "! gsutil cp setup.py $GCS_SETUP_PY" - ] + "! gcloud storage cp setup.py $GCS_SETUP_PY" ] }, { "cell_type": "markdown", @@ -1202,8 +1194,7 @@ "\n", "pipeline.run()\n", "\n", - "! gsutil ls {BUCKET_URI}/exported_data\n", - "\n", + "! gcloud storage ls {BUCKET_URI}/exported_data\n", "\n", "! rm -f dataflow_split.json split.py requirements.txt" ] }, @@ -1258,8 +1249,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb index ac00416dd..4b40b97cc 100644 --- a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb +++ b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb @@ -357,8 +357,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -427,10 +426,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -1386,8 +1384,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb index 453ecace1..85414529c 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb @@ -187,11 +187,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -215,8 +214,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -335,9 +334,7 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", - " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -891,8 +888,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb index 13b825ae6..ec8895474 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb @@ -187,11 +187,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", +# Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -215,7 +214,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -335,9 +335,7 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", - " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -897,8 +895,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb b/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb index 584e45f43..636bc8b42 100644 --- a/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb +++ b/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb @@ -279,8 +279,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -375,12 +374,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -730,8 +727,7 @@ }, "outputs": [], "source": [ - "test_items = !gsutil cat $IMPORT_FILE | head -n2\n", - "if len(str(test_items[0]).split(\",\")) == 3:\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(str(test_items[0]).split(\",\")) == 3:\n", " _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n", " _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n", "else:\n", @@ -764,9 +760,7 @@ "file_1 = test_item_1.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n", "\n", - "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n", - "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n", - "\n", + "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", "\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_2 = BUCKET_URI + \"/\" + file_2" ] @@ -809,8 +803,7 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gsutil cat $gcs_input_uri" - ] + "! gcloud storage cat $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -1066,8 +1059,7 @@ }, "outputs": [], "source": [ - "test_item = !gsutil cat $IMPORT_FILE | head -n1\n", - "if len(str(test_item[0]).split(\",\")) == 3:\n", + "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n", " _, test_item, test_label = str(test_item[0]).split(\",\")\n", "else:\n", " test_item, test_label = str(test_item[0]).split(\",\")\n", @@ -1201,8 +1193,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = True # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/model_monitoring/model_monitoring.ipynb b/notebooks/official/model_monitoring/model_monitoring.ipynb index 183a91170..b2c3b4163 100644 --- a/notebooks/official/model_monitoring/model_monitoring.ipynb +++ b/notebooks/official/model_monitoring/model_monitoring.ipynb @@ -311,8 +311,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1029,8 +1028,7 @@ }, "outputs": [], "source": [ - "!gsutil ls gs://cloud-ai-platform-fdfb4810-148b-4c86-903c-dbdff879f6e1/*/*" - ] + "!gcloud storage ls gs://cloud-ai-platform-fdfb4810-148b-4c86-903c-dbdff879f6e1/*/*" ] }, { "cell_type": "markdown", @@ -1172,8 +1170,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] }, { "cell_type": "markdown", diff --git a/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb b/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb index a2feec79d..fbceff493 100644 --- a/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb +++ b/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb @@ -303,8 +303,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -386,10 +385,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -1088,8 +1086,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb b/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb index f1a299908..24315805b 100644 --- a/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb +++ b/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb @@ -340,8 +340,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -1474,8 +1473,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From d7362fc4fae13492f414fc35c9246058cd8734f8 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 12 Dec 2025 13:57:20 +0000 Subject: [PATCH 2/7] changes for 4319 --- ...ed_with_dataflow_pipeline_components.ipynb | 34 ++++++++++++------- .../get_started_with_vertex_ml_metadata.ipynb | 13 ++++--- ...l_garden_pytorch_mistral_peft_tuning.ipynb | 4 +-- ...l_garden_pytorch_mixtral_peft_tuning.ipynb | 2 -- .../tabnet_on_vertex_pipelines.ipynb | 13 ++++--- 5 files changed, 39 insertions(+), 27 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb index 104486f64..34b9ad5b9 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb @@ -462,7 +462,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -481,7 +482,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -550,9 +552,10 @@ }, "outputs": [], "source": [ - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -798,8 +801,10 @@ "outputs": [], "source": [ "GCS_WC_PY = BUCKET_URI + \"/wc.py\"\n", - "! gcloud storage cp wc.py $GCS_WC_PY\n", "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", - "! gcloud storage cp requirements.txt $GCS_REQUIREMENTS_TXT\n", "\n", + "! gcloud storage cp wc.py $GCS_WC_PY\n", + "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", + "! gcloud storage cp requirements.txt $GCS_REQUIREMENTS_TXT\n", + "\n", "GCS_WC_OUT = BUCKET_URI + \"/wc_out.txt\"" ] }, @@ -867,7 +872,8 @@ "\n", "pipeline.run()\n", "\n", - "! gcloud storage cat {GCS_WC_OUT}* | head -n10\n", "\n", + "! gcloud storage cat {GCS_WC_OUT}* | head -n10\n", + "\n", "! rm -f dataflow_wc.json wc.py requirements.txt" ] }, @@ -1091,10 +1097,12 @@ "outputs": [], "source": [ "GCS_SPLIT_PY = BUCKET_URI + \"/split.py\"\n", - "! gcloud storage cp split.py $GCS_SPLIT_PY\n", "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", + "! gcloud storage cp split.py $GCS_SPLIT_PY\n", + "GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n", "! gcloud storage cp requirements.txt $GCS_REQUIREMENTS_TXT\n", "GCS_SETUP_PY = BUCKET_URI + \"/setup.py\"\n", - "! gcloud storage cp setup.py $GCS_SETUP_PY" ] + "! gcloud storage cp setup.py $GCS_SETUP_PY" + ] }, { "cell_type": "markdown", @@ -1194,7 +1202,8 @@ "\n", "pipeline.run()\n", "\n", - "! gcloud storage ls {BUCKET_URI}/exported_data\n", "\n", + "! gcloud storage ls {BUCKET_URI}/exported_data\n", + "\n", "! rm -f dataflow_split.json split.py requirements.txt" ] }, @@ -1249,7 +1258,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb index 4b40b97cc..21b4f5cdd 100644 --- a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb +++ b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb @@ -357,7 +357,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -426,9 +427,10 @@ }, "outputs": [], "source": [ - "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -1384,7 +1386,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb index 85414529c..cbadda0d8 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb @@ -189,8 +189,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -214,7 +213,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb index ec8895474..d82e747e7 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb @@ -189,7 +189,6 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", -# Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -214,7 +213,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", diff --git a/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb b/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb index fbceff493..7e4f24207 100644 --- a/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb +++ b/notebooks/official/tabular_workflows/tabnet_on_vertex_pipelines.ipynb @@ -303,7 +303,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -385,9 +386,10 @@ }, "outputs": [], "source": [ - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -1086,7 +1088,8 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From 242a45b1737ddd6dfd00e14855ff713f90651be0 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 12 Dec 2025 14:02:55 +0000 Subject: [PATCH 3/7] changes for 4319 --- .../model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb index d82e747e7..14e8afd3c 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb @@ -189,7 +189,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", From a5663a1acd8185c642454d6526e099fb32f9757a Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 16 Dec 2025 12:49:51 +0000 Subject: [PATCH 4/7] Apply automated linter fixes --- ...Natural Language Text Classification.ipynb | 26 +++++++++++++------ .../stage2/get_started_bqml_training.ipynb | 12 ++++++--- ...l_garden_pytorch_mistral_peft_tuning.ipynb | 16 ++++++++---- ...l_garden_pytorch_mixtral_peft_tuning.ipynb | 17 ++++++++---- ...ml-image-classification-batch-online.ipynb | 25 ++++++++++++------ .../model_monitoring/model_monitoring.ipynb | 9 ++++--- .../inventory_prediction.ipynb | 6 +++-- 7 files changed, 76 insertions(+), 35 deletions(-) diff --git a/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb b/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb index d3c34180d..3c6720531 100644 --- a/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb +++ b/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb @@ -336,7 +336,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ] + "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -355,7 +356,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -481,7 +483,8 @@ }, "outputs": [], "source": [ - "! gcloud storage cat $IMPORT_FILE | head -n 10" ] + "! gcloud storage cat $IMPORT_FILE | head -n 10" + ] }, { "cell_type": "markdown", @@ -1137,7 +1140,8 @@ }, "outputs": [], "source": [ - "test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item, test_label = str(test_item[0]).split(\",\")\n", + "test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "test_item, test_label = str(test_item[0]).split(\",\")\n", "\n", "print(test_item, test_label)" ] @@ -1171,7 +1175,9 @@ }, "outputs": [], "source": [ - "! gcloud storage cat $gcs_input_uri\n", "! gcloud storage cat $test_item_uri" ] + "! gcloud storage cat $gcs_input_uri\n", + "! gcloud storage cat $test_item_uri" + ] }, { "cell_type": "markdown", @@ -1319,7 +1325,9 @@ "source": [ "destination_uri = output_config[\"gcs_destination\"][\"output_uri_prefix\"][:-1]\n", "\n", - "! gcloud storage ls $destination_uri/*\n", "! gcloud storage cat $destination_uri/prediction*/*.jsonl" ] + "! gcloud storage ls $destination_uri/*\n", + "! gcloud storage cat $destination_uri/prediction*/*.jsonl" + ] }, { "cell_type": "markdown", @@ -1433,7 +1441,8 @@ }, "outputs": [], "source": [ - "test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item, test_label = str(test_item[0]).split(\",\")" + "test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "test_item, test_label = str(test_item[0]).split(\",\")" ] }, { @@ -1626,7 +1635,8 @@ "\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb b/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb index 3f3bbee0f..018c25f21 100644 --- a/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb @@ -435,7 +435,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -454,7 +455,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -848,7 +850,8 @@ "! bq extract -m $param\n", "\n", "MODEL_DIR = f\"{BUCKET_URI}/{BQ_DATASET_NAME}\"\n", - "! gcloud storage ls $MODEL_DIR" ] + "! gcloud storage ls $MODEL_DIR" + ] }, { "cell_type": "markdown", @@ -1395,7 +1398,8 @@ "delete_storage = False\n", "if delete_storage or os.getenv(\"IS_TESTING\"):\n", " # Delete the created GCS bucket\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb index cbadda0d8..8c7e71ef2 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb @@ -187,9 +187,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -213,7 +215,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -332,7 +335,9 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -886,7 +891,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb index 14e8afd3c..edc232f4e 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb @@ -187,9 +187,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -333,7 +335,9 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -610,7 +614,9 @@ "\n", "# Find Vertex AI prediction supported accelerators and regions [here](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute).\n", "# @markdown Accelerator type to use for serving.\n", - "accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_H100_80GB\"]\n", + "accelerator_type = (\n", + " \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_H100_80GB\"]\n", + ")\n", "\n", "if accelerator_type == \"NVIDIA_L4\":\n", " machine_type = \"g2-standard-96\"\n", @@ -893,7 +899,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb b/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb index 636bc8b42..919222914 100644 --- a/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb +++ b/notebooks/official/migration/sdk-automl-image-classification-batch-online.ipynb @@ -279,7 +279,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {LOCATION} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -374,10 +375,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -727,7 +730,8 @@ }, "outputs": [], "source": [ - "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(str(test_items[0]).split(\",\")) == 3:\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", + "if len(str(test_items[0]).split(\",\")) == 3:\n", " _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n", " _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n", "else:\n", @@ -760,7 +764,9 @@ "file_1 = test_item_1.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n", "\n", - "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", "\n", + "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", + "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", + "\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_2 = BUCKET_URI + \"/\" + file_2" ] @@ -803,7 +809,8 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gcloud storage cat $gcs_input_uri" ] + "! gcloud storage cat $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -1059,7 +1066,8 @@ }, "outputs": [], "source": [ - "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n", + "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", + "if len(str(test_item[0]).split(\",\")) == 3:\n", " _, test_item, test_label = str(test_item[0]).split(\",\")\n", "else:\n", " test_item, test_label = str(test_item[0]).split(\",\")\n", @@ -1193,7 +1201,8 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = True # Set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/model_monitoring/model_monitoring.ipynb b/notebooks/official/model_monitoring/model_monitoring.ipynb index b2c3b4163..4cbd5851a 100644 --- a/notebooks/official/model_monitoring/model_monitoring.ipynb +++ b/notebooks/official/model_monitoring/model_monitoring.ipynb @@ -311,7 +311,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -1028,7 +1029,8 @@ }, "outputs": [], "source": [ - "!gcloud storage ls gs://cloud-ai-platform-fdfb4810-148b-4c86-903c-dbdff879f6e1/*/*" ] + "!gcloud storage ls gs://cloud-ai-platform-fdfb4810-148b-4c86-903c-dbdff879f6e1/*/*" + ] }, { "cell_type": "markdown", @@ -1170,7 +1172,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] }, { "cell_type": "markdown", diff --git a/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb b/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb index 24315805b..0dd3be4f4 100644 --- a/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb +++ b/notebooks/official/workbench/inventory-prediction/inventory_prediction.ipynb @@ -340,7 +340,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -1473,7 +1474,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From f37a2cf6e86877f2092d84c32c17a442b2a79256 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 16 Dec 2025 12:52:19 +0000 Subject: [PATCH 5/7] remove unused import --- ...Natural Language Text Classification.ipynb | 75 +++++++++---------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb b/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb index 3c6720531..dc0609f56 100644 --- a/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb +++ b/notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb @@ -109,27 +109,27 @@ "id": "Af0jTPSgl9yh" }, "source": [ - "## Before you begin\r\n", - "\r\n", - "### GPU run-time\r\n", - "\r\n", - "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\r\n", - "\r\n", - "### Set up your GCP project\r\n", - "\r\n", - "**The following steps are required, regardless of your notebook environment.**\r\n", - "\r\n", - "1. [Select or create a GCP project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\r\n", - "\r\n", - "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\r\n", - "\r\n", - "3. [Enable the AutoML APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component)\r\n", - "\r\n", - "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in AutoML Notebooks.\r\n", - "\r\n", - "5. Enter your project ID in the cell below. Then run the cell to make sure the\r\n", - "Cloud SDK uses the right project for all the commands in this notebook.\r\n", - "\r\n", + "## Before you begin\n", + "\n", + "### GPU run-time\n", + "\n", + "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n", + "\n", + "### Set up your GCP project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a GCP project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", + "\n", + "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n", + "\n", + "3. [Enable the AutoML APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component)\n", + "\n", + "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in AutoML Notebooks.\n", + "\n", + "5. Enter your project ID in the cell below. Then run the cell to make sure the\n", + "Cloud SDK uses the right project for all the commands in this notebook.\n", + "\n", "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands." ] }, @@ -240,11 +240,11 @@ "id": "h_L3MRsOmYED" }, "source": [ - "### Authenticate your GCP account\r\n", - "\r\n", - "**If you are using AutoML Notebooks**, your environment is already\r\n", - "authenticated. Skip this step.\r\n", - "\r\n", + "### Authenticate your GCP account\n", + "\n", + "**If you are using AutoML Notebooks**, your environment is already\n", + "authenticated. Skip this step.\n", + "\n", "*Note: If you are on an AutoML notebook and run the cell, the cell knows to skip executing the authentication steps.*" ] }, @@ -391,7 +391,6 @@ "outputs": [], "source": [ "import json\n", - "import time\n", "\n", "from google.cloud import automl\n", "from google.protobuf.json_format import MessageToJson" @@ -1309,9 +1308,9 @@ "id": "771dDuKzg8Mk" }, "source": [ - "*Example output*:\r\n", - "```\r\n", - "{}\r\n", + "*Example output*:\n", + "```\n", + "{}\n", "```" ] }, @@ -1409,9 +1408,9 @@ "id": "CZ-62obNmBNc" }, "source": [ - "*Example output*:\r\n", - "```\r\n", - "{}\r\n", + "*Example output*:\n", + "```\n", + "{}\n", "```" ] }, @@ -1599,11 +1598,11 @@ "id": "bQ-VVaSxJjkd" }, "source": [ - "# Cleaning up\r\n", - "\r\n", - "To clean up all GCP resources used in this project, you can [delete the GCP\r\n", - "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\r\n", - "\r\n", + "# Cleaning up\n", + "\n", + "To clean up all GCP resources used in this project, you can [delete the GCP\n", + "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", + "\n", "Otherwise, you can delete the individual resources you created in this tutorial." ] }, From ff7f63c1c1a29e2fa03548765997891cf71530fd Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 12:55:40 +0000 Subject: [PATCH 6/7] removed model_garden changes --- ...odel_garden_pytorch_mistral_peft_tuning.ipynb | 12 ++++++------ ...odel_garden_pytorch_mixtral_peft_tuning.ipynb | 16 +++++++--------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb index 8c7e71ef2..453ecace1 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mistral_peft_tuning.ipynb @@ -187,10 +187,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -215,7 +215,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -335,8 +335,8 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", @@ -891,7 +891,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb index edc232f4e..c32860dec 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb @@ -187,10 +187,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -215,7 +215,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -335,8 +335,8 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", @@ -614,9 +614,7 @@ "\n", "# Find Vertex AI prediction supported accelerators and regions [here](https://cloud.google.com/vertex-ai/docs/predictions/configure-compute).\n", "# @markdown Accelerator type to use for serving.\n", - "accelerator_type = (\n", - " \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_H100_80GB\"]\n", - ")\n", + "accelerator_type = \"NVIDIA_L4\" # @param [\"NVIDIA_L4\", \"NVIDIA_TESLA_V100\", \"NVIDIA_H100_80GB\"]\n", "\n", "if accelerator_type == \"NVIDIA_L4\":\n", " machine_type = \"g2-standard-96\"\n", @@ -899,7 +897,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], From 33bd1df374dc9adaf6898b17983dc8a606d7a28f Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 18:27:14 +0530 Subject: [PATCH 7/7] Update model_garden_pytorch_mixtral_peft_tuning.ipynb --- .../model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb index c32860dec..13b825ae6 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_mixtral_peft_tuning.ipynb @@ -187,7 +187,7 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",