From c4d9e75a1620720e642268b39afce1f5e5fc4769 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Thu, 16 Oct 2025 15:50:48 +0000 Subject: [PATCH 1/6] Migrate gsutil usage to gcloud storage --- ...ex_pipelines_sklearn_with_prediction.ipynb | 34 +++++++------------ ...ed_with_airflow_and_vertex_pipelines.ipynb | 32 +++++++---------- ...el_garden_axolotl_gpt_oss_finetuning.ipynb | 15 +++----- ...el_garden_gemma_finetuning_on_vertex.ipynb | 20 +++++------ .../SDK_FBProphet_Forecasting_Online.ipynb | 9 ++--- ...sdk_custom_tabular_regression_online.ipynb | 12 +++---- ..._lightning_custom_container_training.ipynb | 11 +++--- ...stom-scikit-learn-prebuilt-container.ipynb | 12 +++---- ...started_with_model_monitoring_custom.ipynb | 9 ++--- .../chicago_taxi_fare_prediction.ipynb | 6 ++-- 10 files changed, 57 insertions(+), 103 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb b/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb index d07e1ae63..c3d4752a0 100644 --- a/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb @@ -488,8 +488,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -508,8 +507,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -579,10 +577,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -990,8 +987,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_newsaggr.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_newsaggr.tar.gz" ] }, { "cell_type": "markdown", @@ -1206,14 +1202,11 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gsutil cat $EXECUTE_OUTPUT\n", - " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gsutil cat $GCP_RESOURCES\n", - " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gsutil cat $EVAL_METRICS\n", - " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1221,15 +1214,13 @@ "print(\"custompythonpackagetrainingjob-run\")\n", "artifacts = print_pipeline_output(pipeline, \"custompythonpackagetrainingjob-run\")\n", "print(\"\\n\\n\")\n", - "output = !gsutil cat $artifacts\n", - "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n", "print(\"\\n\\n\")\n", "print(\"endpoint-create\")\n", "artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n", "print(\"\\n\\n\")\n", - "output = !gsutil cat $artifacts\n", - "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n", " \"resourceName\"\n", "]\n", @@ -1394,8 +1385,7 @@ "source": [ "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "! rm -rf custom custom.tar.csv" ] } diff --git a/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb index f80b6dcef..ed88a0896 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb @@ -456,8 +456,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -476,8 +475,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -546,10 +544,10 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -910,8 +908,7 @@ }, "outputs": [], "source": [ - "!gsutil cp $COMPOSER_DAG_FILENAME $COMPOSER_DAG_GCS/" - ] + "!gcloud storage cp $COMPOSER_DAG_FILENAME $COMPOSER_DAG_GCS/" ] }, { "cell_type": "code", @@ -921,8 +918,7 @@ }, "outputs": [], "source": [ - "!gsutil ls -l $COMPOSER_DAG_GCS/$COMPOSER_DAG_FILENAME" - ] + "!gcloud storage ls --long $COMPOSER_DAG_GCS/$COMPOSER_DAG_FILENAME" ] }, { "cell_type": "markdown", @@ -1308,14 +1304,11 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gsutil cat $EXECUTE_OUTPUT\n", - " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gsutil cat $GCP_RESOURCES\n", - " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gsutil cat $EVAL_METRICS\n", - " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1389,8 +1382,7 @@ "! bq rm -r -f $PROJECT_ID:$DATASET_NAME\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}\n", - " \n", + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", " \n", "! rm get_composer_config.py data_orchestration_bq_example_dag.py" ] } diff --git a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb index 48572ef3e..2a40bcd96 100644 --- a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb @@ -211,11 +211,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -239,8 +237,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -550,8 +547,7 @@ "vertex_ai_config_path = AXOLOTL_CONFIG_PATH\n", "# Copy the config file to the bucket.\n", "if AXOLOTL_SOURCE == \"LOCAL\":\n", - " ! gsutil -m cp $AXOLOTL_CONFIG_PATH $MODEL_BUCKET/config/\n", - " vertex_ai_config_path = f\"{common_util.gcs_fuse_path(MODEL_BUCKET)}/config/{pathlib.Path(AXOLOTL_CONFIG_PATH).name}\"\n", + " ! gcloud storage cp $AXOLOTL_CONFIG_PATH $MODEL_BUCKET/config/\n", " vertex_ai_config_path = f\"{common_util.gcs_fuse_path(MODEL_BUCKET)}/config/{pathlib.Path(AXOLOTL_CONFIG_PATH).name}\"\n", "\n", "job_name = common_util.get_job_name_with_datetime(\"axolotl-train\")\n", "AXOLOTL_OUTPUT_GCS_URI = f\"{BASE_AXOLOTL_OUTPUT_GCS_URI}/{job_name}\"\n", @@ -1060,8 +1056,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb index b5d314b7a..57dd7a5e1 100644 --- a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb @@ -186,11 +186,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -214,7 +213,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -262,8 +262,7 @@ "\n", " ! mkdir -p ./gemma\n", " ! curl -X GET \"{signed_url}\" | tar -xzvf - -C ./gemma/\n", - " ! gsutil -m cp -R ./gemma/* {MODEL_BUCKET}\n", - "\n", + " ! gcloud storage cp --recursive ./gemma/* {MODEL_BUCKET}\n", "\n", " model_path_prefix = MODEL_BUCKET\n", " HF_TOKEN = \"\"\n", "else:\n", @@ -393,9 +392,7 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", - " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -943,8 +940,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb b/notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb index f80f39dd6..adb25eab6 100644 --- a/notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb +++ b/notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb @@ -348,8 +348,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {REGION} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={REGION} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -742,8 +741,7 @@ "outputs": [], "source": [ "%cd app\n", - "!gsutil cp *.sav *.csv {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", - "%cd .." + "!gcloud storage cp *.sav *.csv {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "%cd .." ] }, { @@ -1541,8 +1539,7 @@ "\n", "# Delete the Cloud Storage bucket\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_custom_tabular_regression_online.ipynb b/notebooks/community/sdk/sdk_custom_tabular_regression_online.ipynb index b9bd12331..a384b225e 100644 --- a/notebooks/community/sdk/sdk_custom_tabular_regression_online.ipynb +++ b/notebooks/community/sdk/sdk_custom_tabular_regression_online.ipynb @@ -475,8 +475,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -495,8 +494,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -921,8 +919,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" ] }, { "cell_type": "markdown", @@ -1427,8 +1424,7 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_pytorch_lightning_custom_container_training.ipynb b/notebooks/community/sdk/sdk_pytorch_lightning_custom_container_training.ipynb index 19b6e3501..651985877 100644 --- a/notebooks/community/sdk/sdk_pytorch_lightning_custom_container_training.ipynb +++ b/notebooks/community/sdk/sdk_pytorch_lightning_custom_container_training.ipynb @@ -505,8 +505,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -525,8 +524,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1521,11 +1519,10 @@ "\n", "CONTENT_DIR = f\"{BUCKET_URI}/{content_name}*\"\n", "# Delete Cloud Storage objects that were created\n", - "! gsutil -m rm -r $CONTENT_DIR\n", + "! gcloud storage rm --recursive $CONTENT_DIR\n", "\n", "if delete_bucket and \"BUCKET_URI\" in globals():\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/migration/sdk-custom-scikit-learn-prebuilt-container.ipynb b/notebooks/official/migration/sdk-custom-scikit-learn-prebuilt-container.ipynb index 865e91014..3e847b9ac 100644 --- a/notebooks/official/migration/sdk-custom-scikit-learn-prebuilt-container.ipynb +++ b/notebooks/official/migration/sdk-custom-scikit-learn-prebuilt-container.ipynb @@ -320,8 +320,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -699,8 +698,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_census.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_census.tar.gz" ] }, { "cell_type": "markdown", @@ -965,8 +963,7 @@ " for i in INSTANCES:\n", " f.write(json.dumps(i) + \"\\n\")\n", "\n", - "! gsutil cat $gcs_input_uri" - ] + "! gcloud storage cat $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -1378,8 +1375,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb b/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb index 612b3da23..ac33d1a28 100644 --- a/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb +++ b/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb @@ -364,8 +364,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -928,8 +927,7 @@ "with open(\"schema.yaml\", \"w\") as f:\n", " f.write(yaml)\n", "\n", - "! gsutil cp schema.yaml {BUCKET_URI}/schema.yaml" - ] + "! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" ] }, { "cell_type": "markdown", @@ -1406,8 +1404,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil rm -rf {BUCKET_URI}\n", - "\n", + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n", "# Delete the locally generated files\n", "! rm -f schema.yaml\n", "\n", diff --git a/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb b/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb index d56b221e6..95f62cd65 100644 --- a/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb +++ b/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb @@ -365,8 +365,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1467,8 +1466,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From 92cd5072a3e18e33b77603c313f9f2a32a57953a Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Thu, 11 Dec 2025 19:33:49 +0000 Subject: [PATCH 2/6] Manual Changes-Migrate gsutil usage to gcloud storage --- ...t_started_vertex_pipelines_sklearn_with_prediction.ipynb | 2 +- .../get_started_with_airflow_and_vertex_pipelines.ipynb | 4 ++-- .../model_garden_axolotl_gpt_oss_finetuning.ipynb | 4 ++-- .../model_garden_gemma_finetuning_on_vertex.ipynb | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb b/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb index c3d4752a0..f6e6e06da 100644 --- a/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb @@ -577,7 +577,7 @@ }, "outputs": [], "source": [ - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, diff --git a/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb index ed88a0896..02bdeb6b9 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb @@ -544,9 +544,9 @@ }, "outputs": [], "source": [ - "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { diff --git a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb index 2a40bcd96..1615fdee2 100644 --- a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb @@ -213,7 +213,7 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -237,7 +237,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" diff --git a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb index 57dd7a5e1..88f02fb81 100644 --- a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb @@ -188,8 +188,8 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -213,7 +213,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", From e46e2580787e0853b5fc5ba682125501d91211ec Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 12 Dec 2025 07:25:48 +0000 Subject: [PATCH 3/6] Manual Changes --- ...t_started_vertex_pipelines_sklearn_with_prediction.ipynb | 1 - .../get_started_with_airflow_and_vertex_pipelines.ipynb | 6 ++---- .../model_garden_axolotl_gpt_oss_finetuning.ipynb | 1 - .../model_garden_gemma_finetuning_on_vertex.ipynb | 2 -- 4 files changed, 2 insertions(+), 8 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb b/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb index f6e6e06da..095719d3c 100644 --- a/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb @@ -577,7 +577,6 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, diff --git a/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb index 02bdeb6b9..3bcdd09b0 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb @@ -544,10 +544,8 @@ }, "outputs": [], "source": [ - - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb index 1615fdee2..370d3b5ed 100644 --- a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb @@ -237,7 +237,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" diff --git a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb index 88f02fb81..7c3ebbb66 100644 --- a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb @@ -188,7 +188,6 @@ " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -213,7 +212,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", From 6118bc4be7949b2f97717ee166fd1f01fc37655d Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Wed, 17 Dec 2025 12:03:44 +0000 Subject: [PATCH 4/6] Fix linter issue for 4323 --- ...ex_pipelines_sklearn_with_prediction.ipynb | 33 ++++++++++++------- ...ed_with_airflow_and_vertex_pipelines.ipynb | 32 +++++++++++------- ...el_garden_axolotl_gpt_oss_finetuning.ipynb | 20 +++++++---- ...el_garden_gemma_finetuning_on_vertex.ipynb | 24 ++++++++++---- .../SDK_FBProphet_Forecasting_Online.ipynb | 9 +++-- ...sdk_custom_tabular_regression_online.ipynb | 12 ++++--- ..._lightning_custom_container_training.ipynb | 9 +++-- ...stom-scikit-learn-prebuilt-container.ipynb | 12 ++++--- ...started_with_model_monitoring_custom.ipynb | 9 +++-- .../chicago_taxi_fare_prediction.ipynb | 6 ++-- 10 files changed, 111 insertions(+), 55 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb b/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb index 095719d3c..04e9023da 100644 --- a/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb @@ -488,7 +488,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -507,7 +508,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -577,8 +579,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -986,7 +990,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_newsaggr.tar.gz" ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_newsaggr.tar.gz" + ] }, { "cell_type": "markdown", @@ -1201,11 +1206,14 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", + " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", + " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", + " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1213,13 +1221,15 @@ "print(\"custompythonpackagetrainingjob-run\")\n", "artifacts = print_pipeline_output(pipeline, \"custompythonpackagetrainingjob-run\")\n", "print(\"\\n\\n\")\n", - "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", + "output = json.loads(output[0])\n", "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n", "print(\"\\n\\n\")\n", "print(\"endpoint-create\")\n", "artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n", "print(\"\\n\\n\")\n", - "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", + "output = json.loads(output[0])\n", "endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n", " \"resourceName\"\n", "]\n", @@ -1384,7 +1394,8 @@ "source": [ "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "! rm -rf custom custom.tar.csv" ] } diff --git a/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb index 3bcdd09b0..32c4f70a5 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb @@ -456,7 +456,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -475,7 +476,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -544,8 +546,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -565,7 +569,6 @@ "outputs": [], "source": [ "from google.cloud import aiplatform\n", - "from kfp import dsl\n", "from kfp.v2 import compiler\n", "from kfp.v2.dsl import Artifact, Output, component" ] @@ -906,7 +909,8 @@ }, "outputs": [], "source": [ - "!gcloud storage cp $COMPOSER_DAG_FILENAME $COMPOSER_DAG_GCS/" ] + "!gcloud storage cp $COMPOSER_DAG_FILENAME $COMPOSER_DAG_GCS/" + ] }, { "cell_type": "code", @@ -916,7 +920,8 @@ }, "outputs": [], "source": [ - "!gcloud storage ls --long $COMPOSER_DAG_GCS/$COMPOSER_DAG_FILENAME" ] + "!gcloud storage ls --long $COMPOSER_DAG_GCS/$COMPOSER_DAG_FILENAME" + ] }, { "cell_type": "markdown", @@ -1147,7 +1152,6 @@ "# You can change the `text` and `emoji_str` parameters here to update the pipeline output\n", "def pipeline():\n", " data_processing_task_dag_name = COMPOSER_DAG_NAME\n", - " data_processing_task = trigger_airflow_dag(\n", " dag_name=data_processing_task_dag_name,\n", " composer_client_id=COMPOSER_CLIENT_ID,\n", " composer_webserver_id=COMPOSER_WEB_URI,\n", @@ -1302,11 +1306,14 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", + " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", + " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", + " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1380,7 +1387,8 @@ "! bq rm -r -f $PROJECT_ID:$DATASET_NAME\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", " \n", + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", + " \n", "! rm get_composer_config.py data_orchestration_bq_example_dag.py" ] } diff --git a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb index 370d3b5ed..e9abe72e7 100644 --- a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb @@ -211,9 +211,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -378,7 +380,9 @@ "# @markdown **Option 1: Hugging Face**\n", "\n", "# @markdown **Hugging Face Dataset Name:**\n", - "HF_DATASET = \"\" # @param {type:\"string\", placeholder: \"e.g. trl-lib/chatbot_arena_completions\"}\n", + "HF_DATASET = (\n", + " \"\" # @param {type:\"string\", placeholder: \"e.g. trl-lib/chatbot_arena_completions\"}\n", + ")\n", "# @markdown **Set the dataset type:** Refer to [Axolotl config file](https://github.com/axolotl-ai-cloud/axolotl/blob/6ba5c0ed2c42a0e069b28c83646ee5a2a6904430/docs/config.qmd#L140) for more details.\n", "HF_DATASET_TYPE = \"\" # @param {type:\"string\", placeholder: \"e.g. chat_template\"}\n", "if HF_DATASET:\n", @@ -509,7 +513,9 @@ " custom_job as gca_custom_job_compat\n", "\n", "# @markdown Acceletor type to use for training.\n", - "training_accelerator_type = \"NVIDIA_H100_80GB\" # @param [\"NVIDIA_H100_80GB\", \"NVIDIA_A100_80GB\"]\n", + "training_accelerator_type = (\n", + " \"NVIDIA_H100_80GB\" # @param [\"NVIDIA_H100_80GB\", \"NVIDIA_A100_80GB\"]\n", + ")\n", "\n", "\n", "replica_count = 1\n", @@ -546,7 +552,8 @@ "vertex_ai_config_path = AXOLOTL_CONFIG_PATH\n", "# Copy the config file to the bucket.\n", "if AXOLOTL_SOURCE == \"LOCAL\":\n", - " ! gcloud storage cp $AXOLOTL_CONFIG_PATH $MODEL_BUCKET/config/\n", " vertex_ai_config_path = f\"{common_util.gcs_fuse_path(MODEL_BUCKET)}/config/{pathlib.Path(AXOLOTL_CONFIG_PATH).name}\"\n", + " ! gcloud storage cp $AXOLOTL_CONFIG_PATH $MODEL_BUCKET/config/\n", + " vertex_ai_config_path = f\"{common_util.gcs_fuse_path(MODEL_BUCKET)}/config/{pathlib.Path(AXOLOTL_CONFIG_PATH).name}\"\n", "\n", "job_name = common_util.get_job_name_with_datetime(\"axolotl-train\")\n", "AXOLOTL_OUTPUT_GCS_URI = f\"{BASE_AXOLOTL_OUTPUT_GCS_URI}/{job_name}\"\n", @@ -1055,7 +1062,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb index 7c3ebbb66..a973050c8 100644 --- a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb @@ -186,9 +186,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -226,7 +228,9 @@ "# @markdown Alternatively, you can also load the original Gemma models for finetuning and serving from Vertex AI after accepting the agreement.\n", "\n", "# @markdown **Select and fill one of the three following sections.**\n", - "LOAD_MODEL_FROM = \"Hugging Face\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + "LOAD_MODEL_FROM = (\n", + " \"Hugging Face\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + ")\n", "\n", "# @markdown ---\n", "\n", @@ -260,7 +264,8 @@ "\n", " ! mkdir -p ./gemma\n", " ! curl -X GET \"{signed_url}\" | tar -xzvf - -C ./gemma/\n", - " ! gcloud storage cp --recursive ./gemma/* {MODEL_BUCKET}\n", "\n", + " ! gcloud storage cp --recursive ./gemma/* {MODEL_BUCKET}\n", + "\n", " model_path_prefix = MODEL_BUCKET\n", " HF_TOKEN = \"\"\n", "else:\n", @@ -390,7 +395,9 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -453,7 +460,9 @@ "labels[\"versioned-mg-tune\"] = f\"{labels['mg-tune']}-{versioned_model_id}\"\n", "\n", "# @markdown Accelerator type to use for training.\n", - "training_accelerator_type = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", + "training_accelerator_type = (\n", + " \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", + ")\n", "\n", "# The pre-built training docker image.\n", "if training_accelerator_type == \"NVIDIA_A100_80GB\":\n", @@ -938,7 +947,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb b/notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb index adb25eab6..3d6c06dce 100644 --- a/notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb +++ b/notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb @@ -348,7 +348,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={REGION} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={REGION} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -741,7 +742,8 @@ "outputs": [], "source": [ "%cd app\n", - "!gcloud storage cp *.sav *.csv {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "%cd .." + "!gcloud storage cp *.sav *.csv {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", + "%cd .." ] }, { @@ -1539,7 +1541,8 @@ "\n", "# Delete the Cloud Storage bucket\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_custom_tabular_regression_online.ipynb b/notebooks/community/sdk/sdk_custom_tabular_regression_online.ipynb index a384b225e..75b9e53be 100644 --- a/notebooks/community/sdk/sdk_custom_tabular_regression_online.ipynb +++ b/notebooks/community/sdk/sdk_custom_tabular_regression_online.ipynb @@ -475,7 +475,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -494,7 +495,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -919,7 +921,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" ] + "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz" + ] }, { "cell_type": "markdown", @@ -1424,7 +1427,8 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_pytorch_lightning_custom_container_training.ipynb b/notebooks/community/sdk/sdk_pytorch_lightning_custom_container_training.ipynb index 651985877..7abcf6762 100644 --- a/notebooks/community/sdk/sdk_pytorch_lightning_custom_container_training.ipynb +++ b/notebooks/community/sdk/sdk_pytorch_lightning_custom_container_training.ipynb @@ -505,7 +505,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -524,7 +525,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -1522,7 +1524,8 @@ "! gcloud storage rm --recursive $CONTENT_DIR\n", "\n", "if delete_bucket and \"BUCKET_URI\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/migration/sdk-custom-scikit-learn-prebuilt-container.ipynb b/notebooks/official/migration/sdk-custom-scikit-learn-prebuilt-container.ipynb index 3e847b9ac..281b96817 100644 --- a/notebooks/official/migration/sdk-custom-scikit-learn-prebuilt-container.ipynb +++ b/notebooks/official/migration/sdk-custom-scikit-learn-prebuilt-container.ipynb @@ -320,7 +320,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -698,7 +699,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_census.tar.gz" ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_census.tar.gz" + ] }, { "cell_type": "markdown", @@ -963,7 +965,8 @@ " for i in INSTANCES:\n", " f.write(json.dumps(i) + \"\\n\")\n", "\n", - "! gcloud storage cat $gcs_input_uri" ] + "! gcloud storage cat $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -1375,7 +1378,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb b/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb index ac33d1a28..06afe99cf 100644 --- a/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb +++ b/notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb @@ -364,7 +364,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -927,7 +928,8 @@ "with open(\"schema.yaml\", \"w\") as f:\n", " f.write(yaml)\n", "\n", - "! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" ] + "! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" + ] }, { "cell_type": "markdown", @@ -1404,7 +1406,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n", + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", + "\n", "# Delete the locally generated files\n", "! rm -f schema.yaml\n", "\n", diff --git a/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb b/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb index 95f62cd65..72af53e31 100644 --- a/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb +++ b/notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb @@ -365,7 +365,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -1466,7 +1467,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From c8bb028efb1ff354d13099aaf8b1c0cb88c36a67 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Mon, 22 Dec 2025 18:02:01 +0000 Subject: [PATCH 5/6] removed changes for model garden --- ...el_garden_axolotl_gpt_oss_finetuning.ipynb | 16 +++++--------- ...el_garden_gemma_finetuning_on_vertex.ipynb | 22 ++++++++----------- 2 files changed, 15 insertions(+), 23 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb index e9abe72e7..a99c7a16e 100644 --- a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb @@ -211,10 +211,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -380,9 +380,7 @@ "# @markdown **Option 1: Hugging Face**\n", "\n", "# @markdown **Hugging Face Dataset Name:**\n", - "HF_DATASET = (\n", - " \"\" # @param {type:\"string\", placeholder: \"e.g. trl-lib/chatbot_arena_completions\"}\n", - ")\n", + "HF_DATASET = \"\" # @param {type:\"string\", placeholder: \"e.g. trl-lib/chatbot_arena_completions\"}\n", "# @markdown **Set the dataset type:** Refer to [Axolotl config file](https://github.com/axolotl-ai-cloud/axolotl/blob/6ba5c0ed2c42a0e069b28c83646ee5a2a6904430/docs/config.qmd#L140) for more details.\n", "HF_DATASET_TYPE = \"\" # @param {type:\"string\", placeholder: \"e.g. chat_template\"}\n", "if HF_DATASET:\n", @@ -513,9 +511,7 @@ " custom_job as gca_custom_job_compat\n", "\n", "# @markdown Acceletor type to use for training.\n", - "training_accelerator_type = (\n", - " \"NVIDIA_H100_80GB\" # @param [\"NVIDIA_H100_80GB\", \"NVIDIA_A100_80GB\"]\n", - ")\n", + "training_accelerator_type = \"NVIDIA_H100_80GB\" # @param [\"NVIDIA_H100_80GB\", \"NVIDIA_A100_80GB\"]\n", "\n", "\n", "replica_count = 1\n", @@ -552,7 +548,7 @@ "vertex_ai_config_path = AXOLOTL_CONFIG_PATH\n", "# Copy the config file to the bucket.\n", "if AXOLOTL_SOURCE == \"LOCAL\":\n", - " ! gcloud storage cp $AXOLOTL_CONFIG_PATH $MODEL_BUCKET/config/\n", + " ! gsutil -m cp $AXOLOTL_CONFIG_PATH $MODEL_BUCKET/config/\n", " vertex_ai_config_path = f\"{common_util.gcs_fuse_path(MODEL_BUCKET)}/config/{pathlib.Path(AXOLOTL_CONFIG_PATH).name}\"\n", "\n", "job_name = common_util.get_job_name_with_datetime(\"axolotl-train\")\n", @@ -1062,7 +1058,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb index a973050c8..b5d314b7a 100644 --- a/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma_finetuning_on_vertex.ipynb @@ -186,10 +186,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -214,7 +214,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -228,9 +228,7 @@ "# @markdown Alternatively, you can also load the original Gemma models for finetuning and serving from Vertex AI after accepting the agreement.\n", "\n", "# @markdown **Select and fill one of the three following sections.**\n", - "LOAD_MODEL_FROM = (\n", - " \"Hugging Face\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", - ")\n", + "LOAD_MODEL_FROM = \"Hugging Face\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", "\n", "# @markdown ---\n", "\n", @@ -264,7 +262,7 @@ "\n", " ! mkdir -p ./gemma\n", " ! curl -X GET \"{signed_url}\" | tar -xzvf - -C ./gemma/\n", - " ! gcloud storage cp --recursive ./gemma/* {MODEL_BUCKET}\n", + " ! gsutil -m cp -R ./gemma/* {MODEL_BUCKET}\n", "\n", " model_path_prefix = MODEL_BUCKET\n", " HF_TOKEN = \"\"\n", @@ -395,8 +393,8 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", @@ -460,9 +458,7 @@ "labels[\"versioned-mg-tune\"] = f\"{labels['mg-tune']}-{versioned_model_id}\"\n", "\n", "# @markdown Accelerator type to use for training.\n", - "training_accelerator_type = (\n", - " \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", - ")\n", + "training_accelerator_type = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", "\n", "# The pre-built training docker image.\n", "if training_accelerator_type == \"NVIDIA_A100_80GB\":\n", @@ -947,7 +943,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], From fe15844d25a6a5db9ce8dba255da248ccabd212c Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 23 Dec 2025 11:12:48 +0530 Subject: [PATCH 6/6] Update model_garden_axolotl_gpt_oss_finetuning.ipynb --- .../model_garden_axolotl_gpt_oss_finetuning.ipynb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb index a99c7a16e..48572ef3e 100644 --- a/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_axolotl_gpt_oss_finetuning.ipynb @@ -214,7 +214,7 @@ " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -239,6 +239,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\""