From 572ccb31361dd79c6366936dc73760b8ae1a3e05 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Fri, 17 Oct 2025 11:57:31 +0000 Subject: [PATCH 1/5] Migrate gsutil usage to gcloud storage --- ...d_with_vertex_ml_metadata_and_automl.ipynb | 15 ++++------ .../stage6/get_started_with_fastapi.ipynb | 15 ++++------ ...ular_classification_model_evaluation.ipynb | 19 +++++------- ...el_garden_pytorch_qwen2_5_finetuning.ipynb | 16 ++++------ ...nts_bqml_pipeline_demand_forecasting.ipynb | 30 +++++++------------ .../sdk_custom_tabular_regression_batch.ipynb | 11 +++---- ...g_for_custom_model_online_prediction.ipynb | 9 ++---- ..._cloud_pipeline_components_bqml_text.ipynb | 23 +++++--------- ...stom_batch_prediction_feature_filter.ipynb | 9 ++---- .../vizier/get_started_vertex_vizier.ipynb | 15 ++++------ 10 files changed, 57 insertions(+), 105 deletions(-) diff --git a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata_and_automl.ipynb b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata_and_automl.ipynb index 4ad30f66e..509b97b5d 100644 --- a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata_and_automl.ipynb +++ b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata_and_automl.ipynb @@ -463,8 +463,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -483,8 +482,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -685,12 +683,10 @@ "source": [ "FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -1468,8 +1464,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_fastapi.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_fastapi.ipynb index 5b0b85e01..7616bfb38 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_fastapi.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_fastapi.ipynb @@ -444,8 +444,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -464,8 +463,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1027,8 +1025,7 @@ "MODEL_DIR=$1\n", "\n", "mkdir -p ./serve/model/\n", - "gsutil cp -r ${MODEL_DIR} ./serve/model/ \n", - "\n", + "gcloud storage cp --recursive ${MODEL_DIR} ./serve/model/ \n", "\n", "cat > ./serve/Dockerfile < Date: Thu, 11 Dec 2025 19:16:28 +0000 Subject: [PATCH 2/5] Manual Changes-Migrate gsutil usage to gcloud storage --- .../automl_tabular_classification_model_evaluation.ipynb | 4 ++-- ...pipeline_components_bqml_pipeline_demand_forecasting.ipynb | 2 +- .../model_monitoring_for_custom_model_online_prediction.ipynb | 2 +- .../google_cloud_pipeline_components_bqml_text.ipynb | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/notebooks/community/model_evaluation/automl_tabular_classification_model_evaluation.ipynb b/notebooks/community/model_evaluation/automl_tabular_classification_model_evaluation.ipynb index 286509988..d81631af8 100644 --- a/notebooks/community/model_evaluation/automl_tabular_classification_model_evaluation.ipynb +++ b/notebooks/community/model_evaluation/automl_tabular_classification_model_evaluation.ipynb @@ -624,10 +624,10 @@ }, "outputs": [], "source": [ - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.objectCreator\"\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { diff --git a/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb b/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb index cc3af6c12..cdb8a82a1 100644 --- a/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb +++ b/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @@ -621,7 +621,7 @@ }, "outputs": [], "source": [ - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding {BUCKET_URI} --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", "! gcloud storage buckets add-iam-policy-binding {BUCKET_URI} --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] diff --git a/notebooks/official/model_monitoring_v2/model_monitoring_for_custom_model_online_prediction.ipynb b/notebooks/official/model_monitoring_v2/model_monitoring_for_custom_model_online_prediction.ipynb index cbc04a511..4ab088d48 100644 --- a/notebooks/official/model_monitoring_v2/model_monitoring_for_custom_model_online_prediction.ipynb +++ b/notebooks/official/model_monitoring_v2/model_monitoring_for_custom_model_online_prediction.ipynb @@ -654,7 +654,7 @@ ")\n", "TRAINING_URI = f\"{BUCKET_URI}/model-monitoring/churn/churn_training.csv\"\n", "\n", - "# TODO: Command \"gsutil copy\" not found in migration guide. Manual review required.\n", "\n", + "! gcloud storage cp $PUBLIC_TRAINING_DATASET $TRAINING_URI", "TRAINING_DATASET = ml_monitoring.spec.MonitoringInput(\n", " gcs_uri=TRAINING_URI, data_format=\"csv\"\n", ")" diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb index ef0f50e30..0368eedf5 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb @@ -411,9 +411,9 @@ }, "outputs": [], "source": [ - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { From e4e7d235d5cbdbed2b46f2aa2a0cc382bc16fe15 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Sun, 14 Dec 2025 04:42:27 +0000 Subject: [PATCH 3/5] Changes for 4327 --- .../automl_tabular_classification_model_evaluation.ipynb | 2 -- ...d_pipeline_components_bqml_pipeline_demand_forecasting.ipynb | 1 - .../pipelines/google_cloud_pipeline_components_bqml_text.ipynb | 2 -- 3 files changed, 5 deletions(-) diff --git a/notebooks/community/model_evaluation/automl_tabular_classification_model_evaluation.ipynb b/notebooks/community/model_evaluation/automl_tabular_classification_model_evaluation.ipynb index d81631af8..8dc162cff 100644 --- a/notebooks/community/model_evaluation/automl_tabular_classification_model_evaluation.ipynb +++ b/notebooks/community/model_evaluation/automl_tabular_classification_model_evaluation.ipynb @@ -624,10 +624,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.objectCreator\"\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { diff --git a/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb b/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb index cdb8a82a1..70546fda5 100644 --- a/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb +++ b/notebooks/community/pipelines/google_cloud_pipeline_components_bqml_pipeline_demand_forecasting.ipynb @@ -621,7 +621,6 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding {BUCKET_URI} --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", "! gcloud storage buckets add-iam-policy-binding {BUCKET_URI} --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb index 0368eedf5..1bdf42a7c 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_bqml_text.ipynb @@ -411,9 +411,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { From ce22c64f7528eaab2a28f06a23d65a07b09a74af Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Wed, 17 Dec 2025 11:45:26 +0000 Subject: [PATCH 4/5] fixing linting erro --- ...d_with_vertex_ml_metadata_and_automl.ipynb | 15 +++++--- .../stage6/get_started_with_fastapi.ipynb | 15 +++++--- ...ular_classification_model_evaluation.ipynb | 15 +++++--- ...el_garden_pytorch_qwen2_5_finetuning.ipynb | 16 ++++++--- ...nts_bqml_pipeline_demand_forecasting.ipynb | 35 ++++++++++++------- .../sdk_custom_tabular_regression_batch.ipynb | 9 +++-- ...g_for_custom_model_online_prediction.ipynb | 9 ++--- ..._cloud_pipeline_components_bqml_text.ipynb | 21 +++++++---- ...stom_batch_prediction_feature_filter.ipynb | 9 +++-- .../vizier/get_started_vertex_vizier.ipynb | 15 +++++--- 10 files changed, 106 insertions(+), 53 deletions(-) diff --git a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata_and_automl.ipynb b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata_and_automl.ipynb index 509b97b5d..148ff865b 100644 --- a/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata_and_automl.ipynb +++ b/notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata_and_automl.ipynb @@ -463,7 +463,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -482,7 +483,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -683,10 +685,12 @@ "source": [ "FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -1464,7 +1468,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_fastapi.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_fastapi.ipynb index 7616bfb38..7efa96d08 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_fastapi.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_fastapi.ipynb @@ -444,7 +444,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -463,7 +464,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -1025,7 +1027,8 @@ "MODEL_DIR=$1\n", "\n", "mkdir -p ./serve/model/\n", - "gcloud storage cp --recursive ${MODEL_DIR} ./serve/model/ \n", "\n", + "gcloud storage cp --recursive ${MODEL_DIR} ./serve/model/ \n", + "\n", "cat > ./serve/Dockerfile < Date: Mon, 22 Dec 2025 17:36:41 +0000 Subject: [PATCH 5/5] removed changes of model garden --- .../model_garden_pytorch_qwen2_5_finetuning.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_qwen2_5_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_qwen2_5_finetuning.ipynb index 998b77311..48d2a1fad 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_qwen2_5_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_qwen2_5_finetuning.ipynb @@ -184,10 +184,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -212,7 +212,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -361,8 +361,8 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", @@ -1031,7 +1031,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ],