From 842f0527c7fdae63fa4375e6e6248d9b25fd9e77 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Thu, 16 Oct 2025 13:03:24 +0000 Subject: [PATCH 1/5] Migrate gsutil usage to gcloud storage --- .../SDK_Custom_Container_XAI.ipynb | 12 +++---- ... Tuning Training Job with TensorFlow.ipynb | 12 +++---- ...ral Language Text Sentiment Analysis.ipynb | 24 +++++--------- .../get_started_vertex_feature_store.ipynb | 4 +-- ...tarted_with_bqml_pipeline_components.ipynb | 31 +++++++------------ ...l_garden_gemma2_deployment_on_vertex.ipynb | 13 +++----- ...del_garden_pytorch_llama3_finetuning.ipynb | 15 +++------ ...onents_TPU_model_train_upload_deploy.ipynb | 14 ++++----- ...tender_vs_champion_deployment_method.ipynb | 14 ++++----- ..._search_create_multimodal_embeddings.ipynb | 9 ++---- 10 files changed, 53 insertions(+), 95 deletions(-) diff --git a/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb b/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb index 1a71bae50..4bd54b517 100644 --- a/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb +++ b/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb @@ -405,8 +405,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -425,8 +424,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -698,8 +696,7 @@ }, "outputs": [], "source": [ - "!gsutil cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" - ] + "!gcloud storage cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] }, { "cell_type": "markdown", @@ -1516,8 +1513,7 @@ "\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/migration/UJ11 legacy HyperParameter Tuning Training Job with TensorFlow.ipynb b/notebooks/community/migration/UJ11 legacy HyperParameter Tuning Training Job with TensorFlow.ipynb index f9d959b9b..16860d24d 100644 --- a/notebooks/community/migration/UJ11 legacy HyperParameter Tuning Training Job with TensorFlow.ipynb +++ b/notebooks/community/migration/UJ11 legacy HyperParameter Tuning Training Job with TensorFlow.ipynb @@ -312,8 +312,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION gs://$BUCKET_NAME" - ] + "! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -332,8 +331,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al gs://$BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -614,8 +612,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz gs://$BUCKET_NAME/hpt_boston_housing.tar.gz" - ] + "! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/hpt_boston_housing.tar.gz" ] }, { "cell_type": "markdown", @@ -1370,8 +1367,7 @@ "delete_bucket = True\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r gs://$BUCKET_NAME" - ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb b/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb index 982b8b3d2..1f445c570 100644 --- a/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb +++ b/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb @@ -326,8 +326,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION gs://$BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -346,8 +345,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al gs://$BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -485,8 +483,7 @@ }, "outputs": [], "source": [ - "! gsutil cat $IMPORT_FILE | head -n 10" - ] + "! gcloud storage cat $IMPORT_FILE | head -n 10" ] }, { "cell_type": "markdown", @@ -1209,15 +1206,12 @@ "gcs_input_uri = \"gs://\" + BUCKET_NAME + \"/test.csv\"\n", "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n", " item_1 = \"gs://cloud-samples-data/language/sentiment-positive.txt\"\n", - " ! gsutil cp $item_1 gs://$BUCKET_NAME\n", - " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-positive.txt\" + \"\\n\")\n", + " ! gcloud storage cp $item_1 gs://$BUCKET_NAME\n", " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-positive.txt\" + \"\\n\")\n", "\n", " item_2 = \"gs://cloud-samples-data/language/sentiment-negative.txt\"\n", - " ! gsutil cp $item_2 gs://$BUCKET_NAME\n", - " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-negative.txt\")\n", + " ! gcloud storage cp $item_2 gs://$BUCKET_NAME\n", " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-negative.txt\")\n", "\n", - "! gsutil cat $gcs_input_uri" - ] + "! gcloud storage cat $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -1381,8 +1375,7 @@ }, "outputs": [], "source": [ - "test_data = ! gsutil cat $IMPORT_FILE | head -n1\n", - "\n", + "test_data = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "\n", "test_item = str(test_data[0]).split(\",\")[0]\n", "test_label = str(test_data[0]).split(\",\")[1]\n", "\n", @@ -1615,8 +1608,7 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r gs://$BUCKET_NAME" - ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb b/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb index 78a4aae0e..47ad5f973 100644 --- a/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb @@ -1017,9 +1017,7 @@ "USERS_AVRO_FN = \"users.avro\"\n", "MOVIES_AVRO_FN = \"movies.avro\"\n", "\n", - "! gsutil cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n", - "! gsutil cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN" - ] + "! gcloud storage cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n", "! gcloud storage cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN" ] }, { "cell_type": "markdown", diff --git a/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb index 63668be8f..680ce7729 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb @@ -452,8 +452,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -472,8 +471,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -545,10 +543,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer ] }, { "cell_type": "markdown", @@ -1052,14 +1049,11 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gsutil cat $EXECUTE_OUTPUT\n", - " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gsutil cat $GCP_RESOURCES\n", - " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gsutil cat $EVAL_METRICS\n", - " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1081,15 +1075,13 @@ "print(\"\\n\\n\")\n", "print(\"model-upload\")\n", "artifacts = print_pipeline_output(pipeline, \"model-upload\")\n", - "output = !gsutil cat $artifacts\n", - "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n", "print(\"\\n\\n\")\n", "print(\"endpoint-create\")\n", "artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n", "print(\"\\n\\n\")\n", - "output = !gsutil cat $artifacts\n", - "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", "endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n", " \"resourceName\"\n", "]\n", @@ -1322,8 +1314,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb index 634df226d..9de86bccc 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb @@ -186,11 +186,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -214,8 +212,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -882,8 +880,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb index 84c4addc4..11c818053 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb @@ -162,11 +162,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -190,8 +188,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -253,8 +250,7 @@ " )\n", " HF_TOKEN = \"\"\n", "\n", - " ! gsutil -m cp -R $VERTEX_AI_MODEL_GARDEN_LLAMA3/* $MODEL_BUCKET\n", - "\n", + " ! gcloud storage cp --recursive $VERTEX_AI_MODEL_GARDEN_LLAMA3/* $MODEL_BUCKET\n", "\n", "# @markdown ---" ] }, @@ -838,8 +834,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb index 614d8e62a..e26416539 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @@ -292,8 +292,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -365,10 +364,10 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -1234,8 +1233,7 @@ "# Warning: Setting this to true deletes everything in your bucket\n", "delete_bucket = True\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "# Delete the Artifact Registry repository\n", "! gcloud artifacts repositories delete $REPOSITORY --location=$LOCATION --quiet\n", "\n", diff --git a/notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb b/notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb index 8c760540a..2d879b45f 100644 --- a/notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb +++ b/notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb @@ -309,8 +309,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -384,10 +383,10 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer ] }, { "cell_type": "markdown", @@ -1239,8 +1238,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = True\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "# Remove the local pipeline package file\n", "! rm multicontender_vs_champion.json" ] diff --git a/notebooks/official/vector_search/sdk_vector_search_create_multimodal_embeddings.ipynb b/notebooks/official/vector_search/sdk_vector_search_create_multimodal_embeddings.ipynb index 63bd6203a..ff8d98d64 100644 --- a/notebooks/official/vector_search/sdk_vector_search_create_multimodal_embeddings.ipynb +++ b/notebooks/official/vector_search/sdk_vector_search_create_multimodal_embeddings.ipynb @@ -275,8 +275,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -1062,8 +1061,7 @@ "source": [ "UNIQUE_FOLDER_NAME = \"embeddings_folder_unique\"\n", "EMBEDDINGS_INITIAL_URI = f\"{BUCKET_URI}/{UNIQUE_FOLDER_NAME}/\"\n", - "! gsutil cp {embeddings_file.name} {EMBEDDINGS_INITIAL_URI}" - ] + "! gcloud storage cp {embeddings_file.name} {EMBEDDINGS_INITIAL_URI}" ] }, { "cell_type": "markdown", @@ -1369,8 +1367,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From 1fa7a809dbd7f571e5ce1f8ac7675d333bbe8483 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 12 Dec 2025 18:31:29 +0000 Subject: [PATCH 2/5] Changes for 4318 --- ...AutoML Natural Language Text Sentiment Analysis.ipynb | 4 +++- .../ml_ops/stage2/get_started_vertex_feature_store.ipynb | 3 ++- .../get_started_with_bqml_pipeline_components.ipynb | 3 +-- .../model_garden_gemma2_deployment_on_vertex.ipynb | 9 ++++----- .../model_garden_pytorch_llama3_finetuning.ipynb | 5 +++-- ...peline_components_TPU_model_train_upload_deploy.ipynb | 3 ++- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb b/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb index 1f445c570..bdc6e01b8 100644 --- a/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb +++ b/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb @@ -1206,7 +1206,9 @@ "gcs_input_uri = \"gs://\" + BUCKET_NAME + \"/test.csv\"\n", "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n", " item_1 = \"gs://cloud-samples-data/language/sentiment-positive.txt\"\n", - " ! gcloud storage cp $item_1 gs://$BUCKET_NAME\n", " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-positive.txt\" + \"\\n\")\n", + " ! gcloud storage cp $item_1 gs://$BUCKET_NAME\n", + " ! gcloud storage cp $item_1 gs://$BUCKET_NAME\n", + " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-positive.txt\" + \"\\n\")\n", "\n", " item_2 = \"gs://cloud-samples-data/language/sentiment-negative.txt\"\n", " ! gcloud storage cp $item_2 gs://$BUCKET_NAME\n", " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-negative.txt\")\n", diff --git a/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb b/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb index 47ad5f973..b9bf0f613 100644 --- a/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb @@ -1017,7 +1017,8 @@ "USERS_AVRO_FN = \"users.avro\"\n", "MOVIES_AVRO_FN = \"movies.avro\"\n", "\n", - "! gcloud storage cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n", "! gcloud storage cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN" ] + "! gcloud storage cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n", + "! gcloud storage cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN" ] }, { "cell_type": "markdown", diff --git a/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb index 680ce7729..03d454e98 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb @@ -544,8 +544,7 @@ "outputs": [], "source": [ "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. - ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer " ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb index 9de86bccc..2a667ea85 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb @@ -186,9 +186,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -212,12 +213,10 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", - "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", - "\n", + "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n","\n", "import vertexai\n", "\n", "vertexai.init(\n", diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb index 11c818053..8a5524279 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb @@ -162,9 +162,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb index e26416539..b3def7fbe 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @@ -292,7 +292,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", From fac4d5ecb055a2f116cc5c8135e374e29043bd3d Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Wed, 17 Dec 2025 17:40:54 +0000 Subject: [PATCH 3/5] fix linter issue for 4318 --- .../SDK_Custom_Container_XAI.ipynb | 12 +++++--- ... Tuning Training Job with TensorFlow.ipynb | 12 +++++--- ...ral Language Text Sentiment Analysis.ipynb | 29 ++++++++++-------- .../get_started_vertex_feature_store.ipynb | 5 ++-- ...tarted_with_bqml_pipeline_components.ipynb | 30 ++++++++++++------- ...l_garden_gemma2_deployment_on_vertex.ipynb | 14 +++++---- ...del_garden_pytorch_llama3_finetuning.ipynb | 14 +++++---- .../mpg_container/trainer/train.py | 3 -- notebooks/notebook_template_review.py | 1 - ...onents_TPU_model_train_upload_deploy.ipynb | 12 ++++---- ...tender_vs_champion_deployment_method.ipynb | 13 ++++---- ..._search_create_multimodal_embeddings.ipynb | 9 ++++-- 12 files changed, 95 insertions(+), 59 deletions(-) diff --git a/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb b/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb index 4bd54b517..96566721a 100644 --- a/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb +++ b/notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb @@ -405,7 +405,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -424,7 +425,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -696,7 +698,8 @@ }, "outputs": [], "source": [ - "!gcloud storage cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] + "!gcloud storage cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" + ] }, { "cell_type": "markdown", @@ -1513,7 +1516,8 @@ "\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/community/migration/UJ11 legacy HyperParameter Tuning Training Job with TensorFlow.ipynb b/notebooks/community/migration/UJ11 legacy HyperParameter Tuning Training Job with TensorFlow.ipynb index 16860d24d..ffa5e65a3 100644 --- a/notebooks/community/migration/UJ11 legacy HyperParameter Tuning Training Job with TensorFlow.ipynb +++ b/notebooks/community/migration/UJ11 legacy HyperParameter Tuning Training Job with TensorFlow.ipynb @@ -312,7 +312,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME" ] + "! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -331,7 +332,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -612,7 +614,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/hpt_boston_housing.tar.gz" ] + "! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/hpt_boston_housing.tar.gz" + ] }, { "cell_type": "markdown", @@ -1367,7 +1370,8 @@ "delete_bucket = True\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb b/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb index bdc6e01b8..0ec6ec283 100644 --- a/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb +++ b/notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb @@ -326,7 +326,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" ] + "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -345,7 +346,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -381,11 +383,9 @@ "import json\n", "import os\n", "import sys\n", - "import time\n", "\n", "from google.cloud import automl\n", - "from google.protobuf.json_format import MessageToJson\n", - "from google.protobuf.struct_pb2 import Value" + "from google.protobuf.json_format import MessageToJson" ] }, { @@ -483,7 +483,8 @@ }, "outputs": [], "source": [ - "! gcloud storage cat $IMPORT_FILE | head -n 10" ] + "! gcloud storage cat $IMPORT_FILE | head -n 10" + ] }, { "cell_type": "markdown", @@ -1206,14 +1207,16 @@ "gcs_input_uri = \"gs://\" + BUCKET_NAME + \"/test.csv\"\n", "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n", " item_1 = \"gs://cloud-samples-data/language/sentiment-positive.txt\"\n", - " ! gcloud storage cp $item_1 gs://$BUCKET_NAME\n", + " ! gcloud storage cp $item_1 gs://$BUCKET_NAME\n", " ! gcloud storage cp $item_1 gs://$BUCKET_NAME\n", " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-positive.txt\" + \"\\n\")\n", "\n", " item_2 = \"gs://cloud-samples-data/language/sentiment-negative.txt\"\n", - " ! gcloud storage cp $item_2 gs://$BUCKET_NAME\n", " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-negative.txt\")\n", + " ! gcloud storage cp $item_2 gs://$BUCKET_NAME\n", + " f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-negative.txt\")\n", "\n", - "! gcloud storage cat $gcs_input_uri" ] + "! gcloud storage cat $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -1377,7 +1380,8 @@ }, "outputs": [], "source": [ - "test_data = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "\n", + "test_data = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "\n", "test_item = str(test_data[0]).split(\",\")[0]\n", "test_label = str(test_data[0]).split(\",\")[1]\n", "\n", @@ -1610,12 +1614,13 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" + ] } ], "metadata": { "colab": { - "name": "UJ8 legacy AutoML Natural Language - Text Sentiment Analysis.ipynb", + "name": "UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb", "toc_visible": true }, "kernelspec": { diff --git a/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb b/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb index b9bf0f613..9eaaa85bb 100644 --- a/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb +++ b/notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb @@ -1017,8 +1017,9 @@ "USERS_AVRO_FN = \"users.avro\"\n", "MOVIES_AVRO_FN = \"movies.avro\"\n", "\n", - "! gcloud storage cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n", - "! gcloud storage cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN" ] + "! gcloud storage cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n", + "! gcloud storage cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN" + ] }, { "cell_type": "markdown", diff --git a/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb index 03d454e98..b4599b047 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb @@ -452,7 +452,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -471,7 +472,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -543,8 +545,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer " ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer " + ] }, { "cell_type": "markdown", @@ -1048,11 +1052,14 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", + " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", + " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", + " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1074,13 +1081,15 @@ "print(\"\\n\\n\")\n", "print(\"model-upload\")\n", "artifacts = print_pipeline_output(pipeline, \"model-upload\")\n", - "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", + "output = json.loads(output[0])\n", "model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n", "print(\"\\n\\n\")\n", "print(\"endpoint-create\")\n", "artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n", "print(\"\\n\\n\")\n", - "output = !gcloud storage cat $artifacts\n", "output = json.loads(output[0])\n", + "output = !gcloud storage cat $artifacts\n", + "output = json.loads(output[0])\n", "endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n", " \"resourceName\"\n", "]\n", @@ -1313,7 +1322,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb index 2a667ea85..48d693040 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb @@ -186,10 +186,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -213,10 +214,12 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", - "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n","\n", + "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", + "\n", "import vertexai\n", "\n", "vertexai.init(\n", @@ -879,7 +882,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb index 8a5524279..5277ea6ee 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb @@ -162,10 +162,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -189,7 +190,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -251,7 +253,8 @@ " )\n", " HF_TOKEN = \"\"\n", "\n", - " ! gcloud storage cp --recursive $VERTEX_AI_MODEL_GARDEN_LLAMA3/* $MODEL_BUCKET\n", "\n", + " ! gcloud storage cp --recursive $VERTEX_AI_MODEL_GARDEN_LLAMA3/* $MODEL_BUCKET\n", + "\n", "# @markdown ---" ] }, @@ -835,7 +838,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/persistent_resource/training_code/mpg_container/trainer/train.py b/notebooks/community/persistent_resource/training_code/mpg_container/trainer/train.py index b61d9884c..cb6534509 100644 --- a/notebooks/community/persistent_resource/training_code/mpg_container/trainer/train.py +++ b/notebooks/community/persistent_resource/training_code/mpg_container/trainer/train.py @@ -14,10 +14,7 @@ import argparse import logging -import os -import numpy as np import pandas as pd -import pathlib import tensorflow as tf from tensorflow import keras diff --git a/notebooks/notebook_template_review.py b/notebooks/notebook_template_review.py index f4f1a7f09..af6113730 100644 --- a/notebooks/notebook_template_review.py +++ b/notebooks/notebook_template_review.py @@ -39,7 +39,6 @@ import json import os import sys -import urllib.request import csv from enum import Enum from abc import ABC, abstractmethod diff --git a/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb b/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb index b3def7fbe..1f2d060e0 100644 --- a/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb +++ b/notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb @@ -292,7 +292,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { @@ -366,9 +366,10 @@ "outputs": [], "source": [ "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -1234,7 +1235,8 @@ "# Warning: Setting this to true deletes everything in your bucket\n", "delete_bucket = True\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "# Delete the Artifact Registry repository\n", "! gcloud artifacts repositories delete $REPOSITORY --location=$LOCATION --quiet\n", "\n", diff --git a/notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb b/notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb index 2d879b45f..286034e13 100644 --- a/notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb +++ b/notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb @@ -309,7 +309,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -384,9 +385,10 @@ "outputs": [], "source": [ "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. - ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -1238,7 +1240,8 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = True\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "# Remove the local pipeline package file\n", "! rm multicontender_vs_champion.json" ] diff --git a/notebooks/official/vector_search/sdk_vector_search_create_multimodal_embeddings.ipynb b/notebooks/official/vector_search/sdk_vector_search_create_multimodal_embeddings.ipynb index ff8d98d64..58fe43f16 100644 --- a/notebooks/official/vector_search/sdk_vector_search_create_multimodal_embeddings.ipynb +++ b/notebooks/official/vector_search/sdk_vector_search_create_multimodal_embeddings.ipynb @@ -275,7 +275,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -1061,7 +1062,8 @@ "source": [ "UNIQUE_FOLDER_NAME = \"embeddings_folder_unique\"\n", "EMBEDDINGS_INITIAL_URI = f\"{BUCKET_URI}/{UNIQUE_FOLDER_NAME}/\"\n", - "! gcloud storage cp {embeddings_file.name} {EMBEDDINGS_INITIAL_URI}" ] + "! gcloud storage cp {embeddings_file.name} {EMBEDDINGS_INITIAL_URI}" + ] }, { "cell_type": "markdown", @@ -1367,7 +1369,8 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From 71c77edae6f72d4d7ffed287658cb2f25896c98a Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Mon, 22 Dec 2025 19:05:59 +0000 Subject: [PATCH 4/5] removed changes for model garden: --- .../model_garden_gemma2_deployment_on_vertex.ipynb | 8 ++++---- .../model_garden_pytorch_llama3_finetuning.ipynb | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb index 48d693040..7e4c69553 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb @@ -186,10 +186,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -L {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -214,7 +214,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -882,7 +882,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb index 5277ea6ee..84c4addc4 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_finetuning.ipynb @@ -162,10 +162,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -190,7 +190,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -253,7 +253,7 @@ " )\n", " HF_TOKEN = \"\"\n", "\n", - " ! gcloud storage cp --recursive $VERTEX_AI_MODEL_GARDEN_LLAMA3/* $MODEL_BUCKET\n", + " ! gsutil -m cp -R $VERTEX_AI_MODEL_GARDEN_LLAMA3/* $MODEL_BUCKET\n", "\n", "# @markdown ---" ] @@ -838,7 +838,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], From 52e54a171257791a5f7e52b9074631dfdcf33fef Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 23 Dec 2025 11:28:40 +0530 Subject: [PATCH 5/5] Update model_garden_gemma2_deployment_on_vertex.ipynb --- .../model_garden/model_garden_gemma2_deployment_on_vertex.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb index 7e4c69553..634df226d 100644 --- a/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_gemma2_deployment_on_vertex.ipynb @@ -189,7 +189,7 @@ " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -L {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n",