Skip to content

Commit 1867536

Browse files
googlyrahmangurusai-voleti
googlyrahman
and
gurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4294)
* Migrate gsutil usage to gcloud storage * updates * removed notes added by agent * removed note * linter changes --------- Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 0edae68 commit 1867536

File tree

10 files changed

+37
-35
lines changed

10 files changed

+37
-35
lines changed

‎community-content/pipeline_components/google-cloud/Vertex_AI/Models/Upload_PyTorch_model_archive/component.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ implementation:
148148
149149
# Downloading the model archive from GCS
150150
# TODO: Fix gsutil bugs (requires project ID, has auth issues) and use gsutil instead.
151-
# gsutil cp "$model_archive_uri" "$model_archive_local_path"
151+
# gcloud storage cp "$model_archive_uri" "$model_archive_local_path"
152152
pip install google-cloud-storage
153153
python -c '
154154
import sys

‎community-content/pytorch_image_classification_single_gpu_with_vertex_sdk_and_torchserve/vertex_prediction_with_custom_torchserve_container.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@
110110
},
111111
"outputs": [],
112112
"source": [
113-
"! gsutil ls $gcs_output_uri_prefix"
113+
"! gcloud storage ls $gcs_output_uri_prefix"
114114
]
115115
},
116116
{
@@ -192,7 +192,7 @@
192192
},
193193
"outputs": [],
194194
"source": [
195-
"! gsutil cp -r $gcs_output_uri_prefix/model ./model_server/"
195+
"! gcloud storage cp --recursive $gcs_output_uri_prefix/model ./model_server/"
196196
]
197197
},
198198
{
@@ -556,7 +556,7 @@
556556
},
557557
"outputs": [],
558558
"source": [
559-
"! gsutil rm -rf $gcs_output_uri_prefix"
559+
"! gcloud storage rm --recursive --continue-on-error $gcs_output_uri_prefix"
560560
]
561561
},
562562
{

‎notebooks/community/model_garden/model_garden_pytorch_fill_mask.ipynb‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,9 @@
125125
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
126126
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
127127
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
128-
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
128+
" ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n",
129129
"else:\n",
130-
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
130+
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
131131
" bucket_region = shell_output[0].strip().lower()\n",
132132
" if bucket_region != REGION:\n",
133133
" raise ValueError(\n",
@@ -145,7 +145,7 @@
145145
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
146146
"\n",
147147
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
148-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
148+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n",
149149
"\n",
150150
"# The pre-built serving docker image. It contains serving scripts and models.\n",
151151
"SERVE_DOCKER_URI = \"us-docker.pkg.dev/deeplearning-platform-release/gcr.io/huggingface-pytorch-inference-cu121.2-2.transformers.4-41.ubuntu2204.py311\"\n",
@@ -185,9 +185,7 @@
185185
" accelerator_count=1,\n",
186186
" deploy_request_timeout=1800,\n",
187187
" service_account=SERVICE_ACCOUNT,\n",
188-
" system_labels={\n",
189-
" \"NOTEBOOK_NAME\": \"model_garden_pytorch_fill_mask.ipynb\"\n",
190-
" },\n",
188+
" system_labels={\"NOTEBOOK_NAME\": \"model_garden_pytorch_fill_mask.ipynb\"},\n",
191189
" )\n",
192190
" return model, endpoint"
193191
]
@@ -283,7 +281,7 @@
283281
"\n",
284282
"delete_bucket = False # @param {type:\"boolean\"}\n",
285283
"if delete_bucket:\n",
286-
" ! gsutil -m rm -r $BUCKET_NAME"
284+
" ! gcloud storage rm --recursive $BUCKET_NAME"
287285
]
288286
}
289287
],

‎notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_xl_turbo.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,10 +148,10 @@
148148
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
149149
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
150150
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
151-
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
151+
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
152152
"else:\n",
153153
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
154-
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
154+
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
155155
" bucket_region = shell_output[0].strip().lower()\n",
156156
" if bucket_region != REGION:\n",
157157
" raise ValueError(\n",
@@ -170,7 +170,7 @@
170170
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
171171
"\n",
172172
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
173-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
173+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
174174
"\n",
175175
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)\n",
176176
"\n",
@@ -338,7 +338,7 @@
338338
"# Delete bucket.\n",
339339
"delete_bucket = False # @param {type:\"boolean\"}\n",
340340
"if delete_bucket:\n",
341-
" ! gsutil -m rm -r $BUCKET_NAME"
341+
" ! gcloud storage rm --recursive $BUCKET_NAME"
342342
]
343343
}
344344
],

‎notebooks/community/model_garden/model_garden_pytorch_vilt_vqa.ipynb‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -135,10 +135,10 @@
135135
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
136136
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
137137
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
138-
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
138+
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
139139
"else:\n",
140140
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
141-
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
141+
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
142142
" bucket_region = shell_output[0].strip().lower()\n",
143143
" if bucket_region != REGION:\n",
144144
" raise ValueError(\n",
@@ -163,7 +163,7 @@
163163
"\n",
164164
"\n",
165165
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
166-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
166+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
167167
"\n",
168168
"! gcloud config set project $PROJECT_ID\n",
169169
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
@@ -271,7 +271,9 @@
271271
"# print(\"Using this existing endpoint from a different session: {aip_endpoint_name}\")\n",
272272
"\n",
273273
"# @markdown ![](http://images.cocodataset.org/val2017/000000039769.jpg?w=1260&h=750)\n",
274-
"image = \"http://images.cocodataset.org/val2017/000000039769.jpg\" # @param {type: \"string\"}\n",
274+
"image = (\n",
275+
" \"http://images.cocodataset.org/val2017/000000039769.jpg\" # @param {type: \"string\"}\n",
276+
")\n",
275277
"question = \"Which cat is bigger?\" # @param {type: \"string\"}\n",
276278
"\n",
277279
"instances = [\n",
@@ -310,7 +312,7 @@
310312
"\n",
311313
"delete_bucket = False # @param {type:\"boolean\"}\n",
312314
"if delete_bucket:\n",
313-
" ! gsutil -m rm -r $BUCKET_NAME"
315+
" ! gcloud storage rm --recursive $BUCKET_NAME"
314316
]
315317
}
316318
],

‎notebooks/community/model_garden/model_garden_pytorch_whisper_large_v3_deployment.ipynb‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -143,10 +143,10 @@
143143
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
144144
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
145145
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
146-
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
146+
" ! gcloud storage buckets create {BUCKET_URI} --location={REGION}\n",
147147
"else:\n",
148148
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
149-
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
149+
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
150150
" bucket_region = shell_output[0].strip().lower()\n",
151151
" if bucket_region != REGION:\n",
152152
" raise ValueError(\n",
@@ -171,7 +171,7 @@
171171
"\n",
172172
"\n",
173173
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
174-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
174+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
175175
"\n",
176176
"! gcloud config set project $PROJECT_ID\n",
177177
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
@@ -191,7 +191,9 @@
191191
"\n",
192192
"# @markdown This section deploys the prebuilt whisper large v3 and whisper large v3 turbo model on a Vertex endpoint. It takes 15 minutes to deploy this model.\n",
193193
"\n",
194-
"model_name = \"whisper-large-v3-turbo\" # @param [\"whisper-large-v3\", \"whisper-large-v3-turbo\"]\n",
194+
"model_name = (\n",
195+
" \"whisper-large-v3-turbo\" # @param [\"whisper-large-v3\", \"whisper-large-v3-turbo\"]\n",
196+
")\n",
195197
"model_id = \"openai/{name}\".format(name=model_name)\n",
196198
"task = \"audio2text\"\n",
197199
"\n",
@@ -303,7 +305,7 @@
303305
" print(\n",
304306
" f\"Provisioning roles/storage.objectViewer permission to the {SERVICE_ACCOUNT} with the {bucket}.\"\n",
305307
" )\n",
306-
" ! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $bucket\n",
308+
" ! gcloud storage buckets add-iam-policy-binding $bucket --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer\n",
307309
"\n",
308310
"instances = [{\"audio\": uri} for uri in gcs_uris]\n",
309311
"\n",
@@ -341,7 +343,7 @@
341343
"\n",
342344
"delete_bucket = False # @param {type:\"boolean\"}\n",
343345
"if delete_bucket:\n",
344-
" ! gsutil -m rm -r $BUCKET_NAME"
346+
" ! gcloud storage rm --recursive $BUCKET_NAME"
345347
]
346348
}
347349
],

‎notebooks/community/sdk/SDK_AutoML_Video_Action_Recognition.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -206,7 +206,7 @@
206206
},
207207
"outputs": [],
208208
"source": [
209-
"!gsutil cp $automl_video_demo_train_data $gcs_source_train"
209+
"!gcloud storage cp $automl_video_demo_train_data $gcs_source_train"
210210
]
211211
},
212212
{
@@ -416,7 +416,7 @@
416416
},
417417
"outputs": [],
418418
"source": [
419-
"!gsutil cp $automl_video_demo_batch_prediction_data $gcs_source_batch_prediction"
419+
"!gcloud storage cp $automl_video_demo_batch_prediction_data $gcs_source_batch_prediction"
420420
]
421421
},
422422
{
@@ -488,7 +488,7 @@
488488
"metadata": {
489489
"colab": {
490490
"collapsed_sections": [],
491-
"name": "AI_Platform_(Unified)_SDK_AutoML_Video_Action_Recognition.ipynb",
491+
"name": "SDK_AutoML_Video_Action_Recognition.ipynb",
492492
"toc_visible": true
493493
},
494494
"kernelspec": {

‎notebooks/community/sdk/SDK_Custom_Training_with_Unmanaged_Image_Dataset.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -338,9 +338,9 @@
338338
},
339339
"outputs": [],
340340
"source": [
341-
"!gsutil -m cp -R gs://cloud-ml-data/img/flower_photos/daisy/14221848160_7f0a37c395.jpg .\n",
342-
"!gsutil -m cp -R gs://cloud-ml-data/img/flower_photos/tulips/13289268363_b9337d751e.jpg .\n",
343-
"!gsutil -m cp -R gs://cloud-ml-data/img/flower_photos/sunflowers/14623719696_1bb7970208_n.jpg ."
341+
"!gcloud storage cp --recursive gs://cloud-ml-data/img/flower_photos/daisy/14221848160_7f0a37c395.jpg .\n",
342+
"!gcloud storage cp --recursive gs://cloud-ml-data/img/flower_photos/tulips/13289268363_b9337d751e.jpg .\n",
343+
"!gcloud storage cp --recursive gs://cloud-ml-data/img/flower_photos/sunflowers/14623719696_1bb7970208_n.jpg ."
344344
]
345345
},
346346
{
@@ -383,7 +383,7 @@
383383
"metadata": {
384384
"colab": {
385385
"collapsed_sections": [],
386-
"name": "AI_Platform_(Unified)_SDK_Custom_Training_with_Unmanaged_Image_Dataset.ipynb",
386+
"name": "SDK_Custom_Training_with_Unmanaged_Image_Dataset.ipynb",
387387
"toc_visible": true
388388
},
389389
"kernelspec": {

‎notebooks/official/generative_ai/text_embedding_api_semantic_search_with_scann.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -328,7 +328,7 @@
328328
},
329329
"outputs": [],
330330
"source": [
331-
"!gsutil cp gs://cloud-samples-data/vertex-ai/dataset-management/datasets/bert_finetuning/wide_and_deep_trainer_container_tests_input.jsonl ."
331+
"!gcloud storage cp gs://cloud-samples-data/vertex-ai/dataset-management/datasets/bert_finetuning/wide_and_deep_trainer_container_tests_input.jsonl ."
332332
]
333333
},
334334
{

‎notebooks/official/ml_metadata/get_started_with_google_artifact_registry.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@
257257
},
258258
"outputs": [],
259259
"source": [
260-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
260+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
261261
]
262262
},
263263
{

0 commit comments

Comments
 (0)