Skip to content

Commit 5b9d04d

Browse files
googlyrahmanbhandarivijay-pnggurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4318)
* Migrate gsutil usage to gcloud storage * Changes for 4318 * fix linter issue for 4318 * removed changes for model garden: * Update model_garden_gemma2_deployment_on_vertex.ipynb --------- Co-authored-by: bhandarivijay <bhandarivijay@google.com> Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 9ed3c2f commit 5b9d04d

10 files changed

+44
-47
lines changed

‎notebooks/community/explainable_ai/SDK_Custom_Container_XAI.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@
405405
},
406406
"outputs": [],
407407
"source": [
408-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
408+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
409409
]
410410
},
411411
{
@@ -425,7 +425,7 @@
425425
},
426426
"outputs": [],
427427
"source": [
428-
"! gsutil ls -al $BUCKET_URI"
428+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
429429
]
430430
},
431431
{
@@ -698,7 +698,7 @@
698698
},
699699
"outputs": [],
700700
"source": [
701-
"!gsutil cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
701+
"!gcloud storage cp app/model.joblib {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
702702
]
703703
},
704704
{
@@ -1516,7 +1516,7 @@
15161516
"\n",
15171517
"delete_bucket = False\n",
15181518
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1519-
" ! gsutil rm -rf {BUCKET_URI}"
1519+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
15201520
]
15211521
}
15221522
],

‎notebooks/community/migration/UJ11 legacy HyperParameter Tuning Training Job with TensorFlow.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@
312312
},
313313
"outputs": [],
314314
"source": [
315-
"! gsutil mb -l $REGION gs://$BUCKET_NAME"
315+
"! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME"
316316
]
317317
},
318318
{
@@ -332,7 +332,7 @@
332332
},
333333
"outputs": [],
334334
"source": [
335-
"! gsutil ls -al gs://$BUCKET_NAME"
335+
"! gcloud storage ls --all-versions --long gs://$BUCKET_NAME"
336336
]
337337
},
338338
{
@@ -614,7 +614,7 @@
614614
"! rm -f custom.tar custom.tar.gz\n",
615615
"! tar cvf custom.tar custom\n",
616616
"! gzip custom.tar\n",
617-
"! gsutil cp custom.tar.gz gs://$BUCKET_NAME/hpt_boston_housing.tar.gz"
617+
"! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/hpt_boston_housing.tar.gz"
618618
]
619619
},
620620
{
@@ -1370,7 +1370,7 @@
13701370
"delete_bucket = True\n",
13711371
"\n",
13721372
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1373-
" ! gsutil rm -r gs://$BUCKET_NAME"
1373+
" ! gcloud storage rm --recursive gs://$BUCKET_NAME"
13741374
]
13751375
}
13761376
],

‎notebooks/community/migration/UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -326,7 +326,7 @@
326326
},
327327
"outputs": [],
328328
"source": [
329-
"! gsutil mb -l $REGION gs://$BUCKET_NAME"
329+
"! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME"
330330
]
331331
},
332332
{
@@ -346,7 +346,7 @@
346346
},
347347
"outputs": [],
348348
"source": [
349-
"! gsutil ls -al gs://$BUCKET_NAME"
349+
"! gcloud storage ls --all-versions --long gs://$BUCKET_NAME"
350350
]
351351
},
352352
{
@@ -383,11 +383,9 @@
383383
"import json\n",
384384
"import os\n",
385385
"import sys\n",
386-
"import time\n",
387386
"\n",
388387
"from google.cloud import automl\n",
389-
"from google.protobuf.json_format import MessageToJson\n",
390-
"from google.protobuf.struct_pb2 import Value"
388+
"from google.protobuf.json_format import MessageToJson"
391389
]
392390
},
393391
{
@@ -485,7 +483,7 @@
485483
},
486484
"outputs": [],
487485
"source": [
488-
"! gsutil cat $IMPORT_FILE | head -n 10"
486+
"! gcloud storage cat $IMPORT_FILE | head -n 10"
489487
]
490488
},
491489
{
@@ -1209,14 +1207,15 @@
12091207
"gcs_input_uri = \"gs://\" + BUCKET_NAME + \"/test.csv\"\n",
12101208
"with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n",
12111209
" item_1 = \"gs://cloud-samples-data/language/sentiment-positive.txt\"\n",
1212-
" ! gsutil cp $item_1 gs://$BUCKET_NAME\n",
1210+
" ! gcloud storage cp $item_1 gs://$BUCKET_NAME\n",
1211+
" ! gcloud storage cp $item_1 gs://$BUCKET_NAME\n",
12131212
" f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-positive.txt\" + \"\\n\")\n",
12141213
"\n",
12151214
" item_2 = \"gs://cloud-samples-data/language/sentiment-negative.txt\"\n",
1216-
" ! gsutil cp $item_2 gs://$BUCKET_NAME\n",
1215+
" ! gcloud storage cp $item_2 gs://$BUCKET_NAME\n",
12171216
" f.write(\"gs://\" + BUCKET_NAME + \"/sentiment-negative.txt\")\n",
12181217
"\n",
1219-
"! gsutil cat $gcs_input_uri"
1218+
"! gcloud storage cat $gcs_input_uri"
12201219
]
12211220
},
12221221
{
@@ -1381,7 +1380,7 @@
13811380
},
13821381
"outputs": [],
13831382
"source": [
1384-
"test_data = ! gsutil cat $IMPORT_FILE | head -n1\n",
1383+
"test_data = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
13851384
"\n",
13861385
"test_item = str(test_data[0]).split(\",\")[0]\n",
13871386
"test_label = str(test_data[0]).split(\",\")[1]\n",
@@ -1615,13 +1614,13 @@
16151614
" print(e)\n",
16161615
"\n",
16171616
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1618-
" ! gsutil rm -r gs://$BUCKET_NAME"
1617+
" ! gcloud storage rm --recursive gs://$BUCKET_NAME"
16191618
]
16201619
}
16211620
],
16221621
"metadata": {
16231622
"colab": {
1624-
"name": "UJ8 legacy AutoML Natural Language - Text Sentiment Analysis.ipynb",
1623+
"name": "UJ8 legacy AutoML Natural Language Text Sentiment Analysis.ipynb",
16251624
"toc_visible": true
16261625
},
16271626
"kernelspec": {

‎notebooks/community/ml_ops/stage2/get_started_vertex_feature_store.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1017,8 +1017,8 @@
10171017
"USERS_AVRO_FN = \"users.avro\"\n",
10181018
"MOVIES_AVRO_FN = \"movies.avro\"\n",
10191019
"\n",
1020-
"! gsutil cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n",
1021-
"! gsutil cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN"
1020+
"! gcloud storage cp $GCS_USERS_AVRO_URI $USERS_AVRO_FN\n",
1021+
"! gcloud storage cp $GCS_MOVIES_AVRO_URI $MOVIES_AVRO_FN"
10221022
]
10231023
},
10241024
{

‎notebooks/community/ml_ops/stage3/get_started_with_bqml_pipeline_components.ipynb‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -452,7 +452,7 @@
452452
},
453453
"outputs": [],
454454
"source": [
455-
"! gsutil mb -l $REGION $BUCKET_URI"
455+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
456456
]
457457
},
458458
{
@@ -472,7 +472,7 @@
472472
},
473473
"outputs": [],
474474
"source": [
475-
"! gsutil ls -al $BUCKET_URI"
475+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
476476
]
477477
},
478478
{
@@ -545,9 +545,9 @@
545545
},
546546
"outputs": [],
547547
"source": [
548-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
548+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
549549
"\n",
550-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
550+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer "
551551
]
552552
},
553553
{
@@ -1052,13 +1052,13 @@
10521052
" + \"/evaluation_metrics\"\n",
10531053
" )\n",
10541054
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
1055-
" ! gsutil cat $EXECUTE_OUTPUT\n",
1055+
" ! gcloud storage cat $EXECUTE_OUTPUT\n",
10561056
" return EXECUTE_OUTPUT\n",
10571057
" elif tf.io.gfile.exists(GCP_RESOURCES):\n",
1058-
" ! gsutil cat $GCP_RESOURCES\n",
1058+
" ! gcloud storage cat $GCP_RESOURCES\n",
10591059
" return GCP_RESOURCES\n",
10601060
" elif tf.io.gfile.exists(EVAL_METRICS):\n",
1061-
" ! gsutil cat $EVAL_METRICS\n",
1061+
" ! gcloud storage cat $EVAL_METRICS\n",
10621062
" return EVAL_METRICS\n",
10631063
"\n",
10641064
" return None\n",
@@ -1081,14 +1081,14 @@
10811081
"print(\"\\n\\n\")\n",
10821082
"print(\"model-upload\")\n",
10831083
"artifacts = print_pipeline_output(pipeline, \"model-upload\")\n",
1084-
"output = !gsutil cat $artifacts\n",
1084+
"output = !gcloud storage cat $artifacts\n",
10851085
"output = json.loads(output[0])\n",
10861086
"model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
10871087
"print(\"\\n\\n\")\n",
10881088
"print(\"endpoint-create\")\n",
10891089
"artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n",
10901090
"print(\"\\n\\n\")\n",
1091-
"output = !gsutil cat $artifacts\n",
1091+
"output = !gcloud storage cat $artifacts\n",
10921092
"output = json.loads(output[0])\n",
10931093
"endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n",
10941094
" \"resourceName\"\n",
@@ -1322,7 +1322,7 @@
13221322
"delete_bucket = False\n",
13231323
"\n",
13241324
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1325-
" ! gsutil rm -r $BUCKET_URI"
1325+
" ! gcloud storage rm --recursive $BUCKET_URI"
13261326
]
13271327
}
13281328
],

‎notebooks/community/persistent_resource/training_code/mpg_container/trainer/train.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,7 @@
1414

1515
import argparse
1616
import logging
17-
import os
18-
import numpy as np
1917
import pandas as pd
20-
import pathlib
2118
import tensorflow as tf
2219

2320
from tensorflow import keras

‎notebooks/notebook_template_review.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939
import json
4040
import os
4141
import sys
42-
import urllib.request
4342
import csv
4443
from enum import Enum
4544
from abc import ABC, abstractmethod

‎notebooks/official/pipelines/google_cloud_pipeline_components_TPU_model_train_upload_deploy.ipynb‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@
292292
},
293293
"outputs": [],
294294
"source": [
295-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
295+
"! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"
296296
]
297297
},
298298
{
@@ -365,9 +365,10 @@
365365
},
366366
"outputs": [],
367367
"source": [
368-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
368+
"# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n",
369+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
369370
"\n",
370-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
371+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
371372
]
372373
},
373374
{
@@ -1234,7 +1235,7 @@
12341235
"# Warning: Setting this to true deletes everything in your bucket\n",
12351236
"delete_bucket = True\n",
12361237
"if delete_bucket:\n",
1237-
" ! gsutil rm -r $BUCKET_URI\n",
1238+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
12381239
"\n",
12391240
"# Delete the Artifact Registry repository\n",
12401241
"! gcloud artifacts repositories delete $REPOSITORY --location=$LOCATION --quiet\n",

‎notebooks/official/pipelines/multicontender_vs_champion_deployment_method.ipynb‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -309,7 +309,7 @@
309309
},
310310
"outputs": [],
311311
"source": [
312-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
312+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
313313
]
314314
},
315315
{
@@ -384,9 +384,10 @@
384384
},
385385
"outputs": [],
386386
"source": [
387-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
387+
"! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n",
388+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
388389
"\n",
389-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
390+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
390391
]
391392
},
392393
{
@@ -1239,7 +1240,7 @@
12391240
"# Delete Cloud Storage objects that were created\n",
12401241
"delete_bucket = True\n",
12411242
"if delete_bucket:\n",
1242-
" ! gsutil -m rm -r $BUCKET_URI\n",
1243+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
12431244
"\n",
12441245
"# Remove the local pipeline package file\n",
12451246
"! rm multicontender_vs_champion.json"

‎notebooks/official/vector_search/sdk_vector_search_create_multimodal_embeddings.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@
275275
},
276276
"outputs": [],
277277
"source": [
278-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
278+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
279279
]
280280
},
281281
{
@@ -1062,7 +1062,7 @@
10621062
"source": [
10631063
"UNIQUE_FOLDER_NAME = \"embeddings_folder_unique\"\n",
10641064
"EMBEDDINGS_INITIAL_URI = f\"{BUCKET_URI}/{UNIQUE_FOLDER_NAME}/\"\n",
1065-
"! gsutil cp {embeddings_file.name} {EMBEDDINGS_INITIAL_URI}"
1065+
"! gcloud storage cp {embeddings_file.name} {EMBEDDINGS_INITIAL_URI}"
10661066
]
10671067
},
10681068
{
@@ -1369,7 +1369,7 @@
13691369
"# Delete Cloud Storage objects that were created\n",
13701370
"delete_bucket = False\n",
13711371
"if delete_bucket:\n",
1372-
" ! gsutil -m rm -r $BUCKET_URI"
1372+
" ! gcloud storage rm --recursive $BUCKET_URI"
13731373
]
13741374
}
13751375
],

0 commit comments

Comments
 (0)