Skip to content

Commit 419f831

Browse files
googlyrahmanbhandarivijay-pnggurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4316)
* Migrate gsutil usage to gcloud storage * Changes for 4316 * Changes for 4316 * fix linter issue for 4316 * removed changes for model garden * removed changes for model garden * Update model_garden_tfvision_image_classification.ipynb * Update model_garden_tfvision_image_classification.ipynb --------- Co-authored-by: bhandarivijay <bhandarivijay@google.com> Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 996b690 commit 419f831

8 files changed

+36
-40
lines changed

‎notebooks/community/migration/UJ11 HyperParameter Tuning Training Job with TensorFlow.ipynb‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@
325325
},
326326
"outputs": [],
327327
"source": [
328-
"! gsutil mb -l $REGION gs://$BUCKET_NAME"
328+
"! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME"
329329
]
330330
},
331331
{
@@ -345,7 +345,7 @@
345345
},
346346
"outputs": [],
347347
"source": [
348-
"! gsutil ls -al gs://$BUCKET_NAME"
348+
"! gcloud storage ls --all-versions --long gs://$BUCKET_NAME"
349349
]
350350
},
351351
{
@@ -383,10 +383,7 @@
383383
"import sys\n",
384384
"import time\n",
385385
"\n",
386-
"from google.cloud.aiplatform import gapic as aip\n",
387-
"from google.protobuf import json_format\n",
388-
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
389-
"from google.protobuf.struct_pb2 import Struct, Value"
386+
"from google.cloud.aiplatform import gapic as aip"
390387
]
391388
},
392389
{
@@ -671,7 +668,7 @@
671668
"! rm -f custom.tar custom.tar.gz\n",
672669
"! tar cvf custom.tar custom\n",
673670
"! gzip custom.tar\n",
674-
"! gsutil cp custom.tar.gz gs://$BUCKET_NAME/hpt_boston_housing.tar.gz"
671+
"! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/hpt_boston_housing.tar.gz"
675672
]
676673
},
677674
{
@@ -1354,13 +1351,13 @@
13541351
" print(e)\n",
13551352
"\n",
13561353
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1357-
" ! gsutil rm -r gs://$BUCKET_NAME"
1354+
" ! gcloud storage rm --recursive gs://$BUCKET_NAME"
13581355
]
13591356
}
13601357
],
13611358
"metadata": {
13621359
"colab": {
1363-
"name": "UJ11 unified HyperParameter Tuning Training Job with TensorFlow.ipynb",
1360+
"name": "UJ11 HyperParameter Tuning Training Job with TensorFlow.ipynb",
13641361
"toc_visible": true
13651362
},
13661363
"kernelspec": {

‎notebooks/community/ml_ops/stage6/get_started_with_custom_text_model_batch.ipynb‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -450,7 +450,7 @@
450450
},
451451
"outputs": [],
452452
"source": [
453-
"! gsutil mb -l $REGION $BUCKET_URI"
453+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
454454
]
455455
},
456456
{
@@ -470,7 +470,7 @@
470470
},
471471
"outputs": [],
472472
"source": [
473-
"! gsutil ls -al $BUCKET_URI"
473+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
474474
]
475475
},
476476
{
@@ -699,7 +699,7 @@
699699
"MODEL_DIR = f\"{BUCKET_URI}\"\n",
700700
"tfhub_model.save(\"model\")\n",
701701
"\n",
702-
"! gsutil cp -r model {MODEL_DIR}\n",
702+
"! gcloud storage cp --recursive model {MODEL_DIR}\n",
703703
"! rm -rf model\n",
704704
"\n",
705705
"MODEL_DIR = MODEL_DIR + \"/model\""
@@ -862,7 +862,7 @@
862862
" data = \"one two\"\n",
863863
" f.write('\"' + data + '\"\\n')\n",
864864
"\n",
865-
"! gsutil cp test.jsonl {gcs_input_uri}"
865+
"! gcloud storage cp test.jsonl {gcs_input_uri}"
866866
]
867867
},
868868
{
@@ -1032,8 +1032,8 @@
10321032
"with open(\"test2.txt\", \"w\") as f:\n",
10331033
" f.write('\"search Google or type an Url\"\\n')\n",
10341034
"\n",
1035-
"! gsutil cp test1.txt {BUCKET_URI}/test1.txt\n",
1036-
"! gsutil cp test2.txt {BUCKET_URI}/test2.txt\n",
1035+
"! gcloud storage cp test1.txt {BUCKET_URI}/test1.txt\n",
1036+
"! gcloud storage cp test2.txt {BUCKET_URI}/test2.txt\n",
10371037
"\n",
10381038
"! rm test1.txt test2.txt\n",
10391039
"\n",
@@ -1043,7 +1043,7 @@
10431043
"\n",
10441044
"gcs_input_uri = f\"{BUCKET_URI}/test.txt\"\n",
10451045
"\n",
1046-
"! gsutil cp test.txt $gcs_input_uri"
1046+
"! gcloud storage cp test.txt $gcs_input_uri"
10471047
]
10481048
},
10491049
{
@@ -1188,7 +1188,7 @@
11881188
" batch_prediction_job.delete()\n",
11891189
"\n",
11901190
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1191-
" ! gsutil rm -rf {BUCKET_URI}"
1191+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
11921192
]
11931193
}
11941194
],

‎notebooks/community/ml_ops/stage6/get_started_with_optimized_tfe_bert.ipynb‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,7 @@
458458
},
459459
"outputs": [],
460460
"source": [
461-
"! gsutil mb -l $REGION $BUCKET_URI"
461+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
462462
]
463463
},
464464
{
@@ -478,7 +478,7 @@
478478
},
479479
"outputs": [],
480480
"source": [
481-
"! gsutil ls -al $BUCKET_URI"
481+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
482482
]
483483
},
484484
{
@@ -504,7 +504,6 @@
504504
"import google.cloud.aiplatform as aip\n",
505505
"import tensorflow as tf\n",
506506
"import tensorflow_hub as hub\n",
507-
"import tensorflow_text\n",
508507
"from official.nlp import optimization # to create AdamW optimizer"
509508
]
510509
},
@@ -1220,7 +1219,7 @@
12201219
" print(e)\n",
12211220
"\n",
12221221
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1223-
" ! gsutil rm -rf {BUCKET_URI}"
1222+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
12241223
]
12251224
}
12261225
],

‎notebooks/community/reduction_server/distributed-training-reduction-server.ipynb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,7 @@
278278
"\n",
279279
"# Get your Google Cloud project ID from gcloud\n",
280280
"if not os.getenv(\"IS_TESTING\"):\n",
281-
" shell_output=!gcloud config list --format 'value(core.project)' 2>/dev/null\n",
281+
" shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n",
282282
" PROJECT_ID = shell_output[0]\n",
283283
" print(\"Project ID: \", PROJECT_ID)"
284284
]
@@ -431,7 +431,7 @@
431431
},
432432
"outputs": [],
433433
"source": [
434-
"! gsutil mb -l $REGION $BUCKET_NAME"
434+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
435435
]
436436
},
437437
{
@@ -451,7 +451,7 @@
451451
},
452452
"outputs": [],
453453
"source": [
454-
"! gsutil ls -al $BUCKET_NAME"
454+
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
455455
]
456456
},
457457
{
@@ -538,7 +538,7 @@
538538
"source": [
539539
"# List the files\n",
540540
"\n",
541-
"! gsutil ls {DATASET_LOCATION}"
541+
"! gcloud storage ls {DATASET_LOCATION}"
542542
]
543543
},
544544
{
@@ -551,7 +551,7 @@
551551
"source": [
552552
"# Examine dataset metadata\n",
553553
"\n",
554-
"! gsutil cat {METADATA_FILE}"
554+
"! gcloud storage cat {METADATA_FILE}"
555555
]
556556
},
557557
{
@@ -1130,7 +1130,7 @@
11301130
"outputs": [],
11311131
"source": [
11321132
"# Delete Cloud Storage objects that were created\n",
1133-
"! gsutil -m rm -r {BUCKET_NAME}\n",
1133+
"! gcloud storage rm --recursive {BUCKET_NAME}\n",
11341134
"\n",
11351135
"# Delete the training job\n",
11361136
"delete_training_job = True\n",

‎notebooks/official/pipelines/challenger_vs_blessed_deployment_method.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -306,7 +306,7 @@
306306
},
307307
"outputs": [],
308308
"source": [
309-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
309+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
310310
]
311311
},
312312
{
@@ -382,9 +382,9 @@
382382
},
383383
"outputs": [],
384384
"source": [
385-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
385+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
386386
"\n",
387-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
387+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
388388
]
389389
},
390390
{
@@ -1164,7 +1164,7 @@
11641164
"# Delete Cloud Storage objects that were created\n",
11651165
"delete_bucket = True\n",
11661166
"if delete_bucket:\n",
1167-
" ! gsutil -m rm -r $BUCKET_URI\n",
1167+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
11681168
"\n",
11691169
"# Delete the locally saved pipeline package file\n",
11701170
"! rm challenger_vs_blessed.json"

‎notebooks/official/ray_on_vertex_ai/spark_on_ray_on_vertex_ai.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -881,7 +881,7 @@
881881
},
882882
"outputs": [],
883883
"source": [
884-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
884+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
885885
]
886886
},
887887
{
@@ -901,7 +901,7 @@
901901
},
902902
"outputs": [],
903903
"source": [
904-
"! curl https://www.datavis.ca/gallery/guerry/guerry.csv | gsutil cp - {BUCKET_URI}/guerry.csv"
904+
"! curl https://www.datavis.ca/gallery/guerry/guerry.csv | gcloud storage cp - {BUCKET_URI}/guerry.csv"
905905
]
906906
},
907907
{

‎notebooks/official/tabular_workflows/bqml_arima_plus.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@
288288
},
289289
"outputs": [],
290290
"source": [
291-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
291+
"! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"
292292
]
293293
},
294294
{
@@ -361,9 +361,9 @@
361361
},
362362
"outputs": [],
363363
"source": [
364-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
364+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.objectCreator\"\n",
365365
"\n",
366-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
366+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
367367
]
368368
},
369369
{
@@ -1107,7 +1107,7 @@
11071107
"\n",
11081108
"delete_bucket = True\n",
11091109
"if delete_bucket:\n",
1110-
" ! gsutil -m rm -r $BUCKET_URI"
1110+
" ! gcloud storage rm --recursive $BUCKET_URI"
11111111
]
11121112
}
11131113
],

‎notebooks/official/training/multi_node_ddp_nccl_vertex_training_with_custom_container.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@
287287
},
288288
"outputs": [],
289289
"source": [
290-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
290+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
291291
]
292292
},
293293
{
@@ -1193,7 +1193,7 @@
11931193
},
11941194
"outputs": [],
11951195
"source": [
1196-
"! gsutil ls $GCS_OUTPUT_URI_PREFIX"
1196+
"! gcloud storage ls $GCS_OUTPUT_URI_PREFIX"
11971197
]
11981198
},
11991199
{
@@ -1236,7 +1236,7 @@
12361236
"delete_bucket = False # Set True for deletion\n",
12371237
"\n",
12381238
"if delete_bucket:\n",
1239-
" ! gsutil rm -r $BUCKET_URI\n",
1239+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
12401240
"\n",
12411241
"# Delete the repository in Artifact Registry\n",
12421242
"! gcloud artifacts repositories delete {PRIVATE_REPO} --location {LOCATION} --quiet\n",

0 commit comments

Comments
 (0)