Skip to content

Commit ca53786

Browse files
googlyrahmanbhandarivijay-pnggurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4322)
* Migrate gsutil usage to gcloud storage * Changes for 4322 * fix linter issue for 4322 * removed changes for model garden * Update model_garden_pytorch_gemma_peft_finetuning_hf.ipynb * Update model_garden_pytorch_gemma_peft_finetuning_hf.ipynb --------- Co-authored-by: bhandarivijay <bhandarivijay@google.com> Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 419f831 commit ca53786

9 files changed

+48
-49
lines changed

‎notebooks/community/ml_ops/stage2/get_started_vertex_training_pytorch.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -483,7 +483,7 @@
483483
},
484484
"outputs": [],
485485
"source": [
486-
"! gsutil mb -l $REGION $BUCKET_URI"
486+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
487487
]
488488
},
489489
{
@@ -503,7 +503,7 @@
503503
},
504504
"outputs": [],
505505
"source": [
506-
"! gsutil ls -al $BUCKET_URI"
506+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
507507
]
508508
},
509509
{
@@ -680,7 +680,7 @@
680680
"The PyTorch package does not have support to save the model to a Cloud Storage location. Instead, you will do the following steps to save to a Cloud Storage location.\n",
681681
"\n",
682682
"1. Save the in-memory model to the local filesystem (e.g., model.pth).\n",
683-
"2. Use gsutil to copy the local copy to the specified Cloud Storage location.\n",
683+
"2. Use gcloud storage to copy the local copy to the specified Cloud Storage location.\n",
684684
"\n",
685685
"*Note*: You can do hyperparameter tuning with a PyTorch model."
686686
]
@@ -1060,7 +1060,7 @@
10601060
"! rm -f custom.tar custom.tar.gz\n",
10611061
"! tar cvf custom.tar custom\n",
10621062
"! gzip custom.tar\n",
1063-
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
1063+
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
10641064
]
10651065
},
10661066
{
@@ -1375,7 +1375,7 @@
13751375
"\n",
13761376
"delete_bucket = False\n",
13771377
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1378-
" ! gsutil rm -r $BUCKET_URI"
1378+
" ! gcloud storage rm --recursive $BUCKET_URI"
13791379
]
13801380
}
13811381
],

‎notebooks/community/ml_ops/stage2/get_started_with_distributed_training_xgboost.ipynb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -448,7 +448,7 @@
448448
},
449449
"outputs": [],
450450
"source": [
451-
"! gsutil mb -l $REGION $BUCKET_URI"
451+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
452452
]
453453
},
454454
{
@@ -468,7 +468,7 @@
468468
},
469469
"outputs": [],
470470
"source": [
471-
"! gsutil ls -al $BUCKET_URI"
471+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
472472
]
473473
},
474474
{
@@ -1136,7 +1136,7 @@
11361136
"! rm -f custom.tar custom.tar.gz\n",
11371137
"! tar cvf custom.tar custom\n",
11381138
"! gzip custom.tar\n",
1139-
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_covertype.tar.gz"
1139+
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_covertype.tar.gz"
11401140
]
11411141
},
11421142
{
@@ -1282,7 +1282,7 @@
12821282
},
12831283
"outputs": [],
12841284
"source": [
1285-
"! gsutil ls {MODEL_DIR}/model"
1285+
"! gcloud storage ls {MODEL_DIR}/model"
12861286
]
12871287
},
12881288
{
@@ -1326,7 +1326,7 @@
13261326
},
13271327
"outputs": [],
13281328
"source": [
1329-
"output = ! gsutil cat {DATASET_FILE} | head -n2\n",
1329+
"output = ! gcloud storage cat {DATASET_FILE} | head -n2\n",
13301330
"\n",
13311331
"print(output[1])\n",
13321332
"\n",
@@ -1423,7 +1423,7 @@
14231423
"source": [
14241424
"delete_bucket = True\n",
14251425
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1426-
" ! gsutil rm -r {BUCKET_URI}\n",
1426+
" ! gcloud storage rm --recursive {BUCKET_URI}\n",
14271427
"\n",
14281428
"! rm -rf custom prediction custom.tar.gz\n",
14291429
"\n",

‎notebooks/community/ml_ops/stage2/get_started_with_visionapi_and_automl.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -512,7 +512,7 @@
512512
},
513513
"outputs": [],
514514
"source": [
515-
"! gsutil mb -l $VERTEX_AI_REGION -p $PROJECT_ID $BUCKET_URI"
515+
"! gcloud storage buckets create --location=$VERTEX_AI_REGION --project=$PROJECT_ID $BUCKET_URI"
516516
]
517517
},
518518
{
@@ -532,7 +532,7 @@
532532
},
533533
"outputs": [],
534534
"source": [
535-
"! gsutil ls -al $BUCKET_URI"
535+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
536536
]
537537
},
538538
{
@@ -1277,7 +1277,7 @@
12771277
"job.delete()\n",
12781278
"\n",
12791279
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1280-
" ! gsutil -m rm -r $BUCKET_URI\n",
1280+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
12811281
"\n",
12821282
"print(\"Clean up completed!\")"
12831283
]

‎notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_model_online.ipynb‎

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,6 @@
349349
},
350350
"outputs": [],
351351
"source": [
352-
"import random\n",
353352
"import string\n",
354353
"\n",
355354
"\n",
@@ -478,7 +477,7 @@
478477
},
479478
"outputs": [],
480479
"source": [
481-
"! gsutil mb -l $REGION $BUCKET_URI"
480+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
482481
]
483482
},
484483
{
@@ -498,7 +497,7 @@
498497
},
499498
"outputs": [],
500499
"source": [
501-
"! gsutil ls -al $BUCKET_URI"
500+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
502501
]
503502
},
504503
{
@@ -680,13 +679,13 @@
680679
},
681680
"outputs": [],
682681
"source": [
683-
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
682+
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
684683
"print(\"Number of Examples\", int(count[0]))\n",
685684
"\n",
686685
"print(\"First 10 rows\")\n",
687-
"! gsutil cat $IMPORT_FILE | head\n",
686+
"! gcloud storage cat $IMPORT_FILE | head\n",
688687
"\n",
689-
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
688+
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
690689
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
691690
"print(\"Label Column Name\", label_column)\n",
692691
"if label_column is None:\n",
@@ -1382,7 +1381,7 @@
13821381
"delete_bucket = False\n",
13831382
"\n",
13841383
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1385-
" ! gsutil rm -rf {BUCKET_URI}"
1384+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
13861385
]
13871386
}
13881387
],

‎notebooks/community/ml_ops/stage6/get_started_with_matching_engine.ipynb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -496,7 +496,7 @@
496496
},
497497
"outputs": [],
498498
"source": [
499-
"! gsutil mb -l $REGION $BUCKET_URI"
499+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
500500
]
501501
},
502502
{
@@ -516,7 +516,7 @@
516516
},
517517
"outputs": [],
518518
"source": [
519-
"! gsutil ls -al $BUCKET_URI"
519+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
520520
]
521521
},
522522
{
@@ -587,7 +587,7 @@
587587
},
588588
"outputs": [],
589589
"source": [
590-
"! gsutil cp gs://cloud-samples-data/vertex-ai/matching_engine/glove-100-angular.hdf5 ."
590+
"! gcloud storage cp gs://cloud-samples-data/vertex-ai/matching_engine/glove-100-angular.hdf5 ."
591591
]
592592
},
593593
{
@@ -667,7 +667,7 @@
667667
"outputs": [],
668668
"source": [
669669
"EMBEDDINGS_INITIAL_URI = f\"{BUCKET_URI}/matching_engine/initial/\"\n",
670-
"! gsutil cp glove100.json {EMBEDDINGS_INITIAL_URI}"
670+
"! gcloud storage cp glove100.json {EMBEDDINGS_INITIAL_URI}"
671671
]
672672
},
673673
{
@@ -762,7 +762,7 @@
762762
"\n",
763763
"EMBEDDINGS_UPDATE_URI = f\"{BUCKET_URI}/matching-engine/incremental/\"\n",
764764
"\n",
765-
"! gsutil cp glove100_incremental.json {EMBEDDINGS_UPDATE_URI}"
765+
"! gcloud storage cp glove100_incremental.json {EMBEDDINGS_UPDATE_URI}"
766766
]
767767
},
768768
{
@@ -1448,7 +1448,7 @@
14481448
"\n",
14491449
"delete_bucket = False\n",
14501450
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1451-
" ! gsutil rm -rf {BUCKET_URI}"
1451+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
14521452
]
14531453
}
14541454
],

‎notebooks/community/workbench/telecom-subscriber-churn-prediction.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -511,7 +511,7 @@
511511
},
512512
"outputs": [],
513513
"source": [
514-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
514+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
515515
]
516516
},
517517
{
@@ -531,7 +531,7 @@
531531
},
532532
"outputs": [],
533533
"source": [
534-
"! gsutil ls -al $BUCKET_URI"
534+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
535535
]
536536
},
537537
{
@@ -1595,7 +1595,7 @@
15951595
"# Delete the Cloud Storage bucket\n",
15961596
"delete_bucket = True\n",
15971597
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1598-
" ! gsutil -m rm -r $BUCKET_URI"
1598+
" ! gcloud storage rm --recursive $BUCKET_URI"
15991599
]
16001600
}
16011601
],

‎notebooks/official/automl/get_started_automl_training.ipynb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,7 @@
313313
},
314314
"outputs": [],
315315
"source": [
316-
"! gsutil mb -l $LOCATION $BUCKET_URI"
316+
"! gcloud storage buckets create --location=$LOCATION $BUCKET_URI"
317317
]
318318
},
319319
{
@@ -583,11 +583,11 @@
583583
"source": [
584584
"FILE = IMPORT_FILE\n",
585585
"\n",
586-
"count = ! gsutil cat $FILE | wc -l\n",
586+
"count = ! gcloud storage cat $FILE | wc -l\n",
587587
"print(\"Number of Examples\", int(count[0]))\n",
588588
"\n",
589589
"print(\"First 10 rows\")\n",
590-
"! gsutil cat $FILE | head"
590+
"! gcloud storage cat $FILE | head"
591591
]
592592
},
593593
{
@@ -811,7 +811,7 @@
811811
},
812812
"outputs": [],
813813
"source": [
814-
"test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
814+
"test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n",
815815
"if len(str(test_item[0]).split(\",\")) == 3:\n",
816816
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
817817
"else:\n",
@@ -864,7 +864,7 @@
864864
"\n",
865865
"# Copy the test image to the Cloud storage bucket as \"test.jpg\"\n",
866866
"test_image_local = \"{}/test.jpg\".format(BUCKET_URI)\n",
867-
"! gsutil cp $test_item $test_image_local\n",
867+
"! gcloud storage cp $test_item $test_image_local\n",
868868
"\n",
869869
"# Download the test image in bytes format\n",
870870
"storage_client = storage.Client(project=PROJECT_ID)\n",
@@ -1453,7 +1453,7 @@
14531453
"# Delete the Cloud Storage bucket\n",
14541454
"delete_bucket = False # Set True for deletion\n",
14551455
"if delete_bucket:\n",
1456-
" ! gsutil rm -r $BUCKET_URI"
1456+
" ! gcloud storage rm --recursive $BUCKET_URI"
14571457
]
14581458
}
14591459
],

‎notebooks/official/pipelines/get_started_with_hpt_pipeline_components.ipynb‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -298,7 +298,7 @@
298298
},
299299
"outputs": [],
300300
"source": [
301-
"! gsutil mb -l $LOCATION $BUCKET_URI"
301+
"! gcloud storage buckets create --location=$LOCATION $BUCKET_URI"
302302
]
303303
},
304304
{
@@ -372,9 +372,9 @@
372372
},
373373
"outputs": [],
374374
"source": [
375-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
375+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
376376
"\n",
377-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
377+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer "
378378
]
379379
},
380380
{
@@ -817,7 +817,7 @@
817817
"! rm -f custom.tar custom.tar.gz\n",
818818
"! tar cvf custom.tar custom\n",
819819
"! gzip custom.tar\n",
820-
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz"
820+
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_horses_or_humans.tar.gz"
821821
]
822822
},
823823
{
@@ -1326,13 +1326,13 @@
13261326
" + \"/evaluation_metrics\"\n",
13271327
" )\n",
13281328
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
1329-
" ! gsutil cat $EXECUTE_OUTPUT\n",
1329+
" ! gcloud storage cat $EXECUTE_OUTPUT\n",
13301330
" return EXECUTE_OUTPUT\n",
13311331
" elif tf.io.gfile.exists(GCP_RESOURCES):\n",
1332-
" ! gsutil cat $GCP_RESOURCES\n",
1332+
" ! gcloud storage cat $GCP_RESOURCES\n",
13331333
" return GCP_RESOURCES\n",
13341334
" elif tf.io.gfile.exists(EVAL_METRICS):\n",
1335-
" ! gsutil cat $EVAL_METRICS\n",
1335+
" ! gcloud storage cat $EVAL_METRICS\n",
13361336
" return EVAL_METRICS\n",
13371337
"\n",
13381338
" return None\n",
@@ -1347,7 +1347,7 @@
13471347
"print(\"getbesttrialop\")\n",
13481348
"artifacts = print_pipeline_output(pipeline, \"getbesttrialop\")\n",
13491349
"print(\"\\n\\n\")\n",
1350-
"output = !gsutil cat $artifacts\n",
1350+
"output = !gcloud storage cat $artifacts\n",
13511351
"output = json.loads(output[0])\n",
13521352
"best_trial = json.loads(output[\"parameters\"][\"Output\"][\"stringValue\"])\n",
13531353
"model_id = best_trial[\"id\"]\n",
@@ -1422,7 +1422,7 @@
14221422
"delete_bucket = False\n",
14231423
"\n",
14241424
"if delete_bucket:\n",
1425-
" ! gsutil rm -r $BUCKET_URI\n",
1425+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
14261426
"\n",
14271427
"# Delete artifact registry repo\n",
14281428
"! gcloud artifacts repositories delete $REPO_NAME --location $LOCATION --quiet"

0 commit comments

Comments
 (0)