Skip to content

Commit 993898b

Browse files
googlyrahmanbhandarivijay-pnggurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4330)
* Migrate gsutil usage to gcloud storage * Manual Changes * Removed blank line and spaces Manual Changes * fix the Linter issue for 4330 * Changes for model garden * Update model_garden_pytorch_llama3_1_finetuning.ipynb --------- Co-authored-by: bhandarivijay <bhandarivijay@google.com> Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 9e9e639 commit 993898b

9 files changed

+55
-55
lines changed

‎notebooks/community/experiments/vertex_ai_model_experimentation.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -537,7 +537,7 @@
537537
},
538538
"outputs": [],
539539
"source": [
540-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
540+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
541541
]
542542
},
543543
{
@@ -557,7 +557,7 @@
557557
},
558558
"outputs": [],
559559
"source": [
560-
"! gsutil ls -al $BUCKET_URI"
560+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
561561
]
562562
},
563563
{
@@ -627,9 +627,9 @@
627627
},
628628
"outputs": [],
629629
"source": [
630-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
630+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
631631
"\n",
632-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
632+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
633633
]
634634
},
635635
{
@@ -1736,7 +1736,7 @@
17361736
"delete_bucket = False\n",
17371737
"\n",
17381738
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1739-
" ! gsutil rm -rf {BUCKET_URI}"
1739+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
17401740
]
17411741
}
17421742
],

‎notebooks/community/migration/UJ10 legacy Custom Training Prebuilt Container SKLearn.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -312,7 +312,7 @@
312312
},
313313
"outputs": [],
314314
"source": [
315-
"! gsutil mb -l $REGION gs://$BUCKET_NAME"
315+
"! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME"
316316
]
317317
},
318318
{
@@ -332,7 +332,7 @@
332332
},
333333
"outputs": [],
334334
"source": [
335-
"! gsutil ls -al gs://$BUCKET_NAME"
335+
"! gcloud storage ls --all-versions --long gs://$BUCKET_NAME"
336336
]
337337
},
338338
{
@@ -668,7 +668,7 @@
668668
"! rm -f custom.tar custom.tar.gz\n",
669669
"! tar cvf custom.tar custom\n",
670670
"! gzip custom.tar\n",
671-
"! gsutil cp custom.tar.gz gs://$BUCKET_NAME/census.tar.gz"
671+
"! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/census.tar.gz"
672672
]
673673
},
674674
{
@@ -1923,7 +1923,7 @@
19231923
" print(e)\n",
19241924
"\n",
19251925
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1926-
" ! gsutil rm -r gs://$BUCKET_NAME"
1926+
" ! gcloud storage rm --recursive gs://$BUCKET_NAME"
19271927
]
19281928
}
19291929
],

‎notebooks/community/ml_ops/stage1/mlops_data_management.ipynb‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -465,7 +465,7 @@
465465
},
466466
"outputs": [],
467467
"source": [
468-
"! gsutil mb -l $REGION $BUCKET_URI"
468+
"! gcloud storage buckets create --location $REGION $BUCKET_URI"
469469
]
470470
},
471471
{
@@ -485,7 +485,7 @@
485485
},
486486
"outputs": [],
487487
"source": [
488-
"! gsutil ls -al $BUCKET_URI"
488+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
489489
]
490490
},
491491
{
@@ -1093,7 +1093,7 @@
10931093
") as f:\n",
10941094
" json.dump(metadata, f)\n",
10951095
"\n",
1096-
"! gsutil cat $BUCKET_URI/metadata.jsonl"
1096+
"! gcloud storage cat $BUCKET_URI/metadata.jsonl"
10971097
]
10981098
},
10991099
{
@@ -1178,7 +1178,7 @@
11781178
") as f:\n",
11791179
" json.dump(metadata, f)\n",
11801180
"\n",
1181-
"! gsutil cat $BUCKET_URI/metadata.jsonl"
1181+
"! gcloud storage cat $BUCKET_URI/metadata.jsonl"
11821182
]
11831183
},
11841184
{
@@ -1527,12 +1527,12 @@
15271527
"run_pipeline(args)\n",
15281528
"print(\"Data preprocessing completed.\")\n",
15291529
"\n",
1530-
"! gsutil ls $TRANSFORMED_DATA_PREFIX/train\n",
1531-
"! gsutil ls $TRANSFORMED_DATA_PREFIX/val\n",
1532-
"! gsutil ls $TRANSFORMED_DATA_PREFIX/test\n",
1533-
"! gsutil ls $TRANSFORM_ARTIFACTS_DIR\n",
1534-
"! gsutil ls {EXPORTED_JSONL_PREFIX}*\n",
1535-
"! gsutil ls $EXPORTED_TFREC_PREFIX"
1530+
"! gcloud storage ls $TRANSFORMED_DATA_PREFIX/train\n",
1531+
"! gcloud storage ls $TRANSFORMED_DATA_PREFIX/val\n",
1532+
"! gcloud storage ls $TRANSFORMED_DATA_PREFIX/test\n",
1533+
"! gcloud storage ls $TRANSFORM_ARTIFACTS_DIR\n",
1534+
"! gcloud storage ls {EXPORTED_JSONL_PREFIX}*\n",
1535+
"! gcloud storage ls $EXPORTED_TFREC_PREFIX"
15361536
]
15371537
},
15381538
{
@@ -1588,7 +1588,7 @@
15881588
") as f:\n",
15891589
" json.dump(metadata, f)\n",
15901590
"\n",
1591-
"! gsutil cat $BUCKET_URI/metadata.jsonl"
1591+
"! gcloud storage cat $BUCKET_URI/metadata.jsonl"
15921592
]
15931593
},
15941594
{
@@ -1626,7 +1626,7 @@
16261626
" print(e)\n",
16271627
"\n",
16281628
" if \"BUCKET_URI\" in globals():\n",
1629-
" ! gsutil rm -r $BUCKET_URI"
1629+
" ! gcloud storage rm --recursive $BUCKET_URI"
16301630
]
16311631
}
16321632
],

‎notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -446,7 +446,7 @@
446446
},
447447
"outputs": [],
448448
"source": [
449-
"! gsutil mb -l $REGION $BUCKET_URI"
449+
"! gcloud storage buckets create --location $REGION $BUCKET_URI"
450450
]
451451
},
452452
{
@@ -466,7 +466,7 @@
466466
},
467467
"outputs": [],
468468
"source": [
469-
"! gsutil ls -al $BUCKET_URI"
469+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
470470
]
471471
},
472472
{
@@ -1098,7 +1098,7 @@
10981098
},
10991099
"outputs": [],
11001100
"source": [
1101-
"! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg"
1101+
"! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg"
11021102
]
11031103
},
11041104
{
@@ -1296,7 +1296,7 @@
12961296
" f.write(\"\\n\")\n",
12971297
" json.dump(instances[0], f)\n",
12981298
"\n",
1299-
"! gsutil cp test.jsonl {BUCKET_URI}/test.jsonl"
1299+
"! gcloud storage cp test.jsonl {BUCKET_URI}/test.jsonl"
13001300
]
13011301
},
13021302
{
@@ -1459,7 +1459,7 @@
14591459
" print(e)\n",
14601460
"\n",
14611461
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1462-
" ! gsutil rm -rf {BUCKET_URI}"
1462+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
14631463
]
14641464
}
14651465
],

‎notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -496,7 +496,7 @@
496496
},
497497
"outputs": [],
498498
"source": [
499-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
499+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
500500
]
501501
},
502502
{
@@ -516,7 +516,7 @@
516516
},
517517
"outputs": [],
518518
"source": [
519-
"! gsutil ls -al $BUCKET_URI"
519+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
520520
]
521521
},
522522
{
@@ -687,8 +687,8 @@
687687
"outputs": [],
688688
"source": [
689689
"%cd ..\n",
690-
"!gsutil cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
691-
"!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
690+
"!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
691+
"!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
692692
]
693693
},
694694
{
@@ -1809,7 +1809,7 @@
18091809
"delete_bucket = False\n",
18101810
"\n",
18111811
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1812-
" ! gsutil rm -r $BUCKET_URI"
1812+
" ! gcloud storage rm --recursive $BUCKET_URI"
18131813
]
18141814
}
18151815
],

‎notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -490,7 +490,7 @@
490490
},
491491
"outputs": [],
492492
"source": [
493-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
493+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
494494
]
495495
},
496496
{
@@ -510,7 +510,7 @@
510510
},
511511
"outputs": [],
512512
"source": [
513-
"! gsutil ls -al $BUCKET_URI"
513+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
514514
]
515515
},
516516
{
@@ -918,8 +918,8 @@
918918
},
919919
"outputs": [],
920920
"source": [
921-
"!gsutil cp -r {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
922-
"!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
921+
"!gcloud storage cp --recursive {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
922+
"!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
923923
]
924924
},
925925
{
@@ -1869,7 +1869,7 @@
18691869
"delete_bucket = False\n",
18701870
"\n",
18711871
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1872-
" ! gsutil rm -r $BUCKET_URI"
1872+
" ! gcloud storage rm --recursive $BUCKET_URI"
18731873
]
18741874
}
18751875
],

‎notebooks/community/sdk/sdk_custom_image_classification_online.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -475,7 +475,7 @@
475475
},
476476
"outputs": [],
477477
"source": [
478-
"! gsutil mb -l $REGION $BUCKET_NAME"
478+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
479479
]
480480
},
481481
{
@@ -495,7 +495,7 @@
495495
},
496496
"outputs": [],
497497
"source": [
498-
"! gsutil ls -al $BUCKET_NAME"
498+
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
499499
]
500500
},
501501
{
@@ -917,7 +917,7 @@
917917
"! rm -f custom.tar custom.tar.gz\n",
918918
"! tar cvf custom.tar custom\n",
919919
"! gzip custom.tar\n",
920-
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
920+
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
921921
]
922922
},
923923
{
@@ -1544,7 +1544,7 @@
15441544
" print(e)\n",
15451545
"\n",
15461546
" if \"BUCKET_NAME\" in globals():\n",
1547-
" ! gsutil rm -r $BUCKET_NAME"
1547+
" ! gcloud storage rm --recursive $BUCKET_NAME"
15481548
]
15491549
}
15501550
],

‎notebooks/community/sdk/sdk_custom_image_classification_online_tfserving.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -475,7 +475,7 @@
475475
},
476476
"outputs": [],
477477
"source": [
478-
"! gsutil mb -l $REGION $BUCKET_NAME"
478+
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
479479
]
480480
},
481481
{
@@ -495,7 +495,7 @@
495495
},
496496
"outputs": [],
497497
"source": [
498-
"! gsutil ls -al $BUCKET_NAME"
498+
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
499499
]
500500
},
501501
{
@@ -944,7 +944,7 @@
944944
"! rm -f custom.tar custom.tar.gz\n",
945945
"! tar cvf custom.tar custom\n",
946946
"! gzip custom.tar\n",
947-
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
947+
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
948948
]
949949
},
950950
{
@@ -1535,7 +1535,7 @@
15351535
" print(e)\n",
15361536
"\n",
15371537
" if \"BUCKET_NAME\" in globals():\n",
1538-
" ! gsutil rm -r $BUCKET_NAME"
1538+
" ! gcloud storage rm --recursive $BUCKET_NAME"
15391539
]
15401540
}
15411541
],

0 commit comments

Comments
 (0)