Skip to content

Commit 9c9f7a6

Browse files
googlyrahmanbhandarivijay-pnggurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4334)
* Migrate gsutil usage to gcloud storage * Manual Changes * changes for 4334 * fix linter issue\ for 4334 * manual changes * Restore gcloud migration code * Remove changes for model garden * update --------- Co-authored-by: bhandarivijay <bhandarivijay@google.com> Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent ede41c2 commit 9c9f7a6

9 files changed

+41
-52
lines changed

‎notebooks/community/gapic/automl/showcase_automl_image_object_detection_online.ipynb‎

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -396,9 +396,7 @@
396396
"import time\n",
397397
"\n",
398398
"from google.cloud.aiplatform import gapic as aip\n",
399-
"from google.protobuf import json_format\n",
400-
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
401-
"from google.protobuf.struct_pb2 import Struct, Value"
399+
"from google.protobuf import json_format"
402400
]
403401
},
404402
{
@@ -722,11 +720,11 @@
722720
"else:\n",
723721
" FILE = IMPORT_FILE\n",
724722
"\n",
725-
"count = ! gsutil cat $FILE | wc -l\n",
723+
"count = ! gcloud storage cat $FILE | wc -l\n",
726724
"print(\"Number of Examples\", int(count[0]))\n",
727725
"\n",
728726
"print(\"First 10 rows\")\n",
729-
"! gsutil cat $FILE | head"
727+
"! gcloud storage cat $FILE | head"
730728
]
731729
},
732730
{
@@ -1345,7 +1343,7 @@
13451343
},
13461344
"outputs": [],
13471345
"source": [
1348-
"test_items = !gsutil cat $IMPORT_FILE | head -n1\n",
1346+
"test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n",
13491347
"cols = str(test_items[0]).split(\",\")\n",
13501348
"if len(cols) == 11:\n",
13511349
" test_item = str(cols[1])\n",
@@ -1574,7 +1572,7 @@
15741572
" print(e)\n",
15751573
"\n",
15761574
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1577-
" ! gsutil rm -r $BUCKET_NAME"
1575+
" ! gcloud storage rm --recursive $BUCKET_NAME"
15781576
]
15791577
}
15801578
],

‎notebooks/community/gapic/automl/showcase_automl_tabular_regression_online_bq.ipynb‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -397,7 +397,6 @@
397397
"\n",
398398
"from google.cloud.aiplatform import gapic as aip\n",
399399
"from google.protobuf import json_format\n",
400-
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
401400
"from google.protobuf.struct_pb2 import Struct, Value"
402401
]
403402
},
@@ -1614,7 +1613,7 @@
16141613
" print(e)\n",
16151614
"\n",
16161615
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1617-
" ! gsutil rm -r $BUCKET_NAME"
1616+
" ! gcloud storage rm --recursive $BUCKET_NAME"
16181617
]
16191618
}
16201619
],

‎notebooks/community/gapic/custom/showcase_local_tabular_regression_online.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@
423423
},
424424
"outputs": [],
425425
"source": [
426-
"! gsutil mb -l $REGION $BUCKET_NAME"
426+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
427427
]
428428
},
429429
{
@@ -443,7 +443,7 @@
443443
},
444444
"outputs": [],
445445
"source": [
446-
"! gsutil ls -al $BUCKET_NAME"
446+
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
447447
]
448448
},
449449
{
@@ -477,12 +477,8 @@
477477
},
478478
"outputs": [],
479479
"source": [
480-
"import time\n",
481-
"\n",
482480
"from google.cloud.aiplatform import gapic as aip\n",
483-
"from google.protobuf import json_format\n",
484-
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
485-
"from google.protobuf.struct_pb2 import Struct, Value"
481+
"from google.protobuf import json_format"
486482
]
487483
},
488484
{
@@ -1639,7 +1635,7 @@
16391635
" print(e)\n",
16401636
"\n",
16411637
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1642-
" ! gsutil rm -r $BUCKET_NAME"
1638+
" ! gcloud storage rm --recursive $BUCKET_NAME"
16431639
]
16441640
}
16451641
],

‎notebooks/community/gapic/custom/showcase_local_text_binary_classification_online.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -423,7 +423,7 @@
423423
},
424424
"outputs": [],
425425
"source": [
426-
"! gsutil mb -l $REGION $BUCKET_NAME"
426+
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
427427
]
428428
},
429429
{
@@ -443,7 +443,7 @@
443443
},
444444
"outputs": [],
445445
"source": [
446-
"! gsutil ls -al $BUCKET_NAME"
446+
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
447447
]
448448
},
449449
{
@@ -477,12 +477,8 @@
477477
},
478478
"outputs": [],
479479
"source": [
480-
"import time\n",
481-
"\n",
482480
"from google.cloud.aiplatform import gapic as aip\n",
483-
"from google.protobuf import json_format\n",
484-
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
485-
"from google.protobuf.struct_pb2 import Struct, Value"
481+
"from google.protobuf import json_format"
486482
]
487483
},
488484
{
@@ -1616,7 +1612,7 @@
16161612
" print(e)\n",
16171613
"\n",
16181614
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1619-
" ! gsutil rm -r $BUCKET_NAME"
1615+
" ! gcloud storage rm --recursive $BUCKET_NAME"
16201616
]
16211617
}
16221618
],

‎notebooks/community/ml_ops/stage6/get_started_with_automl_tabular_model_batch.ipynb‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -487,7 +487,7 @@
487487
},
488488
"outputs": [],
489489
"source": [
490-
"! gsutil mb -l $REGION $BUCKET_URI"
490+
"! gcloud storage buckets create --location $REGION $BUCKET_URI"
491491
]
492492
},
493493
{
@@ -507,7 +507,7 @@
507507
},
508508
"outputs": [],
509509
"source": [
510-
"! gsutil ls -al $BUCKET_URI"
510+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
511511
]
512512
},
513513
{
@@ -634,13 +634,13 @@
634634
},
635635
"outputs": [],
636636
"source": [
637-
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
637+
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
638638
"print(\"Number of Examples\", int(count[0]))\n",
639639
"\n",
640640
"print(\"First 10 rows\")\n",
641-
"! gsutil cat $IMPORT_FILE | head\n",
641+
"! gcloud storage cat $IMPORT_FILE | head\n",
642642
"\n",
643-
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
643+
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
644644
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
645645
"print(\"Label Column Name\", label_column)\n",
646646
"if label_column is None:\n",
@@ -913,14 +913,14 @@
913913
},
914914
"outputs": [],
915915
"source": [
916-
"! gsutil cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
917-
"! gsutil cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
916+
"! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
917+
"! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
918918
"\n",
919919
"! cut -d, -f1-16 tmp.csv > batch.csv\n",
920920
"\n",
921921
"gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
922922
"\n",
923-
"! gsutil cp batch.csv $gcs_input_uri"
923+
"! gcloud storage cp batch.csv $gcs_input_uri"
924924
]
925925
},
926926
{
@@ -1552,7 +1552,7 @@
15521552
"outputs": [],
15531553
"source": [
15541554
"# Create test data w/o the label\n",
1555-
"output = ! gsutil cat $IMPORT_FILE | head -n 10\n",
1555+
"output = ! gcloud storage cat $IMPORT_FILE | head -n 10\n",
15561556
"rows = []\n",
15571557
"for i in range(0, 10):\n",
15581558
" rows.append(output[i][:-2])\n",
@@ -1831,7 +1831,7 @@
18311831
"delete_bucket = False\n",
18321832
"\n",
18331833
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1834-
" ! gsutil rm -rf {BUCKET_URI}"
1834+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
18351835
]
18361836
}
18371837
],

‎notebooks/official/migration/sdk-custom-image-classification-custom-container.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -331,7 +331,7 @@
331331
},
332332
"outputs": [],
333333
"source": [
334-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
334+
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
335335
]
336336
},
337337
{
@@ -892,7 +892,7 @@
892892
"! rm -f custom.tar custom.tar.gz\n",
893893
"! tar cvf custom.tar custom\n",
894894
"! gzip custom.tar\n",
895-
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
895+
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
896896
]
897897
},
898898
{
@@ -1300,8 +1300,8 @@
13001300
},
13011301
"outputs": [],
13021302
"source": [
1303-
"! gsutil cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n",
1304-
"! gsutil cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n",
1303+
"! gcloud storage cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n",
1304+
"! gcloud storage cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n",
13051305
"\n",
13061306
"test_item_1 = BUCKET_URI + \"/tmp1.jpg\"\n",
13071307
"test_item_2 = BUCKET_URI + \"/tmp2.jpg\""
@@ -1695,7 +1695,7 @@
16951695
"# Delete Cloud Storage objects that were created\n",
16961696
"delete_bucket = False # Set True for deletion\n",
16971697
"if delete_bucket:\n",
1698-
" ! gsutil -m rm -r $BUCKET_URI\n",
1698+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
16991699
"\n",
17001700
"# remove the local users and movies avro files\n",
17011701
"! rm -rf custom\n",

‎notebooks/official/migration/sdk-custom-image-classification-prebuilt-container.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -297,7 +297,7 @@
297297
},
298298
"outputs": [],
299299
"source": [
300-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
300+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
301301
]
302302
},
303303
{
@@ -678,7 +678,7 @@
678678
"! rm -f custom.tar custom.tar.gz\n",
679679
"! tar cvf custom.tar custom\n",
680680
"! gzip custom.tar\n",
681-
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
681+
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_cifar10.tar.gz"
682682
]
683683
},
684684
{
@@ -1145,8 +1145,8 @@
11451145
},
11461146
"outputs": [],
11471147
"source": [
1148-
"! gsutil cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n",
1149-
"! gsutil cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n",
1148+
"! gcloud storage cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n",
1149+
"! gcloud storage cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n",
11501150
"\n",
11511151
"test_item_1 = BUCKET_URI + \"/tmp1.jpg\"\n",
11521152
"test_item_2 = BUCKET_URI + \"/tmp2.jpg\""
@@ -1637,7 +1637,7 @@
16371637
"# delete cloud storage bucket\n",
16381638
"delete_bucket = False # set True for deletion\n",
16391639
"if delete_bucket:\n",
1640-
" ! gsutil rm -r $BUCKET_URI"
1640+
" ! gcloud storage rm --recursive $BUCKET_URI"
16411641
]
16421642
}
16431643
],

‎notebooks/official/pipelines/rapid_prototyping_bqml_automl.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -394,7 +394,7 @@
394394
},
395395
"outputs": [],
396396
"source": [
397-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
397+
"! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"
398398
]
399399
},
400400
{
@@ -467,9 +467,9 @@
467467
},
468468
"outputs": [],
469469
"source": [
470-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
470+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
471471
"\n",
472-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
472+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
473473
]
474474
},
475475
{
@@ -602,7 +602,7 @@
602602
},
603603
"outputs": [],
604604
"source": [
605-
"! gsutil cp gs://cloud-samples-data/vertex-ai/community-content/datasets/abalone/abalone.data {RAW_INPUT_DATA}"
605+
"! gcloud storage cp gs://cloud-samples-data/vertex-ai/community-content/datasets/abalone/abalone.data {RAW_INPUT_DATA}"
606606
]
607607
},
608608
{
@@ -1611,7 +1611,7 @@
16111611
"# Delete Cloud Storage bucket\n",
16121612
"delete_bucket = True\n",
16131613
"if delete_bucket:\n",
1614-
" ! gsutil rm -r $BUCKET_URI\n",
1614+
" ! gcloud storage rm --recursive $BUCKET_URI\\\n",
16151615
"\n",
16161616
"# Delete the pipeline package file\n",
16171617
"! rm PIPELINE_YAML_PKG_PATH"

‎notebooks/official/workbench/demand_forecasting/forecasting-retail-demand.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,7 @@
275275
},
276276
"outputs": [],
277277
"source": [
278-
"! gsutil mb -l $LOCATION $BUCKET_URI"
278+
"! gcloud storage buckets create --location=$LOCATION $BUCKET_URI"
279279
]
280280
},
281281
{
@@ -2237,7 +2237,7 @@
22372237
"# Delete Cloud Storage objects that were created\n",
22382238
"delete_bucket = True # Set True for deletion\n",
22392239
"if delete_bucket:\n",
2240-
" ! gsutil -m rm -r $BUCKET_URI"
2240+
" ! gcloud storage rm --recursive $BUCKET_URI"
22412241
]
22422242
}
22432243
],

0 commit comments

Comments
 (0)