Skip to content

Commit 7a61378

Browse files
googlyrahmangooglyrahman
authored
Migrate gsutil usage to gcloud storage (#4333)
1 parent 100243e commit 7a61378

10 files changed

+55
-111
lines changed

‎notebooks/community/gapic/automl/showcase_automl_image_classification_online.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -721,12 +721,10 @@
721721
"else:\n",
722722
" FILE = IMPORT_FILE\n",
723723
"\n",
724-
"count = ! gsutil cat $FILE | wc -l\n",
725-
"print(\"Number of Examples\", int(count[0]))\n",
724+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
726725
"\n",
727726
"print(\"First 10 rows\")\n",
728-
"! gsutil cat $FILE | head"
729-
]
727+
"! gcloud storage cat $FILE | head" ]
730728
},
731729
{
732730
"cell_type": "markdown",
@@ -1342,8 +1340,7 @@
13421340
},
13431341
"outputs": [],
13441342
"source": [
1345-
"test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
1346-
"if len(str(test_item[0]).split(\",\")) == 3:\n",
1343+
"test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n",
13471344
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
13481345
"else:\n",
13491346
" test_item, test_label = str(test_item[0]).split(\",\")\n",
@@ -1568,8 +1565,7 @@
15681565
" print(e)\n",
15691566
"\n",
15701567
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1571-
" ! gsutil rm -r $BUCKET_NAME"
1572-
]
1568+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
15731569
}
15741570
],
15751571
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_image_segmentation_online.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -726,12 +726,10 @@
726726
"else:\n",
727727
" FILE = IMPORT_FILE\n",
728728
"\n",
729-
"count = ! gsutil cat $FILE | wc -l\n",
730-
"print(\"Number of Examples\", int(count[0]))\n",
729+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
731730
"\n",
732731
"print(\"First 10 rows\")\n",
733-
"! gsutil cat $FILE | head"
734-
]
732+
"! gcloud storage cat $FILE | head" ]
735733
},
736734
{
737735
"cell_type": "markdown",
@@ -1338,8 +1336,7 @@
13381336
"source": [
13391337
"import json\n",
13401338
"\n",
1341-
"test_items = !gsutil cat $IMPORT_FILE | head -n1\n",
1342-
"test_data = test_items[0].replace(\"'\", '\"')\n",
1339+
"test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n", "test_data = test_items[0].replace(\"'\", '\"')\n",
13431340
"test_data = json.loads(test_data)\n",
13441341
"try:\n",
13451342
" test_item = test_data[\"image_gcs_uri\"]\n",
@@ -1555,8 +1552,7 @@
15551552
" print(e)\n",
15561553
"\n",
15571554
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1558-
" ! gsutil rm -r $BUCKET_NAME"
1559-
]
1555+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
15601556
}
15611557
],
15621558
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_text_entity_extraction_online.ipynb‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -723,12 +723,10 @@
723723
"else:\n",
724724
" FILE = IMPORT_FILE\n",
725725
"\n",
726-
"count = ! gsutil cat $FILE | wc -l\n",
727-
"print(\"Number of Examples\", int(count[0]))\n",
726+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
728727
"\n",
729728
"print(\"First 10 rows\")\n",
730-
"! gsutil cat $FILE | head"
731-
]
729+
"! gcloud storage cat $FILE | head" ]
732730
},
733731
{
734732
"cell_type": "markdown",
@@ -1544,8 +1542,7 @@
15441542
" print(e)\n",
15451543
"\n",
15461544
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1547-
" ! gsutil rm -r $BUCKET_NAME"
1548-
]
1545+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
15491546
}
15501547
],
15511548
"metadata": {

‎notebooks/community/matching_engine/matching_engine_for_indexing.ipynb‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -477,8 +477,7 @@
477477
},
478478
"outputs": [],
479479
"source": [
480-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_NAME"
481-
]
480+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_NAME" ]
482481
},
483482
{
484483
"cell_type": "markdown",
@@ -497,8 +496,7 @@
497496
},
498497
"outputs": [],
499498
"source": [
500-
"! gsutil ls -al $BUCKET_NAME"
501-
]
499+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
502500
},
503501
{
504502
"cell_type": "markdown",
@@ -579,8 +577,7 @@
579577
},
580578
"outputs": [],
581579
"source": [
582-
"! gsutil cp gs://cloud-samples-data/vertex-ai/matching_engine/glove-100-angular.hdf5 ."
583-
]
580+
"! gcloud storage cp gs://cloud-samples-data/vertex-ai/matching_engine/glove-100-angular.hdf5 ." ]
584581
},
585582
{
586583
"cell_type": "markdown",
@@ -661,8 +658,7 @@
661658
"source": [
662659
"# NOTE: Everything in this Google Cloud Storage directory will be DELETED before uploading the data\n",
663660
"\n",
664-
"! gsutil rm -raf {BUCKET_NAME}/** 2> /dev/null || true"
665-
]
661+
"! gcloud storage rm --recursive --all-versions --continue-on-error {BUCKET_NAME}/** 2> /dev/null || true" ]
666662
},
667663
{
668664
"cell_type": "code",
@@ -672,8 +668,7 @@
672668
},
673669
"outputs": [],
674670
"source": [
675-
"! gsutil cp glove100.json {BUCKET_NAME}/glove100.json"
676-
]
671+
"! gcloud storage cp glove100.json {BUCKET_NAME}/glove100.json" ]
677672
},
678673
{
679674
"cell_type": "code",
@@ -683,8 +678,7 @@
683678
},
684679
"outputs": [],
685680
"source": [
686-
"! gsutil ls {BUCKET_NAME}"
687-
]
681+
"! gcloud storage ls {BUCKET_NAME}" ]
688682
},
689683
{
690684
"cell_type": "markdown",
@@ -966,8 +960,7 @@
966960
},
967961
"outputs": [],
968962
"source": [
969-
"! gsutil cp glove100_incremental.json {BUCKET_NAME}/incremental/glove100.json"
970-
]
963+
"! gcloud storage cp glove100_incremental.json {BUCKET_NAME}/incremental/glove100.json" ]
971964
},
972965
{
973966
"cell_type": "markdown",
@@ -2049,8 +2042,7 @@
20492042
"\n",
20502043
"delete_bucket = False\n",
20512044
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
2052-
" ! gsutil -m rm -r $BUCKET_NAME"
2053-
]
2045+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
20542046
}
20552047
],
20562048
"metadata": {

‎notebooks/community/migration/UJ14 AutoML for vision with Vertex AI Video Classification.ipynb‎

Lines changed: 9 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -325,8 +325,7 @@
325325
},
326326
"outputs": [],
327327
"source": [
328-
"! gsutil mb -l $REGION gs://$BUCKET_NAME"
329-
]
328+
"! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME" ]
330329
},
331330
{
332331
"cell_type": "markdown",
@@ -345,8 +344,7 @@
345344
},
346345
"outputs": [],
347346
"source": [
348-
"! gsutil ls -al gs://$BUCKET_NAME"
349-
]
347+
"! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ]
350348
},
351349
{
352350
"cell_type": "markdown",
@@ -542,8 +540,7 @@
542540
},
543541
"outputs": [],
544542
"source": [
545-
"! gsutil cat $IMPORT_FILE | head -n 10"
546-
]
543+
"! gcloud storage cat $IMPORT_FILE | head -n 10" ]
547544
},
548545
{
549546
"cell_type": "markdown",
@@ -1435,8 +1432,7 @@
14351432
},
14361433
"outputs": [],
14371434
"source": [
1438-
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
1439-
"\n",
1435+
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
14401436
"cols = str(test_items[0]).split(\",\")\n",
14411437
"test_item_1 = str(cols[0])\n",
14421438
"test_label_1 = str(cols[1])\n",
@@ -1510,8 +1506,7 @@
15101506
"\n",
15111507
"print(gcs_input_uri)\n",
15121508
"\n",
1513-
"!gsutil cat $gcs_input_uri"
1514-
]
1509+
"!gcloud storage cat $gcs_input_uri" ]
15151510
},
15161511
{
15171512
"cell_type": "markdown",
@@ -1842,8 +1837,7 @@
18421837
"source": [
18431838
"def get_latest_predictions(gcs_out_dir):\n",
18441839
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
1845-
" folders = !gsutil ls $gcs_out_dir\n",
1846-
" latest = \"\"\n",
1840+
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
18471841
" for folder in folders:\n",
18481842
" subfolder = folder.split(\"/\")[-2]\n",
18491843
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1862,10 +1856,8 @@
18621856
" folder = get_latest_predictions(\n",
18631857
" response.output_config.gcs_destination.output_uri_prefix\n",
18641858
" )\n",
1865-
" ! gsutil ls $folder/prediction*.jsonl\n",
1866-
"\n",
1867-
" ! gsutil cat $folder/prediction*.jsonl\n",
1868-
" break\n",
1859+
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
1860+
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
18691861
" time.sleep(60)"
18701862
]
18711863
},
@@ -1941,8 +1933,7 @@
19411933
" print(e)\n",
19421934
"\n",
19431935
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1944-
" ! gsutil rm -r gs://$BUCKET_NAME"
1945-
]
1936+
" ! gcloud storage rm --recursive gs://$BUCKET_NAME" ]
19461937
}
19471938
],
19481939
"metadata": {

‎notebooks/community/ml_ops/stage1/get_started_vertex_datasets.ipynb‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -489,8 +489,7 @@
489489
},
490490
"outputs": [],
491491
"source": [
492-
"! gsutil mb -l $REGION $BUCKET_URI"
493-
]
492+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI" ]
494493
},
495494
{
496495
"cell_type": "markdown",
@@ -509,8 +508,7 @@
509508
},
510509
"outputs": [],
511510
"source": [
512-
"! gsutil ls -al $BUCKET_URI"
513-
]
511+
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
514512
},
515513
{
516514
"cell_type": "markdown",
@@ -1300,14 +1298,12 @@
13001298
"\n",
13011299
"! bq --location=us extract --destination_format CSV $BQ_PROJECT_DATASET_TABLE $BUCKET_URI/mydata*.csv\n",
13021300
"\n",
1303-
"IMPORT_FILES = ! gsutil ls $BUCKET_URI/mydata*.csv\n",
1304-
"\n",
1301+
"IMPORT_FILES = ! gcloud storage ls $BUCKET_URI/mydata*.csv\n", "\n",
13051302
"print(IMPORT_FILES)\n",
13061303
"\n",
13071304
"EXAMPLE_FILE = IMPORT_FILES[0]\n",
13081305
"\n",
1309-
"! gsutil cat $EXAMPLE_FILE | head"
1310-
]
1306+
"! gcloud storage cat $EXAMPLE_FILE | head" ]
13111307
},
13121308
{
13131309
"cell_type": "markdown",
@@ -1453,8 +1449,7 @@
14531449
"EXPORTED_DIR = f\"{BUCKET_URI}/exported\"\n",
14541450
"exported_files = dataset.export_data(output_dir=EXPORTED_DIR)\n",
14551451
"\n",
1456-
"! gsutil ls $EXPORTED_DIR"
1457-
]
1452+
"! gcloud storage ls $EXPORTED_DIR" ]
14581453
},
14591454
{
14601455
"cell_type": "markdown",
@@ -1483,8 +1478,7 @@
14831478
"source": [
14841479
"jsonl_index = exported_files[0]\n",
14851480
"\n",
1486-
"! gsutil cat $jsonl_index | head"
1487-
]
1481+
"! gcloud storage cat $jsonl_index | head" ]
14881482
},
14891483
{
14901484
"cell_type": "markdown",
@@ -1695,8 +1689,7 @@
16951689
" if n % 10 == 0:\n",
16961690
" print(n, image)\n",
16971691
"\n",
1698-
"listing = ! gsutil ls -la $GCS_TFRECORD_URI\n",
1699-
"print(\"TFRecord File\", listing)"
1692+
"listing = ! gcloud storage ls --long --all-versions $GCS_TFRECORD_URI\n", "print(\"TFRecord File\", listing)"
17001693
]
17011694
},
17021695
{
@@ -1733,8 +1726,7 @@
17331726
"\n",
17341727
"# Delete the bucket\n",
17351728
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1736-
" ! gsutil rm -r $BUCKET_URI"
1737-
]
1729+
" ! gcloud storage rm --recursive $BUCKET_URI" ]
17381730
}
17391731
],
17401732
"metadata": {

‎notebooks/community/sdk/sdk_custom_image_classification_online_custom_training_container.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -475,8 +475,7 @@
475475
},
476476
"outputs": [],
477477
"source": [
478-
"! gsutil mb -l $REGION $BUCKET_NAME"
479-
]
478+
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
480479
},
481480
{
482481
"cell_type": "markdown",
@@ -495,8 +494,7 @@
495494
},
496495
"outputs": [],
497496
"source": [
498-
"! gsutil ls -al $BUCKET_NAME"
499-
]
497+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
500498
},
501499
{
502500
"cell_type": "markdown",
@@ -1057,8 +1055,7 @@
10571055
"! rm -f custom.tar custom.tar.gz\n",
10581056
"! tar cvf custom.tar custom\n",
10591057
"! gzip custom.tar\n",
1060-
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
1061-
]
1058+
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ]
10621059
},
10631060
{
10641061
"cell_type": "markdown",
@@ -1679,8 +1676,7 @@
16791676
" print(e)\n",
16801677
"\n",
16811678
" if \"BUCKET_NAME\" in globals():\n",
1682-
" ! gsutil rm -r $BUCKET_NAME"
1683-
]
1679+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
16841680
}
16851681
],
16861682
"metadata": {

0 commit comments

Comments
 (0)