Skip to content

Commit 6bbe3bc

Browse files
googlyrahmanbhandarivijay-pnggurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4336)
* Migrate gsutil usage to gcloud storage * Manual Changes * removed model garden changes * Update model_garden_llama3_1_finetuning_with_workbench.ipynb --------- Co-authored-by: bhandarivijay <bhandarivijay@google.com> Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 814827a commit 6bbe3bc

9 files changed

+60
-120
lines changed

‎notebooks/community/gapic/automl/showcase_automl_image_classification_online_proxy.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -721,12 +721,10 @@
721721
"else:\n",
722722
" FILE = IMPORT_FILE\n",
723723
"\n",
724-
"count = ! gsutil cat $FILE | wc -l\n",
725-
"print(\"Number of Examples\", int(count[0]))\n",
724+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
726725
"\n",
727726
"print(\"First 10 rows\")\n",
728-
"! gsutil cat $FILE | head"
729-
]
727+
"! gcloud storage cat $FILE | head" ]
730728
},
731729
{
732730
"cell_type": "markdown",
@@ -1342,8 +1340,7 @@
13421340
},
13431341
"outputs": [],
13441342
"source": [
1345-
"test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
1346-
"if len(str(test_item[0]).split(\",\")) == 3:\n",
1343+
"test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n",
13471344
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
13481345
"else:\n",
13491346
" test_item, test_label = str(test_item[0]).split(\",\")\n",
@@ -1747,8 +1744,7 @@
17471744
" print(e)\n",
17481745
"\n",
17491746
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1750-
" ! gsutil rm -r $BUCKET_NAME"
1751-
]
1747+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
17521748
}
17531749
],
17541750
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_tabular_classification_batch.ipynb‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,7 @@
421421
},
422422
"outputs": [],
423423
"source": [
424-
"! gsutil mb -l $REGION $BUCKET_NAME"
425-
]
424+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
426425
},
427426
{
428427
"cell_type": "markdown",
@@ -441,8 +440,7 @@
441440
},
442441
"outputs": [],
443442
"source": [
444-
"! gsutil ls -al $BUCKET_NAME"
445-
]
443+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
446444
},
447445
{
448446
"cell_type": "markdown",
@@ -810,14 +808,11 @@
810808
},
811809
"outputs": [],
812810
"source": [
813-
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
814-
"print(\"Number of Examples\", int(count[0]))\n",
811+
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
815812
"\n",
816813
"print(\"First 10 rows\")\n",
817-
"! gsutil cat $IMPORT_FILE | head\n",
818-
"\n",
819-
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
820-
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
814+
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
815+
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
821816
"print(\"Label Column Name\", label_column)\n",
822817
"if label_column is None:\n",
823818
" raise Exception(\"label column missing\")"
@@ -1363,8 +1358,7 @@
13631358
" f.write(str(INSTANCE_2) + \"\\n\")\n",
13641359
"\n",
13651360
"print(gcs_input_uri)\n",
1366-
"! gsutil cat $gcs_input_uri"
1367-
]
1361+
"! gcloud storage cat $gcs_input_uri" ]
13681362
},
13691363
{
13701364
"cell_type": "markdown",
@@ -1632,8 +1626,7 @@
16321626
"source": [
16331627
"def get_latest_predictions(gcs_out_dir):\n",
16341628
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
1635-
" folders = !gsutil ls $gcs_out_dir\n",
1636-
" latest = \"\"\n",
1629+
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
16371630
" for folder in folders:\n",
16381631
" subfolder = folder.split(\"/\")[-2]\n",
16391632
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1650,10 +1643,8 @@
16501643
" raise Exception(\"Batch Job Failed\")\n",
16511644
" else:\n",
16521645
" folder = get_latest_predictions(predictions)\n",
1653-
" ! gsutil ls $folder/prediction*.csv\n",
1654-
"\n",
1655-
" ! gsutil cat $folder/prediction*.csv\n",
1656-
" break\n",
1646+
" ! gcloud storage ls $folder/prediction*.csv\n", "\n",
1647+
" ! gcloud storage cat $folder/prediction*.csv\n", " break\n",
16571648
" time.sleep(60)"
16581649
]
16591650
},
@@ -1747,8 +1738,7 @@
17471738
" print(e)\n",
17481739
"\n",
17491740
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1750-
" ! gsutil rm -r $BUCKET_NAME"
1751-
]
1741+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
17521742
}
17531743
],
17541744
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_text_classification_batch.ipynb‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,7 @@
421421
},
422422
"outputs": [],
423423
"source": [
424-
"! gsutil mb -l $REGION $BUCKET_NAME"
425-
]
424+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
426425
},
427426
{
428427
"cell_type": "markdown",
@@ -441,8 +440,7 @@
441440
},
442441
"outputs": [],
443442
"source": [
444-
"! gsutil ls -al $BUCKET_NAME"
445-
]
443+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
446444
},
447445
{
448446
"cell_type": "markdown",
@@ -881,12 +879,10 @@
881879
"else:\n",
882880
" FILE = IMPORT_FILE\n",
883881
"\n",
884-
"count = ! gsutil cat $FILE | wc -l\n",
885-
"print(\"Number of Examples\", int(count[0]))\n",
882+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
886883
"\n",
887884
"print(\"First 10 rows\")\n",
888-
"! gsutil cat $FILE | head"
889-
]
885+
"! gcloud storage cat $FILE | head" ]
890886
},
891887
{
892888
"cell_type": "markdown",
@@ -1314,8 +1310,7 @@
13141310
},
13151311
"outputs": [],
13161312
"source": [
1317-
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
1318-
"if len(test_items[0]) == 3:\n",
1313+
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(test_items[0]) == 3:\n",
13191314
" _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n",
13201315
" _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n",
13211316
"else:\n",
@@ -1371,8 +1366,7 @@
13711366
" f.write(json.dumps(data) + \"\\n\")\n",
13721367
"\n",
13731368
"print(gcs_input_uri)\n",
1374-
"! gsutil cat $gcs_input_uri"
1375-
]
1369+
"! gcloud storage cat $gcs_input_uri" ]
13761370
},
13771371
{
13781372
"cell_type": "markdown",
@@ -1644,8 +1638,7 @@
16441638
"source": [
16451639
"def get_latest_predictions(gcs_out_dir):\n",
16461640
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
1647-
" folders = !gsutil ls $gcs_out_dir\n",
1648-
" latest = \"\"\n",
1641+
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
16491642
" for folder in folders:\n",
16501643
" subfolder = folder.split(\"/\")[-2]\n",
16511644
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1662,10 +1655,8 @@
16621655
" raise Exception(\"Batch Job Failed\")\n",
16631656
" else:\n",
16641657
" folder = get_latest_predictions(predictions)\n",
1665-
" ! gsutil ls $folder/prediction*.jsonl\n",
1666-
"\n",
1667-
" ! gsutil cat $folder/prediction*.jsonl\n",
1668-
" break\n",
1658+
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
1659+
! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
16691660
" time.sleep(60)"
16701661
]
16711662
},
@@ -1759,8 +1750,7 @@
17591750
" print(e)\n",
17601751
"\n",
17611752
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1762-
" ! gsutil rm -r $BUCKET_NAME"
1763-
]
1753+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
17641754
}
17651755
],
17661756
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_text_multi-label_classification_batch.ipynb‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,7 @@
421421
},
422422
"outputs": [],
423423
"source": [
424-
"! gsutil mb -l $REGION $BUCKET_NAME"
425-
]
424+
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
426425
},
427426
{
428427
"cell_type": "markdown",
@@ -441,8 +440,7 @@
441440
},
442441
"outputs": [],
443442
"source": [
444-
"! gsutil ls -al $BUCKET_NAME"
445-
]
443+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
446444
},
447445
{
448446
"cell_type": "markdown",
@@ -881,12 +879,10 @@
881879
"else:\n",
882880
" FILE = IMPORT_FILE\n",
883881
"\n",
884-
"count = ! gsutil cat $FILE | wc -l\n",
885-
"print(\"Number of Examples\", int(count[0]))\n",
882+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
886883
"\n",
887884
"print(\"First 10 rows\")\n",
888-
"! gsutil cat $FILE | head"
889-
]
885+
"! gcloud storage cat $FILE | head" ]
890886
},
891887
{
892888
"cell_type": "markdown",
@@ -1314,8 +1310,7 @@
13141310
},
13151311
"outputs": [],
13161312
"source": [
1317-
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
1318-
"\n",
1313+
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
13191314
"cols_1 = str(test_items[0]).split(\",\")\n",
13201315
"cols_2 = str(test_items[1]).split(\",\")\n",
13211316
"test_item_1 = cols_1[0]\n",
@@ -1372,8 +1367,7 @@
13721367
" f.write(json.dumps(data) + \"\\n\")\n",
13731368
"\n",
13741369
"print(gcs_input_uri)\n",
1375-
"! gsutil cat $gcs_input_uri"
1376-
]
1370+
"! gcloud storage cat $gcs_input_uri" ]
13771371
},
13781372
{
13791373
"cell_type": "markdown",
@@ -1645,8 +1639,7 @@
16451639
"source": [
16461640
"def get_latest_predictions(gcs_out_dir):\n",
16471641
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
1648-
" folders = !gsutil ls $gcs_out_dir\n",
1649-
" latest = \"\"\n",
1642+
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
16501643
" for folder in folders:\n",
16511644
" subfolder = folder.split(\"/\")[-2]\n",
16521645
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1663,10 +1656,8 @@
16631656
" raise Exception(\"Batch Job Failed\")\n",
16641657
" else:\n",
16651658
" folder = get_latest_predictions(predictions)\n",
1666-
" ! gsutil ls $folder/prediction*.jsonl\n",
1667-
"\n",
1668-
" ! gsutil cat $folder/prediction*.jsonl\n",
1669-
" break\n",
1659+
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
1660+
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
16701661
" time.sleep(60)"
16711662
]
16721663
},
@@ -1760,8 +1751,7 @@
17601751
" print(e)\n",
17611752
"\n",
17621753
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1763-
" ! gsutil rm -r $BUCKET_NAME"
1764-
]
1754+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
17651755
}
17661756
],
17671757
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_text_sentiment_analysis_batch.ipynb‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,7 @@
421421
},
422422
"outputs": [],
423423
"source": [
424-
"! gsutil mb -l $REGION $BUCKET_NAME"
425-
]
424+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
426425
},
427426
{
428427
"cell_type": "markdown",
@@ -441,8 +440,7 @@
441440
},
442441
"outputs": [],
443442
"source": [
444-
"! gsutil ls -al $BUCKET_NAME"
445-
]
443+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
446444
},
447445
{
448446
"cell_type": "markdown",
@@ -883,12 +881,10 @@
883881
"else:\n",
884882
" FILE = IMPORT_FILE\n",
885883
"\n",
886-
"count = ! gsutil cat $FILE | wc -l\n",
887-
"print(\"Number of Examples\", int(count[0]))\n",
884+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
888885
"\n",
889886
"print(\"First 10 rows\")\n",
890-
"! gsutil cat $FILE | head"
891-
]
887+
"! gcloud storage cat $FILE | head" ]
892888
},
893889
{
894890
"cell_type": "markdown",
@@ -1316,8 +1312,7 @@
13161312
},
13171313
"outputs": [],
13181314
"source": [
1319-
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
1320-
"\n",
1315+
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
13211316
"if len(test_items[0]) == 4:\n",
13221317
" _, test_item_1, test_label_1, _ = str(test_items[0]).split(\",\")\n",
13231318
" _, test_item_2, test_label_2, _ = str(test_items[1]).split(\",\")\n",
@@ -1375,8 +1370,7 @@
13751370
" f.write(json.dumps(data) + \"\\n\")\n",
13761371
"\n",
13771372
"print(gcs_input_uri)\n",
1378-
"! gsutil cat $gcs_input_uri"
1379-
]
1373+
"! gcloud storage cat $gcs_input_uri" ]
13801374
},
13811375
{
13821376
"cell_type": "markdown",
@@ -1643,8 +1637,7 @@
16431637
"source": [
16441638
"def get_latest_predictions(gcs_out_dir):\n",
16451639
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
1646-
" folders = !gsutil ls $gcs_out_dir\n",
1647-
" latest = \"\"\n",
1640+
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
16481641
" for folder in folders:\n",
16491642
" subfolder = folder.split(\"/\")[-2]\n",
16501643
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1661,10 +1654,8 @@
16611654
" raise Exception(\"Batch Job Failed\")\n",
16621655
" else:\n",
16631656
" folder = get_latest_predictions(predictions)\n",
1664-
" ! gsutil ls $folder/prediction*.jsonl\n",
1665-
"\n",
1666-
" ! gsutil cat $folder/prediction*.jsonl\n",
1667-
" break\n",
1657+
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
1658+
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
16681659
" time.sleep(60)"
16691660
]
16701661
},
@@ -1758,8 +1749,7 @@
17581749
" print(e)\n",
17591750
"\n",
17601751
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1761-
" ! gsutil rm -r $BUCKET_NAME"
1762-
]
1752+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
17631753
}
17641754
],
17651755
"metadata": {

0 commit comments

Comments
 (0)