Skip to content

Commit 8d22b22

Browse files
googlyrahmanbhandarivijay-pnggurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4337)
* Migrate gsutil usage to gcloud storage * Manual Changes * Manual Changes * Manual Changes * Manual Changes * Revert "Manual Changes" This reverts commit 3ef23f1. * Manaul Changes * Changes for 4337 * Fix: Resolved linter errors and formatted notebooks * Revert "Fix: Resolved linter errors and formatted notebooks" This reverts commit 7534760. * Revert "Changes for 4337" This reverts commit f497568. * Changes for 4337 * gcloud to gsutil migration * removed changes for model garden * Update model_garden_axolotl_qwen3_finetuning.ipynb * Update model_garden_axolotl_qwen3_finetuning.ipynb --------- Co-authored-by: bhandarivijay <[email protected]> Co-authored-by: gurusai-voleti <[email protected]>
1 parent 2a8ad7c commit 8d22b22

9 files changed

+71
-142
lines changed

‎notebooks/community/gapic/automl/showcase_automl_tabular_binary_classification_batch.ipynb‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,7 @@
421421
},
422422
"outputs": [],
423423
"source": [
424-
"! gsutil mb -l $REGION $BUCKET_NAME"
425-
]
424+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
426425
},
427426
{
428427
"cell_type": "markdown",
@@ -441,8 +440,7 @@
441440
},
442441
"outputs": [],
443442
"source": [
444-
"! gsutil ls -al $BUCKET_NAME"
445-
]
443+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
446444
},
447445
{
448446
"cell_type": "markdown",
@@ -809,14 +807,11 @@
809807
},
810808
"outputs": [],
811809
"source": [
812-
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
813-
"print(\"Number of Examples\", int(count[0]))\n",
810+
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
814811
"\n",
815812
"print(\"First 10 rows\")\n",
816-
"! gsutil cat $IMPORT_FILE | head\n",
817-
"\n",
818-
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
819-
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
813+
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
814+
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
820815
"print(\"Label Column Name\", label_column)\n",
821816
"if label_column is None:\n",
822817
" raise Exception(\"label column missing\")"
@@ -1377,8 +1372,7 @@
13771372
" f.write(str(INSTANCE_2) + \"\\n\")\n",
13781373
"\n",
13791374
"print(gcs_input_uri)\n",
1380-
"! gsutil cat $gcs_input_uri"
1381-
]
1375+
"! gcloud storage cat $gcs_input_uri" ]
13821376
},
13831377
{
13841378
"cell_type": "markdown",
@@ -1646,8 +1640,7 @@
16461640
"source": [
16471641
"def get_latest_predictions(gcs_out_dir):\n",
16481642
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
1649-
" folders = !gsutil ls $gcs_out_dir\n",
1650-
" latest = \"\"\n",
1643+
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
16511644
" for folder in folders:\n",
16521645
" subfolder = folder.split(\"/\")[-2]\n",
16531646
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1664,10 +1657,8 @@
16641657
" raise Exception(\"Batch Job Failed\")\n",
16651658
" else:\n",
16661659
" folder = get_latest_predictions(predictions)\n",
1667-
" ! gsutil ls $folder/prediction*.csv\n",
1668-
"\n",
1669-
" ! gsutil cat $folder/prediction*.csv\n",
1670-
" break\n",
1660+
" ! gcloud storage ls $folder/prediction*.csv\n", "\n",
1661+
" ! gcloud storage cat $folder/prediction*.csv\n", " break\n",
16711662
" time.sleep(60)"
16721663
]
16731664
},
@@ -1761,8 +1752,7 @@
17611752
" print(e)\n",
17621753
"\n",
17631754
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1764-
" ! gsutil rm -r $BUCKET_NAME"
1765-
]
1755+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
17661756
}
17671757
],
17681758
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_tabular_classification_batch_explain.ipynb‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -421,8 +421,7 @@
421421
},
422422
"outputs": [],
423423
"source": [
424-
"! gsutil mb -l $REGION $BUCKET_NAME"
425-
]
424+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
426425
},
427426
{
428427
"cell_type": "markdown",
@@ -441,8 +440,7 @@
441440
},
442441
"outputs": [],
443442
"source": [
444-
"! gsutil ls -al $BUCKET_NAME"
445-
]
443+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
446444
},
447445
{
448446
"cell_type": "markdown",
@@ -810,14 +808,11 @@
810808
},
811809
"outputs": [],
812810
"source": [
813-
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
814-
"print(\"Number of Examples\", int(count[0]))\n",
811+
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
815812
"\n",
816813
"print(\"First 10 rows\")\n",
817-
"! gsutil cat $IMPORT_FILE | head\n",
818-
"\n",
819-
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
820-
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
814+
"! gcloud storage cat $IMPORT_FILE | head\n", "\n",
815+
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
821816
"print(\"Label Column Name\", label_column)\n",
822817
"if label_column is None:\n",
823818
" raise Exception(\"label column missing\")"
@@ -1363,8 +1358,7 @@
13631358
" f.write(str(INSTANCE_2) + \"\\n\")\n",
13641359
"\n",
13651360
"print(gcs_input_uri)\n",
1366-
"! gsutil cat $gcs_input_uri"
1367-
]
1361+
"! gcloud storage cat $gcs_input_uri" ]
13681362
},
13691363
{
13701364
"cell_type": "markdown",
@@ -1631,8 +1625,7 @@
16311625
"source": [
16321626
"def get_latest_predictions(gcs_out_dir):\n",
16331627
" \"\"\" Get the latest prediction subfolder using the timestamp in the subfolder name\"\"\"\n",
1634-
" folders = !gsutil ls $gcs_out_dir\n",
1635-
" latest = \"\"\n",
1628+
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
16361629
" for folder in folders:\n",
16371630
" subfolder = folder.split(\"/\")[-2]\n",
16381631
" if subfolder.startswith(\"prediction-\"):\n",
@@ -1649,10 +1642,8 @@
16491642
" raise Exception(\"Batch Job Failed\")\n",
16501643
" else:\n",
16511644
" folder = get_latest_predictions(predictions)\n",
1652-
" ! gsutil ls $folder/explanation*.csv\n",
1653-
"\n",
1654-
" ! gsutil cat $folder/explanation*.csv\n",
1655-
" break\n",
1645+
" ! gcloud storage ls $folder/explanation*.csv\n", "\n",
1646+
" ! gcloud storage cat $folder/explanation*.csv\n", " break\n",
16561647
" time.sleep(60)"
16571648
]
16581649
},
@@ -1746,8 +1737,7 @@
17461737
" print(e)\n",
17471738
"\n",
17481739
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1749-
" ! gsutil rm -r $BUCKET_NAME"
1750-
]
1740+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
17511741
}
17521742
],
17531743
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_video_action_recognition_batch.ipynb‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -420,8 +420,7 @@
420420
},
421421
"outputs": [],
422422
"source": [
423-
"! gsutil mb -l $REGION $BUCKET_NAME"
424-
]
423+
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
425424
},
426425
{
427426
"cell_type": "markdown",
@@ -440,8 +439,7 @@
440439
},
441440
"outputs": [],
442441
"source": [
443-
"! gsutil ls -al $BUCKET_NAME"
444-
]
442+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
445443
},
446444
{
447445
"cell_type": "markdown",
@@ -888,12 +886,10 @@
888886
"else:\n",
889887
" FILE = IMPORT_FILE\n",
890888
"\n",
891-
"count = ! gsutil cat $FILE | wc -l\n",
892-
"print(\"Number of Examples\", int(count[0]))\n",
889+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
893890
"\n",
894891
"print(\"First 10 rows\")\n",
895-
"! gsutil cat $FILE | head"
896-
]
892+
"! gcloud storage cat $FILE | head" ]
897893
},
898894
{
899895
"cell_type": "markdown",
@@ -1310,8 +1306,7 @@
13101306
"import json\n",
13111307
"\n",
13121308
"import_file = IMPORT_FILES[0]\n",
1313-
"test_items = ! gsutil cat $import_file | head -n2\n",
1314-
"\n",
1309+
"test_items = ! gcloud storage cat $import_file | head -n2\n", "\n",
13151310
"cols = str(test_items[0]).split(',')\n",
13161311
"test_item_1 = str(cols[0])\n",
13171312
"test_label_1 = str(cols[-1])\n",
@@ -1360,8 +1355,7 @@
13601355
" f.write(json.dumps(data) + '\\n')\n",
13611356
"\n",
13621357
"print(gcs_input_uri)\n",
1363-
"! gsutil cat $gcs_input_uri"
1364-
]
1358+
"! gcloud storage cat $gcs_input_uri" ]
13651359
},
13661360
{
13671361
"cell_type": "markdown",
@@ -1637,8 +1631,7 @@
16371631
"source": [
16381632
"def get_latest_predictions(gcs_out_dir):\n",
16391633
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
1640-
" folders = !gsutil ls $gcs_out_dir\n",
1641-
" latest = \"\"\n",
1634+
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
16421635
" for folder in folders:\n",
16431636
" subfolder = folder.split('/')[-2]\n",
16441637
" if subfolder.startswith('prediction-'):\n",
@@ -1655,10 +1648,8 @@
16551648
" raise Exception(\"Batch Job Failed\")\n",
16561649
" else:\n",
16571650
" folder = get_latest_predictions(predictions)\n",
1658-
" ! gsutil ls $folder/prediction*.jsonl\n",
1659-
"\n",
1660-
" ! gsutil cat $folder/prediction*.jsonl\n",
1661-
" break\n",
1651+
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
1652+
" ! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
16621653
" time.sleep(60)"
16631654
]
16641655
},
@@ -1752,8 +1743,7 @@
17521743
" print(e)\n",
17531744
"\n",
17541745
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
1755-
" ! gsutil rm -r $BUCKET_NAME"
1756-
]
1746+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
17571747
}
17581748
],
17591749
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_video_classification_batch.ipynb‎

Lines changed: 10 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -420,8 +420,7 @@
420420
},
421421
"outputs": [],
422422
"source": [
423-
"! gsutil mb -l $REGION $BUCKET_NAME"
424-
]
423+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME" ]
425424
},
426425
{
427426
"cell_type": "markdown",
@@ -440,8 +439,7 @@
440439
},
441440
"outputs": [],
442441
"source": [
443-
"! gsutil ls -al $BUCKET_NAME"
444-
]
442+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
445443
},
446444
{
447445
"cell_type": "markdown",
@@ -885,12 +883,10 @@
885883
"else:\n",
886884
" FILE = IMPORT_FILE\n",
887885
"\n",
888-
"count = ! gsutil cat $FILE | wc -l\n",
889-
"print(\"Number of Examples\", int(count[0]))\n",
886+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
890887
"\n",
891888
"print(\"First 10 rows\")\n",
892-
"! gsutil cat $FILE | head"
893-
]
889+
"! gcloud storage cat $FILE | head" ]
894890
},
895891
{
896892
"cell_type": "markdown",
@@ -1301,8 +1297,7 @@
13011297
},
13021298
"outputs": [],
13031299
"source": [
1304-
"test_items = ! gsutil cat $IMPORT_FILE | head -n2\n",
1305-
"\n",
1300+
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n2\n", "\n",
13061301
"if len(test_items[0]) == 5:\n",
13071302
" _, test_item_1, test_label_1, _, _ = str(test_items[0]).split(',')\n",
13081303
" _, test_item_2, test_label_2, _, _ = str(test_items[1]).split(',')\n",
@@ -1351,8 +1346,7 @@
13511346
" f.write(json.dumps(data) + '\\n')\n",
13521347
"\n",
13531348
"print(gcs_input_uri)\n",
1354-
"! gsutil cat $gcs_input_uri"
1355-
]
1349+
"! gcloud storage cat $gcs_input_uri" ]
13561350
},
13571351
{
13581352
"cell_type": "markdown",
@@ -1630,8 +1624,7 @@
16301624
"source": [
16311625
"def get_latest_predictions(gcs_out_dir):\n",
16321626
" ''' Get the latest prediction subfolder using the timestamp in the subfolder name'''\n",
1633-
" folders = !gsutil ls $gcs_out_dir\n",
1634-
" latest = \"\"\n",
1627+
" folders = !gcloud storage ls $gcs_out_dir\n", " latest = \"\"\n",
16351628
" for folder in folders:\n",
16361629
" subfolder = folder.split('/')[-2]\n",
16371630
" if subfolder.startswith('prediction-'):\n",
@@ -1648,10 +1641,8 @@
16481641
" raise Exception(\"Batch Job Failed\")\n",
16491642
" else:\n",
16501643
" folder = get_latest_predictions(predictions)\n",
1651-
" ! gsutil ls $folder/prediction*.jsonl\n",
1652-
"\n",
1653-
" ! gsutil cat $folder/prediction*.jsonl\n",
1654-
" break\n",
1644+
" ! gcloud storage ls $folder/prediction*.jsonl\n", "\n",
1645+
"! gcloud storage cat $folder/prediction*.jsonl\n", " break\n",
16551646
" time.sleep(60)"
16561647
]
16571648
},
@@ -1745,8 +1736,7 @@
17451736
" print(e)\n",
17461737
"\n",
17471738
"if delete_bucket and 'BUCKET_NAME' in globals():\n",
1748-
" ! gsutil rm -r $BUCKET_NAME"
1749-
]
1739+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
17501740
}
17511741
],
17521742
"metadata": {

0 commit comments

Comments
 (0)