Skip to content

Commit 2a8ad7c

Browse files
googlyrahmangurusai-voleti
googlyrahman
and
gurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4319)
* Migrate gsutil usage to gcloud storage * changes for 4319 * changes for 4319 * Apply automated linter fixes * remove unused import * removed model_garden changes * Update model_garden_pytorch_mixtral_peft_tuning.ipynb --------- Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 3239b30 commit 2a8ad7c

File tree

8 files changed

+85
-86
lines changed

8 files changed

+85
-86
lines changed

‎notebooks/community/migration/UJ6 legacy AutoML Natural Language Text Classification.ipynb‎

Lines changed: 47 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -109,27 +109,27 @@
109109
"id": "Af0jTPSgl9yh"
110110
},
111111
"source": [
112-
"## Before you begin\r\n",
113-
"\r\n",
114-
"### GPU run-time\r\n",
115-
"\r\n",
116-
"*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\r\n",
117-
"\r\n",
118-
"### Set up your GCP project\r\n",
119-
"\r\n",
120-
"**The following steps are required, regardless of your notebook environment.**\r\n",
121-
"\r\n",
122-
"1. [Select or create a GCP project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\r\n",
123-
"\r\n",
124-
"2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\r\n",
125-
"\r\n",
126-
"3. [Enable the AutoML APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component)\r\n",
127-
"\r\n",
128-
"4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in AutoML Notebooks.\r\n",
129-
"\r\n",
130-
"5. Enter your project ID in the cell below. Then run the cell to make sure the\r\n",
131-
"Cloud SDK uses the right project for all the commands in this notebook.\r\n",
132-
"\r\n",
112+
"## Before you begin\n",
113+
"\n",
114+
"### GPU run-time\n",
115+
"\n",
116+
"*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n",
117+
"\n",
118+
"### Set up your GCP project\n",
119+
"\n",
120+
"**The following steps are required, regardless of your notebook environment.**\n",
121+
"\n",
122+
"1. [Select or create a GCP project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n",
123+
"\n",
124+
"2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n",
125+
"\n",
126+
"3. [Enable the AutoML APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component)\n",
127+
"\n",
128+
"4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in AutoML Notebooks.\n",
129+
"\n",
130+
"5. Enter your project ID in the cell below. Then run the cell to make sure the\n",
131+
"Cloud SDK uses the right project for all the commands in this notebook.\n",
132+
"\n",
133133
"**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands."
134134
]
135135
},
@@ -240,11 +240,11 @@
240240
"id": "h_L3MRsOmYED"
241241
},
242242
"source": [
243-
"### Authenticate your GCP account\r\n",
244-
"\r\n",
245-
"**If you are using AutoML Notebooks**, your environment is already\r\n",
246-
"authenticated. Skip this step.\r\n",
247-
"\r\n",
243+
"### Authenticate your GCP account\n",
244+
"\n",
245+
"**If you are using AutoML Notebooks**, your environment is already\n",
246+
"authenticated. Skip this step.\n",
247+
"\n",
248248
"*Note: If you are on an AutoML notebook and run the cell, the cell knows to skip executing the authentication steps.*"
249249
]
250250
},
@@ -336,7 +336,7 @@
336336
},
337337
"outputs": [],
338338
"source": [
339-
"! gsutil mb -l $REGION gs://$BUCKET_NAME"
339+
"! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME"
340340
]
341341
},
342342
{
@@ -356,7 +356,7 @@
356356
},
357357
"outputs": [],
358358
"source": [
359-
"! gsutil ls -al gs://$BUCKET_NAME"
359+
"! gcloud storage ls --all-versions --long gs://$BUCKET_NAME"
360360
]
361361
},
362362
{
@@ -391,7 +391,6 @@
391391
"outputs": [],
392392
"source": [
393393
"import json\n",
394-
"import time\n",
395394
"\n",
396395
"from google.cloud import automl\n",
397396
"from google.protobuf.json_format import MessageToJson"
@@ -483,7 +482,7 @@
483482
},
484483
"outputs": [],
485484
"source": [
486-
"! gsutil cat $IMPORT_FILE | head -n 10"
485+
"! gcloud storage cat $IMPORT_FILE | head -n 10"
487486
]
488487
},
489488
{
@@ -1140,7 +1139,7 @@
11401139
},
11411140
"outputs": [],
11421141
"source": [
1143-
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
1142+
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
11441143
"test_item, test_label = str(test_item[0]).split(\",\")\n",
11451144
"\n",
11461145
"print(test_item, test_label)"
@@ -1175,8 +1174,8 @@
11751174
},
11761175
"outputs": [],
11771176
"source": [
1178-
"! gsutil cat $gcs_input_uri\n",
1179-
"! gsutil cat $test_item_uri"
1177+
"! gcloud storage cat $gcs_input_uri\n",
1178+
"! gcloud storage cat $test_item_uri"
11801179
]
11811180
},
11821181
{
@@ -1309,9 +1308,9 @@
13091308
"id": "771dDuKzg8Mk"
13101309
},
13111310
"source": [
1312-
"*Example output*:\r\n",
1313-
"```\r\n",
1314-
"{}\r\n",
1311+
"*Example output*:\n",
1312+
"```\n",
1313+
"{}\n",
13151314
"```"
13161315
]
13171316
},
@@ -1325,8 +1324,8 @@
13251324
"source": [
13261325
"destination_uri = output_config[\"gcs_destination\"][\"output_uri_prefix\"][:-1]\n",
13271326
"\n",
1328-
"! gsutil ls $destination_uri/*\n",
1329-
"! gsutil cat $destination_uri/prediction*/*.jsonl"
1327+
"! gcloud storage ls $destination_uri/*\n",
1328+
"! gcloud storage cat $destination_uri/prediction*/*.jsonl"
13301329
]
13311330
},
13321331
{
@@ -1409,9 +1408,9 @@
14091408
"id": "CZ-62obNmBNc"
14101409
},
14111410
"source": [
1412-
"*Example output*:\r\n",
1413-
"```\r\n",
1414-
"{}\r\n",
1411+
"*Example output*:\n",
1412+
"```\n",
1413+
"{}\n",
14151414
"```"
14161415
]
14171416
},
@@ -1441,7 +1440,7 @@
14411440
},
14421441
"outputs": [],
14431442
"source": [
1444-
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
1443+
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
14451444
"test_item, test_label = str(test_item[0]).split(\",\")"
14461445
]
14471446
},
@@ -1599,11 +1598,11 @@
15991598
"id": "bQ-VVaSxJjkd"
16001599
},
16011600
"source": [
1602-
"# Cleaning up\r\n",
1603-
"\r\n",
1604-
"To clean up all GCP resources used in this project, you can [delete the GCP\r\n",
1605-
"project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\r\n",
1606-
"\r\n",
1601+
"# Cleaning up\n",
1602+
"\n",
1603+
"To clean up all GCP resources used in this project, you can [delete the GCP\n",
1604+
"project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n",
1605+
"\n",
16071606
"Otherwise, you can delete the individual resources you created in this tutorial."
16081607
]
16091608
},
@@ -1635,7 +1634,7 @@
16351634
"\n",
16361635
"\n",
16371636
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1638-
" ! gsutil rm -r gs://$BUCKET_NAME"
1637+
" ! gcloud storage rm --recursive gs://$BUCKET_NAME"
16391638
]
16401639
}
16411640
],

‎notebooks/community/ml_ops/stage2/get_started_bqml_training.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -435,7 +435,7 @@
435435
},
436436
"outputs": [],
437437
"source": [
438-
"! gsutil mb -l $REGION $BUCKET_URI"
438+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
439439
]
440440
},
441441
{
@@ -455,7 +455,7 @@
455455
},
456456
"outputs": [],
457457
"source": [
458-
"! gsutil ls -al $BUCKET_URI"
458+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
459459
]
460460
},
461461
{
@@ -850,7 +850,7 @@
850850
"! bq extract -m $param\n",
851851
"\n",
852852
"MODEL_DIR = f\"{BUCKET_URI}/{BQ_DATASET_NAME}\"\n",
853-
"! gsutil ls $MODEL_DIR"
853+
"! gcloud storage ls $MODEL_DIR"
854854
]
855855
},
856856
{
@@ -1398,7 +1398,7 @@
13981398
"delete_storage = False\n",
13991399
"if delete_storage or os.getenv(\"IS_TESTING\"):\n",
14001400
" # Delete the created GCS bucket\n",
1401-
" ! gsutil rm -r $BUCKET_URI"
1401+
" ! gcloud storage rm --recursive $BUCKET_URI"
14021402
]
14031403
}
14041404
],

‎notebooks/community/ml_ops/stage3/get_started_with_dataflow_pipeline_components.ipynb‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -462,7 +462,7 @@
462462
},
463463
"outputs": [],
464464
"source": [
465-
"! gsutil mb -l $REGION $BUCKET_URI"
465+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
466466
]
467467
},
468468
{
@@ -482,7 +482,7 @@
482482
},
483483
"outputs": [],
484484
"source": [
485-
"! gsutil ls -al $BUCKET_URI"
485+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
486486
]
487487
},
488488
{
@@ -552,9 +552,9 @@
552552
},
553553
"outputs": [],
554554
"source": [
555-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
555+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
556556
"\n",
557-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
557+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
558558
]
559559
},
560560
{
@@ -801,9 +801,9 @@
801801
"outputs": [],
802802
"source": [
803803
"GCS_WC_PY = BUCKET_URI + \"/wc.py\"\n",
804-
"! gsutil cp wc.py $GCS_WC_PY\n",
804+
"! gcloud storage cp wc.py $GCS_WC_PY\n",
805805
"GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n",
806-
"! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n",
806+
"! gcloud storage cp requirements.txt $GCS_REQUIREMENTS_TXT\n",
807807
"\n",
808808
"GCS_WC_OUT = BUCKET_URI + \"/wc_out.txt\""
809809
]
@@ -872,7 +872,7 @@
872872
"\n",
873873
"pipeline.run()\n",
874874
"\n",
875-
"! gsutil cat {GCS_WC_OUT}* | head -n10\n",
875+
"! gcloud storage cat {GCS_WC_OUT}* | head -n10\n",
876876
"\n",
877877
"! rm -f dataflow_wc.json wc.py requirements.txt"
878878
]
@@ -1097,11 +1097,11 @@
10971097
"outputs": [],
10981098
"source": [
10991099
"GCS_SPLIT_PY = BUCKET_URI + \"/split.py\"\n",
1100-
"! gsutil cp split.py $GCS_SPLIT_PY\n",
1100+
"! gcloud storage cp split.py $GCS_SPLIT_PY\n",
11011101
"GCS_REQUIREMENTS_TXT = BUCKET_URI + \"/requirements.txt\"\n",
1102-
"! gsutil cp requirements.txt $GCS_REQUIREMENTS_TXT\n",
1102+
"! gcloud storage cp requirements.txt $GCS_REQUIREMENTS_TXT\n",
11031103
"GCS_SETUP_PY = BUCKET_URI + \"/setup.py\"\n",
1104-
"! gsutil cp setup.py $GCS_SETUP_PY"
1104+
"! gcloud storage cp setup.py $GCS_SETUP_PY"
11051105
]
11061106
},
11071107
{
@@ -1202,7 +1202,7 @@
12021202
"\n",
12031203
"pipeline.run()\n",
12041204
"\n",
1205-
"! gsutil ls {BUCKET_URI}/exported_data\n",
1205+
"! gcloud storage ls {BUCKET_URI}/exported_data\n",
12061206
"\n",
12071207
"! rm -f dataflow_split.json split.py requirements.txt"
12081208
]
@@ -1258,7 +1258,7 @@
12581258
"delete_bucket = False\n",
12591259
"\n",
12601260
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1261-
" ! gsutil rm -r $BUCKET_URI"
1261+
" ! gcloud storage rm --recursive $BUCKET_URI"
12621262
]
12631263
}
12641264
],

‎notebooks/community/ml_ops/stage4/get_started_with_vertex_ml_metadata.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@
357357
},
358358
"outputs": [],
359359
"source": [
360-
"! gsutil mb -l $REGION $BUCKET_URI"
360+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
361361
]
362362
},
363363
{
@@ -427,9 +427,9 @@
427427
},
428428
"outputs": [],
429429
"source": [
430-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
430+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
431431
"\n",
432-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
432+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
433433
]
434434
},
435435
{
@@ -1386,7 +1386,7 @@
13861386
"delete_bucket = False\n",
13871387
"\n",
13881388
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1389-
" ! gsutil rm -r $BUCKET_URI"
1389+
" ! gcloud storage rm --recursive $BUCKET_URI"
13901390
]
13911391
}
13921392
],

0 commit comments

Comments
 (0)