Skip to content

Commit 9e9e639

Browse files
googlyrahmanbhandarivijay-pnggurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4323)
* Migrate gsutil usage to gcloud storage * Manual Changes-Migrate gsutil usage to gcloud storage * Manual Changes * Fix linter issue for 4323 * removed changes for model garden * Update model_garden_axolotl_gpt_oss_finetuning.ipynb --------- Co-authored-by: bhandarivijay <bhandarivijay@google.com> Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 633cf6a commit 9e9e639

8 files changed

+41
-43
lines changed

‎notebooks/community/ml_ops/stage3/get_started_vertex_pipelines_sklearn_with_prediction.ipynb‎

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -488,7 +488,7 @@
488488
},
489489
"outputs": [],
490490
"source": [
491-
"! gsutil mb -l $REGION $BUCKET_URI"
491+
"! gcloud storage buckets create --location $REGION $BUCKET_URI"
492492
]
493493
},
494494
{
@@ -508,7 +508,7 @@
508508
},
509509
"outputs": [],
510510
"source": [
511-
"! gsutil ls -al $BUCKET_URI"
511+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
512512
]
513513
},
514514
{
@@ -579,9 +579,9 @@
579579
},
580580
"outputs": [],
581581
"source": [
582-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
582+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
583583
"\n",
584-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
584+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
585585
]
586586
},
587587
{
@@ -990,7 +990,7 @@
990990
"! rm -f custom.tar custom.tar.gz\n",
991991
"! tar cvf custom.tar custom\n",
992992
"! gzip custom.tar\n",
993-
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_newsaggr.tar.gz"
993+
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_newsaggr.tar.gz"
994994
]
995995
},
996996
{
@@ -1206,13 +1206,13 @@
12061206
" + \"/evaluation_metrics\"\n",
12071207
" )\n",
12081208
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
1209-
" ! gsutil cat $EXECUTE_OUTPUT\n",
1209+
" ! gcloud storage cat $EXECUTE_OUTPUT\n",
12101210
" return EXECUTE_OUTPUT\n",
12111211
" elif tf.io.gfile.exists(GCP_RESOURCES):\n",
1212-
" ! gsutil cat $GCP_RESOURCES\n",
1212+
" ! gcloud storage cat $GCP_RESOURCES\n",
12131213
" return GCP_RESOURCES\n",
12141214
" elif tf.io.gfile.exists(EVAL_METRICS):\n",
1215-
" ! gsutil cat $EVAL_METRICS\n",
1215+
" ! gcloud storage cat $EVAL_METRICS\n",
12161216
" return EVAL_METRICS\n",
12171217
"\n",
12181218
" return None\n",
@@ -1221,14 +1221,14 @@
12211221
"print(\"custompythonpackagetrainingjob-run\")\n",
12221222
"artifacts = print_pipeline_output(pipeline, \"custompythonpackagetrainingjob-run\")\n",
12231223
"print(\"\\n\\n\")\n",
1224-
"output = !gsutil cat $artifacts\n",
1224+
"output = !gcloud storage cat $artifacts\n",
12251225
"output = json.loads(output[0])\n",
12261226
"model_id = output[\"artifacts\"][\"model\"][\"artifacts\"][0][\"metadata\"][\"resourceName\"]\n",
12271227
"print(\"\\n\\n\")\n",
12281228
"print(\"endpoint-create\")\n",
12291229
"artifacts = print_pipeline_output(pipeline, \"endpoint-create\")\n",
12301230
"print(\"\\n\\n\")\n",
1231-
"output = !gsutil cat $artifacts\n",
1231+
"output = !gcloud storage cat $artifacts\n",
12321232
"output = json.loads(output[0])\n",
12331233
"endpoint_id = output[\"artifacts\"][\"endpoint\"][\"artifacts\"][0][\"metadata\"][\n",
12341234
" \"resourceName\"\n",
@@ -1394,7 +1394,7 @@
13941394
"source": [
13951395
"delete_bucket = False\n",
13961396
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1397-
" ! gsutil rm -r $BUCKET_URI\n",
1397+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
13981398
"\n",
13991399
"! rm -rf custom custom.tar.csv"
14001400
]

‎notebooks/community/ml_ops/stage3/get_started_with_airflow_and_vertex_pipelines.ipynb‎

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -456,7 +456,7 @@
456456
},
457457
"outputs": [],
458458
"source": [
459-
"! gsutil mb -l $REGION $BUCKET_URI"
459+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
460460
]
461461
},
462462
{
@@ -476,7 +476,7 @@
476476
},
477477
"outputs": [],
478478
"source": [
479-
"! gsutil ls -al $BUCKET_URI"
479+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
480480
]
481481
},
482482
{
@@ -546,9 +546,9 @@
546546
},
547547
"outputs": [],
548548
"source": [
549-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
549+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
550550
"\n",
551-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
551+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
552552
]
553553
},
554554
{
@@ -569,7 +569,6 @@
569569
"outputs": [],
570570
"source": [
571571
"from google.cloud import aiplatform\n",
572-
"from kfp import dsl\n",
573572
"from kfp.v2 import compiler\n",
574573
"from kfp.v2.dsl import Artifact, Output, component"
575574
]
@@ -910,7 +909,7 @@
910909
},
911910
"outputs": [],
912911
"source": [
913-
"!gsutil cp $COMPOSER_DAG_FILENAME $COMPOSER_DAG_GCS/"
912+
"!gcloud storage cp $COMPOSER_DAG_FILENAME $COMPOSER_DAG_GCS/"
914913
]
915914
},
916915
{
@@ -921,7 +920,7 @@
921920
},
922921
"outputs": [],
923922
"source": [
924-
"!gsutil ls -l $COMPOSER_DAG_GCS/$COMPOSER_DAG_FILENAME"
923+
"!gcloud storage ls --long $COMPOSER_DAG_GCS/$COMPOSER_DAG_FILENAME"
925924
]
926925
},
927926
{
@@ -1153,7 +1152,6 @@
11531152
"# You can change the `text` and `emoji_str` parameters here to update the pipeline output\n",
11541153
"def pipeline():\n",
11551154
" data_processing_task_dag_name = COMPOSER_DAG_NAME\n",
1156-
" data_processing_task = trigger_airflow_dag(\n",
11571155
" dag_name=data_processing_task_dag_name,\n",
11581156
" composer_client_id=COMPOSER_CLIENT_ID,\n",
11591157
" composer_webserver_id=COMPOSER_WEB_URI,\n",
@@ -1308,13 +1306,13 @@
13081306
" + \"/evaluation_metrics\"\n",
13091307
" )\n",
13101308
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
1311-
" ! gsutil cat $EXECUTE_OUTPUT\n",
1309+
" ! gcloud storage cat $EXECUTE_OUTPUT\n",
13121310
" return EXECUTE_OUTPUT\n",
13131311
" elif tf.io.gfile.exists(GCP_RESOURCES):\n",
1314-
" ! gsutil cat $GCP_RESOURCES\n",
1312+
" ! gcloud storage cat $GCP_RESOURCES\n",
13151313
" return GCP_RESOURCES\n",
13161314
" elif tf.io.gfile.exists(EVAL_METRICS):\n",
1317-
" ! gsutil cat $EVAL_METRICS\n",
1315+
" ! gcloud storage cat $EVAL_METRICS\n",
13181316
" return EVAL_METRICS\n",
13191317
"\n",
13201318
" return None\n",
@@ -1389,7 +1387,7 @@
13891387
"! bq rm -r -f $PROJECT_ID:$DATASET_NAME\n",
13901388
"\n",
13911389
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1392-
" ! gsutil rm -rf {BUCKET_URI}\n",
1390+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n",
13931391
" \n",
13941392
"! rm get_composer_config.py data_orchestration_bq_example_dag.py"
13951393
]

‎notebooks/community/sdk/SDK_FBProphet_Forecasting_Online.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,7 @@
348348
},
349349
"outputs": [],
350350
"source": [
351-
"! gsutil mb -l {REGION} {BUCKET_URI}"
351+
"! gcloud storage buckets create --location={REGION} {BUCKET_URI}"
352352
]
353353
},
354354
{
@@ -742,7 +742,7 @@
742742
"outputs": [],
743743
"source": [
744744
"%cd app\n",
745-
"!gsutil cp *.sav *.csv {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
745+
"!gcloud storage cp *.sav *.csv {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
746746
"%cd .."
747747
]
748748
},
@@ -1541,7 +1541,7 @@
15411541
"\n",
15421542
"# Delete the Cloud Storage bucket\n",
15431543
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1544-
" ! gsutil -m rm -r $BUCKET_URI"
1544+
" ! gcloud storage rm --recursive $BUCKET_URI"
15451545
]
15461546
}
15471547
],

‎notebooks/community/sdk/sdk_custom_tabular_regression_online.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -475,7 +475,7 @@
475475
},
476476
"outputs": [],
477477
"source": [
478-
"! gsutil mb -l $REGION $BUCKET_NAME"
478+
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
479479
]
480480
},
481481
{
@@ -495,7 +495,7 @@
495495
},
496496
"outputs": [],
497497
"source": [
498-
"! gsutil ls -al $BUCKET_NAME"
498+
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
499499
]
500500
},
501501
{
@@ -921,7 +921,7 @@
921921
"! rm -f custom.tar custom.tar.gz\n",
922922
"! tar cvf custom.tar custom\n",
923923
"! gzip custom.tar\n",
924-
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
924+
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_boston.tar.gz"
925925
]
926926
},
927927
{
@@ -1427,7 +1427,7 @@
14271427
" print(e)\n",
14281428
"\n",
14291429
" if \"BUCKET_NAME\" in globals():\n",
1430-
" ! gsutil rm -r $BUCKET_NAME"
1430+
" ! gcloud storage rm --recursive $BUCKET_NAME"
14311431
]
14321432
}
14331433
],

‎notebooks/community/sdk/sdk_pytorch_lightning_custom_container_training.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -505,7 +505,7 @@
505505
},
506506
"outputs": [],
507507
"source": [
508-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
508+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
509509
]
510510
},
511511
{
@@ -525,7 +525,7 @@
525525
},
526526
"outputs": [],
527527
"source": [
528-
"! gsutil ls -al $BUCKET_URI"
528+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
529529
]
530530
},
531531
{
@@ -1521,10 +1521,10 @@
15211521
"\n",
15221522
"CONTENT_DIR = f\"{BUCKET_URI}/{content_name}*\"\n",
15231523
"# Delete Cloud Storage objects that were created\n",
1524-
"! gsutil -m rm -r $CONTENT_DIR\n",
1524+
"! gcloud storage rm --recursive $CONTENT_DIR\n",
15251525
"\n",
15261526
"if delete_bucket and \"BUCKET_URI\" in globals():\n",
1527-
" ! gsutil -m rm -r $BUCKET_URI"
1527+
" ! gcloud storage rm --recursive $BUCKET_URI"
15281528
]
15291529
}
15301530
],

‎notebooks/official/migration/sdk-custom-scikit-learn-prebuilt-container.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@
320320
},
321321
"outputs": [],
322322
"source": [
323-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
323+
"! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"
324324
]
325325
},
326326
{
@@ -699,7 +699,7 @@
699699
"! rm -f custom.tar custom.tar.gz\n",
700700
"! tar cvf custom.tar custom\n",
701701
"! gzip custom.tar\n",
702-
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_census.tar.gz"
702+
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_census.tar.gz"
703703
]
704704
},
705705
{
@@ -965,7 +965,7 @@
965965
" for i in INSTANCES:\n",
966966
" f.write(json.dumps(i) + \"\\n\")\n",
967967
"\n",
968-
"! gsutil cat $gcs_input_uri"
968+
"! gcloud storage cat $gcs_input_uri"
969969
]
970970
},
971971
{
@@ -1378,7 +1378,7 @@
13781378
"# Delete the Cloud Storage bucket\n",
13791379
"delete_bucket = False # set True for deletion\n",
13801380
"if delete_bucket:\n",
1381-
" ! gsutil -m rm -r $BUCKET_URI"
1381+
" ! gcloud storage rm --recursive $BUCKET_URI"
13821382
]
13831383
}
13841384
],

‎notebooks/official/model_monitoring/get_started_with_model_monitoring_custom.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,7 @@
364364
},
365365
"outputs": [],
366366
"source": [
367-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
367+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
368368
]
369369
},
370370
{
@@ -928,7 +928,7 @@
928928
"with open(\"schema.yaml\", \"w\") as f:\n",
929929
" f.write(yaml)\n",
930930
"\n",
931-
"! gsutil cp schema.yaml {BUCKET_URI}/schema.yaml"
931+
"! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml"
932932
]
933933
},
934934
{
@@ -1406,7 +1406,7 @@
14061406
"# Delete the Cloud Storage bucket\n",
14071407
"delete_bucket = False # Set True for deletion\n",
14081408
"if delete_bucket:\n",
1409-
" ! gsutil rm -rf {BUCKET_URI}\n",
1409+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n",
14101410
"\n",
14111411
"# Delete the locally generated files\n",
14121412
"! rm -f schema.yaml\n",

‎notebooks/official/workbench/chicago_taxi_fare_prediction/chicago_taxi_fare_prediction.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,7 @@
365365
},
366366
"outputs": [],
367367
"source": [
368-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
368+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
369369
]
370370
},
371371
{
@@ -1467,7 +1467,7 @@
14671467
"delete_bucket = False\n",
14681468
"\n",
14691469
"if delete_bucket:\n",
1470-
" ! gsutil -m rm -r $BUCKET_URI"
1470+
" ! gcloud storage rm --recursive $BUCKET_URI"
14711471
]
14721472
}
14731473
],

0 commit comments

Comments
 (0)