Skip to content

Commit 814827a

Browse files
googlyrahmangooglyrahman
authored
Migrate gsutil usage to gcloud storage (#4332)
1 parent 7a61378 commit 814827a

10 files changed

+40
-82
lines changed

‎notebooks/community/gapic/automl/showcase_automl_text_classification_online.ipynb‎

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -716,12 +716,10 @@
716716
"else:\n",
717717
" FILE = IMPORT_FILE\n",
718718
"\n",
719-
"count = ! gsutil cat $FILE | wc -l\n",
720-
"print(\"Number of Examples\", int(count[0]))\n",
719+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
721720
"\n",
722721
"print(\"First 10 rows\")\n",
723-
"! gsutil cat $FILE | head"
724-
]
722+
"! gcloud storage cat $FILE | head" ]
725723
},
726724
{
727725
"cell_type": "markdown",
@@ -1327,7 +1325,7 @@
13271325
},
13281326
"outputs": [],
13291327
"source": [
1330-
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
1328+
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
13311329
"if len(test_item[0]) == 3:\n",
13321330
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
13331331
"else:\n",
@@ -1533,8 +1531,7 @@
15331531
" print(e)\n",
15341532
"\n",
15351533
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1536-
" ! gsutil rm -r $BUCKET_NAME"
1537-
]
1534+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
15381535
}
15391536
],
15401537
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_text_multi-label_classification_online.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -716,12 +716,10 @@
716716
"else:\n",
717717
" FILE = IMPORT_FILE\n",
718718
"\n",
719-
"count = ! gsutil cat $FILE | wc -l\n",
720-
"print(\"Number of Examples\", int(count[0]))\n",
719+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
721720
"\n",
722721
"print(\"First 10 rows\")\n",
723-
"! gsutil cat $FILE | head"
724-
]
722+
"! gcloud storage cat $FILE | head" ]
725723
},
726724
{
727725
"cell_type": "markdown",
@@ -1327,8 +1325,7 @@
13271325
},
13281326
"outputs": [],
13291327
"source": [
1330-
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
1331-
"\n",
1328+
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "\n",
13321329
"cols = str(test_item[0]).split(\",\")\n",
13331330
"test_item = cols[0]\n",
13341331
"test_label = cols[1:]\n",
@@ -1533,8 +1530,7 @@
15331530
" print(e)\n",
15341531
"\n",
15351532
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1536-
" ! gsutil rm -r $BUCKET_NAME"
1537-
]
1533+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
15381534
}
15391535
],
15401536
"metadata": {

‎notebooks/community/gapic/automl/showcase_automl_text_sentiment_analysis_online.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -718,12 +718,10 @@
718718
"else:\n",
719719
" FILE = IMPORT_FILE\n",
720720
"\n",
721-
"count = ! gsutil cat $FILE | wc -l\n",
722-
"print(\"Number of Examples\", int(count[0]))\n",
721+
"count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n",
723722
"\n",
724723
"print(\"First 10 rows\")\n",
725-
"! gsutil cat $FILE | head"
726-
]
724+
"! gcloud storage cat $FILE | head" ]
727725
},
728726
{
729727
"cell_type": "markdown",
@@ -1329,8 +1327,7 @@
13291327
},
13301328
"outputs": [],
13311329
"source": [
1332-
"test_item = ! gsutil cat $IMPORT_FILE | head -n1\n",
1333-
"if len(test_item[0]) == 3:\n",
1330+
"test_item = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(test_item[0]) == 3:\n",
13341331
" _, test_item, test_label, max = str(test_item[0]).split(\",\")\n",
13351332
"else:\n",
13361333
" test_item, test_label, max = str(test_item[0]).split(\",\")\n",
@@ -1534,8 +1531,7 @@
15341531
" print(e)\n",
15351532
"\n",
15361533
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1537-
" ! gsutil rm -r $BUCKET_NAME"
1538-
]
1534+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
15391535
}
15401536
],
15411537
"metadata": {

‎notebooks/community/gapic/custom/showcase_tfhub_image_classification_online.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -422,8 +422,7 @@
422422
},
423423
"outputs": [],
424424
"source": [
425-
"! gsutil mb -l $REGION $BUCKET_NAME"
426-
]
425+
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ]
427426
},
428427
{
429428
"cell_type": "markdown",
@@ -442,8 +441,7 @@
442441
},
443442
"outputs": [],
444443
"source": [
445-
"! gsutil ls -al $BUCKET_NAME"
446-
]
444+
"! gcloud storage ls --all-versions --long $BUCKET_NAME" ]
447445
},
448446
{
449447
"cell_type": "markdown",
@@ -1244,8 +1242,7 @@
12441242
"source": [
12451243
"FLOWERS_CSV = \"gs://cloud-ml-data/img/flower_photos/all_data.csv\"\n",
12461244
"\n",
1247-
"test_images = ! gsutil cat $FLOWERS_CSV | head -n1\n",
1248-
"test_image = test_images[0].split(\",\")[0]\n",
1245+
"test_images = ! gcloud storage cat $FLOWERS_CSV | head -n1\n", "test_image = test_images[0].split(\",\")[0]\n",
12491246
"print(test_image)"
12501247
]
12511248
},
@@ -1471,8 +1468,7 @@
14711468
" print(e)\n",
14721469
"\n",
14731470
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1474-
" ! gsutil rm -r $BUCKET_NAME"
1475-
]
1471+
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
14761472
}
14771473
],
14781474
"metadata": {

‎notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_custom.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -561,8 +561,7 @@
561561
},
562562
"outputs": [],
563563
"source": [
564-
"! gsutil mb -l $REGION $BUCKET_URI"
565-
]
564+
"! gcloud storage buckets create --location $REGION $BUCKET_URI" ]
566565
},
567566
{
568567
"cell_type": "markdown",
@@ -581,8 +580,7 @@
581580
},
582581
"outputs": [],
583582
"source": [
584-
"! gsutil ls -al $BUCKET_URI"
585-
]
583+
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
586584
},
587585
{
588586
"cell_type": "markdown",
@@ -1145,8 +1143,7 @@
11451143
"with open(\"schema.yaml\", \"w\") as f:\n",
11461144
" f.write(yaml)\n",
11471145
"\n",
1148-
"! gsutil cp schema.yaml {BUCKET_URI}/schema.yaml"
1149-
]
1146+
"! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" ]
11501147
},
11511148
{
11521149
"cell_type": "markdown",
@@ -1633,8 +1630,7 @@
16331630
"delete_bucket = False\n",
16341631
"\n",
16351632
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1636-
" ! gsutil rm -rf {BUCKET_URI}\n",
1637-
"\n",
1633+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n",
16381634
"! rm -f schema.yaml\n",
16391635
"\n",
16401636
"! bq rm -f {PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}"

‎notebooks/community/ml_ops/stage7/get_started_with_model_monitoring_setup.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -561,8 +561,7 @@
561561
},
562562
"outputs": [],
563563
"source": [
564-
"! gsutil mb -l $REGION $BUCKET_URI"
565-
]
564+
"! gcloud storage buckets create --location $REGION $BUCKET_URI" ]
566565
},
567566
{
568567
"cell_type": "markdown",
@@ -581,8 +580,7 @@
581580
},
582581
"outputs": [],
583582
"source": [
584-
"! gsutil ls -al $BUCKET_URI"
585-
]
583+
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
586584
},
587585
{
588586
"cell_type": "markdown",
@@ -1556,8 +1554,7 @@
15561554
"with open(\"schema.yaml\", \"w\") as f:\n",
15571555
" f.write(yaml)\n",
15581556
"\n",
1559-
"! gsutil cp schema.yaml {BUCKET_URI}/schema.yaml"
1560-
]
1557+
"! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" ]
15611558
},
15621559
{
15631560
"cell_type": "markdown",
@@ -1688,8 +1685,7 @@
16881685
"delete_bucket = True\n",
16891686
"\n",
16901687
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1691-
" ! gsutil rm -rf {BUCKET_URI}\n",
1692-
"\n",
1688+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n",
16931689
"! rm -f schema.yaml\n",
16941690
"\n",
16951691
"! bq rm -r -f {PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}"

‎notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Predict_and_Handler_SDK_Integration.ipynb‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -505,8 +505,7 @@
505505
},
506506
"outputs": [],
507507
"source": [
508-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
509-
]
508+
"! gcloud storage buckets create --location $REGION --project $PROJECT_ID $BUCKET_URI" ]
510509
},
511510
{
512511
"cell_type": "markdown",
@@ -525,8 +524,7 @@
525524
},
526525
"outputs": [],
527526
"source": [
528-
"! gsutil ls -al $BUCKET_URI"
529-
]
527+
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
530528
},
531529
{
532530
"cell_type": "markdown",
@@ -696,9 +694,7 @@
696694
"outputs": [],
697695
"source": [
698696
"%cd ..\n",
699-
"!gsutil cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
700-
"!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
701-
]
697+
"!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ]
702698
},
703699
{
704700
"cell_type": "markdown",
@@ -1899,8 +1895,7 @@
18991895
"delete_bucket = False\n",
19001896
"\n",
19011897
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1902-
" ! gsutil rm -r $BUCKET_URI"
1903-
]
1898+
" ! gcloud storage rm --recursive $BUCKET_URI" ]
19041899
}
19051900
],
19061901
"metadata": {

‎notebooks/community/prediction/custom_prediction_routines/SDK_Pytorch_Custom_Predict.ipynb‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -503,8 +503,7 @@
503503
},
504504
"outputs": [],
505505
"source": [
506-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
507-
]
506+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ]
508507
},
509508
{
510509
"cell_type": "markdown",
@@ -523,8 +522,7 @@
523522
},
524523
"outputs": [],
525524
"source": [
526-
"! gsutil ls -al $BUCKET_URI"
527-
]
525+
"! gcloud storage ls --all-versions --long $BUCKET_URI" ]
528526
},
529527
{
530528
"cell_type": "markdown",
@@ -878,9 +876,7 @@
878876
},
879877
"outputs": [],
880878
"source": [
881-
"!gsutil cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n",
882-
"!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/"
883-
]
879+
"!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ]
884880
},
885881
{
886882
"cell_type": "markdown",
@@ -1952,8 +1948,7 @@
19521948
"delete_bucket = False\n",
19531949
"\n",
19541950
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1955-
" ! gsutil rm -r $BUCKET_URI"
1956-
]
1951+
" ! gcloud storage rm --recursive $BUCKET_URI" ]
19571952
}
19581953
],
19591954
"metadata": {

‎notebooks/official/explainable_ai/sdk_custom_image_classification_batch_explain.ipynb‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -294,8 +294,7 @@
294294
},
295295
"outputs": [],
296296
"source": [
297-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
298-
]
297+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ]
299298
},
300299
{
301300
"cell_type": "markdown",
@@ -700,8 +699,7 @@
700699
"! rm -f custom.tar custom.tar.gz\n",
701700
"! tar cvf custom.tar custom\n",
702701
"! gzip custom.tar\n",
703-
"! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz"
704-
]
702+
"! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ]
705703
},
706704
{
707705
"cell_type": "markdown",
@@ -1326,9 +1324,7 @@
13261324
},
13271325
"outputs": [],
13281326
"source": [
1329-
"! gsutil cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n",
1330-
"! gsutil cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n",
1331-
"\n",
1327+
"! gcloud storage cp tmp1.jpg $BUCKET_URI/tmp1.jpg\n", "! gcloud storage cp tmp2.jpg $BUCKET_URI/tmp2.jpg\n", "\n",
13321328
"test_item_1 = BUCKET_URI + \"/tmp1.jpg\"\n",
13331329
"test_item_2 = BUCKET_URI + \"/tmp2.jpg\""
13341330
]
@@ -1529,8 +1525,7 @@
15291525
" print(e)\n",
15301526
"\n",
15311527
"if delete_bucket:\n",
1532-
" ! gsutil rm -r $BUCKET_URI\n",
1533-
"\n",
1528+
" ! gcloud storage rm --recursive $BUCKET_URI\n", "\n",
15341529
"if delete_application_directory:\n",
15351530
" ! rm -rf custom custom.tar.gz temp1.jpg temp2.jpg"
15361531
]

‎notebooks/official/model_monitoring/get_started_with_model_monitoring_custom_tf_serving.ipynb‎

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -363,8 +363,7 @@
363363
},
364364
"outputs": [],
365365
"source": [
366-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
367-
]
366+
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI" ]
368367
},
369368
{
370369
"cell_type": "markdown",
@@ -747,8 +746,7 @@
747746
"MODEL_ARTIFACT_URI = \"gs://mco-mm/churn\"\n",
748747
"MODEL_DIR = BUCKET_URI + \"/model/1\"\n",
749748
"\n",
750-
"! gsutil cp -r $MODEL_ARTIFACT_URI $MODEL_DIR"
751-
]
749+
"! gcloud storage cp --recursive $MODEL_ARTIFACT_URI $MODEL_DIR" ]
752750
},
753751
{
754752
"cell_type": "markdown",
@@ -1132,8 +1130,7 @@
11321130
"with open(\"schema.yaml\", \"w\") as f:\n",
11331131
" f.write(yaml)\n",
11341132
"\n",
1135-
"! gsutil cp schema.yaml {BUCKET_URI}/schema.yaml"
1136-
]
1133+
"! gcloud storage cp schema.yaml {BUCKET_URI}/schema.yaml" ]
11371134
},
11381135
{
11391136
"cell_type": "markdown",
@@ -1620,8 +1617,7 @@
16201617
"delete_bucket = False\n",
16211618
"\n",
16221619
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1623-
" ! gsutil rm -rf {BUCKET_URI}\n",
1624-
"\n",
1620+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}\n", "\n",
16251621
"! rm -f schema.yaml\n",
16261622
"\n",
16271623
"! bq rm -f {PROJECT_ID}.model_deployment_monitoring_{ENDPOINT_ID}\n",

0 commit comments

Comments
 (0)