Skip to content

Commit f5d341a

Browse files
Migrate gsutil usage to gcloud storage (#4303)
* Migrate gsutil usage to gcloud storage * changes for 4303 * Apply automated linter fixes --------- Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 70770a5 commit f5d341a

10 files changed

+30
-30
lines changed

notebooks/community/model_garden/model_garden_openai_api_llama3_2.ipynb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -251,7 +251,7 @@
251251
},
252252
"outputs": [],
253253
"source": [
254-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
254+
"! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"
255255
]
256256
},
257257
{
@@ -1020,7 +1020,7 @@
10201020
"delete_bucket = False # @param {type:\"boolean\"}\n",
10211021
"\n",
10221022
"if delete_bucket:\n",
1023-
" ! gsutil -m rm -r $BUCKET_NAME"
1023+
" ! gcloud storage rm --recursive $BUCKET_NAME"
10241024
]
10251025
}
10261026
],

notebooks/community/model_garden/model_garden_pytorch_llama2_quantization.ipynb

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -120,11 +120,11 @@
120120
"\n",
121121
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
122122
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}\"\n",
123-
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
123+
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
124124
"else:\n",
125125
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
126126
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
127-
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
127+
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
128128
" bucket_region = shell_output[0].strip().lower()\n",
129129
" if bucket_region != REGION:\n",
130130
" raise ValueError(\n",
@@ -149,7 +149,7 @@
149149
"\n",
150150
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
151151
"BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
152-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
152+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
153153
"\n",
154154
"! gcloud config set project $PROJECT_ID\n",
155155
"\n",
@@ -174,7 +174,7 @@
174174
" MODEL_BUCKET,\n",
175175
")\n",
176176
"\n",
177-
"! gsutil -m cp -R $VERTEX_AI_MODEL_GARDEN_LLAMA2/* $MODEL_BUCKET\n",
177+
"! gcloud storage cp --recursive $VERTEX_AI_MODEL_GARDEN_LLAMA2/* $MODEL_BUCKET\n",
178178
"base_model_path_prefix = MODEL_BUCKET\n",
179179
"\n",
180180
"# The pre-built serving docker images.\n",
@@ -627,7 +627,7 @@
627627
"# Delete Cloud Storage objects that were created\n",
628628
"delete_bucket = False # @param {type: \"boolean\", isTemplate:true}\n",
629629
"if delete_bucket:\n",
630-
" ! gsutil -m rm -r $BUCKET_URI"
630+
" ! gcloud storage rm --recursive $BUCKET_URI"
631631
]
632632
}
633633
],

notebooks/community/pipelines/google_cloud_pipeline_components_cloud_natural_language_pipeline.ipynb

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -479,7 +479,7 @@
479479
},
480480
"outputs": [],
481481
"source": [
482-
"!gsutil mb -l $REGION $BUCKET_URI"
482+
"!gcloud storage buckets create --location $REGION $BUCKET_URI"
483483
]
484484
},
485485
{
@@ -499,7 +499,7 @@
499499
},
500500
"outputs": [],
501501
"source": [
502-
"!gsutil ls -al $BUCKET_URI"
502+
"!gcloud storage ls --all-versions --long $BUCKET_URI"
503503
]
504504
},
505505
{
@@ -809,7 +809,7 @@
809809
" + \"/executor_output.json\"\n",
810810
" )\n",
811811
" if tf.io.gfile.exists(EVAL_METRICS):\n",
812-
" ! gsutil cat $EVAL_METRICS"
812+
" ! gcloud storage cat $EVAL_METRICS"
813813
]
814814
},
815815
{
@@ -832,7 +832,7 @@
832832
"outputs": [],
833833
"source": [
834834
"# Delete GCS bucket.\n",
835-
"!gsutil -m rm -r {BUCKET_URI}"
835+
"!gcloud storage rm --recursive {BUCKET_URI}"
836836
]
837837
},
838838
{

notebooks/community/sdk/SDK_BigQuery_Custom_Container_Training.ipynb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -472,7 +472,7 @@
472472
},
473473
"outputs": [],
474474
"source": [
475-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
475+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
476476
]
477477
},
478478
{
@@ -492,7 +492,7 @@
492492
},
493493
"outputs": [],
494494
"source": [
495-
"! gsutil ls -al $BUCKET_URI"
495+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
496496
]
497497
},
498498
{
@@ -957,7 +957,7 @@
957957
"# Delete bucket\n",
958958
"delete_bucket = False\n",
959959
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
960-
" ! gsutil rm -rf {BUCKET_URI}"
960+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
961961
]
962962
}
963963
],

notebooks/community/vertex_endpoints/torchserve/dreambooth_stablediffusion.ipynb

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -450,7 +450,7 @@
450450
},
451451
"outputs": [],
452452
"source": [
453-
"!gsutil -m cp gs://cloud-samples-data/vertex-ai/model-deployment/models/stable-diffusion/model_artifacts.zip \\\n",
453+
"!gcloud storage cp gs://cloud-samples-data/vertex-ai/model-deployment/models/stable-diffusion/model_artifacts.zip \\\n",
454454
" ."
455455
]
456456
},
@@ -750,7 +750,7 @@
750750
},
751751
"outputs": [],
752752
"source": [
753-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
753+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
754754
]
755755
},
756756
{
@@ -770,7 +770,7 @@
770770
},
771771
"outputs": [],
772772
"source": [
773-
"!gsutil cp -r model_artifacts $BUCKET_URI"
773+
"!gcloud storage cp --recursive model_artifacts $BUCKET_URI"
774774
]
775775
},
776776
{
@@ -968,7 +968,7 @@
968968
"# Delete Cloud Storage objects that were created\n",
969969
"delete_bucket = False\n",
970970
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
971-
" ! gsutil -m rm -r $BUCKET_URI"
971+
" ! gcloud storage rm --recursive $BUCKET_URI"
972972
]
973973
}
974974
],

notebooks/official/automl/sdk_automl_tabular_forecasting_batch.ipynb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@
286286
},
287287
"outputs": [],
288288
"source": [
289-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
289+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
290290
]
291291
},
292292
{
@@ -847,7 +847,7 @@
847847
"delete_bucket = False # set True for deletion\n",
848848
"\n",
849849
"if delete_bucket:\n",
850-
" ! gsutil rm -r $BUCKET_URI"
850+
" ! gcloud storage rm --recursive $BUCKET_URI"
851851
]
852852
}
853853
],

notebooks/official/custom/sdk-custom-image-classification-online.ipynb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@
250250
},
251251
"outputs": [],
252252
"source": [
253-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
253+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
254254
]
255255
},
256256
{
@@ -654,7 +654,7 @@
654654
"outputs": [],
655655
"source": [
656656
"# Download the images\n",
657-
"! gsutil -m cp -r gs://cloud-samples-data/ai-platform-unified/cifar_test_images ."
657+
"! gcloud storage cp --recursive gs://cloud-samples-data/ai-platform-unified/cifar_test_images ."
658658
]
659659
},
660660
{
@@ -804,7 +804,7 @@
804804
"endpoint.delete()\n",
805805
"\n",
806806
"if delete_bucket:\n",
807-
" ! gsutil -m rm -r $BUCKET_URI"
807+
" ! gcloud storage rm --recursive $BUCKET_URI"
808808
]
809809
}
810810
],

notebooks/official/explainable_ai/xai_image_classification_feature_attributions.ipynb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -287,7 +287,7 @@
287287
},
288288
"outputs": [],
289289
"source": [
290-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
290+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
291291
]
292292
},
293293
{
@@ -895,7 +895,7 @@
895895
"# Delete Cloud Storage bucket\n",
896896
"delete_bucket = False # Set True to delete your bucket\n",
897897
"if delete_bucket:\n",
898-
" ! gsutil -m rm -r $BUCKET_URI"
898+
" ! gcloud storage rm --recursive $BUCKET_URI"
899899
]
900900
}
901901
],

notebooks/official/prediction/sdk_private_endpoint.ipynb

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,7 @@
340340
},
341341
"outputs": [],
342342
"source": [
343-
"! gsutil mb -l {REGION} {BUCKET_URI}"
343+
"! gcloud storage buckets create --location={REGION} {BUCKET_URI}"
344344
]
345345
},
346346
{
@@ -479,7 +479,7 @@
479479
"IMPORT_FILE = \"petfinder-tabular-classification-tabnet-with-header.csv\"\n",
480480
"TRAINING_DATA_PATH = f\"{BUCKET_URI}/data/petfinder/train.csv\"\n",
481481
"\n",
482-
"! gsutil cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}"
482+
"! gcloud storage cp gs://cloud-samples-data/ai-platform-unified/datasets/tabular/{IMPORT_FILE} {TRAINING_DATA_PATH}"
483483
]
484484
},
485485
{
@@ -955,7 +955,7 @@
955955
" print(e)\n",
956956
"\n",
957957
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
958-
" ! gsutil rm -r $BUCKET_URI"
958+
" ! gcloud storage rm --recursive $BUCKET_URI"
959959
]
960960
}
961961
],

notebooks/official/tensorboard/tensorboard_profiler_custom_training.ipynb

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@
347347
},
348348
"outputs": [],
349349
"source": [
350-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
350+
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
351351
]
352352
},
353353
{
@@ -899,7 +899,7 @@
899899
"# delete cloud storage bucket\n",
900900
"delete_bucket = False # set True for deletion\n",
901901
"if delete_bucket:\n",
902-
" ! gsutil -m rm -r $BUCKET_URI"
902+
" ! gcloud storage rm --recursive $BUCKET_URI"
903903
]
904904
}
905905
],

0 commit comments

Comments
 (0)