Skip to content

Commit ca7d4e1

Browse files
googlyrahmangurusai-voleti
googlyrahman
and
gurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4297)
* Migrate gsutil usage to gcloud storage * changes for 4297 * Apply automated linter fixes * removed changes from model_garden * removed changes from model_garden --------- Co-authored-by: gurusai-voleti <[email protected]>
1 parent 5b6c766 commit ca7d4e1

File tree

5 files changed

+23
-23
lines changed

5 files changed

+23
-23
lines changed

‎notebooks/official/automl/automl_image_classification_online_prediction.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,7 @@
280280
},
281281
"outputs": [],
282282
"source": [
283-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
283+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
284284
]
285285
},
286286
{
@@ -387,11 +387,11 @@
387387
"else:\n",
388388
" FILE = IMPORT_FILE\n",
389389
"\n",
390-
"count = ! gsutil cat $FILE | wc -l\n",
390+
"count = ! gcloud storage cat $FILE | wc -l\n",
391391
"print(\"Number of Examples\", int(count[0]))\n",
392392
"\n",
393393
"print(\"First 10 rows\")\n",
394-
"! gsutil cat $FILE | head"
394+
"! gcloud storage cat $FILE | head"
395395
]
396396
},
397397
{
@@ -597,7 +597,7 @@
597597
},
598598
"outputs": [],
599599
"source": [
600-
"test_item = !gsutil cat $IMPORT_FILE | head -n1\n",
600+
"test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n",
601601
"if len(str(test_item[0]).split(\",\")) == 3:\n",
602602
" _, test_item, test_label = str(test_item[0]).split(\",\")\n",
603603
"else:\n",
@@ -718,7 +718,7 @@
718718
"# Delete Cloud Storage objects that were created\n",
719719
"delete_bucket = False # Set True for deletion\n",
720720
"if delete_bucket:\n",
721-
" ! gsutil -m rm -r $BUCKET_URI"
721+
" ! gcloud storage rm --recursive $BUCKET_URI"
722722
]
723723
}
724724
],

‎notebooks/official/generative_ai/tune_peft.ipynb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@
286286
},
287287
"outputs": [],
288288
"source": [
289-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
289+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
290290
]
291291
},
292292
{
@@ -359,9 +359,9 @@
359359
},
360360
"outputs": [],
361361
"source": [
362-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
362+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
363363
"\n",
364-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
364+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer\n"
365365
]
366366
},
367367
{
@@ -464,8 +464,8 @@
464464
"outputs": [],
465465
"source": [
466466
"# Download dataset\n",
467-
"! gsutil cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n",
468-
"! gsutil cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl"
467+
"! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n",
468+
"! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl"
469469
]
470470
},
471471
{
@@ -662,7 +662,7 @@
662662
"# Delete bucket\n",
663663
"delete_bucket = True\n",
664664
"if delete_bucket:\n",
665-
" ! gsutil rm -rf {BUCKET_URI}"
665+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
666666
]
667667
}
668668
],

‎notebooks/official/pipelines/google_cloud_pipeline_components_automl_images.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,7 @@
330330
},
331331
"outputs": [],
332332
"source": [
333-
"! gsutil mb -l {LOCATION} {BUCKET_URI}"
333+
"! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}"
334334
]
335335
},
336336
{
@@ -403,9 +403,9 @@
403403
},
404404
"outputs": [],
405405
"source": [
406-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
406+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
407407
"\n",
408-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
408+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
409409
]
410410
},
411411
{
@@ -707,7 +707,7 @@
707707
"delete_bucket = False\n",
708708
"\n",
709709
"if delete_bucket:\n",
710-
" ! gsutil rm -r $BUCKET_URI"
710+
" ! gcloud storage rm --recursive $BUCKET_URI"
711711
]
712712
}
713713
],

‎notebooks/official/pipelines/google_cloud_pipeline_components_automl_tabular.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@
325325
},
326326
"outputs": [],
327327
"source": [
328-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
328+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
329329
]
330330
},
331331
{
@@ -399,9 +399,9 @@
399399
},
400400
"outputs": [],
401401
"source": [
402-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
402+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
403403
"\n",
404-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
404+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer\n"
405405
]
406406
},
407407
{
@@ -492,7 +492,7 @@
492492
"outputs": [],
493493
"source": [
494494
"TRAIN_FILE_NAME = \"california_housing_train.csv\"\n",
495-
"! gsutil cp gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/california_housing/california_housing_train.csv {PIPELINE_ROOT}/data/\n",
495+
"! gcloud storage cp gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/california_housing/california_housing_train.csv {PIPELINE_ROOT}/data/\n",
496496
"\n",
497497
"gcs_csv_path = f\"{PIPELINE_ROOT}/data/{TRAIN_FILE_NAME}\"\n",
498498
"\n",
@@ -682,7 +682,7 @@
682682
"\n",
683683
"\n",
684684
"if delete_bucket:\n",
685-
" ! gsutil rm -r $BUCKET_URI"
685+
" ! gcloud storage rm --recursive $BUCKET_URI"
686686
]
687687
}
688688
],

‎notebooks/official/prediction/get_started_with_raw_predict.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@
286286
},
287287
"outputs": [],
288288
"source": [
289-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
289+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
290290
]
291291
},
292292
{
@@ -439,7 +439,7 @@
439439
"source": [
440440
"MODEL_DIR = BUCKET_URI + \"/model\"\n",
441441
"\n",
442-
"! gsutil cp -r gs://cloud-samples-data/vertex-ai/google-cloud-aiplatform-ci-artifacts/models/penguins/estimator/ {MODEL_DIR}"
442+
"! gcloud storage cp --recursive gs://cloud-samples-data/vertex-ai/google-cloud-aiplatform-ci-artifacts/models/penguins/estimator/ {MODEL_DIR}"
443443
]
444444
},
445445
{
@@ -681,7 +681,7 @@
681681
" print(e)\n",
682682
"\n",
683683
"if delete_bucket:\n",
684-
" ! gsutil rm -rf {BUCKET_URI}"
684+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
685685
]
686686
}
687687
],

0 commit comments

Comments
 (0)