Skip to content

Commit 5b6c766

Browse files
googlyrahmangurusai-voleti
googlyrahman
and
gurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4298)
* Migrate gsutil usage to gcloud storage * update * linter changes for 4298 * Revert "linter changes for 4298" This reverts commit c7a00a9. * Linter fixes * update * update * Update distributed_hyperparameter_tuning.ipynb * removed changes in model_garden folder --------- Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 5efa512 commit 5b6c766

7 files changed

+32
-30
lines changed

‎notebooks/community/vision/video_warehouse_sdk.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -153,8 +153,8 @@
153153
},
154154
"outputs": [],
155155
"source": [
156-
"!gsutil cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n",
157-
"!pip install visionai-0.0.6-py3-none-any.whl --force-reinstall"
156+
"! gcloud storage cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n",
157+
"! pip install visionai-0.0.6-py3-none-any.whl --force-reinstall"
158158
]
159159
},
160160
{

‎notebooks/official/experiments/comparing_local_trained_models.ipynb‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@
286286
},
287287
"outputs": [],
288288
"source": [
289-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
289+
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
290290
]
291291
},
292292
{
@@ -770,7 +770,7 @@
770770
"delete_bucket = False # Set True for deletion\n",
771771
"\n",
772772
"if delete_bucket:\n",
773-
" ! gsutil rm -rf {BUCKET_URI}"
773+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
774774
]
775775
}
776776
],

‎notebooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -300,7 +300,7 @@
300300
},
301301
"outputs": [],
302302
"source": [
303-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
303+
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
304304
]
305305
},
306306
{
@@ -430,9 +430,9 @@
430430
},
431431
"outputs": [],
432432
"source": [
433-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
433+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
434434
"\n",
435-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
435+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
436436
]
437437
},
438438
{
@@ -482,7 +482,7 @@
482482
"SOURCE_DATA_URL = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/uci_glass_preprocessed/glass.csv\"\n",
483483
"DESTINATION_DATA_URL = f\"{BUCKET_URI}/data/glass.csv\"\n",
484484
"\n",
485-
"! gsutil cp $SOURCE_DATA_URL $DESTINATION_DATA_URL"
485+
"! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL"
486486
]
487487
},
488488
{
@@ -807,7 +807,7 @@
807807
"\n",
808808
"# Delete Cloud Storage objects that were created\n",
809809
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
810-
" ! gsutil -m rm -r $BUCKET_URI"
810+
" ! gcloud storage rm --recursive $BUCKET_URI"
811811
]
812812
}
813813
],

‎notebooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -292,7 +292,7 @@
292292
},
293293
"outputs": [],
294294
"source": [
295-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
295+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
296296
]
297297
},
298298
{
@@ -371,13 +371,13 @@
371371
},
372372
"outputs": [],
373373
"source": [
374-
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
374+
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
375375
"print(\"Number of Examples\", int(count[0]))\n",
376376
"\n",
377377
"print(\"First 10 rows\")\n",
378-
"! gsutil cat $IMPORT_FILE | head\n",
378+
"! gcloud storage cat $IMPORT_FILE | head\n",
379379
"\n",
380-
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
380+
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
381381
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
382382
"print(\"Label Column Name\", label_column)\n",
383383
"if label_column is None:\n",
@@ -580,14 +580,14 @@
580580
},
581581
"outputs": [],
582582
"source": [
583-
"! gsutil cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
584-
"! gsutil cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
583+
"! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
584+
"! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
585585
"\n",
586586
"! cut -d, -f1-16 tmp.csv > batch.csv\n",
587587
"\n",
588588
"gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
589589
"\n",
590-
"! gsutil cp batch.csv $gcs_input_uri"
590+
"! gcloud storage cp batch.csv $gcs_input_uri"
591591
]
592592
},
593593
{
@@ -729,7 +729,7 @@
729729
"delete_bucket = False\n",
730730
"\n",
731731
"if delete_bucket:\n",
732-
" ! gsutil rm -r $BUCKET_URI\n",
732+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
733733
"\n",
734734
"# Delete the locally generated files\n",
735735
"! rm batch.csv tmp.csv"

‎notebooks/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@
332332
},
333333
"outputs": [],
334334
"source": [
335-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
335+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
336336
]
337337
},
338338
{
@@ -405,9 +405,9 @@
405405
},
406406
"outputs": [],
407407
"source": [
408-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
408+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
409409
"\n",
410-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
410+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
411411
]
412412
},
413413
{
@@ -506,7 +506,9 @@
506506
},
507507
"outputs": [],
508508
"source": [
509-
"hp_dict: str = '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n",
509+
"hp_dict: str = (\n",
510+
" '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n",
511+
")\n",
510512
"data_dir: str = (\n",
511513
" \"gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/bikes_weather/\"\n",
512514
")\n",
@@ -732,7 +734,7 @@
732734
"source": [
733735
"delete_bucket = False\n",
734736
"if delete_bucket:\n",
735-
" ! gsutil rm -r $BUCKET_URI"
737+
" ! gcloud storage rm --recursive $BUCKET_URI"
736738
]
737739
}
738740
],

‎notebooks/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,7 @@
303303
},
304304
"outputs": [],
305305
"source": [
306-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
306+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
307307
]
308308
},
309309
{
@@ -545,9 +545,9 @@
545545
"source": [
546546
"MODEL_URI = f\"{BUCKET_URI}/{model_name}\"\n",
547547
"\n",
548-
"!gsutil rm -r $MODEL_URI\n",
549-
"!gsutil cp -r $model_path $MODEL_URI\n",
550-
"!gsutil ls -al $MODEL_URI"
548+
"!gcloud storage rm --recursive $MODEL_URI\n",
549+
"!gcloud storage cp --recursive $model_path $MODEL_URI\n",
550+
"!gcloud storage ls --all-versions --long $MODEL_URI"
551551
]
552552
},
553553
{
@@ -823,7 +823,7 @@
823823
"\n",
824824
"delete_bucket = False\n",
825825
"if delete_bucket:\n",
826-
" ! gsutil -m rm -r $BUCKET_URI"
826+
" ! gcloud storage rm --recursive $BUCKET_URI"
827827
]
828828
}
829829
],

‎notebooks/official/training/distributed_hyperparameter_tuning.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -274,7 +274,7 @@
274274
},
275275
"outputs": [],
276276
"source": [
277-
"! gsutil mb -l {LOCATION} {BUCKET_URI}"
277+
"! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}"
278278
]
279279
},
280280
{
@@ -692,15 +692,15 @@
692692
},
693693
"outputs": [],
694694
"source": [
695-
"# Set this to true only if you'd like to delete your bucket\n",
695+
"# Set this to true only if you'd like to delete your bucket.\n",
696696
"delete_bucket = False\n",
697697
"\n",
698698
"hp_job.delete()\n",
699699
"\n",
700700
"!gcloud container images delete $IMAGE_URI --force-delete-tags --quiet\n",
701701
"\n",
702702
"if delete_bucket:\n",
703-
" ! gsutil rm -r $BUCKET_URI"
703+
" ! gcloud storage rm --recursive $BUCKET_URI"
704704
]
705705
}
706706
],

0 commit comments

Comments
 (0)