Skip to content

Commit 5fc93c8

Browse files
googlyrahmanbhandarivijay-pnggurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4328)
* Migrate gsutil usage to gcloud storage * PR changes for 4328 * Fix Linter issue for 4328 * changes removed model garden * Update model_garden_jax_fvlm.ipynb * Update model_garden_jax_fvlm.ipynb --------- Co-authored-by: bhandarivijay <bhandarivijay@google.com> Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 3cf4622 commit 5fc93c8

9 files changed

+51
-59
lines changed

‎notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@
416416
},
417417
"outputs": [],
418418
"source": [
419-
"! gsutil mb -l $REGION $BUCKET_NAME"
419+
"! gcloud storage buckets create --location=$REGION $BUCKET_NAME"
420420
]
421421
},
422422
{
@@ -436,7 +436,7 @@
436436
},
437437
"outputs": [],
438438
"source": [
439-
"! gsutil ls -al $BUCKET_NAME"
439+
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
440440
]
441441
},
442442
{
@@ -474,7 +474,6 @@
474474
"\n",
475475
"from google.cloud.aiplatform import gapic as aip\n",
476476
"from google.protobuf import json_format\n",
477-
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
478477
"from google.protobuf.struct_pb2 import Struct, Value"
479478
]
480479
},
@@ -784,11 +783,11 @@
784783
"else:\n",
785784
" FILE = IMPORT_FILE\n",
786785
"\n",
787-
"count = ! gsutil cat $FILE | wc -l\n",
786+
"count = ! gcloud storage cat $FILE | wc -l\n",
788787
"print(\"Number of Examples\", int(count[0]))\n",
789788
"\n",
790789
"print(\"First 10 rows\")\n",
791-
"! gsutil cat $FILE | head"
790+
"! gcloud storage cat $FILE | head"
792791
]
793792
},
794793
{
@@ -1215,7 +1214,6 @@
12151214
" }\n",
12161215
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
12171216
" print(\"Long running operation:\", response.operation.name)\n",
1218-
" result = response.result(timeout=1800)\n",
12191217
" metadata = response.operation.metadata\n",
12201218
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
12211219
" print(\"Artifact Uri\", artifact_uri)\n",
@@ -1244,9 +1242,9 @@
12441242
},
12451243
"outputs": [],
12461244
"source": [
1247-
"! gsutil ls $model_package\n",
1245+
"! gcloud storage ls $model_package\n",
12481246
"# Download the model artifacts\n",
1249-
"! gsutil cp -r $model_package tflite\n",
1247+
"! gcloud storage cp --recursive $model_package tflite\n",
12501248
"\n",
12511249
"tflite_path = \"tflite/model.tflite\""
12521250
]
@@ -1305,7 +1303,7 @@
13051303
},
13061304
"outputs": [],
13071305
"source": [
1308-
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
1306+
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
13091307
"test_item = test_items[0].split(\",\")[0]\n",
13101308
"\n",
13111309
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1449,7 +1447,7 @@
14491447
" print(e)\n",
14501448
"\n",
14511449
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1452-
" ! gsutil rm -r $BUCKET_NAME"
1450+
" ! gcloud storage rm --recursive $BUCKET_NAME"
14531451
]
14541452
}
14551453
],

‎notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -416,7 +416,7 @@
416416
},
417417
"outputs": [],
418418
"source": [
419-
"! gsutil mb -l $REGION $BUCKET_NAME"
419+
"! gcloud storage buckets create --location $REGION $BUCKET_NAME"
420420
]
421421
},
422422
{
@@ -436,7 +436,7 @@
436436
},
437437
"outputs": [],
438438
"source": [
439-
"! gsutil ls -al $BUCKET_NAME"
439+
"! gcloud storage ls --all-versions --long $BUCKET_NAME"
440440
]
441441
},
442442
{
@@ -473,9 +473,7 @@
473473
"import time\n",
474474
"\n",
475475
"from google.cloud.aiplatform import gapic as aip\n",
476-
"from google.protobuf import json_format\n",
477-
"from google.protobuf.json_format import MessageToJson, ParseDict\n",
478-
"from google.protobuf.struct_pb2 import Struct, Value"
476+
"from google.protobuf import json_format\n"
479477
]
480478
},
481479
{
@@ -785,11 +783,11 @@
785783
"else:\n",
786784
" FILE = IMPORT_FILE\n",
787785
"\n",
788-
"count = ! gsutil cat $FILE | wc -l\n",
786+
"count = ! gcloud storage cat $FILE | wc -l\n",
789787
"print(\"Number of Examples\", int(count[0]))\n",
790788
"\n",
791789
"print(\"First 10 rows\")\n",
792-
"! gsutil cat $FILE | head"
790+
"! gcloud storage cat $FILE | head"
793791
]
794792
},
795793
{
@@ -1218,7 +1216,6 @@
12181216
" }\n",
12191217
" response = clients[\"model\"].export_model(name=name, output_config=output_config)\n",
12201218
" print(\"Long running operation:\", response.operation.name)\n",
1221-
" result = response.result(timeout=1800)\n",
12221219
" metadata = response.operation.metadata\n",
12231220
" artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n",
12241221
" print(\"Artifact Uri\", artifact_uri)\n",
@@ -1247,9 +1244,9 @@
12471244
},
12481245
"outputs": [],
12491246
"source": [
1250-
"! gsutil ls $model_package\n",
1247+
"! gcloud storage ls $model_package\n",
12511248
"# Download the model artifacts\n",
1252-
"! gsutil cp -r $model_package tflite\n",
1249+
"! gcloud storage cp --recursive $model_package tflite\n",
12531250
"\n",
12541251
"tflite_path = \"tflite/model.tflite\""
12551252
]
@@ -1308,7 +1305,7 @@
13081305
},
13091306
"outputs": [],
13101307
"source": [
1311-
"test_items = ! gsutil cat $IMPORT_FILE | head -n1\n",
1308+
"test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
13121309
"test_item = test_items[0].split(\",\")[0]\n",
13131310
"\n",
13141311
"with tf.io.gfile.GFile(test_item, \"rb\") as f:\n",
@@ -1452,7 +1449,7 @@
14521449
" print(e)\n",
14531450
"\n",
14541451
"if delete_bucket and \"BUCKET_NAME\" in globals():\n",
1455-
" ! gsutil rm -r $BUCKET_NAME"
1452+
" ! gcloud storage rm --recursive $BUCKET_NAME"
14561453
]
14571454
}
14581455
],

‎notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb‎

Lines changed: 9 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -430,7 +430,7 @@
430430
},
431431
"outputs": [],
432432
"source": [
433-
"! gsutil mb -l $REGION $BUCKET_URI"
433+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
434434
]
435435
},
436436
{
@@ -450,7 +450,7 @@
450450
},
451451
"outputs": [],
452452
"source": [
453-
"! gsutil ls -al $BUCKET_URI"
453+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
454454
]
455455
},
456456
{
@@ -520,9 +520,8 @@
520520
},
521521
"outputs": [],
522522
"source": [
523-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
524-
"\n",
525-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
523+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n","\n",
524+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
526525
]
527526
},
528527
{
@@ -807,13 +806,13 @@
807806
" + \"/evaluation_metrics\"\n",
808807
" )\n",
809808
" if tf.io.gfile.exists(EXECUTE_OUTPUT):\n",
810-
" ! gsutil cat $EXECUTE_OUTPUT\n",
809+
" ! gcloud storage cat $EXECUTE_OUTPUT\n",
811810
" return EXECUTE_OUTPUT\n",
812811
" elif tf.io.gfile.exists(GCP_RESOURCES):\n",
813-
" ! gsutil cat $GCP_RESOURCES\n",
812+
" ! gcloud storage cat $GCP_RESOURCES\n",
814813
" return GCP_RESOURCES\n",
815814
" elif tf.io.gfile.exists(EVAL_METRICS):\n",
816-
" ! gsutil cat $EVAL_METRICS\n",
815+
" ! gcloud storage cat $EVAL_METRICS\n",
817816
" return EVAL_METRICS\n",
818817
"\n",
819818
" return None\n",
@@ -1167,9 +1166,7 @@
11671166
"\n",
11681167
"\n",
11691168
"@component()\n",
1170-
"def multi_output(\n",
1171-
" text1: str, text2: str\n",
1172-
") -> NamedTuple(\n",
1169+
"def multi_output(text1: str, text2: str) -> NamedTuple(\n",
11731170
" \"Outputs\",\n",
11741171
" [\n",
11751172
" (\"output_1\", str), # Return parameters\n",
@@ -1587,7 +1584,7 @@
15871584
"delete_bucket = False\n",
15881585
"\n",
15891586
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1590-
" ! gsutil rm -r $BUCKET_URI"
1587+
" ! gcloud storage rm --recursive $BUCKET_URI"
15911588
]
15921589
}
15931590
],

‎notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -464,7 +464,7 @@
464464
},
465465
"outputs": [],
466466
"source": [
467-
"! gsutil mb -l $REGION $BUCKET_URI"
467+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
468468
]
469469
},
470470
{
@@ -484,7 +484,7 @@
484484
},
485485
"outputs": [],
486486
"source": [
487-
"! gsutil ls -al $BUCKET_URI"
487+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
488488
]
489489
},
490490
{
@@ -892,7 +892,7 @@
892892
" f.write(json.dumps(instance) + \"\\n\")\n",
893893
"\n",
894894
"# Upload to Cloud Storage bucket\n",
895-
"! gsutil cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n",
895+
"! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n",
896896
"\n",
897897
"print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)"
898898
]
@@ -1108,7 +1108,7 @@
11081108
"outputs": [],
11091109
"source": [
11101110
"data_file = \"gs://cloud-samples-data/ai-platform/iris/iris_data.csv\"\n",
1111-
"! gsutil cat $data_file | head -n 10 > test.csv\n",
1111+
"! gcloud storage cat $data_file | head -n 10 > test.csv\n",
11121112
"\n",
11131113
"! cat test.csv\n",
11141114
"\n",
@@ -1118,7 +1118,7 @@
11181118
" BUCKET_URI + \"/batch_prediction_instances/\" + BATCH_PREDICTION_INSTANCES_FILE\n",
11191119
")\n",
11201120
"\n",
1121-
"! gsutil cp test.csv $BATCH_PREDICTION_GCS_SOURCE"
1121+
"! gcloud storage cp test.csv $BATCH_PREDICTION_GCS_SOURCE"
11221122
]
11231123
},
11241124
{
@@ -1497,7 +1497,7 @@
14971497
"delete_bucket = False\n",
14981498
"\n",
14991499
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1500-
" ! gsutil rm -rf {BUCKET_URI}"
1500+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
15011501
]
15021502
}
15031503
],

‎notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -504,7 +504,7 @@
504504
},
505505
"outputs": [],
506506
"source": [
507-
"! gsutil mb -l $REGION $BUCKET_URI"
507+
"! gcloud storage buckets create --location=$REGION $BUCKET_URI"
508508
]
509509
},
510510
{
@@ -524,7 +524,7 @@
524524
},
525525
"outputs": [],
526526
"source": [
527-
"! gsutil ls -al $BUCKET_URI"
527+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
528528
]
529529
},
530530
{
@@ -1734,7 +1734,7 @@
17341734
"! rm -rf app\n",
17351735
"\n",
17361736
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1737-
" ! gsutil rm -rf {BUCKET_URI}"
1737+
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
17381738
]
17391739
}
17401740
],

‎notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -537,7 +537,7 @@
537537
},
538538
"outputs": [],
539539
"source": [
540-
"! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI"
540+
"! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI"
541541
]
542542
},
543543
{
@@ -557,7 +557,7 @@
557557
},
558558
"outputs": [],
559559
"source": [
560-
"! gsutil ls -al $BUCKET_URI"
560+
"! gcloud storage ls --all-versions --long $BUCKET_URI"
561561
]
562562
},
563563
{
@@ -627,9 +627,9 @@
627627
},
628628
"outputs": [],
629629
"source": [
630-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
630+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
631631
"\n",
632-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
632+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
633633
]
634634
},
635635
{
@@ -1383,7 +1383,7 @@
13831383
"outputs": [],
13841384
"source": [
13851385
"# Load the results\n",
1386-
"attributions = !gsutil cat $feat_attrs_gcs_uri\n",
1386+
"attributions = !gcloud storage cat $feat_attrs_gcs_uri\n",
13871387
"\n",
13881388
"# Print the results obtained\n",
13891389
"attributions = json.loads(attributions[0])\n",
@@ -1463,7 +1463,7 @@
14631463
"# Delete Cloud Storage objects\n",
14641464
"delete_bucket = False\n",
14651465
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
1466-
" ! gsutil -m rm -r $BUCKET_URI"
1466+
" ! gcloud storage rm --recursive $BUCKET_URI"
14671467
]
14681468
}
14691469
],

‎notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -288,7 +288,7 @@
288288
},
289289
"outputs": [],
290290
"source": [
291-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
291+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
292292
]
293293
},
294294
{
@@ -707,7 +707,7 @@
707707
"! rm -f custom.tar custom.tar.gz\n",
708708
"! tar cvf custom.tar custom\n",
709709
"! gzip custom.tar\n",
710-
"! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
710+
"! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz"
711711
]
712712
},
713713
{
@@ -1260,7 +1260,7 @@
12601260
"\n",
12611261
"gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
12621262
"\n",
1263-
"! gsutil cp batch.csv $gcs_input_uri"
1263+
"! gcloud storage cp batch.csv $gcs_input_uri"
12641264
]
12651265
},
12661266
{
@@ -1398,7 +1398,7 @@
13981398
"# Delete the Cloud Storage bucket\n",
13991399
"delete_bucket = False # Set True for deletion\n",
14001400
"if delete_bucket:\n",
1401-
" ! gsutil rm -r $BUCKET_URI\n",
1401+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
14021402
"\n",
14031403
"# Remove the locally generated files\n",
14041404
"! rm -rf batch.csv custom.tar.gz custom"

0 commit comments

Comments
 (0)