Migrate gsutil usage to gcloud storage (#4301)

googlyrahman · gurusai-voleti · web-flow · commit f6c8bcf93785 · 2025-12-24T09:37:11.000-05:00
* Migrate gsutil usage to gcloud storage * changes for 4301 * linter changes * Revert "linter changes" This reverts commit 6665133. * Apply automated linter fixes * Update training-multi-class-classification-model-for-ads-targeting-usecase.ipynb * Update training-multi-class-classification-model-for-ads-targeting-usecase.ipynb * removed model_garden folder changes * Update model_garden_mediapipe_object_detection.ipynb --------- Co-authored-by: gurusai-voleti <gvoleti@google.com>
diff --git a/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb b/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb
@@ -761,7 +761,7 @@
         "\n",
         "BUCKET_ID = \"code-samples-\" + str(uuid.uuid1())\n",
         "\n",
-        "!gsutil mb gs://{BUCKET_ID}"
+        "!gcloud storage buckets create gs://{BUCKET_ID}"
       ]
     },
     {
@@ -874,7 +874,7 @@
       "outputs": [],
       "source": [
         "# # Delete the Google Cloud Storage bucket and files\n",
-        "# ! gsutil rm -r gs://{BUCKET_ID}\n",
+        "# ! gcloud storage rm --recursive gs://{BUCKET_ID}\n",
         "# print(f\"Deleted bucket '{BUCKET_ID}'.\")"
       ]
     }
diff --git a/notebooks/official/datasets/get_started_with_data_labeling.ipynb b/notebooks/official/datasets/get_started_with_data_labeling.ipynb
@@ -381,7 +381,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $REGION $BUCKET_URI"
+        "! gcloud storage buckets create --location=$REGION $BUCKET_URI"
       ]
     },
     {
@@ -584,7 +584,7 @@
         "print(IMPORT_FILE)\n",
         "\n",
         "# printing content of uploaded file\n",
-        "! gsutil cat $IMPORT_FILE"
+        "! gcloud storage cat $IMPORT_FILE"
       ]
     },
     {
@@ -707,7 +707,7 @@
       "outputs": [],
       "source": [
         "# create placeholder file for instructions for data labeling\n",
-        "! echo \"this is instruction\" >> instruction.txt | gsutil cp instruction.txt $BUCKET_URI"
+        "! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI"
       ]
     },
     {
@@ -864,7 +864,7 @@
         "\n",
         "# Delete the bucket created\n",
         "if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "    ! gcloud storage rm --recursive $BUCKET_URI"
       ]
     }
   ],
diff --git a/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb b/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb
@@ -291,7 +291,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
+        "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"
       ]
     },
     {
@@ -372,13 +372,13 @@
       },
       "outputs": [],
       "source": [
-        "count = ! gsutil cat $IMPORT_FILE | wc -l\n",
+        "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
         "print(\"Number of Examples\", int(count[0]))\n",
         "\n",
         "print(\"First 10 rows\")\n",
-        "! gsutil cat $IMPORT_FILE | head\n",
+        "! gcloud storage cat $IMPORT_FILE | head\n",
         "\n",
-        "heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
+        "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
         "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
         "print(\"Label Column Name\", label_column)\n",
         "if label_column is None:\n",
@@ -840,7 +840,7 @@
         "# Delete the Cloud Storage bucket\n",
         "delete_bucket = False  # set True for deletion\n",
         "if delete_bucket:\n",
-        "    ! gsutil rm -r $BUCKET_URI"
+        "    ! gcloud storage rm --recursive $BUCKET_URI"
       ]
     }
   ],
diff --git a/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb b/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb
@@ -217,7 +217,7 @@
       "source": [
         "# Create GCS Bucket\n",
         "BUCKET_URI = \"gs://your-bucket-name-unique\"  # @param {type:\"string\"}\n",
-        "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
+        "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
       ]
     },
     {
@@ -309,7 +309,7 @@
         "IMAGE_URI = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-12:latest\"\n",
         "\n",
         "# Copy Test Models to the Bucket\n",
-        "! gsutil -m cp -r \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n",
+        "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n",
         "\n",
         "model = aiplatform.Model.upload(\n",
         "    display_name=DISPLAY_NAME,\n",
@@ -786,7 +786,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil rm -r {BUCKET_URI}"
+        "! gcloud storage rm --recursive {BUCKET_URI}"
       ]
     },
     {
diff --git a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb
@@ -347,7 +347,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
+        "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"
       ]
     },
     {
@@ -720,9 +720,9 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
+        "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
         "\n",
-        "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
+        "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
       ]
     },
     {
@@ -840,7 +840,7 @@
         "# Delete GCS bucket.\n",
         "delete_bucket = False\n",
         "if delete_bucket:\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "    ! gcloud storage rm --recursive $BUCKET_URI"
       ]
     }
   ],
diff --git a/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb b/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb
@@ -350,7 +350,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
+        "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
       ]
     },
     {
@@ -846,7 +846,7 @@
         "# Delete GCS bucket.\n",
         "delete_bucket = False  # set True for deletion\n",
         "if delete_bucket:\n",
-        "    ! gsutil -m rm -r {BUCKET_URI}"
+        "    ! gcloud storage rm --recursive {BUCKET_URI}"
       ]
     }
   ],
diff --git a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb
@@ -282,7 +282,7 @@
       },
       "outputs": [],
       "source": [
-        "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
+        "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
       ]
     },
     {
@@ -840,7 +840,7 @@
         "\n",
         "delete_bucket = False\n",
         "if delete_bucket:\n",
-        "    ! gsutil -m rm -r $BUCKET_URI"
+        "    ! gcloud storage rm --recursive $BUCKET_URI"
       ]
     }
   ],

Original file line number	Diff line number	Diff line change
`@@ -761,7 +761,7 @@`
`761`	`761`	`"\n",`
`762`	`762`	`"BUCKET_ID = \"code-samples-\" + str(uuid.uuid1())\n",`
`763`	`763`	`"\n",`
`764`		`- "!gsutil mb gs://{BUCKET_ID}"`
	`764`	`+ "!gcloud storage buckets create gs://{BUCKET_ID}"`
`765`	`765`	`]`
`766`	`766`	`},`
`767`	`767`	`{`
`@@ -874,7 +874,7 @@`
`874`	`874`	`"outputs": [],`
`875`	`875`	`"source": [`
`876`	`876`	`"# # Delete the Google Cloud Storage bucket and files\n",`
`877`		`- "# ! gsutil rm -r gs://{BUCKET_ID}\n",`
	`877`	`+ "# ! gcloud storage rm --recursive gs://{BUCKET_ID}\n",`
`878`	`878`	`"# print(f\"Deleted bucket '{BUCKET_ID}'.\")"`
`879`	`879`	`]`
`880`	`880`	`}`
Original file line number	Diff line number	Diff line change
`@@ -381,7 +381,7 @@`
`381`	`381`	`},`
`382`	`382`	`"outputs": [],`
`383`	`383`	`"source": [`
`384`		`- "! gsutil mb -l $REGION $BUCKET_URI"`
	`384`	`+ "! gcloud storage buckets create --location=$REGION $BUCKET_URI"`
`385`	`385`	`]`
`386`	`386`	`},`
`387`	`387`	`{`
`@@ -584,7 +584,7 @@`
`584`	`584`	`"print(IMPORT_FILE)\n",`
`585`	`585`	`"\n",`
`586`	`586`	`"# printing content of uploaded file\n",`
`587`		`- "! gsutil cat $IMPORT_FILE"`
	`587`	`+ "! gcloud storage cat $IMPORT_FILE"`
`588`	`588`	`]`
`589`	`589`	`},`
`590`	`590`	`{`
`@@ -707,7 +707,7 @@`
`707`	`707`	`"outputs": [],`
`708`	`708`	`"source": [`
`709`	`709`	`"# create placeholder file for instructions for data labeling\n",`
`710`		`- "! echo \"this is instruction\" >> instruction.txt \| gsutil cp instruction.txt $BUCKET_URI"`
	`710`	`+ "! echo \"this is instruction\" >> instruction.txt \| gcloud storage cp instruction.txt $BUCKET_URI"`
`711`	`711`	`]`
`712`	`712`	`},`
`713`	`713`	`{`
`@@ -864,7 +864,7 @@`
`864`	`864`	`"\n",`
`865`	`865`	`"# Delete the bucket created\n",`
`866`	`866`	`"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",`
`867`		`- " ! gsutil rm -r $BUCKET_URI"`
	`867`	`+ " ! gcloud storage rm --recursive $BUCKET_URI"`
`868`	`868`	`]`
`869`	`869`	`}`
`870`	`870`	`],`
Original file line number	Diff line number	Diff line change
`@@ -347,7 +347,7 @@`
`347`	`347`	`},`
`348`	`348`	`"outputs": [],`
`349`	`349`	`"source": [`
`350`		`- "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"`
	`350`	`+ "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"`
`351`	`351`	`]`
`352`	`352`	`},`
`353`	`353`	`{`
`@@ -720,9 +720,9 @@`
`720`	`720`	`},`
`721`	`721`	`"outputs": [],`
`722`	`722`	`"source": [`
`723`		`- "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",`
	`723`	`+ "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",`
`724`	`724`	`"\n",`
`725`		`- "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"`
	`725`	`+ "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"`
`726`	`726`	`]`
`727`	`727`	`},`
`728`	`728`	`{`
`@@ -840,7 +840,7 @@`
`840`	`840`	`"# Delete GCS bucket.\n",`
`841`	`841`	`"delete_bucket = False\n",`
`842`	`842`	`"if delete_bucket:\n",`
`843`		`- " ! gsutil -m rm -r $BUCKET_URI"`
	`843`	`+ " ! gcloud storage rm --recursive $BUCKET_URI"`
`844`	`844`	`]`
`845`	`845`	`}`
`846`	`846`	`],`
Original file line number	Diff line number	Diff line change
`@@ -350,7 +350,7 @@`
`350`	`350`	`},`
`351`	`351`	`"outputs": [],`
`352`	`352`	`"source": [`
`353`		`- "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"`
	`353`	`+ "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"`
`354`	`354`	`]`
`355`	`355`	`},`
`356`	`356`	`{`
`@@ -846,7 +846,7 @@`
`846`	`846`	`"# Delete GCS bucket.\n",`
`847`	`847`	`"delete_bucket = False # set True for deletion\n",`
`848`	`848`	`"if delete_bucket:\n",`
`849`		`- " ! gsutil -m rm -r {BUCKET_URI}"`
	`849`	`+ " ! gcloud storage rm --recursive {BUCKET_URI}"`
`850`	`850`	`]`
`851`	`851`	`}`
`852`	`852`	`],`
Original file line number	Diff line number	Diff line change
`@@ -282,7 +282,7 @@`
`282`	`282`	`},`
`283`	`283`	`"outputs": [],`
`284`	`284`	`"source": [`
`285`		`- "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"`
	`285`	`+ "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"`
`286`	`286`	`]`
`287`	`287`	`},`
`288`	`288`	`{`
`@@ -840,7 +840,7 @@`
`840`	`840`	`"\n",`
`841`	`841`	`"delete_bucket = False\n",`
`842`	`842`	`"if delete_bucket:\n",`
`843`		`- " ! gsutil -m rm -r $BUCKET_URI"`
	`843`	`+ " ! gcloud storage rm --recursive $BUCKET_URI"`
`844`	`844`	`]`
`845`	`845`	`}`
`846`	`846`	`],`