Skip to content

Commit 23af537

Browse files
googlyrahmangurusai-voleti
googlyrahman
and
gurusai-voleti
authored
Migrate gsutil usage to gcloud storage (#4295)
* Migrate gsutil usage to gcloud storage * removed notes * Apply automated linter fixes * Revert "Apply automated linter fixes" This reverts commit 3f6c4c0. * linter changes * update * update * update --------- Co-authored-by: gurusai-voleti <gvoleti@google.com>
1 parent 7577c0b commit 23af537

File tree

10 files changed

+32
-32
lines changed

10 files changed

+32
-32
lines changed

‎community-content/vertex_model_garden/model_oss/notebook_util/dataset_validation_util.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,10 @@ def download_gcs_uri_to_local(
102102
if not os.path.exists(destination_dir):
103103
os.mkdir(destination_dir)
104104
subprocess.check_output([
105-
"gsutil",
106-
"-m",
105+
"gcloud",
106+
"storage",
107107
"cp",
108-
"-r",
108+
"--recursive",
109109
gcs_uri,
110110
destination_dir,
111111
])

‎notebooks/community/exploratory_data_analysis/eda_with_r_and_bigquery.ipynb‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -637,20 +637,20 @@
637637
},
638638
"outputs": [],
639639
"source": [
640-
"# Upload CSV data to Cloud Storage by passing gsutil commands to system\n",
640+
"# Upload CSV data to Cloud Storage by passing gcloud storage commands to system\n",
641641
"gcs_url <- paste0(\"gs://\", BUCKET_NAME, \"/\")\n",
642642
"\n",
643-
"command <- paste(\"gsutil mb\", gcs_url)\n",
643+
"command <- paste(\"gcloud storage buckets create\", gcs_url)\n",
644644
"\n",
645645
"system(command)\n",
646646
"\n",
647647
"gcs_data_dir <- paste0(\"gs://\", BUCKET_NAME, \"/data\")\n",
648648
"\n",
649-
"command <- paste(\"gsutil cp data/*_data.csv\", gcs_data_dir)\n",
649+
"command <- paste(\"gcloud storage cp data/*_data.csv\", gcs_data_dir)\n",
650650
"\n",
651651
"system(command)\n",
652652
"\n",
653-
"command <- paste(\"gsutil ls -l\", gcs_data_dir)\n",
653+
"command <- paste(\"gcloud storage ls --long\", gcs_data_dir)\n",
654654
"\n",
655655
"system(command, intern = TRUE)"
656656
]
@@ -679,4 +679,4 @@
679679
},
680680
"nbformat": 4,
681681
"nbformat_minor": 4
682-
}
682+
}

‎notebooks/community/model_garden/docker_source_codes/model_oss/notebook_util/dataset_validation_util.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -103,10 +103,10 @@ def download_gcs_uri_to_local(
103103
if not os.path.exists(destination_dir):
104104
os.mkdir(destination_dir)
105105
subprocess.check_output([
106-
"gsutil",
107-
"-m",
106+
"gcloud",
107+
"storage",
108108
"cp",
109-
"-r",
109+
"--recursive",
110110
gcs_uri,
111111
destination_dir,
112112
])

‎notebooks/community/model_garden/model_garden_pytorch_falcon_evaluation.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -125,9 +125,9 @@
125125
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
126126
" # Create a unique GCS bucket for this notebook, if not specified by the user\n",
127127
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}\"\n",
128-
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
128+
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
129129
"else:\n",
130-
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
130+
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
131131
" bucket_region = shell_output[0].strip().lower()\n",
132132
" if bucket_region != REGION:\n",
133133
" raise ValueError(\n",
@@ -154,7 +154,7 @@
154154
"\n",
155155
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
156156
"BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
157-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
157+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
158158
"\n",
159159
"! gcloud config set project $PROJECT_ID\n",
160160
"\n",
@@ -353,7 +353,7 @@
353353
"\n",
354354
"delete_bucket = False # @param {type:\"boolean\"}\n",
355355
"if delete_bucket:\n",
356-
" ! gsutil -m rm -r $BUCKET_URI\n",
356+
" ! gcloud storage rm --recursive $BUCKET_URI\n",
357357
"\n",
358358
"eval_job.delete()"
359359
]

‎notebooks/community/model_garden/model_garden_pytorch_prompt_guard_deployment.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@
408408
"\n",
409409
"delete_bucket = False # @param {type:\"boolean\"}\n",
410410
"if delete_bucket:\n",
411-
" ! gsutil -m rm -r $BUCKET_NAME"
411+
" ! gcloud storage rm --recursive $BUCKET_NAME"
412412
]
413413
}
414414
],

‎notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_xl_1_0.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -142,10 +142,10 @@
142142
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
143143
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
144144
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
145-
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
145+
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
146146
"else:\n",
147147
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
148-
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
148+
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n",
149149
" bucket_region = shell_output[0].strip().lower()\n",
150150
" if bucket_region != REGION:\n",
151151
" raise ValueError(\n",
@@ -170,7 +170,7 @@
170170
"\n",
171171
"\n",
172172
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
173-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
173+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
174174
"\n",
175175
"! gcloud config set project $PROJECT_ID\n",
176176
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
@@ -474,7 +474,7 @@
474474
"\n",
475475
"delete_bucket = False # @param {type:\"boolean\"}\n",
476476
"if delete_bucket:\n",
477-
" ! gsutil -m rm -r $BUCKET_NAME"
477+
" ! gcloud storage rm --recursive $BUCKET_NAME"
478478
]
479479
}
480480
],

‎notebooks/community/model_garden/model_garden_pytorch_stable_diffusion_xl_lora.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -148,10 +148,10 @@
148148
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
149149
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
150150
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
151-
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
151+
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n",
152152
"else:\n",
153153
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
154-
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
154+
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
155155
" bucket_region = shell_output[0].strip().lower()\n",
156156
" if bucket_region != REGION:\n",
157157
" raise ValueError(\n",
@@ -170,7 +170,7 @@
170170
"print(\"Using this default Service Account:\", SERVICE_ACCOUNT)\n",
171171
"\n",
172172
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
173-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
173+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n",
174174
"\n",
175175
"aiplatform.init(project=PROJECT_ID, location=REGION, staging_bucket=BUCKET_URI)\n",
176176
"\n",
@@ -417,7 +417,7 @@
417417
"# Delete bucket.\n",
418418
"delete_bucket = False # @param {type:\"boolean\"}\n",
419419
"if delete_bucket:\n",
420-
" ! gsutil -m rm -r $BUCKET_NAME"
420+
" ! gcloud storage rm --recursive $BUCKET_NAME"
421421
]
422422
}
423423
],

‎notebooks/community/sdk/sdk_automl_forecasting_evaluating_a_model.ipynb‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -471,7 +471,7 @@
471471
"model.delete(sync=True)\n",
472472
"\n",
473473
"# Delete Cloud Storage objects that were created\n",
474-
"! gsutil -m rm -r $BUCKET_NAME"
474+
"! gcloud storage rm --recursive $BUCKET_NAME"
475475
]
476476
}
477477
],

‎notebooks/official/pipelines/control_flow_kfp.ipynb‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,7 @@
271271
},
272272
"outputs": [],
273273
"source": [
274-
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
274+
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
275275
]
276276
},
277277
{
@@ -344,9 +344,9 @@
344344
},
345345
"outputs": [],
346346
"source": [
347-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
347+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.objectCreator\"\n",
348348
"\n",
349-
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
349+
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
350350
]
351351
},
352352
{
@@ -649,7 +649,7 @@
649649
"! rm control_pipeline.yaml\n",
650650
"\n",
651651
"if delete_bucket:\n",
652-
" ! gsutil rm -r $BUCKET_URI"
652+
" ! gcloud storage rm --recursive $BUCKET_URI"
653653
]
654654
}
655655
],

‎notebooks/official/tabnet/ai-explanations-tabnet-algorithm.ipynb‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@
339339
},
340340
"outputs": [],
341341
"source": [
342-
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
342+
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
343343
]
344344
},
345345
{
@@ -394,7 +394,7 @@
394394
},
395395
"outputs": [],
396396
"source": [
397-
"!gsutil cp gs://cloud-samples-data/ai-platform-unified/datasets/tabnet/tab_net_output/syn2 $BUCKET_URI\n",
397+
"!gcloud storage cp gs://cloud-samples-data/ai-platform-unified/datasets/tabnet/tab_net_output/syn2 $BUCKET_URI\n",
398398
"\n",
399399
"# Replace your the BUCKET_URI and PREDICTION_FILE\n",
400400
"# BUCKET_NAME = \"[<your-bucket-name>]\"\n",
@@ -511,7 +511,7 @@
511511
"outputs": [],
512512
"source": [
513513
"# Delete Cloud Storage that were created\n",
514-
"! gsutil -m rm -r $BUCKET_URI"
514+
"! gcloud storage rm --recursive $BUCKET_URI"
515515
]
516516
},
517517
{

0 commit comments

Comments
 (0)