Skip to content

Commit f2decbe

Browse files
gurusai-voletigurusai-voleti
authored
Merge pull request #96 from bhandarivijay-png/ai-gsutil-migration-b2f2daf91b2145a49af1eebaed6d1fc9
changes removed model garden PR# 4329
2 parents 682761f + 41dad18 commit f2decbe

File tree

3 files changed

+30
-16
lines changed

3 files changed

+30
-16
lines changed

‎notebooks/community/model_garden/model_garden_gemma2_finetuning_on_vertex.ipynb‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -193,9 +193,11 @@
193193
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
194194
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
195195
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
196-
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n",
196+
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
197+
"else:\n",
197198
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
198-
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n",
199+
" shell_output = ! gsutil ls -L {BUCKET_URI} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
200+
" bucket_region = shell_output[0].strip().lower()\n",
199201
" if bucket_region != REGION:\n",
200202
" raise ValueError(\n",
201203
" \"Bucket region %s is different from notebook region %s\"\n",
@@ -219,7 +221,8 @@
219221
"\n",
220222
"\n",
221223
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
222-
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n",
224+
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
225+
"\n",
223226
"! gcloud config set project $PROJECT_ID\n",
224227
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
225228
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n",
@@ -355,7 +358,9 @@
355358
"if dataset_validation_util.is_gcs_path(pretrained_model_id):\n",
356359
" # Download tokenizer.\n",
357360
" ! mkdir tokenizer\n",
358-
" ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n",
361+
" ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n",
362+
" ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n",
363+
" tokenizer_path = \"./tokenizer\"\n",
359364
" access_token = \"\"\n",
360365
"else:\n",
361366
" tokenizer_path = pretrained_model_id\n",
@@ -1059,7 +1064,8 @@
10591064
"\n",
10601065
"delete_bucket = False # @param {type:\"boolean\"}\n",
10611066
"if delete_bucket:\n",
1062-
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
1067+
" ! gsutil -m rm -r $BUCKET_NAME"
1068+
]
10631069
}
10641070
],
10651071
"metadata": {

‎notebooks/community/model_garden/model_garden_pytorch_detectron2.ipynb‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -201,9 +201,11 @@
201201
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
202202
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
203203
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
204-
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n",
204+
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
205+
"else:\n",
205206
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
206-
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n",
207+
" shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
208+
" bucket_region = shell_output[0].strip().lower()\n",
207209
" if bucket_region != REGION:\n",
208210
" raise ValueError(\n",
209211
" \"Bucket region %s is different from notebook region %s\"\n",
@@ -227,8 +229,8 @@
227229
"\n",
228230
"\n",
229231
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
230-
"# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n",
231-
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n",
232+
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n",
233+
"\n",
232234
"! gcloud config set project $PROJECT_ID\n",
233235
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
234236
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n",
@@ -1213,7 +1215,8 @@
12131215
"\n",
12141216
"delete_bucket = False # @param {type:\"boolean\"}\n",
12151217
"if delete_bucket:\n",
1216-
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
1218+
" ! gsutil -m rm -r $BUCKET_NAME"
1219+
]
12171220
}
12181221
],
12191222
"metadata": {

‎notebooks/community/model_garden/model_garden_pytorch_llama3_3_finetuning.ipynb‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@
124124
"\n",
125125
"# @markdown 3. For serving, **[click here](https://console.cloud.google.com/iam-admin/quotas?location=us-central1&metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_l4_gpus)** to check if your project already has the required 1 L4 GPU in the us-central1 region. If yes, then run this notebook in the us-central1 region. If you need more L4 GPUs for your project, then you can follow [these instructions](https://cloud.google.com/docs/quotas/view-manage#viewing_your_quota_console) to request more. Alternatively, if you want to run predictions with A100 80GB or H100 GPUs, we recommend using the regions listed below. **NOTE:** Make sure you have associated quota in selected regions. Click the links to see your current quota for each GPU type: [Nvidia A100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_a100_80gb_gpus), [Nvidia H100 80GB](https://console.cloud.google.com/iam-admin/quotas?metric=aiplatform.googleapis.com%2Fcustom_model_serving_nvidia_h100_gpus).\n",
126126
"\n",
127-
"# @markdown > | Machine Type | Accelerator Type | Recommended Regions |\n",
127+
"# @markdown > | Machin188e Type | Accelerator Type | Recommended Regions |\n",
128128
"# @markdown | ----------- | ----------- | ----------- |\n",
129129
"# @markdown | a2-ultragpu-1g | 1 NVIDIA_A100_80GB | us-central1, us-east4, europe-west4, asia-southeast1, us-east4 |\n",
130130
"# @markdown | a3-highgpu-2g | 2 NVIDIA_H100_80GB | us-west1, asia-southeast1, europe-west4 |\n",
@@ -185,9 +185,11 @@
185185
"if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n",
186186
" BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n",
187187
" BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n",
188-
" ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n",
188+
" ! gsutil mb -l {REGION} {BUCKET_URI}\n",
189+
"else:\n",
189190
" assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n",
190-
" shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n",
191+
" shell_output = ! gsutil ls -b {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n",
192+
" bucket_region = shell_output[0].strip().lower()\n",
191193
" if bucket_region != REGION:\n",
192194
" raise ValueError(\n",
193195
" \"Bucket region %s is different from notebook region %s\"\n",
@@ -211,7 +213,8 @@
211213
"\n",
212214
"\n",
213215
"# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n",
214-
"! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=\"serviceAccount:{SERVICE_ACCOUNT}\" --role=\"roles/storage.admin\"\n", "\n",
216+
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
217+
"\n",
215218
"! gcloud config set project $PROJECT_ID\n",
216219
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n",
217220
"! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\""
@@ -392,7 +395,9 @@
392395
"if dataset_validation_util.is_gcs_path(pretrained_model_id):\n",
393396
" # Download tokenizer.\n",
394397
" ! mkdir tokenizer\n",
395-
" ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n",
398+
" ! gsutil -m cp {pretrained_model_id}/tokenizer.json ./tokenizer\n",
399+
" ! gsutil -m cp {pretrained_model_id}/config.json ./tokenizer\n",
400+
" tokenizer_path = \"./tokenizer\"\n",
396401
" access_token = \"\"\n",
397402
"else:\n",
398403
" tokenizer_path = pretrained_model_id\n",
@@ -1079,7 +1084,7 @@
10791084
"\n",
10801085
"delete_bucket = False # @param {type:\"boolean\"}\n",
10811086
"if delete_bucket:\n",
1082-
" ! gcloud storage rm --recursive $BUCKET_NAME" ]
1087+
" ! gsutil rm -r $BUCKET_NAME" ]
10831088
}
10841089
],
10851090
"metadata": {

0 commit comments

Comments
 (0)