Skip to content

Migrate gsutil usage to gcloud storage #4296

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gericdong merged 10 commits into from
Dec 19, 2025
Merged

Migrate gsutil usage to gcloud storage #4296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9f31198
Migrate gsutil usage to gcloud storage
googlyrahman Oct 15, 2025
9a33ec5
updates
gurusai-voleti Dec 10, 2025
cd8b895
update
gurusai-voleti Dec 10, 2025
18cb75e
Merge pull request #17 from gurusai-voleti/br_4296
googlyrahman Dec 11, 2025
01ccce7
Update llm_streaming_prediction.ipynb
gurusai-voleti Dec 18, 2025
539df5a
revert change
gurusai-voleti Dec 18, 2025
e9ce864
removed model_garden folder changes
gurusai-voleti Dec 19, 2025
c3327f1
Merge pull request #64 from gurusai-voleti/br_4296
gurusai-voleti Dec 19, 2025
ac9c8f3
removed model_garden folder changes
gurusai-voleti Dec 19, 2025
23398c3
Merge pull request #65 from gurusai-voleti/br_4296
gurusai-voleti Dec 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .cloud-build/execute_changed_notebooks_helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -365,7 +365,7 @@ def process_and_execute_notebook(
# Use gcloud to get tail
try:
result.error_message = subprocess.check_output(
["gsutil", "cat", "-r", "-1000", log_file_uri], encoding="UTF-8"
["gcloud", "storage", "cat", "--range", "-1000", log_file_uri], encoding="UTF-8"
)
except Exception as error:
result.error_message = str(error)
Expand Down
8 changes: 4 additions & 4 deletions notebooks/official/generative_ai/distillation.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -323,7 +323,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
Expand All @@ -347,8 +347,8 @@
},
"outputs": [],
"source": [
"! gsutil cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n",
"! gsutil cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl"
"! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_eval_sample.jsonl {BUCKET_URI}/peft_eval_sample.jsonl\n",
"! gcloud storage cp gs://cloud-samples-data/vertex-ai/model-evaluation/peft_train_sample.jsonl {BUCKET_URI}/peft_train_sample.jsonl"
]
},
{
Expand Down Expand Up @@ -554,7 +554,7 @@
"# Delete the Cloud Storage bucket\n",
"delete_bucket = True\n",
"if delete_bucket:\n",
" ! gsutil rm -rf {BUCKET_URI}"
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
]
}
],
Expand Down
6 changes: 3 additions & 3 deletions notebooks/official/ml_metadata/sdk-metric-parameter-tracking-for-custom-jobs.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
Expand Down Expand Up @@ -390,7 +390,7 @@
"outputs": [],
"source": [
"!wget https://storage.googleapis.com/download.tensorflow.org/data/abalone_train.csv\n",
"!gsutil cp abalone_train.csv {BUCKET_URI}/data/\n",
"!gcloud storage cp abalone_train.csv {BUCKET_URI}/data/\n",
"\n",
"gcs_csv_path = f\"{BUCKET_URI}/data/abalone_train.csv\""
]
Expand Down Expand Up @@ -784,7 +784,7 @@
"\n",
"\n",
"if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
Expand Down
8 changes: 4 additions & 4 deletions notebooks/official/prediction/get_started_with_psc_private_endpoint.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -201,7 +201,7 @@
"source": [
"# Create GCS Bucket\n",
"BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n",
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}"
]
},
{
Expand Down Expand Up @@ -237,7 +237,7 @@
"outputs": [],
"source": [
"# Copy Models to the Bucket\n",
"! gsutil cp -r \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/*\" {BUCKET_URI}"
"! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/*\" {BUCKET_URI}"
]
},
{
Expand Down Expand Up @@ -508,7 +508,7 @@
"outputs": [],
"source": [
"# Download the requests files:\n",
"! gsutil cp {BUCKET_URI}/requests/* ./"
"! gcloud storage cp {BUCKET_URI}/requests/* ./"
]
},
{
Expand Down Expand Up @@ -716,7 +716,7 @@
},
"outputs": [],
"source": [
"! gsutil rm -r {BUCKET_URI}"
"! gcloud storage rm --recursive {BUCKET_URI}"
]
},
{
Expand Down
8 changes: 4 additions & 4 deletions notebooks/official/prediction/llm_streaming_prediction.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
Expand Down Expand Up @@ -500,7 +500,7 @@
"source": [
"dataset = \"gs://cloud-samples-data/vertex-ai/prediction/llm/test_table.jsonl\"\n",
"destination_uri_prefix = f\"{BUCKET_URI}/text-bison@001_\"\n",
"! gsutil cp -r gs://cloud-samples-data/vertex-ai/prediction/llm/text-bison@001_/ {destination_uri_prefix}\n",
"! gcloud storage cp --recursive gs://cloud-samples-data/vertex-ai/prediction/llm/text-bison@001_/ {destination_uri_prefix}\n",
"\n",
"\n",
"from vertexai.language_models import TextGenerationModel\n",
Expand Down Expand Up @@ -580,7 +580,7 @@
"outputs": [],
"source": [
"# Tuning model\n",
"! gsutil cp gs://cloud-samples-data/vertex-ai/prediction/llm/q_a_train_with_context.jsonl {BUCKET_URI}/q_a_train_with_context.jsonl\n",
"! gcloud storage cp gs://cloud-samples-data/vertex-ai/prediction/llm/q_a_train_with_context.jsonl {BUCKET_URI}/q_a_train_with_context.jsonl\n",
"\n",
"tuning_job = model3.tune_model(\n",
" training_data=f\"{BUCKET_URI}/q_a_train_with_context.jsonl\",\n",
Expand Down Expand Up @@ -740,7 +740,7 @@
"batch_job_2.delete()\n",
"\n",
"if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
Expand Down