Skip to content

Migrate gsutil usage to gcloud storage #4298

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gericdong merged 13 commits into from
Dec 19, 2025
Merged

Migrate gsutil usage to gcloud storage #4298

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
9c9fee5
Migrate gsutil usage to gcloud storage
googlyrahman Oct 15, 2025
4d47226
update
gurusai-voleti Dec 10, 2025
c7a00a9
linter changes for 4298
gurusai-voleti Dec 12, 2025
00100ff
Revert "linter changes for 4298"
gurusai-voleti Dec 12, 2025
7cbdc7e
Linter fixes
googlyrahman Dec 12, 2025
d6b9d3b
Merge branch 'ai-gsutil-migration-1a72d88702124fbbb218e7db845aa6dc' i…
gurusai-voleti Dec 15, 2025
80438f9
update
gurusai-voleti Dec 15, 2025
d572c98
update
gurusai-voleti Dec 15, 2025
5208bda
Merge pull request #40 from gurusai-voleti/br_4298
googlyrahman Dec 15, 2025
e9fde3f
Merge branch 'GoogleCloudPlatform:main' into ai-gsutil-migration-1a72…
googlyrahman Dec 15, 2025
4f98b23
Update distributed_hyperparameter_tuning.ipynb
gurusai-voleti Dec 18, 2025
354b791
removed changes in model_garden folder
gurusai-voleti Dec 19, 2025
06167ab
Merge pull request #68 from gurusai-voleti/br_4298
gurusai-voleti Dec 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions notebooks/community/vision/video_warehouse_sdk.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,8 @@
},
"outputs": [],
"source": [
"!gsutil cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n",
"!pip install visionai-0.0.6-py3-none-any.whl --force-reinstall"
"! gcloud storage cp gs://visionai-artifacts/visionai-0.0.6-py3-none-any.whl .\n",
"! pip install visionai-0.0.6-py3-none-any.whl --force-reinstall"
]
},
{
Expand Down
4 changes: 2 additions & 2 deletions notebooks/official/experiments/comparing_local_trained_models.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,7 +286,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
]
},
{
Expand Down Expand Up @@ -770,7 +770,7 @@
"delete_bucket = False # Set True for deletion\n",
"\n",
"if delete_bucket:\n",
" ! gsutil rm -rf {BUCKET_URI}"
" ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}"
]
}
],
Expand Down
10 changes: 5 additions & 5 deletions ...ooks/official/experiments/get_started_with_custom_training_autologging_local_script.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location $LOCATION --project $PROJECT_ID $BUCKET_URI"
]
},
{
Expand Down Expand Up @@ -430,9 +430,9 @@
},
"outputs": [],
"source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
]
},
{
Expand Down Expand Up @@ -482,7 +482,7 @@
"SOURCE_DATA_URL = \"gs://cloud-samples-data/vertex-ai/dataset-management/datasets/uci_glass_preprocessed/glass.csv\"\n",
"DESTINATION_DATA_URL = f\"{BUCKET_URI}/data/glass.csv\"\n",
"\n",
"! gsutil cp $SOURCE_DATA_URL $DESTINATION_DATA_URL"
"! gcloud storage cp $SOURCE_DATA_URL $DESTINATION_DATA_URL"
]
},
{
Expand Down Expand Up @@ -807,7 +807,7 @@
"\n",
"# Delete Cloud Storage objects that were created\n",
"if delete_bucket or os.getenv(\"IS_TESTING\"):\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
Expand Down
16 changes: 8 additions & 8 deletions ...ooks/official/explainable_ai/sdk_automl_tabular_binary_classification_batch_explain.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
]
},
{
Expand Down Expand Up @@ -371,13 +371,13 @@
},
"outputs": [],
"source": [
"count = ! gsutil cat $IMPORT_FILE | wc -l\n",
"count = ! gcloud storage cat $IMPORT_FILE | wc -l\n",
"print(\"Number of Examples\", int(count[0]))\n",
"\n",
"print(\"First 10 rows\")\n",
"! gsutil cat $IMPORT_FILE | head\n",
"! gcloud storage cat $IMPORT_FILE | head\n",
"\n",
"heading = ! gsutil cat $IMPORT_FILE | head -n1\n",
"heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n",
"label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n",
"print(\"Label Column Name\", label_column)\n",
"if label_column is None:\n",
Expand Down Expand Up @@ -580,14 +580,14 @@
},
"outputs": [],
"source": [
"! gsutil cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
"! gsutil cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
"! gcloud storage cat $IMPORT_FILE | head -n 1 > tmp.csv\n",
"! gcloud storage cat $IMPORT_FILE | tail -n 10 >> tmp.csv\n",
"\n",
"! cut -d, -f1-16 tmp.csv > batch.csv\n",
"\n",
"gcs_input_uri = BUCKET_URI + \"/test.csv\"\n",
"\n",
"! gsutil cp batch.csv $gcs_input_uri"
"! gcloud storage cp batch.csv $gcs_input_uri"
]
},
{
Expand Down Expand Up @@ -729,7 +729,7 @@
"delete_bucket = False\n",
"\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI\n",
" ! gcloud storage rm --recursive $BUCKET_URI\n",
"\n",
"# Delete the locally generated files\n",
"! rm batch.csv tmp.csv"
Expand Down
12 changes: 7 additions & 5 deletions ...books/official/pipelines/google_cloud_pipeline_components_model_train_upload_deploy.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI"
"! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI"
]
},
{
Expand Down Expand Up @@ -405,9 +405,9 @@
},
"outputs": [],
"source": [
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n",
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n",
"\n",
"! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI"
"! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer"
]
},
{
Expand Down Expand Up @@ -506,7 +506,9 @@
},
"outputs": [],
"source": [
"hp_dict: str = '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n",
"hp_dict: str = (\n",
" '{\"num_hidden_layers\": 3, \"hidden_size\": 32, \"learning_rate\": 0.01, \"epochs\": 1, \"steps_per_epoch\": -1}'\n",
")\n",
"data_dir: str = (\n",
" \"gs://cloud-samples-data/vertex-ai/pipeline-deployment/datasets/bikes_weather/\"\n",
")\n",
Expand Down Expand Up @@ -732,7 +734,7 @@
"source": [
"delete_bucket = False\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
Expand Down
10 changes: 5 additions & 5 deletions ...s/official/prediction/pytorch_image_classification_with_prebuilt_serving_containers.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -303,7 +303,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}"
]
},
{
Expand Down Expand Up @@ -545,9 +545,9 @@
"source": [
"MODEL_URI = f\"{BUCKET_URI}/{model_name}\"\n",
"\n",
"!gsutil rm -r $MODEL_URI\n",
"!gsutil cp -r $model_path $MODEL_URI\n",
"!gsutil ls -al $MODEL_URI"
"!gcloud storage rm --recursive $MODEL_URI\n",
"!gcloud storage cp --recursive $model_path $MODEL_URI\n",
"!gcloud storage ls --all-versions --long $MODEL_URI"
]
},
{
Expand Down Expand Up @@ -823,7 +823,7 @@
"\n",
"delete_bucket = False\n",
"if delete_bucket:\n",
" ! gsutil -m rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
Expand Down
6 changes: 3 additions & 3 deletions notebooks/official/training/distributed_hyperparameter_tuning.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@
},
"outputs": [],
"source": [
"! gsutil mb -l {LOCATION} {BUCKET_URI}"
"! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}"
]
},
{
Expand Down Expand Up @@ -692,15 +692,15 @@
},
"outputs": [],
"source": [
"# Set this to true only if you'd like to delete your bucket\n",
"# Set this to true only if you'd like to delete your bucket.\n",
"delete_bucket = False\n",
"\n",
"hp_job.delete()\n",
"\n",
"!gcloud container images delete $IMAGE_URI --force-delete-tags --quiet\n",
"\n",
"if delete_bucket:\n",
" ! gsutil rm -r $BUCKET_URI"
" ! gcloud storage rm --recursive $BUCKET_URI"
]
}
],
Expand Down