From b278de947c411850307e51e95488f7fd5fb5be59 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Fri, 17 Oct 2025 13:20:00 +0000 Subject: [PATCH 1/6] Migrate gsutil usage to gcloud storage --- .../vertex_ai_model_experimentation.ipynb | 17 +++++----- ... Training Prebuilt Container SKLearn.ipynb | 12 +++---- .../ml_ops/stage1/mlops_data_management.ipynb | 26 +++++----------- .../stage6/get_started_with_tf_serving.ipynb | 15 +++------ ...l_garden_pytorch_llama3_1_finetuning.ipynb | 17 ++++------ .../SDK_Custom_Preprocess.ipynb | 13 +++----- .../SDK_Triton_PyTorch_Local_Prediction.ipynb | 13 +++----- ...k_custom_image_classification_online.ipynb | 12 +++---- ...mage_classification_online_tfserving.ipynb | 12 +++---- ...-image-object-detection-batch-online.ipynb | 31 ++++++------------- 10 files changed, 55 insertions(+), 113 deletions(-) diff --git a/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb b/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb index c4e71b6f4..2db00278e 100644 --- a/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb +++ b/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @@ -537,8 +537,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -557,8 +556,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -627,10 +625,10 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -1736,8 +1734,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/migration/UJ10 legacy Custom Training Prebuilt Container SKLearn.ipynb b/notebooks/community/migration/UJ10 legacy Custom Training Prebuilt Container SKLearn.ipynb index f49bfd55f..65f8e5837 100644 --- a/notebooks/community/migration/UJ10 legacy Custom Training Prebuilt Container SKLearn.ipynb +++ b/notebooks/community/migration/UJ10 legacy Custom Training Prebuilt Container SKLearn.ipynb @@ -312,8 +312,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION gs://$BUCKET_NAME" - ] + "! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -332,8 +331,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al gs://$BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -668,8 +666,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz gs://$BUCKET_NAME/census.tar.gz" - ] + "! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/census.tar.gz" ] }, { "cell_type": "markdown", @@ -1923,8 +1920,7 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r gs://$BUCKET_NAME" - ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb b/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb index f3b38ee64..203304aea 100644 --- a/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb +++ b/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb @@ -465,8 +465,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -485,8 +484,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1093,8 +1091,7 @@ ") as f:\n", " json.dump(metadata, f)\n", "\n", - "! gsutil cat $BUCKET_URI/metadata.jsonl" - ] + "! gcloud storage cat $BUCKET_URI/metadata.jsonl" ] }, { "cell_type": "markdown", @@ -1178,8 +1175,7 @@ ") as f:\n", " json.dump(metadata, f)\n", "\n", - "! gsutil cat $BUCKET_URI/metadata.jsonl" - ] + "! gcloud storage cat $BUCKET_URI/metadata.jsonl" ] }, { "cell_type": "markdown", @@ -1527,13 +1523,7 @@ "run_pipeline(args)\n", "print(\"Data preprocessing completed.\")\n", "\n", - "! gsutil ls $TRANSFORMED_DATA_PREFIX/train\n", - "! gsutil ls $TRANSFORMED_DATA_PREFIX/val\n", - "! gsutil ls $TRANSFORMED_DATA_PREFIX/test\n", - "! gsutil ls $TRANSFORM_ARTIFACTS_DIR\n", - "! gsutil ls {EXPORTED_JSONL_PREFIX}*\n", - "! gsutil ls $EXPORTED_TFREC_PREFIX" - ] + "! gcloud storage ls $TRANSFORMED_DATA_PREFIX/train\n", "! gcloud storage ls $TRANSFORMED_DATA_PREFIX/val\n", "! gcloud storage ls $TRANSFORMED_DATA_PREFIX/test\n", "! gcloud storage ls $TRANSFORM_ARTIFACTS_DIR\n", "! gcloud storage ls {EXPORTED_JSONL_PREFIX}*\n", "! gcloud storage ls $EXPORTED_TFREC_PREFIX" ] }, { "cell_type": "markdown", @@ -1588,8 +1578,7 @@ ") as f:\n", " json.dump(metadata, f)\n", "\n", - "! gsutil cat $BUCKET_URI/metadata.jsonl" - ] + "! gcloud storage cat $BUCKET_URI/metadata.jsonl" ] }, { "cell_type": "markdown", @@ -1626,8 +1615,7 @@ " print(e)\n", "\n", " if \"BUCKET_URI\" in globals():\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb index 9cf8562f3..910d3e2c7 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb @@ -446,8 +446,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] +"! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -466,8 +465,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1098,8 +1096,7 @@ }, "outputs": [], "source": [ - "! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" - ] + "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" ] }, { "cell_type": "code", @@ -1296,8 +1293,7 @@ " f.write(\"\\n\")\n", " json.dump(instances[0], f)\n", "\n", - "! gsutil cp test.jsonl {BUCKET_URI}/test.jsonl" - ] + "! gcloud storage cp test.jsonl {BUCKET_URI}/test.jsonl" ] }, { "cell_type": "markdown", @@ -1459,8 +1455,7 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb index c8e399a70..ccffcea36 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb @@ -185,11 +185,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -213,8 +211,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -395,9 +393,7 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", - " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -1112,8 +1108,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb index a43c0b2cb..7d613dc61 100644 --- a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb +++ b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb @@ -496,8 +496,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -516,8 +515,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -687,9 +685,7 @@ "outputs": [], "source": [ "%cd ..\n", - "!gsutil cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", - "!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" - ] + "!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] }, { "cell_type": "markdown", @@ -1809,8 +1805,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb b/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb index c6b5c8dcc..25f0ed632 100644 --- a/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb +++ b/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb @@ -490,8 +490,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -510,8 +509,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -918,9 +916,7 @@ }, "outputs": [], "source": [ - "!gsutil cp -r {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", - "!gsutil ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" - ] + "!gcloud storage cp --recursive {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] }, { "cell_type": "markdown", @@ -1869,8 +1865,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_custom_image_classification_online.ipynb b/notebooks/community/sdk/sdk_custom_image_classification_online.ipynb index 08283c2da..8248cde60 100644 --- a/notebooks/community/sdk/sdk_custom_image_classification_online.ipynb +++ b/notebooks/community/sdk/sdk_custom_image_classification_online.ipynb @@ -475,8 +475,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -495,8 +494,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -917,8 +915,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ] }, { "cell_type": "markdown", @@ -1544,8 +1541,7 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_custom_image_classification_online_tfserving.ipynb b/notebooks/community/sdk/sdk_custom_image_classification_online_tfserving.ipynb index 0cb244393..a7209a78a 100644 --- a/notebooks/community/sdk/sdk_custom_image_classification_online_tfserving.ipynb +++ b/notebooks/community/sdk/sdk_custom_image_classification_online_tfserving.ipynb @@ -475,8 +475,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -495,8 +494,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -944,8 +942,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ] }, { "cell_type": "markdown", @@ -1535,8 +1532,7 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/official/migration/sdk-automl-image-object-detection-batch-online.ipynb b/notebooks/official/migration/sdk-automl-image-object-detection-batch-online.ipynb index 2925d76c8..42f2cef48 100644 --- a/notebooks/official/migration/sdk-automl-image-object-detection-batch-online.ipynb +++ b/notebooks/official/migration/sdk-automl-image-object-detection-batch-online.ipynb @@ -287,8 +287,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -392,10 +391,8 @@ " + \"/\"\n", ")\n", "\n", - "# Copy images using gsutil commands directly\n", - "for src, dest in zip(df.iloc[:, 0], df[\"destination_path\"]):\n", - " ! gsutil -m cp {src} {dest}\n", - "\n", + "# Copy images using gcloud storage commands directly\n", "for src, dest in zip(df.iloc[:, 0], df[\"destination_path\"]):\n", + " ! gcloud storage cp {src} {dest}\n", "\n", "print(f\"Files copied to {BUCKET_URI}\")" ] }, @@ -475,12 +472,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -832,8 +827,7 @@ }, "outputs": [], "source": [ - "test_items = !gsutil cat $IMPORT_FILE | head -n2\n", - "cols_1 = str(test_items[0]).split(\",\")\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "cols_1 = str(test_items[0]).split(\",\")\n", "cols_2 = str(test_items[1]).split(\",\")\n", "if len(cols_1) == 11:\n", " test_item_1 = str(cols_1[1])\n", @@ -872,9 +866,7 @@ "file_1 = test_item_1.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n", "\n", - "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n", - "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n", - "\n", + "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", "\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_2 = BUCKET_URI + \"/\" + file_2" ] @@ -917,8 +909,7 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gsutil cat $gcs_input_uri" - ] + "! gcloud storage cat $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -1174,8 +1165,7 @@ }, "outputs": [], "source": [ - "test_items = !gsutil cat $IMPORT_FILE | head -n1\n", - "cols = str(test_items[0]).split(\",\")\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n", "cols = str(test_items[0]).split(\",\")\n", "if len(cols) == 11:\n", " test_item = str(cols[1])\n", " test_label = str(cols[2])\n", @@ -1313,8 +1303,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From f28476e152a3bcd16dbabfcfbbc3378a8acb7a7c Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 12 Dec 2025 03:47:19 +0000 Subject: [PATCH 2/6] Manual Changes --- .../experiments/vertex_ai_model_experimentation.ipynb | 4 ++-- .../model_garden_pytorch_llama3_1_finetuning.ipynb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb b/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb index 2db00278e..193bce020 100644 --- a/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb +++ b/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @@ -625,9 +625,9 @@ }, "outputs": [], "source": [ - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb index ccffcea36..9b7604a23 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb @@ -211,7 +211,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", From 5cf02d37852ccfc973ce429d75840862790e6bad Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 12 Dec 2025 07:36:11 +0000 Subject: [PATCH 3/6] Removed blank line and spaces Manual Changes --- .../community/experiments/vertex_ai_model_experimentation.ipynb | 2 -- .../model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb | 1 - 2 files changed, 3 deletions(-) diff --git a/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb b/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb index 193bce020..c5d8895a8 100644 --- a/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb +++ b/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @@ -625,9 +625,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb index 9b7604a23..3fc815e89 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb @@ -211,7 +211,6 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", From 14a5c9b48c666a0bf5953520ee27140709d197c0 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Wed, 17 Dec 2025 15:35:41 +0000 Subject: [PATCH 4/6] fix the Linter issue for 4330 --- .../vertex_ai_model_experimentation.ipynb | 15 ++++++--- ... Training Prebuilt Container SKLearn.ipynb | 12 ++++--- .../ml_ops/stage1/mlops_data_management.ipynb | 26 +++++++++++----- .../stage6/get_started_with_tf_serving.ipynb | 15 ++++++--- ...l_garden_pytorch_llama3_1_finetuning.ipynb | 24 +++++++++----- .../SDK_Custom_Preprocess.ipynb | 13 +++++--- .../SDK_Triton_PyTorch_Local_Prediction.ipynb | 13 +++++--- ...k_custom_image_classification_online.ipynb | 12 ++++--- ...mage_classification_online_tfserving.ipynb | 12 ++++--- ...-image-object-detection-batch-online.ipynb | 31 +++++++++++++------ 10 files changed, 119 insertions(+), 54 deletions(-) diff --git a/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb b/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb index c5d8895a8..f8b13f2ae 100644 --- a/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb +++ b/notebooks/community/experiments/vertex_ai_model_experimentation.ipynb @@ -537,7 +537,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -556,7 +557,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -625,8 +627,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -1732,7 +1736,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/community/migration/UJ10 legacy Custom Training Prebuilt Container SKLearn.ipynb b/notebooks/community/migration/UJ10 legacy Custom Training Prebuilt Container SKLearn.ipynb index 65f8e5837..9ccba7176 100644 --- a/notebooks/community/migration/UJ10 legacy Custom Training Prebuilt Container SKLearn.ipynb +++ b/notebooks/community/migration/UJ10 legacy Custom Training Prebuilt Container SKLearn.ipynb @@ -312,7 +312,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME" ] + "! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -331,7 +332,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -666,7 +668,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/census.tar.gz" ] + "! gcloud storage cp custom.tar.gz gs://$BUCKET_NAME/census.tar.gz" + ] }, { "cell_type": "markdown", @@ -1920,7 +1923,8 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive gs://$BUCKET_NAME" ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb b/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb index 203304aea..ff8465bf7 100644 --- a/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb +++ b/notebooks/community/ml_ops/stage1/mlops_data_management.ipynb @@ -465,7 +465,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -484,7 +485,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -1091,7 +1093,8 @@ ") as f:\n", " json.dump(metadata, f)\n", "\n", - "! gcloud storage cat $BUCKET_URI/metadata.jsonl" ] + "! gcloud storage cat $BUCKET_URI/metadata.jsonl" + ] }, { "cell_type": "markdown", @@ -1175,7 +1178,8 @@ ") as f:\n", " json.dump(metadata, f)\n", "\n", - "! gcloud storage cat $BUCKET_URI/metadata.jsonl" ] + "! gcloud storage cat $BUCKET_URI/metadata.jsonl" + ] }, { "cell_type": "markdown", @@ -1523,7 +1527,13 @@ "run_pipeline(args)\n", "print(\"Data preprocessing completed.\")\n", "\n", - "! gcloud storage ls $TRANSFORMED_DATA_PREFIX/train\n", "! gcloud storage ls $TRANSFORMED_DATA_PREFIX/val\n", "! gcloud storage ls $TRANSFORMED_DATA_PREFIX/test\n", "! gcloud storage ls $TRANSFORM_ARTIFACTS_DIR\n", "! gcloud storage ls {EXPORTED_JSONL_PREFIX}*\n", "! gcloud storage ls $EXPORTED_TFREC_PREFIX" ] + "! gcloud storage ls $TRANSFORMED_DATA_PREFIX/train\n", + "! gcloud storage ls $TRANSFORMED_DATA_PREFIX/val\n", + "! gcloud storage ls $TRANSFORMED_DATA_PREFIX/test\n", + "! gcloud storage ls $TRANSFORM_ARTIFACTS_DIR\n", + "! gcloud storage ls {EXPORTED_JSONL_PREFIX}*\n", + "! gcloud storage ls $EXPORTED_TFREC_PREFIX" + ] }, { "cell_type": "markdown", @@ -1578,7 +1588,8 @@ ") as f:\n", " json.dump(metadata, f)\n", "\n", - "! gcloud storage cat $BUCKET_URI/metadata.jsonl" ] + "! gcloud storage cat $BUCKET_URI/metadata.jsonl" + ] }, { "cell_type": "markdown", @@ -1615,7 +1626,8 @@ " print(e)\n", "\n", " if \"BUCKET_URI\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb index 910d3e2c7..eef36b11f 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_tf_serving.ipynb @@ -446,7 +446,8 @@ }, "outputs": [], "source": [ -"! gcloud storage buckets create --location $REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -465,7 +466,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -1096,7 +1098,8 @@ }, "outputs": [], "source": [ - "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" ] + "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" + ] }, { "cell_type": "code", @@ -1293,7 +1296,8 @@ " f.write(\"\\n\")\n", " json.dump(instances[0], f)\n", "\n", - "! gcloud storage cp test.jsonl {BUCKET_URI}/test.jsonl" ] + "! gcloud storage cp test.jsonl {BUCKET_URI}/test.jsonl" + ] }, { "cell_type": "markdown", @@ -1455,7 +1459,8 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb index 3fc815e89..319ab08a3 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb @@ -185,9 +185,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -211,7 +213,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -236,7 +239,9 @@ "\n", "# @markdown **Only select and fill one of the following sections.**\n", "# It is recommended to use \"Google Cloud\" for 405B model since it can be downloaded faster.\n", - "LOAD_MODEL_FROM = \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + "LOAD_MODEL_FROM = (\n", + " \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + ")\n", "\n", "# @markdown ---\n", "\n", @@ -392,7 +397,9 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", + " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", " tokenizer_path = pretrained_model_id\n", @@ -444,7 +451,9 @@ "# @markdown 1. If `max_steps > 0`, it takes precedence over `epochs`. One can set a small `max_steps` value to quickly check the pipeline.\n", "\n", "# @markdown Acceletor type to use for training.\n", - "training_accelerator_type = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", + "training_accelerator_type = (\n", + " \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", + ")\n", "\n", "# The pre-built training docker image.\n", "if training_accelerator_type == \"NVIDIA_A100_80GB\":\n", @@ -1107,7 +1116,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb index 7d613dc61..c45975233 100644 --- a/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb +++ b/notebooks/community/prediction/custom_prediction_routines/SDK_Custom_Preprocess.ipynb @@ -496,7 +496,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -515,7 +516,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -685,7 +687,9 @@ "outputs": [], "source": [ "%cd ..\n", - "!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] + "!gcloud storage cp {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", + "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" + ] }, { "cell_type": "markdown", @@ -1805,7 +1809,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb b/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb index 25f0ed632..c3c85e994 100644 --- a/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb +++ b/notebooks/community/prediction/custom_prediction_routines/SDK_Triton_PyTorch_Local_Prediction.ipynb @@ -490,7 +490,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -509,7 +510,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -916,7 +918,9 @@ }, "outputs": [], "source": [ - "!gcloud storage cp --recursive {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" ] + "!gcloud storage cp --recursive {LOCAL_MODEL_ARTIFACTS_DIR}/* {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/\n", + "!gcloud storage ls {BUCKET_URI}/{MODEL_ARTIFACT_DIR}/" + ] }, { "cell_type": "markdown", @@ -1865,7 +1869,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_custom_image_classification_online.ipynb b/notebooks/community/sdk/sdk_custom_image_classification_online.ipynb index 8248cde60..5b1cb98ea 100644 --- a/notebooks/community/sdk/sdk_custom_image_classification_online.ipynb +++ b/notebooks/community/sdk/sdk_custom_image_classification_online.ipynb @@ -475,7 +475,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --location $REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -494,7 +495,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -915,7 +917,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ] + "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" + ] }, { "cell_type": "markdown", @@ -1541,7 +1544,8 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_custom_image_classification_online_tfserving.ipynb b/notebooks/community/sdk/sdk_custom_image_classification_online_tfserving.ipynb index a7209a78a..1c2ed6fd4 100644 --- a/notebooks/community/sdk/sdk_custom_image_classification_online_tfserving.ipynb +++ b/notebooks/community/sdk/sdk_custom_image_classification_online_tfserving.ipynb @@ -475,7 +475,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -494,7 +495,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -942,7 +944,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" ] + "! gcloud storage cp custom.tar.gz $BUCKET_NAME/trainer_cifar10.tar.gz" + ] }, { "cell_type": "markdown", @@ -1532,7 +1535,8 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/official/migration/sdk-automl-image-object-detection-batch-online.ipynb b/notebooks/official/migration/sdk-automl-image-object-detection-batch-online.ipynb index 42f2cef48..1fbed93ab 100644 --- a/notebooks/official/migration/sdk-automl-image-object-detection-batch-online.ipynb +++ b/notebooks/official/migration/sdk-automl-image-object-detection-batch-online.ipynb @@ -287,7 +287,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -391,8 +392,10 @@ " + \"/\"\n", ")\n", "\n", - "# Copy images using gcloud storage commands directly\n", "for src, dest in zip(df.iloc[:, 0], df[\"destination_path\"]):\n", - " ! gcloud storage cp {src} {dest}\n", "\n", + "# Copy images using gcloud storage commands directly\n", + "for src, dest in zip(df.iloc[:, 0], df[\"destination_path\"]):\n", + " ! gcloud storage cp {src} {dest}\n", + "\n", "print(f\"Files copied to {BUCKET_URI}\")" ] }, @@ -472,10 +475,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -827,7 +832,8 @@ }, "outputs": [], "source": [ - "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "cols_1 = str(test_items[0]).split(\",\")\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", + "cols_1 = str(test_items[0]).split(\",\")\n", "cols_2 = str(test_items[1]).split(\",\")\n", "if len(cols_1) == 11:\n", " test_item_1 = str(cols_1[1])\n", @@ -866,7 +872,9 @@ "file_1 = test_item_1.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n", "\n", - "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", "\n", + "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", + "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", + "\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_2 = BUCKET_URI + \"/\" + file_2" ] @@ -909,7 +917,8 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gcloud storage cat $gcs_input_uri" ] + "! gcloud storage cat $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -1165,7 +1174,8 @@ }, "outputs": [], "source": [ - "test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n", "cols = str(test_items[0]).split(\",\")\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n1\n", + "cols = str(test_items[0]).split(\",\")\n", "if len(cols) == 11:\n", " test_item = str(cols[1])\n", " test_label = str(cols[2])\n", @@ -1303,7 +1313,8 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From 4c28c66d4afb58699f64522d036b8aba95179b39 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Mon, 22 Dec 2025 18:09:44 +0000 Subject: [PATCH 5/6] Changes for model garden --- .../model_garden_pytorch_llama3_1_finetuning.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb index 319ab08a3..ae6494a44 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb @@ -185,10 +185,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gsutil ls -b {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -213,7 +213,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -397,8 +397,8 @@ "if dataset_validation_util.is_gcs_path(pretrained_model_id):\n", " # Download tokenizer.\n", " ! mkdir tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", - " ! gcloud storage cp {pretrained_model_id}/config.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/tokenizer.json ./tokenizer\n", + " ! gsutil cp {pretrained_model_id}/config.json ./tokenizer\n", " tokenizer_path = \"./tokenizer\"\n", " access_token = \"\"\n", "else:\n", @@ -1116,7 +1116,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], From a5e773e0fbb28d1c82d6dd9c80619f3a2c4a7bcd Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 23 Dec 2025 11:16:02 +0530 Subject: [PATCH 6/6] Update model_garden_pytorch_llama3_1_finetuning.ipynb --- .../model_garden_pytorch_llama3_1_finetuning.ipynb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb index ae6494a44..c8e399a70 100644 --- a/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb +++ b/notebooks/community/model_garden/model_garden_pytorch_llama3_1_finetuning.ipynb @@ -188,7 +188,7 @@ " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -b {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -239,9 +239,7 @@ "\n", "# @markdown **Only select and fill one of the following sections.**\n", "# It is recommended to use \"Google Cloud\" for 405B model since it can be downloaded faster.\n", - "LOAD_MODEL_FROM = (\n", - " \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", - ")\n", + "LOAD_MODEL_FROM = \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", "\n", "# @markdown ---\n", "\n", @@ -451,9 +449,7 @@ "# @markdown 1. If `max_steps > 0`, it takes precedence over `epochs`. One can set a small `max_steps` value to quickly check the pipeline.\n", "\n", "# @markdown Acceletor type to use for training.\n", - "training_accelerator_type = (\n", - " \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", - ")\n", + "training_accelerator_type = \"NVIDIA_A100_80GB\" # @param [\"NVIDIA_A100_80GB\", \"NVIDIA_H100_80GB\"]\n", "\n", "# The pre-built training docker image.\n", "if training_accelerator_type == \"NVIDIA_A100_80GB\":\n",