From 694ac6880e0fecd193ed576c8082a9fcb19159c1 Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Wed, 15 Oct 2025 20:41:56 +0000 Subject: [PATCH 1/9] Migrate gsutil usage to gcloud storage --- .../bq_dataframes_llm_code_generation.ipynb | 6 ++---- .../model_garden_jax_vision_transformer.ipynb | 10 +++------- .../model_garden_keras_yolov8.ipynb | 16 ++++++---------- ...el_garden_mediapipe_object_detection.ipynb | 19 +++++++------------ .../get_started_with_data_labeling.ipynb | 12 ++++-------- ...abular_classification_online_explain.ipynb | 15 +++++---------- .../get_started_with_dedicated_endpoint.ipynb | 9 +++------ ...ustom_training_with_custom_container.ipynb | 13 +++++-------- ...oard_vertex_ai_pipelines_integration.ipynb | 6 ++---- ...tion-model-for-ads-targeting-usecase.ipynb | 6 ++---- 10 files changed, 39 insertions(+), 73 deletions(-) diff --git a/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb b/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb index e5a9f1729..f85c19c4c 100644 --- a/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb +++ b/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb @@ -761,8 +761,7 @@ "\n", "BUCKET_ID = \"code-samples-\" + str(uuid.uuid1())\n", "\n", - "!gsutil mb gs://{BUCKET_ID}" - ] + "!gcloud storage buckets create gs://{BUCKET_ID}" ] }, { "cell_type": "markdown", @@ -874,8 +873,7 @@ "outputs": [], "source": [ "# # Delete the Google Cloud Storage bucket and files\n", - "# ! gsutil rm -r gs://{BUCKET_ID}\n", - "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" + "# ! gcloud storage rm --recursive gs://{BUCKET_ID}\n", "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" ] } ], diff --git a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb index e395180a3..93c1e2344 100644 --- a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb @@ -353,8 +353,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -582,9 +581,7 @@ "source": [ "# Move Flower data from local directory to Cloud Storage.\n", "# This step takes around 2 mins to finish.\n", - "! gsutil -m cp -R $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", - "! gsutil -m cp -R $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" - ] + "! gcloud storage cp --recursive $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", "! gcloud storage cp --recursive $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" ] }, { "cell_type": "markdown", @@ -867,8 +864,7 @@ "# Delete Cloud Storage objects that were created.\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb index a1d8bad58..492225a24 100644 --- a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb @@ -175,11 +175,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -203,8 +202,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -241,8 +239,7 @@ " label_map_yaml_filename = os.path.basename(label_map_yaml_filepath)\n", " local_metrics_path = os.path.join(temp_dir.name, label_map_yaml_filename)\n", "\n", - " ! gsutil cp $label_map_yaml_filepath $local_metrics_path\n", - " with open(local_metrics_path, \"r\") as input_file:\n", + " ! gcloud storage cp $label_map_yaml_filepath $local_metrics_path\n", " with open(local_metrics_path, \"r\") as input_file:\n", " label_map = yaml.safe_load(input_file.read())[\"label_map\"]\n", " temp_dir.cleanup()\n", " return label_map\n", @@ -624,8 +621,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb index 7a311398e..03550d2cc 100644 --- a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb +++ b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb @@ -157,11 +157,9 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -185,8 +183,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -498,8 +496,7 @@ " try:\n", " eval_result_filename = os.path.basename(evaluation_result_path)\n", " subprocess.check_output(\n", - " [\"gsutil\", \"cp\", evaluation_result_path, eval_result_filename],\n", - " stderr=subprocess.STDOUT,\n", + " [\"gcloud\", \"storage\", \"cp\", evaluation_result_path, eval_result_filename],\n", " stderr=subprocess.STDOUT,\n", " )\n", " with open(eval_result_filename, \"r\") as input_file:\n", " evalutation_result = json.loads(input_file.read())\n", @@ -531,8 +528,7 @@ "\n", "# @markdown After fine-tuning and evaluating the model, you can save it as Tensorflow Lite model, try it out in the [Object Detector](https://mediapipe-studio.webapps.google.com/demo/object_detector) demo in MediaPipe Studio or integrate it with your application by following the [Object detection task guide](https://developers.google.com/mediapipe/solutions/vision/object_detector). The exported model also includes metadata and the label map.\n", "\n", - "! gsutil cp $EXPORTED_MODEL_OUTPUT_FILE \"object_detection_model.tflite\"" - ] + "! gcloud storage cp $EXPORTED_MODEL_OUTPUT_FILE \"object_detection_model.tflite\"" ] }, { "cell_type": "markdown", @@ -571,8 +567,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/official/datasets/get_started_with_data_labeling.ipynb b/notebooks/official/datasets/get_started_with_data_labeling.ipynb index 9de4a780d..1837267c6 100644 --- a/notebooks/official/datasets/get_started_with_data_labeling.ipynb +++ b/notebooks/official/datasets/get_started_with_data_labeling.ipynb @@ -381,8 +381,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -584,8 +583,7 @@ "print(IMPORT_FILE)\n", "\n", "# printing content of uploaded file\n", - "! gsutil cat $IMPORT_FILE" - ] + "! gcloud storage cat $IMPORT_FILE" ] }, { "cell_type": "markdown", @@ -707,8 +705,7 @@ "outputs": [], "source": [ "# create placeholder file for instructions for data labeling\n", - "! echo \"this is instruction\" >> instruction.txt | gsutil cp instruction.txt $BUCKET_URI" - ] + "! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" ] }, { "cell_type": "code", @@ -864,8 +861,7 @@ "\n", "# Delete the bucket created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb b/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb index f5baf5f23..c0f29eba2 100644 --- a/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb @@ -291,8 +291,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -372,14 +371,11 @@ }, "outputs": [], "source": [ - "count = ! gsutil cat $IMPORT_FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $IMPORT_FILE | head\n", - "\n", - "heading = ! gsutil cat $IMPORT_FILE | head -n1\n", - "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -840,8 +836,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb b/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb index 5f12b1618..a803a0f89 100644 --- a/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb +++ b/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb @@ -217,8 +217,7 @@ "source": [ "# Create GCS Bucket\n", "BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n", - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -309,8 +308,7 @@ "IMAGE_URI = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-12:latest\"\n", "\n", "# Copy Test Models to the Bucket\n", - "! gsutil -m cp -r \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n", - "\n", + "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n", "\n", "model = aiplatform.Model.upload(\n", " display_name=DISPLAY_NAME,\n", " artifact_uri=ARTIFACT_URI,\n", @@ -786,8 +784,7 @@ }, "outputs": [], "source": [ - "! gsutil rm -r {BUCKET_URI}" - ] + "! gcloud storage rm --recursive {BUCKET_URI}" ] }, { "cell_type": "markdown", diff --git a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb index 6f4823c5f..30cfcba5e 100644 --- a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb +++ b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb @@ -347,8 +347,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -720,10 +719,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", +# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -840,8 +838,7 @@ "# Delete GCS bucket.\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb b/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb index 6c2800621..27aa1c425 100644 --- a/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb +++ b/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb @@ -350,8 +350,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -846,8 +845,7 @@ "# Delete GCS bucket.\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gsutil -m rm -r {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb index 666f96d89..85af168ee 100644 --- a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb +++ b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb @@ -282,8 +282,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $LOCATION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -840,8 +839,7 @@ "\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From f60dd906f506ce7d3f7f04690d99a529453afbc0 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 11 Dec 2025 07:35:58 +0000 Subject: [PATCH 2/9] changes for 4301 --- .../model_garden_keras_yolov8.ipynb | 16 ++++++++++------ ...el_garden_mediapipe_object_detection.ipynb | 19 ++++++++++++------- ...ustom_training_with_custom_container.ipynb | 1 - 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb index 492225a24..b15c71a07 100644 --- a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb @@ -175,10 +175,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -202,7 +203,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -239,7 +241,8 @@ " label_map_yaml_filename = os.path.basename(label_map_yaml_filepath)\n", " local_metrics_path = os.path.join(temp_dir.name, label_map_yaml_filename)\n", "\n", - " ! gcloud storage cp $label_map_yaml_filepath $local_metrics_path\n", " with open(local_metrics_path, \"r\") as input_file:\n", + " ! gcloud storage cp $label_map_yaml_filepath $local_metrics_path\n", + " with open(local_metrics_path, \"r\") as input_file:\n", " label_map = yaml.safe_load(input_file.read())[\"label_map\"]\n", " temp_dir.cleanup()\n", " return label_map\n", @@ -621,7 +624,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb index 03550d2cc..d8c49cb83 100644 --- a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb +++ b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb @@ -157,9 +157,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -183,8 +185,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"\n", @@ -496,7 +498,8 @@ " try:\n", " eval_result_filename = os.path.basename(evaluation_result_path)\n", " subprocess.check_output(\n", - " [\"gcloud\", \"storage\", \"cp\", evaluation_result_path, eval_result_filename],\n", " stderr=subprocess.STDOUT,\n", + " [\"gcloud\", \"storage\", \"cp\", evaluation_result_path, eval_result_filename],\n", + " stderr=subprocess.STDOUT,\n", " )\n", " with open(eval_result_filename, \"r\") as input_file:\n", " evalutation_result = json.loads(input_file.read())\n", @@ -528,7 +531,8 @@ "\n", "# @markdown After fine-tuning and evaluating the model, you can save it as Tensorflow Lite model, try it out in the [Object Detector](https://mediapipe-studio.webapps.google.com/demo/object_detector) demo in MediaPipe Studio or integrate it with your application by following the [Object detection task guide](https://developers.google.com/mediapipe/solutions/vision/object_detector). The exported model also includes metadata and the label map.\n", "\n", - "! gcloud storage cp $EXPORTED_MODEL_OUTPUT_FILE \"object_detection_model.tflite\"" ] + "! gcloud storage cp $EXPORTED_MODEL_OUTPUT_FILE \"object_detection_model.tflite\"" + ] }, { "cell_type": "markdown", @@ -567,7 +571,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb index 30cfcba5e..674d507fb 100644 --- a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb +++ b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb @@ -720,7 +720,6 @@ "outputs": [], "source": [ "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", -# Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { From 6665133b2cf0f16ce2e9be703335f6eb10d6b40f Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 12 Dec 2025 06:32:48 +0000 Subject: [PATCH 3/9] linter changes --- .../bq_dataframes_llm_code_generation.ipynb | 6 ++++-- .../model_garden_jax_vision_transformer.ipynb | 16 +++++++++------- .../model_garden/model_garden_keras_yolov8.ipynb | 2 +- ...model_garden_mediapipe_object_detection.ipynb | 2 +- .../get_started_with_data_labeling.ipynb | 12 ++++++++---- ...l_tabular_classification_online_explain.ipynb | 15 ++++++++++----- .../get_started_with_dedicated_endpoint.ipynb | 9 ++++++--- ...d_custom_training_with_custom_container.ipynb | 12 ++++++++---- ...orboard_vertex_ai_pipelines_integration.ipynb | 6 ++++-- ...ication-model-for-ads-targeting-usecase.ipynb | 6 ++++-- 10 files changed, 55 insertions(+), 31 deletions(-) diff --git a/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb b/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb index f85c19c4c..71e5babd8 100644 --- a/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb +++ b/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb @@ -761,7 +761,8 @@ "\n", "BUCKET_ID = \"code-samples-\" + str(uuid.uuid1())\n", "\n", - "!gcloud storage buckets create gs://{BUCKET_ID}" ] + "!gcloud storage buckets create gs://{BUCKET_ID}" + ] }, { "cell_type": "markdown", @@ -873,7 +874,8 @@ "outputs": [], "source": [ "# # Delete the Google Cloud Storage bucket and files\n", - "# ! gcloud storage rm --recursive gs://{BUCKET_ID}\n", "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" + "# ! gcloud storage rm --recursive gs://{BUCKET_ID}\n", + "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" ] } ], diff --git a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb index 93c1e2344..b8a7d6401 100644 --- a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb @@ -353,7 +353,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -581,7 +582,9 @@ "source": [ "# Move Flower data from local directory to Cloud Storage.\n", "# This step takes around 2 mins to finish.\n", - "! gcloud storage cp --recursive $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", "! gcloud storage cp --recursive $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" ] + "! gcloud storage cp --recursive $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", + "! gcloud storage cp --recursive $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" + ] }, { "cell_type": "markdown", @@ -782,7 +785,7 @@ " serving_container_args=[],\n", " location=REGION,\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/vit-jax\"\n", + " model_garden_source_model_name=\"publishers/google/models/vit-jax\",\n", ")\n", "\n", "jax_vit_endpoint = jax_vit_model.deploy(\n", @@ -793,9 +796,7 @@ " accelerator_count=1,\n", " min_replica_count=1,\n", " max_replica_count=1,\n", - " system_labels={\n", - " \"NOTEBOOK_NAME\": \"model_garden_jax_vision_transformer.ipynb\"\n", - " },\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_jax_vision_transformer.ipynb\"},\n", ")" ] }, @@ -864,7 +865,8 @@ "# Delete Cloud Storage objects that were created.\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb index b15c71a07..ccf7152ca 100644 --- a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb @@ -242,7 +242,7 @@ " local_metrics_path = os.path.join(temp_dir.name, label_map_yaml_filename)\n", "\n", " ! gcloud storage cp $label_map_yaml_filepath $local_metrics_path\n", - " with open(local_metrics_path, \"r\") as input_file:\n", + " with open(local_metrics_path) as input_file:\n", " label_map = yaml.safe_load(input_file.read())[\"label_map\"]\n", " temp_dir.cleanup()\n", " return label_map\n", diff --git a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb index d8c49cb83..84253879c 100644 --- a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb +++ b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb @@ -501,7 +501,7 @@ " [\"gcloud\", \"storage\", \"cp\", evaluation_result_path, eval_result_filename],\n", " stderr=subprocess.STDOUT,\n", " )\n", - " with open(eval_result_filename, \"r\") as input_file:\n", + " with open(eval_result_filename) as input_file:\n", " evalutation_result = json.loads(input_file.read())\n", " return evalutation_result[\"loss\"], evalutation_result[\"coco_metrics\"]\n", " except:\n", diff --git a/notebooks/official/datasets/get_started_with_data_labeling.ipynb b/notebooks/official/datasets/get_started_with_data_labeling.ipynb index 1837267c6..d2846e3c4 100644 --- a/notebooks/official/datasets/get_started_with_data_labeling.ipynb +++ b/notebooks/official/datasets/get_started_with_data_labeling.ipynb @@ -381,7 +381,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -583,7 +584,8 @@ "print(IMPORT_FILE)\n", "\n", "# printing content of uploaded file\n", - "! gcloud storage cat $IMPORT_FILE" ] + "! gcloud storage cat $IMPORT_FILE" + ] }, { "cell_type": "markdown", @@ -705,7 +707,8 @@ "outputs": [], "source": [ "# create placeholder file for instructions for data labeling\n", - "! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" ] + "! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" + ] }, { "cell_type": "code", @@ -861,7 +864,8 @@ "\n", "# Delete the bucket created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb b/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb index c0f29eba2..e758c113d 100644 --- a/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb @@ -291,7 +291,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -371,11 +372,14 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $IMPORT_FILE | head\n", "\n", - "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", + "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -836,7 +840,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb b/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb index a803a0f89..95463bc4c 100644 --- a/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb +++ b/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb @@ -217,7 +217,8 @@ "source": [ "# Create GCS Bucket\n", "BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n", - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -308,7 +309,8 @@ "IMAGE_URI = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-12:latest\"\n", "\n", "# Copy Test Models to the Bucket\n", - "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n", "\n", + "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n", + "\n", "model = aiplatform.Model.upload(\n", " display_name=DISPLAY_NAME,\n", " artifact_uri=ARTIFACT_URI,\n", @@ -784,7 +786,8 @@ }, "outputs": [], "source": [ - "! gcloud storage rm --recursive {BUCKET_URI}" ] + "! gcloud storage rm --recursive {BUCKET_URI}" + ] }, { "cell_type": "markdown", diff --git a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb index 674d507fb..4e25193ac 100644 --- a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb +++ b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb @@ -347,7 +347,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -719,8 +720,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -837,7 +840,8 @@ "# Delete GCS bucket.\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb b/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb index 27aa1c425..58d10fb24 100644 --- a/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb +++ b/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb @@ -350,7 +350,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -845,7 +846,8 @@ "# Delete GCS bucket.\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive {BUCKET_URI}" ] + " ! gcloud storage rm --recursive {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb index 85af168ee..5ae6f0f2a 100644 --- a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb +++ b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb @@ -282,7 +282,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -839,7 +840,8 @@ "\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From c7c9ad33c53d0f8e9c34487fc2b01ffa67782a02 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 12 Dec 2025 09:02:15 +0000 Subject: [PATCH 4/9] Revert "linter changes" This reverts commit 6665133b2cf0f16ce2e9be703335f6eb10d6b40f. --- .../bq_dataframes_llm_code_generation.ipynb | 6 ++---- .../model_garden_jax_vision_transformer.ipynb | 16 +++++++--------- .../model_garden/model_garden_keras_yolov8.ipynb | 2 +- ...model_garden_mediapipe_object_detection.ipynb | 2 +- .../get_started_with_data_labeling.ipynb | 12 ++++-------- ...l_tabular_classification_online_explain.ipynb | 15 +++++---------- .../get_started_with_dedicated_endpoint.ipynb | 9 +++------ ...d_custom_training_with_custom_container.ipynb | 12 ++++-------- ...orboard_vertex_ai_pipelines_integration.ipynb | 6 ++---- ...ication-model-for-ads-targeting-usecase.ipynb | 6 ++---- 10 files changed, 31 insertions(+), 55 deletions(-) diff --git a/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb b/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb index 71e5babd8..f85c19c4c 100644 --- a/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb +++ b/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb @@ -761,8 +761,7 @@ "\n", "BUCKET_ID = \"code-samples-\" + str(uuid.uuid1())\n", "\n", - "!gcloud storage buckets create gs://{BUCKET_ID}" - ] + "!gcloud storage buckets create gs://{BUCKET_ID}" ] }, { "cell_type": "markdown", @@ -874,8 +873,7 @@ "outputs": [], "source": [ "# # Delete the Google Cloud Storage bucket and files\n", - "# ! gcloud storage rm --recursive gs://{BUCKET_ID}\n", - "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" + "# ! gcloud storage rm --recursive gs://{BUCKET_ID}\n", "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" ] } ], diff --git a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb index b8a7d6401..93c1e2344 100644 --- a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb @@ -353,8 +353,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -582,9 +581,7 @@ "source": [ "# Move Flower data from local directory to Cloud Storage.\n", "# This step takes around 2 mins to finish.\n", - "! gcloud storage cp --recursive $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", - "! gcloud storage cp --recursive $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" - ] + "! gcloud storage cp --recursive $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", "! gcloud storage cp --recursive $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" ] }, { "cell_type": "markdown", @@ -785,7 +782,7 @@ " serving_container_args=[],\n", " location=REGION,\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/vit-jax\",\n", + " model_garden_source_model_name=\"publishers/google/models/vit-jax\"\n", ")\n", "\n", "jax_vit_endpoint = jax_vit_model.deploy(\n", @@ -796,7 +793,9 @@ " accelerator_count=1,\n", " min_replica_count=1,\n", " max_replica_count=1,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_jax_vision_transformer.ipynb\"},\n", + " system_labels={\n", + " \"NOTEBOOK_NAME\": \"model_garden_jax_vision_transformer.ipynb\"\n", + " },\n", ")" ] }, @@ -865,8 +864,7 @@ "# Delete Cloud Storage objects that were created.\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb index ccf7152ca..b15c71a07 100644 --- a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb @@ -242,7 +242,7 @@ " local_metrics_path = os.path.join(temp_dir.name, label_map_yaml_filename)\n", "\n", " ! gcloud storage cp $label_map_yaml_filepath $local_metrics_path\n", - " with open(local_metrics_path) as input_file:\n", + " with open(local_metrics_path, \"r\") as input_file:\n", " label_map = yaml.safe_load(input_file.read())[\"label_map\"]\n", " temp_dir.cleanup()\n", " return label_map\n", diff --git a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb index 84253879c..d8c49cb83 100644 --- a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb +++ b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb @@ -501,7 +501,7 @@ " [\"gcloud\", \"storage\", \"cp\", evaluation_result_path, eval_result_filename],\n", " stderr=subprocess.STDOUT,\n", " )\n", - " with open(eval_result_filename) as input_file:\n", + " with open(eval_result_filename, \"r\") as input_file:\n", " evalutation_result = json.loads(input_file.read())\n", " return evalutation_result[\"loss\"], evalutation_result[\"coco_metrics\"]\n", " except:\n", diff --git a/notebooks/official/datasets/get_started_with_data_labeling.ipynb b/notebooks/official/datasets/get_started_with_data_labeling.ipynb index d2846e3c4..1837267c6 100644 --- a/notebooks/official/datasets/get_started_with_data_labeling.ipynb +++ b/notebooks/official/datasets/get_started_with_data_labeling.ipynb @@ -381,8 +381,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -584,8 +583,7 @@ "print(IMPORT_FILE)\n", "\n", "# printing content of uploaded file\n", - "! gcloud storage cat $IMPORT_FILE" - ] + "! gcloud storage cat $IMPORT_FILE" ] }, { "cell_type": "markdown", @@ -707,8 +705,7 @@ "outputs": [], "source": [ "# create placeholder file for instructions for data labeling\n", - "! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" - ] + "! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" ] }, { "cell_type": "code", @@ -864,8 +861,7 @@ "\n", "# Delete the bucket created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb b/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb index e758c113d..c0f29eba2 100644 --- a/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb @@ -291,8 +291,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -372,14 +371,11 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $IMPORT_FILE | head\n", - "\n", - "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", - "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -840,8 +836,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb b/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb index 95463bc4c..a803a0f89 100644 --- a/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb +++ b/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb @@ -217,8 +217,7 @@ "source": [ "# Create GCS Bucket\n", "BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n", - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -309,8 +308,7 @@ "IMAGE_URI = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-12:latest\"\n", "\n", "# Copy Test Models to the Bucket\n", - "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n", - "\n", + "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n", "\n", "model = aiplatform.Model.upload(\n", " display_name=DISPLAY_NAME,\n", " artifact_uri=ARTIFACT_URI,\n", @@ -786,8 +784,7 @@ }, "outputs": [], "source": [ - "! gcloud storage rm --recursive {BUCKET_URI}" - ] + "! gcloud storage rm --recursive {BUCKET_URI}" ] }, { "cell_type": "markdown", diff --git a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb index 4e25193ac..674d507fb 100644 --- a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb +++ b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb @@ -347,8 +347,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -720,10 +719,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", - "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" - ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -840,8 +837,7 @@ "# Delete GCS bucket.\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb b/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb index 58d10fb24..27aa1c425 100644 --- a/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb +++ b/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb @@ -350,8 +350,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -846,8 +845,7 @@ "# Delete GCS bucket.\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb index 5ae6f0f2a..85af168ee 100644 --- a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb +++ b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb @@ -282,8 +282,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -840,8 +839,7 @@ "\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From a9d1b82d4c0743e32e3531fabfb689906dab1764 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 16 Dec 2025 09:49:22 +0000 Subject: [PATCH 5/9] Apply automated linter fixes --- .../bq_dataframes_llm_code_generation.ipynb | 6 ++++-- .../model_garden_jax_vision_transformer.ipynb | 16 +++++++++------- .../model_garden/model_garden_keras_yolov8.ipynb | 2 +- ...model_garden_mediapipe_object_detection.ipynb | 2 +- .../get_started_with_data_labeling.ipynb | 12 ++++++++---- ...l_tabular_classification_online_explain.ipynb | 15 ++++++++++----- .../get_started_with_dedicated_endpoint.ipynb | 9 ++++++--- ...d_custom_training_with_custom_container.ipynb | 12 ++++++++---- ...orboard_vertex_ai_pipelines_integration.ipynb | 6 ++++-- ...ication-model-for-ads-targeting-usecase.ipynb | 6 ++++-- 10 files changed, 55 insertions(+), 31 deletions(-) diff --git a/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb b/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb index f85c19c4c..71e5babd8 100644 --- a/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb +++ b/notebooks/community/generative_ai/bq_dataframes_llm_code_generation.ipynb @@ -761,7 +761,8 @@ "\n", "BUCKET_ID = \"code-samples-\" + str(uuid.uuid1())\n", "\n", - "!gcloud storage buckets create gs://{BUCKET_ID}" ] + "!gcloud storage buckets create gs://{BUCKET_ID}" + ] }, { "cell_type": "markdown", @@ -873,7 +874,8 @@ "outputs": [], "source": [ "# # Delete the Google Cloud Storage bucket and files\n", - "# ! gcloud storage rm --recursive gs://{BUCKET_ID}\n", "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" + "# ! gcloud storage rm --recursive gs://{BUCKET_ID}\n", + "# print(f\"Deleted bucket '{BUCKET_ID}'.\")" ] } ], diff --git a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb index 93c1e2344..b8a7d6401 100644 --- a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb @@ -353,7 +353,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -581,7 +582,9 @@ "source": [ "# Move Flower data from local directory to Cloud Storage.\n", "# This step takes around 2 mins to finish.\n", - "! gcloud storage cp --recursive $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", "! gcloud storage cp --recursive $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" ] + "! gcloud storage cp --recursive $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", + "! gcloud storage cp --recursive $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" + ] }, { "cell_type": "markdown", @@ -782,7 +785,7 @@ " serving_container_args=[],\n", " location=REGION,\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/vit-jax\"\n", + " model_garden_source_model_name=\"publishers/google/models/vit-jax\",\n", ")\n", "\n", "jax_vit_endpoint = jax_vit_model.deploy(\n", @@ -793,9 +796,7 @@ " accelerator_count=1,\n", " min_replica_count=1,\n", " max_replica_count=1,\n", - " system_labels={\n", - " \"NOTEBOOK_NAME\": \"model_garden_jax_vision_transformer.ipynb\"\n", - " },\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_jax_vision_transformer.ipynb\"},\n", ")" ] }, @@ -864,7 +865,8 @@ "# Delete Cloud Storage objects that were created.\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb index b15c71a07..ccf7152ca 100644 --- a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb @@ -242,7 +242,7 @@ " local_metrics_path = os.path.join(temp_dir.name, label_map_yaml_filename)\n", "\n", " ! gcloud storage cp $label_map_yaml_filepath $local_metrics_path\n", - " with open(local_metrics_path, \"r\") as input_file:\n", + " with open(local_metrics_path) as input_file:\n", " label_map = yaml.safe_load(input_file.read())[\"label_map\"]\n", " temp_dir.cleanup()\n", " return label_map\n", diff --git a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb index d8c49cb83..84253879c 100644 --- a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb +++ b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb @@ -501,7 +501,7 @@ " [\"gcloud\", \"storage\", \"cp\", evaluation_result_path, eval_result_filename],\n", " stderr=subprocess.STDOUT,\n", " )\n", - " with open(eval_result_filename, \"r\") as input_file:\n", + " with open(eval_result_filename) as input_file:\n", " evalutation_result = json.loads(input_file.read())\n", " return evalutation_result[\"loss\"], evalutation_result[\"coco_metrics\"]\n", " except:\n", diff --git a/notebooks/official/datasets/get_started_with_data_labeling.ipynb b/notebooks/official/datasets/get_started_with_data_labeling.ipynb index 1837267c6..d2846e3c4 100644 --- a/notebooks/official/datasets/get_started_with_data_labeling.ipynb +++ b/notebooks/official/datasets/get_started_with_data_labeling.ipynb @@ -381,7 +381,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -583,7 +584,8 @@ "print(IMPORT_FILE)\n", "\n", "# printing content of uploaded file\n", - "! gcloud storage cat $IMPORT_FILE" ] + "! gcloud storage cat $IMPORT_FILE" + ] }, { "cell_type": "markdown", @@ -705,7 +707,8 @@ "outputs": [], "source": [ "# create placeholder file for instructions for data labeling\n", - "! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" ] + "! echo \"this is instruction\" >> instruction.txt | gcloud storage cp instruction.txt $BUCKET_URI" + ] }, { "cell_type": "code", @@ -861,7 +864,8 @@ "\n", "# Delete the bucket created\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb b/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb index c0f29eba2..e758c113d 100644 --- a/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_automl_tabular_classification_online_explain.ipynb @@ -291,7 +291,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -371,11 +372,14 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $IMPORT_FILE | head\n", "\n", - "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", + "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -836,7 +840,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb b/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb index a803a0f89..95463bc4c 100644 --- a/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb +++ b/notebooks/official/prediction/get_started_with_dedicated_endpoint.ipynb @@ -217,7 +217,8 @@ "source": [ "# Create GCS Bucket\n", "BUCKET_URI = \"gs://your-bucket-name-unique\" # @param {type:\"string\"}\n", - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -308,7 +309,8 @@ "IMAGE_URI = \"us-docker.pkg.dev/vertex-ai/prediction/tf2-cpu.2-12:latest\"\n", "\n", "# Copy Test Models to the Bucket\n", - "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n", "\n", + "! gcloud storage cp --recursive \"gs://cloud-samples-data/vertex-ai/prediction/test-models-requests/tensorflow/*\" {ARTIFACT_URI}\n", + "\n", "model = aiplatform.Model.upload(\n", " display_name=DISPLAY_NAME,\n", " artifact_uri=ARTIFACT_URI,\n", @@ -784,7 +786,8 @@ }, "outputs": [], "source": [ - "! gcloud storage rm --recursive {BUCKET_URI}" ] + "! gcloud storage rm --recursive {BUCKET_URI}" + ] }, { "cell_type": "markdown", diff --git a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb index 674d507fb..4e25193ac 100644 --- a/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb +++ b/notebooks/official/tensorboard/tensorboard_custom_training_with_custom_container.ipynb @@ -347,7 +347,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {LOCATION} --project {PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -719,8 +720,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -837,7 +840,8 @@ "# Delete GCS bucket.\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb b/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb index 27aa1c425..58d10fb24 100644 --- a/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb +++ b/notebooks/official/tensorboard/tensorboard_vertex_ai_pipelines_integration.ipynb @@ -350,7 +350,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -845,7 +846,8 @@ "# Delete GCS bucket.\n", "delete_bucket = False # set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive {BUCKET_URI}" ] + " ! gcloud storage rm --recursive {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb index 85af168ee..5ae6f0f2a 100644 --- a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb +++ b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb @@ -282,7 +282,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$LOCATION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -839,7 +840,8 @@ "\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From d1e833632c841d12485d468a53d9a745440b1629 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 18 Dec 2025 13:17:40 +0530 Subject: [PATCH 6/9] Update training-multi-class-classification-model-for-ads-targeting-usecase.ipynb --- ...i-class-classification-model-for-ads-targeting-usecase.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb index 5ae6f0f2a..e9c68107c 100644 --- a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb +++ b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb @@ -836,7 +836,7 @@ }, "outputs": [], "source": [ - "# Delete the Cloud Storage bucket\n", + "# Delete the Cloud Storage bucket.\n", "\n", "delete_bucket = False\n", "if delete_bucket:\n", From ff62c50a07caf4a18694ec1623f0f893925b52ae Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Thu, 18 Dec 2025 13:18:38 +0530 Subject: [PATCH 7/9] Update training-multi-class-classification-model-for-ads-targeting-usecase.ipynb --- ...i-class-classification-model-for-ads-targeting-usecase.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb index e9c68107c..5ae6f0f2a 100644 --- a/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb +++ b/notebooks/official/workbench/ads_targetting/training-multi-class-classification-model-for-ads-targeting-usecase.ipynb @@ -836,7 +836,7 @@ }, "outputs": [], "source": [ - "# Delete the Cloud Storage bucket.\n", + "# Delete the Cloud Storage bucket\n", "\n", "delete_bucket = False\n", "if delete_bucket:\n", From 36a0a35a984daf5afb39390ad4443c295ec903b2 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 08:43:09 +0000 Subject: [PATCH 8/9] removed model_garden folder changes --- .../model_garden_jax_vision_transformer.ipynb | 14 ++++++++------ .../model_garden/model_garden_keras_yolov8.ipynb | 12 ++++++------ .../model_garden_mediapipe_object_detection.ipynb | 14 +++++++------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb index b8a7d6401..e395180a3 100644 --- a/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_vision_transformer.ipynb @@ -353,7 +353,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={REGION} --project={PROJECT_ID} {BUCKET_URI}" + "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}" ] }, { @@ -582,8 +582,8 @@ "source": [ "# Move Flower data from local directory to Cloud Storage.\n", "# This step takes around 2 mins to finish.\n", - "! gcloud storage cp --recursive $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", - "! gcloud storage cp --recursive $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" + "! gsutil -m cp -R $local_flower_data_directory/train/* $FLOWER_DATA_GCS_PATH/train/\n", + "! gsutil -m cp -R $local_flower_data_directory/test/* $FLOWER_DATA_GCS_PATH/test/" ] }, { @@ -785,7 +785,7 @@ " serving_container_args=[],\n", " location=REGION,\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/vit-jax\",\n", + " model_garden_source_model_name=\"publishers/google/models/vit-jax\"\n", ")\n", "\n", "jax_vit_endpoint = jax_vit_model.deploy(\n", @@ -796,7 +796,9 @@ " accelerator_count=1,\n", " min_replica_count=1,\n", " max_replica_count=1,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_jax_vision_transformer.ipynb\"},\n", + " system_labels={\n", + " \"NOTEBOOK_NAME\": \"model_garden_jax_vision_transformer.ipynb\"\n", + " },\n", ")" ] }, @@ -865,7 +867,7 @@ "# Delete Cloud Storage objects that were created.\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" + " ! gsutil -m rm -r $BUCKET_URI" ] } ], diff --git a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb index ccf7152ca..a1d8bad58 100644 --- a/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb +++ b/notebooks/community/model_garden/model_garden_keras_yolov8.ipynb @@ -175,10 +175,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -203,7 +203,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -241,8 +241,8 @@ " label_map_yaml_filename = os.path.basename(label_map_yaml_filepath)\n", " local_metrics_path = os.path.join(temp_dir.name, label_map_yaml_filename)\n", "\n", - " ! gcloud storage cp $label_map_yaml_filepath $local_metrics_path\n", - " with open(local_metrics_path) as input_file:\n", + " ! gsutil cp $label_map_yaml_filepath $local_metrics_path\n", + " with open(local_metrics_path, \"r\") as input_file:\n", " label_map = yaml.safe_load(input_file.read())[\"label_map\"]\n", " temp_dir.cleanup()\n", " return label_map\n", @@ -624,7 +624,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] }, { diff --git a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb index 84253879c..54b7a8398 100644 --- a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb +++ b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb @@ -157,10 +157,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location={REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -185,7 +185,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -498,10 +498,10 @@ " try:\n", " eval_result_filename = os.path.basename(evaluation_result_path)\n", " subprocess.check_output(\n", - " [\"gcloud\", \"storage\", \"cp\", evaluation_result_path, eval_result_filename],\n", + " [\"gsutil\", \"cp\", evaluation_result_path, eval_result_filename],\n", " stderr=subprocess.STDOUT,\n", " )\n", - " with open(eval_result_filename) as input_file:\n", + " with open(eval_result_filename, \"r\") as input_file:\n", " evalutation_result = json.loads(input_file.read())\n", " return evalutation_result[\"loss\"], evalutation_result[\"coco_metrics\"]\n", " except:\n", @@ -531,7 +531,7 @@ "\n", "# @markdown After fine-tuning and evaluating the model, you can save it as Tensorflow Lite model, try it out in the [Object Detector](https://mediapipe-studio.webapps.google.com/demo/object_detector) demo in MediaPipe Studio or integrate it with your application by following the [Object detection task guide](https://developers.google.com/mediapipe/solutions/vision/object_detector). The exported model also includes metadata and the label map.\n", "\n", - "! gcloud storage cp $EXPORTED_MODEL_OUTPUT_FILE \"object_detection_model.tflite\"" + "! gsutil cp $EXPORTED_MODEL_OUTPUT_FILE \"object_detection_model.tflite\"" ] }, { @@ -571,7 +571,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], From f67452498dae18ac2aa1141f8480d248044bd69b Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 14:15:59 +0530 Subject: [PATCH 9/9] Update model_garden_mediapipe_object_detection.ipynb --- .../model_garden/model_garden_mediapipe_object_detection.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb index 54b7a8398..7a311398e 100644 --- a/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb +++ b/notebooks/community/model_garden/model_garden_mediapipe_object_detection.ipynb @@ -160,7 +160,7 @@ " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n",