From 8e280af8e05a9371545bf4c74b4aa8ee80ffb47c Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Fri, 17 Oct 2025 12:22:39 +0000 Subject: [PATCH 1/6] Migrate gsutil usage to gcloud storage --- ...oml_image_classification_export_edge.ipynb | 24 ++++++------------ ...l_image_object_detection_export_edge.ipynb | 24 ++++++------------ .../get_started_with_kubeflow_pipelines.ipynb | 25 +++++++------------ ...rted_with_custom_tabular_model_batch.ipynb | 17 +++++-------- ...t_started_with_xai_and_custom_server.ipynb | 9 +++---- ..._tabular_regression_model_evaluation.ipynb | 20 ++++++--------- .../model_garden/model_garden_jax_fvlm.ipynb | 20 ++++----------- ...tom_tabular_regression_batch_explain.ipynb | 12 +++------ ...gression_online_explain_get_metadata.ipynb | 9 +++---- ...lama2_pytorch_finetuning_and_serving.ipynb | 9 +++---- 10 files changed, 57 insertions(+), 112 deletions(-) diff --git a/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb b/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb index 582a7ea5f..69ccd7006 100644 --- a/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb +++ b/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb @@ -416,8 +416,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -436,8 +435,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -784,12 +782,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -1244,10 +1240,8 @@ }, "outputs": [], "source": [ - "! gsutil ls $model_package\n", - "# Download the model artifacts\n", - "! gsutil cp -r $model_package tflite\n", - "\n", + "! gcloud storage ls $model_package\n", "# Download the model artifacts\n", + "! gcloud storage cp --recursive $model_package tflite\n", "\n", "tflite_path = \"tflite/model.tflite\"" ] }, @@ -1305,8 +1299,7 @@ }, "outputs": [], "source": [ - "test_items = ! gsutil cat $IMPORT_FILE | head -n1\n", - "test_item = test_items[0].split(\",\")[0]\n", + "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item = test_items[0].split(\",\")[0]\n", "\n", "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n", " content = f.read()\n", @@ -1449,8 +1442,7 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb b/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb index 2c3a7e11d..01c893ef5 100644 --- a/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb +++ b/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb @@ -416,8 +416,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -436,8 +435,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -785,12 +783,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -1247,10 +1243,8 @@ }, "outputs": [], "source": [ - "! gsutil ls $model_package\n", - "# Download the model artifacts\n", - "! gsutil cp -r $model_package tflite\n", - "\n", + "! gcloud storage ls $model_package\n", "# Download the model artifacts\n", + "! gcloud storage cp --recursive $model_package tflite\n", "\n", "tflite_path = \"tflite/model.tflite\"" ] }, @@ -1308,8 +1302,7 @@ }, "outputs": [], "source": [ - "test_items = ! gsutil cat $IMPORT_FILE | head -n1\n", - "test_item = test_items[0].split(\",\")[0]\n", + "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item = test_items[0].split(\",\")[0]\n", "\n", "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n", " content = f.read()\n", @@ -1452,8 +1445,7 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb index 244a59995..63e1d2cc3 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb @@ -430,8 +430,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -450,8 +449,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -520,10 +518,9 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member="serviceAccount:{SERVICE_ACCOUNT}" --role="roles/storage.objectViewer" ] }, { "cell_type": "markdown", @@ -807,14 +804,11 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gsutil cat $EXECUTE_OUTPUT\n", - " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gsutil cat $GCP_RESOURCES\n", - " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gsutil cat $EVAL_METRICS\n", - " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1587,8 +1581,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb index 3869f7d86..1ac7bf556 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb @@ -464,8 +464,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -484,8 +483,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -892,8 +890,7 @@ " f.write(json.dumps(instance) + \"\\n\")\n", "\n", "# Upload to Cloud Storage bucket\n", - "! gsutil cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", - "\n", + "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", "\n", "print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)" ] }, @@ -1108,7 +1105,7 @@ "outputs": [], "source": [ "data_file = \"gs://cloud-samples-data/ai-platform/iris/iris_data.csv\"\n", - "! gsutil cat $data_file | head -n 10 > test.csv\n", + "! gcloud storage cat $data_file | head -n 10 > test.csv\n", "\n", "! cat test.csv\n", "\n", @@ -1118,8 +1115,7 @@ " BUCKET_URI + \"/batch_prediction_instances/\" + BATCH_PREDICTION_INSTANCES_FILE\n", ")\n", "\n", - "! gsutil cp test.csv $BATCH_PREDICTION_GCS_SOURCE" - ] + "! gcloud storage cp test.csv $BATCH_PREDICTION_GCS_SOURCE" ] }, { "cell_type": "markdown", @@ -1497,8 +1493,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb index 377b45369..aa78dd9bf 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb @@ -504,8 +504,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -524,8 +523,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -1734,8 +1732,7 @@ "! rm -rf app\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb b/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb index 7f70415bc..1a81d3490 100644 --- a/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb +++ b/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb @@ -537,8 +537,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION -p $PROJECT_ID $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -557,8 +556,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -627,10 +625,10 @@ }, "outputs": [], "source": [ - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectCreator $BUCKET_URI\n", - "\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.objectViewer $BUCKET_URI" - ] + "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", + # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. + ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer ] }, { "cell_type": "markdown", @@ -1383,8 +1381,7 @@ "outputs": [], "source": [ "# Load the results\n", - "attributions = !gsutil cat $feat_attrs_gcs_uri\n", - "\n", + "attributions = !gcloud storage cat $feat_attrs_gcs_uri\n", "\n", "# Print the results obtained\n", "attributions = json.loads(attributions[0])\n", "print(attributions)" @@ -1463,8 +1460,7 @@ "# Delete Cloud Storage objects\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb index 489920754..96fd629b0 100644 --- a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb @@ -441,8 +441,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {REGION} --project {PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -965,11 +964,7 @@ }, "outputs": [], "source": [ - "! gsutil -m rm -R -f {GCS_TFRECORD_PATH}\n", - "! gsutil -m cp -R {LOCAL_TFRECORD_DIR} {GCS_TFRECORD_PATH}\n", - "! gsutil -m cp {VAL_TOY_ANNOTATION_PATH} {GCS_VAL_ANNOTATION_FILE}\n", - "! gsutil -m cp {LOCAL_CHECKPOINT_FILE} {GCS_CHECKPOINT_DIR}/checkpoint_0" - ] + "! gcloud storage rm --recursive --continue-on-error {GCS_TFRECORD_PATH}\n", "! gcloud storage cp --recursive {LOCAL_TFRECORD_DIR} {GCS_TFRECORD_PATH}\n", "! gcloud storage cp {VAL_TOY_ANNOTATION_PATH} {GCS_VAL_ANNOTATION_FILE}\n", "! gcloud storage cp {LOCAL_CHECKPOINT_FILE} {GCS_CHECKPOINT_DIR}/checkpoint_0" ] }, { "cell_type": "markdown", @@ -988,8 +983,7 @@ }, "outputs": [], "source": [ - "! gsutil -m cp {LOCAL_CATEGORY_EMBEDDING_PATH} {GCS_CATEGORY_EMBEDDING_PATH}" - ] + "! gcloud storage cp {LOCAL_CATEGORY_EMBEDDING_PATH} {GCS_CATEGORY_EMBEDDING_PATH}" ] }, { "cell_type": "markdown", @@ -1286,10 +1280,7 @@ }, "outputs": [], "source": [ - "! gsutil -m rm -R -f {GCS_CONVERTED_SAVED_MODEL_DIR}\n", - "! gsutil -m cp -R {LOCAL_CONVERTED_SAVED_MODEL_DIR} {GCS_CONVERTED_SAVED_MODEL_DIR}\n", - "! gsutil ls {GCS_CONVERTED_SAVED_MODEL_DIR}" - ] + "! gcloud storage rm --recursive --continue-on-error {GCS_CONVERTED_SAVED_MODEL_DIR}\n", "! gcloud storage cp --recursive {LOCAL_CONVERTED_SAVED_MODEL_DIR} {GCS_CONVERTED_SAVED_MODEL_DIR}\n", "! gcloud storage ls {GCS_CONVERTED_SAVED_MODEL_DIR}" ] }, { "cell_type": "markdown", @@ -1521,8 +1512,7 @@ "# Delete Cloud Storage objects that were created.\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb index a85300e3f..cc89c57ec 100644 --- a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb @@ -288,8 +288,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -707,8 +706,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" ] }, { "cell_type": "markdown", @@ -1260,8 +1258,7 @@ "\n", "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n", "\n", - "! gsutil cp batch.csv $gcs_input_uri" - ] + "! gcloud storage cp batch.csv $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -1398,8 +1395,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "# Remove the locally generated files\n", "! rm -rf batch.csv custom.tar.gz custom" ] diff --git a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb index a14da893f..29ec66640 100644 --- a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb +++ b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb @@ -293,8 +293,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -706,8 +705,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" ] }, { "cell_type": "markdown", @@ -1444,8 +1442,7 @@ "\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gsutil rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb b/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb index 4a894fa66..ef839020d 100644 --- a/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb +++ b/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb @@ -279,8 +279,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} -p {PROJECT_ID} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -1264,8 +1263,7 @@ }, "outputs": [], "source": [ - "!gsutil ls $GCS_MODEL_PATH" - ] + "!gcloud storage ls $GCS_MODEL_PATH" ] }, { "cell_type": "markdown", @@ -1517,8 +1515,7 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil -m rm -r $BUCKET_URI" - ] + " ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { From 4f08882d6dcbf163b72965f6b94c4677dd6e40b9 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Fri, 12 Dec 2025 17:28:44 +0000 Subject: [PATCH 2/6] PR changes for 4328 --- .../ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb | 4 ++-- .../automl_tabular_regression_model_evaluation.ipynb | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb index 63e1d2cc3..c9bd0c78b 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb @@ -519,8 +519,8 @@ "outputs": [], "source": [ "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. - ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member="serviceAccount:{SERVICE_ACCOUNT}" --role="roles/storage.objectViewer" ] + + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", diff --git a/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb b/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb index 1a81d3490..7c78feefd 100644 --- a/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb +++ b/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb @@ -625,10 +625,8 @@ }, "outputs": [], "source": [ - "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop. - ! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] }, { "cell_type": "markdown", From 60851fdec262ea78b9c6f6a39c2c7d8a0568cfe5 Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Wed, 17 Dec 2025 16:11:53 +0000 Subject: [PATCH 3/6] Fix Linter issue for 4328 --- ...oml_image_classification_export_edge.ipynb | 26 ++++++++++------- ...l_image_object_detection_export_edge.ipynb | 29 +++++++++++-------- .../get_started_with_kubeflow_pipelines.ipynb | 28 ++++++++++-------- ...rted_with_custom_tabular_model_batch.ipynb | 15 ++++++---- ...t_started_with_xai_and_custom_server.ipynb | 9 ++++-- ..._tabular_regression_model_evaluation.ipynb | 18 ++++++++---- .../model_garden/model_garden_jax_fvlm.ipynb | 26 +++++++++++------ ...tom_tabular_regression_batch_explain.ipynb | 12 +++++--- ...gression_online_explain_get_metadata.ipynb | 9 ++++-- ...lama2_pytorch_finetuning_and_serving.ipynb | 9 ++++-- 10 files changed, 114 insertions(+), 67 deletions(-) diff --git a/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb b/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb index 69ccd7006..dfccf184d 100644 --- a/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb +++ b/notebooks/community/gapic/automl/showcase_automl_image_classification_export_edge.ipynb @@ -416,7 +416,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -435,7 +436,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -472,7 +474,6 @@ "\n", "from google.cloud.aiplatform import gapic as aip\n", "from google.protobuf import json_format\n", - "from google.protobuf.json_format import MessageToJson, ParseDict\n", "from google.protobuf.struct_pb2 import Struct, Value" ] }, @@ -782,10 +783,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -1211,7 +1214,6 @@ " }\n", " response = clients[\"model\"].export_model(name=name, output_config=output_config)\n", " print(\"Long running operation:\", response.operation.name)\n", - " result = response.result(timeout=1800)\n", " metadata = response.operation.metadata\n", " artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n", " print(\"Artifact Uri\", artifact_uri)\n", @@ -1240,8 +1242,10 @@ }, "outputs": [], "source": [ - "! gcloud storage ls $model_package\n", "# Download the model artifacts\n", - "! gcloud storage cp --recursive $model_package tflite\n", "\n", + "! gcloud storage ls $model_package\n", + "# Download the model artifacts\n", + "! gcloud storage cp --recursive $model_package tflite\n", + "\n", "tflite_path = \"tflite/model.tflite\"" ] }, @@ -1299,7 +1303,8 @@ }, "outputs": [], "source": [ - "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item = test_items[0].split(\",\")[0]\n", + "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "test_item = test_items[0].split(\",\")[0]\n", "\n", "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n", " content = f.read()\n", @@ -1442,7 +1447,8 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb b/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb index 01c893ef5..4a06d69ea 100644 --- a/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb +++ b/notebooks/community/gapic/automl/showcase_automl_image_object_detection_export_edge.ipynb @@ -416,7 +416,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --location $REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -435,7 +436,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -471,9 +473,7 @@ "import time\n", "\n", "from google.cloud.aiplatform import gapic as aip\n", - "from google.protobuf import json_format\n", - "from google.protobuf.json_format import MessageToJson, ParseDict\n", - "from google.protobuf.struct_pb2 import Struct, Value" + "from google.protobuf import json_format\n" ] }, { @@ -783,10 +783,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -1214,7 +1216,6 @@ " }\n", " response = clients[\"model\"].export_model(name=name, output_config=output_config)\n", " print(\"Long running operation:\", response.operation.name)\n", - " result = response.result(timeout=1800)\n", " metadata = response.operation.metadata\n", " artifact_uri = str(metadata.value).split(\"\\\\\")[-1][4:-1]\n", " print(\"Artifact Uri\", artifact_uri)\n", @@ -1243,8 +1244,10 @@ }, "outputs": [], "source": [ - "! gcloud storage ls $model_package\n", "# Download the model artifacts\n", - "! gcloud storage cp --recursive $model_package tflite\n", "\n", + "! gcloud storage ls $model_package\n", + "# Download the model artifacts\n", + "! gcloud storage cp --recursive $model_package tflite\n", + "\n", "tflite_path = \"tflite/model.tflite\"" ] }, @@ -1302,7 +1305,8 @@ }, "outputs": [], "source": [ - "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "test_item = test_items[0].split(\",\")[0]\n", + "test_items = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "test_item = test_items[0].split(\",\")[0]\n", "\n", "with tf.io.gfile.GFile(test_item, \"rb\") as f:\n", " content = f.read()\n", @@ -1445,7 +1449,8 @@ " print(e)\n", "\n", "if delete_bucket and \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb b/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb index c9bd0c78b..51f640e58 100644 --- a/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb +++ b/notebooks/community/ml_ops/stage3/get_started_with_kubeflow_pipelines.ipynb @@ -430,7 +430,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -449,7 +450,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -518,9 +520,9 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n","\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -804,11 +806,14 @@ " + \"/evaluation_metrics\"\n", " )\n", " if tf.io.gfile.exists(EXECUTE_OUTPUT):\n", - " ! gcloud storage cat $EXECUTE_OUTPUT\n", " return EXECUTE_OUTPUT\n", + " ! gcloud storage cat $EXECUTE_OUTPUT\n", + " return EXECUTE_OUTPUT\n", " elif tf.io.gfile.exists(GCP_RESOURCES):\n", - " ! gcloud storage cat $GCP_RESOURCES\n", " return GCP_RESOURCES\n", + " ! gcloud storage cat $GCP_RESOURCES\n", + " return GCP_RESOURCES\n", " elif tf.io.gfile.exists(EVAL_METRICS):\n", - " ! gcloud storage cat $EVAL_METRICS\n", " return EVAL_METRICS\n", + " ! gcloud storage cat $EVAL_METRICS\n", + " return EVAL_METRICS\n", "\n", " return None\n", "\n", @@ -1161,9 +1166,7 @@ "\n", "\n", "@component()\n", - "def multi_output(\n", - " text1: str, text2: str\n", - ") -> NamedTuple(\n", + "def multi_output(text1: str, text2: str) -> NamedTuple(\n", " \"Outputs\",\n", " [\n", " (\"output_1\", str), # Return parameters\n", @@ -1581,7 +1584,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb index 1ac7bf556..9e234689d 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_custom_tabular_model_batch.ipynb @@ -464,7 +464,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -483,7 +484,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -890,7 +892,8 @@ " f.write(json.dumps(instance) + \"\\n\")\n", "\n", "# Upload to Cloud Storage bucket\n", - "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", "\n", + "! gcloud storage cp $BATCH_PREDICTION_INSTANCES_FILE $BATCH_PREDICTION_GCS_SOURCE\n", + "\n", "print(\"Uploaded instances to: \", BATCH_PREDICTION_GCS_SOURCE)" ] }, @@ -1115,7 +1118,8 @@ " BUCKET_URI + \"/batch_prediction_instances/\" + BATCH_PREDICTION_INSTANCES_FILE\n", ")\n", "\n", - "! gcloud storage cp test.csv $BATCH_PREDICTION_GCS_SOURCE" ] + "! gcloud storage cp test.csv $BATCH_PREDICTION_GCS_SOURCE" + ] }, { "cell_type": "markdown", @@ -1493,7 +1497,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb index aa78dd9bf..94c499514 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_xai_and_custom_server.ipynb @@ -504,7 +504,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -523,7 +524,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -1732,7 +1734,8 @@ "! rm -rf app\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb b/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb index 7c78feefd..b527c843f 100644 --- a/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb +++ b/notebooks/community/model_evaluation/automl_tabular_regression_model_evaluation.ipynb @@ -537,7 +537,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION --project=$PROJECT_ID $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -556,7 +557,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -625,8 +627,10 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", "\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" ] + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectCreator\n", + "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_URI --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.objectViewer" + ] }, { "cell_type": "markdown", @@ -1379,7 +1383,8 @@ "outputs": [], "source": [ "# Load the results\n", - "attributions = !gcloud storage cat $feat_attrs_gcs_uri\n", "\n", + "attributions = !gcloud storage cat $feat_attrs_gcs_uri\n", + "\n", "# Print the results obtained\n", "attributions = json.loads(attributions[0])\n", "print(attributions)" @@ -1458,7 +1463,8 @@ "# Delete Cloud Storage objects\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb index 96fd629b0..eed4faeb5 100644 --- a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb @@ -441,7 +441,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {REGION} --project {PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location {REGION} --project {PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -964,7 +965,11 @@ }, "outputs": [], "source": [ - "! gcloud storage rm --recursive --continue-on-error {GCS_TFRECORD_PATH}\n", "! gcloud storage cp --recursive {LOCAL_TFRECORD_DIR} {GCS_TFRECORD_PATH}\n", "! gcloud storage cp {VAL_TOY_ANNOTATION_PATH} {GCS_VAL_ANNOTATION_FILE}\n", "! gcloud storage cp {LOCAL_CHECKPOINT_FILE} {GCS_CHECKPOINT_DIR}/checkpoint_0" ] + "! gcloud storage rm --recursive --continue-on-error {GCS_TFRECORD_PATH}\n", + "! gcloud storage cp --recursive {LOCAL_TFRECORD_DIR} {GCS_TFRECORD_PATH}\n", + "! gcloud storage cp {VAL_TOY_ANNOTATION_PATH} {GCS_VAL_ANNOTATION_FILE}\n", + "! gcloud storage cp {LOCAL_CHECKPOINT_FILE} {GCS_CHECKPOINT_DIR}/checkpoint_0" + ] }, { "cell_type": "markdown", @@ -983,7 +988,8 @@ }, "outputs": [], "source": [ - "! gcloud storage cp {LOCAL_CATEGORY_EMBEDDING_PATH} {GCS_CATEGORY_EMBEDDING_PATH}" ] + "! gcloud storage cp {LOCAL_CATEGORY_EMBEDDING_PATH} {GCS_CATEGORY_EMBEDDING_PATH}" + ] }, { "cell_type": "markdown", @@ -1280,7 +1286,10 @@ }, "outputs": [], "source": [ - "! gcloud storage rm --recursive --continue-on-error {GCS_CONVERTED_SAVED_MODEL_DIR}\n", "! gcloud storage cp --recursive {LOCAL_CONVERTED_SAVED_MODEL_DIR} {GCS_CONVERTED_SAVED_MODEL_DIR}\n", "! gcloud storage ls {GCS_CONVERTED_SAVED_MODEL_DIR}" ] + "! gcloud storage rm --recursive --continue-on-error {GCS_CONVERTED_SAVED_MODEL_DIR}\n", + "! gcloud storage cp --recursive {LOCAL_CONVERTED_SAVED_MODEL_DIR} {GCS_CONVERTED_SAVED_MODEL_DIR}\n", + "! gcloud storage ls {GCS_CONVERTED_SAVED_MODEL_DIR}" + ] }, { "cell_type": "markdown", @@ -1321,7 +1330,7 @@ " serving_container_args=[],\n", " location=REGION,\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/f-vlm-jax\"\n", + " model_garden_source_model_name=\"publishers/google/models/f-vlm-jax\",\n", ")\n", "\n", "jax_fvlm_endpoint = jax_fvlm_model.deploy(\n", @@ -1330,9 +1339,7 @@ " machine_type=\"n1-highmem-16\",\n", " min_replica_count=1,\n", " max_replica_count=1,\n", - " system_labels={\n", - " \"NOTEBOOK_NAME\": \"model_garden_jax_fvlm.ipynb\"\n", - " },\n", + " system_labels={\"NOTEBOOK_NAME\": \"model_garden_jax_fvlm.ipynb\"},\n", ")" ] }, @@ -1512,7 +1519,8 @@ "# Delete Cloud Storage objects that were created.\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb index cc89c57ec..6ad34ad51 100644 --- a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb +++ b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_batch_explain.ipynb @@ -288,7 +288,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -706,7 +707,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" + ] }, { "cell_type": "markdown", @@ -1258,7 +1260,8 @@ "\n", "gcs_input_uri = BUCKET_URI + \"/test.csv\"\n", "\n", - "! gcloud storage cp batch.csv $gcs_input_uri" ] + "! gcloud storage cp batch.csv $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -1395,7 +1398,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "# Remove the locally generated files\n", "! rm -rf batch.csv custom.tar.gz custom" ] diff --git a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb index 29ec66640..d4ebc1769 100644 --- a/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb +++ b/notebooks/official/explainable_ai/sdk_custom_tabular_regression_online_explain_get_metadata.ipynb @@ -293,7 +293,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -705,7 +706,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" + ] }, { "cell_type": "markdown", @@ -1442,7 +1444,8 @@ "\n", "delete_bucket = False # Set True for deletion\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { diff --git a/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb b/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb index ef839020d..ea328a9cb 100644 --- a/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb +++ b/notebooks/official/training/tpuv5e_llama2_pytorch_finetuning_and_serving.ipynb @@ -279,7 +279,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} --project={PROJECT_ID} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -1263,7 +1264,8 @@ }, "outputs": [], "source": [ - "!gcloud storage ls $GCS_MODEL_PATH" ] + "!gcloud storage ls $GCS_MODEL_PATH" + ] }, { "cell_type": "markdown", @@ -1515,7 +1517,8 @@ "# Delete Cloud Storage objects that were created\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" ] + " ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From b8386d37d856dbbac81644e8d8d2d07440c7c06b Mon Sep 17 00:00:00 2001 From: bhandarivijay Date: Mon, 22 Dec 2025 18:48:09 +0000 Subject: [PATCH 4/6] changes removed model garden --- .../model_garden/model_garden_jax_fvlm.ipynb | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb index eed4faeb5..898981aca 100644 --- a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb @@ -441,7 +441,7 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {REGION} --project {PROJECT_ID} {BUCKET_URI}" + "! gsutil mb -l {REGION} -p {PROJECT_ID} {BUCKET_URI}" ] }, { @@ -965,10 +965,10 @@ }, "outputs": [], "source": [ - "! gcloud storage rm --recursive --continue-on-error {GCS_TFRECORD_PATH}\n", - "! gcloud storage cp --recursive {LOCAL_TFRECORD_DIR} {GCS_TFRECORD_PATH}\n", - "! gcloud storage cp {VAL_TOY_ANNOTATION_PATH} {GCS_VAL_ANNOTATION_FILE}\n", - "! gcloud storage cp {LOCAL_CHECKPOINT_FILE} {GCS_CHECKPOINT_DIR}/checkpoint_0" + "! gsutil -m rm -R -f {GCS_TFRECORD_PATH}\n", + "! gsutil -m cp -R {LOCAL_TFRECORD_DIR} {GCS_TFRECORD_PATH}\n", + "! gsutil -m cp {VAL_TOY_ANNOTATION_PATH} {GCS_VAL_ANNOTATION_FILE}\n", + "! gsutil -m cp {LOCAL_CHECKPOINT_FILE} {GCS_CHECKPOINT_DIR}/checkpoint_0" ] }, { @@ -988,7 +988,7 @@ }, "outputs": [], "source": [ - "! gcloud storage cp {LOCAL_CATEGORY_EMBEDDING_PATH} {GCS_CATEGORY_EMBEDDING_PATH}" + "! gsutil -m cp {LOCAL_CATEGORY_EMBEDDING_PATH} {GCS_CATEGORY_EMBEDDING_PATH}" ] }, { @@ -1286,9 +1286,9 @@ }, "outputs": [], "source": [ - "! gcloud storage rm --recursive --continue-on-error {GCS_CONVERTED_SAVED_MODEL_DIR}\n", - "! gcloud storage cp --recursive {LOCAL_CONVERTED_SAVED_MODEL_DIR} {GCS_CONVERTED_SAVED_MODEL_DIR}\n", - "! gcloud storage ls {GCS_CONVERTED_SAVED_MODEL_DIR}" + "! gsutil -m rm -R -f {GCS_CONVERTED_SAVED_MODEL_DIR}\n", + "! gsutil -m cp -R {LOCAL_CONVERTED_SAVED_MODEL_DIR} {GCS_CONVERTED_SAVED_MODEL_DIR}\n", + "! gsutil ls {GCS_CONVERTED_SAVED_MODEL_DIR}" ] }, { @@ -1330,7 +1330,7 @@ " serving_container_args=[],\n", " location=REGION,\n", " serving_container_environment_variables=serving_env,\n", - " model_garden_source_model_name=\"publishers/google/models/f-vlm-jax\",\n", + " model_garden_source_model_name=\"publishers/google/models/f-vlm-jax\"\n", ")\n", "\n", "jax_fvlm_endpoint = jax_fvlm_model.deploy(\n", @@ -1519,7 +1519,7 @@ "# Delete Cloud Storage objects that were created.\n", "delete_bucket = False\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive $BUCKET_URI" + " ! gsutil -m rm -r $BUCKET_URI" ] } ], From 039fea5c54ce074fe463bacbef1abc5aebca0fde Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 23 Dec 2025 11:25:26 +0530 Subject: [PATCH 5/6] Update model_garden_jax_fvlm.ipynb --- notebooks/community/model_garden/model_garden_jax_fvlm.ipynb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb index 898981aca..b28d3aa86 100644 --- a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb @@ -1339,8 +1339,9 @@ " machine_type=\"n1-highmem-16\",\n", " min_replica_count=1,\n", " max_replica_count=1,\n", - " system_labels={\"NOTEBOOK_NAME\": \"model_garden_jax_fvlm.ipynb\"},\n", - ")" + " system_labels={\n", + " \"NOTEBOOK_NAME\": \"model_garden_jax_fvlm.ipynb\"\n", + " },\n", ] }, { From fdeb4e36000f183050ba0bda8b7ebce7ee0c04f7 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 23 Dec 2025 11:26:20 +0530 Subject: [PATCH 6/6] Update model_garden_jax_fvlm.ipynb --- notebooks/community/model_garden/model_garden_jax_fvlm.ipynb | 1 + 1 file changed, 1 insertion(+) diff --git a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb index b28d3aa86..489920754 100644 --- a/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb +++ b/notebooks/community/model_garden/model_garden_jax_fvlm.ipynb @@ -1342,6 +1342,7 @@ " system_labels={\n", " \"NOTEBOOK_NAME\": \"model_garden_jax_fvlm.ipynb\"\n", " },\n", + ")" ] }, {