From 7e165da8b3fe550bbf99442771779035eea87b9b Mon Sep 17 00:00:00 2001 From: Margubur Rahman Date: Thu, 16 Oct 2025 10:53:24 +0000 Subject: [PATCH 1/6] Migrate gsutil usage to gcloud storage --- ... AutoML Vision Video Object Tracking.ipynb | 22 +++++--------- ...tarted_with_automl_image_model_batch.ipynb | 25 +++++----------- ...arted_with_automl_image_model_online.ipynb | 18 ++++------- ...get_started_with_tf_serving_function.ipynb | 12 +++----- ...arden_codegemma_deployment_on_vertex.ipynb | 18 +++++------ ...del_garden_reasoning_engine_llama3_1.ipynb | 3 +- ...y_classification_online_export_cloud.ipynb | 30 +++++++------------ ...abular_classification_online_explain.ipynb | 18 ++++------- .../migration/sdk-hyperparameter-tuning.ipynb | 9 ++---- .../predictive_maintenance_usecase.ipynb | 6 ++-- 10 files changed, 55 insertions(+), 106 deletions(-) diff --git a/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb b/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb index b50933bc5..ff0faf3ba 100644 --- a/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb +++ b/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb @@ -340,8 +340,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION gs://$BUCKET_NAME\n" - ] + "! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME\n" ] }, { "cell_type": "markdown", @@ -361,8 +360,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al gs://$BUCKET_NAME\n" - ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME\n" ] }, { "cell_type": "markdown", @@ -506,8 +504,7 @@ }, "outputs": [], "source": [ - "! gsutil cat $IMPORT_FILE | head -n 10\n" - ] + "! gcloud storage cat $IMPORT_FILE | head -n 10\n" ] }, { "cell_type": "markdown", @@ -1222,8 +1219,7 @@ "source": [ "TRAIN_FILES = \"gs://automl-video-demo-data/traffic_videos/traffic_videos_labels.csv\"\n", "\n", - "test_items = ! gsutil cat $TRAIN_FILES | head -n2\n", - "\n", + "test_items = ! gcloud storage cat $TRAIN_FILES | head -n2\n", "\n", "cols = str(test_items[0]).split(',')\n", "test_item_1 = str(cols[0])\n", "test_label_1 = str(cols[1])\n", @@ -1286,8 +1282,7 @@ " f.write(data + '\\n')\n", " \n", "print(gcs_input_uri)\n", - "!gsutil cat $gcs_input_uri\n" - ] + "!gcloud storage cat $gcs_input_uri\n" ] }, { "cell_type": "markdown", @@ -1452,9 +1447,7 @@ "source": [ "destination_uri = batch_prediction.output_config.gcs_destination.output_uri_prefix[:-1]\n", "\n", - "! gsutil ls $destination_uri/prediction-**\n", - "! gsutil cat $destination_uri/prediction-**" - ] + "! gcloud storage ls $destination_uri/prediction-**\n", "! gcloud storage cat $destination_uri/prediction-**" ] }, { "cell_type": "markdown", @@ -1548,8 +1541,7 @@ " print(e)\n", "\n", "if delete_bucket and 'BUCKET_NAME' in globals():\n", - " ! gsutil rm -r gs://$BUCKET_NAME\n" - ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME\n" ] }, { "cell_type": "code", diff --git a/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb index 0afabf013..21b026d4c 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb @@ -483,8 +483,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -503,8 +502,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -612,12 +610,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -869,8 +865,7 @@ }, "outputs": [], "source": [ - "test_items = !gsutil cat $IMPORT_FILE | head -n2\n", - "if len(str(test_items[0]).split(\",\")) == 3:\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(str(test_items[0]).split(\",\")) == 3:\n", " _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n", " _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n", "else:\n", @@ -903,9 +898,7 @@ "file_1 = test_item_1.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n", "\n", - "! gsutil cp $test_item_1 $BUCKET_URI/$file_1\n", - "! gsutil cp $test_item_2 $BUCKET_URI/$file_2\n", - "\n", + "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", "\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_2 = BUCKET_URI + \"/\" + file_2" ] @@ -948,8 +941,7 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gsutil cat $gcs_input_uri" - ] + "! gcloud storage cat $gcs_input_uri" ] }, { "cell_type": "markdown", @@ -1106,8 +1098,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb index effc08ce0..dc65cb48d 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb @@ -479,8 +479,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -499,8 +498,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -685,12 +683,10 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gsutil cat $FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $FILE | head" - ] + "! gcloud storage cat $FILE | head" ] }, { "cell_type": "markdown", @@ -969,8 +965,7 @@ }, "outputs": [], "source": [ - "test_item = !gsutil cat $IMPORT_FILE | head -n1\n", - "if len(str(test_item[0]).split(\",\")) == 3:\n", + "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n", " _, test_item, test_label = str(test_item[0]).split(\",\")\n", "else:\n", " test_item, test_label = str(test_item[0]).split(\",\")\n", @@ -1122,8 +1117,7 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb index ebe670dea..a052c5c2f 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb @@ -443,8 +443,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_URI" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -463,8 +462,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_URI" - ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" ] }, { "cell_type": "markdown", @@ -947,8 +945,7 @@ }, "outputs": [], "source": [ - "! gsutil cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" - ] + "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" ] }, { "cell_type": "code", @@ -1050,8 +1047,7 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gsutil rm -rf {BUCKET_URI}" - ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb index ad78f4f5d..4595a6762 100644 --- a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb @@ -154,11 +154,10 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gsutil mb -l {REGION} {BUCKET_URI}\n", - "else:\n", + " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", - " bucket_region = shell_output[0].strip().lower()\n", + # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -182,8 +181,9 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", - "\n", + "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", + "! # Note: Conditions: gsutil iam ch does not support modifying IAM policies that contain conditions. gcloud storage commands do support conditions.\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -238,8 +238,7 @@ "\n", " ! mkdir -p ./codegemma\n", " ! curl -X GET \"{signed_url}\" | tar -xzvf - -C ./codegemma/\n", - " ! gsutil -m cp -R ./codegemma/* {BUCKET_URI}\n", - "\n", + " ! gcloud storage cp --recursive ./codegemma/* {BUCKET_URI}\n", "\n", " model_path_prefix = BUCKET_URI.strip(\"/\") + \"/codegemma\"\n", "else:\n", " model_path_prefix = \"google/\"" @@ -820,8 +819,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb b/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb index 687558297..01965cc4e 100644 --- a/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb +++ b/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb @@ -1106,8 +1106,7 @@ "delete_reasoning_engine = False # @param {type:\"boolean\"}\n", "\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_NAME\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_NAME\n", "\n", "if delete_reasoning_engine:\n", " remote_agent.delete()" ] diff --git a/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online_export_cloud.ipynb b/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online_export_cloud.ipynb index e98cf6a14..00acf7b5b 100644 --- a/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online_export_cloud.ipynb +++ b/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online_export_cloud.ipynb @@ -472,8 +472,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -492,8 +491,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -596,14 +594,11 @@ }, "outputs": [], "source": [ - "count = ! gsutil cat $IMPORT_FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $IMPORT_FILE | head\n", - "\n", - "heading = ! gsutil cat $IMPORT_FILE | head -n1\n", - "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -832,15 +827,12 @@ "source": [ "print(\"Model Package:\", model_package)\n", "print(\"Contents:\")\n", - "! gsutil ls $model_package\n", - "\n", + "! gcloud storage ls $model_package\n", "\n", "print(\"\\nTF Saved Model\")\n", "path = model_package + \"/predict\"\n", - "files = ! gsutil ls $path\n", - "saved_dir = files[1]\n", + "files = ! gcloud storage ls $path\n", "saved_dir = files[1]\n", "print(saved_dir)\n", - "! gsutil ls $saved_dir" - ] + "! gcloud storage ls $saved_dir" ] }, { "cell_type": "markdown", @@ -861,8 +853,7 @@ }, "outputs": [], "source": [ - "! gsutil cp -r $model_package ." - ] + "! gcloud storage cp --recursive $model_package ." ] }, { "cell_type": "markdown", @@ -1155,8 +1146,7 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_automl_tabular_classification_online_explain.ipynb b/notebooks/community/sdk/sdk_automl_tabular_classification_online_explain.ipynb index 64ffc7efd..4cf74f233 100644 --- a/notebooks/community/sdk/sdk_automl_tabular_classification_online_explain.ipynb +++ b/notebooks/community/sdk/sdk_automl_tabular_classification_online_explain.ipynb @@ -493,8 +493,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l $REGION $BUCKET_NAME" - ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -513,8 +512,7 @@ }, "outputs": [], "source": [ - "! gsutil ls -al $BUCKET_NAME" - ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] }, { "cell_type": "markdown", @@ -617,14 +615,11 @@ }, "outputs": [], "source": [ - "count = ! gsutil cat $IMPORT_FILE | wc -l\n", - "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gsutil cat $IMPORT_FILE | head\n", - "\n", - "heading = ! gsutil cat $IMPORT_FILE | head -n1\n", - "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -1161,8 +1156,7 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gsutil rm -r $BUCKET_NAME" - ] + " ! gcloud storage rm --recursive $BUCKET_NAME" ] } ], "metadata": { diff --git a/notebooks/official/migration/sdk-hyperparameter-tuning.ipynb b/notebooks/official/migration/sdk-hyperparameter-tuning.ipynb index 87e6657b8..59a8fa8e0 100644 --- a/notebooks/official/migration/sdk-hyperparameter-tuning.ipynb +++ b/notebooks/official/migration/sdk-hyperparameter-tuning.ipynb @@ -270,8 +270,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location {LOCATION} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -645,8 +644,7 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gsutil cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" - ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" ] }, { "cell_type": "markdown", @@ -1092,8 +1090,7 @@ "hpt_job.delete()\n", "\n", "# if delete_bucket:\n", - "# # ! gsutil rm -r $BUCKET_URI" - ] + "# # ! gcloud storage rm --recursive $BUCKET_URI" ] } ], "metadata": { diff --git a/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb b/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb index d06c097e4..36d25c1cb 100644 --- a/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb +++ b/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb @@ -339,8 +339,7 @@ }, "outputs": [], "source": [ - "! gsutil mb -l {LOCATION} {BUCKET_URI}" - ] + "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" ] }, { "cell_type": "markdown", @@ -1217,8 +1216,7 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gsutil -m rm -r $BUCKET_URI\n", - "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", "!rm -rf model.bst" ] } From 7e2db68c8a562ae271023ce22abfbf3549c6af83 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Fri, 12 Dec 2025 14:28:22 +0000 Subject: [PATCH 2/6] changes for 4314 --- ... AutoML Vision Video Object Tracking.ipynb | 22 +++++++++++++------ ...arted_with_automl_image_model_online.ipynb | 18 ++++++++++----- ...arden_codegemma_deployment_on_vertex.ipynb | 18 ++++++++------- .../migration/sdk-hyperparameter-tuning.ipynb | 9 +++++--- 4 files changed, 43 insertions(+), 24 deletions(-) diff --git a/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb b/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb index ff0faf3ba..75b23b051 100644 --- a/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb +++ b/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb @@ -340,7 +340,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION gs://$BUCKET_NAME\n" ] + "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME\n" + ] }, { "cell_type": "markdown", @@ -360,7 +361,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME\n" ] + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME\n" + ] }, { "cell_type": "markdown", @@ -504,7 +506,8 @@ }, "outputs": [], "source": [ - "! gcloud storage cat $IMPORT_FILE | head -n 10\n" ] + "! gcloud storage cat $IMPORT_FILE | head -n 10\n" + ] }, { "cell_type": "markdown", @@ -1219,7 +1222,8 @@ "source": [ "TRAIN_FILES = \"gs://automl-video-demo-data/traffic_videos/traffic_videos_labels.csv\"\n", "\n", - "test_items = ! gcloud storage cat $TRAIN_FILES | head -n2\n", "\n", + "test_items = ! gcloud storage cat $TRAIN_FILES | head -n2\n", + "\n", "cols = str(test_items[0]).split(',')\n", "test_item_1 = str(cols[0])\n", "test_label_1 = str(cols[1])\n", @@ -1282,7 +1286,8 @@ " f.write(data + '\\n')\n", " \n", "print(gcs_input_uri)\n", - "!gcloud storage cat $gcs_input_uri\n" ] + "!gcloud storage cat $gcs_input_uri\n" + ] }, { "cell_type": "markdown", @@ -1447,7 +1452,9 @@ "source": [ "destination_uri = batch_prediction.output_config.gcs_destination.output_uri_prefix[:-1]\n", "\n", - "! gcloud storage ls $destination_uri/prediction-**\n", "! gcloud storage cat $destination_uri/prediction-**" ] + "! gcloud storage ls $destination_uri/prediction-**\n", + "! gcloud storage cat $destination_uri/prediction-**" + ] }, { "cell_type": "markdown", @@ -1541,7 +1548,8 @@ " print(e)\n", "\n", "if delete_bucket and 'BUCKET_NAME' in globals():\n", - " ! gcloud storage rm --recursive gs://$BUCKET_NAME\n" ] + " ! gcloud storage rm --recursive gs://$BUCKET_NAME\n" + ] }, { "cell_type": "code", diff --git a/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb index dc65cb48d..2e7760e37 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_online.ipynb @@ -479,7 +479,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -498,7 +499,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -683,10 +685,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -965,7 +969,8 @@ }, "outputs": [], "source": [ - "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", "if len(str(test_item[0]).split(\",\")) == 3:\n", + "test_item = !gcloud storage cat $IMPORT_FILE | head -n1\n", + "if len(str(test_item[0]).split(\",\")) == 3:\n", " _, test_item, test_label = str(test_item[0]).split(\",\")\n", "else:\n", " test_item, test_label = str(test_item[0]).split(\",\")\n", @@ -1117,7 +1122,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb index 4595a6762..f51d8ced1 100644 --- a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb @@ -154,10 +154,11 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", + " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", + "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - # Note: The format of the full listing output is different. gcloud storage uses a title case for keys and will not display a field if its value is "None". - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", + " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", " \"Bucket region %s is different from notebook region %s\"\n", @@ -181,9 +182,8 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! # Note: Migrating scripts using gsutil iam ch is more complex than get or set. You need to replace the single iam ch command with a series of gcloud storage bucket add-iam-policy-binding and/or gcloud storage bucket remove-iam-policy-binding commands, or replicate the read-modify-write loop.\n", - "! # Note: Conditions: gsutil iam ch does not support modifying IAM policies that contain conditions. gcloud storage commands do support conditions.\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", "\n", + "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/aiplatform.user\"" @@ -238,7 +238,8 @@ "\n", " ! mkdir -p ./codegemma\n", " ! curl -X GET \"{signed_url}\" | tar -xzvf - -C ./codegemma/\n", - " ! gcloud storage cp --recursive ./codegemma/* {BUCKET_URI}\n", "\n", + " ! gcloud storage cp --recursive ./codegemma/* {BUCKET_URI}\n", + "\n", " model_path_prefix = BUCKET_URI.strip(\"/\") + \"/codegemma\"\n", "else:\n", " model_path_prefix = \"google/\"" @@ -819,7 +820,8 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/official/migration/sdk-hyperparameter-tuning.ipynb b/notebooks/official/migration/sdk-hyperparameter-tuning.ipynb index 59a8fa8e0..7d3ee21f2 100644 --- a/notebooks/official/migration/sdk-hyperparameter-tuning.ipynb +++ b/notebooks/official/migration/sdk-hyperparameter-tuning.ipynb @@ -270,7 +270,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location {LOCATION} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -644,7 +645,8 @@ "! rm -f custom.tar custom.tar.gz\n", "! tar cvf custom.tar custom\n", "! gzip custom.tar\n", - "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" ] + "! gcloud storage cp custom.tar.gz $BUCKET_URI/trainer_boston.tar.gz" + ] }, { "cell_type": "markdown", @@ -1090,7 +1092,8 @@ "hpt_job.delete()\n", "\n", "# if delete_bucket:\n", - "# # ! gcloud storage rm --recursive $BUCKET_URI" ] + "# # ! gcloud storage rm --recursive $BUCKET_URI" + ] } ], "metadata": { From 8589909ec7187e0b0c25e3b32f4b52bc6c2fe83b Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 16 Dec 2025 12:34:40 +0000 Subject: [PATCH 3/6] Apply automated linter fixes --- ... AutoML Vision Video Object Tracking.ipynb | 3114 ++++++++--------- ...tarted_with_automl_image_model_batch.ipynb | 25 +- ...get_started_with_tf_serving_function.ipynb | 12 +- ...arden_codegemma_deployment_on_vertex.ipynb | 16 +- ...del_garden_reasoning_engine_llama3_1.ipynb | 20 +- ...y_classification_online_export_cloud.ipynb | 30 +- ...abular_classification_online_explain.ipynb | 18 +- .../predictive_maintenance_usecase.ipynb | 6 +- 8 files changed, 1577 insertions(+), 1664 deletions(-) diff --git a/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb b/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb index 75b23b051..79c9b766f 100644 --- a/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb +++ b/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb @@ -1,1625 +1,1493 @@ { - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "copyright" - }, - "outputs": [], - "source": [ - "# Copyright 2021 Google LLC\n", - "#\n", - "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", - "# you may not use this file except in compliance with the License.\n", - "# You may obtain a copy of the License at\n", - "#\n", - "# https://www.apache.org/licenses/LICENSE-2.0\n", - "#\n", - "# Unless required by applicable law or agreed to in writing, software\n", - "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", - "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", - "# See the License for the specific language governing permissions and\n", - "# limitations under the License.\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "title:migration,new" - }, - "source": [ - "# AutoML SDK: AutoML video object tracking model\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "install_aip" - }, - "source": [ - "## Installation\n", - "\n", - "Install the latest (preview) version of AutoML SDK.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "vQWDe_6HJZJW", - "outputId": "687efd9a-e344-470e-c1fb-18bece046cf2" - }, - "outputs": [], - "source": [ - "! pip3 install -U google-cloud-automl --user\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "install_storage" - }, - "source": [ - "Install the Google *cloud-storage* library as well.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "5nZRXjD0JZJY", - "outputId": "4cf3eea8-952d-4222-cea1-bc631bdb8cb7" - }, - "outputs": [], - "source": [ - "! pip3 install google-cloud-storage\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "restart" - }, - "source": [ - "### Restart the Kernel\n", - "\n", - "Once you've installed the AutoML SDK and Google *cloud-storage*, you need to restart the notebook kernel so it can find the packages.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "3FaDxry1JZJZ" - }, - "outputs": [], - "source": [ - "import os\n", - "\n", - "\n", - "if not os.getenv(\"AUTORUN\"):\n", - " # Automatically restart kernel after installs\n", - " import IPython\n", - " app = IPython.Application.instance()\n", - " app.kernel.do_shutdown(True)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "before_you_begin" - }, - "source": [ - "## Before you begin\n", - "\n", - "### GPU run-time\n", - "\n", - "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n", - "\n", - "### Set up your GCP project\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "1. [Select or create a GCP project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", - "\n", - "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n", - "\n", - "3. [Enable the AutoML APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component)\n", - "\n", - "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in AutoML Notebooks.\n", - "\n", - "5. Enter your project ID in the cell below. Then run the cell to make sure the\n", - "Cloud SDK uses the right project for all the commands in this notebook.\n", - "\n", - "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "set_project_id" - }, - "outputs": [], - "source": [ - "PROJECT_ID = \"[your-project-id]\" #@param {type:\"string\"}\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "autoset_project_id", - "outputId": "1ac7bb1b-ed49-4874-f6d1-5db45456019d" - }, - "outputs": [], - "source": [ - "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n", - " # Get your GCP project id from gcloud\n", - " shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n", - " PROJECT_ID = shell_output[0]\n", - " print(\"Project ID:\", PROJECT_ID)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "set_gcloud_project_id", - "outputId": "0acbf680-cec3-44c5-8d51-1a05a81258ca" - }, - "outputs": [], - "source": [ - "! gcloud config set project $PROJECT_ID\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "region" - }, - "source": [ - "#### Region\n", - "\n", - "You can also change the `REGION` variable, which is used for operations\n", - "throughout the rest of this notebook. Below are regions supported for AutoML. We recommend when possible, to choose the region closest to you.\n", - "\n", - "- Americas: `us-central1`\n", - "- Europe: `europe-west4`\n", - "- Asia Pacific: `asia-east1`\n", - "\n", - "You cannot use a Multi-Regional Storage bucket for training with AutoML. Not all regions provide support for all AutoML services. For the latest support per region, see [Region support for AutoML services]()\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "jVMUAWrJJZJe" - }, - "outputs": [], - "source": [ - "REGION = 'us-central1' #@param {type: \"string\"}\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "timestamp" - }, - "source": [ - "#### Timestamp\n", - "\n", - "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append onto the name of resources which will be created in this tutorial.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "LJuwNh9wJZJe" - }, - "outputs": [], - "source": [ - "from datetime import datetime\n", - "\n", - "\n", - "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gcp_authenticate" - }, - "source": [ - "### Authenticate your GCP account\n", - "\n", - "**If you are using AutoML Notebooks**, your environment is already\n", - "authenticated. Skip this step.\n", - "\n", - "*Note: If you are on an AutoML notebook and run the cell, the cell knows to skip executing the authentication steps.*\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "fEY1QcN_JZJf" - }, - "outputs": [], - "source": [ - "import os\n", - "import sys\n", - "\n", - "# If you are running this notebook in Colab, run this cell and follow the\n", - "# instructions to authenticate your Google Cloud account. This provides access\n", - "# to your Cloud Storage bucket and lets you submit training jobs and prediction\n", - "# requests.\n", - "\n", - "# If on AutoML, then don't execute this code\n", - "if not os.path.exists('/opt/deeplearning/metadata/env_version'):\n", - " if 'google.colab' in sys.modules:\n", - " from google.colab import auth as google_auth\n", - " google_auth.authenticate_user()\n", - "\n", - " # If you are running this tutorial in a notebook locally, replace the string\n", - " # below with the path to your service account key and run this cell to\n", - " # authenticate your Google Cloud account.\n", - " else:\n", - " %env GOOGLE_APPLICATION_CREDENTIALS your_path_to_credentials.json\n", - "\n", - " # Log in to your account on Google Cloud\n", - " ! gcloud auth login\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "bucket:batch_prediction" - }, - "source": [ - "### Create a Cloud Storage bucket\n", - "\n", - "**The following steps are required, regardless of your notebook environment.**\n", - "\n", - "This tutorial is designed to use training data that is in a public Cloud Storage bucket and a local Cloud Storage bucket for your batch predictions. You may alternatively use your own training data that you have stored in a local Cloud Storage bucket.\n", - "\n", - "Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "bucket" - }, - "outputs": [], - "source": [ - "BUCKET_NAME = \"[your-bucket-name]\" #@param {type:\"string\"}\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "autoset_bucket" - }, - "outputs": [], - "source": [ - "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n", - " BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "create_bucket" - }, - "source": [ - "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Dc_4SqrZJZJg", - "outputId": "cfa60d02-7265-421a-b30e-dbcbe987dac8" - }, - "outputs": [], - "source": [ - "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "validate_bucket" - }, - "source": [ - "Finally, validate access to your Cloud Storage bucket by examining its contents:\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "gkxndVXCJZJh" - }, - "outputs": [], - "source": [ - "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "setup_vars" - }, - "source": [ - "### Set up variables\n", - "\n", - "Next, set up some variables used throughout the tutorial.\n", - "### Import libraries and define constants\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "import_aip" - }, - "source": [ - "#### Import AutoML SDK\n", - "\n", - "Import the AutoML SDK into our Python environment.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "x3EpSgrNJZJi" - }, - "outputs": [], - "source": [ - "import json\n", - "import os\n", - "import sys\n", - "import time\n", - "\n", - "\n", - "from google.cloud import automl_v1beta1 as automl\n", - "\n", - "\n", - "from google.protobuf.json_format import MessageToJson\n", - "from google.protobuf.json_format import ParseDict\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "aip_constants" - }, - "source": [ - "#### AutoML constants\n", - "\n", - "Setup up the following constants for AutoML:\n", - "\n", - "- `PARENT`: The AutoML location root path for dataset, model and endpoint resources.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "gIbQItxjJZJj" - }, - "outputs": [], - "source": [ - "# AutoML location root path for your dataset, model and endpoint resources\n", - "PARENT = \"projects/\" + PROJECT_ID + \"/locations/\" + REGION\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "clients" - }, - "source": [ - "## Clients\n", - "\n", - "The AutoML SDK works as a client/server model. On your side (the Python script) you will create a client that sends requests and receives responses from the server (AutoML).\n", - "\n", - "You will use several clients in this tutorial, so set them all up upfront.\n", - "\n", - "- Dataset Service for managed datasets.\n", - "- Model Service for managed models.\n", - "- Pipeline Service for training.\n", - "- Endpoint Service for deployment.\n", - "- Job Service for batch jobs and custom training.\n", - "- Prediction Service for serving. *Note*: Prediction has a different service endpoint.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "eE3k7aqvJZJk", - "outputId": "3b280f73-9cff-4247-f8e2-8e413dfa4491" - }, - "outputs": [], - "source": [ - "def automl_client():\n", - " return automl.AutoMlClient()\n", - "\n", - "def prediction_client():\n", - " return automl.PredictionServiceClient()\n", - "\n", - "def operations_client():\n", - " return automl.AutoMlClient()._transport.operations_client\n", - "\n", - "\n", - "clients = {}\n", - "clients[\"automl\"] = automl_client()\n", - "clients[\"prediction\"] = prediction_client()\n", - "clients[\"operations\"] = operations_client()\n", - "\n", - "for client in clients.items():\n", - " print(client)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "import_file:flowers,csv,icn" - }, - "outputs": [], - "source": [ - "IMPORT_FILE = 'gs://automl-video-demo-data/traffic_videos/traffic_videos.csv'\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "1OCKDsyLJZJl", - "outputId": "cb4b0db7-251a-4928-b944-ee2f109f6855" - }, - "outputs": [], - "source": [ - "! gcloud storage cat $IMPORT_FILE | head -n 10\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "trainingpipelines_create:migration,new,response,icn" - }, - "source": [ - "*Example output*:\n", - "```\n", - "UNASSIGNED,gs://automl-video-demo-data/traffic_videos/traffic_videos_labels.csv\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "create_a_dataset:migration" - }, - "source": [ - "## Create a dataset\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mZwR43s_JZJm" - }, - "source": [ - "### [projects.locations.datasets.create](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.datasets/create)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "request:migration" - }, - "source": [ - "#### Request\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "datasets_create:migration,new,request", - "outputId": "e3f9d4c6-5f20-4201-a1e6-2ea6b5c3b2f1" - }, - "outputs": [], - "source": [ - "dataset = {\n", - " \"display_name\": \"traffic_\" + TIMESTAMP,\n", - " \"video_object_tracking_dataset_metadata\": {}\n", - "}\n", - "\n", - "print(MessageToJson(\n", - " automl.CreateDatasetRequest(\n", - " parent=PARENT,\n", - " dataset=dataset\n", - " ).__dict__[\"_pb\"])\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZZFAAFAkJZJn" - }, - "source": [ - "*Example output*:\n", - "```\n", - "{\n", - " \"parent\": \"projects/migration-ucaip-training/locations/us-central1\",\n", - " \"dataset\": {\n", - " \"displayName\": \"traffic_20210310004803\",\n", - " \"videoObjectTrackingDatasetMetadata\": {}\n", - " }\n", - "}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "call:migration" - }, - "source": [ - "#### Call\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "datasets_create:migration,new,call" - }, - "outputs": [], - "source": [ - "request = clients[\"automl\"].create_dataset(\n", - " parent=PARENT,\n", - " dataset=dataset\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "response:migration" - }, - "source": [ - "#### Response\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "print:migration,new,response", - "outputId": "6b50557f-3012-4076-e5f4-7d40bb5333e2" - }, - "outputs": [], - "source": [ - "result = request\n", - "\n", - "print(MessageToJson(result.__dict__[\"_pb\"]))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "datasets_create:migration,new,response" - }, - "source": [ - "*Example output*:\n", - "```\n", - "{\n", - " \"name\": \"projects/116273516712/locations/us-central1/datasets/VOT4951119001817186304\",\n", - " \"displayName\": \"traffic_20210310004803\",\n", - " \"createTime\": \"2021-03-10T00:48:09.292248Z\",\n", - " \"etag\": \"AB3BwFp3u4a2oy-k3EgK6ci8zwrTqrd91_DmoaY8TYsxnb-N-aXwFefqCIm1z0YTM290\",\n", - " \"videoObjectTrackingDatasetMetadata\": {}\n", - "}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "dataset_id:migration,new,response", - "outputId": "ddb296ce-8c86-4793-a7ba-b5409a7d92d1" - }, - "outputs": [], - "source": [ - "# The full unique ID for the dataset\n", - "dataset_id = result.name\n", - "# The short numeric ID for the dataset\n", - "dataset_short_id = dataset_id.split('/')[-1]\n", - "\n", - "print(dataset_id)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "OOmLNEw3JZJq" - }, - "source": [ - "### [projects.locations.datasets.importData](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.datasets/importData)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "T5yWOG67JZJr" - }, - "source": [ - "#### Request\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "datasets_import:migration,new,request", - "outputId": "5cac21ee-6f18-4f44-c360-d174832d32ff" - }, - "outputs": [], - "source": [ - "input_config = {\n", - " \"gcs_source\": {\n", - " \"input_uris\": [IMPORT_FILE]\n", - " }\n", - "}\n", - "\n", - "print(MessageToJson(\n", - " automl.ImportDataRequest(\n", - " name=dataset_short_id,\n", - " input_config=input_config\n", - " ).__dict__[\"_pb\"])\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Dzg9JangJZJs" - }, - "source": [ - "*Example output*:\n", - "```\n", - "{\n", - " \"name\": \"VOT4951119001817186304\",\n", - " \"inputConfig\": {\n", - " \"gcsSource\": {\n", - " \"inputUris\": [\n", - " \"gs://automl-video-demo-data/traffic_videos/traffic_videos.csv\"\n", - " ]\n", - " }\n", - " }\n", - "}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1shZi6Y4JZJs" - }, - "source": [ - "#### Call\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "datasets_import:migration,new,call" - }, - "outputs": [], - "source": [ - "request = clients[\"automl\"].import_data(\n", - " name=dataset_id,\n", - " input_config=input_config\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "o3_x634HJZJt" - }, - "source": [ - "#### Response\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Dc3YtK0jJZJt", - "outputId": "235c07d2-b693-48aa-a8af-ffc087541ab3" - }, - "outputs": [], - "source": [ - "result = request.result()\n", - "\n", - "print(MessageToJson(result))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "datasets_import:migration,new,response" - }, - "source": [ - "*Example output*:\n", - "```\n", - "{}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "train_a_model:migration" - }, - "source": [ - "## Train a model\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "text_models_create:migration,old" - }, - "source": [ - "### [projects.locations.models.create](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.models/create)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mQD_PyZjJZJu" - }, - "source": [ - "#### Request\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "trainingpipelines_create:migration,new,request,icn", - "outputId": "e900bb95-0e65-4576-efc6-e1ef8bbf7a28" - }, - "outputs": [], - "source": [ - "model = {\n", - " \"display_name\": \"traffic_\" + TIMESTAMP,\n", - " \"dataset_id\": dataset_short_id,\n", - " \"video_object_tracking_model_metadata\": {}\n", - "}\n", - "\n", - "print(MessageToJson(\n", - " automl.CreateModelRequest(\n", - " parent=PARENT,\n", - " model=model\n", - " ).__dict__[\"_pb\"])\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "8atEKPpdJZJv" - }, - "source": [ - "*Example output*:\n", - "```\n", - "{\n", - " \"parent\": \"projects/migration-ucaip-training/locations/us-central1\",\n", - " \"model\": {\n", - " \"displayName\": \"traffic_20210310004803\",\n", - " \"datasetId\": \"VOT4951119001817186304\",\n", - " \"videoObjectTrackingModelMetadata\": {}\n", - " }\n", - "}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ZfHEsMO5JZJv" - }, - "source": [ - "#### Call\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "trainingpipelines_create:migration,new,call" - }, - "outputs": [], - "source": [ - "request = clients[\"automl\"].create_model(\n", - " parent=PARENT,\n", - " model=model\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jYKM6gfvJZJw" - }, - "source": [ - "#### Response\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "print:migration,new,request", - "outputId": "80d54292-09b6-4123-d6eb-0450ce8eb816" - }, - "outputs": [], - "source": [ - "result = request.result()\n", - "\n", - "print(MessageToJson(result.__dict__[\"_pb\"]))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "jdU2ko30JZJw" - }, - "source": [ - "*Example output*:\n", - "```\n", - "{\n", - " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080\"\n", - "}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "training_pipeline_id:migration,new,response", - "outputId": "30468e96-7513-4bf3-decc-3b0ea10534a5" - }, - "outputs": [], - "source": [ - "# The full unique ID for the training pipeline\n", - "model_id = result.name\n", - "# The short numeric ID for the training pipeline\n", - "model_short_id = model_id.split('/')[-1]\n", - "\n", - "print(model_short_id)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "evaluate_the_model:migration" - }, - "source": [ - "## Evaluate the model\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "yfufMwAEJjkX" - }, - "source": [ - "### [projects.locations.models.modelEvaluations.list](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.models.modelEvaluations/list)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_Zv5eHjwJZJx" - }, - "source": [ - "#### Call\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "models_evaluations_list:migration,new,call" - }, - "outputs": [], - "source": [ - "request = clients[\"automl\"].list_model_evaluations(\n", - " parent=model_id, \n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "QtnV8mnJJZJx" - }, - "source": [ - "#### Response\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "models_evaluations_list:migration,new,response", - "outputId": "8a5080de-5c4f-41f8-8930-71dd7afd8405", - "scrolled": true - }, - "outputs": [], - "source": [ - "for evaluation in request:\n", - " print(MessageToJson(evaluation.__dict__[\"_pb\"]))\n", - "\n", - "# The last evaluation slice\n", - "last_evaluation_slice = evaluation.name\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "72tAHlBJJZJy" - }, - "source": [ - "*Example output*:\n", - "```\n", - "{\n", - " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080/modelEvaluations/3861662240237183015\",\n", - " \"annotationSpecId\": \"7080515133784457216\",\n", - " \"createTime\": \"2021-03-10T01:57:44.615737Z\",\n", - " \"evaluatedExampleCount\": 6,\n", - " \"videoObjectTrackingEvaluationMetrics\": {\n", - " \"boundingBoxMetricsEntries\": [\n", - " {\n", - " \"iouThreshold\": 0.5,\n", - " \"meanAveragePrecision\": 0.30026233,\n", - " \"confidenceMetricsEntries\": [\n", - " {\n", - " \"recall\": 1.0,\n", - " \"precision\": 0.37222221,\n", - " \"f1Score\": 0.5425101\n", - " },\n", - " {\n", - " \"confidenceThreshold\": 0.028951555,\n", - " \"recall\": 0.13432837,\n", - " \"precision\": 0.07377049,\n", - " \"f1Score\": 0.0952381\n", - " },\n", - " \n", - " # REMOVED FOR BREVITY\n", - " \n", - " {\n", - " \"confidenceThreshold\": 1.0,\n", - " \"precision\": 1.0\n", - " }\n", - " ]\n", - " }\n", - " ],\n", - " \"boundingBoxMeanAveragePrecision\": 0.30026233\n", - " },\n", - " \"displayName\": \"pickup_suv_van\"\n", - "}\n", - "{\n", - " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080/modelEvaluations/4053501621797068779\",\n", - " \"annotationSpecId\": \"5927593629177610240\",\n", - " \"createTime\": \"2021-03-10T01:57:44.615737Z\",\n", - " \"evaluatedExampleCount\": 5,\n", - " \"videoObjectTrackingEvaluationMetrics\": {\n", - " \"boundingBoxMetricsEntries\": [\n", - " {\n", - " \"iouThreshold\": 0.5,\n", - " \"meanAveragePrecision\": 0.42889464,\n", - " \"confidenceMetricsEntries\": [\n", - " {\n", - " \"recall\": 1.0,\n", - " \"precision\": 0.25490198,\n", - " \"f1Score\": 0.40625\n", - " },\n", - " \n", - " # REMOVED FOR BREVITY\n", - " ],\n", - " \"boundingBoxMeanAveragePrecision\": 0.34359422\n", - " },\n", - " \"displayName\": \"sedan\"\n", - "}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i6T0bzuNJjkY" - }, - "source": [ - "### [projects.locations.models.modelEvaluations.get](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.models.modelEvaluations/get)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Nusdm5_MJZJz" - }, - "source": [ - "#### Call\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "models_evaluations_get:migration,new,call" - }, - "outputs": [], - "source": [ - "request = clients[\"automl\"].get_model_evaluation(\n", - " name=last_evaluation_slice\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "GB8oloXGJZJz" - }, - "source": [ - "#### Response\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "9n1UTT6oJZJ0", - "outputId": "898d5c96-225c-4811-8504-450662cd11fb", - "scrolled": true - }, - "outputs": [], - "source": [ - "print(MessageToJson(request.__dict__[\"_pb\"]))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vqQCboLGJZJ0" - }, - "source": [ - "*Example output*:\n", - "```\n", - "{\n", - " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080/modelEvaluations/6603022638609369541\",\n", - " \"annotationSpecId\": \"1315907610750222336\",\n", - " \"createTime\": \"2021-03-10T01:57:44.615737Z\",\n", - " \"evaluatedExampleCount\": 6,\n", - " \"videoObjectTrackingEvaluationMetrics\": {\n", - " \"boundingBoxMetricsEntries\": [\n", - " {\n", - " \"iouThreshold\": 0.5,\n", - " \"meanAveragePrecision\": 0.34359422,\n", - " \"confidenceMetricsEntries\": [\n", - " {\n", - " \"recall\": 1.0,\n", - " \"precision\": 0.41428572,\n", - " \"f1Score\": 0.5858586\n", - " },\n", - " {\n", - " \"confidenceThreshold\": 0.03328514,\n", - " \"recall\": 0.1724138,\n", - " \"precision\": 0.10869565,\n", - " \"f1Score\": 0.13333334\n", - " },\n", - " \n", - " # REMOVED FOR BREVITY\n", - " ]\n", - " }\n", - " ],\n", - " \"boundingBoxMeanAveragePrecision\": 0.34359422\n", - " },\n", - " \"displayName\": \"sedan\"\n", - "}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "make_batch_predictions:migration" - }, - "source": [ - "## Make batch predictions\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "make_batch_prediction_file:migration,new" - }, - "source": [ - "### Prepare batch prediction data\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "get_test_items:automl,icn,csv", - "outputId": "95539c27-5e06-4d5b-c7ec-c9d199c2c713" - }, - "outputs": [], - "source": [ - "TRAIN_FILES = \"gs://automl-video-demo-data/traffic_videos/traffic_videos_labels.csv\"\n", - "\n", - "test_items = ! gcloud storage cat $TRAIN_FILES | head -n2\n", - "\n", - "cols = str(test_items[0]).split(',')\n", - "test_item_1 = str(cols[0])\n", - "test_label_1 = str(cols[1])\n", - "test_start_time_1 = str(0)\n", - "test_end_time_1 = \"inf\"\n", - "\n", - "print(test_item_1, test_label_1)\n", - "\n", - "cols = str(test_items[1]).split(',')\n", - "test_item_2 = str(cols[0])\n", - "test_label_2 = str(cols[1])\n", - "test_start_time_2 = str(0)\n", - "test_end_time_2 = \"inf\"\n", - "\n", - "print(test_item_2, test_label_2)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "mrOPGyQ-JZJ2" - }, - "source": [ - "*Example output*:\n", - "```\n", - "gs://automl-video-demo-data/traffic_videos/highway_005.mp4 sedan\n", - "gs://automl-video-demo-data/traffic_videos/highway_005.mp4 pickup_suv_van\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "make_batch_file:automl,image" - }, - "source": [ - "### Make the batch input file\n", - "\n", - "To request a batch of predictions from AutoML Video, create a CSV file that lists the Cloud Storage paths to the videos that you want to annotate. You can also specify a start and end time to tell AutoML Video to only annotate a segment (segment-level) of the video. The start time must be zero or greater and must be before the end time. The end time must be greater than the start time and less than or equal to the duration of the video. You can also use inf to indicate the end of a video.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "_EdGkXQPJZJ2", - "outputId": "e9c08c85-e220-47b2-9081-54034471da6f" - }, - "outputs": [], - "source": [ - "import tensorflow as tf\n", - "import json\n", - "\n", - "gcs_input_uri = \"gs://\" + BUCKET_NAME + '/test.csv'\n", - "with tf.io.gfile.GFile(gcs_input_uri, 'w') as f:\n", - " data = f\"{test_item_1}, {test_start_time_1}, {test_end_time_1}\"\n", - " f.write(data + '\\n')\n", - " data = f\"{test_item_2}, {test_start_time_2}, {test_end_time_2}\"\n", - " f.write(data + '\\n')\n", - " \n", - "print(gcs_input_uri)\n", - "!gcloud storage cat $gcs_input_uri\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "PEIM210uJZJ3" - }, - "source": [ - "*Example output*:\n", - "```\n", - "gs://migration-ucaip-trainingaip-20210310004803/test.csv\n", - "gs://automl-video-demo-data/traffic_videos/highway_005.mp4, 0, inf\n", - "gs://automl-video-demo-data/traffic_videos/highway_005.mp4, 0, inf\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "text_models_batchpredict:migration,old" - }, - "source": [ - "### [projects.locations.models.batchPredict](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.models/batchPredict)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "zKU3o21IJZJ4" - }, - "source": [ - "#### Request\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "batchpredictionjobs_create:migration,new,request,icn", - "outputId": "74db149d-12c0-4a96-b99e-a22d226d06ac" - }, - "outputs": [], - "source": [ - "input_config = {\n", - " \"gcs_source\": {\n", - " \"input_uris\": [gcs_input_uri]\n", - " }\n", - "}\n", - " \n", - "output_config = {\n", - " \"gcs_destination\": {\n", - " \"output_uri_prefix\": \"gs://\" + f\"{BUCKET_NAME}/batch_output/\"\n", - " }\n", - "} \n", - "\n", - "batch_prediction = automl.BatchPredictRequest(\n", - " name=model_id,\n", - " input_config=input_config,\n", - " output_config=output_config,\n", - ")\n", - "\n", - "print(MessageToJson(batch_prediction.__dict__[\"_pb\"]))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "1Y0TXjVNJZJ4" - }, - "source": [ - "*Example output*:\n", - "```\n", - "{\n", - " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080\",\n", - " \"inputConfig\": {\n", - " \"gcsSource\": {\n", - " \"inputUris\": [\n", - " \"gs://migration-ucaip-trainingaip-20210310004803/test.csv\"\n", - " ]\n", - " }\n", - " },\n", - " \"outputConfig\": {\n", - " \"gcsDestination\": {\n", - " \"outputUriPrefix\": \"gs://migration-ucaip-trainingaip-20210310004803/batch_output/\"\n", - " }\n", - " }\n", - "}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "ScGH6fbbJZJ5" - }, - "source": [ - "#### Call\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "batchpredictionjobs_create:migration,new,call" - }, - "outputs": [], - "source": [ - "request = clients[\"prediction\"].batch_predict(\n", - " request=batch_prediction\n", - ")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Mh9Jwl8OJZJ5" - }, - "source": [ - "#### Response\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "ep-WdlhxJZJ6", - "outputId": "7f3a4e85-0700-4648-a380-4be7a0da2c3d" - }, - "outputs": [], - "source": [ - "result = request.result()\n", - "\n", - "print(MessageToJson(result.__dict__[\"_pb\"]))\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "batchpredictionjobs_create:migration,new,response,icn" - }, - "source": [ - "*Example output*:\n", - "\n", - "```\n", - "{}\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "GW6GcRaaJZJ6", - "outputId": "5ea11a6f-226c-4ba7-e274-e9f09ecdce48" - }, - "outputs": [], - "source": [ - "destination_uri = batch_prediction.output_config.gcs_destination.output_uri_prefix[:-1]\n", - "\n", - "! gcloud storage ls $destination_uri/prediction-**\n", - "! gcloud storage cat $destination_uri/prediction-**" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "gy3AJcK5JZJ7" - }, - "source": [ - "*Example output*:\n", - "```\n", - "gs://migration-ucaip-trainingaip-20210310004803/batch_output/prediction-traffic_20210310004803-2021-03-10T01:57:52.257240Z/highway_005_1.json\n", - "gs://migration-ucaip-trainingaip-20210310004803/batch_output/prediction-traffic_20210310004803-2021-03-10T01:57:52.257240Z/video_object_tracking.csv\n", - "{\n", - " \"object_annotations\": [ {\n", - " \"annotation_spec\": {\n", - " \"display_name\": \"sedan\",\n", - " \"description\": \"sedan\"\n", - " },\n", - " \"confidence\": 0.52724433,\n", - " \"frames\": [ {\n", - " \"normalized_bounding_box\": {\n", - " \"x_min\": 0.27629745,\n", - " \"y_min\": 0.59244406,\n", - " \"x_max\": 0.53941643,\n", - " \"y_max\": 0.77127469\n", - " },\n", - " \"time_offset\": {\n", - " \n", - " }\n", - " }, {\n", - " \"normalized_bounding_box\": {\n", - " \"x_min\": 0.135607,\n", - " \"y_min\": 0.58437037,\n", - " \"x_max\": 0.42441425,\n", - " \"y_max\": 0.77325606\n", - " },\n", - " \"time_offset\": {\n", - " \"nanos\": 100000000\n", - " }\n", - " }, \n", - " \n", - " # REMOVED FOR BREVITY\n", - " \n", - " }\n", - " } ]\n", - "}\n", - "gs://automl-video-demo-data/traffic_videos/highway_005.mp4,0,315576000000,gs://migration-ucaip-trainingaip-20210310004803/batch_output/prediction-traffic_20210310004803-2021-03-10T01:57:52.257240Z/highway_005_1.json,OK\n", - "```\n", - "\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "cleanup:migration,new" - }, - "source": [ - "# Cleaning up\n", - "\n", - "To clean up all GCP resources used in this project, you can [delete the GCP\n", - "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", - "\n", - "Otherwise, you can delete the individual resources you created in this tutorial.\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "panMCMF8JZJ8", - "outputId": "737f9f35-650f-4329-8b0e-8a9dfe026a1a" - }, - "outputs": [], - "source": [ - "delete_dataset = True\n", - "delete_model = True\n", - "delete_bucket = True\n", - "\n", - "# Delete the dataset using the AutoML fully qualified identifier for the dataset\n", - "try:\n", - " if delete_dataset:\n", - " clients['automl'].delete_dataset(name=dataset_id)\n", - "except Exception as e:\n", - " print(e)\n", - "\n", - "# Delete the model using the AutoML fully qualified identifier for the model\n", - "try:\n", - " if delete_model:\n", - " clients['automl'].delete_model(name=model_id)\n", - "except Exception as e:\n", - " print(e)\n", - "\n", - "if delete_bucket and 'BUCKET_NAME' in globals():\n", - " ! gcloud storage rm --recursive gs://$BUCKET_NAME\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "Z9fbty3uJZJ8" - }, - "outputs": [], - "source": [] - } - ], - "metadata": { - "colab": { - "collapsed_sections": [ - "gcp_authenticate", - "bucket:batch_prediction", - "setup_vars", - "import_aip", - "aip_constants", - "mZwR43s_JZJm", - "request:migration", - "call:migration", - "response:migration", - "OOmLNEw3JZJq", - "T5yWOG67JZJr", - "1shZi6Y4JZJs", - "o3_x634HJZJt", - "text_models_create:migration,old", - "mQD_PyZjJZJu", - "ZfHEsMO5JZJv", - "jYKM6gfvJZJw", - "yfufMwAEJjkX", - "_Zv5eHjwJZJx", - "QtnV8mnJJZJx", - "i6T0bzuNJjkY", - "Nusdm5_MJZJz", - "GB8oloXGJZJz", - "make_batch_prediction_file:migration,new", - "text_models_batchpredict:migration,old", - "zKU3o21IJZJ4", - "ScGH6fbbJZJ5", - "Mh9Jwl8OJZJ5" - ], - "name": "[UJ.15 OLD] AutoML Vision Video Object Tracking.ipynb", - "provenance": [], - "toc_visible": true - }, - "environment": { - "name": "tf2-2-3-gpu.2-3.m55", - "type": "gcloud", - "uri": "gcr.io/deeplearning-platform-release/tf2-2-3-gpu.2-3:m55" - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.8" - } - }, - "nbformat": 4, - "nbformat_minor": 4 + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "copyright" + }, + "outputs": [], + "source": [ + "# Copyright 2021 Google LLC\n", + "#\n", + "# Licensed under the Apache License, Version 2.0 (the \"License\");\n", + "# you may not use this file except in compliance with the License.\n", + "# You may obtain a copy of the License at\n", + "#\n", + "# https://www.apache.org/licenses/LICENSE-2.0\n", + "#\n", + "# Unless required by applicable law or agreed to in writing, software\n", + "# distributed under the License is distributed on an \"AS IS\" BASIS,\n", + "# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n", + "# See the License for the specific language governing permissions and\n", + "# limitations under the License." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "title:migration,new" + }, + "source": [ + "# AutoML SDK: AutoML video object tracking model\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "install_aip" + }, + "source": [ + "## Installation\n", + "\n", + "Install the latest (preview) version of AutoML SDK.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "vQWDe_6HJZJW" + }, + "outputs": [], + "source": [ + "! pip3 install -U google-cloud-automl --user\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "install_storage" + }, + "source": [ + "Install the Google *cloud-storage* library as well.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "5nZRXjD0JZJY" + }, + "outputs": [], + "source": [ + "! pip3 install google-cloud-storage\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "restart" + }, + "source": [ + "### Restart the Kernel\n", + "\n", + "Once you've installed the AutoML SDK and Google *cloud-storage*, you need to restart the notebook kernel so it can find the packages.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "3FaDxry1JZJZ" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "if not os.getenv(\"AUTORUN\"):\n", + " # Automatically restart kernel after installs\n", + " import IPython\n", + "\n", + " app = IPython.Application.instance()\n", + " app.kernel.do_shutdown(True)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "before_you_begin" + }, + "source": [ + "## Before you begin\n", + "\n", + "### GPU run-time\n", + "\n", + "*Make sure you're running this notebook in a GPU runtime if you have that option. In Colab, select* **Runtime > Change Runtime Type > GPU**\n", + "\n", + "### Set up your GCP project\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "1. [Select or create a GCP project](https://console.cloud.google.com/cloud-resource-manager). When you first create an account, you get a $300 free credit towards your compute/storage costs.\n", + "\n", + "2. [Make sure that billing is enabled for your project.](https://cloud.google.com/billing/docs/how-to/modify-project)\n", + "\n", + "3. [Enable the AutoML APIs and Compute Engine APIs.](https://console.cloud.google.com/flows/enableapi?apiid=ml.googleapis.com,compute_component)\n", + "\n", + "4. [Google Cloud SDK](https://cloud.google.com/sdk) is already installed in AutoML Notebooks.\n", + "\n", + "5. Enter your project ID in the cell below. Then run the cell to make sure the\n", + "Cloud SDK uses the right project for all the commands in this notebook.\n", + "\n", + "**Note**: Jupyter runs lines prefixed with `!` as shell commands, and it interpolates Python variables prefixed with `$` into these commands.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "set_project_id" + }, + "outputs": [], + "source": [ + "PROJECT_ID = \"[your-project-id]\" # @param {type:\"string\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "autoset_project_id" + }, + "outputs": [], + "source": [ + "if PROJECT_ID == \"\" or PROJECT_ID is None or PROJECT_ID == \"[your-project-id]\":\n", + " # Get your GCP project id from gcloud\n", + " shell_output = !gcloud config list --format 'value(core.project)' 2>/dev/null\n", + " PROJECT_ID = shell_output[0]\n", + " print(\"Project ID:\", PROJECT_ID)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "set_gcloud_project_id" + }, + "outputs": [], + "source": [ + "! gcloud config set project $PROJECT_ID\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "region" + }, + "source": [ + "#### Region\n", + "\n", + "You can also change the `REGION` variable, which is used for operations\n", + "throughout the rest of this notebook. Below are regions supported for AutoML. We recommend when possible, to choose the region closest to you.\n", + "\n", + "- Americas: `us-central1`\n", + "- Europe: `europe-west4`\n", + "- Asia Pacific: `asia-east1`\n", + "\n", + "You cannot use a Multi-Regional Storage bucket for training with AutoML. Not all regions provide support for all AutoML services. For the latest support per region, see [Region support for AutoML services]()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "jVMUAWrJJZJe" + }, + "outputs": [], + "source": [ + "REGION = \"us-central1\" # @param {type: \"string\"}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "timestamp" + }, + "source": [ + "#### Timestamp\n", + "\n", + "If you are in a live tutorial session, you might be using a shared test account or project. To avoid name collisions between users on resources created, you create a timestamp for each instance session, and append onto the name of resources which will be created in this tutorial.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "LJuwNh9wJZJe" + }, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "\n", + "TIMESTAMP = datetime.now().strftime(\"%Y%m%d%H%M%S\")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gcp_authenticate" + }, + "source": [ + "### Authenticate your GCP account\n", + "\n", + "**If you are using AutoML Notebooks**, your environment is already\n", + "authenticated. Skip this step.\n", + "\n", + "*Note: If you are on an AutoML notebook and run the cell, the cell knows to skip executing the authentication steps.*\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "fEY1QcN_JZJf" + }, + "outputs": [], + "source": [ + "import os\n", + "import sys\n", + "\n", + "# If you are running this notebook in Colab, run this cell and follow the\n", + "# instructions to authenticate your Google Cloud account. This provides access\n", + "# to your Cloud Storage bucket and lets you submit training jobs and prediction\n", + "# requests.\n", + "\n", + "# If on AutoML, then don't execute this code\n", + "if not os.path.exists(\"/opt/deeplearning/metadata/env_version\"):\n", + " if \"google.colab\" in sys.modules:\n", + " from google.colab import auth as google_auth\n", + "\n", + " google_auth.authenticate_user()\n", + "\n", + " # If you are running this tutorial in a notebook locally, replace the string\n", + " # below with the path to your service account key and run this cell to\n", + " # authenticate your Google Cloud account.\n", + " else:\n", + " %env GOOGLE_APPLICATION_CREDENTIALS your_path_to_credentials.json\n", + "\n", + " # Log in to your account on Google Cloud\n", + " ! gcloud auth login" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bucket:batch_prediction" + }, + "source": [ + "### Create a Cloud Storage bucket\n", + "\n", + "**The following steps are required, regardless of your notebook environment.**\n", + "\n", + "This tutorial is designed to use training data that is in a public Cloud Storage bucket and a local Cloud Storage bucket for your batch predictions. You may alternatively use your own training data that you have stored in a local Cloud Storage bucket.\n", + "\n", + "Set the name of your Cloud Storage bucket below. It must be unique across all Cloud Storage buckets.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bucket" + }, + "outputs": [], + "source": [ + "BUCKET_NAME = \"[your-bucket-name]\" # @param {type:\"string\"}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "autoset_bucket" + }, + "outputs": [], + "source": [ + "if BUCKET_NAME == \"\" or BUCKET_NAME is None or BUCKET_NAME == \"[your-bucket-name]\":\n", + " BUCKET_NAME = PROJECT_ID + \"aip-\" + TIMESTAMP" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "create_bucket" + }, + "source": [ + "**Only if your bucket doesn't already exist**: Run the following cell to create your Cloud Storage bucket.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Dc_4SqrZJZJg" + }, + "outputs": [], + "source": [ + "! gcloud storage buckets create --location=$REGION gs://$BUCKET_NAME\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "validate_bucket" + }, + "source": [ + "Finally, validate access to your Cloud Storage bucket by examining its contents:\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gkxndVXCJZJh" + }, + "outputs": [], + "source": [ + "! gcloud storage ls --all-versions --long gs://$BUCKET_NAME\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "setup_vars" + }, + "source": [ + "### Set up variables\n", + "\n", + "Next, set up some variables used throughout the tutorial.\n", + "### Import libraries and define constants\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "import_aip" + }, + "source": [ + "#### Import AutoML SDK\n", + "\n", + "Import the AutoML SDK into our Python environment.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "x3EpSgrNJZJi" + }, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "import sys\n", + "import time\n", + "\n", + "from google.cloud import automl_v1beta1 as automl\n", + "from google.protobuf.json_format import MessageToJson, ParseDict" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "aip_constants" + }, + "source": [ + "#### AutoML constants\n", + "\n", + "Setup up the following constants for AutoML:\n", + "\n", + "- `PARENT`: The AutoML location root path for dataset, model and endpoint resources.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "gIbQItxjJZJj" + }, + "outputs": [], + "source": [ + "# AutoML location root path for your dataset, model and endpoint resources\n", + "PARENT = \"projects/\" + PROJECT_ID + \"/locations/\" + REGION" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "clients" + }, + "source": [ + "## Clients\n", + "\n", + "The AutoML SDK works as a client/server model. On your side (the Python script) you will create a client that sends requests and receives responses from the server (AutoML).\n", + "\n", + "You will use several clients in this tutorial, so set them all up upfront.\n", + "\n", + "- Dataset Service for managed datasets.\n", + "- Model Service for managed models.\n", + "- Pipeline Service for training.\n", + "- Endpoint Service for deployment.\n", + "- Job Service for batch jobs and custom training.\n", + "- Prediction Service for serving. *Note*: Prediction has a different service endpoint.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "eE3k7aqvJZJk" + }, + "outputs": [], + "source": [ + "def automl_client():\n", + " return automl.AutoMlClient()\n", + "\n", + "\n", + "def prediction_client():\n", + " return automl.PredictionServiceClient()\n", + "\n", + "\n", + "def operations_client():\n", + " return automl.AutoMlClient()._transport.operations_client\n", + "\n", + "\n", + "clients = {}\n", + "clients[\"automl\"] = automl_client()\n", + "clients[\"prediction\"] = prediction_client()\n", + "clients[\"operations\"] = operations_client()\n", + "\n", + "for client in clients.items():\n", + " print(client)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "import_file:flowers,csv,icn" + }, + "outputs": [], + "source": [ + "IMPORT_FILE = \"gs://automl-video-demo-data/traffic_videos/traffic_videos.csv\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "1OCKDsyLJZJl" + }, + "outputs": [], + "source": [ + "! gcloud storage cat $IMPORT_FILE | head -n 10\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "trainingpipelines_create:migration,new,response,icn" + }, + "source": [ + "*Example output*:\n", + "```\n", + "UNASSIGNED,gs://automl-video-demo-data/traffic_videos/traffic_videos_labels.csv\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "create_a_dataset:migration" + }, + "source": [ + "## Create a dataset\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mZwR43s_JZJm" + }, + "source": [ + "### [projects.locations.datasets.create](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.datasets/create)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "request:migration" + }, + "source": [ + "#### Request\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "datasets_create:migration,new,request" + }, + "outputs": [], + "source": [ + "dataset = {\n", + " \"display_name\": \"traffic_\" + TIMESTAMP,\n", + " \"video_object_tracking_dataset_metadata\": {},\n", + "}\n", + "\n", + "print(\n", + " MessageToJson(\n", + " automl.CreateDatasetRequest(parent=PARENT, dataset=dataset).__dict__[\"_pb\"]\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZZFAAFAkJZJn" + }, + "source": [ + "*Example output*:\n", + "```\n", + "{\n", + " \"parent\": \"projects/migration-ucaip-training/locations/us-central1\",\n", + " \"dataset\": {\n", + " \"displayName\": \"traffic_20210310004803\",\n", + " \"videoObjectTrackingDatasetMetadata\": {}\n", + " }\n", + "}\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "call:migration" + }, + "source": [ + "#### Call\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "datasets_create:migration,new,call" + }, + "outputs": [], + "source": [ + "request = clients[\"automl\"].create_dataset(parent=PARENT, dataset=dataset)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "response:migration" + }, + "source": [ + "#### Response\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "print:migration,new,response" + }, + "outputs": [], + "source": [ + "result = request\n", + "\n", + "print(MessageToJson(result.__dict__[\"_pb\"]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "datasets_create:migration,new,response" + }, + "source": [ + "*Example output*:\n", + "```\n", + "{\n", + " \"name\": \"projects/116273516712/locations/us-central1/datasets/VOT4951119001817186304\",\n", + " \"displayName\": \"traffic_20210310004803\",\n", + " \"createTime\": \"2021-03-10T00:48:09.292248Z\",\n", + " \"etag\": \"AB3BwFp3u4a2oy-k3EgK6ci8zwrTqrd91_DmoaY8TYsxnb-N-aXwFefqCIm1z0YTM290\",\n", + " \"videoObjectTrackingDatasetMetadata\": {}\n", + "}\n", + "```\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "dataset_id:migration,new,response" + }, + "outputs": [], + "source": [ + "# The full unique ID for the dataset\n", + "dataset_id = result.name\n", + "# The short numeric ID for the dataset\n", + "dataset_short_id = dataset_id.split(\"/\")[-1]\n", + "\n", + "print(dataset_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "OOmLNEw3JZJq" + }, + "source": [ + "### [projects.locations.datasets.importData](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.datasets/importData)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "T5yWOG67JZJr" + }, + "source": [ + "#### Request\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "datasets_import:migration,new,request" + }, + "outputs": [], + "source": [ + "input_config = {\"gcs_source\": {\"input_uris\": [IMPORT_FILE]}}\n", + "\n", + "print(\n", + " MessageToJson(\n", + " automl.ImportDataRequest(\n", + " name=dataset_short_id, input_config=input_config\n", + " ).__dict__[\"_pb\"]\n", + " )\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Dzg9JangJZJs" + }, + "source": [ + "*Example output*:\n", + "```\n", + "{\n", + " \"name\": \"VOT4951119001817186304\",\n", + " \"inputConfig\": {\n", + " \"gcsSource\": {\n", + " \"inputUris\": [\n", + " \"gs://automl-video-demo-data/traffic_videos/traffic_videos.csv\"\n", + " ]\n", + " }\n", + " }\n", + "}\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1shZi6Y4JZJs" + }, + "source": [ + "#### Call\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "datasets_import:migration,new,call" + }, + "outputs": [], + "source": [ + "request = clients[\"automl\"].import_data(name=dataset_id, input_config=input_config)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "o3_x634HJZJt" + }, + "source": [ + "#### Response\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Dc3YtK0jJZJt" + }, + "outputs": [], + "source": [ + "result = request.result()\n", + "\n", + "print(MessageToJson(result))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "datasets_import:migration,new,response" + }, + "source": [ + "*Example output*:\n", + "```\n", + "{}\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "train_a_model:migration" + }, + "source": [ + "## Train a model\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "text_models_create:migration,old" + }, + "source": [ + "### [projects.locations.models.create](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.models/create)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mQD_PyZjJZJu" + }, + "source": [ + "#### Request\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "trainingpipelines_create:migration,new,request,icn" + }, + "outputs": [], + "source": [ + "model = {\n", + " \"display_name\": \"traffic_\" + TIMESTAMP,\n", + " \"dataset_id\": dataset_short_id,\n", + " \"video_object_tracking_model_metadata\": {},\n", + "}\n", + "\n", + "print(\n", + " MessageToJson(automl.CreateModelRequest(parent=PARENT, model=model).__dict__[\"_pb\"])\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "8atEKPpdJZJv" + }, + "source": [ + "*Example output*:\n", + "```\n", + "{\n", + " \"parent\": \"projects/migration-ucaip-training/locations/us-central1\",\n", + " \"model\": {\n", + " \"displayName\": \"traffic_20210310004803\",\n", + " \"datasetId\": \"VOT4951119001817186304\",\n", + " \"videoObjectTrackingModelMetadata\": {}\n", + " }\n", + "}\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZfHEsMO5JZJv" + }, + "source": [ + "#### Call\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "trainingpipelines_create:migration,new,call" + }, + "outputs": [], + "source": [ + "request = clients[\"automl\"].create_model(parent=PARENT, model=model)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jYKM6gfvJZJw" + }, + "source": [ + "#### Response\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "print:migration,new,request" + }, + "outputs": [], + "source": [ + "result = request.result()\n", + "\n", + "print(MessageToJson(result.__dict__[\"_pb\"]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "jdU2ko30JZJw" + }, + "source": [ + "*Example output*:\n", + "```\n", + "{\n", + " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080\"\n", + "}\n", + "```\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "training_pipeline_id:migration,new,response" + }, + "outputs": [], + "source": [ + "# The full unique ID for the training pipeline\n", + "model_id = result.name\n", + "# The short numeric ID for the training pipeline\n", + "model_short_id = model_id.split(\"/\")[-1]\n", + "\n", + "print(model_short_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "evaluate_the_model:migration" + }, + "source": [ + "## Evaluate the model\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yfufMwAEJjkX" + }, + "source": [ + "### [projects.locations.models.modelEvaluations.list](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.models.modelEvaluations/list)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_Zv5eHjwJZJx" + }, + "source": [ + "#### Call\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "models_evaluations_list:migration,new,call" + }, + "outputs": [], + "source": [ + "request = clients[\"automl\"].list_model_evaluations(\n", + " parent=model_id,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "QtnV8mnJJZJx" + }, + "source": [ + "#### Response\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "models_evaluations_list:migration,new,response" + }, + "outputs": [], + "source": [ + "for evaluation in request:\n", + " print(MessageToJson(evaluation.__dict__[\"_pb\"]))\n", + "\n", + "# The last evaluation slice\n", + "last_evaluation_slice = evaluation.name" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "72tAHlBJJZJy" + }, + "source": [ + "*Example output*:\n", + "```\n", + "{\n", + " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080/modelEvaluations/3861662240237183015\",\n", + " \"annotationSpecId\": \"7080515133784457216\",\n", + " \"createTime\": \"2021-03-10T01:57:44.615737Z\",\n", + " \"evaluatedExampleCount\": 6,\n", + " \"videoObjectTrackingEvaluationMetrics\": {\n", + " \"boundingBoxMetricsEntries\": [\n", + " {\n", + " \"iouThreshold\": 0.5,\n", + " \"meanAveragePrecision\": 0.30026233,\n", + " \"confidenceMetricsEntries\": [\n", + " {\n", + " \"recall\": 1.0,\n", + " \"precision\": 0.37222221,\n", + " \"f1Score\": 0.5425101\n", + " },\n", + " {\n", + " \"confidenceThreshold\": 0.028951555,\n", + " \"recall\": 0.13432837,\n", + " \"precision\": 0.07377049,\n", + " \"f1Score\": 0.0952381\n", + " },\n", + " \n", + " # REMOVED FOR BREVITY\n", + " \n", + " {\n", + " \"confidenceThreshold\": 1.0,\n", + " \"precision\": 1.0\n", + " }\n", + " ]\n", + " }\n", + " ],\n", + " \"boundingBoxMeanAveragePrecision\": 0.30026233\n", + " },\n", + " \"displayName\": \"pickup_suv_van\"\n", + "}\n", + "{\n", + " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080/modelEvaluations/4053501621797068779\",\n", + " \"annotationSpecId\": \"5927593629177610240\",\n", + " \"createTime\": \"2021-03-10T01:57:44.615737Z\",\n", + " \"evaluatedExampleCount\": 5,\n", + " \"videoObjectTrackingEvaluationMetrics\": {\n", + " \"boundingBoxMetricsEntries\": [\n", + " {\n", + " \"iouThreshold\": 0.5,\n", + " \"meanAveragePrecision\": 0.42889464,\n", + " \"confidenceMetricsEntries\": [\n", + " {\n", + " \"recall\": 1.0,\n", + " \"precision\": 0.25490198,\n", + " \"f1Score\": 0.40625\n", + " },\n", + " \n", + " # REMOVED FOR BREVITY\n", + " ],\n", + " \"boundingBoxMeanAveragePrecision\": 0.34359422\n", + " },\n", + " \"displayName\": \"sedan\"\n", + "}\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i6T0bzuNJjkY" + }, + "source": [ + "### [projects.locations.models.modelEvaluations.get](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.models.modelEvaluations/get)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Nusdm5_MJZJz" + }, + "source": [ + "#### Call\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "models_evaluations_get:migration,new,call" + }, + "outputs": [], + "source": [ + "request = clients[\"automl\"].get_model_evaluation(name=last_evaluation_slice)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GB8oloXGJZJz" + }, + "source": [ + "#### Response\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "9n1UTT6oJZJ0" + }, + "outputs": [], + "source": [ + "print(MessageToJson(request.__dict__[\"_pb\"]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vqQCboLGJZJ0" + }, + "source": [ + "*Example output*:\n", + "```\n", + "{\n", + " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080/modelEvaluations/6603022638609369541\",\n", + " \"annotationSpecId\": \"1315907610750222336\",\n", + " \"createTime\": \"2021-03-10T01:57:44.615737Z\",\n", + " \"evaluatedExampleCount\": 6,\n", + " \"videoObjectTrackingEvaluationMetrics\": {\n", + " \"boundingBoxMetricsEntries\": [\n", + " {\n", + " \"iouThreshold\": 0.5,\n", + " \"meanAveragePrecision\": 0.34359422,\n", + " \"confidenceMetricsEntries\": [\n", + " {\n", + " \"recall\": 1.0,\n", + " \"precision\": 0.41428572,\n", + " \"f1Score\": 0.5858586\n", + " },\n", + " {\n", + " \"confidenceThreshold\": 0.03328514,\n", + " \"recall\": 0.1724138,\n", + " \"precision\": 0.10869565,\n", + " \"f1Score\": 0.13333334\n", + " },\n", + " \n", + " # REMOVED FOR BREVITY\n", + " ]\n", + " }\n", + " ],\n", + " \"boundingBoxMeanAveragePrecision\": 0.34359422\n", + " },\n", + " \"displayName\": \"sedan\"\n", + "}\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "make_batch_predictions:migration" + }, + "source": [ + "## Make batch predictions\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "make_batch_prediction_file:migration,new" + }, + "source": [ + "### Prepare batch prediction data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "get_test_items:automl,icn,csv" + }, + "outputs": [], + "source": [ + "TRAIN_FILES = \"gs://automl-video-demo-data/traffic_videos/traffic_videos_labels.csv\"\n", + "\n", + "test_items = ! gcloud storage cat $TRAIN_FILES | head -n2\n", + "\n", + "cols = str(test_items[0]).split(\",\")\n", + "test_item_1 = str(cols[0])\n", + "test_label_1 = str(cols[1])\n", + "test_start_time_1 = str(0)\n", + "test_end_time_1 = \"inf\"\n", + "\n", + "print(test_item_1, test_label_1)\n", + "\n", + "cols = str(test_items[1]).split(\",\")\n", + "test_item_2 = str(cols[0])\n", + "test_label_2 = str(cols[1])\n", + "test_start_time_2 = str(0)\n", + "test_end_time_2 = \"inf\"\n", + "\n", + "print(test_item_2, test_label_2)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "mrOPGyQ-JZJ2" + }, + "source": [ + "*Example output*:\n", + "```\n", + "gs://automl-video-demo-data/traffic_videos/highway_005.mp4 sedan\n", + "gs://automl-video-demo-data/traffic_videos/highway_005.mp4 pickup_suv_van\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "make_batch_file:automl,image" + }, + "source": [ + "### Make the batch input file\n", + "\n", + "To request a batch of predictions from AutoML Video, create a CSV file that lists the Cloud Storage paths to the videos that you want to annotate. You can also specify a start and end time to tell AutoML Video to only annotate a segment (segment-level) of the video. The start time must be zero or greater and must be before the end time. The end time must be greater than the start time and less than or equal to the duration of the video. You can also use inf to indicate the end of a video.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_EdGkXQPJZJ2" + }, + "outputs": [], + "source": [ + "import json\n", + "\n", + "import tensorflow as tf\n", + "\n", + "gcs_input_uri = \"gs://\" + BUCKET_NAME + \"/test.csv\"\n", + "with tf.io.gfile.GFile(gcs_input_uri, \"w\") as f:\n", + " data = f\"{test_item_1}, {test_start_time_1}, {test_end_time_1}\"\n", + " f.write(data + \"\\n\")\n", + " data = f\"{test_item_2}, {test_start_time_2}, {test_end_time_2}\"\n", + " f.write(data + \"\\n\")\n", + "\n", + "print(gcs_input_uri)\n", + "!gcloud storage cat $gcs_input_uri" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "PEIM210uJZJ3" + }, + "source": [ + "*Example output*:\n", + "```\n", + "gs://migration-ucaip-trainingaip-20210310004803/test.csv\n", + "gs://automl-video-demo-data/traffic_videos/highway_005.mp4, 0, inf\n", + "gs://automl-video-demo-data/traffic_videos/highway_005.mp4, 0, inf\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "text_models_batchpredict:migration,old" + }, + "source": [ + "### [projects.locations.models.batchPredict](https://cloud.google.com/automl/docs/reference/rest/v1beta1/projects.locations.models/batchPredict)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "zKU3o21IJZJ4" + }, + "source": [ + "#### Request\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "batchpredictionjobs_create:migration,new,request,icn" + }, + "outputs": [], + "source": [ + "input_config = {\"gcs_source\": {\"input_uris\": [gcs_input_uri]}}\n", + "\n", + "output_config = {\n", + " \"gcs_destination\": {\"output_uri_prefix\": \"gs://\" + f\"{BUCKET_NAME}/batch_output/\"}\n", + "}\n", + "\n", + "batch_prediction = automl.BatchPredictRequest(\n", + " name=model_id,\n", + " input_config=input_config,\n", + " output_config=output_config,\n", + ")\n", + "\n", + "print(MessageToJson(batch_prediction.__dict__[\"_pb\"]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "1Y0TXjVNJZJ4" + }, + "source": [ + "*Example output*:\n", + "```\n", + "{\n", + " \"name\": \"projects/116273516712/locations/us-central1/models/VOT6634816000837550080\",\n", + " \"inputConfig\": {\n", + " \"gcsSource\": {\n", + " \"inputUris\": [\n", + " \"gs://migration-ucaip-trainingaip-20210310004803/test.csv\"\n", + " ]\n", + " }\n", + " },\n", + " \"outputConfig\": {\n", + " \"gcsDestination\": {\n", + " \"outputUriPrefix\": \"gs://migration-ucaip-trainingaip-20210310004803/batch_output/\"\n", + " }\n", + " }\n", + "}\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ScGH6fbbJZJ5" + }, + "source": [ + "#### Call\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "batchpredictionjobs_create:migration,new,call" + }, + "outputs": [], + "source": [ + "request = clients[\"prediction\"].batch_predict(request=batch_prediction)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Mh9Jwl8OJZJ5" + }, + "source": [ + "#### Response\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "ep-WdlhxJZJ6" + }, + "outputs": [], + "source": [ + "result = request.result()\n", + "\n", + "print(MessageToJson(result.__dict__[\"_pb\"]))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "batchpredictionjobs_create:migration,new,response,icn" + }, + "source": [ + "*Example output*:\n", + "\n", + "```\n", + "{}\n", + "```\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GW6GcRaaJZJ6" + }, + "outputs": [], + "source": [ + "destination_uri = batch_prediction.output_config.gcs_destination.output_uri_prefix[:-1]\n", + "\n", + "! gcloud storage ls $destination_uri/prediction-**\n", + "! gcloud storage cat $destination_uri/prediction-**" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gy3AJcK5JZJ7" + }, + "source": [ + "*Example output*:\n", + "```\n", + "gs://migration-ucaip-trainingaip-20210310004803/batch_output/prediction-traffic_20210310004803-2021-03-10T01:57:52.257240Z/highway_005_1.json\n", + "gs://migration-ucaip-trainingaip-20210310004803/batch_output/prediction-traffic_20210310004803-2021-03-10T01:57:52.257240Z/video_object_tracking.csv\n", + "{\n", + " \"object_annotations\": [ {\n", + " \"annotation_spec\": {\n", + " \"display_name\": \"sedan\",\n", + " \"description\": \"sedan\"\n", + " },\n", + " \"confidence\": 0.52724433,\n", + " \"frames\": [ {\n", + " \"normalized_bounding_box\": {\n", + " \"x_min\": 0.27629745,\n", + " \"y_min\": 0.59244406,\n", + " \"x_max\": 0.53941643,\n", + " \"y_max\": 0.77127469\n", + " },\n", + " \"time_offset\": {\n", + " \n", + " }\n", + " }, {\n", + " \"normalized_bounding_box\": {\n", + " \"x_min\": 0.135607,\n", + " \"y_min\": 0.58437037,\n", + " \"x_max\": 0.42441425,\n", + " \"y_max\": 0.77325606\n", + " },\n", + " \"time_offset\": {\n", + " \"nanos\": 100000000\n", + " }\n", + " }, \n", + " \n", + " # REMOVED FOR BREVITY\n", + " \n", + " }\n", + " } ]\n", + "}\n", + "gs://automl-video-demo-data/traffic_videos/highway_005.mp4,0,315576000000,gs://migration-ucaip-trainingaip-20210310004803/batch_output/prediction-traffic_20210310004803-2021-03-10T01:57:52.257240Z/highway_005_1.json,OK\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "cleanup:migration,new" + }, + "source": [ + "# Cleaning up\n", + "\n", + "To clean up all GCP resources used in this project, you can [delete the GCP\n", + "project](https://cloud.google.com/resource-manager/docs/creating-managing-projects#shutting_down_projects) you used for the tutorial.\n", + "\n", + "Otherwise, you can delete the individual resources you created in this tutorial.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "panMCMF8JZJ8" + }, + "outputs": [], + "source": [ + "delete_dataset = True\n", + "delete_model = True\n", + "delete_bucket = True\n", + "\n", + "# Delete the dataset using the AutoML fully qualified identifier for the dataset\n", + "try:\n", + " if delete_dataset:\n", + " clients[\"automl\"].delete_dataset(name=dataset_id)\n", + "except Exception as e:\n", + " print(e)\n", + "\n", + "# Delete the model using the AutoML fully qualified identifier for the model\n", + "try:\n", + " if delete_model:\n", + " clients[\"automl\"].delete_model(name=model_id)\n", + "except Exception as e:\n", + " print(e)\n", + "\n", + "if delete_bucket and \"BUCKET_NAME\" in globals():\n", + " ! gcloud storage rm --recursive gs://$BUCKET_NAME" + ] + } + ], + "metadata": { + "colab": { + "collapsed_sections": [ + "gcp_authenticate", + "bucket:batch_prediction", + "setup_vars", + "import_aip", + "aip_constants", + "mZwR43s_JZJm", + "request:migration", + "call:migration", + "response:migration", + "OOmLNEw3JZJq", + "T5yWOG67JZJr", + "1shZi6Y4JZJs", + "o3_x634HJZJt", + "text_models_create:migration,old", + "mQD_PyZjJZJu", + "ZfHEsMO5JZJv", + "jYKM6gfvJZJw", + "yfufMwAEJjkX", + "_Zv5eHjwJZJx", + "QtnV8mnJJZJx", + "i6T0bzuNJjkY", + "Nusdm5_MJZJz", + "GB8oloXGJZJz", + "make_batch_prediction_file:migration,new", + "text_models_batchpredict:migration,old", + "zKU3o21IJZJ4", + "ScGH6fbbJZJ5", + "Mh9Jwl8OJZJ5" + ], + "name": "UJ15 legacy AutoML Vision Video Object Tracking.ipynb", + "toc_visible": true + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + } + }, + "nbformat": 4, + "nbformat_minor": 0 } diff --git a/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb index 21b026d4c..db18dc3b9 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_automl_image_model_batch.ipynb @@ -483,7 +483,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location $REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location $REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -502,7 +503,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -610,10 +612,12 @@ "else:\n", " FILE = IMPORT_FILE\n", "\n", - "count = ! gcloud storage cat $FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $FILE | head" ] + "! gcloud storage cat $FILE | head" + ] }, { "cell_type": "markdown", @@ -865,7 +869,8 @@ }, "outputs": [], "source": [ - "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", "if len(str(test_items[0]).split(\",\")) == 3:\n", + "test_items = !gcloud storage cat $IMPORT_FILE | head -n2\n", + "if len(str(test_items[0]).split(\",\")) == 3:\n", " _, test_item_1, test_label_1 = str(test_items[0]).split(\",\")\n", " _, test_item_2, test_label_2 = str(test_items[1]).split(\",\")\n", "else:\n", @@ -898,7 +903,9 @@ "file_1 = test_item_1.split(\"/\")[-1]\n", "file_2 = test_item_2.split(\"/\")[-1]\n", "\n", - "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", "\n", + "! gcloud storage cp $test_item_1 $BUCKET_URI/$file_1\n", + "! gcloud storage cp $test_item_2 $BUCKET_URI/$file_2\n", + "\n", "test_item_1 = BUCKET_URI + \"/\" + file_1\n", "test_item_2 = BUCKET_URI + \"/\" + file_2" ] @@ -941,7 +948,8 @@ " f.write(json.dumps(data) + \"\\n\")\n", "\n", "print(gcs_input_uri)\n", - "! gcloud storage cat $gcs_input_uri" ] + "! gcloud storage cat $gcs_input_uri" + ] }, { "cell_type": "markdown", @@ -1098,7 +1106,8 @@ "delete_bucket = False\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb b/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb index a052c5c2f..20b406160 100644 --- a/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb +++ b/notebooks/community/ml_ops/stage6/get_started_with_tf_serving_function.ipynb @@ -443,7 +443,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_URI" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -462,7 +463,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_URI" ] + "! gcloud storage ls --all-versions --long $BUCKET_URI" + ] }, { "cell_type": "markdown", @@ -945,7 +947,8 @@ }, "outputs": [], "source": [ - "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" ] + "! gcloud storage cp gs://cloud-ml-data/img/flower_photos/daisy/100080576_f52e8ee070_n.jpg test.jpg" + ] }, { "cell_type": "code", @@ -1047,7 +1050,8 @@ " print(e)\n", "\n", "if delete_bucket or os.getenv(\"IS_TESTING\"):\n", - " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" ] + " ! gcloud storage rm --recursive --continue-on-error {BUCKET_URI}" + ] } ], "metadata": { diff --git a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb index f51d8ced1..529d536bf 100644 --- a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb @@ -203,7 +203,9 @@ "# @markdown If you already obtained access to CodeGemma models on [Hugging Face](https://huggingface.co/), you can load models from there.\n", "# @markdown Alternatively, you can also load the original CodeGemma models for serving from Vertex AI after accepting the agreement.\n", "# @markdown **Select and fill one of the two following sections.**\n", - "LOAD_MODEL_FROM = \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + "LOAD_MODEL_FROM = (\n", + " \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", + ")\n", "\n", "# @markdown #### Access CodeGemma models on Vertex AI\n", "# @markdown Accept the model agreement to access the models:\n", @@ -484,7 +486,9 @@ "# )\n", "# endpoints[\"hexllm_tpu\"] = aiplatform.Endpoint(aip_endpoint_name)\n", "\n", - "prompt = \"Write a function to list n Fibonacci numbers in Python.\" # @param {type: \"string\"}\n", + "prompt = (\n", + " \"Write a function to list n Fibonacci numbers in Python.\" # @param {type: \"string\"}\n", + ")\n", "max_tokens = 500 # @param {type:\"integer\"}\n", "temperature = 1.0 # @param {type:\"number\"}\n", "top_p = 1.0 # @param {type:\"number\"}\n", @@ -540,7 +544,9 @@ "\n", "# @markdown Set the model to deploy.\n", "\n", - "MODEL_ID = \"codegemma-7b-it\" # @param [\"codegemma-2b\", \"codegemma-7b\", \"codegemma-7b-it\"]\n", + "MODEL_ID = (\n", + " \"codegemma-7b-it\" # @param [\"codegemma-2b\", \"codegemma-7b\", \"codegemma-7b-it\"]\n", + ")\n", "model_id = os.path.join(model_path_prefix, MODEL_ID)\n", "\n", "# The pre-built serving docker image for vLLM.\n", @@ -758,7 +764,9 @@ "# )\n", "# endpoints[\"vllm_gpu\"] = aiplatform.Endpoint(aip_endpoint_name)\n", "\n", - "prompt = \"Write a function to list n Fibonacci numbers in Python.\" # @param {type: \"string\"}\n", + "prompt = (\n", + " \"Write a function to list n Fibonacci numbers in Python.\" # @param {type: \"string\"}\n", + ")\n", "max_tokens = 500 # @param {type:\"integer\"}\n", "temperature = 1.0 # @param {type:\"number\"}\n", "top_p = 1.0 # @param {type:\"number\"}\n", diff --git a/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb b/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb index 01965cc4e..44ddaf456 100644 --- a/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb +++ b/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb @@ -862,6 +862,7 @@ "):\n", " \"\"\"Retrieves the exchange rate between two currencies on a specified date.\"\"\"\n", " import requests\n", + "\n", " response = requests.get(\n", " f\"https://api.frankfurter.app/{currency_date}\",\n", " params={\"from\": currency_from, \"to\": currency_to},\n", @@ -909,13 +910,13 @@ "outputs": [], "source": [ "agent = reasoning_engines.LangchainAgent(\n", - " model=MODEL_ID, # Required.\n", - " model_builder=model_builder, # Required.\n", - " tools=[get_exchange_rate], # Optional.\n", + " model=MODEL_ID, # Required.\n", + " model_builder=model_builder, # Required.\n", + " tools=[get_exchange_rate], # Optional.\n", " agent_executor_kwargs={\n", " \"return_intermediate_steps\": True,\n", " \"stream_runnable\": False,\n", - " }, # Optional.\n", + " }, # Optional.\n", ")" ] }, @@ -945,7 +946,9 @@ } ], "source": [ - "response = agent.query(input=\"What's the exchange rate from US dollars to Swedish currency at 2024-07-26?\")\n", + "response = agent.query(\n", + " input=\"What's the exchange rate from US dollars to Swedish currency at 2024-07-26?\"\n", + ")\n", "print(response)" ] }, @@ -1005,7 +1008,9 @@ } ], "source": [ - "response = remote_agent.query(input=\"What's the exchange rate from US dollars to Swedish currency at 2024-07-26?\")\n", + "response = remote_agent.query(\n", + " input=\"What's the exchange rate from US dollars to Swedish currency at 2024-07-26?\"\n", + ")\n", "print(response)" ] }, @@ -1106,7 +1111,8 @@ "delete_reasoning_engine = False # @param {type:\"boolean\"}\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_NAME\n", + "\n", "if delete_reasoning_engine:\n", " remote_agent.delete()" ] diff --git a/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online_export_cloud.ipynb b/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online_export_cloud.ipynb index 00acf7b5b..68578c3b2 100644 --- a/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online_export_cloud.ipynb +++ b/notebooks/community/sdk/sdk_automl_tabular_binary_classification_online_export_cloud.ipynb @@ -472,7 +472,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -491,7 +492,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -594,11 +596,14 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $IMPORT_FILE | head\n", "\n", - "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", + "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -827,12 +832,15 @@ "source": [ "print(\"Model Package:\", model_package)\n", "print(\"Contents:\")\n", - "! gcloud storage ls $model_package\n", "\n", + "! gcloud storage ls $model_package\n", + "\n", "print(\"\\nTF Saved Model\")\n", "path = model_package + \"/predict\"\n", - "files = ! gcloud storage ls $path\n", "saved_dir = files[1]\n", + "files = ! gcloud storage ls $path\n", + "saved_dir = files[1]\n", "print(saved_dir)\n", - "! gcloud storage ls $saved_dir" ] + "! gcloud storage ls $saved_dir" + ] }, { "cell_type": "markdown", @@ -853,7 +861,8 @@ }, "outputs": [], "source": [ - "! gcloud storage cp --recursive $model_package ." ] + "! gcloud storage cp --recursive $model_package ." + ] }, { "cell_type": "markdown", @@ -1146,7 +1155,8 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/community/sdk/sdk_automl_tabular_classification_online_explain.ipynb b/notebooks/community/sdk/sdk_automl_tabular_classification_online_explain.ipynb index 4cf74f233..0d99a83af 100644 --- a/notebooks/community/sdk/sdk_automl_tabular_classification_online_explain.ipynb +++ b/notebooks/community/sdk/sdk_automl_tabular_classification_online_explain.ipynb @@ -493,7 +493,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" ] + "! gcloud storage buckets create --location=$REGION $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -512,7 +513,8 @@ }, "outputs": [], "source": [ - "! gcloud storage ls --all-versions --long $BUCKET_NAME" ] + "! gcloud storage ls --all-versions --long $BUCKET_NAME" + ] }, { "cell_type": "markdown", @@ -615,11 +617,14 @@ }, "outputs": [], "source": [ - "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", "print(\"Number of Examples\", int(count[0]))\n", + "count = ! gcloud storage cat $IMPORT_FILE | wc -l\n", + "print(\"Number of Examples\", int(count[0]))\n", "\n", "print(\"First 10 rows\")\n", - "! gcloud storage cat $IMPORT_FILE | head\n", "\n", - "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", + "! gcloud storage cat $IMPORT_FILE | head\n", + "\n", + "heading = ! gcloud storage cat $IMPORT_FILE | head -n1\n", + "label_column = str(heading).split(\",\")[-1].split(\"'\")[0]\n", "print(\"Label Column Name\", label_column)\n", "if label_column is None:\n", " raise Exception(\"label column missing\")" @@ -1156,7 +1161,8 @@ " print(e)\n", "\n", " if \"BUCKET_NAME\" in globals():\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" ] + " ! gcloud storage rm --recursive $BUCKET_NAME" + ] } ], "metadata": { diff --git a/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb b/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb index 36d25c1cb..8f6d23a54 100644 --- a/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb +++ b/notebooks/official/workbench/predictive_maintainance/predictive_maintenance_usecase.ipynb @@ -339,7 +339,8 @@ }, "outputs": [], "source": [ - "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" ] + "! gcloud storage buckets create --location={LOCATION} {BUCKET_URI}" + ] }, { "cell_type": "markdown", @@ -1216,7 +1217,8 @@ "# Delete the Cloud Storage bucket\n", "delete_bucket = False\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_URI\n", "\n", + " ! gcloud storage rm --recursive $BUCKET_URI\n", + "\n", "!rm -rf model.bst" ] } From 50db8883cb1f5c7ac2ccaeb96adbfc3244e397ce Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Tue, 16 Dec 2025 12:39:11 +0000 Subject: [PATCH 4/6] remove unused imports --- .../UJ15 legacy AutoML Vision Video Object Tracking.ipynb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb b/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb index 79c9b766f..d1fa97205 100644 --- a/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb +++ b/notebooks/community/migration/UJ15 legacy AutoML Vision Video Object Tracking.ipynb @@ -379,13 +379,11 @@ }, "outputs": [], "source": [ - "import json\n", "import os\n", "import sys\n", - "import time\n", "\n", "from google.cloud import automl_v1beta1 as automl\n", - "from google.protobuf.json_format import MessageToJson, ParseDict" + "from google.protobuf.json_format import MessageToJson" ] }, { @@ -1190,8 +1188,6 @@ }, "outputs": [], "source": [ - "import json\n", - "\n", "import tensorflow as tf\n", "\n", "gcs_input_uri = \"gs://\" + BUCKET_NAME + \"/test.csv\"\n", From bb31a00d81d6d7b01c403510d4e1a50e4264565c Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 12:13:36 +0000 Subject: [PATCH 5/6] removed model_garden changes --- ...arden_codegemma_deployment_on_vertex.ipynb | 24 +++++++------------ ...del_garden_reasoning_engine_llama3_1.ipynb | 19 ++++++--------- 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb index 529d536bf..4e1a7faf5 100644 --- a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb @@ -157,7 +157,7 @@ " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", - " shell_output = ! gcloud storage ls --full --buckets {BUCKET_NAME} | grep \"Location Constraint:\" | sed \"s/Location Constraint://\"\n", + " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", " bucket_region = shell_output[0].strip().lower()\n", " if bucket_region != REGION:\n", " raise ValueError(\n", @@ -182,7 +182,7 @@ "\n", "\n", "# Provision permissions to the SERVICE_ACCOUNT with the GCS bucket\n", - "! gcloud storage buckets add-iam-policy-binding $BUCKET_NAME --member=serviceAccount:{SERVICE_ACCOUNT} --role=roles/storage.admin\n", + "! gsutil iam ch serviceAccount:{SERVICE_ACCOUNT}:roles/storage.admin $BUCKET_NAME\n", "\n", "! gcloud config set project $PROJECT_ID\n", "! gcloud projects add-iam-policy-binding --no-user-output-enabled {PROJECT_ID} --member=serviceAccount:{SERVICE_ACCOUNT} --role=\"roles/storage.admin\"\n", @@ -203,9 +203,7 @@ "# @markdown If you already obtained access to CodeGemma models on [Hugging Face](https://huggingface.co/), you can load models from there.\n", "# @markdown Alternatively, you can also load the original CodeGemma models for serving from Vertex AI after accepting the agreement.\n", "# @markdown **Select and fill one of the two following sections.**\n", - "LOAD_MODEL_FROM = (\n", - " \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", - ")\n", + "LOAD_MODEL_FROM = \"Google Cloud\" # @param [\"Hugging Face\", \"Google Cloud\"] {isTemplate:true}\n", "\n", "# @markdown #### Access CodeGemma models on Vertex AI\n", "# @markdown Accept the model agreement to access the models:\n", @@ -240,7 +238,7 @@ "\n", " ! mkdir -p ./codegemma\n", " ! curl -X GET \"{signed_url}\" | tar -xzvf - -C ./codegemma/\n", - " ! gcloud storage cp --recursive ./codegemma/* {BUCKET_URI}\n", + " ! gsutil -m cp -R ./codegemma/* {BUCKET_URI}\n", "\n", " model_path_prefix = BUCKET_URI.strip(\"/\") + \"/codegemma\"\n", "else:\n", @@ -486,9 +484,7 @@ "# )\n", "# endpoints[\"hexllm_tpu\"] = aiplatform.Endpoint(aip_endpoint_name)\n", "\n", - "prompt = (\n", - " \"Write a function to list n Fibonacci numbers in Python.\" # @param {type: \"string\"}\n", - ")\n", + "prompt = \"Write a function to list n Fibonacci numbers in Python.\" # @param {type: \"string\"}\n", "max_tokens = 500 # @param {type:\"integer\"}\n", "temperature = 1.0 # @param {type:\"number\"}\n", "top_p = 1.0 # @param {type:\"number\"}\n", @@ -544,9 +540,7 @@ "\n", "# @markdown Set the model to deploy.\n", "\n", - "MODEL_ID = (\n", - " \"codegemma-7b-it\" # @param [\"codegemma-2b\", \"codegemma-7b\", \"codegemma-7b-it\"]\n", - ")\n", + "MODEL_ID = \"codegemma-7b-it\" # @param [\"codegemma-2b\", \"codegemma-7b\", \"codegemma-7b-it\"]\n", "model_id = os.path.join(model_path_prefix, MODEL_ID)\n", "\n", "# The pre-built serving docker image for vLLM.\n", @@ -764,9 +758,7 @@ "# )\n", "# endpoints[\"vllm_gpu\"] = aiplatform.Endpoint(aip_endpoint_name)\n", "\n", - "prompt = (\n", - " \"Write a function to list n Fibonacci numbers in Python.\" # @param {type: \"string\"}\n", - ")\n", + "prompt = \"Write a function to list n Fibonacci numbers in Python.\" # @param {type: \"string\"}\n", "max_tokens = 500 # @param {type:\"integer\"}\n", "temperature = 1.0 # @param {type:\"number\"}\n", "top_p = 1.0 # @param {type:\"number\"}\n", @@ -828,7 +820,7 @@ "\n", "delete_bucket = False # @param {type:\"boolean\"}\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME" + " ! gsutil -m rm -r $BUCKET_NAME" ] } ], diff --git a/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb b/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb index 44ddaf456..9d7280ffd 100644 --- a/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb +++ b/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb @@ -862,7 +862,6 @@ "):\n", " \"\"\"Retrieves the exchange rate between two currencies on a specified date.\"\"\"\n", " import requests\n", - "\n", " response = requests.get(\n", " f\"https://api.frankfurter.app/{currency_date}\",\n", " params={\"from\": currency_from, \"to\": currency_to},\n", @@ -910,13 +909,13 @@ "outputs": [], "source": [ "agent = reasoning_engines.LangchainAgent(\n", - " model=MODEL_ID, # Required.\n", - " model_builder=model_builder, # Required.\n", - " tools=[get_exchange_rate], # Optional.\n", + " model=MODEL_ID, # Required.\n", + " model_builder=model_builder, # Required.\n", + " tools=[get_exchange_rate], # Optional.\n", " agent_executor_kwargs={\n", " \"return_intermediate_steps\": True,\n", " \"stream_runnable\": False,\n", - " }, # Optional.\n", + " }, # Optional.\n", ")" ] }, @@ -946,9 +945,7 @@ } ], "source": [ - "response = agent.query(\n", - " input=\"What's the exchange rate from US dollars to Swedish currency at 2024-07-26?\"\n", - ")\n", + "response = agent.query(input=\"What's the exchange rate from US dollars to Swedish currency at 2024-07-26?\")\n", "print(response)" ] }, @@ -1008,9 +1005,7 @@ } ], "source": [ - "response = remote_agent.query(\n", - " input=\"What's the exchange rate from US dollars to Swedish currency at 2024-07-26?\"\n", - ")\n", + "response = remote_agent.query(input=\"What's the exchange rate from US dollars to Swedish currency at 2024-07-26?\")\n", "print(response)" ] }, @@ -1111,7 +1106,7 @@ "delete_reasoning_engine = False # @param {type:\"boolean\"}\n", "\n", "if delete_bucket:\n", - " ! gcloud storage rm --recursive $BUCKET_NAME\n", + " ! gsutil -m rm -r $BUCKET_NAME\n", "\n", "if delete_reasoning_engine:\n", " remote_agent.delete()" From cff974d7cbf66872cab13a5f1710605d04594e45 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 22 Dec 2025 12:42:28 +0000 Subject: [PATCH 6/6] updates --- .../model_garden_codegemma_deployment_on_vertex.ipynb | 2 +- .../model_garden/model_garden_reasoning_engine_llama3_1.ipynb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb index 4e1a7faf5..ad78f4f5d 100644 --- a/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb +++ b/notebooks/community/model_garden/model_garden_codegemma_deployment_on_vertex.ipynb @@ -154,7 +154,7 @@ "if BUCKET_URI is None or BUCKET_URI.strip() == \"\" or BUCKET_URI == \"gs://\":\n", " BUCKET_URI = f\"gs://{PROJECT_ID}-tmp-{now}-{str(uuid.uuid4())[:4]}\"\n", " BUCKET_NAME = \"/\".join(BUCKET_URI.split(\"/\")[:3])\n", - " ! gcloud storage buckets create --location {REGION} {BUCKET_URI}\n", + " ! gsutil mb -l {REGION} {BUCKET_URI}\n", "else:\n", " assert BUCKET_URI.startswith(\"gs://\"), \"BUCKET_URI must start with `gs://`.\"\n", " shell_output = ! gsutil ls -Lb {BUCKET_NAME} | grep \"Location constraint:\" | sed \"s/Location constraint://\"\n", diff --git a/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb b/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb index 9d7280ffd..687558297 100644 --- a/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb +++ b/notebooks/community/model_garden/model_garden_reasoning_engine_llama3_1.ipynb @@ -909,7 +909,7 @@ "outputs": [], "source": [ "agent = reasoning_engines.LangchainAgent(\n", - " model=MODEL_ID, # Required.\n", + " model=MODEL_ID, # Required.\n", " model_builder=model_builder, # Required.\n", " tools=[get_exchange_rate], # Optional.\n", " agent_executor_kwargs={\n",