diff --git a/Planetary_Variables_API/.ipynb_checkpoints/Planetary Variables Subscriptions API-checkpoint.ipynb b/Planetary_Variables_API/.ipynb_checkpoints/Planetary Variables Subscriptions API-checkpoint.ipynb new file mode 100644 index 0000000..6df4cc9 --- /dev/null +++ b/Planetary_Variables_API/.ipynb_checkpoints/Planetary Variables Subscriptions API-checkpoint.ipynb @@ -0,0 +1,556 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "59a2ce51", + "metadata": {}, + "source": [ + "# Using the Planetary Variables Subscriptions API\n", + "\n", + "This Jupyter notebook goes over the useage of the Planetary Variables API. As a demonstration, the example of creating a subscription for Biomass Proxy and downloading some of the resulting TIFs is shown.\n", + "\n", + "For more detailed information, you can see the [Planetary Variables API user guide](https://docs.vandersat.com/data_access/api_user_guide.html#data-subscriptions).\n", + "\n", + "Created October 2022" + ] + }, + { + "cell_type": "markdown", + "id": "ea672af9", + "metadata": {}, + "source": [ + "# Part 0 - Getting set up" + ] + }, + { + "cell_type": "markdown", + "id": "7dc5afde", + "metadata": {}, + "source": [ + "### 0.1 Create an environment and install the necessary packages using pip and the requirements.txt file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afdf24ad", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "pip install -r requirements.txt\n" + ] + }, + { + "cell_type": "markdown", + "id": "bb951a3f", + "metadata": {}, + "source": [ + "### 0.2 Import the necessary packages for this notebook" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d73cf03", + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "import logging\n", + "import math\n", + "import os\n", + "from typing import Iterator, List\n", + "import geopandas as gpd\n", + "import json\n", + "import time\n", + "import sys\n", + "\n", + "from requests import Response\n", + "from requests_toolbelt.downloadutils import stream\n", + "from requests_toolbelt.exceptions import StreamingError\n", + "from requests_toolbelt.sessions import BaseUrlSession\n" + ] + }, + { + "cell_type": "markdown", + "id": "9925217b", + "metadata": {}, + "source": [ + "### 0.3 API Functions that Run in the Background" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7229a9e4", + "metadata": {}, + "outputs": [], + "source": [ + "def response_hook(response: Response, *_args, **_kwargs):\n", + " \"\"\"Hook to get detailed error details from the response body.\"\"\"\n", + " if response.status_code >= 400:\n", + " logging.error(\n", + " f\"Error invoking API: url={response.url}; code={response.status_code}; \"\n", + " f\"reason={response.reason}; message={response.text}\"\n", + " )\n", + "\n", + " exit(response.status_code)" + ] + }, + { + "cell_type": "markdown", + "id": "c026623e", + "metadata": {}, + "source": [ + "### 0.4 Enter your credentials for the Planetary Variables API\n", + "It is recommended you do this via the use of environment variables" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8aeac846", + "metadata": {}, + "outputs": [], + "source": [ + "AUTH = (os.getenv(\"PV_DEMO_USER\"), os.getenv(\"PV_DEMO_PASS\"))" + ] + }, + { + "cell_type": "markdown", + "id": "13b2a2fb", + "metadata": {}, + "source": [ + "# Part 1 - Creating a new subscription" + ] + }, + { + "cell_type": "markdown", + "id": "2bb3c760", + "metadata": {}, + "source": [ + "### 1.1 Specify the desired dates, product and geometry for your new subscription\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12b1e069", + "metadata": {}, + "outputs": [], + "source": [ + "# full API name of the desired product, in this case for Biomass Proxy v2.0\n", + "product = \"BIOMASS-PROXY_V2.0_10\"\n", + "\n", + "# between these dates your subscription will be active. the end_date can be in the future\n", + "start_date = \"2022-03-15\"\n", + "end_date = \"2022-09-01\"\n", + "\n", + "# what you would like the same of your data to be\n", + "subscription_name = \"Demo Subscription Biomass Proxy Scotland 1\"\n", + "\n", + "#path to the geometry of your field in .geojson format\n", + "\n", + "field_geom_path = open(os.path.join(sys.path[0], \"Files_for_subscription_demo\", \"demo_field_1.geojson\"))\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "5627bfc9", + "metadata": {}, + "source": [ + "### 1.2 Extract geometry out of geojson file\n", + "This part extracts the geometry of the geojson file provided above so that it is in the correct format for the API to handle." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88955f74", + "metadata": {}, + "outputs": [], + "source": [ + "geodf = gpd.read_file(field_geom_path)\n", + "feature = json.loads(gpd.GeoSeries(geodf.geometry).to_json())\n", + "feature_geometry = feature['features'][0]['geometry']" + ] + }, + { + "cell_type": "markdown", + "id": "f1a3e658", + "metadata": {}, + "source": [ + "### 1.3 Configure the API session" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85894c39", + "metadata": {}, + "outputs": [], + "source": [ + "logging.basicConfig(\n", + " level=logging.INFO, format=\"%(asctime)s %(levelname)s %(message)s\"\n", + " )\n", + "session = BaseUrlSession(base_url=\"https://maps.vandersat.com/api/v2/\")\n", + "session.hooks[\"response\"] = [response_hook]\n", + "session.auth = AUTH" + ] + }, + { + "cell_type": "markdown", + "id": "2bb7972a", + "metadata": {}, + "source": [ + "### 1.4 Create a new subscription\n", + "\n", + "Once run, you will see see from the response from the API that a subscription has been created which has a specific unique id (uuid). You should make a note of this uuid as you will need to use it to retreive the data you have requested.\n", + "#### Note if you are requesting a non-field based product (e.g. soil water content or land surface temperature, the data dictionary must contain the extra \"api_requests_type\" argument\") - see below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e5b0c09", + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# build a dictionary of the parameters for the subscription which can be passed to the API\n", + "\n", + "# use this one if creating a subscription for a field-based product (e.g. biomass proxy)\n", + "data = {\n", + " \"name\": subscription_name,\n", + " \"api_name\": product,\n", + " \"start_date\": start_date,\n", + " \"end_date\": end_date,\n", + " \"geojson\": feature_geometry,\n", + " }\n", + "\n", + "# IMPORTANT NOTE -- \n", + "# if you are creating a subscription for a non-field-based product (i.e. soil water content or land surface temp.)\n", + "# then you need to add an additional entry to the above dictionary e.g.\n", + "\n", + " # \"arguments\": {\"format\": \"gtiff\", \"min_coverage\": 80}\n", + "\n", + " \n", + "result = session.post(url=\"subscriptions\", json=data).json()\n", + "logging.info(f\"Created subscription: {result}\")\n", + "\n", + "# (might take a short while before the response shows below)\n" + ] + }, + { + "cell_type": "markdown", + "id": "62d47140", + "metadata": {}, + "source": [ + "#### Important note: only once a subscription has been made will data processing for the Biomass Proxy begin. Depending on the size of the field, subscription length, and the current processing queue, you will typically have to wait 15 mins + before the data is ready for you to download." + ] + }, + { + "cell_type": "markdown", + "id": "cc5bf013", + "metadata": {}, + "source": [ + "# Part 2 - Downloading data for an exisiting subscription" + ] + }, + { + "cell_type": "markdown", + "id": "0d392af0", + "metadata": {}, + "source": [ + "### 2.1 Configure the API session" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eabb207e", + "metadata": {}, + "outputs": [], + "source": [ + "logging.basicConfig(\n", + " level=logging.INFO, format=\"%(asctime)s %(levelname)s %(message)s\"\n", + " )\n", + "session = BaseUrlSession(base_url=\"https://maps.vandersat.com/api/v2/\")\n", + "session.hooks[\"response\"] = [response_hook]\n", + "session.auth = AUTH" + ] + }, + { + "cell_type": "markdown", + "id": "3b3b3947", + "metadata": {}, + "source": [ + "### 2.2 Functions needed to download via the API\n", + "Ensure this block is ran before proceeding" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "411ba6ba", + "metadata": {}, + "outputs": [], + "source": [ + "def download_files(session: BaseUrlSession, urls: List[str], output_folder: str):\n", + " \"\"\"Save URL(s) using the Content-Disposition header's file name.\"\"\"\n", + " os.makedirs(output_folder, exist_ok=True)\n", + " for url in urls:\n", + " # Find existing files: assume the Content-Disposition header\n", + " # uses the same name as the URL or at most adds a prefix, so a\n", + " # wildcard search suffices. A real application should not rely\n", + " # on that: use the fulfillment date or UUID to track handling.\n", + " name = url.split(\"/\")[-2]\n", + " existing = glob.glob(os.path.join(output_folder, f\"*{name}\"))\n", + " if existing:\n", + " logging.info(f\"Skipped existing file: name={existing[0]}; url={url}\")\n", + " continue\n", + "\n", + " r = session.get(url=url, stream=True)\n", + " try:\n", + " filename = stream.stream_response_to_file(r, path=output_folder)\n", + " logging.info(f\"Downloaded file: name={filename}\")\n", + " except StreamingError as e:\n", + " logging.error(f\"Failed to download file; error={str(e)}; url={url}\")\n", + "\n", + "\n", + "def get_all_pages(\n", + " session: BaseUrlSession, url: str, page_size: int = 50\n", + ") -> Iterator[dict]:\n", + " \"\"\"Get a generator to fetch paginated API results page by page.\"\"\"\n", + " params = {\"page\": 1, \"limit\": page_size}\n", + " first_page = session.get(url=url, params=params).json()\n", + " yield first_page\n", + "\n", + " page_count = math.ceil(first_page[\"total_items\"] / page_size)\n", + " for params[\"page\"] in range(2, page_count + 1):\n", + " next_page = session.get(url=url, params=params).json()\n", + " yield next_page\n", + " \n", + "\n", + "def handle_fulfillment(\n", + " session: BaseUrlSession, subscription_uuid: str, fulfillment: dict\n", + ") -> bool:\n", + " \"\"\"Handle a single fulfillment, like from an HTTP notification.\"\"\"\n", + " if fulfillment[\"status\"] == \"Ready\":\n", + " output_folder = os.path.join(dir_for_data_download, subscription_uuid)\n", + "\n", + " download_files(session, urls=fulfillment[\"files\"], output_folder=output_folder)\n", + "\n", + " return fulfillment[\"status\"] in (\"Ready\", \"Error\")\n", + "\n", + "\n", + "def get_subscription_fulfillments(\n", + " session: BaseUrlSession, subscription_uuid: str\n", + ") -> bool:\n", + " \"\"\"Get fulfillments; not needed when using HTTP notifications.\"\"\"\n", + " url = f\"subscriptions/{subscription_uuid}/fulfillments\"\n", + " pending = None\n", + " for page in get_all_pages(session, url=url):\n", + " logging.info(f\"Fetched page of fulfillments: result={page}\")\n", + " for fulfillment in page[\"fulfillments\"]:\n", + " pending = pending or not handle_fulfillment(\n", + " session, subscription_uuid=subscription_uuid, fulfillment=fulfillment\n", + " )\n", + " # True if fulfillment(s) found and all were handled, False otherwise\n", + " return not pending\n" + ] + }, + { + "cell_type": "markdown", + "id": "1f444777", + "metadata": {}, + "source": [ + "### 2.3 Define which subscription you would like to download data for\n", + "\n", + "Since it takes some time for the Biomass Proxy to actually process after a field is submitted and be ready for download, here is a uuid I prepared earlier for use here:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e098b5e0", + "metadata": {}, + "outputs": [], + "source": [ + "# the uuid of the subscription you want to download the data for\n", + "uuid = \"401d528b-053b-45ba-b3d1-3cecc5aeb650\"" + ] + }, + { + "cell_type": "markdown", + "id": "efb7f0c1", + "metadata": {}, + "source": [ + "### 2.4 Download the tif files of your subscription " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c91a3752", + "metadata": {}, + "outputs": [], + "source": [ + "# the directory where the data should be saved\n", + "dir_for_data_download = os.path.join(sys.path[0], \"bp_tifs_output\")\n", + "\n", + "while True:\n", + " # Get the fulfillments and download the resulting files\n", + " if get_subscription_fulfillments(session, subscription_uuid=uuid):\n", + " break\n", + " logging.info(\"Not done yet; sleeping 10 minutes\")\n", + " time.sleep(10 * 60)\n", + " \n", + "session.close()" + ] + }, + { + "cell_type": "markdown", + "id": "b13f5412", + "metadata": {}, + "source": [ + "# Bonus Part - Visualize a Subscription Geotiff File" + ] + }, + { + "cell_type": "markdown", + "id": "f1cb00ad", + "metadata": {}, + "source": [ + "### 3.0.1 Install required package - rasterio for plotting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1a863c7", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "pip install rasterio" + ] + }, + { + "cell_type": "markdown", + "id": "5e2f58ac", + "metadata": {}, + "source": [ + "### 3.0.2 Import required packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3300a92", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "from rasterio.plot import show" + ] + }, + { + "cell_type": "markdown", + "id": "cf48f278", + "metadata": {}, + "source": [ + "### 3.1 Specify parameters for the plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26ce277b", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# where your tifs are saved\n", + "path_to_saved_tifs = os.path.join(sys.path[0], \"bp_tifs_output\")\n", + "\n", + "# the uuid of the subscription you would like to visualize\n", + "uuid_to_visualize = \"401d528b-053b-45ba-b3d1-3cecc5aeb650\"\n", + "\n", + "# the product associated with this subscription that you will plot\n", + "product = \"BIOMASS-PROXY_V2.0_10\"\n", + "\n", + "# the date you would like to visualize\n", + "date_to_visualize = \"2022-05-01\"" + ] + }, + { + "cell_type": "markdown", + "id": "9838642f", + "metadata": {}, + "source": [ + "### 3.2 Visualize a Single TIF with Colorbar using Matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78e6d282", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "path_to_tif = os.path.join(path_to_saved_tifs, uuid_to_visualize, \"{}_{}_{}.tif\".format(product, uuid_to_visualize, date_to_visualize))\n", + "\n", + "fig, ax = plt.subplots(figsize=(4, 4))\n", + "ax.set_xlabel(\"Degrees Longitude\")\n", + "ax.set_ylabel(\"Degrees Latitude\")\n", + "ax.ticklabel_format(useOffset=False)\n", + "\n", + "# open the tif file using Rasterio\n", + "img = rasterio.open(path_to_tif)\n", + "\n", + "# plot the opened tif\n", + "plot = show(img, \n", + " ax=ax, \n", + " cmap='Greens')\n", + "\n", + "# add colorbar\n", + "im = image.get_images()[0]\n", + "fig.colorbar(im, ax=ax, label=product, fraction=0.046, pad=0.04)\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/Planetary_Variables_API/Planetary Variables Subscriptions API.ipynb b/Planetary_Variables_API/Planetary Variables Subscriptions API.ipynb new file mode 100644 index 0000000..6df4cc9 --- /dev/null +++ b/Planetary_Variables_API/Planetary Variables Subscriptions API.ipynb @@ -0,0 +1,556 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "59a2ce51", + "metadata": {}, + "source": [ + "# Using the Planetary Variables Subscriptions API\n", + "\n", + "This Jupyter notebook goes over the useage of the Planetary Variables API. As a demonstration, the example of creating a subscription for Biomass Proxy and downloading some of the resulting TIFs is shown.\n", + "\n", + "For more detailed information, you can see the [Planetary Variables API user guide](https://docs.vandersat.com/data_access/api_user_guide.html#data-subscriptions).\n", + "\n", + "Created October 2022" + ] + }, + { + "cell_type": "markdown", + "id": "ea672af9", + "metadata": {}, + "source": [ + "# Part 0 - Getting set up" + ] + }, + { + "cell_type": "markdown", + "id": "7dc5afde", + "metadata": {}, + "source": [ + "### 0.1 Create an environment and install the necessary packages using pip and the requirements.txt file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "afdf24ad", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "pip install -r requirements.txt\n" + ] + }, + { + "cell_type": "markdown", + "id": "bb951a3f", + "metadata": {}, + "source": [ + "### 0.2 Import the necessary packages for this notebook" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4d73cf03", + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "import logging\n", + "import math\n", + "import os\n", + "from typing import Iterator, List\n", + "import geopandas as gpd\n", + "import json\n", + "import time\n", + "import sys\n", + "\n", + "from requests import Response\n", + "from requests_toolbelt.downloadutils import stream\n", + "from requests_toolbelt.exceptions import StreamingError\n", + "from requests_toolbelt.sessions import BaseUrlSession\n" + ] + }, + { + "cell_type": "markdown", + "id": "9925217b", + "metadata": {}, + "source": [ + "### 0.3 API Functions that Run in the Background" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7229a9e4", + "metadata": {}, + "outputs": [], + "source": [ + "def response_hook(response: Response, *_args, **_kwargs):\n", + " \"\"\"Hook to get detailed error details from the response body.\"\"\"\n", + " if response.status_code >= 400:\n", + " logging.error(\n", + " f\"Error invoking API: url={response.url}; code={response.status_code}; \"\n", + " f\"reason={response.reason}; message={response.text}\"\n", + " )\n", + "\n", + " exit(response.status_code)" + ] + }, + { + "cell_type": "markdown", + "id": "c026623e", + "metadata": {}, + "source": [ + "### 0.4 Enter your credentials for the Planetary Variables API\n", + "It is recommended you do this via the use of environment variables" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8aeac846", + "metadata": {}, + "outputs": [], + "source": [ + "AUTH = (os.getenv(\"PV_DEMO_USER\"), os.getenv(\"PV_DEMO_PASS\"))" + ] + }, + { + "cell_type": "markdown", + "id": "13b2a2fb", + "metadata": {}, + "source": [ + "# Part 1 - Creating a new subscription" + ] + }, + { + "cell_type": "markdown", + "id": "2bb3c760", + "metadata": {}, + "source": [ + "### 1.1 Specify the desired dates, product and geometry for your new subscription\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12b1e069", + "metadata": {}, + "outputs": [], + "source": [ + "# full API name of the desired product, in this case for Biomass Proxy v2.0\n", + "product = \"BIOMASS-PROXY_V2.0_10\"\n", + "\n", + "# between these dates your subscription will be active. the end_date can be in the future\n", + "start_date = \"2022-03-15\"\n", + "end_date = \"2022-09-01\"\n", + "\n", + "# what you would like the same of your data to be\n", + "subscription_name = \"Demo Subscription Biomass Proxy Scotland 1\"\n", + "\n", + "#path to the geometry of your field in .geojson format\n", + "\n", + "field_geom_path = open(os.path.join(sys.path[0], \"Files_for_subscription_demo\", \"demo_field_1.geojson\"))\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "5627bfc9", + "metadata": {}, + "source": [ + "### 1.2 Extract geometry out of geojson file\n", + "This part extracts the geometry of the geojson file provided above so that it is in the correct format for the API to handle." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88955f74", + "metadata": {}, + "outputs": [], + "source": [ + "geodf = gpd.read_file(field_geom_path)\n", + "feature = json.loads(gpd.GeoSeries(geodf.geometry).to_json())\n", + "feature_geometry = feature['features'][0]['geometry']" + ] + }, + { + "cell_type": "markdown", + "id": "f1a3e658", + "metadata": {}, + "source": [ + "### 1.3 Configure the API session" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85894c39", + "metadata": {}, + "outputs": [], + "source": [ + "logging.basicConfig(\n", + " level=logging.INFO, format=\"%(asctime)s %(levelname)s %(message)s\"\n", + " )\n", + "session = BaseUrlSession(base_url=\"https://maps.vandersat.com/api/v2/\")\n", + "session.hooks[\"response\"] = [response_hook]\n", + "session.auth = AUTH" + ] + }, + { + "cell_type": "markdown", + "id": "2bb7972a", + "metadata": {}, + "source": [ + "### 1.4 Create a new subscription\n", + "\n", + "Once run, you will see see from the response from the API that a subscription has been created which has a specific unique id (uuid). You should make a note of this uuid as you will need to use it to retreive the data you have requested.\n", + "#### Note if you are requesting a non-field based product (e.g. soil water content or land surface temperature, the data dictionary must contain the extra \"api_requests_type\" argument\") - see below" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4e5b0c09", + "metadata": { + "scrolled": false + }, + "outputs": [], + "source": [ + "# build a dictionary of the parameters for the subscription which can be passed to the API\n", + "\n", + "# use this one if creating a subscription for a field-based product (e.g. biomass proxy)\n", + "data = {\n", + " \"name\": subscription_name,\n", + " \"api_name\": product,\n", + " \"start_date\": start_date,\n", + " \"end_date\": end_date,\n", + " \"geojson\": feature_geometry,\n", + " }\n", + "\n", + "# IMPORTANT NOTE -- \n", + "# if you are creating a subscription for a non-field-based product (i.e. soil water content or land surface temp.)\n", + "# then you need to add an additional entry to the above dictionary e.g.\n", + "\n", + " # \"arguments\": {\"format\": \"gtiff\", \"min_coverage\": 80}\n", + "\n", + " \n", + "result = session.post(url=\"subscriptions\", json=data).json()\n", + "logging.info(f\"Created subscription: {result}\")\n", + "\n", + "# (might take a short while before the response shows below)\n" + ] + }, + { + "cell_type": "markdown", + "id": "62d47140", + "metadata": {}, + "source": [ + "#### Important note: only once a subscription has been made will data processing for the Biomass Proxy begin. Depending on the size of the field, subscription length, and the current processing queue, you will typically have to wait 15 mins + before the data is ready for you to download." + ] + }, + { + "cell_type": "markdown", + "id": "cc5bf013", + "metadata": {}, + "source": [ + "# Part 2 - Downloading data for an exisiting subscription" + ] + }, + { + "cell_type": "markdown", + "id": "0d392af0", + "metadata": {}, + "source": [ + "### 2.1 Configure the API session" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eabb207e", + "metadata": {}, + "outputs": [], + "source": [ + "logging.basicConfig(\n", + " level=logging.INFO, format=\"%(asctime)s %(levelname)s %(message)s\"\n", + " )\n", + "session = BaseUrlSession(base_url=\"https://maps.vandersat.com/api/v2/\")\n", + "session.hooks[\"response\"] = [response_hook]\n", + "session.auth = AUTH" + ] + }, + { + "cell_type": "markdown", + "id": "3b3b3947", + "metadata": {}, + "source": [ + "### 2.2 Functions needed to download via the API\n", + "Ensure this block is ran before proceeding" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "411ba6ba", + "metadata": {}, + "outputs": [], + "source": [ + "def download_files(session: BaseUrlSession, urls: List[str], output_folder: str):\n", + " \"\"\"Save URL(s) using the Content-Disposition header's file name.\"\"\"\n", + " os.makedirs(output_folder, exist_ok=True)\n", + " for url in urls:\n", + " # Find existing files: assume the Content-Disposition header\n", + " # uses the same name as the URL or at most adds a prefix, so a\n", + " # wildcard search suffices. A real application should not rely\n", + " # on that: use the fulfillment date or UUID to track handling.\n", + " name = url.split(\"/\")[-2]\n", + " existing = glob.glob(os.path.join(output_folder, f\"*{name}\"))\n", + " if existing:\n", + " logging.info(f\"Skipped existing file: name={existing[0]}; url={url}\")\n", + " continue\n", + "\n", + " r = session.get(url=url, stream=True)\n", + " try:\n", + " filename = stream.stream_response_to_file(r, path=output_folder)\n", + " logging.info(f\"Downloaded file: name={filename}\")\n", + " except StreamingError as e:\n", + " logging.error(f\"Failed to download file; error={str(e)}; url={url}\")\n", + "\n", + "\n", + "def get_all_pages(\n", + " session: BaseUrlSession, url: str, page_size: int = 50\n", + ") -> Iterator[dict]:\n", + " \"\"\"Get a generator to fetch paginated API results page by page.\"\"\"\n", + " params = {\"page\": 1, \"limit\": page_size}\n", + " first_page = session.get(url=url, params=params).json()\n", + " yield first_page\n", + "\n", + " page_count = math.ceil(first_page[\"total_items\"] / page_size)\n", + " for params[\"page\"] in range(2, page_count + 1):\n", + " next_page = session.get(url=url, params=params).json()\n", + " yield next_page\n", + " \n", + "\n", + "def handle_fulfillment(\n", + " session: BaseUrlSession, subscription_uuid: str, fulfillment: dict\n", + ") -> bool:\n", + " \"\"\"Handle a single fulfillment, like from an HTTP notification.\"\"\"\n", + " if fulfillment[\"status\"] == \"Ready\":\n", + " output_folder = os.path.join(dir_for_data_download, subscription_uuid)\n", + "\n", + " download_files(session, urls=fulfillment[\"files\"], output_folder=output_folder)\n", + "\n", + " return fulfillment[\"status\"] in (\"Ready\", \"Error\")\n", + "\n", + "\n", + "def get_subscription_fulfillments(\n", + " session: BaseUrlSession, subscription_uuid: str\n", + ") -> bool:\n", + " \"\"\"Get fulfillments; not needed when using HTTP notifications.\"\"\"\n", + " url = f\"subscriptions/{subscription_uuid}/fulfillments\"\n", + " pending = None\n", + " for page in get_all_pages(session, url=url):\n", + " logging.info(f\"Fetched page of fulfillments: result={page}\")\n", + " for fulfillment in page[\"fulfillments\"]:\n", + " pending = pending or not handle_fulfillment(\n", + " session, subscription_uuid=subscription_uuid, fulfillment=fulfillment\n", + " )\n", + " # True if fulfillment(s) found and all were handled, False otherwise\n", + " return not pending\n" + ] + }, + { + "cell_type": "markdown", + "id": "1f444777", + "metadata": {}, + "source": [ + "### 2.3 Define which subscription you would like to download data for\n", + "\n", + "Since it takes some time for the Biomass Proxy to actually process after a field is submitted and be ready for download, here is a uuid I prepared earlier for use here:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e098b5e0", + "metadata": {}, + "outputs": [], + "source": [ + "# the uuid of the subscription you want to download the data for\n", + "uuid = \"401d528b-053b-45ba-b3d1-3cecc5aeb650\"" + ] + }, + { + "cell_type": "markdown", + "id": "efb7f0c1", + "metadata": {}, + "source": [ + "### 2.4 Download the tif files of your subscription " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c91a3752", + "metadata": {}, + "outputs": [], + "source": [ + "# the directory where the data should be saved\n", + "dir_for_data_download = os.path.join(sys.path[0], \"bp_tifs_output\")\n", + "\n", + "while True:\n", + " # Get the fulfillments and download the resulting files\n", + " if get_subscription_fulfillments(session, subscription_uuid=uuid):\n", + " break\n", + " logging.info(\"Not done yet; sleeping 10 minutes\")\n", + " time.sleep(10 * 60)\n", + " \n", + "session.close()" + ] + }, + { + "cell_type": "markdown", + "id": "b13f5412", + "metadata": {}, + "source": [ + "# Bonus Part - Visualize a Subscription Geotiff File" + ] + }, + { + "cell_type": "markdown", + "id": "f1cb00ad", + "metadata": {}, + "source": [ + "### 3.0.1 Install required package - rasterio for plotting" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c1a863c7", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "pip install rasterio" + ] + }, + { + "cell_type": "markdown", + "id": "5e2f58ac", + "metadata": {}, + "source": [ + "### 3.0.2 Import required packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d3300a92", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "import matplotlib.pyplot as plt\n", + "from rasterio.plot import show" + ] + }, + { + "cell_type": "markdown", + "id": "cf48f278", + "metadata": {}, + "source": [ + "### 3.1 Specify parameters for the plot" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26ce277b", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# where your tifs are saved\n", + "path_to_saved_tifs = os.path.join(sys.path[0], \"bp_tifs_output\")\n", + "\n", + "# the uuid of the subscription you would like to visualize\n", + "uuid_to_visualize = \"401d528b-053b-45ba-b3d1-3cecc5aeb650\"\n", + "\n", + "# the product associated with this subscription that you will plot\n", + "product = \"BIOMASS-PROXY_V2.0_10\"\n", + "\n", + "# the date you would like to visualize\n", + "date_to_visualize = \"2022-05-01\"" + ] + }, + { + "cell_type": "markdown", + "id": "9838642f", + "metadata": {}, + "source": [ + "### 3.2 Visualize a Single TIF with Colorbar using Matplotlib" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78e6d282", + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "path_to_tif = os.path.join(path_to_saved_tifs, uuid_to_visualize, \"{}_{}_{}.tif\".format(product, uuid_to_visualize, date_to_visualize))\n", + "\n", + "fig, ax = plt.subplots(figsize=(4, 4))\n", + "ax.set_xlabel(\"Degrees Longitude\")\n", + "ax.set_ylabel(\"Degrees Latitude\")\n", + "ax.ticklabel_format(useOffset=False)\n", + "\n", + "# open the tif file using Rasterio\n", + "img = rasterio.open(path_to_tif)\n", + "\n", + "# plot the opened tif\n", + "plot = show(img, \n", + " ax=ax, \n", + " cmap='Greens')\n", + "\n", + "# add colorbar\n", + "im = image.get_images()[0]\n", + "fig.colorbar(im, ax=ax, label=product, fraction=0.046, pad=0.04)\n", + "\n", + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/requirements.txt b/requirements.txt index 32817b0..6f2a15a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,4 +10,6 @@ geojsonio matplotlib mercantile scikit-image -scikit-learn \ No newline at end of file +scikit-learn +requests_toolbelt +rasterio