From e574848d0893a4331c1720ad98c1e73858fb50eb Mon Sep 17 00:00:00 2001 From: Quercus Hamlin Date: Fri, 28 Aug 2026 13:43:25 -0400 Subject: [PATCH 1/3] docs / cleanup --- README.md | 104 +++--- docs/builds/POI_gages_builder.md | 1 + docs/builds/divide_attributes.md | 2 + docs/builds/flowpath_attributes.md | 2 + docs/builds/irrigation.md | 79 ----- docs/builds/lakes.md | 2 + docs/builds/quickstart.md | 2 + docs/builds/reservoir_attrs.md | 1 + docs/development/development.md | 54 +-- mkdocs.yml | 7 +- src/hydrofabric_builds/builds/README.md | 15 - src/hydrofabric_builds/builds/irrigation.py | 123 ------- src/hydrofabric_builds/builds/lakes.py | 363 -------------------- src/hydrofabric_builds/builds/network.py | 148 -------- tools/builds/irrigation/run_irrigation.py | 126 ------- tools/builds/lakes/lakes_hydrofabric.py | 77 ----- tools/builds/lakes/usbr/usbr_hydrofabric.py | 85 ----- tools/builds/lakes/usbr/usbr_prep.ipynb | 332 ------------------ 18 files changed, 60 insertions(+), 1463 deletions(-) delete mode 100644 docs/builds/irrigation.md delete mode 100644 src/hydrofabric_builds/builds/README.md delete mode 100644 src/hydrofabric_builds/builds/irrigation.py delete mode 100644 src/hydrofabric_builds/builds/lakes.py delete mode 100644 src/hydrofabric_builds/builds/network.py delete mode 100644 tools/builds/irrigation/run_irrigation.py delete mode 100644 tools/builds/lakes/lakes_hydrofabric.py delete mode 100644 tools/builds/lakes/usbr/usbr_hydrofabric.py delete mode 100644 tools/builds/lakes/usbr/usbr_prep.ipynb diff --git a/README.md b/README.md index 1ee26f0..4d15dad 100644 --- a/README.md +++ b/README.md @@ -3,43 +3,56 @@ Building Hydrofabric & Processing Ancillary Data hydrofabric - -#### Proposed Schema +# About the Data +## Schema The following schema is the proposed data model for NGWPC hydrofabric datasets produced by this repo. +TODO: Update + nhf_v1.1.2_schema.png -##### Flowpaths FACT Table +## Flowpaths FACT Table The central table (or FACT Table) is `Flowpaths`. Each `flowpath` has a downstream, and upstream `nexus` point, allowing for traversal of a river network through a single table. Additionally, there is a 1:1 relationship between `flowpath` and `divide`. -##### NGEN Tables +## NGEN Tables The tables highlighted in green are the infomation needed for lumped modeling to take place. Lumped models require attributes, the shape of the `divide` that is being modeled, and a `nexus` point for flow to be aggregated to. -##### Routing Tables +## Routing Tables The tables highlighted in blue contain the information needed for routing at a high resolution. T-Route is expected to run at a fine-scale (~300m segments) with many `virtual_flowpaths`. Each virtual flowpath is delineated based on the reference fabric, and there should be a many -> one relationship between `virtual_flowpaths` and `flowpaths`, with some `virtual flowpaths` not being represented in the `flowpaths` table. These non-represented `flowpaths` have the parameter of `routing_segment` set to False, and will have flow estimated through flow-scaling. -##### Reference Crosswalks +The `reservoir_da` table encodes crosswalks between lakes and gages with an assigned data assimilation code. The `lakes_polygons` layer mirrors the traditional `lakes` point layer, but includes the polygon representation. This polygon representation is used to derive the flowpaths associated with lakes for routing. The `lake_vfp_crosswalk` table contains the intersection of lake polygons and virtual flowpaths so that T-route treats all lake flowpaths as lakes rather than channels. + +## Reference Crosswalks -The NGWPC Hydrofabric is build using many reference materials: +The NGWPC Hydrofabric is built using many reference materials: - Reference Flowpaths - Reference Reservoirs -- USGS/ENVCA/CADWR/TXDOT Streamflow Gages +- Reference Waterbodies +- NWM v3 Lakes +- National Inventory of Dams +- USGS/ENVCA/CADWR/TXDOT/RFC/USBR/USACE Streamflow Gages - NHD+ To ensure `flowpaths` can be mapped to back to the materials that created them, each of the reference materials is mapped to `flowpaths`, `hydrolocations`, and `virtual flowpaths`. The following IDs pairings are used: - Reference Flowpaths -> `ref_fp_id` - Reference Reservoirs -> `dam_id` -- USGS/ENVCA/CADWR/TXDOT Streamflow Gages -> `site_no` +- Reference Reservoirs -> `ref_fab_wb` is `lake_id` / NHD `COMID` +- Streamflow Gages -> `site_no` - NHD+ -> `nhd_feature_id` -##### Visual Diagram +## Validation +The `validate_hf` task in the pipeline produces a JSON report called `nhf_{version}_validation.json`. This report details various metrics from the built product, such as: number of null divide attributes, number of attributes out of defined minimum and maxium range, and assertions that necessary lakes and gages are present and assigned to flowpaths. + + +## Visual Diagram NHF Diagram + # Development Commands Run these commands from the repository root. @@ -53,56 +66,8 @@ The following command installs the project's base dependencies, the `docs` optio uv sync --all-extras --all-groups ``` -Python 3.12 or newer is required. - -## Sync input data using the `justfile` - -`just` calls series of commands called "recipes" similar to a `make` file. Install on linux with `apt get just` or follow linked readme for other platforms. After installing `just`, you can use the following commands to set up the data sources for `nhf-builds`. You can also use `just` to build hydrofabrics for each domain or specify a config. - -Provide AWS credentials in the current shell: - -```bash -export AWS_DEFAULT_REGION="us-east-1" -export AWS_ACCESS_KEY_ID="..." -export AWS_SECRET_ACCESS_KEY="..." -export AWS_SESSION_TOKEN="..." # Required for temporary credentials -``` - -Verify that AWS recognizes the credentials: - -```bash -aws sts get-caller-identity -``` - -Then sync the input data for the desired domain: - -```bash -just sync # CONUS -just sync-ak # Alaska -just sync-hi # Hawaii -just sync-prvi # Puerto Rico and the US Virgin Islands -``` - -To select a different OCONUS reference-fabric version, pass the `oconus-version` variable: - -```bash -just oconus-version=0.1.8 sync-ak -``` - -> **Warning:** The sync recipes overwrite the corresponding input datasets under `data/`. - -### AWS credential handling - -Exporting AWS credentials in the shell is functionally sufficient because `just` and its child processes inherit those environment variables. The `justfile` also automatically loads variables from a repository-root `.env` file. - -Avoid committing credentials or entering long-lived secrets directly into commands that may be saved in shell history. When available, prefer an AWS SSO or named-profile workflow for data synchronization: - -```bash -aws sso login --profile ngwpc-test -AWS_PROFILE=ngwpc-test just sync -``` - -An AWS profile is sufficient for the `aws s3` commands used by the sync recipes. Some hydrofabric build paths access S3 credentials directly through `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN`, so a profile alone may not be sufficient for every build configuration. +## Unpack Data +Extract the `hydrofabric_builds_data.tar` archive to the `data` folder. This archive includes all data for running the canonical NHF for all domains. ## Run the hydrofabric build @@ -112,7 +77,9 @@ Run the main build script directly with the CONUS example configuration: uv run python scripts/hf_runner.py --config configs/example_config.yaml ``` -Alternatively, use a domain-specific `just` recipe: +Alternatively, use a domain-specific `just` recipe. + +`just` calls series of commands called "recipes" similar to a `make` file. Install on linux with `apt get just` or follow linked readme for other platforms. After installing `just`, you can use the following commands to build the hydrofabric. ```bash just build-conus @@ -127,6 +94,21 @@ Run the build with a custom configuration: just build "configs/my_custom_config.yaml" ``` +## Documentation +This repository has documentation that can be served via [mkdocs](https://www.mkdocs.org/). +Ensure that dependencies are installed: + +```bash +uv sync --extra docs +``` +To serve docs locally, run: + +```bash +mkdocs serve -a localhost:8080 +``` +Navigate to `localhost:8080/` in your browser. + + ## Run tests ### Run the complete test suite diff --git a/docs/builds/POI_gages_builder.md b/docs/builds/POI_gages_builder.md index 18590cd..840cd9e 100644 --- a/docs/builds/POI_gages_builder.md +++ b/docs/builds/POI_gages_builder.md @@ -1,4 +1,5 @@ ## POI/Gages Builder — Integration Guide +TODO: Update This document explains how to assemble a single, canonical gages layer by merging USGS (active + discontinued) and partner gage sources (TXDOT, CADWR, ENVCA, NWM calibration sets, AK/HI/PR supplements). You’ll download the source files from the S3 bucket (see below), place them in your local user directory. ### Download source files: diff --git a/docs/builds/divide_attributes.md b/docs/builds/divide_attributes.md index bee6104..9ca3b66 100644 --- a/docs/builds/divide_attributes.md +++ b/docs/builds/divide_attributes.md @@ -1,5 +1,7 @@ # Divide Attributes Build +TODO: Update + The divide attributes task calculates zonal statistics for divides from a number of rasters. To run: diff --git a/docs/builds/flowpath_attributes.md b/docs/builds/flowpath_attributes.md index fb57a1a..ac5154a 100644 --- a/docs/builds/flowpath_attributes.md +++ b/docs/builds/flowpath_attributes.md @@ -1,5 +1,7 @@ # Flowpath Attributes Build +TODO: Update + The flowpath attributes task calculates metrics per flowpath linestring from multiple sources: DEM, WRF defaults, and RiverML outputs. ## To run: diff --git a/docs/builds/irrigation.md b/docs/builds/irrigation.md deleted file mode 100644 index 3ed9f2f..0000000 --- a/docs/builds/irrigation.md +++ /dev/null @@ -1,79 +0,0 @@ -# Building Irrigation - -## Introduction -Here, irrigation is considered a list of crops that may be irrigated. This build downloads the USDA Cropland Data Layer for requested years, converts to a mask at native resolution (30 meter), and optionally regrids to a coarser grid and resolution where the new raster represents percent of cell irrigated. - -USDA CDL reference as of 8/20/25: https://www.nass.usda.gov/Research_and_Science/Cropland/SARS1a.php - -CDL Crops considered irrigation: -- 1 - corn, -- 3 - rice -- 5 - soybean -- 12 - sweet corn -- 13 - pop or orn corn -- 92 - aquaculture -- 250 - cranberry - -`tools/builds/run_irrigation.py` can perform a full pipeline of: -- download CDL year(s) -- unzip files -- classify to binary raster of irrigated crops -- aggregate temporally to create one raster where any cell with irrigation in requested year range is irrigated will be considered irrigated -- resample and re-align to a coarser resolution grid where each cell represents percent irrigated - -## Running - -!!! warning - Running at the CONUS scale consumes significant memory even when using dask arrays. If classifying and aggregating, **~80 GB RAM** may be used. If regridding at 250 meter resolution, **~100 GB** may be used. - -!!! warning - Downloading and unzipping 10 years of CDL will use **~80 GB** disk space. You can safely delete `.ovr` files (5 GB) to reduce disk space. - -!!! note - Running the full pipeline including re-gridding may create memory leaks. It is recommended to re-grid in a separate call after downloading, unzipping, classifying, and aggregating. This can be done using the `--no-download`, `--no-unzip`, `--no-classify` `--no-aggreegate` flags discussed below. Each layer can take 10+ minutes to regrid. - -#### Pipeline Demo -Run an example of the pipeline (excluding re-gridding) for two years to demonstrate processes: - -1. Create or update virtual environment: `uv sync` - -2. Run -```sh -python tools/builds/irrigation/run_irrigation.py -w ./data --min_yr 2015 --max_yr 2016 -``` - -#### Full pipeline used for the f1 Trainer CONUS CNN: - -1. Download CONUS grid file from s3 (Currently located on `data` account at `s3://fim-services-data/f1/data/conus.tif`) and save to `./data` - -2. Create or update virtual environment: `uv sync` - -3. First download, classify, and aggregate layers. -```sh -python tools/builds/irrigation/run_irrigation.py -w ./data --min_yr 2015 --max_yr 2024 -``` - -4. Then, use skip flags to only do regridding. Regridding can be done with the other steps; however, it can have even higher memory requirements. -```sh -python tools/builds/irrigation/run_irrigation.py -w ./data --min_yr 2015 --max_yr 2024 -g ./data/conus.tif --no_download --no_unzip --no_classify --no_aggregate -``` - -!!! note - To "pick up where you left off": use flags `--no-download`, `--no-unzip`, `--no-classify`, `--no-aggregate` as necessary. Regridding will only be done when `--grid` / `-g` is specified. - - -#### Example: If you already processed all years but only need to regrid, run: -```sh -python tools/builds/irrigation/run_irrigation.py -w ./data --my_yr 2015 --max_yr 2016 -g ./data/conus.tif --no_download --no_unzip --no_classify --no-aggregate -``` - -#### Example: If you already processed all years and only need to regrid **one** layer, run: -```sh -python tools/builds/irrigation/run_irrigation.py -w ./data --my_yr 2015 --max_yr 2016 -g ./data/conus.tif --grid_yr 2015_2016 --no_download --no_unzip --no_classify --no-aggregate -``` -Note that the temporally aggregated "20xx_20xx" string is accepted. Otherwise, use a single year number. - -#### Example: If you already download and unzipped all layers but need to classify and aggregate, run: -```sh -python tools/builds/irrigation/run_irrigation.py -w -./data --min_yr 2015 --max_yr 2025 --no-download --no-unzip -``` diff --git a/docs/builds/lakes.md b/docs/builds/lakes.md index acbb78a..5de422b 100644 --- a/docs/builds/lakes.md +++ b/docs/builds/lakes.md @@ -1,5 +1,7 @@ # Lakes +TODO: Update + The NHF lakes layer integrates lakes and reservoir data from multiple sources. Hydraulic parameters for t-route are calculated. ## Data sources diff --git a/docs/builds/quickstart.md b/docs/builds/quickstart.md index 24aed32..e1372a8 100644 --- a/docs/builds/quickstart.md +++ b/docs/builds/quickstart.md @@ -1,5 +1,7 @@ ### Quickstart +TODO: Replace s3 paths with unpacking file + ## CONUS Below are the data files needed for running a full build of the NHF dataset and their locations. Each of the `aws` commands should be run from the NGWPC Test account diff --git a/docs/builds/reservoir_attrs.md b/docs/builds/reservoir_attrs.md index 88f0b45..77d2e0c 100644 --- a/docs/builds/reservoir_attrs.md +++ b/docs/builds/reservoir_attrs.md @@ -1,5 +1,6 @@ ## Reservoirs Description: +TODO: Update Below is a summary of what the Reservoir Python script is doing and the dependencies it assumes. Potential future action items for each data source has been descussed here for record. This issue is mainly about capturing and solidifying the current workflow so we can (a) understand the data dependencies, and (b) reproduce the same processing in our Python-based toolchain. diff --git a/docs/development/development.md b/docs/development/development.md index 21afe95..ae3967e 100644 --- a/docs/development/development.md +++ b/docs/development/development.md @@ -13,54 +13,8 @@ uv sync --all-extras --all-groups Python 3.12 or newer is required. -## Sync input data using the `justfile` - -`just` calls series of commands called "recipes" similar to a `make` file. Install on linux with `apt get just` or follow linked readme for other platforms. After installing `just`, you can use the following commands to set up the data sources for `nhf-builds`. You can also use `just` to build hydrofabrics for each domain or specify a config. - -Provide AWS credentials in the current shell: - -```bash -export AWS_DEFAULT_REGION="us-east-1" -export AWS_ACCESS_KEY_ID="..." -export AWS_SECRET_ACCESS_KEY="..." -export AWS_SESSION_TOKEN="..." # Required for temporary credentials -``` - -Verify that AWS recognizes the credentials: - -```bash -aws sts get-caller-identity -``` - -Then sync the input data for the desired domain: - -```bash -just sync # CONUS -just sync-ak # Alaska -just sync-hi # Hawaii -just sync-prvi # Puerto Rico and the US Virgin Islands -``` - -To select a different OCONUS reference-fabric version, pass the `oconus-version` variable: - -```bash -just oconus-version=0.1.8 sync-ak -``` - -> **Warning:** The sync recipes overwrite the corresponding input datasets under `data/`. - -### AWS credential handling - -Exporting AWS credentials in the shell is functionally sufficient because `just` and its child processes inherit those environment variables. The `justfile` also automatically loads variables from a repository-root `.env` file. - -Avoid committing credentials or entering long-lived secrets directly into commands that may be saved in shell history. When available, prefer an AWS SSO or named-profile workflow for data synchronization: - -```bash -aws sso login --profile ngwpc-test -AWS_PROFILE=ngwpc-test just sync -``` - -An AWS profile is sufficient for the `aws s3` commands used by the sync recipes. Some hydrofabric build paths access S3 credentials directly through `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN`, so a profile alone may not be sufficient for every build configuration. +## Unpack Data +Extract the `hydrofabric_builds_data.tar` archive to the `data` folder. This archive includes all data for running the canonical NHF for all domains. ## Run the hydrofabric build @@ -70,7 +24,9 @@ Run the main build script directly with the CONUS example configuration: uv run python scripts/hf_runner.py --config configs/example_config.yaml ``` -Alternatively, use a domain-specific `just` recipe: +Alternatively, use a domain-specific `just` recipe. + +`just` calls series of commands called "recipes" similar to a `make` file. Install on linux with `apt get just` or follow linked readme for other platforms. After installing `just`, you can use the following commands to build the hydrofabric. ```bash just build-conus diff --git a/mkdocs.yml b/mkdocs.yml index c34c295..e689b31 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,7 +1,7 @@ site_name: hydrofabric-builds site_description: Building Hydrofabric(s) and processing ancillary data -repo_url: https://github.com/NGWPC/hydrofabric-builds -repo_name: NGWPC/hydrofabric-builds +repo_url: https://github.com/noaa-owp/hydrofabric-builds +repo_name: NOAA-OWP/hydrofabric-builds theme: name: material features: @@ -48,9 +48,6 @@ plugins: nav: - Home: - index.md - - Builds: - - - Irrigation: builds/irrigation.md - Hydrofabric: - builds/index.md - Divide Attributes: builds/divide_attributes.md diff --git a/src/hydrofabric_builds/builds/README.md b/src/hydrofabric_builds/builds/README.md deleted file mode 100644 index 49210a1..0000000 --- a/src/hydrofabric_builds/builds/README.md +++ /dev/null @@ -1,15 +0,0 @@ -Create lakes layer geopackage by running: -python lakes_hydrofabric.py --lakeparm_file --ana_res_file --ext_res_file --med_res_file --short_res_file --hffile --output_path - ---lakeparm_file full path and filename of lakeparms netcdf file, e.g, LAKEPARM_CONUS_216.nc ---ana_res_file full path and filename of ana res netcdf file, e.g., reservoir_index_AnA_309.nc ---ext_res_file full path and filename of extended res netcdf file, e.g., reservoir_index_Extended_AnA.nc ---med_res_file full path and filename of medium range res netcdf file, e.g., reservoir_index_Medium_Range.nc ---short_res_file full path and filename of medium range res netcdf file, e.g., reservoir_index_Short_Range.nc ---hffile full path and filename of hydrofabric geopackage ---output_path full path to directory where outputs will be saved ---domain hydrofabric domain. Only CONUS at this time. - -Check this lake layer against hydrofabric lakes layer by running tools/check_lakes.py -This script checks for lakes that are present in one dataset but not in the other and -returns a csv file showing differences in the lakes layer table. diff --git a/src/hydrofabric_builds/builds/irrigation.py b/src/hydrofabric_builds/builds/irrigation.py deleted file mode 100644 index b7c7780..0000000 --- a/src/hydrofabric_builds/builds/irrigation.py +++ /dev/null @@ -1,123 +0,0 @@ -from pathlib import Path -from zipfile import ZipFile - -import requests -import xarray as xr -from rasterio.enums import Resampling -from tqdm import tqdm - -from hydrofabric_builds.helpers.spatial import regrid_percent -from hydrofabric_builds.schemas.irrigation_constants import CDL_URL_BASE, CROPS - - -def classify_irrigation( - min_yr: int, max_yr: int, wd: Path, download: bool, unzip: bool, classify: bool, aggregate: bool -) -> None: - """Classify the USDA NASS Cropland Data Layer to crops likely to be irrigated. - - The final output is a single boolean layer where 1 represents at least one irrigated crop year and 0 represents None. - Intermediate outputs are a classified layer for each year. - - Includes flags to skip undesired parts of pipeline. If no flags, the entire pipeline will be skipped. - Flags include: - - download: if files should be downloaded - - unzip: if downloaded zips should be unzipped - - classify: if each year's layer should be classified - - aggeregate: if the range of requested years should be temporally aggregated to one layer - - NOTE: .ovr files included in CDL zips can be deleted (5 GB each) - - NOTE: RAM requirements - Running CONUS-scale layers may need 20-40 GB RAM - - Args: - min_yr (int): First CDL year to process - max_yr (int): Last year to process. If only one year needed, set max_yr to equal min_yr - wd (Path): Working directory for data - download (bool): Flag to download files: If True, will download zips. If False, skip downloading - unzip (bool): Flag to unzip files: If True, will unzip. If False, skip unzipping - classify (bool): Flag to classify individual layers based on crop list. If True, all layers will be classified - If False, skipp classification - aggregate (bool): Flag to aggregate all layers temporally. - This will generate one boolean layer where True means at least one year had irrigated crop - """ - # set years - if min_yr == max_yr: - cdl_years = [min_yr] - else: - cdl_years = range(min_yr, max_yr + 1) # type: ignore[assignment] - output_path = wd / f"irrigation_{min_yr}_{max_yr}.tif" - - cdl_urls = [f"{CDL_URL_BASE}{year}_30m_cdls.zip" for year in cdl_years] - - # Download from source - if download: - for i, year in enumerate(tqdm(cdl_years)): - print(f"Downloading {cdl_urls[i]}") - response = requests.get(cdl_urls[i]) - with open(wd / f"{year}_30m_cdls.zip", mode="wb") as f: - f.write(response.content) - - # unzip downloaded files - # NOTE: .ovr files can be deleted if need additional space - if unzip: - for year in tqdm(cdl_years): - print(f"Unzipping {year}") - with ZipFile(wd / f"{year}_30m_cdls.zip") as z: - z.extractall(wd) - - # classify individual years - if classify: - for yr in tqdm(cdl_years): - print(f"Classifying {yr}") - cdl = wd / f"{yr}_30m_cdls.tif" - ds = xr.open_dataset(cdl, engine="rasterio", chunks="auto", masked=True).astype("uint8") - temp_ds = xr.where(ds.astype("uint8").band_data.isin(CROPS), 1, 0).astype("uint8").compute() - - print(f"Writing {yr} to tif") - temp_ds.rio.to_raster(wd / f"irrigation_{yr}.tif", compress="deflate", tiled="YES", crs=5070) - del ds, temp_ds - - # Aggregate all years requested to single boolean layer - if aggregate: - if len(cdl_years) < 2: - print("Temporal aggregation was requested, but only one year was provided. Skipping aggregation.") - return - - ds = xr.open_dataset( - wd / f"irrigation_{cdl_years[0]}.tif", engine="rasterio", chunks="auto", masked=True - ).astype("uint8") - ds_new = ds.copy(deep=True) - crs = ds.rio.crs - del ds - - for yr in tqdm(cdl_years[1:]): - print(f"Aggregating {yr}") - ds = xr.open_dataset( - wd / f"irrigation_{yr}.tif", engine="rasterio", chunks="auto", masked=True - ).astype("uint8") - ds_new = xr.where(ds == 1, 1, ds_new).astype("uint8").compute() - del ds - - print("Saving temporally aggregated raster") - ds_new.rio.write_crs(crs, inplace=True) - ds_new.band_data.rio.to_raster(output_path, compress="deflate", tiled="YES") - del ds_new - print(f"Saved temporally aggregated raster to {output_path}") - - -def regrid_irrigation(wd: Path, grid_path: Path, regrid_list: list[str]) -> None: - """Resample and regrid irrigation layers. Converts fine to coarse resolution where value is percent of irrigated cells. - - Args: - wd (Path): Working directory for data - grid_path (Path_): Path to grid to match - regrid_list (list[str]): List of years to regrid - """ - print(f"Regridding {regrid_list} - each year may take 10+ minutes") - for yr in tqdm(regrid_list): - input_path = wd / f"irrigation_{yr}.tif" - output_path = wd / f"irrigation_{yr}_regrid.tif" - - regrid_percent( - grid_path=grid_path, input_path=input_path, output_path=output_path, resampling=Resampling.nearest - ) diff --git a/src/hydrofabric_builds/builds/lakes.py b/src/hydrofabric_builds/builds/lakes.py deleted file mode 100644 index f650c52..0000000 --- a/src/hydrofabric_builds/builds/lakes.py +++ /dev/null @@ -1,363 +0,0 @@ -from pathlib import Path - -import geopandas as gpd -import pandas as pd -import xarray as xr -from pyproj import Transformer - -from hydrofabric_builds.schemas.hydrofabric import HydrofabricCRS, HydrofabricDomainsGPKG - - -def build_lakes( - lakeparm_file: str, - ana_res_file: str, - ext_res_file: str, - med_res_file: str, - short_res_file: str, - hf_file: str, - output_path: str, - domain: str, -) -> None: - """Builds the hydrofabric lakes layer - - Parameters - ---------- - lakeparm_file : str - full path and filename of lakeparms netcdf file, e.g, LAKEPARM_CONUS_216.nc - ana_res_file : str - full path and filename of ana res netcdf file, e.g., reservoir_index_AnA_309.nc - ext_res_file : str - full path and filename of extended res netcdf file, e.g., reservoir_index_Extended_AnA.nc - med_res_file : str - full path and filename of medium range res netcdf file, e.g., reservoir_index_Medium_Range.nc - short_res_file : str - full path and filename of short range res netcdf file, e.g., reservoir_index_Short_Range.nc - hf_file : str - full path and filename of hydrofabric geopackage - output_path : str - full path to directory where outputs will be saved - domain : str - domain to be created (CONUS, AK, GL, HI, PRVI) - """ - # Set domain and crs - domain = HydrofabricDomainsGPKG[domain].value - domain_crs = HydrofabricCRS[domain].value - - # read lakesparm file into dataframe - lakes = xr.open_dataset(lakeparm_file) - lakes = lakes.to_dataframe() - - # Read data from reservoir files - ana = xr.open_dataset(ana_res_file) - ana_featureid = pd.DataFrame({"lake_id": ana["lake_id"], "reservoir_type": ana["reservoir_type"]}) - ana_usgs_comid = pd.DataFrame({"usgs_lake_id": ana["usgs_lake_id"], "usgs_gage_id": ana["usgs_gage_id"]}) - ana_usace_comid = pd.DataFrame( - {"usace_lake_id": ana["usace_lake_id"], "usace_gage_id": ana["usace_gage_id"]} - ) - ana_rfc_comid = pd.DataFrame({"rfc_lake_id": ana["rfc_lake_id"], "rfc_gage_id": ana["rfc_gage_id"]}) - ana_usgs_comid["usgs_gage_id"] = ana_usgs_comid["usgs_gage_id"].str.decode("utf-8") - ana_usace_comid["usace_gage_id"] = ana_usace_comid["usace_gage_id"].str.decode("utf-8") - ana_rfc_comid["rfc_gage_id"] = ana_rfc_comid["rfc_gage_id"].str.decode("utf-8") - - ext = xr.open_dataset(ext_res_file) - ext = pd.DataFrame({"lake_id": ext["lake_id"], "reservoir_type": ext["reservoir_type"]}) - - med = xr.open_dataset(med_res_file) - med = pd.DataFrame({"lake_id": med["lake_id"], "reservoir_type": med["reservoir_type"]}) - - short = xr.open_dataset(short_res_file) - short = pd.DataFrame({"lake_id": short["lake_id"], "reservoir_type": short["reservoir_type"]}) - - lake_ids = lakes["lake_id"].to_list() - - # create reservoir layer - # get list of lakes in reserviors table where reservoir type > 1. - ana_filtered = ana_featureid[ana_featureid["reservoir_type"] > 1] - ana_res = ana_filtered["lake_id"].to_list() - - ext_filtered = ext[ext["reservoir_type"] > 1] - ext_res = ext_filtered["lake_id"].to_list() - - med_filtered = med[med["reservoir_type"] > 1] - med_res = med_filtered["lake_id"].to_list() - - short_filtered = short[short["reservoir_type"] > 1] - short_res = short_filtered["lake_id"].to_list() - - # read hydrolocations layer from hydrofabric - hl = gpd.read_file(hf_file, layer="hydrolocations") - - # create empty lists for reservoir table rows and for storing the res id cooresponding to a lake id - res_rows = [] - res_id_lakes = [] - - # create reservoir layer - # loop through lake ids cooresponding to reservoirs - for lake in ana_res: - # get lat/lon from lake parm dataframe - x = lakes[lakes["lake_id"] == lake]["lon"].item() - y = lakes[lakes["lake_id"] == lake]["lat"].item() - - # find poi in hydrolocations using lake hl_uri - hl_uri = f"lake-{lake}" - poi = hl[hl["hl_uri"] == hl_uri]["poi_id"] - if not poi.empty: - poi = poi.unique().item() - else: - poi = None - - res = { - "poi_id": poi, - "comid": None, - "domain": domain, - "hl_reference": "lake", - "hl_link": lake, - "x": x, - "y": y, - "hl_source": "NOAAOWP", - } - res_rows.append(res) - - # hl_reference = usgs-gage row - usgs_gage_id = ana_usgs_comid[ana_usgs_comid["usgs_lake_id"] == lake]["usgs_gage_id"] - if not usgs_gage_id.empty: - usgs_gage_id = usgs_gage_id.unique().item() - hl_uri = f"usgs-gage-{usgs_gage_id}" - poi = hl[hl["hl_uri"] == hl_uri]["poi_id"] - if not poi.empty: - poi = poi.unique().item() - else: - poi = None - - res = { - "poi_id": poi, - "comid": None, - "domain": domain, - "hl_reference": "usgs-gage", - "hl_link": usgs_gage_id, - "x": x, - "y": y, - "hl_source": "NOAAOWP", - } - res_rows.append(res) - - # hl_reference = usace-gage row - usace_gage_id = ana_usace_comid[ana_usace_comid["usace_lake_id"] == lake]["usace_gage_id"] - if not usace_gage_id.empty: - usace_gage_id = usace_gage_id.unique().item() - hl_uri = f"usace-gage-{usace_gage_id}" - poi = hl[hl["hl_uri"] == hl_uri]["poi_id"] - if not poi.empty: - poi = poi.unique().item() - else: - poi = None - - res = { - "poi_id": poi, - "comid": None, - "domain": domain, - "hl_reference": "usace-gage", - "hl_link": usace_gage_id, - "x": x, - "y": y, - "hl_source": "NOAAOWP", - } - res_rows.append(res) - - # hl_reference = rfc-gage row - rfc_gage_id = ana_rfc_comid[ana_rfc_comid["rfc_lake_id"] == lake]["rfc_gage_id"] - if not rfc_gage_id.empty: - rfc_gage_id = rfc_gage_id.unique().item() - hl_uri = f"rfc-gage-{usace_gage_id}" - poi = hl[hl["hl_uri"] == hl_uri]["poi_id"] - if not poi.empty: - poi = poi.unique().item() - else: - poi = None - - res = { - "poi_id": poi, - "comid": None, - "domain": domain, - "hl_reference": "rfc-gage", - "hl_link": rfc_gage_id, - "x": x, - "y": y, - "hl_source": "NOAAOWP", - } - res_rows.append(res) - - if lake in ext_res: - hl_uri = f"lake-{lake}" - poi = hl[hl["hl_uri"] == hl_uri]["poi_id"] - if not poi.empty: - poi = poi.unique().item() - res_id = ( - hl.loc[(hl["poi_id"] == poi) & (hl["hl_reference"] == "reservoir"), "hl_link"] - .unique() - .item() - ) - res_id_lake = {"lake_id": lake, "res_id": res_id} - res_id_lakes.append(res_id_lake) - # res_id = hl.loc[(hl['poi_id'] == poi) & (hl['hl_reference'] == 'reservoir'),'hl_link'].item() - else: - poi = None - res_id = None - - res = { - "poi_id": poi, - "comid": None, - "domain": domain, - "hl_reference": "reservoir", - "hl_link": res_id, - "x": x, - "y": y, - "hl_source": "reservoir_index_Extended_Range", - } - res_rows.append(res) - - if lake in med_res: - res = { - "poi_id": poi, - "comid": None, - "domain": domain, - "hl_reference": "reservoir", - "hl_link": res_id, - "x": x, - "y": y, - "hl_source": "reservoir_index_Medium_Range", - } - res_rows.append(res) - - if lake in short_res: - res = { - "poi_id": poi, - "comid": None, - "domain": domain, - "hl_reference": "reservoir", - "hl_link": res_id, - "x": x, - "y": y, - "hl_source": "reservoir_index_Short_Range", - } - res_rows.append(res) - - if lake in ana_res: - res = { - "poi_id": poi, - "comid": None, - "domain": domain, - "hl_reference": "reservoir", - "hl_link": res_id, - "x": x, - "y": y, - "hl_source": "reservoir_index_AnA", - } - res_rows.append(res) - - df = pd.DataFrame(res_rows) - df.to_csv(f"{output_path}/nwm_res.csv") - - res_id_lakes = pd.DataFrame(res_id_lakes) - - # Lakes Layer - - # copy lat and lon columns to y and x - lakes["x"] = lakes["lon"] - lakes["y"] = lakes["lat"] - lakes["domain"] = domain - - # Get pois from hydrolocations layer - lake_ids = [str(x) for x in lake_ids] - pois = hl.loc[(hl["hl_link"].isin(lake_ids)) & (hl["hl_reference"] == "LAKEPARM")][["hl_link", "poi_id"]] - pois = pois.rename(columns={"hl_link": "lake_id"}) - pois["lake_id"] = pois["lake_id"].astype(int) - lakes = lakes.join(pois.set_index("lake_id"), on="lake_id") - - # join res ids to lakes - lakes = lakes.join(res_id_lakes.set_index("lake_id"), on="lake_id") - - # join reservoir type for AnA if greater than 1 - lakes = lakes.join(ana_featureid.set_index("lake_id"), on="lake_id") - lakes = lakes.rename(columns={"reservoir_type": "reservoir_index_AnA"}) - lakes.loc[lakes["reservoir_index_AnA"] == 1, "reservoir_index_AnA"] = None - - # join reservoir type for extended if greater than 1 - lakes = lakes.join(ext.set_index("lake_id"), on="lake_id") - lakes = lakes.rename(columns={"reservoir_type": "reservoir_index_Extended_AnA"}) - lakes.loc[lakes["reservoir_index_Extended_AnA"] == 1, "reservoir_index_Extended_AnA"] = None - - # join reservoir type for medium if greater than 1 - lakes = lakes.join(med.set_index("lake_id"), on="lake_id") - lakes = lakes.rename(columns={"reservoir_type": "reservoir_index_Medium_Range"}) - lakes.loc[lakes["reservoir_index_Medium_Range"] == 1, "reservoir_index_Medium_Range"] = None - - # join reservoir type for short if greater than 1 - lakes = lakes.join(short.set_index("lake_id"), on="lake_id") - lakes = lakes.rename(columns={"reservoir_type": "reservoir_index_Short_Range"}) - lakes.loc[lakes["reservoir_index_Short_Range"] == 1, "reservoir_index_Short_Range"] = None - - # write lakes layer to csv - lakes.to_csv(f"{output_path}/nwm_lakes.csv") - - # Create lakes layer from NWM lakes table - lakes_layer = lakes - - # Convert wgs84 lat/lon to CONUS Albers coordinates for lakes_x and lakes_y columns - transformer = Transformer.from_crs("EPSG:4326", domain_crs, always_xy=True) - - for index, row in lakes_layer.iterrows(): - y = row["y"] - x = row["x"] - conus_albers = transformer.transform(x, y) - lakes_layer.loc[index, "x"] = conus_albers[0] # latitude - lakes_layer.loc[index, "y"] = conus_albers[1] # longitude - - # rename x and y to lake_x and lake_y to match hf data model - lakes_layer = lakes_layer.rename(columns={"y": "lake_y"}) - lakes_layer = lakes_layer.rename(columns={"x": "lake_x"}) - - # add columns for hf_id and vpu_id. Still need to figure out how to populate these - lakes_layer["hf_id"] = None - lakes_layer["vpu_id"] = None - - # remove time, lat, and lon columns to match hf data model. - remove_cols = ["time", "lat", "lon"] - lakes_layer = lakes_layer.drop(remove_cols, axis=1) - - # change order of columns to match hf lakes layer - lakes_cols = [ - "lake_id", - "LkArea", - "LkMxE", - "WeirC", - "WeirL", - "OrificeC", - "OrificeA", - "OrificeE", - "WeirE", - "ifd", - "Dam_Length", - "domain", - "poi_id", - "hf_id", - "reservoir_index_AnA", - "reservoir_index_Extended_AnA", - "reservoir_index_GDL_AK", - "reservoir_index_Medium_Range", - "reservoir_index_Short_Range", - "res_id", - "vpuid", - "lake_x", - "lake_y", - ] - lakes_layer = lakes_layer.reindex(columns=lakes_cols) - - # Convert to a geo data frame and save as a geopackage - gdf = gpd.GeoDataFrame( - lakes_layer, geometry=gpd.points_from_xy(lakes_layer["lake_x"], lakes_layer["lake_y"], crs=domain_crs) - ) - - # Write geopackage - gpkg_path = Path(f"{output_path}/lakes.gpkg") - gdf.to_file(gpkg_path, layer="lakes", driver="GPKG") diff --git a/src/hydrofabric_builds/builds/network.py b/src/hydrofabric_builds/builds/network.py deleted file mode 100644 index 58f6774..0000000 --- a/src/hydrofabric_builds/builds/network.py +++ /dev/null @@ -1,148 +0,0 @@ -import geopandas as gpd -import pandas as pd -from pyarrow import fs -from pyiceberg.catalog import Catalog -from pyiceberg.expressions import In -from tqdm import tqdm - -from hydrofabric_builds.schemas.hydrofabric import HydrofabricDomains - - -def find_usbr_network_rows( - hf_domain: str, - usbr_file: str, - catalog: Catalog, -) -> tuple[pd.DataFrame, pd.DataFrame]: - """Reads the catalog to determine which lake IDs are in the network table - - Parameters - ---------- - hf_domain : str - The domain of the hydrofabric to read - usbr_file : str - the string path to the USBR locations (generated from hydrofabric-builds/tools/builds/lakes/usbr/usbr_prep.ipynb) - catalog : Catalog - The pyiceberg catalog - - Returns - ------- - tuple[pd.DataFrame, pd.DataFrame] - the network table filtered based in lake IDs, the filtered usbr dataframe - """ - usbr_df = gpd.read_parquet(usbr_file) - valid_ids = usbr_df["id"].dropna().values.tolist() - usbr_df_filtered = usbr_df[usbr_df["id"].isin(valid_ids)] - - network_filtered = ( - catalog.load_table(f"{hf_domain}.network").scan(row_filter=In("id", valid_ids)).to_pandas() - ) - return network_filtered, usbr_df_filtered - - -def update_network_hydrolocations_table_usbr( - hf_domain: str, - network_filtered: pd.DataFrame, - usbr_df: pd.DataFrame, - catalog: Catalog, -) -> tuple[pd.DataFrame, pd.DataFrame]: - """Updates the network table to include USBR reservoirs where they already exist - - Parameters - ---------- - hf_domain : str - The domain of the hydrofabric to read - network_filtered : pd.DataFrame - the filtered network table - usbr_df : pd.DataFrame - the dataframe containing usbr lakes - catalog : Catalog - The pyiceberg catalog - - Returns - ------- - tuple[pd.DataFrame, pd.DataFrame] - returns the new network table and hydrolocations - """ - df_lakes = catalog.load_table(f"{hf_domain}.lakes").scan().to_pandas() - df_network = catalog.load_table(f"{hf_domain}.network").scan().to_pandas() - df_hydrolocations = catalog.load_table(f"{hf_domain}.hydrolocations").scan().to_pandas() - print("Loaded icefabric tables") - - existing_pois = network_filtered[~network_filtered["poi_id"].isna()]["poi_id"].unique().astype(int) - filtered_lakes = df_lakes[df_lakes["poi_id"].isin(existing_pois)] - for _, row in tqdm( - filtered_lakes.iterrows(), - total=len(filtered_lakes), - desc="Creating new network table connections for existing USBR lakes", - ): - nidx = df_network.index[-1] + 1 # getting a fresh index for the new entry by appending from the end - hydro_idx = ( - df_hydrolocations.index[-1] + 1 - ) # getting a fresh index for the new entry by appending from the end - network_row = ( - network_filtered[network_filtered["poi_id"] == row.poi_id] - .drop_duplicates(keep="first", subset=["poi_id"]) - .copy() - ) - hydrolocation = ( - df_hydrolocations[df_hydrolocations["poi_id"] == row.poi_id] - .drop_duplicates(keep="first", subset=["poi_id"]) - .copy() - ) - usbr_row = usbr_df[usbr_df["id"] == network_row["id"].values[0]] - try: - location_id = usbr_row["location_id"].item() - except ValueError: - print("Multiple locations found for one divide. Using first location.") - location_id = usbr_row["location_id"].iloc[0].item() - network_row["hl_uri"] = f"usbr-{location_id}" - network_row.index = [nidx] - hydrolocation["hl_reference"] = "usbr" - hydrolocation["hl_uri"] = f"usbr-{location_id}" - hydrolocation["hl_link"] = str(location_id) - hydrolocation["hl_source"] = "USBR" - hydrolocation.index = [hydro_idx] - df_network = pd.concat([df_network, network_row]) - df_hydrolocations = pd.concat([df_hydrolocations, hydrolocation]) - - return df_network, df_hydrolocations - - -def update_network_poi( - catalog: Catalog, hf_domain: str, poi_path: str, s3: fs.S3FileSystem | None = None -) -> pd.DataFrame: - """Update a Hydrofabric network table with new POIs from a POI table - - Parameters - ---------- - catalog : Catalog - Pyicbeger catalog to read from - hf_domain : str - Hydrofabric Domain to use - poi_path : _type_ - Path to new POI parquet - s3 : fs.S3FileSystem | None, optional - An s3 filesystem if reading and writing to s3, by default None - - Returns - ------- - pd.DataFrame - Updated network table with new POIs - """ - print("Updating network table") - df_network = catalog.load_table(f"{HydrofabricDomains[hf_domain].value}.network").scan().to_pandas() - - df_poi = pd.read_parquet(poi_path, filesystem=s3) if s3 else pd.read_parquet(poi_path) - - # merge on nex_id only - df_network_merge = df_network.merge( - df_poi[["poi_id", "nex_id"]], how="left", left_on=["toid"], right_on=["nex_id"] - ) - - # replace old POI with new POI - df_network_merge["poi_id_x"] = df_network_merge["poi_id_y"] - df_network_merge = df_network_merge.drop(columns=["poi_id_y", "nex_id"]).rename( - columns={"poi_id_x": "poi_id"} - ) - - return df_network_merge diff --git a/tools/builds/irrigation/run_irrigation.py b/tools/builds/irrigation/run_irrigation.py deleted file mode 100644 index bbac027..0000000 --- a/tools/builds/irrigation/run_irrigation.py +++ /dev/null @@ -1,126 +0,0 @@ -"""A script to download the USDA Cropland Data Layer and convert to gridded irrigation input for CNN""" - -import argparse -from pathlib import Path - -from hydrofabric_builds.builds.irrigation import classify_irrigation, regrid_irrigation - - -def run_irrigation( - wd: Path, - min_yr: int, - max_yr: int, - download: bool, - unzip: bool, - classify: bool, - aggregate: bool, - grid: Path | None = None, - grid_year: str | int | None = None, -) -> None: - """Run the full irrigation pipeline. - - Here, irrigation is considered a list of crops that may be irrigated. This build downloads the USDA Cropland Data Layer for requested years, converts to a mask at native resolution (30 meter), - and optionally regrids to a coarser grid and resolution where the new raster represents percent irrigated. - - Parameters - ---------- - wd : Path - Working directory for data files - min_yr : str | int - First CDL year to process. If you only want one year, set max_yr to equal min_yr - max_yr : str | int - Last year to process. If you only want one year, set max_yr to equal min_yr - download : bool - Optional flag. Set True to skip downloading files. - unzip : bool - Optional flag. Set True to skip unzipping files. - classify : bool - Optional flag. Set True to skip skip classifying files. - aggregate : bool - Optional flag. Set True to skip skip aggregating files. - grid : Path | None, optional - Optional grid/raster to resample, align, and clip outputs to. Resampling will be percent of - new resolution irrigated, by default None - grid_year : str | int | None, optional - Optionally specify a single year to regrid. Will ignore min/max year arguments. "20xx_20xx" - for temporally aggregated layer is accepted, by default None - """ - classify_irrigation( - min_yr=min_yr, - max_yr=max_yr, - download=download, - unzip=unzip, - classify=classify, - aggregate=aggregate, - wd=wd, - ) - - # regrid if a grid was input - if grid: - # if a single year was input, only regrid this, else regrid min->max and temporally aggregated - if grid_year: - yrs = [grid_year] - else: - yrs = [min_yr] if min_yr == max_yr else list(range(min_yr, max_yr + 1)) + [f"{min_yr}_{max_yr}"] - - regrid_irrigation(wd=wd, grid_path=grid, regrid_list=[str(yr) for yr in yrs]) - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="A script to convert Cropland Data Layer crops into a gridded 'irrigation' mask for use in f1 Trainer" - ) - parser.add_argument( - "-w", "--working_dir", required=True, type=str, help="Working directory for data files" - ) - parser.add_argument("--min_yr", required=True, type=int, help="First CDL year to process") - parser.add_argument( - "--max_yr", - required=True, - type=int, - help="Last year to process. If you only want one year, set max_yr to equal min_yr", - ) - parser.add_argument( - "-g", "--grid", type=str, help="Optional grid to resample, align, and clip outputs to" - ) - parser.add_argument( - "--grid_yr", type=str, help="Optionally specify a single year to regrid. Will ignore min/max year" - ) - parser.add_argument( - "--no_download", - action="store_true", - help="Optional flag. Incldue argument to skip downloading files.", - ) - parser.add_argument( - "--no_unzip", action="store_true", help="Optional flag. Incldue argument to skip unzipping files." - ) - parser.add_argument( - "--no_classify", action="store_true", help="Optional flag. Incldue argumentto skip classifying files." - ) - parser.add_argument( - "--no_aggregate", - action="store_true", - help="Optional flag. Incldue argument to skip aggregating files.", - ) - - args = parser.parse_args() - - wd = Path(args.working_dir) - download = False if args.no_download else True - unzip = False if args.no_unzip else True - classify = False if args.no_classify else True - aggregate = False if args.no_aggregate else True - grid = Path(args.grid) if args.grid else None - grid_yr = args.grid_yr if args.grid_yr else None - - run_irrigation( - wd=wd, - min_yr=int(args.min_yr), - max_yr=int(args.max_yr), - grid=args.grid, - download=download, - unzip=unzip, - classify=classify, - aggregate=aggregate, - grid_year=grid_yr, - ) diff --git a/tools/builds/lakes/lakes_hydrofabric.py b/tools/builds/lakes/lakes_hydrofabric.py deleted file mode 100644 index 754143d..0000000 --- a/tools/builds/lakes/lakes_hydrofabric.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Build the lakes layer for CONUS - -python lakes_hydrofabric.py --lakeparm_file --ana_res_file --ext_res_file --med_res_file --short_res_file --hffile --output_path --domain - -python lakes_hydrofabric.py --lakeparm_file /home/daniel.cumpton/lakes/LAKEPARM_CONUS_216.nc --ana_res_file /home/daniel.cumpton/lakes/reservoir_index_AnA_309.nc --ext_res_file /home/daniel.cumpton/lakes/reservoir_index_Extended_AnA.nc --med_res_file /home/daniel.cumpton/lakes/reservoir_index_Medium_Range.nc --short_res_file /home/daniel.cumpton/lakes/reservoir_index_Short_Range.nc --hffile /home/daniel.cumpton/Hydrofabric/data/hydrofabric/v2.2/nextgen/CONUS/conus_nextgen.gpkg --output_path /home/daniel.cumpton/lakes --domain CONUS - -""" - -import argparse - -from hydrofabric_builds.builds.lakes import build_lakes - -if __name__ == "__main__": - parser = argparse.ArgumentParser() - - parser.add_argument( - "--lakeparm_file", - required=True, - type=str, - help="full path and filename of lakeparms netcdf file, e.g, LAKEPARM_CONUS_216.nc", - ) - - parser.add_argument( - "--ana_res_file", - required=True, - type=str, - help="full path and filename of ana res netcdf file, e.g., reservoir_index_AnA_309.nc", - ) - - parser.add_argument( - "--ext_res_file", - required=True, - type=str, - help="full path and filename of extended res netcdf file, e.g., reservoir_index_Extended_AnA.nc", - ) - - parser.add_argument( - "--med_res_file", - required=True, - type=str, - help="full path and filename of medium range res netcdf file, e.g., reservoir_index_Medium_Range.nc", - ) - - parser.add_argument( - "--short_res_file", - required=True, - type=str, - help="full path and filename of medium range res netcdf file, e.g., reservoir_index_Short_Range.nc", - ) - - parser.add_argument( - "--hffile", required=True, type=str, help="full path and filename of hydrofabric geopackage" - ) - - parser.add_argument( - "--output_path", required=True, type=str, help="full path to directory where outputs will be saved" - ) - - parser.add_argument( - "--domain", required=True, type=str, help="hydrofabric domain (CONUS only at this time)" - ) - - args = parser.parse_args() - - lakeparm_file = args.lakeparm_file - ana_res_file = args.ana_res_file - ext_res_file = args.ext_res_file - med_res_file = args.med_res_file - short_res_file = args.short_res_file - hf_file = args.hffile - output_path = args.output_path - domain = args.domain - - build_lakes( - lakeparm_file, ana_res_file, ext_res_file, med_res_file, short_res_file, hf_file, output_path, domain - ) diff --git a/tools/builds/lakes/usbr/usbr_hydrofabric.py b/tools/builds/lakes/usbr/usbr_hydrofabric.py deleted file mode 100644 index 530add7..0000000 --- a/tools/builds/lakes/usbr/usbr_hydrofabric.py +++ /dev/null @@ -1,85 +0,0 @@ -"""A script to build lakes, network table, POIs, and hydrolocations from a lakes point file - -example usage: python tools/builds/lakes/usbr/usbr_hydrofabric.py --catalog sql --usbr-file /hydrofabric-builds/data/usbr/reservoir_matches.parquet --working-dir /hydrofabric-builds/data/usbr -""" - -import argparse -from pathlib import Path -from typing import Literal - -from hydrofabric_builds.builds.network import find_usbr_network_rows, update_network_hydrolocations_table_usbr -from hydrofabric_builds.helpers.io import setup_glue_catalog, setup_sql_catalog - - -def build_usbr_lakes( - catalog_type: Literal["glue", "sql"], - usbr_file: str, - working_dir: Path, -) -> None: - """Builds the USBR lakes into the network table - - Parameters - ---------- - catalog_type: Literal["glue", "sql"] - The pyiceberg catalog type - usbr_file: str, - the string path to the USBR locations (generated from hydrofabric-builds/tools/builds/lakes/usbr/usbr_prep.ipynb) - working_dir: Path - The working dir to save output parquet files - """ - hf_domain = "conus" # only supporting CONUS for the time being - catalog = setup_glue_catalog() if catalog_type == "glue" else setup_sql_catalog() - network_path = working_dir / "usbr_network.parquet" - hydrolocations_path = working_dir / "usbr_hydrolocations.parquet" - - network_filtered, usbr_df_filtered = find_usbr_network_rows( - hf_domain=f"{hf_domain}_hf", - usbr_file=usbr_file, - catalog=catalog, - ) - - df_network, df_hydrolocations = update_network_hydrolocations_table_usbr( - hf_domain=f"{hf_domain}_hf", - catalog=catalog, - network_filtered=network_filtered, - usbr_df=usbr_df_filtered, - ) - - df_network.to_parquet(network_path) - df_hydrolocations.to_parquet(hydrolocations_path) - print("Saved network and hydrolocations tables with updated USBR references") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="A script to build the network table, and hydrolocations, for USBR reservoirs that are already in the HF" - ) - parser.add_argument( - "--catalog", - required=False, - choices=["sql", "glue"], - type=str, - help="Use 'glue' or 'sql' catalog", - default="sql", - ) - parser.add_argument( - "--usbr-file", - required=True, - type=str, - help="Path to the usbr reservoirs parquet on local disk.", - ) - parser.add_argument( - "--working-dir", - required=False, - type=Path, - default=Path.cwd(), - help="Working directory to save files.", - ) - - args = parser.parse_args() - - build_usbr_lakes( - catalog_type=args.catalog, - usbr_file=args.usbr_file, - working_dir=args.working_dir, - ) diff --git a/tools/builds/lakes/usbr/usbr_prep.ipynb b/tools/builds/lakes/usbr/usbr_prep.ipynb deleted file mode 100644 index 57b7c45..0000000 --- a/tools/builds/lakes/usbr/usbr_prep.ipynb +++ /dev/null @@ -1,332 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# Download and process USBR reservoirs\n", - "\n", - "The following notebook provides code to scrape the USBR reservoir data in order to find what USBR reservoirs are in the hydrofabric" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from pathlib import Path\n", - "\n", - "import yaml\n", - "from dotenv import load_dotenv\n", - "\n", - "# Changes the current working dir to be the project root\n", - "current_working_dir = Path.cwd()\n", - "os.chdir(current_working_dir / \"../../../../\")\n", - "print(\n", - " f\"Changed current working dir from {current_working_dir} to: {Path.cwd()}. This notebook must run at the project root\"\n", - ")\n", - "\n", - "# dir is where the .env file is located\n", - "load_dotenv(dotenv_path=Path.cwd())\n", - "pyiceberg_file = Path.cwd() / \".pyiceberg.yaml\"\n", - "if pyiceberg_file.exists():\n", - " os.environ[\"PYICEBERG_HOME\"] = str(pyiceberg_file)\n", - "else:\n", - " raise FileNotFoundError(\n", - " \"Cannot find .pyiceberg.yaml. Please download this from NGWPC confluence or create \"\n", - " )\n", - "\n", - "# Loading the local pyiceberg config settings\n", - "try:\n", - " with open(Path.cwd() / \".pyiceberg.yaml\", encoding=\"utf-8\") as file:\n", - " pyiceberg_config = yaml.safe_load(file)\n", - "except FileNotFoundError as e:\n", - " raise FileNotFoundError(f\".pyiceberg YAML file not found in cwd: {Path.cwd() / '../../'}\") from e\n", - "except yaml.YAMLError as e:\n", - " raise yaml.YAMLError(f\"Error parsing .pyiceberg YAML file: {e}\") from e" - ] - }, - { - "cell_type": "markdown", - "id": "2", - "metadata": {}, - "source": [ - "Get all of the USBR data. For more inforrmation on the endpoint, see https://data.usbr.gov/rise/" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "import httpx\n", - "\n", - "\"\"\"Get all reservoir locations from USBR RISE API\"\"\"\n", - "base_url = \"https://data.usbr.gov/rise/api/location\"\n", - "all_data = []\n", - "page = 1\n", - "\n", - "print(\"Retrieving USBR locations...\")\n", - "\n", - "with httpx.Client(timeout=30.0) as client:\n", - " while True:\n", - " try:\n", - " response = client.get(\n", - " base_url,\n", - " params={\"page\": page, \"itemsPerPage\": 100},\n", - " headers={\"Accept\": \"application/vnd.api+json\"},\n", - " )\n", - " response.raise_for_status()\n", - "\n", - " content = response.json()\n", - " data_page = content.get(\"data\", [])\n", - "\n", - " if not data_page:\n", - " break\n", - "\n", - " all_data.extend(data_page)\n", - " print(f\"Retrieved page {page}, total entries: {len(all_data)}\")\n", - " page += 1\n", - " except httpx.RequestError as e:\n", - " print(f\"Error retrieving page {page}: {e}\")\n", - " break\n", - " except httpx.HTTPStatusError as e:\n", - " print(f\"HTTP error retrieving page {page}: {e}\")\n", - " break\n", - "\n", - "print(f\"Total entries retrieved: {len(all_data)}\")" - ] - }, - { - "cell_type": "markdown", - "id": "4", - "metadata": {}, - "source": [ - "Process all data from the requests and format them into a list based on which are lakes/reservoirs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "\n", - "# Process the data\n", - "location_type = \"Lake/Reservoir\" # ensuring we only are about lakes and reservoirs\n", - "locations = []\n", - "location_types = {}\n", - "\n", - "for _i, item in enumerate(all_data):\n", - " attrs = item.get(\"attributes\", {})\n", - " coords_data = attrs.get(\"locationCoordinates\", {})\n", - " coords = coords_data.get(\"coordinates\") if coords_data else None\n", - "\n", - " loc_type = attrs.get(\"locationTypeName\")\n", - " if loc_type:\n", - " location_types[loc_type] = location_types.get(loc_type, 0) + 1\n", - "\n", - " if coords and len(coords) == 2:\n", - " locations.append(\n", - " {\n", - " \"locationName\": attrs.get(\"locationName\"),\n", - " \"longitude\": coords[0],\n", - " \"latitude\": coords[1],\n", - " \"locationType\": loc_type,\n", - " \"location_id\": attrs.get(\"_id\"),\n", - " }\n", - " )\n", - "\n", - "print(\"\\nLocation types found:\")\n", - "for loc_type, count in sorted(location_types.items(), key=lambda x: x[1], reverse=True):\n", - " print(f\" {loc_type}: {count}\")\n", - "\n", - "# Convert to DataFrame and filter\n", - "reservoirs_df = pd.DataFrame(locations)\n", - "\n", - "if location_type:\n", - " reservoirs_df = reservoirs_df[reservoirs_df[\"locationType\"] == location_type].copy()\n", - " print(f\"\\nFiltered to {len(reservoirs_df)} '{location_type}' entries\")\n", - "\n", - "reservoirs_df.head()" - ] - }, - { - "cell_type": "markdown", - "id": "6", - "metadata": {}, - "source": [ - "Read in the hydrofabric in order to determine which lakes are already represented in the hydrofabric" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "import geopandas as gpd\n", - "import pandas as pd\n", - "\n", - "\n", - "def to_geopandas(df: pd.DataFrame, crs: str = \"EPSG:5070\") -> gpd.GeoDataFrame:\n", - " \"\"\"Converts the geometries in a pandas df to a geopandas dataframe\n", - "\n", - " Parameters\n", - " ----------\n", - " df: pd.DataFrame\n", - " The iceberg table you are trying to read from\n", - " crs: str, optional\n", - " A string representing the CRS to set in the gdf, by default \"EPSG:5070\"\n", - "\n", - " Returns\n", - " -------\n", - " gpd.DataFrame\n", - " The resulting queried row, but in a geodataframe\n", - "\n", - " Raises\n", - " ------\n", - " ValueError\n", - " Raised if the table does not have a geometry column\n", - " \"\"\"\n", - " if \"geometry\" not in df.columns:\n", - " raise ValueError(\"The provided table does not have a geometry column.\")\n", - "\n", - " return gpd.GeoDataFrame(df, geometry=gpd.GeoSeries.from_wkb(df[\"geometry\"]), crs=crs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [ - "from pyiceberg.catalog import load_catalog\n", - "\n", - "# Loading SQL Catalog\n", - "# This catalog can be downloaded through the icefabric repo. We care about the conus_hf namespace\n", - "catalog = load_catalog(\n", - " name=\"sql\",\n", - " type=pyiceberg_config[\"catalog\"][\"sql\"][\"type\"],\n", - " uri=pyiceberg_config[\"catalog\"][\"sql\"][\"uri\"],\n", - " warehouse=pyiceberg_config[\"catalog\"][\"sql\"][\"warehouse\"],\n", - ")\n", - "\n", - "# Loading Glue Catalog\n", - "# catalog = load_catalog(\"glue\", **{\n", - "# \"type\": \"glue\",\n", - "# \"glue.region\": \"us-east-1\"\n", - "# })\n", - "\n", - "lakes = to_geopandas(catalog.load_table(\"conus_hf.lakes\").scan().to_pandas())\n", - "divides = to_geopandas(catalog.load_table(\"conus_hf.divides\").scan().to_pandas())\n", - "hydrolocations = catalog.load_table(\"conus_hf.hydrolocations\").scan().to_pandas()\n", - "pois = catalog.load_table(\"conus_hf.pois\").scan().to_pandas()\n", - "network = catalog.load_table(\"conus_hf.network\").scan().to_pandas()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "from shapely.geometry import Point\n", - "\n", - "# Create reservoir points\n", - "geometry = [Point(xy) for xy in zip(reservoirs_df.longitude, reservoirs_df.latitude, strict=True)]\n", - "reservoirs_gdf = gpd.GeoDataFrame(reservoirs_df, geometry=geometry, crs=\"EPSG:4326\")\n", - "\n", - "# Ensure matching CRS. This should be EPSG:5070\n", - "if reservoirs_gdf.crs != lakes.crs:\n", - " print(f\"Reprojecting reservoirs from {reservoirs_gdf.crs} to {lakes.crs}\")\n", - " reservoirs_gdf = reservoirs_gdf.to_crs(lakes.crs)\n", - "\n", - "# Buffer the reservoir points\n", - "buffer_meters = 1000 # Adjust this value as needed\n", - "print(f\"Applying {buffer_meters}m buffer to reservoir points...\")\n", - "\n", - "# Apply buffer in projected coordinates\n", - "reservoirs_buffered = reservoirs_gdf.copy()\n", - "reservoirs_buffered.geometry = reservoirs_buffered.geometry.buffer(buffer_meters)\n", - "\n", - "# Spatial join\n", - "print(\"Performing spatial join...\")\n", - "matches = gpd.sjoin(reservoirs_buffered, lakes, how=\"inner\", predicate=\"intersects\")\n", - "\n", - "print(f\"Found {len(matches)} reservoir-lake matches\")\n", - "print(f\"Matched {matches['locationName'].nunique()} unique reservoirs\")\n", - "\n", - "matches.head()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10", - "metadata": {}, - "outputs": [], - "source": [ - "hydrolocations_first = hydrolocations.drop_duplicates(subset=[\"poi_id\"], keep=\"first\")\n", - "\n", - "matches_extended = pd.merge(\n", - " matches,\n", - " hydrolocations_first,\n", - " how=\"inner\",\n", - " on=\"poi_id\", # or whatever your join column is\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "matches_extended.to_parquet(\"data/usbr/reservoir_matches.parquet\")\n", - "matches_extended.to_file(\"data/usbr/reservoir_matches.gpkg\", driver=\"GPKG\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "hydrofabric-builds", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} From 745d35089ffc3901e1de71b123effaaba50a7826 Mon Sep 17 00:00:00 2001 From: Quercus Hamlin Date: Mon, 31 Aug 2026 10:10:42 -0400 Subject: [PATCH 2/3] docs: add json schemas for pydantic models --- .../constants/groundwater_projection_ak.json | 11 + .../groundwater_projection_conus.json | 11 + .../constants/groundwater_projection_hi.json | 11 + .../groundwater_projection_prvi.json | 11 + .../constants/nwm_default_hydraulics.json | 10 + docs/schemas/gages/assign_fp_config.json | 58 +++ docs/schemas/gages/gage_input.json | 84 ++++ docs/schemas/gages/gage_inputs_all.json | 171 +++++++++ docs/schemas/gages/gages_block.json | 257 +++++++++++++ docs/schemas/gages/gages_config.json | 359 ++++++++++++++++++ docs/schemas/gages/gages_target.json | 47 +++ docs/schemas/gages/nldi_upstream_basins.json | 28 ++ docs/schemas/gages/nwm_rfc_input.json | 23 ++ .../adhoc_lake_inputusbr_lake_input.json | 31 ++ docs/schemas/lakes/great_lakes.json | 31 ++ docs/schemas/lakes/great_lakes_mapping.json | 72 ++++ docs/schemas/lakes/lake_dem_input.json | 30 ++ docs/schemas/lakes/lakes_config.json | 129 +++++++ docs/schemas/lakes/nid_input.json | 54 +++ docs/schemas/lakes/nwm_v3_lake_input.json | 155 ++++++++ .../lakes/reference_reservoirs_input.json | 84 ++++ .../lakes/reference_waterbody_input.json | 96 +++++ docs/schemas/network/aggregations.json | 59 +++ .../network/build_hydrofabric_config.json | 44 +++ docs/schemas/network/classifications.json | 82 ++++ .../network/divide_attribute_config.json | 58 +++ .../divide_attribute_model_config.json | 190 +++++++++ .../network/flowpath_attributes_config.json | 207 ++++++++++ .../flowpath_attributes_model_config.json | 54 +++ .../nhd_fp_crosswalk_config.json | 107 ++++++ .../nhd_fp_crosswalk_outputs.json | 13 + .../nhd_fp_crosswalk_reference.json | 12 + .../nhd_fp_crosswalk_target.json | 23 ++ docs/schemas/reservoir_da/active_rfc.json | 18 + .../adhoc_reservoir_da_input.json | 44 +++ .../reservoir_crosswalk_fields.json | 49 +++ .../reservoir_crosswalk_input.json | 78 ++++ .../reservoir_da/reservoir_da_config.json | 326 ++++++++++++++++ .../reservoir_da/reservoir_da_mapping.json | 37 ++ .../usace_reservoir_da_input.json | 32 ++ .../reservoir_da/usbr_reservoir_da_input.json | 32 ++ 41 files changed, 3228 insertions(+) create mode 100644 docs/schemas/constants/groundwater_projection_ak.json create mode 100644 docs/schemas/constants/groundwater_projection_conus.json create mode 100644 docs/schemas/constants/groundwater_projection_hi.json create mode 100644 docs/schemas/constants/groundwater_projection_prvi.json create mode 100644 docs/schemas/constants/nwm_default_hydraulics.json create mode 100644 docs/schemas/gages/assign_fp_config.json create mode 100644 docs/schemas/gages/gage_input.json create mode 100644 docs/schemas/gages/gage_inputs_all.json create mode 100644 docs/schemas/gages/gages_block.json create mode 100644 docs/schemas/gages/gages_config.json create mode 100644 docs/schemas/gages/gages_target.json create mode 100644 docs/schemas/gages/nldi_upstream_basins.json create mode 100644 docs/schemas/gages/nwm_rfc_input.json create mode 100644 docs/schemas/lakes/adhoc_lake_inputusbr_lake_input.json create mode 100644 docs/schemas/lakes/great_lakes.json create mode 100644 docs/schemas/lakes/great_lakes_mapping.json create mode 100644 docs/schemas/lakes/lake_dem_input.json create mode 100644 docs/schemas/lakes/lakes_config.json create mode 100644 docs/schemas/lakes/nid_input.json create mode 100644 docs/schemas/lakes/nwm_v3_lake_input.json create mode 100644 docs/schemas/lakes/reference_reservoirs_input.json create mode 100644 docs/schemas/lakes/reference_waterbody_input.json create mode 100644 docs/schemas/network/aggregations.json create mode 100644 docs/schemas/network/build_hydrofabric_config.json create mode 100644 docs/schemas/network/classifications.json create mode 100644 docs/schemas/network/divide_attribute_config.json create mode 100644 docs/schemas/network/divide_attribute_model_config.json create mode 100644 docs/schemas/network/flowpath_attributes_config.json create mode 100644 docs/schemas/network/flowpath_attributes_model_config.json create mode 100644 docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_config.json create mode 100644 docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_outputs.json create mode 100644 docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_reference.json create mode 100644 docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_target.json create mode 100644 docs/schemas/reservoir_da/active_rfc.json create mode 100644 docs/schemas/reservoir_da/adhoc_reservoir_da_input.json create mode 100644 docs/schemas/reservoir_da/reservoir_crosswalk_fields.json create mode 100644 docs/schemas/reservoir_da/reservoir_crosswalk_input.json create mode 100644 docs/schemas/reservoir_da/reservoir_da_config.json create mode 100644 docs/schemas/reservoir_da/reservoir_da_mapping.json create mode 100644 docs/schemas/reservoir_da/usace_reservoir_da_input.json create mode 100644 docs/schemas/reservoir_da/usbr_reservoir_da_input.json diff --git a/docs/schemas/constants/groundwater_projection_ak.json b/docs/schemas/constants/groundwater_projection_ak.json new file mode 100644 index 0000000..0aacebd --- /dev/null +++ b/docs/schemas/constants/groundwater_projection_ak.json @@ -0,0 +1,11 @@ +{ + "class_name": "GroundWaterProjectionAK", + "values": { + "PROJ4": "+proj=stere +lat_0=90 +lat_ts=60 +lon_0=-135 +x_0=0 +y_0=0 +R=6370000 +units=m +no_defs", + "X_ORIGIN": -1130764.7202253528, + "Y_ORIGIN": -3163389.53353531, + "WIDTH": 3516, + "HEIGHT": 1816, + "DX": 250 + } +} diff --git a/docs/schemas/constants/groundwater_projection_conus.json b/docs/schemas/constants/groundwater_projection_conus.json new file mode 100644 index 0000000..8a4a4be --- /dev/null +++ b/docs/schemas/constants/groundwater_projection_conus.json @@ -0,0 +1,11 @@ +{ + "class_name": "GroundWaterProjectionCONUS", + "values": { + "PROJ4": "+proj=lcc +lat_1=30 +lat_2=60 +lat_0=40.0000076293945 +lon_0=-97 +x_0=0 +y_0=0 +a=6370000 +b=6370000 +units=m +no_defs", + "X_ORIGIN": -2303874.17655, + "Y_ORIGIN": -1919874.66329, + "WIDTH": 18432, + "HEIGHT": 15360, + "DX": 250 + } +} diff --git a/docs/schemas/constants/groundwater_projection_hi.json b/docs/schemas/constants/groundwater_projection_hi.json new file mode 100644 index 0000000..a99026b --- /dev/null +++ b/docs/schemas/constants/groundwater_projection_hi.json @@ -0,0 +1,11 @@ +{ + "class_name": "GroundWaterProjectionHI", + "values": { + "PROJ4": "+proj=lcc +units=m +a=6370000.0 +b=6370000.0 +lat_1=10.0 +lat_2=30.0 +lat_0=20.6 +lon_0=-157.42 +x_0=0 +y_0=0 +k_0=1.0 +nadgrids=@null +wktext +no_defs", + "X_ORIGIN": -294950.07097397465, + "Y_ORIGIN": -194949.36969098, + "WIDTH": 5900, + "HEIGHT": 3900, + "DX": 100 + } +} diff --git a/docs/schemas/constants/groundwater_projection_prvi.json b/docs/schemas/constants/groundwater_projection_prvi.json new file mode 100644 index 0000000..1cd05ea --- /dev/null +++ b/docs/schemas/constants/groundwater_projection_prvi.json @@ -0,0 +1,11 @@ +{ + "class_name": "GroundWaterProjectionPRVI", + "values": { + "PROJ4": "+proj=lcc +units=m +a=6370000.0 +b=6370000.0 +lat_1=18.1 +lat_2=18.1 +lat_0=18.1 +lon_0=-65.91 +x_0=0 +y_0=0 +k_0=1.0 +nadgrids=@null +wktext +no_defs", + "X_ORIGIN": -149949.83, + "Y_ORIGIN": -54948.968, + "WIDTH": 3000, + "HEIGHT": 1100, + "DX": 100 + } +} diff --git a/docs/schemas/constants/nwm_default_hydraulics.json b/docs/schemas/constants/nwm_default_hydraulics.json new file mode 100644 index 0000000..d64db16 --- /dev/null +++ b/docs/schemas/constants/nwm_default_hydraulics.json @@ -0,0 +1,10 @@ +{ + "class_name": "NWMDefaultHydraulics", + "values": { + "WeirC": 0.4, + "WeirL": 10.0, + "OrificeC": 0.1, + "OrificeA": 1.0, + "ifd": 0.899 + } +} diff --git a/docs/schemas/gages/assign_fp_config.json b/docs/schemas/gages/assign_fp_config.json new file mode 100644 index 0000000..c694498 --- /dev/null +++ b/docs/schemas/gages/assign_fp_config.json @@ -0,0 +1,58 @@ +{ + "description": "gages: a class for assigning flowpaths to gages", + "properties": { + "rel_err": { + "default": 0.25, + "title": "Rel Err", + "type": "number" + }, + "buffer_m": { + "default": 500.0, + "title": "Buffer M", + "type": "number" + }, + "parallel": { + "default": false, + "title": "Parallel", + "type": "boolean" + }, + "max_workers": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Max Workers" + }, + "USGS_NLDI_crs": { + "default": "EPSG:4326", + "title": "Usgs Nldi Crs", + "type": "string" + }, + "work_crs": { + "default": "EPSG:5070", + "title": "Work Crs", + "type": "string" + }, + "override_fp_path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A csv with columns `site_no`, `fp_id`, and `virtual_fp_id` to override algorithmically chosen flowpath associations", + "title": "Override Fp Path" + } + }, + "title": "AssignFPConfig", + "type": "object" +} diff --git a/docs/schemas/gages/gage_input.json b/docs/schemas/gages/gage_input.json new file mode 100644 index 0000000..1f57cf4 --- /dev/null +++ b/docs/schemas/gages/gage_input.json @@ -0,0 +1,84 @@ +{ + "description": "gages: gageinput class", + "properties": { + "dir": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dir" + }, + "path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "gage_source_crs": { + "default": "EPSG:4326", + "title": "Gage Source Crs", + "type": "string" + }, + "id_col_name": { + "default": "site_no", + "title": "Id Col Name", + "type": "string" + }, + "x_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "X Col Name" + }, + "y_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Y Col Name" + }, + "area_col_name": { + "default": "area_sqkm", + "title": "Area Col Name", + "type": "string" + }, + "status_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Status Col Name" + } + }, + "title": "GageInput", + "type": "object" +} diff --git a/docs/schemas/gages/gage_inputs_all.json b/docs/schemas/gages/gage_inputs_all.json new file mode 100644 index 0000000..80573de --- /dev/null +++ b/docs/schemas/gages/gage_inputs_all.json @@ -0,0 +1,171 @@ +{ + "$defs": { + "GageInput": { + "description": "gages: gageinput class", + "properties": { + "dir": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dir" + }, + "path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "gage_source_crs": { + "default": "EPSG:4326", + "title": "Gage Source Crs", + "type": "string" + }, + "id_col_name": { + "default": "site_no", + "title": "Id Col Name", + "type": "string" + }, + "x_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "X Col Name" + }, + "y_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Y Col Name" + }, + "area_col_name": { + "default": "area_sqkm", + "title": "Area Col Name", + "type": "string" + }, + "status_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Status Col Name" + } + }, + "title": "GageInput", + "type": "object" + }, + "NWMRFCInput": { + "description": "NWM reservoir index file for retaining RFC and USACE gages. USACE IDs are found in NID.", + "properties": { + "path": { + "default": "rfc/reservoir_index_AnA.nc", + "format": "path", + "title": "Path", + "type": "string" + }, + "rfc_id_col": { + "default": "rfc_gage_id", + "title": "Rfc Id Col", + "type": "string" + }, + "usace_id_col": { + "default": "usace_gage_id", + "title": "Usace Id Col", + "type": "string" + } + }, + "title": "NWMRFCInput", + "type": "object" + } + }, + "description": "gages: configs collection for all inputs", + "properties": { + "usgs_discontinued": { + "$ref": "#/$defs/GageInput" + }, + "usgs_active": { + "$ref": "#/$defs/GageInput" + }, + "txdot_gages": { + "$ref": "#/$defs/GageInput" + }, + "other": { + "$ref": "#/$defs/GageInput" + }, + "CIROH_UA": { + "$ref": "#/$defs/GageInput" + }, + "nwm_calib_gages": { + "$ref": "#/$defs/GageInput" + }, + "routelink": { + "$ref": "#/$defs/GageInput" + }, + "rfc": { + "$ref": "#/$defs/GageInput", + "description": "Table of active NWS gages. Retrieved from https://water.noaa.gov/about/data-and-web-services-catalog on 6/15/26" + }, + "nid": { + "$ref": "#/$defs/GageInput" + }, + "nwm_rfc": { + "$ref": "#/$defs/NWMRFCInput", + "default": { + "path": "rfc/reservoir_index_AnA.nc", + "rfc_id_col": "rfc_gage_id", + "usace_id_col": "usace_gage_id" + }, + "description": "An NWM v3 reservoid index file with RFC gages to retain" + }, + "adhoc_lakes": { + "$ref": "#/$defs/GageInput", + "description": "Adhoc lakes from reference waterbodies" + }, + "canada_great_lakes": { + "default": false, + "description": "Flag to pull Lake Erie and Lake Ontario Canadian gages from GreatLakesMapping class defined in Lakes", + "title": "Canada Great Lakes", + "type": "boolean" + }, + "usbr": { + "$ref": "#/$defs/GageInput", + "description": "USBR lakes" + }, + "usace": { + "$ref": "#/$defs/GageInput", + "description": "USACE gages/reservoirs" + } + }, + "title": "GagesInputs", + "type": "object" +} diff --git a/docs/schemas/gages/gages_block.json b/docs/schemas/gages/gages_block.json new file mode 100644 index 0000000..8716118 --- /dev/null +++ b/docs/schemas/gages/gages_block.json @@ -0,0 +1,257 @@ +{ + "$defs": { + "GageInput": { + "description": "gages: gageinput class", + "properties": { + "dir": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dir" + }, + "path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "gage_source_crs": { + "default": "EPSG:4326", + "title": "Gage Source Crs", + "type": "string" + }, + "id_col_name": { + "default": "site_no", + "title": "Id Col Name", + "type": "string" + }, + "x_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "X Col Name" + }, + "y_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Y Col Name" + }, + "area_col_name": { + "default": "area_sqkm", + "title": "Area Col Name", + "type": "string" + }, + "status_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Status Col Name" + } + }, + "title": "GageInput", + "type": "object" + }, + "GagesInputs": { + "description": "gages: configs collection for all inputs", + "properties": { + "usgs_discontinued": { + "$ref": "#/$defs/GageInput" + }, + "usgs_active": { + "$ref": "#/$defs/GageInput" + }, + "txdot_gages": { + "$ref": "#/$defs/GageInput" + }, + "other": { + "$ref": "#/$defs/GageInput" + }, + "CIROH_UA": { + "$ref": "#/$defs/GageInput" + }, + "nwm_calib_gages": { + "$ref": "#/$defs/GageInput" + }, + "routelink": { + "$ref": "#/$defs/GageInput" + }, + "rfc": { + "$ref": "#/$defs/GageInput", + "description": "Table of active NWS gages. Retrieved from https://water.noaa.gov/about/data-and-web-services-catalog on 6/15/26" + }, + "nid": { + "$ref": "#/$defs/GageInput" + }, + "nwm_rfc": { + "$ref": "#/$defs/NWMRFCInput", + "default": { + "path": "rfc/reservoir_index_AnA.nc", + "rfc_id_col": "rfc_gage_id", + "usace_id_col": "usace_gage_id" + }, + "description": "An NWM v3 reservoid index file with RFC gages to retain" + }, + "adhoc_lakes": { + "$ref": "#/$defs/GageInput", + "description": "Adhoc lakes from reference waterbodies" + }, + "canada_great_lakes": { + "default": false, + "description": "Flag to pull Lake Erie and Lake Ontario Canadian gages from GreatLakesMapping class defined in Lakes", + "title": "Canada Great Lakes", + "type": "boolean" + }, + "usbr": { + "$ref": "#/$defs/GageInput", + "description": "USBR lakes" + }, + "usace": { + "$ref": "#/$defs/GageInput", + "description": "USACE gages/reservoirs" + } + }, + "title": "GagesInputs", + "type": "object" + }, + "GagesTarget": { + "description": "gages: target/output configs", + "properties": { + "crs": { + "default": "EPSG:5070", + "title": "Crs", + "type": "string" + }, + "snap_tolerance_m": { + "default": 100.0, + "title": "Snap Tolerance M", + "type": "number" + }, + "update_existing": { + "default": true, + "title": "Update Existing", + "type": "boolean" + }, + "exclude_ids": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Exclude Ids", + "type": "array" + }, + "out_gpkg": { + "default": "gages.gpkg", + "format": "path", + "title": "Out Gpkg", + "type": "string" + }, + "gpkg_layer_name": { + "default": "gages", + "title": "Gpkg Layer Name", + "type": "string" + } + }, + "title": "GagesTarget", + "type": "object" + }, + "NWMRFCInput": { + "description": "NWM reservoir index file for retaining RFC and USACE gages. USACE IDs are found in NID.", + "properties": { + "path": { + "default": "rfc/reservoir_index_AnA.nc", + "format": "path", + "title": "Path", + "type": "string" + }, + "rfc_id_col": { + "default": "rfc_gage_id", + "title": "Rfc Id Col", + "type": "string" + }, + "usace_id_col": { + "default": "usace_gage_id", + "title": "Usace Id Col", + "type": "string" + } + }, + "title": "NWMRFCInput", + "type": "object" + } + }, + "description": "aggregating all inputs gages classes configs here", + "properties": { + "input_dir": { + "default": "data/gages", + "format": "path", + "title": "Input Dir", + "type": "string" + }, + "inputs": { + "$ref": "#/$defs/GagesInputs" + }, + "target": { + "$ref": "#/$defs/GagesTarget" + }, + "prebuilt_gages": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to a pre-built gages gpkg. If set, skip gage collection (steps 1-8) and use this table for assignment.", + "title": "Prebuilt Gages" + }, + "prebuilt_gages_layer": { + "default": "gages", + "description": "Layer name in the pre-built gages gpkg.", + "title": "Prebuilt Gages Layer", + "type": "string" + } + }, + "title": "GagesBlock", + "type": "object" +} diff --git a/docs/schemas/gages/gages_config.json b/docs/schemas/gages/gages_config.json new file mode 100644 index 0000000..e94ad5f --- /dev/null +++ b/docs/schemas/gages/gages_config.json @@ -0,0 +1,359 @@ +{ + "$defs": { + "AssignFPConfig": { + "description": "gages: a class for assigning flowpaths to gages", + "properties": { + "rel_err": { + "default": 0.25, + "title": "Rel Err", + "type": "number" + }, + "buffer_m": { + "default": 500.0, + "title": "Buffer M", + "type": "number" + }, + "parallel": { + "default": false, + "title": "Parallel", + "type": "boolean" + }, + "max_workers": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Max Workers" + }, + "USGS_NLDI_crs": { + "default": "EPSG:4326", + "title": "Usgs Nldi Crs", + "type": "string" + }, + "work_crs": { + "default": "EPSG:5070", + "title": "Work Crs", + "type": "string" + }, + "override_fp_path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "A csv with columns `site_no`, `fp_id`, and `virtual_fp_id` to override algorithmically chosen flowpath associations", + "title": "Override Fp Path" + } + }, + "title": "AssignFPConfig", + "type": "object" + }, + "GageInput": { + "description": "gages: gageinput class", + "properties": { + "dir": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Dir" + }, + "path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Path" + }, + "gage_source_crs": { + "default": "EPSG:4326", + "title": "Gage Source Crs", + "type": "string" + }, + "id_col_name": { + "default": "site_no", + "title": "Id Col Name", + "type": "string" + }, + "x_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "X Col Name" + }, + "y_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Y Col Name" + }, + "area_col_name": { + "default": "area_sqkm", + "title": "Area Col Name", + "type": "string" + }, + "status_col_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Status Col Name" + } + }, + "title": "GageInput", + "type": "object" + }, + "GagesBlock": { + "description": "aggregating all inputs gages classes configs here", + "properties": { + "input_dir": { + "default": "data/gages", + "format": "path", + "title": "Input Dir", + "type": "string" + }, + "inputs": { + "$ref": "#/$defs/GagesInputs" + }, + "target": { + "$ref": "#/$defs/GagesTarget" + }, + "prebuilt_gages": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to a pre-built gages gpkg. If set, skip gage collection (steps 1-8) and use this table for assignment.", + "title": "Prebuilt Gages" + }, + "prebuilt_gages_layer": { + "default": "gages", + "description": "Layer name in the pre-built gages gpkg.", + "title": "Prebuilt Gages Layer", + "type": "string" + } + }, + "title": "GagesBlock", + "type": "object" + }, + "GagesInputs": { + "description": "gages: configs collection for all inputs", + "properties": { + "usgs_discontinued": { + "$ref": "#/$defs/GageInput" + }, + "usgs_active": { + "$ref": "#/$defs/GageInput" + }, + "txdot_gages": { + "$ref": "#/$defs/GageInput" + }, + "other": { + "$ref": "#/$defs/GageInput" + }, + "CIROH_UA": { + "$ref": "#/$defs/GageInput" + }, + "nwm_calib_gages": { + "$ref": "#/$defs/GageInput" + }, + "routelink": { + "$ref": "#/$defs/GageInput" + }, + "rfc": { + "$ref": "#/$defs/GageInput", + "description": "Table of active NWS gages. Retrieved from https://water.noaa.gov/about/data-and-web-services-catalog on 6/15/26" + }, + "nid": { + "$ref": "#/$defs/GageInput" + }, + "nwm_rfc": { + "$ref": "#/$defs/NWMRFCInput", + "default": { + "path": "rfc/reservoir_index_AnA.nc", + "rfc_id_col": "rfc_gage_id", + "usace_id_col": "usace_gage_id" + }, + "description": "An NWM v3 reservoid index file with RFC gages to retain" + }, + "adhoc_lakes": { + "$ref": "#/$defs/GageInput", + "description": "Adhoc lakes from reference waterbodies" + }, + "canada_great_lakes": { + "default": false, + "description": "Flag to pull Lake Erie and Lake Ontario Canadian gages from GreatLakesMapping class defined in Lakes", + "title": "Canada Great Lakes", + "type": "boolean" + }, + "usbr": { + "$ref": "#/$defs/GageInput", + "description": "USBR lakes" + }, + "usace": { + "$ref": "#/$defs/GageInput", + "description": "USACE gages/reservoirs" + } + }, + "title": "GagesInputs", + "type": "object" + }, + "GagesTarget": { + "description": "gages: target/output configs", + "properties": { + "crs": { + "default": "EPSG:5070", + "title": "Crs", + "type": "string" + }, + "snap_tolerance_m": { + "default": 100.0, + "title": "Snap Tolerance M", + "type": "number" + }, + "update_existing": { + "default": true, + "title": "Update Existing", + "type": "boolean" + }, + "exclude_ids": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Exclude Ids", + "type": "array" + }, + "out_gpkg": { + "default": "gages.gpkg", + "format": "path", + "title": "Out Gpkg", + "type": "string" + }, + "gpkg_layer_name": { + "default": "gages", + "title": "Gpkg Layer Name", + "type": "string" + } + }, + "title": "GagesTarget", + "type": "object" + }, + "NLDIUpstreamBasins": { + "description": "gages: for getting the upstream basins from NLDI USGS API", + "properties": { + "run_NLDI_upstream_basins": { + "default": false, + "title": "Run Nldi Upstream Basins", + "type": "boolean" + }, + "path": { + "default": "nldi_upstream_basins.gpkg", + "format": "path", + "title": "Path", + "type": "string" + }, + "layer_polys": { + "default": "NLDI_upstream_basins", + "title": "Layer Polys", + "type": "string" + }, + "layer_points": { + "default": "sites", + "title": "Layer Points", + "type": "string" + } + }, + "title": "NLDIUpstreamBasins", + "type": "object" + }, + "NWMRFCInput": { + "description": "NWM reservoir index file for retaining RFC and USACE gages. USACE IDs are found in NID.", + "properties": { + "path": { + "default": "rfc/reservoir_index_AnA.nc", + "format": "path", + "title": "Path", + "type": "string" + }, + "rfc_id_col": { + "default": "rfc_gage_id", + "title": "Rfc Id Col", + "type": "string" + }, + "usace_id_col": { + "default": "usace_gage_id", + "title": "Usace Id Col", + "type": "string" + } + }, + "title": "NWMRFCInput", + "type": "object" + } + }, + "description": "Gages config class", + "properties": { + "gages": { + "$ref": "#/$defs/GagesBlock" + }, + "NLDI_upstream_basins": { + "$ref": "#/$defs/NLDIUpstreamBasins" + }, + "assign_fp_to_gages": { + "$ref": "#/$defs/AssignFPConfig" + } + }, + "title": "GagesConfig", + "type": "object" +} diff --git a/docs/schemas/gages/gages_target.json b/docs/schemas/gages/gages_target.json new file mode 100644 index 0000000..c2b192c --- /dev/null +++ b/docs/schemas/gages/gages_target.json @@ -0,0 +1,47 @@ +{ + "description": "gages: target/output configs", + "properties": { + "crs": { + "default": "EPSG:5070", + "title": "Crs", + "type": "string" + }, + "snap_tolerance_m": { + "default": 100.0, + "title": "Snap Tolerance M", + "type": "number" + }, + "update_existing": { + "default": true, + "title": "Update Existing", + "type": "boolean" + }, + "exclude_ids": { + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "integer" + } + ] + }, + "title": "Exclude Ids", + "type": "array" + }, + "out_gpkg": { + "default": "gages.gpkg", + "format": "path", + "title": "Out Gpkg", + "type": "string" + }, + "gpkg_layer_name": { + "default": "gages", + "title": "Gpkg Layer Name", + "type": "string" + } + }, + "title": "GagesTarget", + "type": "object" +} diff --git a/docs/schemas/gages/nldi_upstream_basins.json b/docs/schemas/gages/nldi_upstream_basins.json new file mode 100644 index 0000000..7a40fa1 --- /dev/null +++ b/docs/schemas/gages/nldi_upstream_basins.json @@ -0,0 +1,28 @@ +{ + "description": "gages: for getting the upstream basins from NLDI USGS API", + "properties": { + "run_NLDI_upstream_basins": { + "default": false, + "title": "Run Nldi Upstream Basins", + "type": "boolean" + }, + "path": { + "default": "nldi_upstream_basins.gpkg", + "format": "path", + "title": "Path", + "type": "string" + }, + "layer_polys": { + "default": "NLDI_upstream_basins", + "title": "Layer Polys", + "type": "string" + }, + "layer_points": { + "default": "sites", + "title": "Layer Points", + "type": "string" + } + }, + "title": "NLDIUpstreamBasins", + "type": "object" +} diff --git a/docs/schemas/gages/nwm_rfc_input.json b/docs/schemas/gages/nwm_rfc_input.json new file mode 100644 index 0000000..15eb05c --- /dev/null +++ b/docs/schemas/gages/nwm_rfc_input.json @@ -0,0 +1,23 @@ +{ + "description": "NWM reservoir index file for retaining RFC and USACE gages. USACE IDs are found in NID.", + "properties": { + "path": { + "default": "rfc/reservoir_index_AnA.nc", + "format": "path", + "title": "Path", + "type": "string" + }, + "rfc_id_col": { + "default": "rfc_gage_id", + "title": "Rfc Id Col", + "type": "string" + }, + "usace_id_col": { + "default": "usace_gage_id", + "title": "Usace Id Col", + "type": "string" + } + }, + "title": "NWMRFCInput", + "type": "object" +} diff --git a/docs/schemas/lakes/adhoc_lake_inputusbr_lake_input.json b/docs/schemas/lakes/adhoc_lake_inputusbr_lake_input.json new file mode 100644 index 0000000..5f76c58 --- /dev/null +++ b/docs/schemas/lakes/adhoc_lake_inputusbr_lake_input.json @@ -0,0 +1,31 @@ +{ + "description": "Lakes: Adhoc lakes have been mapped to COMID, site_no (gage), and dam_id (reference reservoirs) when possible. Adhoc lakes that are only in reference waterbodies are flagged to force inclusion.", + "properties": { + "path": { + "default": "input/adhoc_lakes.gpkg", + "description": "Source path. LakesConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "layer": { + "default": "adhoc_lakes", + "title": "Layer", + "type": "string" + }, + "run": { + "default": true, + "description": "Flag to run Adhoc Lake input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "ref_wb_field": { + "default": "ref_waterbodies_only", + "description": "Field in the adhoc lakes table that flags if a lake is only in the reference waterbodies dataset (not in NWM lakes)", + "title": "Ref Wb Field", + "type": "string" + } + }, + "title": "AdhocLakeInput", + "type": "object" +} diff --git a/docs/schemas/lakes/great_lakes.json b/docs/schemas/lakes/great_lakes.json new file mode 100644 index 0000000..2699e0a --- /dev/null +++ b/docs/schemas/lakes/great_lakes.json @@ -0,0 +1,31 @@ +{ + "description": "Lakes: USBR lakes are mapped to COMID/lake_id. Add USBR lakes from reference reservoirs when not included in nwm lakes.", + "properties": { + "path": { + "default": "input/usbr_lake_crosswalk.gpkg", + "description": "Source path. LakesConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "layer": { + "default": "usbr_lake_crosswalk", + "title": "Layer", + "type": "string" + }, + "run": { + "default": true, + "description": "Flag to run USBR Lake input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "ref_wb_field": { + "default": "ref_wb_lake", + "description": "Field in the USBR table that flags if a lake is only in the reference waterbodies dataset (not in NWM lakes)", + "title": "Ref Wb Field", + "type": "string" + } + }, + "title": "USBRLakeInput", + "type": "object" +} diff --git a/docs/schemas/lakes/great_lakes_mapping.json b/docs/schemas/lakes/great_lakes_mapping.json new file mode 100644 index 0000000..2d6d002 --- /dev/null +++ b/docs/schemas/lakes/great_lakes_mapping.json @@ -0,0 +1,72 @@ +{ + "description": "Defines parameters about a Great Lake", + "properties": { + "lake_id": { + "description": "lake_id/NHD 2.2 COMID for Great Lake", + "title": "Lake Id", + "type": "string" + }, + "fp_id": { + "description": "NHF flowpath ID for Great Lake", + "title": "Fp Id", + "type": "number" + }, + "virtual_fp_id": { + "description": "NHF virtual flowpath ID for Great Lake", + "title": "Virtual Fp Id", + "type": "number" + }, + "site_no": { + "description": "Gage ID / site_no for Great Lake", + "title": "Site No", + "type": "string" + }, + "lat": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Add a manual latitude if needed to place gage", + "title": "Lat" + }, + "lon": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Add a manual longitude if needed to place gage", + "title": "Lon" + }, + "data_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Data source of gage", + "title": "Data Source" + } + }, + "required": [ + "lake_id", + "fp_id", + "virtual_fp_id", + "site_no" + ], + "title": "GreatLake", + "type": "object" +} diff --git a/docs/schemas/lakes/lake_dem_input.json b/docs/schemas/lakes/lake_dem_input.json new file mode 100644 index 0000000..4ac2967 --- /dev/null +++ b/docs/schemas/lakes/lake_dem_input.json @@ -0,0 +1,30 @@ +{ + "description": "Lakes: DEM inputs for Lakes", + "properties": { + "path": { + "default": "input/COP90_DEM_SuperCONUS.tif", + "description": "Source path. LakesConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "nodata": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Nodata value. Let rasterio infer if null", + "title": "Nodata" + } + }, + "title": "LakesDEMInputs", + "type": "object" +} diff --git a/docs/schemas/lakes/lakes_config.json b/docs/schemas/lakes/lakes_config.json new file mode 100644 index 0000000..c33bae6 --- /dev/null +++ b/docs/schemas/lakes/lakes_config.json @@ -0,0 +1,129 @@ +{ + "$defs": { + "GreatLake": { + "description": "Defines parameters about a Great Lake", + "properties": { + "lake_id": { + "description": "lake_id/NHD 2.2 COMID for Great Lake", + "title": "Lake Id", + "type": "string" + }, + "fp_id": { + "description": "NHF flowpath ID for Great Lake", + "title": "Fp Id", + "type": "number" + }, + "virtual_fp_id": { + "description": "NHF virtual flowpath ID for Great Lake", + "title": "Virtual Fp Id", + "type": "number" + }, + "site_no": { + "description": "Gage ID / site_no for Great Lake", + "title": "Site No", + "type": "string" + }, + "lat": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Add a manual latitude if needed to place gage", + "title": "Lat" + }, + "lon": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Add a manual longitude if needed to place gage", + "title": "Lon" + }, + "data_source": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Data source of gage", + "title": "Data Source" + } + }, + "required": [ + "lake_id", + "fp_id", + "virtual_fp_id", + "site_no" + ], + "title": "GreatLake", + "type": "object" + } + }, + "description": "All Great Lakes mappings", + "properties": { + "superior": { + "$ref": "#/$defs/GreatLake", + "default": { + "lake_id": "4800002", + "fp_id": 1278348162056612.0, + "virtual_fp_id": 1278346877373953.0, + "site_no": "04127885", + "lat": null, + "lon": null, + "data_source": "USGS" + } + }, + "mi_huron": { + "$ref": "#/$defs/GreatLake", + "default": { + "lake_id": "4800004", + "fp_id": 1276364270499315.0, + "virtual_fp_id": 1276364270423160.0, + "site_no": "04159130", + "lat": null, + "lon": null, + "data_source": "USGS" + } + }, + "erie": { + "$ref": "#/$defs/GreatLake", + "default": { + "lake_id": "4800006", + "fp_id": 1286192735893685.0, + "virtual_fp_id": 1286154743979494.0, + "site_no": "02HA013", + "lat": 42.93028, + "lon": -78.91417, + "data_source": "Environment Canada: https://wateroffice.ec.gc.ca/report/real_time_e.html?stn=02HA013" + } + }, + "ontario": { + "$ref": "#/$defs/GreatLake", + "default": { + "lake_id": "4800007", + "fp_id": 1287248237297035.0, + "virtual_fp_id": 1287248166320950.0, + "site_no": "IJC", + "lat": 45.00639, + "lon": -74.795, + "data_source": "International Lake Ontario-St. Lawrence River Board: https://ijc.org/en/loslrb/watershed/outflow-changes" + } + } + }, + "title": "GreatLakesMapping", + "type": "object" +} diff --git a/docs/schemas/lakes/nid_input.json b/docs/schemas/lakes/nid_input.json new file mode 100644 index 0000000..c5ad1a9 --- /dev/null +++ b/docs/schemas/lakes/nid_input.json @@ -0,0 +1,54 @@ +{ + "description": "Lakes: National Inventory of Dams (NID) inputs to improve hydaulic paramters. NID is available for Reference Reservoirs data.", + "properties": { + "path": { + "default": "input/NID2019_U.csv", + "description": "Source path. When using defaults, WaterbodiesConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "src_crs": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": "EPSG:4326", + "description": "Source CRS", + "title": "Src Crs" + }, + "output_crs": { + "default": "EPSG:5070", + "description": "Output CRS", + "title": "Output Crs", + "type": "string" + }, + "drop_states": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": [ + "AK", + "HI", + "PR", + "GU" + ], + "description": "States to drop from NID", + "title": "Drop States" + } + }, + "title": "NIDInputs", + "type": "object" +} diff --git a/docs/schemas/lakes/nwm_v3_lake_input.json b/docs/schemas/lakes/nwm_v3_lake_input.json new file mode 100644 index 0000000..3c3bf5b --- /dev/null +++ b/docs/schemas/lakes/nwm_v3_lake_input.json @@ -0,0 +1,155 @@ +{ + "description": "Lakes: NWM Lakes is a polygon dataset with COMID. This is the operational lakes layer provided by OWP.", + "properties": { + "path": { + "default": "input/nwm_lakes.gpkg", + "description": "Source path. LakesConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "buffered_path": { + "default": "input/nwm_lakes_sconus_input_500m_buffer.gpkg", + "description": "Source path, with polygons buffered out (for validating duplicate lake points). LakesConfig will inject preceding input path.", + "format": "path", + "title": "Buffered Path", + "type": "string" + }, + "layer": { + "default": "lakes", + "description": "GPKG layer", + "title": "Layer", + "type": "string" + }, + "fp_associated_path": { + "default": "output/fp_associated.gpkg", + "description": "A temporary layer to read lakes from where flowpaths have been associated and attributes joined. Skips flowpath association.", + "format": "path", + "title": "Fp Associated Path", + "type": "string" + }, + "run": { + "default": true, + "description": "Flag to run NWM lakes input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "improve_placement_ref_res": { + "default": true, + "description": "Flag to use reference reservoirs to improve placement of lakes.", + "title": "Improve Placement Ref Res", + "type": "boolean" + }, + "improve_placement_path": { + "default": "output/fp_improved placement.gpkg", + "description": "A temporary layer to read lakes from where flowpaths have been associated, attributes joined, and reference reservoirs folded in to improve placement. Skips reference reservoirs folding.", + "format": "path", + "title": "Improve Placement Path", + "type": "string" + }, + "use_cached_improve_placement": { + "default": false, + "description": "Use the cached layer stored at `improve_placement_path` regardless of whether the process is requested to run.", + "title": "Use Cached Improve Placement", + "type": "boolean" + }, + "associate_flowpaths": { + "default": true, + "description": "Flag to run flowpath association", + "title": "Associate Flowpaths", + "type": "boolean" + }, + "flowpath_association_method": { + "default": "polygon_outlet", + "description": "Type of flowpath association. Options are `polygon_outlet` or `nearest_point`", + "title": "Flowpath Association Method", + "type": "string" + }, + "search_radius_m": { + "default": 1000.0, + "description": "Distance from flowpath to search when associating flowpaths with points.", + "title": "Search Radius M", + "type": "number" + }, + "max_refres_search_distance_m": { + "default": 500.0, + "description": "Distance between NWM lake and reference reservoirs used when improving NWM lake placement.", + "title": "Max Refres Search Distance M", + "type": "number" + }, + "intersection_length_min_m": { + "default": 3.0, + "description": "Minimum prefered intersection lenght when associating polygons with flowpaths. If the flowpath intersection is extremely short, it can sometimes be almost entirely on a long downstream flowpath.", + "title": "Intersection Length Min M", + "type": "number" + }, + "id_field": { + "default": "newID", + "description": "ID field for NWM lakes input", + "title": "Id Field", + "type": "string" + }, + "attrib_src_path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Source file for joining attributes from another file. It will be skipped if null.", + "title": "Attrib Src Path" + }, + "attrib_src_layer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Source file layer for importing attributes", + "title": "Attrib Src Layer" + }, + "attrib_src_key": { + "default": "lake_id", + "description": "Source file key to match when importing attributes", + "title": "Attrib Src Key", + "type": "string" + }, + "fields": { + "default": [ + "lake_id", + "res_id", + "LkArea", + "LkMxE", + "WeirC", + "WeirL", + "WeirE", + "OrificeC", + "OrificeA", + "OrificeE", + "Dam_Length", + "ifd", + "reservoir_index_AnA", + "reservoir_index_Extended_AnA", + "reservoir_index_GDL_AK", + "reservoir_index_Medium_Range", + "reservoir_index_Short_Range" + ], + "description": "Fields to retain from NWM lakes data. IDs and geometry will be kept by default.", + "items": { + "type": "string" + }, + "title": "Fields", + "type": "array" + } + }, + "title": "NWMLakeInput", + "type": "object" +} diff --git a/docs/schemas/lakes/reference_reservoirs_input.json b/docs/schemas/lakes/reference_reservoirs_input.json new file mode 100644 index 0000000..52694d2 --- /dev/null +++ b/docs/schemas/lakes/reference_reservoirs_input.json @@ -0,0 +1,84 @@ +{ + "description": "Lakes: Reference Reservoirs is a point dataset of dams including hydraulic parameters from the National Inventory of Dams (NID).", + "properties": { + "path": { + "default": "input/reference-reservoirs-v1.gpkg", + "description": "Source path. When using defaults, LakesConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "layer": { + "default": "reference-reservoirs-v1", + "description": "GPKG layer", + "title": "Layer", + "type": "string" + }, + "run": { + "default": true, + "description": "Flag to run reference reservoirs input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "src_crs": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Source CRS", + "title": "Src Crs" + }, + "output_crs": { + "default": "EPSG:5070", + "description": "Output CRS", + "title": "Output Crs", + "type": "string" + }, + "distance_to_fp_col": { + "default": "distance_to_fp_m", + "description": "Distance to flowpath (m) column", + "title": "Distance To Fp Col", + "type": "string" + }, + "wb_area_col": { + "default": "wb_areasqkm", + "description": "Area (km2) column", + "title": "Wb Area Col", + "type": "string" + }, + "ref_wb_id_col": { + "default": "ref_fab_wb", + "description": "Reference waterbody ID column", + "title": "Ref Wb Id Col", + "type": "string" + }, + "min_wb_area_sqkm": { + "default": 0.2, + "description": "Minimum waterbody area (km2) to keep for RFC-DA. Use 0 to remove None.", + "title": "Min Wb Area Sqkm", + "type": "number" + }, + "max_distance_m": { + "default": 1000.0, + "description": "max distance of reference reservoir points from column 'distance_to_fp_m'", + "title": "Max Distance M", + "type": "number" + }, + "ref_res_keep": { + "default": [], + "description": "List of reference reservoir dam_id's to keep in RFCDA layer", + "items": { + "type": "string" + }, + "title": "Ref Res Keep", + "type": "array" + } + }, + "title": "ReferenceReservoirsInput", + "type": "object" +} diff --git a/docs/schemas/lakes/reference_waterbody_input.json b/docs/schemas/lakes/reference_waterbody_input.json new file mode 100644 index 0000000..3aea632 --- /dev/null +++ b/docs/schemas/lakes/reference_waterbody_input.json @@ -0,0 +1,96 @@ +{ + "description": "Lakes: Reference Waterbodies is a polygon dataset with COMID. Reference waterbodies will be used for Adhoc Lakes that are only found in Reference Waterbodies. The entire Reference Waterbodies file is not run.", + "properties": { + "path": { + "default": "input/reference_waterbodies.gpkg", + "description": "Source path. LakesConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "run": { + "default": true, + "description": "Flag to run Reference Waterbodies input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "fp_associated_path": { + "default": "input/refwb_tmp.gpkg", + "description": "A temporary layer to read waterbodies from where flowpaths have been associated and attributes joined. Skips flowpath association.", + "format": "path", + "title": "Fp Associated Path", + "type": "string" + }, + "associate_flowpaths": { + "default": true, + "description": "Flag to run flowpath association", + "title": "Associate Flowpaths", + "type": "boolean" + }, + "flowpath_association_method": { + "default": "polygon_outlet", + "description": "Type of flowpath association. Options are `polygon_outlet` or `nearest_point`", + "title": "Flowpath Association Method", + "type": "string" + }, + "search_radius_m": { + "default": 1000.0, + "description": "Distance from flowpath to search when associating flowpaths.", + "title": "Search Radius M", + "type": "number" + }, + "id_field": { + "default": "comid", + "description": "ID field for reference waterbodies", + "title": "Id Field", + "type": "string" + }, + "output_id_field": { + "default": "lake_id", + "description": "ID field to change name to for reference waterbodies", + "title": "Output Id Field", + "type": "string" + }, + "intersection_length_min_m": { + "default": 3.0, + "description": "Minimum prefered intersection length when associating polygons with flowpaths. If the flowpath intersection is extremely short, it can sometimes be almost entirely on a long downstream flowpath.", + "title": "Intersection Length Min M", + "type": "number" + }, + "attrib_src_path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Source file for joining attributes from another file. It will be skipped if null.", + "title": "Attrib Src Path" + }, + "attrib_src_layer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Source file layer for importing attributes", + "title": "Attrib Src Layer" + }, + "attrib_src_key": { + "default": null, + "description": "Source file key to match when importing attributes", + "title": "Attrib Src Key", + "type": "string" + } + }, + "title": "RefWaterbodyInput", + "type": "object" +} diff --git a/docs/schemas/network/aggregations.json b/docs/schemas/network/aggregations.json new file mode 100644 index 0000000..83155d8 --- /dev/null +++ b/docs/schemas/network/aggregations.json @@ -0,0 +1,59 @@ +{ + "description": "A Pydantic BaseModel Container to contain flowpath_id classifications for aggregation", + "properties": { + "aggregates": { + "description": "A list of geometries for aggreagated flowpaths and divides along with upstream and downstream identifiers", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Aggregates", + "type": "array" + }, + "independents": { + "description": "A list of independent segments and their geometries", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Independents", + "type": "array" + }, + "non_nextgen_flowpaths": { + "description": "A list of virtual flowpaths and their geometries", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Non Nextgen Flowpaths", + "type": "array" + }, + "connectors": { + "description": "A list of connection segments and their geometries", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Connectors", + "type": "array" + }, + "non_nextgen_virtual_flowpaths": { + "description": "A list of all non_nextgen virtual flowpaths and their geometries", + "items": { + "additionalProperties": true, + "type": "object" + }, + "title": "Non Nextgen Virtual Flowpaths", + "type": "array" + } + }, + "required": [ + "aggregates", + "independents", + "non_nextgen_flowpaths", + "connectors", + "non_nextgen_virtual_flowpaths" + ], + "title": "Aggregations", + "type": "object" +} diff --git a/docs/schemas/network/build_hydrofabric_config.json b/docs/schemas/network/build_hydrofabric_config.json new file mode 100644 index 0000000..2f19c4b --- /dev/null +++ b/docs/schemas/network/build_hydrofabric_config.json @@ -0,0 +1,44 @@ +{ + "description": "Configs for buld hydrofabric stage", + "properties": { + "reference_divides_path": { + "default": "data/reference_divides.parquet", + "description": "The location of the reference fabric divides.", + "title": "Reference Divides Path", + "type": "string" + }, + "reference_flowpaths_path": { + "default": "data/reference_flowpaths.parquet", + "description": "The location of the reference fabric flowpaths.", + "title": "Reference Flowpaths Path", + "type": "string" + }, + "divide_aggregation_threshold": { + "default": 3.0, + "description": "Threshold for divides to aggreagate into an upstream catchment [km^2]", + "title": "Divide Aggregation Threshold", + "type": "number" + }, + "headwater_virtual_length_threshold": { + "default": 0.3, + "description": "Order-1 headwater flowpaths shorter than this length (km) are virtualized instead of independent", + "title": "Headwater Virtual Length Threshold", + "type": "number" + }, + "debug_outlet_count": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Debug setting to limit the number of outlets processed. None (default) processes all outlets. Set to a positive integer to limit for testing.", + "title": "Debug Outlet Count" + } + }, + "title": "BuildHydrofabricConfig", + "type": "object" +} diff --git a/docs/schemas/network/classifications.json b/docs/schemas/network/classifications.json new file mode 100644 index 0000000..1702e88 --- /dev/null +++ b/docs/schemas/network/classifications.json @@ -0,0 +1,82 @@ +{ + "description": "A Pydantic BaseModel Container to contain flowpath_id classifications for aggregation", + "properties": { + "aggregation_pairs": { + "description": "A list of tuples for flowpaths to be aggregated together. Format: (downstream_id, upstream_id, ...) where downstream merges into upstream", + "items": { + "items": { + "type": "string" + }, + "type": "array" + }, + "title": "Aggregation Pairs", + "type": "array" + }, + "non_nextgen_flowpaths": { + "description": "Reference flowpaths classified as 'virtual' tributaries. These are flowpaths that are stream-order 1, with a total DA of < threshold where routing will not be run", + "items": { + "type": "string" + }, + "title": "Non Nextgen Flowpaths", + "type": "array", + "uniqueItems": true + }, + "independent_flowpaths": { + "description": "Flowpaths that remain independent and are NOT aggregated. These are large catchments (areasqkm > threshold) that form their own divides", + "items": { + "type": "string" + }, + "title": "Independent Flowpaths", + "type": "array", + "uniqueItems": true + }, + "connector_segments": { + "description": "Small flowpaths (areasqkm < threshold) that connect two higher-order streams. These have two upstream flowpaths where both have streamorder > 1. They remain independent despite being small because they serve as connectors between large stream branches, and aggregation would present inconsistencies within routing", + "items": { + "type": "string" + }, + "title": "Connector Segments", + "type": "array" + }, + "non_nextgen_virtual_flowpath_pairs": { + "description": "List of non-NextGen virtual flowpath pairs (source_id, target_id).", + "items": { + "items": { + "type": "string" + }, + "type": "array" + }, + "title": "Non Nextgen Virtual Flowpath Pairs", + "type": "array" + }, + "processed_flowpaths": { + "description": "Set of all flowpath IDs that have been processed during the outlet upstream tracing. Used internally to prevent re-processing flowpaths by mistake (which creates cycles)", + "items": { + "type": "string" + }, + "title": "Processed Flowpaths", + "type": "array", + "uniqueItems": true + }, + "force_queue_flowpaths": { + "description": "flowpaths that are required to be queued. These are only for streams deeply nested in no-divide connectors", + "items": { + "type": "string" + }, + "title": "Force Queue Flowpaths", + "type": "array", + "uniqueItems": true + }, + "aggregation_set": { + "description": "A set flowpaths that have been aggregated together", + "items": { + "type": "string" + }, + "title": "Aggregation Set", + "type": "array", + "uniqueItems": true + } + }, + "title": "Classifications", + "type": "object" +} diff --git a/docs/schemas/network/divide_attribute_config.json b/docs/schemas/network/divide_attribute_config.json new file mode 100644 index 0000000..9d552ca --- /dev/null +++ b/docs/schemas/network/divide_attribute_config.json @@ -0,0 +1,58 @@ +{ + "$defs": { + "AggTypeEnum": { + "description": "Zonal statistics aggregation types", + "enum": [ + "mean", + "mode", + "max", + "weighted_circular_mean", + "quantile_dist", + "quartile_dist", + "weighted_geometric_mean", + "percent", + "groundwater" + ], + "title": "AggTypeEnum", + "type": "string" + } + }, + "description": "Pydantic model for divide attributes attribute configuration", + "properties": { + "data_dir": { + "default": "/mnt/share/projects/hydro/code/nhf-builds2/nhf-builds/data/divide_attributes", + "description": "Top level directory for data layers", + "format": "path", + "title": "Data Dir", + "type": "string" + }, + "agg_type": { + "$ref": "#/$defs/AggTypeEnum", + "description": "Zonal stats aggregation type" + }, + "field_name": { + "description": "Output field name for divide attribute", + "title": "Field Name", + "type": "string" + }, + "file_name": { + "description": "File path of attribute raster", + "format": "path", + "title": "File Name", + "type": "string" + }, + "tmp": { + "description": "Temp file path for parquet", + "format": "path", + "title": "Tmp", + "type": "string" + } + }, + "required": [ + "agg_type", + "field_name", + "file_name" + ], + "title": "DivideAttributeConfig", + "type": "object" +} diff --git a/docs/schemas/network/divide_attribute_model_config.json b/docs/schemas/network/divide_attribute_model_config.json new file mode 100644 index 0000000..64d5c42 --- /dev/null +++ b/docs/schemas/network/divide_attribute_model_config.json @@ -0,0 +1,190 @@ +{ + "$defs": { + "AggTypeEnum": { + "description": "Zonal statistics aggregation types", + "enum": [ + "mean", + "mode", + "max", + "weighted_circular_mean", + "quantile_dist", + "quartile_dist", + "weighted_geometric_mean", + "percent", + "groundwater" + ], + "title": "AggTypeEnum", + "type": "string" + }, + "DivideAttributeConfig": { + "description": "Pydantic model for divide attributes attribute configuration", + "properties": { + "data_dir": { + "default": "nhf-builds/data/divide_attributes", + "description": "Top level directory for data layers", + "format": "path", + "title": "Data Dir", + "type": "string" + }, + "agg_type": { + "$ref": "#/$defs/AggTypeEnum", + "description": "Zonal stats aggregation type" + }, + "field_name": { + "description": "Output field name for divide attribute", + "title": "Field Name", + "type": "string" + }, + "file_name": { + "description": "File path of attribute raster", + "format": "path", + "title": "File Name", + "type": "string" + }, + "tmp": { + "description": "Temp file path for parquet", + "format": "path", + "title": "Tmp", + "type": "string" + } + }, + "required": [ + "agg_type", + "field_name", + "file_name" + ], + "title": "DivideAttributeConfig", + "type": "object" + } + }, + "description": "Pydantic model for divide attributes model configuration", + "properties": { + "hf_path": { + "default": null, + "description": "Path to input and output hydrofabric", + "format": "path", + "title": "Hf Path", + "type": "string" + }, + "crs": { + "default": "EPSG:5070", + "description": "Domain CRS", + "title": "Crs", + "type": "string" + }, + "processes": { + "default": 32, + "description": "Number of processes to use for multiprocessing", + "title": "Processes", + "type": "integer" + }, + "data_dir": { + "default": "nhf-builds/data/divide_attributes", + "description": "Directory of all input data", + "format": "path", + "title": "Data Dir", + "type": "string" + }, + "divide_id": { + "default": "div_id", + "description": "Field name for unique divide id", + "title": "Divide Id", + "type": "string" + }, + "attributes": { + "default": null, + "description": "List of attributes to be computed. Specify in DivideAttributeConfig data model.", + "items": { + "$ref": "#/$defs/DivideAttributeConfig" + }, + "title": "Attributes", + "type": "array" + }, + "divides_path_list": { + "anyOf": [ + { + "items": { + "format": "path", + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of divides paths to use for parallel run. ex. list of VPU subsets.", + "title": "Divides Path List" + }, + "tmp_dir": { + "default": "/tmp/divide-attributes", + "description": "Temp path for saving files", + "format": "path", + "title": "Tmp Dir", + "type": "string" + }, + "split_vpu": { + "anyOf": [ + { + "type": "boolean" + }, + { + "type": "null" + } + ], + "default": false, + "description": "If running in parallel, this will split the domain divides file into separate files.Each VPU can be run separately and will be stitched at end.This will replace anything input to the `divides_path_list`", + "title": "Split Vpu" + }, + "debug": { + "default": false, + "description": "Setting debug to true will save all temporary files. Setting to false will delete files if run fails.", + "title": "Debug", + "type": "boolean" + }, + "domain_mask": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mask a domain to only calculate attributes for a subset. Built to accommodate AK domain being smaller than full state.", + "title": "Domain Mask" + }, + "domain_mask_layer": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "GPKG layer to use for mask", + "title": "Domain Mask Layer" + }, + "divides_masked": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Path to saved masked divides to", + "title": "Divides Masked" + } + }, + "title": "DivideAttributesModelConfig", + "type": "object" +} diff --git a/docs/schemas/network/flowpath_attributes_config.json b/docs/schemas/network/flowpath_attributes_config.json new file mode 100644 index 0000000..77e3e9e --- /dev/null +++ b/docs/schemas/network/flowpath_attributes_config.json @@ -0,0 +1,207 @@ +{ + "description": "Flowpath attributes model to configure and calculate attributes\n\nDefaults from WRF GIS Preprocessor\nSource: https://github.com/NCAR/wrf_hydro_gis_preprocessor/blob/5781ad4788434e8fd4ec16f3a3805d98536a9f82/wrfhydro_gis/wrfhydro_functions.py#L128\nAccessed 10/20/25", + "properties": { + "use_stream_order": { + "default": true, + "description": "Setting to use stream order to calculate Manning's n (n), Bottom Width (BtmWdth), and Channel side slope (ChSlp). When true, calculate these attributes from stream order. When false, use defaults.", + "title": "Stream Order Setting", + "type": "boolean" + }, + "stream_order": { + "anyOf": [ + { + "type": "integer" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Strahler Stream Order 1-10", + "title": "Strahler Stream Order" + }, + "total_da_sqkm": { + "description": "Drainage area (sqkm) from flowpaths", + "title": "Drainage Area (km2)", + "type": "number" + }, + "Y": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Estimated depth associated with TopWdth (m)", + "title": "Estimated Depth" + }, + "r": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Dingmans's r", + "title": "Dingman's r" + }, + "n": { + "default": 0.035, + "description": "Manning's in channel roughness / n. Can be derived from Strahler stream order. Defaults to 0.035 without stream order", + "title": "Mannning's in channel roughness", + "type": "number" + }, + "nCC": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Compound Channel Top Width (m). 2*n", + "title": "Compound Channel Top Width" + }, + "BtmWdth": { + "default": 5, + "description": "Bottom width of channel (m). Can be derived from Strahler stream order. Defaults to 5 without stream order", + "title": "Bottom width of channel", + "type": "number" + }, + "TopWdth": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Top Width (m)", + "title": "Top Width" + }, + "TopWdthCC": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Compound Channel Top Width (m)", + "title": "Compound Channel Top Width" + }, + "ChSlp": { + "default": 0.05, + "description": "Channel side slope. Can be derived from Strahler stream order. Defaults to 0.05 without stream order.", + "title": "Channel Side Slope", + "type": "number" + }, + "alt": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Mean elevation (m) between nodes from 3DEP", + "title": "Elevation" + }, + "So": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Slope (meters/meters) computed from 3DEP", + "title": "Slope" + }, + "MusX": { + "default": 0.2, + "description": "Muskingum Weighting Coefficient. Defaults to 0.2", + "title": "Muskingum Weighting Coeffiecent", + "type": "number" + }, + "MusK": { + "default": 3600, + "description": "Muskingum routing time (seconds). Defaults to 3600", + "title": "Muskingum routing time", + "type": "number" + }, + "Y_ml": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Estimated depth associated with TopWdth (m) calculated from RiverML", + "title": "Estimated Depth ML" + }, + "r_ml": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Dingmans's r calculated from RiverML", + "title": "Dingman's r ML" + }, + "TopWdth_ml": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Top Width (m) calculated from RiverML", + "title": "Top Width ML" + }, + "TopWdthCC_ml": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": null, + "description": "Compound Channel Top Width (m) calculated from RiverML", + "title": "Compound Channel Top Width ML" + } + }, + "required": [ + "total_da_sqkm" + ], + "title": "FlowpathAttributesConfig", + "type": "object" +} diff --git a/docs/schemas/network/flowpath_attributes_model_config.json b/docs/schemas/network/flowpath_attributes_model_config.json new file mode 100644 index 0000000..fc330fa --- /dev/null +++ b/docs/schemas/network/flowpath_attributes_model_config.json @@ -0,0 +1,54 @@ +{ + "description": "Configurations for running flowpath attributes", + "properties": { + "hf_path": { + "default": null, + "description": "Path to input and output hydrofabric", + "format": "path", + "title": "Hydrofabric Path", + "type": "string" + }, + "flowpath_id": { + "default": "fp_id", + "description": "Flowpath ID field", + "title": "Flowpath ID", + "type": "string" + }, + "use_stream_order": { + "default": true, + "description": "Setting to use stream order to calculate Manning's n (n), Bottom Width (BtmWdth), and Channel side slope (ChSlp). When true, calculate these attributes from stream order. When false, use defaults.", + "title": "Stream Order Setting", + "type": "boolean" + }, + "dem_path": { + "default": "nhf-builds/data/usgs_250m_dem_5070.tif", + "description": "Path to DEM", + "format": "path", + "title": "DEM Path", + "type": "string" + }, + "tw_path": { + "default": null, + "description": "Path to RiverML topwidth predictions. If None, it will be skipped.", + "format": "path", + "title": "Topwidth Path", + "type": "string" + }, + "y_path": { + "default": null, + "description": "Path to RiverML Y predictions. If None, it will be skipped.", + "format": "path", + "title": "Y Path", + "type": "string" + }, + "r_path": { + "default": null, + "description": "Path to RiverML R predictions. If None, it will be skipped.", + "format": "path", + "title": "R Path", + "type": "string" + } + }, + "title": "FlowpathAttributesModelConfig", + "type": "object" +} diff --git a/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_config.json b/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_config.json new file mode 100644 index 0000000..2d31347 --- /dev/null +++ b/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_config.json @@ -0,0 +1,107 @@ +{ + "$defs": { + "FPCrosswalkOutputs": { + "description": "fp_crosswalk: final matches table inside a GPKG (no geometry is fine)", + "properties": { + "path": { + "default": "data/fp_crosswalk_matches.parquet", + "format": "path", + "title": "Path", + "type": "string" + } + }, + "title": "FPCrosswalkOutputs", + "type": "object" + }, + "FPCrosswalkReference": { + "description": "fp_crosswalk: reference (file1) network", + "properties": { + "id_col": { + "default": "flowpath_id", + "title": "Id Col", + "type": "string" + } + }, + "title": "FPCrosswalkReference", + "type": "object" + }, + "FPCrosswalkTarget": { + "description": "fp_crosswalk: target (file2) network", + "properties": { + "path": { + "default": "data/nwm_flows.gpkg", + "format": "path", + "title": "Path", + "type": "string" + }, + "layer": { + "default": "nwm_streams", + "title": "Layer", + "type": "string" + }, + "id_col": { + "default": "ID", + "title": "Id Col", + "type": "string" + } + }, + "title": "FPCrosswalkTarget", + "type": "object" + } + }, + "description": "Config class for the `fp_crosswalk:` block in the YAML.", + "properties": { + "reference": { + "$ref": "#/$defs/FPCrosswalkReference" + }, + "target": { + "$ref": "#/$defs/FPCrosswalkTarget" + }, + "outputs": { + "$ref": "#/$defs/FPCrosswalkOutputs" + }, + "search_radius_m": { + "default": 15.0, + "title": "Search Radius M", + "type": "number" + }, + "percent_inside_min": { + "default": 0.005, + "title": "Percent Inside Min", + "type": "number" + }, + "node_snap_tol_m": { + "default": 0.01, + "title": "Node Snap Tol M", + "type": "number" + }, + "endpoint_buffer_m": { + "default": 20.0, + "title": "Endpoint Buffer M", + "type": "number" + }, + "parallel": { + "default": true, + "title": "Parallel", + "type": "boolean" + }, + "n_jobs": { + "default": 3, + "title": "N Jobs", + "type": "integer" + }, + "parallel_chunksize": { + "default": 200, + "title": "Parallel Chunksize", + "type": "integer" + }, + "path": { + "default": "/data/nhd_crosswalk.parquet", + "format": "path", + "title": "Path", + "type": "string" + } + }, + "title": "FPCrosswalkConfig", + "type": "object" +} diff --git a/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_outputs.json b/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_outputs.json new file mode 100644 index 0000000..70da38d --- /dev/null +++ b/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_outputs.json @@ -0,0 +1,13 @@ +{ + "description": "fp_crosswalk: final matches table inside a GPKG (no geometry is fine)", + "properties": { + "path": { + "default": "data/fp_crosswalk_matches.parquet", + "format": "path", + "title": "Path", + "type": "string" + } + }, + "title": "FPCrosswalkOutputs", + "type": "object" +} diff --git a/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_reference.json b/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_reference.json new file mode 100644 index 0000000..04a5ea8 --- /dev/null +++ b/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_reference.json @@ -0,0 +1,12 @@ +{ + "description": "fp_crosswalk: reference (file1) network", + "properties": { + "id_col": { + "default": "flowpath_id", + "title": "Id Col", + "type": "string" + } + }, + "title": "FPCrosswalkReference", + "type": "object" +} diff --git a/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_target.json b/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_target.json new file mode 100644 index 0000000..ebed6c2 --- /dev/null +++ b/docs/schemas/nhd_flowpath_crosswalk/nhd_fp_crosswalk_target.json @@ -0,0 +1,23 @@ +{ + "description": "fp_crosswalk: target (file2) network", + "properties": { + "path": { + "default": "data/nwm_flows.gpkg", + "format": "path", + "title": "Path", + "type": "string" + }, + "layer": { + "default": "nwm_streams", + "title": "Layer", + "type": "string" + }, + "id_col": { + "default": "ID", + "title": "Id Col", + "type": "string" + } + }, + "title": "FPCrosswalkTarget", + "type": "object" +} diff --git a/docs/schemas/reservoir_da/active_rfc.json b/docs/schemas/reservoir_da/active_rfc.json new file mode 100644 index 0000000..ceeb25d --- /dev/null +++ b/docs/schemas/reservoir_da/active_rfc.json @@ -0,0 +1,18 @@ +{ + "description": "Describes table of active NWS gages for reservoir DA. This is optionally used in gages.\n\nRetrieved from https://water.noaa.gov/about/data-and-web-services-catalog on 6/15/26", + "properties": { + "path": { + "default": "input/nwps_all_gauges_report.csv", + "format": "path", + "title": "Path", + "type": "string" + }, + "id_field": { + "default": "nws shef id", + "title": "Id Field", + "type": "string" + } + }, + "title": "ActiveRFC", + "type": "object" +} diff --git a/docs/schemas/reservoir_da/adhoc_reservoir_da_input.json b/docs/schemas/reservoir_da/adhoc_reservoir_da_input.json new file mode 100644 index 0000000..452cedc --- /dev/null +++ b/docs/schemas/reservoir_da/adhoc_reservoir_da_input.json @@ -0,0 +1,44 @@ +{ + "description": "Adhoc lakes input for reservoir DA. Must include lake_id and an rfc_field with the name of RFC gage.", + "properties": { + "path": { + "default": "input/adhoc_lakes.gpkg", + "description": "Source path. ResDAConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "layer": { + "default": "adhoc_lakes", + "description": "Layer in adhoc lakes gpkg", + "title": "Layer", + "type": "string" + }, + "run": { + "default": false, + "description": "Flag to use Adhoc Lake input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "rfc_field": { + "default": "locationId", + "description": "Field containing RFC gage ID", + "title": "Rfc Field", + "type": "string" + }, + "lake_id_field": { + "default": "lake_id", + "description": "Field containing common lake COMID", + "title": "Lake Id Field", + "type": "string" + }, + "null_value": { + "default": -99999, + "description": "Missing data value", + "title": "Null Value", + "type": "integer" + } + }, + "title": "AdhocResDAInput", + "type": "object" +} diff --git a/docs/schemas/reservoir_da/reservoir_crosswalk_fields.json b/docs/schemas/reservoir_da/reservoir_crosswalk_fields.json new file mode 100644 index 0000000..4e3fcd5 --- /dev/null +++ b/docs/schemas/reservoir_da/reservoir_crosswalk_fields.json @@ -0,0 +1,49 @@ +{ + "description": "Fields in reservoir DA crosswalk file (\"reservoir_index_AnA.netcdf).\n\nThese can be changed in config file if needed and will be input read index functions.", + "properties": { + "lake_id_field": { + "default": "lake_id", + "description": "Lake ID field in Res ANA index file", + "title": "Lake Id Field", + "type": "string" + }, + "usgs_gage_id_field": { + "default": "usgs_gage_id", + "description": "USGS gage ID field in reservoir index file", + "title": "Usgs Gage Id Field", + "type": "string" + }, + "usgs_lake_id_field": { + "default": "usgs_lake_id", + "description": "USGS lake ID field in reservoir index file", + "title": "Usgs Lake Id Field", + "type": "string" + }, + "usace_gage_id_field": { + "default": "usace_gage_id", + "description": "USACE gage ID field in reservoir index file", + "title": "Usace Gage Id Field", + "type": "string" + }, + "usace_lake_id_field": { + "default": "usace_lake_id", + "description": "USACE lake ID field in reservoir index file", + "title": "Usace Lake Id Field", + "type": "string" + }, + "rfc_gage_id_field": { + "default": "rfc_gage_id", + "description": "RFC gage ID in reservoir index", + "title": "Rfc Gage Id Field", + "type": "string" + }, + "rfc_lake_id_field": { + "default": "rfc_lake_id", + "description": "RFC lake ID in reservoir index", + "title": "Rfc Lake Id Field", + "type": "string" + } + }, + "title": "ResCrosswalkFields", + "type": "object" +} diff --git a/docs/schemas/reservoir_da/reservoir_crosswalk_input.json b/docs/schemas/reservoir_da/reservoir_crosswalk_input.json new file mode 100644 index 0000000..4cd75cb --- /dev/null +++ b/docs/schemas/reservoir_da/reservoir_crosswalk_input.json @@ -0,0 +1,78 @@ +{ + "$defs": { + "ResCrosswalkFields": { + "description": "Fields in reservoir DA crosswalk file (\"reservoir_index_AnA.netcdf).\n\nThese can be changed in config file if needed and will be input read index functions.", + "properties": { + "lake_id_field": { + "default": "lake_id", + "description": "Lake ID field in Res ANA index file", + "title": "Lake Id Field", + "type": "string" + }, + "usgs_gage_id_field": { + "default": "usgs_gage_id", + "description": "USGS gage ID field in reservoir index file", + "title": "Usgs Gage Id Field", + "type": "string" + }, + "usgs_lake_id_field": { + "default": "usgs_lake_id", + "description": "USGS lake ID field in reservoir index file", + "title": "Usgs Lake Id Field", + "type": "string" + }, + "usace_gage_id_field": { + "default": "usace_gage_id", + "description": "USACE gage ID field in reservoir index file", + "title": "Usace Gage Id Field", + "type": "string" + }, + "usace_lake_id_field": { + "default": "usace_lake_id", + "description": "USACE lake ID field in reservoir index file", + "title": "Usace Lake Id Field", + "type": "string" + }, + "rfc_gage_id_field": { + "default": "rfc_gage_id", + "description": "RFC gage ID in reservoir index", + "title": "Rfc Gage Id Field", + "type": "string" + }, + "rfc_lake_id_field": { + "default": "rfc_lake_id", + "description": "RFC lake ID in reservoir index", + "title": "Rfc Lake Id Field", + "type": "string" + } + }, + "title": "ResCrosswalkFields", + "type": "object" + } + }, + "description": "Reservoir crosswalk file (\"reservoir_index_AnA.netcdf\")", + "properties": { + "path": { + "default": "input/reservoir_index_AnA.nc", + "description": "File with reservoir crosswalks for USGS, USACE, RFC", + "format": "path", + "title": "Path", + "type": "string" + }, + "fields": { + "$ref": "#/$defs/ResCrosswalkFields", + "default": { + "lake_id_field": "lake_id", + "usgs_gage_id_field": "usgs_gage_id", + "usgs_lake_id_field": "usgs_lake_id", + "usace_gage_id_field": "usace_gage_id", + "usace_lake_id_field": "usace_lake_id", + "rfc_gage_id_field": "rfc_gage_id", + "rfc_lake_id_field": "rfc_lake_id" + }, + "description": "All field mappings for reservoir index file" + } + }, + "title": "ResCrossWalkInput", + "type": "object" +} diff --git a/docs/schemas/reservoir_da/reservoir_da_config.json b/docs/schemas/reservoir_da/reservoir_da_config.json new file mode 100644 index 0000000..4138b7a --- /dev/null +++ b/docs/schemas/reservoir_da/reservoir_da_config.json @@ -0,0 +1,326 @@ +{ + "$defs": { + "ActiveRFC": { + "description": "Describes table of active NWS gages for reservoir DA. This is optionally used in gages.\n\nRetrieved from https://water.noaa.gov/about/data-and-web-services-catalog on 6/15/26", + "properties": { + "path": { + "default": "input/nwps_all_gauges_report.csv", + "format": "path", + "title": "Path", + "type": "string" + }, + "id_field": { + "default": "nws shef id", + "title": "Id Field", + "type": "string" + } + }, + "title": "ActiveRFC", + "type": "object" + }, + "AdhocResDAInput": { + "description": "Adhoc lakes input for reservoir DA. Must include lake_id and an rfc_field with the name of RFC gage.", + "properties": { + "path": { + "default": "input/adhoc_lakes.gpkg", + "description": "Source path. ResDAConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "layer": { + "default": "adhoc_lakes", + "description": "Layer in adhoc lakes gpkg", + "title": "Layer", + "type": "string" + }, + "run": { + "default": false, + "description": "Flag to use Adhoc Lake input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "rfc_field": { + "default": "locationId", + "description": "Field containing RFC gage ID", + "title": "Rfc Field", + "type": "string" + }, + "lake_id_field": { + "default": "lake_id", + "description": "Field containing common lake COMID", + "title": "Lake Id Field", + "type": "string" + }, + "null_value": { + "default": -99999, + "description": "Missing data value", + "title": "Null Value", + "type": "integer" + } + }, + "title": "AdhocResDAInput", + "type": "object" + }, + "ResCrossWalkInput": { + "description": "Reservoir crosswalk file (\"reservoir_index_AnA.netcdf\")", + "properties": { + "path": { + "default": "input/reservoir_index_AnA.nc", + "description": "File with reservoir crosswalks for USGS, USACE, RFC", + "format": "path", + "title": "Path", + "type": "string" + }, + "fields": { + "$ref": "#/$defs/ResCrosswalkFields", + "default": { + "lake_id_field": "lake_id", + "usgs_gage_id_field": "usgs_gage_id", + "usgs_lake_id_field": "usgs_lake_id", + "usace_gage_id_field": "usace_gage_id", + "usace_lake_id_field": "usace_lake_id", + "rfc_gage_id_field": "rfc_gage_id", + "rfc_lake_id_field": "rfc_lake_id" + }, + "description": "All field mappings for reservoir index file" + } + }, + "title": "ResCrossWalkInput", + "type": "object" + }, + "ResCrosswalkFields": { + "description": "Fields in reservoir DA crosswalk file (\"reservoir_index_AnA.netcdf).\n\nThese can be changed in config file if needed and will be input read index functions.", + "properties": { + "lake_id_field": { + "default": "lake_id", + "description": "Lake ID field in Res ANA index file", + "title": "Lake Id Field", + "type": "string" + }, + "usgs_gage_id_field": { + "default": "usgs_gage_id", + "description": "USGS gage ID field in reservoir index file", + "title": "Usgs Gage Id Field", + "type": "string" + }, + "usgs_lake_id_field": { + "default": "usgs_lake_id", + "description": "USGS lake ID field in reservoir index file", + "title": "Usgs Lake Id Field", + "type": "string" + }, + "usace_gage_id_field": { + "default": "usace_gage_id", + "description": "USACE gage ID field in reservoir index file", + "title": "Usace Gage Id Field", + "type": "string" + }, + "usace_lake_id_field": { + "default": "usace_lake_id", + "description": "USACE lake ID field in reservoir index file", + "title": "Usace Lake Id Field", + "type": "string" + }, + "rfc_gage_id_field": { + "default": "rfc_gage_id", + "description": "RFC gage ID in reservoir index", + "title": "Rfc Gage Id Field", + "type": "string" + }, + "rfc_lake_id_field": { + "default": "rfc_lake_id", + "description": "RFC lake ID in reservoir index", + "title": "Rfc Lake Id Field", + "type": "string" + } + }, + "title": "ResCrosswalkFields", + "type": "object" + }, + "USACEResDAInput": { + "description": "USACE : lake_id crosswalk for reservoir DA.", + "properties": { + "path": { + "default": "input/usace_crosswalk.gpkg", + "description": "Source path. ResDAConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "run": { + "default": false, + "description": "Flag to use USACE reservoir input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "lake_id_field": { + "default": "lake_id", + "description": "Field containing common lake COMID", + "title": "Lake Id Field", + "type": "string" + }, + "id_field": { + "default": "location", + "description": "Field containing shared reservoir/gage ID.", + "title": "Id Field", + "type": "string" + } + }, + "title": "USACEResDAInput", + "type": "object" + }, + "USBRResDAInput": { + "description": "USBR : lake_id crosswalk for reservoir DA.", + "properties": { + "path": { + "default": "input/usbr_lake_crosswalk.gpkg", + "description": "Source path. ResDAConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "run": { + "default": false, + "description": "Flag to use USBR reservoir input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "lake_id_field": { + "default": "lake_id", + "description": "Field containing common lake COMID", + "title": "Lake Id Field", + "type": "string" + }, + "id_field": { + "default": "locId", + "description": "Field containing shared reservoir/gage ID.", + "title": "Id Field", + "type": "string" + } + }, + "title": "USBRResDAInput", + "type": "object" + } + }, + "description": "Configuration for reservoir DA", + "properties": { + "input_dir": { + "default": "/mnt/share/projects/hydro/code/nhf-builds2/nhf-builds/data/lakes", + "description": "Input directory. This will be prepended to all paths for other inputs", + "format": "path", + "title": "Input Dir", + "type": "string" + }, + "all_level_pool": { + "default": false, + "description": "Flag to make all reservoirs level pool. This can be used in domains with no lake-gage crosswalk available.", + "title": "All Level Pool", + "type": "boolean" + }, + "adhoc": { + "$ref": "#/$defs/AdhocResDAInput", + "default": { + "path": "input/adhoc_lakes.gpkg", + "layer": "adhoc_lakes", + "run": false, + "rfc_field": "locationId", + "lake_id_field": "lake_id", + "null_value": -99999 + }, + "description": "All Adhoc Lakes Input configs. Adhoc lakes have been mapped to COMID/lake_id, site_no (gage), and dam_id (reference reservoirs) when possible." + }, + "lake_id_field": { + "default": "lake_id", + "description": "The common name of 'comid' field that is present in various datasets", + "title": "Lake Id Field", + "type": "string" + }, + "gage_id_field": { + "default": "site_no", + "description": "Name for output gage ID field", + "title": "Gage Id Field", + "type": "string" + }, + "da_type_field": { + "default": "da_type", + "description": "Name for output reservoir DA type field", + "title": "Da Type Field", + "type": "string" + }, + "res_crosswalk": { + "$ref": "#/$defs/ResCrossWalkInput", + "default": { + "path": "input/reservoir_index_AnA.nc", + "fields": { + "lake_id_field": "lake_id", + "rfc_gage_id_field": "rfc_gage_id", + "rfc_lake_id_field": "rfc_lake_id", + "usace_gage_id_field": "usace_gage_id", + "usace_lake_id_field": "usace_lake_id", + "usgs_gage_id_field": "usgs_gage_id", + "usgs_lake_id_field": "usgs_lake_id" + } + }, + "description": "Data for the gage-lake crosswalk" + }, + "great_lakes": { + "default": false, + "description": "Flag to add the Great Lakes mappings to the dataframe", + "title": "Great Lakes", + "type": "boolean" + }, + "generate_additional_crosswalk": { + "default": false, + "description": "Flag to generate lake:gage crosswalks for lakes without RFC or gage information.", + "title": "Generate Additional Crosswalk", + "type": "boolean" + }, + "active_rfc": { + "$ref": "#/$defs/ActiveRFC", + "default": { + "path": "input/nwps_all_gauges_report.csv", + "id_field": "nws shef id" + }, + "description": "Table of active NWS gages used to filter NWM reservoir index." + }, + "usace": { + "$ref": "#/$defs/USACEResDAInput", + "default": { + "path": "input/usace_crosswalk.gpkg", + "run": false, + "lake_id_field": "lake_id", + "id_field": "location" + }, + "description": "Crosswalked table of USACE reservoir/gages to lake_id." + }, + "usbr": { + "$ref": "#/$defs/USBRResDAInput", + "default": { + "path": "input/usbr_lake_crosswalk.gpkg", + "run": false, + "lake_id_field": "lake_id", + "id_field": "locId" + }, + "description": "Crosswalked table of USBR reservoir/gages to lake_id." + }, + "usgs_fix_list": { + "anyOf": [ + { + "items": { + "type": "string" + }, + "type": "array" + }, + { + "type": "null" + } + ], + "default": null, + "description": "List of USGS site_no in the reservoir index that are missing a leading 0. These values will have 0 prepended during the pipeline.", + "title": "Usgs Fix List" + } + }, + "title": "ResDAConfig", + "type": "object" +} diff --git a/docs/schemas/reservoir_da/reservoir_da_mapping.json b/docs/schemas/reservoir_da/reservoir_da_mapping.json new file mode 100644 index 0000000..5a5c5e8 --- /dev/null +++ b/docs/schemas/reservoir_da/reservoir_da_mapping.json @@ -0,0 +1,37 @@ +{ + "description": "Mapping of reservoir DA types to integer code", + "properties": { + "level_pool": { + "default": 1, + "title": "Level Pool", + "type": "integer" + }, + "usgs_persistence": { + "default": 2, + "title": "Usgs Persistence", + "type": "integer" + }, + "usace_persistence": { + "default": 3, + "title": "Usace Persistence", + "type": "integer" + }, + "usbr_persistence": { + "default": 7, + "title": "Usbr Persistence", + "type": "integer" + }, + "rfc_forecast": { + "default": 4, + "title": "Rfc Forecast", + "type": "integer" + }, + "great_lakes": { + "default": 6, + "title": "Great Lakes", + "type": "integer" + } + }, + "title": "ResDAMapping", + "type": "object" +} diff --git a/docs/schemas/reservoir_da/usace_reservoir_da_input.json b/docs/schemas/reservoir_da/usace_reservoir_da_input.json new file mode 100644 index 0000000..c1332e5 --- /dev/null +++ b/docs/schemas/reservoir_da/usace_reservoir_da_input.json @@ -0,0 +1,32 @@ +{ + "description": "USACE : lake_id crosswalk for reservoir DA.", + "properties": { + "path": { + "default": "input/usace_crosswalk.gpkg", + "description": "Source path. ResDAConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "run": { + "default": false, + "description": "Flag to use USACE reservoir input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "lake_id_field": { + "default": "lake_id", + "description": "Field containing common lake COMID", + "title": "Lake Id Field", + "type": "string" + }, + "id_field": { + "default": "location", + "description": "Field containing shared reservoir/gage ID.", + "title": "Id Field", + "type": "string" + } + }, + "title": "USACEResDAInput", + "type": "object" +} diff --git a/docs/schemas/reservoir_da/usbr_reservoir_da_input.json b/docs/schemas/reservoir_da/usbr_reservoir_da_input.json new file mode 100644 index 0000000..fe9747c --- /dev/null +++ b/docs/schemas/reservoir_da/usbr_reservoir_da_input.json @@ -0,0 +1,32 @@ +{ + "description": "USBR : lake_id crosswalk for reservoir DA.", + "properties": { + "path": { + "default": "input/usbr_lake_crosswalk.gpkg", + "description": "Source path. ResDAConfig will inject preceding input path.", + "format": "path", + "title": "Path", + "type": "string" + }, + "run": { + "default": false, + "description": "Flag to use USBR reservoir input. Must be set to false if file is not present.", + "title": "Run", + "type": "boolean" + }, + "lake_id_field": { + "default": "lake_id", + "description": "Field containing common lake COMID", + "title": "Lake Id Field", + "type": "string" + }, + "id_field": { + "default": "locId", + "description": "Field containing shared reservoir/gage ID.", + "title": "Id Field", + "type": "string" + } + }, + "title": "USBRResDAInput", + "type": "object" +} From 4584dac47b1212112493befaa8ec84bc3a59d010 Mon Sep 17 00:00:00 2001 From: Quercus Hamlin Date: Mon, 31 Aug 2026 10:15:47 -0400 Subject: [PATCH 3/3] docs: validation schemas --- .../validation/divides_validation.json | 521 ++++++++++++++++++ .../validation/flowpaths_validation.json | 191 +++++++ docs/schemas/validation/lat_lon_limits.json | 20 + .../validation/validate_hf_config.json | 26 + src/hydrofabric_builds/schemas/hydrofabric.py | 2 +- .../schemas/irrigation_constants.py | 11 - 6 files changed, 759 insertions(+), 12 deletions(-) create mode 100644 docs/schemas/validation/divides_validation.json create mode 100644 docs/schemas/validation/flowpaths_validation.json create mode 100644 docs/schemas/validation/lat_lon_limits.json create mode 100644 docs/schemas/validation/validate_hf_config.json delete mode 100644 src/hydrofabric_builds/schemas/irrigation_constants.py diff --git a/docs/schemas/validation/divides_validation.json b/docs/schemas/validation/divides_validation.json new file mode 100644 index 0000000..b7fea1f --- /dev/null +++ b/docs/schemas/validation/divides_validation.json @@ -0,0 +1,521 @@ +{ + "description": "Pydantic class containing the data type, range (if known) of the divide attributes", + "properties": { + "domain": { + "description": "hydrofabric domain", + "title": "Domain", + "type": "string" + }, + "div_id": { + "description": "unique divide identifier", + "exclusiveMinimum": 0, + "title": "Div Id", + "type": "integer" + }, + "vpu_id": { + "description": "Vector Processing Unit Identifier", + "title": "Vpu Id", + "type": "string" + }, + "type": { + "description": "Divide Type (one of independent, aggregate, connectors)", + "title": "Type", + "type": "string" + }, + "area_sqkm": { + "description": "catchment area in square kilometers", + "exclusiveMinimum": 0.0, + "title": "Area Sqkm", + "type": "number" + }, + "bexp_mode": { + "description": "beta exponent on Clapp-Hornberger (1978) soil water relationship", + "exclusiveMaximum": 15.0, + "exclusiveMinimum": 2.0, + "title": "Bexp Mode", + "type": "number" + }, + "isltyp_mode": { + "description": "dominant soil type catagory", + "maximum": 16, + "minimum": 1, + "title": "Isltyp Mode", + "type": "number" + }, + "ivgtyp_mode": { + "description": "domainant vegetation type category", + "maximum": 16, + "minimum": 1, + "title": "Ivgtyp Mode", + "type": "number" + }, + "dksat_geomean": { + "description": "saturated hydraulic conductivity", + "exclusiveMaximum": 0.00141, + "exclusiveMinimum": 1.95e-07, + "title": "Dksat Geomean", + "type": "number" + }, + "psisat_geomean": { + "description": "saturated capillary head", + "exclusiveMaximum": 0.955, + "exclusiveMinimum": 0.036, + "title": "Psisat Geomean", + "type": "number" + }, + "cwpvt_mean": { + "description": "empirical wind canopy parameter", + "exclusiveMaximum": 0.36, + "exclusiveMinimum": 0.09, + "title": "Cwpvt Mean", + "type": "number" + }, + "mp_mean": { + "description": "slope of conductance to photosynthesis relationship", + "exclusiveMaximum": 12.6, + "exclusiveMinimum": 3.6, + "title": "Mp Mean", + "type": "number" + }, + "mfsno_mean": { + "description": "melt factor for snow depletion curve", + "exclusiveMaximum": 4.0, + "exclusiveMinimum": 0.5, + "title": "Mfsno Mean", + "type": "number" + }, + "quartz_mean": { + "description": "mean soil quartz content", + "exclusiveMaximum": 1.0, + "exclusiveMinimum": 0.0, + "title": "Quartz Mean", + "type": "number" + }, + "refkdt_mean": { + "description": "reference soil infiltration parameter", + "exclusiveMaximum": 4.0, + "exclusiveMinimum": 0.1, + "title": "Refkdt Mean", + "type": "number" + }, + "slope1km_mean": { + "description": "Modifies the gradient of the hydraulic head at the soil bottom", + "exclusiveMaximum": 1.0, + "exclusiveMinimum": 0.0, + "title": "Slope1Km Mean", + "type": "number" + }, + "smcmax_mean": { + "description": "saturated soil moisture content", + "exclusiveMaximum": 0.9, + "exclusiveMinimum": 0.16, + "title": "Smcmax Mean", + "type": "number" + }, + "smcwlt_mean": { + "description": "wilting point soil moisture content", + "exclusiveMaximum": 0.3, + "exclusiveMinimum": 0.05, + "title": "Smcwlt Mean", + "type": "number" + }, + "vcmx_mean": { + "description": "Modifies the gradient of the hydraulic head at the soil bottom", + "exclusiveMaximum": 112.0, + "exclusiveMinimum": 24.0, + "title": "Vcmx Mean", + "type": "number" + }, + "imperv_mean": { + "description": "Modifies the gradient of the hydraulic head at the soil bottom", + "exclusiveMaximum": 1.0, + "exclusiveMinimum": 0.0, + "title": "Imperv Mean", + "type": "number" + }, + "twi_q25": { + "description": "Topographic wetness index 1st quartile", + "title": "Twi Q25", + "type": "number" + }, + "twi_q50": { + "description": "Topographic wetness index 2nd quartile", + "title": "Twi Q50", + "type": "number" + }, + "twi_q75": { + "description": "Topographic wetness index 3rd quartile", + "title": "Twi Q75", + "type": "number" + }, + "twi_q100": { + "description": "Topographic wetness index 4th quartile", + "title": "Twi Q100", + "type": "number" + }, + "twi_q10": { + "description": "Topographic wetness index 10th percentile", + "title": "Twi Q10", + "type": "number" + }, + "twi_q20": { + "description": "Topographic wetness index 20th percentile", + "title": "Twi Q20", + "type": "number" + }, + "twi_q30": { + "description": "Topographic wetness index 30th percentile", + "title": "Twi Q30", + "type": "number" + }, + "twi_q40": { + "description": "Topographic wetness index 40th percentile", + "title": "Twi Q40", + "type": "number" + }, + "twi_q60": { + "description": "Topographic wetness index 60th percentile", + "title": "Twi Q60", + "type": "number" + }, + "twi_q70": { + "description": "Topographic wetness index 70th percentile", + "title": "Twi Q70", + "type": "number" + }, + "twi_q80": { + "description": "Topographic wetness index 80th percentile", + "title": "Twi Q80", + "type": "number" + }, + "twi_q90": { + "description": "Topographic wetness index 10th percentile", + "title": "Twi Q90", + "type": "number" + }, + "elevation_mean": { + "description": "terrain elevation", + "exclusiveMaximum": 4422.0, + "exclusiveMinimum": -86.0, + "title": "Elevation Mean", + "type": "number" + }, + "slope250m_mean": { + "description": "terrain slope", + "exclusiveMaximum": 90.0, + "exclusiveMinimum": 0.0, + "title": "Slope250M Mean", + "type": "number" + }, + "aspect_circmean": { + "description": "terrain aspect", + "exclusiveMaximum": 360.0, + "exclusiveMinimum": 0.0, + "title": "Aspect Circmean", + "type": "number" + }, + "lzfpm_mean": { + "description": "Maximum lower zone free water mean (primary)", + "exclusiveMaximum": 600.0, + "exclusiveMinimum": 40.0, + "title": "Lzfpm Mean", + "type": "number" + }, + "lzpk_mean": { + "description": "Lower zone recession coefficient mean (primary)", + "exclusiveMaximum": 0.015, + "exclusiveMinimum": 0.001, + "title": "Lzpk Mean", + "type": "number" + }, + "lztwm_mean": { + "description": "Maximum lower zone tension water mean", + "exclusiveMaximum": 300.0, + "exclusiveMinimum": 75.0, + "title": "Lztwm Mean", + "type": "number" + }, + "rexp_mean": { + "description": "Percolation equation exponent mean", + "exclusiveMaximum": 3.5, + "exclusiveMinimum": 1.4, + "title": "Rexp Mean", + "type": "number" + }, + "uzk_mean": { + "description": "Upper zone recession coefficient mean", + "exclusiveMaximum": 0.5, + "exclusiveMinimum": 0.2, + "title": "Uzk Mean", + "type": "number" + }, + "zperc_mean": { + "description": "Minimum percolation rate coefficient mean", + "exclusiveMaximum": 360.0, + "exclusiveMinimum": 0.0, + "title": "Zperc Mean", + "type": "number" + }, + "lzfsm_mean": { + "description": "Maximum lower zone free water mean (secondary aka supplemental)", + "exclusiveMaximum": 360.0, + "exclusiveMinimum": 0.0, + "title": "Lzfsm Mean", + "type": "number" + }, + "lzsk_mean": { + "description": "Lower zone recession coefficient mean, (secondary aka supplemental)", + "exclusiveMaximum": 0.2, + "exclusiveMinimum": 0.03, + "title": "Lzsk Mean", + "type": "number" + }, + "pfree_mean": { + "description": "Fraction of water percolating from upper zone directly to lower zone free water storage (mean)", + "exclusiveMaximum": 0.5, + "exclusiveMinimum": 0.0, + "title": "Pfree Mean", + "type": "number" + }, + "uzfwm_mean": { + "description": "Maximum upper zone free water mean", + "exclusiveMaximum": 100.0, + "exclusiveMinimum": 10.0, + "title": "Uzfwm Mean", + "type": "number" + }, + "uztwm_mean": { + "description": "Upper zone tension water maximum storage mean", + "exclusiveMaximum": 125.0, + "exclusiveMinimum": 25.0, + "title": "Uztwm Mean", + "type": "number" + }, + "mfmin_mean": { + "description": "Minimum non-rain melt factor mean", + "exclusiveMaximum": 0.6, + "exclusiveMinimum": 0.01, + "title": "Mfmin Mean", + "type": "number" + }, + "mfmax_mean": { + "description": "Maximum non-rain melt factor mean", + "exclusiveMaximum": 360.0, + "exclusiveMinimum": 0.0, + "title": "Mfmax Mean", + "type": "number" + }, + "uadj_mean": { + "description": "Average wind function for rain on snow (mean)", + "exclusiveMaximum": 0.2, + "exclusiveMinimum": 0.01, + "title": "Uadj Mean", + "type": "number" + }, + "a_xinanjiang_inflection_point_parameter": { + "description": "Inflection point parameter for the Xinanjiang runoff generation model configuration", + "exclusiveMaximum": 0.5, + "exclusiveMinimum": -0.5, + "title": "A Xinanjiang Inflection Point Parameter", + "type": "number" + }, + "b_xinanjiang_shape_parameter": { + "description": "Inflection point parameter for the Xinanjiang runoff generation model configuration", + "exclusiveMaximum": 10.0, + "exclusiveMinimum": 0.01, + "title": "B Xinanjiang Shape Parameter", + "type": "number" + }, + "x_xinanjiang_shape_parameter": { + "description": "Main, exponential shape parameter for the Xinanjiang runoff generation model configuration", + "exclusiveMaximum": 10.0, + "exclusiveMinimum": 0.01, + "title": "X Xinanjiang Shape Parameter", + "type": "number" + }, + "temp_delta_jan_mean": { + "description": "mean temperature difference for January", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Jan Mean", + "type": "number" + }, + "temp_delta_feb_mean": { + "description": "mean temperature difference for February", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Feb Mean", + "type": "number" + }, + "temp_delta_mar_mean": { + "description": "mean temperature difference for March", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Mar Mean", + "type": "number" + }, + "temp_delta_apr_mean": { + "description": "mean temperature difference for April", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Apr Mean", + "type": "number" + }, + "temp_delta_may_mean": { + "description": "mean temperature difference for May", + "exclusiveMinimum": 0.0, + "title": "Temp Delta May Mean", + "type": "number" + }, + "temp_delta_jun_mean": { + "description": "mean temperature difference for June", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Jun Mean", + "type": "number" + }, + "temp_delta_jul_mean": { + "description": "mean temperature difference for July", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Jul Mean", + "type": "number" + }, + "temp_delta_aug_mean": { + "description": "mean temperature difference for August", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Aug Mean", + "type": "number" + }, + "temp_delta_sep_mean": { + "description": "mean temperature difference for September", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Sep Mean", + "type": "number" + }, + "temp_delta_oct_mean": { + "description": "mean temperature difference for October", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Oct Mean", + "type": "number" + }, + "temp_delta_nov_mean": { + "description": "mean temperature difference for November", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Nov Mean", + "type": "number" + }, + "temp_delta_dec_mean": { + "description": "mean temperature difference for December", + "exclusiveMinimum": 0.0, + "title": "Temp Delta Dec Mean", + "type": "number" + }, + "lat": { + "description": "latitude of divide centroid", + "title": "Lat", + "type": "number" + }, + "lon": { + "description": "longitude of divide centroid", + "title": "Lon", + "type": "number" + }, + "glacier_percent": { + "description": "percentage of glacier cover in the divide", + "maximum": 1, + "minimum": 0, + "title": "Glacier Percent", + "type": "number" + }, + "cgw": { + "description": "groundwater coefficient", + "exclusiveMaximum": 0.0018, + "exclusiveMinimum": 1.8e-06, + "title": "Cgw", + "type": "number" + }, + "expon": { + "description": "groundwater exponent", + "exclusiveMaximum": 8.0, + "exclusiveMinimum": 1.0, + "title": "Expon", + "type": "number" + }, + "max_gw_storage": { + "description": "The total height of the baseflow bucket", + "exclusiveMaximum": 0.25, + "exclusiveMinimum": 0.01, + "title": "Max Gw Storage", + "type": "number" + } + }, + "required": [ + "domain", + "div_id", + "vpu_id", + "type", + "area_sqkm", + "bexp_mode", + "isltyp_mode", + "ivgtyp_mode", + "dksat_geomean", + "psisat_geomean", + "cwpvt_mean", + "mp_mean", + "mfsno_mean", + "quartz_mean", + "refkdt_mean", + "slope1km_mean", + "smcmax_mean", + "smcwlt_mean", + "vcmx_mean", + "imperv_mean", + "twi_q25", + "twi_q50", + "twi_q75", + "twi_q100", + "twi_q10", + "twi_q20", + "twi_q30", + "twi_q40", + "twi_q60", + "twi_q70", + "twi_q80", + "twi_q90", + "elevation_mean", + "slope250m_mean", + "aspect_circmean", + "lzfpm_mean", + "lzpk_mean", + "lztwm_mean", + "rexp_mean", + "uzk_mean", + "zperc_mean", + "lzfsm_mean", + "lzsk_mean", + "pfree_mean", + "uzfwm_mean", + "uztwm_mean", + "mfmin_mean", + "mfmax_mean", + "uadj_mean", + "a_xinanjiang_inflection_point_parameter", + "b_xinanjiang_shape_parameter", + "x_xinanjiang_shape_parameter", + "temp_delta_jan_mean", + "temp_delta_feb_mean", + "temp_delta_mar_mean", + "temp_delta_apr_mean", + "temp_delta_may_mean", + "temp_delta_jun_mean", + "temp_delta_jul_mean", + "temp_delta_aug_mean", + "temp_delta_sep_mean", + "temp_delta_oct_mean", + "temp_delta_nov_mean", + "temp_delta_dec_mean", + "lat", + "lon", + "glacier_percent", + "cgw", + "expon", + "max_gw_storage" + ], + "title": "Divides", + "type": "object" +} diff --git a/docs/schemas/validation/flowpaths_validation.json b/docs/schemas/validation/flowpaths_validation.json new file mode 100644 index 0000000..a3f8247 --- /dev/null +++ b/docs/schemas/validation/flowpaths_validation.json @@ -0,0 +1,191 @@ +{ + "description": "Pydantic class containing the data type, range (if known) of the flowpath attributes", + "properties": { + "fp_id": { + "description": "unique flowpath identifier", + "title": "Fp Id", + "type": "integer" + }, + "dn_nex_id": { + "description": "connected downstream nexus identifier", + "title": "Dn Nex Id", + "type": "integer" + }, + "up_nex_id": { + "description": "connected upstream nexus identifier", + "title": "Up Nex Id", + "type": "integer" + }, + "div_id": { + "description": "unique divide identifier", + "title": "Div Id", + "type": "integer" + }, + "vpu_id": { + "description": "Vector Processing Unit Identifier", + "title": "Vpu Id", + "type": "string" + }, + "length_km": { + "description": "flowpath length in kilometers", + "exclusiveMinimum": 0.0, + "title": "Length Km", + "type": "number" + }, + "area_sqkm": { + "description": "incremental area of divide in kilometers", + "exclusiveMinimum": 0.0, + "title": "Area Sqkm", + "type": "number" + }, + "total_da_sqkm": { + "description": "total upstream drainage area in kilometers", + "exclusiveMinimum": 0.0, + "title": "Total Da Sqkm", + "type": "number" + }, + "mainstem_lp": { + "description": "associated upstream drainage area in square kilometers", + "title": "Mainstem Lp", + "type": "integer" + }, + "path_length": { + "description": "downstream path length", + "exclusiveMinimum": 0.0, + "title": "Path Length", + "type": "number" + }, + "dn_hydroseq": { + "description": "downstream hydrologic sequence", + "title": "Dn Hydroseq", + "type": "integer" + }, + "hydroseq": { + "description": "hydrologic sequence", + "title": "Hydroseq", + "type": "integer" + }, + "stream_order": { + "description": "stream order of mapped reference flowpath", + "title": "Stream Order", + "type": "integer" + }, + "mean_elevation": { + "description": "terrain elevation", + "exclusiveMaximum": 4422.0, + "exclusiveMinimum": -86.0, + "title": "Mean Elevation", + "type": "number" + }, + "slope": { + "description": "terrain slope", + "exclusiveMaximum": 90.0, + "exclusiveMinimum": 0.0, + "title": "Slope", + "type": "number" + }, + "n": { + "description": "Manning's in channel roughness", + "title": "N", + "type": "number" + }, + "r": { + "description": "hydrologic radius", + "title": "R", + "type": "number" + }, + "y": { + "description": "estimated depth associated with top width", + "title": "Y", + "type": "number" + }, + "ncc": { + "description": "compound channel top width", + "title": "Ncc", + "type": "number" + }, + "btmwdth": { + "description": "bottom width of channel", + "title": "Btmwdth", + "type": "number" + }, + "chslp": { + "description": "channel side slope", + "title": "Chslp", + "type": "number" + }, + "musx": { + "description": "Muskingum weighting factor", + "title": "Musx", + "type": "number" + }, + "musk": { + "description": "Muskingum routing time", + "title": "Musk", + "type": "integer" + }, + "topwdth": { + "description": "top width", + "title": "Topwdth", + "type": "number" + }, + "topwdthcc": { + "description": "compound channel top width", + "title": "Topwdthcc", + "type": "number" + }, + "topwdthcc_ml": { + "description": "compound channel top width at maximum levee", + "title": "Topwdthcc Ml", + "type": "number" + }, + "topwdth_ml": { + "description": "top width at maximum levee", + "title": "Topwdth Ml", + "type": "number" + }, + "y_ml": { + "description": "estimated depth associated with top wideth at maximum levee", + "title": "Y Ml", + "type": "number" + }, + "r_ml": { + "description": "hydraulic radius at maximum levee", + "title": "R Ml", + "type": "number" + } + }, + "required": [ + "fp_id", + "dn_nex_id", + "up_nex_id", + "div_id", + "vpu_id", + "length_km", + "area_sqkm", + "total_da_sqkm", + "mainstem_lp", + "path_length", + "dn_hydroseq", + "hydroseq", + "stream_order", + "mean_elevation", + "slope", + "n", + "r", + "y", + "ncc", + "btmwdth", + "chslp", + "musx", + "musk", + "topwdth", + "topwdthcc", + "topwdthcc_ml", + "topwdth_ml", + "y_ml", + "r_ml" + ], + "title": "Flowpaths", + "type": "object" +} diff --git a/docs/schemas/validation/lat_lon_limits.json b/docs/schemas/validation/lat_lon_limits.json new file mode 100644 index 0000000..38dbd1e --- /dev/null +++ b/docs/schemas/validation/lat_lon_limits.json @@ -0,0 +1,20 @@ +{ + "class_name": "LatLonLimits", + "values": { + "LAT_MIN_CONUS": 24, + "LAT_MAX_CONUS": 55, + "LON_MIN_CONUS": -125, + "LON_MAX_CONUS": -66, + "LAT_MIN_AK": 51, + "LAT_MAX_AK": 72, + "LON_MIN_AK": -172, + "LAT_MIN_HI": 18, + "LAT_MAX_HI": 23, + "LON_MIN_HI": -161, + "LON_MAX_HI": -155, + "LAT_MIN_PRVI": 17, + "LAT_MAX_PRVI": 19, + "LON_MIN_PRVI": -68, + "LON_MAX_PRVI": -64 + } +} diff --git a/docs/schemas/validation/validate_hf_config.json b/docs/schemas/validation/validate_hf_config.json new file mode 100644 index 0000000..3063fcd --- /dev/null +++ b/docs/schemas/validation/validate_hf_config.json @@ -0,0 +1,26 @@ +{ + "description": "config class for the hf_validate block in the YAML", + "properties": { + "calibration_gages_path": { + "default": "data/gages/validation/calibratable_gages.csv", + "format": "path", + "title": "Calibration Gages Path", + "type": "string" + }, + "routelink_gages_path": { + "anyOf": [ + { + "format": "path", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Routelink Gages Path" + } + }, + "title": "ValidateHFConfig", + "type": "object" +} diff --git a/src/hydrofabric_builds/schemas/hydrofabric.py b/src/hydrofabric_builds/schemas/hydrofabric.py index b1977ca..4e65985 100644 --- a/src/hydrofabric_builds/schemas/hydrofabric.py +++ b/src/hydrofabric_builds/schemas/hydrofabric.py @@ -1267,7 +1267,7 @@ class USACEResDAInput(BaseModel): class USBRResDAInput(BaseModel): - """USACE : lake_id crosswalk for reservoir DA.""" + """USBR : lake_id crosswalk for reservoir DA.""" path: Path = Field( default=Path("input/usbr_lake_crosswalk.gpkg"), diff --git a/src/hydrofabric_builds/schemas/irrigation_constants.py b/src/hydrofabric_builds/schemas/irrigation_constants.py deleted file mode 100644 index 9b7187b..0000000 --- a/src/hydrofabric_builds/schemas/irrigation_constants.py +++ /dev/null @@ -1,11 +0,0 @@ -CROPS = [ - 1, # corn, - 3, # rice - 5, # soybeans - 12, # sweet corn - 13, # pop or orn corn - 92, # aquaculture - 250, # cranberry -] - -CDL_URL_BASE = "https://www.nass.usda.gov/Research_and_Science/Cropland/Release/datasets/"