diff --git a/poetry.lock b/poetry.lock index dc299d2..d96d3c0 100644 --- a/poetry.lock +++ b/poetry.lock @@ -740,6 +740,54 @@ docs = ["furo (>=2024.8.6)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3) testing = ["covdefaults (>=2.3)", "coverage (>=7.6.10)", "diff-cover (>=9.2.1)", "pytest (>=8.3.4)", "pytest-asyncio (>=0.25.2)", "pytest-cov (>=6)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.28.1)"] typing = ["typing-extensions (>=4.12.2) ; python_version < \"3.11\""] +[[package]] +name = "fiona" +version = "1.10.1" +description = "Fiona reads and writes spatial data files" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "fiona-1.10.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:6e2a94beebda24e5db8c3573fe36110d474d4a12fac0264a3e083c75e9d63829"}, + {file = "fiona-1.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fc7366f99bdc18ec99441b9e50246fdf5e72923dc9cbb00267b2bf28edd142ba"}, + {file = "fiona-1.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8c32f424b0641c79f4036b96c2e80322fb181b4e415c8cd02d182baef55e6730"}, + {file = "fiona-1.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:9a67bd88918e87d64168bc9c00d9816d8bb07353594b5ce6c57252979d5dc86e"}, + {file = "fiona-1.10.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:98fe556058b370da07a84f6537c286f87eb4af2343d155fbd3fba5d38ac17ed7"}, + {file = "fiona-1.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:be29044d4aeebae92944b738160dc5f9afc4cdf04f551d59e803c5b910e17520"}, + {file = "fiona-1.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94bd3d448f09f85439e4b77c38b9de1aebe3eef24acc72bd631f75171cdfde51"}, + {file = "fiona-1.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:30594c0cd8682c43fd01e7cdbe000f94540f8fa3b7cb5901e805c88c4ff2058b"}, + {file = "fiona-1.10.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:7338b8c68beb7934bde4ec9f49eb5044e5e484b92d940bc3ec27defdb2b06c67"}, + {file = "fiona-1.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8c77fcfd3cdb0d3c97237965f8c60d1696a64923deeeb2d0b9810286cbe25911"}, + {file = "fiona-1.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:537872cbc9bda7fcdf73851c91bc5338fca2b502c4c17049ccecaa13cde1f18f"}, + {file = "fiona-1.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:41cde2c52c614457e9094ea44b0d30483540789e62fe0fa758c2a2963e980817"}, + {file = "fiona-1.10.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:a00b05935c9900678b2ca660026b39efc4e4b916983915d595964eb381763ae7"}, + {file = "fiona-1.10.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f78b781d5bcbbeeddf1d52712f33458775dbb9fd1b2a39882c83618348dd730f"}, + {file = "fiona-1.10.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:29ceeb38e3cd30d91d68858d0817a1bb0c4f96340d334db4b16a99edb0902d35"}, + {file = "fiona-1.10.1-cp313-cp313-win_amd64.whl", hash = "sha256:15751c90e29cee1e01fcfedf42ab85987e32f0b593cf98d88ed52199ef5ca623"}, + {file = "fiona-1.10.1-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:6f1242f872dc33d3b4269dcaebf1838a359f9097e1cc848b0e11367bce010e4d"}, + {file = "fiona-1.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:65308b7a7e57fcc533de8a5855b0fce798faabc736d1340192dd8673ff61bc4e"}, + {file = "fiona-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:632bc146355af5ff0d77e34ebd1be5072d623b4aedb754b94a3d8c356c4545ac"}, + {file = "fiona-1.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:b7b4c3c97b1d64a1b3321577e9edaebbd36b64006e278f225f300c497cc87c35"}, + {file = "fiona-1.10.1-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:b62aa8d5a0981bd33d81c247219b1eaa1e655e0a0682b3a4759fccc40954bb30"}, + {file = "fiona-1.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f4b19cb5bd22443ef439b39239272349023556994242a8f953a0147684e1c47f"}, + {file = "fiona-1.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa7e7e5ad252ef29905384bf92e7d14dd5374584b525632652c2ab8925304670"}, + {file = "fiona-1.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:4e82d18acbe55230e9cf8ede2a836d99ea96b7c0cc7d2b8b993e6c9f0ac14dc2"}, + {file = "fiona-1.10.1.tar.gz", hash = "sha256:b00ae357669460c6491caba29c2022ff0acfcbde86a95361ea8ff5cd14a86b68"}, +] + +[package.dependencies] +attrs = ">=19.2.0" +certifi = "*" +click = ">=8.0,<9.0" +click-plugins = ">=1.0" +cligj = ">=0.5" + +[package.extras] +all = ["fiona[calc,s3,test]"] +calc = ["pyparsing", "shapely"] +s3 = ["boto3 (>=1.3.1)"] +test = ["aiohttp", "fiona[s3]", "fsspec", "pytest (>=7)", "pytest-cov", "pytz"] + [[package]] name = "fonttools" version = "4.56.0" @@ -1162,6 +1210,75 @@ files = [ {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, ] +[[package]] +name = "jax" +version = "0.8.0" +description = "Differentiate, compile, and transform Numpy code." +optional = false +python-versions = ">=3.11" +groups = ["main"] +files = [ + {file = "jax-0.8.0-py3-none-any.whl", hash = "sha256:d190158bc019756c6a0f6b3d5fc8783471fb407e6deaff559eaac60dd5ee850a"}, + {file = "jax-0.8.0.tar.gz", hash = "sha256:0ea5a7be7068c25934450dfd87d7d80a18a5d30e0a53454e7aade525b23accd5"}, +] + +[package.dependencies] +jaxlib = "0.8.0" +ml_dtypes = ">=0.5.0" +numpy = ">=2.0" +opt_einsum = "*" +scipy = ">=1.13" + +[package.extras] +ci = ["jaxlib (==0.7.2)"] +cuda = ["jax-cuda12-plugin[with-cuda] (==0.8.0)", "jaxlib (==0.8.0)"] +cuda12 = ["jax-cuda12-plugin[with-cuda] (==0.8.0)", "jaxlib (==0.8.0)"] +cuda12-local = ["jax-cuda12-plugin (==0.8.0)", "jaxlib (==0.8.0)"] +cuda13 = ["jax-cuda13-plugin[with-cuda] (==0.8.0)", "jaxlib (==0.8.0)"] +cuda13-local = ["jax-cuda13-plugin (==0.8.0)", "jaxlib (==0.8.0)"] +k8s = ["kubernetes"] +minimum-jaxlib = ["jaxlib (==0.8.0)"] +rocm = ["jax-rocm60-plugin (==0.8.0)", "jaxlib (==0.8.0)"] +tpu = ["jaxlib (==0.8.0)", "libtpu (==0.0.24.*)", "requests"] +xprof = ["xprof"] + +[[package]] +name = "jaxlib" +version = "0.8.0" +description = "XLA library for JAX" +optional = false +python-versions = ">=3.11" +groups = ["main"] +files = [ + {file = "jaxlib-0.8.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bb602a8c24c614cb8ca6eeed3e70a733d9399c6a2f88900a0252623cd67276b5"}, + {file = "jaxlib-0.8.0-cp311-cp311-manylinux_2_27_aarch64.whl", hash = "sha256:41aebddef67a555a6de17427a4e66ce60a528a815847e2dd96dabce579f7acf8"}, + {file = "jaxlib-0.8.0-cp311-cp311-manylinux_2_27_x86_64.whl", hash = "sha256:ff53e8baf978f6b7c4076215af78f0ba969cac434ed2f72565d87e38c23f00e7"}, + {file = "jaxlib-0.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:9cd4c7a8acc5b3dee4ad28a5d101264d89754e29553b0cdb92c79f5b460a511b"}, + {file = "jaxlib-0.8.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f60aac0f64e9e70a5cef341fe292684518695514c71ad00036774bbed5f7312e"}, + {file = "jaxlib-0.8.0-cp312-cp312-manylinux_2_27_aarch64.whl", hash = "sha256:d83ff8cf1b070299639cda4f8427707f69051dc8421e59fbb73305523937570d"}, + {file = "jaxlib-0.8.0-cp312-cp312-manylinux_2_27_x86_64.whl", hash = "sha256:2c8675bf86e391afe4f8d863080be1a024d734dfd3dd137f7aa8e7f22091adcd"}, + {file = "jaxlib-0.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:659d894d93876e3675c2132d13c3d241f204b21172a58f928b96f654f603f6dc"}, + {file = "jaxlib-0.8.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:5fcf33a5639f8f164a473a9c78a1fa0b2e15ac3fcbecd6d96aa0f88bf25ea6bb"}, + {file = "jaxlib-0.8.0-cp313-cp313-manylinux_2_27_aarch64.whl", hash = "sha256:b3eac503b90ffecc68f11fa122133eef2c62c536db28e801e436d7e7a9b67bf8"}, + {file = "jaxlib-0.8.0-cp313-cp313-manylinux_2_27_x86_64.whl", hash = "sha256:66c6f576f54a63ed052f5c469bef4db723f5f050b839ec0c429573011341bd58"}, + {file = "jaxlib-0.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:72759ebbfb40a717349f174712207d306aa28630359f05cd69b091bd4efa0603"}, + {file = "jaxlib-0.8.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:df2781e0fc93fb6f42111b385b90126b9571eafe0e860f033615ff7156b76817"}, + {file = "jaxlib-0.8.0-cp313-cp313t-manylinux_2_27_aarch64.whl", hash = "sha256:7eb3be931de77bfcde27df659ada432719aa1e19a2fa5b835638e7404c74cb63"}, + {file = "jaxlib-0.8.0-cp313-cp313t-manylinux_2_27_x86_64.whl", hash = "sha256:accebe89a36e28306a4db3f68f527a0f87b8a0fd253b3c1556fbd24f16bec22c"}, + {file = "jaxlib-0.8.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ba7e8a2231e4138ccbd8e096debdbbcd82edc5fc1b13c66f32a51bc240651349"}, + {file = "jaxlib-0.8.0-cp314-cp314-manylinux_2_27_aarch64.whl", hash = "sha256:a9bfca27ae597804db08694a2bf7e1cf7fc3fac4ac2e65ace83be8effaa927ea"}, + {file = "jaxlib-0.8.0-cp314-cp314-manylinux_2_27_x86_64.whl", hash = "sha256:bd3219a4d2bfe4b72605900fde395b62126a053c0b99643eb931b7c20e577bf2"}, + {file = "jaxlib-0.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:3320a72d532713c2a31eb20d02c342540a0dec28603a3ac2be0fc0631f086cf2"}, + {file = "jaxlib-0.8.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:248f1ac3acee1fe2cc81e8a668311f3ccb8f28090404391c276869cae8a95daf"}, + {file = "jaxlib-0.8.0-cp314-cp314t-manylinux_2_27_aarch64.whl", hash = "sha256:a5f0656bbbb3f135a360ce0fde55bf34faf73fbc62ab887941e85f0014b3f476"}, + {file = "jaxlib-0.8.0-cp314-cp314t-manylinux_2_27_x86_64.whl", hash = "sha256:61cb2fde154e5a399db2880d560e3443cfa97bda9f074b545c886232ac8fe024"}, +] + +[package.dependencies] +ml_dtypes = ">=0.5.0" +numpy = ">=2.0" +scipy = ">=1.13" + [[package]] name = "jinja2" version = "3.1.5" @@ -1414,6 +1531,162 @@ win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==v0.910) ; python_version < \"3.6\"", "mypy (==v0.971) ; python_version == \"3.6\"", "mypy (==v1.13.0) ; python_version >= \"3.8\"", "mypy (==v1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""] +[[package]] +name = "lxml" +version = "6.0.2" +description = "Powerful and Pythonic XML processing library combining libxml2/libxslt with the ElementTree API." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e77dd455b9a16bbd2a5036a63ddbd479c19572af81b624e79ef422f929eef388"}, + {file = "lxml-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5d444858b9f07cefff6455b983aea9a67f7462ba1f6cbe4a21e8bf6791bf2153"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f952dacaa552f3bb8834908dddd500ba7d508e6ea6eb8c52eb2d28f48ca06a31"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:71695772df6acea9f3c0e59e44ba8ac50c4f125217e84aab21074a1a55e7e5c9"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:17f68764f35fd78d7c4cc4ef209a184c38b65440378013d24b8aecd327c3e0c8"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:058027e261afed589eddcfe530fcc6f3402d7fd7e89bfd0532df82ebc1563dba"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a8ffaeec5dfea5881d4c9d8913a32d10cfe3923495386106e4a24d45300ef79c"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_31_armv7l.whl", hash = "sha256:f2e3b1a6bb38de0bc713edd4d612969dd250ca8b724be8d460001a387507021c"}, + {file = "lxml-6.0.2-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:d6690ec5ec1cce0385cb20896b16be35247ac8c2046e493d03232f1c2414d321"}, + {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f2a50c3c1d11cad0ebebbac357a97b26aa79d2bcaf46f256551152aa85d3a4d1"}, + {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:3efe1b21c7801ffa29a1112fab3b0f643628c30472d507f39544fd48e9549e34"}, + {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:59c45e125140b2c4b33920d21d83681940ca29f0b83f8629ea1a2196dc8cfe6a"}, + {file = "lxml-6.0.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:452b899faa64f1805943ec1c0c9ebeaece01a1af83e130b69cdefeda180bb42c"}, + {file = "lxml-6.0.2-cp310-cp310-win32.whl", hash = "sha256:1e786a464c191ca43b133906c6903a7e4d56bef376b75d97ccbb8ec5cf1f0a4b"}, + {file = "lxml-6.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:dacf3c64ef3f7440e3167aa4b49aa9e0fb99e0aa4f9ff03795640bf94531bcb0"}, + {file = "lxml-6.0.2-cp310-cp310-win_arm64.whl", hash = "sha256:45f93e6f75123f88d7f0cfd90f2d05f441b808562bf0bc01070a00f53f5028b5"}, + {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:13e35cbc684aadf05d8711a5d1b5857c92e5e580efa9a0d2be197199c8def607"}, + {file = "lxml-6.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3b1675e096e17c6fe9c0e8c81434f5736c0739ff9ac6123c87c2d452f48fc938"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8ac6e5811ae2870953390452e3476694196f98d447573234592d30488147404d"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5aa0fc67ae19d7a64c3fe725dc9a1bb11f80e01f78289d05c6f62545affec438"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de496365750cc472b4e7902a485d3f152ecf57bd3ba03ddd5578ed8ceb4c5964"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:200069a593c5e40b8f6fc0d84d86d970ba43138c3e68619ffa234bc9bb806a4d"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7d2de809c2ee3b888b59f995625385f74629707c9355e0ff856445cdcae682b7"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_31_armv7l.whl", hash = "sha256:b2c3da8d93cf5db60e8858c17684c47d01fee6405e554fb55018dd85fc23b178"}, + {file = "lxml-6.0.2-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:442de7530296ef5e188373a1ea5789a46ce90c4847e597856570439621d9c553"}, + {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:2593c77efde7bfea7f6389f1ab249b15ed4aa5bc5cb5131faa3b843c429fbedb"}, + {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:3e3cb08855967a20f553ff32d147e14329b3ae70ced6edc2f282b94afbc74b2a"}, + {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:2ed6c667fcbb8c19c6791bbf40b7268ef8ddf5a96940ba9404b9f9a304832f6c"}, + {file = "lxml-6.0.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b8f18914faec94132e5b91e69d76a5c1d7b0c73e2489ea8929c4aaa10b76bbf7"}, + {file = "lxml-6.0.2-cp311-cp311-win32.whl", hash = "sha256:6605c604e6daa9e0d7f0a2137bdc47a2e93b59c60a65466353e37f8272f47c46"}, + {file = "lxml-6.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:e5867f2651016a3afd8dd2c8238baa66f1e2802f44bc17e236f547ace6647078"}, + {file = "lxml-6.0.2-cp311-cp311-win_arm64.whl", hash = "sha256:4197fb2534ee05fd3e7afaab5d8bfd6c2e186f65ea7f9cd6a82809c887bd1285"}, + {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:a59f5448ba2ceccd06995c95ea59a7674a10de0810f2ce90c9006f3cbc044456"}, + {file = "lxml-6.0.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e8113639f3296706fbac34a30813929e29247718e88173ad849f57ca59754924"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a8bef9b9825fa8bc816a6e641bb67219489229ebc648be422af695f6e7a4fa7f"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:65ea18d710fd14e0186c2f973dc60bb52039a275f82d3c44a0e42b43440ea534"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c371aa98126a0d4c739ca93ceffa0fd7a5d732e3ac66a46e74339acd4d334564"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:700efd30c0fa1a3581d80a748157397559396090a51d306ea59a70020223d16f"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c33e66d44fe60e72397b487ee92e01da0d09ba2d66df8eae42d77b6d06e5eba0"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:90a345bbeaf9d0587a3aaffb7006aa39ccb6ff0e96a57286c0cb2fd1520ea192"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_31_armv7l.whl", hash = "sha256:064fdadaf7a21af3ed1dcaa106b854077fbeada827c18f72aec9346847cd65d0"}, + {file = "lxml-6.0.2-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:fbc74f42c3525ac4ffa4b89cbdd00057b6196bcefe8bce794abd42d33a018092"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6ddff43f702905a4e32bc24f3f2e2edfe0f8fde3277d481bffb709a4cced7a1f"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:6da5185951d72e6f5352166e3da7b0dc27aa70bd1090b0eb3f7f7212b53f1bb8"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:57a86e1ebb4020a38d295c04fc79603c7899e0df71588043eb218722dabc087f"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2047d8234fe735ab77802ce5f2297e410ff40f5238aec569ad7c8e163d7b19a6"}, + {file = "lxml-6.0.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:6f91fd2b2ea15a6800c8e24418c0775a1694eefc011392da73bc6cef2623b322"}, + {file = "lxml-6.0.2-cp312-cp312-win32.whl", hash = "sha256:3ae2ce7d6fedfb3414a2b6c5e20b249c4c607f72cb8d2bb7cc9c6ec7c6f4e849"}, + {file = "lxml-6.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:72c87e5ee4e58a8354fb9c7c84cbf95a1c8236c127a5d1b7683f04bed8361e1f"}, + {file = "lxml-6.0.2-cp312-cp312-win_arm64.whl", hash = "sha256:61cb10eeb95570153e0c0e554f58df92ecf5109f75eacad4a95baa709e26c3d6"}, + {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9b33d21594afab46f37ae58dfadd06636f154923c4e8a4d754b0127554eb2e77"}, + {file = "lxml-6.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6c8963287d7a4c5c9a432ff487c52e9c5618667179c18a204bdedb27310f022f"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:1941354d92699fb5ffe6ed7b32f9649e43c2feb4b97205f75866f7d21aa91452"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb2f6ca0ae2d983ded09357b84af659c954722bbf04dea98030064996d156048"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:eb2a12d704f180a902d7fa778c6d71f36ceb7b0d317f34cdc76a5d05aa1dd1df"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:6ec0e3f745021bfed19c456647f0298d60a24c9ff86d9d051f52b509663feeb1"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:846ae9a12d54e368933b9759052d6206a9e8b250291109c48e350c1f1f49d916"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ef9266d2aa545d7374938fb5c484531ef5a2ec7f2d573e62f8ce722c735685fd"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_31_armv7l.whl", hash = "sha256:4077b7c79f31755df33b795dc12119cb557a0106bfdab0d2c2d97bd3cf3dffa6"}, + {file = "lxml-6.0.2-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:a7c5d5e5f1081955358533be077166ee97ed2571d6a66bdba6ec2f609a715d1a"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:8f8d0cbd0674ee89863a523e6994ac25fd5be9c8486acfc3e5ccea679bad2679"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:2cbcbf6d6e924c28f04a43f3b6f6e272312a090f269eff68a2982e13e5d57659"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:dfb874cfa53340009af6bdd7e54ebc0d21012a60a4e65d927c2e477112e63484"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:fb8dae0b6b8b7f9e96c26fdd8121522ce5de9bb5538010870bd538683d30e9a2"}, + {file = "lxml-6.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:358d9adae670b63e95bc59747c72f4dc97c9ec58881d4627fe0120da0f90d314"}, + {file = "lxml-6.0.2-cp313-cp313-win32.whl", hash = "sha256:e8cd2415f372e7e5a789d743d133ae474290a90b9023197fd78f32e2dc6873e2"}, + {file = "lxml-6.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:b30d46379644fbfc3ab81f8f82ae4de55179414651f110a1514f0b1f8f6cb2d7"}, + {file = "lxml-6.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:13dcecc9946dca97b11b7c40d29fba63b55ab4170d3c0cf8c0c164343b9bfdcf"}, + {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:b0c732aa23de8f8aec23f4b580d1e52905ef468afb4abeafd3fec77042abb6fe"}, + {file = "lxml-6.0.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4468e3b83e10e0317a89a33d28f7aeba1caa4d1a6fd457d115dd4ffe90c5931d"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:abd44571493973bad4598a3be7e1d807ed45aa2adaf7ab92ab7c62609569b17d"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:370cd78d5855cfbffd57c422851f7d3864e6ae72d0da615fca4dad8c45d375a5"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:901e3b4219fa04ef766885fb40fa516a71662a4c61b80c94d25336b4934b71c0"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:a4bf42d2e4cf52c28cc1812d62426b9503cdb0c87a6de81442626aa7d69707ba"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2c7fdaa4d7c3d886a42534adec7cfac73860b89b4e5298752f60aa5984641a0"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:98a5e1660dc7de2200b00d53fa00bcd3c35a3608c305d45a7bbcaf29fa16e83d"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_31_armv7l.whl", hash = "sha256:dc051506c30b609238d79eda75ee9cab3e520570ec8219844a72a46020901e37"}, + {file = "lxml-6.0.2-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:8799481bbdd212470d17513a54d568f44416db01250f49449647b5ab5b5dccb9"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:9261bb77c2dab42f3ecd9103951aeca2c40277701eb7e912c545c1b16e0e4917"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:65ac4a01aba353cfa6d5725b95d7aed6356ddc0a3cd734de00124d285b04b64f"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:b22a07cbb82fea98f8a2fd814f3d1811ff9ed76d0fc6abc84eb21527596e7cc8"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:d759cdd7f3e055d6bc8d9bec3ad905227b2e4c785dc16c372eb5b5e83123f48a"}, + {file = "lxml-6.0.2-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:945da35a48d193d27c188037a05fec5492937f66fb1958c24fc761fb9d40d43c"}, + {file = "lxml-6.0.2-cp314-cp314-win32.whl", hash = "sha256:be3aaa60da67e6153eb15715cc2e19091af5dc75faef8b8a585aea372507384b"}, + {file = "lxml-6.0.2-cp314-cp314-win_amd64.whl", hash = "sha256:fa25afbadead523f7001caf0c2382afd272c315a033a7b06336da2637d92d6ed"}, + {file = "lxml-6.0.2-cp314-cp314-win_arm64.whl", hash = "sha256:063eccf89df5b24e361b123e257e437f9e9878f425ee9aae3144c77faf6da6d8"}, + {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:6162a86d86893d63084faaf4ff937b3daea233e3682fb4474db07395794fa80d"}, + {file = "lxml-6.0.2-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:414aaa94e974e23a3e92e7ca5b97d10c0cf37b6481f50911032c69eeb3991bba"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:48461bd21625458dd01e14e2c38dd0aea69addc3c4f960c30d9f59d7f93be601"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:25fcc59afc57d527cfc78a58f40ab4c9b8fd096a9a3f964d2781ffb6eb33f4ed"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5179c60288204e6ddde3f774a93350177e08876eaf3ab78aa3a3649d43eb7d37"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:967aab75434de148ec80597b75062d8123cadf2943fb4281f385141e18b21338"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d100fcc8930d697c6561156c6810ab4a508fb264c8b6779e6e61e2ed5e7558f9"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2ca59e7e13e5981175b8b3e4ab84d7da57993eeff53c07764dcebda0d0e64ecd"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:957448ac63a42e2e49531b9d6c0fa449a1970dbc32467aaad46f11545be9af1d"}, + {file = "lxml-6.0.2-cp314-cp314t-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b7fc49c37f1786284b12af63152fe1d0990722497e2d5817acfe7a877522f9a9"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e19e0643cc936a22e837f79d01a550678da8377d7d801a14487c10c34ee49c7e"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:1db01e5cf14345628e0cbe71067204db658e2fb8e51e7f33631f5f4735fefd8d"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:875c6b5ab39ad5291588aed6925fac99d0097af0dd62f33c7b43736043d4a2ec"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:cdcbed9ad19da81c480dfd6dd161886db6096083c9938ead313d94b30aadf272"}, + {file = "lxml-6.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:80dadc234ebc532e09be1975ff538d154a7fa61ea5031c03d25178855544728f"}, + {file = "lxml-6.0.2-cp314-cp314t-win32.whl", hash = "sha256:da08e7bb297b04e893d91087df19638dc7a6bb858a954b0cc2b9f5053c922312"}, + {file = "lxml-6.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:252a22982dca42f6155125ac76d3432e548a7625d56f5a273ee78a5057216eca"}, + {file = "lxml-6.0.2-cp314-cp314t-win_arm64.whl", hash = "sha256:bb4c1847b303835d89d785a18801a883436cdfd5dc3d62947f9c49e24f0f5a2c"}, + {file = "lxml-6.0.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a656ca105115f6b766bba324f23a67914d9c728dafec57638e2b92a9dcd76c62"}, + {file = "lxml-6.0.2-cp38-cp38-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c54d83a2188a10ebdba573f16bd97135d06c9ef60c3dc495315c7a28c80a263f"}, + {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:1ea99340b3c729beea786f78c38f60f4795622f36e305d9c9be402201efdc3b7"}, + {file = "lxml-6.0.2-cp38-cp38-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:af85529ae8d2a453feee4c780d9406a5e3b17cee0dd75c18bd31adcd584debc3"}, + {file = "lxml-6.0.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:fe659f6b5d10fb5a17f00a50eb903eb277a71ee35df4615db573c069bcf967ac"}, + {file = "lxml-6.0.2-cp38-cp38-win32.whl", hash = "sha256:5921d924aa5468c939d95c9814fa9f9b5935a6ff4e679e26aaf2951f74043512"}, + {file = "lxml-6.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:0aa7070978f893954008ab73bb9e3c24a7c56c054e00566a21b553dc18105fca"}, + {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:2c8458c2cdd29589a8367c09c8f030f1d202be673f0ca224ec18590b3b9fb694"}, + {file = "lxml-6.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3fee0851639d06276e6b387f1c190eb9d7f06f7f53514e966b26bae46481ec90"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:b2142a376b40b6736dfc214fd2902409e9e3857eff554fed2d3c60f097e62a62"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a6b5b39cc7e2998f968f05309e666103b53e2edd01df8dc51b90d734c0825444"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d4aec24d6b72ee457ec665344a29acb2d35937d5192faebe429ea02633151aad"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_i686.manylinux_2_28_i686.whl", hash = "sha256:b42f4d86b451c2f9d06ffb4f8bbc776e04df3ba070b9fe2657804b1b40277c48"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cdaefac66e8b8f30e37a9b4768a391e1f8a16a7526d5bc77a7928408ef68e93"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_31_armv7l.whl", hash = "sha256:b738f7e648735714bbb82bdfd030203360cfeab7f6e8a34772b3c8c8b820568c"}, + {file = "lxml-6.0.2-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:daf42de090d59db025af61ce6bdb2521f0f102ea0e6ea310f13c17610a97da4c"}, + {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:66328dabea70b5ba7e53d94aa774b733cf66686535f3bc9250a7aab53a91caaf"}, + {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_armv7l.whl", hash = "sha256:e237b807d68a61fc3b1e845407e27e5eb8ef69bc93fe8505337c1acb4ee300b6"}, + {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:ac02dc29fd397608f8eb15ac1610ae2f2f0154b03f631e6d724d9e2ad4ee2c84"}, + {file = "lxml-6.0.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:817ef43a0c0b4a77bd166dc9a09a555394105ff3374777ad41f453526e37f9cb"}, + {file = "lxml-6.0.2-cp39-cp39-win32.whl", hash = "sha256:bc532422ff26b304cfb62b328826bd995c96154ffd2bac4544f37dbb95ecaa8f"}, + {file = "lxml-6.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:995e783eb0374c120f528f807443ad5a83a656a8624c467ea73781fc5f8a8304"}, + {file = "lxml-6.0.2-cp39-cp39-win_arm64.whl", hash = "sha256:08b9d5e803c2e4725ae9e8559ee880e5328ed61aa0935244e0515d7d9dbec0aa"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e748d4cf8fef2526bb2a589a417eba0c8674e29ffcb570ce2ceca44f1e567bf6"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:4ddb1049fa0579d0cbd00503ad8c58b9ab34d1254c77bc6a5576d96ec7853dba"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cb233f9c95f83707dae461b12b720c1af9c28c2d19208e1be03387222151daf5"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bc456d04db0515ce3320d714a1eac7a97774ff0849e7718b492d957da4631dd4"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2613e67de13d619fd283d58bda40bff0ee07739f624ffee8b13b631abf33083d"}, + {file = "lxml-6.0.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:24a8e756c982c001ca8d59e87c80c4d9dcd4d9b44a4cbeb8d9be4482c514d41d"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1c06035eafa8404b5cf475bb37a9f6088b0aca288d4ccc9d69389750d5543700"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c7d13103045de1bdd6fe5d61802565f1a3537d70cd3abf596aa0af62761921ee"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:0a3c150a95fbe5ac91de323aa756219ef9cf7fde5a3f00e2281e30f33fa5fa4f"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:60fa43be34f78bebb27812ed90f1925ec99560b0fa1decdb7d12b84d857d31e9"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-manylinux_2_26_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:21c73b476d3cfe836be731225ec3421fa2f048d84f6df6a8e70433dff1376d5a"}, + {file = "lxml-6.0.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:27220da5be049e936c3aca06f174e8827ca6445a4353a1995584311487fc4e3e"}, + {file = "lxml-6.0.2.tar.gz", hash = "sha256:cd79f3367bd74b317dda655dc8fcfa304d9eb6e4fb06b7168c5cf27f96e0cd62"}, +] + +[package.extras] +cssselect = ["cssselect (>=0.7)"] +html-clean = ["lxml_html_clean"] +html5 = ["html5lib"] +htmlsoup = ["BeautifulSoup4"] + [[package]] name = "markdown" version = "3.7" @@ -1804,6 +2077,82 @@ griffe = ">=0.49" mkdocs-autorefs = ">=1.4" mkdocstrings = ">=0.28.2" +[[package]] +name = "ml-dtypes" +version = "0.5.3" +description = "ml_dtypes is a stand-alone implementation of several NumPy dtype extensions used in machine learning." +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "ml_dtypes-0.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0a1d68a7cb53e3f640b2b6a34d12c0542da3dd935e560fdf463c0c77f339fc20"}, + {file = "ml_dtypes-0.5.3-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cd5a6c711b5350f3cbc2ac28def81cd1c580075ccb7955e61e9d8f4bfd40d24"}, + {file = "ml_dtypes-0.5.3-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdcf26c2dbc926b8a35ec8cbfad7eff1a8bd8239e12478caca83a1fc2c400dc2"}, + {file = "ml_dtypes-0.5.3-cp310-cp310-win_amd64.whl", hash = "sha256:aecbd7c5272c82e54d5b99d8435fd10915d1bc704b7df15e4d9ca8dc3902be61"}, + {file = "ml_dtypes-0.5.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:4a177b882667c69422402df6ed5c3428ce07ac2c1f844d8a1314944651439458"}, + {file = "ml_dtypes-0.5.3-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9849ce7267444c0a717c80c6900997de4f36e2815ce34ac560a3edb2d9a64cd2"}, + {file = "ml_dtypes-0.5.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c3f5ae0309d9f888fd825c2e9d0241102fadaca81d888f26f845bc8c13c1e4ee"}, + {file = "ml_dtypes-0.5.3-cp311-cp311-win_amd64.whl", hash = "sha256:58e39349d820b5702bb6f94ea0cb2dc8ec62ee81c0267d9622067d8333596a46"}, + {file = "ml_dtypes-0.5.3-cp311-cp311-win_arm64.whl", hash = "sha256:66c2756ae6cfd7f5224e355c893cfd617fa2f747b8bbd8996152cbdebad9a184"}, + {file = "ml_dtypes-0.5.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:156418abeeda48ea4797db6776db3c5bdab9ac7be197c1233771e0880c304057"}, + {file = "ml_dtypes-0.5.3-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1db60c154989af253f6c4a34e8a540c2c9dce4d770784d426945e09908fbb177"}, + {file = "ml_dtypes-0.5.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1b255acada256d1fa8c35ed07b5f6d18bc21d1556f842fbc2d5718aea2cd9e55"}, + {file = "ml_dtypes-0.5.3-cp312-cp312-win_amd64.whl", hash = "sha256:da65e5fd3eea434ccb8984c3624bc234ddcc0d9f4c81864af611aaebcc08a50e"}, + {file = "ml_dtypes-0.5.3-cp312-cp312-win_arm64.whl", hash = "sha256:8bb9cd1ce63096567f5f42851f5843b5a0ea11511e50039a7649619abfb4ba6d"}, + {file = "ml_dtypes-0.5.3-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:5103856a225465371fe119f2fef737402b705b810bd95ad5f348e6e1a6ae21af"}, + {file = "ml_dtypes-0.5.3-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4cae435a68861660af81fa3c5af16b70ca11a17275c5b662d9c6f58294e0f113"}, + {file = "ml_dtypes-0.5.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6936283b56d74fbec431ca57ce58a90a908fdbd14d4e2d22eea6d72bb208a7b7"}, + {file = "ml_dtypes-0.5.3-cp313-cp313-win_amd64.whl", hash = "sha256:d0f730a17cf4f343b2c7ad50cee3bd19e969e793d2be6ed911f43086460096e4"}, + {file = "ml_dtypes-0.5.3-cp313-cp313-win_arm64.whl", hash = "sha256:2db74788fc01914a3c7f7da0763427280adfc9cd377e9604b6b64eb8097284bd"}, + {file = "ml_dtypes-0.5.3-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:93c36a08a6d158db44f2eb9ce3258e53f24a9a4a695325a689494f0fdbc71770"}, + {file = "ml_dtypes-0.5.3-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0e44a3761f64bc009d71ddb6d6c71008ba21b53ab6ee588dadab65e2fa79eafc"}, + {file = "ml_dtypes-0.5.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bdf40d2aaabd3913dec11840f0d0ebb1b93134f99af6a0a4fd88ffe924928ab4"}, + {file = "ml_dtypes-0.5.3-cp314-cp314-macosx_10_13_universal2.whl", hash = "sha256:aec640bd94c4c85c0d11e2733bd13cbb10438fb004852996ec0efbc6cacdaf70"}, + {file = "ml_dtypes-0.5.3-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bda32ce212baa724e03c68771e5c69f39e584ea426bfe1a701cb01508ffc7035"}, + {file = "ml_dtypes-0.5.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c205cac07d24a29840c163d6469f61069ce4b065518519216297fc2f261f8db9"}, + {file = "ml_dtypes-0.5.3-cp314-cp314-win_amd64.whl", hash = "sha256:cd7c0bb22d4ff86d65ad61b5dd246812e8993fbc95b558553624c33e8b6903ea"}, + {file = "ml_dtypes-0.5.3-cp314-cp314-win_arm64.whl", hash = "sha256:9d55ea7f7baf2aed61bf1872116cefc9d0c3693b45cae3916897ee27ef4b835e"}, + {file = "ml_dtypes-0.5.3-cp314-cp314t-macosx_10_13_universal2.whl", hash = "sha256:e12e29764a0e66a7a31e9b8bf1de5cc0423ea72979f45909acd4292de834ccd3"}, + {file = "ml_dtypes-0.5.3-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:19f6c3a4f635c2fc9e2aa7d91416bd7a3d649b48350c51f7f715a09370a90d93"}, + {file = "ml_dtypes-0.5.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ab039ffb40f3dc0aeeeba84fd6c3452781b5e15bef72e2d10bcb33e4bbffc39"}, + {file = "ml_dtypes-0.5.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:5ee72568d46b9533ad54f78b1e1f3067c0534c5065120ea8ecc6f210d22748b3"}, + {file = "ml_dtypes-0.5.3-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:01de48de4537dc3c46e684b969a40ec36594e7eeb7c69e9a093e7239f030a28a"}, + {file = "ml_dtypes-0.5.3-cp39-cp39-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8b1a6e231b0770f2894910f1dce6d2f31d65884dbf7668f9b08d73623cdca909"}, + {file = "ml_dtypes-0.5.3-cp39-cp39-win_amd64.whl", hash = "sha256:a4f39b9bf6555fab9bfb536cf5fdd1c1c727e8d22312078702e9ff005354b37f"}, + {file = "ml_dtypes-0.5.3.tar.gz", hash = "sha256:95ce33057ba4d05df50b1f3cfefab22e351868a843b3b15a46c65836283670c9"}, +] + +[package.dependencies] +numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""} + +[package.extras] +dev = ["absl-py", "pyink", "pylint (>=2.6.0)", "pytest", "pytest-xdist"] + +[[package]] +name = "modrover" +version = "0.2.0" +description = "Model space explorer for inference and prediction" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [] +develop = false + +[package.dependencies] +matplotlib = "*" +pplkit = "*" +regmod = "<0.2" + +[package.extras] +docs = ["furo", "sphinx", "sphinx-autodoc-typehints"] +test = ["pytest", "pytest-cov"] + +[package.source] +type = "git" +url = "https://github.com/ihmeuw-msca/modrover.git" +reference = "HEAD" +resolved_reference = "9c5116d7f21a44d0064c039f9bc686c37049ccb0" + [[package]] name = "mpmath" version = "1.3.0" @@ -1822,6 +2171,29 @@ docs = ["sphinx"] gmpy = ["gmpy2 (>=2.1.0a4) ; platform_python_implementation != \"PyPy\""] tests = ["pytest (>=4.6)"] +[[package]] +name = "msca" +version = "0.4.2" +description = "Mathematical sciences and computational algorithms" +optional = false +python-versions = "<3.13,>=3.11" +groups = ["main"] +files = [ + {file = "msca-0.4.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c531d7123d8e9532d791afed2fae97699e4515ca09440e8d4d585ab44f3373"}, + {file = "msca-0.4.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5e8ec9b0decf30b2ff43370b47db1325cf4e09acd6ddb94a908439f58d9bc976"}, +] + +[package.dependencies] +numpy = "*" +pandas = "*" +pydantic = "*" +scikit-learn = "*" +scipy = "*" + +[package.extras] +docs = ["furo", "sphinx", "sphinx-autodoc-typehints"] +test = ["pytest", "pytest-cov"] + [[package]] name = "multidict" version = "6.1.0" @@ -2373,6 +2745,18 @@ rsa = ["cryptography (>=3.0.0)"] signals = ["blinker (>=1.4.0)"] signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] +[[package]] +name = "opt-einsum" +version = "3.4.0" +description = "Path optimization of einsum functions." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd"}, + {file = "opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac"}, +] + [[package]] name = "packaging" version = "24.2" @@ -2455,6 +2839,7 @@ files = [ [package.dependencies] numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""} +pyarrow = {version = ">=10.0.1", optional = true, markers = "extra == \"parquet\""} python-dateutil = ">=2.8.2" pytz = ">=2020.1" tzdata = ">=2022.7" @@ -2679,6 +3064,29 @@ files = [ [package.extras] dill = ["dill (>=0.3.9)"] +[[package]] +name = "pplkit" +version = "0.1.0" +description = "Pipeline building toolkit" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "pplkit-0.1.0-py3-none-any.whl", hash = "sha256:199a71a58e624b2c178afdd63da2eb164b56ae10ed8915dc0514f8cf61cc6d8d"}, + {file = "pplkit-0.1.0.tar.gz", hash = "sha256:5f77fc9ab34cfcffa13d2bef3516028eda61469455fd0247f9fb5a1976b98fb6"}, +] + +[package.dependencies] +dill = "*" +pandas = {version = "*", extras = ["parquet"]} +pyyaml = "*" +tomli = "*" +tomli-w = "*" + +[package.extras] +docs = ["furo", "sphinx", "sphinx-autodoc-typehints"] +test = ["pytest"] + [[package]] name = "pre-commit" version = "4.1.0" @@ -3443,6 +3851,57 @@ pyproj = ">=3.6.0" rasterio = ">=1.3.9" shapely = ">=2.0" +[[package]] +name = "rasterstats" +version = "0.20.0" +description = "Summarize geospatial raster datasets based on vector geometries" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "rasterstats-0.20.0-py3-none-any.whl", hash = "sha256:a7015e99f42807842b8638d489157639ff0cbf8e35aac9647aa3e079982b18ee"}, + {file = "rasterstats-0.20.0.tar.gz", hash = "sha256:5b8ee775e815727767e0d359c03f3dd1c7840876d1d1d0c7a5a88ecf3e492938"}, +] + +[package.dependencies] +affine = "*" +click = ">7.1" +cligj = ">=0.4" +fiona = "*" +numpy = ">=1.9" +rasterio = ">=1.0" +shapely = "*" +simplejson = "*" + +[package.extras] +dev = ["numpydoc", "rasterstats[test]", "twine"] +progress = ["tqdm"] +test = ["coverage", "geopandas", "pyshp (>=1.1.4)", "pytest (>=4.6)", "pytest-cov (>=2.2.0)", "simplejson"] + +[[package]] +name = "regmod" +version = "0.1.3" +description = "General regression models" +optional = false +python-versions = ">=3.10" +groups = ["main"] +files = [ + {file = "regmod-0.1.3-py3-none-any.whl", hash = "sha256:0e00e4eb5e2aacab9d2b8221cdc4bef4eee1defc1853a5eda3bf5e3798fa1e9f"}, + {file = "regmod-0.1.3.tar.gz", hash = "sha256:57779eaca93f99dae1ea7f504559a0eae3f1e12f0028ba7460ecba4afca53bfc"}, +] + +[package.dependencies] +jax = "*" +jaxlib = "*" +msca = "*" +numpy = "*" +pandas = "*" +scipy = "*" +xspline = "0.0.7" + +[package.extras] +test = ["pytest"] + [[package]] name = "requests" version = "2.32.3" @@ -3499,16 +3958,39 @@ files = [ [package.dependencies] requests = ">=2.0.1,<3.0.0" +[[package]] +name = "rioxarray" +version = "0.20.0" +description = "geospatial xarray extension powered by rasterio" +optional = false +python-versions = ">=3.12" +groups = ["main"] +files = [ + {file = "rioxarray-0.20.0-py3-none-any.whl", hash = "sha256:197b0638146dfc6093ef52f8bf8afb42757ca16bc2e0d87b6282ce54170c9799"}, + {file = "rioxarray-0.20.0.tar.gz", hash = "sha256:8bfc7e979edc7e30b4671d638a9be0e5a7d673dab2ea88e2445d3c7745599c02"}, +] + +[package.dependencies] +numpy = ">=2" +packaging = "*" +pyproj = ">=3.3" +rasterio = ">=1.4.3" +xarray = ">=2024.7.0" + +[package.extras] +all = ["scipy"] +interp = ["scipy"] + [[package]] name = "rra-tools" -version = "1.0.25" +version = "1.0.27" description = "Common utilities for IHME Rapid Response team pipelines." optional = false python-versions = ">=3.12" groups = ["main"] files = [ - {file = "rra_tools-1.0.25-py3-none-any.whl", hash = "sha256:21750d88db4e8d89700c7bdb30a7fbca02d245bb86825530cfeadcf9d5ed5fb4"}, - {file = "rra_tools-1.0.25.tar.gz", hash = "sha256:78f6667dc436f8f5e42b7362e081d7c1c3819b8b5905f3fe8f79a7f04fdd2318"}, + {file = "rra_tools-1.0.27-py3-none-any.whl", hash = "sha256:a542eda44461d77bda7d4ae47038a5856cc05ca92eb2f481c00000ef43b10ead"}, + {file = "rra_tools-1.0.27.tar.gz", hash = "sha256:333299a841069c1cba4881e067bacf62b19b85120157cb3eea221ddf255c4e60"}, ] [package.dependencies] @@ -3833,6 +4315,126 @@ numpy = ">=1.14,<3" docs = ["matplotlib", "numpydoc (==1.1.*)", "sphinx", "sphinx-book-theme", "sphinx-remove-toctrees"] test = ["pytest", "pytest-cov"] +[[package]] +name = "simplejson" +version = "3.20.2" +description = "Simple, fast, extensible JSON encoder/decoder for Python" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.5" +groups = ["main"] +files = [ + {file = "simplejson-3.20.2-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:11847093fd36e3f5a4f595ff0506286c54885f8ad2d921dfb64a85bce67f72c4"}, + {file = "simplejson-3.20.2-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:4d291911d23b1ab8eb3241204dd54e3ec60ddcd74dfcb576939d3df327205865"}, + {file = "simplejson-3.20.2-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:da6d16d7108d366bbbf1c1f3274662294859c03266e80dd899fc432598115ea4"}, + {file = "simplejson-3.20.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9ddf9a07694c5bbb4856271cbc4247cc6cf48f224a7d128a280482a2f78bae3d"}, + {file = "simplejson-3.20.2-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:3a0d2337e490e6ab42d65a082e69473717f5cc75c3c3fb530504d3681c4cb40c"}, + {file = "simplejson-3.20.2-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:8ba88696351ed26a8648f8378a1431223f02438f8036f006d23b4f5b572778fa"}, + {file = "simplejson-3.20.2-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:00bcd408a4430af99d1f8b2b103bb2f5133bb688596a511fcfa7db865fbb845e"}, + {file = "simplejson-3.20.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:4fc62feb76f590ccaff6f903f52a01c58ba6423171aa117b96508afda9c210f0"}, + {file = "simplejson-3.20.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:6d7286dc11af60a2f76eafb0c2acde2d997e87890e37e24590bb513bec9f1bc5"}, + {file = "simplejson-3.20.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c01379b4861c3b0aa40cba8d44f2b448f5743999aa68aaa5d3ef7049d4a28a2d"}, + {file = "simplejson-3.20.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a16b029ca25645b3bc44e84a4f941efa51bf93c180b31bd704ce6349d1fc77c1"}, + {file = "simplejson-3.20.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e22a5fb7b1437ffb057e02e1936a3bfb19084ae9d221ec5e9f4cf85f69946b6"}, + {file = "simplejson-3.20.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d8b6ff02fc7b8555c906c24735908854819b0d0dc85883d453e23ca4c0445d01"}, + {file = "simplejson-3.20.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2bfc1c396ad972ba4431130b42307b2321dba14d988580c1ac421ec6a6b7cee3"}, + {file = "simplejson-3.20.2-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a97249ee1aee005d891b5a211faf58092a309f3d9d440bc269043b08f662eda"}, + {file = "simplejson-3.20.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:f1036be00b5edaddbddbb89c0f80ed229714a941cfd21e51386dc69c237201c2"}, + {file = "simplejson-3.20.2-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:5d6f5bacb8cdee64946b45f2680afa3f54cd38e62471ceda89f777693aeca4e4"}, + {file = "simplejson-3.20.2-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:8db6841fb796ec5af632f677abf21c6425a1ebea0d9ac3ef1a340b8dc69f52b8"}, + {file = "simplejson-3.20.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c0a341f7cc2aae82ee2b31f8a827fd2e51d09626f8b3accc441a6907c88aedb7"}, + {file = "simplejson-3.20.2-cp310-cp310-win32.whl", hash = "sha256:27f9c01a6bc581d32ab026f515226864576da05ef322d7fc141cd8a15a95ce53"}, + {file = "simplejson-3.20.2-cp310-cp310-win_amd64.whl", hash = "sha256:c0a63ec98a4547ff366871bf832a7367ee43d047bcec0b07b66c794e2137b476"}, + {file = "simplejson-3.20.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:06190b33cd7849efc413a5738d3da00b90e4a5382fd3d584c841ac20fb828c6f"}, + {file = "simplejson-3.20.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4ad4eac7d858947a30d2c404e61f16b84d16be79eb6fb316341885bdde864fa8"}, + {file = "simplejson-3.20.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b392e11c6165d4a0fde41754a0e13e1d88a5ad782b245a973dd4b2bdb4e5076a"}, + {file = "simplejson-3.20.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:51eccc4e353eed3c50e0ea2326173acdc05e58f0c110405920b989d481287e51"}, + {file = "simplejson-3.20.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:306e83d7c331ad833d2d43c76a67f476c4b80c4a13334f6e34bb110e6105b3bd"}, + {file = "simplejson-3.20.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f820a6ac2ef0bc338ae4963f4f82ccebdb0824fe9caf6d660670c578abe01013"}, + {file = "simplejson-3.20.2-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e7a066528a5451433eb3418184f05682ea0493d14e9aae690499b7e1eb6b81"}, + {file = "simplejson-3.20.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:438680ddde57ea87161a4824e8de04387b328ad51cfdf1eaf723623a3014b7aa"}, + {file = "simplejson-3.20.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:cac78470ae68b8d8c41b6fca97f5bf8e024ca80d5878c7724e024540f5cdaadb"}, + {file = "simplejson-3.20.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:7524e19c2da5ef281860a3d74668050c6986be15c9dd99966034ba47c68828c2"}, + {file = "simplejson-3.20.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:0e9b6d845a603b2eef3394eb5e21edb8626cd9ae9a8361d14e267eb969dbe413"}, + {file = "simplejson-3.20.2-cp311-cp311-win32.whl", hash = "sha256:47d8927e5ac927fdd34c99cc617938abb3624b06ff86e8e219740a86507eb961"}, + {file = "simplejson-3.20.2-cp311-cp311-win_amd64.whl", hash = "sha256:ba4edf3be8e97e4713d06c3d302cba1ff5c49d16e9d24c209884ac1b8455520c"}, + {file = "simplejson-3.20.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:4376d5acae0d1e91e78baeba4ee3cf22fbf6509d81539d01b94e0951d28ec2b6"}, + {file = "simplejson-3.20.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f8fe6de652fcddae6dec8f281cc1e77e4e8f3575249e1800090aab48f73b4259"}, + {file = "simplejson-3.20.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:25ca2663d99328d51e5a138f22018e54c9162438d831e26cfc3458688616eca8"}, + {file = "simplejson-3.20.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:12a6b2816b6cab6c3fd273d43b1948bc9acf708272074c8858f579c394f4cbc9"}, + {file = "simplejson-3.20.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac20dc3fcdfc7b8415bfc3d7d51beccd8695c3f4acb7f74e3a3b538e76672868"}, + {file = "simplejson-3.20.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:db0804d04564e70862ef807f3e1ace2cc212ef0e22deb1b3d6f80c45e5882c6b"}, + {file = "simplejson-3.20.2-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:979ce23ea663895ae39106946ef3d78527822d918a136dbc77b9e2b7f006237e"}, + {file = "simplejson-3.20.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a2ba921b047bb029805726800819675249ef25d2f65fd0edb90639c5b1c3033c"}, + {file = "simplejson-3.20.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:12d3d4dc33770069b780cc8f5abef909fe4a3f071f18f55f6d896a370fd0f970"}, + {file = "simplejson-3.20.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:aff032a59a201b3683a34be1169e71ddda683d9c3b43b261599c12055349251e"}, + {file = "simplejson-3.20.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:30e590e133b06773f0dc9c3f82e567463df40598b660b5adf53eb1c488202544"}, + {file = "simplejson-3.20.2-cp312-cp312-win32.whl", hash = "sha256:8d7be7c99939cc58e7c5bcf6bb52a842a58e6c65e1e9cdd2a94b697b24cddb54"}, + {file = "simplejson-3.20.2-cp312-cp312-win_amd64.whl", hash = "sha256:2c0b4a67e75b945489052af6590e7dca0ed473ead5d0f3aad61fa584afe814ab"}, + {file = "simplejson-3.20.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:90d311ba8fcd733a3677e0be21804827226a57144130ba01c3c6a325e887dd86"}, + {file = "simplejson-3.20.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:feed6806f614bdf7f5cb6d0123cb0c1c5f40407ef103aa935cffaa694e2e0c74"}, + {file = "simplejson-3.20.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6b1d8d7c3e1a205c49e1aee6ba907dcb8ccea83651e6c3e2cb2062f1e52b0726"}, + {file = "simplejson-3.20.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:552f55745044a24c3cb7ec67e54234be56d5d6d0e054f2e4cf4fb3e297429be5"}, + {file = "simplejson-3.20.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c2da97ac65165d66b0570c9e545786f0ac7b5de5854d3711a16cacbcaa8c472d"}, + {file = "simplejson-3.20.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f59a12966daa356bf68927fca5a67bebac0033cd18b96de9c2d426cd11756cd0"}, + {file = "simplejson-3.20.2-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:133ae2098a8e162c71da97cdab1f383afdd91373b7ff5fe65169b04167da976b"}, + {file = "simplejson-3.20.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7977640af7b7d5e6a852d26622057d428706a550f7f5083e7c4dd010a84d941f"}, + {file = "simplejson-3.20.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b530ad6d55e71fa9e93e1109cf8182f427a6355848a4ffa09f69cc44e1512522"}, + {file = "simplejson-3.20.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:bd96a7d981bf64f0e42345584768da4435c05b24fd3c364663f5fbc8fabf82e3"}, + {file = "simplejson-3.20.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:f28ee755fadb426ba2e464d6fcf25d3f152a05eb6b38e0b4f790352f5540c769"}, + {file = "simplejson-3.20.2-cp313-cp313-win32.whl", hash = "sha256:472785b52e48e3eed9b78b95e26a256f59bb1ee38339be3075dad799e2e1e661"}, + {file = "simplejson-3.20.2-cp313-cp313-win_amd64.whl", hash = "sha256:a1a85013eb33e4820286139540accbe2c98d2da894b2dcefd280209db508e608"}, + {file = "simplejson-3.20.2-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:a135941a50795c934bdc9acc74e172b126e3694fe26de3c0c1bc0b33ea17e6ce"}, + {file = "simplejson-3.20.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:25ba488decb18738f5d6bd082018409689ed8e74bc6c4d33a0b81af6edf1c9f4"}, + {file = "simplejson-3.20.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d81f8e982923d5e9841622ff6568be89756428f98a82c16e4158ac32b92a3787"}, + {file = "simplejson-3.20.2-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cdad497ccb1edc5020bef209e9c3e062a923e8e6fca5b8a39f0fb34380c8a66c"}, + {file = "simplejson-3.20.2-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4a3f1db97bcd9fb592928159af7a405b18df7e847cbcc5682a209c5b2ad5d6b1"}, + {file = "simplejson-3.20.2-cp36-cp36m-musllinux_1_2_aarch64.whl", hash = "sha256:215b65b0dc2c432ab79c430aa4f1e595f37b07a83c1e4c4928d7e22e6b49a748"}, + {file = "simplejson-3.20.2-cp36-cp36m-musllinux_1_2_i686.whl", hash = "sha256:ece4863171ba53f086a3bfd87f02ec3d6abc586f413babfc6cf4de4d84894620"}, + {file = "simplejson-3.20.2-cp36-cp36m-musllinux_1_2_ppc64le.whl", hash = "sha256:4a76d7c47d959afe6c41c88005f3041f583a4b9a1783cf341887a3628a77baa0"}, + {file = "simplejson-3.20.2-cp36-cp36m-musllinux_1_2_x86_64.whl", hash = "sha256:e9b0523582a57d9ea74f83ecefdffe18b2b0a907df1a9cef06955883341930d8"}, + {file = "simplejson-3.20.2-cp36-cp36m-win32.whl", hash = "sha256:16366591c8e08a4ac76b81d76a3fc97bf2bcc234c9c097b48d32ea6bfe2be2fe"}, + {file = "simplejson-3.20.2-cp36-cp36m-win_amd64.whl", hash = "sha256:732cf4c4ac1a258b4e9334e1e40a38303689f432497d3caeb491428b7547e782"}, + {file = "simplejson-3.20.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6c3a98e21e5f098e4f982ef302ebb1e681ff16a5d530cfce36296bea58fe2396"}, + {file = "simplejson-3.20.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:10cf9ca1363dc3711c72f4ec7c1caed2bbd9aaa29a8d9122e31106022dc175c6"}, + {file = "simplejson-3.20.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:106762f8aedf3fc3364649bfe8dc9a40bf5104f872a4d2d86bae001b1af30d30"}, + {file = "simplejson-3.20.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b21659898b7496322e99674739193f81052e588afa8b31b6a1c7733d8829b925"}, + {file = "simplejson-3.20.2-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78fa1db6a02bca88829f2b2057c76a1d2dc2fccb8c5ff1199e352f213e9ec719"}, + {file = "simplejson-3.20.2-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:156139d94b660448ec8a4ea89f77ec476597f752c2ff66432d3656704c66b40e"}, + {file = "simplejson-3.20.2-cp37-cp37m-musllinux_1_2_i686.whl", hash = "sha256:b2620ac40be04dff08854baf6f4df10272f67079f61ed1b6274c0e840f2e2ae1"}, + {file = "simplejson-3.20.2-cp37-cp37m-musllinux_1_2_ppc64le.whl", hash = "sha256:9ccef5b5d3e3ac5d9da0a0ca1d2de8cf2b0fb56b06aa0ab79325fa4bcc5a1d60"}, + {file = "simplejson-3.20.2-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:f526304c2cc9fd8b8d18afacb75bc171650f83a7097b2c92ad6a431b5d7c1b72"}, + {file = "simplejson-3.20.2-cp37-cp37m-win32.whl", hash = "sha256:e0f661105398121dd48d9987a2a8f7825b8297b3b2a7fe5b0d247370396119d5"}, + {file = "simplejson-3.20.2-cp37-cp37m-win_amd64.whl", hash = "sha256:dab98625b3d6821e77ea59c4d0e71059f8063825a0885b50ed410e5c8bd5cb66"}, + {file = "simplejson-3.20.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:b8205f113082e7d8f667d6cd37d019a7ee5ef30b48463f9de48e1853726c6127"}, + {file = "simplejson-3.20.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fc8da64929ef0ff16448b602394a76fd9968a39afff0692e5ab53669df1f047f"}, + {file = "simplejson-3.20.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:bfe704864b5fead4f21c8d448a89ee101c9b0fc92a5f40b674111da9272b3a90"}, + {file = "simplejson-3.20.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40ca7cbe7d2f423b97ed4e70989ef357f027a7e487606628c11b79667639dc84"}, + {file = "simplejson-3.20.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cec1868b237fe9fb2d466d6ce0c7b772e005aadeeda582d867f6f1ec9710cad"}, + {file = "simplejson-3.20.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:792debfba68d8dd61085ffb332d72b9f5b38269cda0c99f92c7a054382f55246"}, + {file = "simplejson-3.20.2-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e022b2c4c54cb4855e555f64aa3377e3e5ca912c372fa9e3edcc90ebbad93dce"}, + {file = "simplejson-3.20.2-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:5de26f11d5aca575d3825dddc65f69fdcba18f6ca2b4db5cef16f41f969cef15"}, + {file = "simplejson-3.20.2-cp38-cp38-musllinux_1_2_i686.whl", hash = "sha256:e2162b2a43614727ec3df75baeda8881ab129824aa1b49410d4b6c64f55a45b4"}, + {file = "simplejson-3.20.2-cp38-cp38-musllinux_1_2_ppc64le.whl", hash = "sha256:e11a1d6b2f7e72ca546bdb4e6374b237ebae9220e764051b867111df83acbd13"}, + {file = "simplejson-3.20.2-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:daf7cd18fe99eb427fa6ddb6b437cfde65125a96dc27b93a8969b6fe90a1dbea"}, + {file = "simplejson-3.20.2-cp38-cp38-win32.whl", hash = "sha256:da795ea5f440052f4f497b496010e2c4e05940d449ea7b5c417794ec1be55d01"}, + {file = "simplejson-3.20.2-cp38-cp38-win_amd64.whl", hash = "sha256:6a4b5e7864f952fcce4244a70166797d7b8fd6069b4286d3e8403c14b88656b6"}, + {file = "simplejson-3.20.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:b3bf76512ccb07d47944ebdca44c65b781612d38b9098566b4bb40f713fc4047"}, + {file = "simplejson-3.20.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:214e26acf2dfb9ff3314e65c4e168a6b125bced0e2d99a65ea7b0f169db1e562"}, + {file = "simplejson-3.20.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:2fb1259ca9c385b0395bad59cdbf79535a5a84fb1988f339a49bfbc57455a35a"}, + {file = "simplejson-3.20.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c34e028a2ba8553a208ded1da5fa8501833875078c4c00a50dffc33622057881"}, + {file = "simplejson-3.20.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b538f9d9e503b0dd43af60496780cb50755e4d8e5b34e5647b887675c1ae9fee"}, + {file = "simplejson-3.20.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab998e416ded6c58f549a22b6a8847e75a9e1ef98eb9fbb2863e1f9e61a4105b"}, + {file = "simplejson-3.20.2-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6a8f1c307edf5fbf0c6db3396c5d3471409c4a40c7a2a466fbc762f20d46601a"}, + {file = "simplejson-3.20.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:5a7bbac80bdb82a44303f5630baee140aee208e5a4618e8b9fde3fc400a42671"}, + {file = "simplejson-3.20.2-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:5ef70ec8fe1569872e5a3e4720c1e1dcb823879a3c78bc02589eb88fab920b1f"}, + {file = "simplejson-3.20.2-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:cb11c09c99253a74c36925d461c86ea25f0140f3b98ff678322734ddc0f038d7"}, + {file = "simplejson-3.20.2-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:66f7c78c6ef776f8bd9afaad455e88b8197a51e95617bcc44b50dd974a7825ba"}, + {file = "simplejson-3.20.2-cp39-cp39-win32.whl", hash = "sha256:619ada86bfe3a5aa02b8222ca6bfc5aa3e1075c1fb5b3263d24ba579382df472"}, + {file = "simplejson-3.20.2-cp39-cp39-win_amd64.whl", hash = "sha256:44a6235e09ca5cc41aa5870a952489c06aa4aee3361ae46daa947d8398e57502"}, + {file = "simplejson-3.20.2-py3-none-any.whl", hash = "sha256:3b6bb7fb96efd673eac2e4235200bfffdc2353ad12c54117e1e4e2fc485ac017"}, + {file = "simplejson-3.20.2.tar.gz", hash = "sha256:5fe7a6ce14d1c300d80d08695b7f7e633de6cd72c80644021874d985b3393649"}, +] + [[package]] name = "six" version = "1.17.0" @@ -3937,6 +4539,70 @@ test = ["cmapfile", "czifile", "dask", "defusedxml", "fsspec", "imagecodecs", "l xml = ["defusedxml", "lxml"] zarr = ["fsspec", "zarr (<3)"] +[[package]] +name = "tomli" +version = "2.3.0" +description = "A lil' TOML parser" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45"}, + {file = "tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba"}, + {file = "tomli-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf"}, + {file = "tomli-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441"}, + {file = "tomli-2.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845"}, + {file = "tomli-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c"}, + {file = "tomli-2.3.0-cp311-cp311-win32.whl", hash = "sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456"}, + {file = "tomli-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be"}, + {file = "tomli-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac"}, + {file = "tomli-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22"}, + {file = "tomli-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f"}, + {file = "tomli-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52"}, + {file = "tomli-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8"}, + {file = "tomli-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6"}, + {file = "tomli-2.3.0-cp312-cp312-win32.whl", hash = "sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876"}, + {file = "tomli-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878"}, + {file = "tomli-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b"}, + {file = "tomli-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae"}, + {file = "tomli-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b"}, + {file = "tomli-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf"}, + {file = "tomli-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f"}, + {file = "tomli-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05"}, + {file = "tomli-2.3.0-cp313-cp313-win32.whl", hash = "sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606"}, + {file = "tomli-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999"}, + {file = "tomli-2.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e"}, + {file = "tomli-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3"}, + {file = "tomli-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc"}, + {file = "tomli-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0"}, + {file = "tomli-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879"}, + {file = "tomli-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005"}, + {file = "tomli-2.3.0-cp314-cp314-win32.whl", hash = "sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463"}, + {file = "tomli-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8"}, + {file = "tomli-2.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77"}, + {file = "tomli-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf"}, + {file = "tomli-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530"}, + {file = "tomli-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b"}, + {file = "tomli-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67"}, + {file = "tomli-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f"}, + {file = "tomli-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0"}, + {file = "tomli-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba"}, + {file = "tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b"}, + {file = "tomli-2.3.0.tar.gz", hash = "sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549"}, +] + +[[package]] +name = "tomli-w" +version = "1.2.0" +description = "A lil' TOML writer" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "tomli_w-1.2.0-py3-none-any.whl", hash = "sha256:188306098d013b691fcadc011abd66727d3c414c571bb01b1a174ba8c983cf90"}, + {file = "tomli_w-1.2.0.tar.gz", hash = "sha256:2dd14fac5a47c27be9cd4c976af5a12d87fb1f0b4512f81d69cce3b35ae25021"}, +] + [[package]] name = "torch" version = "2.6.0" @@ -4268,6 +4934,25 @@ parallel = ["dask[complete]"] types = ["pandas-stubs", "scipy-stubs", "types-PyYAML", "types-Pygments", "types-colorama", "types-decorator", "types-defusedxml", "types-docutils", "types-networkx", "types-openpyxl", "types-pexpect", "types-psutil", "types-pycurl", "types-python-dateutil", "types-pytz", "types-setuptools"] viz = ["cartopy (>=0.23)", "matplotlib", "nc-time-axis", "seaborn"] +[[package]] +name = "xspline" +version = "0.0.7" +description = "xspline: Advanced spline tools" +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "xspline-0.0.7-py3-none-any.whl", hash = "sha256:3815d6969c3e1f0f9d8ef7d468aa35c992811e3ad631ba5b9102395ca185a319"}, + {file = "xspline-0.0.7.tar.gz", hash = "sha256:24cb8452ea6b6c83bf23ba72d5b9abac8dcf5eb00e0053050a1ea515f67e90cb"}, +] + +[package.dependencies] +numpy = "*" + +[package.extras] +dev = ["pytest", "pytest-mock"] +test = ["pytest", "pytest-mock"] + [[package]] name = "xyzservices" version = "2025.1.0" @@ -4380,4 +5065,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.1" python-versions = ">=3.12, <3.13" -content-hash = "fc93c2b57729246e9083ef4db483bd252fd61675357600b95558a131d4cd5d9b" +content-hash = "1a2c5125136188c809293fc1d725266dc10970a3602e3e2aaecf50ad2a096a61" diff --git a/pyproject.toml b/pyproject.toml index 9a52c00..948202a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,6 +6,7 @@ license = "BSD-3-Clause" readme = "README.md" authors = [ {name = "James Collins", email = "collijk@uw.edu"}, + {name = "Ryan Barber", email = "rmbarber@uw.edu"}, ] requires-python = ">=3.12" dependencies = [ @@ -13,10 +14,12 @@ dependencies = [ "geopandas>=1.0.1", "numpy>=2.2.1", "pandas>=2.2.3", - "rasterra>=0.6.2", + "rasterra==0.6.2", + # "rasterra (>=0.6.3,<0.7.0)", + "matplotlib>=3.10.1", "shapely>=2.0.6", "pydantic>=2.10.4", - "rra-tools>=1.0.25", + "rra-tools>=1.0.27", "pyyaml>=6.0.2", "scikit-learn>=1.6.0", "pyarrow>=18.1.0", @@ -27,6 +30,10 @@ dependencies = [ "contextily>=1.6.2", "xarray>=2025.7.0", "netCDF4>=1.7.2", + "modrover @ git+https://github.com/ihmeuw-msca/modrover.git", + "rasterstats>=0.20.0", + "rioxarray>=0.20.0", + "lxml>=6.0.2", ] [project.urls] diff --git a/scripts/analyze_admin_distributions.py b/scripts/analyze_admin_distributions.py deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/poverty.py b/scripts/poverty.py new file mode 100644 index 0000000..20b6d7a --- /dev/null +++ b/scripts/poverty.py @@ -0,0 +1,340 @@ +import sys +from pathlib import Path +from typing import List, Dict +from loguru import logger + +import pandas as pd +import numpy as np +import rasterra as rt +import geopandas as gpd +import shapely +from rasterio.features import rasterize + +from jobmon.client.tool import Tool +import uuid +import shutil +from rra_tools.shell_tools import mkdir + +from rra_population_model import constants as pmc +from rra_population_model.model.modeling.datamodel import ModelSpecification +from rra_population_model.data import ( + BuildingDensityData, + PopulationModelData, + save_raster, +) +from rra_population_model.model_prep.features.metadata import FeatureMetadata +from rra_population_model.model_prep.features import utils +from rra_population_model.model_prep.features.metadata import get_feature_metadata + +POVERTY_RATES_PATH = Path("/mnt/share/geospatial/lsae/economic_indicators/ldi/output/20251022_090128/raked_admin2_urban_rural_poverty.csv") + + +def execute_workflow( + resolution: str, + version: str, + block_keys: List[str], + time_points: List[str], + radii: List[int], +): + wf_uuid = uuid.uuid4() + + tool = Tool(name="poverty") + + workflow = tool.create_workflow( + name=f"poverty_{wf_uuid}", + ) + + ## define templates + task_template = tool.get_task_template( + default_compute_resources={ + "queue": "all.q", + "cores": 1, + "memory": "33G", + "runtime": "5m", + "project": "proj_rapidresponse", + }, + template_name="poverty", + default_cluster_name="slurm", + command_template=f"{shutil.which('python')}" + f" {Path(__file__)}" + " worker" + " {resolution}" + " {version}" + " {block_key}" + " {time_point}" + " {radius}", + node_args=["block_key", "time_point", "radius"], + task_args=["resolution", "version"], + op_args=[], + ) + + ## compile tasks + tasks = [] + for block_key in block_keys: + for time_point in time_points: + for radius in radii: + tasks.append( + task_template.create_task( + max_attempts=2, + # resource_scales={ + # "memory": iter([20 , 60 , 100 , 200 , 700 , 900 ]), + # "runtime": iter([5 * 60, 10 * 60, 15 * 60, 30 * 60, 60 * 60, 90 * 60]), + # }, + block_key=block_key, + time_point=time_point, + resolution=resolution, + version=version, + radius=radius, + ) + ) + + workflow.add_tasks(tasks) + workflow.bind() + + logger.info(f"Running workflow with ID {workflow.workflow_id}.") + logger.info("For full information see the Jobmon GUI:") + logger.info(f"https://jobmon-gui.ihme.washington.edu/#/workflow/{workflow.workflow_id}") + + status = workflow.run(fail_fast=False) + logger.info(f"Workflow {workflow.workflow_id} completed with status {status}.") + + +def load_density_mosaic_tile( + feature_metadata: FeatureMetadata, + pm_data: PopulationModelData, + model_spec: ModelSpecification, +) -> rt.RasterArray: + tiles = [] + for buffer_block_key, bounds in feature_metadata.block_bounds.items(): + tile = pm_data.load_raked_prediction( + block_key=buffer_block_key, + model_spec=model_spec, + time_point=feature_metadata.time_point, + subset_bounds=bounds, + ) + area = ( + tile.to_gdf() + .area + .to_numpy() + .astype(np.float32) + .reshape(tile.shape) + ) + area = rt.RasterArray( + data=area, + transform=tile.transform, + crs=tile.crs, + no_data_value=np.nan, + ) + tile /= area # turn into population density + try: + tile = tile.reproject( + dst_crs=feature_metadata.working_crs, + dst_resolution=float(feature_metadata.resolution), + resampling="average", + ) + except ValueError: + # This is kind of a hack, but there"s not a clean way to fix it easily. + # The issue is that the resolution of the tiles do not exactly line up + # to the bounds of the world as defined by the CRS. The southernmost + # have one fewer row of pixels as a whole row would extend past the + # southern edge of the world, causing reprojection issues. The problem + # is that we read the tile with bounds, and those bounds cause the underlying + # rasterio to fill in that missing row. Here we just remove the last row + # of pixels and reproject the tile. + tile._ndarray = tile._ndarray[:-1].copy() # noqa: SLF001 + tile = tile.reproject( + dst_crs=feature_metadata.working_crs, + dst_resolution=float(feature_metadata.resolution), + resampling="average", + ) + tiles.append(tile) + + buffered_measure = utils.suppress_noise(rt.merge(tiles)) + + return buffered_measure + + +def load_poverty_rates(time_point: str, tile: rt.RasterArray) -> Dict[str, rt.RasterArray]: + year_id, quarter = [int(i) for i in time_point.split("q")] + + poverty_rates = pd.read_csv(POVERTY_RATES_PATH).set_index(["year_id", "location_id"]) + poverty_rates = poverty_rates.loc[:, ["urban_poverty_rate", "rural_poverty_rate"]] + poverty_rates = poverty_rates.rename(columns={"urban_poverty_rate": "urban", "rural_poverty_rate": "rural"}) + admin2_shapes = gpd.read_parquet(pmc.MODEL_ROOT / "admin-inputs" / "raking" / "gbd-inputs" / "shapes_lsae_1285_a2.parquet").set_index("location_id") + + year_id = min( + year_id, + poverty_rates.index.get_level_values("year_id").max() + ) + next_year_id = min( + year_id + 1, + poverty_rates.index.get_level_values("year_id").max() + ) + weight = (quarter - 1) / 4 + poverty_rates = ( + poverty_rates.loc[year_id] * (1 - weight) + + poverty_rates.loc[next_year_id] * weight + ) + poverty_rates = admin2_shapes.join(poverty_rates, how="left").fillna(0) + poverty_rates = poverty_rates.to_crs(tile.crs) + + tile_bounds = ( + tile.bounds[0], + tile.bounds[2], + tile.bounds[1], + tile.bounds[3], + ) + poverty_rates = ( + poverty_rates + .overlay( + gpd.GeoDataFrame({"geometry":[shapely.box(*tile_bounds)]}, crs=tile.crs), + how="intersection", + keep_geom_type=True, + ) + ) + poverty_rates_rasters = {} + for urbanicity in ["urban", "rural"]: + shapes = ( + (geom, val) for geom, val in zip(poverty_rates.geometry, poverty_rates[urbanicity]) + ) + poverty_rates_array = rasterize( + shapes=shapes, + out_shape=(tile.height, tile.width), + transform=tile.transform, + fill=0, + dtype=np.float32, + ) + poverty_rates_rasters[urbanicity] = rt.RasterArray( + np.where(tile.no_data_mask, tile.no_data_value, poverty_rates_array).astype(np.float32), + transform=tile.transform, + crs=tile.crs, + no_data_value=tile.no_data_value, + ) + + return poverty_rates_rasters + + +def worker( + resolution: str, + version: str, + block_key: str, + time_point: str, + radius: int, + people_per_km_threshold: int = 300, +): + pm_data = PopulationModelData() + bd_data = BuildingDensityData() + model_spec = pm_data.load_model_specification(resolution, version) + + logger.info("Loading all feature metadata") + feature_metadata = get_feature_metadata( + pm_data, bd_data, resolution, block_key, time_point + ) + if radius > max(pmc.FEATURE_AVERAGE_RADII): + raise ValueError(f"Specified radius ({radius}) is larger than max radius in module {max(pmc.FEATURE_AVERAGE_RADII)}.") + + logger.info("Loading buffered population density tile") + buffered_tile = load_density_mosaic_tile(feature_metadata, pm_data, model_spec) + + logger.info("Calculating spatial average") + average_measure = ( + utils.make_spatial_average( + tile=buffered_tile, + radius=radius, + kernel_type="uniform", + apply_floor=False, + ) + .resample_to(feature_metadata.block_template, "average") + .astype(np.float32) + ) + + tile = pm_data.load_raked_prediction( + block_key=block_key, + model_spec=model_spec, + time_point=feature_metadata.time_point, + ) + average_measure = rt.RasterArray( + np.where(tile.no_data_mask, tile.no_data_value, average_measure), + transform=tile.transform, + crs=tile.crs, + no_data_value=tile.no_data_value, + ) + + time_points = sorted(pm_data.list_raked_prediction_time_points(resolution, version)) + if time_point in [time_points[0], time_points[-1]]: + logger.info("Saving population density") + population_density_path = Path(model_spec.output_root) / "population_density" / time_point / block_key + if not population_density_path.exists(): + mkdir(population_density_path, parents=True, exist_ok=True) + save_raster( + average_measure, + population_density_path / f"{radius}m.tif" + ) + + logger.info("Defining and saving urban mask") + threshold = people_per_km_threshold / 1_000 ** 2 + urban = average_measure.to_numpy() >= threshold + rural = ~urban + urban = rt.RasterArray( + np.where(tile.no_data_mask, tile.no_data_value, urban).astype(np.float32), + transform=tile.transform, + crs=tile.crs, + no_data_value=tile.no_data_value, + ) + rural = rt.RasterArray( + np.where(tile.no_data_mask, tile.no_data_value, rural).astype(np.float32), + transform=tile.transform, + crs=tile.crs, + no_data_value=tile.no_data_value, + ) + urban_path = Path(model_spec.output_root) / "urban" / time_point / block_key + if not urban_path.exists(): + mkdir(urban_path, parents=True, exist_ok=True) + save_raster( + urban, + urban_path / f"{radius}m.tif" + ) + + logger.info("Calculating and saving impoverished population") + poverty_rates = load_poverty_rates(time_point, tile) + poverty = tile * (urban * poverty_rates["urban"] + rural * poverty_rates["rural"]) + poverty_path = Path(model_spec.output_root) / "poverty" / time_point / block_key + if not poverty_path.exists(): + mkdir(poverty_path, parents=True, exist_ok=True) + save_raster( + poverty, + poverty_path / f"{radius}m.tif" + ) + + +def runner(resolution: str, version: str, radii: List[int] = [1_000, 5_000]): + pm_data = PopulationModelData() + modeling_frame = pm_data.load_modeling_frame(resolution) + + block_keys = modeling_frame["block_key"].unique().tolist() + time_points = sorted(pm_data.list_raked_prediction_time_points(resolution, version)) + + logger.info(f"Estimating urbanicity and poverty for {len(block_keys) * len(time_points) * len(radii)} block-key, time-point, radius combinations.") + execute_workflow( + resolution=resolution, + version=version, + block_keys=block_keys, + time_points=time_points, + radii=radii, + ) + + +if __name__ == "__main__": + if sys.argv[1] == "runner": + runner( + resolution=sys.argv[2], + version=sys.argv[3], + ) + elif sys.argv[1] == "worker": + worker( + resolution=sys.argv[2], + version=sys.argv[3], + block_key=sys.argv[4], + time_point=sys.argv[5], + radius=int(sys.argv[6]), + ) diff --git a/scripts/save_early_access_countries.py b/scripts/save_early_access_countries.py new file mode 100644 index 0000000..c4dc041 --- /dev/null +++ b/scripts/save_early_access_countries.py @@ -0,0 +1,245 @@ +import sys +from pathlib import Path +from loguru import logger +from typing import List +import tqdm + +import numpy as np +import pandas as pd +import geopandas as gpd +import rasterra as rt +from affine import Affine + +from jobmon.client.tool import Tool +import uuid +import shutil + +from rra_tools.shell_tools import mkdir + +from rra_population_model.data import ( + PopulationModelData, + save_raster, +) +from rra_population_model.constants import CRSES + +COG_LOCATION_IDS = [ + ## countries that are flagged by the near-antimeridian logic but create problems + 23, # Kiribati + 413, # Tokelau +] + + +def workflow( + location_id_time_points: List[str], + resolution: str, + version: str, +): + wf_uuid = uuid.uuid4() + + tool = Tool(name='pop_ea_countries') + + workflow = tool.create_workflow( + name=f'pop_ea_countries_{wf_uuid}', + ) + + ## define templates + task_template = tool.get_task_template( + default_compute_resources={ + 'queue': 'all.q', + 'memory': '5G', + 'runtime': '3m', + # 'stdout': str(version_root / '_diagnostics' / 'logs' / 'output'), + # 'stderr': str(version_root / '_diagnostics' / 'logs' / 'error'), + 'project': 'proj_rapidresponse', + # 'constraints': 'archive', + }, + template_name='country_pop', + default_cluster_name='slurm', + command_template=f'{shutil.which("python")}' + f' {Path(__file__)}' + ' worker' + ' {location_id_time_point}' + ' {resolution}' + ' {version}', + node_args=['location_id_time_point'], + task_args=['resolution', 'version'], + op_args=[], + ) + + ## compile tasks + tasks = [] + for location_id_time_point in location_id_time_points: + tasks.append( + task_template.create_task( + max_attempts=6, + resource_scales={ + 'memory': iter([50 , 100 , 200 , 500 , 800 ]), + 'runtime': iter([4 * 60, 10 * 60, 20 * 60, 240 * 60, 360 * 60]), + }, + location_id_time_point=location_id_time_point, + resolution=resolution, + version=version, + ) + ) + + workflow.add_tasks(tasks) + workflow.bind() + + logger.info(f'Running workflow with ID {workflow.workflow_id}.') + logger.info('For full information see the Jobmon GUI:') + logger.info(f'https://jobmon-gui.ihme.washington.edu/#/workflow/{workflow.workflow_id}') + + status = workflow.run(fail_fast=False) + logger.info(f'Workflow {workflow.workflow_id} completed with status {status}.') + + +def runner(resolution: str, version: str): + hierarchy = pd.read_parquet( + "/mnt/team/rapidresponse/pub/population-model/admin-inputs/raking/gbd-inputs/hierarchy_gbd_2023.parquet" + ) + is_level_3 = hierarchy['level'] == 3 + location_ids = hierarchy.loc[is_level_3, 'location_id'].to_list() + ihme_loc_ids = hierarchy.loc[is_level_3, 'ihme_loc_id'].to_list() + + pm_data = PopulationModelData() + + output_root = pm_data.root / 'country_data' / f'{resolution}m' / version + mkdir(output_root, exist_ok=True) + for ihme_loc_id in ihme_loc_ids: + mkdir(output_root / ihme_loc_id, exist_ok=True) + + time_points = pm_data.list_compiled_prediction_time_points( + resolution, + version, + measure="population", + ) + time_points = [time_point for time_point in list(sorted(time_points)) if time_point != "2020q1"] + + possible = 0 + running = 0 + location_id_time_points = [] + for location_id, ihme_loc_id in zip(location_ids, ihme_loc_ids): + for time_point in time_points: + possible += 1 + output_path = output_root / ihme_loc_id / f'{time_point}.tif' + if not output_path.exists(): + location_id_time_points.append(f'{location_id}-{time_point}') + running += 1 + complete = possible - running + + logger.info(f'Running {running} location-time points ({complete} already complete).') + workflow( + location_id_time_points=location_id_time_points, + resolution=resolution, + version=version, + ) + + +def shift_to_antimeridian( + raster: rt.RasterArray, +) -> rt.RasterArray: + """Shift a global raster by half the world width (no resampling).""" + if raster.crs != CRSES["equal_area"].code: + raise ValueError("Transformation being applied to world cylindrical (equal area) only.") + shift_px = CRSES["equal_area"].bounds[0] # raster.width // 2 + target_crs = CRSES["equal_area_anti_meridian"].to_pyproj() + + if np.ptp(np.array(raster.bounds)[:2]) >= np.abs(shift_px) * 1.5: + raise ValueError("Raster crosses antimeridian") + + # Update affine transform x origin by half the world width + t = raster.transform + new_transform = Affine(t.a, t.b, t.c - shift_px, t.d, t.e, t.f) + + return rt.RasterArray( + raster.to_numpy(), + transform=new_transform, + crs=target_crs, + no_data_value=raster.no_data_value, + ) + + +def worker( + location_id_time_point: str, + resolution: str, + version: str, + buffer_size: int = 5000, +): + location_id, time_point = location_id_time_point.split('-') + location_id = int(location_id) + logger.info(f'{location_id} - {time_point}') + + logger.info('PREPARING METADATA') + pm_data = PopulationModelData() + model_spec = pm_data.load_model_specification(resolution, version) + modeling_frame = pm_data.load_modeling_frame(resolution) + output_root = pm_data.root / 'country_data' / f'{resolution}m' / version + + hierarchy = pd.read_parquet( + "/mnt/team/rapidresponse/pub/population-model/admin-inputs/raking/gbd-inputs/hierarchy_gbd_2023.parquet" + ) + ihme_loc_id = hierarchy.set_index('location_id').loc[location_id, 'ihme_loc_id'] + + output_path = output_root / ihme_loc_id / f'{time_point}.tif' + if not output_path.exists(): + logger.info('LOADING AND CREATING BUFFERED GEOMETRY') + shapes = gpd.read_parquet( + "/mnt/team/rapidresponse/pub/population-model/admin-inputs/raking/gbd-inputs/shapes_lsae_1285_a0.parquet" + ) + geometry = shapes.to_crs("ESRI:54034").set_index('location_id').loc[location_id, 'geometry'] + buffered_geometry = ( + gpd.GeoSeries(geometry) + .explode(index_parts=True) + .convex_hull.buffer(buffer_size) + .union_all() + ) + + if location_id in COG_LOCATION_IDS: + near_antimeridian = False + else: + block_key_x_max = modeling_frame["block_key"].apply(lambda x: int(x.split("X")[0][-4:])).max() + modeling_frame = modeling_frame.loc[modeling_frame.intersects(buffered_geometry)] + block_key_x = modeling_frame["block_key"].apply(lambda x: int(x.split("X")[0][-4:])) + boundary_blocks = 8 + near_antimeridian = ( + (block_key_x <= boundary_blocks) + | (block_key_x >= block_key_x_max - boundary_blocks) + ).any() + + if near_antimeridian: + logger.info('LOADING RAKED PREDICTION BLOCKS AND REPROJECTING DUE TO ANTIMERIDIAN PROXIMITY') + block_keys = modeling_frame["block_key"].unique().tolist() + raster = [] + for block_key in tqdm.tqdm(block_keys, total=len(block_keys)): + block_raster = pm_data.load_raked_prediction( + block_key, time_point, model_spec + ) + block_raster = block_raster.clip(geometry).mask(geometry) + block_raster = shift_to_antimeridian(block_raster) + raster.append(block_raster) + raster = rt.merge(raster) + else: + logger.info('LOADING COMPILED COGs') + raster = rt.load_raster( + pm_data.compiled_prediction_vrt_path(time_point, model_spec, measure="population"), + buffered_geometry.bounds, + ).clip(geometry).mask(geometry) + + logger.info('SAVING COUNTRY RASTER') + save_raster(raster, output_path) + else: + logger.info('COUNTRY RASTER ALREADY EXISTS') + + +if __name__ == '__main__': + if sys.argv[1] == 'runner': + runner( + resolution=sys.argv[2], + version=sys.argv[3], + ) + elif sys.argv[1] == 'worker': + worker( + location_id_time_point=sys.argv[2], + resolution=sys.argv[3], + version=sys.argv[4], + ) diff --git a/scripts/save_fieldmaps_masks.py b/scripts/save_fieldmaps_masks.py new file mode 100644 index 0000000..1a67af5 --- /dev/null +++ b/scripts/save_fieldmaps_masks.py @@ -0,0 +1,151 @@ +import tqdm + +import pandas as pd +import numpy as np + +import geopandas as gpd +import rasterra as rt +import contextily as ctx +import rasterio as rio + +import matplotlib.pyplot as plt +from matplotlib.backends.backend_pdf import PdfPages + +import rra_population_model.constants as pmc +from rra_population_model.data import PopulationModelData, save_raster + +FIELDMAPS_ROOT = pmc.MODEL_ROOT / 'admin-inputs' / 'itu-masks' / 'fieldmaps' +FIELDMAPS_FILES = { + 'humanitarian': 'adm1_polygons_UNCOD_geoBoundaries.parquet', + 'open': 'adm1_polygons_geoBoundaries.parquet', +} + + +def load_data(dataset_name: str, pm_data: PopulationModelData): + itu_caribbean = pd.read_csv(FIELDMAPS_ROOT / 'datasets' / 'itu_caribbean.csv') + + data = gpd.read_parquet(FIELDMAPS_ROOT / 'datasets' / FIELDMAPS_FILES[dataset_name]) + + data_caribbean = data.loc[data['iso_2'].isin(itu_caribbean['Iso2Code'])] + data_caribbean = data_caribbean.loc[ + # multiple rows for Jamaica, just keep matching admin0 + (data_caribbean['iso_2'] != 'JM') | (data_caribbean['adm0_name'] == 'Jamaica') + ] + data_caribbean = data_caribbean.loc[:, ['adm0_name', 'iso_2', 'iso_3', 'geometry']].dissolve(by=['adm0_name', 'iso_2', 'iso_3']) + + missing = [i for i in itu_caribbean['Iso2Code'] if i not in data_caribbean.index.get_level_values('iso_2')] + if missing: + raise ValueError(f"Missing the following countries: {','.join(missing)}") + + itu_iso3s = pm_data.list_itu_iso3s() + caribbean_iso3s = data_caribbean.index.get_level_values('iso_3').to_list() + itu_iso3s = [i for i in itu_iso3s if i not in caribbean_iso3s] + + data_other = data.loc[data['iso_3'].isin(itu_iso3s)].loc[:, ['adm0_name', 'iso_2', 'iso_3', 'geometry']].dissolve(by=['adm0_name', 'iso_2', 'iso_3']) + + data = pd.concat([data_other, data_caribbean]) + + return data + + +def polygon_to_raster_mask(iso3, gdf, resolution, pm_data): + """ + Creates a binary raster mask from a polygon with a specific resolution. + + Args: + gdf (gpd.GeoDataFrame): GeoDataFrame containing the polygon(s). + resolution (int): The desired resolution in meters. + output_file (str): The path for the output GeoTIFF file. + """ + if iso3 in pm_data.list_itu_iso3s(): + # Use existing mask as template for resolution and extent + template_mask = pm_data.load_itu_mask(iso3) + target_crs = template_mask.crs + else: + # Default to Web Mercator and a reasonable resolution + template_mask = None + target_crs = "EPSG:3857" + + gdf = gdf.loc[:, :, iso3].to_crs(target_crs) + + # Ensure the GeoDataFrame has a projected CRS with meters as units + if gdf.crs is None or gdf.crs.is_geographic: + raise ValueError("Input GeoDataFrame must have a projected CRS (like UTM).") + + # Get the bounding box of the polygon(s) + minx, miny, maxx, maxy = (float(bound) for bound in gdf.total_bounds) + + # Calculate raster dimensions based on resolution + width = int(np.ceil((maxx - minx) / resolution)) + height = int(np.ceil((maxy - miny) / resolution)) + + # Define the georeferencing transform + transform = rio.transform.from_bounds(minx, miny, minx + width * resolution, miny + height * resolution, width, height) + + # Get shapes and values for rasterization + shapes = ((geom, 1.0) for geom in gdf.geometry) + + # Create the binary mask as a NumPy array + binary_mask = rio.features.rasterize( + shapes=shapes, + out_shape=(height, width), + transform=transform, + fill=0.0, + all_touched=True, + dtype=np.float32 + ) + + raster_mask = rt.RasterArray( + data=binary_mask, + transform=transform, + crs=target_crs, + no_data_value=0.0, + ) + + return template_mask, raster_mask + + +def main(): + pm_data = PopulationModelData() + + data_humanitarian = load_data("humanitarian", pm_data) + + with PdfPages(FIELDMAPS_ROOT / "datasets" / "mask_update.pdf") as pdf: + for iso3 in tqdm.tqdm(data_humanitarian.index.get_level_values('iso_3'), total=len(data_humanitarian)): + template_mask, raster_mask = polygon_to_raster_mask( + iso3=iso3, + gdf=data_humanitarian, + resolution=100, + pm_data=pm_data, + ) + save_raster(raster_mask, FIELDMAPS_ROOT / f"{iso3}.tif") + + if template_mask is not None: + diff = rt.RasterArray( + (raster_mask.resample_to(template_mask).to_numpy() - template_mask.to_numpy()), + transform=template_mask.transform, + crs=template_mask.crs, + no_data_value=0.0 + ) + + fig, ax = plt.subplots(1, 3, figsize=(16, 9), sharex=True, sharey=True) + template_mask.plot(ax=ax[0], alpha=0.8) + raster_mask.plot(ax=ax[1], alpha=0.8) + diff.plot(ax=ax[2], alpha=0.8) + if raster_mask.crs != "EPSG:3832": + ctx.add_basemap(ax[0], crs=raster_mask.crs, source=ctx.providers.Esri.WorldImagery, alpha=0.6, attribution=False) + ctx.add_basemap(ax[1], crs=raster_mask.crs, source=ctx.providers.Esri.WorldImagery, alpha=0.6, attribution=False) + ctx.add_basemap(ax[2], crs=raster_mask.crs, source=ctx.providers.Esri.WorldImagery, alpha=0.6, attribution=False) + ax[0].set_title('Old mask') + ax[1].set_title( + f'{iso3}' + '\nNew mask' + ) + ax[2].set_title('Diff') + fig.tight_layout() + pdf.savefig(fig) # saves the current figure into a pdf page + plt.close() + + +if __name__ == "__main__": + main() diff --git a/scripts/urbanicity_admin.py b/scripts/urbanicity_admin.py new file mode 100644 index 0000000..5888d36 --- /dev/null +++ b/scripts/urbanicity_admin.py @@ -0,0 +1,176 @@ +import sys +from pathlib import Path +from typing import List +from loguru import logger +import tqdm + +import numpy as np +import rasterra as rt +from rasterio.windows import WindowError +import pandas as pd +import geopandas as gpd + +from jobmon.client.tool import Tool +import uuid +import shutil +from rra_tools.shell_tools import mkdir + +from rra_population_model import constants as pmc +from rra_population_model.data import PopulationModelData + +LSVID = 1285 +SCRATCH_DIR = "2025-12-03-urban_rural_populations" + + +def execute_workflow( + resolution: str, + version: str, + location_ids: List[int], + radius: int, +): + wf_uuid = uuid.uuid4() + + tool = Tool(name="urban_rural_admin") + + workflow = tool.create_workflow( + name=f"urban_rural_admin_{wf_uuid}", + ) + + ## define templates + task_template = tool.get_task_template( + default_compute_resources={ + "queue": "all.q", + "cores": 1, + "memory": "12G", + "runtime": "4m", + "project": "proj_rapidresponse", + }, + template_name="urban_rural_admin", + default_cluster_name="slurm", + command_template=f"{shutil.which('python')}" + f" {Path(__file__)}" + " worker" + " {resolution}" + " {version}" + " {location_id}" + " {radius}", + node_args=["location_id"], + task_args=["resolution", "version", "radius"], + op_args=[], + ) + + ## compile tasks + tasks = [] + for location_id in location_ids: + tasks.append( + task_template.create_task( + max_attempts=4, + resource_scales={ + "memory": iter([24 , 36 , 300 ]), + "runtime": iter([8 * 60, 12 * 60, 60 * 60]), + }, + location_id=location_id, + resolution=resolution, + version=version, + radius=radius, + ) + ) + + workflow.add_tasks(tasks) + workflow.bind() + + logger.info(f"Running workflow with ID {workflow.workflow_id}.") + logger.info("For full information see the Jobmon GUI:") + logger.info(f"https://jobmon-gui.ihme.washington.edu/#/workflow/{workflow.workflow_id}") + + status = workflow.run(fail_fast=False) + logger.info(f"Workflow {workflow.workflow_id} completed with status {status}.") + + +def worker( + resolution: str, + version: str, + location_id: int, + radius: int, + buffer_size: int = 5_000, +): + pm_data = PopulationModelData() + model_spec = pm_data.load_model_specification(resolution, version) + modeling_frame = pm_data.load_modeling_frame(resolution) + shapes = gpd.read_parquet(f"/mnt/team/rapidresponse/pub/population-model/admin-inputs/raking/gbd-inputs/shapes_lsae_{LSVID}_a2.parquet") + geometry = shapes.loc[shapes['location_id'] == location_id].geometry.to_crs(modeling_frame.crs) + buffered_geometry = ( + geometry + .explode(index_parts=True) + .convex_hull.buffer(buffer_size) + .union_all() + ) + block_keys = modeling_frame.loc[modeling_frame.intersects(buffered_geometry), 'block_key'].unique().tolist() + + time_points = sorted(pm_data.list_raked_prediction_time_points(resolution, version)) + urbanicity = [] + for time_point in tqdm.tqdm(time_points, total=len(time_points)): + for block_key in block_keys: + try: + population = rt.load_raster( + pm_data.raked_prediction_path(block_key, time_point, model_spec) + ).clip(geometry).mask(geometry) + urban_mask = rt.load_raster( + pm_data.model_version_root(resolution, version) / 'urban' / time_point / block_key / f"{radius}m.tif" + ).clip(geometry).mask(geometry) + urbanicity.append( + pd.DataFrame( + { + "urban": np.nansum(population * urban_mask), + "rural": np.nansum(population) - np.nansum(population * urban_mask), + }, + index=pd.MultiIndex.from_tuples([(location_id, block_key, time_point)], names=['location_id', 'block_key', 'time_point']) + ) + ) + except WindowError: + continue + + urbanicity = pd.concat(urbanicity) + urbanicity = urbanicity.groupby(['location_id', 'time_point']).sum() + urbanicity.to_parquet( + pmc.MODEL_ROOT / "scratch" / SCRATCH_DIR / f"{location_id}.parquet" + ) + + +def runner(resolution: str, version: str, overwrite: bool, radius: int = 1_000): + hierarchy = pd.read_parquet(f"/mnt/team/rapidresponse/pub/population-model/admin-inputs/raking/gbd-inputs/hierarchy_lsae_{LSVID}.parquet") + location_ids = hierarchy.loc[hierarchy['most_detailed'] == 1, 'location_id'].to_list() + + if not overwrite: + logger.info("Identifying unwritten location_id files.") + location_ids = [ + location_id for location_id in tqdm.tqdm(location_ids, total=len(location_ids)) + if not (pmc.MODEL_ROOT / "scratch" / SCRATCH_DIR / f"{location_id}.parquet").exists() + ] + + logger.info(f"Calculating urban/rural populations for {len(location_ids)} admin2 locations.") + mkdir(pmc.MODEL_ROOT / "scratch" / SCRATCH_DIR, exist_ok=True) + execute_workflow( + resolution=resolution, + version=version, + location_ids=location_ids, + radius=radius, + ) + + +if __name__ == "__main__": + if sys.argv[1] == "runner": + if sys.argv[4] not in ["True", "False"]: + raise ValueError(f"Invalid overwrite flag: {sys.argv[4]}") + runner( + resolution=sys.argv[2], + version=sys.argv[3], + overwrite=sys.argv[4] == "True", + ) + elif sys.argv[1] == "worker": + worker( + resolution=int(sys.argv[2]), + version=sys.argv[3], + location_id=int(sys.argv[4]), + radius=int(sys.argv[5]), + ) diff --git a/specifications/model_specification.yaml b/specifications/model_specification.yaml index e66d578..99fe4fc 100644 --- a/specifications/model_specification.yaml +++ b/specifications/model_specification.yaml @@ -1,13 +1,13 @@ -resolution: "100" +resolution: "40" split: 0 -denominator: "microsoft_v7_residential_volume" +denominator: "microsoft_v7_1_residential_volume" features: - # - "microsoft_v7_residential_volume_250m" - # - "microsoft_v7_residential_volume_500m" - # - "microsoft_v7_residential_volume_1000m" - # - "microsoft_v7_residential_volume_2500m" - # - "microsoft_v7_residential_volume_5000m" - # - "microsoft_v7_residential_volume_10000m" + - "microsoft_v7_residential_volume_250m" + - "microsoft_v7_residential_volume_500m" + - "microsoft_v7_residential_volume_1000m" + - "microsoft_v7_residential_volume_2500m" + - "microsoft_v7_residential_volume_5000m" + - "microsoft_v7_residential_volume_10000m" training_target: "admin_log_occupancy_rate" loss_target: "admin_log_occupancy_rate" loss_metric: "mse" diff --git a/src/rra_population_model/cli_options.py b/src/rra_population_model/cli_options.py index 6a8502d..a21f2fc 100644 --- a/src/rra_population_model/cli_options.py +++ b/src/rra_population_model/cli_options.py @@ -106,6 +106,20 @@ def with_ntl_option[**P, T]( ) +def with_ga_option[**P, T]( + *, + allow_all: bool = False, +) -> Callable[[Callable[P, T]], Callable[P, T]]: + return with_choice( + "ga_option", + "g", + allow_all=allow_all, + choices=["none", "all"] + [f"{far}m" for far in pmc.FEATURE_AVERAGE_RADII], + help="Geospatial averages of building data to include.", + convert=allow_all, + ) + + def with_version[**P, T]() -> Callable[[Callable[P, T]], Callable[P, T]]: return click.option( "--version", @@ -144,6 +158,20 @@ def with_tile_key[**P, T]() -> Callable[[Callable[P, T]], Callable[P, T]]: ) +def with_purpose[**P, T]( + choices: Collection[str] = pmc.DATA_PURPOSES, + *, + allow_all: bool = False, +) -> Callable[[Callable[P, T]], Callable[P, T]]: + return with_choice( + "purpose", + allow_all=allow_all, + choices=choices, + help="Data purpose to run.", + convert=allow_all, + ) + + __all__ = [ "RUN_ALL", "convert_choice", @@ -158,6 +186,7 @@ def with_tile_key[**P, T]() -> Callable[[Callable[P, T]], Callable[P, T]]: "with_output_directory", "with_overwrite", "with_progress_bar", + "with_purpose", "with_queue", "with_resolution", "with_tile_key", diff --git a/src/rra_population_model/constants.py b/src/rra_population_model/constants.py index 542c26b..92f797a 100644 --- a/src/rra_population_model/constants.py +++ b/src/rra_population_model/constants.py @@ -27,7 +27,7 @@ def to_list(cls) -> list[str]: class BuiltVersion(BaseModel): provider: Literal["ghsl", "microsoft"] - version: Literal["v6", "v7", "v7_1", "r2023a"] + version: Literal["v7_1", "v8", "r2023a"] time_points: list[str] measures: list[str] @@ -54,43 +54,51 @@ def time_points_float(self) -> list[float]: "height", "proportion_residential", "density", - "residential_density", - "nonresidential_density", + # "residential_density", + # "nonresidential_density", "volume", "residential_volume", - "nonresidential_volume", + # "nonresidential_volume", ], ), - "microsoft_v6": BuiltVersion( - provider="microsoft", - version="v6", - time_points=[ - f"{y}q{q}" for y, q in itertools.product(range(2020, 2024), range(1, 5)) - ][1:], - measures=["density"], - ), - "microsoft_v7": BuiltVersion( + "microsoft_v7_1": BuiltVersion( provider="microsoft", - version="v7", + version="v7_1", time_points=[ - f"{y}q{q}" for y, q in itertools.product(range(2020, 2024), range(1, 5)) - ][1:], + f"{y}q{q}" for y, q in itertools.product(range(2020, 2026), range(1, 5)) + ][1:-2], measures=[ "density", "height", ], ), - "microsoft_v7_1": BuiltVersion( + "microsoft_v8": BuiltVersion( provider="microsoft", - version="v7_1", + version="v8", time_points=[ - f"{y}q{q}" for y, q in itertools.product(range(2020, 2025), range(1, 5)) - ][1:-2], + f"{y}q{q}" for y, q in itertools.product(range(2020, 2026), range(1, 5)) + ][1:-1], measures=[ "density", "height", ], ), + # "microsoft_v7_1_d": BuiltVersion( + # provider="microsoft", + # version="v7_1_d", + # time_points=[ + # f"{y}q{q}" for y, q in itertools.product(range(2020, 2025), range(1, 5)) + # ][1:-2], + # measures=["density"], + # ), + # "microsoft_v7_1_h": BuiltVersion( + # provider="microsoft", + # version="v7_1_h", + # time_points=[ + # f"{y}q{q}" for y, q in itertools.product(range(2020, 2025), range(1, 5)) + # ][1:-2], + # measures=["height"], + # ), } DENOMINATORS = [] @@ -98,11 +106,15 @@ def time_points_float(self) -> list[float]: for denominator in [ "density", "volume", - "residential_density", + # "residential_density", "residential_volume", ]: DENOMINATORS.append(f"{built_version.name}_{denominator}") # noqa: PERF401 +DATA_PURPOSES = [ + 'training', + 'inference', +] FEATURE_AVERAGE_RADII = [ 100, @@ -113,6 +125,10 @@ def time_points_float(self) -> list[float]: 10000, ] +BUILT_VERSION_TIME_POINTS = sorted( + set.union(*[set(v.time_points) for v in BUILT_VERSIONS.values()]) +) + ALL_TIME_POINTS = sorted( set.union(*[set(v.time_points) for v in BUILT_VERSIONS.values()]) | {f"{y}q1" for y in range(1975, 2026)} diff --git a/src/rra_population_model/data.py b/src/rra_population_model/data.py index 7e98f56..e7bbf06 100644 --- a/src/rra_population_model/data.py +++ b/src/rra_population_model/data.py @@ -311,13 +311,14 @@ def census(self) -> Path: def census_path(self, iso3: str, year: str) -> Path: return self.census / f"{iso3}_{year}.parquet" - def list_census_data(self) -> list[tuple[str, str]]: + def list_census_data(self) -> list[tuple[str, str, str]]: census_data = [] for path in self.census.glob("*.parquet"): # Some iso3 codes have underscores (e.g. GBR subnats) *iso3_parts, year = path.stem.split("_") iso3 = "_".join(iso3_parts) - census_data.append((iso3, year)) + quarter = '1' + census_data.append((iso3, year, quarter)) return census_data def save_census_data(self, gdf: gpd.GeoDataFrame, iso3: str, year: str) -> None: @@ -550,6 +551,11 @@ def save_tile_training_data( root = self.tile_training_data_root(resolution) / tile_key mkdir(root, exist_ok=True) + for measure, raster in tile_rasters.items(): + raster_path = root / f"{measure}.tif" + touch(raster_path, clobber=True) + save_raster(raster, raster_path) + gdf_path = root / "people_per_structure.parquet" touch(gdf_path, clobber=True) tile_gdf.to_parquet(gdf_path) @@ -558,11 +564,6 @@ def save_tile_training_data( touch(paw_path, clobber=True) tile_area_weights.to_parquet(paw_path) - for measure, raster in tile_rasters.items(): - raster_path = root / f"{measure}.tif" - touch(raster_path, clobber=True) - save_raster(raster, raster_path) - def save_summary_people_per_structure( self, data: pd.DataFrame, @@ -573,6 +574,27 @@ def save_summary_people_per_structure( touch(path, clobber=True) data.to_parquet(path) + def inference_data_root(self, resolution: str) -> Path: + return self.resolution_root(resolution) / "inference-data" + + def tile_inference_data_root(self, resolution: str) -> Path: + return self.inference_data_root(resolution) / "tiles" + + def save_tile_inference_data( + self, + resolution: str, + time_point: str, + tile_key: str, + tile_rasters: dict[str, rt.RasterArray], + ) -> None: + root = self.tile_inference_data_root(resolution) / time_point / tile_key + mkdir(root, exist_ok=True, parents=True) + + for measure, raster in tile_rasters.items(): + raster_path = root / f"{measure}.tif" + touch(raster_path, clobber=True) + save_raster(raster, raster_path) + def load_people_per_structure( self, resolution: str, tile_key: str | None = None ) -> gpd.GeoDataFrame: @@ -607,6 +629,19 @@ def load_tile_training_data( path = self.tile_training_data_root(resolution) / tile_key / f"{measure}.tif" return rt.load_raster(path) + def load_tile_inference_data( + self, + resolution: str, + tile_key: str, + time_point: str, + measure: str, + ) -> rt.RasterArray: + path = self.tile_inference_data_root(resolution) / time_point / tile_key / f"{measure}.tif" + if path.exists(): + return rt.load_raster(path) + else: + return None + def model_root(self, resolution: str) -> Path: return self.resolution_root(resolution) / "models" @@ -860,54 +895,65 @@ def load_raked_prediction( block_key: str, time_point: str, model_spec: "ModelSpecification", + subset_bounds: shapely.Polygon | None = None, ) -> rt.RasterArray: path = self.raked_prediction_path(block_key, time_point, model_spec) - return rt.load_raster(path) + raster = rt.load_raster(path, subset_bounds) + return raster - def compiled_predictions_root(self, resolution: str, version: str) -> Path: - return self.model_version_root(resolution, version) / "compiled_predictions" + def compiled_predictions_root(self, resolution: str, version: str, measure: str = "") -> Path: + return self.model_version_root(resolution, version) / "compiled_predictions" / measure def compiled_prediction_path( - self, group_key: str, time_point: str, model_spec: "ModelSpecification" + self, group_key: str, time_point: str, model_spec: "ModelSpecification", measure: str = "" ) -> Path: resolution = model_spec.resolution version = model_spec.model_version return ( - self.compiled_predictions_root(resolution, version) + self.compiled_predictions_root(resolution, version, measure) / time_point / f"{group_key}.tif" ) def compiled_prediction_vrt_path( - self, time_point: str, model_spec: "ModelSpecification" + self, time_point: str, model_spec: "ModelSpecification", measure: str = "" ) -> Path: resolution = model_spec.resolution version = model_spec.model_version return ( - self.compiled_predictions_root(resolution, version) + self.compiled_predictions_root(resolution, version, measure) / time_point / "index.vrt" ) def list_compiled_prediction_time_points( - self, resolution: str, version: str + self, resolution: str, version: str, measure: str = "" ) -> list[str]: return [ p.name - for p in self.compiled_predictions_root(resolution, version).iterdir() + for p in self.compiled_predictions_root(resolution, version, measure).iterdir() if p.is_dir() ] + def list_compiled_prediction_time_point_group_keys( + self, resolution: str, version: str, time_point: str, measure: str = "" + ) -> list[str]: + return [ + p.name.replace(".tif", "") + for p in (self.compiled_predictions_root(resolution, version, measure) / time_point).glob("*.tif") + ] + def save_compiled_prediction( self, raster: rt.RasterArray, group_key: str, time_point: str, model_spec: "ModelSpecification", + measure: str = "", **save_kwargs: Any, ) -> None: - path = self.compiled_prediction_path(group_key, time_point, model_spec) - mkdir(path.parent, exist_ok=True) + path = self.compiled_prediction_path(group_key, time_point, model_spec, measure) + mkdir(path.parent, exist_ok=True, parents=True) save_raster_to_cog(raster, path, **save_kwargs) def load_compiled_prediction( @@ -915,8 +961,9 @@ def load_compiled_prediction( group_key: str, time_point: str, model_spec: "ModelSpecification", + measure: str = "", ) -> rt.RasterArray: - path = self.compiled_prediction_path(group_key, time_point, model_spec) + path = self.compiled_prediction_path(group_key, time_point, model_spec, measure) return rt.load_raster(path) def validation_root(self, resolution: str, version: str) -> Path: diff --git a/src/rra_population_model/model/inference/runner.py b/src/rra_population_model/model/inference/runner.py index 5581f91..06507cd 100644 --- a/src/rra_population_model/model/inference/runner.py +++ b/src/rra_population_model/model/inference/runner.py @@ -130,15 +130,27 @@ def inference( pm_data = PopulationModelData(output_dir) feature_time_points = pm_data.list_feature_time_points(resolution) time_points = clio.convert_choice(time_point, feature_time_points) - time_points = sorted( - [time_point for time_point in time_points if time_point.startswith("202")] - ) + model_spec = pm_data.load_model_specification(resolution, version) + if model_spec.denominator.startswith("microsoft"): + time_points = sorted( + [time_point for time_point in time_points if time_point.startswith("202")] + ) + elif model_spec.denominator.startswith("ghsl"): + time_points = sorted( + [time_point for time_point in time_points if time_point.endswith("q1")] + ) + else: + msg = f"Unexpected denominator: {model_spec.denominator}" + raise ValueError(msg) + # time_points = ["2020q1", "2020q2"] + # versions = [f"2025_11_08.0{(i + 1):02d}" for i in range(60)] print(f"Running inference for {len(time_points)} time points.") jobmon.run_parallel( runner="pmtask model", task_name="inference", node_args={ + # "version": versions, "time-point": time_points, }, task_args={ @@ -148,9 +160,10 @@ def inference( }, task_resources={ "queue": queue, - "memory": "20G", - "runtime": "480m", + "memory": "40G", + "runtime": "720m", "project": "proj_rapidresponse", }, log_root=pm_data.log_dir("model_inference"), + max_attempts=2, ) diff --git a/src/rra_population_model/model/train/runner.py b/src/rra_population_model/model/train/runner.py index 636ad15..b451807 100644 --- a/src/rra_population_model/model/train/runner.py +++ b/src/rra_population_model/model/train/runner.py @@ -1,4 +1,5 @@ import itertools +import shutil import click from lightning import Trainer @@ -23,27 +24,39 @@ def train_main( version: str, denominator: str, ntl_option: str, + ga_option: str, output_root: str, *, verbose: bool = False, ) -> None: pm_data = PopulationModelData(output_root) + version_root = pm_data.model_version_root(resolution, version) + if version_root.exists(): + shutil.rmtree(version_root) + # First generate the model specification and mint a new version directory print("Setting up model specification") - version_root = pm_data.model_version_root(resolution, version) ntl_feature = { "none": [], "ntl": ["nighttime_lights"], "log_ntl": ["log_nighttime_lights"], }[ntl_option] - bd_features: list[str] = [] + + ga_features = { + "none": [], + "all": [f"{denominator}_{far}m" for far in pmc.FEATURE_AVERAGE_RADII], + } + for far in pmc.FEATURE_AVERAGE_RADII: + ga_features[f"{far}m"] = [f"{denominator}_{far}m"] + ga_features = ga_features[ga_option] + model_spec = ModelSpecification( model_version=version, model_root=str(pm_data.root), output_root=str(version_root), denominator=denominator, resolution=resolution, - features=[*bd_features, *ntl_feature], + features=[*ga_features, *ntl_feature], ) pm_data.save_model_specification(model_spec) @@ -81,6 +94,7 @@ def train_main( @clio.with_version() @clio.with_denominator() @clio.with_ntl_option() +@clio.with_ga_option() @clio.with_output_directory(pmc.MODEL_ROOT) @clio.with_verbose() def train_task( @@ -88,11 +102,12 @@ def train_task( version: str, denominator: str, ntl_option: str, + ga_option: str, output_dir: str, verbose: bool, ) -> None: train_main( - resolution, version, denominator, ntl_option, output_dir, verbose=verbose + resolution, version, denominator, ntl_option, ga_option, output_dir, verbose=verbose ) @@ -100,28 +115,58 @@ def train_task( @clio.with_resolution() @clio.with_denominator(allow_all=True) @clio.with_ntl_option(allow_all=True) +@clio.with_ga_option(allow_all=True) @clio.with_output_directory(pmc.MODEL_ROOT) @clio.with_queue() def train( resolution: str, denominator: list[str], ntl_option: list[str], + ga_option: list[str], output_dir: str, queue: str, ) -> None: + # ########################## + # ## BATCH RUN + # built_versions = [ + # "ghsl_r2023a", + # "microsoft_v7_1", + # "microsoft_v7_1_d", + # "microsoft_v7_1_h", + # ] + # built_measures = [ + # "density", + # "volume", + # "residential_volume", + # ] + # denominator = [ + # f"{bv}_{bm}" for bv, bm in itertools.product(built_versions, built_measures) + # ] + # denominator = [d for d in denominator if d not in ["microsoft_v7_1_d_density", "microsoft_v7_1_h_density"]] + # ntl_option = [ + # "none", + # "ntl", + # "log_ntl", + # ] + # ga_option = [ + # "none", + # "all", + # ] + # ########################## + pm_data = PopulationModelData(output_dir) today, last_version = utils.get_last_run_version(pm_data.model_root(resolution)) node_args = [] - for i, (denom, ntl) in enumerate(itertools.product(denominator, ntl_option)): + for i, (denom, ntl, ga) in enumerate(itertools.product(denominator, ntl_option, ga_option)): version = f"{today}.{last_version + i + 1:03d}" - print(f"{version}: {denom} {ntl}") - node_args.append((version, denom, ntl)) + print(f"{version}: {denom} {ntl} {ga}") + node_args.append((version, denom, ntl, ga)) jobmon.run_parallel( runner="pmtask model", task_name="train", flat_node_args=( - ("version", "denominator", "ntl-option"), + ("version", "denominator", "ntl-option", "ga-option"), node_args, ), task_args={ @@ -131,10 +176,10 @@ def train( task_resources={ "queue": queue, "cores": 1, - "memory": "240G", - "runtime": "150m", + "memory": "320G", + "runtime": "960m", "project": "proj_rapidresponse", }, log_root=pm_data.log_dir("model_train"), - max_attempts=1, + max_attempts=2, ) diff --git a/src/rra_population_model/model_prep/__init__.py b/src/rra_population_model/model_prep/__init__.py index 142851e..76b206f 100644 --- a/src/rra_population_model/model_prep/__init__.py +++ b/src/rra_population_model/model_prep/__init__.py @@ -1,6 +1,8 @@ from rra_population_model.model_prep.features.runner import ( features, features_task, + geospatial_average_features, + geospatial_average_features_task, ) from rra_population_model.model_prep.modeling_frame.runner import ( modeling_frame, @@ -13,11 +15,13 @@ RUNNERS = { "modeling_frame": modeling_frame, "features": features, + "geospatial_average_features": geospatial_average_features, "training_data": training_data, } TASK_RUNNERS = { "modeling_frame": modeling_frame, "features": features_task, + "geospatial_average_features": geospatial_average_features_task, "training_data": training_data_task, } diff --git a/src/rra_population_model/model_prep/features/built.py b/src/rra_population_model/model_prep/features/built.py index 361b8db..e309fc9 100644 --- a/src/rra_population_model/model_prep/features/built.py +++ b/src/rra_population_model/model_prep/features/built.py @@ -148,14 +148,14 @@ def generate_geospatial_averages( ) -> dict[str, Path]: out_paths = {} for feature in features: + print(f"Processing {feature}...") + buffered_measure = mosaic_tile( + measure=feature, + feature_metadata=self.feature_metadata, + pm_data=pm_data, + ) for radius in feature_average_radii: - print(f"Processing {feature} with radius {radius}m.") - buffered_measure = mosaic_tile( - measure=feature, - feature_metadata=self.feature_metadata, - pm_data=pm_data, - ) - print(f"Processing {feature} with radius {radius}m.") + print(f" ... with radius {radius}m.") average_measure = ( utils.make_spatial_average( tile=buffered_measure, @@ -170,7 +170,6 @@ def generate_geospatial_averages( feature_name=f"{feature}_{radius}m", **self.feature_metadata.shared_kwargs, ) - out_paths[f"{feature}_{radius}m"] = pm_data.feature_path( feature_name=f"{feature}_{radius}m", **self.feature_metadata.shared_kwargs, @@ -265,21 +264,37 @@ def _generate_microsoft_derived_measures( "density": "microsoft_v6_density", "height": "ghsl_r2023a_height", "p_residential": "ghsl_r2023a_proportion_residential", + "reference_density": "ghsl_r2023a_density", }, "microsoft_v7": { "density": "microsoft_v7_density", "height": "microsoft_v7_height", "p_residential": "ghsl_r2023a_proportion_residential", + "reference_density": "ghsl_r2023a_density", }, "microsoft_v7_1": { "density": "microsoft_v7_1_density", "height": "microsoft_v7_1_height", "p_residential": "ghsl_r2023a_proportion_residential", + "reference_density": "ghsl_r2023a_density", }, - "microsoft_v7_e101": { - "density": "microsoft_v7_e101_density", - "height": "microsoft_v7_e101_height", + "microsoft_v7_1_d": { + "density": "microsoft_v7_1_d_density", + "height": "ghsl_r2023a_height", "p_residential": "ghsl_r2023a_proportion_residential", + "reference_density": "ghsl_r2023a_density", + }, + "microsoft_v7_1_h": { + "density": "ghsl_r2023a_density", + "height": "microsoft_v7_1_h_height", + "p_residential": "ghsl_r2023a_proportion_residential", + "reference_density": "ghsl_r2023a_density", + }, + "microsoft_v8": { + "density": "microsoft_v8_density", + "height": "microsoft_v8_height", + "p_residential": "ghsl_r2023a_proportion_residential", + "reference_density": "ghsl_r2023a_density", }, }[built_version_name] density = pm_data.load_feature( @@ -296,23 +311,36 @@ def _generate_microsoft_derived_measures( **feature_metadata.shared_kwargs, )._ndarray - # Since we're crosswalking, ensure we have height wherever - # there is density, even if GHSL doesn't think there is density. - height_min = HEIGHT_MIN - if (height_arr > 0).any(): - height_min = float(np.nanmin(height_arr[height_arr > 0])) - density_threshold = 0.01 - density_is_positive = density_arr >= density_threshold - height_is_zero = height_arr == 0 - height_arr[density_is_positive & height_is_zero] = height_min + # CROSSWALKING PROCEDURES + # 1) ensure we have height wherever there is density, even if GHSL doesn't think there is density + if feature_dict["density"].replace("_density", "") != feature_dict["height"].replace("_height", ""): + height_min = HEIGHT_MIN + if (height_arr > 0).any(): + height_min = float(np.nanmin(height_arr[height_arr > 0])) + density_threshold = 0.01 + density_is_positive = density_arr >= density_threshold + height_is_zero = height_arr == 0 + height_arr[density_is_positive & height_is_zero] = height_min + + # 2) if Microsoft places buildings somewhere GHSL does not, call them residential + if "reference_density" in feature_dict.keys(): + reference_density_arr = pm_data.load_feature( + feature_name=feature_dict["reference_density"], + **feature_metadata.shared_kwargs, + )._ndarray + density_threshold = 0.01 + density_is_positive = density_arr >= density_threshold + reference_density_is_zero = reference_density_arr < density_threshold + residential_is_zero = p_residential_arr == 0 + p_residential_arr[density_is_positive & reference_density_is_zero & residential_is_zero] = 1 out_ops = { "density": lambda d, _, __: d, - "residential_density": lambda d, _, p: d * p, - "nonresidential_density": lambda d, _, p: d * (1 - p), + # "residential_density": lambda d, _, p: d * p, + # "nonresidential_density": lambda d, _, p: d * (1 - p), "volume": lambda d, h, _: h * d, "residential_volume": lambda d, h, p: h * d * p, - "nonresidential_volume": lambda d, h, p: h * d * (1 - p), + # "nonresidential_volume": lambda d, h, p: h * d * (1 - p), } for measure, op in out_ops.items(): out = rt.RasterArray( @@ -422,11 +450,13 @@ def mosaic_tile( pm_data: PopulationModelData, ) -> rt.RasterArray: tiles = [] - for bounds in feature_metadata.block_bounds.values(): + for buffer_block_key, bounds in feature_metadata.block_bounds.items(): try: tile = pm_data.load_feature( + block_key=buffer_block_key, feature_name=measure, - **feature_metadata.shared_kwargs, + resolution=feature_metadata.resolution, + time_point=feature_metadata.time_point, subset_bounds=bounds, ) tile = tile.reproject( diff --git a/src/rra_population_model/model_prep/features/metadata.py b/src/rra_population_model/model_prep/features/metadata.py index 70ba7d4..786aece 100644 --- a/src/rra_population_model/model_prep/features/metadata.py +++ b/src/rra_population_model/model_prep/features/metadata.py @@ -12,6 +12,8 @@ PopulationModelData, ) +REFERENCE_BUILDING_VERSION = "microsoft_v8" + class FeatureMetadata(NamedTuple): model_frame: gpd.GeoDataFrame @@ -44,7 +46,7 @@ def get_feature_metadata( working_crs = get_working_crs(block_frame) block_bounds = get_block_bounds(block_frame, model_frame, working_crs) block_template = bd_data.load_tile( # Any provider or measure would do here - provider="microsoft_v4", + provider=REFERENCE_BUILDING_VERSION, measure="density", resolution=resolution, time_point="2023q4", diff --git a/src/rra_population_model/model_prep/features/runner.py b/src/rra_population_model/model_prep/features/runner.py index dc43a78..50550d0 100644 --- a/src/rra_population_model/model_prep/features/runner.py +++ b/src/rra_population_model/model_prep/features/runner.py @@ -16,12 +16,18 @@ from rra_population_model.model_prep.features.ntl import process_ntl # GHSL first, as we need the residential mask for msft -BUILT_VERSIONS = [ - pmc.BUILT_VERSIONS["ghsl_r2023a"], - pmc.BUILT_VERSIONS["microsoft_v6"], - pmc.BUILT_VERSIONS["microsoft_v7"], - pmc.BUILT_VERSIONS["microsoft_v7_1"], -] +BUILT_VERSIONS = { + '40': [ + pmc.BUILT_VERSIONS["ghsl_r2023a"], + pmc.BUILT_VERSIONS["microsoft_v7_1"], + pmc.BUILT_VERSIONS["microsoft_v8"], + ], + '100': [ + pmc.BUILT_VERSIONS["ghsl_r2023a"], + pmc.BUILT_VERSIONS["microsoft_v7_1"], + pmc.BUILT_VERSIONS["microsoft_v8"], + ], +} def features_main( @@ -29,18 +35,18 @@ def features_main( time_point: str, resolution: str, building_density_dir: str | Path, - model_root: str | Path, + output_dir: str | Path, ) -> None: print(f"Processing features for block {block_key} at time {time_point}") bd_data = BuildingDensityData(building_density_dir) - pm_data = PopulationModelData(model_root) + pm_data = PopulationModelData(output_dir) print("Loading all feature metadata") feature_metadata = get_feature_metadata( pm_data, bd_data, resolution, block_key, time_point ) - for built_version in BUILT_VERSIONS: + for built_version in BUILT_VERSIONS[resolution]: print(f"Processing {built_version.name}") strategy, fill_time_points = get_processing_strategy( built_version, feature_metadata @@ -66,11 +72,11 @@ def geospatial_average_features_main( time_point: str, resolution: str, building_density_dir: str | Path, - model_root: str | Path, + output_dir: str | Path, ) -> None: print(f"Processing features for block {block_key} at time {time_point}") bd_data = BuildingDensityData(building_density_dir) - pm_data = PopulationModelData(model_root) + pm_data = PopulationModelData(output_dir) print("Loading all feature metadata") feature_metadata = get_feature_metadata( @@ -79,18 +85,19 @@ def geospatial_average_features_main( features_to_average = [ "density", "volume", - "nonresidential_density", - "nonresidential_volume", - "residential_density", + # "nonresidential_density", + # "nonresidential_volume", + # "residential_density", "residential_volume", ] - for built_version in BUILT_VERSIONS: + for built_version in BUILT_VERSIONS[resolution]: print(f"Processing {built_version.name}") strategy, fill_time_points = get_processing_strategy( built_version, feature_metadata ) + built_version_features = [f"{built_version.name}_{feature}" for feature in features_to_average] feature_paths = strategy.generate_geospatial_averages( - [f"{built_version.name}_{feature}" for feature in features_to_average], + built_version_features, pmc.FEATURE_AVERAGE_RADII, pm_data, ) @@ -174,11 +181,58 @@ def features( task_resources={ "queue": queue, "cores": 1, - "memory": "5G", - "runtime": "10m", + "memory": "6G", + "runtime": "6m", "project": "proj_rapidresponse", "constraints": "archive", }, log_root=pm_data.log_dir("preprocess_features"), max_attempts=3, ) + + +@click.command() +@clio.with_time_point(allow_all=True) +@clio.with_resolution() +@clio.with_input_directory("building-density", pmc.BUILDING_DENSITY_ROOT) +@clio.with_output_directory(pmc.MODEL_ROOT) +@clio.with_queue() +def geospatial_average_features( + time_point: list[str], + resolution: str, + building_density_dir: str, + output_dir: str, + queue: str, +) -> None: + """Prepare model geospatial average features.""" + pm_data = PopulationModelData(output_dir) + print("Loading the modeling frame") + modeling_frame = pm_data.load_modeling_frame(resolution) + block_keys = modeling_frame.block_key.unique().tolist() + + njobs = len(block_keys) * len(time_point) + print(f"Submitting {njobs} jobs to process geospatial average features") + + jobmon.run_parallel( + runner="pmtask model_prep", + task_name="geospatial_average_features", + node_args={ + "block-key": block_keys, + "time-point": time_point, + }, + task_args={ + "building-density-dir": building_density_dir, + "output-dir": output_dir, + "resolution": resolution, + }, + task_resources={ + "queue": queue, + "cores": 1, + "memory": "20G", + "runtime": "20m", + "project": "proj_rapidresponse", + "constraints": "archive", + }, + log_root=pm_data.log_dir("preprocess_geospatial_average_features"), + max_attempts=2, + ) diff --git a/src/rra_population_model/model_prep/features/utils.py b/src/rra_population_model/model_prep/features/utils.py index 2e29be8..401aced 100644 --- a/src/rra_population_model/model_prep/features/utils.py +++ b/src/rra_population_model/model_prep/features/utils.py @@ -103,6 +103,7 @@ def make_spatial_average( tile: rt.RasterArray, radius: int | float, kernel_type: Literal["uniform", "gaussian"] = "uniform", + apply_floor: bool = True, ) -> rt.RasterArray: """Compute a spatial average of a raster. @@ -129,10 +130,11 @@ def make_spatial_average( kernel = make_smoothing_convolution_kernel(tile.x_resolution, radius, kernel_type) out_image = oaconvolve(arr, kernel, mode="same") - # TODO: Figure out why I did this - out_image -= np.nanmin(out_image) - min_value = 0.005 - out_image[out_image < min_value] = 0.0 + if apply_floor: + # TODO: Figure out why I did this + out_image -= np.nanmin(out_image) + min_value = 0.005 + out_image[out_image < min_value] = 0.0 out_image = out_image.reshape(arr.shape) out_raster = rt.RasterArray( diff --git a/src/rra_population_model/model_prep/modeling_frame/runner.py b/src/rra_population_model/model_prep/modeling_frame/runner.py index 37787da..3b55f79 100644 --- a/src/rra_population_model/model_prep/modeling_frame/runner.py +++ b/src/rra_population_model/model_prep/modeling_frame/runner.py @@ -35,7 +35,7 @@ def modeling_frame_main( modeling_frame[["block_key", "geometry"]].dissolve("block_key").reset_index() ) validation_rows = [] - for iso3, year in pm_data.list_census_data(): + for iso3, year, quarter in pm_data.list_census_data(): gdf = pm_data.load_census_data(iso3, year, admin_level=0) intersection_frame = block_frame[ block_frame.intersects(gdf.explode().convex_hull.union_all()) diff --git a/src/rra_population_model/model_prep/training_data/metadata.py b/src/rra_population_model/model_prep/training_data/metadata.py index f469afd..5dd7fb5 100644 --- a/src/rra_population_model/model_prep/training_data/metadata.py +++ b/src/rra_population_model/model_prep/training_data/metadata.py @@ -46,6 +46,7 @@ def get_training_metadata( time_point: str, intersecting_admins: gpd.GeoDataFrame, pm_data: PopulationModelData, + purpose: str, ) -> TrainingMetadata: try: full_shape = intersecting_admins.buffer(0).union_all() @@ -70,7 +71,12 @@ def get_training_metadata( TileMetadata.from_model_frame(model_frame, key) for key in neighborhood_keys ] - features = pm_data.list_features(resolution, tile_meta.block_key, time_point) + denominators = pmc.DENOMINATORS + + if purpose == "inference": + features = denominators + else: + features = pm_data.list_features(resolution, tile_meta.block_key, time_point) return TrainingMetadata( tile_meta=tile_meta, @@ -79,6 +85,6 @@ def get_training_metadata( time_point=time_point, tile_neighborhood=tile_neighborhood, intersecting_admins=intersecting_admins, - denominators=pmc.DENOMINATORS, + denominators=denominators, features=features, ) diff --git a/src/rra_population_model/model_prep/training_data/runner.py b/src/rra_population_model/model_prep/training_data/runner.py index c9938f3..4c27e48 100644 --- a/src/rra_population_model/model_prep/training_data/runner.py +++ b/src/rra_population_model/model_prep/training_data/runner.py @@ -17,21 +17,23 @@ def training_data_main( resolution: str, - iso3_list: str, + iso3_time_point_list: str, tile_key: str, time_point: str, - model_root: str | Path, + purpose: str, + output_dir: str | Path, ) -> None: - """Build the training data for the MEX model for a single tile.""" + """Build the training data for the model for a single tile.""" print("Loading metadata") - pm_data = PopulationModelData(model_root) + pm_data = PopulationModelData(output_dir) model_frame = pm_data.load_modeling_frame(resolution) tile_meta = TileMetadata.from_model_frame(model_frame, tile_key) + + iso3_time_point_list = [i.split(':') for i in iso3_time_point_list.split(",")] print("Finding intersecting admin units") admins = utils.get_intersecting_admins( tile_meta=tile_meta, - iso3_list=iso3_list, - time_point=time_point, + iso3_time_point_list=iso3_time_point_list, pm_data=pm_data, ) if admins.empty: @@ -46,35 +48,86 @@ def training_data_main( time_point=time_point, intersecting_admins=admins, pm_data=pm_data, + purpose=purpose, ) - print("Loading model gdfs") model_gdfs = [] - for n_tile_meta in training_meta.tile_neighborhood: - print(n_tile_meta.key) - n_tile_gdf = utils.get_tile_feature_gdf( - n_tile_meta, - training_meta, - pm_data, - ) - if not n_tile_gdf.empty: - model_gdfs.append(n_tile_gdf) + data_time_point_list = list(set([i[1] for i in iso3_time_point_list] + [time_point])) + for data_time_point in data_time_point_list: + print(f"Loading model gdfs -- {data_time_point}") + time_point_model_gdfs = [] + for n_tile_meta in training_meta.tile_neighborhood: + print(n_tile_meta.key) + n_tile_gdf = utils.get_tile_feature_gdf( + tile_meta=n_tile_meta, + training_meta=training_meta, + pm_data=pm_data, + time_point=data_time_point, + ) + if not n_tile_gdf.empty: + time_point_model_gdfs.append(n_tile_gdf) - print("Processing model gdf") + print(f"Processing model gdf-- {data_time_point}") + time_point_model_gdf = pd.concat( + time_point_model_gdfs, ignore_index=True + ) + time_point_model_gdf = utils.process_model_gdf( + time_point_model_gdf, training_meta + ) + model_gdfs.append(time_point_model_gdf) model_gdf = pd.concat(model_gdfs, ignore_index=True) - - model_gdf = utils.process_model_gdf(model_gdf, training_meta) - admin_gdf = utils.filter_to_admin_gdf(model_gdf, training_meta) tile_gdf = model_gdf[model_gdf["tile_key"] == tile_key] - print("Calculating pixel area weights") - pixel_area_weight = ( - tile_gdf.groupby(["admin_id", "pixel_id"])[["admin_area_weight"]] - .first() - .reset_index() - ) + if purpose == 'training': + model_gdf = model_gdf.loc[model_gdf['time_point'] == time_point] + admin_gdf = utils.filter_to_admin_gdf(model_gdf, training_meta) + + print("Calculating pixel area weights") + pixel_area_weight = ( + tile_gdf.groupby(["admin_id", "pixel_id"])[["admin_area_weight"]] + .first() + .reset_index() + ) + elif purpose == 'inference': + tile_gdfs = [] + for data_time_point in data_time_point_list: + time_point_tile_gdfs = [] + for denominator in training_meta.denominators: + pixel_occupancy_rate = ( + tile_gdf + .loc[(tile_gdf['census_time_point'] == data_time_point) & (tile_gdf['time_point'] == data_time_point)] + .set_index(['admin_id', 'pixel_id']) + .loc[:, f'pixel_occupancy_rate_{denominator}'] + .clip(0, np.inf) + ) + pixel_built = ( + tile_gdf + .loc[(tile_gdf['census_time_point'] == data_time_point) & (tile_gdf['time_point'] == time_point)] + .set_index(['admin_id', 'pixel_id']) + .loc[:, f'pixel_built_{denominator}'] + ) + time_point_tile_gdfs.append( + (pixel_occupancy_rate * pixel_built) + .rename(f'pixel_population_{denominator}') + ) + tile_gdfs.append( + pd.concat(time_point_tile_gdfs, axis=1).reset_index() + ) + multi_tile = ( + tile_gdf + .loc[tile_gdf['time_point'] == time_point] + .loc[:, ['admin_id', 'pixel_id', 'pixel_multi_tile']] + ) + area_weight = ( + tile_gdf + .loc[tile_gdf['time_point'] == time_point] + .loc[:, ['admin_id', 'pixel_id', 'pixel_area_weight']] + ) + tile_gdf = pd.concat(tile_gdfs, ignore_index=True).merge(multi_tile).merge(area_weight) + else: + raise ValueError(f'Invalid purpose: {purpose}') - print("rasterizing features") + print("Rasterizing features") raster_template = pm_data.load_feature( resolution=resolution, block_key=tile_meta.block_key, @@ -83,88 +136,143 @@ def training_data_main( subset_bounds=tile_meta.polygon, ) - out_measures = ["population", "occupancy_rate", "log_occupancy_rate"] + if purpose == 'training': + out_measures = ["population", "occupancy_rate", "log_occupancy_rate"] + elif purpose == 'inference': + out_measures = ["population"] training_rasters = [ f"{m}_{d}" for m, d in itertools.product(out_measures, training_meta.denominators) ] + ["multi_tile"] + if purpose == "inference": + training_rasters.append("area_weight") tile_rasters = {} for raster_name in training_rasters: raster = utils.raster_from_pixel_feature(tile_gdf, raster_name, raster_template) tile_rasters[raster_name] = raster + # import numpy as np + # if purpose == "inference": + # print( + # f"Raster total: {np.nansum(tile_rasters['population_microsoft_v7_1_residential_volume'])}" + # ) + # model_gdf = model_gdf.loc[model_gdf['time_point'] == time_point] + # admin_gdf = utils.filter_to_admin_gdf(model_gdf, training_meta) + # print( + # f"Admin total: {admin_gdf['admin_population'].sum()}" + # ) + print("Saving") - pm_data.save_tile_training_data( - resolution, - tile_key, - admin_gdf, - pixel_area_weight, - tile_rasters, - ) + if purpose == 'training': + pm_data.save_tile_training_data( + resolution, + tile_key, + admin_gdf, + pixel_area_weight, + tile_rasters, + ) + elif purpose == 'inference': + pm_data.save_tile_inference_data( + resolution, + time_point, + tile_key, + tile_rasters, + ) + else: + raise ValueError(f'Unexpected data purpose: {purpose}') @click.command() -@click.option("--iso3-list", type=str, required=True) +@click.option("--iso3-time-point-list", type=str, required=True) @clio.with_resolution() @clio.with_tile_key() @clio.with_time_point() +@clio.with_purpose() @clio.with_output_directory(pmc.MODEL_ROOT) def training_data_task( resolution: str, - iso3_list: str, + iso3_time_point_list: str, tile_key: str, time_point: str, + purpose: str, output_dir: str, ) -> None: """Build the response for a given tile and time point.""" training_data_main( resolution, - iso3_list, + iso3_time_point_list, tile_key, time_point, + purpose, output_dir, ) @click.command() @clio.with_resolution() +@clio.with_purpose() @clio.with_output_directory(pmc.MODEL_ROOT) @clio.with_queue() def training_data( resolution: str, + purpose: str, output_dir: str, queue: str, ) -> None: - """Build the training data for the MEX model.""" + """Build the training data for the model.""" pm_data = PopulationModelData(output_dir) print("Building arg list") - to_run = utils.build_arg_list(resolution, pm_data) - print(f"Building test/train data for {len(to_run)} tiles.") - - status = jobmon.run_parallel( - runner="pmtask model_prep", - task_name="training_data", - flat_node_args=(("tile-key", "time-point", "iso3-list"), to_run), - task_args={ - "output-dir": output_dir, - "resolution": resolution, - }, - task_resources={ - "queue": queue, - "cores": 1, - "memory": "30G", - "runtime": "30m", - "project": "proj_rapidresponse", - }, - max_attempts=5, - log_root=pm_data.log_dir("model_prep_training_data"), - ) + to_run = utils.build_arg_list(resolution, pm_data, purpose) + to_run = [i for i in to_run if i[1].startswith('202')] + + if purpose == "training": + to_run = [ + (i, j, k) for i, j, k in to_run + if not (pm_data.tile_training_data_root(resolution) / i / "pixel_area_weights.parquet").exists() + ] + + time_points = sorted(list(set([i[1] for i in to_run]))) + years = sorted(list(set([time_point.split('q')[0] for time_point in time_points]))) + print(f"Starting annual workflows for each year: {', '.join(years)}") + print("############################################################") + for year in years: + to_run_year = [i for i in to_run if i[1].startswith(year)] + # year = '2023q1-2024q2' + # to_run_year = [i for i in to_run if i[1].startswith('2023') or i[1].startswith('2024')] + print(f"Building {purpose} data for {len(to_run_year)} tiles for {year}.") + status = jobmon.run_parallel( + runner="pmtask model_prep", + task_name="training_data", + flat_node_args=(("tile-key", "time-point", "iso3-time-point-list"), to_run_year), + task_args={ + "output-dir": output_dir, + "resolution": resolution, + "purpose": purpose, + }, + task_resources={ + "queue": queue, + "cores": 1, + "memory": "10G", + "runtime": "5m", + "project": "proj_rapidresponse", + }, + max_attempts=5, + resource_scales={ + "memory": iter([20 , 40 , 80 , 240 ]), # G + "runtime": iter([10 * 60, 20 * 60, 30 * 60, 90 * 60]), # seconds + }, + log_root=pm_data.log_dir("model_prep_training_data"), + ) - if status != "D": - msg = f"Workflow failed with status {status}." - raise RuntimeError(msg) + if status != "D": + msg = f"Workflow failed with status {status}." + raise RuntimeError(msg) + print("############################################################") - print("Building summary datasets.") - people_per_structure = utils.build_summary_people_per_structure(pm_data, resolution) - pm_data.save_summary_people_per_structure(people_per_structure, resolution) + if purpose == 'training': + print("Building summary datasets.") + people_per_structure = utils.build_summary_people_per_structure(pm_data, resolution) + pm_data.save_summary_people_per_structure(people_per_structure, resolution) + elif purpose != 'inference': + raise ValueError(f'Unexpected data purpose: {purpose}') diff --git a/src/rra_population_model/model_prep/training_data/utils.py b/src/rra_population_model/model_prep/training_data/utils.py index a61746a..9722a15 100644 --- a/src/rra_population_model/model_prep/training_data/utils.py +++ b/src/rra_population_model/model_prep/training_data/utils.py @@ -1,4 +1,3 @@ -from collections import defaultdict from typing import Any import geopandas as gpd @@ -8,6 +7,7 @@ import rasterra as rt import tqdm +from rra_population_model import constants as pmc from rra_population_model.data import PopulationModelData from rra_population_model.model_prep.training_data.metadata import ( TileMetadata, @@ -17,20 +17,19 @@ def get_intersecting_admins( tile_meta: TileMetadata, - iso3_list: str, - time_point: str, + iso3_time_point_list: list[list[str]], pm_data: PopulationModelData, ) -> gpd.GeoDataFrame: - year = time_point.split("q")[0] - admin_data = [] - for iso in iso3_list.split(","): - a = pm_data.load_census_data(iso, year, tile_meta.polygon) + for iso3, time_point in iso3_time_point_list: + year = time_point.split("q")[0] + a = pm_data.load_census_data(iso3, year, tile_meta.polygon) # Need to intersect again with the tile poly because we load based on the # intersection with the bounding box. is_max_admin = a.admin_level == a.admin_level.max() intersects_tile = a.intersects(tile_meta.polygon) a = a.loc[is_max_admin & intersects_tile] + a['census_time_point'] = time_point admin_data.append(a) admins = pd.concat(admin_data, ignore_index=True) @@ -42,30 +41,50 @@ def get_intersecting_admins( ) admins["admin_area"] = admins.area admins["geometry"] = admins.buffer(0) - return admins.loc[:, ["admin_id", "admin_population", "admin_area", "geometry"]] + return admins.loc[:, ["admin_id", "admin_population", "admin_area", "geometry", "census_time_point"]] -def get_training_locations_and_years( +def get_data_locations_and_years( pm_data: PopulationModelData, + purpose: str, ) -> list[tuple[str, str, str]]: """Get the locations and years for which we have training data.""" - available_census_years = pm_data.list_census_data() # noqa: F841 - return [ - ("MEX", "2020", "1"), - ("USA", "2020", "1"), - ] + available_census_years = pm_data.list_census_data() + if purpose == 'training': + available_census_years = [ + i for i in available_census_years + if i[0] in ["MEX", "USA"] and i[1] == "2020" + ] + elif purpose == 'inference': + inference_countries = [ + # "DNK", # Denmark + # "FIN", # Finland + # "FRA", # France + "MEX", # Mexico + # "MYS", # Malaysia + # "PAN", # Panama + # "SWE", # Sweden + # "TUR", # Turkey + "USA", # United States + ] + available_census_years = [ + i for i in available_census_years + if i[0] in inference_countries and i[1] == "2020" + ] + return available_census_years def build_arg_list( resolution: str, pm_data: PopulationModelData, + purpose: str, buffer_size: int | float = 5000, ) -> list[tuple[str, str, str]]: modeling_frame = pm_data.load_modeling_frame(resolution) - training_census_years = get_training_locations_and_years(pm_data) + data_years = get_data_locations_and_years(pm_data, purpose) - tile_keys_and_times = defaultdict(list) - for iso3, year, quarter in training_census_years: + tile_keys_and_times = [] + for iso3, year, quarter in data_years: print(f"Processing {iso3} {year}q{quarter}") shape = pm_data.load_census_data(iso3, year) a1 = ( @@ -76,11 +95,55 @@ def build_arg_list( ) a1_intersection = modeling_frame[modeling_frame.intersects(a1)] for tile_key in a1_intersection.tile_key.unique(): - tile_keys_and_times[(tile_key, f"{year}q{quarter}")].append(iso3) + tile_keys_and_times.append( + pd.DataFrame( + {"iso3_time_point": f"{year}q{quarter}", "iso3": iso3}, + index=pd.Index([tile_key], name='tile_key'), + ) + ) + tile_keys_and_times = pd.concat(tile_keys_and_times) + if purpose == 'training': + tile_keys_and_times['time_point'] = tile_keys_and_times['iso3_time_point'] + elif purpose == 'inference': + tile_keys_and_times = pd.concat( + [ + pd.concat([ + tile_keys_and_times, pd.Series(time_point, name='time_point', index=tile_keys_and_times.index) + ], axis=1) + for time_point in pmc.BUILT_VERSION_TIME_POINTS + ] + ) + tile_keys_and_times['year'] = ( + tile_keys_and_times['time_point'].str.split('q').str[0].astype(int) + + (tile_keys_and_times['time_point'].str.split('q').str[1].astype(int) - 1) / 4 + ) + tile_keys_and_times['iso3_year'] = ( + tile_keys_and_times['iso3_time_point'].str.split('q').str[0].astype(int) + + (tile_keys_and_times['iso3_time_point'].str.split('q').str[1].astype(int) - 1) / 4 + ) + tile_keys_and_times['distance'] = (tile_keys_and_times['year'] - tile_keys_and_times['iso3_year']).abs() + tile_keys_and_times = ( + tile_keys_and_times + .sort_values('distance') + .groupby(['tile_key', 'time_point', 'iso3'])['iso3_time_point'] + .first() + .reset_index(['time_point', 'iso3']) + ) + else: + raise ValueError(f'Unexpected purpose: {purpose}') + tile_keys_and_times['iso3_time_point'] = ( + tile_keys_and_times['iso3'] + ':' + tile_keys_and_times['iso3_time_point'] + ) + tile_keys_and_times = ( + tile_keys_and_times + .groupby(['tile_key', 'time_point'])['iso3_time_point'] + .apply(lambda x: ','.join(x.to_list())) + .sort_index() + ) to_run = [ - (tile_key, time_point, ",".join(iso3s)) - for (tile_key, time_point), iso3s in tile_keys_and_times.items() + (tile_key, time_point, iso3_time_points) + for (tile_key, time_point), iso3_time_points in tile_keys_and_times.items() ] return to_run @@ -123,20 +186,18 @@ def get_tile_feature_gdf( tile_meta: TileMetadata, training_meta: TrainingMetadata, pm_data: PopulationModelData, + time_point: str, ) -> gpd.GeoDataFrame: """Load the raster features for the tile and convert to a GeoDataFrame.""" - kwargs = { - "resolution": training_meta.resolution, - "block_key": tile_meta.block_key, - "time_point": training_meta.time_point, - } tile_features = {} for feature_name in training_meta.features: tile_features[feature_name] = pm_data.load_feature( + resolution=training_meta.resolution, + block_key=tile_meta.block_key, feature_name=feature_name, + time_point=time_point, subset_bounds=tile_meta.polygon, - **kwargs, ) default_raster = training_meta.denominators[0] @@ -150,7 +211,7 @@ def get_tile_feature_gdf( feature_gdf["pixel_area"] = feature_gdf.area feature_gdf["block_key"] = tile_meta.block_key feature_gdf["tile_key"] = tile_meta.key - feature_gdf["time_point"] = training_meta.time_point + feature_gdf["time_point"] = time_point for feature_name, feature_raster in tile_features.items(): feature_gdf[f"pixel_{feature_name}"] = feature_raster.to_numpy().flatten() @@ -225,10 +286,12 @@ def process_model_gdf( denominator_df["admin_built"] = denominator_df.groupby("admin_id")[ "isection_built" ].transform("sum") - if denominator[:4] == "msft": - low_density = denominator_df["admin_built"] < min_admin_density - denominator_df.loc[low_density, "admin_built"] = 0.0 - denominator_df.loc[low_density, "isection_built"] = 0.0 + # if denominator.startswith("microsoft"): + # low_density = denominator_df["admin_built"] < min_admin_density + # denominator_df.loc[low_density, "admin_built"] = 0.0 + # denominator_df.loc[low_density, "isection_built"] = 0.0 + # elif not denominator.startswith("ghsl"): + # raise ValueError(f"Unexpected denominator: {denominator}") denominator_df[f"admin_{denominator}"] = safe_divide( denominator_df["admin_built"], model_gdf["admin_area"] @@ -280,6 +343,8 @@ def process_model_gdf( "admin_built", "admin_occupancy_rate", "admin_log_occupancy_rate", + "pixel_built", + "pixel_built_weight", "pixel_population", "pixel_occupancy_rate", "pixel_log_occupancy_rate", @@ -339,10 +404,23 @@ def raster_from_pixel_feature( rt.RasterArray The raster of the pixel feature. """ + if feature_name.startswith("population") or feature_name == "area_weight": + agg_func = "sum" + elif ( + feature_name.startswith("occupancy_rate") + or feature_name.startswith("log_occupancy_rate") + ): + agg_func = "mean" + elif feature_name == "multi_tile": + agg_func = "first" + else: + value_error = f"Unexpected feature name: {feature_name}" + raise ValueError(value_error) + idx = np.arange(raster_template.size) feature_data = ( tile_gdf.groupby("pixel_id")[f"pixel_{feature_name}"] - .first() + .agg(agg_func) .reindex(idx, fill_value=0.0) .to_numpy() .astype(np.float32) diff --git a/src/rra_population_model/postprocess/mosaic/runner.py b/src/rra_population_model/postprocess/mosaic/runner.py index b939b13..dfde47a 100644 --- a/src/rra_population_model/postprocess/mosaic/runner.py +++ b/src/rra_population_model/postprocess/mosaic/runner.py @@ -1,4 +1,5 @@ import itertools +import tqdm import click import rasterra as rt @@ -26,27 +27,53 @@ def mosaic_main( model_spec = pm_data.load_model_specification(resolution, version) block_keys = pm_data.load_modeling_frame(resolution)["block_key"].unique() - paths = [] + pop_paths = [] + denom_paths = [] + # poverty_paths = [] for x, y in itertools.product(range(STRIDE), range(STRIDE)): bx_, by_ = STRIDE * bx + x, STRIDE * by + y block_key = f"B-{bx_:>04}X-{by_:>04}Y" if block_key not in block_keys: continue - paths.append(pm_data.raked_prediction_path(block_key, time_point, model_spec)) + pop_paths.append(pm_data.raked_prediction_path(block_key, time_point, model_spec)) + denom_paths.append(pm_data.feature_path(resolution, block_key, model_spec.denominator, time_point)) + # poverty_paths.append(f"{model_spec.output_root}/poverty/{time_point}/{block_key}/1000m.tif") print("loading rasters") - r = rt.load_mf_raster(paths) + pop_raster = rt.load_mf_raster(pop_paths) + denom_raster = rt.load_mf_raster(denom_paths) + # poverty_raster = rt.load_mf_raster(poverty_paths) print("writing cog") group_key = f"G-{bx:>04}X-{by:>04}Y" pm_data.save_compiled_prediction( - raster=r, + raster=pop_raster, group_key=group_key, time_point=time_point, model_spec=model_spec, + measure="population", num_cores=num_cores, resampling="average", ) + if resolution == "40": + pm_data.save_compiled_prediction( + raster=denom_raster, + group_key=group_key, + time_point=time_point, + model_spec=model_spec, + measure="building", + num_cores=num_cores, + resampling="average", + ) + # pm_data.save_compiled_prediction( + # raster=poverty_raster, + # group_key=group_key, + # time_point=time_point, + # model_spec=model_spec, + # measure="poverty", + # num_cores=num_cores, + # resampling="average", + # ) @click.command() @@ -108,15 +135,14 @@ def mosaic( bys = list(range(y_max // STRIDE + int(bool(y_max % STRIDE)))) print("Compiling") - jobmon.run_parallel( runner="pmtask postprocess", task_name="mosaic", task_resources={ "queue": queue, "cores": num_cores, - "memory": "120G", - "runtime": "30m", + "memory": "160G", + "runtime": "15m", "project": "proj_rapidresponse", }, node_args={ @@ -130,14 +156,31 @@ def mosaic( "num-cores": num_cores, "output-dir": output_dir, }, - max_attempts=1, + max_attempts=2, log_root=pm_data.log_dir("postprocess_mosaic"), ) + if resolution == "40" and all([tp in raked_time_points for tp in time_points]): + print("Calculating and storing change") + utils.save_change( + resolution=resolution, + version=version, + time_points=[sorted(time_points)[0], sorted(time_points)[-1]], + output_dir=output_dir, + num_cores=1, + ) + print("Building VRTs") model_spec = pm_data.load_model_specification(resolution, version) - utils.make_vrts( - time_points, - model_spec=model_spec, - pm_data=pm_data, - ) + if resolution == "40": + measures = ["population", "building", "change"] # , "poverty" + else: + measures = ["population"] # , "poverty" + for measure in measures: + measure_time_points = pm_data.list_compiled_prediction_time_points(resolution, version, measure) + utils.make_vrts( + measure_time_points, + model_spec=model_spec, + pm_data=pm_data, + measure=measure, + ) diff --git a/src/rra_population_model/postprocess/mosaic/utils.py b/src/rra_population_model/postprocess/mosaic/utils.py index 0b1483e..52f0577 100644 --- a/src/rra_population_model/postprocess/mosaic/utils.py +++ b/src/rra_population_model/postprocess/mosaic/utils.py @@ -1,14 +1,74 @@ import shlex import shutil +import functools import subprocess from pathlib import Path from rra_tools.shell_tools import touch +from rra_tools import parallel from rra_population_model.data import PopulationModelData from rra_population_model.model.modeling.datamodel import ModelSpecification +def save_change_group_key( + group_key: str, + pm_data: PopulationModelData, + time_points: list[str], + model_spec: ModelSpecification, + measure: str, +): + start_raster = pm_data.load_compiled_prediction( + group_key, + time_points[0], + model_spec, + measure, + ) + end_raster = pm_data.load_compiled_prediction( + group_key, + time_points[-1], + model_spec, + measure, + ) + change_raster = end_raster - start_raster + + pm_data.save_compiled_prediction( + raster=change_raster, + group_key=group_key, + time_point="-".join(time_points), + model_spec=model_spec, + measure="change", + resampling="average", + ) + + +def save_change( + resolution: str, + version: str, + time_points: list[str], + output_dir: str, + num_cores: int, + measure: str = "population", +): + pm_data = PopulationModelData(output_dir) + model_spec = pm_data.load_model_specification(resolution, version) + group_keys = pm_data.list_compiled_prediction_time_point_group_keys(resolution, version, time_points[-1], measure) + + _save_change_group_key = functools.partial( + save_change_group_key, + pm_data=pm_data, + time_points=time_points, + model_spec=model_spec, + measure=measure, + ) + parallel.run_parallel( + _save_change_group_key, + group_keys, + num_cores=num_cores, + progress_bar=True, + ) + + def make_vrt(vrt_path: Path) -> None: gdalbuildvrt_path = shutil.which("gdalbuildvrt") files = " ".join([str(p) for p in vrt_path.parent.glob("G*.tif")]) @@ -25,8 +85,9 @@ def make_vrts( time_points: list[str], model_spec: ModelSpecification, pm_data: PopulationModelData, + measure: str = "", ) -> None: for tp in time_points: print(tp) - vrt_path = pm_data.compiled_prediction_vrt_path(tp, model_spec) + vrt_path = pm_data.compiled_prediction_vrt_path(tp, model_spec, measure) make_vrt(vrt_path) diff --git a/src/rra_population_model/postprocess/rake/runner.py b/src/rra_population_model/postprocess/rake/runner.py index 29c119d..21f4c66 100644 --- a/src/rra_population_model/postprocess/rake/runner.py +++ b/src/rra_population_model/postprocess/rake/runner.py @@ -3,6 +3,9 @@ import rasterra as rt from rasterra._features import raster_geometry_mask from rra_tools import jobmon +import tqdm +import geopandas as gpd +import shapely from rra_population_model import cli_options as clio from rra_population_model import constants as pmc @@ -13,6 +16,7 @@ def rake_main( resolution: str, version: str, + input_data: str, block_key: str, time_point: str, output_dir: str, @@ -25,9 +29,14 @@ def rake_main( pm_data, resolution, version, time_point ) print("Loading unraked prediction") - unraked_data = pm_data.load_raw_prediction( - block_key, prediction_time_point, model_spec - ) + if input_data in ["raw", "raw_skip"]: + unraked_data = pm_data.load_raw_prediction( + block_key, prediction_time_point, model_spec + ) + elif input_data == "raked": + unraked_data = pm_data.load_raked_prediction( + block_key, prediction_time_point, model_spec + ) print("Loading raking factors") raking_data = pm_data.load_raking_factors( @@ -67,67 +76,191 @@ def rake_main( ) raked = unraked_data * raking_factor + if input_data == "raw": + print("Loading inference data") + model_frame = pm_data.load_modeling_frame(resolution) + model_frame = model_frame.loc[model_frame["block_key"] == block_key] + + census_population = [] + census_weight = [] + for tile_key in tqdm.tqdm(model_frame["tile_key"].to_list()): + tile_census_population = pm_data.load_tile_inference_data( + resolution, + tile_key, + time_point, + f"population_{model_spec.denominator}", + ) + if tile_census_population is None: + tile_census_population = pm_data.load_feature( + resolution=resolution, + block_key=block_key, + feature_name=model_spec.denominator, + time_point=time_point, + subset_bounds=model_frame[model_frame.tile_key == tile_key].geometry.iloc[0], + ) + tile_census_population = rt.RasterArray( + np.zeros_like(tile_census_population), + transform=tile_census_population.transform, + crs=tile_census_population.crs, + no_data_value=np.nan, + ) + tile_census_weight = rt.RasterArray( + np.zeros_like(tile_census_population), + transform=tile_census_population.transform, + crs=tile_census_population.crs, + no_data_value=np.nan, + ) + else: + tile_census_weight = pm_data.load_tile_inference_data( + resolution, + tile_key, + time_point, + "area_weight", + ) + census_population.append(tile_census_population) + census_weight.append(tile_census_weight) + + census_weight = rt.merge(census_weight) + census_population = rt.merge(census_population) + if np.nansum(census_weight) > 0: + if (np.round(census_weight.bounds, 2) != np.round(raked.bounds, 2)).any(): + census_bounds = create_bounds_polygon(census_weight) + raked_bounds = create_bounds_polygon(raked) + + missing = raked_bounds.difference(census_bounds, grid_size=int(resolution)) + missing = raked.clip(missing).mask(missing) + + census_weight = rt.merge([census_weight, missing]) + census_population = rt.merge([census_population, missing]) + if (np.round(census_weight.bounds, 2) != np.round(raked.bounds, 2)).any(): + raise ValueError("Still incompatible after attaching missing pixels") + + array = census_population.to_numpy() + nan_mask = np.isnan(array) + array[nan_mask] = 0 + census_population = rt.RasterArray( + array, + transform=raked.transform, + crs=raked.crs, + no_data_value=np.nan, + ) + + array = census_weight.to_numpy() + nan_mask = np.isnan(array) + array[nan_mask] = 0 + census_weight = rt.RasterArray( + array, + transform=raked.transform, + crs=raked.crs, + no_data_value=np.nan, + ) + + print("Splicing in inference data") + raked = ( + (raked * (1 - census_weight)) + + (census_population * census_weight) + ) + else: + print("No population to splice") + elif input_data not in ["raked", "raw_skip"]: + raise ValueError(f"Invalid `input_data` type: {input_data}") + print("Saving raked prediction") pm_data.save_raked_prediction(raked, block_key, time_point, model_spec) +def create_bounds_polygon(raster_data: rt.RasterArray) -> shapely.Polygon: + bounds = raster_data.bounds + bounds = ( + bounds[0], bounds[2], + bounds[1], bounds[3], + ) + bounds = ( + gpd.GeoSeries( + [shapely.box(*bounds)], + crs=raster_data.crs + ) + .explode(index_parts=True) + .union_all() + ) + + return bounds + + @click.command() @clio.with_resolution() @clio.with_version() +@click.option("--input-data", type=str, required=True) @clio.with_block_key() @clio.with_time_point(choices=None) @clio.with_output_directory(pmc.MODEL_ROOT) def rake_task( resolution: str, version: str, + input_data: str, block_key: str, time_point: str, output_dir: str, ) -> None: - rake_main(resolution, version, block_key, time_point, output_dir) + rake_main(resolution, version, input_data, block_key, time_point, output_dir) @click.command() @clio.with_resolution(allow_all=False) @clio.with_version() +@click.option("--input-data", type=str, required=True) @clio.with_time_point(choices=None, allow_all=True) @clio.with_output_directory(pmc.MODEL_ROOT) @clio.with_queue() def rake( resolution: str, version: str, + input_data: str, time_point: str, output_dir: str, queue: str, ) -> None: pm_data = PopulationModelData(output_dir) + if input_data in ["raw", "raw_skip"]: + if len(list(pm_data.raked_predictions_root(resolution, version).iterdir())) > 0: + raise ValueError(f"Raked predictions already exist, cannot run with `input_data` set to `raw`.") + elif input_data not in ["raked"]: + raise ValueError(f"Invalid `input_data` type: {input_data}") rf_time_points = pm_data.list_raking_factor_time_points(resolution, version) - time_points = clio.convert_choice(time_point, rf_time_points) model_frame = pm_data.load_modeling_frame(resolution) block_keys = model_frame.block_key.unique().tolist() + # versions = [f"2025_11_08.0{(i + 1):02d}" for i in range(60)] + # time_points = ["2020q1", "2020q2"] + print(f"Raking {len(block_keys) * len(time_points)} blocks") + # for time_point in time_points: + # print("##############################################################") + # print(f"Raking {len(block_keys) * len(versions)} blocks for {time_point}") jobmon.run_parallel( runner="pmtask postprocess", task_name="rake", task_resources={ "queue": queue, "cores": 1, - "memory": "50G", - "runtime": "25m", + "memory": "10G", + "runtime": "5m", "project": "proj_rapidresponse", }, node_args={ + # "version": versions, "block-key": block_keys, "time-point": time_points, }, task_args={ "version": version, + # "time-point": time_point, "resolution": resolution, "output-dir": output_dir, + "input-data": input_data, }, max_attempts=3, log_root=pm_data.log_dir("postprocess_rake"), diff --git a/src/rra_population_model/postprocess/rake_itu.py b/src/rra_population_model/postprocess/rake_itu.py index c7f058f..934876d 100644 --- a/src/rra_population_model/postprocess/rake_itu.py +++ b/src/rra_population_model/postprocess/rake_itu.py @@ -92,7 +92,7 @@ def rake_itu_main( # ITU has finer scale boundaries so we need to fill in the gaps # using the nearest location id to_fill_mask = (itu_mask.to_numpy() == 1) & (location_mask == 0) - location_mask = fillnodata(location_mask, ~to_fill_mask, max_search_distance=100) + location_mask = fillnodata(location_mask, ~to_fill_mask, max_search_distance=int(itu_mask.resolution[0])) print("Building modeling frame filters") block_key_x = modeling_frame["block_key"].apply(lambda x: int(x.split("X")[0][-4:])) @@ -219,7 +219,7 @@ def rake_itu( task_resources={ "queue": queue, "cores": 1, - "memory": "75G", + "memory": "80G", "runtime": "60m", "project": "proj_rapidresponse", }, @@ -232,6 +232,10 @@ def rake_itu( "version": version, "output-dir": output_dir, }, - max_attempts=3, + max_attempts=2, + resource_scales={ + "memory": iter([300 ]), # G + "runtime": iter([180 * 60]), # seconds + }, log_root=pm_data.log_dir("postprocess_rake_itu"), ) diff --git a/src/rra_population_model/postprocess/raking_factors/runner.py b/src/rra_population_model/postprocess/raking_factors/runner.py index e318915..05a5728 100644 --- a/src/rra_population_model/postprocess/raking_factors/runner.py +++ b/src/rra_population_model/postprocess/raking_factors/runner.py @@ -29,14 +29,27 @@ def load_admin_populations( # Interpolate the time point population if "q" in time_point: year, quarter = (int(s) for s in time_point.split("q")) + if RAKING_VERSION == "gbd_2023": - next_year = min(year + 1, 2024) + max_data_year = all_pop.index.get_level_values("year_id").max() + next_year = year + 1 + if next_year > 2026: + raise ValueError("Don't project beyond 2025") + if next_year > max_data_year: + prior_year_pop = all_pop.loc[max_data_year - (year - max_data_year)] + next_year_pop = all_pop.loc[max_data_year - (next_year - max_data_year)] + max_data_year_pop = all_pop.loc[max_data_year] + prior_year_pop = max_data_year_pop * (max_data_year_pop / prior_year_pop) + next_year_pop = max_data_year_pop * (max_data_year_pop / next_year_pop) + else: + prior_year_pop = all_pop.loc[year] + next_year_pop = all_pop.loc[next_year] else: next_year = min(year + 1, 2100) - weight = (int(quarter) - 1) / 4 + prior_year_pop = all_pop.loc[year] + next_year_pop = all_pop.loc[next_year] - prior_year_pop = all_pop.loc[year] - next_year_pop = all_pop.loc[next_year] + weight = (int(quarter) - 1) / 4 pop = ((1 - weight) * prior_year_pop + weight * next_year_pop).reset_index() else: @@ -53,6 +66,7 @@ class AggregationArgs(NamedTuple): model_version: str block_key: str time_point: str + input_data: str shape_map: dict[int, shapely.Polygon] tile_poly: shapely.Polygon @@ -61,14 +75,17 @@ def aggregate_unraked_population( aggregate_args: AggregationArgs, ) -> dict[int, float]: est_pop: dict[int, float] = {} - resolution, model_version, block_key, time_point, shape_map, *_ = aggregate_args + resolution, model_version, block_key, time_point, input_data, shape_map, *_ = aggregate_args if not shape_map: return est_pop pm_data = PopulationModelData() model_spec = pm_data.load_model_specification(resolution, model_version) - r = pm_data.load_raw_prediction(block_key, time_point, model_spec) + if input_data == "raw": + r = pm_data.load_raw_prediction(block_key, time_point, model_spec) + elif input_data == "raked": + r = pm_data.load_raked_prediction(block_key, time_point, model_spec) for location_id, geom in shape_map.items(): est_pop[location_id] = np.nansum(r.mask(geom)) # type: ignore[assignment] return est_pop @@ -78,6 +95,7 @@ def raking_factors_main( resolution: str, version: str, time_point: str, + input_data: str, output_dir: str, num_cores: int, progress_bar: bool, @@ -112,6 +130,7 @@ def raking_factors_main( version, block_key, prediction_time_point, + input_data, shape_map, tile_poly, ) @@ -167,6 +186,7 @@ def raking_factors_main( @clio.with_resolution() @clio.with_version() @clio.with_time_point(choices=None) +@click.option("--input-data", type=str, required=True) @clio.with_output_directory(pmc.MODEL_ROOT) @clio.with_num_cores(default=8) @clio.with_progress_bar() @@ -174,12 +194,13 @@ def raking_factors_task( resolution: str, version: str, time_point: str, + input_data: str, output_dir: str, num_cores: int, progress_bar: bool, ) -> None: raking_factors_main( - resolution, version, time_point, output_dir, num_cores, progress_bar + resolution, version, time_point, input_data, output_dir, num_cores, progress_bar ) @@ -189,6 +210,7 @@ def raking_factors_task( @clio.with_copy_from_version() @clio.with_time_point(choices=None, allow_all=True) @click.option("--extrapolate", is_flag=True) +@click.option("--input-data", type=str, required=True) @clio.with_output_directory(pmc.MODEL_ROOT) @clio.with_num_cores(default=8) @clio.with_queue() @@ -197,12 +219,19 @@ def raking_factors( version: str, copy_from_version: str | None, time_point: str, + input_data: str, extrapolate: bool, output_dir: str, num_cores: int, queue: str, ) -> None: pm_data = PopulationModelData(output_dir) + if input_data == "raw": + if len(list(pm_data.raked_predictions_root(resolution, version).iterdir())) > 0: + raise ValueError(f'Raked predictions already exist, cannot run with `input_data` set to `raw`.') + elif input_data not in ["raked"]: + raise ValueError(f'Invalid `input_data` type: {input_data}') + pm_data.maybe_copy_version(resolution, version, copy_from_version) prediction_time_points = pm_data.list_raw_prediction_time_points( @@ -213,26 +242,29 @@ def raking_factors( full_time_series = [f"{y}q1" for y in range(1950, 2101)] time_points = sorted(set(time_points) | set(full_time_series)) - print(f"Building raking factors for {len(time_points)} time points.") + # versions = [f"2025_11_08.0{(i + 1):02d}" for i in range(60)] + # time_points = ["2020q1", "2020q2"] - print("Generating raking factors") + print(f"Building raking factors for {len(time_points)} time points.") jobmon.run_parallel( runner="pmtask postprocess", task_name="raking_factors", task_resources={ "queue": queue, "cores": num_cores, - "memory": f"{num_cores * 15}G", - "runtime": "240m", + "memory": f"{num_cores * 5}G", + "runtime": "60m", "project": "proj_rapidresponse", }, node_args={ + # "version": versions, "time-point": time_points, }, task_args={ "version": version, "resolution": resolution, "num-cores": num_cores, + "input-data": input_data, "output-dir": output_dir, }, max_attempts=1, diff --git a/src/rra_population_model/postprocess/upsample/runner.py b/src/rra_population_model/postprocess/upsample/runner.py index bc42413..7c44b91 100644 --- a/src/rra_population_model/postprocess/upsample/runner.py +++ b/src/rra_population_model/postprocess/upsample/runner.py @@ -85,7 +85,7 @@ def upsample_main( model_spec = pm_data.load_model_specification(resolution, version) gdalwarp_path = shutil.which("gdalwarp") - vrt_path = pm_data.compiled_prediction_vrt_path(time_point, model_spec) + vrt_path = pm_data.compiled_prediction_vrt_path(time_point, model_spec, "population") if "f" in spec_name: out_root = pm_data.figure_results / run_stamp @@ -160,7 +160,7 @@ def upsample( pm_data = PopulationModelData(output_dir) compiled_time_points = pm_data.list_compiled_prediction_time_points( - resolution, version + resolution, version, "population" ) time_points = clio.convert_choice(time_point, compiled_time_points) diff --git a/src/rra_population_model/validate/metrics/runner.py b/src/rra_population_model/validate/metrics/runner.py index 1ccb5f1..a1358a6 100644 --- a/src/rra_population_model/validate/metrics/runner.py +++ b/src/rra_population_model/validate/metrics/runner.py @@ -157,6 +157,7 @@ def metrics( pm_data = PopulationModelData(output_dir) time_points = pm_data.list_raked_prediction_time_points(resolution, version) + time_points = [time_point for time_point in time_points if time_point.endswith("q1")] if time_point not in time_points: msg = ( f"Time point {time_point} not found in {resolution} {version}.\n" @@ -167,6 +168,8 @@ def metrics( validation_frame = pm_data.load_validation_frame(resolution) block_keys = list(validation_frame.block_key.unique()) + # versions = [f"2025_11_08.0{(i + 1):02d}" for i in range(60)] + jobmon.run_parallel( runner="pmtask validate", task_name="pixel_metrics", @@ -179,9 +182,11 @@ def metrics( }, node_args={ "block-key": block_keys, - "version": [f"2025_06_21.00{i}" for i in range(1, 5)], + # "version": versions, + # "time-point": time_points, }, task_args={ + "version": version, "resolution": resolution, "output-dir": output_dir, "time-point": time_point,