[WIP] Add hillslope scripts (to python/ctsm/) and external, plus docs

.git-blame-ignore-revs

@@ -25,9 +25,12 @@ e4d38681df23ccca0ae29581a45f8362574e0630
 a9d96219902cf609636886c7073a84407f450d9a
 d866510188d26d51bcd6d37239283db690af7e82
 0dcd0a3c1abcaffe5529f8d79a6bc34734b195c7
+179e2eeb1ad919539e21f92947fcc80c9b054869
 e096358c832ab292ddfd22dd5878826c7c788968
 475831f0fb0e31e97f630eac4e078c886558b61c
 fd5f177131d63d39e79a13918390bdfb642d781e
+4b03f5a93d781d8ef0ca58fa88fc510784ba9b1a
+# Ran SystemTests and python/ctsm through black python formatter
 a51816e0de380300b69db9fc3e2c7fa83b267b64
 5364ad66eaceb55dde2d3d598fe4ce37ac83a93c
 8056ae649c1b37f5e10aaaac79005d6e3a8b2380
@@ -37,6 +40,11 @@ a51816e0de380300b69db9fc3e2c7fa83b267b64
 183fc26a6691bbdf87f515dc47924a64be3ced9b
 6fccf682eaf718615407d9bacdd3903b8786a03d
 2500534eb0a83cc3aff94b30fb62e915054030bf
+# Ran individual files through black python formatter
+f05fcbc10eda44e574de7554c8ac2b0417215b61
+eb4906d8ee78b4f5ad5b3b990c414f6e113b661f
+63c4873eab8af9fdd1f89bfe0a5416dcc53394fa
+7053b2980da08bb46f82d9cf1a5c4c63b98b37f6
 78d05967c2b027dc9776a884716597db6ef7f57c
 47839a77229c61555e3b8932927bb54cdc511b27
 a0d014fae9550dd9ffbc934abd29ef16176f8208
@@ -67,3 +75,5 @@ cdf40d265cc82775607a1bf25f5f527bacc97405
 3b7a2876933263f8986e4069f5d23bd45635756f
 3dd489af7ebe06566e2c6a1c7ade18550f1eb4ba
 742cfa606039ab89602fde5fef46458516f56fd4
+7fcf6e393377158bd4764e0c5c0698036b0f7879
+29543c6f5e56bfcd1da5e86f622059274aafa365

.github/workflows/docs-build-and-deploy.yml

@@ -12,6 +12,7 @@ on:
       # Include all include::ed files outside doc/ directory!
       - 'src/README.unit_testing'
       - 'tools/README'
+      - 'tools/external/representative-hillslopes/shell_scripts/README.md'
 
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:

.github/workflows/docs-omnibus.yml

@@ -12,6 +12,7 @@ on:
       # Include all include::ed files outside doc/ directory!
       - 'src/README.unit_testing'
       - 'tools/README'
+      - 'tools/external/representative-hillslopes/shell_scripts/README.md'
 
   pull_request:
     # Run on pull requests that change the listed files
@@ -23,6 +24,7 @@ on:
       # Include all include::ed files outside doc/ directory!
       - 'src/README.unit_testing'
       - 'tools/README'
+      - 'tools/external/representative-hillslopes/shell_scripts/README.md'
 
   workflow_dispatch:
 

.github/workflows/docs.yml

@@ -14,6 +14,7 @@ on:
       # Include all include::ed files outside doc/ directory!
       - 'src/README.unit_testing'
       - 'tools/README'
+      - 'tools/external/representative-hillslopes/shell_scripts/README.md'
 
   pull_request:
     # Run on pull requests that change the listed files
@@ -26,6 +27,7 @@ on:
       # Include all include::ed files outside doc/ directory!
       - 'src/README.unit_testing'
       - 'tools/README'
+      - 'tools/external/representative-hillslopes/shell_scripts/README.md'
 
   workflow_dispatch:
 

.gitignore

@@ -61,6 +61,7 @@ buildnmlc
 td.*.status
 td.*.log
 td.*.status.xFail
+.coverage
 
 # mksurfdata output
 surfdata_*.log
@@ -115,4 +116,3 @@ Depends
 # Docs build and testing output
 _build*/
 _publish*/
-doc/.coverage

.gitmodules

@@ -128,3 +128,11 @@ fxtag = v2.2.6
 fxrequired = ToplevelOptional
 # Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
 fxDONOTUSEurl = https://github.com/ESMCI/doc-builder
+
+[submodule "representative-hillslopes"]
+path = tools/external/representative-hillslopes
+url = https://github.com/swensosc/Representative_Hillslopes
+fxtag = f3f980fc40783b9a86143e97bb3773f7761ed29b
+fxrequired = ToplevelOptional
+# Standard Fork to compare to with "git fleximod test" to ensure personal forks aren't committed
+fxDONOTUSEurl = https://github.com/swensosc/Representative_Hillslopes

doc/source/users_guide/using-clm-tools/creating-hillslope-datasets.rst

@@ -0,0 +1,77 @@
+.. _rst_creating_hillslope_datasets:
+
+===========================
+Creating hillslope datasets
+===========================
+
+Creating hillslope datasets for use in CTSM is a multi-step process:
+
+#. Perform the Height Above Nearest Drainage (HAND) analysis, which processes the DEM data for each gridcell, breaking the work into a number of "chunks."
+#. Combine the per-gridcell files from the HAND analysis into one file per chunk.
+#. Combine the per-chunk files into a ``hillslope_file`` for use in CTSM.
+
+HAND analysis
+===============
+
+The HAND analysis uses the optional ``Representative_Hillslopes`` submodule. Before following these instructions, do ``bin/git-fleximod update representative-hillslopes`` to make sure that's been downloaded. Then the instructions assume:
+
+#. You've installed the ``Representative_Hillslopes`` Conda environment, if needed (``conda env create -f tools/external/representative-hillslopes/conda_yml/casper.yml``).
+#. You've activated that environment (``conda activate Representative_Hillslopes``).
+#. You've changed your working directory to ``tools/external/representative-hillslopes/shell_scripts/``.
+
+.. mdinclude:: ../../../../tools/external/representative-hillslopes/shell_scripts/README.md
+
+Gridcell files to chunk files
+=============================
+
+(Instructions here assume you have activated the :ref:`ctsm_pylib<using-ctsm-pylib>` Conda environment.)
+
+This is performed by the ``tools/hillslopes/combine_gridcell_files`` script; do ``tools/hillslopes/combine_gridcell_files --help`` for usage help. Briefly, you need to provide three things:
+
+* ``-i``, input surface dataset: The CTSM surface dataset (``fsurdat``) you used in the HAND analysis above.
+* ``-d``, input directory: The directory the HAND analysis above saved files to (``outdir`` in those instructions).
+* ``-o``, output directory: The directory where you want the chunk files generated by ``combine_gridcell_files`` to be saved.
+
+Example
+--------
+
+This example can be run on the NSF NCAR Casper machine; it assumes you've activated the ``ctsm_pylib`` Conda environment.
+
+.. code-block:: bash
+
+    fsurdat=python/ctsm/test/testinputs/surfdata_5x5_amazon_hist_16pfts_CMIP6_2000_c231031.nc
+    indir=$SCRATCH/hillslopes_5x5_amazon/hand_analysis_global
+    outdir=$SCRATCH/hillslopes_5x5_amazon/hand_analysis_global/combined
+
+    python tools/hillslopes/combine_gridcell_files \
+        -i $fsurdat \
+        -d $indir \
+        -o $outdir
+
+
+Chunk files to ``hillslope_file``
+=================================
+
+(Instructions here assume you have activated the ``ctsm_pylib`` Conda environment.)
+
+This is performed by the ``tools/hillslopes/combine_chunk_files`` script; do ``tools/hillslopes/combine_chunk_files --help`` for usage help. Briefly, you need to provide three things:
+
+* ``-i``, input surface dataset: The CTSM surface dataset (``fsurdat``) you used in the steps above.
+* ``-d``, input directory: The directory where ``combine_gridcell_files`` saved the chunk files (``-o`` in those instructions above).
+* ``-o``, output file: The file path where you want the output ``hillslope_file`` to be saved.
+
+Example
+--------
+
+This example can be run on the NSF NCAR Casper machine; it assumes you've activated the ``ctsm_pylib`` Conda environment.
+
+.. code-block:: bash
+
+    fsurdat=python/ctsm/test/testinputs/surfdata_5x5_amazon_hist_16pfts_CMIP6_2000_c231031.nc
+    indir=$SCRATCH/hillslopes_5x5_amazon/hand_analysis_global/combined
+    outfile=${indir}/$(basename ${fsurdat} | sed "s@surfdata@hilldata@")
+
+    python tools/hillslopes/combine_chunk_files \
+        -i $fsurdat \
+        -d $indir \
+        -o $outfile

doc/source/users_guide/using-clm-tools/index.rst

@@ -19,6 +19,7 @@ Using CLM tools
    using-ctsm-pylib.rst
    creating-input-for-surface-dataset-generation.rst
    creating-surface-datasets.rst
+   creating-hillslope-datasets.rst
    creating-domain-files.rst
    observational-sites-datasets.rst
    cprnc.rst