Merge pull request #135 from xylar/detect-dev-labels-in-bootstrap

Automatically detect E3SM-Unified version and dev labels

Author: xylar

.github/workflows/build_workflow.yml

@@ -44,7 +44,10 @@ jobs:
       - name: Finalize Build Environment
         run: |
           conda install -y conda conda-build
-          conda build -m "recipes/e3sm-unified/configs/mpi_${{ matrix.mpi }}_python${{ matrix.python-version }}.yaml" "recipes/e3sm-unified"
+          cd recipes/e3sm-unified
+          ./build_packages.py --conda ${CONDA_PREFIX} \
+              --python ${{ matrix.python-version }} \
+              --mpi ${{ matrix.mpi }}
 
       - name: Install E3SM-Unified
         run: |

e3sm_supported_machines/bootstrap.py

@@ -14,6 +14,7 @@
 from shared import (
     check_call,
     get_conda_base,
+    get_rc_dev_labels,
     install_miniforge3,
     parse_args,
 )
@@ -132,24 +133,20 @@ def build_env(is_test, recreate, compiler, mpi, conda_mpi, version,
 
     if is_test:
 
-        nco_dev = ('alpha' in nco_spec or 'beta' in nco_spec)
-
         channels = '--override-channels'
         if local_conda_build is not None:
             channels = f'{channels} -c {local_conda_build}'
 
-        if nco_dev:
-            channels = f'{channels} -c conda-forge/label/nco_dev'
-        for package in ['moab']:
-            spec = config.get('spack_specs', package)
-            if 'rc' in spec.lower():
-                channels = f'{channels} -c conda-forge/label/{package}_dev'
-
-        # edit if not using a release candidate for a given package
-        dev_labels = ['e3sm_unified', 'chemdyg', 'e3sm_diags',
-                      'mpas_analysis', 'zppy', 'zstash']
-        for package in dev_labels:
-            channels = f'{channels} -c conda-forge/label/{package}_dev'
+        meta_yaml_path = os.path.join(
+            os.path.dirname(__file__),
+            "..",
+            "recipes",
+            "e3sm-unified",
+            "meta.yaml"
+        )
+        dev_labels = get_rc_dev_labels(meta_yaml_path)
+        for dev_label in dev_labels:
+            channels = f'{channels} -c conda-forge/label/{dev_label}'
         channels = f'{channels} ' \
                    f'-c conda-forge '
     else:

e3sm_supported_machines/shared.py

@@ -8,13 +8,28 @@
 try:
     from urllib.request import urlopen, Request
 except ImportError:
+    # For Python 2 compatibility
     from urllib2 import urlopen, Request
 
+LABELS = {
+     "chemdyg": "chemdyg_dev",
+     "e3sm_diags": "e3sm_diags_dev",
+     "e3sm_to_cmip": "e3sm_to_cmip_dev",
+     "mache": "mache_dev",
+     "moab": "moab_dev",
+     "mpas-analysis": "mpas_analysis_dev",
+     "mpas_tools": "mpas_tools_dev",
+     "nco": "nco_dev",
+     "xcdat": "xcdat_dev",
+     "zppy": "zppy_dev",
+     "zstash": "zstash_dev",
+}
+
 
 def parse_args(bootstrap):
     parser = argparse.ArgumentParser(
         description='Deploy E3SM-Unified')
-    parser.add_argument("--version", dest="version", default="1.11.1",
+    parser.add_argument("--version", dest="version",
                         help="The version of E3SM-Unified to deploy")
     parser.add_argument("--conda", dest="conda_base",
                         help="Path for the  conda base")
@@ -56,6 +71,16 @@ def parse_args(bootstrap):
         raise ValueError('You must supply both or neither of '
                          '--mache_fork and --mache_branch')
 
+    if args.version is None:
+        meta_yaml_path = os.path.join(
+            os.path.dirname(__file__),
+            "..",
+            "recipes",
+            "e3sm-unified",
+            "meta.yaml"
+        )
+        args.version = get_version_from_meta(meta_yaml_path)
+
     return args
 
 
@@ -125,3 +150,64 @@ def get_conda_base(conda_base, config, shared):
     # handle "~" in the path
     conda_base = os.path.abspath(os.path.expanduser(conda_base))
     return conda_base
+
+
+def get_rc_dev_labels(meta_yaml_path):
+    """Parse meta.yaml and return a list of dev labels for RC dependencies."""
+
+    # a rare case where module-level imports are not a good idea because
+    # the deploy_e3sm_unified.py script may be called from an environment
+    # where jinja2 and yaml are not installed.
+    import yaml
+    from jinja2 import Template
+
+    labels_dict = LABELS
+
+    # Render the jinja template with dummy/default values
+    with open(meta_yaml_path) as f:
+        template_text = f.read()
+    # Provide dummy/default values for all jinja variables used in meta.yaml
+    template = Template(template_text)
+    rendered = template.render(
+        mpi='mpich',  # or any valid value
+        py='310',     # or any valid value
+        CONDA_PY='310',  # used in build string
+    )
+    meta = yaml.safe_load(rendered)
+    dev_labels = []
+    run_reqs = meta.get("requirements", {}).get("run", [])
+    for req in run_reqs:
+        # req can be a string like "pkgname version" or just "pkgname"
+        if isinstance(req, str):
+            parts = req.split()
+            pkg = parts[0]
+            version = " ".join(parts[1:]) if len(parts) > 1 else ""
+
+            # NCO is special: it has a dev label for alpha/beta versions
+            if pkg == "nco" and ('alpha' in version or 'beta' in version):
+                label = labels_dict[pkg]
+                if label not in dev_labels:
+                    dev_labels.append(label)
+
+            # Only match 'rc' in version, not in pkg name
+            if "rc" in version and pkg in labels_dict:
+                label = labels_dict[pkg]
+                if label not in dev_labels:
+                    dev_labels.append(label)
+    return dev_labels
+
+
+def get_version_from_meta(meta_yaml_path):
+    """Parse the version from the {% set version = ... %} line in meta.yaml."""
+    with open(meta_yaml_path) as f:
+        for line in f:
+            if line.strip().startswith("{% set version"):
+                # e.g., {% set version = "1.11.1rc1" %}
+                parts = line.split("=")
+                if len(parts) >= 2:
+                    version = (
+                        parts[1].strip().strip('%}').strip().strip(
+                            '"').strip("'")
+                    )
+                    return version
+    raise ValueError("Could not find version in meta.yaml")

recipes/e3sm-unified/build_package.bash

@@ -0,0 +1,102 @@
+#!/usr/bin/env python3
+import os
+import shutil
+import subprocess
+import argparse
+
+from jinja2 import Template
+
+from shared import get_rc_dev_labels, get_version_from_meta
+
+DEV_PYTHON_VERSIONS = ["3.10"]
+DEV_MPI_VERSIONS = ["nompi", "hpc"]
+
+RELEASE_PYTHON_VERSIONS = ["3.9", "3.10"]
+RELEASE_MPI_VERSIONS = ["nompi", "mpich", "openmpi", "hpc"]
+
+
+def generate_matrix_files(dev, python_versions, mpi_versions):
+    with open("configs/template.yaml") as f:
+        template_text = f.read()
+    template = Template(template_text)
+    if python_versions is None:
+        if dev:
+            python_versions = DEV_PYTHON_VERSIONS
+        else:
+            python_versions = RELEASE_PYTHON_VERSIONS
+    if mpi_versions is None:
+        if dev:
+            mpi_versions = DEV_MPI_VERSIONS
+        else:
+            mpi_versions = RELEASE_MPI_VERSIONS
+    matrix_files = []
+    for python in python_versions:
+        for mpi in mpi_versions:
+            script = template.render(python=python, mpi=mpi)
+            filename = f"configs/mpi_{mpi}_python{python}.yaml"
+            with open(filename, "w") as handle:
+                handle.write(script)
+            matrix_files.append(filename)
+    return matrix_files
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Build E3SM-Unified conda packages."
+    )
+    parser.add_argument(
+        "--conda",
+        type=str,
+        default=os.path.expanduser("~/miniforge3"),
+        help="Path to the conda base directory (default: ~/miniforge3)."
+    )
+    parser.add_argument(
+        "--python",
+        nargs="+",
+        help="Python version(s) to build for (overrides default matrix)."
+    )
+    parser.add_argument(
+        "--mpi",
+        nargs="+",
+        help="MPI variant(s) to build for (overrides default matrix)."
+    )
+    args = parser.parse_args()
+
+    conda_dir = os.path.expanduser(args.conda)
+    meta_yaml_path = os.path.join(os.path.dirname(__file__), "meta.yaml")
+    version = get_version_from_meta(meta_yaml_path)
+    dev = "rc" in version
+
+    # Remove conda-bld directory if it exists
+    bld_dir = os.path.join(conda_dir, "conda-bld")
+    if os.path.exists(bld_dir):
+        shutil.rmtree(bld_dir)
+
+    # Generate matrix files on the fly
+    matrix_files = generate_matrix_files(
+        dev,
+        python_versions=args.python,
+        mpi_versions=args.mpi,
+    )
+
+    dev_labels = []
+    if dev:
+        dev_labels = get_rc_dev_labels(meta_yaml_path)
+
+    channels = []
+    for label in dev_labels:
+        channels.extend(["-c", f"conda-forge/label/{label}"])
+    channels += [
+        "-c", "conda-forge"
+    ]
+
+    for file in matrix_files:
+        cmd = [
+            "conda", "build", "-m", file, "--override-channels"
+        ] + channels + ["."]
+        print("Running:", " ".join(cmd))
+        subprocess.run(cmd, check=True)
+
+
+if __name__ == "__main__":
+    main()