flow.cylc

#!jinja2

{# fre version should be programatically set, not hard-coded. #}
{% set FRE_VERSION = "2026.01.alpha2" %}

{# global fre-specific verbosity setting here? -q for error only, default is up to warning, -v for info, -vv for debug #}

{# things written between hash and single braces is a Jinja comment. #}
{# Jinja code between single braces is run.                          #}
{# Jinja code with two braces is printed.                            #}

{# The site-specific file (stored in site/ subdir) is included at the end of this file. When working in the main   #}
{# branch, the ppan site file should be specified. The ppan site comprises of gfdl specific tools. Other available #}
{# sites include ppan, gfdl-ws, and generic. The generic.cylc site file is used for a portable flow.cylc.          #}
{# (Probably, we should use similar mechanisms where sensible, e.g. for shared items among workflows)              #}
{% set SITE = SITE %}

{# Setting STALL_TIMEOUT here provides the ability to override the value if necessary #}
{# A use case for overriding this value is exhibited in the fre-workflows pipeline test_cloud_runner.yaml #}
{# In this pipeline, the stall timeout will be 0 in order for the workflow to immediately abort is stalled #}
{% if STALL_TIMEOUT is not defined %}
    {% set STALL_TIMEOUT = "P1W" %}
{% endif %}

{# Set ANALYSIS_START and ANALYSIS_STOP if they do not exist #}
{% if ANALYSIS_START is not defined %}
    {% set ANALYSIS_START = PP_START %}
{% endif %}
{% if ANALYSIS_STOP is not defined %}
    {% set ANALYSIS_STOP = PP_STOP %}
{% endif %}

{# The combined yaml is the definitive configuration source #}
{% set YAML = EXPERIMENT + '.yaml' %}

{# Retrieve active pp components from the yaml #}
{% set PP_COMPONENTS = YAML | get_components %}

[meta]
    title = "Postprocessing Example 1"
    description = """
    Postprocessing example that includes Bronx-like functionality,
    """
    URL = https://gitlab.gfdl.noaa.gov/fre2/workflows/postprocessing

[scheduler]
    {# Implicit tasks are tasks without explicit runtime definitions in [runtime], often typos. #}
    {# (For prototyping and graphing, change this to True.)                                     #}
    allow implicit tasks = false

    {# Configure the directories and files to be included in the remote file installation #}
    install = app/*, bin/*, etc/*, lib/*
    UTC mode = True

    [[events]]
        # When the stall timeout is reached, a stalled workflow will exit
        # and remove the /xtmp working directory.
        stall timeout = {{ STALL_TIMEOUT }}
{% if DO_WORKFLOW_EMAIL is defined and DO_WORKFLOW_EMAIL %}
        mail events = startup, shutdown, abort, stall, inactivity timeout, restart timeout
{% endif %}

[task parameters]
{# The task parameters (except for component) depend on configuration in Rose apps,                            #}
{# so custom Jinja triggers are used to form the lists. Cylc and custom Jinja triggers have these rules:       #}
{# 1. Triggers live in Jinja2Filters/ and must be python                                       #}
{# 2. The trigger filename must be <trigger-name>.py                                           #}
{# 3. In the file, trigger must be defined as a function with the same name                    #}
{# 4. First argument is passed through as a pipe, and the rest are passed through in the trigger arguments.    #}
{# More info:                                                      #}
{# The python loaded within Cylc has access to some useful Cylc utilities (e.g. metomi.isodatetime.parsers).   #}
{# For development/testing, use "module load conda; conda activate cylc". Then the Cylc python will be loaded. #}
{% set REGRID  =  "regrid-xy" | form_task_parameters('temporal', PP_COMPONENTS, YAML) %}
{% set REGRID_STATIC  =  "regrid-xy" | form_task_parameters('static', PP_COMPONENTS, YAML) %}

{% set DO_REGRID = REGRID|length %}
{% set DO_REGRID_STATIC = REGRID_STATIC|length %}

{% if DO_REGRID %}
    regrid = {{ REGRID }}
{% endif %}

{% if DO_REGRID_STATIC %}
    regrid_static = {{ REGRID_STATIC }}
{% endif %}

{% set NATIVE  =  "native" | form_task_parameters('temporal', PP_COMPONENTS, YAML) %}
{% set NATIVE_STATIC  =  "native" | form_task_parameters('static', PP_COMPONENTS, YAML) %}

{% set DO_NATIVE = NATIVE|length %}
{% set DO_NATIVE_STATIC = NATIVE_STATIC|length %}

{% if DO_NATIVE %}
    native = {{ NATIVE }}
{% endif %}

{% if DO_NATIVE_STATIC %}
    native_static = {{ NATIVE_STATIC }}
{% endif %}

    {# Standard Jinja2 triggers are also available (e.g. "replace") #}
    component = {{ PP_COMPONENTS | replace(' ', ', ') }}



[scheduling]
    initial cycle point = {{ PP_START }}
    final cycle point = {{ PP_STOP }}
    # max number of active cycle points (default 5)
    # this should be something sensible assuming tasks with required outputs
    # (i.e. not marked as "ok to fail") such as analysis scripts.
    # But until the analysis scripts can be marked optional then a workaround
    # is to set this to be very high, ~100 cycle points.
    # However, the cost of keeping many cycle points active at once are starting
    # stage-history tasks first, possibly filling /xtmp before the clean tasks run.
    runahead limit = P20
    [[queues]]
        # Limit the entire workflow to 100 active tasks. Only allow a single cleaning
        # or data-catalog task to run at once, as they can fail if run in parallel.
        [[[default]]]
            limit = 100
        [[[clean]]]
            limit = 1
            members = CLEAN
        [[[data-catalog]]]
            limit = 1
            members = DATA-CATALOG
    [[graph]]
{% if DO_ANALYSIS_ONLY %}
    {{ YAML | get_analysis_info('task-graph', PP_COMPONENTS, PP_DIR, PP_START, PP_STOP, PP_CHUNKS, DO_ANALYSIS_ONLY) }}
{% else %}
        #
        # Every history-file segment: Stage, refineDiag, regrid, split
        #
        {{ HISTORY_SEGMENT }} = """
            pp-starter => STAGE-HISTORY

    {% if DO_REFINEDIAG %}
            stage-history => REFINE-DIAG
            REFINE-DIAG:succeed-all => stage-history-refined
    {% endif %}

    {% if DO_PREANALYSIS %}
            stage-history => PRE-ANALYSIS
    {% endif %}

    {% if DO_REGRID %}
            STAGE-HISTORY:succeed-all => regrid-xy<regrid> => split-netcdf-regrid<regrid>
        {% if DO_ATMOS_PLEVEL_MASKING %}
            STAGE-HISTORY:succeed-all => mask-atmos-plevel<regrid> => regrid-xy<regrid>
        {% endif %}
    {% endif %}

    {% if DO_NATIVE %}
            STAGE-HISTORY:succeed-all => split-netcdf-native<native>
        {% if DO_ATMOS_PLEVEL_MASKING %}
            STAGE-HISTORY:succeed-all => mask-atmos-plevel<native> => split-netcdf-native<native>
        {% endif %}
    {% endif %}

    {% if REFINED_HISTORY_DIR is defined %}
        {% if DO_REGRID %}
            stage-history-refined => regrid-xy<regrid>
        {% endif %}
        {% if DO_NATIVE %}
            stage-history-refined => split-netcdf-native<native>
        {% endif %}
    {% endif %}

    {% if CLEAN_WORK %}
        {% if DO_REGRID %}
            regrid-xy<regrid>                 => clean-history-native
            split-netcdf-regrid<regrid>       => clean-history-regrid-xy & clean-split-regrid-xy
        {% endif %}
        {% if DO_NATIVE %}
            split-netcdf-native<native>       => clean-history-native & clean-split-native
        {% endif %}
    {% endif %}
        """

        #
        # Once at the beginning: Statics
        #
        R1 = """
            pp-starter => STAGE-HISTORY

    {% if DO_REGRID_STATIC %}
            STAGE-HISTORY:succeed-all => regrid-xy<regrid_static> => split-netcdf-regrid<regrid_static>
    {% endif %}

    {% if DO_NATIVE_STATIC %}
            STAGE-HISTORY:succeed-all => split-netcdf-native<native_static>
    {% endif %}

    {% if DO_NATIVE_STATIC and DO_REGRID_STATIC %}
            split-netcdf-regrid<regrid_static> & split-netcdf-native<native_static>
            => remap-pp-components-static<component> => combine-statics
    {% elif DO_NATIVE_STATIC %}
            split-netcdf-native<native_static> => remap-pp-components-static<component> => combine-statics
    {% elif DO_REGRID_STATIC %}
            split-netcdf-regrid<regrid_static> => remap-pp-components-static<component> => combine-statics
    {% endif %}

    {% if REFINED_HISTORY_DIR is defined %}
        {% if DO_REGRID_STATIC %}
            stage-history-refined => regrid-xy<regrid_static>
        {% endif %}

        {% if DO_NATIVE_STATIC %}
            stage-history-refined => split-netcdf-native<native_static>
        {% endif %}
    {% endif %}

    {% if CLEAN_WORK and DO_REGRID_STATIC %}
            regrid-xy<regrid_static> => clean-history-native
            split-netcdf-regrid<regrid_static> => clean-history-regrid-xy & clean-split-regrid-xy
            remap-pp-components-static<component> => clean-shards-static
            combine-statics => clean-pp-statics
    {% endif %}

    {% if CLEAN_WORK and DO_NATIVE_STATIC %}
            split-netcdf-native<native_static> => clean-history-native & clean-split-native
            remap-pp-components-static<component> => clean-shards-static
            combine-statics => clean-pp-statics
    {% endif %}
        """

        #
        # Timeseries
        #
    {% for chunk in PP_CHUNKS | iter_chunks(HISTORY_SEGMENT, PP_START, PP_STOP) %}
        R1/{{ chunk.cycle_point }} = """

        {% if DO_REGRID %}
            split-netcdf-regrid<regrid>
            {% for SEGMENT in chunk.segments[1:] %}
            & split-netcdf-regrid<regrid>[{{ SEGMENT }}]
            {% endfor %}
            {% if HISTORY_SEGMENT != chunk.chunk_size | string %}
                => make-timeseries-regrid-{{ chunk.chunk_size }}<regrid>
            {% endif %}
        {% endif %}

        {% if DO_NATIVE %}
            split-netcdf-native<native>
            {% for SEGMENT in chunk.segments[1:] %}
            & split-netcdf-native<native>[{{ SEGMENT }}]
            {% endfor %}
            {% if HISTORY_SEGMENT != chunk.chunk_size | string %}
                => make-timeseries-native-{{ chunk.chunk_size }}<native>
            {% endif %}
        {% endif %}

        {% if DO_REGRID %}
            {{ "regrid-xy" | form_remap_dep('temporal', chunk.chunk_size, PP_COMPONENTS, 'ts', YAML, HISTORY_SEGMENT) }}
        {% endif %}

        {% if DO_NATIVE %}
            {{ "native" | form_remap_dep('temporal', chunk.chunk_size, PP_COMPONENTS, 'ts', YAML, HISTORY_SEGMENT) }}
        {% endif %}

        {% if CLEAN_WORK %}
            REMAP-PP-COMPONENTS-TS-{{ chunk.chunk_size }}:succeed-all => clean-shards-ts-{{ chunk.chunk_size }}
        {% endif %}
        """

        R1/$ = REMAP-PP-COMPONENTS-TS-{{ chunk.chunk_size }}[{{ chunk.cycle_point }}]:succeed-all => data-catalog-final

        {% if CLEAN_WORK %}
            {% for segment in chunk.segments %}
                R1/{{ segment }} = """
                    REMAP-PP-COMPONENTS-TS-{{ chunk.chunk_size }}[{{ chunk.cycle_point }}]:succeed-all
                    => clean-shards-ts-{{ HISTORY_SEGMENT }}
                """
            {% endfor %}
        {% endif %}
    {% endfor %}

        #
        # Climatologies task graph
        #
    {{ YAML | get_climatology_info('task-graph') }}

        #
        # Analysis task graph
        #
    {% if DO_ANALYSIS %}
        {{ YAML | get_analysis_info('task-graph', PP_COMPONENTS, PP_DIR, PP_START, PP_STOP, PP_CHUNKS, DO_ANALYSIS_ONLY) }}
    {% endif %}

{% endif %}

[runtime]
    [[pp-starter]]
        inherit = PP-STARTER
        # NOTE! script must appear *before* [[[enviroment]]] or else
        # the job scripts will have quoting issues
        script = """
            # Returns 0 if ready, 1 if not
            #
            echo Arguments:
            echo "    INFO: trigger: $targetFile"
            echo "    INFO: cycle point: $CYLC_TASK_CYCLE_POINT"

            echo "    Resolving date strings: YYYY, MM, DD, HH"
            IFS='-' read YYYY MM DD << DATES
            $(cylc cycle-point --template CCYY-MM-DD)
            DATES

            echo "    Resolving target file"
            HH=$(cylc cycle-point --template hh)
            targetFileResolved=$(echo $targetFile | sed -e "s/YYYY/$YYYY/g" -e "s/MM/$MM/g" -e "s/DD/$DD/g" -e "s/HH/$HH/g")
            echo "    trigger (date expanded): $targetFileResolved"

            echo "Looking for resolved target file"
            if ls $targetFileResolved; then
                echo "Target ${targetFileResolved} is ready"
                exit 0
            else
                echo "Error: target ${targetFileResolved} is not ready"
                exit 1
            fi
        """
        [[[environment]]]
            targetFile = {{ HISTORY_DIR }}/YYYYMMDD.nc.tar

    [[STAGE-HISTORY]]
        # Increase time limit for staging jobs to 8 hours
        execution time limit = PT8H
        script = """
            set -euo pipefail

            # Stage the contents of a tarfile to PTMP
            echo Arguments:
            echo "    history dir: $historyDir"
            echo "    ptmp dir: $ptmpDir"
            echo "    cycle point: $CYLC_TASK_CYCLE_POINT"
            echo "    TMPDIR (not used yet): ${TMPDIR:=/tmp}"
            echo Utilities:
            type hsmget
            type cylc

            echo "    Verifying history directory exists: ${historyDir}"
            if [[ ! -d $historyDir ]]; then
                echo "Error: Archive directory ${historyDir} does not exist or isnt a directory"
                exit 1
            fi

            echo "    Resolving history file..."
            file=$historyDir/$(cylc cycle-point --template CCYYMMDD).nc.tar

            echo "    Looking for resolved file: ${file}"
            if ls ${file}; then
                echo "Going to stage ${file} to PTMP"
            else
                echo "Error: Could not locate ${file}"
                exit 1
            fi

            # We want to stage the history files to PTMP without transferring to VFTMP.
            # Calling hsmget on a file with wildcards that doesn't exist does this,
            # e.g. "dummy*" as below, but prints a warning
            echo "    calling hsmget, staging files to PTMP without touching VFTMP"
            if hsmget -v -t -a $historyDir -p $ptmpDir$historyDir -w $TMPDIR$historyDir $(basename -s .tar $file)/dummy\*; then
                echo "History files in ${file} have been staged to PTMP successfully"
            else
                echo "Error: Cant stage history files in ${file} to PTMP"
                exit 1
            fi

            # Check history files for expected number of timesteps
            echo "    checking data integrity with fre pp histval..."
            fre -vv pp histval --warn --history $ptmpDir$historyDir/$(basename -s .tar $file) \
                --date_string $(cylc cycle-point --template CCYYMMDD)

            echo "    Linking staged files to ${CYLC_WORKFLOW_SHARE_DIR}"
            mkdir -p $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/history/native
            cd $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/history/native
            ln -f $ptmpDir/$historyDir/$(cylc cycle-point --template CCYYMMDD).nc/* . \
                || ln -sf $ptmpDir/$historyDir/$(cylc cycle-point --template CCYYMMDD).nc/* .

            # save any diag manifests
            cd $ptmpDir/$historyDir/$(cylc cycle-point --template CCYYMMDD).nc
            if ls *diag_manifest.yaml.*; then
                cp *diag_manifest.yaml.* $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT
                echo "Copied diag manifests to $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT"
            else
                echo "No diag manifests found"
            fi

            echo "Natural end of the history file staging"
            exit 0
        """
    [[stage-history]]
        inherit = STAGE-HISTORY
        [[[environment]]]
            historyDir = {{ HISTORY_DIR }}
            ptmpDir = {{ PTMP_DIR }}


{% if REFINED_HISTORY_DIR is defined %}
    [[stage-history-refined]]
        inherit = STAGE-HISTORY
        post-script = """
        """
        [[[environment]]]
            historyDir = {{ REFINED_HISTORY_DIR }}
            ptmpDir = {{ PTMP_DIR }}

{% endif %}

    [[MAKE-TIMEAVGS]]
        script = """
            fre -vv app gen-time-averages-wrapper \
                --cycle-point     $CYLC_TASK_CYCLE_POINT \
                --dir             $CYLC_WORKFLOW_SHARE_DIR/shards \
                --sources         $sources \
                --output-interval $output_interval \
                --input-interval  $input_interval \
                --grid            $grid \
                --frequency       $frequency
        """
    {% if DO_NATIVE %}
    [[MAKE-TIMEAVGS-NATIVE]]
        [[[environment]]]
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts/native
            outputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/av/native
            component = $CYLC_TASK_PARAM_native
    {% endif %}
    {% if DO_REGRID %}
    [[MAKE-TIMEAVGS-REGRID]]
        [[[environment]]]
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts/regrid-xy
            outputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/av/regrid-xy
            component = $CYLC_TASK_PARAM_regrid
            use_subdirs = 1
    {% endif %}
    {% for PP_CHUNK in PP_CHUNKS %}
    [[MAKE-TIMEAVGS-{{ PP_CHUNK }}]]
        [[[environment]]]
            interval = {{ PP_CHUNK }}
        {% if DO_NATIVE %}
    [[MAKE-TIMEAVGS-NATIVE-{{ PP_CHUNK }}]]
        inherit = MAKE-TIMEAVGS, MAKE-TIMEAVGS-NATIVE, MAKE-TIMEAVGS-{{ PP_CHUNK }}
    [[make-timeavgs-native-{{ PP_CHUNK }}<native>]]
        inherit = MAKE-TIMEAVGS-NATIVE-{{ PP_CHUNK }}, <native>
        {% endif %}
        {% if DO_REGRID %}
    [[MAKE-TIMEAVGS-REGRID-{{ PP_CHUNK }}]]
        inherit = MAKE-TIMEAVGS, MAKE-TIMEAVGS-REGRID, MAKE-TIMEAVGS-{{ PP_CHUNK }}
    [[make-timeavgs-regrid-{{ PP_CHUNK }}<regrid>]]
        inherit = MAKE-TIMEAVGS-REGRID-{{ PP_CHUNK }}, <regrid>
        {% endif %}
    {% endfor %}

{{ YAML | get_climatology_info('task-definitions') }}

{% if DO_REFINEDIAG or DO_PREANALYSIS %}
    [[PRE-ANALYSIS]]
        pre-script = """
            env
            set -x
            mkdir -p $work $tempCache $refineDiagDir
            cd $work/$hsmdate
            ls
        """
        [[[environment]]]
            name = {{ EXPERIMENT }}
            rtsxml = no-more-xml
            work = $TMPDIR/work
            tempCache = $TMPDIR/tempCache
            root = $CYLC_WORKFLOW_RUN_DIR
            archive = is-this-needed
            scriptName = $CYLC_TASK_LOG_DIR/job
            oname = $(cylc cycle-point --template CCYYMMDD)
            hsmdate = $oname.nc
            ptmpDir = {{ PTMP_DIR }}/{{ HISTORY_DIR }}
            histDir = {{ HISTORY_DIR }}
            platform = {{ PLATFORM }}
            target = {{ TARGET }}
            segment_months = {{ HISTORY_SEGMENT }}
            basedate = is-this-needed
            gridspec = {{ PP_GRID_SPEC }}
            refineDiagDir = $TMPDIR/history_refineDiag/$hsmdate
            catalog = {{ PP_DIR }}/catalog.json
{% endif %}
{% if DO_REFINEDIAG %}
    [[REFINE-DIAG]]
        inherit = PRE-ANALYSIS
        post-script = """
            cd $refineDiagDir
            if ls *nc; then
                refinedCount=$(ls -1 *nc | wc -l)
            else
                refinedCount=0
            fi
            if [[ $refinedCount > 0 ]]; then
                for file in $(ls -1 *nc); do
                    list_ncvars.csh -st01234 $file |& tee $CYLC_WORKFLOW_SHARE_DIR/refineDiag.log
                done
            else
                echo ERROR: RefineDiag script did not create any NetCDF files as it was expected to do
                exit 1
            fi
            if [[ -f {{ REFINED_HISTORY_DIR }}/$oname.nc.tar ]]; then
                echo "the contents of {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} is..."
                ls {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}
                echo "the contents of {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}/$oname.nc is..."
                ls {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }}/$oname.nc
                hsmget -v -t -a {{ REFINED_HISTORY_DIR }} -p {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} \
                    -w $TMPDIR/modify_refineDiag $hsmdate/\*
                mv -f * $TMPDIR/modify_refineDiag
                mv -f $TMPDIR/modify_refineDiag/* .
                rm -rf $TMPDIR/modify_refineDiag
            fi
            hsmput -v -t -s tar -a {{ REFINED_HISTORY_DIR }} -p {{ PTMP_DIR }}/{{ REFINED_HISTORY_DIR }} \
                -w $TMPDIR/history_refineDiag $hsmdate
        """
{# The following section executes refineDiag tasks, whether single or multiple refineDiag paths are given #}
{# Create a 'macro' function to reduce code repeat #}
    {% macro exec_refine(script_name) -%}
            inherit = REFINE-DIAG
            script = """
                ls
                echo NOTE: About to source user script
                if csh {{ script_name }}; then
                    echo NOTE: User script exited normally
                else
                    echo ERROR: User script got an error status $?
                    exit 1
                fi
            """
    {%- endmacro %}
{# Use the 'split' function to transform string into an iterable list, including of length 1 #}
    {% set refineDiag_scripts = REFINEDIAG_SCRIPTS.split(' ') %}
    {% for refineDiag_path in refineDiag_scripts %}
            [[refineDiag-{{ refineDiag_path[refineDiag_path.rfind('/')+1:refineDiag_path.rfind('.')] | replace(".", "_") }}]]
        {{ exec_refine(refineDiag_path) }}
    {% endfor %}
{% endif %}

{% if DO_PREANALYSIS and PREANALYSIS_SCRIPT is defined %}
    {% set preanalysis_name = PREANALYSIS_SCRIPT[PREANALYSIS_SCRIPT.rfind('/')+1:PREANALYSIS_SCRIPT.rfind('.')] |
        replace(".", "_") %}
    [[preAnalysis-{{ preanalysis_name }}]]
        inherit = PRE-ANALYSIS
        script = """
            ls
            echo NOTE: About to source user script
            if csh {{ PREANALYSIS_SCRIPT }}; then
                echo NOTE: User script exited normally
            else
                echo ERROR: User script got an error status $?
                exit 1
            fi
        """
{% endif %}

    [[MASK-ATMOS-PLEVEL]]
        script = """
            set -euo pipefail
            echo "component = ${component}"

            echo "changing dir to ${inputDir}"
            cd $inputDir

            echo "trying to find files..."
            files=$(find . -maxdepth 1 \( -name "*.${component}.nc" -o -name "*.${component}.*.nc" \))
            echo "found files at depth 1 = ${files}"
            echo ""

            echo "trying to loop over files: ${files}"
            for file in $files; do
                echo "running: fre -vv app mask-atmos-plevel -i ${file} -o ${file}.masked -p ${file}"
                fre -vv app mask-atmos-plevel --warn-no-ps -i $file -o $file.masked -p $file
                echo "done running: fre -vv app mask-atmos-plevel -i ${file} -o ${file}.masked -p ${file}"
                if [[ -f $file.masked ]]; then
                    echo "renaming ${file}.masked..."
                    mv -f $file.masked $file
                    echo "renamed to ${file}"
                fi
            done
            echo "mask-atmos-plevel task done for ${component}"
        """
        [[[environment]]]
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/history/native
{% if DO_NATIVE %}
    [[mask-atmos-plevel<native>]]
        inherit = MASK-ATMOS-PLEVEL, <native>
        [[[environment]]]
            component = $CYLC_TASK_PARAM_native
{% endif %}
{% if DO_REGRID %}
    [[mask-atmos-plevel<regrid>]]
        inherit = MASK-ATMOS-PLEVEL, <regrid>
        [[[environment]]]
            component = $CYLC_TASK_PARAM_regrid
{% endif %}

    [[SPLIT-NETCDF]]
        script = """
            # Split history files by variable
            mkdir -p $splitOutputDir
            fre -vv pp split-netcdf-wrapper \
                -i $splitInputDir \
                -o $splitOutputDir \
                -s $history_file \
                --split-all-vars \
                $use_subdirs

            # Enable nullglob so the wildcard expands to nothing if no files match
            shopt -s nullglob

            # Initialize an empty array to hold our command-line arguments
            use_diag_manifest=()
            found_manifest="false"

            # Loop through all matching files and append them to the array
            for file in "$CYLC_WORKFLOW_SHARE_DIR"/cycle/$CYLC_TASK_CYCLE_POINT/*diag_manifest.yaml.*; do
                use_diag_manifest+=("--diag-manifest=$file")
                found_manifest="true"
            done

            # If the array has at least one item, copy the files
            if [[ "$found_manifest" == "true" ]]; then
                cp "$CYLC_WORKFLOW_SHARE_DIR"/cycle/$CYLC_TASK_CYCLE_POINT/*diag_manifest.yaml.* .
            fi

            mkdir -p $renameOutputDir

            # Expand the array into separate arguments using "${array[@]}"
            fre -vv pp rename-split \
                --input-dir="$renameInputDir" \
                --output-dir="$renameOutputDir" \
                --component="$history_file" \
                $use_subdirs \
                "${use_diag_manifest[@]}"
        """
        [[[environment]]]
            date = $CYLC_TASK_CYCLE_POINT

{% if DO_NATIVE %}
    [[SPLIT-NETCDF-NATIVE]]
        inherit = SPLIT-NETCDF
        [[[environment]]]
            history_file = $CYLC_TASK_PARAM_native
            use_subdirs = ""

            splitInputDir = $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/history/native
            splitOutputDir = $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/split/native

            renameInputDir = $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/split/native
            renameOutputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts/native
{% endif %}

{% if DO_REGRID %}
    [[SPLIT-NETCDF-REGRID]]
        inherit = SPLIT-NETCDF
        [[[environment]]]
            history_file = $CYLC_TASK_PARAM_regrid
            use_subdirs = --use-subdirs

            splitInputDir = $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/history/regrid-xy
            splitOutputDir = $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/split/regrid-xy

            renameInputDir = $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/split/regrid-xy
            renameOutputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts/regrid-xy
{% endif %}

{% if DO_NATIVE %}
    [[split-netcdf-native<native>]]
        inherit = SPLIT-NETCDF-NATIVE, <native>

    {% if DO_NATIVE_STATIC %}
    [[split-netcdf-native<native_static>]]
        inherit = SPLIT-NETCDF-NATIVE, <native_static>
        [[[environment]]]
            history_file = $CYLC_TASK_PARAM_native_static
    {% endif %}
{% endif %}

{% if DO_REGRID %}
    [[split-netcdf-regrid<regrid>]]
        inherit = SPLIT-NETCDF-REGRID, <regrid>

    {% if DO_REGRID_STATIC %}
    [[split-netcdf-regrid<regrid_static>]]
        inherit = SPLIT-NETCDF-REGRID, <regrid_static>
        [[[environment]]]
            history_file = $CYLC_TASK_PARAM_regrid_static
    {% endif %}
{% endif %}

    [[REMAP-PP-COMPONENTS]]
        [[[environment]]]
            yaml_config = $CYLC_WORKFLOW_RUN_DIR/{{ YAML }}
            components = $CYLC_TASK_PARAM_component

    [[REMAP-PP-COMPONENTS-TS]]
        inherit = REMAP-PP-COMPONENTS
        script = """
            mkdir -p $outputDir
            fre -vv app remap --input-dir=$inputDir \
                                --output-dir=$outputDir \
                                --begin-date=$begin \
                                --current-chunk=$currentChunk \
                                --product=$product \
                                --ts-workaround \
                                --pp-component=$components \
                                --copy-tool=$COPY_TOOL \
                                --yaml-config=$yaml_config \
                                --ens-mem=${ens_mem:-""}
        """
        [[[environment]]]
            components = $CYLC_TASK_PARAM_component
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts
            outputDir = {{ PP_DIR }}
            product = ts
    {% for PP_CHUNK in PP_CHUNKS %}
    [[REMAP-PP-COMPONENTS-TS-{{ PP_CHUNK }}]]
        inherit = REMAP-PP-COMPONENTS-TS
        [[[environment]]]
            begin = $(cylc cycle-point)
            currentChunk = {{ PP_CHUNK }}
    [[remap-pp-components-ts-{{ PP_CHUNK }}<component>]]
        inherit = REMAP-PP-COMPONENTS-TS-{{ PP_CHUNK }}, <component>
    {% endfor %}

    [[REMAP-PP-COMPONENTS-AV]]
        inherit = REMAP-PP-COMPONENTS
        script = """
            mkdir -p $outputDir
            fre -vv app remap --input-dir=$inputDir \
                                --output-dir=$outputDir \
                                --begin-date=$begin \
                                --current-chunk=$currentChunk \
                                --product=$product \
                                --pp-component=$components \
                                --copy-tool=$COPY_TOOL \
                                --yaml-config=$yaml_config \
                                --ens-mem=${ens_mem:-""}
        """
        [[[environment]]]
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/av
            outputDir = $CYLC_WORKFLOW_SHARE_DIR/pp/av
            product = av
    {% for PP_CHUNK in PP_CHUNKS %}
    [[REMAP-PP-COMPONENTS-AV-{{ PP_CHUNK }}]]
        inherit = REMAP-PP-COMPONENTS-AV
        [[[environment]]]
            begin = $(cylc cycle-point)
            currentChunk = {{ PP_CHUNK }}
    [[remap-pp-components-av-{{ PP_CHUNK }}<component>]]
        inherit = REMAP-PP-COMPONENTS-AV-{{ PP_CHUNK }}, <component>
    {% endfor %}

{% if DO_REGRID_STATIC or DO_NATIVE_STATIC %}
    [[remap-pp-components-static<component>]]
        inherit = REMAP-PP-COMPONENTS, <component>
        script = """
            mkdir -p $outputDir
            fre -vv app remap --input-dir=$inputDir \
                                --output-dir=$outputDir \
                                --begin-date=$begin \
                                --current-chunk=$currentChunk \
                                --product=$product \
                                --pp-component=$components \
                                --copy-tool=$COPY_TOOL \
                                --yaml-config=$yaml_config \
                                --ens-mem=${ens_mem:-""}
        """
        [[[environment]]]
            begin = $CYLC_TASK_CYCLE_POINT
            currentChunk = P0Y
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts
            outputDir = $CYLC_WORKFLOW_SHARE_DIR/pp/static
            product = static
{% endif %}

    [[DATA-CATALOG]]
        script = data-catalog
        [[[environment]]]
            PP_DIR = {{ PP_DIR }}
    [[data-catalog-final]]
        inherit = DATA-CATALOG

    [[COMBINE-TIMEAVGS]]
        script = """
            fre -vv app combine-time-averages \
                --in-dir=$inputDir/$component \
                --out-dir=$outputDir \
                --component=$component \
                --begin=$begin \
                --end=$end \
                --frequency=$frequency \
                --interval=$interval
        """
        [[[environment]]]
            begin = $(cylc cycle-point --print-year)
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/pp/av
            outputDir = {{ PP_DIR }}

    [[MAKE-TIMESERIES]]
        script = """
            mkdir -p $outputDir
            rose task-run --verbose --app-key make-timeseries
        """
        [[[environment]]]
            fail_ok_components = grid_spec lumip_Lyr lumip_Lyr_crp lumip_Lyr_psl lumip_Lyr_pst
            pp_stop = {{ PP_STOP }}

{% if DO_NATIVE %}
    [[MAKE-TIMESERIES-NATIVE]]
        [[[environment]]]
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts/native
            outputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts/native
            component = $CYLC_TASK_PARAM_native
{% endif %}

{% if DO_REGRID %}
    [[MAKE-TIMESERIES-REGRID]]
        [[[environment]]]
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts/regrid-xy
            outputDir = $CYLC_WORKFLOW_SHARE_DIR/shards/ts/regrid-xy
            component = $CYLC_TASK_PARAM_regrid
            use_subdirs = 1
{% endif %}

    {% for PP_CHUNK in PP_CHUNKS %}
    [[MAKE-TIMESERIES-{{ PP_CHUNK }}]]
        [[[environment]]]
            begin = $(cylc cycle-point)
            inputChunk = {{ HISTORY_SEGMENT }}
            outputChunk = {{ PP_CHUNK }}
        {% if DO_NATIVE %}
    [[MAKE-TIMESERIES-NATIVE-{{ PP_CHUNK }}]]
        inherit = MAKE-TIMESERIES, MAKE-TIMESERIES-NATIVE, MAKE-TIMESERIES-{{ PP_CHUNK }}
    [[make-timeseries-native-{{ PP_CHUNK }}<native>]]
        inherit = MAKE-TIMESERIES-NATIVE-{{ PP_CHUNK }}, <native>
        {% endif %}
        {% if DO_REGRID %}
    [[MAKE-TIMESERIES-REGRID-{{ PP_CHUNK }}]]
        inherit = MAKE-TIMESERIES, MAKE-TIMESERIES-REGRID, MAKE-TIMESERIES-{{ PP_CHUNK }}
    [[make-timeseries-regrid-{{ PP_CHUNK }}<regrid>]]
        inherit = MAKE-TIMESERIES-REGRID-{{ PP_CHUNK }}, <regrid>
        {% endif %}
    {% endfor %}

{% if DO_REGRID %}
    [[REGRID-XY]]
        script = """
            mkdir -p $outputDir $remapDir

            fre -vv app regrid \
                --yamlfile=$yaml_config \
                --input_dir=$inputDir \
                --output_dir=$outputDir \
                --work_dir=$workDir \
                --remap_dir=$remapDir \
                --source=$source \
                --input_date=$inputDate
        """
        # set back to longer when it works TEMP / TODO
        # there may be a race condition that causes the fast jobs to fail
        execution retry delays = PT1M
        [[[environment]]]
            yaml_config = $CYLC_WORKFLOW_RUN_DIR/{{ YAML }}
            inputDir    = $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/history/native
            outputDir   = $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/history/regrid-xy
            remapDir    = $CYLC_WORKFLOW_SHARE_DIR/fregrid-remap-files
            workDir     = $TMPDIR
            inputDate   = $CYLC_TASK_CYCLE_POINT

    [[regrid-xy<regrid>]]
        inherit = REGRID-XY, <regrid>
        [[[environment]]]
            source = $CYLC_TASK_PARAM_regrid
    {% if DO_REGRID_STATIC %}
    [[regrid-xy<regrid_static>]]
        inherit = REGRID-XY, <regrid_static>
        [[[environment]]]
            source = $CYLC_TASK_PARAM_regrid_static
    {% endif %}
{% endif %}

{% if DO_REGRID_STATIC or DO_NATIVE_STATIC %}
    [[combine-statics]]
        script = """
            mkdir -p $outputDir
            rose task-run --verbose --app-key combine-statics
        """
        [[[environment]]]
            inputDir = $CYLC_WORKFLOW_SHARE_DIR/pp/static
            outputDir = {{ PP_DIR }}
{% endif %}

{% if DO_ANALYSIS %}
    [[PUBLISH-ANALYSIS]]
        [[[environment]]]
            dora_url = dora.gfdl.noaa.gov
    [[BUILD-ANALYSIS]]
        [[[environment]]]
            ANALYSIS_URL = https://github.com/NOAA-GFDL/analysis-scripts.git
    [[ANALYSIS]]
        # Increase time limit for analysis jobs to 24 hours
        execution time limit = PT24H
        [[[environment]]]
            # internally-used vars
            outputDir = $CYLC_WORKFLOW_SHARE_DIR/legacy-analysis-scripts
            # accepted FRE environment variables
            FRE_ANALYSIS_HOME = /home/fms/local/opt/fre-analysis/test
            FRE_ANALYSIS_GIT_URL = file:///home/fms/local/opt/fre-analysis/git
            FRE_ANALYSIS_ARCHIVE = /archive/fms/fre-analysis/test
            # fre-analysis template vars
            out_dir = {{ ANALYSIS_DIR }}
            descriptor = {{ EXPERIMENT }}
            fremodule = fre/{{ FRE_VERSION }}
            WORKDIR = $TMPDIR/{{ EXPERIMENT }}/{{ PLATFORM }}-{{ TARGET }}/$CYLC_TASK_CYCLE_POINT
            hist_dir = {{ HISTORY_DIR }}
            gridspecfile = {{ PP_GRID_SPEC }}
            # new template vars
            catalog = {{ PP_DIR }}/catalog.json
            experiment_yaml = $CYLC_WORKFLOW_RUN_DIR/{{ EXPERIMENT }}.yaml
    {{ YAML | get_analysis_info('task-definitions', PP_COMPONENTS, PP_DIR, PP_START, PP_STOP, PP_CHUNKS, False ) }}
{% endif %}

    [[CLEAN]]
        pre-script = "set -x"

    [[clean-history-native]]
        inherit = CLEAN
        script = "rm -rf $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/history/native"

{% if DO_REGRID %}
    [[clean-history-regrid-xy]]
        inherit = CLEAN
        script = "rm -rf $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/history/regrid-xy"
{% endif %}

{% if DO_REGRID_STATIC or DO_NATIVE_STATIC %}
    [[clean-pp-statics]]
        inherit = CLEAN
        script = "rm -rf $CYLC_WORKFLOW_SHARE_DIR/pp/static"
{% endif %}
{% if DO_NATIVE %}
    [[clean-split-native]]
        inherit = CLEAN
        script = "rm -rf $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/split/native"
{% endif %}
{% if DO_REGRID %}
    [[clean-split-regrid-xy]]
        inherit = CLEAN
        script = "rm -rf $CYLC_WORKFLOW_SHARE_DIR/cycle/$CYLC_TASK_CYCLE_POINT/split/regrid-xy"
{% endif %}

{% if DO_REGRID_STATIC or DO_NATIVE_STATIC %}
    [[clean-shards-static]]
        inherit = CLEAN
        script = """
            rm -rf $CYLC_WORKFLOW_SHARE_DIR/shards/ts/native/*/P0Y
            rm -rf $CYLC_WORKFLOW_SHARE_DIR/shards/ts/regrid-xy/*/*/P0Y
        """
{% endif %}

    [[CLEAN]]
    [[CLEAN-SHARDS-TS]]
        inherit = CLEAN
        script = """
            dirs="$CYLC_WORKFLOW_SHARE_DIR/shards/ts/native/*/*/$duration"
            dirs+=" $CYLC_WORKFLOW_SHARE_DIR/shards/ts/regrid-xy/*/*/*/$duration"
            for dir in $dirs; do
                if [[ -d $dir ]]; then
                    if [[ $duration == P1Y ]]; then
                        find $dir -type f -name "*.$(cylc cycle-point --template CCYY)*.nc" -print -delete
                    else
                        find $dir -type f -name "*.$(cylc cycle-point --template CCYY)*-*.nc" -print -delete
                    fi
                else
                    echo "Skipping '$dir' as it does not exist"
                fi
            done
        """
    [[CLEAN-SHARDS-AV]]
        inherit = CLEAN
        script = """
            dirs="$CYLC_WORKFLOW_SHARE_DIR/shards/av/native/*/*/$duration"
            dirs+=" $CYLC_WORKFLOW_SHARE_DIR/shards/av/regrid-xy/*/*/*/$duration"
            for dir in $dirs; do
                if [[ -d $dir ]]; then
                    if [[ $duration == P1Y ]]; then
                        find $dir -type f -name "*.$(cylc cycle-point --template CCYY)*.nc" -print -delete
                    else
                        find $dir -type f -name "*.$(cylc cycle-point --template CCYY)*-*.nc" -print -delete
                    fi
                else
                    echo "Skipping '$dir' as it does not exist"
                fi
            done
        """
    [[clean-shards-ts-{{ HISTORY_SEGMENT }}]]
        inherit = CLEAN-SHARDS-TS
        [[[environment]]]
            duration = {{ HISTORY_SEGMENT }}

    {% for PP_CHUNK in PP_CHUNKS %}
    [[clean-shards-ts-{{ PP_CHUNK }}]]
        inherit = CLEAN-SHARDS-TS
        [[[environment]]]
            duration = {{ PP_CHUNK }}
    {% endfor %}

    [[CLEAN-PP-TIMEAVGS]]
        inherit = CLEAN
        script = """
            dirs=$CYLC_WORKFLOW_SHARE_DIR/pp/av/*/*/$duration
            for dir in $dirs; do
                if [[ $duration == P1Y ]]; then
                    find $dir -type f -name "*.$(cylc cycle-point --template CCYY)*.nc" -print -delete
                else
                    find $dir -type f -name "*.$(cylc cycle-point --template CCYY)*-*.nc" -print -delete
                fi
            done
        """

# Site-specific settings
{% include 'site/' ~ SITE ~ '.cylc' %}

{# this is required for portability should probably only include #}
{# inherit = <some_task_param> if SITE = ppan_test #}
{# but empty task-family inheritence is inconsequential for now. #}
{% if SITE != 'ppan_test' %}
    {% if DO_REGRID %}
    [[<regrid>]]
        {% if DO_REGRID_STATIC %}
    [[<regrid_static>]]
        {% endif %}
    {% endif %}
    {% if DO_NATIVE %}
    [[<native>]]
        {% if DO_NATIVE_STATIC %}
    [[<native_static>]]
        {% endif %}
    {% endif %}
    [[<component>]]
    [[PP-STARTER]]
{% endif %}