exglobal_archive_tars.py

#!/usr/bin/env python3

import os

from pygfs.task.archive import Archive
from wxflow import AttrDict, Logger, cast_strdict_as_dtypedict, logit, chdir

# initialize root logger
logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=True)


@logit(logger)
def main():

    config = cast_strdict_as_dtypedict(os.environ)

    # Instantiate the Archive object
    archive = Archive(config)

    # update these keys to be 3 digits if they are part of archive.task_config.keys
    for key in ['OCNRES', 'ICERES']:
        try:
            archive.task_config[key] = f"{archive.task_config[key]:03d}"
        except KeyError:
            logger.info(f"key ({key}) not found in archive.task_config!")

    # Pull out all the configuration keys needed to run the rest of archive steps
    keys = ['ATARDIR', 'current_cycle', 'FHMIN', 'FHMAX', 'FHOUT', 'RUN', 'PDY',
            'DO_VERFRAD', 'DO_VMINMON', 'DO_VERFOZN', 'DO_ICE', 'DO_PREP_OBS_AERO',
            'PARMgfs', 'DO_OCN', 'DO_WAVE', 'WRITE_DOPOST', 'PSLOT',
            'DO_JEDISNOWDA', 'DO_ARCHCOM', 'ARCHCOM_TO', 'ROTDIR', 'ARCH_WARMICFREQ',
            'ARCH_FCSTICFREQ', 'ARCH_CYC', 'assim_freq', 'ARCDIR', 'SDATE',
            'FHMIN_GFS', 'FHMAX_GFS', 'FHOUT_GFS', 'ARCH_GAUSSIAN', 'MODE',
            'FHOUT_OCN', 'FHOUT_ICE', 'FHOUT_OCN_GFS', 'FHOUT_ICE_GFS', 'DO_BUFRSND', 'DOHYBVAR',
            'ARCH_GAUSSIAN_FHMAX', 'ARCH_GAUSSIAN_FHINC', 'ARCH_GAUSSIAN_FHINC',
            'DOIAU', 'OCNRES', 'ICERES', 'NUM_SND_COLLECTIVES', 'FHOUT_WAV', 'FHOUT_WAV_GFS',
            'FHOUT_HF_WAV', 'FHMAX_WAV', 'FHMAX_HF_WAV', 'FHMAX_WAV_GFS',
            'restart_interval_gdas', 'restart_interval_gfs', 'DO_ARCHCOM',
            'DO_AERO_ANL', 'DO_AERO_FCST', 'DO_CA', 'DOBNDPNT_WAVE', 'DO_JEDIOCNVAR', 'DOHYBVAR_OCN',
            'DOLETKF_OCN', 'NMEM_ENS', 'DO_JEDIATMVAR', 'FHMAX_FITS', 'waveGRD',
            'IAUFHRS', 'DO_FIT2OBS', 'NET', 'FHOUT_HF_GFS', 'FHMAX_HF_GFS',
            'OFFSET_START_HOUR', 'ARCH_EXPDIR', 'EXPDIR', 'ARCH_EXPDIR_FREQ', 'ARCH_HASHES',
            'ARCH_DIFFS', 'SDATE', 'EDATE', 'HOMEgfs', 'DO_GEMPAK', 'DATASETS_YAML',
            'WAVE_OUT_GRIDS', 'DO_GSISOILDA', 'DO_LAND_IAU', 'TARBALL_TYPE', 'ATMINC_GRID']

    archive_dict = AttrDict()
    for key in keys:
        archive_dict[key] = archive.task_config.get(key)
        if archive_dict[key] is None:
            logger.warning(f"WARNING: key ({key}) not found in task_config!")

    # Also import all COMIN* and COMOUT* directory and template variables
    for key in archive.task_config.keys():
        if key.startswith(("COM_", "COMIN_", "COMOUT_")):
            archive_dict[key] = archive.task_config.get(key)

    with chdir(config.ROTDIR):
        logger.debug(f"Changed working directory to {config.ROTDIR}")

        # Determine which archives to create
        atardir_sets = archive.configure_tars(archive_dict)

        # Create the backup tarballs and store in ATARDIR
        for atardir_set in atardir_sets:
            logger.debug(f"Processing archive set: {atardir_set['name']}")
            archive.execute_backup_dataset(atardir_set)

        # Clean up any temporary files
        logger.debug("Cleaning up temporary files and directories")
        archive.clean()

    logger.info(f"Returned to working directory {os.getcwd()}")


if __name__ == '__main__':
    main()