Skip to content

use metprodcution for eemep #285

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
heikoklein merged 3 commits into from
Aug 12, 2026
Merged

use metprodcution for eemep #285

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
194058b
use metprodcution for eemep
heikoklein Aug 12, 2026
36a8ec8
reduce file-age check from 10min to 10secs
heikoklein Aug 12, 2026
30bb5aa
nicer epoch_time
heikoklein Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion utils/SnapPy/Snappy/EEMEP/Resources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ class Resources(ResourcesCommon):
_ECINPUTDIRS = ["{LF_PROD_DIR}/atom/Meteorology/EC2CWF/"]
ECVLEVELS = "Vertical_levels48.txt"
EC_FILE_PATTERN = "NRPA_EUROPE_0_1_{UTC:02d}/meteo{year:04d}{month:02d}{day:02d}_{dayoffset:02d}.nc"
_OUTPUTDIR = "{LUSTREDIR}/project/fou/kl/eva/eemep/runs/"
_OUTPUTDIR = "{LF_PROD_DIR}/volcanicash/MetModels/"

def __init__(self):
"""
Expand Down
119 changes: 68 additions & 51 deletions utils/SnapPy/Snappy/Resources.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@
import re
import subprocess
import sys
import time as mtime
from collections import OrderedDict
from datetime import date, datetime, time, timedelta
from time import gmtime, strftime
from time import time as epoch_time

from Snappy import read_dosecoefficients_icrp
from Snappy.ResourcesCommon import ResourcesCommon
Expand All @@ -28,7 +28,7 @@ class MetModel(enum.Enum):

def __eq__(self, other):
return self.value == str(other)

def __str__(self):
return self.value

Expand Down Expand Up @@ -114,7 +114,7 @@ def __init__(self):

def getDefaultMetDefinitions(self, metmodel):
"""get the default meteo-definitions as dict to be used as *dict for getSnapInputMetDefinitions"""
if (metmodel == MetModel.NrpaEC0p1):
if metmodel == MetModel.NrpaEC0p1:
return {
"nx": 1 + round(self.ecDomainWidth / self.ecDomainRes),
"ny": 1 + round(self.ecDomainHeight / self.ecDomainRes),
Expand Down Expand Up @@ -401,7 +401,7 @@ def _getSnapInputTemplate(self, metmodel=None):
Keyword arguments:
metmodel
"""
if (metmodel == MetModel.NrpaEC0p1):
if metmodel == MetModel.NrpaEC0p1:
filename = os.path.join(self.directory, "snap.input_nrpa_ec_0p1.tmpl")
elif metmodel == MetModel.EC0p1Global or metmodel == MetModel.EC0p1Europe:
filename = os.path.join(self.directory, "snap.input_ec_0p1.tmpl")
Expand Down Expand Up @@ -559,7 +559,11 @@ def getECRuns(self):
return relevant

def getRequiredMeteorologyFiles(
self, metmodel, dtime: datetime, run_hours: int, fixed_run="best",
self,
metmodel,
dtime: datetime,
run_hours: int,
fixed_run="best",
):
"""Get available meteorology files for the last few days around dtime and run_hours.
Works for MEPS 2.5km and EC Global Meteorology.
Expand All @@ -572,23 +576,23 @@ def getRequiredMeteorologyFiles(
"""

if metmodel == MetModel.Meps2p5:
utc_list=[0, 3, 6, 9, 12, 15, 18, 21]
max_future_day=3
utc_list = [0, 3, 6, 9, 12, 15, 18, 21]
max_future_day = 3
elif metmodel == MetModel.EC0p1Global:
utc_list=[0, 6, 12, 18]
max_future_day=10
utc_list = [0, 6, 12, 18]
max_future_day = 10
else:
raise NotImplementedError(
f"Metmodel='{metmodel}' not implememented for meteorology"
)

relevant_dates = []

# only best currently implemented
if fixed_run == "best":
if run_hours < 0:
start = dtime + timedelta(hours=run_hours)
finish = dtime #start < finish always
finish = dtime # start < finish always
else:
start = dtime # start < finish always
finish = start + timedelta(hours=run_hours)
Expand All @@ -600,91 +604,105 @@ def getRequiredMeteorologyFiles(
logger.debug((f"finish {finish}"))

if (finish - tomorrow).days >= max_future_day:
logger.debug("Runtime exceeds meteorological availability - run will be cut short.")
logger.debug(
"Runtime exceeds meteorological availability - run will be cut short."
)

days = []
tmp = start
while tmp < min(finish + timedelta(days=1), tomorrow):
days.append(tmp)
tmp += timedelta(days=1)
if days==[]:
#If run is purely in future
if days == []:
# If run is purely in future
days = [
today,
]
today,
]

# loop needs to have latest model runs/hindcast runs last
if start.hour < 3:
i=0
cases = [
(u, d)
for d in range(1,6)
for u in utc_list[::-1]
]
filename=None
while filename is None and i < len(cases)-1:
i = 0
cases = [(u, d) for d in range(1, 6) for u in utc_list[::-1]]
filename = None
while filename is None and i < len(cases) - 1:
new_utc = cases[i][0]
dayoffset = cases[i][1]
file = self.MET_FILENAME_PATTERN[metmodel].format(
UTC=new_utc, year=(start- timedelta(days=dayoffset)).year, month=(start - timedelta(days=dayoffset)).month, day=(start - timedelta(days=dayoffset)).day
)

UTC=new_utc,
year=(start - timedelta(days=dayoffset)).year,
month=(start - timedelta(days=dayoffset)).month,
day=(start - timedelta(days=dayoffset)).day,
)

filename = self._findFileInPathes(
file, self.getMetInputDirs(metmodel)
)
)

i+=1
i += 1

if filename is not None:
relevant_dates.append(filename)
else:
logger.debug("Initial file does not exist")
for day in days:
for utc in utc_list:
if day.date() == (finish + timedelta(hours=3)).date() and utc > (finish + timedelta(hours=3)).hour:
if (
day.date() == (finish + timedelta(hours=3)).date()
and utc > (finish + timedelta(hours=3)).hour
):
return relevant_dates
file = self.MET_FILENAME_PATTERN[metmodel].format(
UTC=utc, year=day.year, month=day.month, day=day.day
)
filename = self._findFileInPathes(
file, self.getMetInputDirs(metmodel)
)

if filename is not None:
relevant_dates.append(filename)

elif utc == 0 and day==days[0] and relevant_dates==[]:
elif utc == 0 and day == days[0] and relevant_dates == []:
logger.debug(f"File {file} doesnt exist")

cases = [
(u, d)
for d in range(1,6)
for u in utc_list[::-1]
]
cases = [(u, d) for d in range(1, 6) for u in utc_list[::-1]]
for new_utc, dayoffset in cases:
file = self.MET_FILENAME_PATTERN[metmodel].format(
UTC=new_utc, year=(day- timedelta(days=dayoffset)).year, month=(day - timedelta(days=dayoffset)).month, day=(day - timedelta(days=dayoffset)).day
)

UTC=new_utc,
year=(day - timedelta(days=dayoffset)).year,
month=(day - timedelta(days=dayoffset)).month,
day=(day - timedelta(days=dayoffset)).day,
)

filename = self._findFileInPathes(
file, self.getMetInputDirs(metmodel)
)
)
if filename is not None:
logger.debug(f"Took {file} instead")
relevant_dates.append(filename)
break
if filename is None:
logger.debug("No alternative file exists")

elif metmodel is MetModel.EC0p1Global and day == days[-1] and (utc == 0 or utc == 12):
logger.debug(f"File {file} doesnt exist -- skipping. Note: long runs may be cut short.")

elif (
metmodel is MetModel.EC0p1Global
and day == days[-1]
and (utc == 0 or utc == 12)
):
logger.debug(
f"File {file} doesnt exist -- skipping. Note: long runs may be cut short."
)
else:
logger.debug(f"File {file} doesnt exist -- skipping")

return relevant_dates

def getMeteorologyFiles(
self, metmodel, dtime: datetime, run_hours: int, fixed_run="best", prod_check=True
self,
metmodel,
dtime: datetime,
run_hours: int,
fixed_run="best",
prod_check=True,
):
"""Get available meteorology files for the last few days around dtime and run_hours.

Expand Down Expand Up @@ -736,15 +754,14 @@ def getMeteorologyFiles(
)
if filename is not None:
fmtime = os.stat(filename).st_mtime
if prod_check and (mtime.time() - fmtime) < (
60 * 10
): # file older than 10min -> no longer under production
# file older than 10sec -> no longer under production
if prod_check and (epoch_time() - fmtime) < 10:
pass
else:
relevant_dates.append(filename)

return relevant_dates

def getECMeteorologyFiles(
self, dtime: datetime, run_hours: int, fixed_run="best", pattern=""
) -> list[str]:
Expand Down Expand Up @@ -993,4 +1010,4 @@ def snapNc_convert_to_grib(snapNc, basedir, ident, isotopes, bitmapCompress=Fals
os.remove(basetempfile)

errlog.close()
outlog.close()
outlog.close()
Loading