From cbbccffd23b38d23aff8c64852ebf95f50eab3ac Mon Sep 17 00:00:00 2001 From: Michele Simionato Date: Wed, 29 Jan 2025 09:00:58 +0100 Subject: [PATCH 1/2] Fixed the instantiation of file-dependent GMPEs from the datastore --- debian/changelog | 3 ++- openquake/calculators/tests/scenario_test.py | 11 ++++++++-- openquake/hazardlib/gsim_lt.py | 23 ++++++++++++++++++-- 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/debian/changelog b/debian/changelog index b92e3bb9fb3a..41d9500aac70 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ [Michele Simionato] + * Fixed the instantiation of file-dependent GMPEs from the datastore * Optimized slow tasks in classical and preclassical calculations, - in some situations tripling the speed of the MEX, SAM and USA models + in some situations tripling the speed of the SAM and USA models * Reading the site model in calculations with ruptures.hdf5 and making `minimum_intensity` mandatory * Added parameter `minimum_engine_version` diff --git a/openquake/calculators/tests/scenario_test.py b/openquake/calculators/tests/scenario_test.py index 87cd61dbd238..81f874caecd2 100644 --- a/openquake/calculators/tests/scenario_test.py +++ b/openquake/calculators/tests/scenario_test.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with OpenQuake. If not, see . +import os import numpy from numpy.testing import assert_almost_equal as aae from openquake.qa_tests_data.scenario import ( @@ -404,5 +405,11 @@ def test_case_35(self): self.run_calc(case_35.__file__, 'job.ini') [f] = export(('avg_gmf', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/avg_gmf.csv', f, delta=1E-5) - - + fname = os.path.join(os.path.dirname(case_35.__file__), + 'Wcrust_med_rhypo.hdf5') + try: + os.rename(fname, fname + '.bak') + gsim_lt = self.calc.datastore['full_lt'].gsim_lt + print(gsim_lt) + finally: + os.rename(fname + '.bak', fname) diff --git a/openquake/hazardlib/gsim_lt.py b/openquake/hazardlib/gsim_lt.py index 5b998b341ed8..53b422c497f8 100644 --- a/openquake/hazardlib/gsim_lt.py +++ b/openquake/hazardlib/gsim_lt.py @@ -20,8 +20,10 @@ import ast import copy import json +import shutil import logging import operator +import tempfile import itertools from dataclasses import dataclass from collections import defaultdict @@ -355,18 +357,33 @@ def __toh5__(self): if v is None: # if volc_arc_file is None pass else: + # store in the attribute dictionary the data files fname = os.path.join(dirname, v) with open(fname, 'rb') as f: - dic[os.path.basename(v)] = f.read() + dic[f'{k}:{os.path.basename(v)}'] = f.read() return numpy.array(branches, dt), dic def __fromh5__(self, array, dic): + # Here is a smart trick to retrieve the data files from the + # dictionary of attributes and store them in a temporary directory, + # so that file-dependent GMPEs can be instantiated even if the datastore + # is moved to a different machine. + # NB: the approach may break on macOS for large files since there is + # a limit on the attribute size (unknown at the moment) + data = {tuple(k.split(':')): v for k, v in dic.items() if ':' in k} + if data: + # i.e. {'gmpe_table:Wcrust.hdf5': bytes} in scenario/case_35 + dirname = tempfile.mkdtemp() + for key, name in data: + with open(os.path.join(dirname, name), 'wb') as f: + f.write(data[key, name]) + else: + dirname = os.path.dirname(dic['filename']) self.bsetdict = json.loads(dic['bsetdict']) self.filename = dic['filename'] self.branches = [] self.shortener = {} self.values = defaultdict(list) - dirname = os.path.dirname(dic['filename']) for bsno, branches in enumerate(group_array(array, 'trt').values()): for brno, branch in enumerate(branches): branch = fix_bytes(branch) @@ -380,6 +397,8 @@ def __fromh5__(self, array, dic): bt = GsimBranch(branch['trt'], br_id, gsim, weight, True) self.branches.append(bt) self.shortener[br_id] = keyno(br_id, bsno, brno) + if data: + shutil.rmtree(dirname) def reduce(self, trts): """ From 2344f16faa47639d30fb4617cd270e4efc6776d9 Mon Sep 17 00:00:00 2001 From: Michele Simionato Date: Wed, 29 Jan 2025 09:06:37 +0100 Subject: [PATCH 2/2] Added comment [ci skip] --- openquake/calculators/tests/scenario_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openquake/calculators/tests/scenario_test.py b/openquake/calculators/tests/scenario_test.py index 81f874caecd2..5df4af519697 100644 --- a/openquake/calculators/tests/scenario_test.py +++ b/openquake/calculators/tests/scenario_test.py @@ -408,6 +408,9 @@ def test_case_35(self): fname = os.path.join(os.path.dirname(case_35.__file__), 'Wcrust_med_rhypo.hdf5') try: + # check that even by removing the .hdf5 table + # the GMPETable can be instantiated, since + # GsimLogicTree.__from__hdf5__ reads from the attributes os.rename(fname, fname + '.bak') gsim_lt = self.calc.datastore['full_lt'].gsim_lt print(gsim_lt)