diff --git a/debian/changelog b/debian/changelog index b92e3bb9fb3a..41d9500aac70 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,6 +1,7 @@ [Michele Simionato] + * Fixed the instantiation of file-dependent GMPEs from the datastore * Optimized slow tasks in classical and preclassical calculations, - in some situations tripling the speed of the MEX, SAM and USA models + in some situations tripling the speed of the SAM and USA models * Reading the site model in calculations with ruptures.hdf5 and making `minimum_intensity` mandatory * Added parameter `minimum_engine_version` diff --git a/openquake/calculators/tests/scenario_test.py b/openquake/calculators/tests/scenario_test.py index 87cd61dbd238..5df4af519697 100644 --- a/openquake/calculators/tests/scenario_test.py +++ b/openquake/calculators/tests/scenario_test.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU Affero General Public License # along with OpenQuake. If not, see . +import os import numpy from numpy.testing import assert_almost_equal as aae from openquake.qa_tests_data.scenario import ( @@ -404,5 +405,14 @@ def test_case_35(self): self.run_calc(case_35.__file__, 'job.ini') [f] = export(('avg_gmf', 'csv'), self.calc.datastore) self.assertEqualFiles('expected/avg_gmf.csv', f, delta=1E-5) - - + fname = os.path.join(os.path.dirname(case_35.__file__), + 'Wcrust_med_rhypo.hdf5') + try: + # check that even by removing the .hdf5 table + # the GMPETable can be instantiated, since + # GsimLogicTree.__from__hdf5__ reads from the attributes + os.rename(fname, fname + '.bak') + gsim_lt = self.calc.datastore['full_lt'].gsim_lt + print(gsim_lt) + finally: + os.rename(fname + '.bak', fname) diff --git a/openquake/hazardlib/gsim_lt.py b/openquake/hazardlib/gsim_lt.py index 5b998b341ed8..53b422c497f8 100644 --- a/openquake/hazardlib/gsim_lt.py +++ b/openquake/hazardlib/gsim_lt.py @@ -20,8 +20,10 @@ import ast import copy import json +import shutil import logging import operator +import tempfile import itertools from dataclasses import dataclass from collections import defaultdict @@ -355,18 +357,33 @@ def __toh5__(self): if v is None: # if volc_arc_file is None pass else: + # store in the attribute dictionary the data files fname = os.path.join(dirname, v) with open(fname, 'rb') as f: - dic[os.path.basename(v)] = f.read() + dic[f'{k}:{os.path.basename(v)}'] = f.read() return numpy.array(branches, dt), dic def __fromh5__(self, array, dic): + # Here is a smart trick to retrieve the data files from the + # dictionary of attributes and store them in a temporary directory, + # so that file-dependent GMPEs can be instantiated even if the datastore + # is moved to a different machine. + # NB: the approach may break on macOS for large files since there is + # a limit on the attribute size (unknown at the moment) + data = {tuple(k.split(':')): v for k, v in dic.items() if ':' in k} + if data: + # i.e. {'gmpe_table:Wcrust.hdf5': bytes} in scenario/case_35 + dirname = tempfile.mkdtemp() + for key, name in data: + with open(os.path.join(dirname, name), 'wb') as f: + f.write(data[key, name]) + else: + dirname = os.path.dirname(dic['filename']) self.bsetdict = json.loads(dic['bsetdict']) self.filename = dic['filename'] self.branches = [] self.shortener = {} self.values = defaultdict(list) - dirname = os.path.dirname(dic['filename']) for bsno, branches in enumerate(group_array(array, 'trt').values()): for brno, branch in enumerate(branches): branch = fix_bytes(branch) @@ -380,6 +397,8 @@ def __fromh5__(self, array, dic): bt = GsimBranch(branch['trt'], br_id, gsim, weight, True) self.branches.append(bt) self.shortener[br_id] = keyno(br_id, bsno, brno) + if data: + shutil.rmtree(dirname) def reduce(self, trts): """