Fixed the instantiation of file-dependent GMPEs from the datastore

debian/changelog

@@ -1,6 +1,7 @@
   [Michele Simionato]
+  * Fixed the instantiation of file-dependent GMPEs from the datastore
   * Optimized slow tasks in classical and preclassical calculations,
-    in some situations tripling the speed of the MEX, SAM and USA models
+    in some situations tripling the speed of the SAM and USA models
   * Reading the site model in calculations with ruptures.hdf5 and making
     `minimum_intensity` mandatory
   * Added parameter `minimum_engine_version`

openquake/calculators/tests/scenario_test.py

@@ -16,6 +16,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with OpenQuake. If not, see <http://www.gnu.org/licenses/>.
 
+import os
 import numpy
 from numpy.testing import assert_almost_equal as aae
 from openquake.qa_tests_data.scenario import (
@@ -404,5 +405,14 @@ def test_case_35(self):
         self.run_calc(case_35.__file__, 'job.ini')
         [f] = export(('avg_gmf', 'csv'), self.calc.datastore)
         self.assertEqualFiles('expected/avg_gmf.csv', f, delta=1E-5)
-
-
+        fname = os.path.join(os.path.dirname(case_35.__file__),
+                             'Wcrust_med_rhypo.hdf5')
+        try:
+            # check that even by removing the .hdf5 table
+            # the GMPETable can be instantiated, since
+            # GsimLogicTree.__from__hdf5__ reads from the attributes
+            os.rename(fname, fname + '.bak')
+            gsim_lt = self.calc.datastore['full_lt'].gsim_lt
+            print(gsim_lt)
+        finally:
+            os.rename(fname + '.bak', fname)

openquake/hazardlib/gsim_lt.py

@@ -20,8 +20,10 @@
 import ast
 import copy
 import json
+import shutil
 import logging
 import operator
+import tempfile
 import itertools
 from dataclasses import dataclass
 from collections import defaultdict
@@ -355,18 +357,33 @@ def __toh5__(self):
                             if v is None:  # if volc_arc_file is None
                                 pass
                             else:
+                                # store in the attribute dictionary the data files
                                 fname = os.path.join(dirname, v)
                                 with open(fname, 'rb') as f:
-                                    dic[os.path.basename(v)] = f.read()
+                                    dic[f'{k}:{os.path.basename(v)}'] = f.read()
         return numpy.array(branches, dt), dic
 
     def __fromh5__(self, array, dic):
+        # Here is a smart trick to retrieve the data files from the
+        # dictionary of attributes and store them in a temporary directory,
+        # so that file-dependent GMPEs can be instantiated even if the datastore
+        # is moved to a different machine.
+        # NB: the approach may break on macOS for large files since there is
+        # a limit on the attribute size (unknown at the moment)
+        data = {tuple(k.split(':')): v for k, v in dic.items() if ':' in k}
+        if data:
+            # i.e. {'gmpe_table:Wcrust.hdf5': bytes} in scenario/case_35
+            dirname = tempfile.mkdtemp()
+            for key, name in data:
+                with open(os.path.join(dirname, name), 'wb') as f:
+                    f.write(data[key, name])
+        else:
+            dirname = os.path.dirname(dic['filename'])
         self.bsetdict = json.loads(dic['bsetdict'])
         self.filename = dic['filename']
         self.branches = []
         self.shortener = {}
         self.values = defaultdict(list)
-        dirname = os.path.dirname(dic['filename'])
         for bsno, branches in enumerate(group_array(array, 'trt').values()):
             for brno, branch in enumerate(branches):
                 branch = fix_bytes(branch)
@@ -380,6 +397,8 @@ def __fromh5__(self, array, dic):
                 bt = GsimBranch(branch['trt'], br_id, gsim, weight, True)
                 self.branches.append(bt)
                 self.shortener[br_id] = keyno(br_id, bsno, brno)
+        if data:
+            shutil.rmtree(dirname)
 
     def reduce(self, trts):
         """