From 4f9d9086a452321bf0cc657dcf263aef20fd65b0 Mon Sep 17 00:00:00 2001 From: Michele Simionato Date: Fri, 4 Apr 2025 05:46:18 +0200 Subject: [PATCH] Backported fix to taxonomy mapping with consequences --- debian/changelog | 1 + openquake/commonlib/readinput.py | 2 + .../case_13/consequence_recovery_time.csv | 2 +- .../scenario_damage/case_22/consequences.csv | 12 ++-- openquake/risklib/riskmodels.py | 63 ++++++++----------- 5 files changed, 35 insertions(+), 45 deletions(-) diff --git a/debian/changelog b/debian/changelog index 1ee43c83f541..9de26d2be11e 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,4 +1,5 @@ [Michele Simionato] + * Backported fix to taxonomy mapping with consequences * Backported fix to 64 bit poes critical for multifault sources * Backported fix to workerpool critical for zmq clusters diff --git a/openquake/commonlib/readinput.py b/openquake/commonlib/readinput.py index c77e9cac9d4f..0f7343a221a8 100644 --- a/openquake/commonlib/readinput.py +++ b/openquake/commonlib/readinput.py @@ -1053,6 +1053,8 @@ def get_crmodel(oqparam): # build consdict of the form consequence_by_tagname -> tag -> array loss_dt = oqparam.loss_dt() for by, fnames in oqparam.inputs['consequence'].items(): + if by == 'risk_id': + by = 'taxonomy' if isinstance(fnames, str): # single file fnames = [fnames] # i.e. files collapsed.csv, fatalities.csv, ... with headers like diff --git a/openquake/qa_tests_data/scenario_damage/case_13/consequence_recovery_time.csv b/openquake/qa_tests_data/scenario_damage/case_13/consequence_recovery_time.csv index 4380bdb1ccca..e2fb64ffcfe7 100644 --- a/openquake/qa_tests_data/scenario_damage/case_13/consequence_recovery_time.csv +++ b/openquake/qa_tests_data/scenario_damage/case_13/consequence_recovery_time.csv @@ -1,2 +1,2 @@ taxonomy,consequence,peril,slight,moderate,extensive,complete -MUR-CLBRS-LWAL-HBET-1;2/GOV2,losses,groundshaking,35,90,197,287 +UNM/C_LR/GOV2,losses,groundshaking,35,90,197,287 diff --git a/openquake/qa_tests_data/scenario_damage/case_22/consequences.csv b/openquake/qa_tests_data/scenario_damage/case_22/consequences.csv index 146c5eeafaf6..9b5443bd4bae 100644 --- a/openquake/qa_tests_data/scenario_damage/case_22/consequences.csv +++ b/openquake/qa_tests_data/scenario_damage/case_22/consequences.csv @@ -1,7 +1,7 @@ taxonomy,consequence,peril,slight,moderate,extreme,complete -Concrete1,losses,groundshaking,0.04,0.31,0.6,1 -Wood1,losses,groundshaking,0.04,0.31,0.6,1 -Concrete1,losses,liquefaction,0,0,0,1 -Wood1,losses,liquefaction,0,0,0,1 -Concrete1,losses,landslide,0,0,0,1 -Wood1,losses,landslide,0,0,0,1 +Concrete,losses,groundshaking,0.04,0.31,0.6,1 +Wood,losses,groundshaking,0.04,0.31,0.6,1 +Concrete,losses,liquefaction,0,0,0,1 +Wood,losses,liquefaction,0,0,0,1 +Concrete,losses,landslide,0,0,0,1 +Wood,losses,landslide,0,0,0,1 diff --git a/openquake/risklib/riskmodels.py b/openquake/risklib/riskmodels.py index a32ba2b76d14..4d66d8d9cc7e 100644 --- a/openquake/risklib/riskmodels.py +++ b/openquake/risklib/riskmodels.py @@ -18,7 +18,6 @@ import re import json import copy -import logging import functools import collections import numpy @@ -140,7 +139,7 @@ def groupby_id(self): ddic = AccumDict(accum=AccumDict(accum=[])) dic = AccumDict(accum=[]) for rf in self: - dic[rf.id, rf.peril].append(rf) + dic[rf.id, getattr(rf, 'peril', 'groundshaking')].append(rf) for (riskid, peril), rfs in dic.items(): ddic[riskid][peril] = group_by_lt(rfs) num_perils = {riskid: len(ddic[riskid]) for riskid in ddic} @@ -525,33 +524,6 @@ class ValidationError(Exception): pass -# TODO: if the consequence tag is different from "taxonomy", pass -# the values of the exposure for that tag -def check_consequences(fname, taxonomies, perils): - """ - Check that the taxonomy field (if any) and the peril field (if any) - in the consequence file are consistent with the expected taxonomies - and perils - """ - df = pandas.read_csv(fname) - if 'taxonomy' in df.columns: - csq_taxonomies = set(df['taxonomy']) - extra = csq_taxonomies - taxonomies - missing = taxonomies - csq_taxonomies - if not csq_taxonomies & taxonomies: - raise InvalidFile(f'{fname}: no matching taxonomies') - elif missing: - raise InvalidFile(f'{fname}: missing taxonomies {missing}') - elif extra: - # tested in event_based_damage/case_15 - logging.warning(f'In {fname} there are extra taxonomies missing ' - f'in the exposure: {extra}') - if 'peril' in df.columns: - for line, peril in enumerate(df['peril'], 1): - if peril not in perils: - raise InvalidFile(f'{fname}: unknown {peril=} at {line=}') - - class CompositeRiskModel(collections.abc.Mapping): """ A container (riskid, kind) -> riskmodel @@ -624,12 +596,27 @@ def set_tmap(self, tmap_df, taxidx): else: csq_files.append(fnames) for fname in csq_files: - check_consequences(fname, set(taxidx), self.perils) - for byname, coeffs in self.consdict.items(): - # reduce the consdict to the taxonomies in the exposure - self.consdict[byname] = {taxidx[taxo]: arr - for taxo, arr in coeffs.items() - if taxo in taxidx} + df = pandas.read_csv(fname) + if 'peril' in df.columns: + for line, peril in enumerate(df['peril'], 1): + if peril not in self.perils: + raise InvalidFile( + f'{fname}: unknown {peril=} at {line=}') + + cfs = '\n'.join(csq_files) + df = self.tmap_df + for peril in self.perils: + for byname, coeffs in self.consdict.items(): + # ex. byname = "losses_by_taxonomy" + if len(coeffs): + for per, risk_id, weight in zip( + df.peril, df.risk_id, df.weight): + if (per == '*' or per == peril) and risk_id != '?': + try: + coeffs[risk_id][peril] + except KeyError: + raise InvalidFile( + f'Missing {risk_id=}, {peril=} in\n{cfs}') def check_risk_ids(self, inputs): """ @@ -639,7 +626,8 @@ def check_risk_ids(self, inputs): for riskfunc in self.risklist: ids_by_kind[riskfunc.kind].add(riskfunc.id) kinds = tuple(ids_by_kind) # vulnerability, fragility, ... - fnames = [fname for kind, fname in inputs.items() if kind.endswith(kinds)] + fnames = [fname for kind, fname in inputs.items() + if kind.endswith(kinds)] if len(ids_by_kind) > 1: k = next(iter(ids_by_kind)) base_ids = set(ids_by_kind.pop(k)) @@ -687,7 +675,6 @@ def compute_csq(self, assets, dd5, tmap_df, oq): :returns: a dict consequence_name, loss_type -> array[P, A, E] """ # by construction all assets have the same taxonomy - taxi = assets[0]['taxonomy'] P, A, E, _L, _D = dd5.shape csq = AccumDict(accum=numpy.zeros((P, A, E))) for byname, coeffs in self.consdict.items(): @@ -703,7 +690,7 @@ def compute_csq(self, assets, dd5, tmap_df, oq): else: # assume one weigth per peril [w] = df[df.peril == peril].weight coeff = (dd5[pi, :, :, li, 1:] @ - coeffs[taxi][peril][lt] * w) + coeffs[risk_id][peril][lt] * w) cAE = scientific.consequence( consequence, assets, coeff, lt, oq.time_event) csq[consequence, li][pi] += cAE