Skip to content

No_tag columns in the exposure #10583

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

No_tag columns in the exposure #10583

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9b66052
No_tag columns in the exposure
May 13, 2025
75b0b3b
Fixed test
May 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions debian/changelog
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,6 @@
[Michele Simionato]
* Missing columns in the exposure.csv files are now automatically
filled with 'No_tag'
* Fixed downloading of uncertainty.xml in OQImpact calculations

[Michele Simionato, Paolo Tormene]
Expand Down
2 changes: 1 addition & 1 deletion openquake/commonlib/tests/readinput_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -407,7 +407,7 @@ def test_Lon_instead_of_lon(self):
fname = os.path.join(DATADIR, 'exposure.xml')
with self.assertRaises(InvalidFile) as ctx:
asset.Exposure.read_all([fname])
self.assertIn("missing {'lon'}", str(ctx.exception))
self.assertIn('missing column "lon"', str(ctx.exception))

def test_case_similar(self):
fname = os.path.join(DATADIR, 'exposure2.xml')
Expand Down
24 changes: 12 additions & 12 deletions ...uake/qa_tests_data/scenario_risk/conditioned/Exposure/Aggregated/Exposure_Ind_Albania.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
LONGITUDE,LATITUDE,TAXONOMY,MACRO_TAXONOMY,BUILDINGS,OCCUPANCY,AREA_PER_BUILDING_SQM,COST_PER_AREA_EUR,TOTAL_REPL_COST_EUR,COST_STRUCTURAL_EUR,COST_NONSTRUCTURAL_EUR,COST_CONTENTS_EUR,OCCUPANTS_PER_ASSET,OCCUPANTS_PER_ASSET_DAY,OCCUPANTS_PER_ASSET_NIGHT,OCCUPANTS_PER_ASSET_TRANSIT,OCCUPANTS_PER_ASSET_AVERAGE,ID_1,NAME_1,ID_2,NAME_2,NAME_3,ID_3,COST_PER_AREA_USD,TOTAL_REPL_COST_USD,COST_STRUCTURAL_USD,COST_NONSTRUCTURAL_USD,COST_CONTENTS_USD,ASSET_ID,NAME_0,ID_0,SETTLEMENT,TOTAL_AREA_SQM
19.9685999999999,41.5808,S/LFM+CDM/H:1/IND,"Steel, low rise",19.002721922000003,Ind,315,500.0,2992928.7027150006,448939.30540725007,748232.1756787502,1795757.2216290005,0,104.29,9.48,17.06,43.61,2,Diber,6,Matit,Komsi,42654,570.13,3412689.51,511903.43,853172.38,2047613.71,Ind_19,Albania,ALB,Total,5985.857405430001
20.027,41.0970999999999,S/LFM+CDM/H:1/IND,"Steel, low rise",77.1707656,Ind,315,500.0,12154395.582,1823159.3373,3038598.8955,7292637.349199999,0,98.64,8.96,16.13,41.25,4,Elbasan,9,Elbasanit,Bradashesh,44980,570.13,13859059.96,2078858.99,3464764.99,8315435.97,Ind_124,Albania,ALB,Total,24308.791164
20.0938,41.0720999999999,S/LFM+CDM/H:1/IND,"Steel, low rise",1.5839975366,Ind,315,500.0,249479.6120145,37421.941802175,62369.903003625,149687.76720869998,0,9.17,0.83,1.5,3.83,4,Elbasan,9,Elbasanit,Shirgjan,44998,570.13,284469.34,42670.4,71117.34,170681.6,Ind_269,Albania,ALB,Total,498.959224029
19.9936,41.0138,CR+PC/LFM+CDM/H:1/IND,"Concrete frame, low rise",25.719836415,Ind,315,525.0,4253417.947130625,638012.6920695937,1063354.4867826563,2552050.768278375,0,94.58,8.6,15.47,39.55,4,Elbasan,9,Elbasanit,Gostime,44987,598.63,4849963.45,727494.52,1212490.86,2909978.07,Ind_290,Albania,ALB,Total,8101.748470725
19.9936,40.9971,MUR/LWAL+CDN/H:2/IND,"Unreinforced masonry, low rise",24.91549791,Ind,315,475.0,3727981.374783749,559197.2062175624,931995.3436959372,2236788.8248702497,0,91.63,8.33,14.99,38.31,4,Elbasan,9,Elbasanit,Gostime,44987,541.62,4250833.95,637625.09,1062708.49,2550500.37,Ind_297,Albania,ALB,Total,7848.38184165
19.6597,40.7719,S+SL/LFM+CDM/H:1/IND,"Steel, low rise",19.785874464000003,Ind,315,500.0,3116275.2280800007,467441.284212,779068.8070200002,1869765.136848,0,30.33,2.76,4.96,12.68,5,Fier,13,Fierit,Roskovec,45d140,570.13,3553335.49,533000.32,888333.87,2132001.3,Ind_368,Albania,ALB,Total,6232.550456160001
19.6597,40.7552,S/LFM+CDM/H:1/IND,"Steel, low rise",42.50348063999999,Ind,315,500.0,6694298.200799999,1004144.7301199996,1673574.5501999997,4016578.920479999,0,16.92,1.54,2.77,7.07,5,Fier,13,Fierit,Kuman,45d132,570.13,7633179.25,1144976.89,1908294.81,4579907.55,Ind_394,Albania,ALB,Total,13388.596401599998
19.6597,40.7385999999999,MUR/LWAL+CDN/H:2/IND,"Unreinforced masonry, low rise",33.66059895,Ind,315,475.0,5036467.11789375,755470.0676840623,1259116.7794734377,3021880.270736249,0,13.4,1.22,2.19,5.6,5,Fier,13,Fierit,Kuman,45d132,541.62,5742835.94,861425.39,1435708.98,3445701.56,Ind_427,Albania,ALB,Total,10603.08866925
19.5596,40.7136,MCF/LWAL+CDL/H:1/IND,"Confined or reinforced masonry, low rise",1.4910587436200002,Ind,315,525.0,246583.8397261576,36987.57595892363,61645.959931539386,147950.30383569453,0,14.86,1.35,2.43,6.21,5,Fier,13,Fierit,Portez,45d138,598.63,281167.43,42175.12,70291.86,168700.46,Ind_481,Albania,ALB,Total,469.68350424030007
19.7098,41.6559,S/LFM+CDM/H:1/IND,"Steel, low rise",0.1658874694,Ind,315,500.0,26127.2764305,3919.091464575,6531.819107625,15676.365858299998,0,2.48,0.23,0.41,1.04,9,Lezhe,26,Kurbinit,Milot,49q269,570.13,29791.65,4468.75,7447.91,17874.99,Ind_684,Albania,ALB,Total,52.25455286099999
19.451,40.6051,MUR/LWAL+CDN/H:2/IND,"Unreinforced masonry, low rise",1.11783894255,Ind,315,475.0,167256.65177904375,25088.497766856563,41814.16294476094,100353.99106742624,0,93.29,8.48,15.26,39.01,12,Vlore,37,Vlores,Novosele.,4c_367,541.62,190714.54,28607.18,47678.64,114428.72,Ind_882,Albania,ALB,Total,352.11926690325
LONGITUDE,LATITUDE,TAXONOMY,MACRO_TAXONOMY,BUILDINGS,OCCUPANCY,AREA_PER_BUILDING_SQM,COST_PER_AREA_EUR,TOTAL_REPL_COST_EUR,COST_STRUCTURAL_EUR,COST_NONSTRUCTURAL_EUR,COST_CONTENTS_EUR,OCCUPANTS_PER_ASSET,OCCUPANTS_PER_ASSET_DAY,OCCUPANTS_PER_ASSET_NIGHT,OCCUPANTS_PER_ASSET_TRANSIT,OCCUPANTS_PER_ASSET_AVERAGE,ID_1,NAME_1,ID_2,NAME_2,ID_3,COST_PER_AREA_USD,TOTAL_REPL_COST_USD,COST_STRUCTURAL_USD,COST_NONSTRUCTURAL_USD,COST_CONTENTS_USD,ASSET_ID,NAME_0,ID_0,SETTLEMENT,TOTAL_AREA_SQM
19.9685999999999,41.5808,S/LFM+CDM/H:1/IND,"Steel, low rise",19.002721922000003,Ind,315,500.0,2992928.7027150006,448939.30540725007,748232.1756787502,1795757.2216290005,0,104.29,9.48,17.06,43.61,2,Diber,6,Matit,42654,570.13,3412689.51,511903.43,853172.38,2047613.71,Ind_19,Albania,ALB,Total,5985.857405430001
20.027,41.0970999999999,S/LFM+CDM/H:1/IND,"Steel, low rise",77.1707656,Ind,315,500.0,12154395.582,1823159.3373,3038598.8955,7292637.349199999,0,98.64,8.96,16.13,41.25,4,Elbasan,9,Elbasanit,44980,570.13,13859059.96,2078858.99,3464764.99,8315435.97,Ind_124,Albania,ALB,Total,24308.791164
20.0938,41.0720999999999,S/LFM+CDM/H:1/IND,"Steel, low rise",1.5839975366,Ind,315,500.0,249479.6120145,37421.941802175,62369.903003625,149687.76720869998,0,9.17,0.83,1.5,3.83,4,Elbasan,9,Elbasanit,44998,570.13,284469.34,42670.4,71117.34,170681.6,Ind_269,Albania,ALB,Total,498.959224029
19.9936,41.0138,CR+PC/LFM+CDM/H:1/IND,"Concrete frame, low rise",25.719836415,Ind,315,525.0,4253417.947130625,638012.6920695937,1063354.4867826563,2552050.768278375,0,94.58,8.6,15.47,39.55,4,Elbasan,9,Elbasanit,44987,598.63,4849963.45,727494.52,1212490.86,2909978.07,Ind_290,Albania,ALB,Total,8101.748470725
19.9936,40.9971,MUR/LWAL+CDN/H:2/IND,"Unreinforced masonry, low rise",24.91549791,Ind,315,475.0,3727981.374783749,559197.2062175624,931995.3436959372,2236788.8248702497,0,91.63,8.33,14.99,38.31,4,Elbasan,9,Elbasanit,44987,541.62,4250833.95,637625.09,1062708.49,2550500.37,Ind_297,Albania,ALB,Total,7848.38184165
19.6597,40.7719,S+SL/LFM+CDM/H:1/IND,"Steel, low rise",19.785874464000003,Ind,315,500.0,3116275.2280800007,467441.284212,779068.8070200002,1869765.136848,0,30.33,2.76,4.96,12.68,5,Fier,13,Fierit,45d140,570.13,3553335.49,533000.32,888333.87,2132001.3,Ind_368,Albania,ALB,Total,6232.550456160001
19.6597,40.7552,S/LFM+CDM/H:1/IND,"Steel, low rise",42.50348063999999,Ind,315,500.0,6694298.200799999,1004144.7301199996,1673574.5501999997,4016578.920479999,0,16.92,1.54,2.77,7.07,5,Fier,13,Fierit,45d132,570.13,7633179.25,1144976.89,1908294.81,4579907.55,Ind_394,Albania,ALB,Total,13388.596401599998
19.6597,40.7385999999999,MUR/LWAL+CDN/H:2/IND,"Unreinforced masonry, low rise",33.66059895,Ind,315,475.0,5036467.11789375,755470.0676840623,1259116.7794734377,3021880.270736249,0,13.4,1.22,2.19,5.6,5,Fier,13,Fierit,45d132,541.62,5742835.94,861425.39,1435708.98,3445701.56,Ind_427,Albania,ALB,Total,10603.08866925
19.5596,40.7136,MCF/LWAL+CDL/H:1/IND,"Confined or reinforced masonry, low rise",1.4910587436200002,Ind,315,525.0,246583.8397261576,36987.57595892363,61645.959931539386,147950.30383569453,0,14.86,1.35,2.43,6.21,5,Fier,13,Fierit,45d138,598.63,281167.43,42175.12,70291.86,168700.46,Ind_481,Albania,ALB,Total,469.68350424030007
19.7098,41.6559,S/LFM+CDM/H:1/IND,"Steel, low rise",0.1658874694,Ind,315,500.0,26127.2764305,3919.091464575,6531.819107625,15676.365858299998,0,2.48,0.23,0.41,1.04,9,Lezhe,26,Kurbinit,49q269,570.13,29791.65,4468.75,7447.91,17874.99,Ind_684,Albania,ALB,Total,52.25455286099999
19.451,40.6051,MUR/LWAL+CDN/H:2/IND,"Unreinforced masonry, low rise",1.11783894255,Ind,315,475.0,167256.65177904375,25088.497766856563,41814.16294476094,100353.99106742624,0,93.29,8.48,15.26,39.01,12,Vlore,37,Vlores,4c_367,541.62,190714.54,28607.18,47678.64,114428.72,Ind_882,Albania,ALB,Total,352.11926690325
15 changes: 11 additions & 4 deletions openquake/risklib/asset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -901,7 +901,8 @@ def read_exp_df(fname, calculation_mode='', ignore_missing_costs=(),
df['id'] = asset_prefix + df.id
dfs.append(df)

assets_df = pandas.concat(dfs)
# NB: missing columns in the files are filled with NaN by pandas.concat
assets_df = pandas.concat(dfs).fillna('No_tag')
del fname_dfs # save memory
del dfs # save memory

Expand Down Expand Up @@ -1174,7 +1175,7 @@ def _read_csv(self, errors=None):
'%s: expected %d fields in %s, got %d' %
(fname, len(fields), header, len(header)))
elif missing:
raise InvalidFile('%s: missing %s' % (fname, missing))
logging.warning('%s: missing %s', fname, missing)
conv = {'lon': float, 'lat': float, 'number': float, 'area': float,
'residents': float, 'retrofitted': float, 'ideductible': float,
'occupants_day': float, 'occupants_night': float,
Expand All @@ -1200,8 +1201,14 @@ def _read_csv(self, errors=None):
if len(df) == 0:
continue
add_dupl_fields(df, oqfields)
df['lon'] = numpy.round(df.lon, 5)
df['lat'] = numpy.round(df.lat, 5)
try:
df['lon'] = numpy.round(df.lon, 5)
except AttributeError: # missing lon or lat
raise InvalidFile(f'{fname}: missing column "lon"')
try:
df['lat'] = numpy.round(df.lat, 5)
except AttributeError: # missing lon or lat
raise InvalidFile(f'{fname}: missing column "lat"')
sa = float(os.environ.get('OQ_SAMPLE_ASSETS', 0))
if sa:
df = general.random_filter(df, sa)
Expand Down