Merge pull request #39 from NRCan/dev

new interface and tools for missing DRF entries
rename sel to itemcode add cat lookup label table

Author: cefect

.readthedocs.yaml

@@ -3,7 +3,7 @@ version: "2"
 build:
   os: "ubuntu-22.04"
   tools:
-    python: "3.10"
+    python: "3.12"
 
 python:
   install:

CONTRIBUTING.md

@@ -1,13 +1,33 @@
 # CanCurve development
+guidelines/instructions for contributing to the CanCurve project.
+
+For contributing to the documentation, see `./docs/contributing.MD`
 
 ## Installation
-Typically, development should employ a virtual environment with pyqgis bindings. 
-We usually develop to target the most recent QGIS LTR. 
-Some additional packages used for testing and deployment are specified in `./requirements.txt`
+Typically, development should employ a virtual environment with pyqgis bindings.
+To build this requires:
+- 1) pyqgis bindings (see `./readme.md` for target version)
+- 2) Some additional packages used for testing and deployment are specified in `./requirements.txt`
+- 3) a `./definitions.py` file is needed for some of the testing data (see below for template)
 
 ### PYTHONPATH
 only the source directory should be included (`./CanCurve` not `./CanCurve/cancurve`)
 
+
+### definitions.py
+
+```python
+#machine specific external filepaths
+#See Shared Drive/02_proj/2403_CanCurve/10_IO to download
+#https://drive.google.com/drive/folders/1Z0d40SdXU0bcI0l7CLO2aT6gYGhSu3cQ?usp=drive_link
+
+#directory where test data is stored
+test_data_dir = r's:\02_proj\2403_CanCurve\10_IO\test_data'
+
+#directory with DDFP data from David (for comparison)
+ddfp_data_dir = r's:\02_proj\2403_CanCurve\10_IO\DDF_data'
+```
+
 ## Tests
 pytests are in `./tests`
 

README.md

@@ -10,20 +10,19 @@ CanCurve is an open source tool for developing flood depth-damage (stage-damage)
 
 
 ## Updates
-- 2024-12-04: update survey link [v1.0.3](https://github.com/NRCan/CanCurve/releases/tag/v1.0.3)
-- 2024-11-27: UI and documentation improvements.
-- 2024-10-31: add sphinx documentation. improve treatment of units. update to QGIS 3.34.12 [v1.0.0](https://github.com/NRCan/CanCurve/releases/tag/v1.0.0)
-- 2024-08-22: priority A and B fixes [v0.1.0](https://github.com/NRCan/CanCurve/releases/tag/v0.1.0)
-- 2024-05-09: small fixes and updates based on initial comments
-- 2024-05-01: initial working release (un tested) [v0.0.1](https://github.com/NRCan/CanCurve/releases/tag/v0.0.1)
+- [v1.1.0](https://github.com/NRCan/CanCurve/releases/tag/v1.1.0) post-release improvements
+    - new interface and tools for missing DRF entries
+    - rename sel to itemcode add cat lookup label table
+- [v1.0.3](https://github.com/NRCan/CanCurve/releases/tag/v1.0.3) pre-release
+
 
 
 ## Documentation
 Project documentation is [here](https://cancurve.readthedocs.io/en/latest/)
 
 
 ## Installing for the first time
-- Install [QGIS 3.34.12](https://download.qgis.org/downloads/) (with Qt 5.15.13)
+- Install [QGIS 3.34.14](https://download.qgis.org/downloads/) (with Qt 5.15.13)
 - download the `cancurve.zip` file from the [latest release](https://github.com/NRCan/CanCurve/releases) to your local machine
 - in QGIS, `Manage and Install Plugins...` > `Install from ZIP` > select the downloaded file
 - we recommended to also install the **First Aid** plugin for more detailed error messages 

cancurve/__init__.py

@@ -1,7 +1,7 @@
 #===============================================================================
 # plugin metadata
 #===============================================================================
-__version__='1.0.3'
+__version__='1.1.0'
 
 #===============================================================================
 # plugin entry point
@@ -25,13 +25,28 @@ def classFactory(iface):  # pylint: disable=invalid-name
 
 import importlib, warnings
 
+from packaging import version
+
 def check_package(package_name):
     spec = importlib.util.find_spec(package_name)
     if spec is not None:
-        print(f'module {package_name} is installed')
+        #print(f'module {package_name} is installed')
+        pass
     else:
         warnings.warn(f'module \'{package_name}\' not installed')
 
 
 check_package('openpyxl')
 
+
+try:
+    import pandas as pd
+except ImportError:
+    warnings.warn("pandas is not installed!")
+else:
+    required_version = "2.0.0"
+    current_version = pd.__version__
+    # Skeptically assert that the current pandas version meets the minimum requirement.
+    assert version.parse(current_version) >= version.parse(required_version), (
+        f"pandas version {current_version} is below the required {required_version}"
+    )

cancurve/bldgs/assertions.py

@@ -45,8 +45,8 @@ def assert_ci_df(df, msg=''):
         raise TypeError("Input must be a Pandas DataFrame")
 
     # Check index 
-    if not set(df.index.names).difference(['cat', 'sel']) == set():
-        raise KeyError("Incorrect index names in DataFrame")
+    if not set(df.index.names).difference(['category', 'component']) == set():
+        raise KeyError(f"Incorrect index names in DataFrame\n    {df.index.names}")
 
     # Check data types
     for coln, dstr in df.dtypes.items():
@@ -149,12 +149,12 @@ def assert_drf_df(df):
         raise TypeError("Input must be a Pandas DataFrame")
 
     # Check index 
-    if not set(df.index.names).difference(['cat', 'sel', 'bldg_layout']) == set():
+    if not set(df.index.names).difference(['category', 'component', 'bldg_layout']) == set():
         raise KeyError("Incorrect index names in DataFrame")
 
     # Check the columns all conform to float depths
-    if not 'float' in df.columns.dtype.name:
-        raise TypeError(f'DRF column headers expected as dtype float. instead got \'{df.columns.dtype.name}\'')
+    # if not 'float' in df.columns.dtype.name:
+    #     raise TypeError(f'DRF column headers expected as dtype float. instead got \'{df.columns.dtype.name}\'')
 
     # Check data types (more accurate)
 

cancurve/bldgs/bldg_meta_rqmts.csv

@@ -1,34 +1,34 @@
-varName_core,varName_ui,varName_canflood,type,required_core,required_canflood,default_canflood,widgetName,case1,case2,case3,case4_R2,case5_crawl,01,case1_ci_header_case
-,,tag,str,FALSE,TRUE,?,,test_case1,test_case2,heather_0509,R_2-L-BD-CU_ABCA,R_1-L-C-ST_ABCA,tutorial_01,test_case1
-,currency,impact_units,str,FALSE,TRUE,$CAD,currency_ComboBox,,,,,,,
-,costBasis,impact_var,str,FALSE,FALSE,damage,costBasis_ComboBox,,,Depreciated Costs,,,,
-expo_units,expoUnits,exposure_units,str,FALSE,FALSE,meters,basementHeightUnits_label,meters,meters,meters,feet,meters,meters,meters
-,,exposure_var,str,FALSE,FALSE,flood depth above floor,,,,,,,,
-,,scale_var,str,FALSE,FALSE,building footprint,,,,,,,,
-bldg_layout,buildingLayout,,str,TRUE,FALSE,,buildingLayout_ComboBox,default,default,default,default,,default,default
-basement_height,basementHeight,,float,FALSE,FALSE,,basementHeight_DoubleSpinBox,1.8,2.1,0,10,0,1.8,1.8
-,sizeOrAreaValue,,float,FALSE,FALSE,,sizeOrAreaValue_DoubleSpinBox,232.1,232.2,344,139,232,232.1,232.1
-,sizeOrAreaUnits,scale_units,str,FALSE,FALSE,m2,sizeOrAreaUnits_ComboBox,m2,m2,m2,m2,m2,m2,m2
-scale_value_m2,,,float,TRUE,FALSE,,,232.1,232.2,344,139,232,232.1,232.1
-,,exposure,str,FALSE,TRUE,impact,,,,,,,,
-,occupancyClassification,,str,FALSE,FALSE,,occupancyClassification_ComboBox,,,Commercial,Residential,Residential,,
-,subClassification,,str,FALSE,FALSE,,subClassification_ComboBox,,,Construction,Single Family,,,
-,storys,,str,FALSE,FALSE,,storys_ComboBox,,,1,2,1,,
-,BedroomCount,,int,FALSE,FALSE,,BedroomCount_QSpinBox,,,0,7,,,
-,BathroomCount,,float,FALSE,FALSE,,BathroomCount_DoubleSpinBox,,,1,4,,,
-,heatingType,,str,FALSE,FALSE,,heatingType_ComboBox,,,Forced Air - Gas,Forced Air - Unknown,,,
-,coolingType,,str,FALSE,FALSE,,coolingType_ComboBox,,,Central air,Central air,,,
-,garageType,,str,FALSE,FALSE,,garageType_ComboBox,,,None,None,,,
-,garageSize,,str,FALSE,FALSE,,garageSize_ComboBox,,,,,,,
-,foundationType,,str,FALSE,FALSE,,foundationType_ComboBox,,,Other,Basement,Crawlspace,,
-,qualityOfBuildingMaterials,,str,FALSE,FALSE,,qualityOfBuildingMaterials_ComboBox,,,Average,Custom,Average,,
-,taxesIncluded,,str,FALSE,FALSE,,taxesIncluded_ComboBox,,,,,,,
-,priceListSource,,str,FALSE,FALSE,,priceListSource_LineEdit,,,Xactimate,,,,
-,country,,str,FALSE,FALSE,Canada,country_ComboBox,,,Canada,Canada,Canada,,
-,provinceTerritory,,str,FALSE,FALSE,,provinceTerritory_ComboBox,,,ON,AB,AB,,
-,pricingDate,,str,FALSE,FALSE,,,,,,,,,
-,basementFinish,,float,FALSE,FALSE,,basementFinish_DoubleSpinBox,,,100,100,,,
-,createdBy,,str,FALSE,FALSE,,createdBy_LineEdit,Arcadis,Arcadis,hmcgrath,Arcadis,Arcadis,Arcadis,Arcadis
-,locationCityTownRegion,,str,FALSE,FALSE,,,,,,,,,
-,yearOfBuildingConstruction,,int,FALSE,FALSE,,yearOfBuildingConstruction_SpinBox,,,2500,2000,2000,,
-scale_factor,scaleFactor,,float,TRUE,FALSE,,scaleFactor_DoubleSpinBox,1.1,1,1,1,1,1.1,1.1
+varName_core,varName_ui,varName_canflood,type,required_core,required_canflood,default_canflood,widgetName,case1,case2,case3,case4_R2,case5_crawl,01,02,case1_ci_header_case
+,,tag,str,FALSE,TRUE,?,,test_case1,test_case2,heather_0509,R_2-L-BD-CU_ABCA,R_1-L-C-ST_ABCA,tutorial_01,tutorial_01,test_case1
+,currency,impact_units,str,FALSE,TRUE,$CAD,currency_ComboBox,,,,,,,,
+,costBasis,impact_var,str,FALSE,FALSE,damage,costBasis_ComboBox,,,Depreciated Costs,,,,,
+expo_units,expoUnits,exposure_units,str,FALSE,FALSE,meters,basementHeightUnits_label,meters,meters,meters,feet,meters,meters,meters,meters
+,,exposure_var,str,FALSE,FALSE,flood depth above floor,,,,,,,,,
+,,scale_var,str,FALSE,FALSE,building footprint,,,,,,,,,
+bldg_layout,buildingLayout,,str,TRUE,FALSE,,buildingLayout_ComboBox,default,default,default,default,,default,default,default
+basement_height,basementHeight,,float,FALSE,FALSE,,basementHeight_DoubleSpinBox,1.8,2.1,0,10,0,1.8,1.8,1.8
+,sizeOrAreaValue,,float,FALSE,FALSE,,sizeOrAreaValue_DoubleSpinBox,232.1,232.2,344,139,232,232.1,232.1,232.1
+,sizeOrAreaUnits,scale_units,str,FALSE,FALSE,m2,sizeOrAreaUnits_ComboBox,m2,m2,m2,m2,m2,m2,m2,m2
+scale_value_m2,,,float,TRUE,FALSE,,,232.1,232.2,344,139,232,232.1,232.1,232.1
+,,exposure,str,FALSE,TRUE,impact,,,,,,,,,
+,occupancyClassification,,str,FALSE,FALSE,,occupancyClassification_ComboBox,,,Commercial,Residential,Residential,,,
+,subClassification,,str,FALSE,FALSE,,subClassification_ComboBox,,,Construction,Single Family,,,,
+,storys,,str,FALSE,FALSE,,storys_ComboBox,,,1,2,1,,,
+,BedroomCount,,int,FALSE,FALSE,,BedroomCount_QSpinBox,,,0,7,,,,
+,BathroomCount,,float,FALSE,FALSE,,BathroomCount_DoubleSpinBox,,,1,4,,,,
+,heatingType,,str,FALSE,FALSE,,heatingType_ComboBox,,,Forced Air - Gas,Forced Air - Unknown,,,,
+,coolingType,,str,FALSE,FALSE,,coolingType_ComboBox,,,Central air,Central air,,,,
+,garageType,,str,FALSE,FALSE,,garageType_ComboBox,,,None,None,,,,
+,garageSize,,str,FALSE,FALSE,,garageSize_ComboBox,,,,,,,,
+,foundationType,,str,FALSE,FALSE,,foundationType_ComboBox,,,Other,Basement,Crawlspace,,,
+,qualityOfBuildingMaterials,,str,FALSE,FALSE,,qualityOfBuildingMaterials_ComboBox,,,Average,Custom,Average,,,
+,taxesIncluded,,str,FALSE,FALSE,,taxesIncluded_ComboBox,,,,,,,,
+,priceListSource,,str,FALSE,FALSE,,priceListSource_LineEdit,,,Xactimate,,,,,
+,country,,str,FALSE,FALSE,Canada,country_ComboBox,,,Canada,Canada,Canada,,,
+,provinceTerritory,,str,FALSE,FALSE,,provinceTerritory_ComboBox,,,ON,AB,AB,,,
+,pricingDate,,str,FALSE,FALSE,,,,,,,,,,
+,basementFinish,,float,FALSE,FALSE,,basementFinish_DoubleSpinBox,,,100,100,,,,
+,createdBy,,str,FALSE,FALSE,,createdBy_LineEdit,Arcadis,Arcadis,hmcgrath,Arcadis,Arcadis,Arcadis,Arcadis,Arcadis
+,locationCityTownRegion,,str,FALSE,FALSE,,,,,,,,,,
+,yearOfBuildingConstruction,,int,FALSE,FALSE,,yearOfBuildingConstruction_SpinBox,,,2500,2000,2000,,,
+scale_factor,scaleFactor,,float,TRUE,FALSE,,scaleFactor_DoubleSpinBox,1.1,1,1,1,1,1.1,1.1,1.1