Andy bennett/748 model outputs to pipeline

.github/workflows/eera.yml

@@ -23,6 +23,11 @@ jobs:
     steps:
     - uses: actions/checkout@v2
 
+    - uses: actions/checkout@v2
+      with:
+        repository: ScottishCovidResponse/data_pipeline_api
+        path: data_pipeline_api
+
     - name: Install Dependencies ( Ubuntu )
       run: |
            sudo apt-get update
@@ -39,6 +44,16 @@ jobs:
       run : brew update && brew install gsl cppcheck lcov poppler htmldoc graphviz doxygen
       if: matrix.os == 'macos-latest'
 
+    - name: Data pipeline Dependencies
+      run : |
+            sudo apt-get update && sudo apt-get install -y python3-setuptools python3-venv && sudo rm -rf /var/lib/apt/lists/*
+            python3 -m venv .venv
+            source .venv/bin/activate
+            pip install wheel
+            pip install -r data_pipeline_api/bindings/cpp/requirements.txt
+            pip install data_pipeline_api
+      if: matrix.os == 'ubuntu-20.04'
+
     - name: Format Code ( Ubuntu GCC Master )
       run: |
            git config --local user.email "[email protected]"
@@ -56,18 +71,33 @@ jobs:
         force: false
       if: matrix.os == 'ubuntu-20.04' && matrix.config.compiler == 'gcc' && github.ref == 'refs/heads/master'
 
+    - name: Data pipeline API Compile
+      env:
+              CC: ${{ matrix.config.compiler }}
+              CXX: ${{ matrix.config.compilerpp }}
+      run : |
+            source .venv/bin/activate
+            cd data_pipeline_api/bindings/cpp
+            cmake -H. -Bbuild
+            cmake --build build
+      if: matrix.os == 'ubuntu-20.04'
+
     - name: Compile
       env:
               CC: ${{ matrix.config.compiler }}
               CXX: ${{ matrix.config.compilerpp }}
       run: |
+        source .venv/bin/activate
         mkdir build
         cd build
-        cmake .. -DCODE_COVERAGE=ON -DCLANG_TIDY=ON
+        cmake .. -DCODE_COVERAGE=ON -DCLANG_TIDY=ON -DDATA_PIPELINE=$GITHUB_WORKSPACE/data_pipeline_api
         make 2>&1 | tee clang_tidy_build_results.log
 
     - name: Run regression tests
+      env:
+            PYTHONPATH: data_pipeline_api
       run: |
+        source .venv/bin/activate
         ./scripts/RunRegressionTests.sh 1 24
         if [ $? -eq 0 ]; then
           echo "Regression tests completed successfully"
@@ -77,9 +107,26 @@ jobs:
           exit 1
         fi
 
+    - name: Run data pipeline regression tests
+      env:
+            PYTHONPATH: data_pipeline_api
+      run: |
+        source .venv/bin/activate
+        ./scripts/RunRegressionTests.sh 1 24 -d
+        if [ $? -eq 0 ]; then
+          echo "Data pipeline Regression tests completed successfully"
+          exit 0
+        else
+          echo "Data pipeline Regression tests failed"
+          exit 1
+        fi
+
     - name: Run unit tests
+      env:
+            PYTHONPATH: data_pipeline_api
       run: |
-        ./build/bin/Covid19EERAModel-unit_tests
+        source .venv/bin/activate
+        ./scripts/RunUnitTests.sh
         if [ $? -eq 0 ]; then
           echo "Unit tests completed successfully"
           exit 0

.gitignore

@@ -10,4 +10,6 @@ site/*
 index.html
 src/tclap/
 .DS_Store
-
+test/datapipeline/access-*.yaml
+test/datapipeline/data/outputs
+datapipeline

CMakeLists.txt

@@ -2,9 +2,9 @@ cmake_minimum_required(VERSION 3.10)
 
 set(PROJECT_NAME Covid19EERAModel)
 
-project(${PROJECT_NAME} VERSION 0.10.0 LANGUAGES CXX)
+project(${PROJECT_NAME} VERSION 0.10.1 LANGUAGES CXX)
 
-set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 17)
 set(CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/cmake_modules)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 set(CMAKE_CXX_FLAGS "-DROOT_DIR=\\\"${CMAKE_SOURCE_DIR}\\\" -DVERSION=\\\"${PROJECT_VERSION}\\\" ")

SoftwareChecklist.md

@@ -10,11 +10,11 @@ This checklist is part of ongoing work on a model scoresheet for SCRC models. It
 
 ### Date
 
-> 2020-07-31
+> 2020-08-28
 
 ### Version identifier
 
-> 0.10.0
+> 0.10.1
 
 ## Overall statement
 
@@ -58,8 +58,7 @@ Additionally, for each question please explain the situation and include any rel
 > - [x] Some work remaining or caveats
 > - [ ] Needs to be addressed
 > 
-> We have a set of regression tests which exercise the code in inference mode. These have been used mainly as an aid to refactoring, and are not considered confirmation of the code’s correctness. There are a limited set of unit tests which have been developed to test new software modules that we have developed. However, the core model code has effectively no unit tests. As noted above, implementing effective unit tests would require significant re-design of large portions of the core model code. Both our regression tests and unit tests are automated and run in the GitHub CI. Tests can also be
-run locally on the development host, and there is documentation in the project README on how to do that.
+> We have a set of regression tests which exercise the code in both inference and prediction modes. These have been used mainly as an aid to refactoring, and are not considered confirmation of the code’s correctness. There are a limited set of unit tests which have been developed to test new software modules that we have developed. However, the core model code has effectively no unit tests. As noted above, implementing effective unit tests would require significant re-design of large portions of the core model code. Both our regression tests and unit tests are automated and run in the GitHub CI. Tests can also be run locally on the development host, and there is documentation in the project README on how to do that.
 
 ### Are the scientific results of runs robust to different ways of running the code?
 
@@ -111,7 +110,7 @@ run locally on the development host, and there is documentation in the project R
 > - [ ] Some work remaining or caveats
 > - [ ] Needs to be addressed
 > 
-> Documentation exists at all of these levels. There is high-level documentation (in the `docs/` directory) describing how the model works from an epidemiological point of view. There is user documentation (`README.md`) explaining how to build and run the code, and what inputs are required. There is also inline code documentation which is exported via Doxygen, and can be viewed on the project dashboard (https://scottishcovidresponse.github.io/Covid19_EERAModel/site/doxygen-docs.html).
+> Documentation exists at all of these levels. There is high-level documentation (in the `docs/` directory) describing how the model works from an epidemiological point of view. There is user documentation (`README.md`) explaining how to build and run the code, and what inputs are required, and what outputs are produced. There is also inline code documentation which is exported via Doxygen, and can be viewed on the project dashboard (https://scottishcovidresponse.github.io/Covid19_EERAModel/site/doxygen-docs.html).
 
 ### Is there suitable collaboration infrastructure?
 

cmake/git_watcher.cmake

@@ -88,6 +88,7 @@ set(_state_variable_names
     # >>>
     # 1. Add the name of the additional git variable you're interested in monitoring
     #    to this list.
+    GIT_REMOTE_ORIGIN_URL
 )
 
 
@@ -165,7 +166,10 @@ function(GetGitState _working_dir)
     #    "execute_process()" command. Be sure to set them in
     #    the environment using the same variable name you added
     #    to the "_state_variable_names" list.
-
+    RunGitCommand(config --get remote.origin.url)
+    if(exit_code EQUAL 0)
+        set(ENV{GIT_REMOTE_ORIGIN_URL} "${output}")
+    endif()
 endfunction()
 
 

data/example/parameters.ini

@@ -1,6 +1,10 @@
 [Settings]
 shb_id=15
 tau=1
+nHealthBoards=15
+nAgeGroups=8
+nCfrCategories=4
+nCasesDays=60
 
 [Seed settings]
 seedmethod=background
@@ -59,3 +63,4 @@ prior_lambda_shape2=1e-6
 
 [Prediction Configuration]
 n_sim_steps=1000
+n_iterations=10

data/example/posterior_parameters.csv

@@ -1,4 +1,4 @@
-Index,p_inf,p_hcw,c_hcw,d,q,p_s,rrd,intro,T_lat,juvp_s,T_inf,T_rec,T_sym,T_hos,K,inf_asym
+Index,p_inf,p_hcw,c_hcw,d,q,p_s,rrd,lambda,T_lat,juvp_s,T_inf,T_rec,T_sym,T_hos,K,inf_asym
 0,0.153532,0.60916,37.9059,0.525139,0.313957,0.787278,0.516736,8.50135E-07,4,0.1,1.5,11,7,5,2000,1
 1,0.12602,0.429026,43.9404,0.277644,0.722916,0.470001,3.41006,6.27917E-07,4,0.1,1.5,11,7,5,2000,1
 2,0.133571,0.581229,36.9656,0.150568,0.844921,0.449271,3.65667,3.41873E-07,4,0.1,1.5,11,7,5,2000,1

scripts/CheckDPRunOutputs.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+scriptname=$0
+DPCHECK_SCRIPT=$( dirname "$scriptname" )/dpcheck.py
+
+function usage {
+    echo "usage: $scriptname <actual dp> <expected>"
+    echo "  actual dp   directory containing data pipeline directory used for the model run"
+    echo "  expected    directory containing expected outputs of the model run"
+    exit 1
+}
+
+ACTUALDP_OUTPUTS_DIR=$1
+EXPECTED_OUTPUTS_DIR=$2
+
+if [[ $# -ne 2 ]]; then
+  echo "ERROR: Invalid ($#) number of arguments"
+  usage
+  exit 1
+fi
+
+echo "Checking model outputs..."
+
+if [[ ! -d $ACTUALDP_OUTPUTS_DIR ]]; then
+  echo "ERROR: Cannot find directory $ACTUALDP_OUTPUTS_DIR"
+  usage
+elif [[ ! -d $EXPECTED_OUTPUTS_DIR ]]; then
+  echo "ERROR: Cannot find directory $EXPECTED_OUTPUTS_DIR"
+  usage
+else
+  # Find the data pipeline output file hopefully there should be only 1 present...
+  DPH5FILE=$( find $ACTUALDP_OUTPUTS_DIR/data/outputs -name '*.h5' )
+  EXPECTED_FILE_COUNT=$(find $EXPECTED_OUTPUTS_DIR -maxdepth 1 -type f -name "*.txt" | wc -l)
+
+  echo "Checking for $EXPECTED_FILE_COUNT outputs in $DPH5FILE"
+  difference_count=0
+  for expected_filepath in $EXPECTED_OUTPUTS_DIR/*.txt
+  do
+	filename=$(basename $expected_filepath)
+  component=$(
+		echo $filename |
+		awk -F_ '{ if ($2 == "prediction") { match($3, /([a-z]*).txt/, arr); printf("%s", arr[1]); } else { match($4, /step([0-9]*)/, arr); printf("steps/%s/%s", arr[1], $3); } }'
+	)
+
+	echo "Checking component: $component"
+  $DPCHECK_SCRIPT "$DPH5FILE" "$component" "$expected_filepath"
+	if [[ $? -ne 0 ]]; then
+	  echo "ERROR: Files $expected_filepath and $component are different"
+	  difference_count=$((difference_count+1))
+	else
+      echo "Files $expected_filepath and $component are identical"
+	fi
+  done
+	if [ $difference_count -ne 0 ]; then
+	  echo "SUMMARY - FAILURE: $difference_count components different"
+	  exit 1
+	else
+	  echo "SUMMARY - SUCCESS: Model outputs match expected outputs"
+	  exit 0
+	fi
+  fi
+fi
+

scripts/RunModel.sh

@@ -27,8 +27,9 @@ else
     echo "Warning: Outputs directory $OUTPUT_DIR does not exist; creating it now"
 	mkdir -p $OUTPUT_DIR
   else
-    rm $OUTPUT_DIR/* 2> /dev/null
+    rm -r $OUTPUT_DIR/* 2> /dev/null
   fi
+  mkdir -p $OUTPUT_DIR/logs
 
   $EXEPATH $FLAGS
   if [ $? -ne 0 ]; then

scripts/RunRegressionTests.sh

@@ -4,12 +4,16 @@ scriptname=$0
 
 WORKING_DATA_DIR=./data
 WORKING_OUTPUTS_DIR=./outputs
+WORKING_PIPELINE_DIR=./datapipeline
+PIPELINE_CONFIG=$WORKING_PIPELINE_DIR/config.yaml
 REGRESSION_DIR=./test/regression
-PIPELINE_CONFIG=./test/datapipeline/config.yaml
+PIPELINE_DIR=./test/datapipeline
 EXEPATH=./build/bin/Covid19EERAModel
 SETUP_SCRIPT=./scripts/SetUpModelRun.sh
+SETUP_DP_SCRIPT=./scripts/SetUpDPModelRun.sh
 RUN_SCRIPT=./scripts/RunModel.sh
 CHECK_SCRIPT=./scripts/CheckRunOutputs.sh
+CHECK_DP_SCRIPT=./scripts/CheckDPRunOutputs.sh
 
 function usage {
     echo "usage: $scriptname <first> <last> [-d]"
@@ -81,17 +85,27 @@ for i in $(seq $FIRST $LAST) ; do
 
   regression_test_dir=$REGRESSION_DIR/run$i
 
+  setupdp=0
+  if [[ $USEDATAPIPELINE -eq 1 ]]; then
+    $SETUP_DP_SCRIPT $WORKING_PIPELINE_DIR $PIPELINE_DIR
+    setupdp=$?
+  fi
+
   $SETUP_SCRIPT $WORKING_DATA_DIR $regression_test_dir/data
   setup=$?
 
   echo Command: $RUN_SCRIPT $EXEPATH $WORKING_OUTPUTS_DIR $FLAGS
   $RUN_SCRIPT $EXEPATH $WORKING_OUTPUTS_DIR $FLAGS
   run=$?
 
-  $CHECK_SCRIPT $WORKING_OUTPUTS_DIR $regression_test_dir/outputs
+  if [[ $USEDATAPIPELINE -eq 1 ]]; then
+    $CHECK_DP_SCRIPT $WORKING_PIPELINE_DIR $regression_test_dir/outputs
+  else
+    $CHECK_SCRIPT $WORKING_OUTPUTS_DIR $regression_test_dir/outputs
+  fi
   check=$?
 
-  if [[ setup -ne 0 ]] || [[ run -ne 0 ]] || [[ check -ne 0 ]]; then
+  if [[ setupdp -ne 0 ]] || [[ setup -ne 0 ]] || [[ run -ne 0 ]] || [[ check -ne 0 ]]; then
     failures=$((failures+1))
   else
     successes=$((successes+1))

scripts/RunUnitTests.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+PIPELINE_DIR=./test/datapipeline
+WORKING_PIPELINE_DIR=./datapipeline
+SETUP_DP_SCRIPT=./scripts/SetUpDPModelRun.sh
+UNIT_TESTS=./build/bin/Covid19EERAModel-unit_tests
+
+# Setup the data pipeline directory
+echo "Setting up data pipeline data..."
+$SETUP_DP_SCRIPT $WORKING_PIPELINE_DIR $PIPELINE_DIR
+
+if [[ $? -ne 0 ]]; then
+   echo "Error: Problem downloading data pipeline files"
+   exit 1
+else
+   echo "Running unit tests..."
+   $UNIT_TESTS
+   exit $?
+fi

scripts/SetUpDPModelRun.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+function usage {
+	echo "usage: $scriptname <working_dir> <inputs_dir>"
+    echo "  working_dir     working directory where the model executable expects to find the datapipeline directory"
+    echo "  inputs_dir      directory where the test data pipeline is stored"
+    exit 1
+}
+
+WORKING_DIR=$1
+INPUTS_DIR=$2
+
+if [[ $# -ne 2 ]]; then
+  echo "ERROR: Invalid ($#) number of arguments"
+  usage
+  exit 1
+fi
+
+echo "Setting up model run..."
+
+if [[ ! -d $INPUTS_DIR ]]; then
+  echo "ERROR: Unable to locate test data pipeline directory $INPUTS_DIR"
+  usage
+else
+  if [[ ! -d $WORKING_DIR ]]; then
+    # No datapipeline directory - create and download the data
+    mkdir -p $WORKING_DIR
+    cp $INPUTS_DIR/config.yaml $INPUTS_DIR/parameters.ini $WORKING_DIR
+    python3 -m data_pipeline_api.registry.download --config $WORKING_DIR/config.yaml
+  else
+    # Directory exists - assume still valid and just delete results
+    rm -rf $WORKING_DIR/data/outputs $WORKING_DIR/access*.yaml
+  fi
+
+  echo "Configured data pipeline contents of $WORKING_DIR from $INPUTS_DIR"
+fi