Merge pull request #11 from odtp-org/pipeline_parameter

Pipeline parameter

Author: caviri

Dockerfile

@@ -75,7 +75,6 @@ RUN pip install -r /tmp/requirements.txt
 # ODTP Preparation
 ##################################################
 
-
 # I'm not fully convinced of this folder organization. 
 # Workdir folders should be defined in the startup bash
 

README.md

@@ -17,8 +17,11 @@ ODTP component for running Eqasim.
 
 2. Create your `.env` file with this structure. **If you do not have MONGODB and/or S3 activated omit this step, and just provide the scenario as environmental variable.**
 
+Add the selected scenario `IDF`, or `CH`. Also select the eqasim pipeline you want to run `Synthesis`, or `matsim`.
+
 ```
 SCENARIO=IDF
+PIPELINE=Synthesis
 MONGODB_CLIENT=mongodb://.....
 S3_SERVER=https://....
 S3_ACCESS_KEY=Q0ISQ....
@@ -92,6 +95,18 @@ If you want to kill the session just write `exit`. Also use `tmux ls` to list al
     - Check log.txt and uploads stout to MongoDB
 - s3uploader.py
     - Upload output.zip & workdir.zip to S3. Create a mongodb entry. 
+- parameters.py
+    - Takes care of replacing the placeholder on the config templates.
+
+## Changelog
+
+- v.0.1.0: Version compatible with IDF & CH
+    - Parameters. Now the parameters are taken from the enviroment variables. 
+    - Configuration templates. No needs of copying config.yml file anymore. The file will be built based on the parameters.
+    - Logger: If MongoDB arg. provided the container will log its activity in the db. 
+    - S3 Compatibility: Output content, and snapshots can be uploaded to the S3 server.
+
+- v.0.0.1: Version compatible with IDF
 
 ## Development. 
 

app/ch.sh

@@ -18,8 +18,13 @@ git checkout 4658daa2e441dcda132622e7fcb47da1df8c47d6
 
 # A2A - Prepare parameters & Config File
 # Read placeholders and create config file from Environment  
-python3 /odtp/odtp-app/parameters.py /odtp/odtp-app/config_templates/config_ch.yml /odtp/odtp-workdir/scenario/config.yml
-#cp /odtp/odtp-volume/config.yml /odtp/odtp-workdir/scenario/config.yml
+if [ "$PIPELINE" == "Synthesis" ]; then
+    echo "Running Synthesis PIPELINE"
+    python3 /odtp/odtp-app/parameters.py /odtp/odtp-app/config_templates/config_synthesis_ch.yml /odtp/odtp-workdir/scenario/config.yml
+else
+    echo "Running Matsim PIPELINE"
+    python3 /odtp/odtp-app/parameters.py /odtp/odtp-app/config_templates/config_matsim_ch.yml /odtp/odtp-workdir/scenario/config.yml
+fi
 
 # A2B - Prepare datafolder
 ln -s /odtp/odtp-volume/data /odtp/odtp-workdir/data

app/config_templates/config_matsim_ch.yml

@@ -0,0 +1,27 @@
+# General pipeline settings
+working_directory: /odtp/odtp-workdir/cache
+flowchart_path: /odtp/odtp-workdir/output/flowchart.json
+dryrun: false
+
+# Requested stages
+run:
+  - synthesis.output
+  - matsim.output
+
+# These are configuration options that we use in the pipeline
+config:
+  threads: [threads]
+  osmosis_binary: /tmp/osmosis/bin/osmosis
+  maven_binary: /tmp/maven/apache-maven-3.6.3/bin/mvn
+  random_seed: [random_seed]
+  hot_deck_matching_runners: [hot_deck_matching_runners]
+  disable_progress_bar: false
+  java_memory: [java_memory]
+  input_downsampling: [input_downsampling]
+  enable_scaling: [enable_scaling]
+  scaling_year: [scaling_year]
+  use_freight: [use_freight]
+  hafas_date: [hafas_date]
+  data_path: /odtp/odtp-workdir/data
+  output_path: /odtp/odtp-workdir/output
+  output_id: [output_id]

app/config_templates/config_matsim_idf.yml

@@ -0,0 +1,36 @@
+## Synthetic population pipeline for Île-de-France
+## based on the synpp package
+
+# This is the path to a directory where the pipeline can store temporary data
+working_directory: /odtp/odtp-workdir/cache
+
+# This section defines which parts of the pipeline should be run
+run:
+  - synthesis.output # To create the output population in the output_path (see below)
+  - matsim.output # Uncomment, if you want to run the full simulation (you'll need Java for that)
+
+# Here the configuraiton of the pipeline starts
+config:
+  # Some general configuration
+  processes: [processes]
+  osmosis_binary: /tmp/osmosis/bin/osmosis
+  maven_binary: /tmp/maven/apache-maven-3.6.3/bin/mvn
+
+  # Define whether to use ENTD or EGT as the household travel survey (HTS)
+  hts: [hts]
+
+  # Define sampling rate and random seed for the output population
+  sampling_rate: [sampling_rate]
+  random_seed: [random_seed]
+
+  # Paths to the input data and where the output should be stored
+  data_path: /odtp/odtp-workdir/data
+  output_path: /odtp/odtp-workdir/output
+
+  # Only interesting if you run the simulation
+  java_memory: [java_memory]
+
+  # Uncommented below to enable vehicle fleet generation
+  # generate_vehicles_file: True
+  # generate_vehicles_method: fleet_sample
+  # vehicles_data_year: 2015

app/config_templates/config_ch.yml …config_templates/config_synthesis_ch.ymlapp/config_templates/config_ch.yml renamed to app/config_templates/config_synthesis_ch.yml app/config_templates/config_ch.yml renamed to app/config_templates/config_synthesis_ch.yml

@@ -16,8 +16,13 @@ git checkout b8968c1
 
 # A2A - Prepare parameters & Config File
 # Read placeholders and create config file from Environment  
-python3 /odtp/odtp-app/parameters.py /odtp/odtp-app/config_templates/config_idf.yml /odtp/odtp-workdir/scenario/config.yml
-#cp /odtp/odtp-volume/config.yml /odtp/odtp-workdir/scenario/config.yml
+if [ "$PIPELINE" == "Synthesis" ]; then
+    echo "Running Synthesis PIPELINE"
+    python3 /odtp/odtp-app/parameters.py /odtp/odtp-app/config_templates/config_synthesis_idf.yml /odtp/odtp-workdir/scenario/config.yml
+else
+    echo "Running Matsim PIPELINE"
+    python3 /odtp/odtp-app/parameters.py /odtp/odtp-app/config_templates/config_matsim_idf.yml /odtp/odtp-workdir/scenario/config.yml
+fi
 
 # A2B - Prepare datafolder
 ln -s /odtp/odtp-volume/data /odtp/odtp-workdir/data

app/config_templates/config_idf.yml …onfig_templates/config_synthesis_idf.ymlapp/config_templates/config_idf.yml renamed to app/config_templates/config_synthesis_idf.yml app/config_templates/config_idf.yml renamed to app/config_templates/config_synthesis_idf.yml

@@ -3,6 +3,7 @@
 - name: eqasim
 - repo: https://github.com/odtp-org/odtp-eqasim 
 - author: "Author Test"
+- original_author: Author of the tool
 - description: Synthetic population for Matsim
 - arguments:
   - scenario: [idf, ch]
@@ -14,7 +15,6 @@
   - generted_vehicels_method: fleet_sample
   - vehicles_data_year: 2015
 - volume:
-  - config.yml: /odtp/odtp-volume/config.yml
   - data: /odtp/odtp-volume/data
 - ports:
   - api: 7100