Merge pull request #17 from odtp-org/adding-submodule

v0.3.0

Author: caviri

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "odtp-component-client"]
+	path = odtp-component-client
+	url = https://github.com/odtp-org/odtp-component-client

Dockerfile

@@ -79,7 +79,7 @@ RUN pip install -r /tmp/requirements.txt
 RUN mkdir /odtp \
     /odtp/odtp-config \
     /odtp/odtp-app \
-    /odtp/odtp-client \
+    /odtp/odtp-component-client \
     /odtp/odtp-logs \ 
     /odtp/odtp-input \
     /odtp/odtp-workdir \
@@ -92,9 +92,9 @@ RUN mkdir /odtp/odtp-workdir/cache \
 # This copy all the information for running the ODTP component
 COPY odtp.yml /odtp/odtp-config/odtp.yml
 
-COPY ./odtp-client /odtp/odtp-client
+COPY ./odtp-component-client /odtp/odtp-component-client
 
 COPY ./app /odtp/odtp-app
 WORKDIR /odtp
 
-ENTRYPOINT ["bash", "/odtp/odtp-client/startup.sh"]
+ENTRYPOINT ["bash", "/odtp/odtp-component-client/startup.sh"]

README.md

@@ -14,31 +14,13 @@ ODTP component for running Eqasim.
 
 ## How to run this component in docker. 
 
-1. Prepare manually a folder called volume containing the following datafolder of our selected scenario:
+1. Prepare manually a folder called `odtp-input` containing the following datafolder of our selected scenario:
 
 - data
 
-2. Create your `.env` file with this structure. **If you do not have MONGODB and/or S3 activated omit this step, and just provide the scenario as environmental variable.**
+2. Create your `.env` file with this structure.
 
-Add the selected scenario `IDF`, or `CH`. Also select the eqasim pipeline you want to run `Synthesis`, or `matsim`.
-
-```
-SCENARIO=IDF
-PIPELINE=Synthesis
-MONGODB_CLIENT=mongodb://.....
-S3_SERVER=https://....
-S3_ACCESS_KEY=Q0ISQ....
-S3_SECRET_KEY=OoPthI....
-S3_BUCKET_NAME=13301....
-```
-
-3. Build the dockerfile 
-
-```
-docker build -t odtp-eqasim .
-```
-
-3. Depending on your scenario add the parameters requested:
+Add the selected scenario `IDF`, or `CH`. Also select the eqasim pipeline you want to run `Synthesis`, or `Matsim`.
 
 For Corsica:
 ```
@@ -78,19 +60,40 @@ hafas_date=01.10.2018
 output_id=test
 ```
 
-4. Run the following command. Select the correct volume folder, the `SCENARIO` you want to simulate (`"IDF"`, `CORSICA`, or `"CH"`) and the MONGODB_CLIENT URL. 
+3. Build the dockerfile 
+
+```
+docker build -t odtp-eqasim .
+```
+
+4. Run the following command. Mount the correct volumes for input/output folders. 
 
 ```
-docker run -it --rm -v {PATH_TO_YOUR_INPUT_VOLUME}:/odtp/odtp-input -v {PATH_TO_YOUR_OUTPUT_VOLUME}:/odtp/odtp-output --env-file .env odtp-eqasim
+docker run -it --rm \
+-v {PATH_TO_YOUR_INPUT_VOLUME}:/odtp/odtp-input \
+-v {PATH_TO_YOUR_OUTPUT_VOLUME}:/odtp/odtp-output \
+--env-file .env odtp-eqasim
 ```
 
+
+
 ## Example of tmux session
 
-In this example you will run the isolated container with the CH scenario.
+In this example you will run the isolated container with the CORSICA scenario.
 
 ```
 tmux new -s odtp-test
-docker run -it --rm -v /home/vivar/odtp-tutorial/volume-ch:/odtp/odtp-volume -e SCENARIO=CH --name odtp-test odtp-eqasim
+docker run -it --rm \
+-v {PATH_TO_YOUR_INPUT_VOLUME}:/odtp/odtp-input \
+-v {PATH_TO_YOUR_INPUT_VOLUME}:/odtp/odtp-output \ 
+-e SCENARIO=CORSICA \
+-e PIPELINE=Synthesis \
+-e processes=4 \
+-e hts=entd \
+-e sampling_rate=0.001 \
+-e random_seed=1234 \
+-e java_memory=8GB \
+--name odtp-eqasim odtp-eqasim
 ```
 
 Now you can push Control + B, and then D to dettach from the tmux session. In order to come back to the session you can do: 
@@ -100,25 +103,26 @@ tmux attach-session -t odtp-test
 ```
 If you want to kill the session just write `exit`. Also use `tmux ls` to list all available tmux sessions.
 
+## Usage with ODTP
+
+To connect this docker with an existing implementation of ODTP the following variables should be added to the env file. 
 
-## Description of files
+```
+ODTP_MONGO_SERVER="mongodb://USER:PASSWORD@.....
+ODTP_S3_SERVER=http://....
+ODTP_BUCKET_NAME=...
+ODTP_ACCESS_KEY=....
+ODTP_SECRET_KEY=...
+```
 
-- app/startup.ch
-    - This file is the entrypoint. Check the scenario and keep log of all stout. 
-- app/idf.ch
-    - Pull IDF repo. Launch the pipeline. Activate logger and s3 uploaders. Compress output and workdir. 
-- app/ch.ch
-    - Pull CH repo. Launch the pipeline. Activate logger and s3 uploaders. Compress output and workdir. 
-- logger.py
-    - Check log.txt and uploads stout to MongoDB
-- s3uploader.py
-    - Upload output.zip & workdir.zip to S3. Create a mongodb entry. 
-- parameters.py
-    - Takes care of replacing the placeholder on the config templates.
 
 ## Changelog
 
-- v0.2.0: Version compatible with IDF, CH & Corsica
+- v0.3.0 Corsica compatibility
+    - Updating odtp-component-client to submodule
+    - Output now delivers `eqasim-output` and `cache`. 
+
+- v0.2.0: Version compatible with IDF & CH.
 
 - v0.1.0: Version compatible with IDF & CH
     - Parameters. Now the parameters are taken from the enviroment variables. 

app/app.sh

@@ -11,10 +11,10 @@ if [ "$SCENARIO" == "IDF" ]; then
     # Reading placeholders and create config file from environment variables 
     if [ "$PIPELINE" == "Synthesis" ]; then
         echo "Running Synthesis PIPELINE"
-        python3 /odtp/odtp-client/parameters.py /odtp/odtp-app/config_templates/config_synthesis_idf.yml /odtp/odtp-workdir/scenario/config.yml
+        python3 /odtp/odtp-component-client/parameters.py /odtp/odtp-app/config_templates/config_synthesis_idf.yml /odtp/odtp-workdir/scenario/config.yml
 
         # Preparing input data folder
-        ln -s /odtp/odtp-input/data /odtp/odtp-workdir/data
+        ln -s /odtp/odtp-component-client/data /odtp/odtp-workdir/data
 
     else
         echo "Running Matsim PIPELINE"
@@ -41,14 +41,14 @@ elif [ "$SCENARIO" == "CORSICA" ]; then
     # Reading placeholders and create config file from environment variables
     if [ "$PIPELINE" == "Synthesis" ]; then
         echo "Running Synthesis PIPELINE"
-        python3 /odtp/odtp-client/parameters.py /odtp/odtp-app/config_templates/config_synthesis_corsica.yml /odtp/odtp-workdir/scenario/config.yml
+        python3 /odtp/odtp-component-client/parameters.py /odtp/odtp-app/config_templates/config_synthesis_corsica.yml /odtp/odtp-workdir/scenario/config.yml
 
         # Preparing input data folder
         ln -s /odtp/odtp-input/data /odtp/odtp-workdir/data
 
     else
         echo "Running Matsim PIPELINE"
-        python3 /odtp/odtp-client/parameters.py /odtp/odtp-app/config_templates/config_matsim_corsica.yml /odtp/odtp-workdir/scenario/config.yml
+        python3 /odtp/odtp-component-client/parameters.py /odtp/odtp-app/config_templates/config_matsim_corsica.yml /odtp/odtp-workdir/scenario/config.yml
 
         # Preparing input data folder
         ln -s /odtp/odtp-input/data /odtp/odtp-workdir/data
@@ -73,10 +73,10 @@ else
     # Reading placeholders and create config file from environment variables
     if [ "$PIPELINE" == "Synthesis" ]; then
         echo "Running Synthesis PIPELINE"
-        python3 /odtp/odtp-client/parameters.py /odtp/odtp-app/config_templates/config_synthesis_ch.yml /odtp/odtp-workdir/scenario/config.yml
+        python3 /odtp/odtp-component-client/parameters.py /odtp/odtp-app/config_templates/config_synthesis_ch.yml /odtp/odtp-workdir/scenario/config.yml
 
         # Preparing input data folder
-        ln -s /odtp/odtp-input/data /odtp/odtp-workdir/data
+        ln -s /odtp/odtp-component-client/data /odtp/odtp-workdir/data
 
     else
         echo "Running Matsim PIPELINE"
@@ -93,6 +93,13 @@ fi
 # Running Eqasim pipeline
 python3 -m synpp
 
+if [ "$PIPELINE" == "Matsim" ]; then
+    # For some reason it fails the first time the command is executed with a maven related 
+    # Could not transfer artifact org.geotools:gt-opengis:jar:24.2 from/to osgeo
+    sleep 10
+    python3 -m synpp
+fi
+
 # Copying output in odtp-output
 mkdir /odtp/odtp-output/eqasim-output
 cp -r /odtp/odtp-workdir/output/* /odtp/odtp-output/eqasim-output