Merge pull request #16 from SeQuenC-Consortium/release/27.08.23

Release/27.08.23

Author: GeorgDaniel

.github/workflows/run-pytests.yml

@@ -1,6 +1,6 @@
 name: Runs python tests
 
-on: [push]
+on: [ push ]
 
 jobs:
   run-unittests:
@@ -20,15 +20,5 @@ jobs:
           poetry-version: 1.4.0
       - name: Install requirements with poetry
         run: python -m poetry export --without-hashes --format=requirements.txt -o requirements.txt && python -m pip install -r requirements.txt
-      - name: Setup redis
-        uses: supercharge/[email protected]
-        with:
-          redis-version: 6
-      - name: Install redis cli # so we can test the server
-        run: sudo apt-get install -y redis-tools
-      - name: Verify that redis is up
-        run: redis-cli ping
-      - name: Install pytest-mock
-        run: python -m pip install pytest-mock
       - name: Run unit tests
         run: python -m pytest tests/automated_tests

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9
+FROM python:3.11
 
 # install git and remove caches again in same layer
 ARG DEBIAN_FRONTEND=noninteractive
@@ -23,12 +23,7 @@ ENV CELERY_WORKER_POOL=threads
 RUN mkdir --parents /app/instance \
     && chown --recursive gunicorn /app && chmod --recursive u+rw /app/instance
 
-# Wait for database
-ADD https://github.com/ufoscout/docker-compose-wait/releases/download/2.9.0/wait /wait
-RUN chmod +x /wait
-
-
-RUN python -m pip install poetry gunicorn invoke
+RUN python -m pip install poetry
 
 COPY --chown=gunicorn . /app
 
@@ -40,4 +35,4 @@ EXPOSE 5005
 
 USER gunicorn
 
-ENTRYPOINT ["python","-m", "invoke", "start-docker"]
+ENTRYPOINT ["python","-m", "invoke", "start-docker"]

README.md

@@ -66,6 +66,8 @@ celery worker and execute all tasks synchronously.
     * **PUT /devices/{device_id}/status** *(To check if a device is running)*
     * **PUT /devices/{device_id}/calibration** *(To get some device properties)*
 
+### Run manually
+
 Run the development server with
 
 ```bash
@@ -97,6 +99,18 @@ Trying out the tests -> See tests/README.md
 poetry run pytest .
 ```
 
+### Run using docker-compose
+
+Execute the following command the deployment will be started using docker-compose. This will build the dockerimage
+containing the application and creates all required containers including the database and the message queue.
+
+```bash
+docker-compose up -d
+docker-compose exec server python -m flask create-and-load-db
+```
+
+![Architecture](docker-compose-architecture.svg)
+
 ### Trying out the Template
 
 For a list of all dependencies with their license open <http://localhost:5005/licenses/>.

docker-compose-architecture.svg

@@ -0,0 +1,63 @@
+version: "3"
+services:
+  server:
+    build: .
+    image: qunicorn:local
+    networks:
+      - qunicorn
+    environment:
+      CONTAINER_MODE: server
+      SERVER_PORT: 8080
+      BROKER_URL: "redis://broker:6379"
+      DB_URL: "postgresql://postgres:passwd@postgres/qunicorn"
+    depends_on:
+      - postgres
+      - broker
+    ports:
+      - "8080:8080"
+    labels:
+      kompose.service.expose: "true"
+      kompose.service.type: "loadbalancer"
+
+  worker:
+    build: .
+    image: qunicorn:local
+    networks:
+      - qunicorn
+    environment:
+      CONTAINER_MODE: worker
+      BROKER_URL: "redis://broker:6379"
+      DB_URL: "postgresql://postgres:passwd@postgres/qunicorn"
+    depends_on:
+      - postgres
+      - broker
+    ports:
+      - "6379"
+
+  broker:
+    image: redis:7.0.12
+    networks:
+      - qunicorn
+    ports:
+      - "6379:6379"
+
+  postgres:
+    image: postgres:15.3
+    networks:
+      - qunicorn
+    environment:
+      PGDATA: /var/lib/postgresql/data/pgdata
+      POSTGRES_PASSWORD: passwd
+      POSTGRES_DB: qunicorn
+    volumes:
+      - pgdata:/var/lib/postgresql/data \
+    ports:
+      - "5432:5432"
+
+
+networks:
+  qunicorn:
+
+volumes:
+  pgdata:
+

docker-compose.yaml

@@ -0,0 +1,53 @@
+# How to Map Dtos to Database Objects and wise versa
+
+* Status: accepted
+
+## Context and Problem Statement
+
+We need multiple data transfer objects (DTOs) for different purposes.
+
+* One for the requests
+* One for the internal logic
+* One for the database
+* One for the responses
+
+Normally they need to map from request to core and then back to a response-dto.
+To save the updates to the database they need to be mapepd to the database object.
+To map all of these, we need a mapper.
+
+## Decision Drivers <!-- optional -->
+
+* Have all mapper at one place
+* Clean and readable code
+* Automatically map objects
+
+## Considered Options
+
+* object-mapper
+* py-automapper
+* map-struct
+
+## Decision Outcome
+
+py-automapper
+
+## Description of Changes
+
+A library was added. And used in the "core/mapper/" folder.
+
+### Folder Structure
+
+* core/mapper/
+    * \__init\__.py: Contains all imports for the mapper
+    * general_mapper.py: Contains a helper method which can be used to map objects with automapper
+    * mapper.py: Multiple mapper, one for each model
+
+### Naming Pattern
+
+The naming pattern for the mapper-methods is as follows:
+If a dataclass object needs to be mapped to a dto the name would be dataclass_to_dto.
+If there exists multiple dtos for one dataclass model, the name does not anymore include "dto".
+For example for the job_request_dto the mapper name is: dataclass_to_request.
+
+
+<!-- markdownlint-disable-file MD013 -->