Add plugins infrastructure and factor out the enhancer and GTFS generator

.gitignore

@@ -132,6 +132,7 @@ setup.ipynb
 
 *~
 /.idea/
+/.vscode/
 secrets
 gtfs/*.zip
 gtfs/*.pbf
@@ -142,4 +143,11 @@ data/failed/
 data/trash/
 data/gtfs/
 data/tmp
-
+data/**
+
+#these files are under app/static but they get copied to the outside directory on startup
+logging.conf
+config
+static/**
+templates/**
+conf/**

Dockerfile

@@ -4,17 +4,6 @@ LABEL maintainer="[email protected]"
 
 WORKDIR /app
 
-RUN \
-	apt update \
-	&& apt install -y \
-	# GDAL headers are required for fiona, which is required for geopandas.
-	# Also gcc is used to compile C++ code.
-	libgdal-dev g++ \
-	# libspatialindex is required for rtree.
-	libspatialindex-dev \
-	# Remove package index obtained by `apt update`.
-	&& rm -rf /var/lib/apt/lists/*
-
 ENV ADMIN_TOKEN=''
 ENV RIDE2GO_TOKEN=''
 
@@ -27,13 +16,12 @@ RUN pip install --no-cache-dir --upgrade -r /app/requirements.txt
 ENV MODULE_NAME=amarillo.main
 
 COPY ./amarillo /app/amarillo
-COPY enhancer.py /app
-COPY prestart.sh /app
-COPY ./static /app/static
-COPY ./templates /app/templates
-COPY config /app
-COPY logging.conf /app
-COPY ./conf /app/conf
+COPY ./amarillo/plugins /app/amarillo/plugins
+COPY ./amarillo/static/static /app/static
+COPY ./amarillo/static/templates /app/templates
+COPY ./amarillo/static/config /app
+COPY ./amarillo/static/logging.conf /app
+COPY ./amarillo/static/data /app/data
 
 # This image inherits uvicorn-gunicorn's CMD. If you'd like to start uvicorn, use this instead
 # CMD ["uvicorn", "amarillo.main:app", "--host", "0.0.0.0", "--port", "8000"]

Jenkinsfile

@@ -1,17 +1,21 @@
 pipeline {
-    agent any
+    agent { label 'builtin' }
     environment {
         GITEA_CREDS = credentials('AMARILLO-JENKINS-GITEA-USER')
         PYPI_CREDS = credentials('AMARILLO-JENKINS-PYPI-USER')
         TWINE_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi"
-        PLUGINS_REPO_URL = "git.gerhardt.io/api/packages/amarillo/pypi/simple"
-        DOCKER_REGISTRY_URL = 'https://git.gerhardt.io'
+        PLUGINS_REPO_URL = "https://git.gerhardt.io/api/packages/amarillo/pypi/simple"
+        PYPI_REPO_URL = "https://pypi.org/simple"
+        DOCKER_REGISTRY = 'git.gerhardt.io'
+        DERIVED_DOCKERFILE = 'standard.Dockerfile'
+        MITANAND_DOCKERFILE = 'mitanand.Dockerfile'
         OWNER = 'amarillo'
+        BASE_IMAGE_NAME = 'amarillo-base'
         IMAGE_NAME = 'amarillo'
-        AMARILLO_DISTRIBUTION = '0.2'
-        TAG = "${AMARILLO_DISTRIBUTION}.${BUILD_NUMBER}"
-        PLUGINS = 'amarillo-metrics amarillo-enhancer amarillo-grfs-export'
-        DEPLOY_WEBHOOK_URL = 'http://amarillo.mfdz.de:8888/mitanand'
+        MITANAND_IMAGE_NAME = 'amarillo-mitanand'
+        AMARILLO_DISTRIBUTION = '2.0.0'
+        TAG = "${AMARILLO_DISTRIBUTION}${env.BRANCH_NAME == 'main' ? '' : '-' + env.BRANCH_NAME}-${BUILD_NUMBER}"
+        DEPLOY_WEBHOOK_URL = "http://amarillo.mfdz.de:8888/dev"
         DEPLOY_SECRET = credentials('AMARILLO-JENKINS-DEPLOY-SECRET')
     }
     stages {
@@ -44,52 +48,97 @@ pipeline {
         }
         stage('Publish package to PyPI') {
             when {
-                branch 'release'
+                branch 'main'
             }
             steps {
                 sh 'python3 -m twine upload --verbose --username $PYPI_CREDS_USR --password $PYPI_CREDS_PSW ./dist/*'              
             }
         }
-        stage('Build Mitanand docker image') {
-            when {
-                branch 'mitanand'
-            }
+        stage('Build base docker image') {
             steps {
                 echo 'Building image'
                 script {
-                    docker.build("${OWNER}/${IMAGE_NAME}:${TAG}",
-                    //--no-cache to make sure plugins are updated
-                     "--no-cache --build-arg='PACKAGE_REGISTRY_URL=${PLUGINS_REPO_URL}' --build-arg='PLUGINS=${PLUGINS}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .")
+                    docker.build("${OWNER}/${BASE_IMAGE_NAME}:${TAG}")
                 }
             }
         }
-        stage('Push image to container registry') {
-            when {
-                branch 'mitanand'
+        stage('Push base image to container registry') {
+            steps {
+                echo 'Pushing image to registry'
+                script {
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
+                        def image = docker.image("${OWNER}/${BASE_IMAGE_NAME}:${TAG}")
+                        image.push()
+                        image.push('latest')
+                    }
+                }
             }
+        }
+        stage('Build derived docker image') {
+            steps {
+                echo 'Building image'
+                script {
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
+                        docker.build("${OWNER}/${IMAGE_NAME}:${TAG}",
+                        //--no-cache to make sure plugins are updated
+                        "-f ${DERIVED_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${env.BRANCH_NAME == 'main' ? env.PYPI_REPO_URL : env.PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .")
+                    }
+
+                }
+            }
+        }
+        stage('Push derived image to container registry') {
             steps {
                 echo 'Pushing image to registry'
                 script {
-                    docker.withRegistry(DOCKER_REGISTRY_URL, 'AMARILLO-JENKINS-GITEA-USER'){
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
                         def image = docker.image("${OWNER}/${IMAGE_NAME}:${TAG}")
                         image.push()
                         image.push('latest')
                     }
                 }
             }
         }
-        stage('Notify CD script') {
-            when {
-                branch 'mitanand'
+
+        stage('Build mitanand docker image') {
+            steps {
+                echo 'Building image'
+                script {
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
+                        docker.build("${OWNER}/${MITANAND_IMAGE_NAME}:${TAG}",
+                        //--no-cache to make sure plugins are updated
+                        "-f ${MITANAND_DOCKERFILE} --no-cache --build-arg='PACKAGE_REGISTRY_URL=${env.BRANCH_NAME == 'main' ? env.PYPI_REPO_URL : env.PLUGINS_REPO_URL}' --build-arg='DOCKER_REGISTRY=${DOCKER_REGISTRY}' --secret id=AMARILLO_REGISTRY_CREDENTIALS,env=GITEA_CREDS .")
+                    }
+
+                }
             }
+        }
+        stage('Push mitanand image to container registry') {
             steps {
-                echo 'Triggering deploy webhook'
+                echo 'Pushing image to registry'
                 script {
-                    def response = httpRequest contentType: 'APPLICATION_JSON',
-                        httpMode: 'POST', requestBody: '{}', authentication: 'AMARILLO-JENKINS-DEPLOY-SECRET',
-                        url: "${DEPLOY_WEBHOOK_URL}"
+                    docker.withRegistry("https://${DOCKER_REGISTRY}", 'AMARILLO-JENKINS-GITEA-USER'){
+                        def image = docker.image("${OWNER}/${MITANAND_IMAGE_NAME}:${TAG}")
+                        image.push()
+                        image.push('latest')
+                    }
                 }
             }
         }
+        // stage('Notify CD script') {
+        //     when {
+        //         not {
+        //             branch 'main'
+        //         }
+        //     }
+        //     steps {
+        //         echo 'Triggering deploy webhook'
+        //         script {
+        //             def response = httpRequest contentType: 'APPLICATION_JSON',
+        //                 httpMode: 'POST', requestBody: '{}', authentication: 'AMARILLO-JENKINS-DEPLOY-SECRET',
+        //                 url: "${DEPLOY_WEBHOOK_URL}"
+        //         }
+        //     }
+        // }
     }
-}
+}

MANIFEST.in

@@ -0,0 +1,2 @@
+recursive-include amarillo/static/ *
+recursive-include amarillo/tests/ *

README.md

@@ -6,12 +6,17 @@ An Amarillo is a [yellow-dressed person](https://www.dreamstime.com/sancti-spiri
 
 ## Setup
 
-- Python 3.9.2 with pip
+- Python 3.10 with pip
 - python3-venv
 
-Create a virtual environment `python3 -m venv venv`.
+Create a virtual environment:
+`python3 -m venv venv`.
+
+Activate the environment:
+`. venv/bin/activate`
+
+Install the dependencies: `pip install -r requirements.txt`.
 
-Activate the environment and install the dependencies `pip install -r requirements.txt`.
 
 Run `uvicorn amarillo.main:app`. 
 
@@ -54,13 +59,17 @@ $ protoc --proto_path=. --python_out=../services/gtfsrt gtfs-realtime.proto real
 $ sed 's/import gtfs_realtime_pb2/import amarillo.services.gtfsrt.gtfs_realtime_pb2/g' ../services/gtfsrt/realtime_extension_pb2.py | sponge ../services/gtfsrt/realtime_extension_pb2.py
 ```
 
+### Develop Amarillo and its plugins together
+
+To the develop Amarillo and its plugins concurrently, clone this repo and all of the repos you would like to make changes to. Install the local version of the plugin(s) into the virtual environment, simply using `pip install /path/to/plugin`, or `pip install -e /path/to/plugin` if your environment supports editable installs.
+
 ## Testing
 
 In the top directory, run `pytest amarillo/tests`.
 
 ## Docker
 
-Based on [tiangolo/uvicorn-gunicorn:python3.9-slim](https://github.com/tiangolo/uvicorn-gunicorn-docker)
+Based on [tiangolo/uvicorn-gunicorn:python3.10-slim](https://github.com/tiangolo/uvicorn-gunicorn-docker)
 
 - build `docker build -t amarillo .`
 - run `docker run --rm --name amarillo -p 8000:80 -e MAX_WORKERS="1" -e ADMIN_TOKEN=$ADMIN_TOKEN -e RIDE2GO_TOKEN=$RIDE2GO_TOKEN -e TZ=Europe/Berlin -v $(pwd)/data:/app/data amarillo`

amarillo/configuration.py

@@ -1,21 +1,15 @@
 # separate file so that it can be imported without initializing FastAPI
+from amarillo.routers.carpool import enhance_missing_carpools
 from amarillo.utils.container import container
-import json
 import logging
-from glob import glob
 
-from amarillo.models.Carpool import Agency, Carpool, Region
-from amarillo.services import stops
-from amarillo.services import trips
 from amarillo.services.agencyconf import AgencyConfService, agency_conf_directory
-from amarillo.services.carpools import CarpoolService
 from amarillo.services.agencies import AgencyService
 from amarillo.services.regions import RegionService
 
 from amarillo.services.config import config
 
 from amarillo.utils.utils import assert_folder_exists
-import amarillo.services.gtfs_generator as gtfs_generator
 
 logger = logging.getLogger(__name__)
 
@@ -48,42 +42,7 @@ def configure_services():
     logger.info("Loaded %d regions", len(container['regions'].regions))
 
     create_required_directories()
-
-
-def configure_enhancer_services():
-    configure_services()
-
-    logger.info("Load stops...")
-    with open(config.stop_sources_file) as stop_sources_file:
-        stop_sources = json.load(stop_sources_file)
-        stop_store = stops.StopsStore(stop_sources)
-
-    stop_store.load_stop_sources()
-    container['stops_store'] = stop_store
-    container['trips_store'] = trips.TripStore(stop_store)
-    container['carpools'] = CarpoolService(container['trips_store'])
-
-    logger.info("Restore carpools...")
-
-    for agency_id in container['agencies'].agencies:
-        for carpool_file_name in glob(f'data/carpool/{agency_id}/*.json'):
-            try:
-                with open(carpool_file_name) as carpool_file:
-                    carpool = Carpool(**(json.load(carpool_file)))
-                    container['carpools'].put(carpool.agency, carpool.id, carpool)
-            except Exception as e:
-                logger.warning("Issue during restore of carpool %s: %s", carpool_file_name, repr(e))
-
-        # notify carpool about carpools in trash, as delete notifications must be sent
-        for carpool_file_name in glob(f'data/trash/{agency_id}/*.json'):
-            with open(carpool_file_name) as carpool_file:
-                carpool = Carpool(**(json.load(carpool_file)))
-                container['carpools'].delete(carpool.agency, carpool.id)
-
-    logger.info("Restored carpools: %s", container['carpools'].get_all_ids())
-    logger.info("Starting scheduler")
-    gtfs_generator.start_schedule()
-
+    enhance_missing_carpools()
 
 def configure_admin_token():
     if config.admin_token is None:

amarillo/main.py

@@ -1,21 +1,26 @@
 import logging.config
-
-from amarillo.configuration import configure_services, configure_admin_token
-from amarillo.services.config import config
-
-logging.config.fileConfig('logging.conf', disable_existing_loggers=False)
-logger = logging.getLogger("main")
-
+import importlib
+import pkgutil
 import uvicorn
 import mimetypes
 from starlette.staticfiles import StaticFiles
 
+from amarillo.utils.utils import copy_static_files
+#this has to run before app.configuration is imported, otherwise we get validation error for config because the config file is not copied yet
+copy_static_files(["data", "static", "templates", "logging.conf", "config"]) 
+
+import amarillo.plugins
+from amarillo.services.config import config
+from amarillo.configuration import configure_services, configure_admin_token
 from amarillo.routers import carpool, agency, agencyconf, region
 from fastapi import FastAPI
 
 # https://pydantic-docs.helpmanual.io/usage/settings/
 from amarillo.views import home
 
+logging.config.fileConfig('logging.conf', disable_existing_loggers=False)
+logger = logging.getLogger("main")
+
 logger.info("Hello Amarillo!")
 
 app = FastAPI(title="Amarillo - The Carpooling Intermediary",
@@ -65,6 +70,10 @@
                       "description": "Demo server by MFDZ",
                       "url": "https://amarillo.mfdz.de"
                   },
+                  {
+                      "description": "Mitanand Amarillo service",
+                      "url": "https://mitanand.mfdz.de"
+                  },
                   {
                       "description": "Dev server for development",
                       "url": "https://amarillo-dev.mfdz.de"
@@ -79,10 +88,30 @@
 app.include_router(region.router)
 
 
+def iter_namespace(ns_pkg):
+     # Source: https://packaging.python.org/guides/creating-and-discovering-plugins/
+    return pkgutil.iter_modules(ns_pkg.__path__, ns_pkg.__name__ + ".")
+
+def load_plugins():
+    discovered_plugins = {
+        name: importlib.import_module(name)
+        for finder, name, ispkg
+        in iter_namespace(amarillo.plugins)
+    }
+    logger.info(f"Discovered plugins: {list(discovered_plugins.keys())}")
+
+    for name, module in discovered_plugins.items():
+        if hasattr(module, "setup"):
+            logger.info(f"Running setup function for {name}")
+            module.setup(app)
+
+        else: logger.info(f"Did not find setup function for {name}")
+
 def configure():
     configure_admin_token()
     configure_services()
     configure_routing()
+    load_plugins()
 
 
 def configure_routing():