Dockerize

Author: tomkralidis

Makefile

@@ -1,51 +1,55 @@
-# =================================================================
+###############################################################################
 #
-# Authors: Tom Kralidis <[email protected]>
+# Copyright (C) 2025 Tom Kralidis
 #
-# Copyright (c) 2024 Tom Kralidis
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
 #
-# Permission is hereby granted, free of charge, to any person
-# obtaining a copy of this software and associated documentation
-# files (the "Software"), to deal in the Software without
-# restriction, including without limitation the rights to use,
-# copy, modify, merge, publish, distribute, sublicense, and/or sell
-# copies of the Software, and to permit persons to whom the
-# Software is furnished to do so, subject to the following
-# conditions:
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the 
+# GNU General Public License for more details.
 #
-# The above copyright notice and this permission notice shall be
-# included in all copies or substantial portions of the Software.
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
 #
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-# OTHER DEALINGS IN THE SOFTWARE.
-#
-# =================================================================
+###############################################################################
+
+DOCKER_COMPOSE_ARGS=--project-name msc-wis2node --file docker-compose.yml --file docker-compose.override.yml
+
+build:
+	docker compose $(DOCKER_COMPOSE_ARGS) build
+
+force-build:
+	docker compose $(DOCKER_COMPOSE_ARGS) build --no-cache
 
-# Ubuntu
-SR3_CONFIG=${HOME}/.config/sr3
+up:
+	docker compose $(DOCKER_COMPOSE_ARGS) up
 
-# Mac OSX
-#SR3_CONFIG=${HOME}/Library/Application\ Support/sr3
+down:
+	docker compose $(DOCKER_COMPOSE_ARGS) down
 
-check:
-	@echo "SR3 configuration directory: ${SR3_CONFIG}"
+restart: down up
 
-install: setup
-	cp msc_wis2node/publisher.py $(SR3_CONFIG)/plugins
-	cp deploy/default/sarracenia/dd.weather.gc.ca-all.conf $(SR3_CONFIG)/subscribe
+login:
+	docker exec -it msc-wis2node-management /bin/bash
 
-setup:
-	mkdir -p $(SR3_CONFIG)/plugins
-	mkdir -p $(SR3_CONFIG)/subscribe
+dev:
+	docker compose $(DOCKER_COMPOSE_ARGS) --file docker-compose.dev.yml up
+
+reinit-backend:
+	docker exec -it msc-wis2node-management sh -c "msc-wis2node setup --force"
+
+logs:
+	docker compose $(DOCKER_COMPOSE_ARGS) logs --follow
 
 clean:
-	rm -fr $(SR3_CONFIG)/plugins/publisher.py
-	rm -fr $(SR3_CONFIG)/subscribe/dd.weather.gc.ca-all.conf
+	docker system prune -f
+	docker volume prune -f
+
+rm:
+	docker volume rm $(shell docker volume ls --filter name=msc-wis2node -q)
 
-.PHONY: check install setup clean
+.PHONY: build up dev login down restart reinit-backend force-build logs rm clean

docker-compose.override.yml

@@ -0,0 +1,23 @@
+###############################################################################
+#
+# Copyright (C) 2025 Tom Kralidis
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+services:
+  msc-wis2node-cache:
+    ports:
+        - 11211:11211

docker-compose.yml

@@ -0,0 +1,33 @@
+###############################################################################
+#
+# Copyright (C) 2025 Tom Kralidis
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+services:
+  msc-wis2node-cache:
+    image: memcached
+    container_name: msc-wis2node-cache
+    env_file:
+      - msc-wis2node.env
+  msc-wis2node:
+    container_name: msc-wis2node
+    build:
+      context: msc-wis2node-management
+    env_file:
+      - msc-wis2node.env
+    volumes:
+      - ./datasets.yml:/opt/msc-wis2node/conf/datasets.yml

msc-wis2node-management/Dockerfile

@@ -0,0 +1,58 @@
+###############################################################################
+#
+# Copyright (C) 2025 Tom Kralidis
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+FROM ubuntu:jammy
+
+LABEL maintainer="[email protected]"
+
+ARG SR3_CONFIG /home/msc-wis2node/.config/sr3
+
+ENV TZ="Etc/UTC" \
+    DEBIAN_FRONTEND="noninteractive" \
+    DEBIAN_PACKAGES="bash curl git python3-pip python3-setuptools sudo vim"
+
+# copy the app 
+COPY ./ /app
+
+RUN apt-get update -y && \
+    # install dependencies
+    apt-get install -y ${DEBIAN_PACKAGES} && \
+    pip3 install --no-cache-dir -r /app/requirements.txt && \
+    # install msc-wis2node
+    cd /app && \
+    pip3 install -e . && \
+    chmod +x /app/docker/entrypoint.sh && \
+    # cleanup
+    apt autoremove -y && \
+    apt-get -q clean && \
+    rm -rf /var/lib/apt/lists/* && \
+    # add msc-wis2node user
+    useradd -ms /bin/bash msc-wis2node && \
+    adduser msc-wis2node sudo && \
+    echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && \
+    # install sr3 components
+    mkdir -p $SR3_CONFIG/plugins && \
+    mkdir -p $SR3_CONFIG/subscribe && \
+    cp msc_wis2node/publisher.py $SR3_CONFIG/plugins && \
+    cp deploy/default/sarracenia/dd.weather.gc.ca-all.conf $SR3_CONFIG/subscribe
+
+USER msc-wis2node
+WORKDIR /home/msc-wis2node
+
+ENTRYPOINT [ "/app/docker/entrypoint.sh" ]

msc-wis2node-management/MANIFEST.in

@@ -0,0 +1 @@
+include README.md LICENSE requirements.txt

msc-wis2node-management/README.md

@@ -0,0 +1,3 @@
+# msc-wis2node
+
+Python package to perform MSC WIS2 Node management functions.

debian/changelog msc-wis2node-management/debian/changelog

@@ -0,0 +1,28 @@
+###############################################################################
+#
+# Copyright (C) 2025 Tom Kralidis
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+echo "START /entrypoint.sh"
+
+set +e
+
+echo "starting sr3..."
+
+sr3 --logStdout --debug foreground subscribe/dd.weather.gc.ca-all && sleep infinity
+
+echo "END /entrypoint.sh"

debian/compat msc-wis2node-management/debian/compat

@@ -31,6 +31,7 @@
 DATASET_CONFIG = os.environ.get('MSC_WIS2NODE_DATASET_CONFIG')
 TOPIC_PREFIX = os.environ.get('MSC_WIS2NODE_TOPIC_PREFIX', 'origin/a/wis2')
 DISCOVERY_METADATA_ZIP_URL = os.environ.get('MSC_WIS2NODE_DISCOVERY_METADATA_ZIP_URL')  # noqa
+CACHE = os.environ.get('MSC_WIS2NODE_CACHE')
 CENTRE_ID = os.environ.get('MSC_WIS2NODE_CENTRE_ID')
 WIS2_GDC = os.environ.get('MSC_WIS2NODE_WIS2_GDC')
 

debian/control msc-wis2node-management/debian/control

@@ -17,21 +17,24 @@
 #
 ###############################################################################
 
+from copy import deepcopy
 from datetime import date, datetime, timezone
 from fnmatch import fnmatch
 import json
 import logging
 import re
+import sqlite
 from typing import Union
 import uuid
 
 from paho.mqtt import publish
+from pymemcache.client import base as memcache_base
 from pywis_pubsub.publish import create_message
 from sarracenia.flowcb import FlowCB
 import yaml
 
 from msc_wis2node.env import (BROKER_HOSTNAME, BROKER_PORT, BROKER_USERNAME,
-                              BROKER_PASSWORD, CENTRE_ID, DATASET_CONFIG,
+                              BROKER_PASSWORD, CACHE, CENTRE_ID, DATASET_CONFIG,
                               TOPIC_PREFIX)
 from msc_wis2node.util import get_mqtt_client_id, get_mqtt_tls_settings
 
@@ -74,6 +77,7 @@ class WIS2Publisher:
     def __init__(self):
         """initialize"""
 
+        self.cache = memcache_base.Client(CACHE]
         self.datasets = []
         self.tls = None
 
@@ -178,6 +182,14 @@ def publish_to_wis2(self, dataset: dict, url: str) -> None:
             url=url
         )
 
+        LOGGER.debug('Checking for data update')
+        if message['properties']['data_id'] in self.cache:
+            update_link = deepcopy(message['links'][0])
+            update_link['rel'] = 'update'
+            message['links'].append(update_link)
+        else:
+            self.cache.append(messages['properties']['data_id']
+
         cache = dataset.get('cache', True)
         if not cache:
             LOGGER.info(f'Setting properties.cache={cache}')

debian/copyright msc-wis2node-management/debian/copyright

@@ -3,7 +3,8 @@ export MSC_WIS2NODE_BROKER_PORT=8883
 export MSC_WIS2NODE_BROKER_USERNAME=username
 export MSC_WIS2NODE_BROKER_PASSWORD=password
 export MSC_WIS2NODE_MSC_DATAMART_AMQP=amqps://dd.weather.gc.ca
-export MSC_WIS2NODE_DATASET_CONFIG=/opt/msc-wis2node/conf/datasets.yaml
+export MSC_WIS2NODE_DATASET_CONFIG=/opt/msc-wis2node/conf/datasets.yml
 export MSC_WIS2NODE_DISCOVERY_METADATA_ZIP_URL=https://example.org/discovery-metadata.zip
+export MSC_WIS2NODE_CACHE=msc-wis2node-cache:11211
 export MSC_WIS2NODE_CENTRE_ID=ca-eccc-msc
 export MSC_WIS2NODE_TOPIC_PREFIX=origin/a/wis2