23 limit log files in docker compose (#24)

* feat: add global levels for log config

As suggested by @SBlechmann

* feat: add log config to compose file

* feat: allow volume config for grafana in compose

* 22 adjust context file for data interoperability use case (#26)

* docs: context adjustments skamo / booster use case

* docs: added MultiSensor, SensorPoint, and isPointOf

* implement minimum example for building use case with multi sensor (Brick + SENSE ontologies)

* feat: adapt number of logging files and add logging options for ld setup

---------

Co-authored-by: Sebastian Blechmann <[email protected]>
Co-authored-by: Sebastian Blechmann <[email protected]>

Author: Maghnie

README.md

@@ -127,8 +127,8 @@ CrateDB requires a higher number of memory map areas.
 ***
 
 ### Setup adjustments to your needs:
-We provide a general development setup and try to keep most of the information generic. Most of the configuration parameters of the setup can be changed either in the `.yml` files directly, which we do not recommend. We gathered most of the configuration parameters in `.env` files so the information is well-arranged and can be changed in one place. 
-When using the single computer setup, the `.env` files can be read using a specifig flag. Unfortunately, this does not work for the multi node setup. Portainer is a tool that can help managing your cluster, see following section. When using portainer,  the `.env` files can be used even in a multi node setup. 
+We provide a general development setup and try to keep most of the information generic. Most of the configuration parameters of the setup can be changed either in the `.yml` files directly, which we do not recommend. We gathered most of the configuration parameters in `.env` files so the information is well-arranged and can be changed in one place. For example, to save storage space on the host machine of your platform, log levels can be changed from `DEBUG` to `WARN`. 
+When using the single computer setup, the `.env` files can be read using a specific flag. Unfortunately, this does not work for the multi node setup. Portainer is a tool that can help managing your cluster, see following section. When using portainer,  the `.env` files can be used even in a multi node setup. 
 
 In general, we suggest using portainer in any case due to its nice graphical interface that assists to manage your containerized environments. 
 

general/context-files/building.jsonld

@@ -0,0 +1,46 @@
+{
+    "@context": {
+      "brick": "https://brickschema.org/schema/brick#",
+      "sense": "https://w3id.org/sense#",
+      "unit": "http://qudt.org/vocab/unit/",
+      "esim": "https://w3id.org/esim#",
+      "mm": "https://w3id.org/mm#",
+      "example": "https://www.example.com/",
+      
+      
+      "AirTemperatureSensor": "brick:Air_Temperature_Sensor",
+      "Computed": "mm:Computed",
+      "DataPoint": "sense:DataPoint",
+      "Measured": "mm:Measured",
+      "MultiSensor": "sense:MultiSensor",
+      "Location": "brick:Location",
+      "Heater": "brick:Electric_Radiator",
+      "Humidity": "mm:Humidity",
+      "HumiditySensor": "brick:HumiditySensor",
+      "MetricType": "mm:MetricType",
+      "SensorEquipment": "brick:Sensor_Equipment",
+      "SensorPoint": "brick:Sensor",
+      "Setpoint": "brick:Setpoint",
+      "Space": "brick:Space",
+      "Temperature": "brick:Temperature",
+      "TemperatureSensor": "brick:Temperature_Sensor",
+      "TemperatureSetpoint": "brick:Temperature_Setpoint",
+      "Valve": "brick:Valve",
+      "ValvePosition": "brick:Valve_Position_Sensor",
+      "ValveCommand": "brick:Valve_Position_Command",
+      "WeatherStation": "brick:Weather_Station",
+
+      
+      "hasDataPoint": "sense:hasDataPoint",
+      "isHostedBy": "esim:isHostedBy",
+      "isDataPointOf": "sense:isDataPointOf",
+      "heaterPower": "brick:Heating_Command",
+            
+      
+      "degC": "unit:DEG_C",
+      "percent": "unit:PERCENT"   
+      
+      
+    },
+    "@graph": []
+  }

general/context-files/context.jsonld

@@ -178,8 +178,8 @@ IOTA_JSON_CB_HOST=orion-ld
 IOTA_JSON_CB_PORT=1026 
 # Context configuration
 # Where to find the context file(s), if multiple files shall be used, the URLs can be comma separated
-IOTA_JSON_LD_CONTEXT=http://context-server:80/context.jsonld
-# Port for confugiration of the IoT Agent (internal port)
+IOTA_JSON_LD_CONTEXT=http://context-server:80/building.jsonld
+# Port for confugiration of the IoT Agent.
 IOTA_JSON_NORTH_PORT=4041
 # Whether to hold IoT device info in memory or in a database
 IOTA_JSON_REGISTRY_TYPE=mongodb 
@@ -189,8 +189,6 @@ IOTA_JSON_LOG_LEVEL=DEBUG
 IOTA_JSON_TIMESTAMP=true 
 # v2 for NGSIv2, LD vor NGSI-LD, and mixed if both NGSI versions are used. LD support is still experimental
 IOTA_JSON_CB_NGSI_VERSION=LD
-# The link to the context file(s). If multiple files shall be used, the URLs can be comma separated
-IOTA_JSON_LD_CONTEXT=http://context-server:80/context.jsonld
 # The tenant to use if no tenant is provided in the fiware-service or ngsild-tenant header
 IOTA_JSON_FALLBACK_TENANT=DEFAULT
 # The path to use if no path is provided in the fiware-servicepath or scope header

ld/.env

@@ -1,5 +1,24 @@
 version: "3.7"
 
+# Define levels for log file size limits
+x-logging-low: &logging-low # Used by: mongo-express-ld, mongodb-ld, quantumleap-ld, mqtt-broker-ld, grafana, portainer, context-server
+  driver: "json-file"
+  options:
+      max-file: 2
+      max-size: 5m
+
+x-logging-medium: &logging-medium # Used by: orion-ld
+  driver: "json-file"
+  options:
+      max-file: 2
+      max-size: 10m
+
+x-logging-high: &logging-high # Used by: cratedb-ld, iot-agent-json-ld
+  driver: "json-file"
+  options:
+      max-file: 2
+      max-size: 50m
+
 services:
   orion-ld:
     #Sometimes the newest builds fail. Hence, we fixed a stable version that
@@ -33,6 +52,7 @@ services:
       - ORIONLD_NOTIF_MODE=${ORIONLD_NOTIF_MODE:-persistent}
       - ORIONLD_MUTEX_POLICY=${ORIONLD_MUTEX_POLICY:-all}
       - ORIONLD_CONN_MEMORY=${ORIONLD_CONN_MEMORY:-128}
+    logging: *logging-medium
 
   mongo-db-ld:
     image: mongo:${MONGODB_VERSION:-5.0.24} #Check https://docs.mongodb.com/manual/release-notes/ im Zweifel 4.2
@@ -43,6 +63,7 @@ services:
         - fiware-ld
     volumes:
       - ${MONGODB_VOLUME_PATH:-mongo-db-ld}:${MONGO_DATA_DIR:-/data/db}
+    logging: *logging-low
 
   mongo-express-ld:
     image: mongo-express:${ME_VERSION:-1.0.2}
@@ -59,6 +80,7 @@ services:
         - ME_CONFIG_MONGODB_ENABLE_ADMIN=${ME_CONFIG_MONGODB_ENABLE_ADMIN:-true}
         #- ME_CONFIG_MONGODB_AUTH_DATABASE=admin
         - ME_CONFIG_BASICAUTH=${ME_CONFIG_MONGODB_BASICAUTH:-false}
+    logging: *logging-low
 
 
   iot-agent-json-ld:
@@ -81,7 +103,7 @@ services:
       - IOTA_LOG_LEVEL=${IOTA_JSON_LOG_LEVEL:-DEBUG}
       - IOTA_TIMESTAMP=${IOTA_JSON_TIMESTAMP:-true}
       - IOTA_CB_NGSI_VERSION=${IOTA_JSON_CB_NGSI_VERSION:-LD}
-      - IOTA_JSON_LD_CONTEXT=${IOTA_JSON_LD_CONTEXT:-http://context-server:80/context.jsonld}
+      - IOTA_JSON_LD_CONTEXT=${IOTA_JSON_LD_CONTEXT:-http://context-server:80/building.jsonld}
       # - IOTA_FALLBACK_TENANT=${IOTA_JSON_FALLBACK_TENANT:-default}
       # - IOTA_FALLBACK_PATH=${IOTA_JSON_FALLBACK_PATH:-/}
       - IOTA_AUTOCAST=${IOTA_JSON_AUTOCAST:-false}
@@ -99,6 +121,7 @@ services:
       - IOTA_MQTT_PROTOCOL=${IOTA_JSON_MQTT_PROTOCOL:-mqtt}
       - IOTA_MQTT_KEEPALIVE=${IOTA_JSON_MQTT_KEEPALIVE:-60}
       - IOTA_AMQP_DISABLED=${IOTA_JSON_AMQP_DISABLED:-true}
+    logging: *logging-high
 
   mqtt-broker-ld:
     image: eclipse-mosquitto:${MOSQUITTO_VERSION:-2.0.11}
@@ -111,6 +134,7 @@ services:
         - fiware-ld
     volumes:
         - ../general/mosquitto.conf:/mosquitto/config/mosquitto.conf 
+    logging: *logging-low
 
   quantumleap-ld:
     image: orchestracities/quantumleap:${QL_VERSION:-1.0.0}
@@ -129,6 +153,7 @@ services:
       # - REDIS_HOST=${QL_REDIS_HOST:-redis-cache}
       # - REDIS_PORT=${QL_REDIS_PORT:-6379}
       # - CACHE_QUERIES=${QL_CACHE_QUERIES:-True}
+    logging: *logging-low
 
   crate-ld:
     image: crate:${CRATEDB_VERSION:-4.8.4} # Please check release notes!
@@ -144,6 +169,7 @@ services:
     environment: 
       - TZ=${TIMEZONE:-Europe/Berlin}
       - CRATE_HEAP_SIZE=${CRATEDB_HEAP_SIZE:-1g}
+    logging: *logging-high
 
   grafana:
     image: grafana/grafana:${GRAFANA_VERSION:-9.3.0}
@@ -159,6 +185,7 @@ services:
       - TZ=${TIMEZONE:-Europe/Berlin}
     volumes:
       - "grafana:/var/lib/grafana"
+    logging: *logging-low
 
   portainer:
     image: portainer/portainer-ce:${PORTAINER_VERSION:-2.19.4}
@@ -174,6 +201,7 @@ services:
       - /var/run/docker.sock:/var/run/docker.sock
       - portainer_data:/data
     restart: always
+    logging: *logging-low
 
   context-server:
     image: httpd:alpine
@@ -186,6 +214,7 @@ services:
     volumes:
         - context-files:/usr/local/apache2/htdocs/
     restart: always
+    logging: *logging-low
 
 volumes:
   mongo-db-ld:

ld/docker-compose.yml

@@ -212,6 +212,10 @@ IOTA_JSON_AMQP_DISABLED=true
 GRAFANA_VERSION=10.1.1
 # External port Grafana shall be reached at
 GRAFANA_PORT=3000
+# If no volume is used: Local file path on the node(s), e.g. something like "~/fiware/data/mongodb". Important for multi node setup.
+# GRAFANA_VOLUME_PATH=
+# Where the data is stored within the container
+GRAFANA_DATA_DIR=/var/lib/grafana
 
 # Portainer
 # Portainer version

v2/.env

@@ -1,5 +1,24 @@
 version: "3.8"
 
+# Define levels for log file size limits
+x-logging-low: &logging-low # Used by: mongo-express, mongodb1, mongodb2, mongodb3, quantumleap, mqtt-broker, grafana, portainer
+  driver: "json-file"
+  options:
+      max-file: 2
+      max-size: 5m
+
+x-logging-medium: &logging-medium # Used by: orion
+  driver: "json-file"
+  options:
+      max-file: 2
+      max-size: 10m
+
+x-logging-high: &logging-high # Used by: cratedb, iot-agent-json
+  driver: "json-file"
+  options:
+      max-file: 2
+      max-size: 50m
+
 services:
 
   orion:
@@ -27,6 +46,7 @@ services:
       # - ORION_TRQ_POOL_SIZE={ORION_THREADPOOL_SIZE=-2} # use only if ORION_NOTIFICATION_MODE is set to "threadpool"
       - ORION_MUTEX_POLICY=${ORION_REQUEST_POLICY:-all}
       - ORION_CONN_MEMORY=${ORION_CONNECTION_MEMORY:-128}
+    logging: *logging-medium
 
   mongo-db:
     image: mongo:${MONGODB_VERSION:-5.0.24}
@@ -37,6 +57,7 @@ services:
         - fiware
     volumes:
       - ${MONGODB_VOLUME_PATH:-mongo-db}:${MONGO_DATA_DIR:-/data/db}
+    logging: *logging-low
 
   mongo-express:
     image: mongo-express:${ME_VERSION:-1.0.2}
@@ -53,6 +74,7 @@ services:
         - ME_CONFIG_MONGODB_ENABLE_ADMIN=${ME_CONFIG_MONGODB_ENABLE_ADMIN:-true}
         #- ME_CONFIG_MONGODB_AUTH_DATABASE=admin
         - ME_CONFIG_BASICAUTH=${ME_CONFIG_MONGODB_BASICAUTH:-false}
+    logging: *logging-low
 
   iot-agent-json:
     image: fiware/iotagent-json:${IOTA_JSON_VERSION:-1.19.0}
@@ -89,6 +111,7 @@ services:
       - IOTA_MQTT_PROTOCOL=${IOTA_JSON_MQTT_PROTOCOL:-mqtt}
       - IOTA_MQTT_KEEPALIVE=${IOTA_JSON_MQTT_KEEPALIVE:-60}
       - IOTA_AMQP_DISABLED=${IOTA_JSON_AMQP_DISABLED:-true}
+    logging: *logging-high
 
   mqtt-broker:
     image: eclipse-mosquitto:${MOSQUITTO_VERSION:-2.0.11}
@@ -101,6 +124,7 @@ services:
         - fiware
     volumes:
         - ../general/mosquitto.conf:/mosquitto/config/mosquitto.conf
+    logging: *logging-low
 
   quantumleap:
     image: orchestracities/quantumleap:${QL_VERSION:-1.0.0}
@@ -122,6 +146,7 @@ services:
       # - REDIS_HOST=${QL_REDIS_HOST:-redis-cache}
       # - REDIS_PORT=${QL_REDIS_PORT:-6379}
       # - CACHE_QUERIES=${QL_CACHE_QUERIES:-True}
+    logging: *logging-low
 
   crate:
     image: crate:${CRATEDB_VERSION:-4.8.4} # Please check release notes!
@@ -137,6 +162,7 @@ services:
     environment: 
       - TZ=${TIMEZONE:-Europe/Berlin}
       - CRATE_HEAP_SIZE=${CRATEDB_HEAP_SIZE:-1g}
+    logging: *logging-high
 
   grafana:
     image: grafana/grafana:${GRAFANA_VERSION:-9.3.0}
@@ -151,7 +177,8 @@ services:
       - GF_INSTALL_PLUGINS=grafana-clock-panel,grafana-worldmap-panel,grafana-simple-json-datasource
       - TZ=${TIMEZONE:-Europe/Berlin}
     volumes:
-      - "grafana:/var/lib/grafana"
+      - ${GRAFANA_VOLUME_PATH:-grafana}:${GRAFANA_DATA_DIR:-/var/lib/grafana}
+    logging: *logging-low
 
   portainer:
     image: portainer/portainer-ce:${PORTAINER_VERSION:-2.19.4}
@@ -167,6 +194,7 @@ services:
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
       - portainer_data:/data
+    logging: *logging-low