A possible implementation for managing non-python agents

.github/workflows/develop.yml

@@ -9,6 +9,7 @@ on:
       - '**.md'
       - '.flake8'
       - '.pre-commit-config.yaml'
+      - '.readthedocs.yaml'
 
 jobs:
   test:

.github/workflows/pytest.yml

@@ -8,6 +8,7 @@ on:
       - '**.md'
       - '.flake8'
       - '.pre-commit-config.yaml'
+      - '.readthedocs.yaml'
   workflow_call:
 
 jobs:

.github/workflows/skipped-pytest.yml

@@ -8,6 +8,7 @@ on:
       - '**.md'
       - '.flake8'
       - '.pre-commit-config.yaml'
+      - '.readthedocs.yaml'
 
 jobs:
   test:

.pre-commit-config.yaml

@@ -7,10 +7,10 @@ repos:
     -   id: end-of-file-fixer
     -   id: trailing-whitespace
 -   repo: https://github.com/pre-commit/mirrors-autopep8
-    rev: v2.0.1
+    rev: v2.0.4
     hooks:
     -   id: autopep8
 -   repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 6.1.0
     hooks:
     -   id: flake8

.readthedocs.yaml

@@ -1,10 +1,14 @@
 version: 2
 
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.10"
+
 sphinx:
     configuration: docs/conf.py
 
 python:
-    version: 3.7
     install:
     - requirements: docs/requirements.txt
     - requirements: requirements.txt

docker/crossbar/Dockerfile

@@ -8,8 +8,11 @@ FROM crossbario/crossbar:cpy3-20.8.1
 # Run as root to put config in place and chown
 USER root
 
-# Copy in config and requirements files
-COPY config.json /ocs/.crossbar/config.json
+# Copy in config template and wrapper script
+COPY run-crossbar.sh /ocs/run-crossbar.sh
+RUN chmod a+x /ocs/run-crossbar.sh
+
+COPY config.json.template /ocs/.crossbar/config-with-address.json.template
 RUN chown -R crossbar:crossbar /ocs
 
 # Run image as crossbar user during normal operation
@@ -20,4 +23,4 @@ EXPOSE 8001
 
 # Run crossbar when the container launches
 # User made config.json should be mounted to /ocs/.crossbar/config.json
-ENTRYPOINT ["crossbar", "start", "--cbdir", "/ocs/.crossbar"]
+ENTRYPOINT ["/ocs/run-crossbar.sh"]

docker/crossbar/config.json → docker/crossbar/config.json.template

@@ -10,13 +10,13 @@
             },
             "realms": [
                 {
-                    "name": "test_realm",
+                    "name": "{realm}",
                     "roles": [
                         {
                             "name": "iocs_agent",
                             "permissions": [
                                 {
-                                    "uri": "observatory.",
+                                    "uri": "{address_root}.",
                                     "match": "prefix",
                                     "allow": {
                                         "call": true,
@@ -36,7 +36,7 @@
                             "name": "iocs_controller",
                             "permissions": [
                                 {
-                                    "uri": "observatory.",
+                                    "uri": "{address_root}.",
                                     "match": "prefix",
                                     "allow": {
                                         "call": true,
@@ -60,7 +60,7 @@
 		    "type": "web",
                     "endpoint": {
                         "type": "tcp",
-                        "port": 8001
+                        "port": {port}
                     },
 		    "paths": {
 			"ws": {
@@ -81,7 +81,7 @@
 			},
 			"call": {
 			    "type": "caller",
-			    "realm": "test_realm",
+			    "realm": "{realm}",
 			    "role": "iocs_controller",
 			    "options": {
 				}

docker/crossbar/run-crossbar.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+CONFIG_DIR=/ocs/.crossbar
+OCS_ADDRESS_ROOT=${OCS_ADDRESS_ROOT:-observatory}
+OCS_CROSSBAR_REALM=${OCS_REALM:-test_realm}
+OCS_CROSSBAR_PORT=${OCS_PORT:-8001}
+
+# Did user mount in a config.json?
+if [ -e $CONFIG_DIR/config.json ]; then
+    echo Launching user-provided config.json
+    CONFIG_FILE=$CONFIG_DIR/config.json
+else
+    pattern="
+        s/{address_root}/${OCS_ADDRESS_ROOT}/g
+        s/{realm}/${OCS_CROSSBAR_REALM}/g
+        s/{port}/${OCS_CROSSBAR_PORT}/g
+        "
+    echo "Processing template with replacements:"
+    echo "$pattern"
+    echo
+
+    CONFIG_FILE=$CONFIG_DIR/config-with-address.json
+    sed "$pattern" \
+        $CONFIG_DIR/config-with-address.json.template \
+        > $CONFIG_FILE
+fi
+
+crossbar start --cbdir $CONFIG_DIR --config $CONFIG_FILE

docs/agents/aggregator.rst

@@ -90,7 +90,9 @@ to register a feed so that it will be recorded by the aggregator.
 Unregistered providers will automatically be added when they send data,
 and stale providers will be removed if no data is received in a specified
 time period.
-To do this, the aggregator monitors all feeds in the ``observatory`` namespace to find
+
+To do this, the aggregator monitors all feeds in the namespace defined
+by the `{address_root}` prefix to find
 feeds that should be recorded.  If the aggregator receives data from a feed
 registered with ``record=True``, it will automatically add that feed as a
 Provider, and will start putting incoming data into frames every ``frame_length``

docs/developer/writing_an_agent/docker.rst

@@ -84,7 +84,6 @@ the BarbonesAgent config to the ``ocs-docker`` host.:
       wamp_http: http://localhost:8001/call
       wamp_realm: test_realm
       address_root: observatory
-      registry_address: observatory.registry
 
     hosts:
 

docs/user/centralized_management.rst

@@ -476,6 +476,7 @@ should look something like this::
   ExecStart=/home/ocs/git/ocs-site-configs/my-lab/launcher-hm-server5.sh
   User=ocs
   Restart=always
+  RestartSec=10s
 
   [Install]
   WantedBy=multi-user.target

docs/user/crossbar_config.rst

@@ -12,48 +12,118 @@ the interface to the crossbar server.
 
 .. note::
 
-    For most test deployments of OCS, you should not need to modify this file
-    and can use the one that comes with the ``simonsobs/ocs-crossbar`` Docker
-    Image.
+    For most simple lab deployments of OCS, you should not need to modify this
+    file and can use the one that comes with the ``simonsobs/ocs-crossbar``
+    Docker Image.
 
-Example Config
---------------
-An example of the default OCS crossbar config that is bundled into
-``simonsobs/ocs-crossbar`` can be found in the repository at
-`ocs/docker/crossbar/config.json`_. This is based on the template in
-`ocs/ocs/support/crossbar_config.json`_.
+Configuration File Template
+---------------------------
+The template that the default OCS crossbar config is built with is shown here:
+
+.. literalinclude:: ../../docker/crossbar/config.json.template
 
-The unique parts of this to OCS are the realm name, "test_realm", defined
-roles of "iocs_agent" and "iocs_controller, and "address_root" of
-"observatory.". Additionally, we run on port 8001.
+The variables `realm`, `address_root`, and `port` are all configurable and must
+match configuration options set in your SCF. Keep reading this page to see how
+to configure these variables.
+
+.. note::
+    Changing the `address_root` has implications for the how your data is
+    stored and accessed in tools such as Grafana. It is recommended you pick
+    something reasonable when you first configure your system and do not change it
+    later.
 
 For further details on crossbar server configuration, see the crossbar `Router
 Configuration`_ page.
 
-.. _`ocs/docker/crossbar/config.json`: https://github.com/simonsobs/ocs/blob/main/docker/crossbar/config.json
-.. _`ocs/ocs/support/crossbar_config.json`: https://github.com/simonsobs/ocs/blob/main/ocs/support/crossbar_config.json
 .. _`Router Configuration`: https://crossbar.io/docs/Router-Configuration/
 
+Running with Docker
+===================
+
+We recommend running crossbar within a Docker container. We build the
+``simonsobs/ocs-crossbar`` container from the official `crossbar.io Docker
+image`_, specifically the cpy3 version. Bundled within the container is a
+simple crossbar configuration file template with defaults that are
+compatible with examples in this documentation.
+
+To adjust the crossbar configuration in the container, you can either:
+
+- Use environment variables to alter the most basic settings
+- Generate and mount a new configuration file over
+  ``/ocs/.crossbar/config.json`` with the proper permissions
+
+.. _`crossbar.io Docker image`: https://hub.docker.com/r/crossbario/crossbar
+
+Environment variables in ocs-crossbar
+-------------------------------------
+The following environment variables can be set, to affect the
+generation of the crossbar configuration file when the container
+starts up:
+
+- OCS_ADDRESS_ROOT (default "observatory"): the base URI for OCS
+  entities (this needs to match the `address_root` set in the SCF).
+- OCS_CROSSBAR_REALM (default "test_realm"): the WAMP realm to
+  configure for OCS.
+- OCS_CROSSBAR_PORT (default 8001): the port on which crossbar will
+  accept requests.
+
+Here is an example of a docker-compose entry that overrides the
+OCS_ADDRESS_ROOT::
+
+      crossbar:
+        image: simonsobs/ocs-crossbar:latest
+        ports:
+          - "127.0.0.1:8001:8001" # expose for OCS
+        environment:
+          - PYTHONUNBUFFERED=1
+          - OCS_ADDRESS_ROOT=laboratory
+
+Bind Mounting the Configuration
+-------------------------------
+To instead mount a new configuration into the pre-built image, first chown
+your file to be owned by user and group 242 (the default crossbar UID/GID),
+then mount it appropriately in your docker-compose file. Here we assume you
+put the configuration in the directory ``./dot_crossbar/``::
+
+    $ chown -R 242:242 dot_crossbar/
+
+.. note::
+    If you do not already have a configuration file to modify and use, see the
+    next section on generating one.
+
+Your docker-compose service should then be configured like::
+
+    crossbar:
+      image: simonsobs/ocs-crossbar
+      ports:
+        - "8001:8001" # expose for OCS
+      volumes:
+        - ./dot_crossbar:/ocs/.crossbar
+      environment:
+        - PYTHONUNBUFFERED=1
+
 Generating a New Config File
 ----------------------------
-``ocsbow`` can be used to generate a default configuration file, based on
-options in your OSC file, which can then be modified if needed.
+``ocs-local-support`` can be used to generate a default configuration
+file, based on options in your SCF, which can then be modified if
+needed.
 
 First, we make sure our ``OCS_CONFIG_DIR`` environment variable is set::
 
     $ cd ocs-site-configs/
     $ export OCS_CONFIG_DIR=`pwd`
 
-We should make a directory for the crossbar config, let's call it
-``dot_crossbar/`` (typically a dot directory, but for visibility we'll avoid
-that)::
+We should make a directory for the crossbar config, following along above let's
+call it ``dot_crossbar/`` (typically a dot directory, but for visibility we'll
+avoid that)::
 
     $ mkdir -p ocs-site-configs/dot_crossbar/
 
 This directory needs to be configured as your crossbar 'config-dir' in your
-ocs-site-config file. Now we can generate the config::
+ocs-site-config file. (See example in :ref:`site_config_user`.) Now we can
+generate the config::
 
-    $ ocsbow crossbar generate_config
+    $ ocs-local-support generate_crossbar_config
     The crossbar config-dir is set to:
       ./dot_crossbar/
     Using
@@ -68,54 +138,7 @@ modifications needed for your deployment.
 
 .. note::
 
-    The crossbar 'config-dir' block and the 'agent-instance' block defining the
-    'HostManager' Agent are both required for the system you are running ocsbow
-    on. Be sure to add these to your SCF if they do not exist.
-
-Running with Docker
-===================
-
-We recommend running crossbar within a Docker container. We build the
-``simonsobs/ocs-crossbar`` container from the official `crossbar.io Docker
-image`_, specifically the cpy3 version. Bundled within the container is a
-simple OCS configuration that should work with the configuration
-recommendations in this documentation.
-
-If changes need to be made, then you will need to generate your own
-configuration file as described above. To use a modified configuration in the
-container you can either:
-
-- Edit the default configuration file and rebuild the Docker image
-- Mount the new configuration file over ``/ocs/.crossbar/config.json`` with the
-  proper permissions
-
-.. _`crossbar.io Docker image`: https://hub.docker.com/r/crossbario/crossbar
-
-Rebuilding the Docker Image
----------------------------
-To rebuild the Docker image after modifying ``ocs/docker/config.json`` run::
-
-    $ docker build -t ocs-crossbar .
-
-You should then update your configuration to use the new, local,
-``ocs-crossbar`` image.
-
-Bind Mounting the Configuration
--------------------------------
-To instead mount the new configuration into the pre-built image, first chown
-your file to be owned by user and group 242 (the default crossbar UID/GID),
-then mount it appropriately in your docker-compose file. Here we assume you
-put the configuration in the directory ``./dot_crossbar/``::
-
-    $ chown -R 242:242 dot_crossbar/
-
-Your docker-compose service should then be configured like::
-
-    crossbar:
-    image: simonsobs/ocs-crossbar
-    ports:
-      - "8001:8001" # expose for OCS
-    volumes:
-      - ./dot_crossbar:/ocs/.crossbar
-    environment:
-         - PYTHONUNBUFFERED=1
+    The crossbar 'config-dir' block and the 'agent-instance' block
+    defining the 'HostManager' Agent are both required for the system
+    you are running `ocs-local-support` on. Be sure to add these to
+    your SCF if they do not exist.

docs/user/docker_config.rst

@@ -61,7 +61,7 @@ components)::
         ports:
           - "127.0.0.1:8001:8001" # expose for OCS
         environment:
-             - PYTHONUNBUFFERED=1
+          - PYTHONUNBUFFERED=1
 
       # --------------------------------------------------------------------------
       # OCS Agents
@@ -242,7 +242,7 @@ Where the separate compose files would look something like::
         ports:
           - "127.0.0.1:8001:8001" # expose for OCS
         environment:
-             - PYTHONUNBUFFERED=1
+          - PYTHONUNBUFFERED=1
 
 ::
 

docs/user/quickstart.rst

@@ -48,7 +48,6 @@ structure.
       wamp_http: http://localhost:8001/call
       wamp_realm: test_realm
       address_root: observatory
-      registry_address: observatory.registry
 
     hosts: