docs: update part 1

Author: PeterKraus

docs/apidoc_t/package.rst_t

@@ -15,10 +15,18 @@
 
 {%- if is_namespace %}
 {{- [pkgname, "namespace"] | join(" ") | e | heading }}
-{% elif 'tomato.drivers.dummy' == pkgname %}
-{{- "**dummy**: A dummy driver module" | heading }}
-{% elif 'tomato.drivers.biologic' == pkgname %}
-{{- "**biologic**: Driver for BioLogic potentiostats" | heading }}
+{% elif 'tomato.daemon' == pkgname %}
+{{- "**tomato.daemon**: Functions and modules comprising the tomato daemon" | heading }}
+{% elif 'tomato.driverinterface_2_0' == pkgname %}
+{{- "**tomato.DriverInterface**: version 2.0" | heading }}
+{% elif 'tomato.driverinterface_2_1' == pkgname %}
+{{- "**tomato.DriverInterface**: version 2.1" | heading }}
+{% elif 'tomato.ketchup' == pkgname %}
+{{- "**tomato.ketchup**: CLI and API for the tomato job queue" | heading }}
+{% elif 'tomato.tomato' == pkgname %}
+{{- "**tomato.tomato**: CLI and API for the tomato daemon" | heading }}
+{% elif 'tomato.passata' == pkgname %}
+{{- "**tomato.passata**: CLI and API for tomato drivers and components" | heading }}
 {% else %}
 {{- [pkgname, "package"] | join(" ") | e | heading }}
 {% endif %}

docs/source/quickstart.rst

@@ -141,6 +141,7 @@ The following concepts are used in **tomato**:
 
       pip3[pipeline 3] -.-> c3
 
+.. _settings-file:
 
 Settings file
 `````````````
@@ -212,6 +213,8 @@ Additional, *driver*-specific settings may be provided in this section. Each *dr
 
 Further *driver*-specific settings, such as ``dllpath`` or ``calibration``, can be specified here. All of these *driver*-specific settings are passed to each *driver* when its process is launched and the :class:`DriverInterface` is initialised, and can therefore contain paths to various libraries or other files necessary for the *driver* to function.
 
+.. _devices-file:
+
 Devices file
 ````````````
 This ``yaml``-formatted file contains information about each *device*, corresponding to an individual piece of hardware managed by **tomato**, as well as information about the organisation of the individually-addressable *components* of those *devices* into *pipelines*.
@@ -344,4 +347,4 @@ As of ``tomato-2.0``, the :obj:`task_params` specified in the *payload* are vali
 
 .. |devfile| replace:: *devices file*
 
-.. _devfile: quickstart.html#devices-file
+.. _devfile: quickstart.html#devices-file

src/tomato/__init__.py

@@ -45,7 +45,8 @@ def parse_args(parser, verbose, is_tomato=False):
     if "func" in args:
         ret = args.func(**vars(args), verbosity=verbosity, context=context, **kwargs)
         if args.yaml:
-            print(yaml.dump(ret.dict()))
+            ret.data.model_dump()
+            print(yaml.dump(ret.model_dump()))
         else:
             print(f"{'Success' if ret.success else 'Failure'}: {ret.msg}")
 

src/tomato/daemon/__init__.py

@@ -1,6 +1,4 @@
 """
-**tomato.daemon**: module of functions comprising the tomato daemon
--------------------------------------------------------------------
 .. codeauthor::
     Peter Kraus
 
@@ -24,8 +22,7 @@
 
 def setup_logging(daemon: Daemon):
     """
-    Helper function to set up logging (folder, filename, verbosity, format) based on
-    the passed daemon state.
+    Helper function to set up logging (folder, filename, verbosity, format) based on the passed daemon state.
     """
     logdir = Path(daemon.settings["logdir"])
     logdir.mkdir(parents=True, exist_ok=True)
@@ -39,12 +36,9 @@ def setup_logging(daemon: Daemon):
 
 def tomato_daemon():
     """
-    The function called when `tomato-daemon` is executed.
+    The function called when :obj:`tomato-daemon` is executed.
 
-    Manages the state of the tomato daemon, including recovery of state via
-    :mod:`~tomato.daemon.io`, processing state updates via :mod:`~tomato.daemon.cmd`,
-    and the manager threads for both jobs (:mod:`~tomato.daemon.job`) and drivers
-    (:mod:`~tomato.daemon.driver`).
+    Manages the state of the tomato daemon, spawning manager threads for jobs (:mod:`~tomato.daemon.job`), drivers (:mod:`~tomato.daemon.driver`), and pipelines (:mod:`~tomato.daemon.pip`). Parses the configuration in the :ref:`settings file <settings-file>` and :ref:`devices file <devices-file>`.
     """
     parser = argparse.ArgumentParser(add_help=False)
     parser.add_argument("--port", "-p", type=int, default=1234)
@@ -57,8 +51,7 @@ def tomato_daemon():
     setup_logging(daemon)
     logger.info("logging set up with verbosity %s", daemon.verbosity)
 
-    # TODO: setup should not be a thing really.
-    cmd.setup(msg={}, daemon=daemon)
+    cmd.reload(msg={}, daemon=daemon)
     context = zmq.Context()
     rep = context.socket(zmq.REP)
     logger.debug("binding zmq.REP socket on port %d", daemon.port)
@@ -80,15 +73,15 @@ def tomato_daemon():
         socks = dict(poller.poll(1000))
         if rep in socks:
             msg = rep.recv_pyobj()
-            logger.debug(f"received {msg=}")
+            logger.debug("received msg: %s", msg)
             if "cmd" not in msg:
-                logger.error(f"received msg without cmd: {msg=}")
+                logger.error("received msg without cmd: %s", msg)
                 ret = Reply(success=False, msg="received msg without cmd", data=msg)
             elif hasattr(cmd, msg["cmd"]):
                 ret = getattr(cmd, msg["cmd"])(msg, daemon)
             else:
-                logger.error(f"received msg with an invalid cmd: {msg=}")
-            logger.debug(f"reply with {ret=}")
+                logger.error("received msg with an invalid cmd: %s", msg["cmd"])
+            logger.debug("reply: %s", ret)
             rep.send_pyobj(ret)
         if daemon.status == "stop":
             end = True

src/tomato/daemon/cmd.py

@@ -4,9 +4,7 @@
 .. codeauthor::
     Peter Kraus
 
-All functions in this module expect a :class:`dict` containing the command specification
-and a :class:`~tomato.models.Daemon` object as arguments. The :class:`Daemon` object is
-altered by the command.
+All functions in this module expect a :class:`dict` containing the command specification and a :class:`~tomato.models.Daemon` object as arguments. The :class:`~tomato.models.Daemon` object may be altered by the command.
 
 All functions in this module return a :class:`~tomato.models.Reply`.
 
@@ -27,10 +25,12 @@
 
 
 def status(msg: dict, daemon: Daemon) -> Reply:
+    """Return daemon status, containing the current tomato configuration."""
     return Reply(success=True, msg=daemon.status, data=daemon)
 
 
 def stop(msg: dict, daemon: Daemon) -> Reply:
+    """Stop the tomato daemon."""
     logger = logging.getLogger(f"{__name__}.stop")
     logger.debug("%s", msg)
     dbpath = daemon.settings["jobs"]["dbpath"]
@@ -44,9 +44,16 @@ def stop(msg: dict, daemon: Daemon) -> Reply:
         return Reply(success=True, msg="daemon set to stop")
 
 
-def setup(msg: dict, daemon: Daemon) -> Reply:
-    logger = logging.getLogger(f"{__name__}.setup")
+def reload(msg: dict, daemon: Daemon) -> Reply:
+    """
+    Set-up or reload the tomato daemon using its configuration files.
+
+    .. note::
+
+        When reloading settings, tomato checks whether any running jobs use resources (i.e. drivers, pipelines, components) that would be removed if the new configuration were to be applied. If this is the case, the configuration **will not be updated**.
 
+    """
+    logger = logging.getLogger(f"{__name__}.setup")
     if daemon.status == "bootstrap":
         dbpath = daemon.settings["jobs"]["dbpath"]
         drvs = []
@@ -82,7 +89,6 @@ def setup(msg: dict, daemon: Daemon) -> Reply:
                 success=False,
                 msg="could not parse updated settings",
             )
-        logger.debug(f"{nd=}")
         ndf = nd.devicefile
         # First, check that we're not touching anything associated with a running job
         check_components = set()
@@ -148,9 +154,4 @@ def setup(msg: dict, daemon: Daemon) -> Reply:
         daemon.devicefile = ndf
         logger.info("reload successful with pipelines: '%s'", ndf.pipelines.keys())
 
-    return Reply(success=True, msg="setup successful", data=daemon)
-
-
-def reload(msg: dict, daemon: Daemon, **kwargs: dict) -> Reply:
-    # daemon.settings = toml.load(Path(daemon.appdir) / "settings.toml")
-    return Reply(success=True, msg="daemon settings reloaded", data=daemon.settings)
+    return Reply(success=True, msg="reload successful", data=daemon)

src/tomato/daemon/crates.py

@@ -1,3 +1,10 @@
+"""
+**tomato.daemon.crates**: functions for creating RO-crates
+----------------------------------------------------------
+.. codeauthor::
+    Peter Kraus
+"""
+
 import logging
 from typing import Union
 

src/tomato/daemon/db.py

@@ -1,3 +1,10 @@
+"""
+**tomato.daemon.db**: shared functions using :mod:`sqlite3`
+-----------------------------------------------------------
+.. codeauthor::
+    Peter Kraus
+"""
+
 import logging
 import os
 import sqlite3

src/tomato/daemon/driver.py

@@ -34,8 +34,19 @@
 
 
 def tomato_driver_bootstrap(
-    req: zmq.Socket, logger: logging.Logger, interface: ModelInterface, driver: str
+    req: zmq.Socket,
+    logger: logging.Logger,
+    interface: ModelInterface,
+    driver: str,
 ):
+    """
+    Function that attempts to register all configured components for this driver.
+
+    This helper function is executed when the ``register`` command is set to the driver process. The daemon is first polled for up-to-date configuration, and then each of the returned components is registered, if necessary, using :func:`cmp_register` of the driver interface.
+
+    In case the registration fails, a limited number of retries (as specified by the ``MAX_REGISTER_RETRIES`` constant) can be attempted on subsequent runs of this function.
+
+    """
     logger.debug("getting daemon status")
     req.send_pyobj(dict(cmd="status"))
     daemon: Daemon = req.recv_pyobj().data
@@ -73,6 +84,16 @@ def tomato_driver_bootstrap(
 def perform_idle_measurements(
     interface: ModelInterface, t_last: Union[float, None]
 ) -> Union[float, None]:
+    """
+    Function running idle measurements on the driver.
+
+    This function periodically runs the :func:`cmp_measure` on each component on the driver. The interval is determined from driver configuration using the ``"idle_measurement_interval"`` setting, driver defaults using the :obj:`interface.idle_measurement_interval` object, or tomato default (``IDLE_MEASUREMENT_INTERVAL``).
+
+    .. note::
+
+        How idle measurements are handled is up to the individual driver. By default, the :func:`cmp_measure` function will not submit new measurements when a task or a measurement is already running.
+
+    """
     if not hasattr(interface, "cmp_measure"):
         return t_last
 
@@ -93,17 +114,29 @@ def perform_idle_measurements(
     return t_now
 
 
+def stop_tomato_driver(port: int, context) -> Reply:
+    """
+    The default mechanism for stopping tomato drivers.
+
+    This function is used by the tomato driver manager to gracefully stop the driver, if an existing driver port is known.
+    """
+    req = context.socket(zmq.REQ)
+    req.connect(f"tcp://127.0.0.1:{port}")
+    req.send_pyobj(dict(cmd="stop", sender=f"{__name__}.stop_tomato_driver"))
+    return req.recv_pyobj()
+
+
 def kill_tomato_driver(pid: int):
     """
-    Wrapper around :func:`psutil.terminate`.
+    The backup mechanism for stoping tomato drivers.
+
+    This function is useful if the driver port is unknown or not responsive.
+
+    Wrapper around :func:`psutil.terminate`. Here we kill the (grand)children of the process with the name of `tomato-job`, i.e. the individual task functions. This allows the `tomato-job` process to exit gracefully once the task functions join.
 
-    Here we kill the (grand)children of the process with the name of `tomato-job`,
-    i.e. the individual task functions. This allows the `tomato-job` process to exit
-    gracefully once the task functions join.
+    .. note::
 
-    Note that on Windows, the `tomato-job.exe` process has two children: a `python.exe`
-    which is the actual process running the job, and `conhost.exe`, which we want to
-    avoid killing.
+        On Windows, the `tomato-job.exe` process has two children: a `python.exe` process which is the actual process running the job, and `conhost.exe` process, which we want to avoid killing.
 
     """
     proc = psutil.Process(pid)
@@ -121,17 +154,11 @@ def tomato_driver() -> None:
     """
     The function called when `tomato-driver` is executed.
 
-    This function is responsible for managing all activities involving devices of a
-    single driver type.
+    This function is responsible for managing all activities involving devices of a single driver type.
 
-    First, the list of devices (and their channel/address) for the specified driver is
-    fetched from the `tomato-daemon`. Then, a new instance of the specified driver is
-    spawned, populating its device map using the above list. If successful, the current
-    process information is fed back to the `tomato-daemon`.
+    First, the list of devices (and their channel/address) for the specified driver is fetched from the `tomato-daemon`. Then, a new instance of the specified driver is spawned, populating its device map using the above list. The state of the driver is stored .
 
-    Afterwards, the main loop handles all requests related to each of the devices
-    managed by this driver process, including job commands. Finally, if the driver is
-    instructed to stop, it attempts to perform a teardown before exiting.
+    Afterwards, the main loop handles all requests related to each of the devices managed by this driver process, including job commands. Finally, if the driver is instructed to stop, it attempts to perform a teardown before exiting.
     """
     # ARGUMENT PARSING
     parser = argparse.ArgumentParser()
@@ -299,19 +326,11 @@ def tomato_driver() -> None:
     logger.info("driver '%s' is quitting", args.driver)
 
 
-def stop_tomato_driver(port: int, context) -> Reply:
-    req = context.socket(zmq.REQ)
-    req.connect(f"tcp://127.0.0.1:{port}")
-    req.send_pyobj(dict(cmd="stop", sender=f"{__name__}.stop_tomato_driver"))
-    return req.recv_pyobj()
-
-
 def manager(port: int, timeout: int = 1000):
     """
     The driver manager thread of `tomato-daemon`.
 
-    This manager ensures individual driver processes are (re-)spawned and instructed to
-    quit as necessary.
+    This manager ensures individual driver processes are (re-)spawned and instructed to quit as necessary. The drivers are periodically checked using the ``HEARTBEAT`` constant as the interval. All changes are stored in the drivers table.
     """
     sender = f"{__name__}.manager"
     context = zmq.Context()

src/tomato/daemon/drvdb.py

@@ -1,6 +1,6 @@
 """
 **tomato.daemon.drvdb**: the sqlite database for drivers in tomato
---------------------------------------------------------------------
+------------------------------------------------------------------
 .. codeauthor::
     Peter Kraus
 

src/tomato/daemon/io.py

@@ -1,6 +1,6 @@
 """
-**tomato.daemon.io**: functions for storing and loading data
-------------------------------------------------------------
+**tomato.daemon.io**: functions for storing job data
+----------------------------------------------------
 .. codeauthor::
     Peter Kraus
 
@@ -20,9 +20,7 @@
 
 def merge_netcdfs(job: Job, snapshot=False) -> str:
     """
-    Merges the individual pickled :class:`xr.Datasets` of each Component found in :obj:`job.jobpath`
-    into a single :class:`xr.DataTree`, which is then stored in the NetCDF file,
-    using the Component `role` as the group label.
+    Merges all of the individual pickled :class:`~xarray.Dataset` files from each component found in :obj:`job.jobpath` into a single :class:`~xarray.DataTree`, which is then stored in the NetCDF file. The role of each component is used as the group label.
     """
     logger = logging.getLogger(f"{__name__}.merge_netcdf")
     assert job.jobpath is not None
@@ -52,8 +50,7 @@ def merge_netcdfs(job: Job, snapshot=False) -> str:
 
 def data_to_pickle(ds: xr.Dataset, path: Path, role: str):
     """
-    Dumps the data provided as :class:`xr.Dataset` into a ``pickle``. Concatenates with
-    any existing data stored in the ``pickle``.
+    Dumps the data provided as :class:`~xarray.Dataset` using :mod:`pickle`. Concatenates the new data with any existing data stored in the existing ``.pkl`` file.
     """
     logger = logging.getLogger(f"{__name__}.data_to_pickle")
     ds.attrs["role"] = role