Add developer documentation for managers in NEST

doc/htmldoc/conf.py

@@ -38,6 +38,10 @@
 sys.path.append(pynest_dir)
 
 # -- General configuration ------------------------------------------------
+read_the_docs_build = os.environ.get("READTHEDOCS", None) == "True"
+
+if read_the_docs_build:
+    subprocess.call("doxygen Doxyfile", shell=True)
 
 source_suffix = ".rst"
 master_doc = "index"
@@ -52,6 +56,7 @@
     "sphinxcontrib.mermaid",
     "sphinx.ext.mathjax",
     "sphinx_carousel.carousel",
+    "breathe",
     "sphinxcontrib.plantuml",
     "add_button_notebook",
     "IPython.sphinxext.ipython_console_highlighting",
@@ -67,6 +72,14 @@
 
 autodoc_mock_imports = ["nest.pynestkernel", "nest.ll_api"]
 mathjax_path = "https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js"
+
+# Setup the breathe extension
+breathe_projects = {"NEST Simulator": "./_doxygen/xml"}
+breathe_default_project = "NEST Simulator"
+breathe_domain_by_extension = {
+    "h": "cpp",
+}
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["templates"]
 
@@ -93,10 +106,10 @@
 copybutton_only_copy_prompt_lines = True
 
 mermaid_output_format = "raw"
-mermaid_version = "10.3.0"
-
+# mermaid_version = "10.3.0"
 # disable require js - mermaid doesn't work if require.js is loaded before it
 nbsphinx_requirejs_path = ""
+
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
 # built documents.
@@ -196,8 +209,9 @@
 html_show_sphinx = False
 html_show_copyright = False
 
-# This way works for ReadTheDocs
-# With this local 'make html' is broken!
+# https://www.sphinx-doc.org/en/master/extdev/appapi.html#sphinx-core-events
+
+
 github_doc_root = ""
 
 intersphinx_mapping = {
@@ -210,7 +224,7 @@
     "neat": ("https://nest-neat.readthedocs.io/en/latest/", None),
     "neuromorph": ("https://electronicvisions.github.io/hbp-sp9-guidebook/", None),
     "arbor": ("https://docs.arbor-sim.org/en/latest/", None),
-    "tvb": ("https://docs.thevirtualbrain.org/", None),
+    # "tvb": ("https://docs.thevirtualbrain.org/", None),
     "extmod": ("https://nest-extension-module.readthedocs.io/en/latest/", None),
 }
 

doc/htmldoc/developer_space/index.rst

@@ -110,6 +110,10 @@ Developer guides
 
        * :ref:`vim_sli`
 
+    .. grid-item-card:: Managers in NEST
+
+       * :ref:`Understand how managers work <managers_dev>`
+
 .. toctree::
    :maxdepth: 1
    :hidden:
@@ -123,3 +127,4 @@ Developer guides
    templates/*
    sli_docs/index
    cppcomments
+   managers/index

doc/htmldoc/developer_space/managers/connection_manager.rst

@@ -0,0 +1,13 @@
+.. _connection_manager:
+
+Connection Manager
+==================
+
+
+The ConnectionManager in NEST efficiently manages synaptic connections between neurons and devices using source-table
+structures, supporting dynamic network modifications via structural plasticity and connection rules. It tracks
+connection counts, handles parallel execution resizing, and ensures disabled connections are removed to maintain network
+integrity.
+
+.. doxygenclass:: nest::ConnectionManager
+   :members:

doc/htmldoc/developer_space/managers/event_delivery_manager.rst

@@ -0,0 +1,14 @@
+.. _event_delivery_manager:
+
+Event Delivery Manager
+======================
+
+The EventDeliveryManager coordinates the delivery of events (e.g., spikes, data) between nodes in NEST simulations,
+managing buffers and routing events across MPI processes for distributed computing.
+It handles timing through moduli-based time slicing, optimizes communication via send/receive buffers, and ensures
+accurate event delivery even for off-grid spikes and secondary events like data updates.
+This manager is essential for maintaining simulation accuracy and scalability, particularly in large-scale parallel
+simulations with complex event routing requirements.
+
+.. doxygenclass:: nest::EventDeliveryManager
+    :members:

doc/htmldoc/developer_space/managers/index.rst

@@ -0,0 +1,102 @@
+.. _managers_dev:
+
+Managers in NEST
+================
+
+At the heart of NEST are various
+managers, each responsible for specific aspects of the simulation process.
+
+
+The ManagerInterface provides a common interface for all managers, defining standard methods and properties.
+
+The KernelManager acts as the core, coordinating the other managers that specialize in different aspects of the
+simulation.
+
+:ref:`KernelManager <kernel_manager>` initializes all managers in dependency order:
+
+- :ref:`nest::LoggingManager <logging_manager>`
+- :ref:`nest::MPIManager <mpi_manager>`
+- :ref:`nest::VPManager <vp_manager>`
+- :ref:`nest::ModuleManager <module_manager>`
+- :ref:`nest::RandomManager <random_manager>`
+- :ref:`nest::SimulationManager <simulation_manager>`
+- :ref:`nest::ModelRangeManager <modelrange_manager>`
+- :ref:`nest::ConnectionManager <connection_manager>`
+- :ref:`nest::SPManager <sp_manager>`
+- :ref:`nest::EventDeliveryManager<event_delivery_manager>`
+- :ref:`nest::IOManager <io_manager>`
+- :ref:`nest::ModelManager <model_manager>`
+- :ref:`nest::MUSICManager <music_manager>`
+- :ref:`nest::NodeManager <node_manager>`
+
+
+
+Main control flow
+-----------------
+
+1. **Initialization**:
+   - The Kernel Manager initializes the simulation environment, allocating resources and setting up initial configurations.
+   - The Simulation Manager sets up the simulation based on defined parameters.
+
+2. **Node and Connection Setup**:
+   - The Node Manager creates and configures nodes.
+   - The Connection Manager establishes connections between nodes.
+
+3. **Model Application**:
+   - The Model Manager loads and applies models to nodes and connections.
+   - The SP Manager applies synaptic plasticity rules as needed.
+
+4. **Simulation Execution**:
+   - The Simulation Manager oversees the execution phase, with the Event Delivery Manager ensuring events are processed correctly.
+   - The MPI Manager handles parallel processing tasks if the simulation is distributed.
+
+5. **Data Handling**:
+   - The IO Manager reads configuration files and writes simulation results.
+   - The Logging Manager captures diagnostic information for debugging.
+
+6. **Cleanup**:
+   - The Simulation Manager ensures resources are cleaned up after the simulation completes.
+
+
+Advanced Architectural Features
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- **Policy-based template pattern for manager customization**:
+  - The policy-based template pattern is used to customize specific manager behaviors, allowing developers to extend functionality without modifying core logic.
+
+- **Thread-local storage for performance-critical components**:
+  - Thread-local storage is employed for performance-critical components to minimize contention and improve parallel execution efficiency.
+
+- **Double dispatch pattern for event handling**:
+  - The double dispatch pattern is used for event handling, enabling dynamic and type-safe event processing across different node and connection types.
+
+- **MPI-agnostic interfaces through manager abstraction**:
+  - MPI-agnostic interfaces are provided through manager abstraction, allowing the simulation to run seamlessly with or without MPI, enhancing flexibility and portability.
+
+- **Multiple synchronization strategies**:
+
+  - **MPI_Barrier** is used for global synchronization to ensure all processes reach a certain point before proceeding.
+  - **OpenMP locks** are employed for shared memory parallelism to manage access to shared resources.
+  - **Atomic operations** are used for spike counter updates to ensure thread-safe increments without locks.
+  - **RAII patterns** are used for resource management to ensure proper acquisition and release of resources.
+
+
+
+.. toctree::
+   :hidden:
+
+   connection_manager
+   event_delivery_manager
+   io_manager
+   mpi_manager
+   node_manager
+   music_manager
+   random_manager
+   sp_manager
+   vp_manager
+   logging_manager
+   kernel_manager
+   module_manager
+   modelrange_manager
+   simulation_manager
+   model_manager

doc/htmldoc/developer_space/managers/io_manager.rst

@@ -0,0 +1,14 @@
+.. _io_manager:
+
+IO Manager
+==========
+
+
+The IOManager in NEST coordinates input/output operations, managing data paths, file prefixes, and file-overwrite
+settings. It registers and routes data between recording/stimulation devices and their respective backends (e.g.,
+handling file I/O for spike data or parameter logging). It ensures devices communicate with the correct backend while
+deferring logging responsibilities to the LoggingManager.
+
+
+.. doxygenclass:: nest::IOManager
+   :members:

doc/htmldoc/developer_space/managers/kernel_manager.rst

@@ -0,0 +1,14 @@
+.. _kernel_manager:
+
+Kernel Manager
+==============
+
+The KernelManager in NEST manages the initialization and finalization of all simulation managers in a specific order to
+resolve dependencies and ensure proper setup. It acts as a singleton, providing centralized control over configuration,
+status queries, and simulation resets. Additionally, it tracks a fingerprint to detect changes and maintain consistency
+across multiple runs.
+
+
+
+.. doxygenclass:: nest::KernelManager
+   :members:

doc/htmldoc/developer_space/managers/logging_manager.rst

@@ -0,0 +1,13 @@
+.. _logging_manager:
+
+Logging Manager
+===============
+
+The LoggingManager in NEST centralizes logging by filtering messages based on severity levels and delivering them to
+registered clients, ensuring thread safety with OpenMP. It enforces proper parameter usage by checking dictionary
+entries and supports customizable output through callback functions, such as console logging or external tools.
+
+
+
+.. doxygenclass:: nest::LoggingManager
+   :members:

doc/htmldoc/developer_space/managers/model_manager.rst

@@ -0,0 +1,17 @@
+.. _model_manager:
+
+Model Manager
+=============
+
+
+The ModelManager is responsible for managing node and connection models,
+including their registration, copying, and configuration. It handles the initialization
+and finalization of models, sets default parameters, and manages memory for node models.
+The class also provides functionality for calibrating models and retrieving their status.
+The code is designed to work in a multi-threaded environment, ensuring that
+models are correctly managed across different threads. The ModelManager plays a crucial
+role in the simulation framework by providing a centralized way to handle various models and their configurations.
+
+
+.. doxygenclass:: nest::ModelManager
+   :members:

doc/htmldoc/developer_space/managers/modelrange_manager.rst

@@ -0,0 +1,13 @@
+.. _modelrange_manager:
+
+ModelRange Manager
+==================
+
+
+The ModelRangeManager in NEST groups node IDs into contiguous ranges per model, enabling efficient model lookups via
+binary search and minimizing storage by merging adjacent ranges. It tracks first/last node IDs and provides methods to
+retrieve models for specific nodes, ensuring fast access during simulations.
+
+
+.. doxygenclass:: nest::ModelRangeManager
+   :members:

doc/htmldoc/developer_space/managers/module_manager.rst

@@ -0,0 +1,13 @@
+.. _module_manager:
+
+Module Manager
+==============
+
+The ModuleManager is responsible for managing dynamic modules. Its primary functions include initializing and finalizing
+modules, loading and unloading them dynamically, and handling their status. The class ensures that modules can be
+safely installed and reinitialized, and it manages the search path for dynamic libraries, making it an essential
+component for extending the simulator's functionality with external modules.
+
+
+.. doxygenclass:: nest::ModuleManager
+   :members:

doc/htmldoc/developer_space/managers/mpi_manager.rst

@@ -0,0 +1,12 @@
+.. _mpi_manager:
+
+MPI manager
+===========
+
+The MPIManager in NEST manages MPI initialization and configuration checks to ensure proper setup for distributed
+simulations. It facilitates inter-process communication using MPI functions such as Allgather and Allreduce, and
+synchronizes processes with barriers. Additionally, it handles node-to-process mappings and enforces correct MPI usage
+to prevent runtime errors.
+
+.. doxygenclass:: nest::MPIManager
+   :members: