Docker orchestration logging

Makefile

@@ -69,14 +69,22 @@ install-python-and-uv: prerequisites
 	curl -LsSf https://astral.sh/uv/install.sh | sh
 	. "$$HOME/.local/bin/env" && uv venv --python $(PYTHON_VERSION)
 
-install-dev: install-python-and-uv install-latex install-docker verify-docker
+install-xterm: # needed for external terminal for logging
+	$(log) "Installing xterm..."
+	sudo apt-get install -y xterm
+	$(log) "xterm installed successfully."
+	$(log) "Installing xfonts-base..."
+	sudo apt-get install -y xfonts-base
+	$(log) "xfonts-base installed successfully."
+
+install-dev: install-python-and-uv install-latex install-docker verify-docker install-xterm
 	$(log) "Installing development packages and pre-commit hooks..."
 	PATH="$$HOME/.local/bin:$$PATH" . $(VENV_ACTIVATE) && uv pip install -r requirements-dev.txt
 	PATH="$$HOME/.local/bin:$$PATH" . $(VENV_ACTIVATE) && uv pip install -r requirements-release.txt
 	PATH="$$HOME/.local/bin:$$PATH" . $(VENV_ACTIVATE) && pre-commit install
 	$(log) "Development setup complete."
 
-install-release: install-python-and-uv install-docker verify-docker
+install-release: install-python-and-uv install-docker verify-docker install-xterm
 	$(log) "Installing release packages and pre-commit hooks..."
 	PATH="$$HOME/.local/bin:$$PATH" . $(VENV_ACTIVATE) && uv pip install -r requirements-release.txt
 	PATH="$$HOME/.local/bin:$$PATH" . $(VENV_ACTIVATE) && pre-commit install

pyproject.toml

@@ -8,6 +8,7 @@ requires-python = ">=3.12"
 [logging.output]
 log_to_console = true
 datetime_log_file = false
+external_docker_log_console = true
 
 [tool.coverage.run]
 branch = true
@@ -19,7 +20,7 @@ show_missing = true
 omit = [
     "*/tests/*",
     "*/__init__.py",
-    "src/logger/u_logger.py",
+    "src/logger/*",
     "*_pb2.py",
     "*_pb2_grpc.py",
     "_build_proto.py",
@@ -50,9 +51,9 @@ plugins = [
 
 [tool.pytest.ini_options]
 pythonpath = "src"
-log_file = "log/test.log"
+log_file = "log/pytest_log.log"
 log_file_format = "%(process)d %(thread)d %(asctime)s %(module)s:%(lineno)d %(levelname)s - %(message)s"
-log_level = "DEBUG"
+log_level = "INFO"
 
 retries = 2
 retry_delay = 0.5

src/interfaces/cli.py

@@ -1,7 +1,7 @@
 import argparse
 import json
 
-from logger.u_logger import get_logger
+from logger import get_logger
 from orchestration.risk_management_service import run_risk_management
 
 

src/logger/__init__.py

@@ -0,0 +1,10 @@
+from logger.numeric_logger import get_csv_logger
+from logger.orchestration_logger import log_docker_logs
+from logger.u_logger import configure_logger, get_logger
+
+__all__ = [
+    "get_csv_logger",
+    "configure_logger",
+    "get_logger",
+    "log_docker_logs",
+]

src/logger/orchestration_logger.py

@@ -0,0 +1,135 @@
+import subprocess
+from logging import Logger
+from pathlib import Path
+from typing import List
+
+from logger.utils import get_external_console_logging
+
+TAIL_LINES = 100
+WORKER_PREFIX = "worker"
+SIM_SERVICE_NAME = "simulation_server"
+
+
+def get_services() -> List[str]:
+    """Return a list of all docker compose service names."""
+    try:
+        container_ids = subprocess.check_output(
+            ["docker", "ps", "-q"], text=True
+        ).splitlines()
+
+        services = set()
+        for cid in container_ids:
+            inspect_output = subprocess.check_output(
+                [
+                    "docker",
+                    "inspect",
+                    "--format",
+                    '{{ index .Config.Labels "com.docker.compose.service" }}',
+                    cid,
+                ],
+                text=True,
+            ).strip()
+            if inspect_output and inspect_output != "<no value>":
+                services.add(inspect_output)
+        return list(services)
+    except Exception as e:
+        raise RuntimeError(
+            "Failed to get Docker services. Ensure Docker is running and you have access to it."
+        ) from e
+
+
+def _start_external_xterm_log_terminal(title: str, command: str) -> None:
+    """Start an xterm window with the specified log command."""
+    subprocess.Popen(["xterm", "-hold", "-T", title, "-e", "bash", "-c", command])
+
+
+def _start_external_log_terminal(title: str, command: str) -> None:
+    """Start a tmux session with the specified log command."""
+
+    session_name = f"log_{title.replace(' ', '_').lower()}"
+
+    try:
+        escaped_command = command.replace('"', '\\"')
+        wt_command = (
+            f'wt.exe new-tab --title "{title}" wsl.exe bash -c "{escaped_command}"'
+        )
+        subprocess.run(wt_command, shell=True)
+    except (OSError, subprocess.SubprocessError):
+        _start_external_xterm_log_terminal(
+            title, f"tmux attach-session -t {session_name}"
+        )
+
+
+def _start_file_logging(command: str) -> None:
+    """Start a background process that logs output to a file."""
+    buffered_command = f"stdbuf -oL -eL {command}"
+    subprocess.Popen(
+        ["bash", "-c", buffered_command],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
+
+
+def log_docker_logs(logger: Logger) -> None:
+    """Fetch and log docker logs for workers and simulation service."""
+    assert logger is not None, "Logger must be initialized before logging Docker logs."
+
+    project_root = Path(__file__).resolve().parents[2]
+    log_dir = project_root / "log"
+    log_dir.mkdir(exist_ok=True)
+
+    try:
+        services = get_services()
+    except RuntimeError as e:
+        logger.error(f"Error fetching Docker services: {e}")
+        return
+
+    logger.info(f"Found services: {services}")
+    if not services:
+        logger.warning("No services found. Ensure Docker Compose is running.")
+        return
+
+    worker_services = sorted([s for s in services if s.startswith(WORKER_PREFIX)])
+
+    worker_log_path = (log_dir / "docker_workers.log").resolve()
+    sim_log_path = (log_dir / "docker_simulation_server.log").resolve()
+
+    try:
+        for session in subprocess.check_output(
+            ["tmux", "list-sessions"], text=True, stderr=subprocess.DEVNULL
+        ).splitlines():
+            if session.startswith("log_"):
+                session_name = session.split(":")[0]
+                logger.info(f"Killing existing tmux session: {session_name}")
+                subprocess.Popen(["tmux", "kill-session", "-t", session_name])
+    except subprocess.CalledProcessError:
+        pass
+
+    if worker_services:
+        worker_cmd = f"docker compose logs --tail {TAIL_LINES} --follow {' '.join(worker_services)}"
+
+        if get_external_console_logging():
+            _start_external_log_terminal(
+                "Docker Worker Logs",
+                f"{worker_cmd} | stdbuf -oL -eL tee '{worker_log_path}'",
+            )
+            logger.info(
+                f"Opened external terminal for worker logs (also logging to {worker_log_path})."
+            )
+
+        _start_file_logging(f"{worker_cmd} > '{worker_log_path}'")
+        logger.info(f"Logging worker logs to file: {worker_log_path}")
+
+    sim_cmd = f"docker logs --tail {TAIL_LINES} --follow {SIM_SERVICE_NAME}"
+
+    if get_external_console_logging():
+        _start_external_log_terminal(
+            "Docker Simulation Service Logs",
+            f"{sim_cmd} | stdbuf -oL -eL tee '{sim_log_path}'",
+        )
+        logger.info(
+            f"Opened external terminal for simulation logs (also logging to {sim_log_path})."
+        )
+
+    _start_file_logging(f"{sim_cmd} > '{sim_log_path}'")
+    logger.info(f"Logging simulation logs to file: {sim_log_path}")

src/logger/u_logger.py

@@ -4,9 +4,9 @@
 import sys
 from datetime import datetime
 from logging import Logger
-from typing import Any, Dict, Optional
+from typing import Optional
 
-import tomli
+from logger.utils import get_log_to_console_value, log_to_datetime_log_file
 
 
 def configure_logger() -> None:
@@ -57,47 +57,3 @@ def get_logger(name: Optional[str] = "") -> Logger:
     log_to_console = get_log_to_console_value()
     _effective_name = "aux.console" if log_to_console else "aux" if name else "root"
     return logging.getLogger(_effective_name)
-
-
-def get_log_to_console_value() -> bool:
-    """
-    Get the value of log_to_console from pyproject.toml.
-
-    Returns
-    -------
-    bool
-        The value of log_to_console.
-
-    """
-    return bool(get_logging_output()["log_to_console"])
-
-
-def log_to_datetime_log_file() -> bool:
-    """
-    Get the value of datetime_log_file from pyproject.toml.
-
-    Returns
-    -------
-    bool
-        The value of datetime_log_file.
-
-    """
-    return bool(get_logging_output()["datetime_log_file"])
-
-
-def get_logging_output() -> Dict[str, Any]:
-    """
-    Get the logging output configuration from pyproject.toml.
-
-    Returns
-    -------
-    Dict[str, Any]
-        The logging output configuration.
-
-    """
-    pyproject_toml_path = os.path.join(
-        os.path.abspath(os.path.join(os.path.dirname(__file__), "../../pyproject.toml"))
-    )
-    with open(pyproject_toml_path, "rb") as f:
-        pyproject_toml = tomli.load(f)
-    return dict(pyproject_toml["logging"]["output"])

src/logger/utils.py

@@ -0,0 +1,60 @@
+import os
+from typing import Any, Dict
+
+import tomli
+
+
+def get_log_to_console_value() -> bool:
+    """
+    Get the value of log_to_console from pyproject.toml.
+
+    Returns
+    -------
+    bool
+        The value of log_to_console.
+
+    """
+    return bool(get_logging_output()["log_to_console"])
+
+
+def log_to_datetime_log_file() -> bool:
+    """
+    Get the value of datetime_log_file from pyproject.toml.
+
+    Returns
+    -------
+    bool
+        The value of datetime_log_file.
+
+    """
+    return bool(get_logging_output()["datetime_log_file"])
+
+
+def get_logging_output() -> Dict[str, Any]:
+    """
+    Get the logging output configuration from pyproject.toml.
+
+    Returns
+    -------
+    Dict[str, Any]
+        The logging output configuration.
+
+    """
+    pyproject_toml_path = os.path.join(
+        os.path.abspath(os.path.join(os.path.dirname(__file__), "../../pyproject.toml"))
+    )
+    with open(pyproject_toml_path, "rb") as f:
+        pyproject_toml = tomli.load(f)
+    return dict(pyproject_toml["logging"]["output"])
+
+
+def get_external_console_logging() -> bool:
+    """
+    Get the value of external_docker_log_console from pyproject.toml.
+
+    Returns
+    -------
+    bool
+        The value of external_docker_log_console.
+    """
+    return bool(get_logging_output()["external_docker_log_console"])

src/orchestration/risk_management_service/core/service/risk_management_service.py

@@ -1,7 +1,6 @@
 from typing import Any
 
-from logger.numeric_logger import get_csv_logger
-from logger.u_logger import get_logger
+from logger import get_csv_logger, get_logger
 from orchestration.risk_management_service.core.mappers import ControlVectorMapper
 from services.problem_dispatcher_service import (
     ProblemDispatcherService,

src/services/problem_dispatcher_service/core/service/problem_dispatcher_service.py

@@ -2,7 +2,7 @@
 import random
 from typing import Any, Callable
 
-from logger.u_logger import get_logger
+from logger import get_logger
 from services.problem_dispatcher_service.core.builder import TaskBuilder
 from services.problem_dispatcher_service.core.models import (
     ControlVector,

src/services/simulation_service/core/service/__init__.py

@@ -2,7 +2,7 @@
 import subprocess
 from pathlib import Path
 
-from logger.u_logger import configure_logger, get_logger
+from logger import configure_logger, get_logger
 from services.simulation_service.core.service.simulation_service import (
     SimulationService,
     simulation_cluster_context_manager,

src/services/simulation_service/core/service/simulation_service.py

@@ -6,7 +6,7 @@
 
 import grpc
 
-from logger.u_logger import get_logger
+from logger import get_logger, log_docker_logs
 from services.simulation_service.core.infrastructure.generated import (
     simulation_messaging_pb2 as sm,
 )
@@ -68,7 +68,6 @@ def start_simulation_cluster() -> None:
             SimulationService._SERVER_HOST,
             SimulationService._SERVER_PORT,
         )
-
         with core_directory():
             try:
                 result = subprocess.run(
@@ -86,6 +85,14 @@ def start_simulation_cluster() -> None:
                 )
                 logger.debug("Docker compose output:\n%s", result.stdout)
                 logger.info("Simulation cluster successfully started.")
+                log_docker_logs(logger)
+            except Exception as e:
+                logger.error(
+                    "Error starting the simulation cluster: %s",
+                    getattr(e, "stderr", str(e)),
+                )
+                raise
+
             except subprocess.CalledProcessError as e:
                 logger.error("Error starting the simulation cluster:\n%s", e.stderr)
                 raise

src/services/solution_updater_service/core/service/solution_updater_service.py

@@ -5,7 +5,7 @@
 import numpy as np
 import numpy.typing as npt
 
-from logger.u_logger import get_logger
+from logger import get_logger
 from services.solution_updater_service.core.engines import (
     OptimizationEngineFactory,
     OptimizationEngineInterface,