UX Overhaul: CLI, templates, TUI, local mode, and docs

.env.example

@@ -38,6 +38,10 @@
 # DATA_UVICORN_WORKERS=null
 # DATA_UVICORN_LIMIT_CONCURRENCY=null
 # DATA_UVICORN_LIMIT_MAX_REQUESTS=null
+# DATA_ENABLE_REGISTRY_RESOLUTION=false
+# DATA_MANAGER_NAME=null
+# DATA_MANAGER_DESCRIPTION=null
+# DATA_LAB_URL=null
 # DATA_OTEL_ENABLED=false
 # DATA_OTEL_SERVICE_NAME=null
 # DATA_OTEL_EXPORTER="console"
@@ -71,6 +75,10 @@
 # EVENT_UVICORN_WORKERS=null
 # EVENT_UVICORN_LIMIT_CONCURRENCY=null
 # EVENT_UVICORN_LIMIT_MAX_REQUESTS=null
+# EVENT_ENABLE_REGISTRY_RESOLUTION=false
+# EVENT_MANAGER_NAME=null
+# EVENT_MANAGER_DESCRIPTION=null
+# EVENT_LAB_URL=null
 # EVENT_OTEL_ENABLED=false
 # EVENT_OTEL_SERVICE_NAME=null
 # EVENT_OTEL_EXPORTER="console"
@@ -107,11 +115,16 @@
 # WORKCELL_UVICORN_WORKERS=null
 # WORKCELL_UVICORN_LIMIT_CONCURRENCY=null
 # WORKCELL_UVICORN_LIMIT_MAX_REQUESTS=null
+# WORKCELL_ENABLE_REGISTRY_RESOLUTION=false
+# WORKCELL_MANAGER_NAME=null
+# WORKCELL_MANAGER_DESCRIPTION=null
+# WORKCELL_LAB_URL=null
 # WORKCELL_OTEL_ENABLED=false
 # WORKCELL_OTEL_SERVICE_NAME=null
 # WORKCELL_OTEL_EXPORTER="console"
 # WORKCELL_OTEL_ENDPOINT=null
 # WORKCELL_OTEL_PROTOCOL="grpc"
+# WORKCELL_NODES=null
 # WORKCELLS_DIRECTORY="~/.madsci/workcells"
 # WORKCELL_REDIS_HOST="localhost"
 # WORKCELL_REDIS_PORT=6379
@@ -141,6 +154,10 @@
 # EXPERIMENT_UVICORN_WORKERS=null
 # EXPERIMENT_UVICORN_LIMIT_CONCURRENCY=null
 # EXPERIMENT_UVICORN_LIMIT_MAX_REQUESTS=null
+# EXPERIMENT_ENABLE_REGISTRY_RESOLUTION=false
+# EXPERIMENT_MANAGER_NAME=null
+# EXPERIMENT_MANAGER_DESCRIPTION=null
+# EXPERIMENT_LAB_URL=null
 # EXPERIMENT_OTEL_ENABLED=false
 # EXPERIMENT_OTEL_SERVICE_NAME=null
 # EXPERIMENT_OTEL_EXPORTER="console"
@@ -164,11 +181,16 @@
 # RESOURCE_UVICORN_WORKERS=null
 # RESOURCE_UVICORN_LIMIT_CONCURRENCY=null
 # RESOURCE_UVICORN_LIMIT_MAX_REQUESTS=null
+# RESOURCE_ENABLE_REGISTRY_RESOLUTION=false
+# RESOURCE_MANAGER_NAME=null
+# RESOURCE_MANAGER_DESCRIPTION=null
+# RESOURCE_LAB_URL=null
 # RESOURCE_OTEL_ENABLED=false
 # RESOURCE_OTEL_SERVICE_NAME=null
 # RESOURCE_OTEL_EXPORTER="console"
 # RESOURCE_OTEL_ENDPOINT=null
 # RESOURCE_OTEL_PROTOCOL="grpc"
+# RESOURCE_DEFAULT_TEMPLATES_FILE=null
 # RESOURCE_DB_URL="postgresql://madsci:madsci@localhost:5432/resources"
 
 ### LabManagerSettings
@@ -185,6 +207,10 @@
 # LAB_UVICORN_WORKERS=null
 # LAB_UVICORN_LIMIT_CONCURRENCY=null
 # LAB_UVICORN_LIMIT_MAX_REQUESTS=null
+# LAB_ENABLE_REGISTRY_RESOLUTION=false
+# LAB_MANAGER_NAME=null
+# LAB_MANAGER_DESCRIPTION=null
+# LAB_LAB_URL=null
 # LAB_OTEL_ENABLED=false
 # LAB_OTEL_SERVICE_NAME=null
 # LAB_OTEL_EXPORTER="console"
@@ -206,11 +232,17 @@
 # LOCATION_UVICORN_WORKERS=null
 # LOCATION_UVICORN_LIMIT_CONCURRENCY=null
 # LOCATION_UVICORN_LIMIT_MAX_REQUESTS=null
+# LOCATION_ENABLE_REGISTRY_RESOLUTION=false
+# LOCATION_MANAGER_NAME=null
+# LOCATION_MANAGER_DESCRIPTION=null
+# LOCATION_LAB_URL=null
 # LOCATION_OTEL_ENABLED=false
 # LOCATION_OTEL_SERVICE_NAME=null
 # LOCATION_OTEL_EXPORTER="console"
 # LOCATION_OTEL_ENDPOINT=null
 # LOCATION_OTEL_PROTOCOL="grpc"
+# LOCATION_LOCATIONS_FILE=null
+# LOCATION_TRANSFER_CAPABILITIES_FILE=null
 # LOCATION_REDIS_HOST="localhost"
 # LOCATION_REDIS_PORT=6379
 # LOCATION_REDIS_PASSWORD=null

.github/workflows/pre-commit.yml

@@ -13,12 +13,12 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       name: Checkout code
     - name: Install the latest version of uv
       uses: astral-sh/setup-uv@v7
     - name: Setup PDM
-      uses: pdm-project/setup-pdm@v3
+      uses: pdm-project/setup-pdm@v4
       with:
         python-version: "3.10"
     - name: Install dependencies

.github/workflows/validation.yml

@@ -0,0 +1,116 @@
+# Validation Infrastructure Workflow
+# This workflow runs the E2E test harness and template validation
+# as part of Phase 0 of the UX Overhaul Plan.
+
+name: Validation Infrastructure
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  # Run unit tests for the E2E framework itself
+  framework-tests:
+    name: E2E Framework Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v4
+        with:
+          python-version: "3.12"
+          cache: true
+
+      - name: Install dependencies
+        run: pdm lock && pdm install -G:all
+
+      - name: Run E2E framework tests
+        run: pdm run pytest src/madsci_common/tests/testing/ -v --tb=short
+
+  # Run tutorial tests (no Docker)
+  tutorial-tests:
+    name: Tutorial Tests (No Docker)
+    runs-on: ubuntu-latest
+    needs: framework-tests
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v4
+        with:
+          python-version: "3.12"
+          cache: true
+
+      - name: Install dependencies
+        run: pdm lock && pdm install -G:all
+
+      - name: Run tutorial tests (no Docker)
+        run: pdm run pytest tests/e2e/test_tutorials.py -v --tb=short --tutorial-filter=no-docker
+
+  # Run template validation tests
+  template-validation:
+    name: Template Validation
+    runs-on: ubuntu-latest
+    needs: framework-tests
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v4
+        with:
+          python-version: "3.12"
+          cache: true
+
+      - name: Install dependencies
+        run: pdm lock && pdm install -G:all
+
+      - name: Install ruff for template validation
+        run: pip install ruff
+
+      - name: Run template validation tests
+        run: pdm run pytest src/madsci_common/tests/test_templates/test_template_engine.py -v --tb=short
+
+  # Summary job that requires all others to pass
+  validation-complete:
+    name: Validation Complete
+    runs-on: ubuntu-latest
+    needs:
+      - framework-tests
+      - tutorial-tests
+      - template-validation
+    if: always()
+    steps:
+      - name: Check results
+        run: |
+          if [ "${{ needs.framework-tests.result }}" != "success" ]; then
+            echo "Framework tests failed"
+            exit 1
+          fi
+          if [ "${{ needs.tutorial-tests.result }}" != "success" ]; then
+            echo "Tutorial tests failed"
+            exit 1
+          fi
+          if [ "${{ needs.template-validation.result }}" != "success" ]; then
+            echo "Template validation failed"
+            exit 1
+          fi
+          echo "All validation checks passed!"

.gitignore

@@ -324,5 +324,8 @@ src/madsci_resource_manager/madsci/resource_manager/alembic/versions/*.py
 .claude/
 agent_docs/
 
+# Scratch directory for local testing
+.scratch/
+
 # MyST build outputs
 _build

.justfile

@@ -69,6 +69,10 @@ pdm-install-all:
 pdm-build:
   @pdm build
 
+# Run the full UX verification suite
+verify-ux: checks test
+  @echo "UX verification complete"
+
 # Run automated tests
 test *args:
   @pytest {{args}}

AGENTS.md

@@ -377,3 +377,4 @@ src/madsci_common/
 - Always use pydantic's `AnyUrl` to store URL's, and note that AnyUrl always ensures a trailing forward slash
 - Imports should generally be done at the top of the file, unless there are circular dependencies or other factors which require localized importing.
 - **IMPORTANT**: do not use noqa's or modify the configuration of linters or checks to bypass linter errors without the users _EXPLICIT_ permission.
+- **IMPORTANT**: Use `./.scratch/` for any temporary files, test outputs, or scratch work. Do NOT use `/tmp` or other system-level temporary directories.

CHANGELOG.md

@@ -0,0 +1,97 @@
+# Changelog
+
+All notable changes to the MADSci framework are documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [Unreleased]
+
+### Added
+
+#### CLI (17 commands)
+- `madsci init` - Interactive lab initialization wizard
+- `madsci new` - Component scaffolding from templates (module, interface, node, experiment, workflow, workcell, lab subcommands)
+- `madsci start` - Start lab services (Docker Compose or local mode)
+  - `madsci start manager <name>` - Start a single manager as a subprocess
+  - `madsci start node <path>` - Start a node module as a subprocess
+  - `--wait/--no-wait` flag for health polling after detached start
+- `madsci stop` - Stop lab services
+  - `madsci stop manager <name>` - Stop a background manager process
+  - `madsci stop node <name>` - Stop a background node process
+- `madsci status` - Service health checking with `--watch` and `--json` support
+- `madsci doctor` - Environment diagnostic checks (python, docker, ports)
+- `madsci logs` - Event log viewing with `--follow`, `--level`, `--grep`, `--since` filters
+- `madsci run` - Workflow and experiment execution
+- `madsci validate` - Configuration and definition file validation
+- `madsci config` - Configuration management (export, create, show)
+- `madsci backup` - Database backup creation (PostgreSQL and MongoDB)
+- `madsci registry` - Service registry management
+- `madsci migrate` - Database migration tooling
+- `madsci tui` - Interactive terminal user interface
+- `madsci completion` - Shell completion generation (bash, zsh, fish)
+- `madsci commands` - Command listing
+- `madsci version` - Version display
+- Command aliases: `n`, `s`, `l`, `doc`, `val`, `ui`, `cmd`, `cfg`
+- Lazy command loading for fast CLI startup
+
+#### Templates (26 templates)
+- 6 module templates: basic, device, instrument, liquid_handler, camera, robot_arm
+- 4 interface templates: fake, real, sim, mock
+- 1 node template: basic
+- 4 experiment templates: script, notebook, tui, node
+- 2 workflow templates: basic, multi_step
+- 1 workcell template: basic
+- 3 lab templates: minimal, standard, distributed
+- 5 communication templates: serial, socket, rest, sdk, modbus
+- Template engine with Jinja2, parameter validation, conditional files, and post-generation hooks
+- Template registry with bundled, user, and remote template sources
+
+#### TUI (Terminal User Interface)
+- Dashboard screen with service overview and quick actions
+- Status screen with detailed service health and auto-refresh
+- Logs screen with filterable log viewer
+- Nodes screen with node status and management
+- Workflows screen with workflow monitoring and control
+- CSS theming with light/dark mode support
+- Trogon integration for CLI-to-TUI command exploration
+
+#### Local Mode
+- `madsci start --mode=local` runs all managers in-process without Docker
+- In-memory drop-in backends for all database operations
+- Ephemeral data storage for development and testing
+
+#### Configuration Management
+- `madsci config export` for exporting configuration to YAML
+- `madsci config create` for interactive configuration creation
+- Secret classification with `json_schema_extra={"secret": True}`
+- `model_dump_safe()` method on MadsciBaseModel and MadsciBaseSettings for secret redaction
+- Explicit configuration management (no auto-writing of config files)
+
+#### Node Module Framework
+- `NodeInfo.from_config()` for creating node info from configuration
+- Migration tools for database schema management
+- Service registry for dynamic service discovery
+
+#### Testing
+- 2427+ automated tests
+- 150+ template validation tests
+- CLI tests using Click's CliRunner
+- E2E test harness with tutorial validation
+
+#### Observability
+- OpenTelemetry integration for distributed tracing
+- Structured logging with hierarchical context propagation
+- `event_client_context()` and `get_event_client()` for context management
+- Per-manager OTEL configuration via environment variables
+
+### Changed
+- `update_node_files` defaults to `False` (was `True`)
+- `load_definition()` replaces `load_or_create_definition()` as the primary API
+- Settings consolidation for structural config overrides
+- Opt-in registry resolution in manager base
+
+### Deprecated
+- Definition file auto-writing (use explicit `madsci config` commands instead)
+- `NodeDefinition` files (use `NodeInfo.from_config()` instead)
+- `load_or_create_definition()` (use `load_definition()` instead)