Add live lab runtime demo (#34)

Author: InderdeepBajwa

Makefile

@@ -69,6 +69,12 @@ demo-fixture: build-control-plane dev-db-reset ## Run the fixture-backed evaluat
 demo-fixture-once: build-control-plane dev-db-reset ## Run the fixture-backed evaluator demo and exit cleanly
 	@bash scripts/demo-fixture.sh --once
 
+demo-live-lab: build dev-db-reset ## Run the live lab KVM demo interactively
+	@bash scripts/demo-live-lab.sh
+
+demo-live-lab-once: build dev-db-reset ## Run the live lab KVM demo and exit cleanly
+	@bash scripts/demo-live-lab.sh --once
+
 example-nodes: ## List normalized node summaries
 	@curl -s http://127.0.0.1:9090/api/v1alpha1/nodes
 

README.md

@@ -101,6 +101,16 @@ make example-orphans         # Check for orphaned processes
 make dev-down                # Stop the control plane
 ```
 
+### 4. Live Lab Demo
+
+If you are on an x86_64 Linux host or nested-KVM x86_64 environment with KVM and QEMU, you can run the live lab demo which executes a KVM workload natively:
+
+```bash
+make demo-live-lab
+```
+
+For more information, see the [Live Lab Demo](docs/live-lab.md) documentation.
+
 ## Repository Layout
 
 - `schedune-control-plane/`: The Go-based control plane, API, and orchestration logic.

docs/index.md

@@ -37,4 +37,8 @@ Schedune **is not yet** capable of full workload compatibility discovery/import,
 - **Lifecycle management:** Rigorous state machine with append-only traces, robust restart recovery (rehydration), and real `/proc`-backed orphan sweeping.
 - **Supported Runtimes:** KVM/QEMU (Execution), Cloud Hypervisor (Execution), Firecracker (Validation/Dry-run only).
 
+## Live Lab Demo
+
+To try out a live execution of a KVM/QEMU workload locally, see the [Live Lab Demo](live-lab.md) documentation.
+
 *Note: Expect rapid changes as the product shape stabilizes.*

docs/live-lab.md

@@ -0,0 +1,35 @@
+# Live Lab Demo
+
+The `demo-live-lab` script provides an interactive demonstration of Schedune's `VirtualMachine` runtime lifecycle utilizing KVM and QEMU.
+
+It runs locally on an x86_64 Linux host or nested-KVM x86_64 environment that possesses KVM extensions and the `qemu-system` binaries.
+
+## What it does
+
+The demo proves host node ingestion, runtime lifecycle, and backend readiness by completing the following sequence:
+
+1. **Preflight Checks:** Ensures you have KVM extensions and QEMU.
+2. **Environment Reset:** Drops the local state.
+3. **Agent Inspect & Control Plane Start:** Scrapes host capabilities locally, generating a `node_id`, and pushes them via Schedune Intake.
+4. **Artifact Creation:** Generates a 10MB empty `qcow2` image.
+    * *Note: The demo intentionally uses an empty `qcow2` because its focus is proving the underlying KVM execution and capability engine, not booting a specific Guest OS or Application.*
+5. **Explain & Predict:** Analyzes why an architecture-specific VM requiring KVM maps strictly to the local Node's Holding Pool.
+6. **Validate & Dry-Run:** Confirms the Launch Specification and simulates execution.
+7. **Execute:** Directly spawns a `qemu-system-*` process on the node via Schedune.
+8. **Lifecycle & Readiness:** Polls until the control socket reports the virtual machine is actively listening.
+9. **Trace & Events:** Shows the recorded telemetry trace and state machine events for the Launch.
+10. **Termination:** Issues an API call to tear down the underlying process.
+11. **Orphans:** Confirms no lingering processes exist in the Recovery engine.
+
+## Usage
+
+```bash
+# Build Schedune binaries
+make build
+
+# Run the Live Lab interactively
+make demo-live-lab
+
+# Run the Live Lab and exit automatically
+make demo-live-lab-once
+```

examples/launch-specs/qemu-execute.json

@@ -0,0 +1,18 @@
+{
+  "schema_version": "v1alpha1",
+  "launch_mode": "Execute",
+  "workload_id": "example-qemu-vm",
+  "tenant_id": "tenant-example",
+  "node_id": "node-123",
+  "runtime_class": "VirtualMachine",
+  "runtime_backend_preference": "kvm_qemu",
+  "architecture": "x86_64",
+  "memory_mb": 128,
+  "vcpu": 1,
+  "storage": [
+    {
+      "host_path": "/var/lib/schedune/images/example.qcow2",
+      "format": "qcow2"
+    }
+  ]
+}

mkdocs.yml

@@ -16,4 +16,5 @@ nav:
   - State Machine: state-machine.md
   - Recovery: recovery.md
   - Troubleshooting: troubleshooting.md
+  - Live Lab Demo: live-lab.md
   - GitHub: https://github.com/TechnologyTailors/Schedune

schedune-control-plane/internal/api/launch_handlers.go

@@ -6,7 +6,9 @@ import (
 	"net/http"
 
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/domain"
+	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/domain/lifecycle"
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/runtime"
+	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/runtime/inspect"
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/store"
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/store/sqlite"
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/pkg/schema"
@@ -27,11 +29,29 @@ func (r *StaticExecutorResolver) Resolve(backend string) (runtime.Executor, erro
 	return exec, nil
 }
 
+type InspectorResolver interface {
+	Resolve(backend string) inspect.Inspector
+}
+
+type StaticInspectorResolver struct{}
+
+func (r *StaticInspectorResolver) Resolve(backend string) inspect.Inspector {
+	switch backend {
+	case "kvm_qemu":
+		return &inspect.QemuInspector{}
+	case "cloud_hypervisor":
+		return &inspect.CloudHypervisorInspector{}
+	default:
+		return &inspect.ProcessInspector{}
+	}
+}
+
 type LaunchHandler struct {
-	nodeStore *store.InMemoryStore
-	execStore *sqlite.SQLiteStore
-	resolver  *StaticExecutorResolver
-	orch      *domain.LaunchOrchestrator
+	nodeStore         *store.InMemoryStore
+	execStore         *sqlite.SQLiteStore
+	resolver          *StaticExecutorResolver
+	inspectorResolver InspectorResolver
+	orch              *domain.LaunchOrchestrator
 }
 
 func NewLaunchHandler(nodeStore *store.InMemoryStore, execStore *sqlite.SQLiteStore) *LaunchHandler {
@@ -43,7 +63,7 @@ func NewLaunchHandler(nodeStore *store.InMemoryStore, execStore *sqlite.SQLiteSt
 		},
 	}
 	orch := domain.NewLaunchOrchestrator(nodeStore, execStore, resolver)
-	return &LaunchHandler{nodeStore: nodeStore, execStore: execStore, resolver: resolver, orch: orch}
+	return &LaunchHandler{nodeStore: nodeStore, execStore: execStore, resolver: resolver, inspectorResolver: &StaticInspectorResolver{}, orch: orch}
 }
 
 // ValidateLaunch assesses whether a chosen node is physically capable of executing the requested spec.
@@ -135,6 +155,26 @@ func (h *LaunchHandler) ExecuteLaunch(c *gin.Context) {
 	c.JSON(http.StatusAccepted, record)
 }
 
+func (h *LaunchHandler) reconcileExecution(rec *launch.LaunchExecutionRecord) error {
+	if rec.State == launch.StateExited || rec.State == launch.StateFailed || rec.State == launch.StateTerminated {
+		return nil
+	}
+
+	backend := ""
+	if rec.PreparedState != nil {
+		backend = rec.PreparedState.RuntimeBackend
+	}
+	inspector := h.inspectorResolver.Resolve(backend)
+
+	err := lifecycle.Reconcile(rec, inspector)
+	if err != nil {
+		log.Error().Err(err).Str("execution_id", rec.ExecutionID).Msg("Failed to reconcile execution")
+		return err
+	}
+
+	return h.execStore.SaveExecution(context.Background(), *rec)
+}
+
 // InspectLaunch returns the live status of the execution trace
 func (h *LaunchHandler) InspectLaunch(c *gin.Context) {
 	id := c.Param("id")
@@ -143,6 +183,11 @@ func (h *LaunchHandler) InspectLaunch(c *gin.Context) {
 		c.JSON(http.StatusNotFound, gin.H{"error": "Execution not found"})
 		return
 	}
+
+	if err := h.reconcileExecution(&rec); err != nil {
+		// Logged in reconcileExecution, continue to return current state
+	}
+
 	c.JSON(http.StatusOK, rec)
 }
 
@@ -166,6 +211,10 @@ func (h *LaunchHandler) InspectReadiness(c *gin.Context) {
 		return
 	}
 
+	if err := h.reconcileExecution(&rec); err != nil {
+		// Logged in reconcileExecution, continue to return current state
+	}
+
 	backend := ""
 	if rec.PreparedState != nil {
 		backend = rec.PreparedState.RuntimeBackend

schedune-control-plane/internal/api/launch_handlers_test.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"net/http"
@@ -11,6 +12,7 @@ import (
 
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/domain"
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/runtime"
+	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/runtime/inspect"
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/store"
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/store/sqlite"
 	"github.com/TechnologyTailors/Schedune/schedune-control-plane/pkg/schema"
@@ -248,3 +250,101 @@ func TestDryRunLaunch_PreparationFails(t *testing.T) {
 		t.Errorf("expected preparation reason code, got %v", res.PreparationReasonCode)
 	}
 }
+
+type MockInspector struct {
+	Obs inspect.RuntimeObservation
+	Err error
+}
+
+func (m *MockInspector) Inspect(executionID string, pid *int, prepared launch.PreparedLaunch) (inspect.RuntimeObservation, error) {
+	return m.Obs, m.Err
+}
+
+type MockInspectorResolver struct {
+	Inspector inspect.Inspector
+}
+
+func (m *MockInspectorResolver) Resolve(backend string) inspect.Inspector {
+	return m.Inspector
+}
+
+func setupInspectTestRouter(t *testing.T) (*gin.Engine, *store.InMemoryStore, *sqlite.SQLiteStore, *MockInspector) {
+	gin.SetMode(gin.TestMode)
+	router := gin.New()
+
+	nodeStore := store.NewInMemoryStore()
+	execStore, _ := sqlite.NewSQLiteStore(":memory:")
+
+	resolver := &StaticExecutorResolver{}
+	orch := domain.NewLaunchOrchestrator(nodeStore, execStore, resolver)
+
+	mockInspector := &MockInspector{}
+	mockResolver := &MockInspectorResolver{Inspector: mockInspector}
+
+	handler := &LaunchHandler{
+		nodeStore:         nodeStore,
+		execStore:         execStore,
+		resolver:          resolver,
+		inspectorResolver: mockResolver,
+		orch:              orch,
+	}
+
+	router.GET("/launch/:id", handler.InspectLaunch)
+	router.GET("/launch/:id/readiness", handler.InspectReadiness)
+
+	return router, nodeStore, execStore, mockInspector
+}
+
+func TestInspectReadiness_ReconcilesToReady(t *testing.T) {
+	router, _, execStore, mockInspector := setupInspectTestRouter(t)
+
+	// Insert a Starting execution
+	pid := 1234
+	now := time.Now().Unix()
+	rec := launch.LaunchExecutionRecord{
+		ExecutionID:  "test-exec-1",
+		NodeID:       "test-node",
+		State:        launch.StateStarting,
+		PID:          &pid,
+		StartedAtSec: &now,
+	}
+	execStore.SaveExecution(context.Background(), rec)
+
+	// Mock observation: Process is alive and ready
+	mockInspector.Obs = inspect.RuntimeObservation{
+		ProcessExists:       true,
+		BackendReadySignal:  true,
+		BackendSignalSource: "mock-backend",
+		ObservedAtSec:       now,
+	}
+
+	// Request readiness
+	req, _ := http.NewRequest("GET", "/launch/test-exec-1/readiness", nil)
+	w := httptest.NewRecorder()
+	router.ServeHTTP(w, req)
+
+	if w.Code != http.StatusOK {
+		t.Fatalf("expected status 200, got %d", w.Code)
+	}
+
+	var res launch.LaunchExecutionRecord
+	if err := json.Unmarshal(w.Body.Bytes(), &res); err != nil {
+		t.Fatalf("failed to unmarshal response: %v", err)
+	}
+
+	if res.State != launch.StateRunning {
+		t.Errorf("expected state to be Running, got %s", res.State)
+	}
+	if res.RuntimeReadiness != "Ready" {
+		t.Errorf("expected readiness to be Ready, got %s", res.RuntimeReadiness)
+	}
+
+	// Verify it was persisted
+	persisted, _, err := execStore.GetExecution(context.Background(), "test-exec-1")
+	if err != nil {
+		t.Fatalf("failed to get execution: %v", err)
+	}
+	if persisted.State != launch.StateRunning {
+		t.Errorf("expected persisted state to be Running, got %s", persisted.State)
+	}
+}

schedune-control-plane/internal/api/orphan_handlers.go

@@ -31,6 +31,10 @@ func (h *OrphanHandler) ListOrphans(c *gin.Context) {
 		return
 	}
 
+	if orphans == nil {
+		orphans = []domain.OrphanRecord{}
+	}
+
 	c.JSON(http.StatusOK, gin.H{"orphans": orphans})
 }
 

schedune-control-plane/internal/api/orphan_handlers_test.go

@@ -0,0 +1,54 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/TechnologyTailors/Schedune/schedune-control-plane/internal/store/sqlite"
+	"github.com/gin-gonic/gin"
+)
+
+func TestOrphanHandler_ListOrphans_Empty(t *testing.T) {
+	gin.SetMode(gin.TestMode)
+
+	// Use an in-memory sqlite db for tests
+	dbStore, err := sqlite.NewSQLiteStore(":memory:")
+	if err != nil {
+		t.Fatalf("failed to create sqlite store: %v", err)
+	}
+
+	handler := NewOrphanHandler(dbStore)
+
+	w := httptest.NewRecorder()
+	c, _ := gin.CreateTestContext(w)
+	// Add mock request to context so query params don't panic
+	c.Request, _ = http.NewRequest(http.MethodGet, "/api/v1alpha1/recovery/orphans", nil)
+
+	handler.ListOrphans(c)
+
+	if w.Code != http.StatusOK {
+		t.Errorf("Expected status 200, got %d", w.Code)
+	}
+
+	var response map[string]interface{}
+	if err := json.Unmarshal(w.Body.Bytes(), &response); err != nil {
+		t.Fatalf("Failed to parse response JSON: %v", err)
+	}
+
+	// Since there are no orphans in the fresh DB, it should return an empty slice, not nil
+	orphansInter, ok := response["orphans"]
+	if !ok {
+		t.Fatalf("Expected 'orphans' key in response")
+	}
+
+	orphansList, ok := orphansInter.([]interface{})
+	if !ok {
+		t.Fatalf("Expected 'orphans' to be a list, got %T", orphansInter)
+	}
+
+	if len(orphansList) != 0 {
+		t.Errorf("Expected empty list, got length %d", len(orphansList))
+	}
+}