diff --git a/schedune-control-plane/internal/domain/orchestrator.go b/schedune-control-plane/internal/domain/orchestrator.go index 1a5a4d0..8b0da5d 100644 --- a/schedune-control-plane/internal/domain/orchestrator.go +++ b/schedune-control-plane/internal/domain/orchestrator.go @@ -102,7 +102,7 @@ func (o *LaunchOrchestrator) prepareAndRecord(spec launch.LaunchSpec, rec *launc // We can run ValidateLaunch again or assume the backend is already in ValidationTrace, // but it's cleaner to query the backend from the node + spec using the selector. node, _ := o.nodeStore.GetNode(spec.NodeID) - selectedBackend, _ := SelectBackend(spec, node) + selectedBackend, _, _ := SelectBackend(spec, node) exec, err := o.resolver.Resolve(selectedBackend) if err != nil { diff --git a/schedune-control-plane/internal/domain/runtime_select.go b/schedune-control-plane/internal/domain/runtime_select.go index 9f4c034..a67eb42 100644 --- a/schedune-control-plane/internal/domain/runtime_select.go +++ b/schedune-control-plane/internal/domain/runtime_select.go @@ -38,66 +38,95 @@ func normalizeNetworks(spec launch.LaunchSpec) []launch.NetworkAttachmentSpec { } // SelectBackend determines the appropriate runtime backend for a given launch spec and node. -// It returns the selected backend name, and a map of rejected backends to their rejection reason codes. -func SelectBackend(spec launch.LaunchSpec, node NodeRecord) (string, map[string]string) { +// It returns the selected backend name, the structured rejection evidence, and a legacy map of rejected backends to their rejection reason codes. +func SelectBackend(spec launch.LaunchSpec, node NodeRecord) (string, []launch.BackendRejectionEvidence, map[string]string) { rejected := make(map[string]string) + var evidence []launch.BackendRejectionEvidence + + reject := func(backend, reasonCode string, capName *string, cap *NodeCapabilityRecord) { + rejected[backend] = reasonCode + ev := launch.BackendRejectionEvidence{ + Backend: backend, + ReasonCode: reasonCode, + } + if capName != nil { + ev.CapabilityName = capName + } + if cap != nil { + state := cap.State + ev.CapabilityState = &state + if cap.ReasonCode != "" { + rc := cap.ReasonCode + ev.CapabilityReasonCode = &rc + rejected[backend] += " (" + cap.ReasonCode + ")" + } + stale := cap.IsStale + ev.CapabilityStale = &stale + } + evidence = append(evidence, ev) + } + storage := normalizeStorage(spec) if spec.RuntimeClass == "MicroVM" { // Firecracker path - kvmCap, kvmExists := node.Capabilities["kvm_vm_launch"] + capName := "kvm_vm_launch" + kvmCap, kvmExists := node.Capabilities[capName] if !kvmExists || kvmCap.State != "Supported" || kvmCap.IsStale { - reason := "ERR_LAUNCH_MISSING_CAPABILITY_KVM_QEMU" - if kvmExists && kvmCap.ReasonCode != "" { - reason += " (" + kvmCap.ReasonCode + ")" + var capPtr *NodeCapabilityRecord + if kvmExists { + capPtr = &kvmCap } - rejected["firecracker"] = reason - return "", rejected + reject("firecracker", "ERR_LAUNCH_MISSING_CAPABILITY_KVM_QEMU", &capName, capPtr) + return "", evidence, rejected } - binCap, binExists := node.Capabilities["firecracker_binary_present"] + capName = "firecracker_binary_present" + binCap, binExists := node.Capabilities[capName] if !binExists || binCap.State != "Supported" || binCap.IsStale { - reason := "ERR_LAUNCH_MISSING_CAPABILITY_FC_BINARY" - if binExists && binCap.ReasonCode != "" { - reason += " (" + binCap.ReasonCode + ")" + var capPtr *NodeCapabilityRecord + if binExists { + capPtr = &binCap } - rejected["firecracker"] = reason - return "", rejected + reject("firecracker", "ERR_LAUNCH_MISSING_CAPABILITY_FC_BINARY", &capName, capPtr) + return "", evidence, rejected } - tunCap, tunExists := node.Capabilities["firecracker_tun_ready"] + capName = "firecracker_tun_ready" + tunCap, tunExists := node.Capabilities[capName] if !tunExists || tunCap.State != "Supported" || tunCap.IsStale { - reason := "ERR_LAUNCH_MISSING_CAPABILITY_FC_TUN" - if tunExists && tunCap.ReasonCode != "" { - reason += " (" + tunCap.ReasonCode + ")" + var capPtr *NodeCapabilityRecord + if tunExists { + capPtr = &tunCap } - rejected["firecracker"] = reason - return "", rejected + reject("firecracker", "ERR_LAUNCH_MISSING_CAPABILITY_FC_TUN", &capName, capPtr) + return "", evidence, rejected } - cgCap, cgExists := node.Capabilities["firecracker_cgroups_ready"] + capName = "firecracker_cgroups_ready" + cgCap, cgExists := node.Capabilities[capName] if !cgExists || cgCap.State != "Supported" || cgCap.IsStale { - reason := "ERR_LAUNCH_MISSING_CAPABILITY_FC_CGROUPS" - if cgExists && cgCap.ReasonCode != "" { - reason += " (" + cgCap.ReasonCode + ")" + var capPtr *NodeCapabilityRecord + if cgExists { + capPtr = &cgCap } - rejected["firecracker"] = reason - return "", rejected + reject("firecracker", "ERR_LAUNCH_MISSING_CAPABILITY_FC_CGROUPS", &capName, capPtr) + return "", evidence, rejected } // Artifact validation if len(storage) == 0 { - rejected["firecracker"] = "ERR_LAUNCH_INVALID_FIRECRACKER_ARTIFACT_MODEL" - return "", rejected + reject("firecracker", "ERR_LAUNCH_INVALID_FIRECRACKER_ARTIFACT_MODEL", nil, nil) + return "", evidence, rejected } for _, s := range storage { if s.Format == "qcow2" { - rejected["firecracker"] = "ERR_LAUNCH_INVALID_STORAGE_FORMAT" - return "", rejected + reject("firecracker", "ERR_LAUNCH_INVALID_STORAGE_FORMAT", nil, nil) + return "", evidence, rejected } } - return "firecracker", rejected + return "firecracker", evidence, rejected } if spec.RuntimeClass == "VirtualMachine" { @@ -115,67 +144,73 @@ func SelectBackend(spec launch.LaunchSpec, node NodeRecord) (string, map[string] } } } else { - rejected[spec.RuntimeBackendPreference] = "ERR_LAUNCH_BACKEND_NOT_SUPPORTED" - return "", rejected + reject(spec.RuntimeBackendPreference, "ERR_LAUNCH_BACKEND_NOT_SUPPORTED", nil, nil) + return "", evidence, rejected } } for _, backend := range backends { if backend == "cloud_hypervisor" { - kvmCap, kvmExists := node.Capabilities["kvm_vm_launch"] + capName := "kvm_vm_launch" + kvmCap, kvmExists := node.Capabilities[capName] if !kvmExists || kvmCap.State != "Supported" || kvmCap.IsStale { - reason := "ERR_LAUNCH_MISSING_CAPABILITY_KVM_QEMU" - if kvmExists && kvmCap.ReasonCode != "" { - reason += " (" + kvmCap.ReasonCode + ")" + var capPtr *NodeCapabilityRecord + if kvmExists { + capPtr = &kvmCap } - rejected["cloud_hypervisor"] = reason + reject("cloud_hypervisor", "ERR_LAUNCH_MISSING_CAPABILITY_KVM_QEMU", &capName, capPtr) continue } - binCap, binExists := node.Capabilities["cloud_hypervisor_binary_present"] + capName = "cloud_hypervisor_binary_present" + binCap, binExists := node.Capabilities[capName] if !binExists || binCap.State != "Supported" || binCap.IsStale { - reason := "ERR_LAUNCH_MISSING_CAPABILITY_CH_BINARY" - if binExists && binCap.ReasonCode != "" { - reason += " (" + binCap.ReasonCode + ")" + var capPtr *NodeCapabilityRecord + if binExists { + capPtr = &binCap } - rejected["cloud_hypervisor"] = reason + reject("cloud_hypervisor", "ERR_LAUNCH_MISSING_CAPABILITY_CH_BINARY", &capName, capPtr) continue } if len(storage) == 0 { - rejected["cloud_hypervisor"] = "ERR_LAUNCH_MISSING_ARTIFACT" + reject("cloud_hypervisor", "ERR_LAUNCH_MISSING_ARTIFACT", nil, nil) continue } - return "cloud_hypervisor", rejected + return "cloud_hypervisor", evidence, rejected } if backend == "kvm_qemu" { - cap, exists := node.Capabilities["kvm_vm_launch"] + capName := "kvm_vm_launch" + cap, exists := node.Capabilities[capName] if !exists || cap.State != "Supported" || cap.IsStale { - reason := "ERR_LAUNCH_MISSING_CAPABILITY_KVM_QEMU" - if exists && cap.ReasonCode != "" { - reason += " (" + cap.ReasonCode + ")" + var capPtr *NodeCapabilityRecord + if exists { + capPtr = &cap } - rejected["kvm_qemu"] = reason + reject("kvm_qemu", "ERR_LAUNCH_MISSING_CAPABILITY_KVM_QEMU", &capName, capPtr) continue } - binCap, binExists := node.Capabilities["qemu_binary_present"] + + capName = "qemu_binary_present" + binCap, binExists := node.Capabilities[capName] if !binExists || binCap.State != "Supported" || binCap.IsStale { - reason := "ERR_LAUNCH_MISSING_CAPABILITY_QEMU_BINARY" - if binExists && binCap.ReasonCode != "" { - reason += " (" + binCap.ReasonCode + ")" + var capPtr *NodeCapabilityRecord + if binExists { + capPtr = &binCap } - rejected["kvm_qemu"] = reason + reject("kvm_qemu", "ERR_LAUNCH_MISSING_CAPABILITY_QEMU_BINARY", &capName, capPtr) continue } + if len(storage) == 0 { - rejected["kvm_qemu"] = "ERR_LAUNCH_MISSING_ARTIFACT" + reject("kvm_qemu", "ERR_LAUNCH_MISSING_ARTIFACT", nil, nil) continue } - return "kvm_qemu", rejected + return "kvm_qemu", evidence, rejected } } } - return "", rejected + return "", evidence, rejected } diff --git a/schedune-control-plane/internal/domain/runtime_select_test.go b/schedune-control-plane/internal/domain/runtime_select_test.go index f7fc2b8..c593c26 100644 --- a/schedune-control-plane/internal/domain/runtime_select_test.go +++ b/schedune-control-plane/internal/domain/runtime_select_test.go @@ -23,14 +23,14 @@ func TestSelectBackend_MicroVM(t *testing.T) { RootfsPath: "/tmp/rootfs.ext4", } - backend, rejected := SelectBackend(spec, node) + backend, _, rejected := SelectBackend(spec, node) if backend != "firecracker" { t.Errorf("expected firecracker, got %s. rejections: %v", backend, rejected) } // Missing artifacts spec.KernelImagePath = "" - backend, rejected = SelectBackend(spec, node) + backend, _, rejected = SelectBackend(spec, node) if backend != "" { t.Errorf("expected rejection due to missing artifact, got %s", backend) } @@ -56,7 +56,7 @@ func TestSelectBackend_VirtualMachine(t *testing.T) { } // Should prefer cloud_hypervisor - backend, rejected := SelectBackend(spec, node) + backend, _, rejected := SelectBackend(spec, node) if backend != "cloud_hypervisor" { t.Errorf("expected cloud_hypervisor, got %s. rejections: %v", backend, rejected) } @@ -77,7 +77,7 @@ func TestSelectBackend_Fallback(t *testing.T) { ImageReference: "/tmp/image.qcow2", } - backend, rejected := SelectBackend(spec, node) + backend, _, rejected := SelectBackend(spec, node) if backend != "kvm_qemu" { t.Errorf("expected kvm_qemu fallback, got %s. rejections: %v", backend, rejected) } diff --git a/schedune-control-plane/internal/domain/validate_launch.go b/schedune-control-plane/internal/domain/validate_launch.go index 7549335..d5f08a5 100644 --- a/schedune-control-plane/internal/domain/validate_launch.go +++ b/schedune-control-plane/internal/domain/validate_launch.go @@ -62,8 +62,9 @@ func ValidateLaunch(spec launch.LaunchSpec, node NodeRecord) launch.LaunchValida } // 2. Layer 2 & 3: Backend-specific capability and artifact checks via Runtime Selector - selectedBackend, rejectedBackends := SelectBackend(spec, node) + selectedBackend, evidence, rejectedBackends := SelectBackend(spec, node) result.RejectedBackends = rejectedBackends + result.BackendRejectionEvidence = evidence result.SelectedBackend = selectedBackend if selectedBackend == "" { diff --git a/schedune-control-plane/internal/domain/validate_launch_test.go b/schedune-control-plane/internal/domain/validate_launch_test.go index 54e6f7d..1369d6b 100644 --- a/schedune-control-plane/internal/domain/validate_launch_test.go +++ b/schedune-control-plane/internal/domain/validate_launch_test.go @@ -357,6 +357,20 @@ func TestValidateLaunch_MissingQemuBinary(t *testing.T) { t.Errorf("expected rejected backend kvm_qemu with CAP_QEMU_BINARY_MISSING error, got %v", result.RejectedBackends) } + foundEvidence := false + for _, ev := range result.BackendRejectionEvidence { + if ev.Backend == "kvm_qemu" && ev.ReasonCode == "ERR_LAUNCH_MISSING_CAPABILITY_QEMU_BINARY" { + if ev.CapabilityName != nil && *ev.CapabilityName == "qemu_binary_present" { + if ev.CapabilityReasonCode != nil && *ev.CapabilityReasonCode == "CAP_QEMU_BINARY_MISSING" { + foundEvidence = true + } + } + } + } + if !foundEvidence { + t.Errorf("expected structured evidence for missing qemu binary, got %+v", result.BackendRejectionEvidence) + } + if hint, ok := result.RemediationHints["kvm_qemu_binary"]; !ok || !strings.Contains(hint, "Install qemu-system") { t.Errorf("expected remediation hint for missing qemu binary, got %v", result.RemediationHints) } diff --git a/schedune-control-plane/pkg/schema/launch/v1alpha1.go b/schedune-control-plane/pkg/schema/launch/v1alpha1.go index c1fbdde..c952217 100644 --- a/schedune-control-plane/pkg/schema/launch/v1alpha1.go +++ b/schedune-control-plane/pkg/schema/launch/v1alpha1.go @@ -51,17 +51,28 @@ type LaunchSpec struct { // LaunchValidationResult explains exactly what host-level blockers exist. type LaunchValidationResult struct { - IsValid bool `json:"is_valid"` - SelectedBackend string `json:"selected_backend"` - RejectedBackends map[string]string `json:"rejected_backends"` // Backend -> Reason code - BlockingReasonCodes []string `json:"blocking_reason_codes"` - Warnings []string `json:"warnings"` - RequiredHostFeatures []string `json:"required_host_features"` - MissingHostFeatures []string `json:"missing_host_features"` - RecommendedRuntime string `json:"recommended_runtime"` - ExplainabilityText string `json:"explainability_text"` - ValidationTrace []string `json:"validation_trace"` - RemediationHints map[string]string `json:"remediation_hints,omitempty"` + IsValid bool `json:"is_valid"` + SelectedBackend string `json:"selected_backend"` + RejectedBackends map[string]string `json:"rejected_backends"` // Backend -> Reason code + BackendRejectionEvidence []BackendRejectionEvidence `json:"backend_rejection_evidence,omitempty"` + BlockingReasonCodes []string `json:"blocking_reason_codes"` + Warnings []string `json:"warnings"` + RequiredHostFeatures []string `json:"required_host_features"` + MissingHostFeatures []string `json:"missing_host_features"` + RecommendedRuntime string `json:"recommended_runtime"` + ExplainabilityText string `json:"explainability_text"` + ValidationTrace []string `json:"validation_trace"` + RemediationHints map[string]string `json:"remediation_hints,omitempty"` +} + +type BackendRejectionEvidence struct { + Backend string `json:"backend"` + ReasonCode string `json:"reason_code"` + CapabilityName *string `json:"capability_name,omitempty"` + CapabilityState *string `json:"capability_state,omitempty"` + CapabilityReasonCode *string `json:"capability_reason_code,omitempty"` + CapabilityStale *bool `json:"capability_stale,omitempty"` + Message *string `json:"message,omitempty"` } // LaunchDryRunResult expands on validation to include the generated configuration