diff --git a/operator/e2e/tests/setup.go b/operator/e2e/tests/setup.go index 971432fea..55b6690c1 100644 --- a/operator/e2e/tests/setup.go +++ b/operator/e2e/tests/setup.go @@ -67,6 +67,10 @@ const ( defaultPollTimeout = 4 * time.Minute // defaultPollInterval is the interval for most polling conditions defaultPollInterval = 5 * time.Second + + // Grove label keys + LabelPodClique = "grove.io/podclique" + LabelPodCliqueScalingGroup = "grove.io/podcliquescalinggroup" ) // TestContext holds common test parameters that are shared across many utility functions. diff --git a/operator/e2e/tests/topology_test.go b/operator/e2e/tests/topology_test.go index 762613b7b..1e04148d0 100644 --- a/operator/e2e/tests/topology_test.go +++ b/operator/e2e/tests/topology_test.go @@ -28,10 +28,13 @@ import ( "github.com/ai-dynamo/grove/operator/e2e/utils" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" ) -// deployWorkloadAndGetPods deploys workload, waits for pods to be ready, and returns the pod list -func deployWorkloadAndGetPods(tc TestContext, expectedPods int) ([]v1.Pod, error) { +// DeployWorkloadAndGetPods deploys workload, waits for pods to be ready, and returns the pod list +func DeployWorkloadAndGetPods(tc TestContext, expectedPods int) ([]v1.Pod, error) { if _, err := deployAndVerifyWorkload(tc); err != nil { return nil, fmt.Errorf("failed to deploy workload: %w", err) } @@ -50,6 +53,34 @@ func deployWorkloadAndGetPods(tc TestContext, expectedPods int) ([]v1.Pod, error return podList.Items, nil } +// createTopologyTestContext creates a TestContext for topology tests with standard configuration +func createTopologyTestContext( + t *testing.T, + ctx context.Context, + clientset *kubernetes.Clientset, + restConfig *rest.Config, + dynamicClient dynamic.Interface, + workloadName, yamlPath string, + expectedPods int, +) TestContext { + return TestContext{ + T: t, + Ctx: ctx, + Clientset: clientset, + RestConfig: restConfig, + DynamicClient: dynamicClient, + Namespace: "default", + Timeout: defaultPollTimeout, + Interval: defaultPollInterval, + Workload: &WorkloadConfig{ + Name: workloadName, + YAMLPath: yamlPath, + Namespace: "default", + ExpectedPods: expectedPods, + }, + } +} + // Test_TAS1_TopologyInfrastructure verifies that the operator creates ClusterTopology and KAI Topology CRs at startup // 1. Verify ClusterTopology CR exists with the correct 4-level hierarchy (zone, block, rack, host) // 2. Verify KAI Topology CR exists with matching levels @@ -133,83 +164,294 @@ func Test_TAS2_MultipleCliquesWithDifferentConstraints(t *testing.T) { defer cleanup() expectedPods := 7 // worker-rack: 3 pods, worker-block: 4 pods - tc := TestContext{ - T: t, - Ctx: ctx, - Clientset: clientset, - RestConfig: restConfig, - DynamicClient: dynamicClient, - Namespace: "default", - Timeout: defaultPollTimeout, - Interval: defaultPollInterval, - Workload: &WorkloadConfig{ - Name: "tas-indep-clq", - YAMLPath: "../yaml/tas-indep-clq.yaml", - Namespace: "default", - ExpectedPods: expectedPods, - }, - } + tc := createTopologyTestContext(t, ctx, clientset, restConfig, dynamicClient, + "tas-indep-clq", "../yaml/tas-indep-clq.yaml", expectedPods) logger.Info("2. Deploy workload (TAS2: multiple cliques with different constraints)") - allPods, err := deployWorkloadAndGetPods(tc, expectedPods) + allPods, err := DeployWorkloadAndGetPods(tc, expectedPods) if err != nil { t.Fatalf("Setup failed: %v", err) } logger.Info("3. Verify worker-rack pods (3) are in the same rack") - rackPods := utils.FilterPodsByLabel(allPods, "grove.io/podclique", "tas-indep-clq-0-worker-rack") - if len(rackPods) != 3 { - t.Fatalf("Expected 3 worker-rack pods, got %d", len(rackPods)) - } - - if err := utils.VerifyPodsInSameTopologyDomain(tc.Ctx, tc.Clientset, rackPods, setup.TopologyLabelRack, logger); err != nil { + if err := utils.VerifyLabeledPodsInTopologyDomain(tc.Ctx, tc.Clientset, allPods, LabelPodClique, "tas-indep-clq-0-worker-rack", 3, setup.TopologyLabelRack, logger); err != nil { t.Fatalf("Failed to verify worker-rack pods in same rack: %v", err) } logger.Info("4. Verify worker-block pods (4) are in the same block") - blockPods := utils.FilterPodsByLabel(allPods, "grove.io/podclique", "tas-indep-clq-0-worker-block") - if len(blockPods) != 4 { - t.Fatalf("Expected 4 worker-block pods, got %d", len(blockPods)) - } - - if err := utils.VerifyPodsInSameTopologyDomain(tc.Ctx, tc.Clientset, blockPods, setup.TopologyLabelBlock, logger); err != nil { + if err := utils.VerifyLabeledPodsInTopologyDomain(tc.Ctx, tc.Clientset, allPods, LabelPodClique, "tas-indep-clq-0-worker-block", 4, setup.TopologyLabelBlock, logger); err != nil { t.Fatalf("Failed to verify worker-block pods in same block: %v", err) } logger.Info("5. Verify KAI PodGroup has correct SubGroups with topology constraints") - podGroups, err := utils.WaitForKAIPodGroups(tc.Ctx, tc.DynamicClient, tc.Namespace, "tas-indep-clq", tc.Timeout, tc.Interval, logger) + podGroup, err := utils.GetPodGroupForBasePodGangReplica(tc.Ctx, tc.DynamicClient, tc.Namespace, tc.Workload.Name, 0, tc.Timeout, tc.Interval, logger) if err != nil { - t.Fatalf("Failed to get KAI PodGroups: %v", err) + t.Fatalf("Failed to get PodGroup: %v", err) + } + + // Verify top-level TopologyConstraint is empty (no PCS constraint in this test) + // Verify SubGroups (2 standalone PCLQs - no PCSG) + expectedSubGroups := []utils.ExpectedSubGroup{ + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "worker-rack", 3, setup.TopologyLabelRack), + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "worker-block", 4, setup.TopologyLabelBlock), + } + if err := utils.VerifyPodGroupTopology(podGroup, "", "", expectedSubGroups, logger); err != nil { + t.Fatalf("Failed to verify KAI PodGroup topology: %v", err) } - podGroup, err := utils.FilterPodGroupByOwner(podGroups, "tas-indep-clq-0") + logger.Info("🎉 TAS2: Multiple Cliques with Different Constraints test completed successfully!") +} + +// Test_TAS3_PCSOnlyConstraint tests constraint only at PCS level with no PCSG/PCLQ constraints +// 1. Deploy workload with PCS-only constraint (packDomain: rack) +// - PCSG: NO explicit constraint (nil) +// - PCLQs: NO explicit constraints +// +// 2. Verify all 4 pods are in same rack (inherited from PCS) +// 3. Verify PCSG worker pods (2 total, 1 per replica) +// 4. Verify router pods (2 standalone) +// 5. Verify KAI PodGroup SubGroups: NO PCSG parent groups (because PCSG constraint is nil, per PR #357) +func Test_TAS3_PCSOnlyConstraint(t *testing.T) { + ctx := context.Background() + + logger.Info("1. Initialize a 28-node Grove cluster for topology testing") + clientset, restConfig, dynamicClient, cleanup := prepareTestCluster(ctx, t, 28) + defer cleanup() + + expectedPods := 4 // 2 PCSG workers + 2 router standalone + tc := createTopologyTestContext(t, ctx, clientset, restConfig, dynamicClient, + "tas-sl-pcs-only", "../yaml/tas-sl-pcs-only.yaml", expectedPods) + + logger.Info("2. Deploy workload (TAS3: PCS-only constraint)") + allPods, err := DeployWorkloadAndGetPods(tc, expectedPods) if err != nil { - t.Fatalf("Failed to find PodGroup for PodGang tas-indep-clq-0: %v", err) + t.Fatalf("Setup failed: %v", err) } - // Verify top-level TopologyConstraint is empty (no PCS constraint in this test) - if err := utils.VerifyKAIPodGroupTopologyConstraint(podGroup, "", "", logger); err != nil { - t.Fatalf("Failed to verify KAI PodGroup top-level constraint: %v", err) + logger.Info("3. Verify all 4 pods in same rack (inherited from PCS)") + if err := utils.VerifyPodsInSameTopologyDomain(tc.Ctx, tc.Clientset, allPods, setup.TopologyLabelRack, logger); err != nil { + t.Fatalf("Failed to verify all pods in same rack: %v", err) } - // Verify SubGroups (2 standalone PCLQs - no PCSG) + logger.Info("4. Verify KAI PodGroup has correct SubGroups (PCS-only constraint)") + podGroup, err := utils.GetPodGroupForBasePodGangReplica(tc.Ctx, tc.DynamicClient, tc.Namespace, tc.Workload.Name, 0, tc.Timeout, tc.Interval, logger) + if err != nil { + t.Fatalf("Failed to get PodGroup: %v", err) + } + + // Verify top-level TopologyConstraint (PCS level: rack) + // Verify SubGroups (2 PCLQ children + 1 router standalone = 3 total) + // Note: PCSG parent groups are NOT created when PCSG has nil TopologyConstraint (PR #357) expectedSubGroups := []utils.ExpectedSubGroup{ - { - Name: "tas-indep-clq-0-worker-rack", - MinMember: 3, - Parent: nil, - RequiredTopologyLevel: setup.TopologyLabelRack, - }, - { - Name: "tas-indep-clq-0-worker-block", - MinMember: 4, - Parent: nil, - RequiredTopologyLevel: setup.TopologyLabelBlock, - }, + // Worker PCLQs (directly under PCS constraint, no PCSG parents) + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "workers-0-worker", 1, ""), + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "workers-1-worker", 1, ""), + // Router (standalone) + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "router", 2, ""), } - if err := utils.VerifyKAIPodGroupSubGroups(podGroup, expectedSubGroups, logger); err != nil { - t.Fatalf("Failed to verify KAI PodGroup SubGroups: %v", err) + if err := utils.VerifyPodGroupTopology(podGroup, setup.TopologyLabelRack, "", expectedSubGroups, logger); err != nil { + t.Fatalf("Failed to verify KAI PodGroup topology: %v", err) } - logger.Info("🎉 TAS2: Multiple Cliques with Different Constraints test completed successfully!") + logger.Info("🎉 TAS3: PCS-Only Constraint test completed successfully!") +} + +// Test_TAS4_PCSGOnlyConstraint tests constraint only at PCSG level with no PCS/PCLQ constraints +// 1. Deploy workload with constraint only at PCSG level (packDomain: rack) +// 2. PCS and PCLQs have NO explicit constraints +// 3. Verify PCSG worker pods (2 total) respect rack constraint +// 4. Router pods (2 standalone) are unconstrained +func Test_TAS4_PCSGOnlyConstraint(t *testing.T) { + ctx := context.Background() + + logger.Info("1. Initialize a 28-node Grove cluster for topology testing") + clientset, restConfig, dynamicClient, cleanup := prepareTestCluster(ctx, t, 28) + defer cleanup() + + expectedPods := 4 // 2 PCSG workers + 2 router standalone + tc := createTopologyTestContext(t, ctx, clientset, restConfig, dynamicClient, + "tas-sl-pcsg-only", "../yaml/tas-sl-pcsg-only.yaml", expectedPods) + + logger.Info("2. Deploy workload (TAS4: PCSG-only constraint)") + allPods, err := DeployWorkloadAndGetPods(tc, expectedPods) + if err != nil { + t.Fatalf("Setup failed: %v", err) + } + + logger.Info("3. Verify PCSG worker pods (2 total, 1 per replica) in same rack") + if err := utils.VerifyLabeledPodsInTopologyDomain(tc.Ctx, tc.Clientset, allPods, LabelPodCliqueScalingGroup, "tas-sl-pcsg-only-0-workers", 2, setup.TopologyLabelRack, logger); err != nil { + t.Fatalf("Failed to verify worker pods in same rack: %v", err) + } + + logger.Info("5. Verify KAI PodGroup has correct SubGroups (PCSG-only constraint)") + podGroup, err := utils.GetPodGroupForBasePodGangReplica(tc.Ctx, tc.DynamicClient, tc.Namespace, tc.Workload.Name, 0, tc.Timeout, tc.Interval, logger) + if err != nil { + t.Fatalf("Failed to get PodGroup: %v", err) + } + + // Verify top-level TopologyConstraint (no PCS constraint) + // Verify SubGroups (2 PCSG parents + 2 PCLQ children + 1 router standalone = 5 total) + expectedSubGroups := []utils.ExpectedSubGroup{ + // PCSG replicas (parent groups, rack constraint) + utils.CreateExpectedPCSGParentSubGroup(tc.Workload.Name, 0, "workers", 0, setup.TopologyLabelRack), + utils.CreateExpectedPCSGParentSubGroup(tc.Workload.Name, 0, "workers", 1, setup.TopologyLabelRack), + // Worker PCLQs (children of PCSG replicas) + utils.CreateExpectedPCLQInPCSGSubGroup(tc.Workload.Name, 0, "workers", 0, "worker", 1, ""), + utils.CreateExpectedPCLQInPCSGSubGroup(tc.Workload.Name, 0, "workers", 1, "worker", 1, ""), + // Router (standalone, no constraint) + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "router", 2, ""), + } + if err := utils.VerifyPodGroupTopology(podGroup, "", "", expectedSubGroups, logger); err != nil { + t.Fatalf("Failed to verify KAI PodGroup topology: %v", err) + } + + logger.Info("🎉 TAS4: PCSG-Only Constraint test completed successfully!") +} + +// Test_TAS5_HostLevelConstraint tests PCLQ-only constraint with host-level packing +// 1. Deploy workload with constraint only at PCLQ level (packDomain: host) +// 2. PCS has NO explicit constraint +// 3. Verify all 2 pods on same host (strictest constraint) +func Test_TAS5_HostLevelConstraint(t *testing.T) { + ctx := context.Background() + + logger.Info("1. Initialize a 28-node Grove cluster for topology testing") + clientset, restConfig, dynamicClient, cleanup := prepareTestCluster(ctx, t, 28) + defer cleanup() + + expectedPods := 2 + tc := createTopologyTestContext(t, ctx, clientset, restConfig, dynamicClient, + "tas-host-level", "../yaml/tas-host-level.yaml", expectedPods) + + logger.Info("2. Deploy workload (TAS5: PCLQ-only host constraint)") + allPods, err := DeployWorkloadAndGetPods(tc, expectedPods) + if err != nil { + t.Fatalf("Setup failed: %v", err) + } + + logger.Info("3. Verify all pods on same host") + if err := utils.VerifyPodsInSameTopologyDomain(tc.Ctx, tc.Clientset, allPods, setup.TopologyLabelHostname, logger); err != nil { + t.Fatalf("Failed to verify pods on same host: %v", err) + } + + // Additional check: verify both pods have same node name + if len(allPods) != 2 { + t.Fatalf("Expected 2 pods, got %d", len(allPods)) + } + if allPods[0].Spec.NodeName != allPods[1].Spec.NodeName { + t.Fatalf("Pods not on same node: %s vs %s", allPods[0].Spec.NodeName, allPods[1].Spec.NodeName) + } + + logger.Info("4. Verify KAI PodGroup has correct SubGroups (PCLQ-only host constraint)") + podGroup, err := utils.GetPodGroupForBasePodGangReplica(tc.Ctx, tc.DynamicClient, tc.Namespace, tc.Workload.Name, 0, tc.Timeout, tc.Interval, logger) + if err != nil { + t.Fatalf("Failed to get PodGroup: %v", err) + } + + // Verify top-level TopologyConstraint (no PCS constraint) + // Verify SubGroups (1 standalone PCLQ with host constraint) + expectedSubGroups := []utils.ExpectedSubGroup{ + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "worker", 2, setup.TopologyLabelHostname), + } + if err := utils.VerifyPodGroupTopology(podGroup, "", "", expectedSubGroups, logger); err != nil { + t.Fatalf("Failed to verify KAI PodGroup topology: %v", err) + } + + logger.Info("🎉 TAS5: Host-Level Constraint test completed successfully!") +} + +// Test_TAS6_StandalonePCLQOnlyPCSZoneConstraint tests standalone PCLQ with only PCS zone constraint (no PCSG layer) +// This test differs from TAS3 in two ways: +// 1. Uses zone constraint (wider domain) instead of rack at PCS level +// 2. Has NO PCSG layer - only standalone PCLQ directly under PCS (simpler structure) +// 3. PCLQ itself has NO explicit constraint (inherits from PCS) +// +// 1. Deploy workload with PCS zone constraint and single standalone PCLQ (4 replicas) +// 2. Verify all 4 pods in same zone (PCS constraint inherited) +// 3. Verify KAI PodGroup has zone constraint at top level +// 4. Verify 1 SubGroup (standalone PCLQ) with NO additional constraint +func Test_TAS6_StandalonePCLQOnlyPCSZoneConstraint(t *testing.T) { + ctx := context.Background() + + logger.Info("1. Initialize a 28-node Grove cluster for topology testing") + clientset, restConfig, dynamicClient, cleanup := prepareTestCluster(ctx, t, 28) + defer cleanup() + + expectedPods := 4 + tc := createTopologyTestContext(t, ctx, clientset, restConfig, dynamicClient, + "tas-standalone-pclq", "../yaml/tas-standalone-pclq-only-pcs-zone.yaml", expectedPods) + + logger.Info("2. Deploy workload (TAS6: Standalone PCLQ with only PCS zone constraint)") + allPods, err := DeployWorkloadAndGetPods(tc, expectedPods) + if err != nil { + t.Fatalf("Setup failed: %v", err) + } + + logger.Info("3. Verify all 4 pods in same zone (PCS zone constraint)") + if err := utils.VerifyPodsInSameTopologyDomain(tc.Ctx, tc.Clientset, allPods, setup.TopologyLabelZone, logger); err != nil { + t.Fatalf("Failed to verify pods in same zone: %v", err) + } + + logger.Info("4. Verify KAI PodGroup has correct SubGroups (Standalone PCLQ with PCS zone constraint)") + podGroup, err := utils.GetPodGroupForBasePodGangReplica(tc.Ctx, tc.DynamicClient, tc.Namespace, tc.Workload.Name, 0, tc.Timeout, tc.Interval, logger) + if err != nil { + t.Fatalf("Failed to get PodGroup: %v", err) + } + + // Verify top-level TopologyConstraint (PCS level: zone) + // Verify SubGroups (1 standalone PCLQ with NO constraint - zone is at PCS level) + expectedSubGroups := []utils.ExpectedSubGroup{ + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "worker", 4, ""), + } + if err := utils.VerifyPodGroupTopology(podGroup, setup.TopologyLabelZone, "", expectedSubGroups, logger); err != nil { + t.Fatalf("Failed to verify KAI PodGroup topology: %v", err) + } + + logger.Info("🎉 TAS6: Standalone PCLQ with Only PCS Zone Constraint test completed successfully!") +} + +// Test_TAS7_NoTopologyConstraint tests gang scheduling without any topology constraints +// 1. Deploy workload with no constraints at PCS, PCSG, or PCLQ levels +// 2. Verify all 4 pods scheduled (gang scheduling works) +// 3. Verify KAI PodGroup has 4 SubGroups with NO topology constraints +func Test_TAS7_NoTopologyConstraint(t *testing.T) { + ctx := context.Background() + + logger.Info("1. Initialize a 28-node Grove cluster for topology testing") + clientset, restConfig, dynamicClient, cleanup := prepareTestCluster(ctx, t, 28) + defer cleanup() + + expectedPods := 4 // 2 PCSG replicas × 2 pods each + tc := createTopologyTestContext(t, ctx, clientset, restConfig, dynamicClient, + "tas-no-constraint", "../yaml/tas-no-constraint.yaml", expectedPods) + + logger.Info("2. Deploy workload (TAS7: No topology constraints)") + allPods, err := DeployWorkloadAndGetPods(tc, expectedPods) + if err != nil { + t.Fatalf("Setup failed: %v", err) + } + + logger.Info("3. Verify all 4 pods scheduled (gang scheduling works without constraints)") + if len(allPods) != 4 { + t.Fatalf("Expected 4 pods, got %d", len(allPods)) + } + logger.Info("4. Verify KAI PodGroup has correct SubGroups (no constraints)") + podGroup, err := utils.GetPodGroupForBasePodGangReplica(tc.Ctx, tc.DynamicClient, tc.Namespace, tc.Workload.Name, 0, tc.Timeout, tc.Interval, logger) + if err != nil { + t.Fatalf("Failed to get PodGroup: %v", err) + } + + // Verify top-level TopologyConstraint (no PCS constraint) + // Verify SubGroups (2 PCLQ children, NO constraints) + // Note: PCSG parent groups are NOT created when PCSG has nil TopologyConstraint (PR #357) + expectedSubGroups := []utils.ExpectedSubGroup{ + // Worker PCLQs (directly under PCS, no PCSG parents, no constraints) + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "workers-0-worker", 2, ""), + utils.CreateExpectedStandalonePCLQSubGroup(tc.Workload.Name, 0, "workers-1-worker", 2, ""), + } + if err = utils.VerifyPodGroupTopology(podGroup, "", "", expectedSubGroups, logger); err != nil { + t.Fatalf("Failed to verify KAI PodGroup topology: %v", err) + } + + logger.Info("🎉 TAS7: No Topology Constraint test completed successfully!") } diff --git a/operator/e2e/utils/kai_topology.go b/operator/e2e/utils/kai_topology.go index b1bf34b4c..4d79e127e 100644 --- a/operator/e2e/utils/kai_topology.go +++ b/operator/e2e/utils/kai_topology.go @@ -19,16 +19,14 @@ package utils import ( - "fmt" - - kaischedulingv2alpha2 "github.com/NVIDIA/KAI-scheduler/pkg/apis/scheduling/v2alpha2" - "context" + "fmt" "time" + kaischedulingv2alpha2 "github.com/NVIDIA/KAI-scheduler/pkg/apis/scheduling/v2alpha2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/client-go/dynamic" + "k8s.io/utils/ptr" ) // ExpectedSubGroup defines the expected structure of a KAI PodGroup SubGroup for verification @@ -40,6 +38,43 @@ type ExpectedSubGroup struct { PreferredTopologyLevel string } +// CreateExpectedStandalonePCLQSubGroup creates an ExpectedSubGroup for a standalone PodClique (not in PCSG) +// Name format: -- +func CreateExpectedStandalonePCLQSubGroup(pcsName string, pcsReplica int, cliqueName string, minMember int32, topologyLevel string) ExpectedSubGroup { + name := GetStandalonePCLQSubGroupName(pcsName, pcsReplica, cliqueName) + return ExpectedSubGroup{ + Name: name, + MinMember: minMember, + Parent: nil, + RequiredTopologyLevel: topologyLevel, + } +} + +// CreateExpectedPCSGParentSubGroup creates an ExpectedSubGroup for a PCSG parent (scaling group replica) +// Name format: --- +func CreateExpectedPCSGParentSubGroup(pcsName string, pcsReplica int, sgName string, sgReplica int, topologyLevel string) ExpectedSubGroup { + name := GetPCSGParentSubGroupName(pcsName, pcsReplica, sgName, sgReplica) + return ExpectedSubGroup{ + Name: name, + MinMember: 0, + Parent: nil, + RequiredTopologyLevel: topologyLevel, + } +} + +// CreateExpectedPCLQInPCSGSubGroup creates an ExpectedSubGroup for a PodClique within a PCSG +// Name format: ---- +func CreateExpectedPCLQInPCSGSubGroup(pcsName string, pcsReplica int, sgName string, sgReplica int, cliqueName string, minMember int32, topologyLevel string) ExpectedSubGroup { + name := GetPCLQInPCSGSubGroupName(pcsName, pcsReplica, sgName, sgReplica, cliqueName) + parentName := GetPCSGParentSubGroupName(pcsName, pcsReplica, sgName, sgReplica) + return ExpectedSubGroup{ + Name: name, + MinMember: minMember, + Parent: ptr.To(parentName), + RequiredTopologyLevel: topologyLevel, + } +} + // GetKAIPodGroupsForPCS retrieves all KAI PodGroups for a given PodCliqueSet by label selector // KAI scheduler creates PodGroups with label: app.kubernetes.io/part-of= // Returns a list of PodGroups that tests can filter by owner reference if needed @@ -179,3 +214,48 @@ func VerifyKAIPodGroupSubGroups(podGroup *kaischedulingv2alpha2.PodGroup, expect logger.Infof("KAI PodGroup %s verified with %d SubGroups", podGroup.Name, len(expectedSubGroups)) return nil } + +// GetPodGroupForBasePodGangReplica retrieves the KAI PodGroup of the corresponding PodGang +// which is the base PodGang of specific PodGangSet replica. +// For a PodGangSet workload "my-workload", replica 0's base PodGang is "my-workload-0". +func GetPodGroupForBasePodGangReplica( + ctx context.Context, + dynamicClient dynamic.Interface, + namespace string, + workloadName string, + pgsReplica int, + timeout time.Duration, + interval time.Duration, + logger *Logger, +) (*kaischedulingv2alpha2.PodGroup, error) { + podGroups, err := WaitForKAIPodGroups(ctx, dynamicClient, namespace, workloadName, timeout, interval, logger) + if err != nil { + return nil, fmt.Errorf("failed to get KAI PodGroups: %w", err) + } + + basePodGangName := GetBasePodGangName(workloadName, pgsReplica) + basePodGroup, err := FilterPodGroupByOwner(podGroups, basePodGangName) + if err != nil { + return nil, fmt.Errorf("failed to find PodGroup for PodGang %s: %w", basePodGangName, err) + } + + return basePodGroup, nil +} + +// VerifyPodGroupTopology verifies both top-level topology constraint and SubGroups structure. +func VerifyPodGroupTopology( + podGroup *kaischedulingv2alpha2.PodGroup, + requiredLevel, preferredLevel string, + expectedSubGroups []ExpectedSubGroup, + logger *Logger, +) error { + if err := VerifyKAIPodGroupTopologyConstraint(podGroup, requiredLevel, preferredLevel, logger); err != nil { + return fmt.Errorf("top-level constraint verification failed: %w", err) + } + + if err := VerifyKAIPodGroupSubGroups(podGroup, expectedSubGroups, logger); err != nil { + return fmt.Errorf("SubGroups verification failed: %w", err) + } + + return nil +} diff --git a/operator/e2e/utils/naming.go b/operator/e2e/utils/naming.go new file mode 100644 index 000000000..9b0cbc41a --- /dev/null +++ b/operator/e2e/utils/naming.go @@ -0,0 +1,45 @@ +//go:build e2e + +package utils + +// /* +// Copyright 2025 The Grove Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// */ + +import "fmt" + +// GetBasePodGangName constructs the base PodGang name for a specific PCS replica. +// Format: - +func GetBasePodGangName(workloadName string, pcsReplica int) string { + return fmt.Sprintf("%s-%d", workloadName, pcsReplica) +} + +// GetStandalonePCLQSubGroupName constructs the SubGroup name for a standalone PodClique. +// Format: -- +func GetStandalonePCLQSubGroupName(pcsName string, pcsReplica int, cliqueName string) string { + return fmt.Sprintf("%s-%d-%s", pcsName, pcsReplica, cliqueName) +} + +// GetPCSGParentSubGroupName constructs the SubGroup name for a PCSG parent (scaling group replica). +// Format: --- +func GetPCSGParentSubGroupName(pcsName string, pcsReplica int, sgName string, sgReplica int) string { + return fmt.Sprintf("%s-%d-%s-%d", pcsName, pcsReplica, sgName, sgReplica) +} + +// GetPCLQInPCSGSubGroupName constructs the SubGroup name for a PodClique within a PCSG. +// Format: ---- +func GetPCLQInPCSGSubGroupName(pcsName string, pcsReplica int, sgName string, sgReplica int, cliqueName string) string { + return fmt.Sprintf("%s-%d-%s-%d-%s", pcsName, pcsReplica, sgName, sgReplica, cliqueName) +} diff --git a/operator/e2e/utils/topology.go b/operator/e2e/utils/topology.go index 996de45f4..26315b48b 100644 --- a/operator/e2e/utils/topology.go +++ b/operator/e2e/utils/topology.go @@ -182,3 +182,24 @@ func VerifyPodsInSameTopologyDomain(ctx context.Context, clientset kubernetes.In logger.Infof("Verified %d pods are in same topology domain %s=%s", len(pods), topologyKey, expectedValue) return nil } + +// VerifyLabeledPodsInTopologyDomain filters pods by label, verifies count, and checks topology domain. +func VerifyLabeledPodsInTopologyDomain( + ctx context.Context, + clientset kubernetes.Interface, + allPods []v1.Pod, + labelKey, labelValue string, + expectedCount int, + topologyKey string, + logger *Logger, +) error { + filteredPods := FilterPodsByLabel(allPods, labelKey, labelValue) + if len(filteredPods) != expectedCount { + return fmt.Errorf( + "expected %d pods with %s=%s, got %d", + expectedCount, labelKey, labelValue, len(filteredPods), + ) + } + + return VerifyPodsInSameTopologyDomain(ctx, clientset, filteredPods, topologyKey, logger) +} diff --git a/operator/e2e/yaml/tas-host-level.yaml b/operator/e2e/yaml/tas-host-level.yaml new file mode 100644 index 000000000..05c8d01c0 --- /dev/null +++ b/operator/e2e/yaml/tas-host-level.yaml @@ -0,0 +1,44 @@ +# Workload: Host-Level Packing - PCS with single clique at host level +# Test scenario: PCS with single clique constrained to host level +--- +apiVersion: grove.io/v1alpha1 +kind: PodCliqueSet +metadata: + name: tas-host-level + labels: + app: tas-host-level +spec: + replicas: 1 + template: + cliques: + - name: worker + labels: + kai.scheduler/queue: test + topologyConstraint: + packDomain: host + spec: + roleName: worker + replicas: 2 + minAvailable: 2 + podSpec: + schedulerName: kai-scheduler + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node_role.e2e.grove.nvidia.com + operator: In + values: + - agent + tolerations: + - key: node_role.e2e.grove.nvidia.com + operator: Equal + value: agent + effect: NoSchedule + containers: + - name: worker + image: registry:5001/nginx:alpine-slim + resources: + requests: + memory: 30Mi diff --git a/operator/e2e/yaml/tas-no-constraint.yaml b/operator/e2e/yaml/tas-no-constraint.yaml new file mode 100644 index 000000000..28204b0c2 --- /dev/null +++ b/operator/e2e/yaml/tas-no-constraint.yaml @@ -0,0 +1,48 @@ +# Workload: SL3 - No Topology Constraints +# Test scenario: No constraints at PCS, PCSG, or PCLQ levels (pure gang scheduling) +--- +apiVersion: grove.io/v1alpha1 +kind: PodCliqueSet +metadata: + name: tas-no-constraint + labels: + app: tas-no-constraint +spec: + replicas: 1 + template: + podCliqueScalingGroups: + - name: workers + replicas: 2 + minAvailable: 2 + cliqueNames: + - worker + cliques: + - name: worker + labels: + kai.scheduler/queue: test + spec: + roleName: worker + replicas: 2 + minAvailable: 2 + podSpec: + schedulerName: kai-scheduler + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node_role.e2e.grove.nvidia.com + operator: In + values: + - agent + tolerations: + - key: node_role.e2e.grove.nvidia.com + operator: Equal + value: agent + effect: NoSchedule + containers: + - name: worker + image: registry:5001/nginx:alpine-slim + resources: + requests: + memory: 30Mi diff --git a/operator/e2e/yaml/tas-sl-pcs-only.yaml b/operator/e2e/yaml/tas-sl-pcs-only.yaml new file mode 100644 index 000000000..ccd560b86 --- /dev/null +++ b/operator/e2e/yaml/tas-sl-pcs-only.yaml @@ -0,0 +1,79 @@ +# Workload: PCS-Only Topology - PCS with rack constraint +# Test scenario: PCS with rack constraint, PCSG without constraint, standalone clique +--- +apiVersion: grove.io/v1alpha1 +kind: PodCliqueSet +metadata: + name: tas-sl-pcs-only + labels: + app: tas-sl-pcs-only +spec: + replicas: 1 + template: + topologyConstraint: + packDomain: rack + podCliqueScalingGroups: + - name: workers + replicas: 2 + minAvailable: 2 + cliqueNames: + - worker + cliques: + - name: worker + labels: + kai.scheduler/queue: test + spec: + roleName: worker + replicas: 1 + minAvailable: 1 + podSpec: + schedulerName: kai-scheduler + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node_role.e2e.grove.nvidia.com + operator: In + values: + - agent + tolerations: + - key: node_role.e2e.grove.nvidia.com + operator: Equal + value: agent + effect: NoSchedule + containers: + - name: worker + image: registry:5001/nginx:alpine-slim + resources: + requests: + memory: 30Mi + - name: router + labels: + kai.scheduler/queue: test + spec: + roleName: router + replicas: 2 + minAvailable: 2 + podSpec: + schedulerName: kai-scheduler + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node_role.e2e.grove.nvidia.com + operator: In + values: + - agent + tolerations: + - key: node_role.e2e.grove.nvidia.com + operator: Equal + value: agent + effect: NoSchedule + containers: + - name: router + image: registry:5001/nginx:alpine-slim + resources: + requests: + memory: 30Mi diff --git a/operator/e2e/yaml/tas-sl-pcsg-only.yaml b/operator/e2e/yaml/tas-sl-pcsg-only.yaml new file mode 100644 index 000000000..eabf0ec58 --- /dev/null +++ b/operator/e2e/yaml/tas-sl-pcsg-only.yaml @@ -0,0 +1,79 @@ +# Workload: PCSG-Only Topology - PCSG with rack constraint +# Test scenario: PCS without constraint, PCSG with rack constraint, standalone clique +--- +apiVersion: grove.io/v1alpha1 +kind: PodCliqueSet +metadata: + name: tas-sl-pcsg-only + labels: + app: tas-sl-pcsg-only +spec: + replicas: 1 + template: + podCliqueScalingGroups: + - name: workers + replicas: 2 + minAvailable: 2 + topologyConstraint: + packDomain: rack + cliqueNames: + - worker + cliques: + - name: worker + labels: + kai.scheduler/queue: test + spec: + roleName: worker + replicas: 1 + minAvailable: 1 + podSpec: + schedulerName: kai-scheduler + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node_role.e2e.grove.nvidia.com + operator: In + values: + - agent + tolerations: + - key: node_role.e2e.grove.nvidia.com + operator: Equal + value: agent + effect: NoSchedule + containers: + - name: worker + image: registry:5001/nginx:alpine-slim + resources: + requests: + memory: 30Mi + - name: router + labels: + kai.scheduler/queue: test + spec: + roleName: router + replicas: 2 + minAvailable: 2 + podSpec: + schedulerName: kai-scheduler + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node_role.e2e.grove.nvidia.com + operator: In + values: + - agent + tolerations: + - key: node_role.e2e.grove.nvidia.com + operator: Equal + value: agent + effect: NoSchedule + containers: + - name: router + image: registry:5001/nginx:alpine-slim + resources: + requests: + memory: 30Mi diff --git a/operator/e2e/yaml/tas-standalone-pclq-only-pcs-zone.yaml b/operator/e2e/yaml/tas-standalone-pclq-only-pcs-zone.yaml new file mode 100644 index 000000000..4f4d12879 --- /dev/null +++ b/operator/e2e/yaml/tas-standalone-pclq-only-pcs-zone.yaml @@ -0,0 +1,44 @@ +# Workload: ZL1 - Zone-Level Topology Constraint +# Test scenario: PCS with zone constraint (widest domain), PCLQ without constraint +--- +apiVersion: grove.io/v1alpha1 +kind: PodCliqueSet +metadata: + name: tas-standalone-pclq + labels: + app: tas-standalone-pclq +spec: + replicas: 1 + template: + topologyConstraint: + packDomain: zone + cliques: + - name: worker + labels: + kai.scheduler/queue: test + spec: + roleName: worker + replicas: 4 + minAvailable: 4 + podSpec: + schedulerName: kai-scheduler + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node_role.e2e.grove.nvidia.com + operator: In + values: + - agent + tolerations: + - key: node_role.e2e.grove.nvidia.com + operator: Equal + value: agent + effect: NoSchedule + containers: + - name: worker + image: registry:5001/nginx:alpine-slim + resources: + requests: + memory: 30Mi