Skip to content

Commit f2963ad

Browse files
committed
fix(e2e): resolve agent pods by selector in clique-identity assertion (#437)
assertNodeCliqueIdentities was hardwired to the node-wide scenario's app=gpu-agent selector, so when the IMEX scenario reused it the per-node pod lookup polled for gpu-agent pods that never exist in the imex cluster and timed out ("no running gpu-agent pod on node ..."). Parameterize the assertion with namespace + selector (via the shared agentPodOnNode helper) and update both call sites; the now-redundant nriAgentPodOnNode is removed. Surfaced by e2e-nri once the imex scenario joined the nri label group. Signed-off-by: Giulio Calzolari <gcalzolari@nvidia.com>
1 parent 5ef0b89 commit f2963ad

2 files changed

Lines changed: 11 additions & 33 deletions

File tree

tests/e2e/go/scenario_imex_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ var _ = Describe("nvml-mock NRI IMEX channel injection", Label("imex", "nri"), O
8484
if !p.HasFabric() {
8585
Skip("profile " + name + " declares no device_defaults.fabric block")
8686
}
87-
assertNodeCliqueIdentities(ctx, h, workers)
87+
assertNodeCliqueIdentities(ctx, h, imexWorkloadNS, imexAgentSelector, workers)
8888
})
8989
})
9090
}
@@ -146,8 +146,8 @@ func assertWorkloadSeesImexChannels(ctx context.Context, h *harness.Harness, wor
146146
}
147147
}
148148

149-
// agentPodOnNode generalizes nriAgentPodOnNode with namespace + selector
150-
// parameters so multiple scenarios can locate their per-node agent pod.
149+
// agentPodOnNode locates a running agent pod on the given node by namespace and
150+
// label selector, so multiple scenarios (gpu-agent, imex-agent) can share it.
151151
func agentPodOnNode(ctx context.Context, h *harness.Harness, ns, selector, node string) kube.PodRef {
152152
GinkgoHelper()
153153
var name string

tests/e2e/go/scenario_nri_test.go

Lines changed: 8 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ var _ = Describe("nvml-mock node-wide NRI injection", Label("nri"), Ordered, fun
100100
if !computeDomain {
101101
Skip("profile " + name + " declares no device_defaults.fabric block; ComputeDomain identity is unsupported")
102102
}
103-
assertNodeCliqueIdentities(ctx, h, workers)
103+
assertNodeCliqueIdentities(ctx, h, nriWorkloadNS, nriAgentSelector, workers)
104104
})
105105
})
106106
}
@@ -171,15 +171,17 @@ func assertAgentSeesGPUs(ctx context.Context, h *harness.Harness, expectedGPUs i
171171
}
172172

173173
// assertNodeCliqueIdentities runs the staged `check-fabric` consumer inside the
174-
// gpu-agent pod on every worker and asserts each node reports the clique /
175-
// cluster UUID the topology overlay assigned to it — with no nvidia.com/gpu
176-
// request and no MOCK_* env in the pod spec (identity comes only from NRI).
177-
func assertNodeCliqueIdentities(ctx context.Context, h *harness.Harness, workers []cluster.Node) {
174+
// agent pod (identified by ns/selector) on every worker and asserts each node
175+
// reports the clique / cluster UUID the topology overlay assigned to it — with
176+
// no nvidia.com/gpu request and no MOCK_* env in the pod spec (identity comes
177+
// only from NRI). ns/selector are parameterized so both the node-wide (gpu-agent)
178+
// and IMEX (imex-agent) scenarios can share this assertion.
179+
func assertNodeCliqueIdentities(ctx context.Context, h *harness.Harness, ns, selector string, workers []cluster.Node) {
178180
GinkgoHelper()
179181
cliqueByNode := nriCliqueByNode(workers)
180182
for _, w := range workers {
181183
expected := cliqueByNode[w.Name]
182-
pod := nriAgentPodOnNode(ctx, h, w.Name)
184+
pod := agentPodOnNode(ctx, h, ns, selector, w.Name)
183185
res, err := h.Kube.ExecSh(ctx, pod, "check-fabric 2>&1")
184186
Expect(err).NotTo(HaveOccurred(), "check-fabric on %s: %s", w.Name, res.Combined())
185187
out := res.Combined()
@@ -205,30 +207,6 @@ func firstNRIAgentPod(ctx context.Context, h *harness.Harness) kube.PodRef {
205207
return kube.PodRef{Namespace: nriWorkloadNS, Pod: name}
206208
}
207209

208-
func nriAgentPodOnNode(ctx context.Context, h *harness.Harness, node string) kube.PodRef {
209-
GinkgoHelper()
210-
var name string
211-
Eventually(func() (string, error) {
212-
pods, err := h.Kube.RunningPodNames(ctx, nriWorkloadNS, nriAgentSelector)
213-
if err != nil {
214-
return "", err
215-
}
216-
for _, pod := range pods {
217-
podNode, err := h.Kube.PodNode(ctx, nriWorkloadNS, pod)
218-
if err != nil {
219-
return "", err
220-
}
221-
if podNode == node {
222-
name = pod
223-
return name, nil
224-
}
225-
}
226-
return "", nil
227-
}).WithContext(ctx).WithTimeout(config.ReadyTimeout()).WithPolling(config.PollInterval()).
228-
ShouldNot(BeEmpty(), "no running gpu-agent pod on node %s", node)
229-
return kube.PodRef{Namespace: nriWorkloadNS, Pod: name}
230-
}
231-
232210
func writeNRITopologyValues(workers []cluster.Node) string {
233211
GinkgoHelper()
234212
path, err := assets.WriteTemp("nri-topology-*.yaml", nriTopologyValues(workers))

0 commit comments

Comments
 (0)