Skip to content

Commit 309cdd9

Browse files
ErikJiangErikJiang
committed
e2e tests: use ginkgo.DeferCleanup for consistent resource cleanup
Signed-off-by: bo.jiang <bo.jiang@daocloud.io>
1 parent 1fa3803 commit 309cdd9

1 file changed

Lines changed: 74 additions & 61 deletions

File tree

‎test/e2e/e2e_test.go‎

Lines changed: 74 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -66,10 +66,22 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
6666

6767
epp := createEndPointPicker(simpleConfig)
6868

69-
generateAndCheckLoad(5)
69+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
70+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
71+
72+
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
73+
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
74+
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
75+
76+
nsHdr, podHdr, _ := runCompletion(simplePrompt, simModelName)
77+
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
78+
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
7079

71-
testutils.DeleteObjects(testConfig, epp)
72-
testutils.DeleteObjects(testConfig, modelServers)
80+
nsHdr, podHdr, _ = runChatCompletion(simplePrompt, simModelName)
81+
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
82+
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
83+
84+
generateAndCheckLoad(5)
7385
})
7486

7587
ginkgo.It("should report metrics", func() {
@@ -82,10 +94,10 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
8294

8395
epp := createEndPointPicker(simpleConfig)
8496

85-
verifyMetrics(infPoolName, numTargetPorts)
97+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
98+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
8699

87-
testutils.DeleteObjects(testConfig, epp)
88-
testutils.DeleteObjects(testConfig, modelServers)
100+
verifyMetrics(infPoolName, numTargetPorts)
89101
})
90102
})
91103

@@ -100,11 +112,11 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
100112

101113
epp := createEndPointPickerHelper(simpleConfig, numOfPods, true, false)
102114

115+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
116+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
117+
103118
ginkgo.By("Verifying that exactly one EPP pod is ready")
104119
waitForReadyLeader(numOfPods)
105-
106-
testutils.DeleteObjects(testConfig, epp)
107-
testutils.DeleteObjects(testConfig, modelServers)
108120
})
109121

110122
ginkgo.It("Should successfully failover and serve traffic after the leader pod is deleted", func() {
@@ -119,6 +131,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
119131

120132
epp := createEndPointPickerHelper(simpleConfig, numOfPods, true, false)
121133

134+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
135+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
136+
122137
ginkgo.By("STEP 1: Verifying initial leader is working correctly before failover")
123138
leaderPod := waitForReadyLeader(numOfPods)
124139
generateAndCheckLoad(5)
@@ -142,10 +157,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
142157
ginkgo.By("STEP 4: Verifying the new leader is working correctly after failover")
143158
generateAndCheckLoad(5)
144159
verifyMetrics(infPoolName, numTargetPorts)
145-
146-
testutils.DeleteObjects(testConfig, epp)
147-
testutils.DeleteObjects(testConfig, modelServers)
148-
149160
})
150161
})
151162

@@ -159,6 +170,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
159170

160171
epp := createEndPointPicker(deprecatedPdConfig)
161172

173+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
174+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
175+
162176
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
163177

164178
if k8sContext != "" {
@@ -213,9 +227,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
213227
gomega.Expect(prefillDecodeCountllmDEpp).Should(gomega.Equal(4))
214228
gomega.Expect(decodeOnlyCount).Should(gomega.Equal(2))
215229
gomega.Expect(decodeOnlyCountllmDEpp).Should(gomega.Equal(2))
216-
217-
testutils.DeleteObjects(testConfig, epp)
218-
testutils.DeleteObjects(testConfig, modelServers)
219230
})
220231
})
221232

@@ -239,6 +250,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
239250

240251
epp := createEndPointPicker(config)
241252

253+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
254+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
255+
242256
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
243257
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
244258
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -263,9 +277,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
263277
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
264278
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))
265279
gomega.Expect(podHdr).Should(gomega.Equal(podHdrCompletion))
266-
267-
testutils.DeleteObjects(testConfig, epp)
268-
testutils.DeleteObjects(testConfig, modelServers)
269280
})
270281

271282
ginkgo.It("should run streaming requests successfully", func() {
@@ -277,6 +288,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
277288

278289
epp := createEndPointPicker(config)
279290

291+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
292+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
293+
280294
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
281295
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
282296
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -295,9 +309,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
295309
nsHdr, podHdr = runStreamingCompletion(extraPrompt, simModelName)
296310
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
297311
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))
298-
299-
testutils.DeleteObjects(testConfig, epp)
300-
testutils.DeleteObjects(testConfig, modelServers)
301312
})
302313

303314
ginkgo.It("should handle decode-first success scenario with cache_hit_threshold", func() {
@@ -313,6 +324,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
313324

314325
epp := createEndPointPicker(config)
315326

327+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
328+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
329+
316330
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
317331
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
318332
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -342,9 +356,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
342356
// (decode-first succeeded, so no P/D fallback occurred)
343357
gomega.Expect(prefillCountAfter).Should(gomega.Equal(prefillCountBefore),
344358
"Prefill pod should NOT process requests when cache threshold is met (decode-first success)")
345-
346-
testutils.DeleteObjects(testConfig, epp)
347-
testutils.DeleteObjects(testConfig, modelServers)
348359
})
349360

350361
ginkgo.It("should handle decode-first fallback to P/D when cache threshold not met", func() {
@@ -360,6 +371,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
360371

361372
epp := createEndPointPicker(config)
362373

374+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
375+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
376+
363377
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
364378
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
365379
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -393,9 +407,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
393407
"Prefill pod SHOULD process requests when cache threshold is NOT met (P/D fallback)")
394408
gomega.Expect(prefillCountAfter-prefillCountBefore).Should(gomega.Equal(2),
395409
"Prefill pod should have processed exactly 2 requests (1 regular + 1 streaming)")
396-
397-
testutils.DeleteObjects(testConfig, epp)
398-
testutils.DeleteObjects(testConfig, modelServers)
399410
})
400411
})
401412
}
@@ -410,6 +421,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
410421

411422
epp := createEndPointPicker(pdConfig)
412423

424+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
425+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
426+
413427
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
414428
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
415429
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -421,9 +435,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
421435
nsHdr, podHdr, _ = runChatCompletion(simplePrompt, simModelName)
422436
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
423437
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))
424-
425-
testutils.DeleteObjects(testConfig, epp)
426-
testutils.DeleteObjects(testConfig, modelServers)
427438
})
428439

429440
ginkgo.It("should run streaming requests successfully", func() {
@@ -435,6 +446,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
435446

436447
epp := createEndPointPicker(pdConfig)
437448

449+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
450+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
451+
438452
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
439453
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
440454
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -446,9 +460,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
446460
nsHdr, podHdr = runStreamingChatCompletion(simplePrompt)
447461
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
448462
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))
449-
450-
testutils.DeleteObjects(testConfig, epp)
451-
testutils.DeleteObjects(testConfig, modelServers)
452463
})
453464
})
454465

@@ -463,6 +474,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
463474

464475
epp := createEndPointPicker(pdConfig)
465476

477+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
478+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
479+
466480
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
467481

468482
if k8sContext != "" {
@@ -517,9 +531,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
517531
gomega.Expect(prefillDecodeCountllmDEpp).Should(gomega.Equal(4))
518532
gomega.Expect(decodeOnlyCount).Should(gomega.Equal(2))
519533
gomega.Expect(decodeOnlyCountllmDEpp).Should(gomega.Equal(2))
520-
521-
testutils.DeleteObjects(testConfig, epp)
522-
testutils.DeleteObjects(testConfig, modelServers)
523534
})
524535
})
525536

@@ -531,6 +542,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
531542

532543
epp := createEndPointPicker(decodeOnlyConfig)
533544

545+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
546+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
547+
534548
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
535549
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
536550
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
@@ -542,9 +556,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
542556
nsHdr, podHdr, _ = runChatCompletion(simplePrompt, simModelName)
543557
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
544558
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
545-
546-
testutils.DeleteObjects(testConfig, epp)
547-
testutils.DeleteObjects(testConfig, modelServers)
548559
})
549560
})
550561

@@ -558,6 +569,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
558569

559570
epp := createEndPointPicker(epdEncodeDecodeConfig)
560571

572+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
573+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
574+
561575
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
562576
if k8sContext != "" {
563577
startEPPMetricsPortForward()
@@ -615,9 +629,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
615629
encodeDecodeCountllmDEpp := getCounterMetric(metricsURL, "llm_d_epp_disagg_decision_total", labelFilter)
616630
gomega.Expect(encodeDecodeCount).Should(gomega.Equal(5))
617631
gomega.Expect(encodeDecodeCountllmDEpp).Should(gomega.Equal(5))
618-
619-
testutils.DeleteObjects(testConfig, epp)
620-
testutils.DeleteObjects(testConfig, modelServers)
621632
})
622633
})
623634

@@ -632,6 +643,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
632643

633644
epp := createEndPointPicker(epdConfig)
634645

646+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
647+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
648+
635649
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
636650
if k8sContext != "" {
637651
startEPPMetricsPortForward()
@@ -698,9 +712,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
698712
// gomega.Expect(epdCountllmDEpp).Should(gomega.BeNumerically(">=", 3))
699713
// gomega.Expect(epdCount + edCount).Should(gomega.Equal(4))
700714
// gomega.Expect(epdCountllmDEpp + edCountllmDEpp).Should(gomega.Equal(4))
701-
702-
testutils.DeleteObjects(testConfig, epp)
703-
testutils.DeleteObjects(testConfig, modelServers)
704715
})
705716
})
706717

@@ -717,6 +728,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
717728
// a single pod; multimodal stages will resolve to this same deployment.
718729
epp := createEndPointPicker(epdConfig)
719730

731+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
732+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
733+
720734
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
721735
if k8sContext != "" {
722736
startEPPMetricsPortForward()
@@ -764,9 +778,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
764778
nsHdr, podHdr = runChatCompletionWithImageEmbeds()
765779
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
766780
gomega.Expect(podHdr).Should(gomega.Equal(epdPods[0]))
767-
768-
testutils.DeleteObjects(testConfig, epp)
769-
testutils.DeleteObjects(testConfig, modelServers)
770781
})
771782
})
772783

@@ -777,6 +788,10 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
777788
epp := createEndPointPicker(kvConfig)
778789

779790
modelServers := createModelServersDecodeKV(1)
791+
792+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
793+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
794+
780795
time.Sleep(5 * time.Second) // wait for model server(s) to become ready
781796

782797
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
@@ -788,9 +803,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
788803
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
789804
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
790805
}
791-
792-
testutils.DeleteObjects(testConfig, epp)
793-
testutils.DeleteObjects(testConfig, modelServers)
794806
})
795807
})
796808

@@ -801,6 +813,10 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
801813
epp := createEndPointPicker(kvExternalTokenizerConfig)
802814

803815
modelServers := createModelServersDecodeKV(1)
816+
817+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
818+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
819+
804820
time.Sleep(5 * time.Second) // wait for model server(s) to become ready
805821

806822
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
@@ -823,9 +839,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
823839
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
824840
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
825841
}
826-
827-
testutils.DeleteObjects(testConfig, epp)
828-
testutils.DeleteObjects(testConfig, modelServers)
829842
})
830843
})
831844

@@ -837,6 +850,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
837850

838851
epp := createEndPointPicker(scaleConfig)
839852

853+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
854+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
855+
840856
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
841857
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
842858
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
@@ -879,9 +895,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
879895
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
880896
gomega.Expect(podHdr).Should(gomega.Equal(scaledDownDecodePods[0]))
881897
}
882-
883-
testutils.DeleteObjects(testConfig, epp)
884-
testutils.DeleteObjects(testConfig, modelServers)
885898
})
886899
})
887900

@@ -893,6 +906,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
893906

894907
epp := createEndPointPicker(dataParallelConfig)
895908

909+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
910+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
911+
896912
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
897913
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
898914
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
@@ -928,9 +944,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
928944
}
929945
}
930946
gomega.Expect(parallelPortHdr).ShouldNot(gomega.Equal(portHdr))
931-
932-
testutils.DeleteObjects(testConfig, epp)
933-
testutils.DeleteObjects(testConfig, modelServers)
934947
})
935948
})
936949
})

0 commit comments

Comments
 (0)