Skip to content

Commit 895c168

Browse files
ErikJiangErikJiang
committed
e2e tests: use ginkgo.DeferCleanup for consistent resource cleanup
Signed-off-by: bo.jiang <bo.jiang@daocloud.io>
1 parent f61ab91 commit 895c168

1 file changed

Lines changed: 53 additions & 51 deletions

File tree

‎test/e2e/e2e_test.go‎

Lines changed: 53 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
6060

6161
epp := createEndPointPicker(simpleConfig)
6262

63+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
64+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
65+
6366
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
6467
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
6568
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
@@ -71,9 +74,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
7174
nsHdr, podHdr, _ = runChatCompletion(simplePrompt, simModelName)
7275
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
7376
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
74-
75-
testutils.DeleteObjects(testConfig, epp)
76-
testutils.DeleteObjects(testConfig, modelServers)
7777
})
7878
})
7979

@@ -87,6 +87,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
8787

8888
epp := createEndPointPicker(deprecatedPdConfig)
8989

90+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
91+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
92+
9093
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
9194

9295
if k8sContext != "" {
@@ -141,9 +144,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
141144
gomega.Expect(prefillDecodeCountllmDEpp).Should(gomega.Equal(4))
142145
gomega.Expect(decodeOnlyCount).Should(gomega.Equal(2))
143146
gomega.Expect(decodeOnlyCountllmDEpp).Should(gomega.Equal(2))
144-
145-
testutils.DeleteObjects(testConfig, epp)
146-
testutils.DeleteObjects(testConfig, modelServers)
147147
})
148148
})
149149

@@ -167,6 +167,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
167167

168168
epp := createEndPointPicker(config)
169169

170+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
171+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
172+
170173
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
171174
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
172175
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -191,9 +194,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
191194
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
192195
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))
193196
gomega.Expect(podHdr).Should(gomega.Equal(podHdrCompletion))
194-
195-
testutils.DeleteObjects(testConfig, epp)
196-
testutils.DeleteObjects(testConfig, modelServers)
197197
})
198198

199199
ginkgo.It("should run streaming requests successfully", func() {
@@ -205,6 +205,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
205205

206206
epp := createEndPointPicker(config)
207207

208+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
209+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
210+
208211
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
209212
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
210213
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -223,9 +226,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
223226
nsHdr, podHdr = runStreamingCompletion(extraPrompt, simModelName)
224227
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
225228
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))
226-
227-
testutils.DeleteObjects(testConfig, epp)
228-
testutils.DeleteObjects(testConfig, modelServers)
229229
})
230230

231231
ginkgo.It("should handle decode-first success scenario with cache_hit_threshold", func() {
@@ -241,6 +241,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
241241

242242
epp := createEndPointPicker(config)
243243

244+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
245+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
246+
244247
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
245248
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
246249
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -270,9 +273,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
270273
// (decode-first succeeded, so no P/D fallback occurred)
271274
gomega.Expect(prefillCountAfter).Should(gomega.Equal(prefillCountBefore),
272275
"Prefill pod should NOT process requests when cache threshold is met (decode-first success)")
273-
274-
testutils.DeleteObjects(testConfig, epp)
275-
testutils.DeleteObjects(testConfig, modelServers)
276276
})
277277

278278
ginkgo.It("should handle decode-first fallback to P/D when cache threshold not met", func() {
@@ -288,6 +288,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
288288

289289
epp := createEndPointPicker(config)
290290

291+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
292+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
293+
291294
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
292295
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
293296
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -321,9 +324,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
321324
"Prefill pod SHOULD process requests when cache threshold is NOT met (P/D fallback)")
322325
gomega.Expect(prefillCountAfter-prefillCountBefore).Should(gomega.Equal(2),
323326
"Prefill pod should have processed exactly 2 requests (1 regular + 1 streaming)")
324-
325-
testutils.DeleteObjects(testConfig, epp)
326-
testutils.DeleteObjects(testConfig, modelServers)
327327
})
328328
})
329329
}
@@ -338,6 +338,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
338338

339339
epp := createEndPointPicker(pdConfig)
340340

341+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
342+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
343+
341344
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
342345
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
343346
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -349,9 +352,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
349352
nsHdr, podHdr, _ = runChatCompletion(simplePrompt, simModelName)
350353
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
351354
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))
352-
353-
testutils.DeleteObjects(testConfig, epp)
354-
testutils.DeleteObjects(testConfig, modelServers)
355355
})
356356

357357
ginkgo.It("should run streaming requests successfully", func() {
@@ -363,6 +363,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
363363

364364
epp := createEndPointPicker(pdConfig)
365365

366+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
367+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
368+
366369
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
367370
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
368371
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
@@ -374,9 +377,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
374377
nsHdr, podHdr = runStreamingChatCompletion(simplePrompt)
375378
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
376379
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))
377-
378-
testutils.DeleteObjects(testConfig, epp)
379-
testutils.DeleteObjects(testConfig, modelServers)
380380
})
381381
})
382382

@@ -391,6 +391,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
391391

392392
epp := createEndPointPicker(pdConfig)
393393

394+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
395+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
396+
394397
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
395398

396399
if k8sContext != "" {
@@ -445,9 +448,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
445448
gomega.Expect(prefillDecodeCountllmDEpp).Should(gomega.Equal(4))
446449
gomega.Expect(decodeOnlyCount).Should(gomega.Equal(2))
447450
gomega.Expect(decodeOnlyCountllmDEpp).Should(gomega.Equal(2))
448-
449-
testutils.DeleteObjects(testConfig, epp)
450-
testutils.DeleteObjects(testConfig, modelServers)
451451
})
452452
})
453453

@@ -459,6 +459,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
459459

460460
epp := createEndPointPicker(decodeOnlyConfig)
461461

462+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
463+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
464+
462465
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
463466
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
464467
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
@@ -470,9 +473,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
470473
nsHdr, podHdr, _ = runChatCompletion(simplePrompt, simModelName)
471474
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
472475
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
473-
474-
testutils.DeleteObjects(testConfig, epp)
475-
testutils.DeleteObjects(testConfig, modelServers)
476476
})
477477
})
478478

@@ -486,6 +486,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
486486

487487
epp := createEndPointPicker(epdEncodeDecodeConfig)
488488

489+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
490+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
491+
489492
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
490493
if k8sContext != "" {
491494
startEPPMetricsPortForward()
@@ -543,9 +546,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
543546
encodeDecodeCountllmDEpp := getCounterMetric(metricsURL, "llm_d_epp_disagg_decision_total", labelFilter)
544547
gomega.Expect(encodeDecodeCount).Should(gomega.Equal(5))
545548
gomega.Expect(encodeDecodeCountllmDEpp).Should(gomega.Equal(5))
546-
547-
testutils.DeleteObjects(testConfig, epp)
548-
testutils.DeleteObjects(testConfig, modelServers)
549549
})
550550
})
551551

@@ -560,6 +560,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
560560

561561
epp := createEndPointPicker(epdConfig)
562562

563+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
564+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
565+
563566
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
564567
if k8sContext != "" {
565568
startEPPMetricsPortForward()
@@ -626,9 +629,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
626629
// gomega.Expect(epdCountllmDEpp).Should(gomega.BeNumerically(">=", 3))
627630
// gomega.Expect(epdCount + edCount).Should(gomega.Equal(4))
628631
// gomega.Expect(epdCountllmDEpp + edCountllmDEpp).Should(gomega.Equal(4))
629-
630-
testutils.DeleteObjects(testConfig, epp)
631-
testutils.DeleteObjects(testConfig, modelServers)
632632
})
633633
})
634634

@@ -645,6 +645,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
645645
// a single pod; multimodal stages will resolve to this same deployment.
646646
epp := createEndPointPicker(epdConfig)
647647

648+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
649+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
650+
648651
metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
649652
if k8sContext != "" {
650653
startEPPMetricsPortForward()
@@ -692,9 +695,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
692695
nsHdr, podHdr = runChatCompletionWithImageEmbeds()
693696
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
694697
gomega.Expect(podHdr).Should(gomega.Equal(epdPods[0]))
695-
696-
testutils.DeleteObjects(testConfig, epp)
697-
testutils.DeleteObjects(testConfig, modelServers)
698698
})
699699
})
700700

@@ -705,6 +705,10 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
705705
epp := createEndPointPicker(kvConfig)
706706

707707
modelServers := createModelServersDecodeKV(1)
708+
709+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
710+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
711+
708712
time.Sleep(5 * time.Second) // wait for model server(s) to become ready
709713

710714
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
@@ -716,9 +720,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
716720
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
717721
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
718722
}
719-
720-
testutils.DeleteObjects(testConfig, epp)
721-
testutils.DeleteObjects(testConfig, modelServers)
722723
})
723724
})
724725

@@ -729,6 +730,10 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
729730
epp := createEndPointPicker(kvExternalTokenizerConfig)
730731

731732
modelServers := createModelServersDecodeKV(1)
733+
734+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
735+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
736+
732737
time.Sleep(5 * time.Second) // wait for model server(s) to become ready
733738

734739
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
@@ -751,9 +756,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
751756
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
752757
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
753758
}
754-
755-
testutils.DeleteObjects(testConfig, epp)
756-
testutils.DeleteObjects(testConfig, modelServers)
757759
})
758760
})
759761

@@ -765,6 +767,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
765767

766768
epp := createEndPointPicker(scaleConfig)
767769

770+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
771+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
772+
768773
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
769774
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
770775
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
@@ -807,9 +812,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
807812
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
808813
gomega.Expect(podHdr).Should(gomega.Equal(scaledDownDecodePods[0]))
809814
}
810-
811-
testutils.DeleteObjects(testConfig, epp)
812-
testutils.DeleteObjects(testConfig, modelServers)
813815
})
814816
})
815817

@@ -821,6 +823,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
821823

822824
epp := createEndPointPicker(dataParallelConfig)
823825

826+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
827+
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)
828+
824829
prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
825830
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
826831
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
@@ -856,9 +861,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
856861
}
857862
}
858863
gomega.Expect(parallelPortHdr).ShouldNot(gomega.Equal(portHdr))
859-
860-
testutils.DeleteObjects(testConfig, epp)
861-
testutils.DeleteObjects(testConfig, modelServers)
862864
})
863865
})
864866
})

0 commit comments

Comments
 (0)