Skip to content

e2e tests: use ginkgo.DeferCleanup for consistent resource cleanup #1705

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ErikJiang wants to merge 1 commit into
base: main
Choose a base branch
from
Open

e2e tests: use ginkgo.DeferCleanup for consistent resource cleanup #1705

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 74 additions & 61 deletions test/e2e/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,22 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(simpleConfig)

generateAndCheckLoad(5)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
gomega.Expect(decodePods).Should(gomega.HaveLen(1))

nsHdr, podHdr, _ := runCompletion(simplePrompt, simModelName)
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
nsHdr, podHdr, _ = runChatCompletion(simplePrompt, simModelName)
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))

generateAndCheckLoad(5)
})

ginkgo.It("should report metrics", func() {
Expand All @@ -82,10 +94,10 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(simpleConfig)

verifyMetrics(infPoolName, numTargetPorts)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
verifyMetrics(infPoolName, numTargetPorts)
})
})

Expand All @@ -100,11 +112,11 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPickerHelper(simpleConfig, numOfPods, true, false)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

ginkgo.By("Verifying that exactly one EPP pod is ready")
waitForReadyLeader(numOfPods)

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})

ginkgo.It("Should successfully failover and serve traffic after the leader pod is deleted", func() {
Expand All @@ -119,6 +131,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPickerHelper(simpleConfig, numOfPods, true, false)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

ginkgo.By("STEP 1: Verifying initial leader is working correctly before failover")
leaderPod := waitForReadyLeader(numOfPods)
generateAndCheckLoad(5)
Expand All @@ -142,10 +157,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
ginkgo.By("STEP 4: Verifying the new leader is working correctly after failover")
generateAndCheckLoad(5)
verifyMetrics(infPoolName, numTargetPorts)

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)

})
})

Expand All @@ -159,6 +170,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(deprecatedPdConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)

if k8sContext != "" {
Expand Down Expand Up @@ -213,9 +227,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
gomega.Expect(prefillDecodeCountllmDEpp).Should(gomega.Equal(4))
gomega.Expect(decodeOnlyCount).Should(gomega.Equal(2))
gomega.Expect(decodeOnlyCountllmDEpp).Should(gomega.Equal(2))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -239,6 +250,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(config)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
Expand All @@ -263,9 +277,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))
gomega.Expect(podHdr).Should(gomega.Equal(podHdrCompletion))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})

ginkgo.It("should run streaming requests successfully", func() {
Expand All @@ -277,6 +288,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(config)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
Expand All @@ -295,9 +309,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
nsHdr, podHdr = runStreamingCompletion(extraPrompt, simModelName)
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})

ginkgo.It("should handle decode-first success scenario with cache_hit_threshold", func() {
Expand All @@ -313,6 +324,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(config)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
Expand Down Expand Up @@ -342,9 +356,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
// (decode-first succeeded, so no P/D fallback occurred)
gomega.Expect(prefillCountAfter).Should(gomega.Equal(prefillCountBefore),
"Prefill pod should NOT process requests when cache threshold is met (decode-first success)")

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})

ginkgo.It("should handle decode-first fallback to P/D when cache threshold not met", func() {
Expand All @@ -360,6 +371,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(config)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
Expand Down Expand Up @@ -393,9 +407,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
"Prefill pod SHOULD process requests when cache threshold is NOT met (P/D fallback)")
gomega.Expect(prefillCountAfter-prefillCountBefore).Should(gomega.Equal(2),
"Prefill pod should have processed exactly 2 requests (1 regular + 1 streaming)")

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})
}
Expand All @@ -410,6 +421,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(pdConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
Expand All @@ -421,9 +435,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
nsHdr, podHdr, _ = runChatCompletion(simplePrompt, simModelName)
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})

ginkgo.It("should run streaming requests successfully", func() {
Expand All @@ -435,6 +446,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(pdConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.HaveLen(prefillReplicas))
gomega.Expect(decodePods).Should(gomega.HaveLen(decodeReplicas))
Expand All @@ -446,9 +460,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
nsHdr, podHdr = runStreamingChatCompletion(simplePrompt)
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.BeElementOf(decodePods))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -463,6 +474,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(pdConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)

if k8sContext != "" {
Expand Down Expand Up @@ -517,9 +531,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
gomega.Expect(prefillDecodeCountllmDEpp).Should(gomega.Equal(4))
gomega.Expect(decodeOnlyCount).Should(gomega.Equal(2))
gomega.Expect(decodeOnlyCountllmDEpp).Should(gomega.Equal(2))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -531,6 +542,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(decodeOnlyConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
Expand All @@ -542,9 +556,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
nsHdr, podHdr, _ = runChatCompletion(simplePrompt, simModelName)
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -558,6 +569,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(epdEncodeDecodeConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
if k8sContext != "" {
startEPPMetricsPortForward()
Expand Down Expand Up @@ -615,9 +629,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
encodeDecodeCountllmDEpp := getCounterMetric(metricsURL, "llm_d_epp_disagg_decision_total", labelFilter)
gomega.Expect(encodeDecodeCount).Should(gomega.Equal(5))
gomega.Expect(encodeDecodeCountllmDEpp).Should(gomega.Equal(5))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -632,6 +643,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(epdConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
if k8sContext != "" {
startEPPMetricsPortForward()
Expand Down Expand Up @@ -698,9 +712,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
// gomega.Expect(epdCountllmDEpp).Should(gomega.BeNumerically(">=", 3))
// gomega.Expect(epdCount + edCount).Should(gomega.Equal(4))
// gomega.Expect(epdCountllmDEpp + edCountllmDEpp).Should(gomega.Equal(4))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -717,6 +728,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
// a single pod; multimodal stages will resolve to this same deployment.
epp := createEndPointPicker(epdConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

metricsURL := fmt.Sprintf("http://localhost:%s/metrics", metricsPort)
if k8sContext != "" {
startEPPMetricsPortForward()
Expand Down Expand Up @@ -764,9 +778,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
nsHdr, podHdr = runChatCompletionWithImageEmbeds()
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.Equal(epdPods[0]))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -777,6 +788,10 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
epp := createEndPointPicker(kvConfig)

modelServers := createModelServersDecodeKV(1)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

time.Sleep(5 * time.Second) // wait for model server(s) to become ready

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
Expand All @@ -788,9 +803,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
}

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -801,6 +813,10 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
epp := createEndPointPicker(kvExternalTokenizerConfig)

modelServers := createModelServersDecodeKV(1)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

time.Sleep(5 * time.Second) // wait for model server(s) to become ready

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
Expand All @@ -823,9 +839,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.Equal(decodePods[0]))
}

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -837,6 +850,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(scaleConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
Expand Down Expand Up @@ -879,9 +895,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
gomega.Expect(nsHdr).Should(gomega.Equal(nsName))
gomega.Expect(podHdr).Should(gomega.Equal(scaledDownDecodePods[0]))
}

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})

Expand All @@ -893,6 +906,9 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {

epp := createEndPointPicker(dataParallelConfig)

ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, modelServers)
ginkgo.DeferCleanup(testutils.DeleteObjects, testConfig, epp)

prefillPods, decodePods := getModelServerPods(podSelector, prefillSelector, decodeSelector)
gomega.Expect(prefillPods).Should(gomega.BeEmpty())
gomega.Expect(decodePods).Should(gomega.HaveLen(1))
Expand Down Expand Up @@ -928,9 +944,6 @@ var _ = ginkgo.Describe("Run end to end tests", ginkgo.Ordered, func() {
}
}
gomega.Expect(parallelPortHdr).ShouldNot(gomega.Equal(portHdr))

testutils.DeleteObjects(testConfig, epp)
testutils.DeleteObjects(testConfig, modelServers)
})
})
})
Expand Down
Loading