[Bug] TestRayServiceInPlaceUpdate is flaky (#2620)

Author: kevin85421

.github/workflows/e2e-tests-reusable-workflow.yaml

@@ -74,7 +74,7 @@ jobs:
 
             set -euo pipefail
             cd ${{ inputs.dir-to-test }}
-            go test -timeout 30m -v ./test/e2e -json 2>&1 | tee ${KUBERAY_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
+            go test -timeout 60m -v ./test/e2e -json 2>&1 | tee ${KUBERAY_TEST_OUTPUT_DIR}/gotest.log | gotestfmt
 
         - name: Print KubeRay operator logs
           if: (!inputs.plugin-test) && always() && steps.deploy.outcome == 'success'

ray-operator/test/e2e/rayservice_in_place_update_test.go

@@ -52,12 +52,14 @@ func TestRayServiceInPlaceUpdate(t *testing.T) {
 	}, TestTimeoutShort).Should(WithTransform(sampleyaml.IsPodRunningAndReady, BeTrue()))
 
 	// test the default curl result
-	// curl /fruit
-	stdout, _ := curlRayServicePod(test, rayService, curlPod, curlContainerName, "/fruit", `["MANGO", 2]`)
-	g.Expect(stdout.String()).To(Equal("6"))
-	// curl /calc
-	stdout, _ = curlRayServicePod(test, rayService, curlPod, curlContainerName, "/calc", `["MUL", 3]`)
-	g.Expect(stdout.String()).To(Equal("15 pizzas please!"))
+	g.Eventually(func(g Gomega) {
+		// curl /fruit
+		stdout, _ := curlRayServicePod(test, rayService, curlPod, curlContainerName, "/fruit", `["MANGO", 2]`)
+		g.Expect(stdout.String()).To(Equal("6"))
+		// curl /calc
+		stdout, _ = curlRayServicePod(test, rayService, curlPod, curlContainerName, "/calc", `["MUL", 3]`)
+		g.Expect(stdout.String()).To(Equal("15 pizzas please!"))
+	}, TestTimeoutShort).Should(Succeed())
 
 	// In-place update
 	// Parse ServeConfigV2 and replace the string in the simplest way to update it.
@@ -79,7 +81,7 @@ func TestRayServiceInPlaceUpdate(t *testing.T) {
 	// Test the new price and factor
 	g.Eventually(func(g Gomega) {
 		// curl /fruit
-		stdout, _ = curlRayServicePod(test, rayService, curlPod, curlContainerName, "/fruit", `["MANGO", 2]`)
+		stdout, _ := curlRayServicePod(test, rayService, curlPod, curlContainerName, "/fruit", `["MANGO", 2]`)
 		g.Expect(stdout.String()).To(Equal("8"))
 		// curl /calc
 		stdout, _ = curlRayServicePod(test, rayService, curlPod, curlContainerName, "/calc", `["MUL", 3]`)