Skip to content

Commit e9f3155

Browse files
chiayichiayi
authored
Add e2e test for kubectl ray job submit (#2614)
1 parent e6af2cc commit e9f3155

File tree

7 files changed

+249
-0
lines changed

7 files changed

+249
-0
lines changed

‎.github/workflows/e2e-tests-reusable-workflow.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,9 @@ on:
99
dir-to-test:
1010
required: true
1111
type: string
12+
ray-version:
13+
required: false
14+
type: string
1215

1316
jobs:
1417
build:
@@ -32,6 +35,18 @@ jobs:
3235
- name: Setup and start KinD cluster
3336
uses: ./.github/workflows/actions/kind
3437

38+
- name: Set up Python
39+
if: inputs.plugin-test
40+
uses: actions/setup-python@v5
41+
with:
42+
python-version: '3.12'
43+
44+
- name: Install Ray
45+
if: inputs.plugin-test
46+
run: |
47+
python --version
48+
pip install -U "ray[default]==${{ inputs.ray-version }}"
49+
3550
- name: Build CLI and Add to PATH
3651
if: inputs.plugin-test
3752
run: |

‎.github/workflows/e2e-tests.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,4 @@ jobs:
3838
with:
3939
plugin-test: true
4040
dir-to-test: kubectl-plugin
41+
ray-version: 2.40.0

‎kubectl-plugin/test/e2e/kubectl_ray_job_submit_test.go‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
package e2e
2+
3+
import (
4+
"os/exec"
5+
"path"
6+
"regexp"
7+
8+
. "github.com/onsi/ginkgo/v2"
9+
. "github.com/onsi/gomega"
10+
)
11+
12+
// Directory when running test is kuberay/kubectl-plugin/test/e2e/
13+
const (
14+
rayJobFilePath = "./testdata/ray-job.interactive-mode.yaml"
15+
rayJobNoEnvFilePath = "./testdata/ray-job.interactive-mode-no-runtime-env.yaml"
16+
kubectlRayJobWorkingDir = "./testdata/rayjob-submit-working-dir/"
17+
entrypointSampleFileName = "entrypoint-python-sample.py"
18+
runtimeEnvSampleFileName = "runtime-env-sample.yaml"
19+
)
20+
21+
var _ = Describe("Calling ray plugin `job submit` command on Ray Job", Ordered, func() {
22+
It("succeed in submitting RayJob", func() {
23+
cmd := exec.Command("kubectl", "ray", "job", "submit", "-f", rayJobFilePath, "--working-dir", kubectlRayJobWorkingDir, "--", "python", entrypointSampleFileName)
24+
output, err := cmd.CombinedOutput()
25+
26+
Expect(err).NotTo(HaveOccurred())
27+
// Retrieve the Job ID from the output
28+
regexExp := regexp.MustCompile(`'([^']*raysubmit[^']*)'`)
29+
matches := regexExp.FindStringSubmatch(string(output))
30+
31+
Expect(len(matches)).To(BeNumerically(">=", 2))
32+
cmdOutputJobID := matches[1]
33+
34+
// Use kubectl to check status of the rayjob
35+
// Retrieve Job ID
36+
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
37+
output, err = cmd.CombinedOutput()
38+
Expect(err).ToNot(HaveOccurred())
39+
40+
Expect(cmdOutputJobID).To(Equal(string(output)))
41+
42+
// Retrieve Job Status
43+
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
44+
output, err = cmd.CombinedOutput()
45+
Expect(err).ToNot(HaveOccurred())
46+
47+
Expect(string(output)).To(Equal("SUCCEEDED"))
48+
49+
// Retrieve Job Deployment Status
50+
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
51+
output, err = cmd.CombinedOutput()
52+
Expect(err).ToNot(HaveOccurred())
53+
54+
Expect(string(output)).To(Equal("Complete"))
55+
56+
// Cleanup
57+
cmd = exec.Command("kubectl", "delete", "rayjob", "rayjob-sample")
58+
_, err = cmd.CombinedOutput()
59+
Expect(err).ToNot(HaveOccurred())
60+
})
61+
62+
It("succeed in submitting RayJob with runtime environment set with working dir", func() {
63+
runtimeEnvFilePath := path.Join(kubectlRayJobWorkingDir, runtimeEnvSampleFileName)
64+
cmd := exec.Command("kubectl", "ray", "job", "submit", "-f", rayJobNoEnvFilePath, "--runtime-env", runtimeEnvFilePath, "--", "python", entrypointSampleFileName)
65+
output, err := cmd.CombinedOutput()
66+
67+
Expect(err).NotTo(HaveOccurred())
68+
// Retrieve the Job ID from the output
69+
regexExp := regexp.MustCompile(`'([^']*raysubmit[^']*)'`)
70+
matches := regexExp.FindStringSubmatch(string(output))
71+
72+
Expect(len(matches)).To(BeNumerically(">=", 2))
73+
cmdOutputJobID := matches[1]
74+
75+
// Use kubectl to check status of the rayjob
76+
// Retrieve Job ID
77+
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobId}")
78+
output, err = cmd.CombinedOutput()
79+
Expect(err).ToNot(HaveOccurred())
80+
81+
Expect(cmdOutputJobID).To(Equal(string(output)))
82+
83+
// Retrieve Job Status
84+
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobStatus}")
85+
output, err = cmd.CombinedOutput()
86+
Expect(err).ToNot(HaveOccurred())
87+
88+
Expect(string(output)).To(Equal("SUCCEEDED"))
89+
90+
// Retrieve Job Deployment Status
91+
cmd = exec.Command("kubectl", "get", "rayjob", "rayjob-sample", "-o", "jsonpath={.status.jobDeploymentStatus}")
92+
output, err = cmd.CombinedOutput()
93+
Expect(err).ToNot(HaveOccurred())
94+
95+
Expect(string(output)).To(Equal("Complete"))
96+
97+
// Cleanup
98+
cmd = exec.Command("kubectl", "delete", "rayjob", "rayjob-sample")
99+
_, err = cmd.CombinedOutput()
100+
Expect(err).ToNot(HaveOccurred())
101+
})
102+
})

‎kubectl-plugin/test/e2e/testdata/ray-job.interactive-mode-no-runtime-env.yaml‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
apiVersion: ray.io/v1
2+
kind: RayJob
3+
metadata:
4+
name: rayjob-sample
5+
spec:
6+
submissionMode: 'InteractiveMode'
7+
rayClusterSpec:
8+
rayVersion: '2.39.0'
9+
headGroupSpec:
10+
rayStartParams:
11+
dashboard-host: '0.0.0.0'
12+
template:
13+
spec:
14+
containers:
15+
- name: ray-head
16+
image: rayproject/ray:2.39.0
17+
ports:
18+
- containerPort: 6379
19+
name: gcs-server
20+
- containerPort: 8265
21+
name: dashboard
22+
- containerPort: 10001
23+
name: client
24+
resources:
25+
limits:
26+
cpu: "1"
27+
requests:
28+
cpu: "200m"
29+
workerGroupSpecs:
30+
- replicas: 1
31+
minReplicas: 1
32+
maxReplicas: 5
33+
groupName: small-group
34+
rayStartParams: {}
35+
template:
36+
spec:
37+
containers:
38+
- name: ray-worker
39+
image: rayproject/ray:2.39.0
40+
lifecycle:
41+
preStop:
42+
exec:
43+
command: [ "/bin/sh","-c","ray stop" ]
44+
resources:
45+
limits:
46+
cpu: "1"
47+
requests:
48+
cpu: "200m"

‎kubectl-plugin/test/e2e/testdata/ray-job.interactive-mode.yaml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
apiVersion: ray.io/v1
2+
kind: RayJob
3+
metadata:
4+
name: rayjob-sample
5+
spec:
6+
# The current value is "InteractiveMode", meaning that it will wait for user to submit job and provide the job submission ID
7+
submissionMode: 'InteractiveMode'
8+
runtimeEnvYAML: |
9+
pip:
10+
- emoji==2.14.0
11+
- pyjokes==0.6.0
12+
env_vars:
13+
test_env_var: "first_env_var"
14+
another_env_var: "second_env_var"
15+
16+
rayClusterSpec:
17+
rayVersion: '2.39.0' # should match the Ray version in the image of the containers
18+
headGroupSpec:
19+
rayStartParams:
20+
dashboard-host: '0.0.0.0'
21+
template:
22+
spec:
23+
containers:
24+
- name: ray-head
25+
image: rayproject/ray:2.39.0
26+
ports:
27+
- containerPort: 6379
28+
name: gcs-server
29+
- containerPort: 8265
30+
name: dashboard
31+
- containerPort: 10001
32+
name: client
33+
resources:
34+
limits:
35+
cpu: "1"
36+
requests:
37+
cpu: "200m"
38+
workerGroupSpecs:
39+
- replicas: 1
40+
minReplicas: 1
41+
maxReplicas: 5
42+
groupName: small-group
43+
rayStartParams: {}
44+
template:
45+
spec:
46+
containers:
47+
- name: ray-worker
48+
image: rayproject/ray:2.39.0
49+
lifecycle:
50+
preStop:
51+
exec:
52+
command: [ "/bin/sh","-c","ray stop" ]
53+
resources:
54+
limits:
55+
cpu: "1"
56+
requests:
57+
cpu: "200m"

‎kubectl-plugin/test/e2e/testdata/rayjob-submit-working-dir/entrypoint-python-sample.py‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import ray
2+
import os
3+
import emoji
4+
import pyjokes
5+
6+
ray.init()
7+
8+
@ray.remote
9+
def f():
10+
assert emoji.__version__ == "2.14.0"
11+
assert pyjokes.__version__ == "0.6.0"
12+
13+
first_env_var = os.getenv("test_env_var")
14+
second_env_var = os.getenv("another_env_var")
15+
16+
assert first_env_var == "first_env_var"
17+
assert second_env_var == "second_env_var"
18+
19+
ray.get(f.remote())

‎kubectl-plugin/test/e2e/testdata/rayjob-submit-working-dir/runtime-env-sample.yaml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
pip:
2+
- emoji==2.14.0
3+
- pyjokes==0.6.0
4+
env_vars:
5+
test_env_var: "first_env_var"
6+
another_env_var: "second_env_var"
7+
working_dir: ./testdata/rayjob-submit-working-dir/

0 commit comments

Comments
 (0)