Skip to content

Commit 8ff18be

Browse files
yrahalyrahal
committed
Add helm unit tests for the service orchestrator resources
1 parent 1162aac commit 8ff18be

4 files changed

Lines changed: 783 additions & 0 deletions

‎tests/unit/helm/api-deployment_test.go‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2875,3 +2875,110 @@ func (s *deploymentApiTemplateTest) TestDeploymentUpdateStrategy() {
28752875
})
28762876
}
28772877
}
2878+
2879+
func (s *deploymentApiTemplateTest) TestServiceOrchestratorWiring() {
2880+
testCases := []struct {
2881+
name string
2882+
values map[string]string
2883+
expected func(podSpec corev1.PodSpec)
2884+
}{
2885+
{
2886+
// Feature off (default): no env, no mounts, no volumes.
2887+
"defaultValues",
2888+
nil,
2889+
func(podSpec corev1.PodSpec) {
2890+
for _, env := range podSpec.Containers[0].Env {
2891+
s.NotEqual("FIFTYONE_BUILTIN_SERVICES_PATH", env.Name)
2892+
}
2893+
for _, volumeMount := range podSpec.Containers[0].VolumeMounts {
2894+
s.NotEqual("builtin-services", volumeMount.Name)
2895+
s.NotEqual("service-pod-template", volumeMount.Name)
2896+
}
2897+
for _, volume := range podSpec.Volumes {
2898+
s.NotEqual("builtin-services", volume.Name)
2899+
s.NotEqual("service-pod-template", volume.Name)
2900+
}
2901+
},
2902+
},
2903+
{
2904+
// Feature on: the env var points at the mounted overrides file,
2905+
// and both configmaps are mounted with their generated names.
2906+
"serviceOrchestratorEnabled",
2907+
map[string]string{
2908+
"serviceOrchestrator.enabled": "true",
2909+
},
2910+
func(podSpec corev1.PodSpec) {
2911+
env := map[string]string{}
2912+
for _, envVar := range podSpec.Containers[0].Env {
2913+
env[envVar.Name] = envVar.Value
2914+
}
2915+
s.Equal(
2916+
"/opt/builtin-services/builtin_services.yaml",
2917+
env["FIFTYONE_BUILTIN_SERVICES_PATH"],
2918+
)
2919+
2920+
mounts := map[string]string{}
2921+
for _, volumeMount := range podSpec.Containers[0].VolumeMounts {
2922+
mounts[volumeMount.Name] = volumeMount.MountPath
2923+
}
2924+
s.Equal("/opt/builtin-services", mounts["builtin-services"])
2925+
s.Equal("/opt/service-pod-template", mounts["service-pod-template"])
2926+
2927+
volumes := map[string]string{}
2928+
for _, volume := range podSpec.Volumes {
2929+
if volume.ConfigMap != nil {
2930+
volumes[volume.Name] = volume.ConfigMap.Name
2931+
}
2932+
}
2933+
s.Equal(
2934+
"fiftyone-test-fiftyone-teams-app-builtin-services",
2935+
volumes["builtin-services"],
2936+
)
2937+
s.Equal(
2938+
"fiftyone-test-fiftyone-teams-app-service-pod-template",
2939+
volumes["service-pod-template"],
2940+
)
2941+
},
2942+
},
2943+
{
2944+
// Per-configmap opt-out: no create and no external name means
2945+
// no mount, no volume, no env for that piece.
2946+
"enabledWithBuiltinServicesConfigMapDisabled",
2947+
map[string]string{
2948+
"serviceOrchestrator.enabled": "true",
2949+
"serviceOrchestrator.builtinServices.configMap.create": "false",
2950+
},
2951+
func(podSpec corev1.PodSpec) {
2952+
for _, env := range podSpec.Containers[0].Env {
2953+
s.NotEqual("FIFTYONE_BUILTIN_SERVICES_PATH", env.Name)
2954+
}
2955+
for _, volumeMount := range podSpec.Containers[0].VolumeMounts {
2956+
s.NotEqual("builtin-services", volumeMount.Name)
2957+
}
2958+
mounts := []string{}
2959+
for _, volumeMount := range podSpec.Containers[0].VolumeMounts {
2960+
mounts = append(mounts, volumeMount.Name)
2961+
}
2962+
s.Contains(mounts, "service-pod-template")
2963+
},
2964+
},
2965+
}
2966+
2967+
for _, testCase := range testCases {
2968+
testCase := testCase
2969+
2970+
s.Run(testCase.name, func() {
2971+
subT := s.T()
2972+
subT.Parallel()
2973+
2974+
options := &helm.Options{SetValues: disableTelemetry(testCase.values)}
2975+
2976+
output := helm.RenderTemplate(subT, options, s.chartPath, s.releaseName, s.templates)
2977+
2978+
var deployment appsv1.Deployment
2979+
helm.UnmarshalK8SYaml(subT, output, &deployment)
2980+
2981+
testCase.expected(deployment.Spec.Template.Spec)
2982+
})
2983+
}
2984+
}

‎tests/unit/helm/seed-orchestrators-job_test.go‎

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,256 @@
1+
//go:build kubeall || helm || unit || unitServiceOrchestrator
2+
// +build kubeall helm unit unitServiceOrchestrator
3+
4+
package unit
5+
6+
import (
7+
"encoding/json"
8+
"fmt"
9+
"path/filepath"
10+
"strings"
11+
"testing"
12+
13+
"github.com/gruntwork-io/terratest/modules/helm"
14+
"github.com/gruntwork-io/terratest/modules/random"
15+
"github.com/stretchr/testify/require"
16+
"github.com/stretchr/testify/suite"
17+
18+
batchv1 "k8s.io/api/batch/v1"
19+
)
20+
21+
type seedOrchestratorsJobTemplateTest struct {
22+
suite.Suite
23+
chartPath string
24+
releaseName string
25+
namespace string
26+
templates []string
27+
}
28+
29+
func TestSeedOrchestratorsJobTemplate(t *testing.T) {
30+
t.Parallel()
31+
32+
helmChartPath, err := filepath.Abs(chartPath)
33+
require.NoError(t, err)
34+
35+
suite.Run(t, &seedOrchestratorsJobTemplateTest{
36+
Suite: suite.Suite{},
37+
chartPath: helmChartPath,
38+
releaseName: "fiftyone-test",
39+
namespace: "fiftyone-" + strings.ToLower(random.UniqueId()),
40+
templates: []string{"templates/seed-orchestrators-job.yaml"},
41+
})
42+
}
43+
44+
func seedJobEnv(job batchv1.Job) map[string]string {
45+
env := map[string]string{}
46+
for _, envVar := range job.Spec.Template.Spec.Containers[0].Env {
47+
env[envVar.Name] = envVar.Value
48+
}
49+
return env
50+
}
51+
52+
func (s *seedOrchestratorsJobTemplateTest) TestGating() {
53+
testCases := []struct {
54+
name string
55+
values map[string]string
56+
expected string // empty => the template must not render
57+
}{
58+
{
59+
"defaultValues",
60+
nil,
61+
"",
62+
},
63+
{
64+
// Seeding is independent of serviceOrchestrator: it covers any
65+
// orchestrator environment, so its own flag alone enables it.
66+
"enabledAloneWithoutServiceOrchestrator",
67+
map[string]string{
68+
"seedOrchestrators.enabled": "true",
69+
},
70+
fmt.Sprintf("%s-fiftyone-teams-app-seed-orchestrators", s.releaseName),
71+
},
72+
}
73+
74+
for _, testCase := range testCases {
75+
testCase := testCase
76+
77+
s.Run(testCase.name, func() {
78+
subT := s.T()
79+
subT.Parallel()
80+
81+
options := &helm.Options{SetValues: disableTelemetry(testCase.values)}
82+
83+
if testCase.expected == "" {
84+
_, err := helm.RenderTemplateE(subT, options, s.chartPath, s.releaseName, s.templates)
85+
s.ErrorContains(err, "could not find template templates/seed-orchestrators-job.yaml in chart")
86+
} else {
87+
output := helm.RenderTemplate(subT, options, s.chartPath, s.releaseName, s.templates)
88+
89+
var job batchv1.Job
90+
helm.UnmarshalK8SYaml(subT, output, &job)
91+
92+
s.Equal(testCase.expected, job.ObjectMeta.Name, "Name should be set")
93+
}
94+
})
95+
}
96+
}
97+
98+
func (s *seedOrchestratorsJobTemplateTest) TestHelmHooks() {
99+
options := &helm.Options{
100+
SetValues: disableTelemetry(map[string]string{
101+
"seedOrchestrators.enabled": "true",
102+
}),
103+
}
104+
105+
output := helm.RenderTemplate(s.T(), options, s.chartPath, s.releaseName, s.templates)
106+
107+
var job batchv1.Job
108+
helm.UnmarshalK8SYaml(s.T(), output, &job)
109+
110+
s.Equal(
111+
"post-install,post-upgrade",
112+
job.ObjectMeta.Annotations["helm.sh/hook"],
113+
)
114+
s.Equal(
115+
"before-hook-creation,hook-succeeded",
116+
job.ObjectMeta.Annotations["helm.sh/hook-delete-policy"],
117+
)
118+
}
119+
120+
func (s *seedOrchestratorsJobTemplateTest) TestOrchestratorsEnv() {
121+
options := &helm.Options{
122+
SetValues: disableTelemetry(map[string]string{
123+
"seedOrchestrators.enabled": "true",
124+
}),
125+
SetJsonValues: map[string]string{
126+
"seedOrchestrators.orchestrators": `[
127+
{
128+
"instance_id": "service-orchestrator-cpu",
129+
"description": "Service orchestrator (CPU)",
130+
"environment": "kubernetes-service",
131+
"config": {
132+
"namespace": "svc-ns",
133+
"execution_tmpl_uri": "/opt/service-pod-template/pod.yaml.j2",
134+
"resource_requests": {"cpu": "2", "memory": "8Gi"}
135+
},
136+
"secrets": {"kube_config": ""},
137+
"available_operators": ["@voxel51/operators/run_service"]
138+
},
139+
{
140+
"instance_id": "kubernetes-cpu",
141+
"description": "Kubernetes CPU",
142+
"environment": "kubernetes",
143+
"config": {
144+
"image": "",
145+
"execution_tmpl_uri": "/tmp/do-targets/teamsDoK8sCpu.yaml",
146+
"namespace": "svc-ns"
147+
},
148+
"secrets": {"kube_config": ""}
149+
}
150+
]`,
151+
},
152+
}
153+
154+
output := helm.RenderTemplate(s.T(), options, s.chartPath, s.releaseName, s.templates)
155+
156+
var job batchv1.Job
157+
helm.UnmarshalK8SYaml(s.T(), output, &job)
158+
159+
env := seedJobEnv(job)
160+
161+
// The orchestrator list round-trips as JSON the seeding script consumes
162+
var orchestrators []map[string]interface{}
163+
err := json.Unmarshal([]byte(env["ORCHESTRATORS"]), &orchestrators)
164+
s.NoError(err, "ORCHESTRATORS should be valid JSON")
165+
s.Require().Len(orchestrators, 2)
166+
s.Equal("service-orchestrator-cpu", orchestrators[0]["instance_id"])
167+
config := orchestrators[0]["config"].(map[string]interface{})
168+
s.Equal("/opt/service-pod-template/pod.yaml.j2", config["execution_tmpl_uri"])
169+
// The empty-string image sentinel survives for the script to fill from
170+
// DEFAULT_WORKER_IMAGE
171+
kubernetesConfig := orchestrators[1]["config"].(map[string]interface{})
172+
s.Equal("", kubernetesConfig["image"])
173+
174+
// DEFAULT_WORKER_IMAGE defaults to the delegated-operator worker image
175+
// at the chart's appVersion
176+
cInfo, err := chartInfo(s.T(), s.chartPath)
177+
s.NoError(err)
178+
appVersion, exists := cInfo["appVersion"]
179+
s.True(exists)
180+
s.Equal(
181+
fmt.Sprintf("voxel51/fiftyone-teams-cv-full:%s", appVersion),
182+
env["DEFAULT_WORKER_IMAGE"],
183+
)
184+
}
185+
186+
func (s *seedOrchestratorsJobTemplateTest) TestImageFollowsDelegatedOperatorTemplate() {
187+
options := &helm.Options{
188+
SetValues: disableTelemetry(map[string]string{
189+
"seedOrchestrators.enabled": "true",
190+
"delegatedOperatorJobTemplates.template.image.tag": "v9.9.9",
191+
}),
192+
}
193+
194+
output := helm.RenderTemplate(s.T(), options, s.chartPath, s.releaseName, s.templates)
195+
196+
var job batchv1.Job
197+
helm.UnmarshalK8SYaml(s.T(), output, &job)
198+
199+
container := job.Spec.Template.Spec.Containers[0]
200+
s.Equal("voxel51/fiftyone-teams-cv-full:v9.9.9", container.Image)
201+
s.Equal(
202+
"voxel51/fiftyone-teams-cv-full:v9.9.9",
203+
seedJobEnv(job)["DEFAULT_WORKER_IMAGE"],
204+
)
205+
}
206+
207+
func (s *seedOrchestratorsJobTemplateTest) TestImageOverride() {
208+
options := &helm.Options{
209+
SetValues: disableTelemetry(map[string]string{
210+
"seedOrchestrators.enabled": "true",
211+
"seedOrchestrators.image.repository": "custom/seeder",
212+
"seedOrchestrators.image.tag": "v1.2.3",
213+
}),
214+
}
215+
216+
output := helm.RenderTemplate(s.T(), options, s.chartPath, s.releaseName, s.templates)
217+
218+
var job batchv1.Job
219+
helm.UnmarshalK8SYaml(s.T(), output, &job)
220+
221+
s.Equal("custom/seeder:v1.2.3", job.Spec.Template.Spec.Containers[0].Image)
222+
}
223+
224+
func (s *seedOrchestratorsJobTemplateTest) TestPodSpec() {
225+
options := &helm.Options{
226+
SetValues: disableTelemetry(map[string]string{
227+
"seedOrchestrators.enabled": "true",
228+
}),
229+
}
230+
231+
output := helm.RenderTemplate(s.T(), options, s.chartPath, s.releaseName, s.templates)
232+
233+
var job batchv1.Job
234+
helm.UnmarshalK8SYaml(s.T(), output, &job)
235+
236+
s.Equal(int32(2), *job.Spec.BackoffLimit)
237+
s.Equal(int32(600), *job.Spec.TTLSecondsAfterFinished)
238+
239+
podSpec := job.Spec.Template.Spec
240+
241+
// Non-root, matching the delegated-operator defaults for the same image
242+
s.Require().NotNil(podSpec.SecurityContext)
243+
s.True(*podSpec.SecurityContext.RunAsNonRoot)
244+
s.Equal(int64(1000), *podSpec.SecurityContext.RunAsUser)
245+
246+
container := podSpec.Containers[0]
247+
s.Require().NotNil(container.SecurityContext)
248+
s.True(*container.SecurityContext.ReadOnlyRootFilesystem)
249+
s.False(*container.SecurityContext.AllowPrivilegeEscalation)
250+
251+
// The seeding script arrives inline via .Files.Get
252+
s.Equal([]string{"python", "-c"}, container.Command)
253+
s.Require().Len(container.Args, 1)
254+
s.Contains(container.Args[0], "ORCHESTRATORS")
255+
s.Contains(container.Args[0], "DEFAULT_WORKER_IMAGE")
256+
}

0 commit comments

Comments
 (0)