Skip to content

Commit b4c8746

Browse files
abhay1999abhay1999
committed
driver/kubernetes: add manifest-patch driver option
Add a generic "manifest-patch" driver option that accepts a jq expression, applied to the generated Deployment or StatefulSet manifest after creation. This allows users to set arbitrary manifest fields that are not exposed as dedicated driver options, such as ownerReferences: --driver-opt manifest-patch='.metadata.ownerReferences=[{"apiVersion":"actions.github.com/v1alpha1","kind":"EphemeralRunner","name":"runner-xyz","uid":"..."}]' The jq expression is evaluated using github.com/itchyny/gojq, a pure-Go jq implementation. The expression is validated at driver initialization time, and an error is returned if it is invalid or produces no output. Fixes: #2626 Signed-off-by: abhay1999 <abhaychaurasiya19@gmail.com>
1 parent 43fe71e commit b4c8746

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

58 files changed

+12699
-52
lines changed

‎driver/kubernetes/factory.go‎

Lines changed: 78 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package kubernetes
22

33
import (
44
"context"
5+
"encoding/json"
56
"os"
67
"strconv"
78
"strings"
@@ -13,6 +14,7 @@ import (
1314
"github.com/docker/buildx/driver/kubernetes/kubeclient"
1415
"github.com/docker/buildx/driver/kubernetes/manifest"
1516
"github.com/docker/buildx/driver/kubernetes/podchooser"
17+
"github.com/itchyny/gojq"
1618
dockerclient "github.com/moby/moby/client"
1719
"github.com/pkg/errors"
1820
"github.com/sirupsen/logrus"
@@ -123,7 +125,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
123125
InitConfig: cfg,
124126
}
125127

126-
deploymentOpt, loadbalance, namespace, defaultLoad, timeout, err := f.processDriverOpts(deploymentName, namespace, cfg)
128+
deploymentOpt, loadbalance, namespace, defaultLoad, timeout, manifestPatch, err := f.processDriverOpts(deploymentName, namespace, cfg)
127129
if nil != err {
128130
return nil, err
129131
}
@@ -136,6 +138,19 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
136138
return nil, err
137139
}
138140

141+
if manifestPatch != "" {
142+
if d.deployment != nil {
143+
if d.deployment, err = applyManifestPatch(d.deployment, manifestPatch); err != nil {
144+
return nil, errors.Wrap(err, "failed to apply manifest-patch to Deployment")
145+
}
146+
}
147+
if d.statefulSet != nil {
148+
if d.statefulSet, err = applyManifestPatch(d.statefulSet, manifestPatch); err != nil {
149+
return nil, errors.Wrap(err, "failed to apply manifest-patch to StatefulSet")
150+
}
151+
}
152+
}
153+
139154
d.minReplicas = int(deploymentOpt.Replicas)
140155

141156
clients, err := kubeclient.New(restClientConfig, namespace)
@@ -165,7 +180,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
165180
return d, nil
166181
}
167182

168-
func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg driver.InitConfig) (*manifest.DeploymentOpt, string, string, bool, time.Duration, error) {
183+
func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg driver.InitConfig) (*manifest.DeploymentOpt, string, string, bool, time.Duration, string, error) {
169184
deploymentOpt := &manifest.DeploymentOpt{
170185
Name: deploymentName,
171186
Image: bkimage.DefaultImage,
@@ -180,6 +195,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
180195
timeout := defaultTimeout
181196
deploymentOpt.Qemu.Image = bkimage.QemuImage
182197
loadbalance := LoadbalanceSticky
198+
manifestPatch := ""
183199
var err error
184200

185201
for k, v := range cfg.DriverOpts {
@@ -193,7 +209,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
193209
case k == "replicas":
194210
r, err := strconv.ParseInt(v, 10, 32)
195211
if err != nil {
196-
return nil, "", "", false, 0, err
212+
return nil, "", "", false, 0, "", err
197213
}
198214
deploymentOpt.Replicas = int32(r)
199215
case k == "requests.cpu":
@@ -213,7 +229,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
213229
case k == "rootless":
214230
deploymentOpt.Rootless, err = strconv.ParseBool(v)
215231
if err != nil {
216-
return nil, "", "", false, 0, err
232+
return nil, "", "", false, 0, "", err
217233
}
218234
if _, isImage := cfg.DriverOpts["image"]; !isImage {
219235
deploymentOpt.Image = bkimage.DefaultRootlessImage
@@ -225,17 +241,17 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
225241
case k == "nodeselector":
226242
deploymentOpt.NodeSelector, err = splitMultiValues(v, ",", "=")
227243
if err != nil {
228-
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse node selector")
244+
return nil, "", "", false, 0, "", errors.Wrap(err, "cannot parse node selector")
229245
}
230246
case k == "annotations":
231247
deploymentOpt.CustomAnnotations, err = splitMultiValues(v, ",", "=")
232248
if err != nil {
233-
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse annotations")
249+
return nil, "", "", false, 0, "", errors.Wrap(err, "cannot parse annotations")
234250
}
235251
case k == "labels":
236252
deploymentOpt.CustomLabels, err = splitMultiValues(v, ",", "=")
237253
if err != nil {
238-
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse labels")
254+
return nil, "", "", false, 0, "", errors.Wrap(err, "cannot parse labels")
239255
}
240256
case k == "tolerations":
241257
ts := strings.Split(v, ";")
@@ -260,29 +276,34 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
260276
case "tolerationSeconds":
261277
c, err := strconv.Atoi(kv[1])
262278
if nil != err {
263-
return nil, "", "", false, 0, err
279+
return nil, "", "", false, 0, "", err
264280
}
265281
c64 := int64(c)
266282
t.TolerationSeconds = &c64
267283
default:
268-
return nil, "", "", false, 0, errors.Errorf("invalid tolaration %q", v)
284+
return nil, "", "", false, 0, "", errors.Errorf("invalid tolaration %q", v)
269285
}
270286
}
271287
}
272288

273289
deploymentOpt.Tolerations = append(deploymentOpt.Tolerations, t)
274290
}
291+
case k == "manifest-patch":
292+
if _, err := gojq.Parse(v); err != nil {
293+
return nil, "", "", false, 0, "", errors.Wrap(err, "invalid manifest-patch jq expression")
294+
}
295+
manifestPatch = v
275296
case k == "loadbalance":
276297
switch v {
277298
case LoadbalanceSticky, LoadbalanceRandom:
278299
loadbalance = v
279300
default:
280-
return nil, "", "", false, 0, errors.Errorf("invalid loadbalance %q", v)
301+
return nil, "", "", false, 0, "", errors.Errorf("invalid loadbalance %q", v)
281302
}
282303
case k == "qemu.install":
283304
deploymentOpt.Qemu.Install, err = strconv.ParseBool(v)
284305
if err != nil {
285-
return nil, "", "", false, 0, err
306+
return nil, "", "", false, 0, "", err
286307
}
287308
case k == "qemu.image":
288309
if v != "" {
@@ -295,24 +316,65 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
295316
case k == "default-load":
296317
defaultLoad, err = strconv.ParseBool(v)
297318
if err != nil {
298-
return nil, "", "", false, 0, err
319+
return nil, "", "", false, 0, "", err
299320
}
300321
case k == "timeout":
301322
timeout, err = time.ParseDuration(v)
302323
if err != nil {
303-
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse timeout")
324+
return nil, "", "", false, 0, "", errors.Wrap(err, "cannot parse timeout")
304325
}
305326
case strings.HasPrefix(k, "env."):
306327
envName := strings.TrimPrefix(k, "env.")
307328
if envName == "" {
308-
return nil, "", "", false, 0, errors.Errorf("invalid env option %q, expecting env.FOO=bar", k)
329+
return nil, "", "", false, 0, "", errors.Errorf("invalid env option %q, expecting env.FOO=bar", k)
309330
}
310331
deploymentOpt.Env = append(deploymentOpt.Env, corev1.EnvVar{Name: envName, Value: v})
311332
default:
312-
return nil, "", "", false, 0, errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
333+
return nil, "", "", false, 0, "", errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
313334
}
314335
}
315-
return deploymentOpt, loadbalance, namespace, defaultLoad, timeout, nil
336+
return deploymentOpt, loadbalance, namespace, defaultLoad, timeout, manifestPatch, nil
337+
}
338+
339+
// applyManifestPatch applies a jq expression to patch a Kubernetes manifest object.
340+
// The expression receives the manifest as input and must produce a single object as output.
341+
func applyManifestPatch[T any](obj *T, patch string) (*T, error) {
342+
query, err := gojq.Parse(patch)
343+
if err != nil {
344+
return nil, errors.Wrap(err, "failed to parse manifest-patch expression")
345+
}
346+
code, err := gojq.Compile(query)
347+
if err != nil {
348+
return nil, errors.Wrap(err, "failed to compile manifest-patch expression")
349+
}
350+
351+
b, err := json.Marshal(obj)
352+
if err != nil {
353+
return nil, err
354+
}
355+
var input interface{}
356+
if err := json.Unmarshal(b, &input); err != nil {
357+
return nil, err
358+
}
359+
360+
iter := code.Run(input)
361+
v, ok := iter.Next()
362+
if !ok {
363+
return nil, errors.New("manifest-patch expression produced no output")
364+
}
365+
if err, ok := v.(error); ok {
366+
return nil, errors.Wrap(err, "manifest-patch expression failed")
367+
}
368+
369+
b, err = json.Marshal(v)
370+
if err != nil {
371+
return nil, err
372+
}
373+
var result T
374+
if err := json.Unmarshal(b, &result); err != nil {
375+
return nil, err
376+
}
377+
return &result, nil
316378
}
317379

318380
func splitMultiValues(in string, itemsep string, kvsep string) (map[string]string, error) {

‎driver/kubernetes/factory_test.go‎

Lines changed: 62 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,10 @@ import (
77
"github.com/docker/buildx/driver"
88
"github.com/docker/buildx/driver/bkimage"
99
"github.com/stretchr/testify/require"
10+
appsv1 "k8s.io/api/apps/v1"
1011
v1 "k8s.io/api/core/v1"
12+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
13+
"k8s.io/apimachinery/pkg/types"
1114
"k8s.io/client-go/rest"
1215
)
1316

@@ -55,7 +58,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
5558
"qemu.image": "qemu:latest",
5659
"default-load": "true",
5760
}
58-
r, loadbalance, ns, defaultLoad, timeout, err := f.processDriverOpts(cfg.Name, "test", cfg)
61+
r, loadbalance, ns, defaultLoad, timeout, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
5962

6063
nodeSelectors := map[string]string{
6164
"selector1": "value1",
@@ -113,7 +116,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
113116
"NoOptions", func(t *testing.T) {
114117
cfg.DriverOpts = map[string]string{}
115118

116-
r, loadbalance, ns, defaultLoad, timeout, err := f.processDriverOpts(cfg.Name, "test", cfg)
119+
r, loadbalance, ns, defaultLoad, timeout, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
117120

118121
require.NoError(t, err)
119122

@@ -144,7 +147,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
144147
"loadbalance": "sticky",
145148
}
146149

147-
r, loadbalance, ns, defaultLoad, timeout, err := f.processDriverOpts(cfg.Name, "test", cfg)
150+
r, loadbalance, ns, defaultLoad, timeout, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
148151

149152
require.NoError(t, err)
150153

@@ -173,7 +176,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
173176
cfg.DriverOpts = map[string]string{
174177
"replicas": "invalid",
175178
}
176-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
179+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
177180
require.Error(t, err)
178181
},
179182
)
@@ -183,7 +186,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
183186
cfg.DriverOpts = map[string]string{
184187
"rootless": "invalid",
185188
}
186-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
189+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
187190
require.Error(t, err)
188191
},
189192
)
@@ -193,7 +196,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
193196
cfg.DriverOpts = map[string]string{
194197
"tolerations": "key=foo,value=bar,invalid=foo2",
195198
}
196-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
199+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
197200
require.Error(t, err)
198201
},
199202
)
@@ -203,7 +206,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
203206
cfg.DriverOpts = map[string]string{
204207
"tolerations": "key=foo,value=bar,tolerationSeconds=invalid",
205208
}
206-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
209+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
207210
require.Error(t, err)
208211
},
209212
)
@@ -213,7 +216,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
213216
cfg.DriverOpts = map[string]string{
214217
"annotations": "key,value",
215218
}
216-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
219+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
217220
require.Error(t, err)
218221
},
219222
)
@@ -223,7 +226,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
223226
cfg.DriverOpts = map[string]string{
224227
"labels": "key=value=foo",
225228
}
226-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
229+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
227230
require.Error(t, err)
228231
},
229232
)
@@ -233,7 +236,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
233236
cfg.DriverOpts = map[string]string{
234237
"loadbalance": "invalid",
235238
}
236-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
239+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
237240
require.Error(t, err)
238241
},
239242
)
@@ -243,7 +246,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
243246
cfg.DriverOpts = map[string]string{
244247
"qemu.install": "invalid",
245248
}
246-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
249+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
247250
require.Error(t, err)
248251
},
249252
)
@@ -253,7 +256,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
253256
cfg.DriverOpts = map[string]string{
254257
"invalid": "foo",
255258
}
256-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
259+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
257260
require.Error(t, err)
258261
},
259262
)
@@ -263,8 +266,54 @@ func TestFactory_processDriverOpts(t *testing.T) {
263266
cfg.DriverOpts = map[string]string{
264267
"timeout": "invalid",
265268
}
266-
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
269+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
267270
require.Error(t, err)
268271
},
269272
)
273+
274+
t.Run(
275+
"ManifestPatch", func(t *testing.T) {
276+
cfg.DriverOpts = map[string]string{
277+
"manifest-patch": `.metadata.ownerReferences=[{"apiVersion":"actions.github.com/v1alpha1","kind":"EphemeralRunner","name":"runner-xyz","uid":"b636330d-26b7-417a-8464-c2641438feed"}]`,
278+
}
279+
_, _, _, _, _, patch, err := f.processDriverOpts(cfg.Name, "test", cfg)
280+
require.NoError(t, err)
281+
require.NotEmpty(t, patch)
282+
},
283+
)
284+
285+
t.Run(
286+
"InvalidManifestPatch", func(t *testing.T) {
287+
cfg.DriverOpts = map[string]string{
288+
"manifest-patch": "invalid jq [[[",
289+
}
290+
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
291+
require.Error(t, err)
292+
},
293+
)
294+
}
295+
296+
func TestApplyManifestPatch(t *testing.T) {
297+
t.Run("SetOwnerReferences", func(t *testing.T) {
298+
d := &appsv1.Deployment{
299+
ObjectMeta: metav1.ObjectMeta{
300+
Name: "test",
301+
Namespace: "default",
302+
},
303+
}
304+
patch := `.metadata.ownerReferences=[{"apiVersion":"actions.github.com/v1alpha1","kind":"EphemeralRunner","name":"runner-xyz","uid":"b636330d-26b7-417a-8464-c2641438feed"}]`
305+
result, err := applyManifestPatch(d, patch)
306+
require.NoError(t, err)
307+
require.Len(t, result.OwnerReferences, 1)
308+
require.Equal(t, "actions.github.com/v1alpha1", result.OwnerReferences[0].APIVersion)
309+
require.Equal(t, "EphemeralRunner", result.OwnerReferences[0].Kind)
310+
require.Equal(t, "runner-xyz", result.OwnerReferences[0].Name)
311+
require.Equal(t, types.UID("b636330d-26b7-417a-8464-c2641438feed"), result.OwnerReferences[0].UID)
312+
})
313+
314+
t.Run("InvalidExpression", func(t *testing.T) {
315+
d := &appsv1.Deployment{}
316+
_, err := applyManifestPatch(d, "invalid [[[")
317+
require.Error(t, err)
318+
})
270319
}

0 commit comments

Comments
 (0)