Skip to content

driver/kubernetes: add manifest-patch driver option #3791

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
abhay1999 wants to merge 1 commit into
base: master
Choose a base branch
from
Open

driver/kubernetes: add manifest-patch driver option #3791

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 78 additions & 16 deletions driver/kubernetes/factory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package kubernetes

import (
"context"
"encoding/json"
"os"
"strconv"
"strings"
Expand All @@ -13,6 +14,7 @@ import (
"github.com/docker/buildx/driver/kubernetes/kubeclient"
"github.com/docker/buildx/driver/kubernetes/manifest"
"github.com/docker/buildx/driver/kubernetes/podchooser"
"github.com/itchyny/gojq"
dockerclient "github.com/moby/moby/client"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
Expand Down Expand Up @@ -123,7 +125,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
InitConfig: cfg,
}

deploymentOpt, loadbalance, namespace, defaultLoad, timeout, err := f.processDriverOpts(deploymentName, namespace, cfg)
deploymentOpt, loadbalance, namespace, defaultLoad, timeout, manifestPatch, err := f.processDriverOpts(deploymentName, namespace, cfg)
if nil != err {
return nil, err
}
Expand All @@ -136,6 +138,19 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
return nil, err
}

if manifestPatch != "" {
if d.deployment != nil {
if d.deployment, err = applyManifestPatch(d.deployment, manifestPatch); err != nil {
return nil, errors.Wrap(err, "failed to apply manifest-patch to Deployment")
}
}
if d.statefulSet != nil {
if d.statefulSet, err = applyManifestPatch(d.statefulSet, manifestPatch); err != nil {
return nil, errors.Wrap(err, "failed to apply manifest-patch to StatefulSet")
}
}
}

d.minReplicas = int(deploymentOpt.Replicas)

clients, err := kubeclient.New(restClientConfig, namespace)
Expand Down Expand Up @@ -165,7 +180,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver
return d, nil
}

func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg driver.InitConfig) (*manifest.DeploymentOpt, string, string, bool, time.Duration, error) {
func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg driver.InitConfig) (*manifest.DeploymentOpt, string, string, bool, time.Duration, string, error) {
deploymentOpt := &manifest.DeploymentOpt{
Name: deploymentName,
Image: bkimage.DefaultImage,
Expand All @@ -180,6 +195,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
timeout := defaultTimeout
deploymentOpt.Qemu.Image = bkimage.QemuImage
loadbalance := LoadbalanceSticky
manifestPatch := ""
var err error

for k, v := range cfg.DriverOpts {
Expand All @@ -193,7 +209,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case k == "replicas":
r, err := strconv.ParseInt(v, 10, 32)
if err != nil {
return nil, "", "", false, 0, err
return nil, "", "", false, 0, "", err
}
deploymentOpt.Replicas = int32(r)
case k == "requests.cpu":
Expand All @@ -213,7 +229,7 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case k == "rootless":
deploymentOpt.Rootless, err = strconv.ParseBool(v)
if err != nil {
return nil, "", "", false, 0, err
return nil, "", "", false, 0, "", err
}
if _, isImage := cfg.DriverOpts["image"]; !isImage {
deploymentOpt.Image = bkimage.DefaultRootlessImage
Expand All @@ -225,17 +241,17 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case k == "nodeselector":
deploymentOpt.NodeSelector, err = splitMultiValues(v, ",", "=")
if err != nil {
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse node selector")
return nil, "", "", false, 0, "", errors.Wrap(err, "cannot parse node selector")
}
case k == "annotations":
deploymentOpt.CustomAnnotations, err = splitMultiValues(v, ",", "=")
if err != nil {
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse annotations")
return nil, "", "", false, 0, "", errors.Wrap(err, "cannot parse annotations")
}
case k == "labels":
deploymentOpt.CustomLabels, err = splitMultiValues(v, ",", "=")
if err != nil {
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse labels")
return nil, "", "", false, 0, "", errors.Wrap(err, "cannot parse labels")
}
case k == "tolerations":
ts := strings.Split(v, ";")
Expand All @@ -260,29 +276,34 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case "tolerationSeconds":
c, err := strconv.Atoi(kv[1])
if nil != err {
return nil, "", "", false, 0, err
return nil, "", "", false, 0, "", err
}
c64 := int64(c)
t.TolerationSeconds = &c64
default:
return nil, "", "", false, 0, errors.Errorf("invalid tolaration %q", v)
return nil, "", "", false, 0, "", errors.Errorf("invalid tolaration %q", v)
}
}
}

deploymentOpt.Tolerations = append(deploymentOpt.Tolerations, t)
}
case k == "manifest-patch":
if _, err := gojq.Parse(v); err != nil {
return nil, "", "", false, 0, "", errors.Wrap(err, "invalid manifest-patch jq expression")
}
manifestPatch = v
case k == "loadbalance":
switch v {
case LoadbalanceSticky, LoadbalanceRandom:
loadbalance = v
default:
return nil, "", "", false, 0, errors.Errorf("invalid loadbalance %q", v)
return nil, "", "", false, 0, "", errors.Errorf("invalid loadbalance %q", v)
}
case k == "qemu.install":
deploymentOpt.Qemu.Install, err = strconv.ParseBool(v)
if err != nil {
return nil, "", "", false, 0, err
return nil, "", "", false, 0, "", err
}
case k == "qemu.image":
if v != "" {
Expand All @@ -295,24 +316,65 @@ func (f *factory) processDriverOpts(deploymentName string, namespace string, cfg
case k == "default-load":
defaultLoad, err = strconv.ParseBool(v)
if err != nil {
return nil, "", "", false, 0, err
return nil, "", "", false, 0, "", err
}
case k == "timeout":
timeout, err = time.ParseDuration(v)
if err != nil {
return nil, "", "", false, 0, errors.Wrap(err, "cannot parse timeout")
return nil, "", "", false, 0, "", errors.Wrap(err, "cannot parse timeout")
}
case strings.HasPrefix(k, "env."):
envName := strings.TrimPrefix(k, "env.")
if envName == "" {
return nil, "", "", false, 0, errors.Errorf("invalid env option %q, expecting env.FOO=bar", k)
return nil, "", "", false, 0, "", errors.Errorf("invalid env option %q, expecting env.FOO=bar", k)
}
deploymentOpt.Env = append(deploymentOpt.Env, corev1.EnvVar{Name: envName, Value: v})
default:
return nil, "", "", false, 0, errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
return nil, "", "", false, 0, "", errors.Errorf("invalid driver option %s for driver %s", k, DriverName)
}
}
return deploymentOpt, loadbalance, namespace, defaultLoad, timeout, nil
return deploymentOpt, loadbalance, namespace, defaultLoad, timeout, manifestPatch, nil
}

// applyManifestPatch applies a jq expression to patch a Kubernetes manifest object.
// The expression receives the manifest as input and must produce a single object as output.
func applyManifestPatch[T any](obj *T, patch string) (*T, error) {
query, err := gojq.Parse(patch)
if err != nil {
return nil, errors.Wrap(err, "failed to parse manifest-patch expression")
}
code, err := gojq.Compile(query)
if err != nil {
return nil, errors.Wrap(err, "failed to compile manifest-patch expression")
}

b, err := json.Marshal(obj)
if err != nil {
return nil, err
}
var input interface{}
if err := json.Unmarshal(b, &input); err != nil {
return nil, err
}

iter := code.Run(input)
v, ok := iter.Next()
if !ok {
return nil, errors.New("manifest-patch expression produced no output")
}
if err, ok := v.(error); ok {
return nil, errors.Wrap(err, "manifest-patch expression failed")
}

b, err = json.Marshal(v)
if err != nil {
return nil, err
}
var result T
if err := json.Unmarshal(b, &result); err != nil {
return nil, err
}
return &result, nil
}

func splitMultiValues(in string, itemsep string, kvsep string) (map[string]string, error) {
Expand Down
75 changes: 62 additions & 13 deletions driver/kubernetes/factory_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@ import (
"github.com/docker/buildx/driver"
"github.com/docker/buildx/driver/bkimage"
"github.com/stretchr/testify/require"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/rest"
)

Expand Down Expand Up @@ -55,7 +58,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
"qemu.image": "qemu:latest",
"default-load": "true",
}
r, loadbalance, ns, defaultLoad, timeout, err := f.processDriverOpts(cfg.Name, "test", cfg)
r, loadbalance, ns, defaultLoad, timeout, _, err := f.processDriverOpts(cfg.Name, "test", cfg)

nodeSelectors := map[string]string{
"selector1": "value1",
Expand Down Expand Up @@ -113,7 +116,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
"NoOptions", func(t *testing.T) {
cfg.DriverOpts = map[string]string{}

r, loadbalance, ns, defaultLoad, timeout, err := f.processDriverOpts(cfg.Name, "test", cfg)
r, loadbalance, ns, defaultLoad, timeout, _, err := f.processDriverOpts(cfg.Name, "test", cfg)

require.NoError(t, err)

Expand Down Expand Up @@ -144,7 +147,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
"loadbalance": "sticky",
}

r, loadbalance, ns, defaultLoad, timeout, err := f.processDriverOpts(cfg.Name, "test", cfg)
r, loadbalance, ns, defaultLoad, timeout, _, err := f.processDriverOpts(cfg.Name, "test", cfg)

require.NoError(t, err)

Expand Down Expand Up @@ -173,7 +176,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"replicas": "invalid",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
Expand All @@ -183,7 +186,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"rootless": "invalid",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
Expand All @@ -193,7 +196,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"tolerations": "key=foo,value=bar,invalid=foo2",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
Expand All @@ -203,7 +206,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"tolerations": "key=foo,value=bar,tolerationSeconds=invalid",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
Expand All @@ -213,7 +216,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"annotations": "key,value",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
Expand All @@ -223,7 +226,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"labels": "key=value=foo",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
Expand All @@ -233,7 +236,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"loadbalance": "invalid",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
Expand All @@ -243,7 +246,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"qemu.install": "invalid",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
Expand All @@ -253,7 +256,7 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"invalid": "foo",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
Expand All @@ -263,8 +266,54 @@ func TestFactory_processDriverOpts(t *testing.T) {
cfg.DriverOpts = map[string]string{
"timeout": "invalid",
}
_, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)

t.Run(
"ManifestPatch", func(t *testing.T) {
cfg.DriverOpts = map[string]string{
"manifest-patch": `.metadata.ownerReferences=[{"apiVersion":"actions.github.com/v1alpha1","kind":"EphemeralRunner","name":"runner-xyz","uid":"b636330d-26b7-417a-8464-c2641438feed"}]`,
}
_, _, _, _, _, patch, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.NoError(t, err)
require.NotEmpty(t, patch)
},
)

t.Run(
"InvalidManifestPatch", func(t *testing.T) {
cfg.DriverOpts = map[string]string{
"manifest-patch": "invalid jq [[[",
}
_, _, _, _, _, _, err := f.processDriverOpts(cfg.Name, "test", cfg)
require.Error(t, err)
},
)
}

func TestApplyManifestPatch(t *testing.T) {
t.Run("SetOwnerReferences", func(t *testing.T) {
d := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test",
Namespace: "default",
},
}
patch := `.metadata.ownerReferences=[{"apiVersion":"actions.github.com/v1alpha1","kind":"EphemeralRunner","name":"runner-xyz","uid":"b636330d-26b7-417a-8464-c2641438feed"}]`
result, err := applyManifestPatch(d, patch)
require.NoError(t, err)
require.Len(t, result.OwnerReferences, 1)
require.Equal(t, "actions.github.com/v1alpha1", result.OwnerReferences[0].APIVersion)
require.Equal(t, "EphemeralRunner", result.OwnerReferences[0].Kind)
require.Equal(t, "runner-xyz", result.OwnerReferences[0].Name)
require.Equal(t, types.UID("b636330d-26b7-417a-8464-c2641438feed"), result.OwnerReferences[0].UID)
})

t.Run("InvalidExpression", func(t *testing.T) {
d := &appsv1.Deployment{}
_, err := applyManifestPatch(d, "invalid [[[")
require.Error(t, err)
})
}
Loading
Loading