Skip to content

Commit 97d910e

Browse files
sheidkampdavidjumanijenshuarianaw66
authored
1.18.x mtls for xds in gg2 (#10589)
Co-authored-by: David Jumani <[email protected]> Co-authored-by: Jenny Shu <[email protected]> Co-authored-by: Ariana W. <[email protected]>
1 parent f7fd87c commit 97d910e

25 files changed

+717
-94
lines changed

.github/workflows/pr-kubernetes-tests.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,10 @@ jobs:
8585
go-test-args: '-v -timeout=30m'
8686
go-test-run-regex: '^TestDiscoveryWatchlabels$$|^TestK8sGatewayNoValidation$$|^TestHelm$$|^TestHelmSettings$$|^TestK8sGatewayAws$$|^TestK8sGateway$$/^HTTPRouteServices$$|^TestK8sGateway$$/^TCPRouteServices$$|^TestZeroDowntimeRollout$$'
8787

88-
# Dec 4, 2024: 13 minutes
88+
# Dec 4, 2024: 16 minutes
8989
- cluster-name: 'cluster-seven'
9090
go-test-args: '-v -timeout=25m'
91-
go-test-run-regex: '^TestK8sGateway$$/^CRDCategories$$|^TestK8sGateway$$/^Metrics$$|^TestGloomtlsGatewayEdgeGateway$$|^TestWatchNamespaceSelector$$'
91+
go-test-run-regex: '^TestK8sGateway$$/^CRDCategories$$|^TestK8sGateway$$/^Metrics$$|^TestGloomtlsGatewayEdgeGateway$$|^TestGloomtlsGatewayK8sGateway$$|^TestWatchNamespaceSelector$$'
9292

9393
# In our PR tests, we run the suite of tests using the upper ends of versions that we claim to support
9494
# The versions should mirror: https://docs.solo.io/gloo-edge/latest/reference/support/
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
changelog:
2+
- type: FIX
3+
issueLink: https://github.com/solo-io/solo-projects/issues/6210
4+
resolvesIssue: false
5+
description: >-
6+
Add support for xDS over mTLS for communication between the Gloo pod and the Kubernetes Gateway proxies. This can be enabled by setting the 'global.glooMtls.enabled' helm value to true.

install/helm/gloo/generate/values.go

-1
Original file line numberDiff line numberDiff line change
@@ -340,7 +340,6 @@ type GatewayParameters struct {
340340
AIExtension *GatewayParamsAIExtension `json:"aiExtension,omitempty" desc:"Config used to manage the Gloo Gateway AI extension."`
341341
FloatingUserId *bool `json:"floatingUserId,omitempty" desc:"If true, allows the cluster to dynamically assign a user ID for the processes running in the container. Default is false."`
342342
PodTemplate *GatewayParamsPodTemplate `json:"podTemplate,omitempty" desc:"The template used to generate the gatewayParams pod"`
343-
// TODO(npolshak): Add support for GlooMtls
344343
}
345344

346345
// GatewayProxyPodTemplate contains the Helm API available to configure the PodTemplate on the gateway Deployment

install/helm/gloo/templates/1-gloo-deployment.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -273,6 +273,10 @@ spec:
273273
- name: HEADER_SECRET_REF_NS_MATCHES_US
274274
value: "true"
275275
{{- end}}
276+
{{- if .Values.global.glooMtls.enabled }}
277+
- name: GLOO_MTLS_SDS_ENABLED
278+
value: "true"
279+
{{- end }}
276280
{{- if not .Values.global.glooMtls.enabled }}
277281
readinessProbe:
278282
tcpSocket:

install/helm/gloo/templates/44-rbac.yaml

+6-1
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,14 @@ rules:
2222
- services
2323
- pods
2424
- nodes
25-
- secrets
2625
- namespaces
2726
verbs: ["get", "list", "watch"]
27+
- apiGroups:
28+
- ""
29+
resources:
30+
- secrets
31+
{{/* This is needed as the gateway deployer would need to create / patch the mtls certs if enabled */}}
32+
verbs: ["get", "list", "watch", "create", "patch"]
2833
- apiGroups:
2934
- "discovery.k8s.io"
3035
resources:

pkg/utils/kubeutils/kubectl/cli.go

+6
Original file line numberDiff line numberDiff line change
@@ -287,6 +287,12 @@ func (c *Cli) ExecuteOn(ctx context.Context, kubeContext string, args ...string)
287287
return c.Execute(ctx, args...)
288288
}
289289

290+
// Get executes a `kubectl get` command and returns the contents of stdout, stderr, and any error that occurred while running the command
291+
func (c *Cli) Get(ctx context.Context, args ...string) (string, string, error) {
292+
args = append([]string{"get"}, args...)
293+
return c.Execute(ctx, args...)
294+
}
295+
290296
func (c *Cli) Execute(ctx context.Context, args ...string) (string, string, error) {
291297
if c.kubeContext != "" {
292298
if !slices.Contains(args, "--context") {

projects/gateway2/controller/controller.go

+29-1
Original file line numberDiff line numberDiff line change
@@ -188,6 +188,11 @@ func (c *controllerBuilder) addHttpLisOptIndexes(ctx context.Context) error {
188188
})
189189
}
190190

191+
func (c *controllerBuilder) shouldWatchSecrets() bool {
192+
// watch for secrets if mtls is enabled
193+
return c.cfg.ControlPlane.GlooMtlsEnabled
194+
}
195+
191196
func (c *controllerBuilder) watchGw(ctx context.Context) error {
192197
// setup a deployer
193198
log := log.FromContext(ctx)
@@ -224,8 +229,31 @@ func (c *controllerBuilder) watchGw(ctx context.Context) error {
224229
),
225230
))
226231

227-
// watch for changes in GatewayParameters
228232
cli := c.cfg.Mgr.GetClient()
233+
234+
// watch for secrets if needed
235+
if c.shouldWatchSecrets() {
236+
buildr.Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(
237+
func(ctx context.Context, obj client.Object) []reconcile.Request {
238+
var reqs []reconcile.Request
239+
if obj.GetName() == wellknown.GlooMtlsCertName && obj.GetNamespace() == c.cfg.ControlPlane.Namespace {
240+
var gwList apiv1.GatewayList
241+
err := cli.List(ctx, &gwList, client.InNamespace(corev1.NamespaceAll))
242+
if err != nil {
243+
log.Error(err, "could not list Gateways", "namespace", corev1.NamespaceAll)
244+
return reqs
245+
}
246+
247+
for _, gw := range gwList.Items {
248+
reqs = append(reqs, reconcile.Request{NamespacedName: client.ObjectKey{Namespace: gw.Namespace, Name: gw.Name}})
249+
}
250+
return reqs
251+
}
252+
return reqs
253+
}))
254+
}
255+
256+
// watch for changes in GatewayParameters
229257
buildr.Watches(&v1alpha1.GatewayParameters{}, handler.EnqueueRequestsFromMapFunc(
230258
func(ctx context.Context, obj client.Object) []reconcile.Request {
231259
gwpName := obj.GetName()

projects/gateway2/controller/start.go

+8-3
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"sigs.k8s.io/controller-runtime/pkg/config"
99

1010
glooschemes "github.com/solo-io/gloo/pkg/schemes"
11+
"github.com/solo-io/gloo/pkg/utils/namespaces"
1112
"github.com/solo-io/go-utils/contextutils"
1213

1314
"k8s.io/apimachinery/pkg/util/sets"
@@ -77,6 +78,8 @@ type StartConfig struct {
7778
InitialSettings *glookubev1.Settings
7879
Settings krt.Singleton[glookubev1.Settings]
7980

81+
GlooMtlsEnabled bool
82+
8083
Debugger *krt.DebugHandler
8184
}
8285

@@ -221,7 +224,7 @@ func (c *ControllerBuilder) Start(ctx context.Context) error {
221224
return ctx.Err()
222225
}
223226

224-
logger.Infow("got xds address for deployer", uzap.String("xds_host", xdsHost), uzap.Int32("xds_port", xdsPort))
227+
logger.Infow("got xds address for deployer", uzap.String("xds_host", xdsHost), uzap.Int32("xds_port", xdsPort), uzap.Any("glooMtlsEnabled", c.cfg.GlooMtlsEnabled))
225228

226229
integrationEnabled := c.cfg.InitialSettings.Spec.GetGloo().GetIstioOptions().GetEnableIntegration().GetValue()
227230

@@ -255,8 +258,10 @@ func (c *ControllerBuilder) Start(ctx context.Context) error {
255258
ControllerName: wellknown.GatewayControllerName,
256259
AutoProvision: AutoProvision,
257260
ControlPlane: deployer.ControlPlaneInfo{
258-
XdsHost: xdsHost,
259-
XdsPort: xdsPort,
261+
XdsHost: xdsHost,
262+
XdsPort: xdsPort,
263+
GlooMtlsEnabled: c.cfg.GlooMtlsEnabled,
264+
Namespace: namespaces.GetPodNamespace(),
260265
},
261266
// TODO pass in the settings so that the deloyer can register to it for changes.
262267
IstioIntegrationEnabled: integrationEnabled,

projects/gateway2/deployer/deployer.go

+74-4
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2626
"k8s.io/apimachinery/pkg/runtime"
2727
"k8s.io/apimachinery/pkg/runtime/schema"
28+
"k8s.io/apimachinery/pkg/types"
2829
"k8s.io/apimachinery/pkg/util/yaml"
2930
"k8s.io/utils/ptr"
3031
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -53,6 +54,11 @@ type Deployer struct {
5354
type ControlPlaneInfo struct {
5455
XdsHost string
5556
XdsPort int32
57+
// The data in this struct is static, so is a good place to keep track of if mtls is enabled
58+
// and a bad place to store the actual mtls secret data
59+
GlooMtlsEnabled bool
60+
// We could lookup the pod namespace from the env, but it's cleaner to pass it in
61+
Namespace string
5662
}
5763

5864
type AwsInfo struct {
@@ -103,17 +109,21 @@ func (d *Deployer) GetGvksToWatch(ctx context.Context) ([]schema.GroupVersionKin
103109
// as we only care about the GVKs of the rendered resources)
104110
// - the minimal values that render all the proxy resources (HPA is not included because it's not
105111
// fully integrated/working at the moment)
112+
// - a flag to indicate whether mtls is enabled, so we can render the secret if needed
106113
//
107114
// Note: another option is to hardcode the GVKs here, but rendering the helm chart is a
108115
// _slightly_ more dynamic way of getting the GVKs. It isn't a perfect solution since if
109116
// we add more resources to the helm chart that are gated by a flag, we may forget to
110117
// update the values here to enable them.
118+
// Currently the only resource that is gated by a flag is the mtls secret.
119+
111120
emptyGw := &api.Gateway{
112121
ObjectMeta: metav1.ObjectMeta{
113122
Name: "default",
114123
Namespace: "default",
115124
},
116125
}
126+
117127
// TODO(Law): these must be set explicitly as we don't have defaults for them
118128
// and the internal template isn't robust enough.
119129
// This should be empty eventually -- the template must be resilient against nil-pointers
@@ -124,13 +134,20 @@ func (d *Deployer) GetGvksToWatch(ctx context.Context) ([]schema.GroupVersionKin
124134
"enabled": false,
125135
},
126136
"image": map[string]any{},
137+
// Render the secret based on the mtls flag so we can watch it.
138+
// This is an exception to the "TODO" above as this is not protection against nil-pointers,
139+
// it is determining which resources to render based on ControlPlane configuration.
140+
"glooMtls": map[string]any{
141+
"renderSecret": d.inputs.ControlPlane.GlooMtlsEnabled,
142+
},
127143
},
128144
}
129145

130146
objs, err := d.renderChartToObjects(emptyGw, vals)
131147
if err != nil {
132148
return nil, err
133149
}
150+
134151
var ret []schema.GroupVersionKind
135152
for _, obj := range objs {
136153
gvk := obj.GetObjectKind().GroupVersionKind()
@@ -255,7 +272,7 @@ func (d *Deployer) getGatewayClassFromGateway(ctx context.Context, gw *api.Gatew
255272
return gwc, nil
256273
}
257274

258-
func (d *Deployer) getValues(gw *api.Gateway, gwParam *v1alpha1.GatewayParameters) (*helmConfig, error) {
275+
func (d *Deployer) getValues(ctx context.Context, gw *api.Gateway, gwParam *v1alpha1.GatewayParameters) (*helmConfig, error) {
259276
// construct the default values
260277
vals := &helmConfig{
261278
Gateway: &helmGateway{
@@ -361,9 +378,61 @@ func (d *Deployer) getValues(gw *api.Gateway, gwParam *v1alpha1.GatewayParameter
361378

362379
gateway.Stats = getStatsValues(statsConfig)
363380

381+
// mtls values
382+
gateway.GlooMtls, err = d.getHelmMtlsConfig(ctx)
383+
if err != nil {
384+
return nil, err
385+
}
386+
364387
return vals, nil
365388
}
366389

390+
func (d *Deployer) getHelmMtlsConfig(ctx context.Context) (*helmMtlsConfig, error) {
391+
392+
if !d.inputs.ControlPlane.GlooMtlsEnabled {
393+
return &helmMtlsConfig{
394+
Enabled: ptr.To(false),
395+
}, nil
396+
}
397+
398+
helmTls, err := d.getHelmTlsSecretData(ctx)
399+
400+
if err != nil {
401+
return nil, err
402+
}
403+
404+
return &helmMtlsConfig{
405+
Enabled: ptr.To(true),
406+
TlsSecret: helmTls,
407+
}, nil
408+
}
409+
410+
// getHelmTlsSecretData builds a helmTls object built from the gloo-mtls-certs secret data, which it fetches
411+
// This function does not check if mtls is enabled, and a missing secret will return an error via getGlooMtlsCertsSecret
412+
func (d *Deployer) getHelmTlsSecretData(ctx context.Context) (*helmTlsSecretData, error) {
413+
414+
mtlsSecret := &corev1.Secret{}
415+
mtlsSecretNns := types.NamespacedName{
416+
Name: wellknown.GlooMtlsCertName,
417+
Namespace: d.inputs.ControlPlane.Namespace,
418+
}
419+
err := d.cli.Get(ctx, mtlsSecretNns, mtlsSecret)
420+
421+
if err != nil {
422+
return nil, eris.Wrap(err, "failed to get gloo mtls secret")
423+
}
424+
425+
if mtlsSecret.Type != corev1.SecretTypeTLS {
426+
return nil, eris.New(fmt.Sprintf("unexpected secret type, expected %s and got %s", corev1.SecretTypeTLS, mtlsSecret.Type))
427+
}
428+
429+
return &helmTlsSecretData{
430+
TlsCert: mtlsSecret.Data[corev1.TLSCertKey],
431+
TlsKey: mtlsSecret.Data[corev1.TLSPrivateKeyKey],
432+
CaCert: mtlsSecret.Data[corev1.ServiceAccountRootCAKey],
433+
}, nil
434+
}
435+
367436
// Render relies on a `helm install` to render the Chart with the injected values
368437
// It returns the list of Objects that are rendered, and an optional error if rendering failed,
369438
// or converting the rendered manifests to objects failed.
@@ -392,6 +461,7 @@ func (d *Deployer) Render(name, ns string, vals map[string]any) ([]client.Object
392461
if err != nil {
393462
return nil, fmt.Errorf("failed to convert helm manifest yaml to objects for gateway %s.%s: %w", ns, name, err)
394463
}
464+
395465
return objs, nil
396466
}
397467

@@ -405,6 +475,8 @@ func (d *Deployer) Render(name, ns string, vals map[string]any) ([]client.Object
405475
//
406476
// * returns the objects to be deployed by the caller
407477
func (d *Deployer) GetObjsToDeploy(ctx context.Context, gw *api.Gateway) ([]client.Object, error) {
478+
logger := log.FromContext(ctx)
479+
408480
gwParam, err := d.getGatewayParametersForGateway(ctx, gw)
409481
if err != nil {
410482
return nil, err
@@ -414,9 +486,7 @@ func (d *Deployer) GetObjsToDeploy(ctx context.Context, gw *api.Gateway) ([]clie
414486
return nil, nil
415487
}
416488

417-
logger := log.FromContext(ctx)
418-
419-
vals, err := d.getValues(gw, gwParam)
489+
vals, err := d.getValues(ctx, gw, gwParam)
420490
if err != nil {
421491
return nil, fmt.Errorf("failed to get values to render objects for gateway %s.%s: %w", gw.GetNamespace(), gw.GetName(), err)
422492
}

0 commit comments

Comments
 (0)