Skip to content

Commit 046fe91

Browse files
authored
Merge pull request #25 from yue9944882/feat/copying-seret
Feat: Copying secrets via manager instead of relying on ManagedServiceAccount's projection
2 parents 53c9eec + 04e3944 commit 046fe91

11 files changed

+288
-79
lines changed

charts/addon-manager/crds/clustergatewayconfigurations.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,28 @@ spec:
7070
type: string
7171
installNamespace:
7272
type: string
73+
secretManagement:
74+
properties:
75+
managedServiceAccount:
76+
properties:
77+
name:
78+
default: cluster-gateway
79+
type: string
80+
type: object
81+
type:
82+
default: ManagedServiceAccount
83+
enum:
84+
- Manual
85+
- ManagedServiceAccount
86+
type: string
87+
type: object
7388
secretNamespace:
7489
type: string
7590
required:
7691
- egress
7792
- image
7893
- installNamespace
94+
- secretManagement
7995
- secretNamespace
8096
type: object
8197
status:

charts/addon-manager/templates/clustergatewayconfiguration.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ spec:
66
image: {{ .Values.clusterGateway.image }}
77
installNamespace: {{ .Values.clusterGateway.installNamespace }}
88
secretNamespace: {{ .Values.clusterGateway.secretNamespace }}
9+
secretManagement:
10+
type: ManagedServiceAccount
11+
managedServiceAccount:
12+
name: cluster-gateway
913
egress:
1014
type: ClusterProxy
1115
clusterProxy:

charts/addon-manager/values.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
# Image of the cluster-gateway instances
2-
image: yue9944882/cluster-gateway-addon-manager:v1.1.6.ocm.5
2+
image: yue9944882/cluster-gateway-addon-manager:v1.1.6.sec
33

44
clusterGateway:
55
image: oamdev/cluster-gateway:v1.1.6

go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,6 @@ require (
3030
)
3131

3232
replace (
33-
open-cluster-management.io/managed-serviceaccount v0.0.0 => github.com/yue9944882/managed-serviceaccount v0.0.31
33+
open-cluster-management.io/managed-serviceaccount v0.0.0 => github.com/yue9944882/managed-serviceaccount v0.0.32
3434
sigs.k8s.io/apiserver-network-proxy/konnectivity-client => sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.24
3535
)

go.sum

+2-2
Original file line numberDiff line numberDiff line change
@@ -837,8 +837,8 @@ github.com/yeya24/promlinter v0.1.0/go.mod h1:rs5vtZzeBHqqMwXqFScncpCF6u06lezhZe
837837
github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg=
838838
github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM=
839839
github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc=
840-
github.com/yue9944882/managed-serviceaccount v0.0.31 h1:PDUR80gcc/xecF5BvJzntq3efL+oixfSiRyZDl8UqN8=
841-
github.com/yue9944882/managed-serviceaccount v0.0.31/go.mod h1:pibwxJlKDE0r59JGF8IGw4zaxSNA0Z20t9QmRL5wYaM=
840+
github.com/yue9944882/managed-serviceaccount v0.0.32 h1:QZRO4aisEotalPqtxUCynUBSAYoZcfou8N7qmgvXMAo=
841+
github.com/yue9944882/managed-serviceaccount v0.0.32/go.mod h1:D0DMSNH92z1aVpBexiYXdbrgnv+tG6alxqPSym8DLOk=
842842
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
843843
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
844844
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

hack/crd/bases/proxy.open-cluster-management.io_clustergatewayconfigurations.yaml

+16
Original file line numberDiff line numberDiff line change
@@ -70,12 +70,28 @@ spec:
7070
type: string
7171
installNamespace:
7272
type: string
73+
secretManagement:
74+
properties:
75+
managedServiceAccount:
76+
properties:
77+
name:
78+
default: cluster-gateway
79+
type: string
80+
type: object
81+
type:
82+
default: ManagedServiceAccount
83+
enum:
84+
- Manual
85+
- ManagedServiceAccount
86+
type: string
87+
type: object
7388
secretNamespace:
7489
type: string
7590
required:
7691
- egress
7792
- image
7893
- installNamespace
94+
- secretManagement
7995
- secretNamespace
8096
type: object
8197
status:

pkg/addon/agent/addon.go

-73
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,12 @@
11
package agent
22

33
import (
4-
"context"
5-
"fmt"
6-
"time"
7-
8-
proxyv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/proxy/v1alpha1"
94
"github.com/oam-dev/cluster-gateway/pkg/common"
10-
"github.com/pkg/errors"
11-
apierrors "k8s.io/apimachinery/pkg/api/errors"
12-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
135
"k8s.io/apimachinery/pkg/runtime"
14-
"k8s.io/apimachinery/pkg/types"
156
"k8s.io/client-go/rest"
167
"open-cluster-management.io/addon-framework/pkg/agent"
178
addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
189
clusterv1 "open-cluster-management.io/api/cluster/v1"
19-
ocmauthv1alpha1 "open-cluster-management.io/managed-serviceaccount/api/v1alpha1"
20-
"open-cluster-management.io/managed-serviceaccount/pkg/addon/agent/health"
2110
"sigs.k8s.io/controller-runtime/pkg/client"
2211
)
2312

@@ -36,68 +25,6 @@ type clusterGatewayAddonManager struct {
3625
}
3726

3827
func (c *clusterGatewayAddonManager) Manifests(cluster *clusterv1.ManagedCluster, addon *addonv1alpha1.ManagedClusterAddOn) ([]runtime.Object, error) {
39-
if len(addon.Status.AddOnConfiguration.CRName) == 0 {
40-
return nil, fmt.Errorf("no gateway configuration bond to ManagedClusterAddOn")
41-
}
42-
gatewayConfig := &proxyv1alpha1.ClusterGatewayConfiguration{}
43-
if err := c.client.Get(context.TODO(), types.NamespacedName{
44-
Name: addon.Status.AddOnConfiguration.CRName,
45-
}, gatewayConfig); err != nil {
46-
return nil, fmt.Errorf("failed getting gateway configuration bond to ManagedClusterAddOn")
47-
}
48-
endpointType := "Const"
49-
if gatewayConfig.Spec.Egress.Type == proxyv1alpha1.EgressTypeClusterProxy {
50-
endpointType = "ClusterProxy"
51-
}
52-
msa := &ocmauthv1alpha1.ManagedServiceAccount{
53-
TypeMeta: metav1.TypeMeta{
54-
APIVersion: "authentication.open-cluster-management.io/v1alpha1",
55-
Kind: "ManagedServiceAccount",
56-
},
57-
ObjectMeta: metav1.ObjectMeta{
58-
Namespace: cluster.Name,
59-
Name: common.AddonName,
60-
OwnerReferences: []metav1.OwnerReference{
61-
{
62-
APIVersion: addonv1alpha1.GroupVersion.String(),
63-
Kind: "ManagedClusterAddOn",
64-
UID: addon.UID,
65-
Name: addon.Name,
66-
},
67-
},
68-
},
69-
Spec: ocmauthv1alpha1.ManagedServiceAccountSpec{
70-
Projected: ocmauthv1alpha1.ManagedServiceAccountProjected{
71-
Type: ocmauthv1alpha1.ProjectionTypeSecret,
72-
Secret: &ocmauthv1alpha1.ProjectedSecret{
73-
Labels: map[string]string{
74-
"cluster.core.oam.dev/cluster-credential-type": "ServiceAccountToken",
75-
"cluster.core.oam.dev/cluster-endpoint-type": endpointType,
76-
},
77-
Namespace: gatewayConfig.Spec.SecretNamespace,
78-
Name: cluster.Name,
79-
},
80-
},
81-
Rotation: ocmauthv1alpha1.ManagedServiceAccountRotation{
82-
Enabled: true,
83-
Validity: metav1.Duration{
84-
Duration: time.Hour * 24 * 180,
85-
},
86-
},
87-
},
88-
}
89-
if err := c.client.Create(context.TODO(), msa); err != nil {
90-
if !apierrors.IsAlreadyExists(err) {
91-
return nil, errors.Wrapf(err, "failed creating managed serviceaccount for cluster %v", cluster.Name)
92-
}
93-
}
94-
95-
leaseUpdater, err := health.NewAddonHealthUpdater(c.clientConfig, cluster.Name)
96-
if err != nil {
97-
return nil, errors.Wrapf(err, "failed creating lease updater")
98-
}
99-
go leaseUpdater.Start(context.Background())
100-
10128
return nil, nil
10229
}
10330

pkg/addon/controllers/installer.go

+145-2
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"strconv"
88
"time"
99

10+
clusterv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/cluster/v1alpha1"
1011
proxyv1alpha1 "github.com/oam-dev/cluster-gateway/pkg/apis/proxy/v1alpha1"
1112
"github.com/oam-dev/cluster-gateway/pkg/common"
1213
"github.com/oam-dev/cluster-gateway/pkg/event"
@@ -28,6 +29,7 @@ import (
2829
appsv1 "k8s.io/api/apps/v1"
2930
apierrors "k8s.io/apimachinery/pkg/api/errors"
3031
addonv1alpha1 "open-cluster-management.io/api/addon/v1alpha1"
32+
ocmauthv1alpha1 "open-cluster-management.io/managed-serviceaccount/api/v1alpha1"
3133
"sigs.k8s.io/controller-runtime/pkg/cache"
3234
"sigs.k8s.io/controller-runtime/pkg/client"
3335
"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -55,6 +57,11 @@ func SetupClusterGatewayInstallerWithManager(mgr ctrl.Manager, caPair *crypto.CA
5557
&event.ClusterGatewayConfigurationHandler{
5658
Client: mgr.GetClient(),
5759
}).
60+
Watches(
61+
&source.Kind{
62+
Type: &corev1.Secret{},
63+
},
64+
&event.SecretHandler{}).
5865
Complete(installer)
5966
}
6067

@@ -81,6 +88,11 @@ func (c *ClusterGatewayInstaller) Reconcile(ctx context.Context, request reconci
8188
}
8289
return reconcile.Result{}, errors.Wrapf(err, "failed to get cluster-management-addon: %v", request.Name)
8390
}
91+
if addon.Name != common.AddonName {
92+
// skip
93+
return reconcile.Result{}, nil
94+
}
95+
8496
if addon.Spec.AddOnConfiguration.CRDName != common.ClusterGatewayConfigurationCRDName {
8597
// skip
8698
return reconcile.Result{}, nil
@@ -102,9 +114,12 @@ func (c *ClusterGatewayInstaller) Reconcile(ctx context.Context, request reconci
102114
if err := c.ensureNamespace(clusterGatewayConfiguration.Spec.SecretNamespace); err != nil {
103115
return reconcile.Result{}, errors.Wrapf(err, "failed to ensure required namespace")
104116
}
105-
if err := c.ensureProxySecrets(clusterGatewayConfiguration); err != nil {
117+
if err := c.ensureClusterProxySecrets(clusterGatewayConfiguration); err != nil {
106118
return reconcile.Result{}, errors.Wrapf(err, "failed to ensure required proxy client related credentials")
107119
}
120+
if err := c.ensureSecretManagement(addon, clusterGatewayConfiguration); err != nil {
121+
return reconcile.Result{}, errors.Wrapf(err, "failed to configure secret management")
122+
}
108123

109124
sans := []string{
110125
ServiceNameClusterGateway,
@@ -196,7 +211,7 @@ func (c *ClusterGatewayInstaller) ensureAPIService(addon *addonv1alpha1.ClusterM
196211
return nil
197212
}
198213

199-
func (c *ClusterGatewayInstaller) ensureProxySecrets(config *proxyv1alpha1.ClusterGatewayConfiguration) error {
214+
func (c *ClusterGatewayInstaller) ensureClusterProxySecrets(config *proxyv1alpha1.ClusterGatewayConfiguration) error {
200215
if config.Spec.Egress.Type != proxyv1alpha1.EgressTypeClusterProxy {
201216
return nil
202217
}
@@ -232,6 +247,106 @@ func (c *ClusterGatewayInstaller) ensureProxySecrets(config *proxyv1alpha1.Clust
232247
return nil
233248
}
234249

250+
func (c *ClusterGatewayInstaller) ensureSecretManagement(clusterAddon *addonv1alpha1.ClusterManagementAddOn, config *proxyv1alpha1.ClusterGatewayConfiguration) error {
251+
if config.Spec.SecretManagement.Type != proxyv1alpha1.SecretManagementTypeManagedServiceAccount {
252+
return nil
253+
}
254+
addonList := &addonv1alpha1.ManagedClusterAddOnList{}
255+
if err := c.client.List(context.TODO(), addonList); err != nil {
256+
return errors.Wrapf(err, "failed to list managed cluster addons")
257+
}
258+
clusterGatewayAddon := make([]*addonv1alpha1.ManagedClusterAddOn, 0)
259+
for _, addon := range addonList.Items {
260+
addon := addon
261+
if addon.Name == common.AddonName {
262+
clusterGatewayAddon = append(clusterGatewayAddon, &addon)
263+
}
264+
}
265+
for _, addon := range clusterGatewayAddon {
266+
managedServiceAccount := buildManagedServiceAccount(addon)
267+
if err := c.client.Create(context.TODO(), managedServiceAccount); err != nil {
268+
if !apierrors.IsAlreadyExists(err) {
269+
return errors.Wrapf(err, "failed to create managed serviceaccount")
270+
}
271+
}
272+
273+
if err := c.copySecretForManagedServiceAccount(
274+
clusterAddon,
275+
config,
276+
addon.Namespace); err != nil {
277+
return errors.Wrapf(err, "failed to copy secret from managed serviceaccount")
278+
}
279+
}
280+
return nil
281+
}
282+
283+
func (c *ClusterGatewayInstaller) copySecretForManagedServiceAccount(addon *addonv1alpha1.ClusterManagementAddOn, config *proxyv1alpha1.ClusterGatewayConfiguration, clusterName string) error {
284+
endpointType := clusterv1alpha1.ClusterEndpointTypeConst
285+
if config.Spec.Egress.Type == proxyv1alpha1.EgressTypeClusterProxy {
286+
endpointType = clusterv1alpha1.ClusterEndpointTypeClusterProxy
287+
}
288+
gatewaySecretNamespace := config.Spec.SecretNamespace
289+
secretName := config.Spec.SecretManagement.ManagedServiceAccount.Name
290+
291+
secret, err := c.secretLister.Secrets(clusterName).
292+
Get(secretName)
293+
if err != nil {
294+
if !apierrors.IsNotFound(err) {
295+
return errors.Wrapf(err, "failed to get token secret")
296+
}
297+
return nil
298+
}
299+
currentSecret, err := c.secretLister.Secrets(gatewaySecretNamespace).Get(clusterName)
300+
shouldCreate := false
301+
if err != nil {
302+
if !apierrors.IsNotFound(err) {
303+
return errors.Wrapf(err, "failed to get the cluster secret")
304+
}
305+
shouldCreate = true
306+
}
307+
if shouldCreate {
308+
if _, err := c.nativeClient.CoreV1().Secrets(gatewaySecretNamespace).
309+
Create(context.TODO(),
310+
&corev1.Secret{
311+
ObjectMeta: metav1.ObjectMeta{
312+
Namespace: gatewaySecretNamespace,
313+
Name: clusterName,
314+
Labels: map[string]string{
315+
clusterv1alpha1.LabelKeyClusterCredentialType: string(clusterv1alpha1.CredentialTypeServiceAccountToken),
316+
clusterv1alpha1.LabelKeyClusterEndpointType: endpointType,
317+
},
318+
OwnerReferences: []metav1.OwnerReference{
319+
{
320+
APIVersion: addonv1alpha1.GroupVersion.String(),
321+
Kind: "ClusterManagementAddOn",
322+
UID: addon.UID,
323+
Name: addon.Name,
324+
},
325+
},
326+
},
327+
Type: corev1.SecretTypeOpaque,
328+
Data: map[string][]byte{
329+
corev1.ServiceAccountRootCAKey: secret.Data[corev1.ServiceAccountRootCAKey],
330+
corev1.ServiceAccountTokenKey: secret.Data[corev1.ServiceAccountTokenKey],
331+
},
332+
},
333+
metav1.CreateOptions{}); err != nil {
334+
return errors.Wrapf(err, "failed to create the cluster secret")
335+
}
336+
} else {
337+
if bytes.Equal(secret.Data[corev1.ServiceAccountTokenKey], currentSecret.Data[corev1.ServiceAccountTokenKey]) {
338+
return nil // no need for an update
339+
}
340+
currentSecret.Data[corev1.ServiceAccountRootCAKey] = secret.Data[corev1.ServiceAccountRootCAKey]
341+
currentSecret.Data[corev1.ServiceAccountTokenKey] = secret.Data[corev1.ServiceAccountTokenKey]
342+
if _, err := c.nativeClient.CoreV1().Secrets(gatewaySecretNamespace).
343+
Update(context.TODO(), currentSecret, metav1.UpdateOptions{}); err != nil {
344+
return errors.Wrapf(err, "failed to update the cluster secret")
345+
}
346+
}
347+
return nil
348+
}
349+
235350
func newServiceAccount(addon *addonv1alpha1.ClusterManagementAddOn, namespace string) *corev1.ServiceAccount {
236351
return &corev1.ServiceAccount{
237352
TypeMeta: metav1.TypeMeta{
@@ -567,5 +682,33 @@ func newAPFClusterRoleBinding(addon *addonv1alpha1.ClusterManagementAddOn, names
567682
},
568683
},
569684
}
685+
}
570686

687+
func buildManagedServiceAccount(addon *addonv1alpha1.ManagedClusterAddOn) *ocmauthv1alpha1.ManagedServiceAccount {
688+
return &ocmauthv1alpha1.ManagedServiceAccount{
689+
TypeMeta: metav1.TypeMeta{
690+
APIVersion: "authentication.open-cluster-management.io/v1alpha1",
691+
Kind: "ManagedServiceAccount",
692+
},
693+
ObjectMeta: metav1.ObjectMeta{
694+
Namespace: addon.Namespace,
695+
Name: common.AddonName,
696+
OwnerReferences: []metav1.OwnerReference{
697+
{
698+
APIVersion: addonv1alpha1.GroupVersion.String(),
699+
Kind: "ManagedClusterAddOn",
700+
UID: addon.UID,
701+
Name: addon.Name,
702+
},
703+
},
704+
},
705+
Spec: ocmauthv1alpha1.ManagedServiceAccountSpec{
706+
Rotation: ocmauthv1alpha1.ManagedServiceAccountRotation{
707+
Enabled: true,
708+
Validity: metav1.Duration{
709+
Duration: time.Hour * 24 * 180,
710+
},
711+
},
712+
},
713+
}
571714
}

0 commit comments

Comments
 (0)