Skip to content

Commit b1a21bb

Browse files
driegel1driegel1
committed
support running karmada-agent out-of-cluster
1 parent 1c27857 commit b1a21bb

5 files changed

Lines changed: 278 additions & 37 deletions

File tree

‎cmd/agent/app/agent.go‎

Lines changed: 44 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -159,46 +159,53 @@ func run(ctx context.Context, opts *options.Options) error {
159159
if err != nil {
160160
return fmt.Errorf("error building kubeconfig of member cluster: %w", err)
161161
}
162-
clusterKubeClient := kubeclientset.NewForConfigOrDie(clusterConfig)
163-
controlPlaneKubeClient := kubeclientset.NewForConfigOrDie(controlPlaneRestConfig)
164-
karmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig)
165-
166-
registerOption := util.ClusterRegisterOption{
167-
ClusterNamespace: opts.ClusterNamespace,
168-
ClusterName: opts.ClusterName,
169-
ReportSecrets: opts.ReportSecrets,
170-
ClusterAPIEndpoint: opts.ClusterAPIEndpoint,
171-
ProxyServerAddress: opts.ProxyServerAddress,
172-
ClusterProvider: opts.ClusterProvider,
173-
ClusterRegion: opts.ClusterRegion,
174-
ClusterZones: opts.ClusterZones,
175-
DryRun: false,
176-
ControlPlaneConfig: controlPlaneRestConfig,
177-
ClusterConfig: clusterConfig,
178-
}
162+
if opts.RegisterCluster {
163+
clusterKubeClient := kubeclientset.NewForConfigOrDie(clusterConfig)
164+
controlPlaneKubeClient := kubeclientset.NewForConfigOrDie(controlPlaneRestConfig)
165+
karmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig)
166+
167+
registerOption := util.ClusterRegisterOption{
168+
ClusterNamespace: opts.ClusterNamespace,
169+
ClusterName: opts.ClusterName,
170+
ReportSecrets: opts.ReportSecrets,
171+
ClusterAPIEndpoint: opts.ClusterAPIEndpoint,
172+
ProxyServerAddress: opts.ProxyServerAddress,
173+
ClusterProvider: opts.ClusterProvider,
174+
ClusterRegion: opts.ClusterRegion,
175+
ClusterZones: opts.ClusterZones,
176+
DryRun: false,
177+
ControlPlaneConfig: controlPlaneRestConfig,
178+
ClusterConfig: clusterConfig,
179+
}
179180

180-
registerOption.ClusterID, err = util.ObtainClusterID(clusterKubeClient)
181-
if err != nil {
182-
return err
183-
}
181+
registerOption.ClusterID, err = util.ObtainClusterID(clusterKubeClient)
182+
if err != nil {
183+
return err
184+
}
184185

185-
if err = registerOption.Validate(karmadaClient, true); err != nil {
186-
return err
187-
}
186+
if err = registerOption.Validate(karmadaClient, true); err != nil {
187+
return err
188+
}
188189

189-
clusterSecret, impersonatorSecret, err := util.ObtainCredentialsFromMemberCluster(clusterKubeClient, registerOption)
190-
if err != nil {
191-
return err
192-
}
193-
if clusterSecret != nil {
194-
registerOption.Secret = *clusterSecret
195-
}
196-
if impersonatorSecret != nil {
197-
registerOption.ImpersonatorSecret = *impersonatorSecret
198-
}
199-
err = util.RegisterClusterInControllerPlane(registerOption, controlPlaneKubeClient, generateClusterInControllerPlane)
200-
if err != nil {
201-
return fmt.Errorf("failed to register with karmada control plane: %w", err)
190+
clusterSecret, impersonatorSecret, err := util.ObtainCredentialsFromMemberCluster(clusterKubeClient, registerOption)
191+
if err != nil {
192+
return err
193+
}
194+
if clusterSecret != nil {
195+
registerOption.Secret = *clusterSecret
196+
}
197+
if impersonatorSecret != nil {
198+
registerOption.ImpersonatorSecret = *impersonatorSecret
199+
}
200+
if err = util.RegisterClusterInControllerPlane(registerOption, controlPlaneKubeClient, generateClusterInControllerPlane); err != nil {
201+
return fmt.Errorf("failed to register with karmada control plane: %w", err)
202+
}
203+
} else {
204+
karmadaClient := karmadaclientset.NewForConfigOrDie(controlPlaneRestConfig)
205+
memberKubeClient := kubeclientset.NewForConfigOrDie(clusterConfig)
206+
if err = validateExternallyRegisteredCluster(ctx, opts, karmadaClient, memberKubeClient); err != nil {
207+
return err
208+
}
202209
}
203210

204211
executionSpace := names.GenerateExecutionSpaceName(opts.ClusterName)

‎cmd/agent/app/options/options.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,10 @@ type Options struct {
140140
CertRotationRemainingTimeThreshold float64
141141
// KarmadaKubeconfigNamespace is the namespace of the secret containing karmada-agent certificate.
142142
KarmadaKubeconfigNamespace string
143+
// RegisterCluster indicates whether the agent should register the member cluster with the
144+
// Karmada control plane on startup. When false, the cluster must already be registered in
145+
// Pull mode and the agent only validates the existing registration before running its controllers.
146+
RegisterCluster bool
143147
}
144148

145149
// NewOptions builds an default scheduler options.
@@ -211,6 +215,7 @@ func (o *Options) AddFlags(fs *pflag.FlagSet, allControllers []string) {
211215
fs.DurationVar(&o.CertRotationCheckingInterval, "cert-rotation-checking-interval", 5*time.Minute, "The interval of checking if the certificate need to be rotated. This is only applicable if cert rotation is enabled")
212216
fs.Float64Var(&o.CertRotationRemainingTimeThreshold, "cert-rotation-remaining-time-threshold", 0.2, "The threshold of remaining time of the valid certificate. This is only applicable if cert rotation is enabled.")
213217
fs.StringVar(&o.KarmadaKubeconfigNamespace, "karmada-kubeconfig-namespace", "karmada-system", "Namespace of the secret containing karmada-agent certificate. This is only applicable if cert rotation is enabled.")
218+
fs.BoolVar(&o.RegisterCluster, "register-cluster", true, "Whether to register the member cluster with the Karmada control plane on startup. Set to false when the cluster is pre-registered by an external process; the agent will then only validate the existing Pull-mode registration and run its controllers.")
214219
o.RateLimiterOpts.AddFlags(fs)
215220
features.FeatureGate.AddFlag(fs)
216221
o.ProfileOpts.AddFlags(fs)

‎cmd/agent/app/registration.go‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
Copyright 2021 The Karmada Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package app
18+
19+
import (
20+
"context"
21+
"fmt"
22+
23+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
24+
kubeclientset "k8s.io/client-go/kubernetes"
25+
"k8s.io/klog/v2"
26+
27+
"github.com/karmada-io/karmada/cmd/agent/app/options"
28+
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
29+
karmadaclientset "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
30+
"github.com/karmada-io/karmada/pkg/util"
31+
)
32+
33+
// validateExternallyRegisteredCluster verifies that a cluster registered outside of the agent
34+
// (i.e. with --register-cluster=false) is in a state the agent can run against: it exists, is not
35+
// being deleted, uses Pull sync mode, and its ID matches the member cluster the agent is running for.
36+
func validateExternallyRegisteredCluster(ctx context.Context, opts *options.Options, karmadaClient karmadaclientset.Interface, memberKubeClient kubeclientset.Interface) error {
37+
cluster, err := karmadaClient.ClusterV1alpha1().Clusters().Get(ctx, opts.ClusterName, metav1.GetOptions{})
38+
if err != nil {
39+
return fmt.Errorf("failed to get cluster %q from control plane: %w", opts.ClusterName, err)
40+
}
41+
42+
if !cluster.DeletionTimestamp.IsZero() {
43+
return fmt.Errorf("cluster %q is being deleted", opts.ClusterName)
44+
}
45+
46+
if cluster.Spec.SyncMode != clusterv1alpha1.Pull {
47+
return fmt.Errorf("cluster %q has SyncMode %q, expected %q for an externally registered cluster", opts.ClusterName, cluster.Spec.SyncMode, clusterv1alpha1.Pull)
48+
}
49+
50+
clusterID, err := util.ObtainClusterID(memberKubeClient)
51+
if err != nil {
52+
return fmt.Errorf("failed to obtain cluster ID from member cluster: %w", err)
53+
}
54+
55+
if cluster.Spec.ID != "" && cluster.Spec.ID != clusterID {
56+
return fmt.Errorf("cluster ID mismatch: control plane has %q but member cluster reports %q", cluster.Spec.ID, clusterID)
57+
}
58+
if cluster.Spec.ID == "" {
59+
klog.Warningf("Cluster %q has no ID set in the control plane; consider setting spec.id to %q", opts.ClusterName, clusterID)
60+
}
61+
62+
klog.Infof("Successfully validated externally registered cluster %q", opts.ClusterName)
63+
return nil
64+
}

‎cmd/agent/app/registration_test.go‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
/*
2+
Copyright 2021 The Karmada Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package app
18+
19+
import (
20+
"context"
21+
"strings"
22+
"testing"
23+
"time"
24+
25+
corev1 "k8s.io/api/core/v1"
26+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/types"
28+
kubefake "k8s.io/client-go/kubernetes/fake"
29+
30+
"github.com/karmada-io/karmada/cmd/agent/app/options"
31+
clusterv1alpha1 "github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1"
32+
karmadafake "github.com/karmada-io/karmada/pkg/generated/clientset/versioned/fake"
33+
)
34+
35+
func newTestOpts(clusterName string) *options.Options {
36+
return &options.Options{
37+
ClusterName: clusterName,
38+
}
39+
}
40+
41+
func newKubeSystemNamespace(uid string) *corev1.Namespace {
42+
return &corev1.Namespace{
43+
ObjectMeta: metav1.ObjectMeta{
44+
Name: metav1.NamespaceSystem,
45+
UID: types.UID(uid),
46+
},
47+
}
48+
}
49+
50+
func newCluster(name string, syncMode clusterv1alpha1.ClusterSyncMode, id string) *clusterv1alpha1.Cluster {
51+
return &clusterv1alpha1.Cluster{
52+
ObjectMeta: metav1.ObjectMeta{
53+
Name: name,
54+
},
55+
Spec: clusterv1alpha1.ClusterSpec{
56+
SyncMode: syncMode,
57+
ID: id,
58+
},
59+
}
60+
}
61+
62+
func TestValidateExternallyRegisteredCluster(t *testing.T) {
63+
const (
64+
clusterName = "member1"
65+
clusterUID = "test-uid-12345"
66+
)
67+
68+
tests := []struct {
69+
name string
70+
cluster *clusterv1alpha1.Cluster
71+
nsUID string
72+
noKubeSystemNS bool
73+
wantErr bool
74+
errContains string
75+
}{
76+
{
77+
name: "happy path with matching ID",
78+
cluster: newCluster(clusterName, clusterv1alpha1.Pull, clusterUID),
79+
nsUID: clusterUID,
80+
wantErr: false,
81+
},
82+
{
83+
name: "happy path with empty cluster ID",
84+
cluster: newCluster(clusterName, clusterv1alpha1.Pull, ""),
85+
nsUID: clusterUID,
86+
wantErr: false,
87+
},
88+
{
89+
name: "cluster not found",
90+
cluster: nil,
91+
nsUID: clusterUID,
92+
wantErr: true,
93+
errContains: "failed to get cluster",
94+
},
95+
{
96+
name: "cluster being deleted",
97+
cluster: func() *clusterv1alpha1.Cluster {
98+
c := newCluster(clusterName, clusterv1alpha1.Pull, clusterUID)
99+
now := metav1.NewTime(time.Now())
100+
c.DeletionTimestamp = &now
101+
c.Finalizers = []string{"test-finalizer"}
102+
return c
103+
}(),
104+
nsUID: clusterUID,
105+
wantErr: true,
106+
errContains: "is being deleted",
107+
},
108+
{
109+
name: "wrong sync mode (Push)",
110+
cluster: newCluster(clusterName, clusterv1alpha1.Push, clusterUID),
111+
nsUID: clusterUID,
112+
wantErr: true,
113+
errContains: "SyncMode",
114+
},
115+
{
116+
name: "cluster ID mismatch",
117+
cluster: newCluster(clusterName, clusterv1alpha1.Pull, "different-uid"),
118+
nsUID: clusterUID,
119+
wantErr: true,
120+
errContains: "cluster ID mismatch",
121+
},
122+
{
123+
name: "member cluster missing kube-system namespace",
124+
cluster: newCluster(clusterName, clusterv1alpha1.Pull, clusterUID),
125+
noKubeSystemNS: true,
126+
wantErr: true,
127+
errContains: "failed to obtain cluster ID",
128+
},
129+
}
130+
131+
for _, tt := range tests {
132+
t.Run(tt.name, func(t *testing.T) {
133+
var karmadaClient *karmadafake.Clientset
134+
if tt.cluster != nil {
135+
karmadaClient = karmadafake.NewSimpleClientset(tt.cluster)
136+
} else {
137+
karmadaClient = karmadafake.NewSimpleClientset()
138+
}
139+
140+
var memberKubeClient *kubefake.Clientset
141+
if tt.noKubeSystemNS {
142+
memberKubeClient = kubefake.NewSimpleClientset()
143+
} else {
144+
memberKubeClient = kubefake.NewSimpleClientset(newKubeSystemNamespace(tt.nsUID))
145+
}
146+
opts := newTestOpts(clusterName)
147+
148+
err := validateExternallyRegisteredCluster(context.Background(), opts, karmadaClient, memberKubeClient)
149+
150+
if tt.wantErr {
151+
if err == nil {
152+
t.Fatal("expected error, got nil")
153+
}
154+
if tt.errContains != "" && !strings.Contains(err.Error(), tt.errContains) {
155+
t.Errorf("expected error containing %q, got %q", tt.errContains, err.Error())
156+
}
157+
} else {
158+
if err != nil {
159+
t.Fatalf("unexpected error: %v", err)
160+
}
161+
}
162+
})
163+
}
164+
}

‎docs/command-line-flags/karmada-agent.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@ Generic flags:
108108
--rate-limiter-bucket-size int The bucket size for rate limier. (default 100)
109109
--rate-limiter-max-delay duration The max delay for rate limiter. (default 16m40s)
110110
--rate-limiter-qps int The QPS for rate limier. (default 10)
111+
--register-cluster Whether to register the member cluster with the Karmada control plane on startup. Set to false when the cluster is pre-registered by an external process; the agent will then only validate the existing Pull-mode registration and run its controllers. (default true)
111112
--report-secrets strings The secrets that are allowed to be reported to the Karmada control plane during registering. Valid values are 'KubeCredentials', 'KubeImpersonator' and 'None'. e.g 'KubeCredentials,KubeImpersonator' or 'None'. (default [KubeCredentials,KubeImpersonator])
112113
--resync-period duration Base frequency the informers are resynced.
113114
```

0 commit comments

Comments
 (0)