Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion .github/workflows/e2e-multicluster-ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,8 @@ jobs:
--set apiServerCA="$ca" \
--set apiServerURL="https://$apiServerIP:6443" \
--set clusterNamespace="fleet-default" \
--set token="$token"
--set token="$token" \
--set tokenName="second-token"

echo "waiting for downstream cluster to be registered..."
{ grep -q -m 1 "1/1"; kill $!; } < <(kubectl get cluster -n fleet-default -w)
Expand Down
1 change: 1 addition & 0 deletions charts/fleet-agent/templates/secret.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ data:
systemRegistrationNamespace: "{{b64enc .Values.systemRegistrationNamespace}}"
clusterNamespace: "{{b64enc .Values.clusterNamespace}}"
token: "{{b64enc .Values.token}}"
tokenName: "{{b64enc .Values.tokenName}}"
apiServerURL: "{{b64enc .Values.apiServerURL}}"
apiServerCA: "{{b64enc .Values.apiServerCA}}"
kind: Secret
Expand Down
4 changes: 4 additions & 0 deletions charts/fleet-agent/templates/validate.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,7 @@
{{if not .Values.apiServerURL }}
{{ fail "apiServerURL is required to be set, and most likely also apiServerCA" }}
{{end}}

{{if not .Values.tokenName }}
{{ fail "tokenName is required to be set to the name of the ClusterRegistrationToken used for this registration" }}
{{end}}
3 changes: 3 additions & 0 deletions charts/fleet-agent/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ noProxy: 127.0.0.0/8,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,.svc,.cluster.local
# The cluster registration value
token: ""

# The name of the ClusterRegistrationToken used for this registration
tokenName: ""

# Labels to add to the cluster upon registration only. They are not added after the fact.
# labels:
# foo: bar
Expand Down
2 changes: 2 additions & 0 deletions charts/fleet/templates/deployment.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,8 @@ spec:
- name: FLEET_AGENT_ELECTION_RENEW_DEADLINE
value: {{$.Values.agent.leaderElection.renewDeadline}}
{{- end }}
- name: FLEET_REGISTRATION_TOKEN_TTL_REQUIRED
value: {{ quote $.Values.controller.registrationTokenTTL.required }}
{{- if $.Values.extraEnv }}
{{ toYaml $.Values.extraEnv | indent 8}}
{{- end }}
Expand Down
2 changes: 2 additions & 0 deletions charts/fleet/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ leaderElection:
## Fleet controller configuration
controller:
replicas: 1
registrationTokenTTL:
required: false
reconciler:
# The number of workers that are allowed to each type of reconciler
workers:
Expand Down
6 changes: 6 additions & 0 deletions internal/cmd/agent/register/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ const (
CredName = "fleet-agent" // same as AgentConfigName
Kubeconfig = "kubeconfig"
Token = "token"
TokenName = "tokenName"
Values = "values"
DeploymentNamespace = "deploymentNamespace"
ClusterNamespace = "clusterNamespace"
Expand Down Expand Up @@ -187,11 +188,16 @@ func runRegistration(ctx context.Context, k8s coreInterface, namespace string) (
}
cfg.Labels["fleet.cattle.io/created-by-agent-pod"] = os.Getenv("HOSTNAME")

tokenName := string(values(secret.Data)[TokenName])

logrus.Infof("Creating clusterregistration with id '%s' for new token", clientID)
request, err := fc.Fleet().V1alpha1().ClusterRegistration().Create(&fleet.ClusterRegistration{
ObjectMeta: metav1.ObjectMeta{
GenerateName: "request-",
Namespace: ns,
Labels: map[string]string{
fleet.RegistrationTokenLabel: tokenName,
},
},
Comment thread
0xavi0 marked this conversation as resolved.
Spec: fleet.ClusterRegistrationSpec{
ClientID: clientID,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,6 @@ const (
)

var (
ImportTokenPrefix = "import-token-"

errUnavailableAPIServerURL = errors.New("missing apiServerURL in fleet config for cluster auto registration")
)

Expand Down Expand Up @@ -356,7 +354,7 @@ func (i *importHandler) importCluster(cluster *fleet.Cluster, status fleet.Clust
setID := desiredset.GetSetID(config.AgentBootstrapConfigName, "", cluster.Spec.AgentNamespace)
apply = apply.WithDynamicLookup().WithSetID(setID).WithNoDeleteGVK(fleetns.GVK())

tokenName := names.SafeConcatName(ImportTokenPrefix + cluster.Name)
tokenName := names.SafeConcatName(config.ImportTokenPrefix + cluster.Name)
token, err := i.tokens.Get(cluster.Namespace, tokenName)
if err != nil {
// If token doesn't exist, try to create it
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ type handler struct {
serviceAccountCache corecontrollers.ServiceAccountCache
secretsCache corecontrollers.SecretCache
secrets corecontrollers.SecretController
tokenCache fleetcontrollers.ClusterRegistrationTokenCache
}

func Register(ctx context.Context,
Expand All @@ -61,7 +62,8 @@ func Register(ctx context.Context,
role rbaccontrollers.RoleController,
roleBinding rbaccontrollers.RoleBindingController,
clusterRegistration fleetcontrollers.ClusterRegistrationController,
clusters fleetcontrollers.ClusterController) {
clusters fleetcontrollers.ClusterController,
clusterRegistrationToken fleetcontrollers.ClusterRegistrationTokenController) {
h := &handler{
systemNamespace: systemNamespace,
systemRegistrationNamespace: systemRegistrationNamespace,
Expand All @@ -71,6 +73,7 @@ func Register(ctx context.Context,
serviceAccountCache: serviceAccount.Cache(),
secrets: secret,
secretsCache: secret.Cache(),
tokenCache: clusterRegistrationToken.Cache(),
}

fleetcontrollers.RegisterClusterRegistrationGeneratingHandler(ctx,
Expand Down Expand Up @@ -252,15 +255,14 @@ func (h *handler) OnChange(request *fleet.ClusterRegistration, status fleet.Clus

logrus.Infof("Cluster registration request '%s/%s' granted, creating cluster, request service account, registration secret", request.Namespace, request.Name)

return []runtime.Object{
objs := []runtime.Object{
// the registration secret c-clientID-clientRandom
secret,
// Update the existing service account 'request-UID' in the
// cluster namespace, e.g. 'cluster-fleet-default-NAME-ID'
requestSA(saName, cluster, request),
// Add role bindings to manage bundledeployments and contents,
// the agent could previously only access secrets in
// 'cattle-fleet-clusters-system' and clusterregistrations in
// the agent could previously only access clusterregistrations in
// the cluster registration namespace (e.g. 'fleet-default'). See
// clusterregistrationtoken controller for details.
&rbacv1.Role{
Expand Down Expand Up @@ -346,7 +348,125 @@ func (h *handler) OnChange(request *fleet.ClusterRegistration, status fleet.Clus
Name: resources.ContentClusterRole,
},
},
}, status, nil
}

// For manager-initiated imports, grant the import service account access
// to the specific credential secret it needs to read. The import token
// only exists in manager-initiated flows; agent-initiated flows skip this.
importTokenName := names.SafeConcatName(config.ImportTokenPrefix + cluster.Name)
importToken, err := h.tokenCache.Get(request.Namespace, importTokenName)
if err != nil && !apierrors.IsNotFound(err) {
return nil, status, fmt.Errorf("failed to look up import token %s/%s: %w", request.Namespace, importTokenName, err)
}
if err == nil {
// Grant access scoped to the single credential secret for this cluster.
importSAName := names.SafeConcatName(importTokenName, string(importToken.UID))
credRoleName := names.SafeConcatName(importSAName, "creds")
objs = append(objs,
&rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: credRoleName,
Namespace: h.systemRegistrationNamespace,
Labels: map[string]string{
fleet.ManagedLabel: "true",
},
},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"secrets"},
ResourceNames: []string{secret.Name},
},
},
},
&rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: credRoleName,
Namespace: h.systemRegistrationNamespace,
Labels: map[string]string{
fleet.ManagedLabel: "true",
},
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: importSAName,
Namespace: request.Namespace,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
Kind: "Role",
Name: credRoleName,
},
},
)
} else {
// Agent-initiated: no import-token-<cluster> exists because the cluster
// is auto-created at registration time. The ClusterRegistration carries
// a label identifying the token that was used, so we can grant only
// that token's SA scoped read access to the credential secret.
tokenName := request.Labels[fleet.RegistrationTokenLabel]
if tokenName == "" {
logrus.Warnf("ClusterRegistration '%s/%s' has no %s label; cannot grant credential secret access for agent-initiated registration",
request.Namespace, request.Name, fleet.RegistrationTokenLabel)
Comment thread
0xavi0 marked this conversation as resolved.
} else {
agentToken, tokenErr := h.tokenCache.Get(request.Namespace, tokenName)
if tokenErr != nil && !apierrors.IsNotFound(tokenErr) {
return nil, status, fmt.Errorf("looking up registration token %s/%s: %w", request.Namespace, tokenName, tokenErr)
}
if apierrors.IsNotFound(tokenErr) {
logrus.Warnf("ClusterRegistration '%s/%s': registration token %s/%s not found (may have expired), skipping credential secret access grant",
request.Namespace, request.Name, request.Namespace, tokenName)
return objs, status, nil
}
agentSAName := names.SafeConcatName(tokenName, string(agentToken.UID))
credRoleName := names.SafeConcatName(request.Name, "creds")
objs = append(objs,
&rbacv1.Role{
ObjectMeta: metav1.ObjectMeta{
Name: credRoleName,
Namespace: h.systemRegistrationNamespace,
Labels: map[string]string{
fleet.ManagedLabel: "true",
},
},
Rules: []rbacv1.PolicyRule{
{
Verbs: []string{"get"},
APIGroups: []string{""},
Resources: []string{"secrets"},
ResourceNames: []string{secret.Name},
},
},
},
&rbacv1.RoleBinding{
ObjectMeta: metav1.ObjectMeta{
Name: credRoleName,
Namespace: h.systemRegistrationNamespace,
Labels: map[string]string{
fleet.ManagedLabel: "true",
},
},
Subjects: []rbacv1.Subject{
{
Kind: "ServiceAccount",
Name: agentSAName,
Namespace: request.Namespace,
},
},
RoleRef: rbacv1.RoleRef{
APIGroup: rbacv1.GroupName,
Kind: "Role",
Name: credRoleName,
},
},
)
}
}

return objs, status, nil
}

// shouldDelete returns true for any other cluster registration with the same clientID, but different random and older creation timestamp
Expand Down
Loading