diff --git a/controllers/apps/component_controller.go b/controllers/apps/component_controller.go index 7689e5b8b66..cae17b2e7aa 100644 --- a/controllers/apps/component_controller.go +++ b/controllers/apps/component_controller.go @@ -223,6 +223,7 @@ func (r *ComponentReconciler) setupWithManager(mgr ctrl.Manager) error { Owns(&corev1.ConfigMap{}). Owns(&dpv1alpha1.Backup{}). Owns(&dpv1alpha1.Restore{}). + Watches(&corev1.Secret{}, handler.EnqueueRequestsFromMapFunc(r.filterComponentResources)). Watches(&corev1.PersistentVolumeClaim{}, handler.EnqueueRequestsFromMapFunc(r.filterComponentResources)). Owns(&batchv1.Job{}). Watches(&appsv1alpha1.Configuration{}, handler.EnqueueRequestsFromMapFunc(r.configurationEventHandler)) diff --git a/controllers/apps/transformer_cluster_backup_policy.go b/controllers/apps/transformer_cluster_backup_policy.go index a59f11ed177..8a46d45fc31 100644 --- a/controllers/apps/transformer_cluster_backup_policy.go +++ b/controllers/apps/transformer_cluster_backup_policy.go @@ -22,6 +22,7 @@ package apps import ( "encoding/json" "fmt" + "strings" "golang.org/x/exp/slices" corev1 "k8s.io/api/core/v1" @@ -45,7 +46,11 @@ import ( ) const ( - defaultCronExpression = "0 18 * * *" + defaultCronExpression = "0 18 * * *" + defaultRedisBackupMethod = "datafile" + defaultMySQLBackupMethod = "xtrabackup" + defaultMongoBackupMethod = "dump" + defaultPostgresBackupMethod = "pg-basebackup" ) // clusterBackupPolicyTransformer transforms the backup policy template to the data protection backup policy and backup schedule. @@ -54,6 +59,7 @@ type clusterBackupPolicyTransformer struct { tplCount int tplIdentifier string isDefaultTemplate string + needPatchCluster bool backupPolicyTpl *appsv1alpha1.BackupPolicyTemplate backupPolicy *appsv1alpha1.BackupPolicy @@ -73,6 +79,7 @@ var _ graph.Transformer = &clusterBackupPolicyTransformer{} // Transform transforms the backup policy template to the backup policy and backup schedule. func (r *clusterBackupPolicyTransformer) Transform(ctx graph.TransformContext, dag *graph.DAG) error { r.clusterTransformContext = ctx.(*clusterTransformContext) + r.needPatchCluster = false if model.IsObjectDeleting(r.clusterTransformContext.OrigCluster) { return nil } @@ -182,6 +189,10 @@ func (r *clusterBackupPolicyTransformer) Transform(ctx graph.TransformContext, d } } } + if r.needPatchCluster { + graphCli, _ := r.Client.(model.GraphClient) + graphCli.Patch(dag, r.OrigCluster, r.Cluster, &model.ReplaceIfExistingOption{}) + } return nil } @@ -587,7 +598,7 @@ func (r *clusterBackupPolicyTransformer) mergeClusterBackup( backupPolicy *dpv1alpha1.BackupPolicy, backupSchedule *dpv1alpha1.BackupSchedule, ) *dpv1alpha1.BackupSchedule { - cluster := r.OrigCluster + cluster := r.Cluster backupEnabled := func() bool { return cluster.Spec.Backup != nil && boolValue(cluster.Spec.Backup.Enabled) } @@ -602,6 +613,14 @@ func (r *clusterBackupPolicyTransformer) mergeClusterBackup( } backup := cluster.Spec.Backup + if backup.Method == "" { + backup.Method = r.defaultBackupMethod(comp, backupPolicy) + if backup.Method != "" { + r.needPatchCluster = true + r.V(1).Info("backup method not set, defaulting", + "cluster", cluster.Name, "component", comp.componentName, "method", backup.Method) + } + } method := dputils.GetBackupMethodByName(backup.Method, backupPolicy) // the specified backup method should be in the backup policy, if not, record event and return. if method == nil { @@ -717,6 +736,125 @@ func (r *clusterBackupPolicyTransformer) mergeClusterBackup( return backupSchedule } +func (r *clusterBackupPolicyTransformer) defaultBackupMethod(comp componentItem, backupPolicy *dpv1alpha1.BackupPolicy) string { + // 1) serviceKind / builtinHandler based detection + if method := r.defaultBackupMethodByServiceKind(comp); method != "" { + return method + } + // 2) component definition name fallback + if method := defaultBackupMethodByCompDefName(r.compDefNameFromSpec(comp.compSpec)); method != "" { + return method + } + // 3) best-effort: use known defaults that exist in backup policy + defaultPriority := map[string]int{ + defaultRedisBackupMethod: 0, + defaultMySQLBackupMethod: 1, + defaultMongoBackupMethod: 2, + defaultPostgresBackupMethod: 3, + } + bestMethod := "" + bestPriority := int(^uint(0) >> 1) // max int + if backupPolicy != nil { + for _, method := range backupPolicy.Spec.BackupMethods { + if p, ok := defaultPriority[method.Name]; ok && p < bestPriority { + bestMethod = method.Name + bestPriority = p + } + } + } + if bestMethod != "" { + return bestMethod + } + return "" +} + +func (r *clusterBackupPolicyTransformer) componentServiceKind(comp componentItem) string { + compDefName := r.compDefNameFromSpec(comp.compSpec) + compDef := r.ComponentDefs[compDefName] + if compDef == nil { + return "" + } + if compDef.Spec.ServiceKind != "" { + return strings.ToLower(compDef.Spec.ServiceKind) + } + if compDef.Spec.LifecycleActions != nil { + if handler := builtinHandlerFromLifecycleActions(compDef.Spec.LifecycleActions); handler != "" { + return strings.ToLower(string(handler)) + } + } + return "" +} + +func (r *clusterBackupPolicyTransformer) defaultBackupMethodByServiceKind(comp componentItem) string { + serviceKind := r.componentServiceKind(comp) + switch serviceKind { + case string(appsv1alpha1.RedisBuiltinActionHandler): + return defaultRedisBackupMethod + case string(appsv1alpha1.MySQLBuiltinActionHandler), string(appsv1alpha1.WeSQLBuiltinActionHandler): + return defaultMySQLBackupMethod + case string(appsv1alpha1.MongoDBBuiltinActionHandler): + return defaultMongoBackupMethod + case string(appsv1alpha1.PostgresqlBuiltinActionHandler), + string(appsv1alpha1.OfficialPostgresqlBuiltinActionHandler), + string(appsv1alpha1.ApeCloudPostgresqlBuiltinActionHandler): + return defaultPostgresBackupMethod + default: + return "" + } +} + +func defaultBackupMethodByCompDefName(compDefName string) string { + name := strings.ToLower(compDefName) + switch { + case strings.Contains(name, "redis"): + return defaultRedisBackupMethod + case strings.Contains(name, "mongo"): + return defaultMongoBackupMethod + case strings.Contains(name, "mysql"), strings.Contains(name, "wesql"): + return defaultMySQLBackupMethod + case strings.Contains(name, "postgres"), strings.Contains(name, "pg"): + return defaultPostgresBackupMethod + default: + return "" + } +} + +func hasBackupMethod(backupPolicy *dpv1alpha1.BackupPolicy, method string) bool { + if backupPolicy == nil || method == "" { + return false + } + for _, m := range backupPolicy.Spec.BackupMethods { + if m.Name == method { + return true + } + } + return false +} + +func builtinHandlerFromLifecycleActions(actions *appsv1alpha1.ComponentLifecycleActions) appsv1alpha1.BuiltinActionHandlerType { + if actions == nil { + return "" + } + handlers := []*appsv1alpha1.LifecycleActionHandler{ + actions.PostProvision, + actions.PreTerminate, + actions.MemberJoin, + actions.MemberLeave, + actions.Readonly, + actions.Readwrite, + actions.DataDump, + actions.DataLoad, + actions.Reconfigure, + actions.AccountProvision, + } + for _, handler := range handlers { + if handler != nil && handler.BuiltinHandler != nil { + return *handler.BuiltinHandler + } + } + return "" +} + // getClusterComponentSpec returns the component which matches the componentDef or componentDefRef. func (r *clusterBackupPolicyTransformer) getClusterComponentItems() []componentItem { matchedCompDef := func(compSpec appsv1alpha1.ClusterComponentSpec) bool { diff --git a/controllers/apps/transformer_component_account.go b/controllers/apps/transformer_component_account.go index 5eabb368aba..5a7286e4fd2 100644 --- a/controllers/apps/transformer_component_account.go +++ b/controllers/apps/transformer_component_account.go @@ -58,6 +58,10 @@ func (t *componentAccountTransformer) Transform(ctx graph.TransformContext, dag synthesizeComp := transCtx.SynthesizeComponent graphCli, _ := transCtx.Client.(model.GraphClient) + if err := t.ensureLegacyMongoRootSecret(transCtx, synthesizeComp, graphCli, dag); err != nil { + return err + } + for _, account := range synthesizeComp.SystemAccounts { existSecret, err := t.checkAccountSecretExist(ctx, synthesizeComp, account) if err != nil { @@ -67,6 +71,12 @@ func (t *componentAccountTransformer) Transform(ctx graph.TransformContext, dag if err != nil { return err } + if secret == nil { + if existSecret == nil { + t.emitMissingInitAccountSecretEvent(transCtx, synthesizeComp, account) + } + continue + } if existSecret == nil { graphCli.Create(dag, secret, inUniversalContext4G()) @@ -84,6 +94,59 @@ func (t *componentAccountTransformer) Transform(ctx graph.TransformContext, dag return nil } +func (t *componentAccountTransformer) ensureLegacyMongoRootSecret( + transCtx *componentTransformContext, + synthesizeComp *component.SynthesizedComponent, + graphCli model.GraphClient, + dag *graph.DAG, +) error { + if transCtx == nil || synthesizeComp == nil || graphCli == nil { + return nil + } + compObj := transCtx.ComponentOrig + if compObj == nil { + compObj = transCtx.Component + } + if compObj == nil || !component.IsGenerated(compObj) { + return nil + } + if !t.isMongoComponent(transCtx) { + return nil + } + if hasAccountName(synthesizeComp.SystemAccounts, "root") { + return nil + } + + rootAccount := appsv1alpha1.SystemAccount{ + Name: "root", + InitAccount: true, + } + existSecret, err := t.checkAccountSecretExist(transCtx, synthesizeComp, rootAccount) + if err != nil { + return err + } + if existSecret != nil { + return nil + } + + var password []byte + if transCtx.Cluster != nil { + if restorePwd := factory.GetRestorePassword(transCtx.Cluster, synthesizeComp); restorePwd != "" { + password = []byte(restorePwd) + } + } + if len(password) == 0 { + password = t.getLegacyConnCredentialPassword(transCtx) + } + if len(password) == 0 { + t.emitMissingInitAccountSecretEvent(transCtx, synthesizeComp, rootAccount) + return nil + } + secret := t.buildAccountSecretWithPassword(synthesizeComp, rootAccount, password) + graphCli.Create(dag, secret, inUniversalContext4G()) + return nil +} + func (t *componentAccountTransformer) checkAccountSecretExist(ctx graph.TransformContext, synthesizeComp *component.SynthesizedComponent, account appsv1alpha1.SystemAccount) (*corev1.Secret, error) { secretKey := types.NamespacedName{ @@ -112,7 +175,11 @@ func (t *componentAccountTransformer) buildAccountSecret(ctx *componentTransform return nil, err } default: - password = t.buildPassword(ctx, account) + var ok bool + password, ok = t.buildPassword(ctx, synthesizeComp, account) + if !ok { + return nil, nil + } } return t.buildAccountSecretWithPassword(synthesizeComp, account, password), nil } @@ -132,18 +199,85 @@ func (t *componentAccountTransformer) getPasswordFromSecret(ctx graph.TransformC return secret.Data[constant.AccountPasswdForSecret], nil } -func (t *componentAccountTransformer) buildPassword(ctx *componentTransformContext, account appsv1alpha1.SystemAccount) []byte { +func (t *componentAccountTransformer) buildPassword(ctx *componentTransformContext, synthesizeComp *component.SynthesizedComponent, account appsv1alpha1.SystemAccount) ([]byte, bool) { // get restore password if exists during recovery. password := factory.GetRestoreSystemAccountPassword(ctx.SynthesizeComponent.Annotations, ctx.SynthesizeComponent.Name, account.Name) if account.InitAccount && password == "" { - // initAccount can also restore from factory.GetRestoreSystemAccountPassword(ctx.SynthesizeComponent, account). - // This is compatibility processing. - password = factory.GetRestorePassword(ctx.Cluster, ctx.SynthesizeComponent) + if strings.EqualFold(account.Name, "root") && t.isMongoComponent(ctx) { + password = factory.GetRestorePassword(ctx.Cluster, synthesizeComp) + if password != "" { + return []byte(password), true + } + if legacyPwd := t.getLegacyConnCredentialPassword(ctx); len(legacyPwd) > 0 { + ctx.V(1).Info("using legacy conn-credential password for init account", + "component", ctx.SynthesizeComponent.Name, "account", account.Name) + return legacyPwd, true + } + return nil, false + } else { + // initAccount can also restore from factory.GetRestoreSystemAccountPassword(ctx.SynthesizeComponent, account). + // This is compatibility processing. + password = factory.GetRestorePassword(ctx.Cluster, synthesizeComp) + } } if password == "" { - return t.generatePassword(account) + return t.generatePassword(account), true + } + return []byte(password), true +} + +func hasAccountName(accounts []appsv1alpha1.SystemAccount, name string) bool { + for _, account := range accounts { + if strings.EqualFold(account.Name, name) { + return true + } + } + return false +} + +func (t *componentAccountTransformer) isMongoComponent(ctx *componentTransformContext) bool { + if ctx == nil { + return false + } + if ctx.CompDef != nil && ctx.CompDef.Spec.ServiceKind != "" { + serviceKind := strings.ToLower(ctx.CompDef.Spec.ServiceKind) + for _, alias := range constant.GetMongoDBAlias() { + if serviceKind == alias { + return true + } + } } - return []byte(password) + if ctx.SynthesizeComponent != nil && strings.EqualFold(ctx.SynthesizeComponent.CharacterType, constant.MongoDBCharacterType) { + return true + } + return false +} + +func (t *componentAccountTransformer) getLegacyConnCredentialPassword(ctx *componentTransformContext) []byte { + if ctx == nil || ctx.Cluster == nil { + return nil + } + secretKey := types.NamespacedName{ + Namespace: ctx.Cluster.Namespace, + Name: constant.GenerateDefaultConnCredential(ctx.Cluster.Name), + } + secret := &corev1.Secret{} + if err := ctx.Client.Get(ctx.Context, secretKey, secret); err != nil { + return nil + } + if pwd, ok := secret.Data[constant.AccountPasswdForSecret]; ok && len(pwd) > 0 { + return pwd + } + return nil +} + +func (t *componentAccountTransformer) emitMissingInitAccountSecretEvent(ctx *componentTransformContext, synthesizeComp *component.SynthesizedComponent, account appsv1alpha1.SystemAccount) { + if ctx == nil || ctx.EventRecorder == nil || ctx.Component == nil || synthesizeComp == nil { + return + } + secretName := constant.GenerateAccountSecretName(synthesizeComp.ClusterName, synthesizeComp.Name, account.Name) + ctx.EventRecorder.Eventf(ctx.Component, corev1.EventTypeWarning, "InitAccountSecretMissing", + "missing legacy conn-credential and restore password for init account, please create secret %s manually", secretName) } func (t *componentAccountTransformer) generatePassword(account appsv1alpha1.SystemAccount) []byte {