Merge pull request #1077 from signal18/bug#1076

Add super-read-only in freezing workload #1076

Author: svaroqui

cluster/cluster_fail.go

@@ -365,13 +365,14 @@ func (cluster *Cluster) MasterFailover(fail bool) bool {
 		}
 
 		if cluster.Conf.SwitchDecreaseMaxConn {
-
 			logs, err := dbhelper.SetMaxConnections(cluster.oldMaster.Conn, cluster.oldMaster.maxConn, cluster.oldMaster.DBVersion)
 			cluster.LogSQL(logs, err, cluster.oldMaster.URL, "MasterFailover", config.LvlErr, "Could not set max connection, %s", err)
-
+		}
+		if cluster.Conf.SwitchLockUserOnFreeze {
+			err = cluster.oldMaster.UnLockUsers()
+			cluster.LogSQL(logs, err, cluster.oldMaster.URL, "MasterFailover", config.LvlErr, "Could not unlock users, %s", err)
 		}
 		// Add the old master to the slaves list
-
 		cluster.oldMaster.SetState(stateSlave)
 		if cluster.Conf.MultiMaster == false {
 			cluster.slaves = append(cluster.slaves, cluster.oldMaster)

cluster/cluster_set.go

@@ -823,6 +823,10 @@ func (cluster *Cluster) SetSwitchSync(check bool) {
 	cluster.Conf.SwitchSync = check
 }
 
+func (cluster *Cluster) SetSwitchoverLockUserOnFreeze(check bool) {
+	cluster.Conf.SwitchLockUserOnFreeze = check
+}
+
 func (cluster *Cluster) SetLogLevel(level int) {
 	cluster.Conf.LogLevel = level
 }

cluster/cluster_tgl.go

@@ -13,6 +13,10 @@ import (
 	"github.com/signal18/replication-manager/config"
 )
 
+func (cluster *Cluster) SwitchSwitchoverLockUserOnFreeze() {
+	cluster.Conf.SwitchLockUserOnFreeze = !cluster.Conf.SwitchLockUserOnFreeze
+}
+
 func (cluster *Cluster) SwitchForceSlaveNoGtid() {
 	cluster.Conf.ForceSlaveNoGtid = !cluster.Conf.ForceSlaveNoGtid
 }

cluster/srv.go

@@ -1109,11 +1109,16 @@ func (server *ServerMonitor) freeze() bool {
 	logs, err := server.StopAllSlaves()
 	cluster.LogSQL(logs, err, server.URL, "Freeze", config.LvlErr, "Could not stop replicas source on %s ", server.URL)
 	cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModGeneral, config.LvlInfo, "Freezing writes set read only on %s", server.URL)
-	logs, err = dbhelper.SetReadOnly(server.Conn, true)
+	logs, err = server.SetReadOnly()
 	cluster.LogSQL(logs, err, server.URL, "Freeze", config.LvlInfo, "Could not set %s as read-only: %s", server.URL, err)
 	if err != nil {
 		return false
 	}
+	if cluster.Conf.SwitchLockUserOnFreeze {
+		cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModGeneral, config.LvlInfo, "Freezing writes locking users %s", server.URL)
+		err = server.LockUsers()
+		cluster.LogSQL(logs, err, server.URL, "Freeze", config.LvlErr, "Could not lock all users %s : %s", server.URL, err)
+	}
 	for i := cluster.Conf.SwitchWaitKill; i > 0; i -= 500 {
 		threads, logs, err := dbhelper.CheckLongRunningWrites(server.Conn, 0)
 		cluster.LogSQL(logs, err, server.URL, "Freeze", config.LvlErr, "Could not check long running writes %s as read-only: %s", server.URL, err)
@@ -1467,6 +1472,36 @@ func (server *ServerMonitor) UnInstallPlugin(name string) error {
 	return nil
 }
 
+func (server *ServerMonitor) LockUsers() error {
+	query := ""
+	cluster := server.ClusterGroup
+	for _, u := range server.Users.ToNewMap() {
+		if u.User != cluster.GetDbUser() && u.User != cluster.GetRplUser() && u.User != "root" {
+			query = "ALTER USER '" + u.User + "'@'" + u.Host + "' ACCOUNT LOCK"
+			err := server.ExecQueryNoBinLog(query, 5*time.Second)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (server *ServerMonitor) UnLockUsers() error {
+	query := ""
+	cluster := server.ClusterGroup
+	for _, u := range server.Users.ToNewMap() {
+		if u.User != cluster.GetDbUser() && u.User != cluster.GetRplUser() && u.User != "root" {
+			query = "ALTER USER '" + u.User + "'@'" + u.Host + "' ACCOUNT UNLOCK"
+			err := server.ExecQueryNoBinLog(query, 5*time.Second)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
 func (server *ServerMonitor) Capture(cstate *state.CapturedState) error {
 	cluster := server.ClusterGroup
 	if server.InCaptureMode {

cluster/srv_job.go

@@ -1210,7 +1210,7 @@ func (server *ServerMonitor) JobsCheckRunning() error {
 	}
 
 	if server.Conn == nil {
-		return fmt.Errorf("No connection pool on %s: %s", server.URL)
+		return fmt.Errorf("No connection pool on %s", server.URL)
 	}
 
 	Conn, err := server.GetConnNoBinlog(server.Conn)

config/config.go

@@ -206,6 +206,8 @@ type Config struct {
 	SwitchSlaveWaitRouteChange                int                    `mapstructure:"switchover-wait-route-change" toml:"switchover-wait-route-change" json:"switchoverWaitRouteChange"`
 	SwitchDecreaseMaxConn                     bool                   `mapstructure:"switchover-decrease-max-conn" toml:"switchover-decrease-max-conn" json:"switchoverDecreaseMaxConn"`
 	SwitchDecreaseMaxConnValue                int64                  `mapstructure:"switchover-decrease-max-conn-value" toml:"switchover-decrease-max-conn-value" json:"switchoverDecreaseMaxConnValue"`
+	SwitchLockUserOnFreeze                    bool                   `mapstructure:"switchover-lock-user-on-freeze" toml:"switchover-lock-user-on-freeze" json:"switchoverLockUserOnFreeze"`
+	SwitchRedirectOnFreeze                    bool                   `mapstructure:"switchover-redirect-on-freeze" toml:"switchover-redirect-on-freeze" json:"switchoverRedirectOnFreeze"`
 	FailLimit                                 int                    `mapstructure:"failover-limit" toml:"failover-limit" json:"failoverLimit"`
 	PreScript                                 string                 `mapstructure:"failover-pre-script" toml:"failover-pre-script" json:"failoverPreScript"`
 	PostScript                                string                 `mapstructure:"failover-post-script" toml:"failover-post-script" json:"failoverPostScript"`

server/api_cluster.go

@@ -2072,6 +2072,8 @@ func (repman *ReplicationManager) switchClusterSettings(mycluster *cluster.Clust
 		mycluster.SwitchRejoinForceRestore()
 	case "switchover-at-sync":
 		mycluster.SwitchSwitchoverSync()
+	case "switchover-lock-user-on-freeze":
+		mycluster.SwitchSwitchoverLockUserOnFreeze()
 	case "check-replication-filters":
 		mycluster.SwitchCheckReplicationFilters()
 	case "check-replication-state":
@@ -3015,6 +3017,8 @@ func (repman *ReplicationManager) setClusterSetting(mycluster *cluster.Cluster,
 		mycluster.Conf.AutorejoinForceRestore = isactive
 	case "switchover-at-sync":
 		mycluster.Conf.SwitchSync = isactive
+	case "switchover-lock-user-on-freeze":
+		mycluster.Conf.SwitchLockUserOnFreeze = isactive
 	case "check-replication-filters":
 		mycluster.Conf.CheckReplFilter = isactive
 	case "check-replication-state":

server/server.go

@@ -420,6 +420,9 @@ func (repman *ReplicationManager) AddFlags(flags *pflag.FlagSet, conf *config.Co
 	flags.BoolVar(&conf.SwitchSync, "switchover-at-sync", false, "Switchover Only  when state semisync is sync for last status")
 	flags.BoolVar(&conf.SwitchGtidCheck, "switchover-at-equal-gtid", false, "Switchover only when slaves are fully in sync")
 	flags.BoolVar(&conf.SwitchSlaveWaitCatch, "switchover-slave-wait-catch", true, "Switchover wait for slave to catch with replication, not needed in GTID mode but enable to detect possible issues like witing on old master")
+	flags.BoolVar(&conf.SwitchLockUserOnFreeze, "switchover-lock-user-on-freeze", false, "Switchover lock all non replication-manager users on freeze")
+	//	flags.BoolVar(&conf.SwitchRedirectOnFreeze, "switchover-redirect-on-freeze", false, "Switchover redirect traffic to dummy")
+
 	flags.BoolVar(&conf.SwitchDecreaseMaxConn, "switchover-decrease-max-conn", true, "Switchover decrease max connection on old master")
 	flags.BoolVar(&conf.SwitchoverCopyOldLeaderGtid, "switchover-copy-old-leader-gtid", false, "Switchover copy old leader GTID")
 	flags.Int64Var(&conf.SwitchDecreaseMaxConnValue, "switchover-decrease-max-conn-value", 10, "Switchover decrease max connection to this value different according to flavor")
@@ -788,7 +791,7 @@ func (repman *ReplicationManager) AddFlags(flags *pflag.FlagSet, conf *config.Co
 	flags.StringVar(&conf.BackupMyLoaderPath, "backup-myloader-path", "/usr/bin/myloader", "Path to myloader binary")
 	flags.StringVar(&conf.BackupMyLoaderOptions, "backup-myloader-options", "--overwrite-tables --verbose=3", "Extra options")
 	flags.StringVar(&conf.BackupMyDumperOptions, "backup-mydumper-options", "--chunk-filesize=1000 --compress --less-locking --verbose=3 --triggers --routines --events --trx-consistency-only --kill-long-queries", "Extra options")
-	flags.StringVar(&conf.BackupMyDumperRegex, "backup-mydumper-regex", `^(?!(sys\.|performance_schema\.|information_schema\.|replication_manager_schema\.jobs$))`, "Mydumper regex for backup")
+	flags.StringVar(&conf.BackupMyDumperRegex, "backup-mydumper-regex", `^(?!(sys\.|performance_schema\.|information_schema\.|replication_manager_schema\.jobs|mysql\.gtid_slave_pos$))`, "Mydumper regex for backup")
 	flags.StringVar(&conf.BackupMysqldumpPath, "backup-mysqldump-path", "", "Path to mysqldump binary")
 	flags.StringVar(&conf.BackupMysqldumpOptions, "backup-mysqldump-options", "--hex-blob --single-transaction --verbose --all-databases --routines=true --triggers=true --system=all", "Extra options")
 	flags.StringVar(&conf.BackupMysqlbinlogPath, "backup-mysqlbinlog-path", "", "Path to mysqlbinlog binary")

share/dashboard_react/src/Pages/Settings/RepFailOverSettings.jsx

@@ -227,6 +227,24 @@ function RepFailOverSettings({ selectedCluster, user, openConfirmModal, closeCon
         />
       )
     },
+    {
+      key: 'Switchover lock users on freeze workload ',
+      value: (
+        <RMSwitch
+          confirmTitle={'Confirm switch settings for sswitchover-lock-user-on-freeze?'}
+          onChange={() =>
+            dispatch(
+              switchSetting({
+                clusterName: selectedCluster?.name,
+                setting: 'switchover-lock-user-on-freeze'
+              })
+            )
+          }
+          isDisabled={user?.grants['cluster-settings'] == false}
+          isChecked={selectedCluster?.config?.switchLockUserOnFreeze}
+        />
+      )
+    },
     {
       key: 'Switchover replication maximum delay',
       value: (