Skip to content

Commit b893fc4

Browse files
svaroquisvaroqui
committed
Enable to custumize mydumper myloader extra parameters
myloader add enable binlog that was brecking replication on restaure backup Enable to custumize db job script via parameter to specify script to run Unify export env variable to all ssh execution add REPLICATION_MANAGER_HOST_NAME REPLICATION_MANAGER_HOST_USER REPLICATION_MANAGER_HOST_PASSWORD REPLICATION_MANAGER_HOST_PORT
1 parent a0d767a commit b893fc4

File tree

6 files changed

+64
-69
lines changed

6 files changed

+64
-69
lines changed

‎cluster/prov_onpremise_db.go

+13-33
Original file line numberDiff line numberDiff line change
@@ -98,19 +98,19 @@ func (cluster *Cluster) OnPremiseStopDatabaseService(server *ServerMonitor) erro
9898
}
9999

100100
func (cluster *Cluster) OnPremiseSetEnv(client *sshclient.Client, server *ServerMonitor) error {
101-
adminuser := "admin"
102-
adminpassword := "repman"
103101

104-
if user, ok := server.ClusterGroup.APIUsers[adminuser]; ok {
105-
adminpassword = user.Password
106-
}
107-
buf := strings.NewReader("export MYSQL_ROOT_PASSWORD=\"" + server.Pass + "\";export REPLICATION_MANAGER_URL=\"https://" + server.ClusterGroup.Conf.MonitorAddress + ":" + server.ClusterGroup.Conf.APIPort + "\";export REPLICATION_MANAGER_USER=\"" + adminuser + "\";export REPLICATION_MANAGER_PASSWORD=\"" + adminpassword + "\";export REPLICATION_MANAGER_HOST_NAME=\"" + server.Host + "\";export REPLICATION_MANAGER_HOST_PORT=\"" + server.Port + "\";export REPLICATION_MANAGER_CLUSTER_NAME=\"" + server.ClusterGroup.Name + "\"")
108-
/* REPLICATION_MANAGER_USER
109-
REPLICATION_MANAGER_PASSWORD
110-
REPLICATION_MANAGER_URL
111-
REPLICATION_MANAGER_CLUSTER_NAME
112-
REPLICATION_MANAGER_HOST_NAME
113-
REPLICATION_MANAGER_HOST_PORT */
102+
buf := strings.NewReader(server.GetSshEnv())
103+
/*
104+
REPLICATION_MANAGER_USER
105+
REPLICATION_MANAGER_PASSWORD
106+
REPLICATION_MANAGER_URL
107+
REPLICATION_MANAGER_CLUSTER_NAME
108+
REPLICATION_MANAGER_HOST_NAME
109+
REPLICATION_MANAGER_HOST_USER
110+
REPLICATION_MANAGER_HOST_PASSWORD
111+
REPLICATION_MANAGER_HOST_PORT
112+
113+
*/
114114
var (
115115
stdout bytes.Buffer
116116
stderr bytes.Buffer
@@ -122,15 +122,6 @@ func (cluster *Cluster) OnPremiseSetEnv(client *sshclient.Client, server *Server
122122
}
123123
server.ClusterGroup.LogPrintf(LvlInfo, "OnPremise start database install secret env: %s", stdout.String())
124124

125-
/*out := stdout.String()
126-
out, err := client.Cmd("export MYSQL_ROOT_PASSWORD=" + server.Pass).Cmd("export REPLICATION_MANAGER_URL=https://" + server.ClusterGroup.Conf.MonitorAddress + ":" + server.ClusterGroup.Conf.APIPort).Cmd("export REPLICATION_MANAGER_USER=" + adminuser).Cmd("export REPLICATION_MANAGER_PASSWORD=" + adminpassword).Cmd("export REPLICATION_MANAGER_HOST_NAME=" + server.Host).Cmd("export REPLICATION_MANAGER_HOST_PORT=" + server.Port).Cmd("export REPLICATION_MANAGER_CLUSTER_NAME=" + server.ClusterGroup.Name).SmartOutput()
127-
if err != nil {
128-
server.ClusterGroup.LogPrintf(LvlErr, "OnPremise start database : %s", err)
129-
return err
130-
server.ClusterGroup.LogPrintf(LvlInfo, "OnPremise start database install secret env: %s", string(out))
131-
132-
}*/
133-
134125
return nil
135126
}
136127

@@ -156,12 +147,7 @@ func (cluster *Cluster) OnPremiseStartDatabaseService(server *ServerMonitor) err
156147
buf := new(bytes.Buffer)
157148
buf.ReadFrom(filerc)
158149

159-
adminuser := "admin"
160-
adminpassword := "repman"
161-
if user, ok := server.ClusterGroup.APIUsers[adminuser]; ok {
162-
adminpassword = user.Password
163-
}
164-
buf2 := strings.NewReader("export MYSQL_ROOT_PASSWORD=\"" + server.Pass + "\";export REPLICATION_MANAGER_URL=\"https://" + server.ClusterGroup.Conf.MonitorAddress + ":" + server.ClusterGroup.Conf.APIPort + "\";export REPLICATION_MANAGER_USER=\"" + adminuser + "\";export REPLICATION_MANAGER_PASSWORD=\"" + adminpassword + "\";export REPLICATION_MANAGER_HOST_NAME=\"" + server.Host + "\";export REPLICATION_MANAGER_HOST_PORT=\"" + server.Port + "\";export REPLICATION_MANAGER_CLUSTER_NAME=\"" + server.ClusterGroup.Name + "\"\n")
150+
buf2 := strings.NewReader(server.GetSshEnv())
165151
r := io.MultiReader(buf2, buf)
166152

167153
var (
@@ -174,12 +160,6 @@ func (cluster *Cluster) OnPremiseStartDatabaseService(server *ServerMonitor) err
174160
out := stdout.String()
175161

176162
server.ClusterGroup.LogPrintf(LvlInfo, "OnPremise start script: %s ,out: %s ,err: %s", cmd, out, stderr.String())
177-
/* out, err := client.Cmd(cmd).SmartOutput()
178-
if err != nil {
179-
server.ClusterGroup.LogPrintf(LvlErr, "OnPremise start database : %s", err)
180-
return err
181-
}
182163

183-
*/
184164
return nil
185165
}

‎cluster/srv_get.go

+20
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,26 @@ func (server *ServerMonitor) GetProcessList() []dbhelper.Processlist {
3131
return server.FullProcessList
3232
}
3333

34+
func (server *ServerMonitor) GetSshEnv() string {
35+
/*
36+
REPLICATION_MANAGER_USER
37+
REPLICATION_MANAGER_PASSWORD
38+
REPLICATION_MANAGER_URL
39+
REPLICATION_MANAGER_CLUSTER_NAME
40+
REPLICATION_MANAGER_HOST_NAME
41+
REPLICATION_MANAGER_HOST_USER
42+
REPLICATION_MANAGER_HOST_PASSWORD
43+
REPLICATION_MANAGER_HOST_PORT
44+
45+
*/
46+
adminuser := "admin"
47+
adminpassword := "repman"
48+
if user, ok := server.ClusterGroup.APIUsers[adminuser]; ok {
49+
adminpassword = user.Password
50+
}
51+
return "export REPLICATION_MANAGER_HOST_USER=\"" + server.User + "\";export REPLICATION_MANAGER_HOST_PASSWORD=\"" + server.Pass + "\";export MYSQL_ROOT_PASSWORD=\"" + server.Pass + "\";export REPLICATION_MANAGER_URL=\"https://" + server.ClusterGroup.Conf.MonitorAddress + ":" + server.ClusterGroup.Conf.APIPort + "\";export REPLICATION_MANAGER_USER=\"" + adminuser + "\";export REPLICATION_MANAGER_PASSWORD=\"" + adminpassword + "\";export REPLICATION_MANAGER_HOST_NAME=\"" + server.Host + "\";export REPLICATION_MANAGER_HOST_PORT=\"" + server.Port + "\";export REPLICATION_MANAGER_CLUSTER_NAME=\"" + server.ClusterGroup.Name + "\"\n"
52+
}
53+
3454
func (server *ServerMonitor) GetUniversalGtidServerID() uint64 {
3555

3656
if server.IsMariaDB() {

‎cluster/srv_job.go

+16-36
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,11 @@ func (server *ServerMonitor) JobZFSSnapBack() (int64, error) {
376376
func (server *ServerMonitor) JobReseedMyLoader() {
377377

378378
threads := strconv.Itoa(server.ClusterGroup.Conf.BackupLogicalLoadThreads)
379-
dumpCmd := exec.Command(server.ClusterGroup.GetMyLoaderPath(), "--overwrite-tables", "--directory="+server.ClusterGroup.master.GetMasterBackupDirectory(), "--verbose=3", "--threads="+threads, "--host="+misc.Unbracket(server.Host), "--port="+server.Port, "--user="+server.ClusterGroup.dbUser, "--password="+server.ClusterGroup.dbPass)
379+
380+
myargs := strings.Split(strings.ReplaceAll(server.ClusterGroup.Conf.BackupMyLoaderOptions, " ", " "), " ")
381+
myargs = append(myargs, "--directory="+server.ClusterGroup.master.GetMasterBackupDirectory(), "--threads="+threads, "--host="+misc.Unbracket(server.Host), "--port="+server.Port, "--user="+server.ClusterGroup.dbUser, "--password="+server.ClusterGroup.dbPass)
382+
dumpCmd := exec.Command(server.ClusterGroup.GetMyLoaderPath(), myargs...)
383+
380384
server.ClusterGroup.LogPrintf(LvlInfo, "Command: %s", strings.Replace(dumpCmd.String(), server.ClusterGroup.dbPass, "XXXX", 1))
381385

382386
stdoutIn, _ := dumpCmd.StdoutPipe()
@@ -664,9 +668,7 @@ func (server *ServerMonitor) JobBackupLogical() error {
664668
}
665669

666670
dumpargs := strings.Split(strings.ReplaceAll("--defaults-file="+file+" "+server.ClusterGroup.getDumpParameter()+" "+dumpslave+" "+usegtid+" "+events, " ", " "), " ")
667-
668671
dumpargs = append(dumpargs, "--apply-slave-statements", "--host="+misc.Unbracket(server.Host), "--port="+server.Port, "--user="+server.ClusterGroup.dbUser /*"--log-error="+server.GetMyBackupDirectory()+"dump_error.log"*/)
669-
670672
dumpCmd := exec.Command(server.ClusterGroup.GetMysqlDumpPath(), dumpargs...)
671673

672674
server.ClusterGroup.LogPrintf(LvlInfo, "Command: %s ", strings.Replace(dumpCmd.String(), server.ClusterGroup.dbPass, "XXXX", -1))
@@ -738,27 +740,11 @@ func (server *ServerMonitor) JobBackupLogical() error {
738740
// --no-schemas --regex '^(?!(mysql))'
739741

740742
threads := strconv.Itoa(server.ClusterGroup.Conf.BackupLogicalDumpThreads)
741-
dumpCmd := exec.Command(server.ClusterGroup.GetMyDumperPath(), "--outputdir="+server.GetMyBackupDirectory(), "--chunk-filesize=1000", "--compress", "--less-locking", "--verbose=3", "--triggers", "--routines", "--events", "--trx-consistency-only", "--kill-long-queries", "--threads="+threads, "--host="+misc.Unbracket(server.Host), "--port="+server.Port, "--user="+server.ClusterGroup.dbUser, "--password="+server.ClusterGroup.dbPass)
742-
server.ClusterGroup.LogPrintf(LvlInfo, "%s", strings.Replace(dumpCmd.String(), server.ClusterGroup.dbPass, "XXXX", 1))
743-
/* pr, pw := io.Pipe()
744-
defer pw.Close()
745-
746-
// tell the command to write to our pipe
747-
748-
dumpCmd.Stdout = pw
749-
dumpCmd.Stderr = pw
743+
myargs := strings.Split(strings.ReplaceAll(server.ClusterGroup.Conf.BackupMyLoaderOptions, " ", " "), " ")
744+
myargs = append(myargs, "--outputdir="+server.GetMyBackupDirectory(), "--threads="+threads, "--host="+misc.Unbracket(server.Host), "--port="+server.Port, "--user="+server.ClusterGroup.dbUser, "--password="+server.ClusterGroup.dbPass)
745+
dumpCmd := exec.Command(server.ClusterGroup.GetMyDumperPath(), myargs...)
750746

751-
buf := new(bytes.Buffer)
752-
go func() {
753-
defer pr.Close()
754-
// copy the data written to the PipeReader via the cmd to stdout
755-
if _, err := io.Copy(buf, pr); err != nil {
756-
server.ClusterGroup.LogPrintf(LvlErr, "MyDumper: %s", err)
757-
}
758-
server.ClusterGroup.LogPrintf(LvlInfo, "%s", buf.String())
759-
}()
760-
761-
*/
747+
server.ClusterGroup.LogPrintf(LvlInfo, "%s", strings.Replace(dumpCmd.String(), server.ClusterGroup.dbPass, "XXXX", 1))
762748
stdoutIn, _ := dumpCmd.StdoutPipe()
763749
stderrIn, _ := dumpCmd.StderrPipe()
764750
dumpCmd.Start()
@@ -884,7 +870,11 @@ func (server *ServerMonitor) JobRunViaSSH() error {
884870
stdout bytes.Buffer
885871
stderr bytes.Buffer
886872
)
887-
filerc, err2 := os.Open(server.Datadir + "/init/init/dbjobs_new")
873+
scriptpath := server.Datadir + "/init/init/dbjobs_new"
874+
if server.GetCluster().GetConf().OnPremiseSSHDbJobScript == "" {
875+
scriptpath = server.GetCluster().GetConf().OnPremiseSSHDbJobScript
876+
}
877+
filerc, err2 := os.Open(scriptpath)
888878
if err2 != nil {
889879
server.ClusterGroup.LogPrintf(LvlErr, "JobRunViaSSH %s", err2)
890880
return errors.New("Cancel dbjob can't open script")
@@ -893,20 +883,10 @@ func (server *ServerMonitor) JobRunViaSSH() error {
893883
defer filerc.Close()
894884
buf := new(bytes.Buffer)
895885
buf.ReadFrom(filerc)
896-
/*
897-
adminuser := "admin"
898-
adminpassword := "repman"
899-
if user, ok := server.ClusterGroup.APIUsers[adminuser]; ok {
900-
adminpassword = user.Password
901-
}
902-
_, err = client.Cmd("export MYSQL_ROOT_PASSWORD=" + server.Pass).Cmd("export REPLICATION_MANAGER_URL=" + server.ClusterGroup.Conf.MonitorAddress + ":" + server.ClusterGroup.Conf.APIPort).Cmd("export REPLICATION_MANAGER_USER=" + adminuser).Cmd("export REPLICATION_MANAGER_PASSWORD=" + adminpassword).Cmd("export REPLICATION_MANAGER_HOST_NAME=" + server.Host).Cmd("export REPLICATION_MANAGER_HOST_PORT=" + server.Port).Cmd("export REPLICATION_MANAGER_CLUSTER_NAME=" +
903-
server.ClusterGroup.Name).SmartOutput()
904-
if err != nil {
905-
return errors.New("JobRunViaSSH Setup env variables via SSH %s" + err.Error())
906-
}*/
907886

908-
buf2 := strings.NewReader("export MYSQL_ROOT_PASSWORD=\"" + server.Pass + "\"\n")
887+
buf2 := strings.NewReader(server.GetSshEnv())
909888
r := io.MultiReader(buf2, buf)
889+
910890
if client.Shell().SetStdio(r, &stdout, &stderr).Start(); err != nil {
911891
server.ClusterGroup.LogPrintf(LvlErr, "Database jobs run via SSH: %s", stderr.String())
912892
}

‎config/config.go

+3
Original file line numberDiff line numberDiff line change
@@ -334,6 +334,7 @@ type Config struct {
334334
OnPremiseSSHPrivateKey string `mapstructure:"onpremise-ssh-private-key" toml:"onpremise-ssh-private-key" json:"onpremiseSshPrivateKey"`
335335
OnPremiseSSHStartDbScript string `mapstructure:"onpremise-ssh-start-db-script" toml:"onpremise-ssh-start-db-script" json:"onpremiseSshStartDbScript"`
336336
OnPremiseSSHStartProxyScript string `mapstructure:"onpremise-ssh-start-proxy-script" toml:"onpremise-ssh-start-proxy-script" json:"onpremiseSshStartProxyScript"`
337+
OnPremiseSSHDbJobScript string `mapstructure:"onpremise-ssh-db-job-script" toml:"onpremise-ssh-db-job-script" json:"onpremiseSshDbJobScript"`
337338
ProvOpensvcP12Certificate string `mapstructure:"opensvc-p12-certificate" toml:"opensvc-p12-certificat" json:"opensvcP12Certificate"`
338339
ProvOpensvcP12Secret string `mapstructure:"opensvc-p12-secret" toml:"opensvc-p12-secret" json:"opensvcP12Secret"`
339340
ProvOpensvcUseCollectorAPI bool `mapstructure:"opensvc-use-collector-api" toml:"opensvc-use-collector-api" json:"opensvcUseCollectorApi"`
@@ -498,6 +499,8 @@ type Config struct {
498499
BackupMysqldumpOptions string `mapstructure:"backup-mysqldump-options" toml:"backup-mysqldump-options" json:"backupMysqldumpOptions"`
499500
BackupMyDumperPath string `mapstructure:"backup-mydumper-path" toml:"backup-mydumper-path" json:"backupMydumperPath"`
500501
BackupMyLoaderPath string `mapstructure:"backup-myloader-path" toml:"backup-myloader-path" json:"backupMyloaderPath"`
502+
BackupMyLoaderOptions string `mapstructure:"backup-myloader-options" toml:"backup-myloader-options" json:"backupMyloaderOptions"`
503+
BackupMyDumperOptions string `mapstructure:"backup-mydumper-options" toml:"backup-mydumper-options" json:"backupMyDumperOptions"`
501504
BackupMysqlbinlogPath string `mapstructure:"backup-mysqlbinlog-path" toml:"backup-mysqlbinlog-path" json:"backupMysqlbinlogPath"`
502505
BackupMysqlclientPath string `mapstructure:"backup-mysqlclient-path" toml:"backup-mysqlclient-path" json:"backupMysqlclientgPath"`
503506
BackupBinlogs bool `mapstructure:"backup-binlogs" toml:"backup-binlogs" json:"backupBinlogs"`

‎server/server_monitor.go

+4
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,8 @@ func init() {
365365

366366
monitorCmd.Flags().StringVar(&conf.BackupMyDumperPath, "backup-mydumper-path", "/usr/bin/mydumper", "Path to mydumper binary")
367367
monitorCmd.Flags().StringVar(&conf.BackupMyLoaderPath, "backup-myloader-path", "/usr/bin/myloader", "Path to myloader binary")
368+
monitorCmd.Flags().StringVar(&conf.BackupMyLoaderOptions, "backup-myloader-options", "--overwrite-tables --enable-binlog --verbose=3", "Extra options")
369+
monitorCmd.Flags().StringVar(&conf.BackupMyDumperOptions, "backup-mydumper-options", "--chunk-filesize=1000 --compress --less-locking --verbose=3 --triggers --routines --events --trx-consistency-only --kill-long-queries", "Extra options")
368370
monitorCmd.Flags().StringVar(&conf.BackupMysqldumpPath, "backup-mysqldump-path", "", "Path to mysqldump binary")
369371
monitorCmd.Flags().StringVar(&conf.BackupMysqldumpOptions, "backup-mysqldump-options", "--hex-blob --single-transaction --verbose --all-databases --routines=true --triggers=true --system=all", "Extra options")
370372
monitorCmd.Flags().StringVar(&conf.BackupMysqlbinlogPath, "backup-mysqlbinlog-path", "", "Path to mysqlbinlog binary")
@@ -434,6 +436,8 @@ func init() {
434436
monitorCmd.Flags().StringVar(&conf.OnPremiseSSHCredential, "onpremise-ssh-credential", "root:", "User:password for ssh if no password using current user private key")
435437
monitorCmd.Flags().StringVar(&conf.OnPremiseSSHStartDbScript, "onpremise-ssh-start-db-script", "", "Run via ssh a custom script to start database")
436438
monitorCmd.Flags().StringVar(&conf.OnPremiseSSHStartProxyScript, "onpremise-ssh-start-proxy-script", "", "Run via ssh a custom script to start proxy")
439+
monitorCmd.Flags().StringVar(&conf.OnPremiseSSHDbJobScript, "onpremise-ssh-db-job-script", "", "Run via ssh a custom script to execute database jobs")
440+
437441
if WithProvisioning == "ON" {
438442
monitorCmd.Flags().StringVar(&conf.ProvDatadirVersion, "prov-db-datadir-version", "10.2", "Empty datadir to deploy for localtest")
439443
monitorCmd.Flags().StringVar(&conf.ProvDiskSystemSize, "prov-db-disk-system-size", "2", "Disk in g for micro service VM")

0 commit comments

Comments
 (0)