Skip to content

Commit 6d4bc95

Browse files
svaroquisvaroqui
authored
Merge branch 'develop' into gzip
2 parents c00a557 + 9dbc0a6 commit 6d4bc95

25 files changed

+706
-59
lines changed

‎clients/client_cmd.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -276,11 +276,15 @@ func init() {
276276
rootClientCmd.AddCommand(switchoverCmd)
277277
initSwitchoverFlags(switchoverCmd)
278278
initClusterFlags(switchoverCmd)
279+
279280
rootClientCmd.AddCommand(failoverCmd)
280281
initFailoverFlags(failoverCmd)
281282
initClusterFlags(failoverCmd)
283+
282284
rootClientCmd.AddCommand(topologyCmd)
283285
initClusterFlags(topologyCmd)
286+
initServerApiFlags(topologyCmd)
287+
284288
rootClientCmd.AddCommand(apiCmd)
285289
initApiFlags(apiCmd)
286290
initClusterFlags(apiCmd)

‎cluster/cluster.go

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ type Cluster struct {
9898
IsNotMonitoring bool `json:"isNotMonitoring"`
9999
IsCapturing bool `json:"isCapturing"`
100100
IsGitPull bool `json:"isGitPull"`
101+
IsAlertDisable bool `json:"isAlertDisable"`
101102
Conf config.Config `json:"config"`
102103
Confs *config.ConfVersion `json:"-"`
103104
CleanAll bool `json:"cleanReplication"` //used in testing
@@ -184,6 +185,7 @@ type Cluster struct {
184185
idSchedulerRollingRestart cron.EntryID `json:"-"`
185186
idSchedulerDbsjobsSsh cron.EntryID `json:"-"`
186187
idSchedulerRollingReprov cron.EntryID `json:"-"`
188+
idSchedulerAlertDisable cron.EntryID `json:"-"`
187189
WaitingRejoin int `json:"waitingRejoin"`
188190
WaitingSwitchover int `json:"waitingSwitchover"`
189191
WaitingFailover int `json:"waitingFailover"`
@@ -405,9 +407,10 @@ func (cluster *Cluster) InitFromConf() {
405407
cluster.LogPrintf("START", "Replication manager started with version: %s", cluster.Conf.Version)
406408

407409
if cluster.Conf.MailTo != "" {
408-
msg := "Replication manager started with version: " + cluster.Conf.Version
410+
msg := "Replication-Manager started\nVersion: " + cluster.Conf.Version
409411
subj := "Replication-Manager started"
410412
alert := alert.Alert{}
413+
alert.Cluster = cluster.Name
411414
go alert.EmailMessage(msg, subj, cluster.Conf)
412415
}
413416

@@ -517,6 +520,7 @@ func (cluster *Cluster) initScheduler() {
517520
cluster.SetSchedulerSlaRotate()
518521
cluster.SetSchedulerRollingRestart()
519522
cluster.SetSchedulerDbJobsSsh()
523+
cluster.SetSchedulerAlertDisable()
520524
cluster.scheduler.Start()
521525
}
522526

@@ -754,11 +758,12 @@ func (cluster *Cluster) Stop() {
754758
}
755759

756760
func (cluster *Cluster) Save() error {
761+
//Needed to preserve diretory before Pull
757762
if !cluster.IsGitPull && cluster.Conf.Cloud18 {
758763
return nil
759764
}
760765
_, file, no, ok := runtime.Caller(1)
761-
if ok {
766+
if ok && cluster.GetLogLevel() > 3 {
762767
cluster.LogPrintf(LvlInfo, "Saved called from %s#%d\n", file, no)
763768
}
764769
type Save struct {

‎cluster/cluster_acl.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,22 @@ func (cluster *Cluster) IsURLPassDatabasesACL(strUser string, URL string) bool {
162162
return true
163163
}
164164
}
165+
if cluster.APIUsers[strUser].Grants[config.GrantClusterSwitchover] {
166+
if strings.Contains(URL, "/actions/switchover") {
167+
return true
168+
}
169+
}
170+
if cluster.APIUsers[strUser].Grants[config.GrantClusterFailover] {
171+
if strings.Contains(URL, "/actions/set-prefered") {
172+
return true
173+
}
174+
if strings.Contains(URL, "/actions/set-unrated") {
175+
return true
176+
}
177+
if strings.Contains(URL, "/actions/set-ignored") {
178+
return true
179+
}
180+
}
165181
if cluster.APIUsers[strUser].Grants[config.GrantDBKill] {
166182
if strings.Contains(URL, "/actions/kill") {
167183
return true

‎cluster/cluster_chk.go

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -481,8 +481,8 @@ func (cluster *Cluster) CheckAlert(state state.State) {
481481

482482
if strings.Contains(cluster.Conf.MonitoringAlertTrigger, state.ErrKey) {
483483
a := alert.Alert{
484-
State: state.ErrKey,
485-
Origin: cluster.Name,
484+
State: state.ErrKey,
485+
Cluster: cluster.Name,
486486
}
487487

488488
err := cluster.SendAlert(a)
@@ -494,13 +494,18 @@ func (cluster *Cluster) CheckAlert(state state.State) {
494494
}
495495

496496
func (cluster *Cluster) SendAlert(alert alert.Alert) error {
497+
if cluster.IsAlertDisable {
498+
cluster.LogPrintf(LvlInfo, "Cancel alert caused by alert disabled from scheduler")
499+
return nil
500+
}
497501
if cluster.Conf.MailTo != "" {
498502
go alert.EmailMessage("", "", cluster.Conf)
499503
}
504+
500505
if cluster.Conf.AlertScript != "" {
501506
cluster.LogPrintf("INFO", "Calling alert script")
502507
var out []byte
503-
out, err := exec.Command(cluster.Conf.AlertScript, alert.Origin, alert.PrevState, alert.State).CombinedOutput()
508+
out, err := exec.Command(cluster.Conf.AlertScript, alert.Cluster, alert.Host, alert.PrevState, alert.State).CombinedOutput()
504509
if err != nil {
505510
cluster.LogPrintf("ERROR", "%s", err)
506511
}

‎cluster/cluster_get.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,27 @@ func (cluster *Cluster) GetGroupReplicationWhiteList() string {
359359
return strings.Join(gcomms, ",")
360360
}
361361

362+
func (cluster *Cluster) GetPreferedMasterList() string {
363+
var prefmaster []string
364+
for _, server := range cluster.Servers {
365+
if server.Prefered {
366+
prefmaster = append(prefmaster, server.URL)
367+
}
368+
}
369+
return strings.Join(prefmaster, ",")
370+
}
371+
372+
373+
func (cluster *Cluster) GetIgnoredHostList() string {
374+
var prevIgnored []string
375+
for _, server := range cluster.Servers {
376+
if server.Ignored {
377+
prevIgnored = append(prevIgnored, server.URL)
378+
}
379+
}
380+
return strings.Join(prevIgnored, ",")
381+
}
382+
362383
func (cluster *Cluster) GetGComm() string {
363384
var gcomms []string
364385
for _, server := range cluster.Servers {

‎cluster/cluster_has.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -348,7 +348,7 @@ func (cluster *Cluster) IsDiscovered() bool {
348348
}
349349

350350
func (cluster *Cluster) IsMultiMaster() bool {
351-
if cluster.GetTopology() != topoMultiMasterWsrep || cluster.GetTopology() != topoMultiMaster || cluster.GetTopology() != topoMultiMasterRing {
351+
if cluster.GetTopology() == topoMultiMasterWsrep || cluster.GetTopology() == topoMultiMaster || cluster.GetTopology() == topoMultiMasterRing {
352352
return true
353353
}
354354
return false

‎cluster/cluster_sec.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,9 +197,10 @@ func (cluster *Cluster) RotatePasswords() error {
197197
cluster.LogPrintf("ALERT", msg)
198198
}
199199
if cluster.Conf.MailTo != "" {
200-
msg := "A password rotation has been made on Replication-Manager " + cluster.Name + " cluster. Check the new password on " + cluster.Conf.VaultServerAddr + " website on path " + cluster.Conf.VaultMount + cluster.Conf.User + " and " + cluster.Conf.VaultMount + cluster.Conf.RplUser + "."
200+
msg := "A password rotation has been made\nCheck the new password on " + cluster.Conf.VaultServerAddr + " website on path " + cluster.Conf.VaultMount + cluster.Conf.User + " and " + cluster.Conf.VaultMount + cluster.Conf.RplUser + "."
201201
subj := "Password Rotation Replication-Manager"
202202
alert := alert.Alert{}
203+
alert.Cluster=cluster.Name
203204
go alert.EmailMessage(msg, subj, cluster.Conf)
204205
}
205206

@@ -309,9 +310,10 @@ func (cluster *Cluster) RotatePasswords() error {
309310
cluster.LogPrintf("ALERT", msg)
310311
}
311312
if cluster.Conf.MailTo != "" {
312-
msg := "A password rotation has been made on Replication-Manager " + cluster.Name + " cluster. Check the new password on " + cluster.Conf.VaultServerAddr + " website on path " + cluster.Conf.VaultMount + cluster.Conf.User + " and " + cluster.Conf.VaultMount + cluster.Conf.RplUser + "."
313+
msg := "A password rotation has been made\nCheck the new password on " + cluster.Conf.VaultServerAddr + " website on path " + cluster.Conf.VaultMount + cluster.Conf.User + " and " + cluster.Conf.VaultMount + cluster.Conf.RplUser + "."
313314
subj := "Password Rotation Replication-Manager"
314315
alert := alert.Alert{}
316+
alert.Cluster=cluster.Name
315317
go alert.EmailMessage(msg, subj, cluster.Conf)
316318
}
317319

‎cluster/cluster_set.go

Lines changed: 52 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,31 @@ func (cluster *Cluster) SetSchedulerDbJobsSsh() {
262262
}
263263
}
264264

265+
266+
func (cluster *Cluster) SetSchedulerAlertDisable() {
267+
if cluster.HasSchedulerEntry("alertdisable") {
268+
cluster.LogPrintf(LvlInfo, "Stopping scheduler to disable alert")
269+
cluster.scheduler.Remove(cluster.idSchedulerAlertDisable)
270+
delete(cluster.Schedule, "alertdisable")
271+
}
272+
if cluster.Conf.SchedulerAlertDisable {
273+
var err error
274+
cluster.LogPrintf(LvlInfo, "Schedule disable alert at: %s", cluster.Conf.SchedulerAlertDisableCron)
275+
cluster.idSchedulerAlertDisable, err = cluster.scheduler.AddFunc(cluster.Conf.SchedulerAlertDisableCron, func() {
276+
cluster.LogPrintf(LvlInfo, "Alerting is disabled from scheduler")
277+
cluster.IsAlertDisable = true
278+
go cluster.WaitAlertDisable()
279+
})
280+
if err == nil {
281+
cluster.Schedule["alertdisable"] = cluster.scheduler.Entry(cluster.idSchedulerAlertDisable)
282+
}
283+
}
284+
}
285+
286+
func (cluster *Cluster) CompressBackups() {
287+
//cluster.LogPrintf(LvlInfo, "COUCOU compress backups")
288+
}
289+
265290
func (cluster *Cluster) SetCfgGroupDisplay(cfgGroup string) {
266291
cluster.cfgGroupDisplay = cfgGroup
267292
}
@@ -358,15 +383,33 @@ func (cluster *Cluster) SetBenchMethod(m string) {
358383
}
359384

360385
// SetPrefMaster is used by regtest test_switchover_semisync_switchback_prefmaster_norplcheck and API to force a server
361-
func (cluster *Cluster) SetPrefMaster(PrefMaster string) {
362-
cluster.Conf.PrefMaster = PrefMaster
386+
func (cluster *Cluster) SetPrefMaster(PrefMasterURL string) {
387+
var prefmasterlist []string
363388
for _, srv := range cluster.Servers {
364-
if strings.Contains(PrefMaster, srv.URL) {
389+
if strings.Contains(PrefMasterURL, srv.URL) {
365390
srv.SetPrefered(true)
391+
prefmasterlist = append(prefmasterlist, strings.Replace(srv.URL, srv.Domain + ":3306","", -1))
366392
} else {
367393
srv.SetPrefered(false)
368394
}
369395
}
396+
cluster.Conf.PrefMaster = strings.Join(prefmasterlist, ",")
397+
// fmt.Printf("Update config prefered Master: " + cluster.Conf.PrefMaster + "\n")
398+
}
399+
400+
// SetIgnoredHost
401+
func (cluster *Cluster) SetIgnoreSrv(IgnoredHostURL string) {
402+
var ignoresrvlist []string
403+
for _, srv := range cluster.Servers {
404+
if strings.Contains(IgnoredHostURL, srv.URL) {
405+
srv.SetIgnored(true)
406+
ignoresrvlist = append(ignoresrvlist, strings.Replace(srv.URL, srv.Domain + ":3306","", -1))
407+
} else {
408+
srv.SetIgnored(false)
409+
}
410+
}
411+
cluster.Conf.IgnoreSrv = strings.Join(ignoresrvlist, ",")
412+
// fmt.Printf("Update config ignored server: " + cluster.Conf.IgnoreSrv + "\n")
370413
}
371414

372415
func (cluster *Cluster) SetFailoverCtr(failoverCtr int) {
@@ -1186,6 +1229,12 @@ func (cluster *Cluster) SetSchedulerJobsSshCron(value string) error {
11861229
return nil
11871230
}
11881231

1232+
func (cluster *Cluster) SetSchedulerAlertDisableCron(value string) error {
1233+
cluster.Conf.SchedulerAlertDisableCron = value
1234+
cluster.SetSchedulerAlertDisable()
1235+
return nil
1236+
}
1237+
11891238
func (cluster *Cluster) SetDbServerHosts(value string) error {
11901239
cluster.Conf.Hosts = value
11911240
cluster.hostList = strings.Split(value, ",")

‎cluster/cluster_tgl.go

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,42 @@ func (cluster *Cluster) SwitchDBApplyDynamicConfig() {
2424
cluster.Conf.ProvDBApplyDynamicConfig = !cluster.Conf.ProvDBApplyDynamicConfig
2525
}
2626

27+
func (cluster *Cluster) SwitchForceSlaveReadOnly() {
28+
cluster.Conf.ForceSlaveReadOnly = !cluster.Conf.ForceSlaveReadOnly
29+
}
30+
31+
func (cluster *Cluster) SwitchForceBinlogRow() {
32+
cluster.Conf.ForceBinlogRow = !cluster.Conf.ForceBinlogRow
33+
}
34+
35+
func (cluster *Cluster) SwitchForceSlaveSemisync() {
36+
cluster.Conf.ForceSlaveSemisync = !cluster.Conf.ForceSlaveSemisync
37+
}
38+
39+
func (cluster *Cluster) SwitchForceSlaveHeartbeat() {
40+
cluster.Conf.ForceSlaveHeartbeat = !cluster.Conf.ForceSlaveHeartbeat
41+
}
42+
43+
func (cluster *Cluster) SwitchForceSlaveGtid() {
44+
cluster.Conf.ForceSlaveGtid = !cluster.Conf.ForceSlaveGtid
45+
}
46+
47+
func (cluster *Cluster) SwitchForceSlaveGtidStrict() {
48+
cluster.Conf.ForceSlaveGtidStrict = !cluster.Conf.ForceSlaveGtidStrict
49+
}
50+
51+
func (cluster *Cluster) SwitchForceBinlogCompress() {
52+
cluster.Conf.ForceBinlogCompress = !cluster.Conf.ForceBinlogCompress
53+
}
54+
55+
func (cluster *Cluster) SwitchForceBinlogAnnotate() {
56+
cluster.Conf.ForceBinlogAnnotate = !cluster.Conf.ForceBinlogAnnotate
57+
}
58+
59+
func (cluster *Cluster) SwitchForceBinlogSlowqueries() {
60+
cluster.Conf.ForceBinlogSlowqueries = !cluster.Conf.ForceBinlogSlowqueries
61+
}
62+
2763
func (cluster *Cluster) SwitchServerMaintenance(serverid uint64) {
2864
server := cluster.GetServerFromId(serverid)
2965
server.SwitchMaintenance()
@@ -174,6 +210,11 @@ func (cluster *Cluster) SwitchSchedulerRollingReprov() {
174210
cluster.SetSchedulerRollingReprov()
175211
}
176212

213+
func (cluster *Cluster) SwitchSchedulerAlertDisable() {
214+
cluster.Conf.SchedulerAlertDisable = !cluster.Conf.SchedulerAlertDisable
215+
216+
}
217+
177218
func (cluster *Cluster) SwitchGraphiteEmbedded() {
178219
cluster.Conf.GraphiteEmbedded = !cluster.Conf.GraphiteEmbedded
179220
}

‎cluster/cluster_wait.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -326,3 +326,9 @@ func (cluster *Cluster) WaitDatabaseCanConn() error {
326326
}
327327
return nil
328328
}
329+
330+
func (cluster *Cluster) WaitAlertDisable() {
331+
time.Sleep(time.Duration(cluster.Conf.SchedulerAlertDisableTime) * time.Second)
332+
cluster.LogPrintf(LvlErr, "Alerting is enabled from scheduler")
333+
cluster.IsAlertDisable = false
334+
}

‎cluster/srv_chk.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,7 @@ func (server *ServerMonitor) CheckSlaveSettings() {
168168
if server.ClusterGroup.Conf.ForceSlaveReadOnly && sl.ReadOnly == "OFF" && !server.ClusterGroup.IsInIgnoredReadonly(server) && !server.ClusterGroup.IsMultiMaster() {
169169
// In non-multimaster mode, enforce read-only flag if the option is set
170170
sl.SetReadOnly()
171-
server.ClusterGroup.LogPrintf("INFO", "Enforce read only on slave %s", sl.URL)
171+
server.ClusterGroup.LogPrintf("INFO", "Enforce read only on slave %s, ReadOnly:%s, InIgnored:%t MultiMaster:%t", sl.URL, sl.ReadOnly, server.ClusterGroup.IsInIgnoredReadonly(server), server.ClusterGroup.IsMultiMaster())
172172
}
173173
if server.ClusterGroup.Conf.ForceSlaveHeartbeat && sl.GetReplicationHearbeatPeriod() > 1 {
174174
dbhelper.SetSlaveHeartbeat(sl.Conn, "1", server.ClusterGroup.Conf.MasterConn, server.DBVersion)

‎cluster/srv_snd.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,8 @@ func (server *ServerMonitor) SendAlert() error {
104104
a := alert.Alert{
105105
State: server.State,
106106
PrevState: server.PrevState,
107-
Origin: server.URL,
107+
Host: server.URL,
108+
Cluster: server.GetCluster().Name,
108109
}
109110

110111
return server.ClusterGroup.SendAlert(a)

‎config/config.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -508,6 +508,9 @@ type Config struct {
508508
SchedulerBackupPhysical bool `mapstructure:"scheduler-db-servers-physical-backup" toml:"scheduler-db-servers-physical-backup" json:"schedulerDbServersPhysicalBackup"`
509509
SchedulerDatabaseLogs bool `mapstructure:"scheduler-db-servers-logs" toml:"scheduler-db-servers-logs" json:"schedulerDbServersLogs"`
510510
SchedulerDatabaseOptimize bool `mapstructure:"scheduler-db-servers-optimize" toml:"scheduler-db-servers-optimize" json:"schedulerDbServersOptimize"`
511+
SchedulerAlertDisable bool `mapstructure:"scheduler-alert-disable" toml:"scheduler-alert-disable" json:"schedulerAlertDisable"`
512+
SchedulerAlertDisableCron string `mapstructure:"scheduler-alert-disable-cron" toml:"scheduler-alert-disable-cron" json:"schedulerAlertDisableCron"`
513+
SchedulerAlertDisableTime int `mapstructure:"scheduler-alert-disable-time" toml:"scheduler-alert-disable-time" json:"schedulerAlertDisableTime"`
511514
BackupLogicalCron string `mapstructure:"scheduler-db-servers-logical-backup-cron" toml:"scheduler-db-servers-logical-backup-cron" json:"schedulerDbServersLogicalBackupCron"`
512515
BackupPhysicalCron string `mapstructure:"scheduler-db-servers-physical-backup-cron" toml:"scheduler-db-servers-physical-backup-cron" json:"schedulerDbServersPhysicalBackupCron"`
513516
BackupDatabaseLogCron string `mapstructure:"scheduler-db-servers-logs-cron" toml:"scheduler-db-servers-logs-cron" json:"schedulerDbServersLogsCron"`

‎doc/api.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ OUPUT:
6363
/api/clusters/{clusterName}/status
6464

6565
# API protected endpoints
66+
/api/clusters/{clusterName}/actions/switchover/{serverName}
6667

6768
/api/clusters/{clusterName}/actions/switchover
6869

‎etc/opensvc/cluster-api/cluster-demo/stephane.toml

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ monitoring-save-config=true
4646

4747
mail-from = "[email protected]"
4848
mail-to = "[email protected]"
49-
#mail-smtp-addr = "in-v3.mailjet.com:587"
49+
mail-smtp-addr = "in-v3.mailjet.com:587"
5050
mail-smtp-user = "53b0c2c80298023abf083a106261c789"
5151
mail-smtp-password = "hash_497b7124700d1a7ffaacbc88d672f92714a4bfda2f0ae3747a8647c8bf75db2809513f4f54463d081ab2813ac8c45e8d"
5252
alert-pushover-app-token="hash_046d7d15e81ebbfe4e6e55ac1a3bb3312612c2a52a32cf3fde26b49260b32d043f113904e8e124f9e01ea12e8fd0"
@@ -66,3 +66,4 @@ backup-streaming-aws-access-secret = "hash_7522e94c4ceb43505c950e856742345a473a3
6666
backup-streaming-endpoint= "https://s3.signal18.io/"
6767
backup-streaming-region= "fr-1"
6868
backup-streaming-bucket= "repman"
69+
backup-mysqldump-path = "/usr/local/bin/mysqldump"

0 commit comments

Comments
 (0)