Skip to content

Commit 08cd3a8

Browse files
caffeinated92caffeinated92
authored
Merge pull request #807 from signal18/jobs
Add option to keep old backup temporary before valid
2 parents 335039a + 31f5ded commit 08cd3a8

File tree

9 files changed

+192
-12
lines changed

9 files changed

+192
-12
lines changed

‎cluster/cluster_tgl.go

+4
Original file line numberDiff line numberDiff line change
@@ -619,3 +619,7 @@ func (cluster *Cluster) SwitchForceWriteConfig() {
619619
cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModGeneral, config.LvlInfo, "Configurator force write config files de-activated. Will create config files with suffix (.new) for conflicting files on next provision.")
620620
}
621621
}
622+
623+
func (cluster *Cluster) SwitchBackupKeepUntilValid() {
624+
cluster.Conf.BackupKeepUntilValid = !cluster.Conf.BackupKeepUntilValid
625+
}

‎cluster/srv_bck.go

+37
Original file line numberDiff line numberDiff line change
@@ -92,3 +92,40 @@ func (server *ServerMonitor) ReadLastMetadata(method string) (*config.BackupMeta
9292

9393
return meta, nil
9494
}
95+
96+
func (server *ServerMonitor) GetLatestMeta(method string) (int64, *config.BackupMetadata) {
97+
cluster := server.ClusterGroup
98+
var latest int64 = 0
99+
var meta *config.BackupMetadata
100+
cluster.BackupMetaMap.Range(func(k, v any) bool {
101+
m := v.(*config.BackupMetadata)
102+
valid := false
103+
switch method {
104+
case "logical":
105+
if m.BackupMethod == config.BackupMethodLogical {
106+
valid = true
107+
}
108+
case "physical":
109+
if m.BackupMethod == config.BackupMethodPhysical {
110+
valid = true
111+
}
112+
default:
113+
if m.BackupTool == method {
114+
valid = true
115+
}
116+
}
117+
118+
if m.Source != server.URL {
119+
valid = false
120+
}
121+
122+
if valid && latest < m.Id {
123+
latest = m.Id
124+
meta = m
125+
}
126+
127+
return true
128+
})
129+
130+
return latest, meta
131+
}

‎cluster/srv_job.go

+45-8
Original file line numberDiff line numberDiff line change
@@ -311,6 +311,17 @@ func (server *ServerMonitor) JobBackupPhysical() (int64, error) {
311311

312312
now := time.Now()
313313
// Reset last backup meta
314+
var prevId int64
315+
prev := cluster.BackupMetaMap.GetPreviousBackup(cluster.Conf.BackupPhysicalType, server.URL)
316+
if prev != nil {
317+
prevId = prev.Id
318+
}
319+
320+
// Remove from backup list, since the file will be replaced
321+
if !cluster.Conf.BackupKeepUntilValid {
322+
cluster.BackupMetaMap.Delete(prevId)
323+
}
324+
314325
server.LastBackupMeta.Physical = &config.BackupMetadata{
315326
Id: now.Unix(),
316327
StartTime: now,
@@ -320,6 +331,7 @@ func (server *ServerMonitor) JobBackupPhysical() (int64, error) {
320331
Source: server.URL,
321332
Dest: dest,
322333
Compressed: cluster.Conf.CompressBackups,
334+
Previous: prevId,
323335
}
324336

325337
cluster.BackupMetaMap.Set(server.LastBackupMeta.Physical.Id, server.LastBackupMeta.Physical)
@@ -1888,15 +1900,29 @@ func (server *ServerMonitor) JobBackupLogical() error {
18881900

18891901
cluster.SetInLogicalBackupState(true)
18901902
start := time.Now()
1903+
var prevId int64
1904+
prev := cluster.BackupMetaMap.GetPreviousBackup(cluster.Conf.BackupLogicalType, server.URL)
1905+
if prev != nil {
1906+
prevId = prev.Id
1907+
}
1908+
1909+
// Remove from backup list, since the file will be replaced
1910+
if !cluster.Conf.BackupKeepUntilValid {
1911+
cluster.BackupMetaMap.Delete(prevId)
1912+
}
1913+
18911914
server.LastBackupMeta.Logical = &config.BackupMetadata{
18921915
Id: start.Unix(),
18931916
StartTime: start,
18941917
BackupMethod: config.BackupMethodLogical,
18951918
BackupTool: cluster.Conf.BackupLogicalType,
18961919
BackupStrategy: config.BackupStrategyFull,
18971920
Source: server.URL,
1921+
Previous: prevId,
18981922
}
18991923

1924+
cluster.BackupMetaMap.Set(server.LastBackupMeta.Logical.Id, server.LastBackupMeta.Logical)
1925+
19001926
// Removing previous valid backup state and start
19011927
server.DelBackupLogicalCookie()
19021928

@@ -1934,10 +1960,10 @@ func (server *ServerMonitor) JobBackupLogical() error {
19341960
if e2 := server.JobsUpdateState(task, "Backup completed", 3, 1); e2 != nil {
19351961
cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlWarn, "Task only updated in runtime. Error while writing to jobs table: %s", e2.Error())
19361962
}
1937-
finfo, e3 := os.Stat(filename)
1963+
_, e3 := os.Stat(filename)
19381964
if e3 == nil {
19391965
server.LastBackupMeta.Logical.EndTime = time.Now()
1940-
server.LastBackupMeta.Logical.Size = finfo.Size()
1966+
server.LastBackupMeta.Logical.GetSize()
19411967
server.LastBackupMeta.Logical.Completed = true
19421968
server.SetBackupLogicalCookie(config.ConstBackupLogicalTypeMysqldump)
19431969
}
@@ -1959,10 +1985,10 @@ func (server *ServerMonitor) JobBackupLogical() error {
19591985
if e2 := server.JobsUpdateState(task, "Backup completed", 3, 1); e2 != nil {
19601986
cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlWarn, "Task only updated in runtime. Error while writing to jobs table: %s", e2.Error())
19611987
}
1962-
finfo, e3 := os.Stat(outputdir)
1988+
_, e3 := os.Stat(outputdir)
19631989
if e3 == nil {
19641990
server.LastBackupMeta.Logical.EndTime = time.Now()
1965-
server.LastBackupMeta.Logical.Size = finfo.Size()
1991+
server.LastBackupMeta.Logical.GetSize()
19661992
server.LastBackupMeta.Logical.Completed = true
19671993
server.SetBackupLogicalCookie(config.ConstBackupLogicalTypeDumpling)
19681994
}
@@ -1985,10 +2011,10 @@ func (server *ServerMonitor) JobBackupLogical() error {
19852011
cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlWarn, "Task only updated in runtime. Error while writing to jobs table: %s", e2.Error())
19862012
}
19872013

1988-
finfo, e3 := os.Stat(outputdir)
2014+
_, e3 := os.Stat(outputdir)
19892015
if e3 == nil {
19902016
server.LastBackupMeta.Logical.EndTime = time.Now()
1991-
server.LastBackupMeta.Logical.Size = finfo.Size()
2017+
server.LastBackupMeta.Logical.GetSize()
19922018
server.LastBackupMeta.Logical.Completed = true
19932019
server.SetBackupLogicalCookie(config.ConstBackupLogicalTypeDumpling)
19942020
}
@@ -2834,8 +2860,8 @@ func (server *ServerMonitor) WriteBackupMetadata(backtype config.BackupMethod) {
28342860
return
28352861
}
28362862

2837-
if finfo, err := os.Stat(lastmeta.Dest); err == nil {
2838-
lastmeta.Size = finfo.Size()
2863+
if _, err := os.Stat(lastmeta.Dest); err == nil {
2864+
lastmeta.GetSize()
28392865
lastmeta.EndTime = time.Now()
28402866
}
28412867

@@ -2875,13 +2901,24 @@ func (server *ServerMonitor) WriteBackupMetadata(backtype config.BackupMethod) {
28752901
//Don't change river
28762902
if cluster.Conf.BackupKeepUntilValid && lastmeta.BackupTool != config.ConstBackupLogicalTypeRiver {
28772903
if lastmeta.Completed {
2904+
// Delete previous meta with same type
2905+
cluster.BackupMetaMap.Delete(lastmeta.Previous)
28782906
cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlInfo, "Backup valid, removing old backup.")
28792907
exec.Command("rm", "-r", lastmeta.Dest+".old").Run()
28802908
} else {
28812909
cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModTask, config.LvlInfo, "Error occured in backup, rolling back to old backup.")
28822910
exec.Command("mv", lastmeta.Dest, lastmeta.Dest+".err").Run()
28832911
exec.Command("mv", lastmeta.Dest+".old", lastmeta.Dest).Run()
28842912
exec.Command("rm", "-r", lastmeta.Dest+".err").Run()
2913+
2914+
// Revert to previous meta with same type
2915+
cluster.BackupMetaMap.Delete(lastmeta.Id)
2916+
switch backtype {
2917+
case config.BackupMethodLogical:
2918+
_, server.LastBackupMeta.Logical = server.GetLatestMeta("logical")
2919+
case config.BackupMethodPhysical:
2920+
_, server.LastBackupMeta.Physical = server.GetLatestMeta("physical")
2921+
}
28852922
}
28862923
}
28872924
}

‎config/backup.go

+18-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package config
22

3-
import "time"
3+
import (
4+
"os"
5+
"path/filepath"
6+
"time"
7+
)
48

59
type BackupMethod int
610

@@ -37,4 +41,17 @@ type BackupMetadata struct {
3741
BinLogFilePos uint64 `json:"binLogFilePos"`
3842
BinLogGtid string `json:"binLogUuid"`
3943
Completed bool `json:"completed"`
44+
Previous int64 `json:"previous"`
45+
}
46+
47+
func (bm *BackupMetadata) GetSize() error {
48+
var size int64 = 0
49+
err := filepath.Walk(bm.Dest, func(_ string, info os.FileInfo, err error) error {
50+
if err == nil && !info.IsDir() {
51+
size += info.Size()
52+
}
53+
return err
54+
})
55+
bm.Size = size
56+
return err
4057
}

‎config/maps.go

+15
Original file line numberDiff line numberDiff line change
@@ -880,3 +880,18 @@ func FromBackupMetaMap(m *BackupMetaMap, c *BackupMetaMap) *BackupMetaMap {
880880

881881
return m
882882
}
883+
884+
// GetBackupsByToolAndSource retrieves backups with the same backupTool and source.
885+
func (b *BackupMetaMap) GetPreviousBackup(backupTool string, source string) *BackupMetadata {
886+
var result *BackupMetadata
887+
b.Map.Range(func(key, value interface{}) bool {
888+
if backup, ok := value.(*BackupMetadata); ok {
889+
if backup.BackupTool == backupTool && backup.Source == source {
890+
result = backup
891+
return false
892+
}
893+
}
894+
return true
895+
})
896+
return result
897+
}

‎server/api_cluster.go

+2
Original file line numberDiff line numberDiff line change
@@ -1256,6 +1256,8 @@ func (repman *ReplicationManager) switchSettings(mycluster *cluster.Cluster, set
12561256
mycluster.SwitchReplicationNoRelay()
12571257
case "prov-db-force-write-config":
12581258
mycluster.SwitchForceWriteConfig()
1259+
case "backup-keep-until-valid":
1260+
mycluster.SwitchBackupKeepUntilValid()
12591261
}
12601262
}
12611263

‎share/dashboard/app/dashboard.js

+17
Original file line numberDiff line numberDiff line change
@@ -375,6 +375,23 @@ app.controller('DashboardController', function (
375375
return t.state === 0 || (t.start < Math.floor((Date.now() - 300000) / 1000) && !t.end && t.state < 3)
376376
}
377377

378+
$scope.getBackupMethod = function(method) {
379+
switch(method) {
380+
case 1: return 'Logical';
381+
case 2: return 'Physical';
382+
default: return 'Unknown';
383+
}
384+
};
385+
386+
$scope.getBackupStrategy = function(strategy) {
387+
switch(strategy) {
388+
case 1: return 'Full';
389+
case 2: return 'Incremental';
390+
case 3: return 'Differential';
391+
default: return 'Unknown';
392+
}
393+
};
394+
378395
$scope.SetApiTokenTimeout = function (val) {
379396
if ($scope.roApiTokenTimeout) {
380397
$scope.selectedApiTokenTimeout = Number(val)

‎share/dashboard/static/card-setting-backup.html

+14
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,20 @@
4646
</span>
4747
</td>
4848
</tr>
49+
<tr>
50+
<th colspan=4>Backup keep previous until valid</th>
51+
</tr>
52+
<tr>
53+
<td colspan=4>
54+
<md-switch ng-disabled="selectedCluster.apiUsers[user].grants['cluster-settings']==false"
55+
ng-true-value="true" ng-false-value="false" ng-model="selectedCluster.config.backupKeepUntilValid"
56+
ng-click="switchsettings('backup-keep-until-valid')" aria-label="Keep previous backup until next backup is valid">
57+
<span ng-if="selectedCluster.config.backupKeepUntilValid" class="label label-primary">On</span><span
58+
ng-if="!selectedCluster.config.backupKeepUntilValid" class="label label-warning">Off</span>
59+
60+
</md-switch>
61+
</td>
62+
</tr>
4963
</table>
5064
</md-card>
5165
<md-card ng-if="selectedCluster">

‎share/dashboard/static/tab-cluster-backups.html

+40-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,45 @@
11
<md-content class="md-padding">
22
<md-card>
33
<md-card-content>
4-
<p class="sectionheader">Backups</p>
4+
<p class="sectionheader">Current Backups</p>
5+
</md-card-content>
6+
<md-card>
7+
<table ng-if="selectedCluster.backupList" class="table">
8+
<thead>
9+
<tr>
10+
<th>Id</th>
11+
<th>Start Time</th>
12+
<th>End Time</th>
13+
<th>Backup Method</th>
14+
<th>Backup Strategy</th>
15+
<th>Source</th>
16+
<th>Destination</th>
17+
<th>Size</th>
18+
<th>Compressed</th>
19+
<th>Encrypted</th>
20+
<th>Completed</th>
21+
</tr>
22+
</thead>
23+
<tbody>
24+
<tr ng-repeat="backup in selectedCluster.backupList">
25+
<td>{{backup.id}}</td>
26+
<td>{{backup.startTime | date:'yyyy-MM-dd HH:mm:ss'}}</td>
27+
<td>{{backup.endTime | date:'yyyy-MM-dd HH:mm:ss'}}</td>
28+
<td>{{getBackupMethod(backup.backupMethod)}}</td>
29+
<td>{{getBackupStrategy(backup.backupStrategy)}}</td>
30+
<td>{{backup.source}}</td>
31+
<td>{{backup.dest}}</td>
32+
<td>{{formatBytes(backup.size)}}</td>
33+
<td>{{backup.compressed ? 'Yes' : 'No'}}</td>
34+
<td>{{backup.encrypted ? 'Yes' : 'No'}}</td>
35+
<td>{{backup.completed ? 'Yes' : 'No'}}</td>
36+
</tr>
37+
</tbody>
38+
</table>
39+
</md-card>
40+
<md-card-content>
41+
<p class="sectionheader">Backup History</p>
542
</md-card-content>
6-
743
<md-card>
844
<table ng-if="backups" class="table">
945
<thead>
@@ -22,7 +58,6 @@
2258
</tbody>
2359
</table>
2460
</md-card>
25-
2661
<md-card>
2762
<table ng-if="backups" class="table">
2863
<thead>
@@ -45,5 +80,7 @@
4580
</tbody>
4681
</table>
4782
</md-card>
83+
84+
4885
</md-card>
4986
</md-content>

0 commit comments

Comments
 (0)