Skip to content

Commit 0cbd515

Browse files
c4urselfc4urself
committed
IMP add sync/dump commands
This adds a code-first approach where you can sync Consul KV from a git repository
1 parent befabf8 commit 0cbd515

File tree

7 files changed

+355
-43
lines changed

7 files changed

+355
-43
lines changed

‎README.md‎

Lines changed: 79 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,98 @@
11
# Consul Bak [![Build Status](https://travis-ci.org/Tubular/consul-bak.png)](https://travis-ci.org/Tubular/consul-bak)
22

3-
Backs up and restores KV pairs in a Consul cluster using the consul-api Go library.
3+
Backs up, restores, syncs and dumps KV pairs in a Consul cluster using the consul-api Go library.
44

55
## Changelog / Releases
66

77
[See the changelog here](CHANGELOG.md)
88

9+
10+
## Modes
11+
12+
13+
### Backup
14+
15+
This backups your Consul master's KV store into a file. You can run this via a crontab and sync
16+
it to S3 for example:
17+
18+
```
19+
filename="$(date +%Y%m%d).txt"
20+
/usr/local/bin/consul-bak backup --leader-only $filename
21+
/usr/local/bin/aws s3 mv $filename s3://<my_bucket>/
22+
```
23+
24+
25+
### Restore
26+
27+
This restores a file created by consul-bak into the Consul master's KV store. Example:
28+
29+
```
30+
/usr/local/bin/consul-bak restore $filename
31+
```
32+
33+
34+
### Syncgit
35+
36+
This synchronises a filesystem representation of the KV tree into a Consul master's KV store. The
37+
filesystem representation is expected to be a git repository. This is especially useful if you want
38+
a code-first approach to creating configuration in Consul. Example:
39+
40+
```
41+
/usr/local/bin/consul-bak sync git@github.com:Tubular/consul-bak.git|path/to/root/of/kv/tree
42+
```
43+
44+
45+
### Dumptree
46+
47+
This dumps the values of the Consul master's KV as a tree onto the filesystem. If you want to start
48+
using sync, you could use this to kickstart your project. Example:
49+
50+
```
51+
/usr/local/bin/consul-bak dumptree /tmp/kv_tree
52+
```
53+
54+
955
## Usage examples
1056

1157
```sh
1258
Usage:
13-
consul-bak [-i IP] [--http-port HTTPPORT] [--rpc-port RPCPORT]
14-
[-l] [-t TOKEN] [-a] [-b ACLBACKUPFILE] [-n INPREFIX]...
15-
[-x EXPREFIX]... [--restore] [--no-prompt] <filename>
59+
consul-bak (backup|restore|aclbackup)
60+
[--leader-only]
61+
[--rpc-port RPCPORT]
62+
[--http-port HTTPPORT]
63+
[--address IP]
64+
[--include-prefix INPREFIX]...
65+
[--exclude-prefix EXPREFIX]...
66+
[--token TOKEN]
67+
[--no-prompt]
68+
<filename>
69+
consul-bak dumptree
70+
[--leader-only]
71+
[--rpc-port RPCPORT]
72+
[--http-port HTTPPORT]
73+
[--address IP]
74+
<pathname>
75+
consul-bak syncgit
76+
[--leader-only]
77+
[--rpc-port RPCPORT]
78+
[--http-port HTTPPORT]
79+
[--address IP]
80+
<git-url>
1681
consul-bak -h | --help
17-
consul-bak --version
82+
consul-bak -v | --version
1883

1984
Options:
20-
-h --help Show this screen.
21-
--version Show version.
22-
-l, --leader-only Create backup only on consul leader.
85+
-h, --help Show this screen.
86+
-v, --version Show version.
87+
--mode=MODE Set mode, can be one of backup,restore,syncgit,dumptree,aclbackup [default: backup]
88+
--leader-only Only run on consul leader.
2389
--rpc-port=RPCPORT RPC port [default: 8400].
2490
--http-port=HTTPPORT HTTP endpoint port [default: 8500].
25-
-i, --address=IP The HTTP endpoint of Consul [default: 127.0.0.1].
26-
-t, --token=TOKEN An ACL Token with proper permissions in Consul [default: ].
27-
-a, --aclbackup Backup ACLs, does nothing in restore mode. ACL restore not available at this time.
28-
-b, --aclbackupfile=ACLBACKUPFILE ACL Backup Filename [default: acl.bkp].
29-
-x, --exclude-prefix=[EXPREFIX] Repeatable option for keys starting with prefix to exclude from the backup.
30-
-n, --include-prefix=[INPREFIX] Repeatable option for keys starting with prefix to include in the backup.
31-
-r, --restore Activate restore mode.
32-
--no-prompt Don't prompt, force overwrite in restore mode.
91+
--address=IP The HTTP endpoint of Consul [default: 127.0.0.1].
92+
--include-prefix=[INPREFIX] Repeatable option for keys starting with prefix to include in the backup.
93+
--exclude-prefix=[EXPREFIX] Repeatable option for keys starting with prefix to exclude from the backup.
94+
--token=TOKEN An ACL Token with proper permissions in Consul [default: ].
95+
--force Don't prompt, force overwrite.
3396
```
3497
3598

‎dump.go‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
"github.com/hashicorp/consul/api"
6+
"os"
7+
"path/filepath"
8+
"sort"
9+
"strings"
10+
)
11+
12+
func Dump(ipAddress string, token string, rootPath string) {
13+
14+
var standardDirPerms os.FileMode = 0775
15+
logger.Infof("Starting dump from %s to %s.", ipAddress, rootPath)
16+
17+
config := api.DefaultConfig()
18+
config.Address = ipAddress
19+
config.Token = token
20+
21+
client, _ := api.NewClient(config)
22+
kv := client.KV()
23+
24+
pairs, _, err := kv.List("/", nil)
25+
if err != nil {
26+
panic(err)
27+
}
28+
29+
sort.Sort(ByCreateIndex(pairs))
30+
31+
if len(pairs) == 0 {
32+
logger.Info("No KV pairs found. Nothing to do. Exiting.")
33+
os.Exit(0)
34+
} else {
35+
logger.Infof("Found %d KV pairs", len(pairs))
36+
}
37+
38+
oldDir, _ := os.Getwd()
39+
defer os.Chdir(oldDir)
40+
41+
absPath, pathErr := filepath.Abs(rootPath)
42+
logger.Debugf("Path to dump kv in is: %s", absPath)
43+
Check(pathErr)
44+
ensureErr := EnsureDir(absPath, standardDirPerms)
45+
Check(ensureErr)
46+
chdirErr := os.Chdir(absPath)
47+
Check(chdirErr)
48+
baseDir, _ := os.Getwd()
49+
logger.Debugf("Changed directory to: %s", baseDir)
50+
51+
for _, element := range pairs {
52+
var path string
53+
path = fmt.Sprintf("%s/%s", baseDir, element.Key)
54+
logger.Infof("Writing %s", path)
55+
if strings.HasSuffix(path, "/") {
56+
err := EnsureDir(path, standardDirPerms)
57+
Check(err)
58+
} else {
59+
// ensure directory is present before creating file
60+
err := EnsureDir(filepath.Dir(path), standardDirPerms)
61+
Check(err)
62+
63+
f, createErr := os.Create(path)
64+
Check(createErr)
65+
f.Write(element.Value)
66+
}
67+
}
68+
}

‎logging.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import (
66
)
77

88
var logger = logging.MustGetLogger("consul-bak")
9-
var format = logging.MustStringFormatter(`%{time:2006-01-02 15:04:05.000} %{color}%{level}%{color:reset} (consul-bak): %{message}`)
9+
var format = logging.MustStringFormatter(`%{time:2006-01-02 15:04:05.000} %{color}%{level}%{color:reset} (consul-bak.%{shortfile}): %{message}`)
1010

1111
// SetupLogging setup up logging infra, should be called once on start
1212
func SetupLogging() {

‎main.go‎

Lines changed: 47 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -9,31 +9,48 @@ import (
99

1010
func main() {
1111

12-
usage := fmt.Sprintf(`Consul KV and ACL Backup with KV Restore tool.
12+
usage := fmt.Sprintf(`Consul KV and ACL Backup and Sync tool.
1313
1414
Version: %s (Commit: %s)
1515
1616
Usage:
17-
consul-bak [-i IP] [--http-port HTTPPORT] [--rpc-port RPCPORT]
18-
[-l] [-t TOKEN] [-a] [-b ACLBACKUPFILE] [-n INPREFIX]...
19-
[-x EXPREFIX]... [--restore] [--no-prompt] <filename>
17+
consul-bak (backup|restore|aclbackup)
18+
[--leader-only]
19+
[--rpc-port RPCPORT]
20+
[--http-port HTTPPORT]
21+
[--address IP]
22+
[--include-prefix INPREFIX]...
23+
[--exclude-prefix EXPREFIX]...
24+
[--token TOKEN]
25+
[--no-prompt]
26+
<filename>
27+
consul-bak dumptree
28+
[--leader-only]
29+
[--rpc-port RPCPORT]
30+
[--http-port HTTPPORT]
31+
[--address IP]
32+
<pathname>
33+
consul-bak syncgit
34+
[--leader-only]
35+
[--rpc-port RPCPORT]
36+
[--http-port HTTPPORT]
37+
[--address IP]
38+
<git-url>
2039
consul-bak -h | --help
21-
consul-bak --version
40+
consul-bak -v | --version
2241
2342
Options:
24-
-h --help Show this screen.
25-
--version Show version.
26-
-l, --leader-only Create backup only on consul leader.
43+
-h, --help Show this screen.
44+
-v, --version Show version.
45+
--mode=MODE Set mode, can be one of backup,restore,syncgit,dumptree,aclbackup [default: backup]
46+
--leader-only Only run on consul leader.
2747
--rpc-port=RPCPORT RPC port [default: 8400].
2848
--http-port=HTTPPORT HTTP endpoint port [default: 8500].
29-
-i, --address=IP The HTTP endpoint of Consul [default: 127.0.0.1].
30-
-t, --token=TOKEN An ACL Token with proper permissions in Consul [default: ].
31-
-a, --aclbackup Backup ACLs, does nothing in restore mode. ACL restore not available at this time.
32-
-b, --aclbackupfile=ACLBACKUPFILE ACL Backup Filename [default: acl.bkp].
33-
-x, --exclude-prefix=[EXPREFIX] Repeatable option for keys starting with prefix to exclude from the backup.
34-
-n, --include-prefix=[INPREFIX] Repeatable option for keys starting with prefix to include in the backup.
35-
-r, --restore Activate restore mode.
36-
--no-prompt Don't prompt, force overwrite in restore mode.`, Version, GitCommit)
49+
--address=IP The HTTP endpoint of Consul [default: 127.0.0.1].
50+
--include-prefix=[INPREFIX] Repeatable option for keys starting with prefix to include in the backup.
51+
--exclude-prefix=[EXPREFIX] Repeatable option for keys starting with prefix to exclude from the backup.
52+
--token=TOKEN An ACL Token with proper permissions in Consul [default: ].
53+
--force Don't prompt, force overwrite.`, Version, GitCommit)
3754

3855
arguments, _ := docopt.Parse(usage, nil, true, fmt.Sprintf("consul-bak %s (%s)", Version, GitCommit), false)
3956
SetupLogging()
@@ -50,7 +67,7 @@ Options:
5067
CheckSocket(rpcEndpoint)
5168

5269
if arguments["--leader-only"] == true {
53-
logger.Info("Running in leader only mode, only running backup/restore on Consul leader.")
70+
logger.Info("Running in leader only mode, only running on Consul leader.")
5471
// if consul client is not available we keep running
5572
if Which("consul") {
5673
var out = ConsulBinaryCall("info", rpcOptString)
@@ -66,19 +83,19 @@ Options:
6683
}
6784
}
6885

69-
if arguments["--restore"] == true {
86+
if arguments["restore"].(bool) {
7087
logger.Info("Running in restore mode.")
7188
if (len(arguments["--exclude-prefix"].([]string)) > 0) || (len(arguments["--include-prefix"].([]string)) > 0) {
7289
logger.Error("--exclude-prefix, -x and --include-prefix, -n can be used only for backups")
7390
os.Exit(1)
7491
}
75-
if arguments["--no-prompt"] == false {
92+
if !arguments["--force"].(bool) {
7693
fmt.Printf("\nWarning! This will overwrite existing kv. Press [enter] to continue; CTL-C to exit")
7794
fmt.Scanln()
7895
}
7996
logger.Infof("Restoring KV from file: %s", arguments["<filename>"].(string))
8097
Restore(httpEndpoint, arguments["--token"].(string), arguments["<filename>"].(string))
81-
} else {
98+
} else if arguments["backup"].(bool) {
8299
logger.Info("Running in backup mode.")
83100
if (len(arguments["--exclude-prefix"].([]string)) > 0) && (len(arguments["--include-prefix"].([]string)) > 0) {
84101
logger.Error("--exclude-prefix and --include-prefix cannot be used together")
@@ -92,10 +109,15 @@ Options:
92109
}
93110
logger.Infof("KV store will be backed up to file: %s", arguments["<filename>"].(string))
94111
Backup(httpEndpoint, arguments["--token"].(string), arguments["<filename>"].(string), arguments["--exclude-prefix"].([]string), arguments["--include-prefix"].([]string))
95-
if arguments["--aclbackup"] == true {
96-
97-
logger.Infof("ACL Tokens will be backed up to file: %s", arguments["--aclbackupfile"].(string))
98-
BackupACLs(httpEndpoint, arguments["--token"].(string), arguments["--aclbackupfile"].(string))
99-
}
112+
} else if arguments["aclbackup"].(bool) {
113+
logger.Infof("ACL Tokens will be backed up to file: %s", arguments["<filename>"].(string))
114+
BackupACLs(httpEndpoint, arguments["--token"].(string), arguments["<filename>"].(string))
115+
} else if arguments["syncgit"].(bool) {
116+
logger.Info("Running in sync mode.")
117+
Sync(httpEndpoint, arguments["--token"].(string), arguments["<git-url>"].(string))
118+
} else if arguments["dumptree"].(bool) {
119+
var path string = arguments["<pathname>"].(string)
120+
logger.Infof("Running in dump mode. Dumping to %s", path)
121+
Dump(httpEndpoint, arguments["--token"].(string), path)
100122
}
101123
}

0 commit comments

Comments
 (0)