Skip to content

More configuration options for rsync #208

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Feb 27, 2025
Merged

More configuration options for rsync #208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
27e4307
Implement rsync_override_only in mirrorConfig
Harry-Chen Feb 27, 2025
a64557b
Implement global.rsync_options (fixes #206)
Harry-Chen Feb 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 18 additions & 14 deletions worker/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,9 @@ type globalConfig struct {
Retry int `toml:"retry"`
Timeout int `toml:"timeout"`

// appended to the options generated by rsync_provider, but before mirror-specific options
RsyncOptions []string `toml:"rsync_options"`

ExecOnSuccess []string `toml:"exec_on_success"`
ExecOnFailure []string `toml:"exec_on_failure"`
}
Expand Down Expand Up @@ -162,23 +165,24 @@ type mirrorConfig struct {
ExecOnSuccess []string `toml:"exec_on_success"`
ExecOnFailure []string `toml:"exec_on_failure"`

// These two options the global options
// These two options are appended to the global options
ExecOnSuccessExtra []string `toml:"exec_on_success_extra"`
ExecOnFailureExtra []string `toml:"exec_on_failure_extra"`

Command string `toml:"command"`
FailOnMatch string `toml:"fail_on_match"`
SizePattern string `toml:"size_pattern"`
UseIPv6 bool `toml:"use_ipv6"`
UseIPv4 bool `toml:"use_ipv4"`
ExcludeFile string `toml:"exclude_file"`
Username string `toml:"username"`
Password string `toml:"password"`
RsyncNoTimeo bool `toml:"rsync_no_timeout"`
RsyncTimeout int `toml:"rsync_timeout"`
RsyncOptions []string `toml:"rsync_options"`
RsyncOverride []string `toml:"rsync_override"`
Stage1Profile string `toml:"stage1_profile"`
Command string `toml:"command"`
FailOnMatch string `toml:"fail_on_match"`
SizePattern string `toml:"size_pattern"`
UseIPv6 bool `toml:"use_ipv6"`
UseIPv4 bool `toml:"use_ipv4"`
ExcludeFile string `toml:"exclude_file"`
Username string `toml:"username"`
Password string `toml:"password"`
RsyncNoTimeo bool `toml:"rsync_no_timeout"`
RsyncTimeout int `toml:"rsync_timeout"`
RsyncOptions []string `toml:"rsync_options"`
RsyncOverride []string `toml:"rsync_override"`
RsyncOverrideOnly bool `toml:"rsync_override_only"` // only use provided overridden options if true
Stage1Profile string `toml:"stage1_profile"`

MemoryLimit MemBytes `toml:"memory_limit"`

Expand Down
118 changes: 118 additions & 0 deletions worker/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,4 +403,122 @@ use_ipv6 = true
So(rp.WorkingDir(), ShouldEqual, "/data/mirrors/debian-cd")
So(p.Timeout(), ShouldEqual, 86400*time.Second)
})

Convey("rsync_override_only should work", t, func() {
tmpfile, err := os.CreateTemp("", "tunasync")
So(err, ShouldEqual, nil)
defer os.Remove(tmpfile.Name())

cfgBlob1 := `
[global]
name = "test_worker"
log_dir = "/var/log/tunasync/{{.Name}}"
mirror_dir = "/data/mirrors"
concurrent = 10
interval = 240
retry = 3
timeout = 86400

[manager]
api_base = "https://127.0.0.1:5000"
token = "some_token"

[server]
hostname = "worker1.example.com"
listen_addr = "127.0.0.1"
listen_port = 6000
ssl_cert = "/etc/tunasync.d/worker1.cert"
ssl_key = "/etc/tunasync.d/worker1.key"

[[mirrors]]
name = "foo"
provider = "rsync"
upstream = "rsync://foo.bar/"
interval = 720
retry = 2
timeout = 3600
mirror_dir = "/data/foo"
rsync_override = ["--bar", "baz"]
rsync_override_only = true
`

err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
So(err, ShouldEqual, nil)
defer tmpfile.Close()

cfg, err := LoadConfig(tmpfile.Name())
So(err, ShouldBeNil)

providers := map[string]mirrorProvider{}
for _, m := range cfg.Mirrors {
p := newMirrorProvider(m, cfg)
providers[p.Name()] = p
}

p, ok := providers["foo"].(*rsyncProvider)
So(ok, ShouldBeTrue)
So(p.options, ShouldResemble, []string{"--bar", "baz"})
})

Convey("rsync global options should work", t, func() {
tmpfile, err := os.CreateTemp("", "tunasync")
So(err, ShouldEqual, nil)
defer os.Remove(tmpfile.Name())

cfgBlob1 := `
[global]
name = "test_worker"
log_dir = "/var/log/tunasync/{{.Name}}"
mirror_dir = "/data/mirrors"
concurrent = 10
interval = 240
retry = 3
timeout = 86400
rsync_options = ["--global"]

[manager]
api_base = "https://127.0.0.1:5000"
token = "some_token"

[server]
hostname = "worker1.example.com"
listen_addr = "127.0.0.1"
listen_port = 6000
ssl_cert = "/etc/tunasync.d/worker1.cert"
ssl_key = "/etc/tunasync.d/worker1.key"

[[mirrors]]
name = "foo"
provider = "rsync"
upstream = "rsync://foo.bar/"
interval = 720
retry = 2
timeout = 3600
mirror_dir = "/data/foo"
rsync_override = ["--override"]
rsync_options = ["--local"]
`

err = os.WriteFile(tmpfile.Name(), []byte(cfgBlob1), 0644)
So(err, ShouldEqual, nil)
defer tmpfile.Close()

cfg, err := LoadConfig(tmpfile.Name())
So(err, ShouldBeNil)

providers := map[string]mirrorProvider{}
for _, m := range cfg.Mirrors {
p := newMirrorProvider(m, cfg)
providers[p.Name()] = p
}

p, ok := providers["foo"].(*rsyncProvider)
So(ok, ShouldBeTrue)
So(p.options, ShouldResemble, []string{
"--override", // from mirror.rsync_override
"--timeout=120", // generated by newRsyncProvider
"--global", // from global.rsync_options
"--local", // from mirror.rsync_options
})
})
}
2 changes: 2 additions & 0 deletions worker/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,9 @@ func newMirrorProvider(mirror mirrorConfig, cfg *Config) mirrorProvider {
extraOptions: mirror.RsyncOptions,
rsyncNeverTimeout: mirror.RsyncNoTimeo,
rsyncTimeoutValue: mirror.RsyncTimeout,
globalOptions: cfg.Global.RsyncOptions,
overriddenOptions: mirror.RsyncOverride,
useOverrideOnly: mirror.RsyncOverrideOnly,
rsyncEnv: mirror.Env,
workingDir: mirrorDir,
logDir: logDir,
Expand Down
45 changes: 29 additions & 16 deletions worker/rsync_provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@ type rsyncConfig struct {
rsyncCmd string
upstreamURL, username, password, excludeFile string
extraOptions []string
globalOptions []string
overriddenOptions []string
useOverrideOnly bool
rsyncNeverTimeout bool
rsyncTimeoutValue int
rsyncEnv map[string]string
Expand Down Expand Up @@ -75,25 +77,36 @@ func newRsyncProvider(c rsyncConfig) (*rsyncProvider, error) {
options = c.overriddenOptions
}

if !c.rsyncNeverTimeout {
timeo := 120
if c.rsyncTimeoutValue > 0 {
timeo = c.rsyncTimeoutValue
if c.useOverrideOnly == true {
if c.overriddenOptions == nil {
return nil, errors.New("rsync_override_only is set but no rsync_override provided")
}
// use overridden options only
} else {
if !c.rsyncNeverTimeout {
timeo := 120
if c.rsyncTimeoutValue > 0 {
timeo = c.rsyncTimeoutValue
}
options = append(options, fmt.Sprintf("--timeout=%d", timeo))
}
options = append(options, fmt.Sprintf("--timeout=%d", timeo))
}

if c.useIPv6 {
options = append(options, "-6")
} else if c.useIPv4 {
options = append(options, "-4")
}
if c.useIPv6 {
options = append(options, "-6")
} else if c.useIPv4 {
options = append(options, "-4")
}

if c.excludeFile != "" {
options = append(options, "--exclude-from", c.excludeFile)
}
if c.extraOptions != nil {
options = append(options, c.extraOptions...)
if c.excludeFile != "" {
options = append(options, "--exclude-from", c.excludeFile)
}

if c.globalOptions != nil {
options = append(options, c.globalOptions...)
}
if c.extraOptions != nil {
options = append(options, c.extraOptions...)
}
}
provider.options = options

Expand Down