Skip to content

Commit 3d1b4a6

Browse files
authored
feat(storage): add provider replacement progress engine (#13)
1 parent 6976224 commit 3d1b4a6

76 files changed

Lines changed: 4277 additions & 1143 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

.env.example

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,4 +9,7 @@
99
# SYNAPS3_FILECOIN_PRIVATE_KEY=0x...
1010
# SYNAPS3_FILECOIN_RPC_URL=https://api.calibration.node.glif.io/rpc/v1
1111
# SYNAPS3_CACHE_MAX_SIZE_GB=100
12+
# SYNAPS3_WORKER_PROVIDER_REPLACEMENT_CONCURRENCY=4
13+
# SYNAPS3_WORKER_PROVIDER_REPLACEMENT_POLL_INTERVAL=5s
14+
# SYNAPS3_WORKER_PROVIDER_REPLACEMENT_MAX_RETRIES=5
1215
# SYNAPS3_ADMIN_AUTH_USERNAME=admin

cmd/synaps3/admin.go

Lines changed: 36 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -838,9 +838,10 @@ type adminSettingsCacheConfig struct {
838838
}
839839

840840
type adminSettingsWorkerConfig struct {
841-
Upload adminSettingsWorkerPoolConfig `json:"upload"`
842-
Evictor adminSettingsWorkerPoolConfig `json:"evictor"`
843-
StorageCleanup adminSettingsWorkerPoolConfig `json:"storage_cleanup"`
841+
Upload adminSettingsWorkerPoolConfig `json:"upload"`
842+
ProviderReplacement adminSettingsWorkerPoolConfig `json:"provider_replacement"`
843+
Evictor adminSettingsWorkerPoolConfig `json:"evictor"`
844+
StorageCleanup adminSettingsWorkerPoolConfig `json:"storage_cleanup"`
844845
}
845846

846847
type adminSettingsWorkerPoolConfig struct {
@@ -903,26 +904,35 @@ type adminSettingSpec struct {
903904
}
904905

905906
var adminEditableSettings = map[string]adminSettingSpec{
906-
"server.port": {path: []string{"server", "port"}, kind: adminSettingString},
907-
"server.max_connections": {path: []string{"server", "max_connections"}, kind: adminSettingInt},
908-
"server.max_requests": {path: []string{"server", "max_requests"}, kind: adminSettingInt},
909-
"server.tls.enabled": {path: []string{"server", "tls", "enabled"}, kind: adminSettingBool},
910-
"server.tls.cert_file": {path: []string{"server", "tls", "cert_file"}, kind: adminSettingString},
911-
"server.tls.key_file": {path: []string{"server", "tls", "key_file"}, kind: adminSettingString},
912-
"s3.region": {path: []string{"s3", "region"}, kind: adminSettingString},
913-
"filecoin.network": {path: []string{"filecoin", "network"}, kind: adminSettingString},
914-
"filecoin.rpc_url": {path: []string{"filecoin", "rpc_url"}, kind: adminSettingString},
915-
"filecoin.with_cdn": {path: []string{"filecoin", "with_cdn"}, kind: adminSettingBool},
916-
"filecoin.allow_private_networks": {path: []string{"filecoin", "allow_private_networks"}, kind: adminSettingBool},
917-
"filecoin.default_copies": {path: []string{"filecoin", "default_copies"}, kind: adminSettingInt},
918-
"cache.dir": {path: []string{"cache", "dir"}, kind: adminSettingString},
919-
"cache.max_size_gb": {path: []string{"cache", "max_size_gb"}, kind: adminSettingInt},
920-
"cache.eviction_policy": {path: []string{"cache", "eviction_policy"}, kind: adminSettingString},
921-
"cache.lru_high_watermark_percent": {path: []string{"cache", "lru_high_watermark_percent"}, kind: adminSettingInt},
922-
"cache.lru_low_watermark_percent": {path: []string{"cache", "lru_low_watermark_percent"}, kind: adminSettingInt},
923-
"worker.upload.concurrency": {path: []string{"worker", "upload", "concurrency"}, kind: adminSettingInt},
924-
"worker.upload.poll_interval": {path: []string{"worker", "upload", "poll_interval"}, kind: adminSettingString},
925-
"worker.upload.max_retries": {path: []string{"worker", "upload", "max_retries"}, kind: adminSettingInt},
907+
"server.port": {path: []string{"server", "port"}, kind: adminSettingString},
908+
"server.max_connections": {path: []string{"server", "max_connections"}, kind: adminSettingInt},
909+
"server.max_requests": {path: []string{"server", "max_requests"}, kind: adminSettingInt},
910+
"server.tls.enabled": {path: []string{"server", "tls", "enabled"}, kind: adminSettingBool},
911+
"server.tls.cert_file": {path: []string{"server", "tls", "cert_file"}, kind: adminSettingString},
912+
"server.tls.key_file": {path: []string{"server", "tls", "key_file"}, kind: adminSettingString},
913+
"s3.region": {path: []string{"s3", "region"}, kind: adminSettingString},
914+
"filecoin.network": {path: []string{"filecoin", "network"}, kind: adminSettingString},
915+
"filecoin.rpc_url": {path: []string{"filecoin", "rpc_url"}, kind: adminSettingString},
916+
"filecoin.with_cdn": {path: []string{"filecoin", "with_cdn"}, kind: adminSettingBool},
917+
"filecoin.allow_private_networks": {path: []string{"filecoin", "allow_private_networks"}, kind: adminSettingBool},
918+
"filecoin.default_copies": {path: []string{"filecoin", "default_copies"}, kind: adminSettingInt},
919+
"cache.dir": {path: []string{"cache", "dir"}, kind: adminSettingString},
920+
"cache.max_size_gb": {path: []string{"cache", "max_size_gb"}, kind: adminSettingInt},
921+
"cache.eviction_policy": {path: []string{"cache", "eviction_policy"}, kind: adminSettingString},
922+
"cache.lru_high_watermark_percent": {path: []string{"cache", "lru_high_watermark_percent"}, kind: adminSettingInt},
923+
"cache.lru_low_watermark_percent": {path: []string{"cache", "lru_low_watermark_percent"}, kind: adminSettingInt},
924+
"worker.upload.concurrency": {path: []string{"worker", "upload", "concurrency"}, kind: adminSettingInt},
925+
"worker.upload.poll_interval": {path: []string{"worker", "upload", "poll_interval"}, kind: adminSettingString},
926+
"worker.upload.max_retries": {path: []string{"worker", "upload", "max_retries"}, kind: adminSettingInt},
927+
"worker.provider_replacement.concurrency": {
928+
path: []string{"worker", "provider_replacement", "concurrency"}, kind: adminSettingInt,
929+
},
930+
"worker.provider_replacement.poll_interval": {
931+
path: []string{"worker", "provider_replacement", "poll_interval"}, kind: adminSettingString,
932+
},
933+
"worker.provider_replacement.max_retries": {
934+
path: []string{"worker", "provider_replacement", "max_retries"}, kind: adminSettingInt,
935+
},
926936
"worker.evictor.concurrency": {path: []string{"worker", "evictor", "concurrency"}, kind: adminSettingInt},
927937
"worker.evictor.poll_interval": {path: []string{"worker", "evictor", "poll_interval"}, kind: adminSettingString},
928938
"worker.evictor.max_retries": {path: []string{"worker", "evictor", "max_retries"}, kind: adminSettingInt},
@@ -1279,6 +1289,9 @@ func writeAdminSettingsSummary(w io.Writer, settings adminSettingsResponse) erro
12791289
{Name: "worker.upload.concurrency", Value: strconv.Itoa(settings.Config.Worker.Upload.Concurrency)},
12801290
{Name: "worker.upload.poll_interval", Value: settings.Config.Worker.Upload.PollInterval},
12811291
{Name: "worker.upload.max_retries", Value: strconv.Itoa(settings.Config.Worker.Upload.MaxRetries)},
1292+
{Name: "worker.provider_replacement.concurrency", Value: strconv.Itoa(settings.Config.Worker.ProviderReplacement.Concurrency)},
1293+
{Name: "worker.provider_replacement.poll_interval", Value: settings.Config.Worker.ProviderReplacement.PollInterval},
1294+
{Name: "worker.provider_replacement.max_retries", Value: strconv.Itoa(settings.Config.Worker.ProviderReplacement.MaxRetries)},
12821295
{Name: "worker.evictor.concurrency", Value: strconv.Itoa(settings.Config.Worker.Evictor.Concurrency)},
12831296
{Name: "worker.evictor.poll_interval", Value: settings.Config.Worker.Evictor.PollInterval},
12841297
{Name: "worker.evictor.max_retries", Value: strconv.Itoa(settings.Config.Worker.Evictor.MaxRetries)},

cmd/synaps3/admin_test.go

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -581,6 +581,7 @@ func TestAdminSettingsSetValidationAndPayload(t *testing.T) {
581581
"100.00 GiB",
582582
"cache.lru_high_watermark_percent",
583583
"cache.lru_low_watermark_percent",
584+
"worker.provider_replacement.concurrency",
584585
"Logging",
585586
} {
586587
if !strings.Contains(out, want) {
@@ -724,6 +725,11 @@ func TestAdminSettingsSetValidationAndPayload(t *testing.T) {
724725
if cache["lru_low_watermark_percent"] != float64(70) {
725726
t.Fatalf("cache.lru_low_watermark_percent = %#v, want 70", cache["lru_low_watermark_percent"])
726727
}
728+
worker := body["worker"].(map[string]any)
729+
providerReplacement := worker["provider_replacement"].(map[string]any)
730+
if providerReplacement["poll_interval"] != "9s" {
731+
t.Fatalf("worker.provider_replacement.poll_interval = %#v, want 9s", providerReplacement["poll_interval"])
732+
}
727733
filecoin := body["filecoin"].(map[string]any)
728734
if filecoin["with_cdn"] != true {
729735
t.Fatalf("filecoin.with_cdn = %#v, want true", filecoin["with_cdn"])
@@ -749,7 +755,8 @@ func TestAdminSettingsSetValidationAndPayload(t *testing.T) {
749755
out, err := runAdminCommand(t, []string{
750756
"synaps3", "admin", "--admin-url", ts.URL,
751757
"settings", "set", "cache.max_size_gb=8", "cache.lru_high_watermark_percent=85",
752-
"cache.lru_low_watermark_percent=70", "filecoin.with_cdn=true", "logging.level=debug",
758+
"cache.lru_low_watermark_percent=70", "worker.provider_replacement.poll_interval=9s",
759+
"filecoin.with_cdn=true", "logging.level=debug",
753760
"logging.s3_access.enabled=false", "logging.s3_access.level=debug",
754761
})
755762
if err != nil {
@@ -995,8 +1002,10 @@ func adminTestSettings(network string, allowPrivate bool) map[string]any {
9951002
"lru_low_watermark_percent": 80,
9961003
},
9971004
"worker": map[string]any{
998-
"upload": map[string]any{"concurrency": 4, "poll_interval": "5s", "max_retries": 5},
999-
"evictor": map[string]any{"concurrency": 2, "poll_interval": "1m0s", "max_retries": 3},
1005+
"upload": map[string]any{"concurrency": 4, "poll_interval": "5s", "max_retries": 5},
1006+
"provider_replacement": map[string]any{"concurrency": 4, "poll_interval": "5s", "max_retries": 5},
1007+
"evictor": map[string]any{"concurrency": 2, "poll_interval": "1m0s", "max_retries": 3},
1008+
"storage_cleanup": map[string]any{"concurrency": 2, "poll_interval": "1m0s", "max_retries": 5},
10001009
},
10011010
"logging": map[string]any{
10021011
"level": "info",

cmd/synaps3/main.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -432,6 +432,9 @@ func setupModeAllowedField(field string) bool {
432432
"worker.upload.concurrency",
433433
"worker.upload.poll_interval",
434434
"worker.upload.max_retries",
435+
"worker.provider_replacement.concurrency",
436+
"worker.provider_replacement.poll_interval",
437+
"worker.provider_replacement.max_retries",
435438
"worker.evictor.concurrency",
436439
"worker.evictor.poll_interval",
437440
"worker.evictor.max_retries",

cmd/synaps3/setup_mode_test.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ func TestShouldStartSetupModeAllowsEditableConfigErrors(t *testing.T) {
3838
cfg.Cache.MaxSizeGB = 0
3939
cfg.Worker.Upload.PollInterval = 0
4040
cfg.Worker.Upload.MaxRetries = -1
41+
cfg.Worker.ProviderReplacement.Concurrency = 0
42+
cfg.Worker.ProviderReplacement.PollInterval = 0
43+
cfg.Worker.ProviderReplacement.MaxRetries = -1
4144
cfg.Logging.Level = "verbose"
4245
cfg.Logging.S3Access.Level = "verbose"
4346

docs/en/concepts/filecoin-storage-flow.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,9 +66,9 @@ When a provider becomes permanently unavailable, or you plan to move away from o
6666

6767
One confirmation covers the whole move. SynapS3 creates the new storage service, switches new uploads to it once it is ready, copies existing data across, and only then shuts down the old provider. Objects copy from another replica or from local cache. An object with neither cannot be copied, and the old provider is not shut down. Data that can still be read from the old provider stays readable until every retained version is readable on the new one.
6868

69-
While replacements run, the replica shows every move that still needs progress or operator attention, including parallel moves on other replicas. Progress is counted in stored items, since content shared by several versions is copied once. Once copying is finished, the dashboard separates content that was transferred from content that was deleted before it needed to move.
69+
While replacements run, the replica shows every move that still needs progress or operator attention, including parallel moves on other replicas. Discovery uses an indeterminate progress bar because the final count is not known yet. Once discovery finishes, progress uses the processed share of the final total. Progress counts unique stored content, so content shared by several versions is copied once, and separates content transferred from content deleted before it needed to move. The same compact progress appears in Tasks; the Data Sets card shows the full breakdown.
7070

71-
Some steps wait rather than fail. The dashboard distinguishes creating the new service, waiting for it to become writable, an unreachable provider, wallet funds, and missing readable content. Most waits resume on their own. If an object has no other replica and no local cache, replacement stays waiting until one is available. Retry from the same Data Sets list when work has run out of attempts, or when shutting down the old provider needs a payment settled first. A target already in use cannot be retried; choose another provider.
71+
Some steps wait rather than fail. The dashboard distinguishes creating the new service, waiting for it to become writable, an unreachable provider, wallet funds, and missing readable content. Most waits resume on their own. If an object has no other replica and no local cache, replacement waits until a source is available; this does not count toward the retry limit. Temporary copy failures resume after a restart and do not stop other content or another replacement. Use **Retry replacement** from the Data Sets list when the dashboard shows that content needs attention, or when shutting down the old provider needs a payment settled first. A target already in use cannot be retried; choose another provider.
7272

7373
## What Users See
7474

docs/en/configuration/environment.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,9 @@ Configuration environment variables use the `SYNAPS3_` prefix and map underscore
3939
| `SYNAPS3_WORKER_UPLOAD_CONCURRENCY` | `worker.upload.concurrency` |
4040
| `SYNAPS3_WORKER_UPLOAD_POLL_INTERVAL` | `worker.upload.poll_interval` |
4141
| `SYNAPS3_WORKER_UPLOAD_MAX_RETRIES` | `worker.upload.max_retries` |
42+
| `SYNAPS3_WORKER_PROVIDER_REPLACEMENT_CONCURRENCY` | `worker.provider_replacement.concurrency` |
43+
| `SYNAPS3_WORKER_PROVIDER_REPLACEMENT_POLL_INTERVAL` | `worker.provider_replacement.poll_interval` |
44+
| `SYNAPS3_WORKER_PROVIDER_REPLACEMENT_MAX_RETRIES` | `worker.provider_replacement.max_retries` |
4245
| `SYNAPS3_WORKER_EVICTOR_CONCURRENCY` | `worker.evictor.concurrency` |
4346
| `SYNAPS3_WORKER_EVICTOR_POLL_INTERVAL` | `worker.evictor.poll_interval` |
4447
| `SYNAPS3_WORKER_EVICTOR_MAX_RETRIES` | `worker.evictor.max_retries` |

docs/en/configuration/model.md

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ SQLite is the default and recommended database for SynapS3 single-node deploymen
7272
| `database` | SQLite or PostgreSQL metadata database. |
7373
| `cache` | Local object cache directory, capacity, and eviction policy. |
7474
| `worker.upload` | Background Filecoin storage concurrency, polling, and retries. |
75+
| `worker.provider_replacement` | Provider-replacement transfer concurrency, polling, and copy retries. |
7576
| `worker.evictor` | Local cache eviction tasks. |
7677
| `worker.storage_cleanup` | Remote copy cleanup tasks. |
7778
| `logging` | Runtime log level, format, and S3 access logs. |
@@ -96,12 +97,17 @@ SQLite is the default and recommended database for SynapS3 single-node deploymen
9697
| `cache.lru_low_watermark_percent` | `80` |
9798
| `worker.upload.concurrency` | `4` |
9899
| `worker.upload.max_retries` | `5` |
100+
| `worker.provider_replacement.concurrency` | `4` |
101+
| `worker.provider_replacement.poll_interval` | `5s` |
102+
| `worker.provider_replacement.max_retries` | `5` |
99103
| `admin.addr` | `127.0.0.1:9090` |
100104
| `admin.trusted_proxies` | `[]` |
101105
| `admin.auth.enabled` | `true` |
102106
| `admin.auth.username` | `admin` |
103107
| `admin.auth.session_ttl` | `12h` |
104108

109+
`worker.provider_replacement` settings do not affect ordinary uploads. Changing `worker.provider_replacement.max_retries` applies to replacement work discovered afterward and content retried with **Retry replacement**; work already in progress keeps its current limit. Changes to provider replacement concurrency, polling, or retries require a SynapS3 restart.
110+
105111
## Admin Session Lifetime
106112

107113
`admin.auth.session_ttl` controls the lifetime of each standard Admin UI session token. It is neither a server-enforced idle timeout nor an absolute cap on a login. After the earlier of five minutes or half the token lifetime, the server permits renewal. The official dashboard requests renewal only after a trusted pointer, click, keyboard, or wheel interaction; background polling and returning to a visible tab do not trigger it. Any client holding the valid session cookie and matching CSRF token can call the refresh endpoint after `refresh_after`. If no client requests renewal, the token expires at `expires_at`.

docs/en/operations/troubleshooting.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ Retry only after RPC connectivity, storage provider availability, wallet funds,
132132
synaps3 admin task retry 42
133133
```
134134

135-
Provider replacement work is the exception: do not retry it from Tasks. Finished or stopped replacement tasks provide **Open Data Sets**, which opens the affected bucket directly at **Details****Storage****Data Sets**. Use **Retry replacement** only when that action is shown. If the selected provider already stores this bucket, choose a different provider instead.
135+
Provider replacement work is the exception: do not retry it from Tasks. Copy retries and waits for readable content resume automatically, including after a restart. Finished or stopped replacement tasks provide **Open Data Sets**, which opens the affected bucket directly at **Details****Storage****Data Sets**. Use **Retry replacement** only when that action is shown. It retries content that needs attention with the current `worker.provider_replacement.max_retries` setting and keeps completed work. If the selected provider already stores this bucket, choose a different provider instead.
136136

137137
## Provider or RPC Issues
138138

docs/en/operations/upgrade-recovery.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ Receive write -> save object -> record metadata -> return success -> continue ba
6262
| Database full | Free space or scale the database. |
6363
| Cache disk full | Increase disk, raise `cache.max_size_gb`, or restore upload and eviction progress. |
6464
| Provider is permanently unavailable, or must be evacuated | Open the bucket, choose **Details**, then **Storage****Data Sets**, and replace the provider. New uploads move to the new provider once it is ready. Existing objects copy from another replica or from local cache; an object with neither cannot be copied, and the old provider is not shut down. If the selected target is already in use, choose another provider rather than retrying it. |
65-
| Process crash | Restart the service, then verify health and task statistics; unfinished tasks become eligible to continue. If shutting down an unused replacement service was already submitted, SynapS3 continues checking that request instead of submitting it again. |
65+
| Process crash | Restart the service, then verify health and task statistics. Unfinished provider replacement work resumes from saved progress, including scheduled retries and waits for readable content. If the process stopped before SynapS3 recorded a provider copy result, restarting may repeat that copy request. Recorded service-shutdown transactions are checked before another shutdown is submitted. |
6666

6767
A provider becoming unavailable after a copy has already been stored does not necessarily create a retryable task. Use storage-health views to identify affected copies. Restoring the target copy count is part of [Planned Replica Repair](../concepts/filecoin-storage-flow.md#planned-replica-repair).
6868

0 commit comments

Comments
 (0)