From 9068dbf45af6fb57ee97e93627360e979b0f3dd5 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Sat, 22 Aug 2026 18:35:55 +0800 Subject: [PATCH] feat(storage): support operator-approved provider replacement --- cmd/synaps3/main.go | 4 + docs/en/concepts/filecoin-storage-flow.md | 12 +- docs/en/operations/troubleshooting.md | 2 + docs/en/operations/upgrade-recovery.md | 3 +- docs/en/reference/admin-api.md | 68 + docs/en/reference/cli-api.md | 2 + docs/zh/concepts/filecoin-storage-flow.md | 12 +- docs/zh/operations/troubleshooting.md | 2 + docs/zh/operations/upgrade-recovery.md | 3 +- docs/zh/reference/admin-api.md | 68 + docs/zh/reference/cli-api.md | 2 + internal/admin/api_buckets.go | 19 +- internal/admin/api_buckets_test.go | 7 +- internal/admin/api_replacement.go | 524 ++++++++ internal/admin/api_replacement_test.go | 463 +++++++ internal/admin/api_tasks.go | 35 +- internal/admin/api_tasks_test.go | 31 +- internal/admin/replacement_candidates.go | 88 ++ internal/admin/replacement_candidates_test.go | 95 ++ internal/admin/replacement_selector.go | 98 ++ internal/admin/replacement_selector_test.go | 165 +++ internal/admin/server.go | 30 +- internal/app/runtime.go | 14 +- internal/db/db_test.go | 21 +- ...2026082101_storage_data_set_generations.go | 264 ++++ .../storage_data_set_generations_test.go | 300 +++++ internal/db/repository/bucket_repo.go | 19 + internal/db/repository/cache_eviction_repo.go | 30 +- internal/db/repository/errors.go | 5 + internal/db/repository/interfaces.go | 163 ++- internal/db/repository/repos.go | 2 + .../db/repository/storage_cleanup_repo.go | 48 +- internal/db/repository/storage_health_sql.go | 34 +- .../storage_health_sql_internal_test.go | 64 + .../repository/storage_readable_copy_sql.go | 64 + .../db/repository/storage_replacement_gate.go | 367 +++++ .../storage_replacement_gate_internal_test.go | 55 + .../repository/storage_replacement_items.go | 580 ++++++++ .../storage_replacement_postgres_test.go | 423 ++++++ .../db/repository/storage_replacement_repo.go | 736 ++++++++++ .../storage_replacement_repo_test.go | 1196 +++++++++++++++++ .../repository/storage_upload_health_repo.go | 16 +- internal/db/repository/storage_upload_repo.go | 278 +++- .../db/repository/storage_upload_repo_test.go | 216 ++- internal/db/repository/task_repo.go | 42 +- internal/db/repository/tx.go | 19 + internal/model/storage.go | 6 + internal/storagereplacement/codes.go | 51 + internal/storagereplacement/errors.go | 58 + internal/storagereplacement/failurereason.go | 14 + internal/storagereplacement/record.go | 72 + internal/storagereplacement/status.go | 125 ++ internal/storagereplacement/task.go | 229 ++++ internal/storagereplacement/task_test.go | 169 +++ internal/storagereplacement/transition.go | 113 ++ .../storagereplacement/transition_test.go | 206 +++ internal/storagereplacement/waitreason.go | 72 + internal/synapse/epochs.go | 43 + internal/synapse/interfaces.go | 21 + internal/synapse/storage_client.go | 5 +- internal/synapse/terminate.go | 110 ++ internal/synapse/terminate_test.go | 223 +++ internal/systemtest/filecoin.go | 69 +- internal/systemtest/harness.go | 1 + internal/testutil/mocks.go | 35 +- internal/worker/manager.go | 66 + internal/worker/manager_test.go | 2 +- internal/worker/storage_cleanup.go | 35 +- .../worker/storage_cleanup_replacement.go | 530 ++++++++ internal/worker/uploader.go | 429 ++++-- internal/worker/uploader_replacement.go | 871 ++++++++++++ .../uploader_replacement_internal_test.go | 338 +++++ internal/worker/uploader_replacement_test.go | 954 +++++++++++++ internal/worker/uploader_replica_repair.go | 25 +- internal/worker/uploader_test.go | 6 +- tests/system/golden_test.go | 5 +- tests/system/provider_replacement_test.go | 213 +++ ui/src/api/client.ts | 72 + .../app/DangerActionAlertDialog.tsx | 4 +- ui/src/components/app/ProviderSelect.tsx | 124 ++ ui/src/hooks/queries.ts | 57 + ui/src/lib/bucket-route-search.ts | 2 + ui/src/lib/provider-replacement.ts | 283 ++++ ui/src/lib/storage-status-labels.ts | 17 + ui/src/routes/buckets.$name.tsx | 376 +++++- ui/src/routes/tasks.tsx | 56 +- ui/test/provider-replacement.test.ts | 366 +++++ ui/test/storage-status-labels.test.ts | 11 + 88 files changed, 12866 insertions(+), 287 deletions(-) create mode 100644 internal/admin/api_replacement.go create mode 100644 internal/admin/api_replacement_test.go create mode 100644 internal/admin/replacement_candidates.go create mode 100644 internal/admin/replacement_candidates_test.go create mode 100644 internal/admin/replacement_selector.go create mode 100644 internal/admin/replacement_selector_test.go create mode 100644 internal/db/migrations/2026082101_storage_data_set_generations.go create mode 100644 internal/db/migrations/storage_data_set_generations_test.go create mode 100644 internal/db/repository/storage_health_sql_internal_test.go create mode 100644 internal/db/repository/storage_readable_copy_sql.go create mode 100644 internal/db/repository/storage_replacement_gate.go create mode 100644 internal/db/repository/storage_replacement_gate_internal_test.go create mode 100644 internal/db/repository/storage_replacement_items.go create mode 100644 internal/db/repository/storage_replacement_postgres_test.go create mode 100644 internal/db/repository/storage_replacement_repo.go create mode 100644 internal/db/repository/storage_replacement_repo_test.go create mode 100644 internal/db/repository/tx.go create mode 100644 internal/storagereplacement/codes.go create mode 100644 internal/storagereplacement/errors.go create mode 100644 internal/storagereplacement/failurereason.go create mode 100644 internal/storagereplacement/record.go create mode 100644 internal/storagereplacement/status.go create mode 100644 internal/storagereplacement/task.go create mode 100644 internal/storagereplacement/task_test.go create mode 100644 internal/storagereplacement/transition.go create mode 100644 internal/storagereplacement/transition_test.go create mode 100644 internal/storagereplacement/waitreason.go create mode 100644 internal/synapse/epochs.go create mode 100644 internal/synapse/terminate.go create mode 100644 internal/synapse/terminate_test.go create mode 100644 internal/worker/storage_cleanup_replacement.go create mode 100644 internal/worker/uploader_replacement.go create mode 100644 internal/worker/uploader_replacement_internal_test.go create mode 100644 internal/worker/uploader_replacement_test.go create mode 100644 tests/system/provider_replacement_test.go create mode 100644 ui/src/components/app/ProviderSelect.tsx create mode 100644 ui/src/lib/provider-replacement.ts create mode 100644 ui/test/provider-replacement.test.ts diff --git a/cmd/synaps3/main.go b/cmd/synaps3/main.go index 28b68bc..a6db9c8 100644 --- a/cmd/synaps3/main.go +++ b/cmd/synaps3/main.go @@ -319,6 +319,10 @@ func runServe(ctx context.Context, src config.Source) error { Receipts: walletReceiptClient, Readiness: filecoinReadiness, Observability: observabilityChecker, + Terminator: storageClient, + // The epoch comes from the same node that reports wallet receipts, + // so replacement adds no new RPC connection. + Epochs: synapse.NewChainEpochReader(walletReceiptClient), }, ProviderIdentity: admin.NewProviderIdentityResolver(client.SPRegistry(), cfg.Filecoin.RPCURL, logger), Logger: logger, diff --git a/docs/en/concepts/filecoin-storage-flow.md b/docs/en/concepts/filecoin-storage-flow.md index 73fdaeb..40e1f13 100644 --- a/docs/en/concepts/filecoin-storage-flow.md +++ b/docs/en/concepts/filecoin-storage-flow.md @@ -52,7 +52,7 @@ Retry after restoring RPC connectivity, storage provider reachability, wallet fu Health checks record storage provider and local data set status. The dashboard uses those results to show copies that are `unavailable`, `degraded`, or `unknown`. -If an established provider becomes temporarily unavailable while the initial copies are still being stored, SynapS3 keeps using the other assigned writable copies. The unfinished copy waits without consuming retries and resumes automatically when the original provider becomes reachable again. SynapS3 does not automatically select a replacement provider. Repairing copies that became unavailable after storage completed remains part of the planned replica repair feature below. +If an established provider becomes temporarily unavailable while the initial copies are still being stored, SynapS3 keeps using the other assigned writable copies. The unfinished copy waits without consuming retries and resumes automatically when the original provider becomes reachable again. SynapS3 does not automatically select a replacement provider; see [Replace a Storage Provider](#replace-a-storage-provider) for the operator-approved path. Repairing copies that became unavailable after storage completed remains part of the planned replica repair feature below. ## Target and Minimum Replicas @@ -60,6 +60,16 @@ The target replica count is frozen when an upload starts. By default, **Release Changing the target affects new uploads. Changing the minimum also re-evaluates retained cache for current uploads. Increasing the minimum does not move versions that are already stored back to an earlier state and cannot restore cache that has already been deleted. +## Replace a Storage Provider + +When a provider becomes permanently unavailable, or you plan to move away from one, open the bucket in the dashboard, choose **Details**, then **Storage** → **Data Sets**, and replace the provider. This is always an explicit decision: SynapS3 never swaps a provider on its own, because doing so creates a new paid service and changes where your data lives. + +One confirmation covers the whole move. SynapS3 creates the new storage service, switches new uploads to it once it is ready, copies existing data across, and only then shuts down the old provider. Objects copy from another replica or from local cache. An object with neither cannot be copied, and the old provider is not shut down. Data that can still be read from the old provider stays readable until every retained version is readable on the new one. + +While replacements run, the replica shows every move that still needs progress or operator attention, including parallel moves on other replicas. Progress is counted in stored items, since content shared by several versions is copied once. Once copying is finished, the dashboard separates content that was transferred from content that was deleted before it needed to move. + +Some steps wait rather than fail. The dashboard distinguishes creating the new service, waiting for it to become writable, an unreachable provider, wallet funds, and missing readable content. Most waits resume on their own. If an object has no other replica and no local cache, replacement stays waiting until one is available. Retry from the same Data Sets list when work has run out of attempts, or when shutting down the old provider needs a payment settled first. A target already in use cannot be retried; choose another provider. + ## What Users See - S3 upload can succeed before Filecoin storage finishes. diff --git a/docs/en/operations/troubleshooting.md b/docs/en/operations/troubleshooting.md index 222a2be..c5724fb 100644 --- a/docs/en/operations/troubleshooting.md +++ b/docs/en/operations/troubleshooting.md @@ -132,6 +132,8 @@ Retry only after RPC connectivity, storage provider availability, wallet funds, synaps3 admin task retry 42 ``` +Provider replacement work is the exception: do not retry it from Tasks. Finished or stopped replacement tasks provide **Open Data Sets**, which opens the affected bucket directly at **Details** → **Storage** → **Data Sets**. Use **Retry replacement** only when that action is shown. If the selected provider already stores this bucket, choose a different provider instead. + ## Provider or RPC Issues Check provider health and Filecoin readiness in the dashboard, or inspect the Admin API: diff --git a/docs/en/operations/upgrade-recovery.md b/docs/en/operations/upgrade-recovery.md index 0ca521e..d0720f9 100644 --- a/docs/en/operations/upgrade-recovery.md +++ b/docs/en/operations/upgrade-recovery.md @@ -61,7 +61,8 @@ Receive write -> save object -> record metadata -> return success -> continue ba | Private provider URL blocked | Keep blocked by default; enable `filecoin.allow_private_networks` only for trusted private deployments. | | Database full | Free space or scale the database. | | Cache disk full | Increase disk, raise `cache.max_size_gb`, or restore upload and eviction progress. | -| Process crash | Restart the service, then verify health and task statistics; unfinished tasks become eligible to continue. | +| Provider is permanently unavailable, or must be evacuated | Open the bucket, choose **Details**, then **Storage** → **Data Sets**, and replace the provider. New uploads move to the new provider once it is ready. Existing objects copy from another replica or from local cache; an object with neither cannot be copied, and the old provider is not shut down. If the selected target is already in use, choose another provider rather than retrying it. | +| Process crash | Restart the service, then verify health and task statistics; unfinished tasks become eligible to continue. If shutting down an unused replacement service was already submitted, SynapS3 continues checking that request instead of submitting it again. | A provider becoming unavailable after a copy has already been stored does not necessarily create a retryable task. Use storage-health views to identify affected copies. Restoring the target copy count is part of [Planned Replica Repair](../concepts/filecoin-storage-flow.md#planned-replica-repair). diff --git a/docs/en/reference/admin-api.md b/docs/en/reference/admin-api.md index 09d2f21..afc0ef0 100644 --- a/docs/en/reference/admin-api.md +++ b/docs/en/reference/admin-api.md @@ -93,6 +93,7 @@ Treat these endpoints as change-window operations. They can change data, credent | S3 users | `POST /api/v1/s3-users`, `PUT /api/v1/s3-users/{accessKey}`, `POST /api/v1/s3-users/{accessKey}/secret`, `DELETE /api/v1/s3-users/{accessKey}` | Changes client access or invalidates credentials. | | Buckets and objects | bucket create, owner/copy-policy updates, object upload/download/delete/restore/permanent-delete | Changes or exposes user-visible S3 data and metadata. | | Tasks and storage health | task retry, diagnostic refresh, storage provider and data set refresh | Requeues work or refreshes operational status. | +| Provider replacement | `POST /api/v1/buckets/{name}/data-sets/{id}/replacement`, `POST /api/v1/storage-replacements/{id}/retry` | Creates a new paid storage service, moves a replica to it, and ends the old service. | ## Health and Metrics @@ -130,6 +131,9 @@ Treat these endpoints as change-window operations. They can change data, credent | `POST` | `/api/v1/buckets/{name}/objects/permanent-delete` | Permanently delete an object version. | | `POST` | `/api/v1/buckets/{name}/objects/deleted/permanent-delete` | Permanently delete a deleted object version. | | `GET` | `/api/v1/buckets/{name}/storage-health/affected-versions` | List versions affected by storage health issues. | +| `GET` | `/api/v1/buckets/{name}/data-sets/{id}/replacement/providers` | List the providers this replica can move to, and why the others cannot take it. | +| `POST` | `/api/v1/buckets/{name}/data-sets/{id}/replacement` | Authorize replacing the storage provider behind a replica. | +| `POST` | `/api/v1/storage-replacements/{id}/retry` | Resume a failed or attention-holding provider replacement. | For object upload, the HTTP `Content-Type` is the uploaded object's content type. It is not a JSON request marker. @@ -189,6 +193,68 @@ A missing or permanently deleted source returns `404 Not Found`; insufficient ca The restore streams synchronously for up to one hour and requires enough cache capacity for the new destination version. +### Replace a Storage Provider + +`POST /api/v1/buckets/{name}/data-sets/{id}/replacement` is the only way to replace the storage provider behind a replica. One confirmation authorizes all of it: a new paid storage service, moving new uploads to it, copying existing data across, and ending the old service once every retained version is readable on the new provider. Objects copy from another replica or from local cache. An object with neither cannot be copied, and the old provider is not shut down. + +Choose the new provider automatically: + +```json +{ "mode": "automatic", "client_request_id": "019d2e22-8c36-7d5b-a6be-5f7fa6d6f584" } +``` + +Automatic selection excludes every provider the bucket has ever used, including retired ones. Or name the provider yourself: + +```json +{ "mode": "manual", "provider_id": "202", "client_request_id": "019d2e22-8c36-7d5b-a6be-5f7fa6d6f584" } +``` + +A named provider must appear in the complete available, active, PDP-capable provider inventory. It may be one the bucket used before, provided that earlier service has already been retired. The provider being replaced, and any provider still holding a live generation of this bucket, are rejected. + +`client_request_id` is required after trimming and must contain 1–128 characters. The first successful request returns `201 Created`. Replaying the same bucket, source, mode, and manual provider with the same ID returns the original record and `200 OK`, even after the replica has switched. Reusing the ID with different parameters returns `409 Conflict` with `replacement_idempotency_conflict`. An automatic replay is resolved before reading the provider inventory, so a later inventory change cannot choose a different provider. + +Confirming again for the same replica supersedes the earlier request and returns `201 Created`; it is not a conflict. The unused provider from the earlier request is shut down. `replacement_active` means something else: the replica is the target of another unfinished replacement, which has to be resolved first. + +Only the replica that currently receives writes can be replaced; a historical generation is reported with `"replaceable": false` in `GET /api/v1/buckets/{name}`. + +A successful first confirmation returns `201 Created` with the replacement record; an exact replay returns `200 OK`. `GET /api/v1/buckets/{name}` returns the bucket's recent replacements in `replacements`, newest first, up to the 50 most recent. + +Replacement moves through these states: + +| Status | Meaning | +| --- | --- | +| `preparing_target` | The new service is being created. Writes still go to the current provider. | +| `migrating` | The new provider receives new uploads while existing data is copied across. | +| `waiting` | Paused. `wait_reason` and `wait_message` distinguish service creation (`target_creating`), writable confirmation (`target_writable`), an unreachable provider (`target`), funding, source availability, and safe retirement waits. Most waits resume without action. | +| `retiring` | Everything is copied and the old service is being ended. | +| `cleanup_attention` | Ending the old service needs an operator decision, such as settling payment debt. | +| `failed` | Work ran out of attempts and needs to be retried. | +| `completed` | The old service is ended and the replica now lives on the new provider. | +| `superseded` | A later confirmation replaced this request. | + +`last_error` is set only for `failed` and `cleanup_attention`, and is cleared by a retry. Waiting never sets it, because waiting is not a failure. A failed response may also include `failure_reason`. `target_in_use` is permanent for that approved target: choose a different provider; the retry endpoint returns a conflict. + +`items_total` and `items_copied` count unique stored content, not object versions: content shared by many versions is copied once. Content deleted while migration is in progress is no longer needed and is not counted as copied. After copying finishes, the response reports how much content was copied and how much no longer needed to move; `items_copied/items_total` is not a completion percentage. The confirmation dialog instead counts referenced versions and total size. + +`POST /api/v1/storage-replacements/{id}/retry` resumes a `failed` or `cleanup_attention` replacement on the same approved provider. + +Choosing a different provider requires a new confirmation, and is only available while the retiring provider still holds the replica. Once the new provider has taken the replica over, each generation holds data the other does not, so confirming again on either one is refused (`replacement_source_not_current` on the old, `replacement_active` on the new) and the approved copy has to be finished with retry. + +Conflicts return `409 Conflict` with a stable code: + +```json +{ + "error": "that provider already stores a replica of this bucket", + "code": "replacement_target_in_use" +} +``` + +The codes are `replacement_active`, `replacement_target_in_use`, `replacement_target_unavailable`, `replacement_no_eligible_provider`, `replacement_source_not_current`, `replacement_superseded`, `replacement_not_retryable`, `replacement_task_running`, and `replacement_idempotency_conflict`. An invalid provider choice returns `400 Bad Request` with `replacement_target_invalid`; a currently unavailable manual target returns `400` with `replacement_target_unavailable`; an unknown bucket, data set, or replacement returns `404 Not Found`; an unavailable storage service returns `503 Service Unavailable`; internal failures return `500 Internal Server Error`. + +`GET /api/v1/buckets/{name}/data-sets/{id}/replacement/providers` lists every provider currently reported available with `eligible`, an `ineligible_reason` of `current_source` or `already_serves_bucket`, and `previously_used` for a provider this bucket has used and fully retired. Providers that cannot take the replica are listed rather than omitted, so an operator can see why one they expected is unavailable. It is the same inventory the storage topology reports under the `Available` filter, so a provider listed there is offered here and an unreachable one is offered in neither. Eligibility is the same rule the confirmation enforces. Automatic selection is stricter still: it never returns to a provider this bucket has used, which a manual choice may. + +Confirmation only checks what SynapS3 has recorded. A provider that still runs a storage service for this bucket on chain is detected when the replacement prepares its target: the replacement stops at `failed` with the provider and data set named, and the operator confirms again on a different provider. The replica has not moved at that point, so nothing is at risk. A provider whose earlier service for this bucket was retired normally can be chosen again. + ## Tasks | Method | Path | Purpose | @@ -202,6 +268,8 @@ The restore streams synchronously for up to one hour and requires enough cache c | `GET` | `/admin/exhausted-tasks` | List exhausted tasks. Supports `limit` up to `1000`. | | `POST` | `/admin/exhausted-tasks/{id}/retry` | Retry an exhausted task (legacy path). | +Replacement and retirement task responses that refer to a bucket include `bucket_name` in list and reference-detail responses. Retrying replacement work from the task queue returns `409 Conflict` with `"code": "replacement_task_retry_unsupported"`. When a replacement task has completed or stopped, use **Open Data Sets**, or open the bucket and go to Details → Storage → Data Sets. A `target_in_use` failure has no Retry action because it requires a different provider. + ## Wallet and Filecoin | Method | Path | Purpose | diff --git a/docs/en/reference/cli-api.md b/docs/en/reference/cli-api.md index 7cdff03..e72fc7a 100644 --- a/docs/en/reference/cli-api.md +++ b/docs/en/reference/cli-api.md @@ -97,6 +97,8 @@ Admin global flags must appear after `admin` and before the subcommand: Task listing supports `--type`, `--stage`, `--status`, `--limit`, and `--offset`. `--stage` requires `--type`. +`synaps3 admin task retry` does not retry provider replacement work. Use **Open Data Sets** from a finished or stopped replacement task, or open the bucket and go to **Details** → **Storage** → **Data Sets**. If the selected provider already stores this bucket, choose a different provider instead of retrying. + Cache eviction policy accepts `lru`, `after_upload`, or `none`. LRU watermarks must satisfy `0 <= low < high <= 100`; these settings are retained but inactive under the other policies. ## Settings Safety diff --git a/docs/zh/concepts/filecoin-storage-flow.md b/docs/zh/concepts/filecoin-storage-flow.md index e2d09e2..c11781a 100644 --- a/docs/zh/concepts/filecoin-storage-flow.md +++ b/docs/zh/concepts/filecoin-storage-flow.md @@ -52,7 +52,7 @@ synaps3 admin task retry 42 健康检查会记录存储提供方和本地数据集的状态。仪表盘会使用这些结果，标出 `unavailable`、`degraded` 或 `unknown` 的存储副本。 -如果已建立的存储提供方在首次副本尚未全部完成时暂时不可用，SynapS3 会继续使用其他已分配且可写的副本。未完成副本会等待且不消耗重试次数，并在原存储提供方恢复可达后自动继续；系统不会自动选择替代提供方。已完成存储的副本随后变为不可用时，其修复仍属于下面计划支持的副本修复功能。 +如果已建立的存储提供方在首次副本尚未全部完成时暂时不可用，SynapS3 会继续使用其他已分配且可写的副本。未完成副本会等待且不消耗重试次数，并在原存储提供方恢复可达后自动继续；系统不会自动选择替代提供方，运营者批准的替换路径见[替换存储提供方](#替换存储提供方)。已完成存储的副本随后变为不可用时，其修复仍属于下面计划支持的副本修复功能。 ## 目标副本与最低耐久副本 @@ -60,6 +60,16 @@ synaps3 admin task retry 42 修改目标副本数只影响新上传。修改最低耐久副本数也会重新评估当前上传仍保留的缓存。提高门槛不会让已经进入已存储状态的版本回退，也无法恢复已经删除的缓存。 +## 替换存储提供方 + +当某个存储提供方永久不可用，或者你计划迁离它时，在仪表盘打开该存储桶，选择 **Details**，然后到 **Storage** → **Data Sets** 替换提供方。这始终是一次明确的决定：SynapS3 不会自行更换提供方，因为这会创建新的付费服务，并改变数据的存放位置。 + +一次确认覆盖整个迁移过程。SynapS3 会创建新的存储服务，新提供方就绪后把新上传切换过去，复制已有数据，然后才关闭旧提供方。对象从其他副本或本地缓存复制。两者都没有的对象无法复制，旧提供方也不会被关闭。仍能从旧提供方读取的数据会保持可读，直到每个保留版本都能从新提供方读取。 + +替换进行期间，副本会显示仍需推进或需要操作的所有迁移，包括其他副本上的并行迁移。进度以已存储条目计数，因为被多个版本共享的内容只复制一次。复制完成后，仪表盘会分别显示已迁移的内容，以及在迁移前已被删除、无需再迁移的内容。 + +有些步骤是等待而不是失败。仪表盘会区分创建新服务、等待服务可写、提供方不可达、钱包资金和缺少可读内容。多数等待会自行继续。如果某个对象既没有其他副本、也没有本地缓存，替换会一直等待，直到有可用来源。重试次数用尽，或关闭旧提供方需要先结清欠费时，在同一个 Data Sets 列表里重试。目标已被占用时不能重试，请改选提供方。 + ## 用户能看到什么 - S3 上传可以在 Filecoin 存储完成前成功。 diff --git a/docs/zh/operations/troubleshooting.md b/docs/zh/operations/troubleshooting.md index 9c611a9..0ffbd40 100644 --- a/docs/zh/operations/troubleshooting.md +++ b/docs/zh/operations/troubleshooting.md @@ -132,6 +132,8 @@ synaps3 admin task list --status exhausted --limit 100 synaps3 admin task retry 42 ``` +提供方替换工作是例外：不要从 Tasks 重试。已完成或已停止的替换任务会提供 **Open Data Sets**，直接打开受影响存储桶的 **Details** → **Storage** → **Data Sets**。仅在页面显示 **Retry replacement** 时使用它；如果所选提供方已经存储该桶，请改选其他提供方。 + ## 存储提供方或 RPC 问题 在仪表盘查看存储提供方健康状态和 Filecoin readiness，或检查 Admin API： diff --git a/docs/zh/operations/upgrade-recovery.md b/docs/zh/operations/upgrade-recovery.md index f73fad4..df66a5f 100644 --- a/docs/zh/operations/upgrade-recovery.md +++ b/docs/zh/operations/upgrade-recovery.md @@ -61,7 +61,8 @@ synaps3 admin task stats | 私有存储提供方 URL 被阻止 | 默认保持阻止；只在可信私有部署中开启 `filecoin.allow_private_networks`。 | | 数据库空间不足 | 释放空间或扩容数据库。 | | 缓存磁盘空间不足 | 扩容磁盘、提高 `cache.max_size_gb`，或恢复上传和淘汰进度。 | -| 进程崩溃 | 重启服务，再检查健康状态和任务统计；未完成任务会重新进入可继续处理的状态。 | +| 存储提供方永久不可用，或需要计划性迁离 | 打开该存储桶，选择 **Details**，然后到 **Storage** → **Data Sets** 替换提供方。新提供方就绪后，新上传会切过去。已有对象从其他副本或本地缓存复制；两者都没有的对象无法复制，旧提供方也不会被关闭。所选目标已被占用时，请改选提供方，而不是重试。 | +| 进程崩溃 | 重启服务，再检查健康状态和任务统计；未完成任务会重新进入可继续处理的状态。如果关闭未使用替换服务的请求已经提交，SynapS3 会继续检查该请求，不会再次提交。 | 副本完成存储后，如果存储提供方变为不可用，不一定会产生可重试任务。使用存储健康视图识别受影响副本；恢复目标副本数属于[计划支持的副本修复](../concepts/filecoin-storage-flow.md#计划支持的副本修复)。 diff --git a/docs/zh/reference/admin-api.md b/docs/zh/reference/admin-api.md index ac6efb0..a0d4f79 100644 --- a/docs/zh/reference/admin-api.md +++ b/docs/zh/reference/admin-api.md @@ -93,6 +93,7 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` | S3 用户 | `POST /api/v1/s3-users`、`PUT /api/v1/s3-users/{accessKey}`、`POST /api/v1/s3-users/{accessKey}/secret`、`DELETE /api/v1/s3-users/{accessKey}` | 改变客户端访问权限，或让已有凭据失效。 | | 存储桶和对象 | 创建存储桶、更新 owner/copy-policy，以及上传、下载、删除、恢复或永久删除对象 | 改变或暴露用户可见的 S3 数据和元数据。 | | 后台任务和存储健康 | 任务重试、诊断刷新、存储提供方和数据集刷新 | 重新入队任务，或刷新运维状态。 | +| 提供方替换 | `POST /api/v1/buckets/{name}/data-sets/{id}/replacement`、`POST /api/v1/storage-replacements/{id}/retry` | 创建新的付费存储服务，把副本迁移过去，并终止旧服务。 | ## 健康检查和指标 @@ -130,6 +131,9 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` | `POST` | `/api/v1/buckets/{name}/objects/permanent-delete` | 永久删除对象版本。 | | `POST` | `/api/v1/buckets/{name}/objects/deleted/permanent-delete` | 永久删除已删除对象版本。 | | `GET` | `/api/v1/buckets/{name}/storage-health/affected-versions` | 列出受存储健康问题影响的版本。 | +| `GET` | `/api/v1/buckets/{name}/data-sets/{id}/replacement/providers` | 列出该副本可以迁往的提供方，以及其余提供方不能接管的原因。 | +| `POST` | `/api/v1/buckets/{name}/data-sets/{id}/replacement` | 授权替换某个副本背后的存储提供方。 | +| `POST` | `/api/v1/storage-replacements/{id}/retry` | 恢复处于 `failed` 或 `cleanup_attention` 的提供方替换。 | 对象上传时，HTTP `Content-Type` 表示上传对象的内容类型，不是 JSON 请求标记。 @@ -189,6 +193,68 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` 恢复操作同步流式执行，最长一小时，并且缓存必须能容纳新的目标版本。 +### 替换存储提供方 + +`POST /api/v1/buckets/{name}/data-sets/{id}/replacement` 是替换某个副本背后存储提供方的唯一入口。一次确认即授权全部动作：新建付费存储服务、把新上传切换过去、复制已有数据，并在每个保留版本都能从新提供方读取之后关闭旧提供方。对象从其他副本或本地缓存复制。两者都没有的对象无法复制，旧提供方也不会被关闭。 + +自动选择新的提供方： + +```json +{ "mode": "automatic", "client_request_id": "019d2e22-8c36-7d5b-a6be-5f7fa6d6f584" } +``` + +自动选择会排除该存储桶用过的所有提供方，包括已退休的。也可以指定提供方： + +```json +{ "mode": "manual", "provider_id": "202", "client_request_id": "019d2e22-8c36-7d5b-a6be-5f7fa6d6f584" } +``` + +指定的提供方必须出现在完整的可用、活跃且支持 PDP 的提供方清单中。它可以是该存储桶以前用过的，前提是那次服务已经退休。正在被替换的提供方，以及任何仍持有该存储桶活跃代的提供方，都会被拒绝。 + +`client_request_id` 为必填项，trim 后长度必须为 1–128 个字符。首次成功返回 `201 Created`。同一存储桶、来源、模式和手动提供方使用同一个 ID 精确重放时，会返回原记录和 `200 OK`，即使副本已经切换也一样。同一个 ID 携带不同参数会返回 `409 Conflict` 和 `replacement_idempotency_conflict`。自动模式的重放会在读取提供方清单前命中原记录，因此清单后续变化不会改选提供方。 + +对同一副本再次确认会取代先前的请求并返回 `201 Created`，不是冲突。先前请求里尚未使用的提供方会被关闭。`replacement_active` 表示的是另一件事：该副本是另一次未完成替换的目标，必须先处理那一次。 + +只有当前接收写入的副本可以被替换；历史代在 `GET /api/v1/buckets/{name}` 中返回 `"replaceable": false`。 + +首次确认成功返回 `201 Created` 和替换记录；精确重放返回 `200 OK`。`GET /api/v1/buckets/{name}` 在 `replacements` 中返回该存储桶最近的替换记录，最新的在前，最多 50 条。 + +替换会经历以下状态： + +| 状态 | 含义 | +| --- | --- | +| `preparing_target` | 正在创建新服务。写入仍然发往当前提供方。 | +| `migrating` | 新提供方开始接收新上传，同时复制已有数据。 | +| `waiting` | 已暂停。`wait_reason` 与 `wait_message` 会区分服务创建（`target_creating`）、可写确认（`target_writable`）、提供方不可达（`target`）、资金、来源可用性和安全退休等待。多数等待无需操作即可继续。 | +| `retiring` | 数据已复制完毕，正在终止旧服务。 | +| `cleanup_attention` | 终止旧服务需要运营者决定，例如结清欠费。 | +| `failed` | 重试次数用尽，需要重新发起。 | +| `completed` | 旧服务已终止，该副本已落在新提供方上。 | +| `superseded` | 更晚的一次确认取代了本次请求。 | + +`last_error` 只在 `failed` 和 `cleanup_attention` 时设置，重试会清空它。等待状态从不设置它，因为等待不是失败。失败响应还可能包含 `failure_reason`。`target_in_use` 对当前已批准目标是永久失败：需要改选提供方，重试接口会返回冲突。 + +`items_total` 与 `items_copied` 统计的是唯一的已存储内容，而不是对象版本：被多个版本共享的内容只复制一次。迁移期间删除的内容已不再需要，不会算作已复制。复制结束后，响应会分别说明已复制的内容，以及已无需迁移的内容；`items_copied/items_total` 不是完成百分比。确认页统计的是引用版本数和数据量。 + +`POST /api/v1/storage-replacements/{id}/retry` 在同一个已批准的提供方上恢复 `failed` 或 `cleanup_attention` 的替换。 + +更换提供方需要重新确认，且仅在旧提供方仍持有该副本时可用。新提供方接管副本之后，两代各自持有对方没有的数据，因此对任意一代再次确认都会被拒绝（旧代返回 `replacement_source_not_current`，新代返回 `replacement_active`），此时只能用重试完成已批准的复制。 + +冲突返回 `409 Conflict` 并附带稳定的 code： + +```json +{ + "error": "that provider already stores a replica of this bucket", + "code": "replacement_target_in_use" +} +``` + +这些 code 包括 `replacement_active`、`replacement_target_in_use`、`replacement_target_unavailable`、`replacement_no_eligible_provider`、`replacement_source_not_current`、`replacement_superseded`、`replacement_not_retryable`、`replacement_task_running` 和 `replacement_idempotency_conflict`。无效提供方选择返回 `400 Bad Request` 和 `replacement_target_invalid`；当前不可用的手动目标返回 `400` 和 `replacement_target_unavailable`；未知的存储桶、数据集或替换记录返回 `404 Not Found`；存储服务不可用返回 `503 Service Unavailable`；内部失败返回 `500 Internal Server Error`。 + +`GET /api/v1/buckets/{name}/data-sets/{id}/replacement/providers` 列出当前探测为可用的提供方，附带 `eligible`、取值为 `current_source` 或 `already_serves_bucket` 的 `ineligible_reason`，以及标记该存储桶用过并已完全退休的 `previously_used`。不能接管该副本的提供方会照常列出而不是省略，便于运营者看清预期中的提供方为何不可用。这份清单与存储拓扑页在 `Available` 过滤下读取的是同一份：在那里可用的提供方这里会提供，探测不通的两边都不会出现。可选性判定与确认阶段完全一致。自动选择更严格：它绝不会回到该存储桶用过的提供方，而手动选择可以。 + +确认阶段只能检查 SynapS3 已记录的信息。如果某提供方在链上仍为该存储桶运行着存储服务，会在替换准备目标时被发现：替换停在 `failed`，并写明提供方与数据集，运营者改选另一个提供方重新确认即可。此时副本尚未迁移，没有任何风险。此前已正常退休的提供方可以再次选择。 + ## 任务 | Method | Path | 用途 | @@ -202,6 +268,8 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` | `GET` | `/admin/exhausted-tasks` | 列出 exhausted 任务。支持最大为 `1000` 的 `limit`。 | | `POST` | `/admin/exhausted-tasks/{id}/retry` | 重试 exhausted 任务（遗留路径）。 | +引用存储桶的替换和退休任务会在列表与引用详情响应中包含 `bucket_name`。从任务队列重试提供方替换工作会返回 `409 Conflict` 和 `"code": "replacement_task_retry_unsupported"`。替换任务完成或停止后，可以使用 **Open Data Sets**，或打开存储桶并前往 Details → Storage → Data Sets。`target_in_use` 失败不会显示 Retry，因为它需要改选提供方。 + ## 钱包和 Filecoin | Method | Path | 用途 | diff --git a/docs/zh/reference/cli-api.md b/docs/zh/reference/cli-api.md index 576d77a..483c0c5 100644 --- a/docs/zh/reference/cli-api.md +++ b/docs/zh/reference/cli-api.md @@ -97,6 +97,8 @@ Admin 全局 flags 必须放在 `admin` 之后、子命令之前： 列出后台任务时支持 `--type`、`--stage`、`--status`、`--limit` 和 `--offset`。`--stage` 必须与 `--type` 一起使用。 +`synaps3 admin task retry` 不会重试提供方替换工作。请在已完成或已停止的替换任务上使用 **Open Data Sets**，或打开存储桶并前往 **Details** → **Storage** → **Data Sets**。如果所选提供方已经存储该桶，请改选其他提供方，而不是重试。 + 缓存淘汰策略可设为 `lru`、`after_upload` 或 `none`。LRU 水位必须满足 `0 <= low < high <= 100`；其他策略会保留这些设置，但不使用它们。 ## 设置安全 diff --git a/internal/admin/api_buckets.go b/internal/admin/api_buckets.go index 720666c..6c5164d 100644 --- a/internal/admin/api_buckets.go +++ b/internal/admin/api_buckets.go @@ -91,6 +91,8 @@ type bucketDetailResponse struct { VersioningStatus string `json:"versioning_status"` VersioningEnforced bool `json:"versioning_enforced"` DataSets []storageDataSetSummaryResponse `json:"data_sets"` + // Replacements is the bucket's full history, newest first. + Replacements []providerReplacementResponse `json:"replacements"` } type storageDataSetSummaryResponse struct { @@ -98,6 +100,9 @@ type storageDataSetSummaryResponse struct { BucketID int64 `json:"bucket_id"` BucketName string `json:"bucket_name,omitempty"` CopyIndex int `json:"copy_index"` + Generation int `json:"generation"` + IsCurrent bool `json:"is_current"` + Replaceable bool `json:"replaceable"` ProviderID string `json:"provider_id"` ProviderIdentity *providerIdentityResponse `json:"provider_identity,omitempty"` DataSetID *string `json:"data_set_id,omitempty"` @@ -379,6 +384,7 @@ func (s *Server) handleAPIGetBucket(w http.ResponseWriter, r *http.Request) { VersioningStatus: "Enabled", VersioningEnforced: true, DataSets: dataSets, + Replacements: s.bucketReplacementResponses(ctx, bucket.Name, bucket.ID), }) } @@ -613,10 +619,15 @@ func (s *Server) storageDataSetSummaryResponses(ctx context.Context, summaries [ storageHealth = dataSetStorageHealthQueryFailureInfo() } out = append(out, storageDataSetSummaryResponse{ - ID: summary.ID, - BucketID: summary.BucketID, - BucketName: summary.BucketName, - CopyIndex: summary.CopyIndex, + ID: summary.ID, + BucketID: summary.BucketID, + BucketName: summary.BucketName, + CopyIndex: summary.CopyIndex, + Generation: summary.Generation, + IsCurrent: summary.IsCurrent, + // Only the generation that receives writes can be replaced; + // replacing a historical one would move nothing. + Replaceable: summary.IsCurrent && summary.Status != model.StorageDataSetStatusRetired, ProviderID: summary.ProviderID.String(), ProviderIdentity: providerIdentityFromSnapshot(identities, summary.ProviderID), DataSetID: onChainIDStringPtr(summary.DataSetID), diff --git a/internal/admin/api_buckets_test.go b/internal/admin/api_buckets_test.go index 47775da..a08cf40 100644 --- a/internal/admin/api_buckets_test.go +++ b/internal/admin/api_buckets_test.go @@ -97,6 +97,9 @@ func newBucketAPIMux(srv *Server) *http.ServeMux { mux.HandleFunc("POST /api/v1/buckets/{name}/objects/versions/restore", srv.handleAPIRestoreObjectVersion) mux.HandleFunc("GET /api/v1/buckets/{name}/objects/download", srv.handleAPIDownloadObject) mux.HandleFunc("POST /api/v1/buckets/{name}/objects/upload", srv.handleAPIUploadObject) + mux.HandleFunc("POST /api/v1/buckets/{name}/data-sets/{id}/replacement", srv.handleAPIStartDataSetReplacement) + mux.HandleFunc("GET /api/v1/buckets/{name}/data-sets/{id}/replacement/providers", srv.handleAPIListDataSetReplacementProviders) + mux.HandleFunc("POST /api/v1/storage-replacements/{id}/retry", srv.handleAPIRetryStorageReplacement) return mux } @@ -565,7 +568,7 @@ func bindAdminPartialUpload(t *testing.T, repos *repository.Repositories, versio }); err != nil { t.Fatalf("bind primary committed upload: %v", err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, upload.ID, 1, "secondary pull: timeout"); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 1, LastError: "secondary pull: timeout"}); err != nil { t.Fatalf("mark secondary failed: %v", err) } return upload @@ -654,7 +657,7 @@ func markAdminFailedUpload(t *testing.T, repos *repository.Repositories, version }}); err != nil { t.Fatalf("create failed upload copy: %v", err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, upload.ID, 0, message); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 0, LastError: message}); err != nil { t.Fatalf("mark failed upload copy: %v", err) } return upload diff --git a/internal/admin/api_replacement.go b/internal/admin/api_replacement.go new file mode 100644 index 0000000..ff41143 --- /dev/null +++ b/internal/admin/api_replacement.go @@ -0,0 +1,524 @@ +package admin + +import ( + "context" + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + idtypes "github.com/strahe/synaps3/internal/types" +) + +// providerReplacementSelector resolves the provider an automatic replacement +// should use. It is injected so the admin API stays testable without a live +// storage service. +type providerReplacementSelector interface { + // ListReplacementProviders returns the approved, active providers a + // replacement may use, in a stable order. Which of them a given replica can + // actually take is decided by replacementProviderCandidates. + ListReplacementProviders(ctx context.Context) ([]idtypes.OnChainID, error) +} + +// WithProviderReplacement enables the provider replacement endpoints. Without it +// they report that the storage service is unavailable. +func (s *Server) WithProviderReplacement(selector providerReplacementSelector) *Server { + s.replacementSelector = selector + return s +} + +type startReplacementRequest struct { + Mode string `json:"mode"` + ProviderID string `json:"provider_id"` + ClientRequestID string `json:"client_request_id"` +} + +// providerReplacementResponse is the operator-facing view of one replacement. +// Counts are deliberately named for what they measure: a confirmation talks +// about referenced versions, progress talks about stored content. +type providerReplacementResponse struct { + ID int64 `json:"id"` + BucketName string `json:"bucket_name"` + CopyIndex int `json:"copy_index"` + Status string `json:"status"` + WaitReason string `json:"wait_reason,omitempty"` + WaitMessage string `json:"wait_message,omitempty"` + FailureReason string `json:"failure_reason,omitempty"` + SelectionMode string `json:"selection_mode"` + + Source replacementDataSetResponse `json:"source"` + Target replacementDataSetResponse `json:"target"` + + ItemsTotal int `json:"items_total"` + ItemsCopied int `json:"items_copied"` + LastError *string `json:"last_error"` + TerminationEpoch *int64 `json:"termination_epoch"` + + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` +} + +type replacementDataSetResponse struct { + ID int64 `json:"id"` + Generation int `json:"generation"` + IsCurrent bool `json:"is_current"` + Status string `json:"status"` + ProviderID string `json:"provider_id"` + DataSetID *string `json:"data_set_id"` + ProviderIdentity *providerIdentityResponse `json:"provider_identity,omitempty"` +} + +// handleAPIStartDataSetReplacement is the only entry point that authorizes a +// provider replacement. One confirmation covers the new paid service, the +// topology switch, the migration, and retirement of the old service once the +// safety gate passes. +func (s *Server) handleAPIStartDataSetReplacement(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if !bucketNameRe.MatchString(name) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid bucket name"}) + return + } + dataSetID, err := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err != nil || dataSetID <= 0 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid data set id"}) + return + } + var req startReplacementRequest + if !decodeBucketStrictJSON(w, r, &req) { + return + } + req.ClientRequestID = strings.TrimSpace(req.ClientRequestID) + if req.ClientRequestID == "" || len(req.ClientRequestID) > 128 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "client_request_id is required"}) + return + } + + ctx := r.Context() + bucket, source, ok := s.replacementSubject(w, ctx, name, dataSetID) + if !ok { + return + } + + mode := storagereplacement.SelectionMode(req.Mode) + if !mode.Valid() { + s.writeReplacementError(w, storagereplacement.ErrInvalidTarget, name) + return + } + existing, err := s.repos.Replacements.GetByClientRequestID(ctx, bucket.ID, req.ClientRequestID) + if err != nil { + s.writeReplacementError(w, err, name) + return + } + if existing != nil { + if !replacementReplayMatches(existing, source.ID, mode, req.ProviderID) { + s.writeReplacementError(w, storagereplacement.ErrIdempotencyConflict, name) + return + } + response, responseErr := s.providerReplacementResponse(ctx, bucket.Name, existing) + if responseErr != nil { + s.logger.Error("api: failed to build replacement replay response", "error", responseErr, "replacementID", existing.ID) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return + } + writeJSON(w, http.StatusOK, response) + return + } + targetProvider, err := s.resolveReplacementProvider(ctx, bucket, source, mode, req.ProviderID) + if err != nil { + s.writeReplacementError(w, err, name) + return + } + + row, created, err := s.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, + SourceDataSetID: source.ID, + SelectionMode: mode, + TargetProviderID: targetProvider, + ClientRequestID: req.ClientRequestID, + MaxRetries: s.uploadMaxRetries, + }) + if err != nil { + s.writeReplacementError(w, err, name) + return + } + response, err := s.providerReplacementResponse(ctx, bucket.Name, row) + if err != nil { + s.logger.Error("api: failed to build replacement response", "error", err, "replacementID", row.ID) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return + } + status := http.StatusOK + if created { + status = http.StatusCreated + } + writeJSON(w, status, response) +} + +func replacementReplayMatches( + row *storagereplacement.Replacement, + sourceDataSetID int64, + mode storagereplacement.SelectionMode, + requested string, +) bool { + if row.SourceDataSetID != sourceDataSetID || row.SelectionMode != mode { + return false + } + if mode == storagereplacement.SelectionModeAutomatic { + return strings.TrimSpace(requested) == "" + } + providerID, err := idtypes.ParseOnChainID("provider_id", requested) + return err == nil && row.RequestedProviderID != nil && row.RequestedProviderID.Equal(providerID) +} + +// resolveReplacementProvider turns the operator's choice into one provider. +// Automatic selection excludes every provider the bucket has used, including +// retired generations; a supplied Provider ID is used exactly as approved. +func (s *Server) resolveReplacementProvider( + ctx context.Context, + bucket *model.Bucket, + source *model.StorageDataSet, + mode storagereplacement.SelectionMode, + requested string, +) (idtypes.OnChainID, error) { + switch mode { + case storagereplacement.SelectionModeManual: + providerID, err := idtypes.ParseOnChainID("provider_id", requested) + if err != nil || providerID.IsZero() { + return idtypes.OnChainID{}, storagereplacement.ErrInvalidTarget + } + if providerID.Equal(source.ProviderID) { + return idtypes.OnChainID{}, storagereplacement.ErrInvalidTarget + } + candidates, err := s.replacementProviderCandidates(ctx, bucket, source) + if err != nil { + return idtypes.OnChainID{}, err + } + for _, candidate := range candidates { + if !candidate.ProviderID.Equal(providerID) { + continue + } + if candidate.Eligible { + return providerID, nil + } + if candidate.IneligibleReason == providerIneligibleServesBucket { + return idtypes.OnChainID{}, storagereplacement.ErrTargetInUse + } + return idtypes.OnChainID{}, storagereplacement.ErrInvalidTarget + } + return idtypes.OnChainID{}, storagereplacement.ErrTargetUnavailable + case storagereplacement.SelectionModeAutomatic: + if requested != "" { + return idtypes.OnChainID{}, storagereplacement.ErrInvalidTarget + } + candidates, err := s.replacementProviderCandidates(ctx, bucket, source) + if err != nil { + return idtypes.OnChainID{}, err + } + providerID, ok := firstAutomaticChoice(candidates) + if !ok { + return idtypes.OnChainID{}, storagereplacement.ErrNoEligibleProvider + } + return providerID, nil + default: + return idtypes.OnChainID{}, storagereplacement.ErrInvalidTarget + } +} + +// replacementProviderCandidates lists every approved provider together with +// whether this replica can move to it. Both the automatic choice and the +// dashboard's provider list read it, so neither can offer what Authorize +// refuses. +func (s *Server) replacementProviderCandidates( + ctx context.Context, + bucket *model.Bucket, + source *model.StorageDataSet, +) ([]replacementProviderCandidate, error) { + if s.replacementSelector == nil { + return nil, errReplacementUnavailable + } + providers, err := s.replacementSelector.ListReplacementProviders(ctx) + if err != nil { + return nil, err + } + bindings, err := s.repos.Uploads.ListDataSetBindings(ctx, bucket.ID) + if err != nil { + return nil, err + } + return replacementProviderCandidates(providers, bindings, source), nil +} + +var errReplacementUnavailable = errors.New("storage service is unavailable") + +// handleAPIRetryStorageReplacement resumes work an operator owns. Choosing a +// different provider needs a new confirmation, so this endpoint never changes +// the approved target. +func (s *Server) handleAPIRetryStorageReplacement(w http.ResponseWriter, r *http.Request) { + id, err := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err != nil || id <= 0 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid replacement id"}) + return + } + ctx := r.Context() + row, err := s.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ + ReplacementID: id, + MaxRetries: s.uploadMaxRetries, + }) + if err != nil { + s.writeReplacementError(w, err, "") + return + } + bucket, err := s.repos.Buckets.GetByID(ctx, row.BucketID) + if err != nil || bucket == nil { + s.logger.Error("api: failed to load bucket for replacement retry", "error", err, "bucketID", row.BucketID) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return + } + response, err := s.providerReplacementResponse(ctx, bucket.Name, row) + if err != nil { + s.logger.Error("api: failed to build replacement response", "error", err, "replacementID", row.ID) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return + } + writeJSON(w, http.StatusOK, response) +} + +func (s *Server) writeReplacementError(w http.ResponseWriter, err error, bucketName string) { + code := storagereplacement.Code(err) + switch { + case errors.Is(err, errReplacementUnavailable): + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"error": "storage service is unavailable"}) + case errors.Is(err, storagereplacement.ErrInvalidTarget): + writeJSON(w, http.StatusBadRequest, map[string]string{ + "error": "choose a different provider for this replica", + "code": code, + }) + case errors.Is(err, repository.ErrInvalidInput): + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid replacement request"}) + case errors.Is(err, repository.ErrNotFound): + writeJSON(w, http.StatusNotFound, map[string]string{"error": "not found"}) + case errors.Is(err, storagereplacement.ErrActiveReplacement): + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "this replica is already being replaced", + "code": code, + }) + case errors.Is(err, storagereplacement.ErrTargetInUse): + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "that provider already stores a replica of this bucket", + "code": code, + }) + case errors.Is(err, storagereplacement.ErrNoEligibleProvider): + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "no unused storage provider is available right now", + "code": code, + }) + case errors.Is(err, storagereplacement.ErrTargetUnavailable): + writeJSON(w, http.StatusBadRequest, map[string]string{ + "error": "that provider is not currently available for replacement", + "code": code, + }) + case errors.Is(err, storagereplacement.ErrIdempotencyConflict): + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "client_request_id already belongs to a different replacement request", + "code": code, + }) + case errors.Is(err, storagereplacement.ErrSourceNotCurrent): + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "this replica no longer receives writes, so replacing it would change nothing", + "code": code, + }) + case errors.Is(err, storagereplacement.ErrSuperseded): + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "a newer replacement has taken over this replica", + "code": code, + }) + case errors.Is(err, storagereplacement.ErrNotRetryable): + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "this replacement is still progressing on its own", + "code": code, + }) + case errors.Is(err, storagereplacement.ErrTaskRunning): + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "replacement work is still running; try again shortly", + "code": code, + }) + case errors.Is(err, repository.ErrConflict): + writeJSON(w, http.StatusConflict, map[string]string{"error": "replacement state changed; refresh and try again"}) + default: + s.logger.Error("api: provider replacement failed", "error", err, "bucket", bucketName) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + } +} + +func (s *Server) providerReplacementResponse(ctx context.Context, bucketName string, row *storagereplacement.Replacement) (providerReplacementResponse, error) { + source, err := s.repos.Uploads.GetDataSetBindingByID(ctx, row.SourceDataSetID) + if err != nil { + return providerReplacementResponse{}, err + } + target, err := s.repos.Uploads.GetDataSetBindingByID(ctx, row.TargetDataSetID) + if err != nil { + return providerReplacementResponse{}, err + } + identities := s.providerIdentities(replacementProviderIDs(source, target)) + response := providerReplacementResponse{ + ID: row.ID, + BucketName: bucketName, + CopyIndex: row.CopyIndex, + Status: string(row.Status), + SelectionMode: string(row.SelectionMode), + Source: replacementDataSetView(source, identities), + Target: replacementDataSetView(target, identities), + ItemsTotal: row.ItemsTotal, + ItemsCopied: row.ItemsCopied, + LastError: row.LastError, + TerminationEpoch: row.TerminationEpoch, + CreatedAt: row.CreatedAt, + UpdatedAt: row.UpdatedAt, + } + if row.WaitReason != nil { + response.WaitReason = string(*row.WaitReason) + response.WaitMessage = row.WaitReason.Message() + } + if row.FailureReason != nil { + response.FailureReason = string(*row.FailureReason) + } + return response, nil +} + +func replacementProviderIDs(sets ...*model.StorageDataSet) []idtypes.OnChainID { + ids := make([]idtypes.OnChainID, 0, len(sets)) + for _, set := range sets { + if set != nil { + ids = append(ids, set.ProviderID) + } + } + return ids +} + +func replacementDataSetView(set *model.StorageDataSet, identities map[string]*providerIdentityResponse) replacementDataSetResponse { + if set == nil { + return replacementDataSetResponse{} + } + view := replacementDataSetResponse{ + ID: set.ID, + Generation: set.Generation, + IsCurrent: set.IsCurrent, + Status: string(set.Status), + ProviderID: set.ProviderID.String(), + DataSetID: onChainIDStringPtr(set.DataSetID), + } + if identity, ok := identities[set.ProviderID.String()]; ok { + view.ProviderIdentity = identity + } + return view +} + +// bucketReplacementResponses returns the bucket's whole replacement history, +// newest first, so the dashboard can show what happened as well as what is +// happening. +func (s *Server) bucketReplacementResponses(ctx context.Context, bucketName string, bucketID int64) []providerReplacementResponse { + rows, err := s.repos.Replacements.ListForBucket(ctx, bucketID, 50) + if err != nil { + s.logger.Error("api: failed to list provider replacements", "error", err, "bucketID", bucketID) + return nil + } + out := make([]providerReplacementResponse, 0, len(rows)) + for i := range rows { + response, err := s.providerReplacementResponse(ctx, bucketName, &rows[i]) + if err != nil { + s.logger.Error("api: failed to build replacement response", "error", err, "replacementID", rows[i].ID) + continue + } + out = append(out, response) + } + return out +} + +// replacementSubject resolves the bucket and the replica a replacement acts on, +// writing the response itself when either is missing. +func (s *Server) replacementSubject( + w http.ResponseWriter, + ctx context.Context, + name string, + dataSetID int64, +) (*model.Bucket, *model.StorageDataSet, bool) { + bucket, err := s.repos.Buckets.GetByName(ctx, name) + if err != nil { + s.logger.Error("api: failed to load bucket for replacement", "error", err, "bucket", name) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return nil, nil, false + } + if bucket == nil || !bucket.Status.IsAdminVisible() { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "bucket not found"}) + return nil, nil, false + } + source, err := s.repos.Uploads.GetDataSetBindingByID(ctx, dataSetID) + if err != nil { + s.logger.Error("api: failed to load data set for replacement", "error", err, "dataSetID", dataSetID) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return nil, nil, false + } + if source == nil || source.BucketID != bucket.ID { + writeJSON(w, http.StatusNotFound, map[string]string{"error": "data set not found"}) + return nil, nil, false + } + return bucket, source, true +} + +// replacementProviderResponse is one row of the provider chooser. Ineligible +// providers are included with the reason, so an operator can see why the +// provider they were looking for cannot take this replica. +type replacementProviderResponse struct { + ProviderID string `json:"provider_id"` + Eligible bool `json:"eligible"` + IneligibleReason string `json:"ineligible_reason,omitempty"` + PreviouslyUsed bool `json:"previously_used"` + ProviderIdentity *providerIdentityResponse `json:"provider_identity,omitempty"` +} + +// handleAPIListDataSetReplacementProviders lists the providers this replica can +// move to. It reports the same eligibility the confirmation enforces, so the +// chooser never offers something the confirmation would reject. +func (s *Server) handleAPIListDataSetReplacementProviders(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if !bucketNameRe.MatchString(name) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid bucket name"}) + return + } + dataSetID, err := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err != nil || dataSetID <= 0 { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "invalid data set id"}) + return + } + ctx := r.Context() + bucket, source, ok := s.replacementSubject(w, ctx, name, dataSetID) + if !ok { + return + } + candidates, err := s.replacementProviderCandidates(ctx, bucket, source) + if err != nil { + s.writeReplacementError(w, err, name) + return + } + + providerIDs := make([]idtypes.OnChainID, 0, len(candidates)) + for _, candidate := range candidates { + providerIDs = append(providerIDs, candidate.ProviderID) + } + identities := s.providerIdentities(providerIDs) + + providers := make([]replacementProviderResponse, 0, len(candidates)) + for _, candidate := range candidates { + providers = append(providers, replacementProviderResponse{ + ProviderID: candidate.ProviderID.String(), + Eligible: candidate.Eligible, + IneligibleReason: candidate.IneligibleReason, + PreviouslyUsed: candidate.PreviouslyUsed, + ProviderIdentity: providerIdentityFromSnapshot(identities, candidate.ProviderID), + }) + } + writeJSON(w, http.StatusOK, map[string]any{"providers": providers}) +} diff --git a/internal/admin/api_replacement_test.go b/internal/admin/api_replacement_test.go new file mode 100644 index 0000000..676544f --- /dev/null +++ b/internal/admin/api_replacement_test.go @@ -0,0 +1,463 @@ +package admin + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + idtypes "github.com/strahe/synaps3/internal/types" +) + +type stubProviderSelector struct { + providers []string + err error + calls int +} + +func onChainIDValue(value string) idtypes.OnChainID { + id, err := idtypes.ParseOnChainID("test id", value) + if err != nil { + panic(err) + } + return id +} + +func (s *stubProviderSelector) ListReplacementProviders(context.Context) ([]idtypes.OnChainID, error) { + s.calls++ + if s.err != nil { + return nil, s.err + } + providers := make([]idtypes.OnChainID, 0, len(s.providers)) + for _, id := range s.providers { + providers = append(providers, onChainIDValue(id)) + } + return providers, nil +} + +type replacementAPIFixture struct { + srv *Server + mux *http.ServeMux + bucket *model.Bucket + source *model.StorageDataSet + request int +} + +func newReplacementAPIFixture(t *testing.T, selector providerReplacementSelector) *replacementAPIFixture { + t.Helper() + srv, _ := newBucketAPITestServer(t) + if selector != nil { + srv.WithProviderReplacement(selector) + } + ctx := context.Background() + bucket := &model.Bucket{Name: "replacement-bucket", Status: model.BucketStatusActive} + if err := srv.repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Create bucket: %v", err) + } + binding, err := srv.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, + ProviderID: onChainIDValue("101"), + CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := srv.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, + DataSetID: onChainIDValue("1001"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + source, err := srv.repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) + if err != nil || source == nil { + t.Fatalf("GetDataSetBindingByID: %#v err=%v", source, err) + } + return &replacementAPIFixture{srv: srv, mux: newBucketAPIMux(srv), bucket: bucket, source: source} +} + +func (f *replacementAPIFixture) start(t *testing.T, body string) *httptest.ResponseRecorder { + t.Helper() + f.request++ + if !strings.Contains(body, `"client_request_id"`) && strings.HasSuffix(body, "}") { + body = strings.TrimSuffix(body, "}") + if body != "{" { + body += "," + } + body += `"client_request_id":"test-request-` + strconv.Itoa(f.request) + `"}` + } + return f.startRaw(t, body) +} + +func (f *replacementAPIFixture) startRaw(t *testing.T, body string) *httptest.ResponseRecorder { + t.Helper() + path := "/api/v1/buckets/" + f.bucket.Name + "/data-sets/" + + strconv.FormatInt(f.source.ID, 10) + "/replacement" + req := httptest.NewRequest(http.MethodPost, path, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + f.mux.ServeHTTP(rec, req) + return rec +} + +func decodeReplacement(t *testing.T, rec *httptest.ResponseRecorder) providerReplacementResponse { + t.Helper() + var body providerReplacementResponse + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode replacement response: %v", err) + } + return body +} + +func decodeAPIError(t *testing.T, rec *httptest.ResponseRecorder) map[string]string { + t.Helper() + var body map[string]string + if err := json.NewDecoder(rec.Body).Decode(&body); err != nil { + t.Fatalf("decode error response: %v", err) + } + return body +} + +func TestAPIStartDataSetReplacementManualMode(t *testing.T) { + fixture := newReplacementAPIFixture(t, &stubProviderSelector{providers: []string{"202"}}) + + rec := fixture.start(t, `{"mode":"manual","provider_id":"202"}`) + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d body=%s, want 201", rec.Code, rec.Body.String()) + } + body := decodeReplacement(t, rec) + if body.Status != string(storagereplacement.StatusPreparingTarget) { + t.Fatalf("status = %s, want preparing_target", body.Status) + } + if body.Source.ProviderID != "101" || body.Target.ProviderID != "202" { + t.Fatalf("providers = %s -> %s, want 101 -> 202", body.Source.ProviderID, body.Target.ProviderID) + } + // Preparing the target must not move writes. + if !body.Source.IsCurrent || body.Target.IsCurrent { + t.Fatalf("currency = source:%v target:%v, want the source still current", body.Source.IsCurrent, body.Target.IsCurrent) + } + if body.ItemsTotal != 0 || body.ItemsCopied != 0 { + t.Fatalf("progress = %d/%d, want no migration work yet", body.ItemsCopied, body.ItemsTotal) + } +} + +// Automatic selection must exclude every provider the bucket has ever used. +func TestAPIStartDataSetReplacementAutomaticMode(t *testing.T) { + // 101 already serves this bucket, so automatic selection has to walk past it. + selector := &stubProviderSelector{providers: []string{"101", "303"}} + fixture := newReplacementAPIFixture(t, selector) + + rec := fixture.start(t, `{"mode":"automatic"}`) + if rec.Code != http.StatusCreated { + t.Fatalf("status = %d body=%s, want 201", rec.Code, rec.Body.String()) + } + body := decodeReplacement(t, rec) + if body.Target.ProviderID != "303" || body.SelectionMode != string(storagereplacement.SelectionModeAutomatic) { + t.Fatalf("body = %#v, want the automatically selected provider", body) + } +} + +func TestAPIStartDataSetReplacementIsIdempotent(t *testing.T) { + selector := &stubProviderSelector{providers: []string{"202", "303"}} + fixture := newReplacementAPIFixture(t, selector) + body := `{"mode":"manual","provider_id":"202","client_request_id":"same-confirmation"}` + + first := fixture.startRaw(t, body) + if first.Code != http.StatusCreated { + t.Fatalf("first status = %d body=%s, want 201", first.Code, first.Body.String()) + } + firstReplacement := decodeReplacement(t, first) + ctx := context.Background() + if err := fixture.srv.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: firstReplacement.Target.ID, DataSetID: onChainIDValue("2002"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + if err := fixture.srv.repos.Replacements.Activate(ctx, firstReplacement.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + replay := fixture.startRaw(t, body) + if replay.Code != http.StatusOK { + t.Fatalf("replay status = %d body=%s, want 200", replay.Code, replay.Body.String()) + } + if got := decodeReplacement(t, replay); got.ID != firstReplacement.ID { + t.Fatalf("replay replacement = %d, want original %d", got.ID, firstReplacement.ID) + } + if selector.calls != 1 { + t.Fatalf("provider inventory calls = %d, want replay before inventory lookup", selector.calls) + } + + conflict := fixture.startRaw(t, `{"mode":"manual","provider_id":"303","client_request_id":"same-confirmation"}`) + if conflict.Code != http.StatusConflict { + t.Fatalf("conflict status = %d body=%s, want 409", conflict.Code, conflict.Body.String()) + } + if got := decodeAPIError(t, conflict)["code"]; got != storagereplacement.CodeIdempotencyConflict { + t.Fatalf("conflict code = %q, want %q", got, storagereplacement.CodeIdempotencyConflict) + } +} + +func TestAPIStartDataSetReplacementAutomaticReplayIgnoresRegistryDrift(t *testing.T) { + selector := &stubProviderSelector{providers: []string{"202"}} + fixture := newReplacementAPIFixture(t, selector) + body := `{"mode":"automatic","client_request_id":"automatic-replay"}` + first := fixture.startRaw(t, body) + if first.Code != http.StatusCreated { + t.Fatalf("first status = %d body=%s, want 201", first.Code, first.Body.String()) + } + original := decodeReplacement(t, first) + selector.providers = []string{"303"} + replay := fixture.startRaw(t, body) + if replay.Code != http.StatusOK { + t.Fatalf("replay status = %d body=%s, want 200", replay.Code, replay.Body.String()) + } + got := decodeReplacement(t, replay) + if got.ID != original.ID || got.Target.ProviderID != original.Target.ProviderID { + t.Fatalf("replay = replacement %d provider %s, want %d/%s", got.ID, got.Target.ProviderID, original.ID, original.Target.ProviderID) + } + if selector.calls != 1 { + t.Fatalf("provider inventory calls = %d, want the replay to avoid registry drift", selector.calls) + } +} + +func TestAPIStartDataSetReplacementRejections(t *testing.T) { + cases := []struct { + name string + body string + selector providerReplacementSelector + status int + code string + }{ + {name: "unknown mode", body: `{"mode":"guess"}`, status: http.StatusBadRequest, code: storagereplacement.CodeTargetInvalid}, + {name: "manual without provider", body: `{"mode":"manual"}`, status: http.StatusBadRequest, code: storagereplacement.CodeTargetInvalid}, + {name: "manual naming the source", body: `{"mode":"manual","provider_id":"101"}`, status: http.StatusBadRequest, code: storagereplacement.CodeTargetInvalid}, + {name: "manual provider unavailable", body: `{"mode":"manual","provider_id":"202"}`, selector: &stubProviderSelector{providers: []string{"303"}}, status: http.StatusBadRequest, code: storagereplacement.CodeTargetUnavailable}, + {name: "automatic with a provider", body: `{"mode":"automatic","provider_id":"202"}`, status: http.StatusBadRequest, code: storagereplacement.CodeTargetInvalid}, + {name: "unknown field", body: `{"mode":"manual","provider":"202"}`, status: http.StatusBadRequest}, + {name: "automatic without a storage service", body: `{"mode":"automatic"}`, status: http.StatusServiceUnavailable}, + { + name: "no eligible provider", + body: `{"mode":"automatic"}`, + selector: &stubProviderSelector{providers: []string{"101"}}, + status: http.StatusConflict, + code: storagereplacement.CodeNoEligibleProvider, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fixture := newReplacementAPIFixture(t, tc.selector) + rec := fixture.start(t, tc.body) + if rec.Code != tc.status { + t.Fatalf("status = %d body=%s, want %d", rec.Code, rec.Body.String(), tc.status) + } + if tc.code != "" { + if got := decodeAPIError(t, rec)["code"]; got != tc.code { + t.Fatalf("code = %q, want %q", got, tc.code) + } + } + }) + } +} + +func TestAPIStartDataSetReplacementRequiresClientRequestID(t *testing.T) { + fixture := newReplacementAPIFixture(t, &stubProviderSelector{providers: []string{"202"}}) + rec := fixture.startRaw(t, `{"mode":"manual","provider_id":"202"}`) + if rec.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s, want 400", rec.Code, rec.Body.String()) + } +} + +func TestAPIStartDataSetReplacementRejectsSecondConfirmation(t *testing.T) { + fixture := newReplacementAPIFixture(t, &stubProviderSelector{providers: []string{"202", "303"}}) + if rec := fixture.start(t, `{"mode":"manual","provider_id":"202"}`); rec.Code != http.StatusCreated { + t.Fatalf("first confirmation status = %d", rec.Code) + } + // A second confirmation supersedes the first rather than colliding, which is + // how an operator changes their mind about the target. + rec := fixture.start(t, `{"mode":"manual","provider_id":"303"}`) + if rec.Code != http.StatusCreated { + t.Fatalf("second confirmation status = %d body=%s, want 201", rec.Code, rec.Body.String()) + } +} + +func TestAPIRetryStorageReplacementStates(t *testing.T) { + fixture := newReplacementAPIFixture(t, &stubProviderSelector{providers: []string{"202"}}) + ctx := context.Background() + rec := fixture.start(t, `{"mode":"manual","provider_id":"202"}`) + if rec.Code != http.StatusCreated { + t.Fatalf("confirmation status = %d", rec.Code) + } + created := decodeReplacement(t, rec) + + retry := func() *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodPost, "/api/v1/storage-replacements/"+strconv.FormatInt(created.ID, 10)+"/retry", nil) + out := httptest.NewRecorder() + fixture.mux.ServeHTTP(out, req) + return out + } + + // Work that is still progressing is not the operator's to retry. + out := retry() + if out.Code != http.StatusConflict { + t.Fatalf("retry while preparing = %d, want 409", out.Code) + } + if got := decodeAPIError(t, out)["code"]; got != storagereplacement.CodeNotRetryable { + t.Fatalf("code = %q, want %q", got, storagereplacement.CodeNotRetryable) + } + + if err := fixture.srv.repos.Replacements.MarkFailed(ctx, created.ID, nil, "target creation exhausted"); err != nil { + t.Fatalf("MarkFailed: %v", err) + } + out = retry() + if out.Code != http.StatusOK { + t.Fatalf("retry after failure = %d body=%s, want 200", out.Code, out.Body.String()) + } + if body := decodeReplacement(t, out); body.LastError != nil { + t.Fatalf("last_error = %v, want it cleared by the retry", *body.LastError) + } + + req := httptest.NewRequest(http.MethodPost, "/api/v1/storage-replacements/999999/retry", nil) + out = httptest.NewRecorder() + fixture.mux.ServeHTTP(out, req) + if out.Code != http.StatusNotFound { + t.Fatalf("retry unknown replacement = %d, want 404", out.Code) + } +} + +func TestAPIRetryStorageReplacementRejectsPermanentTargetConflict(t *testing.T) { + fixture := newReplacementAPIFixture(t, &stubProviderSelector{providers: []string{"202"}}) + body := `{"mode":"manual","provider_id":"202","client_request_id":"permanent-failure"}` + rec := fixture.startRaw(t, body) + if rec.Code != http.StatusCreated { + t.Fatalf("confirmation status = %d body=%s", rec.Code, rec.Body.String()) + } + created := decodeReplacement(t, rec) + reason := storagereplacement.FailureReasonTargetInUse + if err := fixture.srv.repos.Replacements.MarkFailed(context.Background(), created.ID, &reason, "target already serves the bucket"); err != nil { + t.Fatalf("MarkFailed: %v", err) + } + + replay := fixture.startRaw(t, body) + if replay.Code != http.StatusOK { + t.Fatalf("replay status = %d body=%s, want 200", replay.Code, replay.Body.String()) + } + if got := decodeReplacement(t, replay).FailureReason; got != string(reason) { + t.Fatalf("failure_reason = %q, want %q", got, reason) + } + + req := httptest.NewRequest(http.MethodPost, "/api/v1/storage-replacements/"+strconv.FormatInt(created.ID, 10)+"/retry", nil) + retry := httptest.NewRecorder() + fixture.mux.ServeHTTP(retry, req) + if retry.Code != http.StatusConflict { + t.Fatalf("retry status = %d body=%s, want 409", retry.Code, retry.Body.String()) + } + if got := decodeAPIError(t, retry)["code"]; got != storagereplacement.CodeTargetInUse { + t.Fatalf("retry code = %q, want %q", got, storagereplacement.CodeTargetInUse) + } +} + +// Replacement work carries state the task queue knows nothing about, so the +// generic retry must refuse it and point the operator at the Data Sets surface. +func TestRetryExhaustedRejectsReplacementCoordinator(t *testing.T) { + srv, _ := newBucketAPITestServer(t) + ctx := context.Background() + stage := storagereplacement.StageMigrate + task := &model.Task{ + Type: model.TaskTypeUpload, + Stage: &stage, + RefType: "bucket", + RefID: 1, + IdempotencyKey: storagereplacement.MigrateTaskKey(42), + Payload: storagereplacement.NewMigratePayload(42, 0, 0), + Status: model.TaskStatusQueued, + MaxRetries: 1, + } + if err := srv.repos.Tasks.Create(ctx, task); err != nil { + t.Fatalf("Create task: %v", err) + } + if _, err := srv.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("status = ?", model.TaskStatusExhausted). + Where("id = ?", task.ID). + Exec(ctx); err != nil { + t.Fatalf("mark exhausted: %v", err) + } + + err := srv.repos.Tasks.RetryExhausted(ctx, task.ID) + if !errors.Is(err, repository.ErrReplacementRetryUnsupported) { + t.Fatalf("RetryExhausted = %v, want the replacement rejection", err) + } + + mux := http.NewServeMux() + mux.HandleFunc("POST /api/v1/tasks/{id}/retry", srv.handleRetryExhausted) + req := httptest.NewRequest(http.MethodPost, "/api/v1/tasks/"+strconv.FormatInt(task.ID, 10)+"/retry", nil) + rec := httptest.NewRecorder() + mux.ServeHTTP(rec, req) + if rec.Code != http.StatusConflict { + t.Fatalf("status = %d body=%s, want 409", rec.Code, rec.Body.String()) + } + body := decodeAPIError(t, rec) + if body["code"] != storagereplacement.CodeTaskRetryUnsupported { + t.Fatalf("code = %q, want %q", body["code"], storagereplacement.CodeTaskRetryUnsupported) + } + if !strings.Contains(body["error"], "Data Sets") { + t.Fatalf("error = %q, want it to point at the Data Sets surface", body["error"]) + } +} + +func (f *replacementAPIFixture) listProviders(t *testing.T) *httptest.ResponseRecorder { + t.Helper() + path := "/api/v1/buckets/" + f.bucket.Name + "/data-sets/" + + strconv.FormatInt(f.source.ID, 10) + "/replacement/providers" + rec := httptest.NewRecorder() + f.mux.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, path, nil)) + return rec +} + +// The chooser reports the same eligibility the confirmation enforces, and keeps +// ineligible providers visible with the reason. +func TestAPIListDataSetReplacementProviders(t *testing.T) { + fixture := newReplacementAPIFixture(t, &stubProviderSelector{providers: []string{"101", "303"}}) + rec := fixture.listProviders(t) + if rec.Code != http.StatusOK { + t.Fatalf("status = %d body=%s, want 200", rec.Code, rec.Body.String()) + } + var body struct { + Providers []replacementProviderResponse `json:"providers"` + } + if err := json.Unmarshal(rec.Body.Bytes(), &body); err != nil { + t.Fatalf("decode providers: %v", err) + } + if len(body.Providers) != 2 { + t.Fatalf("providers = %#v, want every approved provider listed", body.Providers) + } + source, replacement := body.Providers[0], body.Providers[1] + if source.ProviderID != "101" || source.Eligible || + source.IneligibleReason != providerIneligibleCurrentSource { + t.Fatalf("source provider = %#v, want it listed but not choosable", source) + } + if replacement.ProviderID != "303" || !replacement.Eligible || replacement.IneligibleReason != "" { + t.Fatalf("replacement provider = %#v, want it choosable", replacement) + } +} + +func TestAPIListDataSetReplacementProvidersRejections(t *testing.T) { + t.Run("without a storage service", func(t *testing.T) { + fixture := newReplacementAPIFixture(t, nil) + if rec := fixture.listProviders(t); rec.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d body=%s, want 503", rec.Code, rec.Body.String()) + } + }) + t.Run("unknown data set", func(t *testing.T) { + fixture := newReplacementAPIFixture(t, &stubProviderSelector{providers: []string{"303"}}) + fixture.source.ID = 987654 + if rec := fixture.listProviders(t); rec.Code != http.StatusNotFound { + t.Fatalf("status = %d body=%s, want 404", rec.Code, rec.Body.String()) + } + }) +} diff --git a/internal/admin/api_tasks.go b/internal/admin/api_tasks.go index 07628e0..eef518e 100644 --- a/internal/admin/api_tasks.go +++ b/internal/admin/api_tasks.go @@ -20,6 +20,7 @@ type taskListItem struct { CopyIndex *int `json:"copy_index,omitempty"` RefType string `json:"ref_type"` RefID int64 `json:"ref_id"` + BucketName string `json:"bucket_name,omitempty"` RefVersionID string `json:"ref_version_id"` Status string `json:"status"` Progress *uploadProgressResponse `json:"progress,omitempty"` @@ -44,6 +45,7 @@ type taskRefDetailResponse struct { RefType string `json:"ref_type"` RefID int64 `json:"ref_id"` RefVersionID string `json:"ref_version_id"` + BucketName string `json:"bucket_name,omitempty"` Object *taskRefObjectDetail `json:"object"` StorageCleanup *taskRefStorageCleanupDetail `json:"storage_cleanup,omitempty"` } @@ -121,10 +123,30 @@ func (s *Server) handleAPITasks(w http.ResponseWriter, r *http.Request) { return } progressByTaskID := s.taskUploadProgresses(ctx, tasks) + bucketIDs := make([]int64, 0) + seenBucketIDs := make(map[int64]struct{}) + for i := range tasks { + if tasks[i].RefType == "bucket" && tasks[i].RefID > 0 { + if _, ok := seenBucketIDs[tasks[i].RefID]; !ok { + seenBucketIDs[tasks[i].RefID] = struct{}{} + bucketIDs = append(bucketIDs, tasks[i].RefID) + } + } + } + bucketNames, err := s.repos.Buckets.GetNamesByIDs(ctx, bucketIDs) + if err != nil { + s.logger.Error("api: failed to load task bucket names", "error", err) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return + } items := make([]taskListItem, 0, len(tasks)) for i := range tasks { - items = append(items, taskListItemFromModel(&tasks[i], progressByTaskID[tasks[i].ID])) + item := taskListItemFromModel(&tasks[i], progressByTaskID[tasks[i].ID]) + if tasks[i].RefType == "bucket" { + item.BucketName = bucketNames[tasks[i].RefID] + } + items = append(items, item) } writeJSON(w, http.StatusOK, taskListResponse{ @@ -325,6 +347,17 @@ func (s *Server) handleAPITaskRefDetail(w http.ResponseWriter, r *http.Request) RefID: task.RefID, RefVersionID: task.RefVersionID, } + if task.RefType == "bucket" { + bucket, bucketErr := s.repos.Buckets.GetByID(ctx, task.RefID) + if bucketErr != nil { + s.logger.Error("api: failed to load task bucket ref", "error", bucketErr, "taskID", id) + writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + return + } + if bucket != nil { + resp.BucketName = bucket.Name + } + } if task.RefType == "storage_upload" && task.Type == model.TaskTypeStorageCleanup { copies, err := s.repos.StorageCleanup.ListCopiesForTask(ctx, task.ID) if err != nil { diff --git a/internal/admin/api_tasks_test.go b/internal/admin/api_tasks_test.go index 853ec58..73750a0 100644 --- a/internal/admin/api_tasks_test.go +++ b/internal/admin/api_tasks_test.go @@ -73,6 +73,35 @@ func TestAPIListExhaustedUsesTaskListDTO(t *testing.T) { } } +func TestAPIListTasksIncludesBucketNameForBucketReferences(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := testutil.SeedBucket(t, db, "replacement-task-bucket") + stage := "retire_abandoned_target" + task := &model.Task{ + Type: model.TaskTypeStorageCleanup, Stage: &stage, RefType: "bucket", RefID: bucket.ID, + IdempotencyKey: "replacement-task-bucket-ref", Status: model.TaskStatusCompleted, + ScheduledAt: time.Now(), CompletedAt: func() *time.Time { now := time.Now(); return &now }(), + } + if err := repos.Tasks.Create(ctx, task); err != nil { + t.Fatalf("Create task: %v", err) + } + srv := newTestServer(":0", db, nil, 0, repos, nil, nil, config.DefaultFilecoinCopies, testLogger()) + rr := httptest.NewRecorder() + srv.handleAPITasks(rr, httptest.NewRequest(http.MethodGet, "/api/v1/tasks", nil)) + if rr.Code != http.StatusOK { + t.Fatalf("status = %d body=%s, want 200", rr.Code, rr.Body.String()) + } + var body taskListResponse + if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { + t.Fatalf("Decode: %v", err) + } + if len(body.Tasks) != 1 || body.Tasks[0].BucketName != bucket.Name { + t.Fatalf("tasks = %#v, want bucket name %q", body.Tasks, bucket.Name) + } +} + func TestAPIRetryExhaustedHTTPStatuses(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) @@ -649,7 +678,7 @@ func TestAPITaskRefDetailNonObject(t *testing.T) { if status != http.StatusOK { t.Fatalf("status = %d, want 200", status) } - if body.RefType != "bucket" || body.RefID != bucket.ID || body.Object != nil { + if body.RefType != "bucket" || body.RefID != bucket.ID || body.BucketName != bucket.Name || body.Object != nil { t.Fatalf("ref detail = %#v, want bucket ref without object", body) } } diff --git a/internal/admin/replacement_candidates.go b/internal/admin/replacement_candidates.go new file mode 100644 index 0000000..9c95847 --- /dev/null +++ b/internal/admin/replacement_candidates.go @@ -0,0 +1,88 @@ +package admin + +import ( + "github.com/strahe/synaps3/internal/model" + idtypes "github.com/strahe/synaps3/internal/types" +) + +// Reasons a provider cannot take over a replica. They mirror what Authorize +// rejects, so the dashboard never offers a choice the API refuses. +const ( + providerIneligibleCurrentSource = "current_source" + providerIneligibleServesBucket = "already_serves_bucket" +) + +// replacementProviderCandidate is one provider offered for a replacement. +// Ineligible providers are still listed: an operator looking for a provider +// they expected to see needs to know why it cannot be chosen, rather than +// finding it silently missing. +type replacementProviderCandidate struct { + ProviderID idtypes.OnChainID + Eligible bool + IneligibleReason string + // PreviouslyUsed marks a provider this bucket has used before and fully + // retired. It can be chosen again, which automatic selection avoids and an + // operator may still want. + PreviouslyUsed bool +} + +// replacementProviderCandidates applies the target rules once, so automatic +// selection and the dashboard's provider list can never disagree about what is +// choosable. +// +// The rules follow StorageReplacementRepository.Authorize: the generation being +// replaced cannot replace itself, and a provider still holding an unretired +// generation of this bucket would collide with it. A provider whose earlier +// generations are all retired is choosable again. +func replacementProviderCandidates( + providers []idtypes.OnChainID, + bindings []model.StorageDataSet, + source *model.StorageDataSet, +) []replacementProviderCandidate { + serving := make(map[string]bool, len(bindings)) + used := make(map[string]bool, len(bindings)) + for i := range bindings { + key := bindings[i].ProviderID.String() + used[key] = true + if bindings[i].Status != model.StorageDataSetStatusRetired { + serving[key] = true + } + } + sourceKey := "" + if source != nil { + sourceKey = source.ProviderID.String() + } + + candidates := make([]replacementProviderCandidate, 0, len(providers)) + for _, providerID := range providers { + key := providerID.String() + candidate := replacementProviderCandidate{ + ProviderID: providerID, + Eligible: true, + PreviouslyUsed: used[key], + } + switch { + case sourceKey != "" && key == sourceKey: + candidate.Eligible = false + candidate.IneligibleReason = providerIneligibleCurrentSource + case serving[key]: + candidate.Eligible = false + candidate.IneligibleReason = providerIneligibleServesBucket + } + candidates = append(candidates, candidate) + } + return candidates +} + +// firstAutomaticChoice picks the provider an automatic replacement should use. +// It goes further than eligibility and skips every provider the bucket has ever +// used, so an automatic choice never lands back where the operator moved away +// from. Choosing one of those again stays available as a manual decision. +func firstAutomaticChoice(candidates []replacementProviderCandidate) (idtypes.OnChainID, bool) { + for _, candidate := range candidates { + if candidate.Eligible && !candidate.PreviouslyUsed { + return candidate.ProviderID, true + } + } + return idtypes.OnChainID{}, false +} diff --git a/internal/admin/replacement_candidates_test.go b/internal/admin/replacement_candidates_test.go new file mode 100644 index 0000000..aaaac36 --- /dev/null +++ b/internal/admin/replacement_candidates_test.go @@ -0,0 +1,95 @@ +package admin + +import ( + "testing" + + "github.com/strahe/synaps3/internal/model" + idtypes "github.com/strahe/synaps3/internal/types" +) + +func candidateProviders(ids ...string) []idtypes.OnChainID { + out := make([]idtypes.OnChainID, 0, len(ids)) + for _, id := range ids { + out = append(out, onChainIDValue(id)) + } + return out +} + +func candidateBinding(providerID string, status model.StorageDataSetStatus) model.StorageDataSet { + return model.StorageDataSet{ProviderID: onChainIDValue(providerID), Status: status} +} + +func candidateByProvider(t *testing.T, candidates []replacementProviderCandidate, providerID string) replacementProviderCandidate { + t.Helper() + for _, candidate := range candidates { + if candidate.ProviderID.String() == providerID { + return candidate + } + } + t.Fatalf("provider %s missing from candidates %v", providerID, candidates) + return replacementProviderCandidate{} +} + +// The rules here are the same ones Authorize enforces. If they drift, the +// dashboard offers a provider the confirmation then rejects. +func TestReplacementProviderCandidates_MirrorsWhatAuthorizeAccepts(t *testing.T) { + source := candidateBinding("101", model.StorageDataSetStatusReady) + candidates := replacementProviderCandidates( + candidateProviders("101", "202", "303", "404"), + []model.StorageDataSet{ + source, + candidateBinding("202", model.StorageDataSetStatusDraining), + candidateBinding("303", model.StorageDataSetStatusRetired), + }, + &source, + ) + + if got := candidateByProvider(t, candidates, "101"); got.Eligible || got.IneligibleReason != providerIneligibleCurrentSource { + t.Fatalf("source provider = %#v, want ineligible as the current source", got) + } + if got := candidateByProvider(t, candidates, "202"); got.Eligible || got.IneligibleReason != providerIneligibleServesBucket { + t.Fatalf("serving provider = %#v, want ineligible while it still holds a generation", got) + } + // A provider whose earlier service was retired is free to take the replica + // again; only automatic selection avoids it. + if got := candidateByProvider(t, candidates, "303"); !got.Eligible || !got.PreviouslyUsed { + t.Fatalf("retired provider = %#v, want it choosable and marked as used before", got) + } + if got := candidateByProvider(t, candidates, "404"); !got.Eligible || got.PreviouslyUsed { + t.Fatalf("unused provider = %#v, want it plainly choosable", got) + } +} + +// Ineligible providers stay in the list. An operator hunting for a provider +// they expected needs the reason, not a silent absence. +func TestReplacementProviderCandidates_KeepsIneligibleProvidersVisible(t *testing.T) { + source := candidateBinding("101", model.StorageDataSetStatusReady) + candidates := replacementProviderCandidates( + candidateProviders("101", "202"), + []model.StorageDataSet{source}, + &source, + ) + if len(candidates) != 2 { + t.Fatalf("candidates = %d, want every approved provider listed", len(candidates)) + } +} + +// Automatic selection goes further than eligibility: it never returns to a +// provider this bucket has moved away from, even once that service is retired. +func TestFirstAutomaticChoice_SkipsEveryProviderTheBucketHasUsed(t *testing.T) { + source := candidateBinding("101", model.StorageDataSetStatusReady) + bindings := []model.StorageDataSet{source, candidateBinding("202", model.StorageDataSetStatusRetired)} + + got, ok := firstAutomaticChoice(replacementProviderCandidates( + candidateProviders("101", "202", "303"), bindings, &source)) + if !ok || got.String() != "303" { + t.Fatalf("automatic choice = %v ok=%v, want the provider the bucket has never used", got, ok) + } + + // With nothing new left, automatic selection reports it rather than + // silently reusing a retired provider. + if _, ok := firstAutomaticChoice(replacementProviderCandidates( + candidateProviders("101", "202"), bindings, &source)); ok { + t.Fatal("automatic selection reused a provider the bucket had moved away from") + } +} diff --git a/internal/admin/replacement_selector.go b/internal/admin/replacement_selector.go new file mode 100644 index 0000000..eba4066 --- /dev/null +++ b/internal/admin/replacement_selector.go @@ -0,0 +1,98 @@ +package admin + +import ( + "context" + "slices" + "time" + + "github.com/strahe/synaps3/internal/observability" + idtypes "github.com/strahe/synaps3/internal/types" +) + +// replacementProviderListTimeout bounds the inventory read so a confirmation +// dialog cannot hang. +const replacementProviderListTimeout = 15 * time.Second + +// replacementProviderPageSize caps one inventory read. The registry holds tens +// of providers, not thousands. +const replacementProviderPageSize = 200 + +// ProviderRegistry lists the storage providers a replacement can move to. It is +// an interface so provider selection stays testable without live observability. +type ProviderRegistry interface { + ListProviderObservations(ctx context.Context, opts observability.ListOptions) (observability.ProviderObservationPage, error) +} + +// StorageProviderSelector reports which storage providers a replacement can +// use. It resolves candidates only; the paid service itself is created later by +// the worker, after the operator has confirmed. +type StorageProviderSelector struct { + providers ProviderRegistry +} + +// NewStorageProviderSelector builds the provider inventory used by replacement. +func NewStorageProviderSelector(providers ProviderRegistry) *StorageProviderSelector { + return &StorageProviderSelector{providers: providers} +} + +// ListReplacementProviders returns the providers currently reachable and able +// to take a data set, lowest registry ID first so the same inventory always +// yields the same choice. Which of them a given replica can take is decided by +// replacementProviderCandidates, which owns that rule alone. +// +// It reads the same provider observations the storage topology reports and +// applies the same availability filter, so the two views cannot disagree: a +// provider an operator sees listed as available there is offered here, and one +// that is unreachable is offered in neither. Reading the registry directly +// would drop the health signal and offer providers that cannot answer. +// +// The inventory is deliberately not narrowed to the storage service's +// approved-provider list. That list governs the SDK's automatic selection only +// -- naming a provider outright bypasses it -- so filtering on it here would +// hide providers replacement can in fact use, and it is small enough for one +// bucket to exhaust. +func (s *StorageProviderSelector) ListReplacementProviders(ctx context.Context) ([]idtypes.OnChainID, error) { + if s == nil || s.providers == nil { + return nil, errReplacementUnavailable + } + listCtx, cancel := context.WithTimeout(ctx, replacementProviderListTimeout) + defer cancel() + + providers := make([]idtypes.OnChainID, 0, replacementProviderPageSize) + seen := make(map[string]struct{}, replacementProviderPageSize) + for offset := 0; ; { + page, err := s.providers.ListProviderObservations(listCtx, observability.ListOptions{ + Status: observability.StatusAvailable, + Limit: replacementProviderPageSize, + Offset: offset, + }) + if err != nil { + return nil, err + } + added := 0 + for _, item := range page.Items { + facts := item.Facts + key := facts.ProviderID.String() + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + added++ + // A provider without an active PDP offering cannot hold a data set. + if facts.ProviderID.IsZero() || facts.Active == nil || !*facts.Active || facts.HasPDP == nil || !*facts.HasPDP { + continue + } + providers = append(providers, facts.ProviderID) + } + next := offset + len(page.Items) + if len(page.Items) == 0 || len(page.Items) < replacementProviderPageSize || + (page.Total > 0 && next >= page.Total) || next <= offset || added == 0 { + break + } + offset = next + } + slices.SortFunc(providers, func(a, b idtypes.OnChainID) int { + return a.SDK().Cmp(b.SDK()) + }) + return providers, nil +} diff --git a/internal/admin/replacement_selector_test.go b/internal/admin/replacement_selector_test.go new file mode 100644 index 0000000..89f2b06 --- /dev/null +++ b/internal/admin/replacement_selector_test.go @@ -0,0 +1,165 @@ +package admin + +import ( + "context" + "errors" + "strconv" + "testing" + + "github.com/strahe/synaps3/internal/observability" +) + +type stubProviderRegistry struct { + items []observability.ProviderObservation + err error + opts observability.ListOptions + calls []observability.ListOptions + list func(observability.ListOptions) observability.ProviderObservationPage +} + +func (s *stubProviderRegistry) ListProviderObservations( + _ context.Context, + opts observability.ListOptions, +) (observability.ProviderObservationPage, error) { + s.opts = opts + s.calls = append(s.calls, opts) + if s.err != nil { + return observability.ProviderObservationPage{}, s.err + } + if s.list != nil { + return s.list(opts), nil + } + start := min(opts.Offset, len(s.items)) + end := min(start+opts.Limit, len(s.items)) + return observability.ProviderObservationPage{Items: s.items[start:end], Total: len(s.items)}, nil +} + +func observedProvider(id string, active, hasPDP bool) observability.ProviderObservation { + return observability.ProviderObservation{ + Facts: observability.ProviderFacts{ + ProviderID: onChainIDValue(id), + Active: &active, + HasPDP: &hasPDP, + }, + } +} + +func selectorWithRegistry(items ...observability.ProviderObservation) *StorageProviderSelector { + return NewStorageProviderSelector(&stubProviderRegistry{items: items}) +} + +func providerIDStrings(t *testing.T, selector *StorageProviderSelector) []string { + t.Helper() + providers, err := selector.ListReplacementProviders(context.Background()) + if err != nil { + t.Fatalf("ListReplacementProviders: %v", err) + } + out := make([]string, 0, len(providers)) + for _, id := range providers { + out = append(out, id.String()) + } + return out +} + +// A provider without an active PDP offering cannot hold a data set, so offering +// it would only fail later inside the worker. +func TestListReplacementProviders_SkipsProvidersThatCannotHoldADataSet(t *testing.T) { + got := providerIDStrings(t, selectorWithRegistry( + observedProvider("101", false, true), + observedProvider("202", true, false), + observedProvider("303", true, true), + )) + if len(got) != 1 || got[0] != "303" { + t.Fatalf("providers = %v, want only the usable one", got) + } +} + +// Registry order is not guaranteed, so the inventory is sorted. Without it the +// same confirmation could land on a different provider each time it is retried. +func TestListReplacementProviders_IsStableAcrossRegistryOrder(t *testing.T) { + forward := providerIDStrings(t, selectorWithRegistry(observedProvider("303", true, true), observedProvider("202", true, true))) + reverse := providerIDStrings(t, selectorWithRegistry(observedProvider("202", true, true), observedProvider("303", true, true))) + if len(forward) != 2 || forward[0] != "202" || forward[1] != "303" { + t.Fatalf("providers = %v, want lowest registry ID first", forward) + } + if len(reverse) != len(forward) || reverse[0] != forward[0] || reverse[1] != forward[1] { + t.Fatalf("providers = %v then %v, want the same order regardless of registry order", forward, reverse) + } +} + +// Registry IDs are numbers, not strings: 9 sorts before 23. +func TestListReplacementProviders_SortsNumerically(t *testing.T) { + got := providerIDStrings(t, selectorWithRegistry( + observedProvider("23", true, true), + observedProvider("9", true, true), + observedProvider("101", true, true), + )) + want := []string{"9", "23", "101"} + for i := range want { + if i >= len(got) || got[i] != want[i] { + t.Fatalf("providers = %v, want %v", got, want) + } + } +} + +func TestListReplacementProviders_ReportsAnUnavailableRegistry(t *testing.T) { + if _, err := (*StorageProviderSelector)(nil).ListReplacementProviders(context.Background()); err == nil { + t.Fatal("listing without an inventory returned no error") + } + selector := NewStorageProviderSelector(&stubProviderRegistry{err: errors.New("registry unreachable")}) + if _, err := selector.ListReplacementProviders(context.Background()); err == nil { + t.Fatal("a failing inventory read returned no error") + } +} + +// The storage topology lists providers whose health probe says available. The +// chooser has to ask for exactly that, or it offers providers the operator was +// told are unreachable -- and hides ones they were told are fine. +func TestListReplacementProviders_AsksForTheSameSetTheTopologyShows(t *testing.T) { + registry := &stubProviderRegistry{items: []observability.ProviderObservation{observedProvider("2", true, true)}} + if _, err := NewStorageProviderSelector(registry).ListReplacementProviders(context.Background()); err != nil { + t.Fatalf("ListReplacementProviders: %v", err) + } + if registry.opts.Status != observability.StatusAvailable { + t.Fatalf("requested status = %q, want only the providers reported available", registry.opts.Status) + } + if registry.opts.Limit <= 0 { + t.Fatalf("requested limit = %d, want a bounded page", registry.opts.Limit) + } +} + +func TestListReplacementProviders_ReadsPastTheFirstPage(t *testing.T) { + items := make([]observability.ProviderObservation, 0, replacementProviderPageSize+1) + for i := 1; i <= replacementProviderPageSize+1; i++ { + items = append(items, observedProvider(strconv.Itoa(i), true, i == replacementProviderPageSize+1)) + } + registry := &stubProviderRegistry{items: items} + got := providerIDStrings(t, NewStorageProviderSelector(registry)) + if len(got) != 1 || got[0] != strconv.Itoa(replacementProviderPageSize+1) { + t.Fatalf("providers = %v, want the eligible provider on page two", got) + } + if len(registry.calls) != 2 || registry.calls[1].Offset != replacementProviderPageSize { + t.Fatalf("page requests = %#v, want offsets 0 and %d", registry.calls, replacementProviderPageSize) + } +} + +func TestListReplacementProviders_StopsWhenPaginationDoesNotAdvance(t *testing.T) { + page := make([]observability.ProviderObservation, 0, replacementProviderPageSize) + for i := 1; i <= replacementProviderPageSize; i++ { + page = append(page, observedProvider(strconv.Itoa(i), true, true)) + } + registry := &stubProviderRegistry{ + list: func(observability.ListOptions) observability.ProviderObservationPage { + // Simulate a drifting registry that ignores the requested offset and + // returns the same full page forever. + return observability.ProviderObservationPage{Items: page, Total: replacementProviderPageSize * 2} + }, + } + got := providerIDStrings(t, NewStorageProviderSelector(registry)) + if len(got) != replacementProviderPageSize { + t.Fatalf("providers = %d, want one de-duplicated page", len(got)) + } + if len(registry.calls) != 2 { + t.Fatalf("page requests = %d, want the duplicate page to stop pagination", len(registry.calls)) + } +} diff --git a/internal/admin/server.go b/internal/admin/server.go index 3bc4cc9..d0b3cc3 100644 --- a/internal/admin/server.go +++ b/internal/admin/server.go @@ -22,6 +22,7 @@ import ( "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/objectreader" "github.com/strahe/synaps3/internal/observability" + "github.com/strahe/synaps3/internal/storagereplacement" "github.com/strahe/synaps3/internal/synapse" "github.com/strahe/synaps3/ui" "github.com/uptrace/bun" @@ -62,9 +63,13 @@ type Server struct { filecoinDefaultCopies int evictMaxRetries int storageCleanupMaxRetries int - setupOnly bool - logger *slog.Logger - startedAt time.Time + uploadMaxRetries int + // replacementSelector resolves an automatic replacement provider. Nil means + // only an explicit Provider ID can be confirmed. + replacementSelector providerReplacementSelector + setupOnly bool + logger *slog.Logger + startedAt time.Time // Track previously seen label sets to zero stale entries on refresh. prevTaskLabels map[[2]string]struct{} @@ -108,6 +113,7 @@ func New( filecoinDefaultCopies: boundedBucketCopies(filecoinDefaultCopies), evictMaxRetries: 5, storageCleanupMaxRetries: 5, + uploadMaxRetries: 5, logger: logger, startedAt: time.Now(), } @@ -206,6 +212,14 @@ func (s *Server) WithS3IAM(iam auth.IAMService, rootAccess string) *Server { return s } +// WithUploadMaxRetries sets the retry budget replacement coordinators inherit. +func (s *Server) WithUploadMaxRetries(maxRetries int) *Server { + if maxRetries > 0 { + s.uploadMaxRetries = maxRetries + } + return s +} + // WithStorageCleanupMaxRetries configures max retries for storage cleanup tasks created by admin actions. func (s *Server) WithStorageCleanupMaxRetries(maxRetries int) *Server { s.storageCleanupMaxRetries = maxRetries @@ -257,6 +271,9 @@ func (s *Server) Serve(ctx context.Context, listener net.Listener) error { mux.HandleFunc("GET /api/v1/buckets/{name}", s.handleAPIGetBucket) mux.HandleFunc("PUT /api/v1/buckets/{name}/owner", s.handleAPIUpdateBucketOwner) mux.HandleFunc("PUT /api/v1/buckets/{name}/copy-policy", s.handleAPIUpdateBucketCopyPolicy) + mux.HandleFunc("POST /api/v1/buckets/{name}/data-sets/{id}/replacement", s.handleAPIStartDataSetReplacement) + mux.HandleFunc("GET /api/v1/buckets/{name}/data-sets/{id}/replacement/providers", s.handleAPIListDataSetReplacementProviders) + mux.HandleFunc("POST /api/v1/storage-replacements/{id}/retry", s.handleAPIRetryStorageReplacement) mux.HandleFunc("DELETE /api/v1/buckets/{name}", s.handleAPIDeleteBucket) mux.HandleFunc("GET /api/v1/buckets/{name}/objects", s.handleAPIBucketObjects) mux.HandleFunc("DELETE /api/v1/buckets/{name}/objects", s.handleAPIDeleteBucketObject) @@ -463,7 +480,12 @@ func (s *Server) handleRetryExhausted(w http.ResponseWriter, r *http.Request) { if err := s.repos.Tasks.RetryExhausted(r.Context(), id); err != nil { s.logger.Error("failed to retry exhausted task", "taskID", id, "error", err) - if errors.Is(err, repository.ErrNotFound) { + if errors.Is(err, repository.ErrReplacementRetryUnsupported) { + writeJSON(w, http.StatusConflict, map[string]string{ + "error": "Retry this replacement from the bucket Details page, under Storage → Data Sets.", + "code": storagereplacement.CodeTaskRetryUnsupported, + }) + } else if errors.Is(err, repository.ErrNotFound) { writeJSON(w, http.StatusNotFound, map[string]string{"error": "not found or not in exhausted state"}) } else { writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) diff --git a/internal/app/runtime.go b/internal/app/runtime.go index 9e2e50a..1a13635 100644 --- a/internal/app/runtime.go +++ b/internal/app/runtime.go @@ -53,6 +53,10 @@ type FilecoinServices struct { Receipts worker.WalletReceiptChecker Readiness ReadinessProbe Observability observability.RefreshChecker + // Terminator ends a replaced storage service. Epochs observes the chain so + // the gateway can tell an accepted termination from a completed one. + Terminator synapse.ServiceTerminator + Epochs synapse.ChainEpochReader } // RuntimeOptions configures the application composition root. Database, @@ -172,7 +176,7 @@ func NewRuntime(ctx context.Context, opts RuntimeOptions) (_ *Runtime, err error cfg.Worker.Evictor.MaxRetries, )), worker.NewStorageCleanupWorker(repos, opts.Filecoin.Storage, - cfg.Worker.StorageCleanup.Concurrency, cfg.Worker.StorageCleanup.PollInterval, logger), + cfg.Worker.StorageCleanup.Concurrency, cfg.Worker.StorageCleanup.PollInterval, logger, worker.WithServiceTermination(opts.Filecoin.Terminator, opts.Filecoin.Epochs)), worker.NewWalletOperationRunner(repos, opts.Filecoin.Wallet, opts.Filecoin.Receipts, 5*time.Second, logger, worker.WithWalletOperationEventPublisher(events)), ).WithTaskMaxRetries(cfg.Worker.Upload.MaxRetries, cfg.Worker.Evictor.MaxRetries) @@ -191,10 +195,16 @@ func NewRuntime(ctx context.Context, opts RuntimeOptions) (_ *Runtime, err error WithObservability(observabilityService). WithEvictMaxRetries(cfg.Worker.Evictor.MaxRetries). WithStorageCleanupMaxRetries(cfg.Worker.StorageCleanup.MaxRetries). + WithUploadMaxRetries(cfg.Worker.Upload.MaxRetries). WithS3IAM(iamService, rootAccount.Access) if opts.ProviderIdentity != nil { adminServer.WithProviderIdentityResolver(opts.ProviderIdentity) } + if observabilityService != nil { + // Replacement offers the same providers the storage topology reports, + // which is the same service that reports them. + adminServer.WithProviderReplacement(admin.NewStorageProviderSelector(observabilityService)) + } if err := adminServer.WithTrustedProxies(cfg.Admin.TrustedProxies); err != nil { return nil, fmt.Errorf("initializing admin trusted proxies: %w", err) } @@ -247,6 +257,8 @@ func validateOptions(opts RuntimeOptions) error { {name: "filecoin receipts", value: opts.Filecoin.Receipts}, {name: "filecoin readiness", value: opts.Filecoin.Readiness}, {name: "filecoin observability", value: opts.Filecoin.Observability}, + {name: "filecoin service terminator", value: opts.Filecoin.Terminator}, + {name: "filecoin chain epochs", value: opts.Filecoin.Epochs}, } for _, dependency := range dependencies { if isNil(dependency.value) { diff --git a/internal/db/db_test.go b/internal/db/db_test.go index 4531379..0c8f9e2 100644 --- a/internal/db/db_test.go +++ b/internal/db/db_test.go @@ -221,18 +221,27 @@ func TestRunMigrations_StorageProvenanceConstraints(t *testing.T) { mustExec(t, db, `INSERT INTO buckets (id, name) VALUES (2, 'bucket-b')`) mustExec(t, db, `INSERT INTO storage_uploads (id, bucket_id, source_version_id, content_size, checksum, status, piece_cid, requested_copies) VALUES (1, 1, 'v1', 10, 'sum-1', 'complete', 'bafk2bzacefake', 2)`) mustExec(t, db, `INSERT INTO storage_uploads (id, bucket_id, source_version_id, content_size, checksum, status, requested_copies) VALUES (2, 2, 'v2', 10, 'sum-2', 'running', 3)`) - mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, data_set_id, status, created_by_upload_id, last_used_upload_id) VALUES (1, 1, '101', 0, '1001', 'ready', 1, 1)`) - if _, err := db.ExecContext(ctx, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, data_set_id, status, created_by_upload_id, last_used_upload_id) VALUES (2, '101', 0, '1001', 'ready', 2, 2)`); err == nil { + mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_by_upload_id, last_used_upload_id) VALUES (1, 1, '101', 0, 1, TRUE, '1001', 'ready', 1, 1)`) + if _, err := db.ExecContext(ctx, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_by_upload_id, last_used_upload_id) VALUES (2, '101', 0, 1, TRUE, '1001', 'ready', 2, 2)`); err == nil { t.Fatal("expected provider/data_set reuse across buckets to fail") } - if _, err := db.ExecContext(ctx, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, status, created_by_upload_id, last_used_upload_id) VALUES (1, '202', 0, 'pending', 1, 1)`); err == nil { + if _, err := db.ExecContext(ctx, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, status, created_by_upload_id, last_used_upload_id) VALUES (1, '202', 0, 1, TRUE, 'pending', 1, 1)`); err == nil { t.Fatal("expected duplicate bucket/copy_index binding to fail") } - mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, status, created_by_upload_id, last_used_upload_id) VALUES (2, 1, '202', 1, 'pending', 1, 1)`) + mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, generation, is_current, status, created_by_upload_id, last_used_upload_id) VALUES (2, 1, '202', 1, 1, TRUE, 'pending', 1, 1)`) mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id) VALUES (1, 0, '101', '2001', 'ingress', 'committed', 'https://provider.example/piece', 1)`) - if _, err := db.ExecContext(ctx, `INSERT INTO storage_upload_copies (upload_id, copy_index, transfer_method) VALUES (1, 0, 'peer_pull')`); err == nil { - t.Fatal("expected duplicate copy_index for upload to fail") + if _, err := db.ExecContext(ctx, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, transfer_method, storage_data_set_id) VALUES (1, 0, '101', 'peer_pull', 1)`); err == nil { + t.Fatal("expected duplicate copy for one data set to fail") + } + // A replica slot holds both generations while a provider replacement migrates. + mustExec(t, db, `INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, generation, is_current, status, created_by_upload_id, last_used_upload_id) VALUES (3, 1, '303', 0, 2, FALSE, 'pending', 1, 1)`) + mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, transfer_method, storage_data_set_id) VALUES (1, 0, '303', 'peer_pull', 3)`) + // Unbound copies still cannot duplicate a slot, which distinct NULL data set + // ids would otherwise allow. + mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, transfer_method) VALUES (1, 3, 'peer_pull')`) + if _, err := db.ExecContext(ctx, `INSERT INTO storage_upload_copies (upload_id, copy_index, transfer_method) VALUES (1, 3, 'peer_pull')`); err == nil { + t.Fatal("expected duplicate unbound copy for one slot to fail") } mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, transfer_method, storage_data_set_id) VALUES (1, 1, '202', 'peer_pull', 2)`) if _, err := db.ExecContext(ctx, `UPDATE storage_upload_copies SET status = 'committed' WHERE upload_id = 1 AND copy_index = 1`); err == nil { diff --git a/internal/db/migrations/2026082101_storage_data_set_generations.go b/internal/db/migrations/2026082101_storage_data_set_generations.go new file mode 100644 index 0000000..808ba5c --- /dev/null +++ b/internal/db/migrations/2026082101_storage_data_set_generations.go @@ -0,0 +1,264 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func init() { + Migrations.MustRegister(up2026082101StorageDataSetGenerations, down2026082101StorageDataSetGenerations) +} + +// A replica slot may now own several data set generations so an operator can +// replace a provider while the previous generation stays readable. Both +// directions run in one transaction because bun marks a migration applied +// before it executes; a half-applied schema would otherwise be recorded as +// complete. +func up2026082101StorageDataSetGenerations(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + if err := assertStorageGenerationPreconditions2026082101(ctx, tx); err != nil { + return err + } + pg := db.Dialect().Name() == dialect.PG + + isCurrentColumn := "INTEGER NOT NULL DEFAULT 1" + if pg { + isCurrentColumn = "BOOLEAN NOT NULL DEFAULT TRUE" + } + // Neither column carries a CHECK: SQLite cannot add one later and a + // column-level CHECK blocks DROP COLUMN on rollback. The partial unique + // indexes below enforce the invariants on both dialects instead. + // + // The initial migration builds storage_data_sets from the runtime model, + // so a database created after this change already has both columns while + // an upgraded one does not. + var statements []string + for column, definition := range map[string]string{ + "is_current": isCurrentColumn, + "generation": "INTEGER NOT NULL DEFAULT 1", + } { + exists, err := storageDataSetColumnExists2026082101(ctx, tx, pg, column) + if err != nil { + return err + } + if !exists { + statements = append(statements, fmt.Sprintf("ALTER TABLE storage_data_sets ADD COLUMN %s %s", column, definition)) + } + } + + statements = append(statements, + "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_copy_index", + "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_provider", + "DROP INDEX IF EXISTS idx_storage_upload_copies_upload_index", + + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_copy_current + ON storage_data_sets (bucket_id, copy_index) WHERE is_current`, + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_copy_generation + ON storage_data_sets (bucket_id, copy_index, generation)`, + // A historical generation never blocks reusing its provider; only a + // live slot does. + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_provider_current + ON storage_data_sets (bucket_id, provider_id) WHERE is_current`, + + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_upload_copies_upload_data_set + ON storage_upload_copies (upload_id, storage_data_set_id) WHERE storage_data_set_id IS NOT NULL`, + // NULLs compare distinct in unique indexes on both dialects, so the + // index above would let unbound duplicates accumulate per slot. + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_upload_copies_upload_slot_unbound + ON storage_upload_copies (upload_id, copy_index) WHERE storage_data_set_id IS NULL`, + `CREATE INDEX IF NOT EXISTS idx_storage_upload_copies_upload_slot + ON storage_upload_copies (upload_id, copy_index)`, + + storageReplacementsTableSQL2026082101(pg), + storageReplacementItemsTableSQL2026082101(pg), + + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_replacements_active_source + ON storage_replacements (source_data_set_id) WHERE status NOT IN ('completed', 'superseded')`, + `CREATE INDEX IF NOT EXISTS idx_storage_replacements_bucket_slot + ON storage_replacements (bucket_id, copy_index, id)`, + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_replacements_bucket_request + ON storage_replacements (bucket_id, client_request_id)`, + `CREATE INDEX IF NOT EXISTS idx_storage_replacement_items_next + ON storage_replacement_items (replacement_id, status, id)`, + ) + for _, query := range statements { + if _, err := tx.ExecContext(ctx, query); err != nil { + return fmt.Errorf("adding storage data set generations: %w", err) + } + } + return nil + }) +} + +// Rollback only succeeds while every slot still owns a single generation. Once +// a replacement has produced a second generation the original unique indexes +// cannot be restored, and refusing is the correct outcome. +func down2026082101StorageDataSetGenerations(ctx context.Context, db *bun.DB) error { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + statements := []string{ + "DROP TABLE IF EXISTS storage_replacement_items", + "DROP TABLE IF EXISTS storage_replacements", + + // SQLite refuses to drop a column named by an index or by a partial + // index predicate, so the indexes go first. + "DROP INDEX IF EXISTS idx_storage_upload_copies_upload_slot", + "DROP INDEX IF EXISTS idx_storage_upload_copies_upload_slot_unbound", + "DROP INDEX IF EXISTS idx_storage_upload_copies_upload_data_set", + "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_provider_current", + "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_copy_generation", + "DROP INDEX IF EXISTS idx_storage_data_sets_bucket_copy_current", + + "ALTER TABLE storage_data_sets DROP COLUMN generation", + "ALTER TABLE storage_data_sets DROP COLUMN is_current", + + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_copy_index + ON storage_data_sets (bucket_id, copy_index)`, + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_data_sets_bucket_provider + ON storage_data_sets (bucket_id, provider_id)`, + `CREATE UNIQUE INDEX IF NOT EXISTS idx_storage_upload_copies_upload_index + ON storage_upload_copies (upload_id, copy_index)`, + } + for _, query := range statements { + if _, err := tx.ExecContext(ctx, query); err != nil { + return fmt.Errorf("removing storage data set generations: %w", err) + } + } + return nil + }) +} + +func storageDataSetColumnExists2026082101(ctx context.Context, tx bun.Tx, pg bool, column string) (bool, error) { + query := `SELECT COUNT(*) FROM pragma_table_info('storage_data_sets') WHERE name = ?` + if pg { + query = `SELECT COUNT(*) FROM information_schema.columns + WHERE table_schema = current_schema() + AND table_name = 'storage_data_sets' + AND column_name = ?` + } + var count int + if err := tx.NewRaw(query, column).Scan(ctx, &count); err != nil { + return false, fmt.Errorf("checking storage_data_sets.%s: %w", column, err) + } + return count > 0, nil +} + +// The new unique indexes would fail mid-migration on a database that violated +// an invariant the old schema never enforced. Reporting the offending rows up +// front keeps the failure actionable. +func assertStorageGenerationPreconditions2026082101(ctx context.Context, tx bun.Tx) error { + checks := []struct { + description string + query string + }{ + { + description: "storage upload copies sharing one data set", + query: `SELECT COUNT(*) FROM ( + SELECT upload_id, storage_data_set_id FROM storage_upload_copies + WHERE storage_data_set_id IS NOT NULL + GROUP BY upload_id, storage_data_set_id HAVING COUNT(*) > 1 + ) AS duplicates`, + }, + { + description: "unbound storage upload copies sharing one replica slot", + query: `SELECT COUNT(*) FROM ( + SELECT upload_id, copy_index FROM storage_upload_copies + WHERE storage_data_set_id IS NULL + GROUP BY upload_id, copy_index HAVING COUNT(*) > 1 + ) AS duplicates`, + }, + { + description: "storage data sets sharing one replica slot", + query: `SELECT COUNT(*) FROM ( + SELECT bucket_id, copy_index FROM storage_data_sets + GROUP BY bucket_id, copy_index HAVING COUNT(*) > 1 + ) AS duplicates`, + }, + } + for _, check := range checks { + var count int + if err := tx.NewRaw(check.query).Scan(ctx, &count); err != nil { + return fmt.Errorf("checking %s: %w", check.description, err) + } + if count > 0 { + return fmt.Errorf("cannot add storage data set generations: found %d %s", count, check.description) + } + } + return nil +} + +func storageReplacementsTableSQL2026082101(pg bool) string { + identity := "id INTEGER PRIMARY KEY AUTOINCREMENT" + reference := "INTEGER" + timestamp := "TIMESTAMP" + boolean := "INTEGER NOT NULL DEFAULT 0" + if pg { + identity = "id BIGSERIAL PRIMARY KEY" + reference = "BIGINT" + timestamp = "TIMESTAMPTZ" + boolean = "BOOLEAN NOT NULL DEFAULT FALSE" + } + return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS storage_replacements ( + %[1]s, + bucket_id %[2]s NOT NULL REFERENCES buckets (id) ON UPDATE CASCADE ON DELETE RESTRICT, + copy_index INTEGER NOT NULL, + source_data_set_id %[2]s NOT NULL REFERENCES storage_data_sets (id) ON UPDATE CASCADE ON DELETE RESTRICT, + target_data_set_id %[2]s NOT NULL REFERENCES storage_data_sets (id) ON UPDATE CASCADE ON DELETE RESTRICT, + selection_mode TEXT NOT NULL, + requested_provider_id TEXT, + client_request_id TEXT NOT NULL, + status TEXT NOT NULL, + wait_reason TEXT, + failure_reason TEXT, + last_error TEXT, + items_total INTEGER NOT NULL DEFAULT 0, + items_copied INTEGER NOT NULL DEFAULT 0, + seed_cursor_upload_id %[2]s NOT NULL DEFAULT 0, + seeding_complete %[4]s, + termination_tx_hash TEXT, + termination_epoch %[2]s, + termination_observed_at %[3]s, + abandoned_termination_tx_hash TEXT, + abandoned_termination_epoch %[2]s, + abandoned_termination_observed_at %[3]s, + superseded_by_id %[2]s REFERENCES storage_replacements (id) ON UPDATE CASCADE ON DELETE SET NULL, + confirmed_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, + created_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT chk_storage_replacements_copy_index CHECK (copy_index >= 0), + CONSTRAINT chk_storage_replacements_selection_mode CHECK (selection_mode IN ('automatic', 'manual')), + CONSTRAINT chk_storage_replacements_status CHECK (status IN ('preparing_target', 'migrating', 'waiting', 'retiring', 'cleanup_attention', 'failed', 'completed', 'superseded')), + CONSTRAINT chk_storage_replacements_wait_reason CHECK (wait_reason IS NULL OR wait_reason IN ('readable_source', 'target', 'target_creating', 'target_writable', 'funding', 'provider', 'termination_epoch', 'source_writes', 'coverage')), + CONSTRAINT chk_storage_replacements_failure_reason CHECK (failure_reason IS NULL OR failure_reason IN ('target_in_use')), + CONSTRAINT chk_storage_replacements_client_request_id CHECK (length(client_request_id) BETWEEN 1 AND 128), + CONSTRAINT chk_storage_replacements_distinct_data_sets CHECK (source_data_set_id <> target_data_set_id), + CONSTRAINT chk_storage_replacements_items CHECK (items_total >= 0 AND items_copied >= 0 AND items_copied <= items_total) + )`, identity, reference, timestamp, boolean) +} + +func storageReplacementItemsTableSQL2026082101(pg bool) string { + identity := "id INTEGER PRIMARY KEY AUTOINCREMENT" + reference := "INTEGER" + timestamp := "TIMESTAMP" + if pg { + identity = "id BIGSERIAL PRIMARY KEY" + reference = "BIGINT" + timestamp = "TIMESTAMPTZ" + } + return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS storage_replacement_items ( + %[1]s, + replacement_id %[2]s NOT NULL REFERENCES storage_replacements (id) ON UPDATE CASCADE ON DELETE CASCADE, + upload_id %[2]s NOT NULL REFERENCES storage_uploads (id) ON UPDATE CASCADE ON DELETE RESTRICT, + target_copy_id %[2]s REFERENCES storage_upload_copies (id) ON UPDATE CASCADE ON DELETE SET NULL, + status TEXT NOT NULL, + attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at %[3]s NOT NULL DEFAULT CURRENT_TIMESTAMP, + CONSTRAINT chk_storage_replacement_items_status CHECK (status IN ('pending', 'running', 'waiting_source', 'copied', 'cancelled')), + CONSTRAINT chk_storage_replacement_items_attempts CHECK (attempts >= 0), + CONSTRAINT uq_storage_replacement_items_upload UNIQUE (replacement_id, upload_id) + )`, identity, reference, timestamp) +} diff --git a/internal/db/migrations/storage_data_set_generations_test.go b/internal/db/migrations/storage_data_set_generations_test.go new file mode 100644 index 0000000..3c91061 --- /dev/null +++ b/internal/db/migrations/storage_data_set_generations_test.go @@ -0,0 +1,300 @@ +package migrations + +import ( + "context" + "database/sql" + "fmt" + "strings" + "testing" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/sqlitedialect" + + _ "modernc.org/sqlite" +) + +func TestStorageDataSetGenerationsMigrationPreservesExistingSlots(t *testing.T) { + ctx := context.Background() + db := newGenerationsTestDB(t, "generations_preserve") + seedLegacyDataSet(t, db, 1, 1, 0, "101") + + if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { + t.Fatalf("up migration: %v", err) + } + + for _, column := range []string{"is_current", "generation"} { + if !sqliteColumnExists(t, db, "storage_data_sets", column) { + t.Fatalf("storage_data_sets.%s column missing", column) + } + } + var isCurrent bool + var generation int + if err := db.QueryRow("SELECT is_current, generation FROM storage_data_sets WHERE id = 1"). + Scan(&isCurrent, &generation); err != nil { + t.Fatalf("select migrated data set: %v", err) + } + if !isCurrent || generation != 1 { + t.Fatalf("migrated data set is_current=%v generation=%d, want true/1", isCurrent, generation) + } + + for _, index := range []string{ + "idx_storage_data_sets_bucket_copy_index", + "idx_storage_data_sets_bucket_provider", + "idx_storage_upload_copies_upload_index", + } { + if sqliteIndexExists(t, db, index) { + t.Fatalf("index %s should have been replaced", index) + } + } + for _, table := range []string{"storage_replacements", "storage_replacement_items"} { + if !sqliteTableExists(t, db, table) { + t.Fatalf("table %s missing", table) + } + } + for _, column := range []string{ + "client_request_id", "failure_reason", "abandoned_termination_tx_hash", + "abandoned_termination_epoch", "abandoned_termination_observed_at", + } { + if !sqliteColumnExists(t, db, "storage_replacements", column) { + t.Fatalf("storage_replacements.%s column missing", column) + } + } + if !sqliteIndexExists(t, db, "idx_storage_replacements_bucket_request") { + t.Fatal("bucket-scoped replacement idempotency index missing") + } +} + +func TestStorageDataSetGenerationsMigrationEnforcesSlotInvariants(t *testing.T) { + ctx := context.Background() + db := newGenerationsTestDB(t, "generations_invariants") + seedLegacyDataSet(t, db, 1, 1, 0, "101") + if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { + t.Fatalf("up migration: %v", err) + } + + // A slot keeps exactly one writable generation... + if err := insertDataSet(db, 2, 1, 0, "202", true, 2); err == nil { + t.Fatal("second current generation for one slot was accepted, want unique violation") + } + // ...but may retain historical ones. + if err := insertDataSet(db, 2, 1, 0, "202", false, 2); err != nil { + t.Fatalf("historical generation rejected: %v", err) + } + if err := insertDataSet(db, 3, 1, 0, "303", false, 2); err == nil { + t.Fatal("duplicate generation number for one slot was accepted, want unique violation") + } + + // A provider that only holds a historical generation can be selected again, + // which is what lets an operator reuse a previously used provider. + mustExecMigrationTest(t, db, "UPDATE storage_data_sets SET is_current = 0 WHERE id = 1") + if err := insertDataSet(db, 4, 1, 0, "101", true, 3); err != nil { + t.Fatalf("reusing a historical provider rejected: %v", err) + } + // Provider 101 now serves slot 0, so it cannot also take slot 1. + if err := insertDataSet(db, 5, 1, 1, "101", true, 1); err == nil { + t.Fatal("provider bound to two current slots was accepted, want unique violation") + } +} + +func TestStorageDataSetGenerationsMigrationClosesUnboundCopyHole(t *testing.T) { + ctx := context.Background() + db := newGenerationsTestDB(t, "generations_copy_hole") + seedLegacyDataSet(t, db, 1, 1, 0, "101") + mustExecMigrationTest(t, db, "INSERT INTO storage_uploads (id, bucket_id) VALUES (1, 1)") + if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { + t.Fatalf("up migration: %v", err) + } + + mustExecMigrationTest(t, db, + "INSERT INTO storage_upload_copies (id, upload_id, copy_index, storage_data_set_id) VALUES (1, 1, 0, 1)") + if err := insertUploadCopy(db, 2, 1, 0, sql.NullInt64{Int64: 1, Valid: true}); err == nil { + t.Fatal("duplicate copy for one data set was accepted, want unique violation") + } + + // NULL compares distinct in a unique index, so unbound copies need their own + // partial index or a slot could accumulate duplicates. + mustExecMigrationTest(t, db, + "INSERT INTO storage_upload_copies (id, upload_id, copy_index, storage_data_set_id) VALUES (3, 1, 1, NULL)") + if err := insertUploadCopy(db, 4, 1, 1, sql.NullInt64{}); err == nil { + t.Fatal("duplicate unbound copy for one slot was accepted, want unique violation") + } +} + +func TestStorageDataSetGenerationsMigrationLimitsActiveReplacements(t *testing.T) { + ctx := context.Background() + db := newGenerationsTestDB(t, "generations_active_replacement") + seedLegacyDataSet(t, db, 1, 1, 0, "101") + if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { + t.Fatalf("up migration: %v", err) + } + mustExecMigrationTest(t, db, "UPDATE storage_data_sets SET is_current = 0 WHERE id = 1") + if err := insertDataSet(db, 2, 1, 0, "202", true, 2); err != nil { + t.Fatalf("seed target generation: %v", err) + } + + if err := insertReplacement(db, 1, 1, 2, "migrating"); err != nil { + t.Fatalf("first replacement rejected: %v", err) + } + if err := insertReplacement(db, 2, 1, 2, "preparing_target"); err == nil { + t.Fatal("second active replacement for one source was accepted, want unique violation") + } + // A terminal replacement releases its source for a later confirmation. + mustExecMigrationTest(t, db, "UPDATE storage_replacements SET status = 'superseded' WHERE id = 1") + if err := insertReplacement(db, 2, 1, 2, "preparing_target"); err != nil { + t.Fatalf("replacement after supersede rejected: %v", err) + } + if err := insertReplacement(db, 3, 1, 1, "preparing_target"); err == nil { + t.Fatal("replacement onto itself was accepted, want check violation") + } +} + +func TestStorageDataSetGenerationsMigrationRejectsIncompatibleData(t *testing.T) { + ctx := context.Background() + db := newGenerationsTestDB(t, "generations_incompatible") + seedLegacyDataSet(t, db, 1, 1, 0, "101") + mustExecMigrationTest(t, db, "INSERT INTO storage_uploads (id, bucket_id) VALUES (1, 1)") + // The old schema never enforced this, so a database damaged by an earlier + // bug must fail loudly instead of part way through the index rebuild. + mustExecMigrationTest(t, db, "DROP INDEX idx_storage_upload_copies_upload_index") + mustExecMigrationTest(t, db, + "INSERT INTO storage_upload_copies (id, upload_id, copy_index, storage_data_set_id) VALUES (1, 1, 0, 1), (2, 1, 1, 1)") + + err := up2026082101StorageDataSetGenerations(ctx, db) + if err == nil { + t.Fatal("migration accepted duplicate copies for one data set, want failure") + } + if !strings.Contains(err.Error(), "sharing one data set") { + t.Fatalf("error = %v, want it to name the offending invariant", err) + } + if sqliteColumnExists(t, db, "storage_data_sets", "is_current") { + t.Fatal("failed migration left is_current behind, want a rolled back transaction") + } +} + +func TestStorageDataSetGenerationsMigrationDown(t *testing.T) { + ctx := context.Background() + db := newGenerationsTestDB(t, "generations_down") + seedLegacyDataSet(t, db, 1, 1, 0, "101") + if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { + t.Fatalf("up migration: %v", err) + } + if err := down2026082101StorageDataSetGenerations(ctx, db); err != nil { + t.Fatalf("down migration: %v", err) + } + for _, column := range []string{"is_current", "generation"} { + if sqliteColumnExists(t, db, "storage_data_sets", column) { + t.Fatalf("storage_data_sets.%s survived rollback", column) + } + } + for _, index := range []string{ + "idx_storage_data_sets_bucket_copy_index", + "idx_storage_data_sets_bucket_provider", + "idx_storage_upload_copies_upload_index", + } { + if !sqliteIndexExists(t, db, index) { + t.Fatalf("index %s was not restored", index) + } + } + if sqliteTableExists(t, db, "storage_replacements") { + t.Fatal("storage_replacements survived rollback") + } +} + +// Rolling back is only meaningful while every slot still owns one generation. +func TestStorageDataSetGenerationsMigrationDownRefusesMultipleGenerations(t *testing.T) { + ctx := context.Background() + db := newGenerationsTestDB(t, "generations_down_dirty") + seedLegacyDataSet(t, db, 1, 1, 0, "101") + if err := up2026082101StorageDataSetGenerations(ctx, db); err != nil { + t.Fatalf("up migration: %v", err) + } + if err := insertDataSet(db, 2, 1, 0, "202", false, 2); err != nil { + t.Fatalf("seed historical generation: %v", err) + } + + if err := down2026082101StorageDataSetGenerations(ctx, db); err == nil { + t.Fatal("rollback accepted two generations for one slot, want failure") + } + if !sqliteColumnExists(t, db, "storage_data_sets", "generation") { + t.Fatal("failed rollback dropped generation, want a rolled back transaction") + } +} + +func newGenerationsTestDB(t *testing.T, name string) *bun.DB { + t.Helper() + sqldb, err := sql.Open("sqlite", "file:"+name+"?mode=memory&cache=shared&_pragma=foreign_keys(1)") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + sqldb.SetMaxOpenConns(1) + db := bun.NewDB(sqldb, sqlitedialect.New()) + t.Cleanup(func() { _ = db.Close() }) + + // Stand-ins carrying only what this migration reads or rewrites. + schema := []string{ + `CREATE TABLE buckets (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL UNIQUE)`, + `CREATE TABLE storage_uploads (id INTEGER PRIMARY KEY AUTOINCREMENT, bucket_id INTEGER NOT NULL REFERENCES buckets (id))`, + `CREATE TABLE storage_data_sets ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + bucket_id INTEGER NOT NULL REFERENCES buckets (id), + provider_id TEXT NOT NULL, + copy_index INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'ready' + )`, + `CREATE TABLE storage_upload_copies ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + upload_id INTEGER NOT NULL REFERENCES storage_uploads (id), + copy_index INTEGER NOT NULL, + storage_data_set_id INTEGER REFERENCES storage_data_sets (id) + )`, + `CREATE UNIQUE INDEX idx_storage_data_sets_bucket_copy_index ON storage_data_sets (bucket_id, copy_index)`, + `CREATE UNIQUE INDEX idx_storage_data_sets_bucket_provider ON storage_data_sets (bucket_id, provider_id)`, + `CREATE UNIQUE INDEX idx_storage_upload_copies_upload_index ON storage_upload_copies (upload_id, copy_index)`, + `INSERT INTO buckets (id, name) VALUES (1, 'replacement-bucket')`, + } + for _, query := range schema { + mustExecMigrationTest(t, db, query) + } + return db +} + +func seedLegacyDataSet(t *testing.T, db *bun.DB, id, bucketID int64, copyIndex int, providerID string) { + t.Helper() + mustExecMigrationTest(t, db, fmt.Sprintf( + "INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index) VALUES (%d, %d, '%s', %d)", + id, bucketID, providerID, copyIndex, + )) +} + +func insertDataSet(db *bun.DB, id, bucketID int64, copyIndex int, providerID string, isCurrent bool, generation int) error { + _, err := db.Exec( + "INSERT INTO storage_data_sets (id, bucket_id, provider_id, copy_index, is_current, generation) VALUES (?, ?, ?, ?, ?, ?)", + id, bucketID, providerID, copyIndex, isCurrent, generation, + ) + return err +} + +func insertUploadCopy(db *bun.DB, id, uploadID int64, copyIndex int, dataSetID sql.NullInt64) error { + _, err := db.Exec( + "INSERT INTO storage_upload_copies (id, upload_id, copy_index, storage_data_set_id) VALUES (?, ?, ?, ?)", + id, uploadID, copyIndex, dataSetID, + ) + return err +} + +func insertReplacement(db *bun.DB, id, sourceDataSetID, targetDataSetID int64, status string) error { + _, err := db.Exec( + `INSERT INTO storage_replacements + (id, bucket_id, copy_index, source_data_set_id, target_data_set_id, selection_mode, client_request_id, status) + VALUES (?, 1, 0, ?, ?, 'automatic', ?, ?)`, + id, sourceDataSetID, targetDataSetID, fmt.Sprintf("migration-request-%d", id), status, + ) + return err +} + +func mustExecMigrationTest(t *testing.T, db *bun.DB, query string) { + t.Helper() + if _, err := db.Exec(query); err != nil { + t.Fatalf("exec %q: %v", query, err) + } +} diff --git a/internal/db/repository/bucket_repo.go b/internal/db/repository/bucket_repo.go index 693e2f6..83d5556 100644 --- a/internal/db/repository/bucket_repo.go +++ b/internal/db/repository/bucket_repo.go @@ -58,6 +58,25 @@ func (r *BunBucketRepo) GetByID(ctx context.Context, id int64) (*model.Bucket, e return bucket, nil } +func (r *BunBucketRepo) GetNamesByIDs(ctx context.Context, ids []int64) (map[int64]string, error) { + out := make(map[int64]string, len(ids)) + if len(ids) == 0 { + return out, nil + } + var buckets []model.Bucket + if err := r.db.NewSelect(). + Model(&buckets). + Column("id", "name"). + Where("id IN (?)", bun.List(ids)). + Scan(ctx); err != nil { + return nil, fmt.Errorf("selecting bucket names: %w", err) + } + for i := range buckets { + out[buckets[i].ID] = buckets[i].Name + } + return out, nil +} + func (r *BunBucketRepo) ListActive(ctx context.Context) ([]model.Bucket, error) { var buckets []model.Bucket err := r.db.NewSelect(). diff --git a/internal/db/repository/cache_eviction_repo.go b/internal/db/repository/cache_eviction_repo.go index a76826c..71489e8 100644 --- a/internal/db/repository/cache_eviction_repo.go +++ b/internal/db/repository/cache_eviction_repo.go @@ -730,7 +730,7 @@ func requireMinimumDurability( return cacheeviction.ErrDurabilityThreshold } minimum := minimumDurableCopiesForUpload(bucket, upload.RequestedCopies) - readable, err := countReadableCommittedCopies(ctx, db, upload.ID) + readable, err := countReadableReplicaSlots(ctx, db, upload.ID) if err != nil { return err } @@ -802,28 +802,17 @@ func nextBucketDurabilityCandidate( return version, nil } +// Keep this threshold aligned with minimumDurableCopiesForUpload; both compare +// readable replica slots, not physical data set generations, against the +// bucket's effective minimum. func minimumDurabilityMetSQL(uploadAlias, bucketAlias string) string { - return fmt.Sprintf(`( - SELECT COUNT(*) - FROM storage_upload_copies AS durable_copy - JOIN storage_data_sets AS durable_data_set ON durable_data_set.id = durable_copy.storage_data_set_id - WHERE durable_copy.upload_id = %s.id - AND durable_copy.status = '%s' - AND durable_copy.storage_data_set_id IS NOT NULL - AND durable_copy.provider_id IS NOT NULL AND durable_copy.provider_id <> '' - AND durable_data_set.data_set_id IS NOT NULL AND durable_data_set.data_set_id <> '' - AND durable_data_set.status IN (%s) - AND durable_copy.piece_id IS NOT NULL AND durable_copy.piece_id <> '' - AND durable_copy.retrieval_url IS NOT NULL AND durable_copy.retrieval_url <> '' - ) >= CASE + return fmt.Sprintf(`%s >= CASE WHEN %s.minimum_durable_copies IS NULL OR %s.minimum_durable_copies >= %s.requested_copies THEN %s.requested_copies ELSE %s.minimum_durable_copies END`, - uploadAlias, - model.StorageUploadCopyStatusCommitted, - storageHealthReadyDataSetStatusListSQL(), + distinctReadableSlotCountSQL("durable_copy", "durable_data_set", uploadAlias+".id"), bucketAlias, bucketAlias, uploadAlias, @@ -857,10 +846,5 @@ func cacheDeletionAuthorizedSQL(dialectName dialect.Name) string { } func (r *BunCacheEvictionRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) error { - if db, ok := r.db.(*bun.DB); ok { - return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - return fn(tx) - }) - } - return fn(r.db) + return runMaybeTx(ctx, r.db, fn) } diff --git a/internal/db/repository/errors.go b/internal/db/repository/errors.go index 1cf0070..6f0a7fa 100644 --- a/internal/db/repository/errors.go +++ b/internal/db/repository/errors.go @@ -36,6 +36,11 @@ var ErrUploadTaskCancelled = errors.New("upload task cancelled") // ErrTaskClaimLost reports that a worker no longer owns the running task claim. var ErrTaskClaimLost = errors.New("task claim lost") +// ErrReplacementRetryUnsupported means the task belongs to an operator-approved +// provider replacement, which resumes only through its own retry action so the +// replacement record and the task never disagree. It wraps ErrConflict. +var ErrReplacementRetryUnsupported = fmt.Errorf("provider replacement work cannot be retried from the task queue: %w", ErrConflict) + // ErrAlreadyCurrent is returned when a restore would not change the current object representation. var ErrAlreadyCurrent = errors.New("already current") diff --git a/internal/db/repository/interfaces.go b/internal/db/repository/interfaces.go index 8a9cac9..14fda66 100644 --- a/internal/db/repository/interfaces.go +++ b/internal/db/repository/interfaces.go @@ -6,6 +6,7 @@ import ( "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/observability" + "github.com/strahe/synaps3/internal/storagereplacement" "github.com/strahe/synaps3/internal/types" "github.com/versity/versitygw/auth" ) @@ -15,6 +16,7 @@ type BucketRepository interface { Create(ctx context.Context, bucket *model.Bucket) error GetByName(ctx context.Context, name string) (*model.Bucket, error) GetByID(ctx context.Context, id int64) (*model.Bucket, error) + GetNamesByIDs(ctx context.Context, ids []int64) (map[int64]string, error) ListActive(ctx context.Context) ([]model.Bucket, error) // List returns all buckets regardless of status. List(ctx context.Context) ([]model.Bucket, error) @@ -226,6 +228,8 @@ type StorageDataSetSummary struct { BucketID int64 `bun:"bucket_id"` BucketName string `bun:"bucket_name"` CopyIndex int `bun:"copy_index"` + Generation int `bun:"generation"` + IsCurrent bool `bun:"is_current"` ProviderID types.OnChainID `bun:"provider_id"` DataSetID *types.OnChainID `bun:"data_set_id"` ClientDataSetID *types.OnChainID `bun:"client_data_set_id"` @@ -349,8 +353,17 @@ type UploadCopyBindingInput struct { ProviderID types.OnChainID } +// StorageUploadCopyID names the exact copy row to write. A task that was +// queued before the current data set generation took over must still land on +// the generation it actually stored to, so addressing is separate from the +// eligibility guards below. +// +// RequireEligibleCopy makes the write refuse a failed copy, a deleted object, +// or a copy that no longer matches, instead of reporting no rows. Coordinators +// that own one specific copy set it; ordinary upload stages stay idempotent. type MarkUploadCopyPieceReadyInput struct { StorageUploadCopyID int64 + RequireEligibleCopy bool UploadID int64 CopyIndex int PieceCID string @@ -360,13 +373,30 @@ type MarkUploadCopyPieceReadyInput struct { type MarkUploadCopyCommittingInput struct { StorageUploadCopyID int64 + RequireEligibleCopy bool UploadID int64 CopyIndex int CommitExtraDataHex string CommitTransactionID string } +// MarkUploadCopyFailedInput names the copy that failed. Without the id the +// failure resolves through the replica slot, which after an activation is a +// different generation than the one the write was bound to: the failure would +// either land on the replacement's copy or update nothing at all, leaving the +// original stuck mid-transfer and holding retirement open. +type MarkUploadCopyFailedInput struct { + StorageUploadCopyID int64 + UploadID int64 + CopyIndex int + LastError string +} + type ResetRejectedUploadCopyCommitInput struct { + // StorageUploadCopyID names the exact copy whose commit was rejected. Without + // it the reset resolves through the replica slot, which after an activation + // points at a different generation than the one that submitted the commit. + StorageUploadCopyID int64 UploadID int64 CopyIndex int CommitTransactionID string @@ -375,6 +405,7 @@ type ResetRejectedUploadCopyCommitInput struct { type MarkUploadCopyCommittedInput struct { StorageUploadCopyID int64 + RequireEligibleCopy bool UploadID int64 CopyIndex int PieceCID string @@ -467,6 +498,7 @@ type StorageUploadRepository interface { GetUploadProvenance(ctx context.Context, uploadID int64) (*StorageUploadProvenance, error) AppendUploadFailure(ctx context.Context, input AppendUploadFailureInput) error ListCopies(ctx context.Context, uploadID int64) ([]model.StorageUploadCopy, error) + CountCurrentGenerationCopySlots(ctx context.Context, uploadID int64) (int, error) ListReadableCommittedCopies(ctx context.Context, uploadID int64) ([]ReadableStorageCopy, error) HasReadableCommittedCopy(ctx context.Context, uploadID int64) (bool, error) ListBucketStorageHealthSummaries(ctx context.Context, bucketID int64, staleBefore time.Time, affectedVersionCap int) ([]BucketStorageHealthSummary, error) @@ -486,6 +518,8 @@ type StorageUploadRepository interface { CreateUploadCopiesForBindings(ctx context.Context, uploadID int64, copies []UploadCopyBindingInput) error GetUploadCopy(ctx context.Context, uploadID int64, copyIndex int) (*model.StorageUploadCopy, error) GetUploadCopyByID(ctx context.Context, id int64) (*model.StorageUploadCopy, error) + // GetUploadCopyForDataSet addresses one concrete data set generation. + GetUploadCopyForDataSet(ctx context.Context, uploadID, storageDataSetID int64) (*model.StorageUploadCopy, error) AcquireUploadTask(ctx context.Context, input AcquireUploadTaskInput) error AcquireReplicaRepairItem(ctx context.Context, input AcquireReplicaRepairItemInput) (*ReplicaRepairItem, error) NextIncompleteCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageUploadCopy, error) @@ -497,7 +531,7 @@ type StorageUploadRepository interface { MarkUploadCopyCommitting(ctx context.Context, input MarkUploadCopyCommittingInput) error ResetRejectedUploadCopyCommit(ctx context.Context, input ResetRejectedUploadCopyCommitInput) error MarkUploadCopyCommitted(ctx context.Context, input MarkUploadCopyCommittedInput) error - MarkUploadCopyFailed(ctx context.Context, uploadID int64, copyIndex int, lastError string) error + MarkUploadCopyFailed(ctx context.Context, input MarkUploadCopyFailedInput) error BindReadableUploadForContent(ctx context.Context, input BindReadableUploadInput) ([]ObjectVersionRef, error) BindReadableUploadForVersion(ctx context.Context, input BindReadableUploadForVersionInput) ([]ObjectVersionRef, error) FinalizeUploadIfTargetCopiesMet(ctx context.Context, input FinalizeUploadInput) (bool, []ObjectVersionRef, error) @@ -516,6 +550,130 @@ type BucketACLSnapshot struct { ACL []byte `bun:"acl"` } +// StorageReplacementRepository owns operator-approved provider replacement: +// its state machine, the bounded migration cursor, and the retirement safety +// gate. Every state change is a compare-and-set so a superseded or stale caller +// is refused rather than silently applied. +type StorageReplacementRepository interface { + // Authorize records one confirmed replacement, creates the target data set + // generation, supersedes any earlier replacement of the same source, and + // queues the migration coordinator, all in one transaction. + Authorize(ctx context.Context, input AuthorizeReplacementInput) (*storagereplacement.Replacement, bool, error) + // Retry resumes failed or cleanup-attention work on the same approved + // target. Choosing a different provider requires a new authorization. + Retry(ctx context.Context, input RetryReplacementInput) (*storagereplacement.Replacement, error) + + GetByID(ctx context.Context, id int64) (*storagereplacement.Replacement, error) + GetByClientRequestID(ctx context.Context, bucketID int64, clientRequestID string) (*storagereplacement.Replacement, error) + ListForBucket(ctx context.Context, bucketID int64, limit int) ([]storagereplacement.Replacement, error) + GetActiveForDataSet(ctx context.Context, dataSetID int64) (*storagereplacement.Replacement, error) + // HeldItemCopyID reports the target copy the coordinator is writing right + // now, or zero when it holds no item. + HeldItemCopyID(ctx context.Context, replacementID int64) (int64, error) + // HasInProgressForDataSet reports whether recovery must leave this data set + // alone. Terminally failed work does not count, so a stuck slot can still + // repair in place. + HasInProgressForDataSet(ctx context.Context, dataSetID int64) (bool, error) + ListActive(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) + ListSupersededCleanupCandidates(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) + + // Activate makes the target the write target and marks the source draining + // in one transaction. It touches a fixed number of rows regardless of how + // much history the bucket holds. + Activate(ctx context.Context, replacementID int64) error + // SeedMigrationBatch inserts one bounded batch of migration work and + // advances the cursor. done reports that the whole history has been scanned. + SeedMigrationBatch(ctx context.Context, replacementID int64, limit int) (inserted int, done bool, err error) + NextExecutableItem(ctx context.Context, replacementID int64) (*storagereplacement.Item, error) + // AcquireItem re-derives a consistent snapshot and revalidates the worker + // claim. No provider call may start before it returns. + AcquireItem(ctx context.Context, input AcquireReplacementItemInput) (*ReplacementItemSnapshot, error) + AttachTargetCopy(ctx context.Context, input AttachReplacementTargetCopyInput) (*model.StorageUploadCopy, error) + MarkItemCopied(ctx context.Context, itemID int64) error + MarkItemWaitingSource(ctx context.Context, itemID int64, lastError string) error + + MarkMigrating(ctx context.Context, replacementID int64) error + MarkWaiting(ctx context.Context, replacementID int64, reason storagereplacement.WaitReason) error + MarkFailed(ctx context.Context, replacementID int64, reason *storagereplacement.FailureReason, lastError string) error + MarkCleanupAttention(ctx context.Context, replacementID int64, lastError string) error + BeginRetirement(ctx context.Context, replacementID int64) error + RecordTerminationEpoch(ctx context.Context, input RecordTerminationEpochInput) error + RecordAbandonedTerminationEpoch(ctx context.Context, input RecordTerminationEpochInput) error + CompleteAbandonedTargetTermination(ctx context.Context, replacementID int64, observedAt time.Time) error + // EvaluateRetirementGate reports every blocker by name so the API and UI can + // explain why a source is still held. + EvaluateRetirementGate(ctx context.Context, replacementID int64, observedEpoch *int64) (RetirementGate, error) + // CompleteRetirement re-runs the whole gate inside its own transaction and + // refuses premature completion even when called outside the worker. + CompleteRetirement(ctx context.Context, replacementID int64, observedEpoch int64) error + + // CountAbandonedTargetSoleCopies and RetireAbandonedTarget clean up a target + // a later confirmation replaced. They retire the opposite generation from + // CompleteRetirement and never change the replacement record. + CountAbandonedTargetSoleCopies(ctx context.Context, targetDataSetID int64) (int, error) + RetireAbandonedTarget(ctx context.Context, replacementID int64) error +} + +// AuthorizeReplacementInput is one operator confirmation. +type AuthorizeReplacementInput struct { + BucketID int64 + SourceDataSetID int64 + SelectionMode storagereplacement.SelectionMode + TargetProviderID types.OnChainID + ClientRequestID string + MaxRetries int +} + +type RetryReplacementInput struct { + ReplacementID int64 + MaxRetries int +} + +type AcquireReplacementItemInput struct { + ReplacementID int64 + ItemID int64 + TaskID int64 + TaskClaimedAt time.Time +} + +// ReplacementItemSnapshot is the consistent view one migration item needs. +type ReplacementItemSnapshot struct { + Replacement storagereplacement.Replacement + Item storagereplacement.Item + Source model.StorageDataSet + Target model.StorageDataSet + Upload model.StorageUpload + Version model.ObjectVersion +} + +type AttachReplacementTargetCopyInput struct { + ReplacementID int64 + ItemID int64 + UploadID int64 +} + +type RecordTerminationEpochInput struct { + ReplacementID int64 + TxHash string + Epoch int64 +} + +// RetirementGate reports each safety predicate separately so a recoverable +// block can wait while a structural one raises operator attention. +type RetirementGate struct { + CoverageGaps int + SourceWrites int + WaitingItems int + SlotOwned bool + EpochReached bool + TerminationEpoch *int64 + // Blockers names the failing predicates in evaluation order. + Blockers []string +} + +// Passed reports whether every predicate is satisfied. +func (g RetirementGate) Passed() bool { return len(g.Blockers) == 0 } + // TaskRepository defines persistence operations for Task entities. type TaskRepository interface { Create(ctx context.Context, task *model.Task) error @@ -523,6 +681,9 @@ type TaskRepository interface { // completed row with the supplied payload. Active, failed, exhausted, and // cancelled rows are left unchanged. EnsureRecurring(ctx context.Context, task *model.Task) (bool, error) + // ResumeCoordinator revives a singleton coordinator on an operator's + // request, including one that exhausted its retries or failed. + ResumeCoordinator(ctx context.Context, task *model.Task) (bool, error) GetByID(ctx context.Context, id int64) (*model.Task, error) GetByIdempotencyKey(ctx context.Context, idempotencyKey string) (*model.Task, error) HasActiveByIdempotencyKey(ctx context.Context, idempotencyKey string) (bool, error) diff --git a/internal/db/repository/repos.go b/internal/db/repository/repos.go index 2119770..1d87dc2 100644 --- a/internal/db/repository/repos.go +++ b/internal/db/repository/repos.go @@ -16,6 +16,7 @@ type Repositories struct { S3Accounts S3AccountRepository Objects ObjectRepository Uploads StorageUploadRepository + Replacements StorageReplacementRepository StorageCleanup StorageCleanupRepository Tasks TaskRepository CacheEvictions CacheEvictionRepository @@ -33,6 +34,7 @@ func NewRepositories(db bun.IDB) *Repositories { S3Accounts: &BunS3AccountRepo{db: db}, Objects: &BunObjectRepo{db: db}, Uploads: &BunStorageUploadRepo{db: db}, + Replacements: &BunStorageReplacementRepo{db: db}, StorageCleanup: &BunStorageCleanupRepo{db: db}, Tasks: &BunTaskRepo{db: db}, CacheEvictions: &BunCacheEvictionRepo{db: db}, diff --git a/internal/db/repository/storage_cleanup_repo.go b/internal/db/repository/storage_cleanup_repo.go index 206493f..e656b0a 100644 --- a/internal/db/repository/storage_cleanup_repo.go +++ b/internal/db/repository/storage_cleanup_repo.go @@ -5,9 +5,11 @@ import ( "database/sql" "errors" "fmt" + "slices" "time" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" "github.com/uptrace/bun" ) @@ -174,6 +176,45 @@ func (r *BunStorageCleanupRepo) DeleteUploadProvenanceIfUnreferenced(ctx context if hasRefs { return nil } + // Replacement items retain upload provenance while they are executable. + // Settle and remove them explicitly so the RESTRICT foreign key remains a + // guard against bypassing the replacement progress transaction. + var items []storagereplacement.Item + if err := db.NewRaw( + `UPDATE storage_replacement_items SET updated_at = updated_at WHERE upload_id = ? RETURNING *`, + uploadID, + ).Scan(ctx, &items); err != nil && err != sql.ErrNoRows { + return fmt.Errorf("locking replacement items before deleting provenance: %w", err) + } + lockedReplacements := make(map[int64]struct{}, len(items)) + replacementIDs := make([]int64, 0, len(items)) + for i := range items { + if _, ok := lockedReplacements[items[i].ReplacementID]; ok { + continue + } + lockedReplacements[items[i].ReplacementID] = struct{}{} + replacementIDs = append(replacementIDs, items[i].ReplacementID) + } + slices.Sort(replacementIDs) + for _, replacementID := range replacementIDs { + if _, err := lockReplacementByID(ctx, db, replacementID); err != nil { + return err + } + } + for i := range items { + item := &items[i] + if err := settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled); err != nil { + return err + } + } + if len(items) > 0 { + if _, err := db.NewDelete(). + Model((*storagereplacement.Item)(nil)). + Where("upload_id = ?", uploadID). + Exec(ctx); err != nil { + return fmt.Errorf("deleting settled replacement items: %w", err) + } + } if _, err := db.NewDelete(). Model((*model.StorageUpload)(nil)). Where("id = ?", uploadID). @@ -185,12 +226,7 @@ func (r *BunStorageCleanupRepo) DeleteUploadProvenanceIfUnreferenced(ctx context } func (r *BunStorageCleanupRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) error { - if db, ok := r.db.(*bun.DB); ok { - return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - return fn(tx) - }) - } - return fn(r.db) + return runMaybeTx(ctx, r.db, fn) } func storageCleanupCopyUpdateResult(res sql.Result, err error, op string) error { diff --git a/internal/db/repository/storage_health_sql.go b/internal/db/repository/storage_health_sql.go index c314696..2a04f47 100644 --- a/internal/db/repository/storage_health_sql.go +++ b/internal/db/repository/storage_health_sql.go @@ -59,17 +59,39 @@ func storageHealthSQLLiteralList(values ...string) string { return strings.Join(out, ", ") } -func storageHealthSQLLiteral(value string) string { - switch value { - case string(model.StorageDataSetStatusReady), +// Enum values share spellings across domains, so the allowlist is a set rather +// than a switch. +var storageHealthSQLLiterals = func() map[string]struct{} { + values := []string{ + string(model.StorageDataSetStatusPending), + string(model.StorageDataSetStatusCreating), + string(model.StorageDataSetStatusReady), + string(model.StorageDataSetStatusFailed), + string(model.StorageDataSetStatusUnavailable), string(model.StorageDataSetStatusDraining), + string(model.StorageDataSetStatusRetired), + string(model.StorageUploadCopyStatusPending), + string(model.StorageUploadCopyStatusPieceReady), + string(model.StorageUploadCopyStatusCommitting), string(model.StorageUploadCopyStatusCommitted), + string(model.StorageUploadCopyStatusFailed), string(observability.StatusAvailable), string(observability.StatusDegraded), string(observability.StatusUnavailable), - string(observability.StatusUnknown): - return "'" + strings.ReplaceAll(value, "'", "''") + "'" - default: + string(observability.StatusUnknown), + } + set := make(map[string]struct{}, len(values)) + for _, value := range values { + set[value] = struct{}{} + } + return set +}() + +// Only enum values that already exist as domain constants may be inlined into +// raw SQL. The panic guards against interpolating caller-supplied text. +func storageHealthSQLLiteral(value string) string { + if _, ok := storageHealthSQLLiterals[value]; !ok { panic("unsupported storage health SQL literal") } + return "'" + strings.ReplaceAll(value, "'", "''") + "'" } diff --git a/internal/db/repository/storage_health_sql_internal_test.go b/internal/db/repository/storage_health_sql_internal_test.go new file mode 100644 index 0000000..684dfe8 --- /dev/null +++ b/internal/db/repository/storage_health_sql_internal_test.go @@ -0,0 +1,64 @@ +package repository + +import ( + "testing" + + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/observability" +) + +// Every status these queries can encounter must be inlinable. A value missing +// from the allowlist panics when the query is built, which would take down the +// process at request time rather than failing a query. +func TestStorageHealthSQLLiteralAcceptsEveryDomainStatus(t *testing.T) { + dataSetStatuses := []model.StorageDataSetStatus{ + model.StorageDataSetStatusPending, + model.StorageDataSetStatusCreating, + model.StorageDataSetStatusReady, + model.StorageDataSetStatusFailed, + model.StorageDataSetStatusUnavailable, + model.StorageDataSetStatusDraining, + model.StorageDataSetStatusRetired, + } + copyStatuses := []model.StorageUploadCopyStatus{ + model.StorageUploadCopyStatusPending, + model.StorageUploadCopyStatusPieceReady, + model.StorageUploadCopyStatusCommitting, + model.StorageUploadCopyStatusCommitted, + model.StorageUploadCopyStatusFailed, + } + observationStatuses := []observability.Status{ + observability.StatusAvailable, + observability.StatusDegraded, + observability.StatusUnavailable, + observability.StatusUnknown, + } + + values := make([]string, 0, len(dataSetStatuses)+len(copyStatuses)+len(observationStatuses)) + for _, status := range dataSetStatuses { + values = append(values, string(status)) + } + for _, status := range copyStatuses { + values = append(values, string(status)) + } + for _, status := range observationStatuses { + values = append(values, string(status)) + } + + for _, value := range values { + t.Run(value, func(t *testing.T) { + if got := storageHealthSQLLiteral(value); got != "'"+value+"'" { + t.Fatalf("storageHealthSQLLiteral(%q) = %q, want %q", value, got, "'"+value+"'") + } + }) + } +} + +func TestStorageHealthSQLLiteralRejectsUnknownValue(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatal("storageHealthSQLLiteral accepted an unknown value, want panic") + } + }() + storageHealthSQLLiteral("'; DROP TABLE storage_data_sets; --") +} diff --git a/internal/db/repository/storage_readable_copy_sql.go b/internal/db/repository/storage_readable_copy_sql.go new file mode 100644 index 0000000..d091ff2 --- /dev/null +++ b/internal/db/repository/storage_readable_copy_sql.go @@ -0,0 +1,64 @@ +package repository + +import "fmt" + +// A readable committed copy is one this node recorded as committed with a +// complete retrieval identity, on a data set that still serves reads. It +// reflects local bookkeeping and is not a live data-safety guarantee. +func readableCommittedCopyPredicateSQL(copyAlias, dataSetAlias string) string { + return readableCommittedCopyPredicateWithDataSetStatusSQL( + copyAlias, + dataSetAlias, + fmt.Sprintf("%s.status IN (%s)", dataSetAlias, storageHealthReadyDataSetStatusListSQL()), + ) +} + +// dataSetStatusCondition widens the data set status test for callers that must +// also treat a data set as its own readable source while it is being finalized. +func readableCommittedCopyPredicateWithDataSetStatusSQL(copyAlias, dataSetAlias, dataSetStatusCondition string) string { + return fmt.Sprintf(`%[1]s.status = %[3]s + AND %[1]s.storage_data_set_id IS NOT NULL + AND %[1]s.provider_id IS NOT NULL AND %[1]s.provider_id <> '' + AND %[2]s.data_set_id IS NOT NULL AND %[2]s.data_set_id <> '' + AND %[4]s + AND %[1]s.piece_id IS NOT NULL AND %[1]s.piece_id <> '' + AND %[1]s.retrieval_url IS NOT NULL AND %[1]s.retrieval_url <> ''`, + copyAlias, + dataSetAlias, + storageHealthCommittedCopyStatusSQL(), + dataSetStatusCondition, + ) +} + +// A replica slot can hold several data set generations during a provider +// replacement, so a lookup that knows only the upload and the slot must resolve +// to the generation that currently owns the slot. A copy with no data set yet +// belongs to the slot until one is assigned. +func currentGenerationCopySQL(copyAlias string) string { + return fmt.Sprintf(`( + %[1]s.storage_data_set_id IS NULL + OR EXISTS ( + SELECT 1 FROM storage_data_sets AS current_slot_data_set + WHERE current_slot_data_set.id = %[1]s.storage_data_set_id + AND current_slot_data_set.is_current + ) + )`, copyAlias) +} + +// Durability is measured in logical replica slots. One slot can hold several +// physical data set generations while a provider replacement is in flight, and +// those generations must never count as separate replicas. +func distinctReadableSlotCountSQL(copyAlias, dataSetAlias, uploadIDExpr string) string { + return fmt.Sprintf(`( + SELECT COUNT(DISTINCT %[2]s.copy_index) + FROM storage_upload_copies AS %[1]s + JOIN storage_data_sets AS %[2]s ON %[2]s.id = %[1]s.storage_data_set_id + WHERE %[1]s.upload_id = %[3]s + AND %[4]s + )`, + copyAlias, + dataSetAlias, + uploadIDExpr, + readableCommittedCopyPredicateSQL(copyAlias, dataSetAlias), + ) +} diff --git a/internal/db/repository/storage_replacement_gate.go b/internal/db/repository/storage_replacement_gate.go new file mode 100644 index 0000000..059e3e9 --- /dev/null +++ b/internal/db/repository/storage_replacement_gate.go @@ -0,0 +1,367 @@ +package repository + +import ( + "context" + "fmt" + "time" + + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +// taskPayloadCopyIDSQL extracts the concrete copy a task targets. Tasks queued +// before copy ids were recorded yield NULL and are handled separately. +func taskPayloadCopyIDSQL(dialectName dialect.Name) func(alias string) string { + if dialectName == dialect.PG { + return func(alias string) string { + return "CAST(" + alias + ".payload ->> 'storage_upload_copy_id' AS BIGINT)" + } + } + return func(alias string) string { + return "CAST(json_extract(" + alias + ".payload, '$.storage_upload_copy_id') AS INTEGER)" + } +} + +// Blocker names are stable so the API and UI can explain a held source without +// inventing their own vocabulary. +const ( + retirementBlockerCoverage = "coverage" + retirementBlockerSourceWrites = "source_writes" + retirementBlockerWaitingItems = "waiting_items" + retirementBlockerSlotOwnership = "slot_ownership" + retirementBlockerEpoch = "termination_epoch" +) + +// EvaluateRetirementGate answers one question: can this source stop existing +// without losing anything? Pass a nil observedEpoch to evaluate everything +// except the epoch, which must be read outside a transaction. +func (r *BunStorageReplacementRepo) EvaluateRetirementGate(ctx context.Context, replacementID int64, observedEpoch *int64) (RetirementGate, error) { + return evaluateRetirementGate(ctx, r.db, replacementID, observedEpoch) +} + +func evaluateRetirementGate(ctx context.Context, db bun.IDB, replacementID int64, observedEpoch *int64) (RetirementGate, error) { + gate := RetirementGate{} + row := new(storagereplacement.Replacement) + if err := db.NewSelect().Model(row).Where("id = ?", replacementID).Scan(ctx); err != nil { + return gate, fmt.Errorf("loading provider replacement for retirement: %w", err) + } + gate.TerminationEpoch = row.TerminationEpoch + + uploads := &BunStorageUploadRepo{db: db} + source, err := uploads.GetDataSetBindingByID(ctx, row.SourceDataSetID) + if err != nil { + return gate, err + } + target, err := uploads.GetDataSetBindingByID(ctx, row.TargetDataSetID) + if err != nil { + return gate, err + } + if source == nil || target == nil { + return gate, fmt.Errorf("loading retirement data sets: %w", ErrNotFound) + } + + // G1: every still-accessible version stored on the source must be readable + // on the generation that replaced it. + gaps, err := countRetirementCoverageGaps(ctx, db, source.ID, target.ID) + if err != nil { + return gate, err + } + gate.CoverageGaps = gaps + if gaps > 0 { + gate.Blockers = append(gate.Blockers, retirementBlockerCoverage) + } + + // G2: nothing may still be writing to the source. + writes, err := countRetirementSourceWrites(ctx, db, row.BucketID, source.ID) + if err != nil { + return gate, err + } + gate.SourceWrites = writes + if writes > 0 { + gate.Blockers = append(gate.Blockers, retirementBlockerSourceWrites) + } + + // G3: no migration item may still be owed. + waiting, err := db.NewSelect(). + Model((*storagereplacement.Item)(nil)). + Where("replacement_id = ?", row.ID). + Where("status IN (?, ?, ?)", + storagereplacement.ItemStatusPending, + storagereplacement.ItemStatusRunning, + storagereplacement.ItemStatusWaitingSource). + Count(ctx) + if err != nil { + return gate, fmt.Errorf("counting outstanding replacement items: %w", err) + } + gate.WaitingItems = waiting + if waiting > 0 { + gate.Blockers = append(gate.Blockers, retirementBlockerWaitingItems) + } + + // G4: the slot must have genuinely moved on. This one is structural: if it + // fails, something reordered the generations and retrying will not help. + gate.SlotOwned = !source.IsCurrent && + target.IsCurrent && + target.Status == model.StorageDataSetStatusReady && + target.Generation > source.Generation + if !gate.SlotOwned { + gate.Blockers = append(gate.Blockers, retirementBlockerSlotOwnership) + } + + // G5: the chain must have reached the recorded end of term. + gate.EpochReached = row.TerminationEpoch != nil && observedEpoch != nil && *observedEpoch >= *row.TerminationEpoch + if observedEpoch != nil && !gate.EpochReached { + gate.Blockers = append(gate.Blockers, retirementBlockerEpoch) + } + return gate, nil +} + +// A gap is an upload the source still holds, whose content some live version +// still needs, and which the target cannot serve. +func countRetirementCoverageGaps(ctx context.Context, db bun.IDB, sourceDataSetID, targetDataSetID int64) (int, error) { + var count int + if err := db.NewRaw(retirementCoverageGapsSQL(), sourceDataSetID, false, targetDataSetID).Scan(ctx, &count); err != nil { + return 0, fmt.Errorf("counting retirement coverage gaps: %w", err) + } + return count, nil +} + +// retirementCoverageGapsSQL is shared with the query-plan regression test so +// the tested plan cannot drift away from the production retirement gate. +func retirementCoverageGapsSQL() string { + return fmt.Sprintf(`SELECT COUNT(*) FROM storage_uploads AS retiring_upload + WHERE EXISTS ( + SELECT 1 FROM storage_upload_copies AS source_copy + WHERE source_copy.upload_id = retiring_upload.id + AND source_copy.storage_data_set_id = ? + AND source_copy.status = %[1]s + ) + AND EXISTS ( + SELECT 1 FROM object_versions AS live_version + WHERE %[2]s + AND live_version.is_delete_marker = ? + ) + AND NOT EXISTS ( + SELECT 1 FROM storage_upload_copies AS target_copy + JOIN storage_data_sets AS target_data_set ON target_data_set.id = target_copy.storage_data_set_id + WHERE target_copy.upload_id = retiring_upload.id + AND target_copy.storage_data_set_id = ? + AND %[3]s + )`, + storageHealthCommittedCopyStatusSQL(), + objectVersionReferencesStorageUploadSQL("live_version", "retiring_upload"), + readableCommittedCopyPredicateSQL("target_copy", "target_data_set"), + ) +} + +// Source writes are counted from both directions: copy rows that are still +// mid-transfer, and tasks that could still produce one. A running task with no +// recorded copy predates copy-id addressing, so it is treated as a possible +// source write; queued work is not, because it will resolve to the generation +// that owns the slot by the time it runs. +func countRetirementSourceWrites(ctx context.Context, db bun.IDB, bucketID, sourceDataSetID int64) (int, error) { + inFlight, err := db.NewSelect(). + Model((*model.StorageUploadCopy)(nil)). + Where("storage_data_set_id = ?", sourceDataSetID). + Where("status IN (?, ?, ?)", + model.StorageUploadCopyStatusPending, + model.StorageUploadCopyStatusPieceReady, + model.StorageUploadCopyStatusCommitting). + Count(ctx) + if err != nil { + return 0, fmt.Errorf("counting in-flight source copies: %w", err) + } + + copyIDExpr := taskPayloadCopyIDSQL(db.Dialect().Name()) + var bound int + if err := db.NewRaw(`SELECT COUNT(*) FROM tasks AS active_task + JOIN storage_upload_copies AS bound_copy + ON bound_copy.id = `+copyIDExpr("active_task")+` + WHERE active_task.type = ? + AND active_task.status IN (?, ?, ?, ?) + AND bound_copy.storage_data_set_id = ?`, + model.TaskTypeUpload, + model.TaskStatusQueued, model.TaskStatusScheduled, model.TaskStatusWaiting, model.TaskStatusRunning, + sourceDataSetID, + ).Scan(ctx, &bound); err != nil { + return 0, fmt.Errorf("counting source-bound upload tasks: %w", err) + } + + var legacyRunning int + if err := db.NewRaw(`SELECT COUNT(*) FROM tasks AS legacy_task + WHERE legacy_task.type = ? + AND legacy_task.status = ? + AND legacy_task.ref_type = ? + AND `+copyIDExpr("legacy_task")+` IS NULL + AND EXISTS ( + SELECT 1 FROM object_versions AS task_version + WHERE task_version.version_id = legacy_task.ref_version_id + AND task_version.bucket_id = ? + )`, + model.TaskTypeUpload, model.TaskStatusRunning, "object", bucketID, + ).Scan(ctx, &legacyRunning); err != nil { + return 0, fmt.Errorf("counting legacy running upload tasks: %w", err) + } + return inFlight + bound + legacyRunning, nil +} + +// CountAbandonedTargetSoleCopies reports how many uploads would lose their only +// readable copy if this generation's service ended. An abandoned target holds +// partially migrated data that the retiring source should still have, so the +// answer is normally zero; anything else means terminating it would destroy the +// last copy of something. +func (r *BunStorageReplacementRepo) CountAbandonedTargetSoleCopies(ctx context.Context, targetDataSetID int64) (int, error) { + predicate := readableCommittedCopyPredicateSQL("abandoned_copy", "abandoned_data_set") + elsewhere := readableCommittedCopyPredicateSQL("other_copy", "other_data_set") + query := fmt.Sprintf(`SELECT COUNT(*) + FROM storage_upload_copies AS abandoned_copy + JOIN storage_data_sets AS abandoned_data_set ON abandoned_data_set.id = abandoned_copy.storage_data_set_id + JOIN storage_uploads AS abandoned_upload ON abandoned_upload.id = abandoned_copy.upload_id + WHERE abandoned_copy.storage_data_set_id = ? + AND %[1]s + AND EXISTS ( + SELECT 1 FROM object_versions AS live_version + WHERE %[2]s + AND live_version.is_delete_marker = ? + ) + AND NOT EXISTS ( + SELECT 1 FROM storage_upload_copies AS other_copy + JOIN storage_data_sets AS other_data_set ON other_data_set.id = other_copy.storage_data_set_id + WHERE other_copy.upload_id = abandoned_copy.upload_id + AND other_copy.storage_data_set_id <> ? + AND %[3]s + )`, + predicate, + objectVersionReferencesStorageUploadSQL("live_version", "abandoned_upload"), + elsewhere, + ) + var count int + if err := r.db.NewRaw(query, targetDataSetID, false, targetDataSetID).Scan(ctx, &count); err != nil { + return 0, fmt.Errorf("counting abandoned target sole copies: %w", err) + } + return count, nil +} + +// RetireAbandonedTarget marks an abandoned generation retired. It never touches +// the replacement record, which stays superseded, and it refuses a generation +// that still owns its slot. +func (r *BunStorageReplacementRepo) RetireAbandonedTarget(ctx context.Context, replacementID int64) error { + return r.retireAbandonedTarget(ctx, replacementID, nil) +} + +// CompleteAbandonedTargetTermination retires the superseded target and records +// epoch observation in the same transaction. +func (r *BunStorageReplacementRepo) CompleteAbandonedTargetTermination( + ctx context.Context, + replacementID int64, + observedAt time.Time, +) error { + if observedAt.IsZero() { + return fmt.Errorf("completing abandoned target termination: %w", ErrInvalidInput) + } + return r.retireAbandonedTarget(ctx, replacementID, &observedAt) +} + +func (r *BunStorageReplacementRepo) retireAbandonedTarget( + ctx context.Context, + replacementID int64, + observedAt *time.Time, +) error { + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + row, err := lockReplacementByID(ctx, db, replacementID) + if err != nil { + return err + } + sole, err := (&BunStorageReplacementRepo{db: db}).CountAbandonedTargetSoleCopies(ctx, row.TargetDataSetID) + if err != nil { + return err + } + if sole > 0 { + return fmt.Errorf("retiring abandoned target of replacement %d holds %d sole copies: %w", + replacementID, sole, storagereplacement.ErrPrematureComplete) + } + if observedAt != nil { + if row.Status != storagereplacement.StatusSuperseded || row.AbandonedTerminationEpoch == nil { + return fmt.Errorf("completing abandoned target termination: %w", ErrConflict) + } + } + res, err := db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusRetired). + Set("updated_at = ?", time.Now()). + Where("id = ? AND is_current = ?", row.TargetDataSetID, false). + Where("status <> ?", model.StorageDataSetStatusRetired). + Exec(ctx) + if err != nil { + return fmt.Errorf("retiring abandoned target: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("retiring abandoned target: %w", ErrConflict) + } + if observedAt != nil { + res, err = db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("abandoned_termination_observed_at = ?", *observedAt). + Set("updated_at = ?", *observedAt). + Where("id = ? AND status = ?", replacementID, storagereplacement.StatusSuperseded). + Where("abandoned_termination_epoch IS NOT NULL"). + Exec(ctx) + if err != nil { + return fmt.Errorf("recording abandoned target termination observation: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("recording abandoned target termination observation: %w", ErrConflict) + } + } + return nil + }) +} + +// CompleteRetirement re-runs every predicate inside its own transaction, so a +// caller outside the worker cannot retire a source that is still needed. +func (r *BunStorageReplacementRepo) CompleteRetirement(ctx context.Context, replacementID int64, observedEpoch int64) error { + if replacementID <= 0 { + return fmt.Errorf("completing provider replacement: %w", ErrInvalidInput) + } + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + row, err := lockReplacementByID(ctx, db, replacementID) + if err != nil { + return err + } + if row.Status == storagereplacement.StatusCompleted { + return nil + } + gate, err := evaluateRetirementGate(ctx, db, replacementID, &observedEpoch) + if err != nil { + return err + } + if !gate.Passed() { + return fmt.Errorf("completing provider replacement %d blocked by %v: %w", + replacementID, gate.Blockers, storagereplacement.ErrPrematureComplete) + } + now := time.Now() + res, err := db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusRetired). + Set("updated_at = ?", now). + Where("id = ? AND is_current = ?", row.SourceDataSetID, false). + Where("status <> ?", model.StorageDataSetStatusRetired). + Exec(ctx) + if err != nil { + return fmt.Errorf("retiring replacement source: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("retiring replacement source: %w", ErrConflict) + } + return transitionReplacement(ctx, db, replacementID, + []storagereplacement.Status{storagereplacement.StatusRetiring}, + storagereplacement.StatusCompleted, + func(q *bun.UpdateQuery) *bun.UpdateQuery { + return q.Set("wait_reason = NULL"). + Set("last_error = NULL"). + Set("termination_observed_at = ?", now) + }, now) + }) +} diff --git a/internal/db/repository/storage_replacement_gate_internal_test.go b/internal/db/repository/storage_replacement_gate_internal_test.go new file mode 100644 index 0000000..ce60598 --- /dev/null +++ b/internal/db/repository/storage_replacement_gate_internal_test.go @@ -0,0 +1,55 @@ +package repository + +import ( + "context" + "database/sql" + "path/filepath" + "strings" + "testing" + + synaps3db "github.com/strahe/synaps3/internal/db" + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/sqlitedialect" + _ "modernc.org/sqlite" +) + +func TestRetirementCoverageProductionSQLUsesIndexes(t *testing.T) { + sqldb, err := sql.Open("sqlite", "file:"+filepath.Join(t.TempDir(), "retirement-plan.db")+"?_pragma=foreign_keys(1)") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + db := bun.NewDB(sqldb, sqlitedialect.New()) + t.Cleanup(func() { _ = db.Close() }) + + ctx := context.Background() + if err := synaps3db.RunMigrations(ctx, db); err != nil { + t.Fatalf("RunMigrations: %v", err) + } + rows, err := sqldb.QueryContext(ctx, "EXPLAIN QUERY PLAN "+retirementCoverageGapsSQL(), int64(1), false, int64(2)) + if err != nil { + t.Fatalf("EXPLAIN production retirement coverage query: %v", err) + } + defer func() { _ = rows.Close() }() + + var details []string + for rows.Next() { + var id, parent, notUsed int + var detail string + if err := rows.Scan(&id, &parent, ¬Used, &detail); err != nil { + t.Fatalf("scan query plan: %v", err) + } + details = append(details, detail) + } + if err := rows.Err(); err != nil { + t.Fatalf("read query plan: %v", err) + } + plan := strings.Join(details, "\n") + for _, index := range []string{ + "idx_storage_upload_copies_status_data_set_upload", + "idx_object_versions_storage_upload", + } { + if !strings.Contains(plan, index) { + t.Fatalf("production retirement coverage plan =\n%s\nwant %s", plan, index) + } + } +} diff --git a/internal/db/repository/storage_replacement_items.go b/internal/db/repository/storage_replacement_items.go new file mode 100644 index 0000000..f41440f --- /dev/null +++ b/internal/db/repository/storage_replacement_items.go @@ -0,0 +1,580 @@ +package repository + +import ( + "context" + "database/sql" + "fmt" + "time" + + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/uptrace/bun" +) + +// SeedMigrationBatch scans one bounded window of the bucket's upload history. +// The cursor advances over every upload it examined, not only the ones it +// inserted, so a window full of ineligible uploads still makes progress. +func (r *BunStorageReplacementRepo) SeedMigrationBatch(ctx context.Context, replacementID int64, limit int) (int, bool, error) { + if replacementID <= 0 || limit <= 0 { + return 0, false, fmt.Errorf("seeding replacement migration: %w", ErrInvalidInput) + } + row, err := r.GetByID(ctx, replacementID) + if err != nil { + return 0, false, err + } + if row == nil { + return 0, false, fmt.Errorf("provider replacement %d: %w", replacementID, ErrNotFound) + } + if row.SeedingComplete { + return 0, true, nil + } + + // Deciding which uploads need migrating is a read over bucket history. It + // runs outside the write transaction so a replacement never holds SQLite's + // single writer while it scans. + eligible, cursor, scanned, err := r.scanMigrationCandidates(ctx, row, limit) + if err != nil { + return 0, false, err + } + if scanned == 0 { + return 0, true, markSeedingComplete(ctx, r.db, row.ID, row.SeedCursorUploadID) + } + + inserted := 0 + done := scanned < limit + err = runMaybeTx(ctx, r.db, func(db bun.IDB) error { + // Re-lock and re-check the cursor: another pass may have advanced it + // while this one was reading. + locked, err := lockReplacementByID(ctx, db, replacementID) + if err != nil { + return err + } + if locked.SeedCursorUploadID != row.SeedCursorUploadID { + return fmt.Errorf("advancing replacement migration cursor: %w", ErrConflict) + } + if len(eligible) > 0 { + items := make([]storagereplacement.Item, 0, len(eligible)) + now := time.Now() + for _, uploadID := range eligible { + items = append(items, storagereplacement.Item{ + ReplacementID: row.ID, + UploadID: uploadID, + Status: storagereplacement.ItemStatusPending, + CreatedAt: now, + UpdatedAt: now, + }) + } + res, err := db.NewInsert(). + Model(&items). + On("CONFLICT (replacement_id, upload_id) DO NOTHING"). + Exec(ctx) + if err != nil { + return fmt.Errorf("seeding replacement migration items: %w", err) + } + affected, _ := res.RowsAffected() + inserted = int(affected) + } + + q := db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("seed_cursor_upload_id = ?", cursor). + Set("items_total = items_total + ?", inserted). + Set("updated_at = ?", time.Now()). + Where("id = ? AND seed_cursor_upload_id = ?", row.ID, row.SeedCursorUploadID) + if done { + q = q.Set("seeding_complete = ?", true) + } + res, err := q.Exec(ctx) + if err != nil { + return fmt.Errorf("advancing replacement migration cursor: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("advancing replacement migration cursor: %w", ErrConflict) + } + return nil + }) + if err != nil { + return 0, false, err + } + return inserted, done, nil +} + +// scanMigrationCandidates reads one bounded window of upload history and reports +// which uploads still need a copy on the new provider. The cursor advances over +// every upload examined, not only the eligible ones, so a window full of +// ineligible uploads still makes progress. +func (r *BunStorageReplacementRepo) scanMigrationCandidates( + ctx context.Context, + row *storagereplacement.Replacement, + limit int, +) (eligible []int64, cursor int64, scanned int, err error) { + var candidates []int64 + if err := r.db.NewSelect(). + Model((*model.StorageUpload)(nil)). + Column("id"). + Where("bucket_id = ? AND id > ?", row.BucketID, row.SeedCursorUploadID). + OrderExpr("id ASC"). + Limit(limit). + Scan(ctx, &candidates); err != nil { + return nil, 0, 0, fmt.Errorf("scanning replacement migration candidates: %w", err) + } + if len(candidates) == 0 { + return nil, row.SeedCursorUploadID, 0, nil + } + cursor = candidates[len(candidates)-1] + + // Migration is keyed by stored content, so content shared by many object + // versions is copied once. + // + // Seeding deliberately does not ask whether the retiring generation already + // holds a committed copy. An upload still in flight would answer "no" at this + // instant, commit to the source moments later, and never be revisited once + // the cursor moved past it, leaving the retirement coverage gate blocked + // forever. AcquireItem asks that question instead, at a point where it can + // settle the item either way. + query := fmt.Sprintf(`SELECT candidate.id + FROM storage_uploads AS candidate + WHERE candidate.id IN (?) + AND EXISTS ( + SELECT 1 FROM object_versions AS live_version + WHERE %s + AND live_version.is_delete_marker = ? + ) + ORDER BY candidate.id ASC`, + objectVersionReferencesStorageUploadSQL("live_version", "candidate"), + ) + if err := r.db.NewRaw(query, bun.List(candidates), false). + Scan(ctx, &eligible); err != nil { + return nil, 0, 0, fmt.Errorf("selecting replacement migration items: %w", err) + } + return eligible, cursor, len(candidates), nil +} + +func markSeedingComplete(ctx context.Context, db bun.IDB, replacementID, cursor int64) error { + _, err := db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("seeding_complete = ?", true). + Set("updated_at = ?", time.Now()). + Where("id = ? AND seed_cursor_upload_id = ?", replacementID, cursor). + Exec(ctx) + if err != nil { + return fmt.Errorf("completing replacement migration seeding: %w", err) + } + return nil +} + +// NextExecutableItem prefers work that has never been attempted, and only then +// revisits items that were parked for want of a readable source. +func (r *BunStorageReplacementRepo) NextExecutableItem(ctx context.Context, replacementID int64) (*storagereplacement.Item, error) { + item := new(storagereplacement.Item) + err := r.db.NewSelect(). + Model(item). + Where("replacement_id = ?", replacementID). + Where("status IN (?, ?)", storagereplacement.ItemStatusPending, storagereplacement.ItemStatusRunning). + OrderExpr("id ASC"). + Limit(1). + Scan(ctx) + if err == nil { + return item, nil + } + if err != sql.ErrNoRows { + return nil, fmt.Errorf("selecting next replacement item: %w", err) + } + waiting := new(storagereplacement.Item) + err = r.db.NewSelect(). + Model(waiting). + Where("replacement_id = ?", replacementID). + Where("status = ?", storagereplacement.ItemStatusWaitingSource). + OrderExpr("updated_at ASC, id ASC"). + Limit(1). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting waiting replacement item: %w", err) + } + return waiting, nil +} + +// AcquireItem re-derives every identity the item depends on inside one +// transaction and revalidates the worker's claim, so no provider call can start +// from a stale snapshot. +// +// It also decides, at this moment rather than at seeding time, whether the item +// still needs migrating at all. An item that cannot or need not be migrated is +// settled to a terminal status here; leaving it executable would make the +// coordinator pick it up forever and hold retirement open. +func (r *BunStorageReplacementRepo) AcquireItem(ctx context.Context, input AcquireReplacementItemInput) (*ReplacementItemSnapshot, error) { + if input.ReplacementID <= 0 || input.ItemID <= 0 || input.TaskID <= 0 || input.TaskClaimedAt.IsZero() { + return nil, fmt.Errorf("acquiring replacement item: %w", ErrInvalidInput) + } + var snapshot *ReplacementItemSnapshot + // settled and deferred are reported after the transaction commits, so the + // status this call writes survives; returning an error would roll it back. + settled := false + deferred := false + err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { + item := new(storagereplacement.Item) + err := db.NewRaw( + `UPDATE storage_replacement_items SET updated_at = updated_at WHERE id = ? RETURNING *`, + input.ItemID, + ).Scan(ctx, item) + if err != nil { + if err == sql.ErrNoRows { + return fmt.Errorf("replacement item %d: %w", input.ItemID, ErrNotFound) + } + return fmt.Errorf("locking replacement item: %w", err) + } + if item.ReplacementID != input.ReplacementID { + return fmt.Errorf("replacement item %d belongs to another replacement: %w", item.ID, ErrConflict) + } + if !item.Status.Executable() { + settled = true + return nil + } + // The claim is revalidated before any provider call so a lost lease can + // never race a second worker into the same transfer. + task := new(model.Task) + err = db.NewRaw(`UPDATE tasks + SET status = status + WHERE id = ? + AND status = ? + AND claimed_at = ? + AND lease_until IS NOT NULL + AND lease_until > ? + RETURNING *`, input.TaskID, model.TaskStatusRunning, input.TaskClaimedAt, time.Now()).Scan(ctx, task) + if err != nil { + if err == sql.ErrNoRows { + return ErrTaskClaimLost + } + return fmt.Errorf("locking replacement item task claim: %w", err) + } + + replacement, err := lockReplacementByID(ctx, db, input.ReplacementID) + if err != nil { + return err + } + if !replacement.Status.Active() { + // The replacement itself is finished or superseded; its items are no + // longer this coordinator's business and must not be rewritten. + settled = true + return nil + } + uploads := &BunStorageUploadRepo{db: db} + source, err := uploads.GetDataSetBindingByID(ctx, replacement.SourceDataSetID) + if err != nil { + return err + } + target, err := uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) + if err != nil { + return err + } + if source == nil || target == nil { + return fmt.Errorf("acquiring replacement item: data set: %w", ErrNotFound) + } + if !target.IsCurrent || target.CopyIndex != replacement.CopyIndex || source.CopyIndex != replacement.CopyIndex { + return fmt.Errorf("acquiring replacement item: replica slot changed: %w", ErrConflict) + } + upload, err := uploads.GetByID(ctx, item.UploadID) + if err != nil { + return err + } + if upload == nil || upload.BucketID != replacement.BucketID { + return fmt.Errorf("acquiring replacement item: upload %d: %w", item.UploadID, ErrNotFound) + } + version, err := selectLiveObjectVersionForStorageUpload(ctx, db, upload, nil) + if err != nil { + return err + } + if version == nil { + // Nothing references this content any more, so the new provider does + // not need it. + settled = true + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled) + } + owed, inFlight, err := sourceCopyState(ctx, db, upload.ID, source.ID) + if err != nil { + return err + } + if !owed { + if inFlight { + // The retiring generation is still writing this content. It is not + // copyable yet and must not be cancelled: the write will commit, + // and the coverage gate would then block on content with no item + // behind it. Park it and revisit. + deferred = true + return r.parkItemWaitingSource(ctx, db, item, "the retiring provider has not finished storing this content") + } + // The retiring generation never stored this content and never will, so + // the slot owes the target nothing for it. + settled = true + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCancelled) + } + covered, err := targetHoldsReadableCopy(ctx, db, upload.ID, target.ID) + if err != nil { + return err + } + if covered { + // Already migrated, most likely by an ordinary upload that landed on + // the target after activation. + settled = true + return settleReplacementItem(ctx, db, item, storagereplacement.ItemStatusCopied) + } + if item.Status != storagereplacement.ItemStatusRunning { + if _, err := db.NewUpdate(). + Model((*storagereplacement.Item)(nil)). + Set("status = ?", storagereplacement.ItemStatusRunning). + Set("attempts = attempts + 1"). + Set("updated_at = ?", time.Now()). + Where("id = ?", item.ID). + Exec(ctx); err != nil { + return fmt.Errorf("claiming replacement item: %w", err) + } + item.Status = storagereplacement.ItemStatusRunning + } + snapshot = &ReplacementItemSnapshot{ + Replacement: *replacement, + Item: *item, + Source: *source, + Target: *target, + Upload: *upload, + Version: *version, + } + return nil + }) + if err != nil { + return nil, err + } + if deferred { + return nil, storagereplacement.ErrItemDeferred + } + if settled { + return nil, storagereplacement.ErrItemCancelled + } + return snapshot, nil +} + +func (r *BunStorageReplacementRepo) parkItemWaitingSource(ctx context.Context, db bun.IDB, item *storagereplacement.Item, reason string) error { + _, err := db.NewUpdate(). + Model((*storagereplacement.Item)(nil)). + Set("status = ?", storagereplacement.ItemStatusWaitingSource). + Set("last_error = ?", reason). + Set("updated_at = ?", time.Now()). + Where("id = ?", item.ID). + Where("status NOT IN (?, ?)", storagereplacement.ItemStatusCopied, storagereplacement.ItemStatusCancelled). + Exec(ctx) + if err != nil { + return fmt.Errorf("parking replacement item: %w", err) + } + return nil +} + +// settleReplacementItem moves an item to a terminal status and keeps the +// replacement's progress counter in step. +func settleReplacementItem(ctx context.Context, db bun.IDB, item *storagereplacement.Item, status storagereplacement.ItemStatus) error { + res, err := db.NewUpdate(). + Model((*storagereplacement.Item)(nil)). + Set("status = ?", status). + Set("updated_at = ?", time.Now()). + Where("id = ?", item.ID). + Where("status NOT IN (?, ?)", storagereplacement.ItemStatusCopied, storagereplacement.ItemStatusCancelled). + Exec(ctx) + if err != nil { + return fmt.Errorf("settling replacement item: %w", err) + } + if rows, _ := res.RowsAffected(); rows == 0 { + return nil + } + if status != storagereplacement.ItemStatusCopied { + return nil + } + if _, err := db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("items_copied = items_copied + 1"). + Set("updated_at = ?", time.Now()). + Where("id = ? AND items_copied < items_total", item.ReplacementID). + Exec(ctx); err != nil { + return fmt.Errorf("recording replacement progress: %w", err) + } + return nil +} + +// sourceCopyState answers two different questions that must not be collapsed: +// whether the retiring generation already stored this content, and whether it is +// still in the middle of storing it. Treating "not committed yet" as "never +// stored" cancels work that the coverage gate will later demand. +func sourceCopyState(ctx context.Context, db bun.IDB, uploadID, sourceDataSetID int64) (owed bool, inFlight bool, err error) { + var copies []model.StorageUploadCopy + if err := db.NewSelect(). + Model(&copies). + Where("upload_id = ? AND storage_data_set_id = ?", uploadID, sourceDataSetID). + Scan(ctx); err != nil { + return false, false, fmt.Errorf("checking retiring generation copy: %w", err) + } + for i := range copies { + switch copies[i].Status { + case model.StorageUploadCopyStatusCommitted: + return true, false, nil + case model.StorageUploadCopyStatusPending, + model.StorageUploadCopyStatusPieceReady, + model.StorageUploadCopyStatusCommitting: + inFlight = true + } + } + return false, inFlight, nil +} + +// targetHoldsReadableCopy answers the same question the retirement coverage gate +// asks, so an item is never left owing work the gate already considers done. +func targetHoldsReadableCopy(ctx context.Context, db bun.IDB, uploadID, targetDataSetID int64) (bool, error) { + query := fmt.Sprintf(`SELECT COUNT(*) + FROM storage_upload_copies AS target_copy + JOIN storage_data_sets AS target_data_set ON target_data_set.id = target_copy.storage_data_set_id + WHERE target_copy.upload_id = ? + AND target_copy.storage_data_set_id = ? + AND %s`, readableCommittedCopyPredicateSQL("target_copy", "target_data_set")) + var count int + if err := db.NewRaw(query, uploadID, targetDataSetID).Scan(ctx, &count); err != nil { + return false, fmt.Errorf("checking replacement target coverage: %w", err) + } + return count > 0, nil +} + +// AttachTargetCopy creates the copy row on the target generation, or returns +// the existing one so a retried item reuses the same concrete row. +func (r *BunStorageReplacementRepo) AttachTargetCopy(ctx context.Context, input AttachReplacementTargetCopyInput) (*model.StorageUploadCopy, error) { + if input.ReplacementID <= 0 || input.ItemID <= 0 || input.UploadID <= 0 { + return nil, fmt.Errorf("attaching replacement target copy: %w", ErrInvalidInput) + } + var attached *model.StorageUploadCopy + err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { + replacement, err := lockReplacementByID(ctx, db, input.ReplacementID) + if err != nil { + return err + } + uploads := &BunStorageUploadRepo{db: db} + target, err := uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) + if err != nil { + return err + } + if target == nil { + return fmt.Errorf("attaching replacement target copy: data set: %w", ErrNotFound) + } + if err := uploads.CreateUploadCopiesForBindings(ctx, input.UploadID, []UploadCopyBindingInput{{ + StorageDataSetID: target.ID, + CopyIndex: target.CopyIndex, + // Migration pulls from a remote replica whenever one is readable. + TransferMethod: model.StorageCopyTransferMethodPeerPull, + ProviderID: target.ProviderID, + }}); err != nil { + return err + } + copyRow, err := uploads.GetUploadCopyForDataSet(ctx, input.UploadID, target.ID) + if err != nil { + return err + } + if copyRow == nil { + return fmt.Errorf("attaching replacement target copy: %w", ErrNotFound) + } + if _, err := db.NewUpdate(). + Model((*storagereplacement.Item)(nil)). + Set("target_copy_id = ?", copyRow.ID). + Set("updated_at = ?", time.Now()). + Where("id = ? AND replacement_id = ?", input.ItemID, input.ReplacementID). + Exec(ctx); err != nil { + return fmt.Errorf("recording replacement target copy: %w", err) + } + attached = copyRow + return nil + }) + if err != nil { + return nil, err + } + return attached, nil +} + +func (r *BunStorageReplacementRepo) MarkItemCopied(ctx context.Context, itemID int64) error { + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + item := new(storagereplacement.Item) + if err := db.NewSelect().Model(item).Where("id = ?", itemID).Scan(ctx); err != nil { + if err == sql.ErrNoRows { + // Provenance cleanup can remove an item after its content stops + // being referenced while a provider call is still returning. + return nil + } + return fmt.Errorf("selecting replacement item: %w", err) + } + if item.Status == storagereplacement.ItemStatusCopied || item.Status == storagereplacement.ItemStatusCancelled { + return nil + } + res, err := db.NewUpdate(). + Model((*storagereplacement.Item)(nil)). + Set("status = ?", storagereplacement.ItemStatusCopied). + Set("last_error = NULL"). + Set("updated_at = ?", time.Now()). + Where("id = ?", itemID). + Where("status NOT IN (?, ?)", storagereplacement.ItemStatusCopied, storagereplacement.ItemStatusCancelled). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking replacement item copied: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("marking replacement item copied: %w", ErrConflict) + } + if _, err := db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("items_copied = items_copied + 1"). + Set("updated_at = ?", time.Now()). + Where("id = ? AND items_copied < items_total", item.ReplacementID). + Exec(ctx); err != nil { + return fmt.Errorf("recording replacement progress: %w", err) + } + return nil + }) +} + +// MarkItemWaitingSource parks one item so the coordinator can move on to other +// content instead of blocking the whole replacement. +func (r *BunStorageReplacementRepo) MarkItemWaitingSource(ctx context.Context, itemID int64, lastError string) error { + res, err := r.db.NewUpdate(). + Model((*storagereplacement.Item)(nil)). + Set("status = ?", storagereplacement.ItemStatusWaitingSource). + Set("last_error = ?", nullableString(lastError)). + Set("updated_at = ?", time.Now()). + Where("id = ?", itemID). + Where("status <> ?", storagereplacement.ItemStatusCopied). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking replacement item waiting: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("marking replacement item waiting: %w", ErrConflict) + } + return nil +} + +// HeldItemCopyID reports the target copy the coordinator is currently writing, +// or zero when it holds no item. Mutual exclusion is per copy row: between +// items the coordinator writes nothing, so nothing needs to stand down for it. +func (r *BunStorageReplacementRepo) HeldItemCopyID(ctx context.Context, replacementID int64) (int64, error) { + item := new(storagereplacement.Item) + err := r.db.NewSelect(). + Model(item). + Column("target_copy_id"). + Where("replacement_id = ?", replacementID). + Where("status = ?", storagereplacement.ItemStatusRunning). + Where("target_copy_id IS NOT NULL"). + Limit(1). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return 0, nil + } + return 0, fmt.Errorf("selecting held replacement item copy: %w", err) + } + if item.TargetCopyID == nil { + return 0, nil + } + return *item.TargetCopyID, nil +} diff --git a/internal/db/repository/storage_replacement_postgres_test.go b/internal/db/repository/storage_replacement_postgres_test.go new file mode 100644 index 0000000..91b59b2 --- /dev/null +++ b/internal/db/repository/storage_replacement_postgres_test.go @@ -0,0 +1,423 @@ +package repository_test + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/strahe/synaps3/internal/config" + appdb "github.com/strahe/synaps3/internal/db" + "github.com/strahe/synaps3/internal/db/migrations" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/uptrace/bun" + "github.com/uptrace/bun/migrate" +) + +type replacementLockContextKey struct{} + +type replacementBucketLockBarrier struct { + winner string + locked chan struct{} + release chan struct{} + competitor chan struct{} + lockedOnce sync.Once + attemptOnce sync.Once +} + +func (h *replacementBucketLockBarrier) BeforeQuery(ctx context.Context, event *bun.QueryEvent) context.Context { + if replacementBucketLockQuery(event.Query) && ctx.Value(replacementLockContextKey{}) != h.winner { + h.attemptOnce.Do(func() { close(h.competitor) }) + } + return ctx +} + +func (h *replacementBucketLockBarrier) AfterQuery(ctx context.Context, event *bun.QueryEvent) { + if !replacementBucketLockQuery(event.Query) || ctx.Value(replacementLockContextKey{}) != h.winner { + return + } + h.lockedOnce.Do(func() { + close(h.locked) + <-h.release + }) +} + +func replacementBucketLockQuery(query string) bool { + query = strings.ToLower(query) + return strings.Contains(query, "update") && strings.Contains(query, "buckets") && + strings.Contains(query, "updated_at = updated_at") +} + +// The generation indexes rely on partial-index semantics and on ON CONFLICT +// inferring a partial index. Those differ enough between SQLite and PostgreSQL +// that the guarantees have to be checked on both. +func TestPostgresStorageReplacementSchemaParity(t *testing.T) { + dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") + if dsn == "" { + t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") + } + ctx := context.Background() + db := newPostgresReplacementDB(t, ctx, dsn) + repos := repository.NewRepositories(db) + + bucket := &model.Bucket{Name: "pg-replacement", Status: model.BucketStatusActive} + if _, err := db.NewInsert().Model(bucket).Exec(ctx); err != nil { + t.Fatalf("seed bucket: %v", err) + } + source, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, DataSetID: onChainID(t, "1001"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + + row, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, + SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), + ClientRequestID: "postgres-replacement", + MaxRetries: 5, + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + + // A second live replacement for one source must be impossible. + if _, err := db.NewInsert().Model(&storagereplacement.Replacement{ + BucketID: bucket.ID, CopyIndex: 0, + SourceDataSetID: source.ID, TargetDataSetID: row.TargetDataSetID, + SelectionMode: storagereplacement.SelectionModeManual, + Status: storagereplacement.StatusPreparingTarget, + ConfirmedAt: time.Now(), CreatedAt: time.Now(), UpdatedAt: time.Now(), + }).Exec(ctx); err == nil { + t.Fatal("PostgreSQL accepted a second active replacement for one source") + } + + // The slot may hold several generations, but only one current one. + if _, err := db.NewInsert().Model(&model.StorageDataSet{ + BucketID: bucket.ID, ProviderID: onChainID(t, "303"), CopyIndex: 0, + Generation: 5, IsCurrent: true, Status: model.StorageDataSetStatusPending, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + }).Exec(ctx); err == nil { + t.Fatal("PostgreSQL accepted a second current generation for one slot") + } + if _, err := db.NewInsert().Model(&model.StorageDataSet{ + BucketID: bucket.ID, ProviderID: onChainID(t, "303"), CopyIndex: 0, + Generation: 5, IsCurrent: false, Status: model.StorageDataSetStatusPending, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + }).Exec(ctx); err != nil { + t.Fatalf("PostgreSQL rejected a historical generation: %v", err) + } + + upload := &model.StorageUpload{ + BucketID: bucket.ID, SourceVersionID: "01J00000000000000000000PG1", + ContentSize: 10, Checksum: "pg-sum", Status: model.StorageUploadStatusRunning, RequestedCopies: 1, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + if _, err := db.NewInsert().Model(upload).Exec(ctx); err != nil { + t.Fatalf("seed upload: %v", err) + } + // ON CONFLICT must infer the partial index, so a repeated bind is a no-op + // rather than an error or a duplicate. + for range 2 { + if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: source.ID, CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: onChainID(t, "101"), + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + } + count, err := db.NewSelect().Model((*model.StorageUploadCopy)(nil)). + Where("upload_id = ?", upload.ID).Count(ctx) + if err != nil { + t.Fatalf("count copies: %v", err) + } + if count != 1 { + t.Fatalf("copies = %d, want the repeated bind to be a no-op", count) + } + + // Unbound copies are distinct under NULL, so they need their own guard. + if _, err := db.NewInsert().Model(&model.StorageUploadCopy{ + UploadID: upload.ID, CopyIndex: 3, + TransferMethod: model.StorageCopyTransferMethodPeerPull, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + }).Exec(ctx); err != nil { + t.Fatalf("seed unbound copy: %v", err) + } + if _, err := db.NewInsert().Model(&model.StorageUploadCopy{ + UploadID: upload.ID, CopyIndex: 3, + TransferMethod: model.StorageCopyTransferMethodPeerPull, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + }).Exec(ctx); err == nil { + t.Fatal("PostgreSQL accepted a duplicate unbound copy for one slot") + } +} + +func TestPostgresAuthorizeAndActivateSerializeOnBucket(t *testing.T) { + dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") + if dsn == "" { + t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") + } + + for _, winner := range []string{"activate", "authorize"} { + t.Run(winner+" wins", func(t *testing.T) { + ctx := context.Background() + db := newPostgresReplacementDB(t, ctx, dsn) + fixture := seedPostgresReplacement(t, db, "pg-replacement-lock-"+winner) + barrier := &replacementBucketLockBarrier{ + winner: winner, locked: make(chan struct{}), release: make(chan struct{}), competitor: make(chan struct{}), + } + defer func() { + select { + case <-barrier.release: + default: + close(barrier.release) + } + }() + db.AddQueryHook(barrier) + + activateResult := make(chan error, 1) + authorizeResult := make(chan error, 1) + var successor *storagereplacement.Replacement + startActivate := func() { + go func() { + activateCtx := context.WithValue(ctx, replacementLockContextKey{}, "activate") + activateResult <- fixture.repos.Replacements.Activate(activateCtx, fixture.row.ID) + }() + } + startAuthorize := func() { + go func() { + authorizeCtx := context.WithValue(ctx, replacementLockContextKey{}, "authorize") + row, _, err := fixture.repos.Replacements.Authorize(authorizeCtx, repository.AuthorizeReplacementInput{ + BucketID: fixture.bucket.ID, SourceDataSetID: fixture.source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "303"), ClientRequestID: "concurrent-successor", MaxRetries: 5, + }) + successor = row + authorizeResult <- err + }() + } + if winner == "activate" { + startActivate() + } else { + startAuthorize() + } + waitReplacementSignal(t, barrier.locked, "winning bucket lock") + if winner == "activate" { + startAuthorize() + } else { + startActivate() + } + waitReplacementSignal(t, barrier.competitor, "competing bucket lock") + close(barrier.release) + + activateErr := waitReplacementResult(t, activateResult, "Activate") + authorizeErr := waitReplacementResult(t, authorizeResult, "Authorize") + current, err := fixture.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, fixture.bucket.ID, fixture.source.CopyIndex) + if err != nil || current == nil { + t.Fatalf("current generation = %#v err=%v", current, err) + } + original, err := fixture.repos.Replacements.GetByID(ctx, fixture.row.ID) + if err != nil || original == nil { + t.Fatalf("original replacement = %#v err=%v", original, err) + } + + if winner == "activate" { + if activateErr != nil { + t.Fatalf("Activate winner: %v", activateErr) + } + if !errors.Is(authorizeErr, storagereplacement.ErrSourceNotCurrent) { + t.Fatalf("Authorize after activation = %v, want ErrSourceNotCurrent", authorizeErr) + } + if current.ID != fixture.target.ID || original.Status != storagereplacement.StatusMigrating { + t.Fatalf("winner state = current:%d replacement:%s, want target %d migrating", current.ID, original.Status, fixture.target.ID) + } + return + } + if authorizeErr != nil || successor == nil { + t.Fatalf("Authorize winner = %#v err=%v", successor, authorizeErr) + } + if !errors.Is(activateErr, repository.ErrConflict) { + t.Fatalf("Activate superseded replacement = %v, want ErrConflict", activateErr) + } + if current.ID != fixture.source.ID || original.Status != storagereplacement.StatusSuperseded { + t.Fatalf("winner state = current:%d replacement:%s, want source %d and superseded", current.ID, original.Status, fixture.source.ID) + } + if successor.TargetDataSetID == current.ID { + t.Fatalf("superseding target %d became current before activation", successor.TargetDataSetID) + } + }) + } +} + +func TestPostgresAuthorizeConcurrentIdempotentReplay(t *testing.T) { + dsn := os.Getenv("SYNAPS3_POSTGRES_TEST_DSN") + if dsn == "" { + t.Skip("SYNAPS3_POSTGRES_TEST_DSN is not set") + } + ctx := context.Background() + db := newPostgresReplacementDB(t, ctx, dsn) + repos := repository.NewRepositories(db) + bucket := &model.Bucket{Name: "pg-concurrent-idempotency", Status: model.BucketStatusActive} + if _, err := db.NewInsert().Model(bucket).Exec(ctx); err != nil { + t.Fatalf("seed bucket: %v", err) + } + source := &model.StorageDataSet{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + Generation: 1, IsCurrent: true, Status: model.StorageDataSetStatusReady, + DataSetID: onChainIDPtr(t, "1001"), CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + if _, err := db.NewInsert().Model(source).Exec(ctx); err != nil { + t.Fatalf("seed source: %v", err) + } + input := repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), ClientRequestID: "concurrent-confirmation", MaxRetries: 5, + } + type result struct { + row *storagereplacement.Replacement + created bool + err error + } + start := make(chan struct{}) + results := make(chan result, 2) + for range 2 { + go func() { + <-start + row, created, err := repos.Replacements.Authorize(ctx, input) + results <- result{row: row, created: created, err: err} + }() + } + close(start) + first := <-results + second := <-results + if first.err != nil || second.err != nil || first.row == nil || second.row == nil { + t.Fatalf("concurrent results = %#v and %#v", first, second) + } + if first.row.ID != second.row.ID || first.created == second.created { + t.Fatalf("concurrent results = ids %d/%d created %v/%v, want one shared row and one creator", + first.row.ID, second.row.ID, first.created, second.created) + } + count, err := db.NewSelect().Model((*storagereplacement.Replacement)(nil)). + Where("bucket_id = ? AND client_request_id = ?", bucket.ID, input.ClientRequestID).Count(ctx) + if err != nil || count != 1 { + t.Fatalf("persisted replacements = %d err=%v, want one", count, err) + } +} + +func waitReplacementSignal(t *testing.T, signal <-chan struct{}, name string) { + t.Helper() + select { + case <-signal: + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for %s", name) + } +} + +func waitReplacementResult(t *testing.T, result <-chan error, name string) error { + t.Helper() + select { + case err := <-result: + return err + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for %s", name) + return nil + } +} + +type postgresReplacementFixture struct { + repos *repository.Repositories + bucket *model.Bucket + source *model.StorageDataSet + target *model.StorageDataSet + row *storagereplacement.Replacement +} + +func seedPostgresReplacement(t *testing.T, db *bun.DB, name string) *postgresReplacementFixture { + t.Helper() + ctx := context.Background() + repos := repository.NewRepositories(db) + bucket := &model.Bucket{Name: name, Status: model.BucketStatusActive} + if _, err := db.NewInsert().Model(bucket).Exec(ctx); err != nil { + t.Fatalf("seed bucket: %v", err) + } + source, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, ProviderID: onChainID(t, "101"), CopyIndex: 0, + }) + if err != nil { + t.Fatalf("seed source: %v", err) + } + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, DataSetID: onChainID(t, "1001"), + }); err != nil { + t.Fatalf("ready source: %v", err) + } + row, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), ClientRequestID: "concurrent-original", MaxRetries: 5, + }) + if err != nil { + t.Fatalf("authorize replacement: %v", err) + } + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: row.TargetDataSetID, DataSetID: onChainID(t, "2002"), + }); err != nil { + t.Fatalf("ready target: %v", err) + } + target, err := repos.Uploads.GetDataSetBindingByID(ctx, row.TargetDataSetID) + if err != nil || target == nil { + t.Fatalf("target = %#v err=%v", target, err) + } + return &postgresReplacementFixture{repos: repos, bucket: bucket, source: source, target: target, row: row} +} + +func newPostgresReplacementDB(t *testing.T, ctx context.Context, dsn string) *bun.DB { + t.Helper() + db, err := appdb.New(config.DatabaseConfig{ + Driver: "postgres", DSN: dsn, MaxOpenConns: 4, MaxIdleConns: 2, + }) + if err != nil { + t.Fatalf("opening postgres test db: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + schema := fmt.Sprintf("synaps3_replacement_%d", time.Now().UnixNano()) + quoted := `"` + schema + `"` + if _, err := db.ExecContext(ctx, "CREATE SCHEMA "+quoted); err != nil { + t.Fatalf("creating schema: %v", err) + } + t.Cleanup(func() { + dropCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if _, err := db.ExecContext(dropCtx, "DROP SCHEMA "+quoted+" CASCADE"); err != nil { + t.Logf("dropping schema %s: %v", schema, err) + } + }) + if _, err := db.ExecContext(ctx, "SET search_path TO "+quoted); err != nil { + t.Fatalf("setting search_path: %v", err) + } + + migrator := migrate.NewMigrator(db, migrations.Migrations) + if err := migrator.Init(ctx); err != nil { + t.Fatalf("migrator init: %v", err) + } + if _, err := migrator.Migrate(ctx); err != nil { + t.Fatalf("running postgres migrations: %v", err) + } + return db +} diff --git a/internal/db/repository/storage_replacement_repo.go b/internal/db/repository/storage_replacement_repo.go new file mode 100644 index 0000000..954f12b --- /dev/null +++ b/internal/db/repository/storage_replacement_repo.go @@ -0,0 +1,736 @@ +package repository + +import ( + "context" + "database/sql" + "fmt" + "strings" + "time" + + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/uptrace/bun" +) + +var _ StorageReplacementRepository = (*BunStorageReplacementRepo)(nil) + +// BunStorageReplacementRepo persists operator-approved provider replacements. +type BunStorageReplacementRepo struct { + db bun.IDB +} + +// Authorize is the only way a replacement comes into existence. It runs as one +// transaction so the superseded predecessor, the new target generation, the +// replacement record, and its coordinator either all exist or none do. +func (r *BunStorageReplacementRepo) Authorize(ctx context.Context, input AuthorizeReplacementInput) (*storagereplacement.Replacement, bool, error) { + input.ClientRequestID = strings.TrimSpace(input.ClientRequestID) + if input.BucketID <= 0 || input.SourceDataSetID <= 0 || input.TargetProviderID.IsZero() || + input.ClientRequestID == "" || len(input.ClientRequestID) > 128 { + return nil, false, fmt.Errorf("authorizing provider replacement: %w", ErrInvalidInput) + } + if !input.SelectionMode.Valid() { + return nil, false, fmt.Errorf("authorizing provider replacement: unknown selection mode: %w", ErrInvalidInput) + } + var result *storagereplacement.Replacement + created := false + err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { + bucket, err := lockBucketByID(ctx, db, input.BucketID) + if err != nil { + return err + } + if bucket == nil { + return fmt.Errorf("authorizing provider replacement: bucket %d: %w", input.BucketID, ErrNotFound) + } + existing, err := getReplacementByClientRequestID(ctx, db, input.BucketID, input.ClientRequestID) + if err != nil { + return err + } + if existing != nil { + if !replacementRequestMatches(existing, input) { + return fmt.Errorf("authorizing provider replacement: %w", storagereplacement.ErrIdempotencyConflict) + } + result = existing + return nil + } + source, err := (&BunStorageUploadRepo{db: db}).GetDataSetBindingByID(ctx, input.SourceDataSetID) + if err != nil { + return err + } + if source == nil || source.BucketID != input.BucketID { + return fmt.Errorf("authorizing provider replacement: data set %d: %w", input.SourceDataSetID, ErrNotFound) + } + // Replacing a generation that no longer owns the slot would not move any + // writes, so it is refused rather than silently accepted. + if !source.IsCurrent { + return fmt.Errorf("authorizing provider replacement: %w", storagereplacement.ErrSourceNotCurrent) + } + if source.ProviderID.Equal(input.TargetProviderID) { + return fmt.Errorf("authorizing provider replacement: %w", storagereplacement.ErrInvalidTarget) + } + // A generation that some unfinished replacement is still migrating into + // cannot become a source of its own. That replacement's safety gate + // requires this generation to keep owning the slot, so handing the slot + // to a third generation would strand it and leave the original source + // unable to retire. + pending, err := db.NewSelect(). + Model((*storagereplacement.Replacement)(nil)). + Where("target_data_set_id = ?", source.ID). + Where("status NOT IN (?, ?)", storagereplacement.StatusCompleted, storagereplacement.StatusSuperseded). + Count(ctx) + if err != nil { + return fmt.Errorf("checking replacements targeting this data set: %w", err) + } + if pending > 0 { + return fmt.Errorf("authorizing provider replacement: %w", storagereplacement.ErrActiveReplacement) + } + // Any generation that has not been retired still holds this provider's + // data set for the bucket. Preparing a second one would make the SDK + // hand back the same data set and collide on the provider/data set + // uniqueness, so the choice is refused up front with a typed error + // rather than failing later inside the worker. + inUse, err := db.NewSelect(). + Model((*model.StorageDataSet)(nil)). + Where("bucket_id = ? AND provider_id = ?", input.BucketID, input.TargetProviderID). + Where("status <> ?", model.StorageDataSetStatusRetired). + Count(ctx) + if err != nil { + return fmt.Errorf("checking replacement target provider: %w", err) + } + if inUse > 0 { + return fmt.Errorf("authorizing provider replacement: %w", storagereplacement.ErrTargetInUse) + } + + now := time.Now() + // The single-active-replacement index rejects a second live row for one + // source, so the predecessor must step down before the successor exists. + superseded, err := supersedeEarlierReplacements(ctx, db, source.ID, now) + if err != nil { + return err + } + generation, err := nextDataSetGeneration(ctx, db, input.BucketID, source.CopyIndex) + if err != nil { + return err + } + target := &model.StorageDataSet{ + BucketID: input.BucketID, + ProviderID: input.TargetProviderID, + CopyIndex: source.CopyIndex, + Generation: generation, + // The target only takes the slot once it is writable, so preparing it + // does not change where uploads go. + IsCurrent: false, + Status: model.StorageDataSetStatusPending, + CreatedAt: now, + UpdatedAt: now, + } + if _, err := db.NewInsert().Model(target).Exec(ctx); err != nil { + if isUniqueViolation(err) { + return fmt.Errorf("authorizing provider replacement: %w", storagereplacement.ErrTargetInUse) + } + return fmt.Errorf("creating replacement target data set: %w", err) + } + + replacement := &storagereplacement.Replacement{ + BucketID: input.BucketID, + CopyIndex: source.CopyIndex, + SourceDataSetID: source.ID, + TargetDataSetID: target.ID, + SelectionMode: input.SelectionMode, + ClientRequestID: input.ClientRequestID, + Status: storagereplacement.StatusPreparingTarget, + ConfirmedAt: now, + CreatedAt: now, + UpdatedAt: now, + } + if input.SelectionMode == storagereplacement.SelectionModeManual { + providerID := input.TargetProviderID + replacement.RequestedProviderID = &providerID + } + if _, err := db.NewInsert().Model(replacement).Exec(ctx); err != nil { + if isUniqueViolation(err) { + return fmt.Errorf("authorizing provider replacement: %w", storagereplacement.ErrActiveReplacement) + } + return fmt.Errorf("creating provider replacement: %w", err) + } + if err := linkSupersededReplacements(ctx, db, superseded, replacement.ID, now); err != nil { + return err + } + tasks := &BunTaskRepo{db: db} + for _, id := range superseded { + // Leftover targets keep costing money until their own coordinator + // ends them. Queue that work in this transaction so cleanup does + // not wait for the next process start. + if _, err := tasks.EnsureRecurring(ctx, storagereplacement.NewAbandonedTargetTask( + id, input.BucketID, input.MaxRetries, now, + )); err != nil { + return fmt.Errorf("queueing abandoned replacement cleanup: %w", err) + } + } + if _, err := tasks.EnsureRecurring(ctx, storagereplacement.NewMigrateTask( + replacement.ID, input.BucketID, "", input.MaxRetries, now, + )); err != nil { + return fmt.Errorf("queueing replacement migration coordinator: %w", err) + } + result = replacement + created = true + return nil + }) + if err != nil { + return nil, false, err + } + return result, created, nil +} + +func replacementRequestMatches(row *storagereplacement.Replacement, input AuthorizeReplacementInput) bool { + if row.BucketID != input.BucketID || row.SourceDataSetID != input.SourceDataSetID || row.SelectionMode != input.SelectionMode { + return false + } + if input.SelectionMode != storagereplacement.SelectionModeManual { + return true + } + return row.RequestedProviderID != nil && row.RequestedProviderID.Equal(input.TargetProviderID) +} + +func getReplacementByClientRequestID( + ctx context.Context, + db bun.IDB, + bucketID int64, + clientRequestID string, +) (*storagereplacement.Replacement, error) { + row := new(storagereplacement.Replacement) + err := db.NewSelect(). + Model(row). + Where("bucket_id = ? AND client_request_id = ?", bucketID, clientRequestID). + Scan(ctx) + if err == sql.ErrNoRows { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("selecting provider replacement by client request id: %w", err) + } + return row, nil +} + +// A later confirmation takes over from an earlier one in the same transaction, +// which is what the single-active-replacement index relies on. +func supersedeEarlierReplacements(ctx context.Context, db bun.IDB, sourceDataSetID int64, now time.Time) ([]int64, error) { + var ids []int64 + if err := db.NewSelect(). + Model((*storagereplacement.Replacement)(nil)). + Column("id"). + Where("source_data_set_id = ?", sourceDataSetID). + Where("status NOT IN (?, ?)", storagereplacement.StatusCompleted, storagereplacement.StatusSuperseded). + Scan(ctx, &ids); err != nil { + return nil, fmt.Errorf("selecting earlier provider replacements: %w", err) + } + if len(ids) == 0 { + return nil, nil + } + if _, err := db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("status = ?", storagereplacement.StatusSuperseded). + Set("wait_reason = NULL"). + Set("updated_at = ?", now). + Where("id IN (?)", bun.List(ids)). + Exec(ctx); err != nil { + return nil, fmt.Errorf("superseding earlier provider replacement: %w", err) + } + return ids, nil +} + +// The successor id is recorded once it exists, so an operator can follow the +// chain from an abandoned confirmation to the one that replaced it. +func linkSupersededReplacements(ctx context.Context, db bun.IDB, ids []int64, successorID int64, now time.Time) error { + if len(ids) == 0 { + return nil + } + if _, err := db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("superseded_by_id = ?", successorID). + Set("updated_at = ?", now). + Where("id IN (?)", bun.List(ids)). + Exec(ctx); err != nil { + return fmt.Errorf("linking superseded provider replacement: %w", err) + } + return nil +} + +// Retry resumes the same approved target. The phase is re-derived from the +// data rather than remembered, so a retry always restarts at the stage the +// replacement actually reached. +func (r *BunStorageReplacementRepo) Retry(ctx context.Context, input RetryReplacementInput) (*storagereplacement.Replacement, error) { + if input.ReplacementID <= 0 { + return nil, fmt.Errorf("retrying provider replacement: %w", ErrInvalidInput) + } + var resumed *storagereplacement.Replacement + err := runMaybeTx(ctx, r.db, func(db bun.IDB) error { + row, err := lockReplacementByID(ctx, db, input.ReplacementID) + if err != nil { + return err + } + if row.Status == storagereplacement.StatusSuperseded { + return fmt.Errorf("retrying provider replacement: %w", storagereplacement.ErrSuperseded) + } + if row.FailureReason != nil && *row.FailureReason == storagereplacement.FailureReasonTargetInUse { + return fmt.Errorf("retrying provider replacement: %w", storagereplacement.ErrTargetInUse) + } + if !row.Status.Retryable() { + return fmt.Errorf("retrying provider replacement: %w", storagereplacement.ErrNotRetryable) + } + running, err := db.NewSelect(). + Model((*model.Task)(nil)). + Where("idempotency_key IN (?, ?)", + storagereplacement.MigrateTaskKey(row.ID), + storagereplacement.RetireTaskKey(row.ID)). + Where("status = ?", model.TaskStatusRunning). + Count(ctx) + if err != nil { + return fmt.Errorf("checking replacement coordinator tasks: %w", err) + } + if running > 0 { + return fmt.Errorf("retrying provider replacement: %w", storagereplacement.ErrTaskRunning) + } + + target, err := (&BunStorageUploadRepo{db: db}).GetDataSetBindingByID(ctx, row.TargetDataSetID) + if err != nil { + return err + } + if target == nil { + return fmt.Errorf("retrying provider replacement: target data set %d: %w", row.TargetDataSetID, ErrNotFound) + } + next := storagereplacement.StatusPreparingTarget + switch { + case row.Status == storagereplacement.StatusCleanupAttention: + next = storagereplacement.StatusRetiring + case target.IsCurrent: + next = storagereplacement.StatusMigrating + } + now := time.Now() + if err := transitionReplacement(ctx, db, row.ID, []storagereplacement.Status{row.Status}, next, func(q *bun.UpdateQuery) *bun.UpdateQuery { + return q.Set("last_error = NULL").Set("wait_reason = NULL").Set("failure_reason = NULL") + }, now); err != nil { + return err + } + task := storagereplacement.NewMigrateTask(row.ID, row.BucketID, "", input.MaxRetries, now) + if next == storagereplacement.StatusRetiring { + task = storagereplacement.NewRetireTask(row.ID, row.BucketID, input.MaxRetries, now) + } + // The worker marks the coordinator exhausted or failed on its way into a + // retryable state, and automatic recurrence deliberately leaves those + // alone. Resuming is the operator's explicit request to undo that. + if _, err := (&BunTaskRepo{db: db}).ResumeCoordinator(ctx, task); err != nil { + return fmt.Errorf("requeueing replacement coordinator: %w", err) + } + row.Status = next + row.LastError = nil + row.WaitReason = nil + row.FailureReason = nil + row.UpdatedAt = now + resumed = row + return nil + }) + if err != nil { + return nil, err + } + return resumed, nil +} + +// Activate is the single atomic switch: the target starts receiving writes and +// the source starts draining. It touches three rows whatever the bucket holds. +func (r *BunStorageReplacementRepo) Activate(ctx context.Context, replacementID int64) error { + if replacementID <= 0 { + return fmt.Errorf("activating provider replacement: %w", ErrInvalidInput) + } + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + var bucketID int64 + if err := db.NewSelect(). + Model((*storagereplacement.Replacement)(nil)). + Column("bucket_id"). + Where("id = ?", replacementID). + Scan(ctx, &bucketID); err != nil { + if err == sql.ErrNoRows { + return fmt.Errorf("provider replacement %d: %w", replacementID, ErrNotFound) + } + return fmt.Errorf("selecting replacement bucket: %w", err) + } + if _, err := lockBucketByID(ctx, db, bucketID); err != nil { + return err + } + row, err := lockReplacementByID(ctx, db, replacementID) + if err != nil { + return err + } + if row.Status != storagereplacement.StatusPreparingTarget && row.Status != storagereplacement.StatusWaiting { + return fmt.Errorf("activating provider replacement %d from %s: %w", replacementID, row.Status, ErrConflict) + } + if row.BucketID != bucketID { + return fmt.Errorf("activating provider replacement: bucket changed: %w", ErrConflict) + } + now := time.Now() + res, err := db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("is_current = ?", false). + Set("status = ?", model.StorageDataSetStatusDraining). + Set("updated_at = ?", now). + Where("id = ? AND is_current = ?", row.SourceDataSetID, true). + Exec(ctx) + if err != nil { + return fmt.Errorf("draining replacement source: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("draining replacement source: %w", ErrConflict) + } + res, err = db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("is_current = ?", true). + Set("updated_at = ?", now). + Where("id = ? AND is_current = ?", row.TargetDataSetID, false). + Where("status = ?", model.StorageDataSetStatusReady). + Exec(ctx) + if err != nil { + return fmt.Errorf("activating replacement target: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("activating replacement target: %w", ErrConflict) + } + return transitionReplacement(ctx, db, replacementID, + []storagereplacement.Status{row.Status}, storagereplacement.StatusMigrating, + func(q *bun.UpdateQuery) *bun.UpdateQuery { return q.Set("wait_reason = NULL") }, now) + }) +} + +func (r *BunStorageReplacementRepo) MarkMigrating(ctx context.Context, replacementID int64) error { + return r.transition(ctx, replacementID, + []storagereplacement.Status{storagereplacement.StatusWaiting, storagereplacement.StatusMigrating}, + storagereplacement.StatusMigrating, + func(q *bun.UpdateQuery) *bun.UpdateQuery { return q.Set("wait_reason = NULL") }) +} + +// MarkWaiting records a recoverable pause. It never consumes retry budget and +// is never reported as a failure. +func (r *BunStorageReplacementRepo) MarkWaiting(ctx context.Context, replacementID int64, reason storagereplacement.WaitReason) error { + if !reason.Valid() { + return fmt.Errorf("marking replacement waiting: unknown reason: %w", ErrInvalidInput) + } + return r.transition(ctx, replacementID, + []storagereplacement.Status{ + storagereplacement.StatusPreparingTarget, + storagereplacement.StatusMigrating, + storagereplacement.StatusWaiting, + storagereplacement.StatusRetiring, + }, + storagereplacement.StatusWaiting, + func(q *bun.UpdateQuery) *bun.UpdateQuery { return q.Set("wait_reason = ?", reason) }) +} + +func (r *BunStorageReplacementRepo) MarkFailed( + ctx context.Context, + replacementID int64, + reason *storagereplacement.FailureReason, + lastError string, +) error { + if reason != nil && !reason.Valid() { + return fmt.Errorf("marking replacement failed: unknown reason: %w", ErrInvalidInput) + } + return r.transition(ctx, replacementID, + []storagereplacement.Status{ + storagereplacement.StatusPreparingTarget, + storagereplacement.StatusMigrating, + storagereplacement.StatusWaiting, + }, + storagereplacement.StatusFailed, + func(q *bun.UpdateQuery) *bun.UpdateQuery { + return q.Set("last_error = ?", lastError). + Set("wait_reason = NULL"). + Set("failure_reason = ?", reason) + }) +} + +// MarkCleanupAttention is committed in the same transaction that stops the +// coordinator task, so automatic retry can never resume suppressed cleanup. +func (r *BunStorageReplacementRepo) MarkCleanupAttention(ctx context.Context, replacementID int64, lastError string) error { + return r.transition(ctx, replacementID, + []storagereplacement.Status{storagereplacement.StatusRetiring, storagereplacement.StatusWaiting}, + storagereplacement.StatusCleanupAttention, + func(q *bun.UpdateQuery) *bun.UpdateQuery { + return q.Set("last_error = ?", lastError).Set("wait_reason = NULL") + }) +} + +func (r *BunStorageReplacementRepo) BeginRetirement(ctx context.Context, replacementID int64) error { + return r.transition(ctx, replacementID, + []storagereplacement.Status{ + storagereplacement.StatusMigrating, + storagereplacement.StatusWaiting, + storagereplacement.StatusCleanupAttention, + }, + storagereplacement.StatusRetiring, + func(q *bun.UpdateQuery) *bun.UpdateQuery { return q.Set("wait_reason = NULL") }) +} + +// RecordTerminationEpoch persists the end of term before the remote service is +// treated as terminated, so a crash in between re-reads it instead of +// terminating a second time. +func (r *BunStorageReplacementRepo) RecordTerminationEpoch(ctx context.Context, input RecordTerminationEpochInput) error { + if input.ReplacementID <= 0 { + return fmt.Errorf("recording replacement termination epoch: %w", ErrInvalidInput) + } + res, err := r.db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("termination_epoch = ?", input.Epoch). + Set("termination_tx_hash = ?", nullableString(input.TxHash)). + Set("updated_at = ?", time.Now()). + Where("id = ?", input.ReplacementID). + Where("status IN (?, ?, ?)", + storagereplacement.StatusRetiring, + storagereplacement.StatusWaiting, + storagereplacement.StatusCleanupAttention). + Where("termination_epoch IS NULL"). + Exec(ctx) + if err != nil { + return fmt.Errorf("recording replacement termination epoch: %w", err) + } + if rows, _ := res.RowsAffected(); rows != 1 { + return fmt.Errorf("recording replacement termination epoch: %w", ErrConflict) + } + return nil +} + +// RecordAbandonedTerminationEpoch persists termination of a superseded target +// before waiting for its end epoch. A restarted worker observes this value +// instead of paying for another termination transaction. +func (r *BunStorageReplacementRepo) RecordAbandonedTerminationEpoch( + ctx context.Context, + input RecordTerminationEpochInput, +) error { + if input.ReplacementID <= 0 || input.Epoch < 0 { + return fmt.Errorf("recording abandoned target termination epoch: %w", ErrInvalidInput) + } + res, err := r.db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("abandoned_termination_epoch = ?", input.Epoch). + Set("abandoned_termination_tx_hash = ?", nullableString(input.TxHash)). + Set("updated_at = ?", time.Now()). + Where("id = ?", input.ReplacementID). + Where("status = ?", storagereplacement.StatusSuperseded). + Where("abandoned_termination_epoch IS NULL"). + Exec(ctx) + if err != nil { + return fmt.Errorf("recording abandoned target termination epoch: %w", err) + } + if rows, _ := res.RowsAffected(); rows == 1 { + return nil + } + row, getErr := r.GetByID(ctx, input.ReplacementID) + if getErr != nil { + return getErr + } + if row != nil && row.Status == storagereplacement.StatusSuperseded && + row.AbandonedTerminationEpoch != nil && *row.AbandonedTerminationEpoch == input.Epoch { + return nil + } + return fmt.Errorf("recording abandoned target termination epoch: %w", ErrConflict) +} + +func (r *BunStorageReplacementRepo) GetByID(ctx context.Context, id int64) (*storagereplacement.Replacement, error) { + row := new(storagereplacement.Replacement) + err := r.db.NewSelect().Model(row).Where("id = ?", id).Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting provider replacement: %w", err) + } + return row, nil +} + +func (r *BunStorageReplacementRepo) GetByClientRequestID( + ctx context.Context, + bucketID int64, + clientRequestID string, +) (*storagereplacement.Replacement, error) { + clientRequestID = strings.TrimSpace(clientRequestID) + if bucketID <= 0 || clientRequestID == "" || len(clientRequestID) > 128 { + return nil, fmt.Errorf("selecting provider replacement by client request id: %w", ErrInvalidInput) + } + return getReplacementByClientRequestID(ctx, r.db, bucketID, clientRequestID) +} + +// ListForBucket returns the whole replacement history, newest first, so the API +// can present a stable order. +func (r *BunStorageReplacementRepo) ListForBucket(ctx context.Context, bucketID int64, limit int) ([]storagereplacement.Replacement, error) { + var rows []storagereplacement.Replacement + q := r.db.NewSelect(). + Model(&rows). + Where("bucket_id = ?", bucketID). + OrderExpr("id DESC") + if limit > 0 { + q = q.Limit(limit) + } + if err := q.Scan(ctx); err != nil { + return nil, fmt.Errorf("listing bucket provider replacements: %w", err) + } + return rows, nil +} + +func (r *BunStorageReplacementRepo) GetActiveForDataSet(ctx context.Context, dataSetID int64) (*storagereplacement.Replacement, error) { + row := new(storagereplacement.Replacement) + err := r.db.NewSelect(). + Model(row). + // Either generation is owned by the replacement. After activation the + // coordinator writes the target, so a caller asking about the target has + // to see the replacement too. + Where("source_data_set_id = ? OR target_data_set_id = ?", dataSetID, dataSetID). + Where("status NOT IN (?, ?)", storagereplacement.StatusCompleted, storagereplacement.StatusSuperseded). + Limit(1). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting active provider replacement: %w", err) + } + return row, nil +} + +// HasInProgressForDataSet covers both generations a replacement owns. The +// target matters as much as the source: in-place recovery would otherwise queue +// repair work on the generation the coordinator is actively writing, and the +// two would race over the same copy row. It excludes terminally failed and +// attention states so a generation whose replacement gave up can still recover +// and repair in place. +func (r *BunStorageReplacementRepo) HasInProgressForDataSet(ctx context.Context, dataSetID int64) (bool, error) { + count, err := r.db.NewSelect(). + Model((*storagereplacement.Replacement)(nil)). + Where("source_data_set_id = ? OR target_data_set_id = ?", dataSetID, dataSetID). + Where("status IN (?, ?, ?, ?)", + storagereplacement.StatusPreparingTarget, + storagereplacement.StatusMigrating, + storagereplacement.StatusWaiting, + storagereplacement.StatusRetiring). + Count(ctx) + if err != nil { + return false, fmt.Errorf("checking in-progress provider replacement: %w", err) + } + return count > 0, nil +} + +func (r *BunStorageReplacementRepo) ListActive(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) { + var rows []storagereplacement.Replacement + if err := r.db.NewSelect(). + Model(&rows). + Where("id > ?", afterID). + Where("status IN (?, ?, ?, ?)", + storagereplacement.StatusPreparingTarget, + storagereplacement.StatusMigrating, + storagereplacement.StatusWaiting, + storagereplacement.StatusRetiring). + OrderExpr("id ASC"). + Limit(limit). + Scan(ctx); err != nil { + return nil, fmt.Errorf("listing active provider replacements: %w", err) + } + return rows, nil +} + +// A superseded replacement leaves behind a target generation that holds partly +// migrated data and no coverage obligation. It still needs its own retirement +// so the abandoned service does not keep costing money. +func (r *BunStorageReplacementRepo) ListSupersededCleanupCandidates(ctx context.Context, afterID int64, limit int) ([]storagereplacement.Replacement, error) { + var rows []storagereplacement.Replacement + if err := r.db.NewSelect(). + Model(&rows). + Where("id > ?", afterID). + Where("status = ?", storagereplacement.StatusSuperseded). + Where(`EXISTS ( + SELECT 1 FROM storage_data_sets AS abandoned_target + WHERE abandoned_target.id = storage_replacement.target_data_set_id + AND abandoned_target.is_current = ? + AND abandoned_target.status <> ? + )`, false, model.StorageDataSetStatusRetired). + OrderExpr("id ASC"). + Limit(limit). + Scan(ctx); err != nil { + return nil, fmt.Errorf("listing superseded replacement cleanup candidates: %w", err) + } + return rows, nil +} + +func (r *BunStorageReplacementRepo) transition( + ctx context.Context, + replacementID int64, + from []storagereplacement.Status, + to storagereplacement.Status, + mutate func(*bun.UpdateQuery) *bun.UpdateQuery, +) error { + if replacementID <= 0 { + return fmt.Errorf("updating provider replacement: %w", ErrInvalidInput) + } + return runMaybeTx(ctx, r.db, func(db bun.IDB) error { + return transitionReplacement(ctx, db, replacementID, from, to, mutate, time.Now()) + }) +} + +// transitionReplacement refuses any move the state machine does not define, so +// an illegal combination fails loudly instead of corrupting the record. +func transitionReplacement( + ctx context.Context, + db bun.IDB, + replacementID int64, + from []storagereplacement.Status, + to storagereplacement.Status, + mutate func(*bun.UpdateQuery) *bun.UpdateQuery, + now time.Time, +) error { + allowed := make([]storagereplacement.Status, 0, len(from)) + for _, candidate := range from { + if candidate == to || storagereplacement.Allowed(candidate, to) { + allowed = append(allowed, candidate) + } + } + if len(allowed) == 0 { + return fmt.Errorf("replacement %d to %s: %w", replacementID, to, storagereplacement.ErrIllegalTransition) + } + q := db.NewUpdate(). + Model((*storagereplacement.Replacement)(nil)). + Set("status = ?", to). + Set("updated_at = ?", now). + Where("id = ?", replacementID). + Where("status IN (?)", bun.List(allowed)) + if mutate != nil { + q = mutate(q) + } + res, err := q.Exec(ctx) + if err != nil { + return fmt.Errorf("updating provider replacement: %w", err) + } + if rows, _ := res.RowsAffected(); rows == 1 { + return nil + } + count, countErr := db.NewSelect(). + Model((*storagereplacement.Replacement)(nil)). + Where("id = ?", replacementID). + Count(ctx) + if countErr != nil { + return fmt.Errorf("updating provider replacement: %w", countErr) + } + if count == 0 { + return fmt.Errorf("provider replacement %d: %w", replacementID, ErrNotFound) + } + return fmt.Errorf("updating provider replacement %d to %s: %w", replacementID, to, ErrConflict) +} + +func lockReplacementByID(ctx context.Context, db bun.IDB, replacementID int64) (*storagereplacement.Replacement, error) { + row := new(storagereplacement.Replacement) + err := db.NewRaw( + `UPDATE storage_replacements SET updated_at = updated_at WHERE id = ? RETURNING *`, + replacementID, + ).Scan(ctx, row) + if err != nil { + if err == sql.ErrNoRows { + return nil, fmt.Errorf("provider replacement %d: %w", replacementID, ErrNotFound) + } + return nil, fmt.Errorf("locking provider replacement: %w", err) + } + return row, nil +} diff --git a/internal/db/repository/storage_replacement_repo_test.go b/internal/db/repository/storage_replacement_repo_test.go new file mode 100644 index 0000000..d491863 --- /dev/null +++ b/internal/db/repository/storage_replacement_repo_test.go @@ -0,0 +1,1196 @@ +package repository_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/uptrace/bun" +) + +type replacementFixture struct { + db *bun.DB + repos *repository.Repositories + bucket *model.Bucket + upload *model.StorageUpload + version *model.ObjectVersion + source *model.StorageDataSet + request int +} + +func newReplacementFixture(t *testing.T, name, versionID string) *replacementFixture { + t.Helper() + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, name) + + version := newObjectVersion(bucket.ID, "file.txt", versionID, 10) + version.Checksum = name + "-checksum" + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 1) + source := commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://source.example/piece") + bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) + return &replacementFixture{db: db, repos: repos, bucket: bucket, upload: upload, version: version, source: source} +} + +func (f *replacementFixture) authorize(t *testing.T, provider string) *storagereplacement.Replacement { + t.Helper() + f.request++ + row, _, err := f.repos.Replacements.Authorize(context.Background(), repository.AuthorizeReplacementInput{ + BucketID: f.bucket.ID, + SourceDataSetID: f.source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, provider), + ClientRequestID: fmt.Sprintf("fixture-%d", f.request), + MaxRetries: 5, + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + return row +} + +// readyTarget brings the approved target to the point where it can take over. +func (f *replacementFixture) readyTarget(t *testing.T, row *storagereplacement.Replacement, dataSetID string) *model.StorageDataSet { + t.Helper() + ctx := context.Background() + if err := f.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: row.TargetDataSetID, + UploadID: f.upload.ID, + DataSetID: onChainID(t, dataSetID), + }); err != nil { + t.Fatalf("MarkDataSetReady target: %v", err) + } + target, err := f.repos.Uploads.GetDataSetBindingByID(ctx, row.TargetDataSetID) + if err != nil || target == nil { + t.Fatalf("GetDataSetBindingByID target = %#v err=%v", target, err) + } + return target +} + +func TestStorageReplacementRepo_AuthorizeCreatesTargetGenerationWithoutMovingWrites(t *testing.T) { + f := newReplacementFixture(t, "replacement-authorize", "01J000000000000000000RPL01") + ctx := context.Background() + + row := f.authorize(t, "202") + if row.Status != storagereplacement.StatusPreparingTarget || row.CopyIndex != 0 { + t.Fatalf("replacement = %#v, want preparing_target on slot 0", row) + } + + // Writes must keep going to the source until the target is actually usable. + current, err := f.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, f.bucket.ID, 0) + if err != nil || current == nil || current.ID != f.source.ID { + t.Fatalf("current binding = %#v err=%v, want the source until activation", current, err) + } + target, err := f.repos.Uploads.GetDataSetBindingByID(ctx, row.TargetDataSetID) + if err != nil || target == nil { + t.Fatalf("target binding = %#v err=%v", target, err) + } + if target.IsCurrent || target.Generation != f.source.Generation+1 || target.Status != model.StorageDataSetStatusPending { + t.Fatalf("target = %#v, want a pending next generation that is not current", target) + } + + task, err := f.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.MigrateTaskKey(row.ID)) + if err != nil || task == nil { + t.Fatalf("migration coordinator = %#v err=%v, want it queued with the confirmation", task, err) + } +} + +func TestStorageReplacementRepo_AuthorizeRejections(t *testing.T) { + f := newReplacementFixture(t, "replacement-reject", "01J000000000000000000RPL02") + ctx := context.Background() + base := repository.AuthorizeReplacementInput{ + BucketID: f.bucket.ID, + SourceDataSetID: f.source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), + MaxRetries: 5, + ClientRequestID: "reject-request", + } + + t.Run("target is the source", func(t *testing.T) { + input := base + input.TargetProviderID = f.source.ProviderID + if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, storagereplacement.ErrInvalidTarget) { + t.Fatalf("error = %v, want ErrInvalidTarget", err) + } + }) + + t.Run("target already serves the bucket", func(t *testing.T) { + other := commitStorageHealthCopy(t, f.repos, f.bucket.ID, f.upload.ID, 1, "303", "3003", "3003", "https://other.example/piece") + input := base + input.TargetProviderID = other.ProviderID + if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, storagereplacement.ErrTargetInUse) { + t.Fatalf("error = %v, want ErrTargetInUse", err) + } + }) + + // A provider whose earlier generation is draining still owns its data set for + // this bucket, so preparing a second one would collide on the provider/data + // set uniqueness inside the worker. + t.Run("target still holds a draining generation", func(t *testing.T) { + drainingProvider := commitStorageHealthCopy(t, f.repos, f.bucket.ID, f.upload.ID, 2, "404", "4004", "4004", "https://draining.example/piece") + mustExec(t, f.db, `UPDATE storage_data_sets SET is_current = FALSE, status = ? WHERE id = ?`, + model.StorageDataSetStatusDraining, drainingProvider.ID) + input := base + input.TargetProviderID = drainingProvider.ProviderID + if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, storagereplacement.ErrTargetInUse) { + t.Fatalf("error = %v, want ErrTargetInUse", err) + } + }) + + t.Run("unknown data set", func(t *testing.T) { + input := base + input.SourceDataSetID = 9999 + if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, repository.ErrNotFound) { + t.Fatalf("error = %v, want ErrNotFound", err) + } + }) + + t.Run("unknown selection mode", func(t *testing.T) { + input := base + input.SelectionMode = "guess" + if _, _, err := f.repos.Replacements.Authorize(ctx, input); !errors.Is(err, repository.ErrInvalidInput) { + t.Fatalf("error = %v, want ErrInvalidInput", err) + } + }) +} + +// A later confirmation takes over in the same transaction, so a source never +// holds two live replacements. +func TestStorageReplacementRepo_AuthorizeSupersedesEarlierConfirmation(t *testing.T) { + f := newReplacementFixture(t, "replacement-supersede", "01J000000000000000000RPL03") + ctx := context.Background() + + first := f.authorize(t, "202") + second := f.authorize(t, "303") + if second.ID == first.ID { + t.Fatal("second confirmation reused the first replacement") + } + + got, err := f.repos.Replacements.GetByID(ctx, first.ID) + if err != nil || got == nil { + t.Fatalf("GetByID first = %#v err=%v", got, err) + } + if got.Status != storagereplacement.StatusSuperseded { + t.Fatalf("first replacement status = %s, want superseded", got.Status) + } + if got.SupersededByID == nil || *got.SupersededByID != second.ID { + t.Fatalf("first.SupersededByID = %v, want %d", got.SupersededByID, second.ID) + } + active, err := f.repos.Replacements.GetActiveForDataSet(ctx, f.source.ID) + if err != nil || active == nil || active.ID != second.ID { + t.Fatalf("active replacement = %#v err=%v, want the newest confirmation", active, err) + } + + abandoned, err := f.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.AbandonedTargetTaskKey(first.ID)) + if err != nil || abandoned == nil { + t.Fatalf("abandoned-target cleanup = %#v err=%v, want it queued with the later confirmation", abandoned, err) + } + claimed, err := f.repos.Tasks.ClaimReady(ctx, model.TaskTypeStorageCleanup, time.Minute) + if err != nil || claimed == nil || claimed.ID != abandoned.ID { + t.Fatalf("ClaimReady abandoned cleanup = %#v err=%v, want the leftover terminator", claimed, err) + } +} + +func TestStorageReplacementRepo_ActivateSwitchesTheSlotAtomically(t *testing.T) { + f := newReplacementFixture(t, "replacement-activate", "01J000000000000000000RPL04") + ctx := context.Background() + row := f.authorize(t, "202") + + // A target that is not writable yet cannot take the slot. + if err := f.repos.Replacements.Activate(ctx, row.ID); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("activate before ready = %v, want conflict", err) + } + f.readyTarget(t, row, "2002") + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + + current, err := f.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, f.bucket.ID, 0) + if err != nil || current == nil || current.ID != row.TargetDataSetID { + t.Fatalf("current binding = %#v err=%v, want the target", current, err) + } + source, err := f.repos.Uploads.GetDataSetBindingByID(ctx, f.source.ID) + if err != nil || source == nil || source.IsCurrent || source.Status != model.StorageDataSetStatusDraining { + t.Fatalf("source after activation = %#v err=%v, want draining and not current", source, err) + } + got, err := f.repos.Replacements.GetByID(ctx, row.ID) + if err != nil || got == nil || got.Status != storagereplacement.StatusMigrating { + t.Fatalf("replacement after activation = %#v err=%v, want migrating", got, err) + } + if err := f.repos.Replacements.Activate(ctx, row.ID); !errors.Is(err, repository.ErrConflict) { + t.Fatalf("second activate = %v, want conflict", err) + } +} + +// Recovery must not revive a generation an operator is actively replacing, but +// a replacement that has given up should not hold the slot hostage. +func TestStorageReplacementRepo_RecoveryYieldsToInProgressReplacementOnly(t *testing.T) { + f := newReplacementFixture(t, "replacement-recovery", "01J000000000000000000RPL05") + ctx := context.Background() + row := f.authorize(t, "202") + + if err := f.repos.Uploads.MarkDataSetUnavailable(ctx, f.source.ID, "provider unreachable"); err != nil { + t.Fatalf("MarkDataSetUnavailable: %v", err) + } + recovered, err := f.repos.Uploads.RecoverDataSet(ctx, repository.MarkDataSetReadyInput{ + ID: f.source.ID, + UploadID: f.upload.ID, + DataSetID: onChainID(t, "1001"), + }) + if err != nil { + t.Fatalf("RecoverDataSet during replacement: %v", err) + } + if recovered { + t.Fatal("recovery revived a generation with an in-progress replacement") + } + + if err := f.repos.Replacements.MarkFailed(ctx, row.ID, nil, "target creation exhausted"); err != nil { + t.Fatalf("MarkFailed: %v", err) + } + recovered, err = f.repos.Uploads.RecoverDataSet(ctx, repository.MarkDataSetReadyInput{ + ID: f.source.ID, + UploadID: f.upload.ID, + DataSetID: onChainID(t, "1001"), + }) + if err != nil { + t.Fatalf("RecoverDataSet after failure: %v", err) + } + if !recovered { + t.Fatal("recovery stayed blocked after the replacement terminally failed") + } +} + +func TestStorageReplacementRepo_SeedMigrationBatchIsBounded(t *testing.T) { + f := newReplacementFixture(t, "replacement-seed", "01J000000000000000000RPL06") + ctx := context.Background() + + // Extra stored content on the same source, so seeding has to page. + for i, versionID := range []string{ + "01J000000000000000000RPL07", + "01J000000000000000000RPL08", + "01J000000000000000000RPL09", + } { + version := newObjectVersion(f.bucket.ID, "file.txt", versionID, 10) + version.Checksum = versionID + if _, err := f.repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent %d: %v", i, err) + } + upload := startCopyHealthUpload(t, f.repos, f.bucket.ID, version.VersionID, version.Size, version.Checksum, 1) + if err := f.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: f.source.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: f.source.ProviderID, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings %d: %v", i, err) + } + if err := f.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + UploadID: upload.ID, + CopyIndex: 0, + PieceCID: "bafk2bzacestorhealth", + PieceID: onChainIDPtr(t, "700"+versionID[len(versionID)-1:]), + RetrievalURL: "https://source.example/piece-" + versionID, + }); err != nil { + t.Fatalf("MarkUploadCopyCommitted %d: %v", i, err) + } + bindStorageHealthVersion(t, f.repos, f.bucket.ID, upload.ID, version) + } + + row := f.authorize(t, "202") + f.readyTarget(t, row, "2002") + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + + total := 0 + passes := 0 + for { + inserted, done, err := f.repos.Replacements.SeedMigrationBatch(ctx, row.ID, 2) + if err != nil { + t.Fatalf("SeedMigrationBatch: %v", err) + } + if inserted > 2 { + t.Fatalf("seeded %d items in one pass, want at most the batch limit", inserted) + } + total += inserted + passes++ + if done { + break + } + if passes > 10 { + t.Fatal("seeding never completed, cursor is not advancing") + } + } + if total != 4 { + t.Fatalf("seeded %d items, want one per stored upload", total) + } + if passes < 2 { + t.Fatalf("seeding finished in %d pass, want it to page", passes) + } + + got, err := f.repos.Replacements.GetByID(ctx, row.ID) + if err != nil || got == nil || !got.SeedingComplete || got.ItemsTotal != 4 { + t.Fatalf("replacement after seeding = %#v err=%v, want complete with 4 items", got, err) + } + // Re-running is a no-op rather than a duplicate. + inserted, done, err := f.repos.Replacements.SeedMigrationBatch(ctx, row.ID, 2) + if err != nil || inserted != 0 || !done { + t.Fatalf("re-seed = (%d, %v, %v), want no new work", inserted, done, err) + } +} + +// Parked items are revisited only after fresh work runs out, so one unreachable +// piece of content cannot stall the whole replacement. +func TestStorageReplacementRepo_NextExecutableItemPrefersFreshWork(t *testing.T) { + f := newReplacementFixture(t, "replacement-next-item", "01J000000000000000000RPL10") + ctx := context.Background() + later := newObjectVersion(f.bucket.ID, "other.txt", "01J000000000000000000RPL10B", 10) + later.Checksum = "replacement-next-item-later" + if _, err := f.repos.Objects.CreateVersionAndSetCurrent(ctx, later); err != nil { + t.Fatalf("CreateVersionAndSetCurrent later: %v", err) + } + laterUpload := startCopyHealthUpload(t, f.repos, f.bucket.ID, later.VersionID, later.Size, later.Checksum, 1) + if err := f.repos.Uploads.CreateUploadCopiesForBindings(ctx, laterUpload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: f.source.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: f.source.ProviderID, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings later: %v", err) + } + if err := f.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + UploadID: laterUpload.ID, + CopyIndex: 0, + PieceCID: "bafk2bzacepreferfresh", + PieceID: onChainIDPtr(t, "7010"), + RetrievalURL: "https://source.example/piece-later", + }); err != nil { + t.Fatalf("MarkUploadCopyCommitted later: %v", err) + } + bindStorageHealthVersion(t, f.repos, f.bucket.ID, laterUpload.ID, later) + + row := f.authorize(t, "202") + f.readyTarget(t, row, "2002") + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + if _, _, err := f.repos.Replacements.SeedMigrationBatch(ctx, row.ID, 10); err != nil { + t.Fatalf("SeedMigrationBatch: %v", err) + } + + first, err := f.repos.Replacements.NextExecutableItem(ctx, row.ID) + if err != nil || first == nil { + t.Fatalf("NextExecutableItem = %#v err=%v", first, err) + } + if err := f.repos.Replacements.MarkItemWaitingSource(ctx, first.ID, "no readable source"); err != nil { + t.Fatalf("MarkItemWaitingSource: %v", err) + } + fresh, err := f.repos.Replacements.NextExecutableItem(ctx, row.ID) + if err != nil || fresh == nil || fresh.ID == first.ID { + t.Fatalf("NextExecutableItem after parking = %#v err=%v, want pending work ahead of the parked item", fresh, err) + } + if fresh.Status != storagereplacement.ItemStatusPending { + t.Fatalf("fresh item status = %s, want pending", fresh.Status) + } + + if err := f.repos.Replacements.MarkItemCopied(ctx, fresh.ID); err != nil { + t.Fatalf("MarkItemCopied: %v", err) + } + parked, err := f.repos.Replacements.NextExecutableItem(ctx, row.ID) + if err != nil || parked == nil || parked.ID != first.ID { + t.Fatalf("NextExecutableItem after fresh work = %#v err=%v, want the parked item revisited", parked, err) + } + if parked.Status != storagereplacement.ItemStatusWaitingSource { + t.Fatalf("parked item status = %s, want waiting_source", parked.Status) + } +} + +func TestStorageReplacementRepo_RetryOnlyResumesOperatorAttentionStates(t *testing.T) { + f := newReplacementFixture(t, "replacement-retry", "01J000000000000000000RPL11") + ctx := context.Background() + row := f.authorize(t, "202") + + if _, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ReplacementID: row.ID, MaxRetries: 5}); !errors.Is(err, storagereplacement.ErrNotRetryable) { + t.Fatalf("retry while preparing = %v, want ErrNotRetryable", err) + } + if err := f.repos.Replacements.MarkFailed(ctx, row.ID, nil, "creation exhausted"); err != nil { + t.Fatalf("MarkFailed: %v", err) + } + resumed, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ReplacementID: row.ID, MaxRetries: 5}) + if err != nil { + t.Fatalf("Retry: %v", err) + } + // The target never activated, so the retry resumes preparation. + if resumed.Status != storagereplacement.StatusPreparingTarget || resumed.LastError != nil { + t.Fatalf("resumed = %#v, want preparing_target with the error cleared", resumed) + } + + superseded := f.authorize(t, "303") + if _, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ReplacementID: row.ID, MaxRetries: 5}); !errors.Is(err, storagereplacement.ErrSuperseded) { + t.Fatalf("retry after supersede = %v, want ErrSuperseded", err) + } + if superseded.ID == row.ID { + t.Fatal("supersede reused the replacement row") + } +} + +func TestStorageCleanupRepo_SettlesReplacementItemBeforeDeletingUpload(t *testing.T) { + f := newReplacementFixture(t, "replacement-provenance-cleanup", model.NewVersionID()) + ctx := context.Background() + replacement := f.authorize(t, "202") + orphan, err := f.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ + BucketID: f.bucket.ID, SourceVersionID: model.NewVersionID(), ContentSize: 10, + Checksum: "orphaned-replacement-content", RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("StartObjectUploadAttempt: %v", err) + } + item := &storagereplacement.Item{ + ReplacementID: replacement.ID, UploadID: orphan.ID, + Status: storagereplacement.ItemStatusRunning, CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + if _, err := f.db.NewInsert().Model(item).Exec(ctx); err != nil { + t.Fatalf("insert replacement item: %v", err) + } + if _, err := f.db.NewUpdate().Model((*storagereplacement.Replacement)(nil)). + Set("items_total = ?", 1).Where("id = ?", replacement.ID).Exec(ctx); err != nil { + t.Fatalf("set replacement total: %v", err) + } + + if err := f.repos.StorageCleanup.DeleteUploadProvenanceIfUnreferenced(ctx, orphan.ID); err != nil { + t.Fatalf("DeleteUploadProvenanceIfUnreferenced: %v", err) + } + if got, err := f.repos.Uploads.GetByID(ctx, orphan.ID); err != nil || got != nil { + t.Fatalf("upload after cleanup = %#v err=%v, want deleted", got, err) + } + itemCount, err := f.db.NewSelect().Model((*storagereplacement.Item)(nil)). + Where("id = ?", item.ID).Count(ctx) + if err != nil { + t.Fatalf("count replacement item: %v", err) + } + if itemCount != 0 { + t.Fatalf("replacement item count = %d, want deleted after settlement", itemCount) + } + if err := f.repos.Replacements.MarkItemCopied(ctx, item.ID); err != nil { + t.Fatalf("late MarkItemCopied: %v", err) + } + got, err := f.repos.Replacements.GetByID(ctx, replacement.ID) + if err != nil || got == nil || got.ItemsCopied != 0 || got.ItemsTotal != 1 { + t.Fatalf("replacement progress after late completion = %#v err=%v, want copied 0 of historical total 1", got, err) + } +} + +// The retirement gate is the last thing standing between a replacement and +// permanent data loss, so every predicate is checked independently and +// CompleteRetirement refuses on its own, whoever calls it. +func TestStorageReplacementRepo_RetirementGateBlocksEachUnsafeCondition(t *testing.T) { + f := newReplacementFixture(t, "replacement-gate", "01J000000000000000000RPL12") + ctx := context.Background() + row := f.authorize(t, "202") + target := f.readyTarget(t, row, "2002") + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + if _, _, err := f.repos.Replacements.SeedMigrationBatch(ctx, row.ID, 10); err != nil { + t.Fatalf("SeedMigrationBatch: %v", err) + } + + // Outstanding migration work blocks retirement. + gate, err := f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) + if err != nil { + t.Fatalf("EvaluateRetirementGate: %v", err) + } + if gate.Passed() || gate.WaitingItems != 1 { + t.Fatalf("gate = %#v, want it blocked by one outstanding item", gate) + } + if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 9999); !errors.Is(err, storagereplacement.ErrPrematureComplete) { + t.Fatalf("CompleteRetirement with outstanding work = %v, want ErrPrematureComplete", err) + } + + item, err := f.repos.Replacements.NextExecutableItem(ctx, row.ID) + if err != nil || item == nil { + t.Fatalf("NextExecutableItem: %#v err=%v", item, err) + } + if err := f.repos.Replacements.MarkItemCopied(ctx, item.ID); err != nil { + t.Fatalf("MarkItemCopied: %v", err) + } + + // The content is not actually on the new provider yet. + gate, err = f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) + if err != nil { + t.Fatalf("EvaluateRetirementGate: %v", err) + } + if gate.Passed() || gate.CoverageGaps != 1 { + t.Fatalf("gate = %#v, want it blocked by a coverage gap", gate) + } + + mustExec(t, f.db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id, created_at, updated_at) + VALUES (?, 0, '202', '3002', ?, ?, 'https://target.example/piece', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + f.upload.ID, model.StorageCopyTransferMethodPeerPull, model.StorageUploadCopyStatusCommitted, target.ID) + + // Everything is covered, but the service has not been terminated yet. + gate, err = f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) + if err != nil { + t.Fatalf("EvaluateRetirementGate: %v", err) + } + if !gate.Passed() { + t.Fatalf("gate = %#v, want everything except the epoch satisfied", gate) + } + if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 9999); !errors.Is(err, storagereplacement.ErrPrematureComplete) { + t.Fatalf("CompleteRetirement before termination = %v, want ErrPrematureComplete", err) + } + + if err := f.repos.Replacements.BeginRetirement(ctx, row.ID); err != nil { + t.Fatalf("BeginRetirement: %v", err) + } + if err := f.repos.Replacements.RecordTerminationEpoch(ctx, repository.RecordTerminationEpochInput{ + ReplacementID: row.ID, TxHash: "0xterminate", Epoch: 5000, + }); err != nil { + t.Fatalf("RecordTerminationEpoch: %v", err) + } + + // The chain has not reached the end of term. + if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 4999); !errors.Is(err, storagereplacement.ErrPrematureComplete) { + t.Fatalf("CompleteRetirement before the epoch = %v, want ErrPrematureComplete", err) + } + + // Another upload still writing to the source blocks retirement even now. + inFlight := startCopyHealthUpload(t, f.repos, f.bucket.ID, "01J000000000000000000RPL13", 10, "in-flight-checksum", 1) + mustExec(t, f.db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, transfer_method, status, storage_data_set_id, created_at, updated_at) + VALUES (?, 0, '101', ?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + inFlight.ID, model.StorageCopyTransferMethodPeerPull, model.StorageUploadCopyStatusCommitting, f.source.ID) + if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 5000); !errors.Is(err, storagereplacement.ErrPrematureComplete) { + t.Fatalf("CompleteRetirement with an in-flight source write = %v, want ErrPrematureComplete", err) + } + mustExec(t, f.db, `DELETE FROM storage_upload_copies WHERE upload_id = ?`, inFlight.ID) + + if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 5000); err != nil { + t.Fatalf("CompleteRetirement: %v", err) + } + source, err := f.repos.Uploads.GetDataSetBindingByID(ctx, f.source.ID) + if err != nil || source == nil || source.Status != model.StorageDataSetStatusRetired { + t.Fatalf("source = %#v err=%v, want retired", source, err) + } + done, err := f.repos.Replacements.GetByID(ctx, row.ID) + if err != nil || done == nil || done.Status != storagereplacement.StatusCompleted { + t.Fatalf("replacement = %#v err=%v, want completed", done, err) + } + // Completing twice is harmless, which keeps a retried task safe. + if err := f.repos.Replacements.CompleteRetirement(ctx, row.ID, 5000); err != nil { + t.Fatalf("second CompleteRetirement: %v", err) + } +} + +// An upload that is still in flight when seeding runs must still get migrated. +// Judging eligibility at seeding time skipped it forever, and the retirement +// coverage gate then blocked on content no item was ever created for. +func TestStorageReplacementRepo_SeedingCoversUploadsStillInFlight(t *testing.T) { + f := newReplacementFixture(t, "replacement-inflight", "01J000000000000000000RPL14") + ctx := context.Background() + row := f.authorize(t, "202") + f.readyTarget(t, row, "2002") + + // A second upload starts before activation and has not committed yet. + inFlightVersion := newObjectVersion(f.bucket.ID, "later.txt", "01J000000000000000000RPL15", 10) + inFlightVersion.Checksum = "in-flight-checksum" + if _, err := f.repos.Objects.CreateVersionAndSetCurrent(ctx, inFlightVersion); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + inFlight := startCopyHealthUpload(t, f.repos, f.bucket.ID, inFlightVersion.VersionID, 10, inFlightVersion.Checksum, 1) + if err := f.repos.Uploads.CreateUploadCopiesForBindings(ctx, inFlight.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: f.source.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: f.source.ProviderID, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + if _, _, err := f.repos.Replacements.SeedMigrationBatch(ctx, row.ID, 10); err != nil { + t.Fatalf("SeedMigrationBatch: %v", err) + } + + // The in-flight upload commits to the retiring generation afterwards. + inFlightCopy, err := f.repos.Uploads.GetUploadCopyForDataSet(ctx, inFlight.ID, f.source.ID) + if err != nil || inFlightCopy == nil { + t.Fatalf("GetUploadCopyForDataSet = %#v err=%v", inFlightCopy, err) + } + if err := f.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + StorageUploadCopyID: inFlightCopy.ID, + UploadID: inFlight.ID, + CopyIndex: 0, + PieceCID: "bafk2bzacestorhealth", + PieceID: onChainIDPtr(t, "7001"), + RetrievalURL: "https://source.example/in-flight", + }); err != nil { + t.Fatalf("MarkUploadCopyCommitted: %v", err) + } + + var seeded int + if err = f.db.NewRaw(`SELECT COUNT(*) FROM storage_replacement_items WHERE replacement_id = ? AND upload_id = ?`, + row.ID, inFlight.ID).Scan(ctx, &seeded); err != nil { + t.Fatalf("count seeded items: %v", err) + } + if seeded != 1 { + t.Fatal("the upload that was still in flight during seeding was never given migration work") + } + + // The retirement gate must therefore see it as owed work, not as a + // permanent coverage gap with no item behind it. + gate, err := f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) + if err != nil { + t.Fatalf("EvaluateRetirementGate: %v", err) + } + if gate.WaitingItems == 0 { + t.Fatalf("gate = %#v, want the in-flight upload counted as outstanding work", gate) + } +} + +// An item that can never be satisfied must reach a terminal status. Leaving it +// executable made the coordinator pick the same item forever and held the +// retirement gate open. +func TestStorageReplacementRepo_UnsatisfiableItemsSettleTerminally(t *testing.T) { + f := newReplacementFixture(t, "replacement-settle", "01J000000000000000000RPL16") + ctx := context.Background() + row := f.authorize(t, "202") + f.readyTarget(t, row, "2002") + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + if _, _, err := f.repos.Replacements.SeedMigrationBatch(ctx, row.ID, 10); err != nil { + t.Fatalf("SeedMigrationBatch: %v", err) + } + item, err := f.repos.Replacements.NextExecutableItem(ctx, row.ID) + if err != nil || item == nil { + t.Fatalf("NextExecutableItem = %#v err=%v", item, err) + } + + // The content stops being referenced before the item runs. + mustExec(t, f.db, `DELETE FROM object_versions WHERE version_id = ?`, f.version.VersionID) + + task := seedClaimedReplacementTask(t, f, row.ID) + if _, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ + ReplacementID: row.ID, + ItemID: item.ID, + TaskID: task.ID, + TaskClaimedAt: *task.ClaimedAt, + }); !errors.Is(err, storagereplacement.ErrItemCancelled) { + t.Fatalf("AcquireItem = %v, want ErrItemCancelled", err) + } + + // The decisive part: the item must not come back. + next, err := f.repos.Replacements.NextExecutableItem(ctx, row.ID) + if err != nil { + t.Fatalf("NextExecutableItem: %v", err) + } + if next != nil { + t.Fatalf("item %d is still executable after being cancelled, so the coordinator would loop on it", next.ID) + } + gate, err := f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) + if err != nil { + t.Fatalf("EvaluateRetirementGate: %v", err) + } + if gate.WaitingItems != 0 { + t.Fatalf("gate = %#v, want no outstanding items once they are settled", gate) + } +} + +// A commit rejected on the retiring generation must be reset on that generation, +// not on whichever one currently owns the slot. +func TestStorageReplacementRepo_ResetRejectedCommitTargetsTheRecordedCopy(t *testing.T) { + f := newReplacementFixture(t, "replacement-reset", "01J000000000000000000RPL17") + ctx := context.Background() + sourceCopy, err := f.repos.Uploads.GetUploadCopyForDataSet(ctx, f.upload.ID, f.source.ID) + if err != nil || sourceCopy == nil { + t.Fatalf("GetUploadCopyForDataSet = %#v err=%v", sourceCopy, err) + } + mustExec(t, f.db, `UPDATE storage_upload_copies SET status = ?, commit_transaction_id = '0xrejected' WHERE id = ?`, + model.StorageUploadCopyStatusCommitting, sourceCopy.ID) + + row := f.authorize(t, "202") + f.readyTarget(t, row, "2002") + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + + // Without the recorded copy id this resolves to the new generation and fails, + // leaving the old one stuck in committing and the retirement gate blocked. + if err := f.repos.Uploads.ResetRejectedUploadCopyCommit(ctx, repository.ResetRejectedUploadCopyCommitInput{ + StorageUploadCopyID: sourceCopy.ID, + UploadID: f.upload.ID, + CopyIndex: 0, + CommitTransactionID: "0xrejected", + LastError: "provider rejected the commit", + }); err != nil { + t.Fatalf("ResetRejectedUploadCopyCommit: %v", err) + } + got, err := f.repos.Uploads.GetUploadCopyByID(ctx, sourceCopy.ID) + if err != nil || got == nil || got.Status != model.StorageUploadCopyStatusPieceReady { + t.Fatalf("retiring copy = %#v err=%v, want it reset to piece_ready", got, err) + } +} + +// A target a later confirmation abandoned keeps costing money until its own +// service ends. Retiring it must never touch the source, which the successor +// still depends on. +func TestStorageReplacementRepo_AbandonedTargetRetiresWithoutTouchingTheSource(t *testing.T) { + f := newReplacementFixture(t, "replacement-abandoned", "01J000000000000000000RPL18") + ctx := context.Background() + first := f.authorize(t, "202") + f.readyTarget(t, first, "2002") + second := f.authorize(t, "303") + + superseded, err := f.repos.Replacements.GetByID(ctx, first.ID) + if err != nil || superseded == nil || superseded.Status != storagereplacement.StatusSuperseded { + t.Fatalf("first replacement = %#v err=%v, want superseded", superseded, err) + } + candidates, err := f.repos.Replacements.ListSupersededCleanupCandidates(ctx, 0, 10) + if err != nil { + t.Fatalf("ListSupersededCleanupCandidates: %v", err) + } + if len(candidates) != 1 || candidates[0].ID != first.ID { + t.Fatalf("cleanup candidates = %#v, want the superseded replacement", candidates) + } + + // The abandoned target holds nothing anyone depends on. + sole, err := f.repos.Replacements.CountAbandonedTargetSoleCopies(ctx, first.TargetDataSetID) + if err != nil { + t.Fatalf("CountAbandonedTargetSoleCopies: %v", err) + } + if sole != 0 { + t.Fatalf("sole copies = %d, want none on an unused target", sole) + } + if err := f.repos.Replacements.RetireAbandonedTarget(ctx, first.ID); err != nil { + t.Fatalf("RetireAbandonedTarget: %v", err) + } + + abandoned, err := f.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) + if err != nil || abandoned == nil || abandoned.Status != model.StorageDataSetStatusRetired { + t.Fatalf("abandoned target = %#v err=%v, want retired", abandoned, err) + } + source, err := f.repos.Uploads.GetDataSetBindingByID(ctx, f.source.ID) + if err != nil || source == nil || source.Status == model.StorageDataSetStatusRetired || !source.IsCurrent { + t.Fatalf("source = %#v err=%v, want it untouched and still current", source, err) + } + // The replacement record stays superseded; cleanup never rewrites it. + got, err := f.repos.Replacements.GetByID(ctx, first.ID) + if err != nil || got == nil || got.Status != storagereplacement.StatusSuperseded { + t.Fatalf("replacement = %#v err=%v, want it left superseded", got, err) + } + if second.ID == first.ID { + t.Fatal("the later confirmation reused the superseded record") + } +} + +func seedClaimedReplacementTask(t *testing.T, f *replacementFixture, replacementID int64) *model.Task { + t.Helper() + ctx := context.Background() + task := storagereplacement.NewMigrateTask(replacementID, f.bucket.ID, "", 5, time.Now()) + task.IdempotencyKey += ":acquire-test" + if err := f.repos.Tasks.Create(ctx, task); err != nil { + t.Fatalf("Create coordinator task: %v", err) + } + claimed, err := f.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) + if err != nil || claimed == nil { + t.Fatalf("ClaimReady = %#v err=%v", claimed, err) + } + return claimed +} + +// Content the retiring generation is still writing is not "never stored". A +// cancelled item never migrates, the write then commits, and the coverage gate +// blocks forever on content no item exists for. +func TestStorageReplacementRepo_InFlightSourceCopyIsParkedNotCancelled(t *testing.T) { + f := newReplacementFixture(t, "replacement-inflight-acquire", "01J000000000000000000RPL19") + ctx := context.Background() + row := f.authorize(t, "202") + f.readyTarget(t, row, "2002") + + inFlightVersion := newObjectVersion(f.bucket.ID, "pending.txt", "01J000000000000000000RPL20", 10) + inFlightVersion.Checksum = "pending-checksum" + if _, err := f.repos.Objects.CreateVersionAndSetCurrent(ctx, inFlightVersion); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + inFlight := startCopyHealthUpload(t, f.repos, f.bucket.ID, inFlightVersion.VersionID, 10, inFlightVersion.Checksum, 1) + if err := f.repos.Uploads.CreateUploadCopiesForBindings(ctx, inFlight.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: f.source.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: f.source.ProviderID, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + if _, _, err := f.repos.Replacements.SeedMigrationBatch(ctx, row.ID, 10); err != nil { + t.Fatalf("SeedMigrationBatch: %v", err) + } + + var itemID int64 + if err := f.db.NewRaw(`SELECT id FROM storage_replacement_items WHERE replacement_id = ? AND upload_id = ?`, + row.ID, inFlight.ID).Scan(ctx, &itemID); err != nil { + t.Fatalf("select in-flight item: %v", err) + } + task := seedClaimedReplacementTask(t, f, row.ID) + if _, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ + ReplacementID: row.ID, + ItemID: itemID, + TaskID: task.ID, + TaskClaimedAt: *task.ClaimedAt, + }); !errors.Is(err, storagereplacement.ErrItemDeferred) { + t.Fatalf("AcquireItem while the source is still writing = %v, want ErrItemDeferred", err) + } + + // Parked, not cancelled: it must come back once the source finishes. + var status storagereplacement.ItemStatus + if err := f.db.NewRaw(`SELECT status FROM storage_replacement_items WHERE id = ?`, itemID).Scan(ctx, &status); err != nil { + t.Fatalf("select item status: %v", err) + } + if status != storagereplacement.ItemStatusWaitingSource { + t.Fatalf("item status = %s, want waiting_source so the coordinator revisits it", status) + } + gate, err := f.repos.Replacements.EvaluateRetirementGate(ctx, row.ID, nil) + if err != nil { + t.Fatalf("EvaluateRetirementGate: %v", err) + } + if gate.WaitingItems == 0 { + t.Fatalf("gate = %#v, want the parked item to keep retirement open", gate) + } +} + +// An upload the retiring generation never stored, and never will, owes the +// target nothing. +func TestStorageReplacementRepo_ItemWithNoSourceCopyIsCancelled(t *testing.T) { + f := newReplacementFixture(t, "replacement-nosource", "01J000000000000000000RPL21") + ctx := context.Background() + row := f.authorize(t, "202") + f.readyTarget(t, row, "2002") + + elsewhereVersion := newObjectVersion(f.bucket.ID, "elsewhere.txt", "01J000000000000000000RPL22", 10) + elsewhereVersion.Checksum = "elsewhere-checksum" + if _, err := f.repos.Objects.CreateVersionAndSetCurrent(ctx, elsewhereVersion); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + elsewhere := startCopyHealthUpload(t, f.repos, f.bucket.ID, elsewhereVersion.VersionID, 10, elsewhereVersion.Checksum, 1) + + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + if _, _, err := f.repos.Replacements.SeedMigrationBatch(ctx, row.ID, 10); err != nil { + t.Fatalf("SeedMigrationBatch: %v", err) + } + var itemID int64 + if err := f.db.NewRaw(`SELECT id FROM storage_replacement_items WHERE replacement_id = ? AND upload_id = ?`, + row.ID, elsewhere.ID).Scan(ctx, &itemID); err != nil { + t.Fatalf("select item: %v", err) + } + task := seedClaimedReplacementTask(t, f, row.ID) + if _, err := f.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ + ReplacementID: row.ID, + ItemID: itemID, + TaskID: task.ID, + TaskClaimedAt: *task.ClaimedAt, + }); !errors.Is(err, storagereplacement.ErrItemCancelled) { + t.Fatalf("AcquireItem = %v, want ErrItemCancelled", err) + } + var status storagereplacement.ItemStatus + if err := f.db.NewRaw(`SELECT status FROM storage_replacement_items WHERE id = ?`, itemID).Scan(ctx, &status); err != nil { + t.Fatalf("select item status: %v", err) + } + if status != storagereplacement.ItemStatusCancelled { + t.Fatalf("item status = %s, want cancelled", status) + } +} + +// A generation some unfinished replacement is migrating into cannot become a +// source: that replacement needs it to keep owning the slot. +func TestStorageReplacementRepo_AuthorizeRejectsReplacingALiveTarget(t *testing.T) { + f := newReplacementFixture(t, "replacement-chain", "01J000000000000000000RPL23") + ctx := context.Background() + first := f.authorize(t, "202") + target := f.readyTarget(t, first, "2002") + if err := f.repos.Replacements.Activate(ctx, first.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + if err := f.repos.Replacements.MarkFailed(ctx, first.ID, nil, "migration exhausted"); err != nil { + t.Fatalf("MarkFailed: %v", err) + } + + if _, _, err := f.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: f.bucket.ID, + SourceDataSetID: target.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "404"), + ClientRequestID: "replace-live-target", + MaxRetries: 5, + }); !errors.Is(err, storagereplacement.ErrActiveReplacement) { + t.Fatalf("replacing a live replacement's target = %v, want ErrActiveReplacement", err) + } +} + +// After an activation a failure bound to the retiring generation must land on +// that generation. Resolving through the slot would either mark the replacement +// copy failed or update nothing, leaving the original stuck mid-transfer and +// holding retirement open. +func TestStorageUploadRepo_CopyFailureFollowsTheRecordedCopy(t *testing.T) { + f := newReplacementFixture(t, "replacement-failure", "01J000000000000000000RPL24") + ctx := context.Background() + sourceCopy, err := f.repos.Uploads.GetUploadCopyForDataSet(ctx, f.upload.ID, f.source.ID) + if err != nil || sourceCopy == nil { + t.Fatalf("GetUploadCopyForDataSet = %#v err=%v", sourceCopy, err) + } + mustExec(t, f.db, `UPDATE storage_upload_copies SET status = ? WHERE id = ?`, + model.StorageUploadCopyStatusPieceReady, sourceCopy.ID) + + row := f.authorize(t, "202") + target := f.readyTarget(t, row, "2002") + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + if err := f.repos.Uploads.CreateUploadCopiesForBindings(ctx, f.upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: target.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodPeerPull, + ProviderID: target.ProviderID, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings target: %v", err) + } + targetCopy, err := f.repos.Uploads.GetUploadCopyForDataSet(ctx, f.upload.ID, target.ID) + if err != nil || targetCopy == nil { + t.Fatalf("GetUploadCopyForDataSet target = %#v err=%v", targetCopy, err) + } + + if err := f.repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{ + StorageUploadCopyID: sourceCopy.ID, + UploadID: f.upload.ID, + CopyIndex: 0, + LastError: "ingress store: provider rejected the piece", + }); err != nil { + t.Fatalf("MarkUploadCopyFailed: %v", err) + } + gotSource, err := f.repos.Uploads.GetUploadCopyByID(ctx, sourceCopy.ID) + if err != nil || gotSource == nil || gotSource.Status != model.StorageUploadCopyStatusFailed { + t.Fatalf("retiring copy = %#v err=%v, want failed", gotSource, err) + } + gotTarget, err := f.repos.Uploads.GetUploadCopyByID(ctx, targetCopy.ID) + if err != nil || gotTarget == nil || gotTarget.Status != model.StorageUploadCopyStatusPending { + t.Fatalf("replacement copy = %#v err=%v, want it untouched", gotTarget, err) + } +} + +// The dedicated retry exists to resume work the worker gave up on, and the +// worker gives up by driving the coordinator task to a terminal status: +// migration exhausts its retries, cleanup fails outright. Moving the record +// back to a working status without reviving that task leaves the replacement +// permanently stuck — displayed as in progress, with the retry hidden because +// the status is no longer retryable, and nothing queued to make progress. +func TestStorageReplacementRepo_RetryMakesTheCoordinatorClaimableAgain(t *testing.T) { + for _, tc := range []struct { + name string + markStuck func(t *testing.T, f *replacementFixture, row *storagereplacement.Replacement) + taskStatus model.TaskStatus + taskType model.TaskType + key func(int64) string + }{ + { + name: "migration exhausted its retries", + markStuck: func(t *testing.T, f *replacementFixture, row *storagereplacement.Replacement) { + if err := f.repos.Replacements.MarkFailed(context.Background(), row.ID, nil, "copy replacement item: exhausted"); err != nil { + t.Fatalf("MarkFailed: %v", err) + } + }, + taskStatus: model.TaskStatusExhausted, + taskType: model.TaskTypeUpload, + key: storagereplacement.MigrateTaskKey, + }, + { + name: "cleanup could not end the service", + markStuck: func(t *testing.T, f *replacementFixture, row *storagereplacement.Replacement) { + if err := f.repos.Replacements.BeginRetirement(context.Background(), row.ID); err != nil { + t.Fatalf("BeginRetirement: %v", err) + } + if err := f.repos.Replacements.MarkCleanupAttention(context.Background(), row.ID, "payment debt"); err != nil { + t.Fatalf("MarkCleanupAttention: %v", err) + } + }, + taskStatus: model.TaskStatusFailed, + taskType: model.TaskTypeStorageCleanup, + key: storagereplacement.RetireTaskKey, + }, + } { + t.Run(tc.name, func(t *testing.T) { + f := newReplacementFixture(t, "retry-revives-"+model.NewVersionID()[:8], model.NewVersionID()) + ctx := context.Background() + row := f.authorize(t, "202") + f.readyTarget(t, row, "2002") + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + tc.markStuck(t, f, row) + + // Put the coordinator where the worker leaves it on the way in. + key := tc.key(row.ID) + if _, err := f.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewRetireTask(row.ID, f.bucket.ID, 5, time.Now())); err != nil { + t.Fatalf("seed retire coordinator: %v", err) + } + if _, err := f.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("status = ?", tc.taskStatus). + Set("retry_count = 5"). + Where("idempotency_key = ?", key). + Exec(ctx); err != nil { + t.Fatalf("mark coordinator %s: %v", tc.taskStatus, err) + } + + if _, err := f.repos.Replacements.Retry(ctx, repository.RetryReplacementInput{ReplacementID: row.ID, MaxRetries: 5}); err != nil { + t.Fatalf("Retry: %v", err) + } + + // The only assertion that matters: a worker can pick the work up. + claimed, err := f.repos.Tasks.ClaimReady(ctx, tc.taskType, time.Minute) + if err != nil { + t.Fatalf("ClaimReady %s: %v", tc.taskType, err) + } + if claimed == nil || claimed.IdempotencyKey != key { + t.Fatalf("claimed = %#v, want the resumed coordinator %s", claimed, key) + } + if claimed.RetryCount != 0 { + t.Fatalf("retry count = %d, want the operator's retry to restore the budget", claimed.RetryCount) + } + }) + } +} + +// Automatic recurrence must keep its own rule: a coordinator that gave up is +// not restarted just because recovery ran again. +func TestTaskRepo_StartupRecoveryDoesNotRestartAbandonedCoordinators(t *testing.T) { + f := newReplacementFixture(t, "startup-vs-exhausted", model.NewVersionID()) + ctx := context.Background() + row := f.authorize(t, "202") + key := storagereplacement.MigrateTaskKey(row.ID) + if _, err := f.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("status = ?", model.TaskStatusExhausted). + Where("idempotency_key = ?", key). + Exec(ctx); err != nil { + t.Fatalf("mark coordinator exhausted: %v", err) + } + if _, err := f.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewMigrateTask(row.ID, f.bucket.ID, "", 5, time.Now())); err != nil { + t.Fatalf("EnsureRecurring: %v", err) + } + claimed, err := f.repos.Tasks.ClaimReady(ctx, model.TaskTypeUpload, time.Minute) + if err != nil { + t.Fatalf("ClaimReady: %v", err) + } + if claimed != nil { + t.Fatal("startup recovery restarted a coordinator that had given up and needs an operator") + } +} + +// Abandoned-target cleanup cannot use the Data Sets retry: the replacement is +// superseded. ResumeCoordinator is the only way a failed leftover terminator +// becomes claimable again after a restart. +func TestTaskRepo_ResumeCoordinatorRevivesAbandonedTargetCleanup(t *testing.T) { + f := newReplacementFixture(t, "resume-abandoned-cleanup", model.NewVersionID()) + ctx := context.Background() + task := storagereplacement.NewAbandonedTargetTask(11, f.bucket.ID, 5, time.Now()) + if err := f.repos.Tasks.Create(ctx, task); err != nil { + t.Fatalf("Create: %v", err) + } + if _, err := f.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("status = ?", model.TaskStatusFailed). + Set("retry_count = 5"). + Where("id = ?", task.ID). + Exec(ctx); err != nil { + t.Fatalf("mark failed: %v", err) + } + if _, err := f.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewAbandonedTargetTask(11, f.bucket.ID, 5, time.Now())); err != nil { + t.Fatalf("EnsureRecurring: %v", err) + } + claimed, err := f.repos.Tasks.ClaimReady(ctx, model.TaskTypeStorageCleanup, time.Minute) + if err != nil { + t.Fatalf("ClaimReady after EnsureRecurring: %v", err) + } + if claimed != nil { + t.Fatal("automatic recurrence revived abandoned cleanup that had failed") + } + if _, err := f.repos.Tasks.ResumeCoordinator(ctx, storagereplacement.NewAbandonedTargetTask(11, f.bucket.ID, 5, time.Now())); err != nil { + t.Fatalf("ResumeCoordinator: %v", err) + } + claimed, err = f.repos.Tasks.ClaimReady(ctx, model.TaskTypeStorageCleanup, time.Minute) + if err != nil { + t.Fatalf("ClaimReady after ResumeCoordinator: %v", err) + } + if claimed == nil || claimed.IdempotencyKey != storagereplacement.AbandonedTargetTaskKey(11) { + t.Fatalf("claimed = %#v, want the abandoned-target coordinator", claimed) + } +} + +// Mutual exclusion is per copy row. A pending item, or a running item that has +// not attached its target copy yet, must not make every other upload on the +// replica stand down. +func TestStorageReplacementRepo_HeldItemCopyIDIsTheRunningTargetCopy(t *testing.T) { + f := newReplacementFixture(t, "replacement-held-copy", "01J000000000000000000RPL21") + ctx := context.Background() + row := f.authorize(t, "202") + f.readyTarget(t, row, "2002") + if err := f.repos.Replacements.Activate(ctx, row.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + if _, _, err := f.repos.Replacements.SeedMigrationBatch(ctx, row.ID, 10); err != nil { + t.Fatalf("SeedMigrationBatch: %v", err) + } + + held, err := f.repos.Replacements.HeldItemCopyID(ctx, row.ID) + if err != nil || held != 0 { + t.Fatalf("HeldItemCopyID before claim = %d err=%v, want 0", held, err) + } + + item, err := f.repos.Replacements.NextExecutableItem(ctx, row.ID) + if err != nil || item == nil { + t.Fatalf("NextExecutableItem = %#v err=%v", item, err) + } + copyRow, err := f.repos.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ + ReplacementID: row.ID, + ItemID: item.ID, + UploadID: item.UploadID, + }) + if err != nil || copyRow == nil { + t.Fatalf("AttachTargetCopy = %#v err=%v", copyRow, err) + } + held, err = f.repos.Replacements.HeldItemCopyID(ctx, row.ID) + if err != nil || held != 0 { + t.Fatalf("HeldItemCopyID after attach = %d err=%v, want 0 until the item is running", held, err) + } + + mustExec(t, f.db, `UPDATE storage_replacement_items SET status = ? WHERE id = ?`, + storagereplacement.ItemStatusRunning, item.ID) + held, err = f.repos.Replacements.HeldItemCopyID(ctx, row.ID) + if err != nil || held != copyRow.ID { + t.Fatalf("HeldItemCopyID while running = %d err=%v, want copy %d", held, err, copyRow.ID) + } + + if err := f.repos.Replacements.MarkItemCopied(ctx, item.ID); err != nil { + t.Fatalf("MarkItemCopied: %v", err) + } + held, err = f.repos.Replacements.HeldItemCopyID(ctx, row.ID) + if err != nil || held != 0 { + t.Fatalf("HeldItemCopyID after copy = %d err=%v, want 0", held, err) + } +} diff --git a/internal/db/repository/storage_upload_health_repo.go b/internal/db/repository/storage_upload_health_repo.go index 72beacd..d122145 100644 --- a/internal/db/repository/storage_upload_health_repo.go +++ b/internal/db/repository/storage_upload_health_repo.go @@ -230,7 +230,7 @@ func (r *BunStorageUploadRepo) listBucketStorageHealthReadableAlternativeCounts( } query := fmt.Sprintf(`SELECT object_version.version_id, - COUNT(*) AS readable_alternative_count + COUNT(DISTINCT readable_data_set.copy_index) AS readable_alternative_count FROM object_versions AS object_version JOIN storage_upload_copies AS readable_copy ON readable_copy.upload_id = object_version.storage_upload_id JOIN storage_uploads AS readable_upload @@ -311,19 +311,13 @@ func validateBucketStorageHealthAffectedVersionMarkers(input BucketStorageHealth } // This records locally known readable committed copies; it is not a data-safety guarantee. +// The single placeholder binds the observation freshness cutoff. func readableCommittedStorageCopySQL() string { return fmt.Sprintf(`readable_upload.piece_cid IS NOT NULL AND readable_upload.piece_cid <> '' - AND readable_copy.status = %s - AND readable_copy.storage_data_set_id IS NOT NULL - AND readable_copy.provider_id IS NOT NULL AND readable_copy.provider_id <> '' - AND readable_data_set.data_set_id IS NOT NULL AND readable_data_set.data_set_id <> '' - AND readable_data_set.status IN (%s) + AND %s AND readable_observation.status = %s - AND readable_observation.last_checked_at >= ? - AND readable_copy.piece_id IS NOT NULL AND readable_copy.piece_id <> '' - AND readable_copy.retrieval_url IS NOT NULL AND readable_copy.retrieval_url <> ''`, - storageHealthCommittedCopyStatusSQL(), - storageHealthReadyDataSetStatusListSQL(), + AND readable_observation.last_checked_at >= ?`, + readableCommittedCopyPredicateSQL("readable_copy", "readable_data_set"), storageHealthAvailableObservationStatusSQL(), ) } diff --git a/internal/db/repository/storage_upload_repo.go b/internal/db/repository/storage_upload_repo.go index 10a90eb..fd29b3b 100644 --- a/internal/db/repository/storage_upload_repo.go +++ b/internal/db/repository/storage_upload_repo.go @@ -9,6 +9,7 @@ import ( "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/observability" + "github.com/strahe/synaps3/internal/storagereplacement" "github.com/strahe/synaps3/internal/types" "github.com/uptrace/bun" ) @@ -231,6 +232,22 @@ func (r *BunStorageUploadRepo) ListCopies(ctx context.Context, uploadID int64) ( return copies, nil } +// CountCurrentGenerationCopySlots counts logical replica slots, not physical +// generation rows. An unbound copy belongs to its slot until it is assigned. +func (r *BunStorageUploadRepo) CountCurrentGenerationCopySlots(ctx context.Context, uploadID int64) (int, error) { + if uploadID <= 0 { + return 0, fmt.Errorf("counting current upload copy slots: %w", ErrInvalidInput) + } + var count int + query := fmt.Sprintf(`SELECT COUNT(DISTINCT storage_copy.copy_index) + FROM storage_upload_copies AS storage_copy + WHERE storage_copy.upload_id = ? AND %s`, currentGenerationCopySQL("storage_copy")) + if err := r.db.NewRaw(query, uploadID).Scan(ctx, &count); err != nil { + return 0, fmt.Errorf("counting current upload copy slots: %w", err) + } + return count, nil +} + func shouldRetryUploadFailureAppend(err error) bool { return isUniqueViolation(err) || isSQLiteBusy(err) } @@ -275,16 +292,10 @@ func (r *BunStorageUploadRepo) ListReadableCommittedCopies(ctx context.Context, JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id WHERE storage_copy.upload_id = ? AND storage_upload.piece_cid IS NOT NULL AND storage_upload.piece_cid <> '' - AND storage_copy.status = ? - AND storage_copy.storage_data_set_id IS NOT NULL - AND storage_copy.provider_id IS NOT NULL AND storage_copy.provider_id <> '' - AND storage_data_set.data_set_id IS NOT NULL AND storage_data_set.data_set_id <> '' - AND storage_data_set.status IN (%s) - AND storage_copy.piece_id IS NOT NULL AND storage_copy.piece_id <> '' - AND storage_copy.retrieval_url IS NOT NULL AND storage_copy.retrieval_url <> ''`, - storageHealthReadyDataSetStatusListSQL(), + AND %s`, + readableCommittedCopyPredicateSQL("storage_copy", "storage_data_set"), ) - args := []interface{}{uploadID, model.StorageUploadCopyStatusCommitted} + args := []interface{}{uploadID} query += " ORDER BY storage_copy.copy_index ASC" if err := r.db.NewRaw(query, args...).Scan(ctx, &copies); err != nil { if err == sql.ErrNoRows { @@ -296,7 +307,7 @@ func (r *BunStorageUploadRepo) ListReadableCommittedCopies(ctx context.Context, } func (r *BunStorageUploadRepo) HasReadableCommittedCopy(ctx context.Context, uploadID int64) (bool, error) { - count, err := countReadableCommittedCopies(ctx, r.db, uploadID) + count, err := countReadableReplicaSlots(ctx, r.db, uploadID) if err != nil { return false, err } @@ -550,12 +561,14 @@ func (r *BunStorageUploadRepo) listBucketStorageHealthReasonCodes(ctx context.Co return out, nil } +// ListDataSetBindings returns every generation, including retired ones, because +// callers need the full provider history as well as the current write targets. func (r *BunStorageUploadRepo) ListDataSetBindings(ctx context.Context, bucketID int64) ([]model.StorageDataSet, error) { var bindings []model.StorageDataSet if err := r.db.NewSelect(). Model(&bindings). Where("bucket_id = ?", bucketID). - OrderExpr("copy_index ASC"). + OrderExpr("copy_index ASC, generation ASC"). Scan(ctx); err != nil { return nil, fmt.Errorf("listing storage data set bindings: %w", err) } @@ -569,6 +582,8 @@ func (r *BunStorageUploadRepo) ListDataSetSummaries(ctx context.Context, bucketI storage_data_set.bucket_id, bucket.name AS bucket_name, storage_data_set.copy_index, + storage_data_set.generation, + storage_data_set.is_current, storage_data_set.provider_id, storage_data_set.data_set_id, storage_data_set.client_data_set_id, @@ -619,7 +634,7 @@ func (r *BunStorageUploadRepo) ListDataSetSummaries(ctx context.Context, bucketI GROUP BY storage_copy.storage_data_set_id ) AS version_stats ON version_stats.storage_data_set_id = storage_data_set.id WHERE (? = 0 OR storage_data_set.bucket_id = ?) - ORDER BY bucket.name ASC, storage_data_set.copy_index ASC`, + ORDER BY bucket.name ASC, storage_data_set.copy_index ASC, storage_data_set.generation ASC`, storageHealthReadyDataSetStatusListSQL(), storageHealthCommittedCopyStatusSQL(), storageHealthCommittedCopyStatusSQL(), @@ -630,11 +645,13 @@ func (r *BunStorageUploadRepo) ListDataSetSummaries(ctx context.Context, bucketI return summaries, nil } +// GetDataSetBindingByCopyIndex returns the generation that currently owns the +// slot. Historical generations stay readable but never receive new writes. func (r *BunStorageUploadRepo) GetDataSetBindingByCopyIndex(ctx context.Context, bucketID int64, copyIndex int) (*model.StorageDataSet, error) { binding := new(model.StorageDataSet) err := r.db.NewSelect(). Model(binding). - Where("bucket_id = ? AND copy_index = ?", bucketID, copyIndex). + Where("bucket_id = ? AND copy_index = ? AND is_current", bucketID, copyIndex). Scan(ctx) if err != nil { if err == sql.ErrNoRows { @@ -698,6 +715,11 @@ func (r *BunStorageUploadRepo) MarkDataSetReady(ctx context.Context, input MarkD }) } +// RecoverDataSet restores a quarantined binding once storage confirms it is +// usable again. A generation an operator is actively replacing is never +// revived, because bringing it back would fight the approved migration. +// Terminally failed or attention-holding replacements do not block recovery, so +// a slot whose replacement gave up can still repair in place. func (r *BunStorageUploadRepo) RecoverDataSet(ctx context.Context, input MarkDataSetReadyInput) (bool, error) { if input.ID <= 0 || input.DataSetID.IsZero() { return false, fmt.Errorf("recovering storage data set: %w", ErrInvalidInput) @@ -716,6 +738,15 @@ func (r *BunStorageUploadRepo) RecoverDataSet(ctx context.Context, input MarkDat model.StorageDataSetStatusUnavailable, model.StorageDataSetStatusReady, })). + Where(`NOT EXISTS ( + SELECT 1 FROM storage_replacements AS blocking_replacement + WHERE blocking_replacement.source_data_set_id = storage_data_set.id + AND blocking_replacement.status IN (?, ?, ?, ?) + )`, + storagereplacement.StatusPreparingTarget, + storagereplacement.StatusMigrating, + storagereplacement.StatusWaiting, + storagereplacement.StatusRetiring). Exec(ctx) if err != nil { return false, fmt.Errorf("recovering storage data set: %w", err) @@ -908,9 +939,11 @@ func (r *BunStorageUploadRepo) CreateUploadCopiesForBindings(ctx context.Context StorageDataSetID: &input.StorageDataSetID, IsNewDataSet: isNewDataSet, } + // Copies are unique per concrete data set so one upload can hold both + // generations of a slot while a replacement migrates. if _, err := db.NewInsert(). Model(copyRow). - On("CONFLICT (upload_id, copy_index) DO NOTHING"). + On("CONFLICT (upload_id, storage_data_set_id) WHERE storage_data_set_id IS NOT NULL DO NOTHING"). Exec(ctx); err != nil { return fmt.Errorf("creating storage upload copy row: %w", err) } @@ -919,11 +952,17 @@ func (r *BunStorageUploadRepo) CreateUploadCopiesForBindings(ctx context.Context }) } +// GetUploadCopy resolves a slot to the copy on its current generation, so a +// task that carries only (upload, slot) can never address a replaced or a +// not-yet-activated generation. A bound copy wins over an unbound one. func (r *BunStorageUploadRepo) GetUploadCopy(ctx context.Context, uploadID int64, copyIndex int) (*model.StorageUploadCopy, error) { copyRow := new(model.StorageUploadCopy) err := r.db.NewSelect(). Model(copyRow). - Where("upload_id = ? AND copy_index = ?", uploadID, copyIndex). + Where("storage_upload_copy.upload_id = ? AND storage_upload_copy.copy_index = ?", uploadID, copyIndex). + Where(currentGenerationCopySQL("storage_upload_copy")). + OrderExpr("(storage_upload_copy.storage_data_set_id IS NULL) ASC"). + Limit(1). Scan(ctx) if err != nil { if err == sql.ErrNoRows { @@ -934,6 +973,26 @@ func (r *BunStorageUploadRepo) GetUploadCopy(ctx context.Context, uploadID int64 return copyRow, nil } +// GetUploadCopyForDataSet addresses one concrete generation, which is how +// replacement work targets the new provider while the old one still exists. +func (r *BunStorageUploadRepo) GetUploadCopyForDataSet(ctx context.Context, uploadID, storageDataSetID int64) (*model.StorageUploadCopy, error) { + if uploadID <= 0 || storageDataSetID <= 0 { + return nil, fmt.Errorf("selecting storage upload copy for data set: %w", ErrInvalidInput) + } + copyRow := new(model.StorageUploadCopy) + err := r.db.NewSelect(). + Model(copyRow). + Where("upload_id = ? AND storage_data_set_id = ?", uploadID, storageDataSetID). + Scan(ctx) + if err != nil { + if err == sql.ErrNoRows { + return nil, nil + } + return nil, fmt.Errorf("selecting storage upload copy for data set: %w", err) + } + return copyRow, nil +} + func (r *BunStorageUploadRepo) GetUploadCopyByID(ctx context.Context, id int64) (*model.StorageUploadCopy, error) { copyRow := new(model.StorageUploadCopy) err := r.db.NewSelect().Model(copyRow).Where("id = ?", id).Scan(ctx) @@ -1176,20 +1235,19 @@ func (r *BunStorageUploadRepo) NextFinalizableCopyForDataSet(ctx context.Context AND pending_version.state = ? ) AND ( - SELECT COUNT(*) + SELECT COUNT(DISTINCT readable_data_set.copy_index) FROM storage_upload_copies AS readable_copy JOIN storage_data_sets AS readable_data_set ON readable_data_set.id = readable_copy.storage_data_set_id WHERE readable_copy.upload_id = storage_upload.id - AND readable_copy.status = ? - AND readable_copy.storage_data_set_id IS NOT NULL - AND readable_copy.provider_id IS NOT NULL AND readable_copy.provider_id <> '' - AND readable_data_set.data_set_id IS NOT NULL AND readable_data_set.data_set_id <> '' - AND (readable_data_set.status IN (%s) OR readable_data_set.id = ?) - AND readable_copy.piece_id IS NOT NULL AND readable_copy.piece_id <> '' - AND readable_copy.retrieval_url IS NOT NULL AND readable_copy.retrieval_url <> '' + AND %s ) >= storage_upload.requested_copies ORDER BY storage_copy.id ASC - LIMIT 1`, storageHealthReadyDataSetStatusListSQL()) + LIMIT 1`, readableCommittedCopyPredicateWithDataSetStatusSQL( + "readable_copy", + "readable_data_set", + fmt.Sprintf("(readable_data_set.status IN (%s) OR readable_data_set.id = ?)", + storageHealthReadyDataSetStatusListSQL()), + )) err := r.db.NewRaw( query, storageDataSetID, @@ -1198,7 +1256,6 @@ func (r *BunStorageUploadRepo) NextFinalizableCopyForDataSet(ctx context.Context model.StorageUploadStatusIngressReady, model.StorageUploadStatusReadable, model.ObjectStateReplicating, - model.StorageUploadCopyStatusCommitted, storageDataSetID, ).Scan(ctx, copyRow) if err != nil { @@ -1306,6 +1363,7 @@ func (r *BunStorageUploadRepo) ReassignIngressCopy(ctx context.Context, uploadID Where("storage_upload_copy.status = ?", model.StorageUploadCopyStatusPending). Where("storage_upload_copy.transfer_method = ?", model.StorageCopyTransferMethodPeerPull). Where("storage_data_set.status = ?", model.StorageDataSetStatusReady). + Where("storage_data_set.is_current"). OrderExpr("storage_upload_copy.copy_index ASC"). Limit(1). Scan(ctx) @@ -1315,10 +1373,14 @@ func (r *BunStorageUploadRepo) ReassignIngressCopy(ctx context.Context, uploadID if err != nil { return fmt.Errorf("selecting alternate ingress copy: %w", err) } + unavailableCopyID, err := resolveSlotCopyID(ctx, db, uploadID, unavailableCopyIndex) + if err != nil { + return err + } res, err := db.NewUpdate(). Model((*model.StorageUploadCopy)(nil)). Set("transfer_method = ?", model.StorageCopyTransferMethodPeerPull). - Where("upload_id = ? AND copy_index = ?", uploadID, unavailableCopyIndex). + Where("id = ?", unavailableCopyID). Where("transfer_method = ?", model.StorageCopyTransferMethodIngress). Where("status <> ?", model.StorageUploadCopyStatusCommitted). Where("NOT (status = ? AND commit_transaction_id IS NOT NULL AND commit_transaction_id <> '')", model.StorageUploadCopyStatusCommitting). @@ -1363,6 +1425,10 @@ func (r *BunStorageUploadRepo) MarkUploadCopyPieceReady(ctx context.Context, inp if err := lockStorageUploadForCopyMutation(ctx, db, input.UploadID); err != nil { return fmt.Errorf("locking storage upload for piece-ready copy: %w", err) } + copyID, err := slotCopyTarget(ctx, db, input.StorageUploadCopyID, input.UploadID, input.CopyIndex) + if err != nil { + return err + } now := time.Now() q := db.NewUpdate(). Model((*model.StorageUploadCopy)(nil)). @@ -1371,11 +1437,10 @@ func (r *BunStorageUploadRepo) MarkUploadCopyPieceReady(ctx context.Context, inp Set("retrieval_url = COALESCE(?, retrieval_url)", nullableString(input.RetrievalURL)). Set("last_error = NULL"). Set("updated_at = ?", now). - Where("upload_id = ? AND copy_index = ?", input.UploadID, input.CopyIndex). + Where("id = ?", copyID). Where("status <> ?", model.StorageUploadCopyStatusCommitted) - if input.StorageUploadCopyID > 0 { + if input.RequireEligibleCopy { q = q. - Where("id = ?", input.StorageUploadCopyID). Where("status <> ?", model.StorageUploadCopyStatusFailed). Where(liveObjectVersionExistsForUploadSQL(), input.UploadID, false) } @@ -1385,7 +1450,7 @@ func (r *BunStorageUploadRepo) MarkUploadCopyPieceReady(ctx context.Context, inp } rows, _ := res.RowsAffected() if rows == 0 { - if input.StorageUploadCopyID > 0 { + if input.RequireEligibleCopy { return fmt.Errorf("marking storage upload copy piece ready: %w", ErrConflict) } return nil @@ -1417,6 +1482,10 @@ func (r *BunStorageUploadRepo) MarkUploadCopyCommitting(ctx context.Context, inp if err := lockStorageUploadForCopyMutation(ctx, db, input.UploadID); err != nil { return fmt.Errorf("locking storage upload for committing copy: %w", err) } + copyID, err := slotCopyTarget(ctx, db, input.StorageUploadCopyID, input.UploadID, input.CopyIndex) + if err != nil { + return err + } q := db.NewUpdate(). Model((*model.StorageUploadCopy)(nil)). Set("status = CASE WHEN ? IS NOT NULL THEN ? ELSE status END", nullableString(input.CommitTransactionID), model.StorageUploadCopyStatusCommitting). @@ -1424,10 +1493,9 @@ func (r *BunStorageUploadRepo) MarkUploadCopyCommitting(ctx context.Context, inp Set("commit_transaction_id = COALESCE(?, commit_transaction_id)", nullableString(input.CommitTransactionID)). Set("last_error = NULL"). Set("updated_at = ?", time.Now()). - Where("upload_id = ? AND copy_index = ?", input.UploadID, input.CopyIndex) - if input.StorageUploadCopyID > 0 { + Where("id = ?", copyID) + if input.RequireEligibleCopy { q = q. - Where("id = ?", input.StorageUploadCopyID). Where("status <> ?", model.StorageUploadCopyStatusFailed). Where(liveObjectVersionExistsForUploadSQL(), input.UploadID, false) } @@ -1435,7 +1503,7 @@ func (r *BunStorageUploadRepo) MarkUploadCopyCommitting(ctx context.Context, inp if err != nil { return fmt.Errorf("marking storage upload copy committing: %w", err) } - if input.StorageUploadCopyID > 0 { + if input.RequireEligibleCopy { rows, _ := res.RowsAffected() if rows == 0 { return fmt.Errorf("marking storage upload copy committing: %w", ErrConflict) @@ -1449,6 +1517,10 @@ func (r *BunStorageUploadRepo) ResetRejectedUploadCopyCommit(ctx context.Context if input.UploadID <= 0 || input.CopyIndex < 0 || input.CommitTransactionID == "" { return fmt.Errorf("resetting rejected storage upload commit: %w", ErrInvalidInput) } + copyID, err := slotCopyTarget(ctx, r.db, input.StorageUploadCopyID, input.UploadID, input.CopyIndex) + if err != nil { + return err + } res, err := r.db.NewUpdate(). Model((*model.StorageUploadCopy)(nil)). Set("status = ?", model.StorageUploadCopyStatusPieceReady). @@ -1456,7 +1528,7 @@ func (r *BunStorageUploadRepo) ResetRejectedUploadCopyCommit(ctx context.Context Set("commit_transaction_id = NULL"). Set("last_error = ?", input.LastError). Set("updated_at = ?", time.Now()). - Where("upload_id = ? AND copy_index = ?", input.UploadID, input.CopyIndex). + Where("id = ?", copyID). Where("status = ?", model.StorageUploadCopyStatusCommitting). Where("commit_transaction_id = ?", input.CommitTransactionID). Exec(ctx) @@ -1486,6 +1558,10 @@ func (r *BunStorageUploadRepo) MarkUploadCopyCommitted(ctx context.Context, inpu if err != nil { return err } + copyID, err := slotCopyTarget(ctx, db, input.StorageUploadCopyID, input.UploadID, input.CopyIndex) + if err != nil { + return err + } q := db.NewUpdate(). Model((*model.StorageUploadCopy)(nil)). Set("status = ?", model.StorageUploadCopyStatusCommitted). @@ -1496,10 +1572,9 @@ func (r *BunStorageUploadRepo) MarkUploadCopyCommitted(ctx context.Context, inpu Set("commit_transaction_id = COALESCE(?, commit_transaction_id)", nullableString(input.CommitTransactionID)). Set("last_error = NULL"). Set("updated_at = ?", now). - Where("upload_id = ? AND copy_index = ?", input.UploadID, input.CopyIndex) - if input.StorageUploadCopyID > 0 { + Where("id = ?", copyID) + if input.RequireEligibleCopy { q = q. - Where("id = ?", input.StorageUploadCopyID). Where("status <> ?", model.StorageUploadCopyStatusFailed). Where(liveObjectVersionExistsForUploadSQL(), input.UploadID, false) } @@ -1509,7 +1584,7 @@ func (r *BunStorageUploadRepo) MarkUploadCopyCommitted(ctx context.Context, inpu } rows, _ := res.RowsAffected() if rows == 0 { - if input.StorageUploadCopyID > 0 { + if input.RequireEligibleCopy { return fmt.Errorf("marking storage upload copy committed: %w", ErrConflict) } return fmt.Errorf("marking storage upload copy committed: %w", ErrNotFound) @@ -1532,18 +1607,23 @@ func liveObjectVersionExistsForUploadSQL() string { )` } -func (r *BunStorageUploadRepo) MarkUploadCopyFailed(ctx context.Context, uploadID int64, copyIndex int, lastError string) error { +func (r *BunStorageUploadRepo) MarkUploadCopyFailed(ctx context.Context, input MarkUploadCopyFailedInput) error { + uploadID, copyIndex, lastError := input.UploadID, input.CopyIndex, input.LastError return r.runMaybeTx(ctx, func(db bun.IDB) error { if err := lockStorageUploadForCopyMutation(ctx, db, uploadID); err != nil { return fmt.Errorf("locking storage upload for failed copy: %w", err) } + copyID, err := slotCopyTarget(ctx, db, input.StorageUploadCopyID, uploadID, copyIndex) + if err != nil { + return err + } now := time.Now() res, err := db.NewUpdate(). Model((*model.StorageUploadCopy)(nil)). Set("status = ?", model.StorageUploadCopyStatusFailed). Set("last_error = ?", lastError). Set("updated_at = ?", now). - Where("upload_id = ? AND copy_index = ?", uploadID, copyIndex). + Where("id = ?", copyID). Where("status <> ?", model.StorageUploadCopyStatusCommitted). Where("NOT (status = ? AND commit_transaction_id IS NOT NULL AND commit_transaction_id <> '')", model.StorageUploadCopyStatusCommitting). Exec(ctx) @@ -1557,7 +1637,7 @@ func (r *BunStorageUploadRepo) MarkUploadCopyFailed(ctx context.Context, uploadI if rows == 0 { submittedCount, countErr := db.NewSelect(). Model((*model.StorageUploadCopy)(nil)). - Where("upload_id = ? AND copy_index = ?", uploadID, copyIndex). + Where("id = ?", copyID). Where("status = ?", model.StorageUploadCopyStatusCommitting). Where("commit_transaction_id IS NOT NULL AND commit_transaction_id <> ''"). Count(ctx) @@ -1569,7 +1649,7 @@ func (r *BunStorageUploadRepo) MarkUploadCopyFailed(ctx context.Context, uploadI } return nil } - readableCount, err := countReadableCommittedCopies(ctx, db, uploadID) + readableCount, err := countReadableReplicaSlots(ctx, db, uploadID) if err != nil { return err } @@ -1742,7 +1822,7 @@ func (r *BunStorageUploadRepo) FinalizeUploadIfTargetCopiesMet(ctx context.Conte if upload == nil || upload.Status == model.StorageUploadStatusRejected || upload.Status == model.StorageUploadStatusSuperseded { return fmt.Errorf("storage upload %d cannot be finalized: %w", input.UploadID, ErrConflict) } - readable, err := countReadableCommittedCopies(ctx, db, input.UploadID) + readable, err := countReadableReplicaSlots(ctx, db, input.UploadID) if err != nil { return err } @@ -1822,7 +1902,7 @@ func requireCurrentMinimumDurableCopies(ctx context.Context, db bun.IDB, upload return fmt.Errorf("loading storage upload bucket durability policy: %w", err) } minimum := minimumDurableCopiesForUpload(bucket, upload.RequestedCopies) - readable, err := countReadableCommittedCopies(ctx, db, upload.ID) + readable, err := countReadableReplicaSlots(ctx, db, upload.ID) if err != nil { return err } @@ -1892,12 +1972,7 @@ func (r *BunStorageUploadRepo) SetAcceptError(ctx context.Context, uploadID int6 } func (r *BunStorageUploadRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) error { - if db, ok := r.db.(*bun.DB); ok { - return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { - return fn(tx) - }) - } - return fn(r.db) + return runMaybeTx(ctx, r.db, fn) } func (r *BunStorageUploadRepo) findActiveUploadBySourceVersion(ctx context.Context, versionID string) (*model.StorageUpload, error) { @@ -1918,14 +1993,30 @@ func (r *BunStorageUploadRepo) findActiveUploadBySourceVersion(ctx context.Conte return upload, nil } +// Generations are numbered per replica slot and never reused, so a retired +// generation stays distinguishable from the one that replaced it. +func nextDataSetGeneration(ctx context.Context, db bun.IDB, bucketID int64, copyIndex int) (int, error) { + var generation int + err := db.NewRaw( + `SELECT COALESCE(MAX(generation), 0) + 1 FROM storage_data_sets WHERE bucket_id = ? AND copy_index = ?`, + bucketID, copyIndex, + ).Scan(ctx, &generation) + if err != nil { + return 0, fmt.Errorf("selecting next storage data set generation: %w", err) + } + return generation, nil +} + func ensureDataSetBinding(ctx context.Context, db bun.IDB, input EnsureDataSetBindingInput) (*model.StorageDataSet, error) { if input.BucketID == 0 || input.ProviderID.IsZero() || input.CopyIndex < 0 { return nil, fmt.Errorf("invalid storage data set binding input: %w", ErrInvalidInput) } + // Only live generations reserve a provider or a slot; a retired generation + // leaves both free so an operator can reuse a provider they used before. existingByProvider := new(model.StorageDataSet) err := db.NewSelect(). Model(existingByProvider). - Where("bucket_id = ? AND provider_id = ?", input.BucketID, input.ProviderID). + Where("bucket_id = ? AND provider_id = ? AND is_current", input.BucketID, input.ProviderID). Scan(ctx) if err == nil { if existingByProvider.CopyIndex != input.CopyIndex { @@ -1939,7 +2030,7 @@ func ensureDataSetBinding(ctx context.Context, db bun.IDB, input EnsureDataSetBi existingByIndex := new(model.StorageDataSet) err = db.NewSelect(). Model(existingByIndex). - Where("bucket_id = ? AND copy_index = ?", input.BucketID, input.CopyIndex). + Where("bucket_id = ? AND copy_index = ? AND is_current", input.BucketID, input.CopyIndex). Scan(ctx) if err == nil { return nil, fmt.Errorf("copy_index %d already bound to provider %s: %w", input.CopyIndex, existingByIndex.ProviderID, ErrAlreadyExists) @@ -1947,11 +2038,17 @@ func ensureDataSetBinding(ctx context.Context, db bun.IDB, input EnsureDataSetBi if err != sql.ErrNoRows { return nil, fmt.Errorf("selecting storage data set by copy index: %w", err) } + generation, err := nextDataSetGeneration(ctx, db, input.BucketID, input.CopyIndex) + if err != nil { + return nil, err + } now := time.Now() binding := &model.StorageDataSet{ BucketID: input.BucketID, ProviderID: input.ProviderID, CopyIndex: input.CopyIndex, + Generation: generation, + IsCurrent: true, Status: model.StorageDataSetStatusPending, CreatedByUploadID: nullableInt64(input.CreatedByUploadID), LastUsedUploadID: nullableInt64(input.CreatedByUploadID), @@ -1970,7 +2067,7 @@ func ensureDataSetBinding(ctx context.Context, db bun.IDB, input EnsureDataSetBi existing := new(model.StorageDataSet) selectErr := db.NewSelect(). Model(existing). - Where("bucket_id = ? AND provider_id = ?", input.BucketID, input.ProviderID). + Where("bucket_id = ? AND provider_id = ? AND is_current", input.BucketID, input.ProviderID). Scan(ctx) if selectErr == nil && existing.CopyIndex == input.CopyIndex { return existing, nil @@ -2023,33 +2120,25 @@ func markDataSetReady(ctx context.Context, db bun.IDB, id int64, uploadID int64, return fmt.Errorf("provider data set already bound to another bucket: %w", ErrAlreadyExists) } -func countReadableCommittedCopies(ctx context.Context, db bun.IDB, uploadID int64) (int, error) { +// The result counts logical replica slots, so several data set generations of +// one slot never inflate an upload's durability. +func countReadableReplicaSlots(ctx context.Context, db bun.IDB, uploadID int64) (int, error) { var row struct { Count int `bun:"count"` } - err := db.NewRaw(fmt.Sprintf(`SELECT COUNT(*) AS count - FROM storage_upload_copies AS storage_copy - JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - WHERE storage_copy.upload_id = ? - AND storage_copy.status = ? - AND storage_copy.storage_data_set_id IS NOT NULL - AND storage_copy.provider_id IS NOT NULL AND storage_copy.provider_id <> '' - AND storage_data_set.data_set_id IS NOT NULL AND storage_data_set.data_set_id <> '' - AND storage_data_set.status IN (%s) - AND storage_copy.piece_id IS NOT NULL AND storage_copy.piece_id <> '' - AND storage_copy.retrieval_url IS NOT NULL AND storage_copy.retrieval_url <> ''`, - storageHealthReadyDataSetStatusListSQL(), + err := db.NewRaw(fmt.Sprintf(`SELECT %s AS count`, + distinctReadableSlotCountSQL("storage_copy", "storage_data_set", "?"), ), - uploadID, model.StorageUploadCopyStatusCommitted, + uploadID, ).Scan(ctx, &row) if err != nil { - return 0, fmt.Errorf("counting readable storage upload copies: %w", err) + return 0, fmt.Errorf("counting readable replica slots: %w", err) } return row.Count, nil } func requireReadableCommittedCopy(ctx context.Context, db bun.IDB, uploadID int64) error { - count, err := countReadableCommittedCopies(ctx, db, uploadID) + count, err := countReadableReplicaSlots(ctx, db, uploadID) if err != nil { return err } @@ -2059,6 +2148,43 @@ func requireReadableCommittedCopy(ctx context.Context, db bun.IDB, uploadID int6 return nil } +// Addressing a copy by slot alone became ambiguous once a slot can own several +// generations, so every write resolves to one concrete row first. Returning +// zero means the slot has no copy yet; an ambiguous slot is a conflict rather +// than a silent multi-row update. +// slotCopyTarget picks the concrete copy a mutation must touch. Zero means the +// slot has no copy, which every caller already handles as "no rows updated". +func slotCopyTarget(ctx context.Context, db bun.IDB, copyID, uploadID int64, copyIndex int) (int64, error) { + if copyID > 0 { + return copyID, nil + } + return resolveSlotCopyID(ctx, db, uploadID, copyIndex) +} + +func resolveSlotCopyID(ctx context.Context, db bun.IDB, uploadID int64, copyIndex int) (int64, error) { + var ids []int64 + err := db.NewSelect(). + Model((*model.StorageUploadCopy)(nil)). + Column("id"). + Where("storage_upload_copy.upload_id = ? AND storage_upload_copy.copy_index = ?", uploadID, copyIndex). + Where(currentGenerationCopySQL("storage_upload_copy")). + Scan(ctx, &ids) + if err != nil { + return 0, fmt.Errorf("resolving storage upload copy for slot: %w", err) + } + switch len(ids) { + case 0: + return 0, nil + case 1: + return ids[0], nil + default: + return 0, fmt.Errorf( + "storage upload %d replica slot %d matches %d copies: %w", + uploadID, copyIndex, len(ids), ErrConflict, + ) + } +} + func uploadCopyTransferMethod(ctx context.Context, db bun.IDB, uploadID int64, copyIndex int) (model.StorageCopyTransferMethod, error) { var row struct { TransferMethod model.StorageCopyTransferMethod `bun:"transfer_method"` @@ -2066,7 +2192,10 @@ func uploadCopyTransferMethod(ctx context.Context, db bun.IDB, uploadID int64, c err := db.NewSelect(). Model((*model.StorageUploadCopy)(nil)). Column("transfer_method"). - Where("upload_id = ? AND copy_index = ?", uploadID, copyIndex). + Where("storage_upload_copy.upload_id = ? AND storage_upload_copy.copy_index = ?", uploadID, copyIndex). + Where(currentGenerationCopySQL("storage_upload_copy")). + OrderExpr("(storage_upload_copy.storage_data_set_id IS NULL) ASC"). + Limit(1). Scan(ctx, &row) if err != nil { if err == sql.ErrNoRows { @@ -2081,13 +2210,16 @@ func uploadCopyDataSetCreatedByUpload(ctx context.Context, db bun.IDB, uploadID var row struct { IsNewDataSet bool `bun:"is_new_data_set"` } - err := db.NewRaw(`SELECT CASE + err := db.NewRaw(fmt.Sprintf(`SELECT CASE WHEN storage_data_set.created_by_upload_id = storage_copy.upload_id THEN TRUE ELSE FALSE END AS is_new_data_set FROM storage_upload_copies AS storage_copy LEFT JOIN storage_data_sets AS storage_data_set ON storage_data_set.id = storage_copy.storage_data_set_id - WHERE storage_copy.upload_id = ? AND storage_copy.copy_index = ?`, + WHERE storage_copy.upload_id = ? AND storage_copy.copy_index = ? + AND %s + ORDER BY (storage_copy.storage_data_set_id IS NULL) ASC + LIMIT 1`, currentGenerationCopySQL("storage_copy")), uploadID, copyIndex, ).Scan(ctx, &row) if err != nil { diff --git a/internal/db/repository/storage_upload_repo_test.go b/internal/db/repository/storage_upload_repo_test.go index 5d24cee..4c1ad21 100644 --- a/internal/db/repository/storage_upload_repo_test.go +++ b/internal/db/repository/storage_upload_repo_test.go @@ -1501,7 +1501,7 @@ func TestStorageUploadRepo_FinalizeUploadIfTargetCopiesMetMovesReplicatingToStor t.Fatalf("partial state = %s, want replicating", got.State) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, upload.ID, 1, "peer pull: dataset unavailable"); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 1, LastError: "peer pull: dataset unavailable"}); err != nil { t.Fatalf("MarkUploadCopyFailed peer: %v", err) } replacement, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{BucketID: bucket.ID, ProviderID: onChainID(t, "303"), CopyIndex: 2, CreatedByUploadID: upload.ID}) @@ -1913,7 +1913,7 @@ func TestStorageUploadRepo_PrimaryCopyFailureMarksUploadFailed(t *testing.T) { t.Fatalf("CreateUploadCopiesForBindings: %v", err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, upload.ID, 0, "ingress store: provider rejected piece"); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 0, LastError: "ingress store: provider rejected piece"}); err != nil { t.Fatalf("MarkUploadCopyFailed: %v", err) } got, err := repos.Uploads.GetByID(ctx, upload.ID) @@ -1943,7 +1943,7 @@ func TestStorageUploadRepo_PrimaryCopyFailureMarksUploadFailed(t *testing.T) { t.Fatalf("upload status after store retry = %s, want ingress_ready", got.Status) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, upload.ID, 0, "ingress commit: provider rejected piece"); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 0, LastError: "ingress commit: provider rejected piece"}); err != nil { t.Fatalf("MarkUploadCopyFailed after store retry: %v", err) } if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ @@ -2174,7 +2174,7 @@ func TestStorageUploadRepo_CommittedCopyIgnoresStaleStatusUpdates(t *testing.T) }); err != nil { t.Fatalf("MarkUploadCopyPieceReady stale: %v", err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, upload.ID, 1, "secondary pull: stale failure"); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 1, LastError: "secondary pull: stale failure"}); err != nil { t.Fatalf("MarkUploadCopyFailed stale: %v", err) } @@ -2311,14 +2311,14 @@ func TestStorageUploadRepo_UnavailableDataSetRecoveryUsesIncompleteCopies(t *tes if err != nil || firstCopy == nil || firstCopy.UploadID != firstUpload.ID { t.Fatalf("first incomplete copy = %#v err=%v", firstCopy, err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, firstUpload.ID, firstCopy.CopyIndex, "skip completed repair item"); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: firstUpload.ID, CopyIndex: firstCopy.CopyIndex, LastError: "skip completed repair item"}); err != nil { t.Fatalf("MarkUploadCopyFailed: %v", err) } secondCopy, err := repos.Uploads.NextIncompleteCopyForDataSet(ctx, binding.ID) if err != nil || secondCopy == nil || secondCopy.UploadID != secondUpload.ID { t.Fatalf("second incomplete copy = %#v err=%v", secondCopy, err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, secondUpload.ID, secondCopy.CopyIndex, "skip second repair item"); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: secondUpload.ID, CopyIndex: secondCopy.CopyIndex, LastError: "skip second repair item"}); err != nil { t.Fatalf("MarkUploadCopyFailed second: %v", err) } next, err := repos.Uploads.NextIncompleteCopyForDataSet(ctx, binding.ID) @@ -2506,7 +2506,7 @@ func TestStorageUploadRepo_DiscardFailedCandidateIsAtomicWithSharedReferences(t if err := repos.Uploads.MarkDataSetFailed(ctx, binding.ID, "creation rejected"); err != nil { t.Fatalf("MarkDataSetFailed: %v", err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, first.ID, 0, "creation rejected"); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: first.ID, CopyIndex: 0, LastError: "creation rejected"}); err != nil { t.Fatalf("MarkUploadCopyFailed: %v", err) } discarded, err := repos.Uploads.DiscardFailedDataSetCandidate(ctx, first.ID, 0, binding.ID) @@ -2721,3 +2721,205 @@ func affectedVersionByID(versions []repository.BucketStorageHealthAffectedVersio func strPtr(v string) *string { return &v } + +// A provider replacement gives one replica slot two data set generations that +// can both hold a committed copy. Counting them as two replicas would release +// cache while only one provider actually holds the data. +func TestStorageUploadRepo_DurabilityCountsDistinctSlotsAcrossGenerations(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "generation-durability-bucket") + + version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000GEN01", 10) + version.Checksum = "generation-durability-checksum" + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 3) + commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") + commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 1, "202", "2002", "2002", "https://two.example/piece") + bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) + + source, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) + if err != nil || source == nil { + t.Fatalf("GetDataSetBindingByCopyIndex = %#v err=%v", source, err) + } + // Stand in for an activated replacement until the replacement repository + // owns this transition. + mustExec(t, db, `UPDATE storage_data_sets SET is_current = FALSE, status = ? WHERE id = ?`, + model.StorageDataSetStatusDraining, source.ID) + mustExec(t, db, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_at, updated_at) + VALUES (?, '909', 0, 2, TRUE, '9009', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + bucket.ID, model.StorageDataSetStatusReady) + target, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) + if err != nil || target == nil || target.ID == source.ID || target.Generation != 2 { + t.Fatalf("current binding after activation = %#v err=%v, want the second generation", target, err) + } + mustExec(t, db, `INSERT INTO storage_upload_copies (upload_id, copy_index, provider_id, piece_id, transfer_method, status, retrieval_url, storage_data_set_id, created_at, updated_at) + VALUES (?, 0, '909', '9001', ?, ?, 'https://three.example/piece', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + upload.ID, model.StorageCopyTransferMethodPeerPull, model.StorageUploadCopyStatusCommitted, target.ID) + + // All three physical copies stay retrievable, including the retiring one. + copies, err := repos.Uploads.ListReadableCommittedCopies(ctx, upload.ID) + if err != nil { + t.Fatalf("ListReadableCommittedCopies: %v", err) + } + if len(copies) != 3 { + t.Fatalf("readable copies = %d, want 3 physical copies", len(copies)) + } + + // Durability still sees two slots, so the third requested copy is owed. + done, refs, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}) + if err != nil { + t.Fatalf("FinalizeUploadIfTargetCopiesMet: %v", err) + } + if done { + t.Fatalf("two generations of one slot satisfied a 3-copy target, want them counted as one replica (refs=%#v)", refs) + } + gotVersion, err := repos.Objects.GetVersionByID(ctx, version.VersionID) + if err != nil || gotVersion == nil || gotVersion.State == model.ObjectStateStored { + t.Fatalf("version after activation = %#v err=%v, want it to stay short of the durability threshold", gotVersion, err) + } +} + +// A staged task records the copy it stored to. If the slot's current generation +// changes before the task runs, the write must still land on the generation +// that actually holds the piece rather than on its replacement. +func TestStorageUploadRepo_CopyWritesFollowTheRecordedCopyNotTheCurrentGeneration(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "generation-addressing-bucket") + + version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000ADR01", 10) + version.Checksum = "generation-addressing-checksum" + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 1) + source := ensureCopyHealthBinding(t, repos, bucket.ID, upload.ID, 0, "101") + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, + UploadID: upload.ID, + DataSetID: onChainID(t, "1001"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: source.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: onChainID(t, "101"), + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + sourceCopy, err := repos.Uploads.GetUploadCopyForDataSet(ctx, upload.ID, source.ID) + if err != nil || sourceCopy == nil { + t.Fatalf("GetUploadCopyForDataSet source = %#v err=%v", sourceCopy, err) + } + + // Stand in for an activated replacement that also staged its own copy. + mustExec(t, db, `UPDATE storage_data_sets SET is_current = FALSE, status = ? WHERE id = ?`, + model.StorageDataSetStatusDraining, source.ID) + mustExec(t, db, `INSERT INTO storage_data_sets (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_at, updated_at) + VALUES (?, '909', 0, 2, TRUE, '9009', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + bucket.ID, model.StorageDataSetStatusReady) + target, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) + if err != nil || target == nil || target.ID == source.ID { + t.Fatalf("current binding after activation = %#v err=%v, want the new generation", target, err) + } + if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: target.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodPeerPull, + ProviderID: onChainID(t, "909"), + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings target: %v", err) + } + targetCopy, err := repos.Uploads.GetUploadCopyForDataSet(ctx, upload.ID, target.ID) + if err != nil || targetCopy == nil { + t.Fatalf("GetUploadCopyForDataSet target = %#v err=%v", targetCopy, err) + } + + if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ + StorageUploadCopyID: sourceCopy.ID, + UploadID: upload.ID, + CopyIndex: 0, + PieceCID: "bafk2bzacegeneration", + PieceID: onChainIDPtr(t, "2001"), + RetrievalURL: "https://source.example/piece", + }); err != nil { + t.Fatalf("MarkUploadCopyPieceReady: %v", err) + } + + gotSource, err := repos.Uploads.GetUploadCopyByID(ctx, sourceCopy.ID) + if err != nil || gotSource == nil || gotSource.Status != model.StorageUploadCopyStatusPieceReady { + t.Fatalf("recorded copy = %#v err=%v, want piece_ready on the generation that stored it", gotSource, err) + } + gotTarget, err := repos.Uploads.GetUploadCopyByID(ctx, targetCopy.ID) + if err != nil || gotTarget == nil || gotTarget.Status != model.StorageUploadCopyStatusPending { + t.Fatalf("replacement copy = %#v err=%v, want it untouched", gotTarget, err) + } + + // A task queued before copy ids existed still resolves through its slot, + // which names the current generation. + if err := repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ + UploadID: upload.ID, + CopyIndex: 0, + PieceCID: "bafk2bzacegeneration", + PieceID: onChainIDPtr(t, "9001"), + RetrievalURL: "https://target.example/piece", + }); err != nil { + t.Fatalf("MarkUploadCopyPieceReady legacy: %v", err) + } + gotTarget, err = repos.Uploads.GetUploadCopyByID(ctx, targetCopy.ID) + if err != nil || gotTarget == nil || gotTarget.Status != model.StorageUploadCopyStatusPieceReady { + t.Fatalf("legacy addressed copy = %#v err=%v, want the current generation", gotTarget, err) + } +} + +func TestStorageUploadRepo_CountCurrentGenerationCopySlotsIgnoresHistoricalCopies(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "current-generation-copy-count") + upload := startCopyHealthUpload(t, repos, bucket.ID, model.NewVersionID(), 10, "generation-count", 2) + source := ensureCopyHealthBinding(t, repos, bucket.ID, upload.ID, 0, "101") + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, UploadID: upload.ID, DataSetID: onChainID(t, "1001"), + }); err != nil { + t.Fatalf("MarkDataSetReady source: %v", err) + } + if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: source.ID, CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, ProviderID: onChainID(t, "101"), + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings source: %v", err) + } + + mustExec(t, db, `UPDATE storage_data_sets SET is_current = FALSE, status = ? WHERE id = ?`, + model.StorageDataSetStatusDraining, source.ID) + mustExec(t, db, `INSERT INTO storage_data_sets + (bucket_id, provider_id, copy_index, generation, is_current, data_set_id, status, created_at, updated_at) + VALUES (?, '202', 0, 2, TRUE, '2002', ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)`, + bucket.ID, model.StorageDataSetStatusReady) + target, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) + if err != nil || target == nil || target.ID == source.ID { + t.Fatalf("current target = %#v err=%v", target, err) + } + if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: target.ID, CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodPeerPull, ProviderID: onChainID(t, "202"), + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings target: %v", err) + } + + count, err := repos.Uploads.CountCurrentGenerationCopySlots(ctx, upload.ID) + if err != nil { + t.Fatalf("CountCurrentGenerationCopySlots: %v", err) + } + if count != 1 { + t.Fatalf("current logical slots = %d, want one occupied slot and one missing slot", count) + } +} diff --git a/internal/db/repository/task_repo.go b/internal/db/repository/task_repo.go index 35a4c2d..79cf49a 100644 --- a/internal/db/repository/task_repo.go +++ b/internal/db/repository/task_repo.go @@ -4,9 +4,11 @@ import ( "context" "database/sql" "fmt" + "slices" "time" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" "github.com/uptrace/bun" "github.com/uptrace/bun/dialect" ) @@ -48,6 +50,25 @@ func (r *BunTaskRepo) Create(ctx context.Context, task *model.Task) error { } func (r *BunTaskRepo) EnsureRecurring(ctx context.Context, task *model.Task) (bool, error) { + // Automatic recurrence never revives work that gave up. Exhausted and failed + // coordinators wait for an operator, who resumes them through ResumeCoordinator. + return r.ensureRecurringTask(ctx, task, model.TaskStatusCompleted) +} + +// ResumeCoordinator restarts a singleton coordinator on an operator's request. +// It differs from EnsureRecurring in exactly one way: it also revives a task +// that exhausted its retries or failed outright, which is the state the +// dedicated replacement retry exists to recover from. Without it the retry +// would move the replacement back into a working status with nothing queued to +// do the work, and the record would never leave it. +func (r *BunTaskRepo) ResumeCoordinator(ctx context.Context, task *model.Task) (bool, error) { + if task != nil && !storagereplacement.IsCoordinatorTask(task.Type, task.Stage) { + return false, fmt.Errorf("resuming a task that is not a coordinator: %w", ErrInvalidInput) + } + return r.ensureRecurringTask(ctx, task, model.TaskStatusCompleted, model.TaskStatusExhausted, model.TaskStatusFailed) +} + +func (r *BunTaskRepo) ensureRecurringTask(ctx context.Context, task *model.Task, revivable ...model.TaskStatus) (bool, error) { if task == nil || task.IdempotencyKey == "" || task.Type == "" || task.RefType == "" { return false, fmt.Errorf("recurring task identity is required: %w", ErrInvalidInput) } @@ -85,7 +106,7 @@ func (r *BunTaskRepo) EnsureRecurring(ctx context.Context, task *model.Task) (bo if err != nil { return fmt.Errorf("loading recurring task: %w", err) } - if existing.Status != model.TaskStatusCompleted { + if !slices.Contains(revivable, existing.Status) { return nil } now := time.Now() @@ -107,7 +128,7 @@ func (r *BunTaskRepo) EnsureRecurring(ctx context.Context, task *model.Task) (bo Set("lease_until = NULL"). Set("started_at = NULL"). Set("completed_at = NULL"). - Where("id = ? AND status = ?", existing.ID, model.TaskStatusCompleted). + Where("id = ? AND status = ?", existing.ID, existing.Status). Exec(ctx) if err != nil { return fmt.Errorf("reactivating recurring task: %w", err) @@ -480,8 +501,13 @@ func (r *BunTaskRepo) LockRunningClaim(ctx context.Context, claimedTask *model.T } func (r *BunTaskRepo) ContinueRunning(ctx context.Context, claimedTask *model.Task, refVersionID string, payload map[string]interface{}) error { - if refVersionID == "" || payload == nil { - return fmt.Errorf("continuation version and payload are required: %w", ErrInvalidInput) + if payload == nil { + return fmt.Errorf("continuation payload is required: %w", ErrInvalidInput) + } + // An object coordinator must keep naming the version it is working on. A + // bucket-scoped coordinator legitimately has none between items. + if refVersionID == "" && claimedTask != nil && claimedTask.RefType == "object" { + return fmt.Errorf("continuation version is required: %w", ErrInvalidInput) } taskID, claimedAt, err := runningTaskClaim(claimedTask) if err != nil { @@ -490,7 +516,7 @@ func (r *BunTaskRepo) ContinueRunning(ctx context.Context, claimedTask *model.Ta now := time.Now() res, err := r.db.NewUpdate(). Model((*model.Task)(nil)). - Set("ref_version_id = ?", refVersionID). + Set("ref_version_id = COALESCE(NULLIF(?, ''), ref_version_id)", refVersionID). Set("payload = ?", payload). Set("status = ?", model.TaskStatusQueued). Set("retry_count = 0"). @@ -665,6 +691,12 @@ func (r *BunTaskRepo) RetryExhausted(ctx context.Context, taskID int64) error { return fmt.Errorf("loading exhausted task: %w", err) } + // Replacement work carries state the generic queue knows nothing about, + // so it must resume through the dedicated replacement action instead. + if storagereplacement.IsCoordinatorTask(task.Type, task.Stage) { + return ErrReplacementRetryUnsupported + } + now := time.Now() if err := resetFailedObjectForTaskRetry(ctx, db, task, now); err != nil { return err diff --git a/internal/db/repository/tx.go b/internal/db/repository/tx.go new file mode 100644 index 0000000..96b3d0d --- /dev/null +++ b/internal/db/repository/tx.go @@ -0,0 +1,19 @@ +package repository + +import ( + "context" + + "github.com/uptrace/bun" +) + +// runMaybeTx runs fn in a transaction when db is a connection pool, and inline +// when it is already a transaction. Repository methods use it so they compose +// under WithTx without nesting transactions. +func runMaybeTx(ctx context.Context, db bun.IDB, fn func(bun.IDB) error) error { + if pool, ok := db.(*bun.DB); ok { + return pool.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return fn(tx) + }) + } + return fn(db) +} diff --git a/internal/model/storage.go b/internal/model/storage.go index 0086391..924d531 100644 --- a/internal/model/storage.go +++ b/internal/model/storage.go @@ -95,6 +95,10 @@ type StorageUpload struct { } // StorageDataSet records the bucket ownership of a provider-scoped data set. +// CopyIndex names the logical replica slot, which can own several physical +// generations while a provider replacement is in flight. IsCurrent selects the +// generation that accepts new writes; replaced generations stay readable until +// they are verifiably retired. type StorageDataSet struct { bun.BaseModel `bun:"table:storage_data_sets"` @@ -102,6 +106,8 @@ type StorageDataSet struct { BucketID int64 `bun:",notnull"` ProviderID types.OnChainID `bun:"type:text,notnull"` CopyIndex int `bun:",notnull"` + Generation int `bun:",notnull"` + IsCurrent bool `bun:",notnull"` DataSetID *types.OnChainID `bun:"type:text"` ClientDataSetID *types.OnChainID `bun:"type:text"` Status StorageDataSetStatus `bun:",notnull,default:'pending'"` diff --git a/internal/storagereplacement/codes.go b/internal/storagereplacement/codes.go new file mode 100644 index 0000000..de39535 --- /dev/null +++ b/internal/storagereplacement/codes.go @@ -0,0 +1,51 @@ +package storagereplacement + +import "errors" + +// Stable machine-readable codes returned alongside API errors. Clients branch +// on these, so treat them as part of the public contract. +const ( + CodeActive = "replacement_active" + CodeSuperseded = "replacement_superseded" + CodeNotRetryable = "replacement_not_retryable" + CodeTaskRunning = "replacement_task_running" + CodeTargetInvalid = "replacement_target_invalid" + CodeTargetInUse = "replacement_target_in_use" + CodeNoEligibleProvider = "replacement_no_eligible_provider" + CodeTargetUnavailable = "replacement_target_unavailable" + CodeIdempotencyConflict = "replacement_idempotency_conflict" + CodeSourceNotCurrent = "replacement_source_not_current" + + // CodeTaskRetryUnsupported is returned by the generic exhausted-task retry + // endpoint when the task belongs to a replacement. + CodeTaskRetryUnsupported = "replacement_task_retry_unsupported" +) + +// Code maps a replacement error to its stable API code. It returns an empty +// string for errors that carry no client-facing code. +func Code(err error) string { + switch { + case errors.Is(err, ErrActiveReplacement): + return CodeActive + case errors.Is(err, ErrSuperseded): + return CodeSuperseded + case errors.Is(err, ErrNotRetryable): + return CodeNotRetryable + case errors.Is(err, ErrTaskRunning): + return CodeTaskRunning + case errors.Is(err, ErrInvalidTarget): + return CodeTargetInvalid + case errors.Is(err, ErrTargetInUse): + return CodeTargetInUse + case errors.Is(err, ErrNoEligibleProvider): + return CodeNoEligibleProvider + case errors.Is(err, ErrTargetUnavailable): + return CodeTargetUnavailable + case errors.Is(err, ErrIdempotencyConflict): + return CodeIdempotencyConflict + case errors.Is(err, ErrSourceNotCurrent): + return CodeSourceNotCurrent + default: + return "" + } +} diff --git a/internal/storagereplacement/errors.go b/internal/storagereplacement/errors.go new file mode 100644 index 0000000..018b1eb --- /dev/null +++ b/internal/storagereplacement/errors.go @@ -0,0 +1,58 @@ +package storagereplacement + +import "errors" + +var ( + // ErrActiveReplacement means the source data set already has a replacement + // that has not reached a terminal state. + ErrActiveReplacement = errors.New("data set already has an active replacement") + + // ErrSuperseded means a later confirmation took ownership of this work. + ErrSuperseded = errors.New("replacement has been superseded") + + // ErrNotRetryable means the replacement is progressing or finished, so the + // dedicated retry action does not apply. + ErrNotRetryable = errors.New("replacement is not in a retryable state") + + // ErrTaskRunning means a coordinator task still holds this replacement. + ErrTaskRunning = errors.New("replacement task is still running") + + // ErrInvalidTarget means the requested provider cannot serve as the target, + // for example because it is the source itself. + ErrInvalidTarget = errors.New("requested provider cannot replace this data set") + + // ErrTargetInUse means the requested provider already owns another current + // data set in the same bucket. + ErrTargetInUse = errors.New("requested provider already serves this bucket") + + // ErrNoEligibleProvider means automatic selection found no provider the + // bucket has not already used. + ErrNoEligibleProvider = errors.New("no eligible replacement provider is available") + + // ErrTargetUnavailable means the requested provider is not currently an + // active PDP-capable replacement candidate. + ErrTargetUnavailable = errors.New("requested provider is not available for replacement") + + // ErrIdempotencyConflict means a client request id was reused with different + // replacement parameters. + ErrIdempotencyConflict = errors.New("replacement idempotency key conflicts with an earlier request") + + // ErrSourceNotCurrent means the data set no longer owns its replica slot, + // so replacing it would not change where writes go. + ErrSourceNotCurrent = errors.New("data set is not the current replica") + + // ErrItemCancelled means this migration item no longer has executable work. + ErrItemCancelled = errors.New("replacement item is no longer executable") + + // ErrItemDeferred means the item cannot run yet but must be revisited. The + // coordinator moves on to other content rather than blocking on it. + ErrItemDeferred = errors.New("replacement item is waiting for its source") + + // ErrPrematureComplete means a retirement safety gate still reports a + // blocker. Repositories return it even when called outside the worker. + ErrPrematureComplete = errors.New("replacement cannot complete while a safety gate blocks it") + + // ErrIllegalTransition means the requested state change is not in the + // state machine. + ErrIllegalTransition = errors.New("illegal replacement state transition") +) diff --git a/internal/storagereplacement/failurereason.go b/internal/storagereplacement/failurereason.go new file mode 100644 index 0000000..9ef1d40 --- /dev/null +++ b/internal/storagereplacement/failurereason.go @@ -0,0 +1,14 @@ +package storagereplacement + +// FailureReason identifies a permanent failure whose recovery action differs +// from retrying the same approved target. +type FailureReason string + +const ( + FailureReasonTargetInUse FailureReason = "target_in_use" +) + +// Valid reports whether the value is a known permanent failure reason. +func (r FailureReason) Valid() bool { + return r == FailureReasonTargetInUse +} diff --git a/internal/storagereplacement/record.go b/internal/storagereplacement/record.go new file mode 100644 index 0000000..b3602fa --- /dev/null +++ b/internal/storagereplacement/record.go @@ -0,0 +1,72 @@ +package storagereplacement + +import ( + "time" + + "github.com/strahe/synaps3/internal/types" + "github.com/uptrace/bun" +) + +// Replacement is one operator-approved provider replacement for a single +// bucket replica slot. It is created only by an explicit confirmation and +// authorizes a new paid service, the topology switch, migration, and +// termination of the old service once the safety gate passes. +type Replacement struct { + bun.BaseModel `bun:"table:storage_replacements,alias:storage_replacement"` + + ID int64 `bun:",pk,autoincrement"` + BucketID int64 `bun:",notnull"` + // CopyIndex is the logical replica slot both generations belong to. + CopyIndex int `bun:",notnull"` + SourceDataSetID int64 `bun:",notnull"` + TargetDataSetID int64 `bun:",notnull"` + SelectionMode SelectionMode `bun:",notnull"` + RequestedProviderID *types.OnChainID `bun:"type:text"` + ClientRequestID string `bun:",notnull"` + Status Status `bun:",notnull"` + WaitReason *WaitReason `bun:",nullzero"` + FailureReason *FailureReason `bun:",nullzero"` + LastError *string `bun:",nullzero"` + // ItemsTotal and ItemsCopied are maintained inside the transactions that + // seed and complete items, so progress never needs a history-sized count. + ItemsTotal int `bun:",notnull,default:0"` + ItemsCopied int `bun:",notnull,default:0"` + // SeedCursorUploadID advances through storage uploads in bounded batches so + // no single transaction scales with retained bucket history. + SeedCursorUploadID int64 `bun:",notnull,default:0"` + SeedingComplete bool `bun:",notnull,default:false"` + // TerminationEpoch is recorded before the old service is treated as + // terminated, so a crash between termination and observation re-reads it + // instead of terminating twice. + TerminationTxHash *string `bun:",nullzero"` + TerminationEpoch *int64 `bun:",nullzero"` + TerminationObservedAt *time.Time `bun:",nullzero"` + // AbandonedTerminationEpoch records termination of a superseded target. + // It is separate from the source termination fields above because the two + // services belong to opposite generations. + AbandonedTerminationTxHash *string `bun:",nullzero"` + AbandonedTerminationEpoch *int64 `bun:",nullzero"` + AbandonedTerminationObservedAt *time.Time `bun:",nullzero"` + SupersededByID *int64 `bun:",nullzero"` + ConfirmedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` +} + +// Item is one unit of migration work. Items are keyed by storage upload, not by +// object version, so content shared by many versions is copied once. +type Item struct { + bun.BaseModel `bun:"table:storage_replacement_items,alias:storage_replacement_item"` + + ID int64 `bun:",pk,autoincrement"` + ReplacementID int64 `bun:",notnull"` + UploadID int64 `bun:",notnull"` + // TargetCopyID is the concrete copy row on the target generation. Tasks + // address it directly so they can never write the wrong generation. + TargetCopyID *int64 `bun:",nullzero"` + Status ItemStatus `bun:",notnull"` + Attempts int `bun:",notnull,default:0"` + LastError *string `bun:",nullzero"` + CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` +} diff --git a/internal/storagereplacement/status.go b/internal/storagereplacement/status.go new file mode 100644 index 0000000..56f7dc1 --- /dev/null +++ b/internal/storagereplacement/status.go @@ -0,0 +1,125 @@ +// Package storagereplacement owns the operator-approved provider replacement +// record: its state machine, wait reasons, stable API codes, and task payload +// contract. It holds no database or provider dependencies. +package storagereplacement + +// Status is the lifecycle of one approved replacement. +type Status string + +const ( + // StatusPreparingTarget means the approved target service is being created. + // Writes still go to the source. + StatusPreparingTarget Status = "preparing_target" + // StatusMigrating means the target owns the slot for new writes and stored + // content is being copied across. + StatusMigrating Status = "migrating" + // StatusWaiting means progress is blocked on a recoverable dependency. It + // consumes no retry budget and is never a failure. + StatusWaiting Status = "waiting" + // StatusRetiring means migration finished and the source is going through + // the retirement safety gate. + StatusRetiring Status = "retiring" + // StatusCleanupAttention means retirement cannot proceed without an + // operator decision. Automatic task retry is suppressed. + StatusCleanupAttention Status = "cleanup_attention" + // StatusFailed means the work exhausted its retries and needs the operator + // to retry it from the Data Sets surface. + StatusFailed Status = "failed" + // StatusCompleted means the source service was terminated and observed as + // terminated. + StatusCompleted Status = "completed" + // StatusSuperseded means a later confirmation replaced this one. + StatusSuperseded Status = "superseded" +) + +// Active reports whether a coordinator should still advance this replacement. +func (s Status) Active() bool { + switch s { + case StatusPreparingTarget, StatusMigrating, StatusWaiting, StatusRetiring: + return true + default: + return false + } +} + +// Terminal reports whether the replacement can never change again. +func (s Status) Terminal() bool { + return s == StatusCompleted || s == StatusSuperseded +} + +// Retryable reports whether the dedicated retry action accepts this status. +// Waiting work resumes on its own and is deliberately excluded. +func (s Status) Retryable() bool { + return s == StatusFailed || s == StatusCleanupAttention +} + +// HoldsSource reports whether the replacement still owns its source data set, +// which is what stops a second replacement from starting on the same source. +func (s Status) HoldsSource() bool { + return !s.Terminal() +} + +// Valid reports whether the value is a known status. +func (s Status) Valid() bool { + switch s { + case StatusPreparingTarget, StatusMigrating, StatusWaiting, StatusRetiring, + StatusCleanupAttention, StatusFailed, StatusCompleted, StatusSuperseded: + return true + default: + return false + } +} + +// SelectionMode records how the operator chose the target provider. +type SelectionMode string + +const ( + // SelectionModeAutomatic picks an eligible provider the bucket has not used. + SelectionModeAutomatic SelectionMode = "automatic" + // SelectionModeManual uses exactly the provider the operator supplied. + SelectionModeManual SelectionMode = "manual" +) + +// Valid reports whether the value is a known selection mode. +func (m SelectionMode) Valid() bool { + return m == SelectionModeAutomatic || m == SelectionModeManual +} + +// ItemStatus is the lifecycle of one unit of migration work. An item is keyed +// by stored content, not by object version, so shared content migrates once. +type ItemStatus string + +const ( + // ItemStatusPending is seeded work not yet attempted. + ItemStatusPending ItemStatus = "pending" + // ItemStatusRunning is the one item currently held by the coordinator. + ItemStatusRunning ItemStatus = "running" + // ItemStatusWaitingSource means no readable copy and no cached content is + // available yet. The coordinator moves on and revisits it later. + ItemStatusWaitingSource ItemStatus = "waiting_source" + // ItemStatusCopied means the target holds a committed readable copy. + ItemStatusCopied ItemStatus = "copied" + // ItemStatusCancelled means the content no longer needs migrating. + ItemStatusCancelled ItemStatus = "cancelled" +) + +// Executable reports whether the coordinator may pick this item up. +func (s ItemStatus) Executable() bool { + return s == ItemStatusPending || s == ItemStatusRunning || s == ItemStatusWaitingSource +} + +// Blocking reports whether the item prevents the source from being retired. +func (s ItemStatus) Blocking() bool { + return s.Executable() +} + +// Valid reports whether the value is a known item status. +func (s ItemStatus) Valid() bool { + switch s { + case ItemStatusPending, ItemStatusRunning, ItemStatusWaitingSource, + ItemStatusCopied, ItemStatusCancelled: + return true + default: + return false + } +} diff --git a/internal/storagereplacement/task.go b/internal/storagereplacement/task.go new file mode 100644 index 0000000..3247219 --- /dev/null +++ b/internal/storagereplacement/task.go @@ -0,0 +1,229 @@ +package storagereplacement + +import ( + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/strahe/synaps3/internal/model" +) + +const ( + // StageMigrate advances replacement migration through the Upload worker. + StageMigrate = "replace_provider" + // StageRetire runs the retirement safety gate through the storage cleanup + // worker. + StageRetire = "retire_data_set" + // StageRetireAbandonedTarget ends the service of a target that a later + // confirmation replaced before it ever took over the slot. + StageRetireAbandonedTarget = "retire_abandoned_target" + + replacementIDPayloadKey = "replacement_id" + itemIDPayloadKey = "item_id" + copyIDPayloadKey = "storage_upload_copy_id" + + migrateTaskKeyPrefix = "upload:storage-replacement:" + retireTaskKeyPrefix = "storage_cleanup:storage-replacement:" +) + +// MigrateTaskKey identifies the single migration coordinator for one +// replacement. Exactly one such task exists, which is what limits a +// replacement to one executable item at a time. +func MigrateTaskKey(replacementID int64) string { + return fmt.Sprintf("%s%d:migrate", migrateTaskKeyPrefix, replacementID) +} + +// RetireTaskKey identifies the single retirement coordinator for one +// replacement. It retires the generation being replaced. +func RetireTaskKey(replacementID int64) string { + return fmt.Sprintf("%s%d:cleanup", retireTaskKeyPrefix, replacementID) +} + +// AbandonedTargetTaskKey identifies the cleanup of a target a later +// confirmation abandoned. It is a separate coordinator because it retires the +// opposite generation and answers a different safety question. +func AbandonedTargetTaskKey(replacementID int64) string { + return fmt.Sprintf("%s%d:abandoned-target", retireTaskKeyPrefix, replacementID) +} + +// NewMigrateTask builds the singleton migration coordinator for one +// replacement. Exactly one exists per replacement, which is what keeps a +// replacement to one executable item and lets ordinary uploads interleave +// through the normal queue order. +func NewMigrateTask(replacementID, bucketID int64, versionID string, maxRetries int, scheduledAt time.Time) *model.Task { + stage := StageMigrate + return &model.Task{ + Type: model.TaskTypeUpload, + Stage: &stage, + RefType: "bucket", + RefID: bucketID, + RefVersionID: versionID, + IdempotencyKey: MigrateTaskKey(replacementID), + Payload: NewMigratePayload(replacementID, 0, 0), + Status: model.TaskStatusQueued, + MaxRetries: maxRetries, + ScheduledAt: scheduledAt, + } +} + +// NewRetireTask builds the singleton retirement coordinator for one +// replacement. It runs on the storage cleanup worker because terminating a +// service is a destructive lifecycle action, not upload work. +func NewRetireTask(replacementID, bucketID int64, maxRetries int, scheduledAt time.Time) *model.Task { + stage := StageRetire + return &model.Task{ + Type: model.TaskTypeStorageCleanup, + Stage: &stage, + RefType: "bucket", + RefID: bucketID, + IdempotencyKey: RetireTaskKey(replacementID), + Payload: NewRetirePayload(replacementID), + Status: model.TaskStatusQueued, + MaxRetries: maxRetries, + ScheduledAt: scheduledAt, + } +} + +// MigratePayload is the persisted state of the migration coordinator. It holds +// identifiers only; every claim re-derives the rest from the database so a +// restart cannot act on a stale snapshot. +type MigratePayload struct { + ReplacementID int64 + // ItemID is zero when no item is currently assigned. + ItemID int64 + // CopyID is zero until the target copy row exists. + CopyID int64 +} + +// NewMigratePayload builds the coordinator payload. +func NewMigratePayload(replacementID, itemID, copyID int64) map[string]any { + payload := map[string]any{replacementIDPayloadKey: replacementID} + if itemID > 0 { + payload[itemIDPayloadKey] = itemID + } + if copyID > 0 { + payload[copyIDPayloadKey] = copyID + } + return payload +} + +// ParseMigratePayload decodes a migration coordinator payload. These tasks only +// ever exist after the upgrade that introduced them, so the replacement ID is +// required rather than inferred. +func ParseMigratePayload(task *model.Task) (MigratePayload, error) { + if task == nil { + return MigratePayload{}, errors.New("nil replacement migration task") + } + replacementID, err := payloadInt64(task.Payload, replacementIDPayloadKey) + if err != nil { + return MigratePayload{}, err + } + if replacementID <= 0 { + return MigratePayload{}, fmt.Errorf("replacement migration task %s must be positive", replacementIDPayloadKey) + } + itemID, err := optionalPayloadInt64(task.Payload, itemIDPayloadKey) + if err != nil { + return MigratePayload{}, err + } + copyID, err := optionalPayloadInt64(task.Payload, copyIDPayloadKey) + if err != nil { + return MigratePayload{}, err + } + return MigratePayload{ReplacementID: replacementID, ItemID: itemID, CopyID: copyID}, nil +} + +// NewRetirePayload builds the retirement coordinator payload. +func NewRetirePayload(replacementID int64) map[string]any { + return map[string]any{replacementIDPayloadKey: replacementID} +} + +// NewAbandonedTargetTask builds the coordinator that ends the paid service of a +// target no confirmation uses any more. Without it the abandoned service keeps +// costing money after a later confirmation takes over. +func NewAbandonedTargetTask(replacementID, bucketID int64, maxRetries int, scheduledAt time.Time) *model.Task { + stage := StageRetireAbandonedTarget + return &model.Task{ + Type: model.TaskTypeStorageCleanup, + Stage: &stage, + RefType: "bucket", + RefID: bucketID, + IdempotencyKey: AbandonedTargetTaskKey(replacementID), + Payload: NewRetirePayload(replacementID), + Status: model.TaskStatusQueued, + MaxRetries: maxRetries, + ScheduledAt: scheduledAt, + } +} + +// ParseRetirePayload decodes a retirement coordinator payload. +func ParseRetirePayload(task *model.Task) (int64, error) { + if task == nil { + return 0, errors.New("nil replacement retirement task") + } + replacementID, err := payloadInt64(task.Payload, replacementIDPayloadKey) + if err != nil { + return 0, err + } + if replacementID <= 0 { + return 0, fmt.Errorf("replacement retirement task %s must be positive", replacementIDPayloadKey) + } + return replacementID, nil +} + +// IsCoordinatorTask reports whether a task belongs to a replacement. Generic +// exhausted-task retry uses it to refuse work that must resume through the +// dedicated replacement action. +func IsCoordinatorTask(taskType model.TaskType, stage *string) bool { + if stage == nil { + return false + } + switch { + case taskType == model.TaskTypeUpload && *stage == StageMigrate: + return true + case taskType == model.TaskTypeStorageCleanup && *stage == StageRetire: + return true + case taskType == model.TaskTypeStorageCleanup && *stage == StageRetireAbandonedTarget: + return true + default: + return false + } +} + +func optionalPayloadInt64(payload map[string]any, key string) (int64, error) { + if payload == nil { + return 0, nil + } + if _, ok := payload[key]; !ok { + return 0, nil + } + return payloadInt64(payload, key) +} + +// Payload values survive a JSON round trip through the task table, so an +// integer can come back as float64 or json.Number depending on the driver. +func payloadInt64(payload map[string]any, key string) (int64, error) { + if payload == nil { + return 0, fmt.Errorf("replacement task payload is missing %s", key) + } + raw, ok := payload[key] + if !ok { + return 0, fmt.Errorf("replacement task payload is missing %s", key) + } + switch value := raw.(type) { + case int64: + return value, nil + case int: + return int64(value), nil + case float64: + return int64(value), nil + case json.Number: + parsed, err := value.Int64() + if err != nil { + return 0, fmt.Errorf("replacement task payload %s is not an integer: %w", key, err) + } + return parsed, nil + default: + return 0, fmt.Errorf("replacement task payload %s has type %T, want an integer", key, raw) + } +} diff --git a/internal/storagereplacement/task_test.go b/internal/storagereplacement/task_test.go new file mode 100644 index 0000000..4f8211f --- /dev/null +++ b/internal/storagereplacement/task_test.go @@ -0,0 +1,169 @@ +package storagereplacement + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/strahe/synaps3/internal/model" +) + +func TestMigratePayloadRoundTrip(t *testing.T) { + task := &model.Task{Payload: NewMigratePayload(7, 42, 99)} + got, err := ParseMigratePayload(task) + if err != nil { + t.Fatalf("ParseMigratePayload: %v", err) + } + want := MigratePayload{ReplacementID: 7, ItemID: 42, CopyID: 99} + if got != want { + t.Fatalf("payload = %+v, want %+v", got, want) + } +} + +// Between items the coordinator holds no item and no copy, and must still be +// decodable. +func TestMigratePayloadWithoutAssignedItem(t *testing.T) { + task := &model.Task{Payload: NewMigratePayload(7, 0, 0)} + got, err := ParseMigratePayload(task) + if err != nil { + t.Fatalf("ParseMigratePayload: %v", err) + } + if got.ItemID != 0 || got.CopyID != 0 { + t.Fatalf("payload = %+v, want zero item and copy", got) + } +} + +// Payloads survive a JSON round trip through the task table, so integers come +// back as float64 or json.Number depending on the driver. +func TestMigratePayloadSurvivesJSONRoundTrip(t *testing.T) { + encoded, err := json.Marshal(NewMigratePayload(7, 42, 99)) + if err != nil { + t.Fatalf("marshal payload: %v", err) + } + for _, useNumber := range []bool{false, true} { + decoded := map[string]any{} + decoder := json.NewDecoder(strings.NewReader(string(encoded))) + if useNumber { + decoder.UseNumber() + } + if err := decoder.Decode(&decoded); err != nil { + t.Fatalf("decode payload: %v", err) + } + got, err := ParseMigratePayload(&model.Task{Payload: decoded}) + if err != nil { + t.Fatalf("ParseMigratePayload(useNumber=%v): %v", useNumber, err) + } + if got.ReplacementID != 7 || got.ItemID != 42 || got.CopyID != 99 { + t.Fatalf("payload = %+v, want 7/42/99", got) + } + } +} + +func TestMigratePayloadRejectsMissingReplacement(t *testing.T) { + cases := map[string]*model.Task{ + "nil task": nil, + "nil payload": {}, + "empty payload": {Payload: map[string]any{}}, + "zero id": {Payload: map[string]any{"replacement_id": 0}}, + "wrong type": {Payload: map[string]any{"replacement_id": "seven"}}, + "only item id": {Payload: map[string]any{"item_id": 42}}, + "negative value": {Payload: map[string]any{"replacement_id": -1}}, + } + for name, task := range cases { + if _, err := ParseMigratePayload(task); err == nil { + t.Fatalf("ParseMigratePayload(%s) succeeded, want an error", name) + } + } +} + +func TestRetirePayloadRoundTrip(t *testing.T) { + replacementID, err := ParseRetirePayload(&model.Task{Payload: NewRetirePayload(11)}) + if err != nil { + t.Fatalf("ParseRetirePayload: %v", err) + } + if replacementID != 11 { + t.Fatalf("replacementID = %d, want 11", replacementID) + } + if _, err := ParseRetirePayload(&model.Task{}); err == nil { + t.Fatal("ParseRetirePayload accepted an empty payload, want an error") + } +} + +func TestCoordinatorTaskKeysAreDistinctSingletons(t *testing.T) { + if MigrateTaskKey(1) == MigrateTaskKey(2) { + t.Fatal("migration keys collide across replacements") + } + if MigrateTaskKey(1) == RetireTaskKey(1) { + t.Fatal("migration and retirement keys collide") + } + if AbandonedTargetTaskKey(1) == RetireTaskKey(1) { + t.Fatal("abandoned-target and source retirement keys collide") + } + if AbandonedTargetTaskKey(1) == MigrateTaskKey(1) { + t.Fatal("abandoned-target and migration keys collide") + } + for _, key := range []string{MigrateTaskKey(1), RetireTaskKey(1), AbandonedTargetTaskKey(1)} { + if !strings.Contains(key, "storage-replacement:1:") { + t.Fatalf("key %q does not identify replacement 1", key) + } + } +} + +// Generic exhausted-task retry must refuse replacement work and send the +// operator back to the Data Sets surface. +func TestIsCoordinatorTask(t *testing.T) { + migrate := StageMigrate + retire := StageRetire + abandoned := StageRetireAbandonedTarget + ingress := "ingress_store" + cases := []struct { + name string + taskType model.TaskType + stage *string + want bool + }{ + {"migration coordinator", model.TaskTypeUpload, &migrate, true}, + {"retirement coordinator", model.TaskTypeStorageCleanup, &retire, true}, + {"abandoned-target coordinator", model.TaskTypeStorageCleanup, &abandoned, true}, + {"ordinary upload", model.TaskTypeUpload, &ingress, false}, + {"no stage", model.TaskTypeUpload, nil, false}, + {"stage on the wrong task type", model.TaskTypeStorageCleanup, &migrate, false}, + {"retire stage on the wrong task type", model.TaskTypeUpload, &retire, false}, + } + for _, tc := range cases { + if got := IsCoordinatorTask(tc.taskType, tc.stage); got != tc.want { + t.Fatalf("IsCoordinatorTask(%s) = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestCodeMapsReplacementErrors(t *testing.T) { + cases := map[error]string{ + ErrActiveReplacement: CodeActive, + ErrSuperseded: CodeSuperseded, + ErrNotRetryable: CodeNotRetryable, + ErrTaskRunning: CodeTaskRunning, + ErrInvalidTarget: CodeTargetInvalid, + ErrTargetInUse: CodeTargetInUse, + ErrNoEligibleProvider: CodeNoEligibleProvider, + ErrSourceNotCurrent: CodeSourceNotCurrent, + } + seen := make(map[string]error, len(cases)) + for err, want := range cases { + if got := Code(err); got != want { + t.Fatalf("Code(%v) = %q, want %q", err, got, want) + } + // Wrapping is how repositories add context, so it must not lose the code. + if got := Code(errors.Join(errors.New("context"), err)); got != want { + t.Fatalf("Code(wrapped %v) = %q, want %q", err, got, want) + } + if other, ok := seen[want]; ok { + t.Fatalf("%v and %v share code %q", err, other, want) + } + seen[want] = err + } + if got := Code(errors.New("unrelated")); got != "" { + t.Fatalf("Code(unrelated) = %q, want an empty string", got) + } +} diff --git a/internal/storagereplacement/transition.go b/internal/storagereplacement/transition.go new file mode 100644 index 0000000..40a5d0f --- /dev/null +++ b/internal/storagereplacement/transition.go @@ -0,0 +1,113 @@ +package storagereplacement + +// Phase is the worker responsibility a status maps to. It keeps the coordinator +// dispatch and the state machine from drifting apart. +type Phase string + +const ( + // PhasePrepare creates the approved target service and activates it. + PhasePrepare Phase = "prepare" + // PhaseMigrate copies stored content to the target. + PhaseMigrate Phase = "migrate" + // PhaseRetire runs the safety gate and terminates the old service. + PhaseRetire Phase = "retire" + // PhaseNone means no coordinator should act. + PhaseNone Phase = "none" +) + +// allowedTransitions is the complete state machine. A transition absent here is +// a bug, not an edge case, and the repository refuses it. +var allowedTransitions = map[Status]map[Status]bool{ + StatusPreparingTarget: { + StatusMigrating: true, + StatusWaiting: true, + StatusFailed: true, + StatusSuperseded: true, + }, + StatusMigrating: { + StatusWaiting: true, + StatusRetiring: true, + StatusFailed: true, + StatusSuperseded: true, + }, + StatusWaiting: { + StatusPreparingTarget: true, + StatusMigrating: true, + StatusRetiring: true, + StatusFailed: true, + StatusCleanupAttention: true, + StatusSuperseded: true, + }, + StatusRetiring: { + StatusWaiting: true, + StatusCompleted: true, + StatusCleanupAttention: true, + StatusSuperseded: true, + }, + // Retry re-derives the phase from data, so a failed replacement can resume + // at whichever stage it actually reached. + StatusFailed: { + StatusPreparingTarget: true, + StatusMigrating: true, + StatusRetiring: true, + StatusSuperseded: true, + }, + StatusCleanupAttention: { + StatusRetiring: true, + StatusSuperseded: true, + }, + StatusCompleted: {}, + StatusSuperseded: {}, +} + +// Allowed reports whether the state machine permits this transition. A status +// never transitions to itself; callers update fields in place instead. +func Allowed(from, to Status) bool { + return allowedTransitions[from][to] +} + +// NextStates lists the statuses reachable from one status. +func NextStates(from Status) []Status { + targets := allowedTransitions[from] + out := make([]Status, 0, len(targets)) + for _, candidate := range []Status{ + StatusPreparingTarget, StatusMigrating, StatusWaiting, StatusRetiring, + StatusCleanupAttention, StatusFailed, StatusCompleted, StatusSuperseded, + } { + if targets[candidate] { + out = append(out, candidate) + } + } + return out +} + +// PhaseFor maps a status to the work a coordinator should perform. +func PhaseFor(status Status) Phase { + switch status { + case StatusPreparingTarget: + return PhasePrepare + case StatusMigrating: + return PhaseMigrate + case StatusRetiring: + return PhaseRetire + default: + // Waiting work resumes through the status it was waiting in, which the + // caller recovers from the record rather than from the status alone. + return PhaseNone + } +} + +// OnTaskExhausted maps a status to the state a terminally exhausted coordinator +// task leaves behind. Retirement goes to operator attention rather than failed +// because its remaining work is a cleanup decision, not a retryable copy. +// The second return value reports whether any change is warranted. +func OnTaskExhausted(status Status) (Status, bool) { + switch status { + case StatusPreparingTarget, StatusMigrating, StatusWaiting: + return StatusFailed, true + case StatusRetiring: + return StatusCleanupAttention, true + default: + return status, false + } +} diff --git a/internal/storagereplacement/transition_test.go b/internal/storagereplacement/transition_test.go new file mode 100644 index 0000000..2571ce6 --- /dev/null +++ b/internal/storagereplacement/transition_test.go @@ -0,0 +1,206 @@ +package storagereplacement + +import "testing" + +var allStatuses = []Status{ + StatusPreparingTarget, StatusMigrating, StatusWaiting, StatusRetiring, + StatusCleanupAttention, StatusFailed, StatusCompleted, StatusSuperseded, +} + +func TestTerminalStatusesNeverTransition(t *testing.T) { + for _, from := range []Status{StatusCompleted, StatusSuperseded} { + for _, to := range allStatuses { + if Allowed(from, to) { + t.Fatalf("Allowed(%s, %s) = true, want false for a terminal status", from, to) + } + } + } +} + +func TestEveryUnfinishedStatusCanBeSuperseded(t *testing.T) { + for _, from := range allStatuses { + if from.Terminal() { + continue + } + if !Allowed(from, StatusSuperseded) { + t.Fatalf("Allowed(%s, superseded) = false, want true so a later confirmation can take over", from) + } + } +} + +func TestNoStatusTransitionsToItself(t *testing.T) { + for _, status := range allStatuses { + if Allowed(status, status) { + t.Fatalf("Allowed(%s, %s) = true, want false", status, status) + } + } +} + +// Retirement is the only path to completion, so no earlier phase may shortcut +// the safety gate. +func TestOnlyRetiringReachesCompleted(t *testing.T) { + for _, from := range allStatuses { + want := from == StatusRetiring + if got := Allowed(from, StatusCompleted); got != want { + t.Fatalf("Allowed(%s, completed) = %v, want %v", from, got, want) + } + } +} + +func TestForbiddenShortcuts(t *testing.T) { + forbidden := []struct { + from Status + to Status + why string + }{ + {StatusPreparingTarget, StatusRetiring, "content must migrate before the source is retired"}, + {StatusPreparingTarget, StatusCleanupAttention, "cleanup attention belongs to retirement"}, + {StatusMigrating, StatusPreparingTarget, "the target is already activated"}, + {StatusMigrating, StatusCleanupAttention, "cleanup attention belongs to retirement"}, + {StatusCleanupAttention, StatusFailed, "cleanup attention is resolved by retrying retirement"}, + {StatusFailed, StatusCompleted, "a retry must re-run the safety gate"}, + {StatusFailed, StatusWaiting, "retry re-derives a working phase, not a wait"}, + } + for _, tc := range forbidden { + if Allowed(tc.from, tc.to) { + t.Fatalf("Allowed(%s, %s) = true, want false: %s", tc.from, tc.to, tc.why) + } + } +} + +// A failed replacement resumes at whatever phase it actually reached. +func TestFailedResumesAtAnyWorkingPhase(t *testing.T) { + for _, to := range []Status{StatusPreparingTarget, StatusMigrating, StatusRetiring} { + if !Allowed(StatusFailed, to) { + t.Fatalf("Allowed(failed, %s) = false, want true", to) + } + } +} + +func TestOnTaskExhausted(t *testing.T) { + cases := []struct { + status Status + want Status + changed bool + }{ + {StatusPreparingTarget, StatusFailed, true}, + {StatusMigrating, StatusFailed, true}, + {StatusWaiting, StatusFailed, true}, + // Retirement needs an operator decision, not another copy attempt. + {StatusRetiring, StatusCleanupAttention, true}, + {StatusCleanupAttention, StatusCleanupAttention, false}, + {StatusFailed, StatusFailed, false}, + {StatusCompleted, StatusCompleted, false}, + {StatusSuperseded, StatusSuperseded, false}, + } + for _, tc := range cases { + got, changed := OnTaskExhausted(tc.status) + if got != tc.want || changed != tc.changed { + t.Fatalf("OnTaskExhausted(%s) = (%s, %v), want (%s, %v)", tc.status, got, changed, tc.want, tc.changed) + } + } + for _, tc := range cases { + if !tc.changed { + continue + } + if !Allowed(tc.status, tc.want) { + t.Fatalf("exhaustion moves %s to %s, but the state machine forbids it", tc.status, tc.want) + } + } +} + +func TestStatusClassification(t *testing.T) { + cases := []struct { + status Status + active bool + terminal bool + retryable bool + }{ + {StatusPreparingTarget, true, false, false}, + {StatusMigrating, true, false, false}, + {StatusWaiting, true, false, false}, + {StatusRetiring, true, false, false}, + {StatusCleanupAttention, false, false, true}, + {StatusFailed, false, false, true}, + {StatusCompleted, false, true, false}, + {StatusSuperseded, false, true, false}, + } + for _, tc := range cases { + if got := tc.status.Active(); got != tc.active { + t.Fatalf("%s.Active() = %v, want %v", tc.status, got, tc.active) + } + if got := tc.status.Terminal(); got != tc.terminal { + t.Fatalf("%s.Terminal() = %v, want %v", tc.status, got, tc.terminal) + } + if got := tc.status.Retryable(); got != tc.retryable { + t.Fatalf("%s.Retryable() = %v, want %v", tc.status, got, tc.retryable) + } + if !tc.status.Valid() { + t.Fatalf("%s.Valid() = false, want true", tc.status) + } + } + if Status("unknown").Valid() { + t.Fatal("unknown status reported as valid") + } +} + +func TestPhaseForMatchesCoordinatorWork(t *testing.T) { + cases := map[Status]Phase{ + StatusPreparingTarget: PhasePrepare, + StatusMigrating: PhaseMigrate, + StatusRetiring: PhaseRetire, + StatusWaiting: PhaseNone, + StatusCleanupAttention: PhaseNone, + StatusFailed: PhaseNone, + StatusCompleted: PhaseNone, + StatusSuperseded: PhaseNone, + } + for status, want := range cases { + if got := PhaseFor(status); got != want { + t.Fatalf("PhaseFor(%s) = %s, want %s", status, got, want) + } + } +} + +func TestItemStatusBlocksRetirementWhileExecutable(t *testing.T) { + cases := map[ItemStatus]bool{ + ItemStatusPending: true, + ItemStatusRunning: true, + ItemStatusWaitingSource: true, + ItemStatusCopied: false, + ItemStatusCancelled: false, + } + for status, want := range cases { + if got := status.Blocking(); got != want { + t.Fatalf("%s.Blocking() = %v, want %v", status, got, want) + } + if !status.Valid() { + t.Fatalf("%s.Valid() = false, want true", status) + } + } +} + +func TestWaitReasonsCarryOperatorMessages(t *testing.T) { + reasons := []WaitReason{ + WaitReasonReadableSource, WaitReasonTarget, WaitReasonTargetCreating, WaitReasonTargetWritable, + WaitReasonFunding, WaitReasonProvider, + WaitReasonTerminationEpoch, WaitReasonSourceWrites, WaitReasonCoverage, + } + seen := make(map[string]WaitReason, len(reasons)) + for _, reason := range reasons { + if !reason.Valid() { + t.Fatalf("%s.Valid() = false, want true", reason) + } + message := reason.Message() + if message == "" { + t.Fatalf("%s has no operator message", reason) + } + if other, ok := seen[message]; ok { + t.Fatalf("%s and %s share the message %q, so an operator cannot tell them apart", reason, other, message) + } + seen[message] = reason + } + if WaitReason("unknown").Valid() { + t.Fatal("unknown wait reason reported as valid") + } +} diff --git a/internal/storagereplacement/waitreason.go b/internal/storagereplacement/waitreason.go new file mode 100644 index 0000000..051e9be --- /dev/null +++ b/internal/storagereplacement/waitreason.go @@ -0,0 +1,72 @@ +package storagereplacement + +// WaitReason explains a recoverable pause so an operator can tell "nothing is +// wrong yet" from "something needs me". It is only meaningful while the status +// is StatusWaiting. +type WaitReason string + +const ( + // WaitReasonReadableSource means no replica and no cached copy can supply + // the content this item needs. + WaitReasonReadableSource WaitReason = "readable_source" + // WaitReasonTarget means the approved target provider is unreachable. + WaitReasonTarget WaitReason = "target" + // WaitReasonTargetCreating means the paid target service has not been + // created yet. + WaitReasonTargetCreating WaitReason = "target_creating" + // WaitReasonTargetWritable means the created service is not writable yet. + WaitReasonTargetWritable WaitReason = "target_writable" + // WaitReasonFunding means the wallet cannot yet pay for the target service. + WaitReasonFunding WaitReason = "funding" + // WaitReasonProvider means a provider call failed in a way that resolves on + // its own. + WaitReasonProvider WaitReason = "provider" + // WaitReasonTerminationEpoch means the chain has not yet reached the epoch + // at which the old service ends. + WaitReasonTerminationEpoch WaitReason = "termination_epoch" + // WaitReasonSourceWrites means a write to the old provider is still in + // flight, so it cannot be retired. + WaitReasonSourceWrites WaitReason = "source_writes" + // WaitReasonCoverage means some retained version is not yet readable on the + // new provider. + WaitReasonCoverage WaitReason = "coverage" +) + +// Valid reports whether the value is a known wait reason. +func (r WaitReason) Valid() bool { + switch r { + case WaitReasonReadableSource, WaitReasonTarget, WaitReasonTargetCreating, WaitReasonTargetWritable, + WaitReasonFunding, WaitReasonProvider, + WaitReasonTerminationEpoch, WaitReasonSourceWrites, WaitReasonCoverage: + return true + default: + return false + } +} + +// Message is the operator-facing explanation of the pause. It says what is +// being waited on, never an internal state name. +func (r WaitReason) Message() string { + switch r { + case WaitReasonReadableSource: + return "Waiting for a readable copy of some stored content" + case WaitReasonTarget: + return "Waiting for the replacement provider to become reachable" + case WaitReasonTargetCreating: + return "Waiting for the replacement storage service to be created" + case WaitReasonTargetWritable: + return "Waiting for the replacement storage service to become writable" + case WaitReasonFunding: + return "Waiting for wallet funds to cover the replacement service" + case WaitReasonProvider: + return "Waiting for the storage provider to respond" + case WaitReasonTerminationEpoch: + return "Waiting for the old service to reach its end of term" + case WaitReasonSourceWrites: + return "Waiting for in-flight writes to the old provider to finish" + case WaitReasonCoverage: + return "Waiting for every retained version to be readable on the new provider" + default: + return "Waiting for a dependency" + } +} diff --git a/internal/synapse/epochs.go b/internal/synapse/epochs.go new file mode 100644 index 0000000..5615842 --- /dev/null +++ b/internal/synapse/epochs.go @@ -0,0 +1,43 @@ +package synapse + +import ( + "context" + "errors" + "fmt" + "math" +) + +// blockNumberSource is satisfied by an Ethereum JSON-RPC client. On Filecoin the +// EVM block number is the chain epoch, which is the same source the storage SDK +// reads. +type blockNumberSource interface { + BlockNumber(ctx context.Context) (uint64, error) +} + +type chainEpochReader struct { + source blockNumberSource +} + +// NewChainEpochReader reads the epoch from the chain head. +// +// The wall-clock estimate available elsewhere in the SDK derives an epoch from +// genesis time and can run ahead of the chain, which would let a replacement +// retire a source before its service has really ended. Retirement therefore +// only trusts an observed block number. +func NewChainEpochReader(source blockNumberSource) ChainEpochReader { + return &chainEpochReader{source: source} +} + +func (r *chainEpochReader) CurrentEpoch(ctx context.Context) (int64, error) { + if r == nil || r.source == nil { + return 0, errors.New("chain epoch reader is not configured") + } + height, err := r.source.BlockNumber(ctx) + if err != nil { + return 0, fmt.Errorf("reading chain head epoch: %w", NormalizeProviderOperationError(ctx, err)) + } + if height > math.MaxInt64 { + return 0, fmt.Errorf("chain head epoch %d is out of range", height) + } + return int64(height), nil +} diff --git a/internal/synapse/interfaces.go b/internal/synapse/interfaces.go index 3b90036..6855591 100644 --- a/internal/synapse/interfaces.go +++ b/internal/synapse/interfaces.go @@ -43,6 +43,27 @@ type StorageClient interface { CreateCleanupContext(ctx context.Context, opts *storage.CreateContextOptions) (CleanupContext, error) } +// ServiceTerminator is the destructive service-lifecycle boundary. It is used +// only after replacement cleanup authorization and the retirement safety gate +// have both passed. +type ServiceTerminator interface { + TerminateService(ctx context.Context, dataSetID sdktypes.BigInt) (*TerminationResult, error) +} + +// TerminationResult records what the chain agreed to. EndEpoch is the epoch at +// which the service actually stops, which is why retirement waits for the chain +// to reach it rather than trusting the call returning. +type TerminationResult struct { + TxHash string + EndEpoch int64 +} + +// ChainEpochReader observes the chain head. Replacement uses it to decide when +// a terminated service has genuinely ended. +type ChainEpochReader interface { + CurrentEpoch(ctx context.Context) (int64, error) +} + // WalletQuerier provides on-chain wallet state for the admin dashboard. type WalletQuerier interface { GetWalletInfo(ctx context.Context) (*WalletInfo, error) diff --git a/internal/synapse/storage_client.go b/internal/synapse/storage_client.go index d7b3eb8..d2a3eff 100644 --- a/internal/synapse/storage_client.go +++ b/internal/synapse/storage_client.go @@ -13,11 +13,12 @@ import ( // SynapS3's testable staged storage interface. It exists because Go does not // allow []*storage.Context to satisfy []UploadContext directly. type StorageServiceAdapter struct { - service *storage.Service + service *storage.Service + terminator storageServiceTerminator } func AdaptStorageService(service *storage.Service) *StorageServiceAdapter { - return &StorageServiceAdapter{service: service} + return &StorageServiceAdapter{service: service, terminator: service} } func (s *StorageServiceAdapter) Download(ctx context.Context, pieceCID cid.Cid, opts *storage.DownloadOptions) (io.ReadCloser, error) { diff --git a/internal/synapse/terminate.go b/internal/synapse/terminate.go new file mode 100644 index 0000000..3289f1e --- /dev/null +++ b/internal/synapse/terminate.go @@ -0,0 +1,110 @@ +package synapse + +import ( + "context" + "errors" + "fmt" + "math/big" + "time" + + "github.com/strahe/synapse-go/pdp" + "github.com/strahe/synapse-go/storage" + sdktypes "github.com/strahe/synapse-go/types" +) + +const providerTerminationWaitTimeout = 30 * time.Second + +type storageServiceTerminator interface { + TerminateService(context.Context, sdktypes.BigInt, *storage.TerminateServiceOptions) (*storage.TerminateServiceResult, error) +} + +// TerminationBlockedError means the service cannot be terminated until an +// operator resolves something the gateway must not decide on its own, such as +// settling outstanding payment debt. It is never retried automatically. +type TerminationBlockedError struct { + Reason string + Shortfall *big.Int + Err error +} + +func (e *TerminationBlockedError) Error() string { + if e == nil { + return "" + } + if e.Shortfall != nil { + return fmt.Sprintf("service termination blocked (%s): outstanding amount %s", e.Reason, e.Shortfall) + } + return fmt.Sprintf("service termination blocked (%s)", e.Reason) +} + +func (e *TerminationBlockedError) Unwrap() error { + if e == nil { + return nil + } + return e.Err +} + +// IsTerminationBlocked reports whether termination needs operator action rather +// than another attempt. +func IsTerminationBlocked(err error) bool { + var blocked *TerminationBlockedError + return errors.As(err, &blocked) +} + +// TerminateService ends the storage service for one data set. It is the +// destructive boundary of provider replacement and must only be called after +// the retirement safety gate passes. +// +// Termination is relayed through the provider first. A provider-side failure +// falls back to the direct FWSS transaction path; a pending relay does not, +// because the provider has already accepted it. An already-terminated service +// reports its recorded end epoch rather than failing. +func (s *StorageServiceAdapter) TerminateService(ctx context.Context, dataSetID sdktypes.BigInt) (*TerminationResult, error) { + if s == nil || s.terminator == nil { + return nil, errors.New("storage service terminator is not configured") + } + res, err := s.terminator.TerminateService(ctx, dataSetID, &storage.TerminateServiceOptions{ + ProviderWaitTimeout: providerTerminationWaitTimeout, + }) + if err != nil { + providerErr := normalizeTerminationError(ctx, err) + if terminationPending(err) || ctx.Err() != nil { + return nil, providerErr + } + res, err = s.terminator.TerminateService(ctx, dataSetID, &storage.TerminateServiceOptions{SkipProvider: true}) + if err != nil { + directErr := NormalizeProviderOperationError(ctx, err) + return nil, fmt.Errorf("direct termination failed after provider relay error (%v): %w", providerErr, directErr) + } + } + if res == nil { + return nil, errors.New("storage service returned no termination result") + } + out := &TerminationResult{EndEpoch: int64(res.EndEpoch)} + if res.TxHash != nil { + out.TxHash = res.TxHash.Hex() + } + return out, nil +} + +func terminationPending(err error) bool { + var pending *pdp.TerminateServicePendingError + return errors.As(err, &pending) +} + +func normalizeTerminationError(ctx context.Context, err error) error { + if err == nil { + return nil + } + var debt *storage.TerminateServiceDebtError + if errors.As(err, &debt) { + return &TerminationBlockedError{Reason: "payment_debt", Shortfall: debt.Shortfall, Err: err} + } + // The provider accepted the request but has not published it yet. That + // resolves on its own, so it is a dependency wait rather than a failure. + var pending *pdp.TerminateServicePendingError + if errors.As(err, &pending) { + return &ProviderUnavailableError{Cause: err} + } + return NormalizeProviderOperationError(ctx, err) +} diff --git a/internal/synapse/terminate_test.go b/internal/synapse/terminate_test.go new file mode 100644 index 0000000..2df4a76 --- /dev/null +++ b/internal/synapse/terminate_test.go @@ -0,0 +1,223 @@ +package synapse + +import ( + "context" + "errors" + "fmt" + "math" + "math/big" + "strings" + "testing" + + "github.com/strahe/synapse-go/pdp" + "github.com/strahe/synapse-go/storage" + sdktypes "github.com/strahe/synapse-go/types" +) + +type terminatorCall struct { + opts *storage.TerminateServiceOptions +} + +type stubStorageServiceTerminator struct { + calls []terminatorCall + results []*storage.TerminateServiceResult + errors []error +} + +func (s *stubStorageServiceTerminator) TerminateService( + _ context.Context, + _ sdktypes.BigInt, + opts *storage.TerminateServiceOptions, +) (*storage.TerminateServiceResult, error) { + s.calls = append(s.calls, terminatorCall{opts: opts}) + index := len(s.calls) - 1 + var result *storage.TerminateServiceResult + if index < len(s.results) { + result = s.results[index] + } + if index < len(s.errors) { + return result, s.errors[index] + } + return result, nil +} + +func TestStorageServiceAdapterTerminationFallsBackToDirect(t *testing.T) { + stub := &stubStorageServiceTerminator{ + results: []*storage.TerminateServiceResult{nil, {EndEpoch: 84}}, + errors: []error{ + &storage.TerminateServiceDebtError{Shortfall: big.NewInt(1)}, + nil, + }, + } + adapter := &StorageServiceAdapter{terminator: stub} + got, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) + if err != nil { + t.Fatalf("TerminateService: %v", err) + } + if got == nil || got.EndEpoch != 84 { + t.Fatalf("result = %#v, want end epoch 84", got) + } + if len(stub.calls) != 2 || stub.calls[0].opts.SkipProvider || !stub.calls[1].opts.SkipProvider { + t.Fatalf("calls = %#v, want provider relay followed by direct termination", stub.calls) + } + if stub.calls[0].opts.ProviderWaitTimeout != providerTerminationWaitTimeout { + t.Fatalf("provider wait timeout = %s, want %s", stub.calls[0].opts.ProviderWaitTimeout, providerTerminationWaitTimeout) + } +} + +func TestStorageServiceAdapterTerminationFallsBackAfterProviderSubTimeout(t *testing.T) { + stub := &stubStorageServiceTerminator{ + results: []*storage.TerminateServiceResult{nil, {EndEpoch: 91}}, + errors: []error{context.DeadlineExceeded, nil}, + } + adapter := &StorageServiceAdapter{terminator: stub} + got, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) + if err != nil { + t.Fatalf("TerminateService: %v", err) + } + if got == nil || got.EndEpoch != 91 { + t.Fatalf("result = %#v, want end epoch 91", got) + } + if len(stub.calls) != 2 || !stub.calls[1].opts.SkipProvider { + t.Fatalf("calls = %#v, want provider timeout followed by direct termination", stub.calls) + } +} + +func TestStorageServiceAdapterTerminationDoesNotDuplicatePendingRelay(t *testing.T) { + stub := &stubStorageServiceTerminator{errors: []error{&pdp.TerminateServicePendingError{Message: "queued"}}} + adapter := &StorageServiceAdapter{terminator: stub} + _, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) + if !IsProviderUnavailable(err) { + t.Fatalf("error = %T %v, want dependency wait", err, err) + } + if len(stub.calls) != 1 { + t.Fatalf("calls = %d, want no direct fallback for pending relay", len(stub.calls)) + } +} + +func TestStorageServiceAdapterTerminationHonorsCallerCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + stub := &stubStorageServiceTerminator{errors: []error{context.Canceled}} + adapter := &StorageServiceAdapter{terminator: stub} + _, err := adapter.TerminateService(ctx, sdktypes.NewBigInt(42)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v, want context cancellation", err) + } + if len(stub.calls) != 1 { + t.Fatalf("calls = %d, want no fallback after caller cancellation", len(stub.calls)) + } +} + +func TestStorageServiceAdapterTerminationPreservesDirectFailure(t *testing.T) { + directErr := errors.New("rpc unavailable") + stub := &stubStorageServiceTerminator{ + errors: []error{errors.New("provider unavailable"), directErr}, + } + adapter := &StorageServiceAdapter{terminator: stub} + _, err := adapter.TerminateService(context.Background(), sdktypes.NewBigInt(42)) + if !errors.Is(err, directErr) || !strings.Contains(err.Error(), "provider unavailable") { + t.Fatalf("error = %v, want both provider and direct evidence", err) + } +} + +// Outstanding debt is a decision for the operator, not something to retry, so +// it must never look like a transient provider problem. +func TestNormalizeTerminationErrorReportsPaymentDebtAsBlocked(t *testing.T) { + t.Parallel() + + shortfall := big.NewInt(4200) + got := normalizeTerminationError(context.Background(), fmt.Errorf( + "storage.Service.TerminateService: %w", + &storage.TerminateServiceDebtError{Shortfall: shortfall}, + )) + if !IsTerminationBlocked(got) { + t.Fatalf("error = %T %v, want a blocked termination", got, got) + } + var blocked *TerminationBlockedError + if !errors.As(got, &blocked) { + t.Fatalf("error = %T, want *TerminationBlockedError", got) + } + if blocked.Reason != "payment_debt" || blocked.Shortfall.Cmp(shortfall) != 0 { + t.Fatalf("blocked = %+v, want the payment shortfall preserved", blocked) + } + if IsProviderUnavailable(got) { + t.Fatal("payment debt reported as provider unavailability, which would retry forever") + } +} + +// A termination the provider has accepted but not yet published resolves on its +// own, so it must wait rather than raise operator attention. +func TestNormalizeTerminationErrorTreatsPendingPublicationAsRetryable(t *testing.T) { + t.Parallel() + + got := normalizeTerminationError(context.Background(), fmt.Errorf( + "storage.Service.TerminateService: %w", + &pdp.TerminateServicePendingError{Message: "queued"}, + )) + if !IsProviderUnavailable(got) { + t.Fatalf("error = %T %v, want provider unavailability", got, got) + } + if IsTerminationBlocked(got) { + t.Fatal("pending publication reported as blocked, which would need operator action") + } +} + +func TestNormalizeTerminationErrorPassesThroughUnknownFailures(t *testing.T) { + t.Parallel() + + if got := normalizeTerminationError(context.Background(), nil); got != nil { + t.Fatalf("nil error = %v, want nil", got) + } + unknown := errors.New("provider exploded") + got := normalizeTerminationError(context.Background(), unknown) + if IsTerminationBlocked(got) { + t.Fatalf("unknown error = %T, want it left retryable", got) + } + if !errors.Is(got, unknown) { + t.Fatalf("unknown error = %v, want the cause preserved", got) + } +} + +type stubBlockNumberSource struct { + height uint64 + err error +} + +func (s stubBlockNumberSource) BlockNumber(context.Context) (uint64, error) { + return s.height, s.err +} + +func TestChainEpochReaderReportsChainHead(t *testing.T) { + t.Parallel() + + epoch, err := NewChainEpochReader(stubBlockNumberSource{height: 4096}).CurrentEpoch(context.Background()) + if err != nil { + t.Fatalf("CurrentEpoch: %v", err) + } + if epoch != 4096 { + t.Fatalf("epoch = %d, want the observed block number", epoch) + } +} + +// Retirement must never treat an unreadable chain as "the epoch was reached". +func TestChainEpochReaderRefusesToGuess(t *testing.T) { + t.Parallel() + + cases := map[string]ChainEpochReader{ + "rpc failure": NewChainEpochReader(stubBlockNumberSource{err: errors.New("rpc down")}), + "out of range": NewChainEpochReader(stubBlockNumberSource{height: math.MaxUint64}), + "unconfigured": NewChainEpochReader(nil), + } + for name, reader := range cases { + t.Run(name, func(t *testing.T) { + epoch, err := reader.CurrentEpoch(context.Background()) + if err == nil { + t.Fatalf("CurrentEpoch = %d, want an error", epoch) + } + if epoch != 0 { + t.Fatalf("epoch = %d on failure, want 0", epoch) + } + }) + } +} diff --git a/internal/systemtest/filecoin.go b/internal/systemtest/filecoin.go index 29ad0ef..e30e9f4 100644 --- a/internal/systemtest/filecoin.go +++ b/internal/systemtest/filecoin.go @@ -51,15 +51,31 @@ type MemoryFilecoin struct { pieces map[string]*memoryPiece nextDataSet map[string]uint64 nextPiece uint64 -} - -// NewMemoryFilecoin creates three deterministic active storage providers. + // terminated records the epoch at which each data set's service ends, and + // epoch is the observed chain head. Tests advance the head to prove that + // retirement waits for the chain rather than for the call returning. + terminated map[string]int64 + epoch int64 + // terminationDelay is how far ahead of the chain head a terminated service + // ends. Zero means the end of term is already reached. + terminationDelay int64 +} + +// MemoryFilecoinProviders is how many providers the fake offers. One more than +// a bucket's default replica count, so an approved replacement always has an +// unused provider to move to. +const MemoryFilecoinProviders = 4 + +// NewMemoryFilecoin creates deterministic active storage providers. func NewMemoryFilecoin() *MemoryFilecoin { return &MemoryFilecoin{ + // The fourth provider is never used by a fresh bucket, so an approved + // replacement always has somewhere to move to. providers: []sdktypes.BigInt{ sdktypes.NewBigInt(101), sdktypes.NewBigInt(102), sdktypes.NewBigInt(103), + sdktypes.NewBigInt(104), }, dataSets: make(map[string]*memoryDataSet), pendingDataSets: make(map[string]sdktypes.BigInt), @@ -67,6 +83,8 @@ func NewMemoryFilecoin() *MemoryFilecoin { pieces: make(map[string]*memoryPiece), nextDataSet: make(map[string]uint64), nextPiece: 1, + terminated: make(map[string]int64), + epoch: 1000, } } @@ -458,6 +476,51 @@ func copyBigIntPtr(value sdktypes.BigInt) *sdktypes.BigInt { } // GetWalletInfo returns a complete, funded wallet snapshot. +// TerminateService ends a data set's service. The fake reports an end of term +// the chain has already reached, because a system test exercises the operator +// flow rather than chain timing; use terminationDelay to make retirement wait. +func (m *MemoryFilecoin) TerminateService(_ context.Context, dataSetID sdktypes.BigInt) (*synapse.TerminationResult, error) { + m.mu.Lock() + defer m.mu.Unlock() + key := dataSetID.String() + endEpoch, ok := m.terminated[key] + if !ok { + endEpoch = m.epoch + m.terminationDelay + m.terminated[key] = endEpoch + } + return &synapse.TerminationResult{TxHash: "0xterminate" + key, EndEpoch: endEpoch}, nil +} + +// SetTerminationDelay controls how many epochs pass between termination and the +// end of term, so a test can decide whether retirement has to wait. +func (m *MemoryFilecoin) SetTerminationDelay(epochs int64) { + m.mu.Lock() + defer m.mu.Unlock() + m.terminationDelay = epochs +} + +func (m *MemoryFilecoin) CurrentEpoch(context.Context) (int64, error) { + m.mu.RLock() + defer m.mu.RUnlock() + return m.epoch, nil +} + +// AdvanceEpoch moves the observed chain head forward. +func (m *MemoryFilecoin) AdvanceEpoch(delta int64) { + m.mu.Lock() + defer m.mu.Unlock() + m.epoch += delta +} + +// TerminationEpoch reports the recorded end of term, or false when the service +// was never terminated. +func (m *MemoryFilecoin) TerminationEpoch(dataSetID string) (int64, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + epoch, ok := m.terminated[dataSetID] + return epoch, ok +} + func (m *MemoryFilecoin) GetWalletInfo(context.Context) (*synapse.WalletInfo, error) { nonce := uint64(1) return &synapse.WalletInfo{ diff --git a/internal/systemtest/harness.go b/internal/systemtest/harness.go index 1c0735d..f97f1c3 100644 --- a/internal/systemtest/harness.go +++ b/internal/systemtest/harness.go @@ -159,6 +159,7 @@ func newHarness(ctx context.Context, logger *slog.Logger, s3Address string) (_ * Filecoin: app.FilecoinServices{ Storage: filecoin, WalletQuery: filecoin, Wallet: filecoin, Receipts: filecoin, Readiness: filecoin, Observability: filecoin, + Terminator: filecoin, Epochs: filecoin, }, S3Addresses: []string{socketPath}, ShutdownTimeout: 5 * time.Second, }) diff --git a/internal/testutil/mocks.go b/internal/testutil/mocks.go index d1bf5a0..c9f511c 100644 --- a/internal/testutil/mocks.go +++ b/internal/testutil/mocks.go @@ -14,9 +14,11 @@ import ( // Compile-time interface checks. var ( - _ synapse.StorageClient = (*MockStorageClient)(nil) - _ synapse.WalletQuerier = (*MockWalletQuerier)(nil) - _ cache.Cache = (*MockCache)(nil) + _ synapse.StorageClient = (*MockStorageClient)(nil) + _ synapse.WalletQuerier = (*MockWalletQuerier)(nil) + _ synapse.ServiceTerminator = (*MockServiceTerminator)(nil) + _ synapse.ChainEpochReader = (*MockChainEpochReader)(nil) + _ cache.Cache = (*MockCache)(nil) ) // MockStorageClient is a configurable test double for synapse.StorageClient. @@ -263,3 +265,30 @@ func (m *MockCache) DeleteUpload(ctx context.Context, uploadID string) error { } return nil } + +// MockServiceTerminator is a configurable test double for +// synapse.ServiceTerminator. +type MockServiceTerminator struct { + TerminateServiceFunc func(ctx context.Context, dataSetID sdktypes.BigInt) (*synapse.TerminationResult, error) +} + +func (m *MockServiceTerminator) TerminateService(ctx context.Context, dataSetID sdktypes.BigInt) (*synapse.TerminationResult, error) { + if m.TerminateServiceFunc != nil { + return m.TerminateServiceFunc(ctx, dataSetID) + } + return nil, errors.New("MockServiceTerminator.TerminateService not configured") +} + +// MockChainEpochReader is a configurable test double for +// synapse.ChainEpochReader. An unconfigured reader reports epoch zero, which +// keeps a retirement gate waiting rather than letting it pass by accident. +type MockChainEpochReader struct { + CurrentEpochFunc func(ctx context.Context) (int64, error) +} + +func (m *MockChainEpochReader) CurrentEpoch(ctx context.Context) (int64, error) { + if m.CurrentEpochFunc != nil { + return m.CurrentEpochFunc(ctx) + } + return 0, nil +} diff --git a/internal/worker/manager.go b/internal/worker/manager.go index 865e409..a0225ff 100644 --- a/internal/worker/manager.go +++ b/internal/worker/manager.go @@ -13,6 +13,7 @@ import ( "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/state" + "github.com/strahe/synaps3/internal/storagereplacement" ) // Worker defines a background processing unit. @@ -122,6 +123,7 @@ func (m *Manager) recoverOnStartup(ctx context.Context) { m.reconcileStagedUploads(ctx) m.reconcileIncompleteReadableUploads(ctx) m.reconcileUnavailableDataSets(ctx) + m.reconcileProviderReplacements(ctx) // Log exhausted task count for operator awareness exhaustedTasks, err := m.repos.Tasks.ListExhausted(ctx, 100) @@ -172,6 +174,70 @@ func (m *Manager) reconcileUnavailableDataSets(ctx context.Context) { } } +// reconcileProviderReplacements re-issues the coordinator for every replacement +// that still has work. Both coordinator keys are idempotent singletons, so this +// is safe to run on every boot, and it also picks up an abandoned target left by +// a superseded confirmation. +func (m *Manager) reconcileProviderReplacements(ctx context.Context) { + afterID := int64(0) + for { + rows, err := m.repos.Replacements.ListActive(ctx, afterID, reconcileBatchSize) + if err != nil { + m.logger.Error("failed to list active provider replacements for recovery", "error", err) + break + } + for i := range rows { + row := &rows[i] + m.enqueueReplacementCoordinator(ctx, row) + afterID = row.ID + } + if len(rows) < reconcileBatchSize { + break + } + } + + afterID = 0 + for { + rows, err := m.repos.Replacements.ListSupersededCleanupCandidates(ctx, afterID, reconcileBatchSize) + if err != nil { + m.logger.Error("failed to list superseded provider replacements for cleanup", "error", err) + return + } + for i := range rows { + row := &rows[i] + // A superseded replacement's leftover is its unused target, not its + // source; the source still belongs to whoever took over the slot. + task := storagereplacement.NewAbandonedTargetTask(row.ID, row.BucketID, m.uploadMaxRetries, time.Now()) + // Leftover paid services have no Data Sets retry: the replacement is + // superseded. Revive a coordinator that failed or exhausted so an + // unused target does not stay reserved across restarts. + if _, err := m.repos.Tasks.ResumeCoordinator(ctx, task); err != nil { + m.logger.Error("failed to ensure superseded replacement cleanup", "replacementID", row.ID, "error", err) + } + afterID = row.ID + } + if len(rows) < reconcileBatchSize { + return + } + } +} + +// A replacement resumes through whichever coordinator owns its current phase. +func (m *Manager) enqueueReplacementCoordinator(ctx context.Context, row *storagereplacement.Replacement) { + var task *model.Task + // A recorded termination epoch means the old service has already been asked + // to end, so this replacement is past migration even if a wait moved it out + // of the retiring status. + if row.Status == storagereplacement.StatusRetiring || row.TerminationEpoch != nil { + task = storagereplacement.NewRetireTask(row.ID, row.BucketID, m.uploadMaxRetries, time.Now()) + } else { + task = storagereplacement.NewMigrateTask(row.ID, row.BucketID, "", m.uploadMaxRetries, time.Now()) + } + if _, err := m.repos.Tasks.EnsureRecurring(ctx, task); err != nil { + m.logger.Error("failed to ensure provider replacement coordinator", "replacementID", row.ID, "error", err) + } +} + // reconcileTasks finds object versions in the given state and ensures each has a corresponding // queued task. Uses idempotency keys to safely skip objects that already have tasks. // keyPrefix must match the prefix used by the normal task creation path for deduplication. diff --git a/internal/worker/manager_test.go b/internal/worker/manager_test.go index e4bfdae..6207c06 100644 --- a/internal/worker/manager_test.go +++ b/internal/worker/manager_test.go @@ -733,7 +733,7 @@ func TestManager_RecoverOnStartup_DoesNotReplaceAssignedFailedPeer(t *testing.T) }); err != nil { t.Fatalf("BindReadableUploadForContent: %v", err) } - if err := repos.Uploads.MarkUploadCopyFailed(ctx, upload.ID, 2, "peer pull: provider failed"); err != nil { + if err := repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: upload.ID, CopyIndex: 2, LastError: "peer pull: provider failed"}); err != nil { t.Fatalf("MarkUploadCopyFailed: %v", err) } diff --git a/internal/worker/storage_cleanup.go b/internal/worker/storage_cleanup.go index e392d12..1d9d046 100644 --- a/internal/worker/storage_cleanup.go +++ b/internal/worker/storage_cleanup.go @@ -12,12 +12,17 @@ import ( "github.com/strahe/synaps3/internal/admin" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" "github.com/strahe/synaps3/internal/synapse" "github.com/strahe/synapse-go/storage" ) type StorageCleanupWorker struct { - repos *repository.Repositories + repos *repository.Repositories + // terminator and epochs are only used by provider replacement retirement, + // which is the sole destructive service-lifecycle path. + terminator synapse.ServiceTerminator + epochs synapse.ChainEpochReader storage synapse.StorageClient concurrency int pollInterval time.Duration @@ -33,8 +38,20 @@ const ( var errStorageCleanupCopyUnsupported = errors.New("storage cleanup copy unsupported") -func NewStorageCleanupWorker(repos *repository.Repositories, storageClient synapse.StorageClient, concurrency int, pollInterval time.Duration, logger *slog.Logger) *StorageCleanupWorker { - return &StorageCleanupWorker{ +// StorageCleanupOption configures optional cleanup dependencies. +type StorageCleanupOption func(*StorageCleanupWorker) + +// WithServiceTermination enables provider replacement retirement. Without it +// the worker still removes pieces but refuses to end any service. +func WithServiceTermination(terminator synapse.ServiceTerminator, epochs synapse.ChainEpochReader) StorageCleanupOption { + return func(w *StorageCleanupWorker) { + w.terminator = terminator + w.epochs = epochs + } +} + +func NewStorageCleanupWorker(repos *repository.Repositories, storageClient synapse.StorageClient, concurrency int, pollInterval time.Duration, logger *slog.Logger, opts ...StorageCleanupOption) *StorageCleanupWorker { + w := &StorageCleanupWorker{ repos: repos, storage: storageClient, concurrency: concurrency, @@ -43,6 +60,10 @@ func NewStorageCleanupWorker(repos *repository.Repositories, storageClient synap logger: logger, livenessTracker: newLivenessTracker(pollInterval), } + for _, opt := range opts { + opt(w) + } + return w } func (w *StorageCleanupWorker) Name() string { return "storage_cleanup" } @@ -122,6 +143,14 @@ func (w *StorageCleanupWorker) processTask(ctx context.Context, task *model.Task admin.WorkerTaskDuration.WithLabelValues("storage_cleanup").Observe(time.Since(start).Seconds()) }() + if task.Stage != nil { + switch *task.Stage { + case storagereplacement.StageRetire, storagereplacement.StageRetireAbandonedTarget: + w.processReplacementRetirementTask(ctx, task) + return + } + } + logger := w.logger.With("taskID", task.ID, "uploadID", task.RefID) hasRefs, err := w.repos.StorageCleanup.UploadHasObjectReferences(ctx, task.RefID) if err != nil { diff --git a/internal/worker/storage_cleanup_replacement.go b/internal/worker/storage_cleanup_replacement.go new file mode 100644 index 0000000..c0f8d55 --- /dev/null +++ b/internal/worker/storage_cleanup_replacement.go @@ -0,0 +1,530 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/strahe/synaps3/internal/admin" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/strahe/synaps3/internal/synapse" + idtypes "github.com/strahe/synaps3/internal/types" + "github.com/strahe/synapse-go/storage" +) + +// processReplacementRetirementTask ends a replaced storage service. Every step +// is ordered so the destructive call sits between transactions and can never +// run against a stale view: +// +// tx A evaluate the safety gate, wait if anything still needs the source +// -- terminate the service with the provider +// tx B record the epoch at which the service ends +// -- observe the chain head +// tx C re-evaluate the whole gate, then retire and complete +// +// A blocker that can clear on its own returns to waiting; only a structural +// problem or a payment decision raises operator attention. +func (w *StorageCleanupWorker) processReplacementRetirementTask(ctx context.Context, task *model.Task) { + logger := w.logger.With("taskID", task.ID) + replacementID, err := storagereplacement.ParseRetirePayload(task) + if err != nil { + w.failReplacementRetirement(ctx, task, logger, "parse replacement retirement task", err) + return + } + logger = logger.With("replacementID", replacementID) + + replacement, err := w.repos.Replacements.GetByID(ctx, replacementID) + if err != nil { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "load provider replacement", err) + return + } + if replacement == nil { + w.completeTask(ctx, task, logger, "Replacement already retired") + return + } + if task.Stage != nil && *task.Stage == storagereplacement.StageRetireAbandonedTarget { + w.retireAbandonedTarget(ctx, task, replacement, logger) + return + } + if replacement.Status == storagereplacement.StatusCompleted { + w.completeTask(ctx, task, logger, "Replacement already retired") + return + } + if replacement.Status == storagereplacement.StatusSuperseded { + // A later confirmation owns this slot now. Retiring this replacement's + // source would terminate a service the successor still depends on. + w.completeTask(ctx, task, logger, "Replacement superseded") + return + } + if replacement.Status == storagereplacement.StatusCleanupAttention { + // Automatic retry is suppressed until an operator acts, so the task must + // not keep re-running on its own. + w.completeTask(ctx, task, logger, "Waiting for operator action") + return + } + if w.terminator == nil || w.epochs == nil { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "retire replaced service", + errors.New("service termination is not configured")) + return + } + source, err := w.repos.Uploads.GetDataSetBindingByID(ctx, replacement.SourceDataSetID) + if err != nil { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "load retiring data set", err) + return + } + if source == nil || source.DataSetID == nil || source.DataSetID.IsZero() { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "load retiring data set", + fmt.Errorf("data set %d has no on-chain identity: %w", replacement.SourceDataSetID, repository.ErrNotFound)) + return + } + if source.Status == model.StorageDataSetStatusRetired { + w.completeTask(ctx, task, logger, "Replaced provider already retired") + return + } + + // tx A: nothing may still need the source before it is terminated. + gate, err := w.repos.Replacements.EvaluateRetirementGate(ctx, replacementID, nil) + if err != nil { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "evaluate retirement gate", err) + return + } + if !gate.Passed() { + w.waitForReplacementRetirement(ctx, task, replacementID, gate, logger) + return + } + // Nothing needs the source any more, so the record should say retiring + // rather than waiting while the destructive work runs. + if replacement.Status != storagereplacement.StatusRetiring { + if err := w.repos.Replacements.BeginRetirement(ctx, replacementID); err != nil { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "resume replacement retirement", err) + return + } + replacement.Status = storagereplacement.StatusRetiring + } + + if replacement.TerminationEpoch == nil { + result, err := w.terminator.TerminateService(ctx, source.DataSetID.SDK()) + if err != nil { + w.handleTerminationError(ctx, task, replacementID, logger, err) + return + } + if result == nil { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "terminate replaced service", + errors.New("storage service returned no termination result")) + return + } + // tx B: the end of term is recorded before the service is treated as + // terminated, so a crash here re-reads it instead of terminating twice. + if err := w.repos.Replacements.RecordTerminationEpoch(ctx, repository.RecordTerminationEpochInput{ + ReplacementID: replacementID, + TxHash: result.TxHash, + Epoch: result.EndEpoch, + }); err != nil { + // A conflict means someone recorded an epoch first. Trust the stored + // value rather than the one this attempt just observed, but never + // continue without one. + stored, loadErr := w.repos.Replacements.GetByID(ctx, replacementID) + if loadErr != nil || stored == nil || stored.TerminationEpoch == nil { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "record termination epoch", err) + return + } + replacement.TerminationEpoch = stored.TerminationEpoch + } else { + replacement.TerminationEpoch = &result.EndEpoch + } + logger.Info("replaced storage service termination submitted", + "dataSetID", source.DataSetID.String(), "endEpoch", *replacement.TerminationEpoch) + } + + observed, err := w.epochs.CurrentEpoch(ctx) + if err != nil { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "read chain epoch", err) + return + } + if replacement.TerminationEpoch == nil || observed < *replacement.TerminationEpoch { + w.waitForReplacementEpoch(ctx, task, replacementID, logger) + return + } + + // tx C: the gate is evaluated again from scratch, this time including the + // epoch, and the repository refuses the change if anything regressed. + if err := w.repos.Replacements.CompleteRetirement(ctx, replacementID, observed); err != nil { + w.handleRetirementCompletionError(ctx, task, replacementID, logger, err) + return + } + logger.Info("replaced storage service retired", "dataSetID", source.DataSetID.String()) + w.completeTask(ctx, task, logger, "Replaced provider retired") +} + +// retireAbandonedTarget ends the paid service of a target a later confirmation +// replaced. It asks a narrower question than source retirement: the abandoned +// generation only holds partly migrated data, so the gate is simply that +// nothing depends on it as its last readable copy. The replacement record stays +// superseded throughout. +func (w *StorageCleanupWorker) retireAbandonedTarget( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + logger *slog.Logger, +) { + if w.terminator == nil || w.epochs == nil { + // Abandoned cleanup has no operator retry: the replacement is superseded. + // Waiting keeps the coordinator claimable until termination is configured. + w.waitForAbandonedTarget(ctx, task, logger, "Waiting until unused storage services can be ended") + return + } + target, err := w.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) + if err != nil { + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") + return + } + if target == nil || target.Status == model.StorageDataSetStatusRetired { + w.completeTask(ctx, task, logger, "Abandoned provider already retired") + return + } + if target.IsCurrent { + // It took over the slot after all, so it is not abandoned. + w.completeTask(ctx, task, logger, "Target is in use") + return + } + if target.DataSetID == nil || target.DataSetID.IsZero() { + if dataSetBindingHasCreationEvidence(target) { + // A creation was submitted and may still land a paid service on this + // provider. Retiring the local row now would release the provider + // while that service exists and nothing is left watching for it, so + // the submission is resolved first. + w.resolveAbandonedTargetCreation(ctx, task, replacement, target, logger) + return + } + // Nothing was ever asked of this provider, but the local generation still + // holds it for this bucket: an unretired generation makes the provider + // unavailable to both automatic selection and a manual choice. Release it. + if err := w.releaseAbandonedTargetWithoutService(ctx, replacement, target, + "provider replacement was superseded before a storage service was created"); err != nil { + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to release the unused storage provider") + return + } + w.completeTask(ctx, task, logger, "Abandoned provider released without a service") + return + } + + sole, err := w.repos.Replacements.CountAbandonedTargetSoleCopies(ctx, target.ID) + if err != nil { + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") + return + } + if sole > 0 { + logger.Warn("abandoned target still holds the only readable copy of some content", + "dataSetID", target.DataSetID.String(), "count", sole) + w.waitForAbandonedTarget(ctx, task, logger, "Waiting for content on the abandoned provider to be copied elsewhere") + return + } + + endEpoch := replacement.AbandonedTerminationEpoch + if endEpoch == nil { + result, terminateErr := w.terminator.TerminateService(ctx, target.DataSetID.SDK()) + if terminateErr != nil { + switch { + case synapse.IsTerminationBlocked(terminateErr): + w.waitForAbandonedTarget(ctx, task, logger, "Waiting for outstanding payment to be settled") + case synapse.IsProviderUnavailable(terminateErr): + w.waitForAbandonedTarget(ctx, task, logger, storagereplacement.WaitReasonProvider.Message()) + default: + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") + } + return + } + if result == nil { + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to record the unused storage service end epoch") + return + } + if err := w.repos.Replacements.RecordAbandonedTerminationEpoch(ctx, repository.RecordTerminationEpochInput{ + ReplacementID: replacement.ID, + TxHash: result.TxHash, + Epoch: result.EndEpoch, + }); err != nil { + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to record the unused storage service end epoch") + return + } + endEpoch = &result.EndEpoch + } + observed, err := w.epochs.CurrentEpoch(ctx) + if err != nil { + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") + return + } + if observed < *endEpoch { + w.waitForAbandonedTarget(ctx, task, logger, storagereplacement.WaitReasonTerminationEpoch.Message()) + return + } + if err := w.repos.Replacements.CompleteAbandonedTargetTermination(ctx, replacement.ID, time.Now()); err != nil { + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to end the unused storage service") + return + } + logger.Info("abandoned replacement target retired", "dataSetID", target.DataSetID.String()) + w.completeTask(ctx, task, logger, "Abandoned provider retired") +} + +// A recoverable blocker keeps the replacement waiting; the operator sees which +// predicate is holding it. +func (w *StorageCleanupWorker) waitForReplacementRetirement( + ctx context.Context, + task *model.Task, + replacementID int64, + gate repository.RetirementGate, + logger *slog.Logger, +) { + reason := retirementWaitReason(gate) + if reason == storagereplacement.WaitReasonCoverage && !gate.SlotOwned { + // The generations are not in the shape retirement assumes. Retrying + // cannot fix that, so it needs an operator. + w.raiseCleanupAttention(ctx, task, replacementID, logger, + fmt.Sprintf("replica slot ownership is inconsistent: %v", gate.Blockers)) + return + } + if err := w.repos.Replacements.MarkWaiting(ctx, replacementID, reason); err != nil { + logger.Warn("failed to record retirement wait reason", "reason", reason, "error", err) + } + logger.Debug("retirement gate is blocked", "blockers", gate.Blockers) + w.waitForReferences(ctx, task, logger, reason.Message()) +} + +// retirementWaitReason names the first blocker so the operator sees the reason +// closest to the source of the delay. +func retirementWaitReason(gate repository.RetirementGate) storagereplacement.WaitReason { + if !gate.SlotOwned { + return storagereplacement.WaitReasonCoverage + } + switch { + case gate.WaitingItems > 0: + return storagereplacement.WaitReasonReadableSource + case gate.CoverageGaps > 0: + return storagereplacement.WaitReasonCoverage + case gate.SourceWrites > 0: + return storagereplacement.WaitReasonSourceWrites + default: + return storagereplacement.WaitReasonTerminationEpoch + } +} + +func (w *StorageCleanupWorker) waitForReplacementEpoch(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger) { + if err := w.repos.Replacements.MarkWaiting(ctx, replacementID, storagereplacement.WaitReasonTerminationEpoch); err != nil { + logger.Warn("failed to record termination epoch wait", "error", err) + } + if err := w.repos.Tasks.WaitRunning(ctx, task, model.TaskWaitReasonExternalConfirmation, + storagereplacement.WaitReasonTerminationEpoch.Message(), storageCleanupConfirmationDelay); err != nil { + logger.Error("failed to wait for termination epoch", "taskID", task.ID, "error", err) + admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() + return + } + admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "success").Inc() +} + +func (w *StorageCleanupWorker) handleTerminationError(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger, err error) { + switch { + case synapse.IsTerminationBlocked(err): + // Settling debt is the operator's decision, never the gateway's. + w.raiseCleanupAttention(ctx, task, replacementID, logger, err.Error()) + case synapse.IsProviderUnavailable(err): + if markErr := w.repos.Replacements.MarkWaiting(ctx, replacementID, storagereplacement.WaitReasonProvider); markErr != nil { + logger.Warn("failed to record provider wait reason", "error", markErr) + } + w.waitForReferences(ctx, task, logger, storagereplacement.WaitReasonProvider.Message()) + default: + w.retryReplacementRetirement(ctx, task, replacementID, logger, "terminate replaced service", err) + } +} + +// CompleteRetirement re-checks everything itself, so a refusal here means the +// world changed between the first gate and the final transaction. +func (w *StorageCleanupWorker) handleRetirementCompletionError(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger, err error) { + if errors.Is(err, storagereplacement.ErrPrematureComplete) { + gate, gateErr := w.repos.Replacements.EvaluateRetirementGate(ctx, replacementID, nil) + if gateErr != nil { + w.retryReplacementRetirement(ctx, task, replacementID, logger, "re-evaluate retirement gate", gateErr) + return + } + w.waitForReplacementRetirement(ctx, task, replacementID, gate, logger) + return + } + w.retryReplacementRetirement(ctx, task, replacementID, logger, "complete replacement retirement", err) +} + +// raiseCleanupAttention stops the task in the same step that records why, so +// automatic retry can never resume suppressed cleanup. +func (w *StorageCleanupWorker) raiseCleanupAttention(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger, message string) { + err := w.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + if err := txRepos.Replacements.MarkCleanupAttention(ctx, replacementID, message); err != nil { + return err + } + return txRepos.Tasks.FailRunning(ctx, task, message) + }) + if err != nil { + logger.Error("failed to raise replacement cleanup attention", "error", err) + admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() + return + } + logger.Warn("replacement retirement needs operator action", "reason", message) + admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() +} + +func (w *StorageCleanupWorker) retryReplacementRetirement( + ctx context.Context, + task *model.Task, + replacementID int64, + logger *slog.Logger, + stage string, + err error, +) { + logger.Error(stage+" failed", "error", err) + status := scheduleTaskRetry(ctx, w.repos, task, "storage_cleanup", logger, err) + // A zero replacement id means the caller owns no replacement state to move, + // which is the case for abandoned target cleanup. + if status == model.TaskStatusExhausted && replacementID > 0 { + // Cleanup that ran out of attempts is an operator decision, not a + // failure to retry, and the record must land even during shutdown. + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), terminalFailureCleanupTimeout) + defer cancel() + if markErr := w.repos.Replacements.MarkCleanupAttention(cleanupCtx, replacementID, + fmt.Sprintf("%s: %v (max retries reached)", stage, err)); markErr != nil { + logger.Error("failed to record replacement cleanup attention", "error", markErr) + } + } + admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() +} + +func (w *StorageCleanupWorker) failReplacementRetirement(ctx context.Context, task *model.Task, logger *slog.Logger, stage string, err error) { + logger.Error(stage+" failed", "error", err) + if failErr := w.repos.Tasks.FailRunning(ctx, task, err.Error()); failErr != nil { + logger.Error("failed to stop replacement retirement task", "error", failErr) + } + admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() +} + +// releaseAbandonedTargetWithoutService retires a local generation that never +// received an on-chain identity. Leaving it unretired would block that provider +// from every later confirmation for the bucket. +func (w *StorageCleanupWorker) releaseAbandonedTargetWithoutService( + ctx context.Context, + replacement *storagereplacement.Replacement, + target *model.StorageDataSet, + lastError string, +) error { + if target != nil { + switch target.Status { + case model.StorageDataSetStatusPending, model.StorageDataSetStatusCreating, model.StorageDataSetStatusFailed: + if err := w.repos.Uploads.MarkDataSetFailed(ctx, target.ID, lastError); err != nil && !errors.Is(err, repository.ErrConflict) { + return err + } + } + } + return w.repos.Replacements.RetireAbandonedTarget(ctx, replacement.ID) +} + +// Observation failures never produced a data set id. After the last retry the +// provider must be released: superseded replacements have no Data Sets retry. +func (w *StorageCleanupWorker) retryAbandonedTargetObservation( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + target *model.StorageDataSet, + logger *slog.Logger, + stage string, + err error, +) { + if synapse.IsProviderUnavailable(err) { + w.waitForReferences(ctx, task, logger, storagereplacement.WaitReasonProvider.Message()) + return + } + logger.Error(stage+" failed", "error", err) + status := scheduleTaskRetry(ctx, w.repos, task, "storage_cleanup", logger, err) + if status == model.TaskStatusExhausted { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), terminalFailureCleanupTimeout) + defer cancel() + if releaseErr := w.releaseAbandonedTargetWithoutService(cleanupCtx, replacement, target, + fmt.Sprintf("%s: %v (max retries reached)", stage, err)); releaseErr != nil { + logger.Error("failed to release abandoned target after observation retries", "error", releaseErr) + } + } + admin.WorkerTasksProcessed.WithLabelValues("storage_cleanup", "failure").Inc() +} + +// resolveAbandonedTargetCreation finishes a data set creation the superseded +// confirmation left in flight. Until it resolves, the generation has neither a +// service to end nor proof that none exists, and the provider stays reserved. +func (w *StorageCleanupWorker) resolveAbandonedTargetCreation( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + target *model.StorageDataSet, + logger *slog.Logger, +) { + if target.CreateTransactionID == nil || target.CreateStatusURL == nil || target.ClientDataSetID == nil { + // The submission left no way to observe its outcome. Treating it as a + // live service would reserve the provider forever, so it is recorded as + // failed and the generation released. + if err := w.releaseAbandonedTargetWithoutService(ctx, replacement, target, + "provider replacement was superseded before the service could be observed"); err != nil { + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to release the unused storage provider") + return + } + w.completeTask(ctx, task, logger, "Abandoned provider released; its service could not be observed") + return + } + bucket, err := w.repos.Buckets.GetByID(ctx, replacement.BucketID) + if err != nil || bucket == nil { + if err == nil { + err = fmt.Errorf("bucket %d: %w", replacement.BucketID, repository.ErrNotFound) + } + w.retryAbandonedTargetObservation(ctx, task, replacement, target, logger, "load bucket for abandoned target", err) + return + } + if w.storage == nil { + w.retryAbandonedTargetObservation(ctx, task, replacement, target, logger, "open abandoned target context", + errors.New("storage client is not configured")) + return + } + storageCtx, err := w.storage.CreateContext(ctx, &storage.CreateContextOptions{ + ProviderID: sdkBigIntPtr(&target.ProviderID), + DataSetMetadata: map[string]string{"bucket": bucket.Name}, + }) + if err != nil || storageCtx == nil { + if err == nil { + err = errors.New("storage context resolver returned no context") + } + w.retryAbandonedTargetObservation(ctx, task, replacement, target, logger, "open abandoned target context", err) + return + } + result, err := storageCtx.WaitForDataSetCreated(ctx, storage.CreateDataSetSubmission{ + TransactionID: *target.CreateTransactionID, + StatusURL: *target.CreateStatusURL, + ClientDataSetID: sdkBigIntPtr(target.ClientDataSetID), + }) + if err != nil { + w.retryAbandonedTargetObservation(ctx, task, replacement, target, logger, "resolve abandoned target creation", err) + return + } + if err := w.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: target.ID, + DataSetID: idtypes.OnChainIDFromSDK(result.DataSetID), + ClientDataSetID: onChainIDPtrFromSDK(result.ClientDataSetID), + }); err != nil { + // The chain service exists. Releasing the local row would orphan it, and + // exhausting the task would leave no resume path, so wait and persist again. + w.waitForAbandonedTarget(ctx, task, logger, "Waiting to record the unused storage service") + return + } + // The service now has an identity, so the next run ends it through the gate. + w.waitForAbandonedTarget(ctx, task, logger, "Ending the unused storage service") +} + +// waitForAbandonedTarget parks leftover cleanup without burning retries or +// moving the superseded replacement. Startup can still see a waiting task, and +// ResumeCoordinator revives one that previously failed or exhausted. +func (w *StorageCleanupWorker) waitForAbandonedTarget(ctx context.Context, task *model.Task, logger *slog.Logger, message string) { + w.waitForReferences(ctx, task, logger, message) +} diff --git a/internal/worker/uploader.go b/internal/worker/uploader.go index 483f994..c122079 100644 --- a/internal/worker/uploader.go +++ b/internal/worker/uploader.go @@ -22,6 +22,7 @@ import ( "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/objectlimits" "github.com/strahe/synaps3/internal/state" + "github.com/strahe/synaps3/internal/storagereplacement" "github.com/strahe/synaps3/internal/synapse" idtypes "github.com/strahe/synaps3/internal/types" "github.com/strahe/synapse-go/pdp" @@ -78,6 +79,8 @@ const ( uploadStagePeerPull = "peer_pull" uploadStagePeerCommit = "peer_commit" uploadStageRepairReplica = "repair_replica" + // Provider replacement advances through the same queue as ordinary uploads. + uploadStageReplaceProvider = storagereplacement.StageMigrate ) // UploaderOption configures uploader behavior. @@ -427,9 +430,15 @@ func (u *Uploader) processTask(ctx context.Context, task *model.Task) { admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() return } - if uploadTaskStage(task) == uploadStageRepairReplica { + switch uploadTaskStage(task) { + case uploadStageRepairReplica: u.processReplicaRepairTask(ctx, task, logger) return + case uploadStageReplaceProvider: + // The coordinator tracks a bucket slot rather than one object version, + // so it is dispatched before any version lookup. + u.processReplacementTask(ctx, task, logger) + return } version, err := u.repos.Objects.GetVersionByID(ctx, task.RefVersionID) @@ -591,6 +600,9 @@ func (u *Uploader) processStagedTask(ctx context.Context, task *model.Task, vers if u.deferToReplicaRepair(ctx, task, bucket.ID, uploadID, copyIndex, logger) { return } + if u.deferToReplacement(ctx, task, bucket.ID, uploadID, copyIndex, logger) { + return + } u.ingressStore(ctx, task, version, bucket, uploadID, copyIndex, logger) case uploadStageIngressCommit: uploadID, copyIndex, err := uploadStageIDs(task, true) @@ -601,6 +613,9 @@ func (u *Uploader) processStagedTask(ctx context.Context, task *model.Task, vers if u.deferToReplicaRepair(ctx, task, bucket.ID, uploadID, copyIndex, logger) { return } + if u.deferToReplacement(ctx, task, bucket.ID, uploadID, copyIndex, logger) { + return + } u.ingressCommit(ctx, task, version, bucket, uploadID, copyIndex, logger) case uploadStagePeerPull: uploadID, copyIndex, err := uploadStageIDs(task, true) @@ -611,6 +626,9 @@ func (u *Uploader) processStagedTask(ctx context.Context, task *model.Task, vers if u.deferToReplicaRepair(ctx, task, bucket.ID, uploadID, copyIndex, logger) { return } + if u.deferToReplacement(ctx, task, bucket.ID, uploadID, copyIndex, logger) { + return + } u.peerPull(ctx, task, version, bucket, uploadID, copyIndex, logger) case uploadStagePeerCommit: uploadID, copyIndex, err := uploadStageIDs(task, true) @@ -621,6 +639,9 @@ func (u *Uploader) processStagedTask(ctx context.Context, task *model.Task, vers if u.deferToReplicaRepair(ctx, task, bucket.ID, uploadID, copyIndex, logger) { return } + if u.deferToReplacement(ctx, task, bucket.ID, uploadID, copyIndex, logger) { + return + } u.peerCommit(ctx, task, version, bucket, uploadID, copyIndex, logger) default: u.handleTaskFailure(ctx, task, logger, "parse upload task payload", fmt.Errorf("unknown upload stage %q", stage)) @@ -644,16 +665,22 @@ func (u *Uploader) deferToReplicaRepair(ctx context.Context, task *model.Task, b if repairTask == nil || repairTask.Status != model.TaskStatusRunning { return false } - copyRow, err := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) - if err != nil || copyRow == nil { - if err == nil { - err = fmt.Errorf("upload copy %d not found", copyIndex) + // A task queued before copy ids were recorded still resolves through its + // replica slot, which now names the current generation. + claimedCopyID := uploadStageCopyID(task) + if claimedCopyID == 0 { + copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) + if err != nil || copyRow == nil { + if err == nil { + err = fmt.Errorf("upload copy %d not found", copyIndex) + } + u.handleTaskFailure(ctx, task, logger, "load upload copy for replica repair coordination", err) + return true } - u.handleTaskFailure(ctx, task, logger, "load upload copy for replica repair coordination", err) - return true + claimedCopyID = copyRow.ID } repairCopyID, err := payloadInt64(repairTask.Payload, replicaRepairCopyIDKey) - if err == nil && repairCopyID != copyRow.ID { + if err == nil && repairCopyID != claimedCopyID { return false } if !taskClaimPrecedes(repairTask, task) { @@ -711,6 +738,11 @@ func (u *Uploader) prepareStagedUpload(ctx context.Context, task *model.Task, ve u.handleTaskFailure(ctx, task, logger, "ensure provider bindings", planErr) return } + plan, err = u.preserveInFlightUploadBindings(ctx, upload.ID, plan) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "preserve in-flight provider bindings", err) + return + } copyInputs, err := u.uploadCopyInputs(ctx, upload.ID, plan.bindings) if err != nil { u.handleTaskFailure(ctx, task, logger, "plan upload copy rows", err) @@ -746,7 +778,7 @@ func (u *Uploader) prepareStagedUpload(ctx context.Context, task *model.Task, ve u.waitForStorageDependency(ctx, task, logger, "Waiting for an assigned storage provider to recover") return } - if err := u.enqueueUploadStage(ctx, task, uploadStageEnsureDataSet, upload.ID, ingress.CopyIndex, model.StorageCopyTransferMethodIngress); err != nil { + if err := u.enqueueUploadStageForCopy(ctx, task, uploadStageEnsureDataSet, upload.ID, ingress.CopyIndex, model.StorageCopyTransferMethodIngress, ingress.ID); err != nil { u.handleTaskFailure(ctx, task, logger, "enqueue ingress dataset task", err) return } @@ -759,9 +791,15 @@ func (u *Uploader) uploadCopyInputs(ctx context.Context, uploadID int64, binding return nil, err } existingByIndex := make(map[int]model.StorageUploadCopy, len(existingCopies)) + // A slot can hold a copy per generation during a replacement, so the copy + // that matches a binding is found by data set rather than by slot. + existingByDataSet := make(map[int64]model.StorageUploadCopy, len(existingCopies)) ingressCopyIndex := -1 for _, copyRow := range existingCopies { existingByIndex[copyRow.CopyIndex] = copyRow + if copyRow.StorageDataSetID != nil { + existingByDataSet[*copyRow.StorageDataSetID] = copyRow + } if copyRow.TransferMethod == model.StorageCopyTransferMethodIngress { ingressCopyIndex = copyRow.CopyIndex } @@ -784,12 +822,16 @@ func (u *Uploader) uploadCopyInputs(ctx context.Context, uploadID int64, binding } inputs := make([]repository.UploadCopyBindingInput, 0, len(bindings)) for _, binding := range bindings { - if _, exists := existingByIndex[binding.CopyIndex]; !exists && !uploadTracksDataSetBinding(uploadID, &binding) { + existingCopy, hasCopy := existingByDataSet[binding.ID] + if !hasCopy { + existingCopy, hasCopy = existingByIndex[binding.CopyIndex] + } + if !hasCopy && !uploadTracksDataSetBinding(uploadID, &binding) { continue } transferMethod := model.StorageCopyTransferMethodPeerPull - if existing, ok := existingByIndex[binding.CopyIndex]; ok { - transferMethod = existing.TransferMethod + if hasCopy { + transferMethod = existingCopy.TransferMethod } else if binding.CopyIndex == ingressCopyIndex { transferMethod = model.StorageCopyTransferMethodIngress } @@ -855,6 +897,11 @@ func (u *Uploader) prepareReadableUploadRepair(ctx context.Context, task *model. u.handleTaskFailure(ctx, task, logger, "ensure repair provider bindings", planErr) return } + plan, err = u.preserveInFlightUploadBindings(ctx, uploadID, plan) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "preserve in-flight repair bindings", err) + return + } inputs, err := u.uploadCopyInputs(ctx, uploadID, plan.bindings) if err != nil { u.handleTaskFailure(ctx, task, logger, "plan repair upload copy rows", err) @@ -906,7 +953,7 @@ func (u *Uploader) prepareReadableUploadRepair(ctx context.Context, task *model. binding := plan.byCopyIndex[copyRow.CopyIndex] switch { case fundedBindings[copyRow.CopyIndex] != nil && uploadCanUseDataSetBinding(uploadID, binding): - if err := u.enqueueUploadStage(ctx, task, uploadStageEnsureDataSet, uploadID, copyRow.CopyIndex, model.StorageCopyTransferMethodPeerPull); err != nil { + if err := u.enqueueUploadStageForCopy(ctx, task, uploadStageEnsureDataSet, uploadID, copyRow.CopyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { u.handleTaskFailure(ctx, task, logger, "enqueue existing repair copy", err) return } @@ -957,6 +1004,79 @@ func newBucketBindingPlan(bindings []model.StorageDataSet, uploadID int64) bucke return plan } +// preserveInFlightUploadBindings keeps unfinished copy rows on the generation +// they were created for. A provider replacement may have moved the slot to a +// new current generation while funding or task enqueueing was waiting. +func (u *Uploader) preserveInFlightUploadBindings( + ctx context.Context, + uploadID int64, + plan bucketBindingPlan, +) (bucketBindingPlan, error) { + copies, err := u.repos.Uploads.ListCopies(ctx, uploadID) + if err != nil { + return bucketBindingPlan{}, err + } + chosen := make(map[int]model.StorageUploadCopy) + for i := range copies { + copyRow := copies[i] + if copyRow.StorageDataSetID == nil || copyRow.Status == model.StorageUploadCopyStatusFailed || copyCommitted(©Row) { + continue + } + previous, ok := chosen[copyRow.CopyIndex] + if !ok || (copyRow.TransferMethod == model.StorageCopyTransferMethodIngress && previous.TransferMethod != model.StorageCopyTransferMethodIngress) || + (copyRow.TransferMethod == previous.TransferMethod && copyRow.ID < previous.ID) { + chosen[copyRow.CopyIndex] = copyRow + } + } + if len(chosen) == 0 { + return plan, nil + } + authoritative := make(map[int]model.StorageDataSet, len(chosen)) + for copyIndex, copyRow := range chosen { + binding, err := u.repos.Uploads.GetDataSetBindingByID(ctx, *copyRow.StorageDataSetID) + if err != nil { + return bucketBindingPlan{}, err + } + if binding == nil || binding.CopyIndex != copyIndex || + (binding.Status != model.StorageDataSetStatusReady && binding.Status != model.StorageDataSetStatusDraining) { + continue + } + authoritative[copyIndex] = *binding + } + if len(authoritative) == 0 { + return plan, nil + } + bindings := make([]model.StorageDataSet, 0, len(plan.bindings)+len(authoritative)) + seen := make(map[int]struct{}, len(plan.bindings)+len(authoritative)) + for i := range plan.bindings { + binding := plan.bindings[i] + if inFlight, ok := authoritative[binding.CopyIndex]; ok { + binding = inFlight + } + if _, ok := seen[binding.CopyIndex]; ok { + continue + } + seen[binding.CopyIndex] = struct{}{} + bindings = append(bindings, binding) + } + for copyIndex, binding := range authoritative { + if _, ok := seen[copyIndex]; ok { + continue + } + bindings = append(bindings, binding) + } + out := newBucketBindingPlan(bindings, uploadID) + out.complete = plan.complete + out.deferredContext = plan.deferredContext + for i := range out.bindings { + binding := &out.bindings[i] + if binding.Status == model.StorageDataSetStatusDraining { + out.writable = append(out.writable, *binding) + } + } + return out, nil +} + func (u *Uploader) ensureWritableIngressCopy(ctx context.Context, uploadID int64, fundedBindings map[int]*model.StorageDataSet) (*model.StorageUploadCopy, error) { copies, err := u.repos.Uploads.ListCopies(ctx, uploadID) if err != nil { @@ -1001,8 +1121,14 @@ func (u *Uploader) ensureBucketProviderBindings(ctx context.Context, bucket *mod inFlight := 0 for i := range bindings { binding := bindings[i] - existing[binding.CopyIndex] = binding + // Automatic selection excludes every provider this bucket has ever used, + // including retired generations, so a replacement never lands back on a + // provider the operator already moved away from. excluded = append(excluded, binding.ProviderID.SDK()) + if !binding.IsCurrent { + continue + } + existing[binding.CopyIndex] = binding if dataSetBindingCanOccupyUploadSlot(uploadID, &binding) { if len(selected) < targetCopies { selected = append(selected, binding) @@ -1107,7 +1233,7 @@ func (u *Uploader) ensureUploadFundingReady( storageCtx synapse.UploadContext err error ) - if binding.Status == model.StorageDataSetStatusReady && binding.DataSetID != nil && !binding.DataSetID.IsZero() { + if binding.DataSetID != nil && !binding.DataSetID.IsZero() { storageCtx, err = u.contextForReadyBinding(ctx, binding, bucket.Name) } else { storageCtx, err = u.contextForBindingProvider(ctx, binding, bucket.Name) @@ -1217,7 +1343,7 @@ func uploadFundingWaitMessage(costs *storage.MultiContextCosts) string { } func (u *Uploader) ensureUploadDataSet(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - copyRow, err := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) + copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) if err != nil || copyRow == nil { if err == nil { err = fmt.Errorf("upload copy %d not found", copyIndex) @@ -1343,7 +1469,7 @@ func (u *Uploader) ensureUploadDataSet(ctx context.Context, task *model.Task, ve if copyRow.TransferMethod == model.StorageCopyTransferMethodIngress { nextStage = uploadStageIngressStore } - if err := u.enqueueUploadStage(ctx, task, nextStage, uploadID, copyIndex, copyRow.TransferMethod); err != nil { + if err := u.enqueueUploadStageForCopy(ctx, task, nextStage, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { u.handleTaskFailure(ctx, task, logger, "enqueue next upload stage", err) return } @@ -1351,7 +1477,7 @@ func (u *Uploader) ensureUploadDataSet(ctx context.Context, task *model.Task, ve } func (u *Uploader) ingressStore(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - binding, storageCtx, err := u.readyContextForCopy(ctx, bucket, copyIndex) + binding, storageCtx, err := u.readyContextForCopy(ctx, task, bucket, uploadID, copyIndex) if err != nil { u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "ingress context", err) return @@ -1360,7 +1486,7 @@ func (u *Uploader) ingressStore(ctx context.Context, task *model.Task, version * u.handleTaskFailure(ctx, task, logger, "ingress context", errors.New("ingress dataset binding not found")) return } - copyRow, err := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) + copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) if err != nil { u.handleTaskFailure(ctx, task, logger, "load ingress copy", err) return @@ -1372,7 +1498,7 @@ func (u *Uploader) ingressStore(ctx context.Context, task *model.Task, version * return } } - if err := u.enqueueUploadStage(ctx, task, uploadStageIngressCommit, uploadID, copyIndex, model.StorageCopyTransferMethodIngress); err != nil { + if err := u.enqueueUploadStageForCopy(ctx, task, uploadStageIngressCommit, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { u.handleTaskFailure(ctx, task, logger, "enqueue ingress commit", err) return } @@ -1403,10 +1529,11 @@ func (u *Uploader) ingressStore(ctx context.Context, task *model.Task, version * progress.Flush(version.Size, true) pieceCID := result.PieceCID.String() if err := u.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: uploadID, - CopyIndex: copyIndex, - PieceCID: pieceCID, - RetrievalURL: storageCtx.PieceURL(result.PieceCID), + StorageUploadCopyID: uploadStageCopyID(task), + UploadID: uploadID, + CopyIndex: copyIndex, + PieceCID: pieceCID, + RetrievalURL: storageCtx.PieceURL(result.PieceCID), }); err != nil { u.handleTaskFailure(ctx, task, logger, "mark ingress piece ready", err) return @@ -1417,7 +1544,7 @@ func (u *Uploader) ingressStore(ctx context.Context, task *model.Task, version * return } } - if err := u.enqueueUploadStage(ctx, task, uploadStageIngressCommit, uploadID, copyIndex, model.StorageCopyTransferMethodIngress); err != nil { + if err := u.enqueueUploadStageForCopy(ctx, task, uploadStageIngressCommit, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { u.handleTaskFailure(ctx, task, logger, "enqueue ingress commit", err) return } @@ -1425,7 +1552,7 @@ func (u *Uploader) ingressStore(ctx context.Context, task *model.Task, version * } func (u *Uploader) ingressCommit(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - copyRow, err := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) + copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) if err != nil { u.handleTaskFailure(ctx, task, logger, "load ingress copy", err) return @@ -1434,7 +1561,7 @@ func (u *Uploader) ingressCommit(ctx context.Context, task *model.Task, version u.finishCommittedIngress(ctx, task, version, bucket, uploadID, copyRow, logger) return } - binding, storageCtx, err := u.readyContextForCopy(ctx, bucket, copyIndex) + binding, storageCtx, err := u.readyContextForCopy(ctx, task, bucket, uploadID, copyIndex) if err != nil { u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "ingress commit context", err) return @@ -1460,7 +1587,7 @@ func (u *Uploader) ingressCommit(ctx context.Context, task *model.Task, version return } if errors.Is(err, errCommitRejected) { - if resetErr := u.resetRejectedSubmittedCommit(ctx, uploadID, copyIndex, *copyRow.CommitTransactionID, err); resetErr != nil { + if resetErr := u.resetRejectedSubmittedCommit(ctx, copyRow.ID, uploadID, copyIndex, *copyRow.CommitTransactionID, err); resetErr != nil { u.handleTaskFailure(ctx, task, logger, "reset rejected ingress commit", resetErr) return } @@ -1475,6 +1602,7 @@ func (u *Uploader) ingressCommit(ctx context.Context, task *model.Task, version pieceID = onChainIDPtrFromSDK(result.PieceIDs[0]) } if err := u.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + StorageUploadCopyID: uploadStageCopyID(task), UploadID: uploadID, CopyIndex: copyIndex, PieceCID: *upload.PieceCID, @@ -1489,7 +1617,7 @@ func (u *Uploader) ingressCommit(ctx context.Context, task *model.Task, version u.finishReadable(ctx, task, version, uploadID, logger) return } - extraData, extraHex, err := u.extraDataForCopy(ctx, storageCtx, uploadID, copyIndex, pieces) + extraData, extraHex, err := u.extraDataForCopy(ctx, storageCtx, copyRow, pieces) if err != nil { u.handleIngressDataSetFailure(ctx, task, version, uploadID, copyIndex, binding.ID, logger, "ingress presign", err) return @@ -1502,6 +1630,7 @@ func (u *Uploader) ingressCommit(ctx context.Context, task *model.Task, version OnSubmitted: func(txHash string) { submittedTx = txHash submitErr = u.repos.Uploads.MarkUploadCopyCommitting(ctx, repository.MarkUploadCopyCommittingInput{ + StorageUploadCopyID: uploadStageCopyID(task), UploadID: uploadID, CopyIndex: copyIndex, CommitExtraDataHex: extraHex, @@ -1526,6 +1655,7 @@ func (u *Uploader) ingressCommit(ctx context.Context, task *model.Task, version pieceID = onChainIDPtrFromSDK(result.PieceIDs[0]) } if err := u.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + StorageUploadCopyID: uploadStageCopyID(task), UploadID: uploadID, CopyIndex: copyIndex, PieceCID: *upload.PieceCID, @@ -1702,18 +1832,28 @@ func (u *Uploader) scheduleRemainingPeerCopies( if upload == nil { return nil, false, fmt.Errorf("storage upload %d not found", uploadID) } - needsPreparation := len(copies) < boundedTargetCopies(upload.RequestedCopies) + currentSlots, err := u.repos.Uploads.CountCurrentGenerationCopySlots(ctx, uploadID) + if err != nil { + return nil, false, err + } + needsPreparation := currentSlots < boundedTargetCopies(upload.RequestedCopies) for _, copyRow := range copies { if copyRow.TransferMethod != model.StorageCopyTransferMethodPeerPull || copyCommitted(©Row) || copyRow.Status == model.StorageUploadCopyStatusFailed { continue } - binding, err := u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucketID, copyRow.CopyIndex) + var binding *model.StorageDataSet + if copyRow.StorageDataSetID != nil { + binding, err = u.repos.Uploads.GetDataSetBindingByID(ctx, *copyRow.StorageDataSetID) + } else { + binding, err = u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucketID, copyRow.CopyIndex) + } if err != nil { return nil, false, err } switch { - case binding != nil && binding.Status == model.StorageDataSetStatusReady: - if err := u.enqueueUploadStageForVersion(ctx, ref, maxRetries, uploadStageEnsureDataSet, uploadID, copyRow.CopyIndex, copyRow.TransferMethod); err != nil { + case binding != nil && (binding.Status == model.StorageDataSetStatusReady || + (copyRow.StorageDataSetID != nil && binding.Status == model.StorageDataSetStatusDraining)): + if err := u.enqueueUploadStageForVersionCopy(ctx, ref, maxRetries, uploadStageEnsureDataSet, uploadID, copyRow.CopyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { return nil, false, err } case binding != nil && binding.Status == model.StorageDataSetStatusUnavailable: @@ -1742,12 +1882,12 @@ func (u *Uploader) repairReadableBinding(ctx context.Context, task *model.Task, } func (u *Uploader) peerPull(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - binding, storageCtx, err := u.readyContextForCopy(ctx, bucket, copyIndex) + binding, storageCtx, err := u.readyContextForCopy(ctx, task, bucket, uploadID, copyIndex) if err != nil { u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "peer pull context", err) return } - copyRow, err := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) + copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) if err != nil { u.handleTaskFailure(ctx, task, logger, "load peer copy", err) return @@ -1757,7 +1897,7 @@ func (u *Uploader) peerPull(ctx context.Context, task *model.Task, version *mode return } if copyHasPiece(copyRow) { - if err := u.enqueueUploadStage(ctx, task, uploadStagePeerCommit, uploadID, copyIndex, model.StorageCopyTransferMethodPeerPull); err != nil { + if err := u.enqueueUploadStageForCopy(ctx, task, uploadStagePeerCommit, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { u.handleTaskFailure(ctx, task, logger, "enqueue peer commit", err) return } @@ -1784,7 +1924,7 @@ func (u *Uploader) peerPull(ctx context.Context, task *model.Task, version *mode } pieceCIDString = sourceCopy.PieceCID pieces := []storage.PieceInput{{PieceCID: pieceCID}} - extraData, encodedExtra, err := u.extraDataForCopy(ctx, storageCtx, uploadID, copyIndex, pieces) + extraData, encodedExtra, err := u.extraDataForCopy(ctx, storageCtx, copyRow, pieces) if err != nil { u.handlePeerDataSetFailure(ctx, task, bucket, uploadID, copyIndex, binding.ID, logger, "peer presign", err) return @@ -1831,30 +1971,32 @@ func (u *Uploader) peerPull(ctx context.Context, task *model.Task, version *mode } pieceCID = result.PieceCID pieceCIDString = pieceCID.String() - _, extraHex, err = u.extraDataForCopy(ctx, storageCtx, uploadID, copyIndex, []storage.PieceInput{{PieceCID: pieceCID}}) + _, extraHex, err = u.extraDataForCopy(ctx, storageCtx, copyRow, []storage.PieceInput{{PieceCID: pieceCID}}) if err != nil { u.handlePeerDataSetFailure(ctx, task, bucket, uploadID, copyIndex, binding.ID, logger, "peer presign", err) return } } if err := u.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ - UploadID: uploadID, - CopyIndex: copyIndex, - PieceCID: pieceCIDString, - RetrievalURL: storageCtx.PieceURL(pieceCID), + StorageUploadCopyID: uploadStageCopyID(task), + UploadID: uploadID, + CopyIndex: copyIndex, + PieceCID: pieceCIDString, + RetrievalURL: storageCtx.PieceURL(pieceCID), }); err != nil { u.handleTaskFailure(ctx, task, logger, "mark peer piece ready", err) return } if err := u.repos.Uploads.MarkUploadCopyCommitting(ctx, repository.MarkUploadCopyCommittingInput{ - UploadID: uploadID, - CopyIndex: copyIndex, - CommitExtraDataHex: extraHex, + StorageUploadCopyID: uploadStageCopyID(task), + UploadID: uploadID, + CopyIndex: copyIndex, + CommitExtraDataHex: extraHex, }); err != nil { u.handleTaskFailure(ctx, task, logger, "save peer extra data", err) return } - if err := u.enqueueUploadStage(ctx, task, uploadStagePeerCommit, uploadID, copyIndex, model.StorageCopyTransferMethodPeerPull); err != nil { + if err := u.enqueueUploadStageForCopy(ctx, task, uploadStagePeerCommit, uploadID, copyIndex, copyRow.TransferMethod, copyRow.ID); err != nil { u.handleTaskFailure(ctx, task, logger, "enqueue peer commit", err) return } @@ -1862,12 +2004,12 @@ func (u *Uploader) peerPull(ctx context.Context, task *model.Task, version *mode } func (u *Uploader) peerCommit(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, logger *slog.Logger) { - binding, storageCtx, err := u.readyContextForCopy(ctx, bucket, copyIndex) + binding, storageCtx, err := u.readyContextForCopy(ctx, task, bucket, uploadID, copyIndex) if err != nil { u.markDataSetStageFailed(ctx, task, version, bucket, uploadID, copyIndex, binding, logger, "peer commit context", err) return } - copyRow, err := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) + copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) if err != nil { u.handleTaskFailure(ctx, task, logger, "load peer copy", err) return @@ -1911,7 +2053,7 @@ func (u *Uploader) peerCommit(ctx context.Context, task *model.Task, version *mo return } if errors.Is(err, errCommitRejected) { - if resetErr := u.resetRejectedSubmittedCommit(ctx, uploadID, copyIndex, *copyRow.CommitTransactionID, err); resetErr != nil { + if resetErr := u.resetRejectedSubmittedCommit(ctx, copyRow.ID, uploadID, copyIndex, *copyRow.CommitTransactionID, err); resetErr != nil { u.handleTaskFailure(ctx, task, logger, "reset rejected peer commit", resetErr) return } @@ -1930,6 +2072,7 @@ func (u *Uploader) peerCommit(ctx context.Context, task *model.Task, version *mo pieceID = onChainIDPtrFromSDK(result.PieceIDs[0]) } if err := u.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + StorageUploadCopyID: uploadStageCopyID(task), UploadID: uploadID, CopyIndex: copyIndex, PieceCID: pieceCIDString, @@ -1944,7 +2087,7 @@ func (u *Uploader) peerCommit(ctx context.Context, task *model.Task, version *mo u.finishPeerCopy(ctx, task, version, uploadID, logger) return } - extraData, extraHex, err := u.extraDataForCopy(ctx, storageCtx, uploadID, copyIndex, pieces) + extraData, extraHex, err := u.extraDataForCopy(ctx, storageCtx, copyRow, pieces) if err != nil { u.handlePeerDataSetFailure(ctx, task, bucket, uploadID, copyIndex, binding.ID, logger, "peer presign", err) return @@ -1957,6 +2100,7 @@ func (u *Uploader) peerCommit(ctx context.Context, task *model.Task, version *mo OnSubmitted: func(txHash string) { submittedTx = txHash submitErr = u.repos.Uploads.MarkUploadCopyCommitting(ctx, repository.MarkUploadCopyCommittingInput{ + StorageUploadCopyID: uploadStageCopyID(task), UploadID: uploadID, CopyIndex: copyIndex, CommitExtraDataHex: extraHex, @@ -1971,7 +2115,7 @@ func (u *Uploader) peerCommit(ctx context.Context, task *model.Task, version *mo return } if errors.Is(err, errCommitRejected) { - if resetErr := u.resetRejectedSubmittedCommit(ctx, uploadID, copyIndex, submittedTx, err); resetErr != nil { + if resetErr := u.resetRejectedSubmittedCommit(ctx, copyRow.ID, uploadID, copyIndex, submittedTx, err); resetErr != nil { u.handleTaskFailure(ctx, task, logger, "reset rejected peer commit", resetErr) return } @@ -1993,6 +2137,7 @@ func (u *Uploader) peerCommit(ctx context.Context, task *model.Task, version *mo pieceID = onChainIDPtrFromSDK(result.PieceIDs[0]) } if err := u.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + StorageUploadCopyID: uploadStageCopyID(task), UploadID: uploadID, CopyIndex: copyIndex, PieceCID: pieceCIDString, @@ -2030,32 +2175,73 @@ func (u *Uploader) finishPeerCopy(ctx context.Context, task *model.Task, version completeWorkerTask(ctx, u.repos, task, "uploader", logger) } -func (u *Uploader) enqueueUploadStage(ctx context.Context, parent *model.Task, stage string, uploadID int64, copyIndex int, transferMethod model.StorageCopyTransferMethod) error { +func (u *Uploader) enqueueUploadStageForCopy( + ctx context.Context, + parent *model.Task, + stage string, + uploadID int64, + copyIndex int, + transferMethod model.StorageCopyTransferMethod, + copyID int64, +) error { + if parent == nil || copyID <= 0 { + return fmt.Errorf("enqueueing concrete upload copy: %w", repository.ErrInvalidInput) + } ref := repository.ObjectVersionRef{ObjectID: parent.RefID, VersionID: parent.RefVersionID} - return u.enqueueUploadStageForVersion(ctx, ref, parent.MaxRetries, stage, uploadID, copyIndex, transferMethod) + return u.enqueueUploadStageForVersionCopy(ctx, ref, parent.MaxRetries, stage, uploadID, copyIndex, transferMethod, copyID) } -func (u *Uploader) enqueueUploadStageForVersion(ctx context.Context, ref repository.ObjectVersionRef, maxRetries int, stage string, uploadID int64, copyIndex int, transferMethod model.StorageCopyTransferMethod) error { - return enqueueUploadStageForVersion(ctx, u.repos, ref, maxRetries, stage, uploadID, copyIndex, transferMethod) -} - -func enqueueUploadStageForVersion(ctx context.Context, repos *repository.Repositories, ref repository.ObjectVersionRef, maxRetries int, stage string, uploadID int64, copyIndex int, transferMethod model.StorageCopyTransferMethod) error { - task := newUploadStageTask(ref, maxRetries, stage, uploadID, copyIndex, transferMethod) - if err := repos.Tasks.Create(ctx, task); err != nil && !errors.Is(err, repository.ErrAlreadyExists) { +func (u *Uploader) enqueueUploadStageForVersionCopy( + ctx context.Context, + ref repository.ObjectVersionRef, + maxRetries int, + stage string, + uploadID int64, + copyIndex int, + transferMethod model.StorageCopyTransferMethod, + copyID int64, +) error { + if copyID <= 0 { + return fmt.Errorf("enqueueing concrete upload copy: %w", repository.ErrInvalidInput) + } + task := newUploadStageTask(ref, maxRetries, stage, uploadID, copyIndex, transferMethod, copyID) + if err := u.repos.Tasks.Create(ctx, task); err != nil && !errors.Is(err, repository.ErrAlreadyExists) { return err } return nil } -func newUploadStageTask(ref repository.ObjectVersionRef, maxRetries int, stage string, uploadID int64, copyIndex int, transferMethod model.StorageCopyTransferMethod) *model.Task { +// Recording the concrete copy lets the replacement safety gate tell a write +// bound for the retiring generation from one bound for its replacement. A task +// queued before this existed simply carries no copy id. +func stageTaskCopyID(ctx context.Context, repos *repository.Repositories, uploadID int64, copyIndex int, transferMethod model.StorageCopyTransferMethod) (int64, error) { + if transferMethod == "" { + return 0, nil + } + copyRow, err := repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) + if err != nil { + return 0, err + } + if copyRow == nil { + return 0, nil + } + return copyRow.ID, nil +} + +func newUploadStageTask(ref repository.ObjectVersionRef, maxRetries int, stage string, uploadID int64, copyIndex int, transferMethod model.StorageCopyTransferMethod, copyID int64) *model.Task { payload := map[string]interface{}{ "upload_id": uploadID, } + // The key stays keyed by slot so tasks queued before copy ids existed keep + // their identity. key := fmt.Sprintf("upload:%s:%s:%d", ref.VersionID, stage, uploadID) if transferMethod != "" { payload["copy_index"] = copyIndex payload["transfer_method"] = string(transferMethod) key = fmt.Sprintf("%s:%d", key, copyIndex) + if copyID > 0 { + payload[replicaRepairCopyIDKey] = copyID + } } return &model.Task{ Type: model.TaskTypeUpload, @@ -2072,7 +2258,11 @@ func newUploadStageTask(ref repository.ObjectVersionRef, maxRetries int, stage s } func ensureIngressHandoffTask(ctx context.Context, repos *repository.Repositories, ref repository.ObjectVersionRef, maxRetries int, uploadID int64, copyIndex int) error { - task := newUploadStageTask(ref, maxRetries, uploadStageEnsureDataSet, uploadID, copyIndex, model.StorageCopyTransferMethodIngress) + copyID, err := stageTaskCopyID(ctx, repos, uploadID, copyIndex, model.StorageCopyTransferMethodIngress) + if err != nil { + return err + } + task := newUploadStageTask(ref, maxRetries, uploadStageEnsureDataSet, uploadID, copyIndex, model.StorageCopyTransferMethodIngress, copyID) created, err := repos.Tasks.EnsureRecurring(ctx, task) if err != nil || created { return err @@ -2189,6 +2379,59 @@ func uploadStageIDs(task *model.Task, needsCopyIndex bool) (int64, int, error) { return uploadID, copyIndex, nil } +// taskUploadCopy loads the copy a staged task is bound to. When the task records +// a concrete copy it wins over the replica slot, so reads and writes stay on the +// same generation after a replacement activates. +func (u *Uploader) taskUploadCopy(ctx context.Context, task *model.Task, uploadID int64, copyIndex int) (*model.StorageUploadCopy, error) { + if copyID := uploadStageCopyID(task); copyID > 0 { + copyRow, err := u.repos.Uploads.GetUploadCopyByID(ctx, copyID) + if err != nil { + return nil, err + } + if copyRow == nil { + return nil, fmt.Errorf("storage upload copy %d: %w", copyID, repository.ErrNotFound) + } + transferMethod, _ := task.Payload["transfer_method"].(string) + if copyRow.UploadID != uploadID || copyRow.CopyIndex != copyIndex || + (transferMethod != "" && string(copyRow.TransferMethod) != transferMethod) { + return nil, fmt.Errorf("storage upload copy %d does not match task payload: %w", copyID, repository.ErrConflict) + } + return copyRow, nil + } + return u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) +} + +// taskCopyDataSet resolves the data set a task's copy is bound to. Tasks queued +// before copy ids existed, and copies not yet bound to a data set, fall back to +// the slot's current generation, which is where they belong. +func (u *Uploader) taskCopyDataSet(ctx context.Context, task *model.Task, bucketID, uploadID int64, copyIndex int) (*model.StorageDataSet, error) { + copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) + if err != nil { + return nil, err + } + if copyRow != nil && copyRow.StorageDataSetID != nil && *copyRow.StorageDataSetID > 0 { + return u.repos.Uploads.GetDataSetBindingByID(ctx, *copyRow.StorageDataSetID) + } + return u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucketID, copyIndex) +} + +// uploadStageCopyID reports the concrete copy a staged task targets. Tasks +// queued before copy ids were recorded return zero and stay resolvable through +// their replica slot. +func uploadStageCopyID(task *model.Task) int64 { + if task == nil || task.Payload == nil { + return 0 + } + if _, ok := task.Payload[replicaRepairCopyIDKey]; !ok { + return 0 + } + copyID, err := payloadInt64(task.Payload, replicaRepairCopyIDKey) + if err != nil || copyID < 0 { + return 0 + } + return copyID +} + func payloadInt64(payload map[string]interface{}, key string) (int64, error) { raw, ok := payload[key] if !ok { @@ -2211,7 +2454,7 @@ func payloadInt64(payload map[string]interface{}, key string) (int64, error) { } func (u *Uploader) markDataSetStageFailed(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, copyIndex int, binding *model.StorageDataSet, logger *slog.Logger, stage string, err error) { - copyRow, copyErr := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) + copyRow, copyErr := u.taskUploadCopy(ctx, task, uploadID, copyIndex) if copyErr != nil { u.handleTaskFailure(ctx, task, logger, "load failed upload copy", copyErr) return @@ -2284,7 +2527,12 @@ func (u *Uploader) markPeerFailed(ctx context.Context, task *model.Task, uploadI if status == model.TaskStatusExhausted { cleanupCtx, cancel := context.WithTimeout(context.Background(), terminalFailureCleanupTimeout) defer cancel() - if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, uploadID, copyIndex, fmt.Sprintf("%s: %v", stage, err)); markErr != nil { + if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, repository.MarkUploadCopyFailedInput{ + StorageUploadCopyID: uploadStageCopyID(task), + UploadID: uploadID, + CopyIndex: copyIndex, + LastError: fmt.Sprintf("%s: %v", stage, err), + }); markErr != nil { logger.Warn("failed to mark peer upload copy failed", "uploadID", uploadID, "copyIndex", copyIndex, "error", markErr) } discarded := false @@ -2315,7 +2563,7 @@ func (u *Uploader) handlePeerDataSetFailure(ctx context.Context, task *model.Tas } } if dataSetFailureEnded(err, binding) || dataSetFailureUnavailable(err, binding) { - copyRow, loadErr := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) + copyRow, loadErr := u.taskUploadCopy(ctx, task, uploadID, copyIndex) if loadErr != nil || copyRow == nil { if loadErr == nil { loadErr = errors.New("peer upload copy not found") @@ -2445,7 +2693,12 @@ func (u *Uploader) handleDataSetCreationFailure( } cleanupCtx, cancel := context.WithTimeout(context.Background(), terminalFailureCleanupTimeout) defer cancel() - if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, uploadID, copyRow.CopyIndex, fmt.Sprintf("%s: %v", stage, err)); markErr != nil { + if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, repository.MarkUploadCopyFailedInput{ + StorageUploadCopyID: copyRow.ID, + UploadID: uploadID, + CopyIndex: copyRow.CopyIndex, + LastError: fmt.Sprintf("%s: %v", stage, err), + }); markErr != nil { logger.Warn("failed to mark rejected data set copy failed", "uploadID", uploadID, "copyIndex", copyRow.CopyIndex, "error", markErr) } discarded, discardErr := u.repos.Uploads.DiscardFailedDataSetCandidate(cleanupCtx, uploadID, copyRow.CopyIndex, binding.ID) @@ -2472,7 +2725,7 @@ func (u *Uploader) handleDataSetCreationFailure( if reassignErr != nil { logger.Warn("failed to reassign ingress after data set rejection", "uploadID", uploadID, "copyIndex", copyRow.CopyIndex, "error", reassignErr) } else if reassigned != nil { - if enqueueErr := u.enqueueUploadStage(cleanupCtx, task, uploadStageEnsureDataSet, uploadID, reassigned.CopyIndex, model.StorageCopyTransferMethodIngress); enqueueErr != nil { + if enqueueErr := u.enqueueUploadStageForCopy(cleanupCtx, task, uploadStageEnsureDataSet, uploadID, reassigned.CopyIndex, reassigned.TransferMethod, reassigned.ID); enqueueErr != nil { logger.Warn("failed to enqueue ingress after data set rejection", "uploadID", uploadID, "copyIndex", reassigned.CopyIndex, "error", enqueueErr) } } @@ -2590,15 +2843,24 @@ func (u *Uploader) markDataSetStatus(ctx context.Context, binding *model.Storage return latest, nil } -func (u *Uploader) readyContextForCopy(ctx context.Context, bucket *model.Bucket, copyIndex int) (*model.StorageDataSet, synapse.UploadContext, error) { - binding, err := u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, copyIndex) +// readyContextForCopy opens the provider context for the data set this task's +// copy is actually bound to, not for whichever generation currently owns the +// slot. A provider replacement can take the slot while an upload is mid-flight; +// resolving by slot would then store the piece on the new provider while the +// copy row being updated still belongs to the old one. +func (u *Uploader) readyContextForCopy(ctx context.Context, task *model.Task, bucket *model.Bucket, uploadID int64, copyIndex int) (*model.StorageDataSet, synapse.UploadContext, error) { + binding, err := u.taskCopyDataSet(ctx, task, bucket.ID, uploadID, copyIndex) if err != nil { return nil, nil, err } if binding == nil { return nil, nil, nil } - if binding.Status != model.StorageDataSetStatusReady || binding.DataSetID == nil || binding.DataSetID.IsZero() { + concrete := uploadStageCopyID(task) > 0 + usableStatus := binding.Status == model.StorageDataSetStatusReady || + concrete && binding.Status == model.StorageDataSetStatusDraining + if !usableStatus || + binding.DataSetID == nil || binding.DataSetID.IsZero() { return binding, nil, fmt.Errorf("dataset binding %d is not ready", binding.ID) } storageCtx, err := u.contextForReadyBinding(ctx, binding, bucket.Name) @@ -2644,11 +2906,10 @@ func (u *Uploader) contextForReadyBinding(ctx context.Context, binding *model.St return storageCtx, nil } -func (u *Uploader) extraDataForCopy(ctx context.Context, storageCtx synapse.UploadContext, uploadID int64, copyIndex int, pieces []storage.PieceInput) ([]byte, string, error) { - copyRow, err := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) - if err != nil { - return nil, "", err - } +// extraDataForCopy takes the concrete copy rather than a replica slot: the +// cached presign blob belongs to one generation, and resolving it by slot would +// hand back the replacement's blob after an activation. +func (u *Uploader) extraDataForCopy(ctx context.Context, storageCtx synapse.UploadContext, copyRow *model.StorageUploadCopy, pieces []storage.PieceInput) ([]byte, string, error) { if copyRow != nil && copyRow.CommitExtraDataHex != nil && *copyRow.CommitExtraDataHex != "" { extraData, err := hex.DecodeString(*copyRow.CommitExtraDataHex) return extraData, strings.ToLower(*copyRow.CommitExtraDataHex), err @@ -2739,12 +3000,14 @@ func (u *Uploader) waitForPendingSubmittedCommit(ctx context.Context, task *mode func (u *Uploader) resetRejectedSubmittedCommit( ctx context.Context, + copyID int64, uploadID int64, copyIndex int, transactionID string, commitErr error, ) error { return u.repos.Uploads.ResetRejectedUploadCopyCommit(ctx, repository.ResetRejectedUploadCopyCommitInput{ + StorageUploadCopyID: copyID, UploadID: uploadID, CopyIndex: copyIndex, CommitTransactionID: transactionID, @@ -2942,7 +3205,12 @@ func (u *Uploader) handleIngressFailure(ctx context.Context, task *model.Task, v if status == model.TaskStatusExhausted && !currentSubmitted { cleanupCtx, cancel := context.WithTimeout(context.Background(), terminalFailureCleanupTimeout) defer cancel() - if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, uploadID, copyIndex, fmt.Sprintf("%s: %v", stage, err)); markErr != nil { + if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, repository.MarkUploadCopyFailedInput{ + StorageUploadCopyID: uploadStageCopyID(task), + UploadID: uploadID, + CopyIndex: copyIndex, + LastError: fmt.Sprintf("%s: %v", stage, err), + }); markErr != nil { logger.Warn("failed to mark alternate ingress upload copy failed", "uploadID", uploadID, "copyIndex", copyIndex, "error", markErr) } } @@ -2953,7 +3221,12 @@ func (u *Uploader) handleIngressFailure(ctx context.Context, task *model.Task, v if status == model.TaskStatusExhausted { cleanupCtx, cancel := context.WithTimeout(context.Background(), terminalFailureCleanupTimeout) defer cancel() - if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, uploadID, copyIndex, fmt.Sprintf("%s: %v", stage, err)); markErr != nil { + if markErr := u.repos.Uploads.MarkUploadCopyFailed(cleanupCtx, repository.MarkUploadCopyFailedInput{ + StorageUploadCopyID: uploadStageCopyID(task), + UploadID: uploadID, + CopyIndex: copyIndex, + LastError: fmt.Sprintf("%s: %v", stage, err), + }); markErr != nil { logger.Warn("failed to mark ingress upload copy failed", "uploadID", uploadID, "copyIndex", copyIndex, "error", markErr) } } @@ -2970,7 +3243,7 @@ func (u *Uploader) handleIngressDataSetFailure(ctx context.Context, task *model. } } if dataSetFailureEnded(err, binding) || dataSetFailureUnavailable(err, binding) { - copyRow, loadErr := u.repos.Uploads.GetUploadCopy(ctx, uploadID, copyIndex) + copyRow, loadErr := u.taskUploadCopy(ctx, task, uploadID, copyIndex) if loadErr != nil || copyRow == nil { if loadErr == nil { loadErr = errors.New("ingress upload copy not found") diff --git a/internal/worker/uploader_replacement.go b/internal/worker/uploader_replacement.go new file mode 100644 index 0000000..4f20741 --- /dev/null +++ b/internal/worker/uploader_replacement.go @@ -0,0 +1,871 @@ +package worker + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "time" + + "github.com/ipfs/go-cid" + "github.com/strahe/synaps3/internal/admin" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/objectlimits" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/strahe/synaps3/internal/synapse" + idtypes "github.com/strahe/synaps3/internal/types" + "github.com/strahe/synapse-go/storage" +) + +// One bounded window of upload history per seeding pass. Small enough that the +// write stays short on SQLite's single writer, large enough that a long history +// still drains in a reasonable number of passes. +const replacementSeedBatchSize = 200 + +// processReplacementTask advances one approved replacement by exactly one step. +// The coordinator is a singleton per replacement, so a replacement can never +// have more than one item in flight and ordinary uploads keep their place in +// the queue. +func (u *Uploader) processReplacementTask(ctx context.Context, task *model.Task, logger *slog.Logger) { + if task == nil || task.ClaimedAt == nil { + return + } + payload, err := storagereplacement.ParseMigratePayload(task) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "parse replacement migration task", err) + return + } + replacement, err := u.repos.Replacements.GetByID(ctx, payload.ReplacementID) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "load provider replacement", err) + return + } + if replacement == nil || replacement.Status == storagereplacement.StatusCompleted || replacement.Status.Retryable() { + // A finished or operator-owned replacement has no work the coordinator + // may do on its own. + completeWorkerTask(ctx, u.repos, task, "uploader", logger) + return + } + if replacement.Status == storagereplacement.StatusSuperseded { + // The successor owns the slot. This coordinator's leftover is the unused + // target, which must still be ended even if confirmation already queued it. + if err := u.ensureAbandonedTargetTask(ctx, replacement.ID, replacement.BucketID, task.MaxRetries); err != nil { + u.handleReplacementTaskFailure(ctx, task, 0, logger, "queue abandoned replacement cleanup", err) + return + } + completeWorkerTask(ctx, u.repos, task, "uploader", logger) + return + } + target, err := u.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) + if err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "load replacement target", err) + return + } + if target == nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "load replacement target", + fmt.Errorf("target data set %d: %w", replacement.TargetDataSetID, repository.ErrNotFound)) + return + } + + switch replacementPhase(replacement.Status, target) { + case storagereplacement.PhasePrepare: + u.prepareReplacementTarget(ctx, task, replacement, target, logger) + case storagereplacement.PhaseMigrate: + u.migrateReplacementItem(ctx, task, replacement, target, payload, logger) + case storagereplacement.PhaseRetire: + // Retirement belongs to the cleanup worker; the migration coordinator's + // job is finished once it has handed over. + if err := u.ensureReplacementRetirementTask(ctx, replacement.ID, replacement.BucketID, task.MaxRetries); err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "queue replacement retirement", err) + return + } + completeWorkerTask(ctx, u.repos, task, "uploader", logger) + default: + // An unrecognised phase is a bug, not an outage. Stop the task without + // touching the replacement so the record still describes reality. + u.failReplacementTaskWithoutMutation(ctx, task, logger, + fmt.Errorf("replacement %d has no coordinator work in status %s", replacement.ID, replacement.Status)) + } +} + +// A waiting replacement resumes at the phase it was waiting in, which is +// recovered from the data rather than remembered in the status. +func replacementPhase(status storagereplacement.Status, target *model.StorageDataSet) storagereplacement.Phase { + if phase := storagereplacement.PhaseFor(status); phase != storagereplacement.PhaseNone { + return phase + } + if status != storagereplacement.StatusWaiting { + return storagereplacement.PhaseNone + } + if target != nil && target.IsCurrent { + return storagereplacement.PhaseMigrate + } + return storagereplacement.PhasePrepare +} + +// prepareReplacementTarget creates the approved service and, once it is +// writable, switches the slot over. Until that switch every write still goes to +// the source, so a failure here costs nothing but time. +func (u *Uploader) prepareReplacementTarget( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + target *model.StorageDataSet, + logger *slog.Logger, +) { + bucket, err := u.repos.Buckets.GetByID(ctx, replacement.BucketID) + if err != nil || bucket == nil { + if err == nil { + err = fmt.Errorf("bucket %d: %w", replacement.BucketID, repository.ErrNotFound) + } + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "load replacement bucket", err) + return + } + if target.Status != model.StorageDataSetStatusReady { + if target.Status == model.StorageDataSetStatusPending || target.Status == model.StorageDataSetStatusFailed { + if !u.ensureReplacementFundingReady(ctx, task, replacement, target, bucket, logger) { + return + } + } + ready, err := u.createReplacementDataSet(ctx, replacement, target, bucket) + if err != nil { + if errors.Is(err, storagereplacement.ErrTargetInUse) { + // Retrying cannot change this: the operator has to pick a + // provider that is free. The source still owns the replica at + // this point, so a new confirmation is available to them. + u.failReplacementTarget(ctx, task, replacement.ID, logger, err.Error()) + return + } + u.handleReplacementProviderFailure(ctx, task, replacement, logger, "create replacement service", err) + return + } + if !ready { + u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonTargetCreating, logger, + storagereplacement.WaitReasonTargetCreating.Message()) + return + } + } + if err := u.repos.Replacements.Activate(ctx, replacement.ID); err != nil { + if errors.Is(err, repository.ErrConflict) { + u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonTargetWritable, logger, + storagereplacement.WaitReasonTargetWritable.Message()) + return + } + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "activate replacement target", err) + return + } + logger.Info("provider replacement activated", + "replacementID", replacement.ID, "bucketID", replacement.BucketID, "copyIndex", replacement.CopyIndex) + u.continueReplacementTask(ctx, task, replacement.ID, 0, 0, "", logger) +} + +// createReplacementDataSet drives one data set creation step and reports +// whether the service is ready. It mirrors the ordinary upload path but does +// not touch any copy row, because the target holds no data yet. +func (u *Uploader) createReplacementDataSet( + ctx context.Context, + replacement *storagereplacement.Replacement, + target *model.StorageDataSet, + bucket *model.Bucket, +) (bool, error) { + storageCtx, err := u.contextForBindingProvider(ctx, target, bucket.Name) + if err != nil { + return false, err + } + switch target.Status { + case model.StorageDataSetStatusPending, model.StorageDataSetStatusFailed: + // Nothing has been submitted for this generation yet, so a context that + // already carries a data set can only be somebody else's: this provider + // still runs a live service for this bucket, a generation released + // locally without being terminated on chain. A replacement must open its + // own paid service -- attaching here would leave it paying for, and + // later retiring, a service it does not own. + // + // A creation this replacement did submit is resumed by the creating + // branch below, which resolves the recorded transaction instead. + if dataSetID := storageCtx.DataSetID(); dataSetID != nil { + return false, fmt.Errorf("provider %s already runs data set %s for this bucket: %w", + target.ProviderID.String(), idtypes.OnChainIDFromSDK(*dataSetID).String(), + storagereplacement.ErrTargetInUse) + } + var submitted storage.CreateDataSetSubmission + var submitErr error + result, err := storageCtx.CreateDataSet(ctx, &storage.CreateDataSetOptions{ + OnSubmitted: func(sub storage.CreateDataSetSubmission) { + submitted = sub + submitErr = u.repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ + ID: target.ID, + TransactionID: sub.TransactionID, + StatusURL: sub.StatusURL, + ClientDataSetID: onChainIDPtrFromSDKPtr(sub.ClientDataSetID), + }) + }, + }) + if submitted.TransactionID != "" && submitErr != nil { + return false, fmt.Errorf("save replacement service submission: %w", submitErr) + } + if err != nil { + return false, err + } + return true, u.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: target.ID, + DataSetID: idtypes.OnChainIDFromSDK(result.DataSetID), + ClientDataSetID: onChainIDPtrFromSDK(result.ClientDataSetID), + }) + case model.StorageDataSetStatusCreating: + if target.CreateTransactionID == nil || target.CreateStatusURL == nil || target.ClientDataSetID == nil { + return false, errDataSetCreationIncomplete + } + result, err := storageCtx.WaitForDataSetCreated(ctx, storage.CreateDataSetSubmission{ + TransactionID: *target.CreateTransactionID, + StatusURL: *target.CreateStatusURL, + ClientDataSetID: sdkBigIntPtr(target.ClientDataSetID), + }) + if err != nil { + return false, err + } + return true, u.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: target.ID, + DataSetID: idtypes.OnChainIDFromSDK(result.DataSetID), + ClientDataSetID: onChainIDPtrFromSDK(result.ClientDataSetID), + }) + default: + return false, fmt.Errorf("replacement %d target status %s cannot be prepared", replacement.ID, target.Status) + } +} + +// migrateReplacementItem copies one piece of stored content to the target. It +// seeds work lazily so a large history never blocks the first transfer, and it +// holds exactly one item at a time. +func (u *Uploader) migrateReplacementItem( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + target *model.StorageDataSet, + payload storagereplacement.MigratePayload, + logger *slog.Logger, +) { + if !replacement.SeedingComplete { + if _, _, err := u.repos.Replacements.SeedMigrationBatch(ctx, replacement.ID, replacementSeedBatchSize); err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "seed replacement migration", err) + return + } + } + item, err := u.nextReplacementItem(ctx, replacement.ID, payload.ItemID) + if err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "select replacement item", err) + return + } + if item == nil { + u.finishReplacementMigration(ctx, task, replacement, logger) + return + } + snapshot, err := u.repos.Replacements.AcquireItem(ctx, repository.AcquireReplacementItemInput{ + ReplacementID: replacement.ID, + ItemID: item.ID, + TaskID: task.ID, + TaskClaimedAt: *task.ClaimedAt, + }) + if err != nil { + switch { + case errors.Is(err, storagereplacement.ErrItemDeferred) && item.Status == storagereplacement.ItemStatusWaitingSource: + // Already-parked work is only reached once nothing executable is + // left, so re-queueing here would spin the coordinator against the + // same item and starve ordinary uploads. Wait for the source + // instead; the item is rechecked on the next tick. + u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonReadableSource, logger, + storagereplacement.WaitReasonReadableSource.Message()) + case errors.Is(err, storagereplacement.ErrItemCancelled), errors.Is(err, storagereplacement.ErrItemDeferred): + // Either this content no longer needs migrating, or it just moved + // out of the executable set. Both mean: make progress elsewhere. + u.continueReplacementTask(ctx, task, replacement.ID, 0, 0, "", logger) + case errors.Is(err, repository.ErrTaskClaimLost): + return + default: + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "acquire replacement item", err) + } + return + } + + // An ordinary upload already writing this slot owns it first; the + // coordinator waits rather than racing it. + ordinaryRunning, err := u.repos.Tasks.HasEarlierRunningUploadCopyTask(ctx, task, snapshot.Upload.ID, replacement.CopyIndex) + if err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "check ordinary upload copy task", err) + return + } + if ordinaryRunning { + u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonSourceWrites, logger, + storagereplacement.WaitReasonSourceWrites.Message()) + return + } + + copyRow, err := u.repos.Replacements.AttachTargetCopy(ctx, repository.AttachReplacementTargetCopyInput{ + ReplacementID: replacement.ID, + ItemID: item.ID, + UploadID: snapshot.Upload.ID, + }) + if err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "attach replacement target copy", err) + return + } + if copyCommitted(copyRow) { + u.completeReplacementItem(ctx, task, replacement, &snapshot.Upload, item.ID, logger) + return + } + + bucket, err := u.repos.Buckets.GetByID(ctx, replacement.BucketID) + if err != nil || bucket == nil { + if err == nil { + err = fmt.Errorf("bucket %d: %w", replacement.BucketID, repository.ErrNotFound) + } + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "load replacement bucket", err) + return + } + storageCtx, err := u.contextForReadyBinding(ctx, target, bucket.Name) + if err != nil { + u.handleReplacementProviderFailure(ctx, task, replacement, logger, "open replacement target context", err) + return + } + if err := u.copyReplacementItem(ctx, task, replacement, snapshot, copyRow, storageCtx, bucket, logger); err != nil { + u.handleReplacementProviderFailure(ctx, task, replacement, logger, "copy replacement item", err) + } +} + +// nextReplacementItem keeps the item the payload already names, so a retry +// resumes the same transfer instead of starting a different one. +func (u *Uploader) nextReplacementItem(ctx context.Context, replacementID, assignedItemID int64) (*storagereplacement.Item, error) { + if assignedItemID > 0 { + item, err := u.repos.Replacements.NextExecutableItem(ctx, replacementID) + if err != nil { + return nil, err + } + if item != nil && item.ID == assignedItemID { + return item, nil + } + } + return u.repos.Replacements.NextExecutableItem(ctx, replacementID) +} + +func (u *Uploader) copyReplacementItem( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + snapshot *repository.ReplacementItemSnapshot, + copyRow *model.StorageUploadCopy, + storageCtx synapse.UploadContext, + bucket *model.Bucket, + logger *slog.Logger, +) error { + upload := &snapshot.Upload + version := &snapshot.Version + var pieceCID cid.Cid + var pieceCIDString string + extraHex := derefString(copyRow.CommitExtraDataHex) + + if !copyHasPiece(copyRow) { + readable, err := u.repos.Uploads.ListReadableCommittedCopies(ctx, upload.ID) + if err != nil { + return err + } + pulled, pullErr := u.pullReplacementItem(ctx, storageCtx, upload, copyRow, + orderReplacementSources(readable, snapshot.Target.CopyIndex)) + if pullErr != nil { + return pullErr + } + if pulled != nil { + pieceCID = pulled.pieceCID + pieceCIDString = pulled.pieceCIDString + extraHex = pulled.extraHex + } else { + // No remote replica could serve the content, so fall back to data this + // node still holds. + stored, storedPieceCID, err := u.storeReplacementItemFromCache(ctx, storageCtx, bucket, version, logger) + if err != nil { + return err + } + if !stored { + // Nothing can supply this content yet. Park the item so the + // coordinator keeps making progress on everything else, and + // revisit it later. + if err := u.repos.Replacements.MarkItemWaitingSource(ctx, snapshot.Item.ID, + "no readable replica or retained cache data"); err != nil { + return err + } + u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonReadableSource, logger, + storagereplacement.WaitReasonReadableSource.Message()) + return nil + } + pieceCID = storedPieceCID + pieceCIDString = pieceCID.String() + _, extraHex, err = u.extraDataForCopy(ctx, storageCtx, copyRow, + []storage.PieceInput{{PieceCID: pieceCID}}) + if err != nil { + return err + } + } + if err := u.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ + StorageUploadCopyID: copyRow.ID, + RequireEligibleCopy: true, + UploadID: upload.ID, + CopyIndex: copyRow.CopyIndex, + PieceCID: pieceCIDString, + RetrievalURL: storageCtx.PieceURL(pieceCID), + }); err != nil { + return err + } + if err := u.repos.Uploads.MarkUploadCopyCommitting(ctx, repository.MarkUploadCopyCommittingInput{ + StorageUploadCopyID: copyRow.ID, + RequireEligibleCopy: true, + UploadID: upload.ID, + CopyIndex: copyRow.CopyIndex, + CommitExtraDataHex: extraHex, + }); err != nil { + return err + } + copyRow.Status = model.StorageUploadCopyStatusPieceReady + copyRow.CommitExtraDataHex = &extraHex + } else { + if upload.PieceCID == nil || *upload.PieceCID == "" { + return fmt.Errorf("storage upload %d has no piece CID", upload.ID) + } + pieceCIDString = *upload.PieceCID + decoded, err := cid.Decode(pieceCIDString) + if err != nil { + return fmt.Errorf("decode stored piece CID: %w", err) + } + pieceCID = decoded + } + + pieces := []storage.PieceInput{{PieceCID: pieceCID}} + result, err := u.commitReplicaRepairCopy(ctx, upload, &snapshot.Target, copyRow, storageCtx, pieces) + if err != nil { + return err + } + if result == nil || len(result.PieceIDs) == 0 { + return errors.New("replacement commit returned no piece ID") + } + pieceID := idtypes.OnChainIDFromSDK(result.PieceIDs[0]) + if err := u.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + StorageUploadCopyID: copyRow.ID, + RequireEligibleCopy: true, + UploadID: upload.ID, + CopyIndex: copyRow.CopyIndex, + PieceCID: pieceCIDString, + PieceID: &pieceID, + RetrievalURL: storageCtx.PieceURL(pieceCID), + CommitExtraDataHex: derefString(copyRow.CommitExtraDataHex), + CommitTransactionID: result.TransactionID, + }); err != nil { + return err + } + u.completeReplacementItem(ctx, task, replacement, upload, snapshot.Item.ID, logger) + return nil +} + +type pulledReplacementPiece struct { + pieceCID cid.Cid + pieceCIDString string + extraHex string +} + +// pullReplacementItem tries each readable replica in turn. A provider that +// cannot serve the piece is skipped rather than retried, because another +// replica or the local cache may still have it. A nil result with no error +// means no remote replica could supply the content. +func (u *Uploader) pullReplacementItem( + ctx context.Context, + storageCtx synapse.UploadContext, + upload *model.StorageUpload, + copyRow *model.StorageUploadCopy, + sources []repository.ReadableStorageCopy, +) (*pulledReplacementPiece, error) { + var lastErr error + for i := range sources { + source := &sources[i] + pieceCID, err := cid.Decode(source.PieceCID) + if err != nil { + lastErr = fmt.Errorf("decode source piece CID: %w", err) + continue + } + extraData, extraHex, err := u.extraDataForCopy(ctx, storageCtx, copyRow, + []storage.PieceInput{{PieceCID: pieceCID}}) + if err != nil { + return nil, err + } + // Provider-to-provider transfer keeps the bytes off this node. + if _, err := storageCtx.Pull(ctx, storage.PullRequest{ + Pieces: []cid.Cid{pieceCID}, + ExtraData: extraData, + From: func(cid.Cid) string { + return source.RetrievalURL + }, + }); err != nil { + if replacementSourceUnusable(err) { + lastErr = err + continue + } + return nil, err + } + return &pulledReplacementPiece{pieceCID: pieceCID, pieceCIDString: source.PieceCID, extraHex: extraHex}, nil + } + if lastErr != nil && len(sources) > 0 { + // Every candidate refused. Let the caller try retained cache data before + // deciding the content has no source at all. + return nil, nil + } + return nil, nil +} + +// A source that cannot serve a read is not a reason to fail the migration; it +// only means this replica is not usable right now. +func replacementSourceUnusable(err error) bool { + return synapse.IsProviderUnavailable(err) || + synapse.IsDataSetServiceEnded(err) || + synapse.IsNoProviderCandidates(err) +} + +// orderReplacementSources prefers a replica on another slot over the draining +// source, and never offers the target itself. A source that recovered +// mid-migration therefore becomes usable again with no special case. +func orderReplacementSources(copies []repository.ReadableStorageCopy, targetCopyIndex int) []repository.ReadableStorageCopy { + preferred := make([]repository.ReadableStorageCopy, 0, len(copies)) + sameSlot := make([]repository.ReadableStorageCopy, 0, len(copies)) + for i := range copies { + candidate := copies[i] + if candidate.PieceCID == "" || candidate.RetrievalURL == "" { + continue + } + if candidate.CopyIndex == targetCopyIndex { + sameSlot = append(sameSlot, candidate) + continue + } + preferred = append(preferred, candidate) + } + return append(preferred, sameSlot...) +} + +func (u *Uploader) storeReplacementItemFromCache( + ctx context.Context, + storageCtx synapse.UploadContext, + bucket *model.Bucket, + version *model.ObjectVersion, + logger *slog.Logger, +) (bool, cid.Cid, error) { + if version == nil || version.CacheKey == "" { + return false, cid.Undef, nil + } + rc, _, err := u.cache.Get(ctx, bucket.Name, version.CacheKey) + if err != nil { + if os.IsNotExist(err) { + if version.InCache { + if markErr := u.repos.Objects.SetVersionCachePresence(ctx, version.VersionID, false); markErr != nil { + logger.Warn("failed to mark cache location absent", "versionID", version.VersionID, "error", markErr) + } + } + return false, cid.Undef, nil + } + return false, cid.Undef, fmt.Errorf("open retained cache data: %w", err) + } + result, storeErr := storageCtx.Store(ctx, rc, &storage.StoreOptions{}) + closeErr := rc.Close() + if storeErr != nil { + return false, cid.Undef, storeErr + } + if closeErr != nil { + return false, cid.Undef, fmt.Errorf("close retained cache data: %w", closeErr) + } + if result == nil || !result.PieceCID.Defined() { + return false, cid.Undef, errors.New("replacement store returned no piece CID") + } + return true, result.PieceCID, nil +} + +func (u *Uploader) completeReplacementItem( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + upload *model.StorageUpload, + itemID int64, + logger *slog.Logger, +) { + if err := u.repos.Replacements.MarkItemCopied(ctx, itemID); err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "record replacement item copied", err) + return + } + // The target now holds another readable slot, which can complete the + // upload's durability target. + if _, _, err := u.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, u.finalizeUploadInput(upload.ID)); err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "finalize migrated upload", err) + return + } + // The coordinator is bucket-scoped. Recording the migrated version here + // would make the task list point at one arbitrary object. + u.continueReplacementTask(ctx, task, replacement.ID, 0, 0, "", logger) +} + +// finishReplacementMigration hands over to retirement once nothing is owed. +// Seeding must have finished first, otherwise "no items" only means "none +// discovered yet". +func (u *Uploader) finishReplacementMigration( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + logger *slog.Logger, +) { + if !replacement.SeedingComplete { + u.continueReplacementTask(ctx, task, replacement.ID, 0, 0, "", logger) + return + } + if err := u.repos.Replacements.BeginRetirement(ctx, replacement.ID); err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "begin replacement retirement", err) + return + } + if err := u.ensureReplacementRetirementTask(ctx, replacement.ID, replacement.BucketID, task.MaxRetries); err != nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "queue replacement retirement", err) + return + } + logger.Info("provider replacement migration complete", "replacementID", replacement.ID) + completeWorkerTask(ctx, u.repos, task, "uploader", logger) +} + +func (u *Uploader) ensureReplacementRetirementTask(ctx context.Context, replacementID, bucketID int64, maxRetries int) error { + _, err := u.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewRetireTask(replacementID, bucketID, maxRetries, time.Now())) + if err != nil { + return fmt.Errorf("ensure replacement retirement task for replacement %d: %w", replacementID, err) + } + return nil +} + +func (u *Uploader) ensureAbandonedTargetTask(ctx context.Context, replacementID, bucketID int64, maxRetries int) error { + _, err := u.repos.Tasks.EnsureRecurring(ctx, storagereplacement.NewAbandonedTargetTask(replacementID, bucketID, maxRetries, time.Now())) + if err != nil { + return fmt.Errorf("ensure abandoned replacement cleanup for replacement %d: %w", replacementID, err) + } + return nil +} + +// ensureReplacementFundingReady waits, without burning retries, until the wallet +// can pay for the approved service. Ordinary uploads already do this; creating +// a replacement service is the same kind of paid work. +func (u *Uploader) ensureReplacementFundingReady( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + target *model.StorageDataSet, + bucket *model.Bucket, + logger *slog.Logger, +) bool { + storageCtx, err := u.contextForBindingProvider(ctx, target, bucket.Name) + if err != nil { + u.handleReplacementProviderFailure(ctx, task, replacement, logger, "open replacement funding context", err) + return false + } + costs, err := u.storage.PrepareUpload(ctx, uint64(objectlimits.MinFOCUploadSize), []synapse.UploadContext{storageCtx}) + if err != nil { + u.handleReplacementProviderFailure(ctx, task, replacement, logger, "prepare replacement funding", err) + return false + } + if costs == nil { + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, "prepare replacement funding", + errors.New("missing storage cost estimate")) + return false + } + if costs.Ready { + return true + } + u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonFunding, logger, + uploadFundingWaitMessage(costs)) + return false +} + +// continueReplacementTask re-queues the coordinator at the tail of the upload +// queue. Ordinary uploads that were already due are claimed first, which is how +// migration interleaves without a priority system. +func (u *Uploader) continueReplacementTask( + ctx context.Context, + task *model.Task, + replacementID, itemID, copyID int64, + versionID string, + logger *slog.Logger, +) { + err := u.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + if err := txRepos.Tasks.LockRunningClaim(ctx, task); err != nil { + return err + } + return txRepos.Tasks.ContinueRunning(ctx, task, versionID, + storagereplacement.NewMigratePayload(replacementID, itemID, copyID)) + }) + if err != nil { + u.handleReplacementTaskFailure(ctx, task, replacementID, logger, "advance replacement task", err) + return + } + admin.WorkerTasksProcessed.WithLabelValues("uploader", "success").Inc() +} + +// waitForReplacementDependency records why progress paused without consuming +// retry budget. Waiting is never a failure. +func (u *Uploader) waitForReplacementDependency( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + reason storagereplacement.WaitReason, + logger *slog.Logger, + message string, +) { + if err := u.repos.Replacements.MarkWaiting(ctx, replacement.ID, reason); err != nil { + logger.Warn("failed to record replacement wait reason", + "replacementID", replacement.ID, "reason", reason, "error", err) + } + u.waitForStorageDependency(ctx, task, logger, message) +} + +// handleReplacementProviderFailure keeps a recoverable provider problem in +// waiting and reserves retries for genuinely unknown failures. +func (u *Uploader) handleReplacementProviderFailure( + ctx context.Context, + task *model.Task, + replacement *storagereplacement.Replacement, + logger *slog.Logger, + stage string, + err error, +) { + if u.waitForPendingSubmittedCommit(ctx, task, logger, err) { + return + } + switch { + case synapse.IsProviderUnavailable(err), synapse.IsNoProviderCandidates(err): + u.waitForReplacementDependency(ctx, task, replacement, storagereplacement.WaitReasonTarget, logger, + storagereplacement.WaitReasonTarget.Message()) + case dataSetWriteBlockedError(err), synapse.IsDataSetServiceEnded(err): + // The approved target itself ended. That needs a new confirmation, so it + // is operator work rather than another attempt. + u.markReplacementFailed(ctx, replacement.ID, logger, fmt.Sprintf("%s: %v", stage, err)) + u.failReplacementTaskWithoutMutation(ctx, task, logger, err) + default: + u.handleReplacementTaskFailure(ctx, task, replacement.ID, logger, stage, err) + } +} + +// handleReplacementTaskFailure retries, and records the replacement as failed +// only once the task has genuinely run out of attempts. +func (u *Uploader) handleReplacementTaskFailure( + ctx context.Context, + task *model.Task, + replacementID int64, + logger *slog.Logger, + stage string, + err error, +) { + logger.Error(stage+" failed", "replacementID", replacementID, "error", err) + status := scheduleTaskRetry(ctx, u.repos, task, "uploader", logger, err) + if status == model.TaskStatusExhausted && replacementID > 0 { + // Use an independent context so the record still lands during shutdown. + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), terminalFailureCleanupTimeout) + defer cancel() + u.markReplacementFailed(cleanupCtx, replacementID, logger, + fmt.Sprintf("%s: %v (max retries reached)", stage, err)) + } + admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() +} + +func (u *Uploader) markReplacementFailed(ctx context.Context, replacementID int64, logger *slog.Logger, message string) { + if err := u.repos.Replacements.MarkFailed(ctx, replacementID, nil, message); err != nil { + logger.Error("failed to record provider replacement failure", "replacementID", replacementID, "error", err) + } +} + +// failReplacementTarget records an approved target that can never work and +// stops its coordinator in the same transaction, so the record never shows work +// in progress with nothing queued to do it. Retrying is pointless here, so the +// retry budget is not spent first. +func (u *Uploader) failReplacementTarget(ctx context.Context, task *model.Task, replacementID int64, logger *slog.Logger, message string) { + err := u.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + reason := storagereplacement.FailureReasonTargetInUse + if err := txRepos.Replacements.MarkFailed(ctx, replacementID, &reason, message); err != nil { + return err + } + return txRepos.Tasks.FailRunning(ctx, task, message) + }) + if err != nil { + logger.Error("failed to record unusable replacement target", + "replacementID", replacementID, "error", err) + } else { + logger.Warn("provider replacement needs a different provider", + "replacementID", replacementID, "reason", message) + } + admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() +} + +// failReplacementTaskWithoutMutation stops a task that must not retry while +// leaving the replacement record exactly as it is. +func (u *Uploader) failReplacementTaskWithoutMutation(ctx context.Context, task *model.Task, logger *slog.Logger, err error) { + logger.Error("replacement coordinator stopped", "taskID", task.ID, "error", err) + if failErr := u.repos.Tasks.FailRunning(ctx, task, err.Error()); failErr != nil { + logger.Error("failed to stop replacement coordinator", "taskID", task.ID, "error", failErr) + } + admin.WorkerTasksProcessed.WithLabelValues("uploader", "failure").Inc() +} + +// deferToReplacement yields an ordinary upload stage to the replacement +// coordinator when both target the same copy. It mirrors the recovered-replica +// gate so the two coordinators and normal uploads never write one row at once. +func (u *Uploader) deferToReplacement(ctx context.Context, task *model.Task, bucketID, uploadID int64, copyIndex int, logger *slog.Logger) bool { + copyRow, err := u.taskUploadCopy(ctx, task, uploadID, copyIndex) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "load upload copy for replacement coordination", err) + return true + } + // The write belongs to the generation its copy is bound to. Asking the slot + // instead would stop deferring the moment the replacement takes the slot, + // which is exactly when the two writers overlap. + binding, err := u.taskCopyDataSet(ctx, task, bucketID, uploadID, copyIndex) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "load upload data set", err) + return true + } + if binding == nil { + return false + } + replacement, err := u.repos.Replacements.GetActiveForDataSet(ctx, binding.ID) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "check provider replacement", err) + return true + } + if replacement == nil { + return false + } + coordinator, err := u.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.MigrateTaskKey(replacement.ID)) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "check replacement coordinator task", err) + return true + } + if coordinator == nil || coordinator.Status != model.TaskStatusRunning { + return false + } + if copyRow == nil { + u.handleTaskFailure(ctx, task, logger, "load upload copy for replacement coordination", + fmt.Errorf("upload copy %d not found", copyIndex)) + return true + } + claimedCopyID := copyRow.ID + // Mutual exclusion is per copy row, and the coordinator's row is recorded on + // the item it holds. Standing down for anything else would idle every + // ordinary upload on this bucket for the length of each transfer, which also + // keeps the retiring service billable for longer. + heldCopyID, err := u.repos.Replacements.HeldItemCopyID(ctx, replacement.ID) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "check replacement item in progress", err) + return true + } + if heldCopyID == 0 || heldCopyID != claimedCopyID { + return false + } + if !taskClaimPrecedes(coordinator, task) { + return false + } + u.waitForStorageDependency(ctx, task, logger, "Waiting for the approved provider replacement") + return true +} diff --git a/internal/worker/uploader_replacement_internal_test.go b/internal/worker/uploader_replacement_internal_test.go new file mode 100644 index 0000000..01a219c --- /dev/null +++ b/internal/worker/uploader_replacement_internal_test.go @@ -0,0 +1,338 @@ +package worker + +import ( + "context" + "log/slog" + "testing" + + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/strahe/synaps3/internal/testutil" + "github.com/strahe/synaps3/internal/types" +) + +// In-place recovery and an approved replacement both want to finish the same +// generation's work. The replacement wins while it is running, otherwise the +// two would race each other over one copy row. +func TestEnsureReplicaRepairTaskStandsDownForApprovedReplacement(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := testutil.SeedBucket(t, db, "repair-vs-replacement") + + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), + BucketID: bucket.ID, + Key: "file.txt", + Size: 11, + ETag: "etag", + Checksum: "sum", + CacheKey: ".versions/repair-vs-replacement", + } + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ + BucketID: bucket.ID, + SourceVersionID: version.VersionID, + ContentSize: version.Size, + Checksum: version.Checksum, + RequestedCopies: 2, + }) + if err != nil { + t.Fatalf("StartObjectUploadAttempt: %v", err) + } + providerID := mustOnChainID(t, "101") + binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, + ProviderID: providerID, + CopyIndex: 0, + CreatedByUploadID: upload.ID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, + UploadID: upload.ID, + DataSetID: mustOnChainID(t, "1001"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: providerID, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + established, err := repos.Uploads.GetDataSetBindingByID(ctx, binding.ID) + if err != nil || established == nil { + t.Fatalf("GetDataSetBindingByID: %#v err=%v", established, err) + } + + // Without a replacement the generation repairs itself as usual. + created, err := ensureReplicaRepairTask(ctx, repos, established, 5) + if err != nil { + t.Fatalf("ensureReplicaRepairTask: %v", err) + } + if !created { + t.Fatal("recovery did not queue repair work for an unfinished copy") + } + if _, err := db.NewDelete(). + Model((*model.Task)(nil)). + Where("idempotency_key = ?", replicaRepairTaskKey(established.ID)). + Exec(ctx); err != nil { + t.Fatalf("clear repair task: %v", err) + } + + replacement, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, + SourceDataSetID: established.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: mustOnChainID(t, "202"), + ClientRequestID: "internal-replacement-1", + MaxRetries: 5, + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + created, err = ensureReplicaRepairTask(ctx, repos, established, 5) + if err != nil { + t.Fatalf("ensureReplicaRepairTask during replacement: %v", err) + } + if created { + t.Fatal("recovery queued repair work while an approved replacement owns the generation") + } + + // The generation being written *to* needs the same protection: repairing the + // target in place would race the coordinator over one copy row, and the two + // gates key off different task payloads so neither would see the other. + target, err := repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) + if err != nil || target == nil { + t.Fatalf("GetDataSetBindingByID target = %#v err=%v", target, err) + } + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: target.ID, DataSetID: mustOnChainID(t, "2002"), + }); err != nil { + t.Fatalf("MarkDataSetReady target: %v", err) + } + if err := repos.Uploads.MarkDataSetUnavailable(ctx, target.ID, "target provider unreachable"); err != nil { + t.Fatalf("MarkDataSetUnavailable target: %v", err) + } + unavailableTarget, err := repos.Uploads.GetDataSetBindingByID(ctx, target.ID) + if err != nil || unavailableTarget == nil { + t.Fatalf("reload target = %#v err=%v", unavailableTarget, err) + } + created, err = ensureReplicaRepairTask(ctx, repos, unavailableTarget, 5) + if err != nil { + t.Fatalf("ensureReplicaRepairTask for target: %v", err) + } + if created { + t.Fatal("recovery queued repair work on the generation the replacement is writing to") + } + + // A replacement that has given up must not hold the slot hostage. + if err := repos.Replacements.MarkFailed(ctx, replacement.ID, nil, "target creation exhausted"); err != nil { + t.Fatalf("MarkFailed: %v", err) + } + created, err = ensureReplicaRepairTask(ctx, repos, established, 5) + if err != nil { + t.Fatalf("ensureReplicaRepairTask after failure: %v", err) + } + if !created { + t.Fatal("recovery stayed blocked after the replacement terminally failed") + } +} + +func mustOnChainID(t *testing.T, value string) types.OnChainID { + t.Helper() + id, err := types.ParseOnChainID("test id", value) + if err != nil { + t.Fatalf("parse on-chain id %q: %v", value, err) + } + return id +} + +// An upload already in flight belongs to the generation its copy was bound to. +// Resolving the data set by replica slot instead would follow the slot to the +// replacement target the moment it activates, storing the piece on the new +// provider while the copy row being updated still points at the old one. +func TestTaskCopyDataSetFollowsTheCopyNotTheSlot(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := testutil.SeedBucket(t, db, "in-flight-across-activation") + + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), + BucketID: bucket.ID, + Key: "file.txt", + Size: 11, + ETag: "etag", + Checksum: "sum", + CacheKey: ".versions/in-flight-across-activation", + } + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ + BucketID: bucket.ID, + SourceVersionID: version.VersionID, + ContentSize: version.Size, + Checksum: version.Checksum, + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("StartObjectUploadAttempt: %v", err) + } + source, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, + ProviderID: mustOnChainID(t, "101"), + CopyIndex: 0, + CreatedByUploadID: upload.ID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: source.ID, UploadID: upload.ID, DataSetID: mustOnChainID(t, "1001"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: source.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: mustOnChainID(t, "101"), + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + inFlight, err := repos.Uploads.GetUploadCopy(ctx, upload.ID, 0) + if err != nil || inFlight == nil { + t.Fatalf("GetUploadCopy = %#v err=%v", inFlight, err) + } + + // The task was queued while the source still owned the slot. + task := newUploadStageTask( + repository.ObjectVersionRef{ObjectID: version.ObjectID, VersionID: version.VersionID}, + 5, uploadStageIngressStore, upload.ID, 0, model.StorageCopyTransferMethodIngress, inFlight.ID) + + replacement, _, err := repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: bucket.ID, + SourceDataSetID: source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: mustOnChainID(t, "202"), + ClientRequestID: "internal-replacement-2", + MaxRetries: 5, + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: replacement.TargetDataSetID, DataSetID: mustOnChainID(t, "2002"), + }); err != nil { + t.Fatalf("MarkDataSetReady target: %v", err) + } + if err := repos.Replacements.Activate(ctx, replacement.ID); err != nil { + t.Fatalf("Activate: %v", err) + } + + current, err := repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, 0) + if err != nil || current == nil || current.ID != replacement.TargetDataSetID { + t.Fatalf("slot owner after activation = %#v err=%v, want the target", current, err) + } + + u := &Uploader{repos: repos} + // A retry after activation must keep the source binding whether it resumes + // immediately after copy creation, after storing the piece, or while commit + // confirmation is in progress. + for _, status := range []model.StorageUploadCopyStatus{ + model.StorageUploadCopyStatusPending, + model.StorageUploadCopyStatusPieceReady, + model.StorageUploadCopyStatusCommitting, + } { + if _, err := db.NewUpdate().Model((*model.StorageUploadCopy)(nil)). + Set("status = ?", status). + Where("id = ?", inFlight.ID). + Exec(ctx); err != nil { + t.Fatalf("set in-flight status %s: %v", status, err) + } + preserved, err := u.preserveInFlightUploadBindings( + ctx, + upload.ID, + newBucketBindingPlan([]model.StorageDataSet{*current}, upload.ID), + ) + if err != nil { + t.Fatalf("preserve binding at %s: %v", status, err) + } + if got := preserved.byCopyIndex[0]; got == nil || got.ID != source.ID { + t.Fatalf("binding at %s = %#v, want source generation %d", status, got, source.ID) + } + } + resolved, err := u.taskCopyDataSet(ctx, task, bucket.ID, upload.ID, 0) + if err != nil || resolved == nil { + t.Fatalf("taskCopyDataSet = %#v err=%v", resolved, err) + } + if resolved.ID != source.ID { + t.Fatalf("resolved data set %d, want the retiring generation %d the copy is bound to", resolved.ID, source.ID) + } + + // A task queued before copy ids existed has nothing to anchor to and stays + // resolvable through the slot, which is where it belongs. + legacy := newUploadStageTask( + repository.ObjectVersionRef{ObjectID: version.ObjectID, VersionID: version.VersionID}, + 5, uploadStageIngressStore, upload.ID, 0, model.StorageCopyTransferMethodIngress, 0) + viaSlot, err := u.taskCopyDataSet(ctx, legacy, bucket.ID, upload.ID, 0) + if err != nil || viaSlot == nil { + t.Fatalf("taskCopyDataSet legacy = %#v err=%v", viaSlot, err) + } + if viaSlot.ID != replacement.TargetDataSetID { + t.Fatalf("legacy task resolved to %d, want the current generation %d", viaSlot.ID, replacement.TargetDataSetID) + } +} + +// A replacement that has already asked the old service to end is past +// migration, even when a dependency wait has moved it out of the retiring +// status. Restarting it as a migration would re-run the copy pass and, worse, +// leave nothing driving the termination it already started. +func TestEnqueueReplacementCoordinatorResumesRetirementAfterTermination(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + m := &Manager{repos: repos, logger: slog.Default(), uploadMaxRetries: 5} + + epoch := int64(4200) + for _, tc := range []struct { + name string + row storagereplacement.Replacement + wantKey func(int64) string + }{ + {"migrating", storagereplacement.Replacement{ID: 1, BucketID: 9, Status: storagereplacement.StatusMigrating}, storagereplacement.MigrateTaskKey}, + {"waiting before termination", storagereplacement.Replacement{ID: 2, BucketID: 9, Status: storagereplacement.StatusWaiting}, storagereplacement.MigrateTaskKey}, + {"retiring", storagereplacement.Replacement{ID: 3, BucketID: 9, Status: storagereplacement.StatusRetiring}, storagereplacement.RetireTaskKey}, + {"waiting after termination", storagereplacement.Replacement{ID: 4, BucketID: 9, Status: storagereplacement.StatusWaiting, TerminationEpoch: &epoch}, storagereplacement.RetireTaskKey}, + } { + t.Run(tc.name, func(t *testing.T) { + row := tc.row + m.enqueueReplacementCoordinator(ctx, &row) + task, err := repos.Tasks.GetByIdempotencyKey(ctx, tc.wantKey(row.ID)) + if err != nil || task == nil { + t.Fatalf("coordinator for %s = %#v err=%v, want %s", tc.name, task, err, tc.wantKey(row.ID)) + } + }) + } + + // The retirement cases must not also have queued a migration pass. + for _, id := range []int64{3, 4} { + task, err := repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.MigrateTaskKey(id)) + if err != nil { + t.Fatalf("GetByIdempotencyKey: %v", err) + } + if task != nil { + t.Fatalf("replacement %d was restarted as a migration after its service was already ended", id) + } + } +} diff --git a/internal/worker/uploader_replacement_test.go b/internal/worker/uploader_replacement_test.go new file mode 100644 index 0000000..63cea90 --- /dev/null +++ b/internal/worker/uploader_replacement_test.go @@ -0,0 +1,954 @@ +package worker_test + +import ( + "context" + "errors" + "fmt" + "log/slog" + "math/big" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/storagereplacement" + "github.com/strahe/synaps3/internal/synapse" + "github.com/strahe/synaps3/internal/testutil" + "github.com/strahe/synaps3/internal/worker" + "github.com/strahe/synapse-go/storage" + sdktypes "github.com/strahe/synapse-go/types" +) + +type replacementEnv struct { + env *testWorkerEnv + bucket *model.Bucket + upload *model.StorageUpload + versionID string + source *model.StorageDataSet + sourceCtx *fakeUploadContext + targetCtx *fakeUploadContext +} + +// seedReplacementEnv stores one object on a single replica slot and prepares +// fake contexts for both the retiring provider and its replacement. +func seedReplacementEnv(t *testing.T) *replacementEnv { + t.Helper() + env := newTestWorkerEnv(t) + ctx := context.Background() + bucket, _, versionID := seedCachedObject(t, env) + + version, err := env.repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil { + t.Fatalf("GetVersionByID: %v", err) + } + upload, err := env.repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ + BucketID: bucket.ID, + SourceVersionID: versionID, + ContentSize: version.Size, + Checksum: version.Checksum, + RequestedCopies: 1, + }) + if err != nil { + t.Fatalf("StartObjectUploadAttempt: %v", err) + } + source := seedReadyBinding(t, env, bucket.ID, upload.ID, 0, "101", "1001") + if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: source.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: onChainID(t, "101"), + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + UploadID: upload.ID, + CopyIndex: 0, + PieceCID: testCID(t).String(), + PieceID: onChainIDPtr(t, "2001"), + RetrievalURL: "https://source.example/piece", + }); err != nil { + t.Fatalf("MarkUploadCopyCommitted: %v", err) + } + if _, err := env.repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + UploadID: upload.ID, + BucketID: bucket.ID, + ContentSize: version.Size, + Checksum: version.Checksum, + }); err != nil { + t.Fatalf("BindReadableUploadForContent: %v", err) + } + + sourceCtx := readyFakeUploadContext(sdktypes.NewBigInt(101), sdktypes.NewBigInt(1001), sdktypes.NewBigInt(2001), testCID(t)) + targetCtx := newFakeUploadContext(sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3002), testCID(t)) + env.storage.CreateContextFunc = func(_ context.Context, opts *storage.CreateContextOptions) (synapse.UploadContext, error) { + switch { + case createContextDataSetIDEqual(opts, sdktypes.NewBigInt(1001)): + return sourceCtx, nil + case createContextDataSetIDEqual(opts, sdktypes.NewBigInt(2002)), createContextProviderIDEqual(opts, sdktypes.NewBigInt(202)): + return targetCtx, nil + } + return nil, fmt.Errorf("unexpected CreateContext opts: %#v", opts) + } + return &replacementEnv{ + env: env, bucket: bucket, upload: upload, versionID: versionID, + source: source, sourceCtx: sourceCtx, targetCtx: targetCtx, + } +} + +func (r *replacementEnv) authorize(t *testing.T, provider string) *storagereplacement.Replacement { + t.Helper() + row, _, err := r.env.repos.Replacements.Authorize(context.Background(), repository.AuthorizeReplacementInput{ + BucketID: r.bucket.ID, + SourceDataSetID: r.source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, provider), + ClientRequestID: "worker-replacement-" + provider, + MaxRetries: 5, + }) + if err != nil { + t.Fatalf("Authorize: %v", err) + } + return row +} + +func (r *replacementEnv) coordinatorTask(t *testing.T, replacementID int64) *model.Task { + t.Helper() + task, err := r.env.repos.Tasks.GetByIdempotencyKey(context.Background(), storagereplacement.MigrateTaskKey(replacementID)) + if err != nil || task == nil { + t.Fatalf("migration coordinator = %#v err=%v", task, err) + } + return task +} + +func (r *replacementEnv) abandonedTargetTask(t *testing.T, replacementID int64) *model.Task { + t.Helper() + task, err := r.env.repos.Tasks.GetByIdempotencyKey(context.Background(), storagereplacement.AbandonedTargetTaskKey(replacementID)) + if err != nil || task == nil { + t.Fatalf("abandoned-target cleanup = %#v err=%v, want it queued with the later confirmation", task, err) + } + return task +} + +// runUploaderUntil drives the uploader until cond holds. Waiting and retrying +// both leave the task in an active status, which runWorkerUntilTask never +// returns on. +func (r *replacementEnv) runUploaderUntil(t *testing.T, cond func() bool, timeout time.Duration) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + uploader := r.newUploader() + go func() { + defer close(done) + _ = uploader.Run(ctx) + }() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + cancel() + <-done + return + } + time.Sleep(20 * time.Millisecond) + } + cancel() + <-done + t.Fatal("uploader did not reach the expected state before the timeout") +} + +func (r *replacementEnv) newUploader() *worker.Uploader { + return worker.NewUploader(r.env.repos, r.env.cache, r.env.storage, nil, r.env.sm, + cache.EvictionPolicyAfterUpload, 1, 1, 10*time.Millisecond, slog.Default()) +} + +// The whole approved flow: prepare the new service, switch the slot, copy the +// stored content across, and hand over to retirement. +func TestUploader_ReplacementPreparesActivatesAndMigrates(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + row := fixture.authorize(t, "202") + task := fixture.coordinatorTask(t, row.ID) + + final := runWorkerUntilTask(t, fixture.env, fixture.newUploader(), task.ID, 20*time.Second) + if final == nil || final.Status != model.TaskStatusCompleted { + t.Fatalf("coordinator task = %#v, want completed", final) + } + + current, err := fixture.env.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, fixture.bucket.ID, 0) + if err != nil || current == nil || current.ID != row.TargetDataSetID { + t.Fatalf("current binding = %#v err=%v, want the replacement target", current, err) + } + source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) + if err != nil || source == nil || source.IsCurrent || source.Status != model.StorageDataSetStatusDraining { + t.Fatalf("source = %#v err=%v, want draining and not current", source, err) + } + + // The content must actually exist on the new provider, pulled rather than + // re-uploaded from this node. + targetCopy, err := fixture.env.repos.Uploads.GetUploadCopyForDataSet(ctx, fixture.upload.ID, row.TargetDataSetID) + if err != nil || targetCopy == nil || targetCopy.Status != model.StorageUploadCopyStatusCommitted { + t.Fatalf("target copy = %#v err=%v, want committed", targetCopy, err) + } + if fixture.targetCtx.pullCalls.Load() == 0 { + t.Fatal("migration did not pull from a remote replica") + } + if fixture.targetCtx.storeCalls.Load() != 0 { + t.Fatal("migration uploaded from local cache while a readable replica existed") + } + + got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) + if err != nil || got == nil || got.Status != storagereplacement.StatusRetiring { + t.Fatalf("replacement = %#v err=%v, want retiring", got, err) + } + if got.ItemsCopied != got.ItemsTotal || got.ItemsTotal != 1 { + t.Fatalf("progress = %d/%d, want 1/1", got.ItemsCopied, got.ItemsTotal) + } + retire, err := fixture.env.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.RetireTaskKey(row.ID)) + if err != nil || retire == nil { + t.Fatalf("retirement coordinator = %#v err=%v, want it queued", retire, err) + } +} + +// With no readable replica and no retained cache data the item is parked and +// the replacement waits, without burning retry budget. +func TestUploader_ReplacementWaitsWhenNoSourceIsReadable(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + row := fixture.authorize(t, "202") + task := fixture.coordinatorTask(t, row.ID) + + // The retiring provider can no longer serve reads and the cached copy is + // gone, so this content has nowhere to come from. + fixture.targetCtx.pullErr = &synapse.ProviderUnavailableError{Cause: errors.New("source provider unreachable")} + if err := fixture.env.cache.Delete(ctx, fixture.bucket.Name, ".versions/"+fixture.versionID); err != nil { + t.Fatalf("cache delete: %v", err) + } + + fixture.runUploaderUntil(t, func() bool { + got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) + return err == nil && got != nil && got.Status == storagereplacement.StatusWaiting + }, 20*time.Second) + + final, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) + if err != nil || final == nil { + t.Fatalf("GetByID task: %#v err=%v", final, err) + } + if final.RetryCount != 0 { + t.Fatalf("retry count = %d, want waiting to leave the retry budget untouched", final.RetryCount) + } + got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) + if err != nil || got == nil { + t.Fatalf("GetByID: %#v err=%v", got, err) + } + if got.WaitReason == nil || *got.WaitReason != storagereplacement.WaitReasonReadableSource { + t.Fatalf("wait reason = %v, want readable_source", got.WaitReason) + } + if got.LastError != nil { + t.Fatalf("last_error = %v, want waiting to record no failure", *got.LastError) + } + items, err := fixture.env.repos.Replacements.NextExecutableItem(ctx, row.ID) + if err != nil || items == nil || items.Status != storagereplacement.ItemStatusWaitingSource { + t.Fatalf("parked item = %#v err=%v, want waiting_source", items, err) + } +} + +func TestUploader_ReplacementWaitsForFundingBeforeCreatingTheService(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + row := fixture.authorize(t, "202") + task := fixture.coordinatorTask(t, row.ID) + var createCalls atomic.Int32 + fixture.targetCtx.createCalls = &createCalls + fixture.env.storage.PrepareUploadFunc = func(context.Context, uint64, []synapse.UploadContext) (*storage.MultiContextCosts, error) { + return &storage.MultiContextCosts{Ready: false}, nil + } + + fixture.runUploaderUntil(t, func() bool { + got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) + return err == nil && got != nil && got.Status == storagereplacement.StatusWaiting + }, 20*time.Second) + + final, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) + if err != nil || final == nil { + t.Fatalf("GetByID task: %#v err=%v", final, err) + } + if final.RetryCount != 0 { + t.Fatalf("retry count = %d, want funding wait to leave the retry budget untouched", final.RetryCount) + } + got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) + if err != nil || got == nil { + t.Fatalf("GetByID: %#v err=%v", got, err) + } + if got.WaitReason == nil || *got.WaitReason != storagereplacement.WaitReasonFunding { + t.Fatalf("wait reason = %v, want funding", got.WaitReason) + } + if createCalls.Load() != 0 { + t.Fatalf("CreateDataSet calls = %d, want none while funding is not ready", createCalls.Load()) + } +} + +// A coordinator that cannot make progress retries the task, but must leave the +// replacement record describing reality rather than inventing a state. +func TestUploader_ReplacementRetriesWithoutRewritingTheRecord(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + row := fixture.authorize(t, "202") + task := fixture.coordinatorTask(t, row.ID) + + // Put the approved target into a state no preparation step can advance. + if _, err := fixture.env.db.NewUpdate(). + Model((*model.StorageDataSet)(nil)). + Set("status = ?", model.StorageDataSetStatusRetired). + Where("id = ?", row.TargetDataSetID). + Exec(ctx); err != nil { + t.Fatalf("retire target: %v", err) + } + + fixture.runUploaderUntil(t, func() bool { + got, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) + return err == nil && got != nil && (got.RetryCount > 0 || got.Status == model.TaskStatusFailed) + }, 20*time.Second) + + got, err := fixture.env.repos.Replacements.GetByID(ctx, row.ID) + if err != nil || got == nil { + t.Fatalf("GetByID: %#v err=%v", got, err) + } + if got.Status != storagereplacement.StatusPreparingTarget { + t.Fatalf("replacement status = %s, want it untouched at preparing_target", got.Status) + } +} + +// retirementFixture runs migration to completion so retirement has a realistic +// starting point: a drained source and a target that already holds the data. +type retirementFixture struct { + *replacementEnv + replacement *storagereplacement.Replacement + terminator *testutil.MockServiceTerminator + epochs *testutil.MockChainEpochReader + epoch atomic.Int64 + terminated atomic.Int32 +} + +func seedRetirementFixture(t *testing.T) *retirementFixture { + t.Helper() + base := seedReplacementEnv(t) + row := base.authorize(t, "202") + migrate := base.coordinatorTask(t, row.ID) + if final := runWorkerUntilTask(t, base.env, base.newUploader(), migrate.ID, 20*time.Second); final == nil || + final.Status != model.TaskStatusCompleted { + t.Fatalf("migration coordinator = %#v, want completed", final) + } + + fixture := &retirementFixture{replacementEnv: base, replacement: row} + fixture.epoch.Store(1000) + fixture.terminator = &testutil.MockServiceTerminator{ + TerminateServiceFunc: func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { + fixture.terminated.Add(1) + return &synapse.TerminationResult{TxHash: "0xterminate", EndEpoch: fixture.epoch.Load() + 5}, nil + }, + } + fixture.epochs = &testutil.MockChainEpochReader{ + CurrentEpochFunc: func(context.Context) (int64, error) { return fixture.epoch.Load(), nil }, + } + return fixture +} + +func (f *retirementFixture) newCleanupWorker() *worker.StorageCleanupWorker { + return worker.NewStorageCleanupWorker(f.env.repos, f.env.storage, 1, 10*time.Millisecond, slog.Default(), + worker.WithServiceTermination(f.terminator, f.epochs)) +} + +func (f *retirementFixture) runCleanupUntil(t *testing.T, cond func() bool, timeout time.Duration) { + t.Helper() + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan struct{}) + w := f.newCleanupWorker() + go func() { + defer close(done) + _ = w.Run(ctx) + }() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if cond() { + cancel() + <-done + return + } + time.Sleep(20 * time.Millisecond) + } + cancel() + <-done + t.Fatal("storage cleanup did not reach the expected state before the timeout") +} + +// releaseWaitingTasks brings a parked task's schedule forward so a test does not +// have to sleep through the real wait interval. +func (f *retirementFixture) releaseWaitingTasks(t *testing.T) { + t.Helper() + if _, err := f.env.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("scheduled_at = ?", time.Now().Add(-time.Second)). + Where("status = ?", model.TaskStatusWaiting). + Exec(context.Background()); err != nil { + t.Fatalf("release waiting tasks: %v", err) + } +} + +func (f *retirementFixture) reload(t *testing.T) *storagereplacement.Replacement { + t.Helper() + got, err := f.env.repos.Replacements.GetByID(context.Background(), f.replacement.ID) + if err != nil || got == nil { + t.Fatalf("GetByID: %#v err=%v", got, err) + } + return got +} + +// The old service is only treated as gone once the chain has actually reached +// the epoch it ends at. +func TestStorageCleanup_RetirementWaitsForTheTerminationEpoch(t *testing.T) { + fixture := seedRetirementFixture(t) + ctx := context.Background() + + fixture.runCleanupUntil(t, func() bool { + got, err := fixture.env.repos.Replacements.GetByID(ctx, fixture.replacement.ID) + return err == nil && got != nil && got.WaitReason != nil && + *got.WaitReason == storagereplacement.WaitReasonTerminationEpoch + }, 20*time.Second) + + waiting := fixture.reload(t) + if waiting.Status != storagereplacement.StatusWaiting { + t.Fatalf("status = %s, want waiting for the end of term", waiting.Status) + } + if waiting.TerminationEpoch == nil { + t.Fatal("termination epoch was not recorded before waiting for it") + } + source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) + if err != nil || source == nil || source.Status == model.StorageDataSetStatusRetired { + t.Fatalf("source = %#v err=%v, want it not retired before the epoch is reached", source, err) + } + + // Let the chain catch up. The task is parked for the epoch re-check delay, + // so bring it forward rather than sleeping through it. + fixture.epoch.Store(*waiting.TerminationEpoch) + fixture.releaseWaitingTasks(t) + fixture.runCleanupUntil(t, func() bool { + got, err := fixture.env.repos.Replacements.GetByID(ctx, fixture.replacement.ID) + return err == nil && got != nil && got.Status == storagereplacement.StatusCompleted + }, 20*time.Second) + + done := fixture.reload(t) + if done.TerminationObservedAt == nil { + t.Fatal("completed replacement recorded no observation time") + } + source, err = fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) + if err != nil || source == nil || source.Status != model.StorageDataSetStatusRetired || source.IsCurrent { + t.Fatalf("source = %#v err=%v, want retired", source, err) + } + if got := fixture.terminated.Load(); got != 1 { + t.Fatalf("termination calls = %d, want the service terminated exactly once", got) + } +} + +// A version the new provider cannot serve must keep the old one alive. +func TestStorageCleanup_RetirementRefusesWhileCoverageIsIncomplete(t *testing.T) { + fixture := seedRetirementFixture(t) + ctx := context.Background() + + // Take the migrated copy away, leaving the retained version with no home on + // the new provider. + if _, err := fixture.env.db.NewUpdate(). + Model((*model.StorageUploadCopy)(nil)). + Set("status = ?", model.StorageUploadCopyStatusFailed). + Where("storage_data_set_id = ?", fixture.replacement.TargetDataSetID). + Exec(ctx); err != nil { + t.Fatalf("break target coverage: %v", err) + } + + fixture.runCleanupUntil(t, func() bool { + got, err := fixture.env.repos.Replacements.GetByID(ctx, fixture.replacement.ID) + return err == nil && got != nil && got.WaitReason != nil + }, 20*time.Second) + + got := fixture.reload(t) + if got.Status != storagereplacement.StatusWaiting { + t.Fatalf("status = %s, want waiting", got.Status) + } + if *got.WaitReason != storagereplacement.WaitReasonCoverage { + t.Fatalf("wait reason = %s, want coverage", *got.WaitReason) + } + if fixture.terminated.Load() != 0 { + t.Fatal("terminated the old service while a version was still uncovered") + } + source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) + if err != nil || source == nil || source.Status == model.StorageDataSetStatusRetired { + t.Fatalf("source = %#v err=%v, want it kept alive", source, err) + } +} + +// Settling payment debt is the operator's call. The replacement must stop and +// say so instead of retrying, and the two facts must be recorded together. +func TestStorageCleanup_RetirementRaisesAttentionOnPaymentDebt(t *testing.T) { + fixture := seedRetirementFixture(t) + ctx := context.Background() + fixture.terminator.TerminateServiceFunc = func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { + fixture.terminated.Add(1) + return nil, &synapse.TerminationBlockedError{Reason: "payment_debt", Shortfall: big.NewInt(500)} + } + + fixture.runCleanupUntil(t, func() bool { + got, err := fixture.env.repos.Replacements.GetByID(ctx, fixture.replacement.ID) + return err == nil && got != nil && got.Status == storagereplacement.StatusCleanupAttention + }, 20*time.Second) + + got := fixture.reload(t) + if got.LastError == nil || !strings.Contains(*got.LastError, "payment_debt") { + t.Fatalf("last_error = %v, want it to name the payment debt", got.LastError) + } + retire, err := fixture.env.repos.Tasks.GetByIdempotencyKey(ctx, storagereplacement.RetireTaskKey(fixture.replacement.ID)) + if err != nil || retire == nil { + t.Fatalf("retirement task = %#v err=%v", retire, err) + } + if retire.Status != model.TaskStatusFailed { + t.Fatalf("retirement task status = %s, want it stopped rather than retrying", retire.Status) + } + source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) + if err != nil || source == nil || source.Status == model.StorageDataSetStatusRetired { + t.Fatalf("source = %#v err=%v, want it kept alive", source, err) + } +} + +// The abandoned target coordinator existed but was never dispatched: the +// cleanup worker only routed the source retirement stage, so the task fell into +// ordinary replica cleanup, found nothing, and completed without ending the +// paid service it was created to end. +func TestStorageCleanup_AbandonedTargetIsDispatchedAndTerminated(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + first := fixture.authorize(t, "202") + + // Bring the first target's service into existence, then supersede it. + if err := fixture.env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: first.TargetDataSetID, + DataSetID: onChainID(t, "2002"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + second := fixture.authorize(t, "303") + if second.ID == first.ID { + t.Fatal("the later confirmation reused the record") + } + + var terminated atomic.Int64 + terminator := &testutil.MockServiceTerminator{ + TerminateServiceFunc: func(_ context.Context, dataSetID sdktypes.BigInt) (*synapse.TerminationResult, error) { + if dataSetID.String() != "2002" { + t.Errorf("terminated data set %s, want the abandoned target 2002", dataSetID.String()) + } + terminated.Add(1) + return &synapse.TerminationResult{TxHash: "0xabandon", EndEpoch: 1000}, nil + }, + } + epochs := &testutil.MockChainEpochReader{ + CurrentEpochFunc: func(context.Context) (int64, error) { return 5000, nil }, + } + task := fixture.abandonedTargetTask(t, first.ID) + + worker := worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, + slog.Default(), worker.WithServiceTermination(terminator, epochs)) + runWorkerUntilTask(t, fixture.env, worker, task.ID, 20*time.Second) + + if terminated.Load() != 1 { + t.Fatalf("termination calls = %d, want the abandoned service ended exactly once", terminated.Load()) + } + abandoned, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) + if err != nil || abandoned == nil || abandoned.Status != model.StorageDataSetStatusRetired { + t.Fatalf("abandoned target = %#v err=%v, want retired", abandoned, err) + } + source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) + if err != nil || source == nil || !source.IsCurrent || source.Status == model.StorageDataSetStatusRetired { + t.Fatalf("source = %#v err=%v, want it untouched and still current", source, err) + } +} + +func TestStorageCleanup_AbandonedTargetRestartObservesStoredTermination(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + first := fixture.authorize(t, "202") + if err := fixture.env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: first.TargetDataSetID, DataSetID: onChainID(t, "2002"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + fixture.authorize(t, "303") + + var terminationCalls atomic.Int64 + terminator := &testutil.MockServiceTerminator{ + TerminateServiceFunc: func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { + terminationCalls.Add(1) + return &synapse.TerminationResult{TxHash: "0xpersisted", EndEpoch: 5000}, nil + }, + } + var observedEpoch atomic.Int64 + observedEpoch.Store(1000) + epochs := &testutil.MockChainEpochReader{ + CurrentEpochFunc: func(context.Context) (int64, error) { return observedEpoch.Load(), nil }, + } + task := fixture.abandonedTargetTask(t, first.ID) + cleanup := worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, + slog.Default(), worker.WithServiceTermination(terminator, epochs)) + runWorkerUntilTaskStatus(t, fixture.env, cleanup, task.ID, model.TaskStatusWaiting, 20*time.Second) + if terminationCalls.Load() != 1 { + t.Fatalf("termination calls before restart = %d, want 1", terminationCalls.Load()) + } + replacement, err := fixture.env.repos.Replacements.GetByID(ctx, first.ID) + if err != nil || replacement == nil || replacement.AbandonedTerminationEpoch == nil || + *replacement.AbandonedTerminationEpoch != 5000 { + t.Fatalf("stored abandoned termination = %#v err=%v, want epoch 5000", replacement, err) + } + + observedEpoch.Store(6000) + if _, err := fixture.env.db.NewUpdate().Model((*model.Task)(nil)). + Set("scheduled_at = ?", time.Now()).Where("id = ?", task.ID).Exec(ctx); err != nil { + t.Fatalf("release epoch wait: %v", err) + } + cleanup = worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, + slog.Default(), worker.WithServiceTermination(terminator, epochs)) + runWorkerUntilTaskStatus(t, fixture.env, cleanup, task.ID, model.TaskStatusCompleted, 20*time.Second) + if terminationCalls.Load() != 1 { + t.Fatalf("termination calls after restart = %d, want the stored epoch to prevent a second call", terminationCalls.Load()) + } + replacement, err = fixture.env.repos.Replacements.GetByID(ctx, first.ID) + if err != nil || replacement == nil || replacement.AbandonedTerminationObservedAt == nil { + t.Fatalf("observed abandoned termination = %#v err=%v", replacement, err) + } +} + +func TestStorageCleanup_AbandonedTargetWaitsOnPaymentDebtThenRetires(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + first := fixture.authorize(t, "202") + if err := fixture.env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: first.TargetDataSetID, + DataSetID: onChainID(t, "2002"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + if second := fixture.authorize(t, "303"); second.ID == first.ID { + t.Fatal("the later confirmation reused the record") + } + + var blocked atomic.Bool + blocked.Store(true) + var terminated atomic.Int64 + terminator := &testutil.MockServiceTerminator{ + TerminateServiceFunc: func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { + terminated.Add(1) + if blocked.Load() { + return nil, &synapse.TerminationBlockedError{Reason: "payment_debt", Shortfall: big.NewInt(500)} + } + return &synapse.TerminationResult{TxHash: "0xabandon", EndEpoch: 1000}, nil + }, + } + epochs := &testutil.MockChainEpochReader{ + CurrentEpochFunc: func(context.Context) (int64, error) { return 5000, nil }, + } + task := fixture.abandonedTargetTask(t, first.ID) + + cleanup := worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, + slog.Default(), worker.WithServiceTermination(terminator, epochs)) + runWorkerUntilTaskStatus(t, fixture.env, cleanup, task.ID, model.TaskStatusWaiting, 20*time.Second) + + got, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) + if err != nil || got == nil || got.Status != model.TaskStatusWaiting { + t.Fatalf("task = %#v err=%v, want waiting so cleanup can resume after the debt is settled", got, err) + } + abandoned, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) + if err != nil || abandoned == nil || abandoned.Status == model.StorageDataSetStatusRetired { + t.Fatalf("abandoned target = %#v err=%v, want it kept until termination succeeds", abandoned, err) + } + + blocked.Store(false) + if _, err := fixture.env.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("scheduled_at = ?", time.Now()). + Where("id = ?", task.ID). + Exec(ctx); err != nil { + t.Fatalf("release wait schedule: %v", err) + } + cleanup = worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, + slog.Default(), worker.WithServiceTermination(terminator, epochs)) + runWorkerUntilTaskStatus(t, fixture.env, cleanup, task.ID, model.TaskStatusCompleted, 20*time.Second) + + abandoned, err = fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) + if err != nil || abandoned == nil || abandoned.Status != model.StorageDataSetStatusRetired { + t.Fatalf("abandoned target = %#v err=%v, want retired after the debt cleared", abandoned, err) + } + if terminated.Load() < 2 { + t.Fatalf("termination calls = %d, want a blocked attempt and a successful one", terminated.Load()) + } +} + +func TestStorageCleanup_AbandonedTargetWithoutOnChainServiceReleasesTheProvider(t *testing.T) { + for _, tc := range []struct { + name string + maxRetries int + prepare func(*testing.T, *replacementEnv, *storagereplacement.Replacement) + }{ + { + name: "service was never created", + maxRetries: 5, + }, + { + name: "creation observation exhausts", + maxRetries: 1, + prepare: func(t *testing.T, fixture *replacementEnv, first *storagereplacement.Replacement) { + t.Helper() + clientDataSetID := onChainIDPtr(t, "9202") + if err := fixture.env.repos.Uploads.MarkDataSetCreating(context.Background(), repository.MarkDataSetCreatingInput{ + ID: first.TargetDataSetID, + TransactionID: "0xabandoned-create", + StatusURL: "https://provider-202.example/status/create", + ClientDataSetID: clientDataSetID, + }); err != nil { + t.Fatalf("MarkDataSetCreating: %v", err) + } + fixture.targetCtx.waitErr = errors.New("rpc timeout") + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + first := fixture.authorize(t, "202") + if tc.prepare != nil { + tc.prepare(t, fixture, first) + } + second := fixture.authorize(t, "303") + if second.ID == first.ID { + t.Fatal("the later confirmation reused the record") + } + + var terminated atomic.Int64 + terminator := &testutil.MockServiceTerminator{ + TerminateServiceFunc: func(context.Context, sdktypes.BigInt) (*synapse.TerminationResult, error) { + terminated.Add(1) + return &synapse.TerminationResult{TxHash: "0xabandon", EndEpoch: 1000}, nil + }, + } + epochs := &testutil.MockChainEpochReader{ + CurrentEpochFunc: func(context.Context) (int64, error) { return 5000, nil }, + } + task := fixture.abandonedTargetTask(t, first.ID) + if tc.maxRetries != task.MaxRetries { + if _, err := fixture.env.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("max_retries = ?", tc.maxRetries). + Where("id = ?", task.ID). + Exec(ctx); err != nil { + t.Fatalf("set abandoned-target max retries: %v", err) + } + task.MaxRetries = tc.maxRetries + } + + cleanup := worker.NewStorageCleanupWorker(fixture.env.repos, fixture.env.storage, 1, 10*time.Millisecond, + slog.Default(), worker.WithServiceTermination(terminator, epochs)) + final := runWorkerUntilTask(t, fixture.env, cleanup, task.ID, 20*time.Second) + if final == nil { + t.Fatal("abandoned target task was not processed") + } + + if terminated.Load() != 0 { + t.Fatalf("termination calls = %d, want none when no on-chain service exists", terminated.Load()) + } + abandoned, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, first.TargetDataSetID) + if err != nil || abandoned == nil || abandoned.Status != model.StorageDataSetStatusRetired { + t.Fatalf("abandoned target = %#v err=%v, want retired", abandoned, err) + } + if _, _, err := fixture.env.repos.Replacements.Authorize(ctx, repository.AuthorizeReplacementInput{ + BucketID: fixture.bucket.ID, + SourceDataSetID: fixture.source.ID, + SelectionMode: storagereplacement.SelectionModeManual, + TargetProviderID: onChainID(t, "202"), + ClientRequestID: "worker-reuse-202", + MaxRetries: 5, + }); err != nil { + t.Fatalf("reusing the released provider: %v", err) + } + }) + } +} + +// Parked work is only reached once nothing executable is left. Re-queueing the +// coordinator there spins it against the same item at queue-tail rate and +// starves ordinary uploads, so it has to wait for the source instead. +func TestUploader_OnlyParkedItemsPutTheCoordinatorIntoAWait(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + replacement := fixture.authorize(t, "202") + if err := fixture.env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: replacement.TargetDataSetID, + DataSetID: onChainID(t, "2002"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + + // The retiring generation is still writing this content, so there is + // nothing to copy from yet and nothing else to do. + if _, err := fixture.env.db.NewUpdate(). + Model((*model.StorageUploadCopy)(nil)). + Set("status = ?", model.StorageUploadCopyStatusPending). + Where("upload_id = ? AND storage_data_set_id = ?", fixture.upload.ID, fixture.source.ID). + Exec(ctx); err != nil { + t.Fatalf("put the source copy back in flight: %v", err) + } + + task := fixture.coordinatorTask(t, replacement.ID) + fixture.runUploaderUntil(t, func() bool { + row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) + return err == nil && row != nil && row.Status == storagereplacement.StatusWaiting + }, 20*time.Second) + + row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) + if err != nil || row == nil { + t.Fatalf("GetByID = %#v err=%v", row, err) + } + if row.WaitReason == nil || *row.WaitReason != storagereplacement.WaitReasonReadableSource { + t.Fatalf("wait reason = %v, want readable_source", row.WaitReason) + } + settled, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) + if err != nil || settled == nil { + t.Fatalf("GetByID task = %#v err=%v", settled, err) + } + // Waiting must not burn the retry budget; the item is rechecked next tick. + if settled.RetryCount != 0 { + t.Fatalf("retry count = %d, want waiting to leave the budget untouched", settled.RetryCount) + } + if settled.ScheduledAt.Before(time.Now()) { + t.Fatal("the coordinator was re-queued immediately instead of waiting for the source") + } +} + +// A replacement has to open its own paid service. If the chosen provider still +// runs a live service for this bucket, the SDK hands back a context already +// bound to it -- and attaching there would leave the replacement paying for, +// and later retiring, a service it does not own. +// +// The fixture's target context is deliberately unbound, so this branch was +// never reached by any existing test. +func TestUploader_ReplacementNeverAttachesToAnExistingService(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + + // The target provider already runs a data set carrying this bucket's + // metadata: a generation released locally without being terminated on chain. + var createCalls atomic.Int32 + existing := readyFakeUploadContext( + sdktypes.NewBigInt(202), sdktypes.NewBigInt(9002), sdktypes.NewBigInt(3002), testCID(t)) + existing.createCalls = &createCalls + fixture.env.storage.CreateContextFunc = func(_ context.Context, opts *storage.CreateContextOptions) (synapse.UploadContext, error) { + if createContextProviderIDEqual(opts, sdktypes.NewBigInt(202)) { + return existing, nil + } + return fixture.sourceCtx, nil + } + + replacement := fixture.authorize(t, "202") + task := fixture.coordinatorTask(t, replacement.ID) + fixture.runUploaderUntil(t, func() bool { + row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) + return err == nil && row != nil && row.Status == storagereplacement.StatusFailed + }, 20*time.Second) + + if calls := createCalls.Load(); calls != 0 { + t.Fatalf("CreateDataSet calls = %d, want the replacement to refuse rather than reuse", calls) + } + target, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) + if err != nil || target == nil { + t.Fatalf("GetDataSetBindingByID = %#v err=%v", target, err) + } + if target.DataSetID != nil { + t.Fatalf("target adopted data set %s, want no service recorded", target.DataSetID.String()) + } + if target.Status == model.StorageDataSetStatusReady { + t.Fatal("target was marked ready without a service of its own") + } + + row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) + if err != nil || row == nil || row.LastError == nil { + t.Fatalf("replacement = %#v err=%v, want a recorded reason", row, err) + } + // The operator has to be able to act on it: name the provider and the service. + if !strings.Contains(*row.LastError, "202") || !strings.Contains(*row.LastError, "9002") { + t.Fatalf("last error = %q, want the provider and data set named", *row.LastError) + } + + // The source still owns the replica, so a new confirmation is the way out. + source, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, fixture.source.ID) + if err != nil || source == nil || !source.IsCurrent { + t.Fatalf("source = %#v err=%v, want it to still own the replica", source, err) + } + // Retrying cannot make a taken provider free, so the budget is not spent. + settled, err := fixture.env.repos.Tasks.GetByID(ctx, task.ID) + if err != nil || settled == nil { + t.Fatalf("GetByID task = %#v err=%v", settled, err) + } + if settled.RetryCount != 0 { + t.Fatalf("retry count = %d, want no retries spent on an unusable provider", settled.RetryCount) + } + if settled.Status != model.TaskStatusFailed { + t.Fatalf("task status = %s, want failed", settled.Status) + } +} + +// Refusing a bound context must not refuse this replacement's own service. Once +// a creation has been submitted, the recorded transaction is what identifies +// the service -- and by then the SDK resolves a bound context for it, which is +// exactly the state a crash between submitting and recording leaves behind. +func TestUploader_ReplacementResumesItsOwnSubmittedCreation(t *testing.T) { + fixture := seedReplacementEnv(t) + ctx := context.Background() + replacement := fixture.authorize(t, "202") + + if err := fixture.env.repos.Uploads.MarkDataSetCreating(ctx, repository.MarkDataSetCreatingInput{ + ID: replacement.TargetDataSetID, + TransactionID: "0xcreate2002", + StatusURL: "https://provider-202.example/status/create", + ClientDataSetID: onChainIDPtr(t, "12002"), + }); err != nil { + t.Fatalf("MarkDataSetCreating: %v", err) + } + // The service this replacement created is now visible to the resolver. + bound := readyFakeUploadContext( + sdktypes.NewBigInt(202), sdktypes.NewBigInt(2002), sdktypes.NewBigInt(3002), testCID(t)) + fixture.env.storage.CreateContextFunc = func(_ context.Context, opts *storage.CreateContextOptions) (synapse.UploadContext, error) { + if createContextProviderIDEqual(opts, sdktypes.NewBigInt(202)) || + createContextDataSetIDEqual(opts, sdktypes.NewBigInt(2002)) { + return bound, nil + } + return fixture.sourceCtx, nil + } + + fixture.runUploaderUntil(t, func() bool { + target, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) + return err == nil && target != nil && target.Status == model.StorageDataSetStatusReady + }, 20*time.Second) + + target, err := fixture.env.repos.Uploads.GetDataSetBindingByID(ctx, replacement.TargetDataSetID) + if err != nil || target == nil || target.DataSetID == nil { + t.Fatalf("target = %#v err=%v, want the submitted service recorded", target, err) + } + if target.DataSetID.String() != "2002" { + t.Fatalf("target recorded data set %s, want the one it submitted", target.DataSetID.String()) + } + row, err := fixture.env.repos.Replacements.GetByID(ctx, replacement.ID) + if err != nil || row == nil { + t.Fatalf("GetByID = %#v err=%v", row, err) + } + if row.Status == storagereplacement.StatusFailed { + t.Fatalf("replacement failed on its own submitted service: %v", row.LastError) + } +} diff --git a/internal/worker/uploader_replica_repair.go b/internal/worker/uploader_replica_repair.go index 6ec930d..6e53713 100644 --- a/internal/worker/uploader_replica_repair.go +++ b/internal/worker/uploader_replica_repair.go @@ -71,6 +71,16 @@ func ensureReplicaRepairTask(ctx context.Context, repos *repository.Repositories if binding.Status != model.StorageDataSetStatusUnavailable && binding.Status != model.StorageDataSetStatusReady { return false, nil } + // An approved replacement already owns this generation's remaining work. + // Repairing it in place would fight the migration, so recovery stands down + // until the replacement finishes or terminally fails. + replacing, err := repos.Replacements.HasInProgressForDataSet(ctx, binding.ID) + if err != nil { + return false, fmt.Errorf("check provider replacement for data set %d: %w", binding.ID, err) + } + if replacing { + return false, nil + } copyRow, err := repos.Uploads.NextFinalizableCopyForDataSet(ctx, binding.ID) if err != nil { return false, fmt.Errorf("select replica finalization copy for data set %d: %w", binding.ID, err) @@ -265,7 +275,7 @@ func (u *Uploader) repairReplicaCopy( } pieceCIDString = sourceCopy.PieceCID pieces := []storage.PieceInput{{PieceCID: pieceCID}} - extraData, encodedExtra, err := u.extraDataForCopy(ctx, storageCtx, upload.ID, copyRow.CopyIndex, pieces) + extraData, encodedExtra, err := u.extraDataForCopy(ctx, storageCtx, copyRow, pieces) if err != nil { return err } @@ -313,8 +323,7 @@ func (u *Uploader) repairReplicaCopy( _, extraHex, err = u.extraDataForCopy( ctx, storageCtx, - upload.ID, - copyRow.CopyIndex, + copyRow, []storage.PieceInput{{PieceCID: pieceCID}}, ) if err != nil { @@ -323,6 +332,7 @@ func (u *Uploader) repairReplicaCopy( } if err := u.repos.Uploads.MarkUploadCopyPieceReady(ctx, repository.MarkUploadCopyPieceReadyInput{ StorageUploadCopyID: copyRow.ID, + RequireEligibleCopy: true, UploadID: upload.ID, CopyIndex: copyRow.CopyIndex, PieceCID: pieceCIDString, @@ -332,6 +342,7 @@ func (u *Uploader) repairReplicaCopy( } if err := u.repos.Uploads.MarkUploadCopyCommitting(ctx, repository.MarkUploadCopyCommittingInput{ StorageUploadCopyID: copyRow.ID, + RequireEligibleCopy: true, UploadID: upload.ID, CopyIndex: copyRow.CopyIndex, CommitExtraDataHex: extraHex, @@ -362,6 +373,7 @@ func (u *Uploader) repairReplicaCopy( pieceID := idtypes.OnChainIDFromSDK(result.PieceIDs[0]) if err := u.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ StorageUploadCopyID: copyRow.ID, + RequireEligibleCopy: true, UploadID: upload.ID, CopyIndex: copyRow.CopyIndex, PieceCID: pieceCIDString, @@ -462,13 +474,13 @@ func (u *Uploader) commitReplicaRepairCopy( transactionID := *copyRow.CommitTransactionID result, err := u.waitForSubmittedCommit(ctx, storageCtx, binding, transactionID, len(pieces)) if errors.Is(err, errCommitRejected) { - if resetErr := u.resetRejectedSubmittedCommit(ctx, upload.ID, copyRow.CopyIndex, transactionID, err); resetErr != nil { + if resetErr := u.resetRejectedSubmittedCommit(ctx, copyRow.ID, upload.ID, copyRow.CopyIndex, transactionID, err); resetErr != nil { return nil, fmt.Errorf("reset rejected replica repair commit: %w", resetErr) } } return result, err } - extraData, extraHex, err := u.extraDataForCopy(ctx, storageCtx, upload.ID, copyRow.CopyIndex, pieces) + extraData, extraHex, err := u.extraDataForCopy(ctx, storageCtx, copyRow, pieces) if err != nil { return nil, err } @@ -481,6 +493,7 @@ func (u *Uploader) commitReplicaRepairCopy( submittedTx = txHash submitErr = u.repos.Uploads.MarkUploadCopyCommitting(ctx, repository.MarkUploadCopyCommittingInput{ StorageUploadCopyID: copyRow.ID, + RequireEligibleCopy: true, UploadID: upload.ID, CopyIndex: copyRow.CopyIndex, CommitExtraDataHex: extraHex, @@ -492,7 +505,7 @@ func (u *Uploader) commitReplicaRepairCopy( return nil, fmt.Errorf("save replica repair commit submission: %w", submitErr) } if errors.Is(err, errCommitRejected) && submittedTx != "" { - if resetErr := u.resetRejectedSubmittedCommit(ctx, upload.ID, copyRow.CopyIndex, submittedTx, err); resetErr != nil { + if resetErr := u.resetRejectedSubmittedCommit(ctx, copyRow.ID, upload.ID, copyRow.CopyIndex, submittedTx, err); resetErr != nil { return nil, fmt.Errorf("reset rejected replica repair commit: %w", resetErr) } } diff --git a/internal/worker/uploader_test.go b/internal/worker/uploader_test.go index 225b640..6945bfb 100644 --- a/internal/worker/uploader_test.go +++ b/internal/worker/uploader_test.go @@ -3133,7 +3133,7 @@ func TestUploader_SubmittedPeerMismatchedStatusRemainsRecoverableAfterExhaustion if err != nil || !copyCommitSubmittedForTest(copyRow) || *copyRow.CommitTransactionID != fakeSubmittedCommitTxHash { t.Fatalf("peer copy after mismatched status = %#v err=%v, want recoverable submitted commit", copyRow, err) } - if err := env.repos.Uploads.MarkUploadCopyFailed(ctx, fixture.upload.ID, 1, "late generic failure"); !errors.Is(err, repository.ErrConflict) { + if err := env.repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: fixture.upload.ID, CopyIndex: 1, LastError: "late generic failure"}); !errors.Is(err, repository.ErrConflict) { t.Fatalf("MarkUploadCopyFailed submitted peer error = %v, want conflict", err) } version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) @@ -4592,7 +4592,7 @@ func TestUploader_RepairPreparePreservesAssignedPeerSlots(t *testing.T) { name: "failed copy", mark: func(ctx context.Context, t *testing.T, env *testWorkerEnv, fixture readableUploadWithPendingPeerFixture) { t.Helper() - if err := env.repos.Uploads.MarkUploadCopyFailed(ctx, fixture.upload.ID, 1, "peer pull: provider failed"); err != nil { + if err := env.repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: fixture.upload.ID, CopyIndex: 1, LastError: "peer pull: provider failed"}); err != nil { t.Fatalf("MarkUploadCopyFailed: %v", err) } }, @@ -5021,7 +5021,7 @@ func TestUploader_EvidenceFreeDataSetCandidateIsNotSharedAcrossUploads(t *testin if err := env.repos.Uploads.MarkDataSetFailed(ctx, candidate.ID, "creation rejected"); err != nil { t.Fatalf("MarkDataSetFailed(candidate): %v", err) } - if err := env.repos.Uploads.MarkUploadCopyFailed(ctx, firstUpload.ID, 0, "creation rejected"); err != nil { + if err := env.repos.Uploads.MarkUploadCopyFailed(ctx, repository.MarkUploadCopyFailedInput{UploadID: firstUpload.ID, CopyIndex: 0, LastError: "creation rejected"}); err != nil { t.Fatalf("MarkUploadCopyFailed(first): %v", err) } discarded, err := env.repos.Uploads.DiscardFailedDataSetCandidate(ctx, firstUpload.ID, 0, candidate.ID) diff --git a/tests/system/golden_test.go b/tests/system/golden_test.go index 66474df..c777d7d 100644 --- a/tests/system/golden_test.go +++ b/tests/system/golden_test.go @@ -144,7 +144,10 @@ func TestSystemGoldenPath(t *testing.T) { if err != nil { return providerRaw + "\n" + dataSetRaw, false, err } - return providerRaw + "\n" + dataSetRaw, providers.Summary.Available == 3 && dataSets.Summary.Available == 3, nil + // Every configured provider is observed; the bucket's three replicas + // account for the data sets. + return providerRaw + "\n" + dataSetRaw, + providers.Summary.Available == systemtest.MemoryFilecoinProviders && dataSets.Summary.Available == 3, nil }) closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) diff --git a/tests/system/provider_replacement_test.go b/tests/system/provider_replacement_test.go new file mode 100644 index 0000000..5dd7a4f --- /dev/null +++ b/tests/system/provider_replacement_test.go @@ -0,0 +1,213 @@ +//go:build systemtest + +package system_test + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "log/slog" + "os" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + awss3 "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/strahe/synaps3/internal/systemtest" + "github.com/strahe/synaps3/tests/testutil/e2e" +) + +type replacementDataSet struct { + ID int64 `json:"id"` + CopyIndex int `json:"copy_index"` + Generation int `json:"generation"` + IsCurrent bool `json:"is_current"` + Replaceable bool `json:"replaceable"` + ProviderID string `json:"provider_id"` + Status string `json:"status"` +} + +type replacementSummary struct { + ID int64 `json:"id"` + Status string `json:"status"` + WaitReason string `json:"wait_reason"` + ItemsTotal int `json:"items_total"` + ItemsCopied int `json:"items_copied"` + Source struct { + ProviderID string `json:"provider_id"` + Status string `json:"status"` + } `json:"source"` + Target struct { + ProviderID string `json:"provider_id"` + IsCurrent bool `json:"is_current"` + } `json:"target"` +} + +type bucketReplacementView struct { + DataSets []replacementDataSet `json:"data_sets"` + Replacements []replacementSummary `json:"replacements"` +} + +// A whole approved replacement, driven only through the public Admin and S3 +// surfaces: confirm, migrate, retire, with reads working throughout. +func TestSystemProviderReplacement(t *testing.T) { + logger := slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelWarn})) + harness, err := systemtest.NewHarness(t.Context(), logger) + if err != nil { + t.Fatalf("NewHarness: %v", err) + } + t.Cleanup(func() { + closeCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := harness.Close(closeCtx); err != nil { + t.Errorf("Close harness: %v", err) + } + }) + + admin := e2e.NewAdminClient(t, harness.AdminURL) + admin.Login(t, t.Context(), systemtest.AdminUsername, systemtest.AdminPassword) + credentials := admin.CreateS3User(t, t.Context()) + s3Client := e2e.NewUnixSocketS3Client(harness.S3SocketPath(), credentials.AccessKey, credentials.SecretKey) + + bucket, key := "system-replacement", "objects/replaceable.bin" + if _, err := s3Client.CreateBucket(t.Context(), &awss3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { + t.Fatalf("CreateBucket: %v", err) + } + content := bytes.Repeat([]byte("synaps3-provider-replacement\n"), 2000) + checksum := sha256.Sum256(content) + if _, err := s3Client.PutObject(t.Context(), &awss3.PutObjectInput{ + Bucket: aws.String(bucket), Key: aws.String(key), Body: bytes.NewReader(content), + }); err != nil { + t.Fatalf("PutObject: %v", err) + } + e2e.AssertS3Object(t, t.Context(), s3Client, bucket, key, content, checksum) + + // Wait until the object is durably stored on its replicas. + view := e2e.Eventually(t, t.Context(), 30*time.Second, "bucket replicas to become ready", + func(ctx context.Context) (bucketReplacementView, bool, error) { + var got bucketReplacementView + if _, err := admin.GetJSON(ctx, "/api/v1/buckets/"+bucket, &got); err != nil { + return got, false, err + } + ready := 0 + for _, set := range got.DataSets { + if set.IsCurrent && set.Status == "ready" { + ready++ + } + } + return got, ready > 0, nil + }) + + var source replacementDataSet + for _, set := range view.DataSets { + if set.Replaceable { + source = set + break + } + } + if source.ID == 0 { + t.Fatalf("no replaceable data set in %+v", view.DataSets) + } + if source.Generation != 1 { + t.Fatalf("initial generation = %d, want 1", source.Generation) + } + + // One confirmation authorizes the new service, the switch, the migration, + // and retirement of the old service. + var confirmed replacementSummary + admin.PostJSON(t, t.Context(), + fmt.Sprintf("/api/v1/buckets/%s/data-sets/%d/replacement", bucket, source.ID), + map[string]string{ + "mode": "automatic", + "client_request_id": "system-provider-replacement", + }, &confirmed) + if confirmed.Status != "preparing_target" { + t.Fatalf("confirmed status = %s, want preparing_target", confirmed.Status) + } + if confirmed.Target.ProviderID == source.ProviderID { + t.Fatalf("automatic selection reused the retiring provider %s", source.ProviderID) + } + + // Reads must keep working while the replacement runs. + e2e.AssertS3Object(t, t.Context(), s3Client, bucket, key, content, checksum) + + // A coordinator that hits any dependency wait parks for uploadDependencyWaitDelay, + // which is a minute. The budget has to clear one of those plus the work either + // side of it, or a single transient wait fails the test rather than delaying it. + final := e2e.Eventually(t, t.Context(), 150*time.Second, "provider replacement to finish", + func(ctx context.Context) (bucketReplacementView, bool, error) { + var got bucketReplacementView + if _, err := admin.GetJSON(ctx, "/api/v1/buckets/"+bucket, &got); err != nil { + return got, false, err + } + completed := false + for _, row := range got.Replacements { + if row.ID == confirmed.ID { + completed = row.Status == "completed" + break + } + } + if !completed { + return got, false, nil + } + sourceRetired, targetCurrent := false, false + for _, set := range got.DataSets { + sourceRetired = sourceRetired || set.ID == source.ID && set.Status == "retired" + targetCurrent = targetCurrent || set.CopyIndex == source.CopyIndex && set.IsCurrent && set.ProviderID == confirmed.Target.ProviderID + } + return got, sourceRetired && targetCurrent, nil + }) + + var completed replacementSummary + for _, row := range final.Replacements { + if row.ID == confirmed.ID { + completed = row + } + } + if completed.ItemsTotal == 0 || completed.ItemsCopied != completed.ItemsTotal { + t.Fatalf("migration progress = %d/%d, want everything copied", completed.ItemsCopied, completed.ItemsTotal) + } + if completed.Source.Status != "retired" { + t.Fatalf("source status = %s, want retired", completed.Source.Status) + } + if !completed.Target.IsCurrent { + t.Fatal("target did not take over the replica slot") + } + + // The slot now has two generations: the retired original and its successor. + var current, retired *replacementDataSet + for i := range final.DataSets { + set := &final.DataSets[i] + if set.CopyIndex != source.CopyIndex { + continue + } + if set.IsCurrent { + current = set + } else if set.Status == "retired" { + retired = set + } + } + if current == nil || retired == nil { + t.Fatalf("slot generations = %+v, want one current and one retired", final.DataSets) + } + if current.Generation <= retired.Generation { + t.Fatalf("generations = current:%d retired:%d, want the successor to be newer", current.Generation, retired.Generation) + } + if retired.Replaceable { + t.Fatal("a retired generation is still offered for replacement") + } + + // The object is still readable from the new provider, and new writes land + // there too. + e2e.AssertS3Object(t, t.Context(), s3Client, bucket, key, content, checksum) + newKey := "objects/after-replacement.bin" + newContent := bytes.Repeat([]byte("after-replacement\n"), 500) + newChecksum := sha256.Sum256(newContent) + if _, err := s3Client.PutObject(t.Context(), &awss3.PutObjectInput{ + Bucket: aws.String(bucket), Key: aws.String(newKey), Body: bytes.NewReader(newContent), + }); err != nil { + t.Fatalf("PutObject after replacement: %v", err) + } + e2e.AssertS3Object(t, t.Context(), s3Client, bucket, newKey, newContent, newChecksum) +} diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index 1306baa..7a48ddf 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -208,6 +208,10 @@ export interface StorageDataSetSummary { bucket_id: number bucket_name?: string copy_index: number + generation: number + is_current: boolean + /** Only the generation that receives writes can be replaced. */ + replaceable: boolean provider_id: string provider_identity?: ProviderIdentity data_set_id?: string @@ -290,6 +294,48 @@ export interface BucketDetail extends BucketItem { versioning_status: string versioning_enforced: boolean data_sets: StorageDataSetSummary[] + /** Full replacement history for this bucket, newest first. */ + replacements: ProviderReplacement[] +} + +export type ProviderReplacementStatus = + | 'preparing_target' + | 'migrating' + | 'waiting' + | 'retiring' + | 'cleanup_attention' + | 'failed' + | 'completed' + | 'superseded' + +export interface ProviderReplacementDataSet { + id: number + generation: number + is_current: boolean + status: string + provider_id: string + data_set_id: string | null + provider_identity?: ProviderIdentity +} + +export interface ProviderReplacement { + id: number + bucket_name: string + copy_index: number + status: ProviderReplacementStatus + wait_reason?: string + wait_message?: string + failure_reason?: 'target_in_use' + selection_mode: 'automatic' | 'manual' + source: ProviderReplacementDataSet + target: ProviderReplacementDataSet + /** Progress counts unique stored content, not object versions. */ + items_total: number + items_copied: number + last_error: string | null + termination_epoch: number | null + created_at: string + updated_at: string } export interface BucketMutationResponse { @@ -496,6 +542,15 @@ export interface ObjectStatusDetail { export type ObjectUploadCopyStatus = 'pending' | 'piece_ready' | 'committing' | 'committed' | 'failed' +/** One provider offered for a replacement, with why it cannot be chosen. */ +export interface ReplacementProviderCandidate { + provider_id: string + eligible: boolean + ineligible_reason?: string + previously_used: boolean + provider_identity?: ProviderIdentity +} + export interface ProviderIdentity { registry_provider_id: string name?: string @@ -554,6 +609,7 @@ export interface TaskItem { copy_index?: number ref_type: string ref_id: number + bucket_name?: string ref_version_id: string status: string progress?: UploadTransferProgress @@ -751,6 +807,7 @@ export interface TaskRefDetail { ref_type: string ref_id: number ref_version_id: string + bucket_name?: string object: TaskRefObjectDetail | null storage_cleanup?: TaskStorageCleanupDetail } @@ -1072,6 +1129,21 @@ export const api = { method: 'PUT', body: JSON.stringify(policy), }), + startProviderReplacement: ( + name: string, + dataSetID: number, + request: { mode: 'automatic' | 'manual'; provider_id?: string; client_request_id: string } + ) => + fetchJSON(`/buckets/${encodeURIComponent(name)}/data-sets/${dataSetID}/replacement`, { + method: 'POST', + body: JSON.stringify(request), + }), + retryProviderReplacement: (replacementID: number) => + fetchJSON(`/storage-replacements/${replacementID}/retry`, { method: 'POST' }), + getReplacementProviders: (name: string, dataSetID: number) => + fetchJSON<{ providers: ReplacementProviderCandidate[] }>( + `/buckets/${encodeURIComponent(name)}/data-sets/${dataSetID}/replacement/providers` + ), getBucketObjects: (name: string, params: { prefix?: string; delimiter?: string; after?: string; limit?: number }) => { const sp = new URLSearchParams() if (params.prefix) sp.set('prefix', params.prefix) diff --git a/ui/src/components/app/DangerActionAlertDialog.tsx b/ui/src/components/app/DangerActionAlertDialog.tsx index 1fd6d1c..4128b31 100644 --- a/ui/src/components/app/DangerActionAlertDialog.tsx +++ b/ui/src/components/app/DangerActionAlertDialog.tsx @@ -23,6 +23,7 @@ export interface DangerActionAlertDialogProps { confirmLabel: string onConfirm: () => void pending?: boolean + confirmDisabled?: boolean error?: string | null typedTarget?: string typedTargetLabel?: string @@ -38,6 +39,7 @@ export function DangerActionAlertDialog({ confirmLabel, onConfirm, pending = false, + confirmDisabled = false, error, typedTarget, typedTargetLabel = 'Type to confirm', @@ -50,7 +52,7 @@ export function DangerActionAlertDialog({ const needsTypedConfirmation = typedTarget !== undefined const typedConfirmationValid = !needsTypedConfirmation || (typedTarget.length > 0 && confirmationMatches(confirmInput, typedTarget)) - const canConfirm = !pending && typedConfirmationValid + const canConfirm = !pending && !confirmDisabled && typedConfirmationValid useEffect(() => { if (!open) setConfirmInput('') diff --git a/ui/src/components/app/ProviderSelect.tsx b/ui/src/components/app/ProviderSelect.tsx new file mode 100644 index 0000000..c27b7b7 --- /dev/null +++ b/ui/src/components/app/ProviderSelect.tsx @@ -0,0 +1,124 @@ +import { Check, ChevronsUpDown, Search } from 'lucide-react' +import { useMemo, useState } from 'react' + +import type { ReplacementProviderCandidate } from '@/api/client' +import { Button } from '@/components/ui/button' +import { Input } from '@/components/ui/input' +import { Popover, PopoverContent, PopoverTrigger } from '@/components/ui/popover' +import { + providerCandidateDisabledReason, + providerCandidateLabel, + providerCandidateMatches, + providerCandidateNote, + providerCandidateRegistryLine, +} from '@/lib/provider-replacement' +import { cn } from '@/lib/utils' + +interface ProviderSelectProps { + id?: string + candidates: ReplacementProviderCandidate[] + value: string + onChange: (providerID: string) => void + disabled?: boolean +} + +/** + * A provider chooser that searches by name or registry ID. Providers that + * cannot take the replica stay in the list, disabled and labelled with the + * reason, so an operator looking for one they expected sees why rather than + * finding it missing. + */ +export function ProviderSelect({ id, candidates, value, onChange, disabled }: ProviderSelectProps) { + const [open, setOpen] = useState(false) + const [query, setQuery] = useState('') + + const matches = useMemo( + () => candidates.filter((candidate) => providerCandidateMatches(candidate, query)), + [candidates, query] + ) + const selected = candidates.find((candidate) => candidate.provider_id === value) + + return ( + { + setOpen(next) + if (!next) setQuery('') + }} + > + + + + +
+ + setQuery(event.target.value)} + placeholder="Search by name or ID" + autoComplete="off" + className="h-8 border-0 px-0 shadow-none focus-visible:ring-0" + /> +
+
+ {matches.length === 0 && ( +

No provider matches that search.

+ )} + {matches.map((candidate) => { + const disabledReason = providerCandidateDisabledReason(candidate) + const note = providerCandidateNote(candidate) + const registryLine = providerCandidateRegistryLine(candidate) + return ( + + ) + })} +
+ + + ) +} diff --git a/ui/src/hooks/queries.ts b/ui/src/hooks/queries.ts index 604c734..5bdbb58 100644 --- a/ui/src/hooks/queries.ts +++ b/ui/src/hooks/queries.ts @@ -163,6 +163,63 @@ export function useUpdateBucketCopyPolicy() { }) } +/** + * The provider chooser, loaded only while the operator is actually choosing. + * It reaches the registry, so it is not worth fetching for a dialog that is + * closed or set to pick automatically. + */ +export function useReplacementProviderCandidates(bucket: string, dataSetID: number | null, enabled: boolean) { + return useQuery({ + queryKey: ['replacement-providers', bucket, dataSetID], + queryFn: () => api.getReplacementProviders(bucket, dataSetID as number), + enabled: enabled && dataSetID !== null, + staleTime: 30_000, + }) +} + +export function useStartProviderReplacement() { + const qc = useQueryClient() + + return useMutation({ + mutationFn: ({ + bucket, + dataSetID, + mode, + providerID, + clientRequestID, + }: { + bucket: string + dataSetID: number + mode: 'automatic' | 'manual' + providerID?: string + clientRequestID: string + }) => + api.startProviderReplacement(bucket, dataSetID, { + mode, + client_request_id: clientRequestID, + ...(mode === 'manual' && providerID ? { provider_id: providerID } : {}), + }), + onSuccess: (_, variables) => { + qc.invalidateQueries({ queryKey: ['bucket', variables.bucket] }) + qc.invalidateQueries({ queryKey: ['buckets'] }) + qc.invalidateQueries({ queryKey: ['tasks'] }) + }, + }) +} + +export function useRetryProviderReplacement() { + const qc = useQueryClient() + + return useMutation({ + mutationFn: ({ replacementID }: { bucket: string; replacementID: number }) => + api.retryProviderReplacement(replacementID), + onSuccess: (_, variables) => { + qc.invalidateQueries({ queryKey: ['bucket', variables.bucket] }) + qc.invalidateQueries({ queryKey: ['tasks'] }) + }, + }) +} + export function useDeleteBucketObject() { const qc = useQueryClient() diff --git a/ui/src/lib/bucket-route-search.ts b/ui/src/lib/bucket-route-search.ts index a21fe67..013ef46 100644 --- a/ui/src/lib/bucket-route-search.ts +++ b/ui/src/lib/bucket-route-search.ts @@ -10,6 +10,7 @@ export type BucketRouteSearch = { risk_created_at_marker?: string risk_stale_before?: string view?: 'objects' | 'deleted' | 'storage-risk' + details?: 'storage' } export function normalizeBucketRouteSearch(search: Record): BucketRouteSearch { @@ -25,6 +26,7 @@ export function normalizeBucketRouteSearch(search: Record): Buc risk_created_at_marker: normalizeSearchString(search.risk_created_at_marker), risk_stale_before: normalizeSearchString(search.risk_stale_before), view: search.view === 'deleted' || search.view === 'storage-risk' ? search.view : undefined, + details: search.details === 'storage' ? 'storage' : undefined, } } diff --git a/ui/src/lib/provider-replacement.ts b/ui/src/lib/provider-replacement.ts new file mode 100644 index 0000000..0e538c9 --- /dev/null +++ b/ui/src/lib/provider-replacement.ts @@ -0,0 +1,283 @@ +import type { StatusTone } from '@/components/app/StatusBadge' +import type { + ProviderReplacement, + ProviderReplacementStatus, + ReplacementProviderCandidate, + StorageDataSetSummary, +} from '../api/client' +import { APIError } from '../api/client.ts' +import { formatBytes, formatNumber } from './utils.ts' + +const replacementStatusLabels: Record = { + preparing_target: 'Preparing new provider', + migrating: 'Copying data', + waiting: 'Waiting', + retiring: 'Retiring old provider', + cleanup_attention: 'Could not end the old provider', + failed: 'Could not finish the replacement', + completed: 'Completed', + superseded: 'Replaced by a newer request', +} + +export function replacementStatusLabel(status: ProviderReplacementStatus) { + return replacementStatusLabels[status] ?? 'Unknown' +} + +export function replacementStatusTone(status: ProviderReplacementStatus): StatusTone { + switch (status) { + case 'completed': + return 'success' + case 'failed': + case 'cleanup_attention': + return 'danger' + case 'waiting': + return 'warning' + case 'superseded': + return 'neutral' + default: + return 'info' + } +} + +/** A replacement is still doing something on its own. */ +export function replacementInProgress(replacement: ProviderReplacement) { + return ['preparing_target', 'migrating', 'waiting', 'retiring'].includes(replacement.status) +} + +/** Only these states are resumed by the operator; the rest resume themselves. */ +export function replacementRetryable(replacement: ProviderReplacement) { + if (replacement.failure_reason === 'target_in_use') return false + return replacement.status === 'failed' || replacement.status === 'cleanup_attention' +} + +/** A replacement worth showing above the table: still running, or waiting for a decision. */ +export function activeReplacements(replacements: ProviderReplacement[] | undefined) { + if (!replacements?.length) return [] + return replacements.filter( + (row) => replacementInProgress(row) || row.status === 'failed' || row.status === 'cleanup_attention' + ) +} + +export function replacementForDataSet(replacements: ProviderReplacement[] | undefined, dataSetID: number) { + if (!replacements?.length) return undefined + return replacements.find( + (row) => + (row.source.id === dataSetID || row.target.id === dataSetID) && + (replacementInProgress(row) || row.status === 'failed' || row.status === 'cleanup_attention') + ) +} + +/** + * A replacement that is still progressing owns the replica. One that stopped and + * is waiting for the operator does not: choosing a different provider is how you + * move on from a target that will not work, and that needs a new confirmation + * rather than a retry of the old one. + * + * That only applies to the replica being replaced. A provider that is itself a + * replacement's target cannot be replaced again while that replacement is + * unfinished, because the original source still owes it the data it has not + * migrated yet. Offering the action there would only produce a conflict. + */ +export function dataSetReplaceable(dataSet: StorageDataSetSummary, replacements: ProviderReplacement[] | undefined) { + if (!dataSet.replaceable) return false + const active = replacementForDataSet(replacements, dataSet.id) + if (!active) return true + if (active.target.id === dataSet.id) return false + if (active.status === 'failed') return true + return replacementRetryable(active) +} + +/** + * Progress is measured in unique stored content, which is what actually gets + * copied. Many object versions can share one item. + */ +export function replacementProgressLabel(replacement: ProviderReplacement) { + if (replacement.status === 'preparing_target') { + return 'Creating the new storage service' + } + if (replacement.items_total === 0) { + if (replacement.status === 'retiring' || replacement.status === 'cleanup_attention') { + return 'Stored content migration is complete' + } + return 'Preparing the list of stored items' + } + if ( + replacement.status === 'retiring' || + replacement.status === 'cleanup_attention' || + replacement.status === 'completed' + ) { + const noLongerNeeded = Math.max(0, replacement.items_total - replacement.items_copied) + const suffix = noLongerNeeded > 0 ? `; ${formatNumber(noLongerNeeded)} no longer needed` : '' + return `Stored content migration is complete · ${formatNumber(replacement.items_copied)} copied${suffix}` + } + return `Migrated ${formatNumber(replacement.items_copied)} of ${formatNumber(replacement.items_total)} stored items` +} + +/** + * A confirmation counts referenced versions and total size, which is what the + * operator is authorizing storage for. + */ +export function replacementConfirmationSummary(dataSet: StorageDataSetSummary) { + return `${formatNumber(dataSet.referenced_version_count)} versions · ${formatBytes(dataSet.physical_bytes)}` +} + +export function replacementGenerationLabel(dataSet: StorageDataSetSummary) { + return `Generation ${dataSet.generation}` +} + +/** A generation that has no service yet is the incoming replacement target. */ +function dataSetIsBeingPrepared(dataSet: StorageDataSetSummary) { + return !dataSet.is_current && (dataSet.status === 'pending' || dataSet.status === 'creating') +} + +export function dataSetGenerationTone(dataSet: StorageDataSetSummary): StatusTone { + if (dataSet.is_current) return 'success' + if (dataSet.status === 'retired') return 'neutral' + // Setting up the replacement is ordinary progress, not something to act on. + if (dataSetIsBeingPrepared(dataSet)) return 'info' + return 'warning' +} + +/** + * The replica column is narrow, so these stay about as short as "Current" and + * "Retired". A label that does not fit is truncated to something unreadable, + * which is worse than a terser word. + */ +export function dataSetGenerationLabel(dataSet: StorageDataSetSummary) { + if (dataSet.is_current) return 'Current' + if (dataSet.status === 'retired') return 'Retired' + // The pair reads as a direction: one generation is on its way in, the other + // on its way out. + if (dataSet.status === 'draining') return 'Outgoing' + // A generation still being set up is the replacement's destination. Calling it + // historical points at the past when it is the one arriving. + if (dataSetIsBeingPrepared(dataSet)) return 'Incoming' + if (dataSet.status === 'failed') return 'Failed' + return 'Historical' +} + +const replacementErrorMessages: Record = { + replacement_active: 'This replica is already being replaced. Wait for it to finish or retry it below.', + replacement_target_in_use: 'That provider already stores a replica of this bucket. Choose a different one.', + replacement_target_invalid: 'Choose a provider other than the one being replaced.', + replacement_no_eligible_provider: + 'No unused storage provider is available right now. Try again later or choose an available provider.', + replacement_target_unavailable: 'That provider is not available for replacement right now. Choose another provider.', + replacement_idempotency_conflict: 'This confirmation changed after it was submitted. Close it and try again.', + replacement_source_not_current: 'This replica no longer receives writes, so replacing it would change nothing.', + replacement_superseded: 'A newer request has taken over this replica.', + replacement_not_retryable: 'This replacement is still progressing on its own.', + replacement_task_running: 'Replacement work is still running. Try again shortly.', +} + +/** + * Provider replacement work resumes from the Data Sets view, never from the task + * queue, so the Tasks page must not offer a Retry the API will refuse. + */ +export function taskRetryableFromQueue(task: { stage?: string | null }) { + return ( + task.stage !== 'replace_provider' && task.stage !== 'retire_data_set' && task.stage !== 'retire_abandoned_target' + ) +} + +export function taskHasDataSetsRecovery(task: { stage?: string | null; status: string; bucket_name?: string | null }) { + return ( + Boolean(task.bucket_name) && + ['replace_provider', 'retire_data_set', 'retire_abandoned_target'].includes(task.stage ?? '') && + ['completed', 'failed', 'exhausted', 'cancelled'].includes(task.status) + ) +} + +/** + * What the operator should do next, in their terms. The recorded error is + * developer diagnostics and belongs behind a detail view, not on the card. + */ +export function replacementNextStep(replacement: ProviderReplacement) { + switch (replacement.status) { + case 'failed': + if (replacement.failure_reason === 'target_in_use') { + return 'This provider is already in use. Choose a different provider.' + } + // Choosing a different provider is only open while the old provider still + // holds the replica. Once the new one has taken it over, the only way + // forward is to finish the copy that was started. + return replacement.source.is_current + ? 'The replacement has not finished. Retry, or choose a different provider.' + : 'The replacement has not finished. Retry to continue where it stopped.' + case 'cleanup_attention': + return 'The data is on the new provider. The old one could not be shut down and is still being paid for.' + case 'waiting': + return replacement.wait_message ?? 'Waiting.' + default: + return null + } +} + +/** Turn a failed request into something the operator can act on. */ +export function replacementErrorMessage(error: unknown) { + if (error instanceof APIError) { + const known = error.code ? replacementErrorMessages[error.code] : undefined + if (known) return known + if (error.status === 503) { + return 'Filecoin storage is unavailable right now. Try again once it recovers.' + } + // A 5xx body carries internal diagnostics, which say nothing an operator can + // act on. Anything the operator can fix arrives as a typed 4xx above. + if (error.status >= 500) { + return 'The request could not be completed. Check the server logs for details.' + } + return error.message + } + if (error instanceof Error) return error.message + return 'Could not complete the request.' +} + +const providerIneligibleReasons: Record = { + current_source: 'This is the provider being replaced', + already_serves_bucket: 'Already stores a replica of this bucket', +} + +/** + * Why a provider cannot take this replica. Ineligible providers stay in the + * list: an operator looking for one they expected needs the reason, not a + * silent absence. + */ +export function providerCandidateDisabledReason(candidate: ReplacementProviderCandidate) { + if (candidate.eligible) return null + return providerIneligibleReasons[candidate.ineligible_reason ?? ''] ?? 'Cannot take this replica' +} + +/** A provider reads as its name; the registry ID identifies it. */ +export function providerCandidateLabel(candidate: ReplacementProviderCandidate) { + const name = candidate.provider_identity?.name?.trim() + return name || `Registry ${candidate.provider_id}` +} + +/** + * The registry ID line under a provider's name. A bare number there reads as an + * index or a rank; the prefix names what it is, matching how a provider ID is + * written everywhere else in the dashboard. Nothing is shown when the name is + * already the ID, which would only repeat it. + */ +export function providerCandidateRegistryLine(candidate: ReplacementProviderCandidate) { + const registryLine = `Registry ${candidate.provider_id}` + return providerCandidateLabel(candidate) === registryLine ? null : registryLine +} + +/** Extra context for a choosable provider, or null when there is nothing to add. */ +export function providerCandidateNote(candidate: ReplacementProviderCandidate) { + if (!candidate.eligible || !candidate.previously_used) return null + return 'Used by this bucket before' +} + +/** + * Operators search by whichever they have to hand: the name they know the + * provider by, or the registry ID from a task or log line. + */ +export function providerCandidateMatches(candidate: ReplacementProviderCandidate, query: string) { + const needle = query.trim().toLowerCase() + if (!needle) return true + if (candidate.provider_id.toLowerCase().includes(needle)) return true + const identity = candidate.provider_identity + return Boolean(identity?.name?.toLowerCase().includes(needle) || identity?.location?.toLowerCase().includes(needle)) +} diff --git a/ui/src/lib/storage-status-labels.ts b/ui/src/lib/storage-status-labels.ts index d88bf4c..cb26a1c 100644 --- a/ui/src/lib/storage-status-labels.ts +++ b/ui/src/lib/storage-status-labels.ts @@ -11,6 +11,7 @@ export const taskStageOptions = [ 'peer_pull', 'peer_commit', 'repair_replica', + 'replace_provider', 'reconcile_bucket_durability', ] as const @@ -24,10 +25,22 @@ const taskStageLabels: Record | '', string> = { peer_pull: 'Sync peer replica', peer_commit: 'Register peer replica on-chain', repair_replica: 'Resume replica upload', + replace_provider: 'Replace provider', reconcile_bucket_durability: 'Apply cache policy', '': 'Upload', } +// The retirement stages run on the storage cleanup worker, so they are labelled +// but never offered in the Upload stage filter, which would return nothing. +const replacementCleanupStageLabels: Record = { + retire_data_set: 'Retire replaced provider', + retire_abandoned_target: 'Retire unused provider', +} + +export function replacementCleanupStageLabel(stage?: string | null) { + return stage ? replacementCleanupStageLabels[stage] : undefined +} + export function taskTypeLabel(type?: string) { switch (type) { case 'all': @@ -106,6 +119,10 @@ export function storageCleanupCopyStatusTone(status?: string): StatusTone { } function taskOperationBaseLabel(type: string | undefined, stage: string) { + // Retiring a replaced provider ends a paid service; it is not the ordinary + // replica deletion the storage_cleanup type otherwise means. + const replacementLabel = replacementCleanupStageLabel(stage) + if (replacementLabel) return replacementLabel const stageLabel = taskStageLabels[stage as keyof typeof taskStageLabels] if (stageLabel && stage !== '') return stageLabel switch (type) { diff --git a/ui/src/routes/buckets.$name.tsx b/ui/src/routes/buckets.$name.tsx index 6d2d5ad..d8a68cd 100644 --- a/ui/src/routes/buckets.$name.tsx +++ b/ui/src/routes/buckets.$name.tsx @@ -13,6 +13,7 @@ import { Loader2, MoreHorizontal, RefreshCw, + Repeat2, RotateCcw, Trash2, TriangleAlert, @@ -40,6 +41,7 @@ import { type ObjectUploadStatus, type ObjectVersionItem, objectVersionAlreadyCurrentCode, + type ProviderReplacement, type StorageDataSetSummary, type StorageHealthStatus, type UploadTransferProgress, @@ -53,6 +55,7 @@ import { DetailTextDialog } from '@/components/app/DetailTextDialog' import { PageErrorState } from '@/components/app/PageErrorState' import { PageHeader } from '@/components/app/PageHeader' import { ProviderIdentityCell } from '@/components/app/ProviderIdentityCell' +import { ProviderSelect } from '@/components/app/ProviderSelect' import { ReviewDetails } from '@/components/app/ReviewDetails' import { bucketStatusTone, StatusBadge, type StatusTone } from '@/components/app/StatusBadge' import { UploadProgressRing, uploadProgressPercent } from '@/components/app/UploadProgress' @@ -106,10 +109,13 @@ import { usePermanentDeleteBucketObjectVersion, usePermanentDeleteDeletedBucketObject, useRefreshDataSetStorageHealth, + useReplacementProviderCandidates, useRestoreBucketObject, useRestoreBucketObjectVersion, + useRetryProviderReplacement, useS3Users, useSettings, + useStartProviderReplacement, useUpdateBucketCopyPolicy, useUpdateBucketOwner, } from '@/hooks/queries' @@ -155,6 +161,19 @@ import { copyHealthSummaryTitle, } from '@/lib/copy-health' import { dataSetStorageHealthDetailParts, dataSetStorageHealthRefreshErrorMessage } from '@/lib/data-set-storage-health' +import { + activeReplacements, + dataSetGenerationLabel, + dataSetGenerationTone, + dataSetReplaceable, + replacementConfirmationSummary, + replacementErrorMessage, + replacementNextStep, + replacementProgressLabel, + replacementRetryable, + replacementStatusLabel, + replacementStatusTone, +} from '@/lib/provider-replacement' import { ownerLabel } from '@/lib/s3-owner' import { type BucketPrefixCrumb, bucketPrefixCrumbs, duplicateObjectUploadKeys, objectUploadKey } from '@/lib/s3-prefix' import { objectStateLabel, replicaLabel, transferMethodLabel, uploadStatusLabel } from '@/lib/storage-status-labels' @@ -1182,6 +1201,22 @@ function ObjectBrowserPage() { ) const qc = useQueryClient() + useEffect(() => { + if (search.details === 'storage') setDetailsOpen(true) + }, [search.details]) + + const handleDetailsOpenChange = (open: boolean) => { + setDetailsOpen(open) + if (!open && search.details === 'storage') { + navigate({ + to: '/buckets/$name', + params: { name }, + search: { ...search, details: undefined }, + replace: true, + }) + } + } + const pathCrumbs = bucketPrefixCrumbs(prefix) const navigateToPrefix = (newPrefix: string) => { @@ -1446,7 +1481,8 @@ function ObjectBrowserPage() { navigateToStorageRisk()} onReviewStorageDataSetRisk={navigateToStorageRiskDataSet} @@ -1769,6 +1805,7 @@ function BucketDetailsSheet({ onChangeOwner, onReviewStorageRisk, onReviewStorageDataSetRisk, + focusStorage, }: { bucket: NonNullable['data']> open: boolean @@ -1776,9 +1813,19 @@ function BucketDetailsSheet({ onChangeOwner: () => void onReviewStorageRisk: () => void onReviewStorageDataSetRisk: (dataSetID: number) => void + focusStorage: boolean }) { const [storageHealthError, setStorageHealthError] = useState(null) + // Replacement diagnostics are not replica-health diagnostics; sharing one + // dialog labelled them as the wrong kind of fault. + const [replacementError, setReplacementError] = useState(null) + const [replacementTarget, setReplacementTarget] = useState(null) const titleRef = useRef(null) + const storageRef = useRef(null) + + useEffect(() => { + if (open && focusStorage) storageRef.current?.scrollIntoView({ block: 'start' }) + }, [focusStorage, open]) return ( <> @@ -1811,13 +1858,22 @@ function BucketDetailsSheet({ onOpenLastError={setStorageHealthError} onReviewVersions={onReviewStorageRisk} /> - - - +
+ + + + +
@@ -1830,10 +1886,247 @@ function BucketDetailsSheet({ text={storageHealthError} onClose={() => setStorageHealthError(null)} /> + setReplacementError(null)} + /> + setReplacementTarget(null)} + /> ) } +function ReplaceProviderDialog({ + bucketName, + dataSet, + onClose, +}: { + bucketName: string + dataSet: StorageDataSetSummary | null + onClose: () => void +}) { + const startReplacement = useStartProviderReplacement() + const [mode, setMode] = useState<'automatic' | 'manual'>('automatic') + const [providerID, setProviderID] = useState('') + const [clientRequestID, setClientRequestID] = useState('') + const [error, setError] = useState(null) + const dataSetID = dataSet?.id + // The registry is only worth reading while the operator is actually choosing. + const candidates = useReplacementProviderCandidates( + bucketName, + dataSet?.id ?? null, + Boolean(dataSet) && mode === 'manual' + ) + + useEffect(() => { + if (dataSetID !== undefined) { + setMode('automatic') + setProviderID('') + setError(null) + setClientRequestID(crypto.randomUUID()) + } + }, [dataSetID]) + + if (!dataSet) return null + + const namedProvider = dataSet.provider_identity?.name?.trim() + const providerLabel = namedProvider || (dataSet.provider_id ? `Registry ${dataSet.provider_id}` : '—') + const canChooseProvider = + mode === 'automatic' || (!candidates.isLoading && !candidates.isError && providerID.trim().length > 0) + + const submit = () => { + if (!canChooseProvider || startReplacement.isPending) return + const requestID = clientRequestID || crypto.randomUUID() + if (!clientRequestID) setClientRequestID(requestID) + setError(null) + startReplacement.mutate( + { bucket: bucketName, dataSetID: dataSet.id, mode, providerID: providerID.trim(), clientRequestID: requestID }, + { + onSuccess: () => onClose(), + onError: (mutationError) => setError(replacementErrorMessage(mutationError)), + } + ) + } + + return ( + { + if (!open) onClose() + }} + title="Replace provider" + description="This starts paying a new provider, and new uploads go there. The old provider is ended after existing objects are readable on the new one." + confirmLabel="Replace provider" + typedTarget="replace" + pending={startReplacement.isPending} + confirmDisabled={!canChooseProvider} + error={error} + contentClassName="data-[size=default]:max-w-lg data-[size=default]:sm:max-w-lg" + onConfirm={submit} + > + + + + Existing objects copy from another replica or from cache. If an object has neither, the old provider stays. + + + + + New provider + + {mode === 'automatic' && Picks a provider this bucket has never used.} + + {mode === 'manual' && ( + + Provider + {candidates.isError ? ( + Provider choices couldn’t be loaded. Try again. + ) : ( + { + setProviderID(value) + setClientRequestID(crypto.randomUUID()) + }} + disabled={startReplacement.isPending || candidates.isLoading} + /> + )} + + )} + + + ) +} + +/** + * ProviderReplacementProgress shows the replacement that still needs something, + * including a retry the operator owns. A failed retry request is reported here + * rather than disappearing. + */ +function ProviderReplacementProgress({ + bucketName, + replacements, + onOpenLastError, +}: { + bucketName: string + replacements: ProviderReplacement[] + onOpenLastError: (text: string) => void +}) { + const active = activeReplacements(replacements) + if (active.length === 0) return null + return ( +
+ {active.map((replacement) => ( + + ))} +
+ ) +} + +function ProviderReplacementProgressCard({ + bucketName, + replacement, + onOpenLastError, +}: { + bucketName: string + replacement: ProviderReplacement + onOpenLastError: (text: string) => void +}) { + const retryReplacement = useRetryProviderReplacement() + const [retryError, setRetryError] = useState(null) + const nextStep = replacementNextStep(replacement) + + return ( +
+
+ + Replacing {replicaLabel(replacement.copy_index)} · {replacement.source.provider_id} →{' '} + {replacement.target.provider_id} + + + {replacementStatusLabel(replacement.status)} + +
+
{replacementProgressLabel(replacement)}
+ {nextStep &&
{nextStep}
} + {replacement.last_error && ( + onOpenLastError(replacement.last_error ?? '')} + /> + )} + {retryError && ( + + {retryError} + + )} + {replacementRetryable(replacement) && ( +
+ +
+ )} +
+ ) +} + function BucketDetailsSection({ title, children }: { title: string; children: ReactNode }) { return (
@@ -1970,11 +2263,15 @@ function BucketDetailAction({ label, value, onClick }: { label: string; value: s function BucketStorageDataSets({ bucketName, dataSets, + replacements, onReviewStorageRisk, + onReplaceProvider, }: { bucketName: string dataSets: StorageDataSetSummary[] + replacements: ProviderReplacement[] onReviewStorageRisk: (dataSetID: number) => void + onReplaceProvider: (dataSet: StorageDataSetSummary) => void }) { const refreshStorageHealth = useRefreshDataSetStorageHealth() const [refreshError, setRefreshError] = useState(null) @@ -2057,8 +2354,17 @@ function BucketStorageDataSets({ return ( - - {replicaLabel(dataSet.copy_index)} + + + {replicaLabel(dataSet.copy_index)} + + + {dataSetGenerationLabel(dataSet)} + @@ -2093,22 +2399,40 @@ function BucketStorageDataSets({ {formatNumber(dataSet.referenced_version_count)} - {dataSetNeedsStorageRiskReview(dataSet) && ( - - - - - Affected versions - - )} +
+ {dataSetNeedsStorageRiskReview(dataSet) && ( + + + + + Affected versions + + )} + {dataSetReplaceable(dataSet, replacements) && ( + + + + + Replace provider + + )} +
) diff --git a/ui/src/routes/tasks.tsx b/ui/src/routes/tasks.tsx index d7b61ed..f9ddde5 100644 --- a/ui/src/routes/tasks.tsx +++ b/ui/src/routes/tasks.tsx @@ -1,6 +1,15 @@ import { useMutation, useQueryClient } from '@tanstack/react-query' -import { createFileRoute } from '@tanstack/react-router' -import { ChevronDown, ListTodo, Loader2, RefreshCw, RotateCcw, Stethoscope, TriangleAlert } from 'lucide-react' +import { createFileRoute, Link } from '@tanstack/react-router' +import { + ChevronDown, + ExternalLink, + ListTodo, + Loader2, + RefreshCw, + RotateCcw, + Stethoscope, + TriangleAlert, +} from 'lucide-react' import { type ReactNode, useEffect, useState } from 'react' import { api, type TaskDiagnostic, type TaskItem, type TaskStorageCleanupDetail } from '@/api/client' import { CopyableValue } from '@/components/app/CopyableValue' @@ -27,7 +36,9 @@ import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@ import { Tabs, TabsList, TabsTrigger } from '@/components/ui/tabs' import { Tooltip, TooltipContent, TooltipTrigger } from '@/components/ui/tooltip' import { useTaskRefDetail, useTasks } from '@/hooks/queries' +import { taskHasDataSetsRecovery, taskRetryableFromQueue } from '@/lib/provider-replacement' import { + replacementCleanupStageLabel, storageCleanupCopyStatusLabel, storageCleanupCopyStatusTone, storageCleanupStatusLabel, @@ -116,7 +127,16 @@ function taskDetailTitle(task: TaskItem) { function TaskRefCell({ task }: { task: TaskItem }) { const [detailsOpen, setDetailsOpen] = useState(false) const detail = useTaskRefDetail(task.id, detailsOpen) - const refLabel = task.type === 'storage_cleanup' ? 'Deleted object' : `${task.ref_type}:${task.ref_id}` + // Replacement cleanup ends a storage service; calling it a deleted object + // would send the operator looking in the wrong place. + const replacementCleanup = replacementCleanupStageLabel(task.stage) + const refLabel = replacementCleanup + ? task.bucket_name + ? `${replacementCleanup} · ${task.bucket_name}` + : replacementCleanup + : task.type === 'storage_cleanup' + ? 'Deleted object' + : `${task.ref_type}:${task.ref_id}` return ( @@ -148,6 +168,14 @@ function TaskRefPopoverContent({ detail, enabled }: { detail: ReturnType{enabled ? 'Loading details' : ''} } if (!detail.data.object && !detail.data.storage_cleanup) { + if (detail.data.bucket_name) { + return ( +
+ Bucket + {detail.data.bucket_name} +
+ ) + } return Details unavailable } @@ -324,8 +352,9 @@ function TaskActionsCell({ onOpenDiagnostic: (task: TaskItem) => void }) { const showDiagnostic = task.type === 'upload' - const showRetry = task.status === 'exhausted' - if (!showDiagnostic && !showRetry) return — + const showRetry = task.status === 'exhausted' && taskRetryableFromQueue(task) + const showDataSets = taskHasDataSetsRecovery(task) + if (!showDiagnostic && !showRetry && !showDataSets) return — return (
@@ -362,6 +391,23 @@ function TaskActionsCell({ Retry )} + {showDataSets && task.bucket_name && ( + + + + + Open Data Sets + + )}
) } diff --git a/ui/test/provider-replacement.test.ts b/ui/test/provider-replacement.test.ts new file mode 100644 index 0000000..91e6209 --- /dev/null +++ b/ui/test/provider-replacement.test.ts @@ -0,0 +1,366 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { + APIError, + type ProviderReplacement, + type ReplacementProviderCandidate, + type StorageDataSetSummary, +} from '../src/api/client.ts' +import { + activeReplacements, + dataSetGenerationLabel, + dataSetGenerationTone, + dataSetReplaceable, + providerCandidateDisabledReason, + providerCandidateLabel, + providerCandidateMatches, + providerCandidateNote, + providerCandidateRegistryLine, + replacementConfirmationSummary, + replacementErrorMessage, + replacementNextStep, + replacementProgressLabel, + replacementRetryable, + replacementStatusLabel, + replacementStatusTone, + taskHasDataSetsRecovery, + taskRetryableFromQueue, +} from '../src/lib/provider-replacement.ts' + +function dataSet(overrides: Partial = {}): StorageDataSetSummary { + return { + id: 1, + bucket_id: 1, + copy_index: 0, + generation: 1, + is_current: true, + replaceable: true, + provider_id: '101', + status: 'ready', + committed_copies: 3, + readable_copies: 3, + physical_bytes: 1024 * 1024, + referenced_version_count: 12, + current_version_count: 4, + created_at: '2026-08-01T00:00:00Z', + updated_at: '2026-08-01T00:00:00Z', + ...overrides, + } +} + +function replacement(overrides: Partial = {}): ProviderReplacement { + return { + id: 1, + bucket_name: 'bucket', + copy_index: 0, + status: 'migrating', + selection_mode: 'automatic', + source: { id: 1, generation: 1, is_current: false, status: 'draining', provider_id: '101', data_set_id: '1001' }, + target: { id: 2, generation: 2, is_current: true, status: 'ready', provider_id: '202', data_set_id: '2002' }, + items_total: 10, + items_copied: 3, + last_error: null, + termination_epoch: null, + created_at: '2026-08-01T00:00:00Z', + updated_at: '2026-08-01T00:00:00Z', + ...overrides, + } +} + +// A replica already being replaced must not offer a second confirmation. +test('a data set in flight is not replaceable again', () => { + const set = dataSet() + assert.equal(dataSetReplaceable(set, []), true) + assert.equal(dataSetReplaceable(set, [replacement({ source: { ...replacement().source, id: set.id } })]), false) + // A finished replacement releases the slot. + assert.equal(dataSetReplaceable(set, [replacement({ status: 'completed' })]), true) +}) + +test('a historical generation is never offered for replacement', () => { + assert.equal(dataSetReplaceable(dataSet({ is_current: false, replaceable: false }), []), false) +}) + +// Confirmation counts versions; progress counts stored content. Mixing the two +// would tell the operator the wrong thing about what is being copied. +test('confirmation and progress use their own units', () => { + assert.equal(replacementConfirmationSummary(dataSet()), '12 versions · 1 MB') + assert.equal(replacementProgressLabel(replacement()), 'Migrated 3 of 10 stored items') + assert.equal( + replacementProgressLabel(replacement({ status: 'preparing_target', items_total: 0, items_copied: 0 })), + 'Creating the new storage service' + ) + assert.equal( + replacementProgressLabel(replacement({ items_total: 0, items_copied: 0 })), + 'Preparing the list of stored items' + ) +}) + +test('only operator-owned states are retryable', () => { + assert.equal(replacementRetryable(replacement({ status: 'failed' })), true) + assert.equal(replacementRetryable(replacement({ status: 'cleanup_attention' })), true) + assert.equal(replacementRetryable(replacement({ status: 'failed', failure_reason: 'target_in_use' })), false) + for (const status of ['preparing_target', 'migrating', 'waiting', 'retiring', 'completed', 'superseded'] as const) { + assert.equal(replacementRetryable(replacement({ status })), false, status) + } +}) + +test('a permanent target conflict stays visible and allows a new provider choice', () => { + const source = dataSet({ id: 1, is_current: true, replaceable: true }) + const stopped = replacement({ + status: 'failed', + failure_reason: 'target_in_use', + source: { ...replacement().source, id: 1, is_current: true, status: 'ready' }, + target: { ...replacement().target, id: 2, is_current: false }, + }) + assert.deepEqual( + activeReplacements([stopped]).map((row) => row.id), + [stopped.id] + ) + assert.equal(replacementRetryable(stopped), false) + assert.equal(dataSetReplaceable(source, [stopped]), true) +}) + +test('every replacement that still needs something is surfaced', () => { + assert.deepEqual(activeReplacements([]), []) + assert.deepEqual(activeReplacements([replacement({ status: 'completed' })]), []) + assert.deepEqual(activeReplacements([replacement({ status: 'superseded' })]), []) + assert.deepEqual( + activeReplacements([ + replacement({ id: 7, status: 'waiting' }), + replacement({ id: 8, status: 'cleanup_attention' }), + ]).map((row) => row.id), + [7, 8] + ) +}) + +test('terminal migration progress separates copied content from content no longer needed', () => { + assert.equal( + replacementProgressLabel(replacement({ status: 'retiring', items_total: 10, items_copied: 8 })), + 'Stored content migration is complete · 8 copied; 2 no longer needed' + ) +}) + +test('states needing attention are visually distinct from progress', () => { + assert.equal(replacementStatusTone('failed'), 'danger') + assert.equal(replacementStatusTone('cleanup_attention'), 'danger') + assert.equal(replacementStatusTone('waiting'), 'warning') + assert.equal(replacementStatusTone('completed'), 'success') + assert.equal(replacementStatusTone('migrating'), 'info') +}) + +test('generation labels describe the slot without internal state names', () => { + assert.equal(dataSetGenerationLabel(dataSet()), 'Current') + assert.equal(dataSetGenerationLabel(dataSet({ is_current: false, status: 'draining' })), 'Outgoing') + assert.equal(dataSetGenerationLabel(dataSet({ is_current: false, status: 'retired' })), 'Retired') + assert.equal(dataSetGenerationLabel(dataSet({ is_current: false, status: 'ready' })), 'Historical') +}) + +// The replica column is narrow. A label longer than about "Historical" is cut +// mid-word, which reads as nothing at all. +test('generation labels are short enough for the replica column', () => { + const statuses = ['ready', 'pending', 'creating', 'failed', 'draining', 'retired'] + for (const status of statuses) { + for (const isCurrent of [true, false]) { + const label = dataSetGenerationLabel(dataSet({ is_current: isCurrent, status })) + assert.ok(label.length <= 10, `${status}/${isCurrent} → ${label} (${label.length} chars)`) + } + } +}) + +// The replacement target has no service yet, so it is not current. Falling +// through to "Historical" pointed at the past for the generation that is +// arriving, and coloured ordinary progress as something to act on. +test('the generation being prepared is not labelled as a past one', () => { + for (const status of ['pending', 'creating']) { + const target = dataSet({ is_current: false, status, generation: 2 }) + assert.equal(dataSetGenerationLabel(target), 'Incoming', status) + assert.equal(dataSetGenerationTone(target), 'info', status) + } + // A target whose service was never created is not "historical" either. + assert.equal(dataSetGenerationLabel(dataSet({ is_current: false, status: 'failed' })), 'Failed') + // The live generation is unaffected even before its own service settles. + assert.equal(dataSetGenerationLabel(dataSet({ is_current: true, status: 'creating' })), 'Current') +}) + +// A typed conflict must tell the operator what to do, not echo a status code. +test('errors explain what to do next', () => { + assert.equal( + replacementErrorMessage(new APIError('conflict', 409, 'replacement_target_in_use')), + 'That provider already stores a replica of this bucket. Choose a different one.' + ) + assert.equal( + replacementErrorMessage(new APIError('unavailable', 503)), + 'Filecoin storage is unavailable right now. Try again once it recovers.' + ) + assert.equal(replacementErrorMessage(new APIError('boom', 400)), 'boom') + assert.equal(replacementErrorMessage(new Error('network down')), 'network down') + assert.equal(replacementErrorMessage('nope'), 'Could not complete the request.') +}) + +// #314 says choosing a different target after a failure needs a new +// confirmation. Treating a stopped replacement as owning the replica left the +// operator with nothing but Retry on a provider that will not work. +test('a stopped replacement does not block choosing a different provider', () => { + const set = dataSet() + const owned = { ...replacement().source, id: set.id } + for (const status of ['preparing_target', 'migrating', 'waiting', 'retiring'] as const) { + assert.equal(dataSetReplaceable(set, [replacement({ status, source: owned })]), false, status) + } + for (const status of ['failed', 'cleanup_attention'] as const) { + assert.equal(dataSetReplaceable(set, [replacement({ status, source: owned })]), true, status) + } +}) + +// The API refuses a queue retry for replacement work, so offering the button +// would only produce a 409. +test('replacement work is not retryable from the task queue', () => { + for (const stage of ['replace_provider', 'retire_data_set', 'retire_abandoned_target']) { + assert.equal(taskRetryableFromQueue({ stage }), false, stage) + } + for (const stage of ['ingress_store', 'peer_pull', 'repair_replica']) { + assert.equal(taskRetryableFromQueue({ stage }), true, stage) + } + assert.equal(taskRetryableFromQueue({}), true) +}) + +test('terminal replacement tasks can recover through Data Sets', () => { + for (const stage of ['replace_provider', 'retire_data_set', 'retire_abandoned_target']) { + assert.equal(taskHasDataSetsRecovery({ stage, status: 'completed', bucket_name: 'bucket' }), true) + assert.equal(taskHasDataSetsRecovery({ stage, status: 'failed', bucket_name: 'bucket' }), true) + assert.equal(taskHasDataSetsRecovery({ stage, status: 'running', bucket_name: 'bucket' }), false) + assert.equal(taskHasDataSetsRecovery({ stage, status: 'failed' }), false) + } +}) + +// The two attention states have different consequences: one means the data has +// not moved, the other means it has and the old provider is still being paid +// for. Calling both "needs your attention" hid that difference. +test('failure and cleanup attention say different things', () => { + const failed = replacementStatusLabel('failed') + const attention = replacementStatusLabel('cleanup_attention') + assert.notEqual(failed, attention) + assert.match(replacementNextStep(replacement({ status: 'failed' })) ?? '', /finished/) + assert.match(replacementNextStep(replacement({ status: 'cleanup_attention' })) ?? '', /still being paid for/) +}) + +// The card shows what happened and what to do; the recorded error is developer +// diagnostics and stays behind a detail view. +test('the next step never echoes the recorded error', () => { + const row = replacement({ + status: 'failed', + last_error: 'copy replacement item: pull: provider timed out (max retries reached)', + }) + const step = replacementNextStep(row) ?? '' + assert.ok(step.length > 0) + assert.ok(!step.includes('max retries reached')) + assert.equal(replacementNextStep(replacement({ status: 'migrating' })), null) +}) + +// The API refuses to replace a data set another replacement is still migrating +// into, because the original source still owes it the data it has not copied. +// Offering the action there produced a guaranteed 409. +test('the target of a stopped replacement is not offered for replacement', () => { + const target = dataSet({ id: 2, generation: 2 }) + const row = replacement({ + status: 'failed', + source: { ...replacement().source, id: 1, is_current: false }, + target: { ...replacement().target, id: 2 }, + }) + assert.equal(dataSetReplaceable(target, [row]), false) + // The replica being replaced keeps the re-target path #314 requires. + const source = dataSet({ id: 1 }) + assert.equal(dataSetReplaceable(source, [row]), true) +}) + +// Changing provider is only open while the old one still holds the replica. +// Once the new one has taken it over, the copy has to be finished, not restarted +// somewhere else. +test('the next step offers a different provider only when one can be chosen', () => { + const beforeHandover = replacement({ status: 'failed', source: { ...replacement().source, is_current: true } }) + assert.match(replacementNextStep(beforeHandover) ?? '', /different provider/) + const afterHandover = replacement({ status: 'failed', source: { ...replacement().source, is_current: false } }) + const step = replacementNextStep(afterHandover) ?? '' + assert.match(step, /Retry/) + assert.ok(!step.includes('different provider')) +}) + +// A 5xx body is internal diagnostics. Anything the operator can act on arrives +// as a typed 4xx. +test('server faults do not surface internal text', () => { + const message = replacementErrorMessage(new APIError('sql: transaction has already been committed', 500)) + assert.ok(!message.includes('sql:')) + assert.match(message, /server logs/) +}) + +function candidate(overrides: Partial = {}): ReplacementProviderCandidate { + return { provider_id: '303', eligible: true, previously_used: false, ...overrides } +} + +// The API refuses these choices, so the chooser has to say so rather than +// letting the operator find out from a 409. +test('a provider that cannot take the replica says why', () => { + assert.equal( + providerCandidateDisabledReason(candidate({ eligible: false, ineligible_reason: 'current_source' })), + 'This is the provider being replaced' + ) + assert.equal( + providerCandidateDisabledReason(candidate({ eligible: false, ineligible_reason: 'already_serves_bucket' })), + 'Already stores a replica of this bucket' + ) + // An unrecognised reason still has to read as a reason, not as a code. + assert.equal( + providerCandidateDisabledReason(candidate({ eligible: false, ineligible_reason: 'something_new' })), + 'Cannot take this replica' + ) + assert.equal(providerCandidateDisabledReason(candidate()), null) +}) + +test('a provider reads as its name and falls back to its registry ID', () => { + assert.equal( + providerCandidateLabel(candidate({ provider_identity: { registry_provider_id: '303', name: 'Acme' } })), + 'Acme' + ) + assert.equal(providerCandidateLabel(candidate()), 'Registry 303') + // A blank name is not a name. + assert.equal( + providerCandidateLabel(candidate({ provider_identity: { registry_provider_id: '303', name: ' ' } })), + 'Registry 303' + ) +}) + +// Automatic selection avoids a provider the bucket moved away from; choosing it +// manually stays allowed, so the note explains rather than blocks. +test('a previously used provider is choosable but flagged', () => { + assert.equal(providerCandidateNote(candidate({ previously_used: true })), 'Used by this bucket before') + assert.equal(providerCandidateNote(candidate()), null) + assert.equal(providerCandidateNote(candidate({ eligible: false, previously_used: true })), null) +}) + +// Operators search by whichever they have to hand. +test('providers are searchable by name and by registry ID', () => { + const acme = candidate({ + provider_id: '303', + provider_identity: { registry_provider_id: '303', name: 'Acme Storage', location: 'Berlin' }, + }) + assert.equal(providerCandidateMatches(acme, 'acme'), true) + assert.equal(providerCandidateMatches(acme, '303'), true) + assert.equal(providerCandidateMatches(acme, 'berlin'), true) + assert.equal(providerCandidateMatches(acme, ''), true) + assert.equal(providerCandidateMatches(acme, 'zzz'), false) + // A provider with no identity is still findable by ID. + assert.equal(providerCandidateMatches(candidate({ provider_id: '404' }), '404'), true) + assert.equal(providerCandidateMatches(candidate({ provider_id: '404' }), 'acme'), false) +}) + +// A bare number under the name reads as an index or a rank. The dashboard +// writes a provider ID as "Registry N" everywhere else. +test('the id line under a provider name says what the number is', () => { + assert.equal( + providerCandidateRegistryLine( + candidate({ provider_id: '5', provider_identity: { registry_provider_id: '5', name: 'Mongo2Stor' } }) + ), + 'Registry 5' + ) + // With no name the heading is already "Registry 5"; repeating it says nothing. + assert.equal(providerCandidateRegistryLine(candidate({ provider_id: '5' })), null) +}) diff --git a/ui/test/storage-status-labels.test.ts b/ui/test/storage-status-labels.test.ts index 62db0fe..aed0a74 100644 --- a/ui/test/storage-status-labels.test.ts +++ b/ui/test/storage-status-labels.test.ts @@ -89,3 +89,14 @@ test('replica cleanup copy status tones expose each cleanup state', () => { assert.equal(storageCleanupCopyStatusTone('failed'), 'danger') assert.equal(storageCleanupCopyStatusTone('unsupported'), 'danger') }) + +// Retiring a replaced provider ends a paid service. Reading "Delete remote +// replicas" there suggests the object data is being removed. +test('retiring a provider is not labelled as replica deletion', () => { + assert.equal(taskOperationLabel({ type: 'storage_cleanup', stage: 'retire_data_set' }), 'Retire replaced provider') + assert.equal( + taskOperationLabel({ type: 'storage_cleanup', stage: 'retire_abandoned_target' }), + 'Retire unused provider' + ) + assert.equal(taskOperationLabel({ type: 'storage_cleanup', stage: '' }), 'Delete remote replicas') +})