From 45d25e3c12a14537751bf6b2400e6222ec802504 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:11:14 +0800 Subject: [PATCH 1/5] feat(storage): configure minimum durable copies --- docs/en/concepts/filecoin-storage-flow.md | 10 +- docs/en/concepts/write-path-cache.md | 6 +- docs/en/configuration/model.md | 2 +- docs/en/operations/troubleshooting.md | 4 +- docs/en/reference/admin-api.md | 15 +- docs/zh/concepts/filecoin-storage-flow.md | 10 +- docs/zh/concepts/write-path-cache.md | 6 +- docs/zh/configuration/model.md | 2 +- docs/zh/operations/troubleshooting.md | 4 +- docs/zh/reference/admin-api.md | 15 +- internal/admin/api_buckets.go | 271 ++++--- internal/admin/api_buckets_test.go | 151 +++- internal/admin/api_settings.go | 31 +- internal/admin/api_settings_test.go | 16 +- internal/admin/server.go | 8 + internal/app/runtime.go | 1 + internal/cacheeviction/task.go | 71 +- ...026081901_bucket_minimum_durable_copies.go | 31 + .../bucket_minimum_durable_copies_test.go | 57 ++ internal/db/repository/bucket_repo.go | 80 +- internal/db/repository/bucket_repo_test.go | 49 +- internal/db/repository/cache_eviction_repo.go | 724 ++++++++++++++++-- .../db/repository/cache_eviction_repo_test.go | 261 +++++++ internal/db/repository/interfaces.go | 21 +- internal/db/repository/object_repo.go | 29 +- internal/db/repository/object_repo_test.go | 59 ++ .../db/repository/storage_upload_reference.go | 23 +- internal/db/repository/storage_upload_repo.go | 162 +++- .../db/repository/storage_upload_repo_test.go | 119 +++ internal/model/bucket.go | 17 +- internal/worker/evictor.go | 41 +- internal/worker/evictor_after_upload.go | 57 +- .../evictor_after_upload_failure_test.go | 115 ++- internal/worker/evictor_after_upload_test.go | 22 +- internal/worker/evictor_concurrency_test.go | 4 +- internal/worker/evictor_finalize.go | 48 +- internal/worker/evictor_lru.go | 53 +- internal/worker/evictor_lru_capacity_test.go | 26 +- internal/worker/evictor_lru_finalize_test.go | 6 +- internal/worker/evictor_reconcile.go | 144 ++++ .../worker/evictor_reconcile_internal_test.go | 170 ++++ internal/worker/manager.go | 22 +- internal/worker/manager_test.go | 87 +++ internal/worker/uploader.go | 95 ++- internal/worker/uploader_test.go | 208 +++++ ui/src/api/client.ts | 19 +- ui/src/hooks/queries.ts | 23 +- ui/src/lib/bucket-copy-policy.ts | 52 +- ui/src/lib/storage-status-labels.ts | 2 + ui/src/routes/buckets.$name.tsx | 130 +++- ui/src/routes/buckets.index.tsx | 67 +- ui/test/api-client.test.ts | 57 ++ ui/test/bucket-copy-policy.test.ts | 82 +- ui/test/storage-status-labels.test.ts | 1 + 54 files changed, 3340 insertions(+), 446 deletions(-) create mode 100644 internal/db/migrations/2026081901_bucket_minimum_durable_copies.go create mode 100644 internal/db/migrations/bucket_minimum_durable_copies_test.go create mode 100644 internal/worker/evictor_reconcile.go create mode 100644 internal/worker/evictor_reconcile_internal_test.go diff --git a/docs/en/concepts/filecoin-storage-flow.md b/docs/en/concepts/filecoin-storage-flow.md index 12c3575..3b2b9dd 100644 --- a/docs/en/concepts/filecoin-storage-flow.md +++ b/docs/en/concepts/filecoin-storage-flow.md @@ -31,7 +31,7 @@ flowchart TD | `uploading` | A background task is preparing remote storage or uploading bytes. | | `committing` | The provider has a piece ready and the commit step is in progress. | | `replicating` | At least one readable copy exists while target copies are still being completed. | -| `stored` | Target remote copy policy is satisfied and metadata is available. | +| `stored` | The bucket's minimum durable copies are readable and committed; remaining target copies may still be syncing. | | `failed` | The active lifecycle step failed and may be retried. | | `cache_evicted` | Local cache has been removed after remote durability. | @@ -54,12 +54,18 @@ Health checks record storage provider and local data set status. The dashboard u If an established provider becomes temporarily unavailable while the initial copies are still being stored, SynapS3 keeps using the other assigned writable copies. The unfinished copy waits without consuming retries and resumes automatically when the original provider becomes reachable again. SynapS3 does not automatically select a replacement provider. Repairing copies that became unavailable after storage completed remains part of the planned replica repair feature below. +## Target and Minimum Replicas + +The target replica count is frozen when an upload starts. By default, cache release remains strict: every target replica must be readable and committed. A bucket can instead set a lower minimum durable replica count. Once that minimum is met, the version becomes stored and its cache follows the configured eviction policy, while the upload continues filling its original replica slots until the target is reached. + +Changing the target affects new uploads. Changing the minimum also re-evaluates retained cache for current uploads. Increasing the minimum does not move versions that are already stored back to an earlier state and cannot restore cache that has already been deleted. + ## What Users See - S3 upload can succeed before Filecoin storage finishes. - Dashboard task and topology views show storage progress. - Reads prefer local cache. If remote metadata exists, SynapS3 can retrieve the object from the provider. -- Cache eviction is an operational optimization, not the write acceptance point. `after_upload` removes a version after storage completes, `lru` waits for capacity pressure, and `none` retains it. +- Cache eviction is an operational optimization, not the write acceptance point. `after_upload` removes a version after its minimum durable replicas are ready, `lru` waits for capacity pressure, and `none` retains it. ## Planned Replica Repair diff --git a/docs/en/concepts/write-path-cache.md b/docs/en/concepts/write-path-cache.md index e69e249..ceb6c0b 100644 --- a/docs/en/concepts/write-path-cache.md +++ b/docs/en/concepts/write-path-cache.md @@ -39,10 +39,12 @@ Repeated reads of the same version coalesce access-time updates to at most one d | Policy | Behavior | | --- | --- | | `lru` | At the high capacity watermark, queue the least recently accessed remotely safe versions until planned usage reaches the low watermark. | -| `after_upload` | Queue each version for removal after all target remote copies commit. | +| `after_upload` | Queue each version for removal after its bucket's minimum durable copies commit. | | `none` | Do not create or run automatic cache eviction work. | -Only versions with a readable committed remote copy are eligible. Eviction waits for active reads of the same version to close. Because cleanup is asynchronous, writes can still return `507 Insufficient Storage` when cleanup cannot keep pace or no safe candidate exists. +Each bucket defaults to strict cache release, so the minimum equals the target replicas frozen for each upload. An operator can lower the minimum for a bucket to release retained cache while the original replica slots continue syncing. The minimum is clamped to each upload's target. Raising it affects cache that still exists; it cannot recreate cache that has already been deleted. + +Only versions that currently meet their minimum and have a readable committed remote copy are eligible. Eviction checks the current minimum again before authorizing deletion and waits for active reads of the same version to close. Because cleanup is asynchronous, writes can still return `507 Insufficient Storage` when cleanup cannot keep pace or no safe candidate exists. ## Multipart Uploads diff --git a/docs/en/configuration/model.md b/docs/en/configuration/model.md index f4acb6e..ff5e52f 100644 --- a/docs/en/configuration/model.md +++ b/docs/en/configuration/model.md @@ -121,7 +121,7 @@ The login page uses a browser-session cookie by default. Selecting **Keep me sig Cache eviction policies have these user-visible results: - `lru`: when cache usage reaches the high watermark, SynapS3 removes the least recently accessed remotely safe entries until usage reaches the low watermark. -- `after_upload`: after all target remote copies commit, SynapS3 queues that version for removal at the next Evictor poll. A later remote read can restore the cache, and that restored entry is not immediately removed again. +- `after_upload`: after a version meets its bucket's minimum durable copies, SynapS3 queues it for removal at the next Evictor poll. A later remote read can restore the cache, and that restored entry is not immediately removed again. - `none`: SynapS3 does not automatically remove local cache data. The LRU watermarks must always satisfy `0 <= low < high <= 100`. They remain saved but have no effect under `after_upload` or `none`. diff --git a/docs/en/operations/troubleshooting.md b/docs/en/operations/troubleshooting.md index d6b3dff..222a2be 100644 --- a/docs/en/operations/troubleshooting.md +++ b/docs/en/operations/troubleshooting.md @@ -109,10 +109,10 @@ Recovery options: - Confirm the host has free disk space, then increase `cache.max_size_gb` if capacity allows. - Restore storage provider connectivity and background task progress so queued uploads can complete and cache eviction can run. - Use the default `lru` policy for capacity-based cleanup. Lower the high watermark to leave more write headroom, and keep `0 <= low < high <= 100`. -- Use `after_upload` only when each version should be removed on the next Evictor poll after all target remote copies commit. +- Use `after_upload` only when each version should be removed on the next Evictor poll after its bucket's minimum durable copies commit. - Use `none` when automatic removal must be disabled. -LRU cannot remove multipart staging data, versions that are not remotely durable, or versions without a readable committed remote copy. A write does not synchronously run eviction, so `507 Insufficient Storage` can continue until the Evictor catches up or safe candidates become available. +LRU cannot remove multipart staging data, versions below their bucket's minimum durable copies, or versions without a readable committed remote copy. A write does not synchronously run eviction, so `507 Insufficient Storage` can continue until the Evictor catches up or safe candidates become available. Failed LRU deletion tasks remain visible as exhausted work and become eligible again after a one-hour cooldown. Fix the reported filesystem or database problem first; use `synaps3 admin task retry ` to retry sooner. diff --git a/docs/en/reference/admin-api.md b/docs/en/reference/admin-api.md index 6e3c50e..09d2f21 100644 --- a/docs/en/reference/admin-api.md +++ b/docs/en/reference/admin-api.md @@ -114,7 +114,7 @@ Treat these endpoints as change-window operations. They can change data, credent | `POST` | `/api/v1/buckets` | Create a bucket. | | `GET` | `/api/v1/buckets/{name}` | Read bucket detail. | | `PUT` | `/api/v1/buckets/{name}/owner` | Update bucket owner. | -| `PUT` | `/api/v1/buckets/{name}/copy-policy` | Update default copy policy. | +| `PUT` | `/api/v1/buckets/{name}/copy-policy` | Update target replicas and/or the cache-release threshold. | | `DELETE` | `/api/v1/buckets/{name}` | Not supported. Returns `501 Not Implemented`. | | `GET` | `/api/v1/buckets/{name}/objects` | List objects. | | `DELETE` | `/api/v1/buckets/{name}/objects` | Create an object delete marker. | @@ -133,6 +133,17 @@ Treat these endpoints as change-window operations. They can change data, credent For object upload, the HTTP `Content-Type` is the uploaded object's content type. It is not a JSON request marker. +### Bucket Copy Policy + +`POST /api/v1/buckets` accepts optional `default_copies` and `minimum_durable_copies` fields. Bucket list, detail, create, and policy-update responses include: + +- `minimum_durable_copies`: the explicit bucket value, or `null` for strict per-upload behavior; +- `effective_minimum_durable_copies`: the current display value after clamping the bucket minimum to the current target. + +`PUT /api/v1/buckets/{name}/copy-policy` accepts `default_copies` and `minimum_durable_copies` independently. An omitted field is unchanged. `default_copies: null` inherits the current runtime target for new uploads. `minimum_durable_copies: null` requires every replica frozen for each upload before releasing its cache. An explicit minimum must be between `1` and `8` and cannot exceed the target produced by the same request. An empty request or an invalid final combination returns `400 Bad Request`. + +Target changes affect new uploads. Minimum changes also re-evaluate retained cache for current uploads. Increasing the minimum cannot restore cache that has already been deleted. + ### Permanently Delete Object Versions `POST /api/v1/buckets/{name}/objects/permanent-delete` accepts `key` and `version_id`. `POST /api/v1/buckets/{name}/objects/deleted/permanent-delete` accepts `key` and `delete_marker_version_id`. @@ -222,6 +233,8 @@ The restore streams synchronously for up to one hour and requires enough cache c Cache settings expose `eviction_policy`, `lru_high_watermark_percent`, and `lru_low_watermark_percent` under `cache`. Valid policies are `lru`, `after_upload`, and `none`. Watermarks must satisfy `0 <= low < high <= 100` and only affect `lru`. +When the full runtime is available, `GET /api/v1/settings` also returns `runtime_filecoin_default_copies`. This is the value used by the current process. `config.filecoin.default_copies` remains the saved value that takes effect after the next restart. + After saving settings, restart SynapS3, check `/healthz`, and read settings again to confirm the effective values. ## Write Example diff --git a/docs/zh/concepts/filecoin-storage-flow.md b/docs/zh/concepts/filecoin-storage-flow.md index 3107d6a..ba5e6df 100644 --- a/docs/zh/concepts/filecoin-storage-flow.md +++ b/docs/zh/concepts/filecoin-storage-flow.md @@ -31,7 +31,7 @@ flowchart TD | `uploading` | 后台任务正在准备远端存储或上传对象数据。 | | `committing` | 存储提供方已有 piece，commit 步骤正在进行。 | | `replicating` | 至少已有一个可读副本，目标副本数仍在补齐。 | -| `stored` | 目标远端副本策略已满足，并且已有存储元数据。 | +| `stored` | 存储桶要求的最低耐久副本已可读并提交；其余目标副本可能仍在补齐。 | | `failed` | 正在执行的生命周期步骤失败，可重试。 | | `cache_evicted` | 远端持久化后，本地缓存已清理。 | @@ -54,12 +54,18 @@ synaps3 admin task retry 42 如果已建立的存储提供方在首次副本尚未全部完成时暂时不可用，SynapS3 会继续使用其他已分配且可写的副本。未完成副本会等待且不消耗重试次数，并在原存储提供方恢复可达后自动继续；系统不会自动选择替代提供方。已完成存储的副本随后变为不可用时，其修复仍属于下面计划支持的副本修复功能。 +## 目标副本与最低耐久副本 + +上传开始时会冻结目标副本数。默认缓存释放策略保持严格：所有目标副本都必须可读并完成提交。存储桶也可以设置较低的最低耐久副本数。达到该门槛后，版本进入已存储状态，缓存按已配置的淘汰策略处理；上传仍会继续补齐原有副本位，直到达到目标副本数。 + +修改目标副本数只影响新上传。修改最低耐久副本数也会重新评估当前上传仍保留的缓存。提高门槛不会让已经进入已存储状态的版本回退，也无法恢复已经删除的缓存。 + ## 用户能看到什么 - S3 上传可以在 Filecoin 存储完成前成功。 - 仪表盘的任务和拓扑视图会展示存储进度。 - 读取优先使用本地缓存；已有远端元数据时，可以从存储提供方取回对象。 -- 缓存淘汰是运维优化，不是写入接受点。`after_upload` 会在存储完成后清理版本，`lru` 等待容量压力，`none` 则保留本地缓存。 +- 缓存淘汰是运维优化，不是写入接受点。`after_upload` 会在最低耐久副本就绪后清理版本，`lru` 等待容量压力，`none` 则保留本地缓存。 ## 计划支持的副本修复 diff --git a/docs/zh/concepts/write-path-cache.md b/docs/zh/concepts/write-path-cache.md index 3168920..11a37fc 100644 --- a/docs/zh/concepts/write-path-cache.md +++ b/docs/zh/concepts/write-path-cache.md @@ -39,10 +39,12 @@ SynapS3 会校验请求，保存对象及其元数据，再返回 S3 兼容的 E | 策略 | 行为 | | --- | --- | | `lru` | 达到容量高水位后，按最近访问时间为远端安全的版本排队，直到计划使用量降至低水位。 | -| `after_upload` | 所有目标远端副本提交后，为该版本排队清理。 | +| `after_upload` | 存储桶要求的最低耐久副本提交后，为该版本排队清理。 | | `none` | 不创建或执行自动缓存淘汰任务。 | -只有存在可读已提交远端副本的版本才可淘汰。淘汰会等待同一版本正在进行的读取关闭。清理是异步流程，因此清理追赶不及时或没有安全候选时，写入仍可能返回 `507 Insufficient Storage`。 +每个存储桶默认采用严格缓存释放策略，因此最低耐久副本数等于每次上传冻结的目标副本数。运维人员可以降低某个存储桶的门槛，在原副本位继续补齐期间释放仍保留的缓存。该门槛不会超过单次上传的目标副本数。提高门槛只影响尚未删除的缓存，无法恢复已经删除的缓存。 + +只有当前满足最低耐久副本数且存在可读已提交远端副本的版本才可淘汰。系统会在授权删除前再次检查当前门槛，并等待同一版本正在进行的读取关闭。清理是异步流程，因此清理追赶不及时或没有安全候选时，写入仍可能返回 `507 Insufficient Storage`。 ## 分段上传 diff --git a/docs/zh/configuration/model.md b/docs/zh/configuration/model.md index 649b1eb..d082049 100644 --- a/docs/zh/configuration/model.md +++ b/docs/zh/configuration/model.md @@ -121,7 +121,7 @@ SQLite 是 SynapS3 单机部署的默认且推荐数据库。已有 PostgreSQL 缓存淘汰策略会产生以下用户可见结果： - `lru`：缓存使用量达到高水位后，SynapS3 按最近访问时间淘汰最久未使用且远端安全的条目，直到降至低水位。 -- `after_upload`：所有目标远端副本提交后，该版本会在下一次 Evictor 轮询时加入清理。之后从远端读取并回填的缓存不会再次被立即删除。 +- `after_upload`：版本达到其存储桶要求的最低耐久副本数后，会在下一次 Evictor 轮询时加入清理。之后从远端读取并回填的缓存不会再次被立即删除。 - `none`：SynapS3 不会自动清理本地缓存。 LRU 水位始终必须满足 `0 <= low < high <= 100`。在 `after_upload` 或 `none` 下仍会保存这些值，但不会生效。 diff --git a/docs/zh/operations/troubleshooting.md b/docs/zh/operations/troubleshooting.md index 2a813af..9c611a9 100644 --- a/docs/zh/operations/troubleshooting.md +++ b/docs/zh/operations/troubleshooting.md @@ -109,10 +109,10 @@ synaps3 admin settings get cache.lru_low_watermark_percent - 先确认主机仍有可用磁盘空间，再按容量增大 `cache.max_size_gb`。 - 恢复存储提供方连接和后台任务进度，让排队上传完成并触发缓存淘汰。 - 默认的 `lru` 适合按容量自动清理。降低高水位可以为新写入保留更多余量，并始终满足 `0 <= low < high <= 100`。 -- 只有希望所有目标远端副本提交后，在下一次 Evictor 轮询中删除对应版本时，才使用 `after_upload`。 +- 只有希望版本达到存储桶要求的最低耐久副本数后，在下一次 Evictor 轮询中删除对应版本时，才使用 `after_upload`。 - 需要完全禁用自动清理时使用 `none`。 -LRU 无法清理 multipart 暂存数据、尚未远端持久化的版本，或没有可读已提交远端副本的版本。写入不会同步触发淘汰，因此在 Evictor 追赶完成或出现安全候选前，仍可能继续返回 `507 Insufficient Storage`。 +LRU 无法清理 multipart 暂存数据、未达到存储桶最低耐久副本数的版本，或没有可读已提交远端副本的版本。写入不会同步触发淘汰，因此在 Evictor 追赶完成或出现安全候选前，仍可能继续返回 `507 Insufficient Storage`。 LRU 删除失败后，任务仍会作为 exhausted 工作保留，并在一小时冷却后重新具备执行资格。先修复任务中报告的文件系统或数据库问题；需要提前重试时，运行 `synaps3 admin task retry `。 diff --git a/docs/zh/reference/admin-api.md b/docs/zh/reference/admin-api.md index fc62dab..ac6efb0 100644 --- a/docs/zh/reference/admin-api.md +++ b/docs/zh/reference/admin-api.md @@ -114,7 +114,7 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` | `POST` | `/api/v1/buckets` | 创建存储桶。 | | `GET` | `/api/v1/buckets/{name}` | 读取存储桶详情。 | | `PUT` | `/api/v1/buckets/{name}/owner` | 更新存储桶 owner。 | -| `PUT` | `/api/v1/buckets/{name}/copy-policy` | 更新默认 copy policy。 | +| `PUT` | `/api/v1/buckets/{name}/copy-policy` | 更新目标副本数和/或缓存释放门槛。 | | `DELETE` | `/api/v1/buckets/{name}` | 不支持，返回 `501 Not Implemented`。 | | `GET` | `/api/v1/buckets/{name}/objects` | 列出对象。 | | `DELETE` | `/api/v1/buckets/{name}/objects` | 创建对象 delete marker。 | @@ -133,6 +133,17 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` 对象上传时，HTTP `Content-Type` 表示上传对象的内容类型，不是 JSON 请求标记。 +### 存储桶副本策略 + +`POST /api/v1/buckets` 接受可选的 `default_copies` 和 `minimum_durable_copies` 字段。存储桶列表、详情、创建和策略更新响应包含： + +- `minimum_durable_copies`：存储桶显式设置的值；`null` 表示按每次上传采用严格策略； +- `effective_minimum_durable_copies`：将当前存储桶门槛限制在当前目标副本数以内后，用于展示的值。 + +`PUT /api/v1/buckets/{name}/copy-policy` 可以独立接收 `default_copies` 和 `minimum_durable_copies`。字段缺省时保持不变。`default_copies: null` 表示新上传继承当前运行时目标副本数。`minimum_durable_copies: null` 表示必须完成单次上传冻结的所有副本后才能释放缓存。显式门槛必须在 `1` 到 `8` 之间，且不能超过同一请求产生的最终目标副本数。空请求或无效的最终组合返回 `400 Bad Request`。 + +目标副本数变更只影响新上传。最低耐久副本数变更还会重新评估当前上传仍保留的缓存。提高门槛无法恢复已经删除的缓存。 + ### 永久删除对象版本 `POST /api/v1/buckets/{name}/objects/permanent-delete` 接受 `key` 和 `version_id`。`POST /api/v1/buckets/{name}/objects/deleted/permanent-delete` 接受 `key` 和 `delete_marker_version_id`。 @@ -222,6 +233,8 @@ Admin 响应包含 `Content-Security-Policy`、`X-Content-Type-Options: nosniff` 缓存设置在 `cache` 下提供 `eviction_policy`、`lru_high_watermark_percent` 和 `lru_low_watermark_percent`。有效策略为 `lru`、`after_upload` 和 `none`。水位必须满足 `0 <= low < high <= 100`，且只在 `lru` 策略下生效。 +完整运行时可用时，`GET /api/v1/settings` 还会返回 `runtime_filecoin_default_copies`，表示当前进程实际使用的值。`config.filecoin.default_copies` 仍表示已保存、下次重启后生效的值。 + 保存设置后，重启 SynapS3，检查 `/healthz`，再读取设置以确认实际生效值。 ## 写请求示例 diff --git a/internal/admin/api_buckets.go b/internal/admin/api_buckets.go index 9d81e88..720666c 100644 --- a/internal/admin/api_buckets.go +++ b/internal/admin/api_buckets.go @@ -42,48 +42,55 @@ const ( ) type bucketListItem struct { - ID int64 `json:"id"` - Name string `json:"name"` - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - Status string `json:"status"` - ObjectCount int64 `json:"object_count"` - TotalSizeBytes int64 `json:"total_size_bytes"` - StorageHealth bucketStorageHealthSummaryResponse `json:"storage_health"` - CreatedAt string `json:"created_at"` + ID int64 `json:"id"` + Name string `json:"name"` + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies *int `json:"default_copies"` + EffectiveCopies int `json:"effective_copies"` + MinimumDurableCopies *int `json:"minimum_durable_copies"` + EffectiveMinimumDurableCopies int `json:"effective_minimum_durable_copies"` + Status string `json:"status"` + ObjectCount int64 `json:"object_count"` + TotalSizeBytes int64 `json:"total_size_bytes"` + StorageHealth bucketStorageHealthSummaryResponse `json:"storage_health"` + CreatedAt string `json:"created_at"` } type bucketCreateRequest struct { - Name string `json:"name"` - OwnerAccessKey string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` + Name string `json:"name"` + OwnerAccessKey string `json:"owner_access_key"` + DefaultCopies *int `json:"default_copies"` + MinimumDurableCopies *int `json:"minimum_durable_copies"` } type bucketMutationResponse struct { - ID int64 `json:"id"` - Name string `json:"name"` - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - Status string `json:"status"` + ID int64 `json:"id"` + Name string `json:"name"` + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies *int `json:"default_copies"` + EffectiveCopies int `json:"effective_copies"` + MinimumDurableCopies *int `json:"minimum_durable_copies"` + EffectiveMinimumDurableCopies int `json:"effective_minimum_durable_copies"` + Status string `json:"status"` } type bucketDetailResponse struct { - ID int64 `json:"id"` - Name string `json:"name"` - OwnerAccessKey *string `json:"owner_access_key"` - DefaultCopies *int `json:"default_copies"` - EffectiveCopies int `json:"effective_copies"` - Status string `json:"status"` - ObjectCount int64 `json:"object_count"` - TotalSizeBytes int64 `json:"total_size_bytes"` - StorageHealth bucketStorageHealthSummaryResponse `json:"storage_health"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - VersioningStatus string `json:"versioning_status"` - VersioningEnforced bool `json:"versioning_enforced"` - DataSets []storageDataSetSummaryResponse `json:"data_sets"` + ID int64 `json:"id"` + Name string `json:"name"` + OwnerAccessKey *string `json:"owner_access_key"` + DefaultCopies *int `json:"default_copies"` + EffectiveCopies int `json:"effective_copies"` + MinimumDurableCopies *int `json:"minimum_durable_copies"` + EffectiveMinimumDurableCopies int `json:"effective_minimum_durable_copies"` + Status string `json:"status"` + ObjectCount int64 `json:"object_count"` + TotalSizeBytes int64 `json:"total_size_bytes"` + StorageHealth bucketStorageHealthSummaryResponse `json:"storage_health"` + CreatedAt string `json:"created_at"` + UpdatedAt string `json:"updated_at"` + VersioningStatus string `json:"versioning_status"` + VersioningEnforced bool `json:"versioning_enforced"` + DataSets []storageDataSetSummaryResponse `json:"data_sets"` } type storageDataSetSummaryResponse struct { @@ -122,7 +129,8 @@ type bucketOwnerUpdateRequest struct { } type bucketCopyPolicyUpdateRequest struct { - DefaultCopies json.RawMessage `json:"default_copies"` + DefaultCopies json.RawMessage `json:"default_copies"` + MinimumDurableCopies json.RawMessage `json:"minimum_durable_copies"` } func (s *Server) effectiveBucketCopies(bucket *model.Bucket) int { @@ -132,6 +140,14 @@ func (s *Server) effectiveBucketCopies(bucket *model.Bucket) int { return boundedBucketCopies(s.filecoinDefaultCopies) } +func (s *Server) effectiveBucketMinimumDurableCopies(bucket *model.Bucket) int { + target := s.effectiveBucketCopies(bucket) + if bucket == nil || bucket.MinimumDurableCopies == nil || *bucket.MinimumDurableCopies > target { + return target + } + return *bucket.MinimumDurableCopies +} + func boundedBucketCopies(copies int) int { return model.ClampStorageCopies(copies) } @@ -146,21 +162,31 @@ func validateBucketDefaultCopies(copies *int) error { return nil } -func parseBucketDefaultCopies(raw json.RawMessage) (*int, error) { +func validateBucketMinimumDurableCopies(copies *int) error { + if copies == nil { + return nil + } + if !model.ValidStorageCopies(*copies) { + return fmt.Errorf("minimum_durable_copies must be between %d and %d", model.StorageCopiesMin, model.StorageCopiesMax) + } + return nil +} + +func parseBucketCopyPolicyValue(raw json.RawMessage, field string) (*int, bool, error) { if len(raw) == 0 { - return nil, fmt.Errorf("default_copies is required") + return nil, false, nil } if bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { - return nil, nil + return nil, true, nil } var value int if err := json.Unmarshal(raw, &value); err != nil { - return nil, fmt.Errorf("default_copies must be an integer or null") + return nil, true, fmt.Errorf("%s must be an integer or null", field) } - if err := validateBucketDefaultCopies(&value); err != nil { - return nil, err + if !model.ValidStorageCopies(value) { + return nil, true, fmt.Errorf("%s must be between %d and %d", field, model.StorageCopiesMin, model.StorageCopiesMax) } - return &value, nil + return &value, true, nil } func (s *Server) handleAPIListBuckets(w http.ResponseWriter, r *http.Request) { @@ -188,16 +214,18 @@ func (s *Server) handleAPIListBuckets(w http.ResponseWriter, r *http.Request) { } stats := statsMap[b.ID] items = append(items, bucketListItem{ - ID: b.ID, - Name: b.Name, - OwnerAccessKey: s.adminOwnerAccessKey(b.OwnerAccessKey), - DefaultCopies: b.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(&b), - Status: string(b.Status), - ObjectCount: stats.Count, - TotalSizeBytes: stats.TotalSize, - StorageHealth: bucketStorageHealthSummaryForBucket(storageHealthMap, b.ID, storageHealthFailed), - CreatedAt: b.CreatedAt.Format(time.RFC3339), + ID: b.ID, + Name: b.Name, + OwnerAccessKey: s.adminOwnerAccessKey(b.OwnerAccessKey), + DefaultCopies: b.DefaultCopies, + EffectiveCopies: s.effectiveBucketCopies(&b), + MinimumDurableCopies: b.MinimumDurableCopies, + EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(&b), + Status: string(b.Status), + ObjectCount: stats.Count, + TotalSizeBytes: stats.TotalSize, + StorageHealth: bucketStorageHealthSummaryForBucket(storageHealthMap, b.ID, storageHealthFailed), + CreatedAt: b.CreatedAt.Format(time.RFC3339), }) } @@ -227,6 +255,18 @@ func (s *Server) handleAPICreateBucket(w http.ResponseWriter, r *http.Request) { writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } + if err := validateBucketMinimumDurableCopies(req.MinimumDurableCopies); err != nil { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) + return + } + targetCopies := s.filecoinDefaultCopies + if req.DefaultCopies != nil { + targetCopies = *req.DefaultCopies + } + if req.MinimumDurableCopies != nil && *req.MinimumDurableCopies > targetCopies { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "minimum_durable_copies cannot exceed the effective replica target"}) + return + } ownerAccessKey := strings.TrimSpace(req.OwnerAccessKey) actualOwnerAccessKey, ok := s.resolveS3BucketOwner(w, ownerAccessKey, http.StatusBadRequest) if !ok { @@ -249,11 +289,12 @@ func (s *Server) handleAPICreateBucket(w http.ResponseWriter, r *http.Request) { return auth.ErrNoSuchUser } bucket = &model.Bucket{ - Name: name, - ACL: acl, - OwnerAccessKey: &actualOwnerAccessKey, - DefaultCopies: req.DefaultCopies, - Status: model.BucketStatusActive, + Name: name, + ACL: acl, + OwnerAccessKey: &actualOwnerAccessKey, + DefaultCopies: req.DefaultCopies, + MinimumDurableCopies: req.MinimumDurableCopies, + Status: model.BucketStatusActive, } return txRepos.Buckets.Create(r.Context(), bucket) }) @@ -273,12 +314,14 @@ func (s *Server) handleAPICreateBucket(w http.ResponseWriter, r *http.Request) { s.bucketLifecycle.EnsureCacheBucketDir(r.Context(), name) writeJSON(w, http.StatusCreated, bucketMutationResponse{ - ID: bucket.ID, - Name: bucket.Name, - OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), - DefaultCopies: bucket.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(bucket), - Status: string(bucket.Status), + ID: bucket.ID, + Name: bucket.Name, + OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), + DefaultCopies: bucket.DefaultCopies, + EffectiveCopies: s.effectiveBucketCopies(bucket), + MinimumDurableCopies: bucket.MinimumDurableCopies, + EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(bucket), + Status: string(bucket.Status), }) } @@ -320,20 +363,22 @@ func (s *Server) handleAPIGetBucket(w http.ResponseWriter, r *http.Request) { storageHealthMap, storageHealthFailed := s.bucketStorageHealthSummaries(ctx, bucket.ID) writeJSON(w, http.StatusOK, bucketDetailResponse{ - ID: bucket.ID, - Name: bucket.Name, - OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), - DefaultCopies: bucket.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(bucket), - Status: string(bucket.Status), - ObjectCount: stats.Count, - TotalSizeBytes: stats.TotalSize, - StorageHealth: bucketStorageHealthSummaryForBucket(storageHealthMap, bucket.ID, storageHealthFailed), - CreatedAt: bucket.CreatedAt.Format(time.RFC3339), - UpdatedAt: bucket.UpdatedAt.Format(time.RFC3339), - VersioningStatus: "Enabled", - VersioningEnforced: true, - DataSets: dataSets, + ID: bucket.ID, + Name: bucket.Name, + OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), + DefaultCopies: bucket.DefaultCopies, + EffectiveCopies: s.effectiveBucketCopies(bucket), + MinimumDurableCopies: bucket.MinimumDurableCopies, + EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(bucket), + Status: string(bucket.Status), + ObjectCount: stats.Count, + TotalSizeBytes: stats.TotalSize, + StorageHealth: bucketStorageHealthSummaryForBucket(storageHealthMap, bucket.ID, storageHealthFailed), + CreatedAt: bucket.CreatedAt.Format(time.RFC3339), + UpdatedAt: bucket.UpdatedAt.Format(time.RFC3339), + VersioningStatus: "Enabled", + VersioningEnforced: true, + DataSets: dataSets, }) } @@ -393,12 +438,14 @@ func (s *Server) handleAPIUpdateBucketOwner(w http.ResponseWriter, r *http.Reque } writeJSON(w, http.StatusOK, bucketMutationResponse{ - ID: bucket.ID, - Name: bucket.Name, - OwnerAccessKey: s.adminOwnerAccessKey(&actualOwnerAccessKey), - DefaultCopies: bucket.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(bucket), - Status: string(bucket.Status), + ID: bucket.ID, + Name: bucket.Name, + OwnerAccessKey: s.adminOwnerAccessKey(&actualOwnerAccessKey), + DefaultCopies: bucket.DefaultCopies, + EffectiveCopies: s.effectiveBucketCopies(bucket), + MinimumDurableCopies: bucket.MinimumDurableCopies, + EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(bucket), + Status: string(bucket.Status), }) } @@ -414,36 +461,68 @@ func (s *Server) handleAPIUpdateBucketCopyPolicy(w http.ResponseWriter, r *http. if !decodeBucketStrictJSON(w, r, &req) { return } - copies, err := parseBucketDefaultCopies(req.DefaultCopies) + defaultCopies, setDefaultCopies, err := parseBucketCopyPolicyValue(req.DefaultCopies, "default_copies") if err != nil { writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } - - bucket, err := s.repos.Buckets.GetByName(ctx, bucketName) + minimumCopies, setMinimumCopies, err := parseBucketCopyPolicyValue(req.MinimumDurableCopies, "minimum_durable_copies") if err != nil { - s.logger.Error("api: failed to get bucket for copy policy update", "error", err, "name", bucketName) - writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) + writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } - if bucket == nil || !bucket.Status.IsAdminVisible() { + if !setDefaultCopies && !setMinimumCopies { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "copy policy update requires at least one field"}) + return + } + + var bucket *model.Bucket + err = s.repos.WithTx(ctx, func(txRepos *repository.Repositories) error { + updated, err := txRepos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucketName, + SetDefaultCopies: setDefaultCopies, + DefaultCopies: defaultCopies, + SetMinimumDurableCopies: setMinimumCopies, + MinimumDurableCopies: minimumCopies, + }) + if err != nil { + return err + } + if updated == nil || !updated.Status.IsAdminVisible() { + return repository.ErrNotFound + } + if updated.MinimumDurableCopies != nil && *updated.MinimumDurableCopies > s.effectiveBucketCopies(updated) { + return fmt.Errorf("minimum_durable_copies cannot exceed the effective replica target: %w", repository.ErrInvalidInput) + } + if _, err := txRepos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, updated.ID, s.evictMaxRetries); err != nil { + return err + } + bucket = updated + return nil + }) + if errors.Is(err, repository.ErrNotFound) { writeJSON(w, http.StatusNotFound, map[string]string{"error": "bucket not found"}) return } - if err := s.repos.Buckets.SetDefaultCopies(ctx, bucketName, copies); err != nil { + if errors.Is(err, repository.ErrInvalidInput) { + writeJSON(w, http.StatusBadRequest, map[string]string{"error": "minimum_durable_copies cannot exceed the effective replica target"}) + return + } + if err != nil { s.logger.Error("api: failed to update bucket copy policy", "error", err, "name", bucketName) writeJSON(w, http.StatusInternalServerError, map[string]string{"error": "internal"}) return } - bucket.DefaultCopies = copies writeJSON(w, http.StatusOK, bucketMutationResponse{ - ID: bucket.ID, - Name: bucket.Name, - OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), - DefaultCopies: bucket.DefaultCopies, - EffectiveCopies: s.effectiveBucketCopies(bucket), - Status: string(bucket.Status), + ID: bucket.ID, + Name: bucket.Name, + OwnerAccessKey: s.adminOwnerAccessKey(bucket.OwnerAccessKey), + DefaultCopies: bucket.DefaultCopies, + EffectiveCopies: s.effectiveBucketCopies(bucket), + MinimumDurableCopies: bucket.MinimumDurableCopies, + EffectiveMinimumDurableCopies: s.effectiveBucketMinimumDurableCopies(bucket), + Status: string(bucket.Status), }) } diff --git a/internal/admin/api_buckets_test.go b/internal/admin/api_buckets_test.go index 7938c7d..47775da 100644 --- a/internal/admin/api_buckets_test.go +++ b/internal/admin/api_buckets_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/config" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" @@ -734,7 +735,7 @@ func (r *recordingObjectListRepo) list(prefix string, include func(string) bool, func TestHandleAPIBuckets_CreateBucket(t *testing.T) { srv, repos := newBucketAPITestServerWithS3UsersAndRuntimeCopies(t, 2, "owner-access") - req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets", strings.NewReader(`{"name":"admin-create-bucket","owner_access_key":"owner-access","default_copies":4}`)) + req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets", strings.NewReader(`{"name":"admin-create-bucket","owner_access_key":"owner-access","default_copies":4,"minimum_durable_copies":2}`)) req.Header.Set("Content-Type", "application/json") setBucketWriteHeaders(req) rr := httptest.NewRecorder() @@ -766,12 +767,17 @@ func TestHandleAPIBuckets_CreateBucket(t *testing.T) { if bucket.DefaultCopies == nil || *bucket.DefaultCopies != 4 { t.Fatalf("bucket default_copies = %v, want 4", bucket.DefaultCopies) } + if bucket.MinimumDurableCopies == nil || *bucket.MinimumDurableCopies != 2 { + t.Fatalf("bucket minimum_durable_copies = %v, want 2", bucket.MinimumDurableCopies) + } var body struct { Name string `json:"name"` OwnerAccessKey *string `json:"owner_access_key"` DefaultCopies *int `json:"default_copies"` EffectiveCopies int `json:"effective_copies"` + MinimumCopies *int `json:"minimum_durable_copies"` + EffectiveMin int `json:"effective_minimum_durable_copies"` } if err := json.NewDecoder(rr.Body).Decode(&body); err != nil { t.Fatalf("Decode response: %v", err) @@ -782,6 +788,9 @@ func TestHandleAPIBuckets_CreateBucket(t *testing.T) { if body.DefaultCopies == nil || *body.DefaultCopies != 4 || body.EffectiveCopies != 4 { t.Fatalf("copy policy response = default:%v effective:%d, want 4/4", body.DefaultCopies, body.EffectiveCopies) } + if body.MinimumCopies == nil || *body.MinimumCopies != 2 || body.EffectiveMin != 2 { + t.Fatalf("minimum copy policy response = minimum:%v effective:%d, want 2/2", body.MinimumCopies, body.EffectiveMin) + } } func TestHandleAPIBuckets_CreateBucketAllowsInternalRootOwner(t *testing.T) { @@ -877,6 +886,30 @@ func TestHandleAPIBuckets_CreateBucketRejectsMalformedStrictJSON(t *testing.T) { } } +func TestHandleAPIBuckets_CreateBucketRejectsMinimumAboveTarget(t *testing.T) { + srv, repos := newBucketAPITestServerWithS3UsersAndRuntimeCopies(t, 3, "owner-access") + for _, body := range []string{ + `{"name":"invalid-explicit-minimum","owner_access_key":"owner-access","default_copies":2,"minimum_durable_copies":3}`, + `{"name":"invalid-inherited-minimum","owner_access_key":"owner-access","minimum_durable_copies":4}`, + } { + req := httptest.NewRequest(http.MethodPost, "/api/v1/buckets", strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + srv.handleAPICreateBucket(rr, req) + if rr.Code != http.StatusBadRequest { + t.Fatalf("status = %d body=%s, want bad request", rr.Code, rr.Body.String()) + } + } + buckets, err := repos.Buckets.List(context.Background()) + if err != nil { + t.Fatalf("Buckets.List: %v", err) + } + if len(buckets) != 0 { + t.Fatalf("invalid requests created buckets: %#v", buckets) + } +} + func TestAPIBucketDetail(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() @@ -2563,6 +2596,119 @@ func TestAPIBucketCopyPolicy_UpdateAndClear(t *testing.T) { } } +func TestAPIBucketCopyPolicy_IndependentFieldsValidateFinalPolicyAndUseOneCoordinator(t *testing.T) { + srv, repos := newBucketAPITestServerWithRuntimeCopies(t, 5) + ctx := context.Background() + bucket := &model.Bucket{Name: "independent-copy-policy-bucket", Status: model.BucketStatusActive} + if err := repos.Buckets.Create(ctx, bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) + } + + update := func(body string) *httptest.ResponseRecorder { + t.Helper() + req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/independent-copy-policy-bucket/copy-policy", strings.NewReader(body)) + req.SetPathValue("name", bucket.Name) + req.Header.Set("Content-Type", "application/json") + setBucketWriteHeaders(req) + rr := httptest.NewRecorder() + srv.handleAPIUpdateBucketCopyPolicy(rr, req) + return rr + } + assertStored := func(wantTarget, wantMinimum *int) { + t.Helper() + got, err := repos.Buckets.GetByName(ctx, bucket.Name) + if err != nil || got == nil { + t.Fatalf("GetByName: bucket=%#v err=%v", got, err) + } + if !reflect.DeepEqual(got.DefaultCopies, wantTarget) || !reflect.DeepEqual(got.MinimumDurableCopies, wantMinimum) { + t.Fatalf("stored policy = target:%v minimum:%v, want target:%v minimum:%v", got.DefaultCopies, got.MinimumDurableCopies, wantTarget, wantMinimum) + } + } + + targetFour, minimumTwo := 4, 2 + if rr := update(`{"default_copies":4,"minimum_durable_copies":2}`); rr.Code != http.StatusOK { + t.Fatalf("joint update status = %d body=%s", rr.Code, rr.Body.String()) + } + assertStored(&targetFour, &minimumTwo) + + minimumThree := 3 + if rr := update(`{"minimum_durable_copies":3}`); rr.Code != http.StatusOK { + t.Fatalf("minimum-only update status = %d body=%s", rr.Code, rr.Body.String()) + } + assertStored(&targetFour, &minimumThree) + + targetThree := 3 + if rr := update(`{"default_copies":3}`); rr.Code != http.StatusOK { + t.Fatalf("target-only update status = %d body=%s", rr.Code, rr.Body.String()) + } + assertStored(&targetThree, &minimumThree) + + if rr := update(`{"default_copies":2}`); rr.Code != http.StatusBadRequest { + t.Fatalf("invalid target reduction status = %d body=%s", rr.Code, rr.Body.String()) + } + assertStored(&targetThree, &minimumThree) + + if rr := update(`{"minimum_durable_copies":null}`); rr.Code != http.StatusOK { + t.Fatalf("clear minimum status = %d body=%s", rr.Code, rr.Body.String()) + } else { + var response struct { + Minimum *int `json:"minimum_durable_copies"` + Effective int `json:"effective_minimum_durable_copies"` + Target *int `json:"default_copies"` + TargetValue int `json:"effective_copies"` + } + if err := json.NewDecoder(rr.Body).Decode(&response); err != nil { + t.Fatalf("Decode clear minimum response: %v", err) + } + if response.Minimum != nil || response.Effective != 3 || response.Target == nil || *response.Target != 3 || response.TargetValue != 3 { + t.Fatalf("clear minimum response = %#v, want strict 3/3", response) + } + } + assertStored(&targetThree, nil) + + tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeEvictCache), cacheeviction.StageReconcileBucketDurability, "", 10, 0) + if err != nil { + t.Fatalf("List coordinator tasks: %v", err) + } + if total != 1 || len(tasks) != 1 || tasks[0].RefType != "bucket" || tasks[0].RefID != bucket.ID { + t.Fatalf("coordinator tasks total=%d tasks=%#v, want one bucket task", total, tasks) + } +} + +func TestAPIBucketCopyPolicy_EffectiveMinimumClampsWithoutRewritingStoredValue(t *testing.T) { + srv, repos := newBucketAPITestServerWithRuntimeCopies(t, 2) + minimum := 5 + bucket := &model.Bucket{ + Name: "clamped-copy-policy-bucket", + MinimumDurableCopies: &minimum, + Status: model.BucketStatusActive, + } + if err := repos.Buckets.Create(context.Background(), bucket); err != nil { + t.Fatalf("Buckets.Create: %v", err) + } + req := httptest.NewRequest(http.MethodGet, "/api/v1/buckets/clamped-copy-policy-bucket", nil) + req.SetPathValue("name", bucket.Name) + rr := httptest.NewRecorder() + srv.handleAPIGetBucket(rr, req) + if rr.Code != http.StatusOK { + t.Fatalf("GET status = %d body=%s", rr.Code, rr.Body.String()) + } + var response struct { + Minimum *int `json:"minimum_durable_copies"` + Effective int `json:"effective_minimum_durable_copies"` + } + if err := json.NewDecoder(rr.Body).Decode(&response); err != nil { + t.Fatalf("Decode response: %v", err) + } + if response.Minimum == nil || *response.Minimum != 5 || response.Effective != 2 { + t.Fatalf("minimum response = stored:%v effective:%d, want 5 clamped to 2", response.Minimum, response.Effective) + } + stored, err := repos.Buckets.GetByName(context.Background(), bucket.Name) + if err != nil || stored == nil || stored.MinimumDurableCopies == nil || *stored.MinimumDurableCopies != 5 { + t.Fatalf("stored bucket after GET = %#v err=%v, want unchanged minimum 5", stored, err) + } +} + func TestAPIBucketCopyPolicy_RejectsInvalidPayloads(t *testing.T) { srv, repos := newBucketAPITestServer(t) ctx := context.Background() @@ -2581,6 +2727,9 @@ func TestAPIBucketCopyPolicy_RejectsInvalidPayloads(t *testing.T) { {name: "fractional copies", body: `{"default_copies":3.5}`}, {name: "zero copies", body: `{"default_copies":0}`}, {name: "too many copies", body: `{"default_copies":9}`}, + {name: "zero minimum", body: `{"minimum_durable_copies":0}`}, + {name: "too many minimum", body: `{"minimum_durable_copies":9}`}, + {name: "minimum exceeds target", body: `{"default_copies":2,"minimum_durable_copies":3}`}, } { t.Run(tc.name, func(t *testing.T) { req := httptest.NewRequest(http.MethodPut, "/api/v1/buckets/invalid-copy-policy-bucket/copy-policy", strings.NewReader(tc.body)) diff --git a/internal/admin/api_settings.go b/internal/admin/api_settings.go index 7bd89d9..7c20d95 100644 --- a/internal/admin/api_settings.go +++ b/internal/admin/api_settings.go @@ -326,19 +326,20 @@ func cloneConfig(cfg *config.Config) *config.Config { } type settingsResponse struct { - Mode string `json:"mode"` - ConfigPath string `json:"config_path"` - Writable bool `json:"writable"` - RuntimeAvailable bool `json:"runtime_available"` - RestartRequired bool `json:"restart_required"` - S3Users settingsS3UsersStatus `json:"s3_users"` - Config settingsEditableConfig `json:"config"` - Manual settingsManualConfig `json:"manual"` - Secrets settingsSecretStatus `json:"secrets"` - Metadata map[string]config.FieldMetadata `json:"metadata"` - Defaults settingsDefaults `json:"defaults"` - EnvManaged map[string]string `json:"env_managed"` - ValidationErrors []config.FieldError `json:"validation_errors,omitempty"` + Mode string `json:"mode"` + ConfigPath string `json:"config_path"` + Writable bool `json:"writable"` + RuntimeAvailable bool `json:"runtime_available"` + RuntimeFilecoinDefaultCopies *int `json:"runtime_filecoin_default_copies,omitempty"` + RestartRequired bool `json:"restart_required"` + S3Users settingsS3UsersStatus `json:"s3_users"` + Config settingsEditableConfig `json:"config"` + Manual settingsManualConfig `json:"manual"` + Secrets settingsSecretStatus `json:"secrets"` + Metadata map[string]config.FieldMetadata `json:"metadata"` + Defaults settingsDefaults `json:"defaults"` + EnvManaged map[string]string `json:"env_managed"` + ValidationErrors []config.FieldError `json:"validation_errors,omitempty"` } type settingsDefaults struct { @@ -776,6 +777,10 @@ func (s *Server) readSettingsUpdateRequest(w http.ResponseWriter, r *http.Reques func (s *Server) decorateSettingsResponse(resp settingsResponse) settingsResponse { resp.S3Users = s.s3UsersStatus() resp.RuntimeAvailable = !s.setupOnly + if resp.RuntimeAvailable { + copies := s.filecoinDefaultCopies + resp.RuntimeFilecoinDefaultCopies = &copies + } return resp } diff --git a/internal/admin/api_settings_test.go b/internal/admin/api_settings_test.go index 70ec4a3..91aa8e3 100644 --- a/internal/admin/api_settings_test.go +++ b/internal/admin/api_settings_test.go @@ -139,14 +139,24 @@ func TestSettingsGETReportsRuntimeObservability(t *testing.T) { } runtimeSrv := &Server{ - addr: "127.0.0.1:9090", - settings: setupSrv.settings, - logger: testLogger(), + addr: "127.0.0.1:9090", + settings: setupSrv.settings, + filecoinDefaultCopies: 4, + logger: testLogger(), } runtimeResp := getSettingsResponse(t, runtimeSrv) if !runtimeResp.RuntimeAvailable { t.Fatalf("runtime runtime_available = false, want true") } + if setupResp.RuntimeFilecoinDefaultCopies != nil { + t.Fatalf("setup runtime copies = %v, want unavailable", setupResp.RuntimeFilecoinDefaultCopies) + } + if runtimeResp.RuntimeFilecoinDefaultCopies == nil || *runtimeResp.RuntimeFilecoinDefaultCopies != 4 { + t.Fatalf("runtime copies = %v, want current process value 4", runtimeResp.RuntimeFilecoinDefaultCopies) + } + if runtimeResp.Config.Filecoin.DefaultCopies == 4 { + t.Fatal("saved next-start value unexpectedly matches injected runtime value; test no longer proves the distinction") + } } func TestSettingsGETIncludesFilecoinRPCDefaults(t *testing.T) { diff --git a/internal/admin/server.go b/internal/admin/server.go index 921edbc..3bc4cc9 100644 --- a/internal/admin/server.go +++ b/internal/admin/server.go @@ -60,6 +60,7 @@ type Server struct { s3IAM auth.IAMService s3RootAccess string filecoinDefaultCopies int + evictMaxRetries int storageCleanupMaxRetries int setupOnly bool logger *slog.Logger @@ -105,6 +106,7 @@ func New( taskDiagnosticChecker: synapse.NewPDPStatusChecker(synapse.PDPStatusCheckerOptions{}), events: newAdminEventHub(), filecoinDefaultCopies: boundedBucketCopies(filecoinDefaultCopies), + evictMaxRetries: 5, storageCleanupMaxRetries: 5, logger: logger, startedAt: time.Now(), @@ -210,6 +212,12 @@ func (s *Server) WithStorageCleanupMaxRetries(maxRetries int) *Server { return s } +// WithEvictMaxRetries configures max retries for eviction tasks created by admin actions. +func (s *Server) WithEvictMaxRetries(maxRetries int) *Server { + s.evictMaxRetries = maxRetries + return s +} + // Run starts the admin HTTP server at its configured address. func (s *Server) Run(ctx context.Context) error { listener, err := net.Listen("tcp", s.addr) diff --git a/internal/app/runtime.go b/internal/app/runtime.go index b3a5c37..9e2e50a 100644 --- a/internal/app/runtime.go +++ b/internal/app/runtime.go @@ -189,6 +189,7 @@ func NewRuntime(ctx context.Context, opts RuntimeOptions) (_ *Runtime, err error WithSettings(opts.Settings). WithFilecoinReadiness(opts.Filecoin.Readiness). WithObservability(observabilityService). + WithEvictMaxRetries(cfg.Worker.Evictor.MaxRetries). WithStorageCleanupMaxRetries(cfg.Worker.StorageCleanup.MaxRetries). WithS3IAM(iamService, rootAccount.Access) if opts.ProviderIdentity != nil { diff --git a/internal/cacheeviction/task.go b/internal/cacheeviction/task.go index 7e13d2c..2077313 100644 --- a/internal/cacheeviction/task.go +++ b/internal/cacheeviction/task.go @@ -9,14 +9,26 @@ import ( ) const ( - StageLRU = "lru" - StageAfterUpload = "after_upload" + StageLRU = "lru" + StageAfterUpload = "after_upload" + StageReconcileBucketDurability = "reconcile_bucket_durability" - lruAccessedAtPayloadKey = "cache_accessed_at" - lruTaskKeyPrefix = "evict_cache:lru:" - afterUploadTaskKeyPrefix = "evict_cache:" + lruAccessedAtPayloadKey = "cache_accessed_at" + deleteAuthorizedPayloadKey = "delete_authorized" + lruTaskKeyPrefix = "evict_cache:lru:" + afterUploadTaskKeyPrefix = "evict_cache:" + bucketDurabilityTaskKeyPrefix = "evict_cache:bucket_durability:" ) +// ErrDurabilityThreshold means the current Bucket policy does not authorize deletion. +var ErrDurabilityThreshold = errors.New("minimum durable copies not met") + +// ErrNoLongerEligible means a planned cache entry no longer matches the deletion contract. +var ErrNoLongerEligible = errors.New("cache entry is no longer eligible") + +// ErrAccessChanged means an LRU candidate was accessed after it was planned. +var ErrAccessChanged = errors.New("cache access snapshot changed") + // Candidate is the persisted snapshot needed to plan one LRU eviction. type Candidate struct { ObjectID int64 `bun:"object_id"` @@ -30,6 +42,13 @@ type LRUTaskPayload struct { AccessedAt time.Time } +// AuthorizedDeletion is the persisted decision needed to remove one cache +// entry outside the database transaction that approved it. +type AuthorizedDeletion struct { + Version model.ObjectVersion + BucketName string +} + // NormalizeAccessTime matches the timestamp precision supported by both // PostgreSQL and SQLite persistence paths. func NormalizeAccessTime(value time.Time) time.Time { @@ -72,6 +91,21 @@ func NewAfterUploadTask(objectID int64, versionID string, maxRetries int, schedu } } +// NewBucketDurabilityTask builds the singleton reconciliation task for one bucket. +func NewBucketDurabilityTask(bucketID int64, maxRetries int, scheduledAt time.Time) *model.Task { + stage := StageReconcileBucketDurability + return &model.Task{ + Type: model.TaskTypeEvictCache, + Stage: &stage, + RefType: "bucket", + RefID: bucketID, + IdempotencyKey: fmt.Sprintf("%s%d", bucketDurabilityTaskKeyPrefix, bucketID), + Status: model.TaskStatusQueued, + MaxRetries: maxRetries, + ScheduledAt: scheduledAt, + } +} + // ParseLRUTaskPayload validates and decodes the persisted LRU access snapshot. func ParseLRUTaskPayload(task *model.Task) (LRUTaskPayload, error) { if task == nil { @@ -92,6 +126,33 @@ func ParseLRUTaskPayload(task *model.Task) (LRUTaskPayload, error) { return LRUTaskPayload{AccessedAt: NormalizeAccessTime(accessedAt)}, nil } +// DeleteAuthorized reports whether the task has crossed the durable deletion +// authorization boundary. +func DeleteAuthorized(task *model.Task) (bool, error) { + if task == nil || task.Payload == nil { + return false, nil + } + raw, ok := task.Payload[deleteAuthorizedPayloadKey] + if !ok { + return false, nil + } + authorized, ok := raw.(bool) + if !ok { + return false, fmt.Errorf("cache eviction task delete_authorized has type %T, want bool", raw) + } + return authorized, nil +} + +// WithDeleteAuthorization copies payload before recording an authorization. +func WithDeleteAuthorization(payload map[string]any) map[string]any { + out := make(map[string]any, len(payload)+1) + for key, value := range payload { + out[key] = value + } + out[deleteAuthorizedPayloadKey] = true + return out +} + func (p LRUTaskPayload) taskPayload() map[string]any { return map[string]any{ lruAccessedAtPayloadKey: NormalizeAccessTime(p.AccessedAt).Format(time.RFC3339Nano), diff --git a/internal/db/migrations/2026081901_bucket_minimum_durable_copies.go b/internal/db/migrations/2026081901_bucket_minimum_durable_copies.go new file mode 100644 index 0000000..eb326e8 --- /dev/null +++ b/internal/db/migrations/2026081901_bucket_minimum_durable_copies.go @@ -0,0 +1,31 @@ +package migrations + +import ( + "context" + "fmt" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" +) + +func init() { + Migrations.MustRegister(up2026081901BucketMinimumDurableCopies, down2026081901BucketMinimumDurableCopies) +} + +func up2026081901BucketMinimumDurableCopies(ctx context.Context, db *bun.DB) error { + query := "ALTER TABLE buckets ADD COLUMN minimum_durable_copies INTEGER CHECK (minimum_durable_copies IS NULL OR (minimum_durable_copies >= 1 AND minimum_durable_copies <= 8))" + if db.Dialect().Name() == dialect.PG { + query = "ALTER TABLE buckets ADD COLUMN minimum_durable_copies INTEGER CONSTRAINT chk_buckets_minimum_durable_copies CHECK (minimum_durable_copies IS NULL OR (minimum_durable_copies >= 1 AND minimum_durable_copies <= 8))" + } + if _, err := db.ExecContext(ctx, query); err != nil { + return fmt.Errorf("adding buckets.minimum_durable_copies: %w", err) + } + return nil +} + +func down2026081901BucketMinimumDurableCopies(ctx context.Context, db *bun.DB) error { + if _, err := db.ExecContext(ctx, "ALTER TABLE buckets DROP COLUMN minimum_durable_copies"); err != nil { + return fmt.Errorf("dropping buckets.minimum_durable_copies: %w", err) + } + return nil +} diff --git a/internal/db/migrations/bucket_minimum_durable_copies_test.go b/internal/db/migrations/bucket_minimum_durable_copies_test.go new file mode 100644 index 0000000..7bda13b --- /dev/null +++ b/internal/db/migrations/bucket_minimum_durable_copies_test.go @@ -0,0 +1,57 @@ +package migrations + +import ( + "context" + "database/sql" + "testing" + + "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect/sqlitedialect" + + _ "modernc.org/sqlite" +) + +func TestBucketMinimumDurableCopiesMigrationAddsNullableBoundedColumn(t *testing.T) { + sqldb, err := sql.Open("sqlite", "file:bucket_minimum_durable_copies?mode=memory&cache=shared") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + sqldb.SetMaxOpenConns(1) + db := bun.NewDB(sqldb, sqlitedialect.New()) + t.Cleanup(func() { _ = db.Close() }) + ctx := context.Background() + + if _, err := db.ExecContext(ctx, `CREATE TABLE buckets (id INTEGER PRIMARY KEY, name TEXT NOT NULL)`); err != nil { + t.Fatalf("create buckets: %v", err) + } + if _, err := db.ExecContext(ctx, `INSERT INTO buckets (id, name) VALUES (1, 'existing')`); err != nil { + t.Fatalf("insert existing bucket: %v", err) + } + if err := up2026081901BucketMinimumDurableCopies(ctx, db); err != nil { + t.Fatalf("up migration: %v", err) + } + if !sqliteColumnExists(t, db, "buckets", "minimum_durable_copies") { + t.Fatal("buckets.minimum_durable_copies column missing") + } + var minimum sql.NullInt64 + if err := db.NewRaw(`SELECT minimum_durable_copies FROM buckets WHERE id = 1`).Scan(ctx, &minimum); err != nil { + t.Fatalf("select existing bucket minimum: %v", err) + } + if minimum.Valid { + t.Fatalf("existing bucket minimum = %d, want NULL", minimum.Int64) + } + for _, invalid := range []int{0, 9} { + if _, err := db.ExecContext(ctx, `UPDATE buckets SET minimum_durable_copies = ? WHERE id = 1`, invalid); err == nil { + t.Fatalf("minimum_durable_copies=%d accepted, want check failure", invalid) + } + } + if _, err := db.ExecContext(ctx, `UPDATE buckets SET minimum_durable_copies = 2 WHERE id = 1`); err != nil { + t.Fatalf("set valid minimum: %v", err) + } + if err := down2026081901BucketMinimumDurableCopies(ctx, db); err != nil { + t.Fatalf("down migration: %v", err) + } + if sqliteColumnExists(t, db, "buckets", "minimum_durable_copies") { + t.Fatal("buckets.minimum_durable_copies still exists after down migration") + } +} diff --git a/internal/db/repository/bucket_repo.go b/internal/db/repository/bucket_repo.go index 196881b..693e2f6 100644 --- a/internal/db/repository/bucket_repo.go +++ b/internal/db/repository/bucket_repo.go @@ -133,21 +133,87 @@ func (r *BunBucketRepo) SetOwnerAndACL(ctx context.Context, name string, ownerAc return nil } -func (r *BunBucketRepo) SetDefaultCopies(ctx context.Context, name string, copies *int) error { - res, err := r.db.NewUpdate(). +func (r *BunBucketRepo) UpdateCopyPolicy(ctx context.Context, input UpdateBucketCopyPolicyInput) (*model.Bucket, error) { + bucket, err := lockBucketByName(ctx, r.db, input.Name) + if err != nil || bucket == nil { + return bucket, err + } + + update := r.db.NewUpdate(). Model((*model.Bucket)(nil)). - Set("default_copies = ?", copies). Set("updated_at = ?", time.Now().UTC()). + Where("id = ?", bucket.ID) + if input.SetDefaultCopies { + bucket.DefaultCopies = input.DefaultCopies + update = update.Set("default_copies = ?", input.DefaultCopies) + } + if input.SetMinimumDurableCopies { + bucket.MinimumDurableCopies = input.MinimumDurableCopies + update = update.Set("minimum_durable_copies = ?", input.MinimumDurableCopies) + } + if _, err := update.Exec(ctx); err != nil { + return nil, fmt.Errorf("updating bucket copy policy: %w", err) + } + return bucket, nil +} + +func (r *BunBucketRepo) SetDefaultCopies(ctx context.Context, name string, copies *int) error { + bucket, err := r.UpdateCopyPolicy(ctx, UpdateBucketCopyPolicyInput{ + Name: name, + SetDefaultCopies: true, + DefaultCopies: copies, + }) + if err != nil { + return err + } + if bucket == nil { + return fmt.Errorf("setting bucket default copies: bucket %q not found", name) + } + return nil +} + +func lockBucketByName(ctx context.Context, db bun.IDB, name string) (*model.Bucket, error) { + lockResult, err := db.NewUpdate(). + Model((*model.Bucket)(nil)). + Set("updated_at = updated_at"). Where("name = ?", name). Exec(ctx) if err != nil { - return fmt.Errorf("setting bucket default copies: %w", err) + return nil, fmt.Errorf("locking bucket copy policy: %w", err) } - rows, _ := res.RowsAffected() + rows, _ := lockResult.RowsAffected() if rows == 0 { - return fmt.Errorf("setting bucket default copies: bucket %q not found", name) + return nil, nil } - return nil + + bucket := new(model.Bucket) + if err := db.NewSelect(). + Model(bucket). + Where("name = ?", name). + Scan(ctx); err != nil { + return nil, fmt.Errorf("locking bucket copy policy: %w", err) + } + return bucket, nil +} + +func lockBucketByID(ctx context.Context, db bun.IDB, id int64) (*model.Bucket, error) { + lockResult, err := db.NewUpdate(). + Model((*model.Bucket)(nil)). + Set("updated_at = updated_at"). + Where("id = ?", id). + Exec(ctx) + if err != nil { + return nil, fmt.Errorf("locking bucket %d: %w", id, err) + } + rows, _ := lockResult.RowsAffected() + if rows == 0 { + return nil, nil + } + bucket := new(model.Bucket) + if err := db.NewSelect().Model(bucket).Where("id = ?", id).Scan(ctx); err != nil { + return nil, fmt.Errorf("loading locked bucket %d: %w", id, err) + } + return bucket, nil } func (r *BunBucketRepo) CountByOwner(ctx context.Context, ownerAccessKey string) (int, error) { diff --git a/internal/db/repository/bucket_repo_test.go b/internal/db/repository/bucket_repo_test.go index 2b97394..8678515 100644 --- a/internal/db/repository/bucket_repo_test.go +++ b/internal/db/repository/bucket_repo_test.go @@ -69,7 +69,7 @@ func TestBucketRepo_GetByID(t *testing.T) { } } -func TestBucketRepo_SetDefaultCopies(t *testing.T) { +func TestBucketRepo_UpdateCopyPolicy(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) ctx := context.Background() @@ -80,37 +80,62 @@ func TestBucketRepo_SetDefaultCopies(t *testing.T) { } copies := 4 - if err := repos.Buckets.SetDefaultCopies(ctx, bucket.Name, &copies); err != nil { - t.Fatalf("SetDefaultCopies set: %v", err) + minimum := 2 + updated, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetDefaultCopies: true, + DefaultCopies: &copies, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimum, + }) + if err != nil { + t.Fatalf("UpdateCopyPolicy set: %v", err) + } + if updated == nil || updated.DefaultCopies == nil || *updated.DefaultCopies != copies || + updated.MinimumDurableCopies == nil || *updated.MinimumDurableCopies != minimum { + t.Fatalf("UpdateCopyPolicy result = %#v, want target/minimum %d/%d", updated, copies, minimum) } got, err := repos.Buckets.GetByName(ctx, bucket.Name) if err != nil { t.Fatalf("GetByName after set: %v", err) } - if got == nil || got.DefaultCopies == nil || *got.DefaultCopies != copies { - t.Fatalf("DefaultCopies after set = %#v, want %d", got, copies) + if got == nil || got.DefaultCopies == nil || *got.DefaultCopies != copies || + got.MinimumDurableCopies == nil || *got.MinimumDurableCopies != minimum { + t.Fatalf("copy policy after set = %#v, want target/minimum %d/%d", got, copies, minimum) } - if err := repos.Buckets.SetDefaultCopies(ctx, bucket.Name, nil); err != nil { - t.Fatalf("SetDefaultCopies clear: %v", err) + updated, err = repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + }) + if err != nil { + t.Fatalf("UpdateCopyPolicy clear minimum: %v", err) + } + if updated == nil || updated.DefaultCopies == nil || *updated.DefaultCopies != copies || updated.MinimumDurableCopies != nil { + t.Fatalf("UpdateCopyPolicy clear result = %#v, want target %d and strict minimum", updated, copies) } got, err = repos.Buckets.GetByName(ctx, bucket.Name) if err != nil { t.Fatalf("GetByName after clear: %v", err) } - if got == nil || got.DefaultCopies != nil { - t.Fatalf("DefaultCopies after clear = %#v, want nil", got) + if got == nil || got.DefaultCopies == nil || *got.DefaultCopies != copies || got.MinimumDurableCopies != nil { + t.Fatalf("copy policy after minimum clear = %#v, want target %d and strict minimum", got, copies) } } -func TestBucketRepo_SetDefaultCopiesMissingBucket(t *testing.T) { +func TestBucketRepo_UpdateCopyPolicyMissingBucket(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) ctx := context.Background() copies := 3 - if err := repos.Buckets.SetDefaultCopies(ctx, "missing-copies-policy", &copies); err == nil { - t.Fatal("SetDefaultCopies missing bucket succeeded, want error") + updated, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: "missing-copies-policy", + SetDefaultCopies: true, + DefaultCopies: &copies, + }) + if err != nil || updated != nil { + t.Fatalf("UpdateCopyPolicy missing bucket = %#v, %v; want nil, nil", updated, err) } } diff --git a/internal/db/repository/cache_eviction_repo.go b/internal/db/repository/cache_eviction_repo.go index cbab261..a76826c 100644 --- a/internal/db/repository/cache_eviction_repo.go +++ b/internal/db/repository/cache_eviction_repo.go @@ -2,6 +2,7 @@ package repository import ( "context" + "database/sql" "errors" "fmt" "time" @@ -9,16 +10,304 @@ import ( "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/model" "github.com/uptrace/bun" + "github.com/uptrace/bun/dialect" ) +func (r *BunCacheEvictionRepo) AuthorizeDeletion( + ctx context.Context, + task *model.Task, + expectedAccess *time.Time, +) (*cacheeviction.AuthorizedDeletion, error) { + if task == nil || task.RefVersionID == "" { + return nil, fmt.Errorf("cache eviction task target is required: %w", ErrInvalidInput) + } + preflight, err := r.objectVersionByID(ctx, r.db, task.RefVersionID) + if err != nil { + return nil, err + } + if preflight.StorageUploadID == nil { + return nil, cacheeviction.ErrNoLongerEligible + } + + var authorized *cacheeviction.AuthorizedDeletion + err = r.runMaybeTx(ctx, func(db bun.IDB) error { + bucket, upload, version, lockedTask, err := lockCacheEvictionContext( + ctx, + db, + task, + preflight.BucketID, + *preflight.StorageUploadID, + preflight.VersionID, + ) + if err != nil { + return err + } + if lockedTask.RefVersionID != version.VersionID { + return fmt.Errorf("cache eviction task target changed: %w", ErrConflict) + } + alreadyAuthorized, err := cacheeviction.DeleteAuthorized(lockedTask) + if err != nil { + return err + } + if !alreadyAuthorized { + if lockedTask.RefType != "object" { + return fmt.Errorf("cache eviction task is not an object deletion: %w", ErrConflict) + } + if err := requireMinimumDurability(ctx, db, bucket, upload); err != nil { + return err + } + if !cacheDeletionStateEligible(version) { + return cacheeviction.ErrNoLongerEligible + } + if expectedAccess != nil && !cacheeviction.NormalizeAccessTime(cacheAccessTime(version)).Equal(cacheeviction.NormalizeAccessTime(*expectedAccess)) { + return cacheeviction.ErrAccessChanged + } + payload := cacheeviction.WithDeleteAuthorization(lockedTask.Payload) + if err := updateRunningEvictionTask(ctx, db, task, lockedTask.RefVersionID, payload); err != nil { + return err + } + task.Payload = payload + } + authorized = &cacheeviction.AuthorizedDeletion{Version: *version, BucketName: bucket.Name} + return nil + }) + return authorized, err +} + +func (r *BunCacheEvictionRepo) NextBucketDurabilityCandidate( + ctx context.Context, + bucketID int64, +) (*model.ObjectVersion, error) { + return nextBucketDurabilityCandidate(ctx, r.db, bucketID) +} + +func (r *BunCacheEvictionRepo) PromoteBucketDurabilityCandidate( + ctx context.Context, + task *model.Task, + versionID string, + authorizeDelete bool, +) (*cacheeviction.AuthorizedDeletion, error) { + if task == nil || task.RefType != "bucket" || task.RefID <= 0 || versionID == "" { + return nil, fmt.Errorf("bucket durability task and candidate are required: %w", ErrInvalidInput) + } + preflight, err := r.objectVersionByID(ctx, r.db, versionID) + if err != nil { + return nil, err + } + if preflight.BucketID != task.RefID || preflight.StorageUploadID == nil { + return nil, cacheeviction.ErrNoLongerEligible + } + + var deletion *cacheeviction.AuthorizedDeletion + err = r.runMaybeTx(ctx, func(db bun.IDB) error { + bucket, upload, version, lockedTask, err := lockCacheEvictionContext( + ctx, + db, + task, + preflight.BucketID, + *preflight.StorageUploadID, + versionID, + ) + if err != nil { + return err + } + if lockedTask.RefType != "bucket" || lockedTask.RefID != bucket.ID { + return fmt.Errorf("bucket durability task target changed: %w", ErrConflict) + } + if version.BucketID != bucket.ID || version.StorageUploadID == nil || *version.StorageUploadID != upload.ID || + version.State != model.ObjectStateReplicating || !version.InCache || version.IsDeleteMarker { + return cacheeviction.ErrNoLongerEligible + } + if err := requireMinimumDurability(ctx, db, bucket, upload); err != nil { + return err + } + now := time.Now() + res, err := db.NewUpdate(). + Model((*model.ObjectVersion)(nil)). + Set("state = ?", model.ObjectStateStored). + Set("updated_at = ?", now). + Where("version_id = ? AND state = ? AND in_cache = ?", version.VersionID, model.ObjectStateReplicating, true). + Exec(ctx) + if err != nil { + return fmt.Errorf("promoting bucket durability candidate: %w", err) + } + rows, _ := res.RowsAffected() + if rows != 1 { + return cacheeviction.ErrNoLongerEligible + } + version.State = model.ObjectStateStored + version.UpdatedAt = now + if authorizeDelete { + payload := cacheeviction.WithDeleteAuthorization(lockedTask.Payload) + if err := updateRunningEvictionTask(ctx, db, task, version.VersionID, payload); err != nil { + return err + } + task.RefVersionID = version.VersionID + task.Payload = payload + deletion = &cacheeviction.AuthorizedDeletion{Version: *version, BucketName: bucket.Name} + } + return nil + }) + return deletion, err +} + +func (r *BunCacheEvictionRepo) CompleteBucketDurabilityReconciliation( + ctx context.Context, + task *model.Task, +) (bool, error) { + if task == nil || task.RefType != "bucket" || task.RefID <= 0 { + return false, fmt.Errorf("bucket durability task is required: %w", ErrInvalidInput) + } + completed := false + err := r.runMaybeTx(ctx, func(db bun.IDB) error { + bucket, err := lockBucketByID(ctx, db, task.RefID) + if err != nil { + return err + } + if bucket == nil { + return cacheeviction.ErrNoLongerEligible + } + candidate, err := nextBucketDurabilityCandidate(ctx, db, bucket.ID) + if err != nil { + return err + } + if candidate != nil { + return nil + } + if err := (&BunTaskRepo{db: db}).LockRunningClaim(ctx, task); err != nil { + return err + } + if err := (&BunTaskRepo{db: db}).Complete(ctx, task); err != nil { + return err + } + completed = true + return nil + }) + return completed, err +} + +func (r *BunCacheEvictionRepo) RecordAuthorizedDeletion(ctx context.Context, task *model.Task) error { + if task == nil || task.RefVersionID == "" { + return fmt.Errorf("authorized cache eviction task is required: %w", ErrInvalidInput) + } + preflight, err := r.objectVersionByID(ctx, r.db, task.RefVersionID) + if err != nil { + if errors.Is(err, ErrNotFound) && task.RefType == "bucket" { + return r.clearMissingBucketDurabilityAuthorization(ctx, task) + } + return err + } + if preflight.StorageUploadID == nil { + return cacheeviction.ErrNoLongerEligible + } + return r.runMaybeTx(ctx, func(db bun.IDB) error { + _, _, version, lockedTask, err := lockCacheEvictionContext( + ctx, + db, + task, + preflight.BucketID, + *preflight.StorageUploadID, + preflight.VersionID, + ) + if err != nil { + return err + } + authorized, err := cacheeviction.DeleteAuthorized(lockedTask) + if err != nil { + return err + } + if !authorized || lockedTask.RefVersionID != version.VersionID { + return fmt.Errorf("cache deletion was not authorized: %w", ErrConflict) + } + switch version.State { + case model.ObjectStateStored: + _, err = db.NewUpdate(). + Model((*model.ObjectVersion)(nil)). + Set("state = ?", model.ObjectStateCacheEvicted). + Set("in_cache = ?", false). + Set("updated_at = ?", time.Now()). + Where("version_id = ? AND state = ?", version.VersionID, model.ObjectStateStored). + Exec(ctx) + case model.ObjectStateCacheEvicted: + _, err = db.NewUpdate(). + Model((*model.ObjectVersion)(nil)). + Set("in_cache = ?", false). + Where("version_id = ?", version.VersionID). + Exec(ctx) + default: + return cacheeviction.ErrNoLongerEligible + } + if err != nil { + return fmt.Errorf("recording authorized cache deletion: %w", err) + } + if lockedTask.RefType == "bucket" { + if err := updateRunningEvictionTask(ctx, db, task, "", nil); err != nil { + return err + } + task.RefVersionID = "" + task.Payload = nil + } + return nil + }) +} + +func (r *BunCacheEvictionRepo) clearMissingBucketDurabilityAuthorization( + ctx context.Context, + task *model.Task, +) error { + if task == nil || task.RefType != "bucket" || task.RefID <= 0 || task.RefVersionID == "" { + return fmt.Errorf("bucket durability deletion authorization is required: %w", ErrInvalidInput) + } + return r.runMaybeTx(ctx, func(db bun.IDB) error { + bucket, err := lockBucketByID(ctx, db, task.RefID) + if err != nil { + return err + } + if bucket == nil { + return cacheeviction.ErrNoLongerEligible + } + tasks := &BunTaskRepo{db: db} + if err := tasks.LockRunningClaim(ctx, task); err != nil { + return err + } + lockedTask, err := tasks.GetByID(ctx, task.ID) + if err != nil { + return err + } + if lockedTask == nil || lockedTask.RefType != "bucket" || lockedTask.RefID != bucket.ID || + lockedTask.RefVersionID != task.RefVersionID { + return fmt.Errorf("bucket durability deletion target changed: %w", ErrConflict) + } + authorized, err := cacheeviction.DeleteAuthorized(lockedTask) + if err != nil { + return err + } + if !authorized { + return fmt.Errorf("bucket durability deletion was not authorized: %w", ErrConflict) + } + if err := updateRunningEvictionTask(ctx, db, task, "", nil); err != nil { + return err + } + task.RefVersionID = "" + task.Payload = nil + return nil + }) +} + // CacheEvictionRepository owns persistence operations used only by cache // eviction planning and policy reconciliation. type CacheEvictionRepository interface { EnsureAfterUploadTask(ctx context.Context, objectID int64, versionID string, maxRetries int) (bool, error) + EnsureBucketDurabilityReconciliation(ctx context.Context, bucketID int64, maxRetries int) (bool, error) ListLRUCandidates(ctx context.Context, terminalSince time.Time, limit int) ([]cacheeviction.Candidate, error) PlanLRU(ctx context.Context, candidate cacheeviction.Candidate, maxRetries int, terminalBefore time.Time) (bool, error) ActiveLRUBytes(ctx context.Context) (int64, error) CancelActiveTasksExcept(ctx context.Context, keepStage string, message string) (int, error) + AuthorizeDeletion(ctx context.Context, task *model.Task, expectedAccess *time.Time) (*cacheeviction.AuthorizedDeletion, error) + NextBucketDurabilityCandidate(ctx context.Context, bucketID int64) (*model.ObjectVersion, error) + PromoteBucketDurabilityCandidate(ctx context.Context, task *model.Task, versionID string, authorizeDelete bool) (*cacheeviction.AuthorizedDeletion, error) + CompleteBucketDurabilityReconciliation(ctx context.Context, task *model.Task) (bool, error) + RecordAuthorizedDeletion(ctx context.Context, task *model.Task) error } // BunCacheEvictionRepo implements cache eviction planning and reconciliation @@ -42,6 +331,89 @@ func (r *BunCacheEvictionRepo) EnsureAfterUploadTask( }) } +func (r *BunCacheEvictionRepo) EnsureBucketDurabilityReconciliation( + ctx context.Context, + bucketID int64, + maxRetries int, +) (bool, error) { + task := cacheeviction.NewBucketDurabilityTask(bucketID, maxRetries, time.Now()) + requestedMaxRetries := task.MaxRetries + activated := false + err := r.runMaybeTx(ctx, func(db bun.IDB) error { + existing, err := loadAndLockTaskByIdempotencyKey(ctx, db, task.IdempotencyKey) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("loading bucket durability task: %w", err) + } + res, err := db.NewInsert().Model(task).On("CONFLICT (idempotency_key) DO NOTHING").Exec(ctx) + if err != nil { + return fmt.Errorf("creating bucket durability task: %w", err) + } + rows, _ := res.RowsAffected() + activated = rows == 1 + if activated && requestedMaxRetries == 0 { + // Bun otherwise substitutes the SQL default for this zero-valued field. + if _, err := db.NewUpdate(). + Model((*model.Task)(nil)). + Set("max_retries = ?", 0). + Where("id = ?", task.ID). + Exec(ctx); err != nil { + return fmt.Errorf("preserving bucket durability task zero retries: %w", err) + } + task.MaxRetries = requestedMaxRetries + } + return nil + } + switch existing.Status { + case model.TaskStatusCompleted, + model.TaskStatusFailed, + model.TaskStatusExhausted, + model.TaskStatusCancelled: + default: + return nil + } + + preserveAuthorization, err := cacheeviction.DeleteAuthorized(existing) + if err != nil { + return fmt.Errorf("reading bucket durability task authorization: %w", err) + } + refVersionID := task.RefVersionID + payload := task.Payload + if preserveAuthorization && existing.Status != model.TaskStatusCompleted { + refVersionID = existing.RefVersionID + payload = existing.Payload + } + now := time.Now() + res, err := db.NewUpdate(). + Model((*model.Task)(nil)). + Set("stage = ?", task.Stage). + Set("ref_type = ?", task.RefType). + Set("ref_id = ?", task.RefID). + Set("ref_version_id = ?", refVersionID). + Set("payload = ?", payload). + Set("status = ?", model.TaskStatusQueued). + Set("retry_count = 0"). + Set("max_retries = ?", requestedMaxRetries). + Set("scheduled_at = ?", now). + Set("claimed_at = NULL"). + Set("lease_until = NULL"). + Set("started_at = NULL"). + Set("completed_at = NULL"). + Set("last_error = NULL"). + Set("wait_reason = NULL"). + Set("status_message = NULL"). + Where("id = ? AND status = ?", existing.ID, existing.Status). + Exec(ctx) + if err != nil { + return fmt.Errorf("reactivating bucket durability task %d: %w", bucketID, err) + } + rows, _ := res.RowsAffected() + activated = rows == 1 + return nil + }) + return activated, err +} + func (r *BunCacheEvictionRepo) ListLRUCandidates( ctx context.Context, terminalSince time.Time, @@ -55,6 +427,7 @@ func (r *BunCacheEvictionRepo) ListLRUCandidates( ColumnExpr("object_version.size"). ColumnExpr("object_version.cache_accessed_at"). Join("JOIN storage_uploads AS storage_upload ON storage_upload.id = object_version.storage_upload_id"). + Join("JOIN buckets AS durability_bucket ON durability_bucket.id = storage_upload.bucket_id"). Where("object_version.in_cache = ?", true). Where("object_version.is_delete_marker = ?", false). Where("object_version.size > 0"). @@ -63,8 +436,11 @@ func (r *BunCacheEvictionRepo) ListLRUCandidates( model.ObjectStateStored, model.ObjectStateCacheEvicted, })). - Where("storage_upload.status = ?", model.StorageUploadStatusComplete). - Where(usableCopyExistsSQL("object_version.storage_upload_id")). + Where("storage_upload.status IN (?)", bun.List([]model.StorageUploadStatus{ + model.StorageUploadStatusReadable, + model.StorageUploadStatusComplete, + })). + Where(minimumDurabilityMetSQL("storage_upload", "durability_bucket")). Where(`NOT EXISTS ( SELECT 1 FROM tasks AS eviction_task WHERE eviction_task.type = ? @@ -132,64 +508,84 @@ func (r *BunCacheEvictionRepo) createOrReactivate( task *model.Task, rule taskReactivationRule, ) (bool, error) { - tasks := &BunTaskRepo{db: r.db} - if err := tasks.Create(ctx, task); err == nil { - return true, nil - } else if !errors.Is(err, ErrAlreadyExists) { - return false, err - } + activated := false + err := r.runMaybeTx(ctx, func(db bun.IDB) error { + existing, err := loadAndLockTaskByIdempotencyKey(ctx, db, task.IdempotencyKey) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return err + } + res, err := db.NewInsert().Model(task).On("CONFLICT (idempotency_key) DO NOTHING").Exec(ctx) + if err != nil { + return err + } + rows, _ := res.RowsAffected() + activated = rows == 1 + return nil + } + eligible := taskStatusIn(existing.Status, rule.immediateStatuses) + if !eligible && taskStatusIn(existing.Status, rule.cooledStatuses) { + if rule.terminalBefore == nil { + return errors.New("reactivating task: terminal cutoff is required") + } + eligible = existing.CompletedAt != nil && !existing.CompletedAt.After(*rule.terminalBefore) + } + if !eligible { + return nil + } - now := time.Now() - q := r.reactivationQuery(task, now) - if len(rule.cooledStatuses) == 0 { - q = q.Where("status IN (?)", bun.List(rule.immediateStatuses)) - } else { - if rule.terminalBefore == nil { - return false, errors.New("reactivating task: terminal cutoff is required") - } - q = q.Where( - `( - status IN (?) - OR ( - status IN (?) - AND completed_at IS NOT NULL - AND completed_at <= ? - ) - )`, - bun.List(rule.immediateStatuses), - bun.List(rule.cooledStatuses), - *rule.terminalBefore, - ) - } - res, err := q.Exec(ctx) + refVersionID := task.RefVersionID + payload := task.Payload + preserveAuthorization, err := cacheeviction.DeleteAuthorized(existing) + if err != nil { + return err + } + if preserveAuthorization && existing.Status != model.TaskStatusCompleted { + refVersionID = existing.RefVersionID + payload = existing.Payload + } + now := time.Now() + res, err := db.NewUpdate(). + Model((*model.Task)(nil)). + Set("type = ?", task.Type). + Set("stage = ?", task.Stage). + Set("ref_type = ?", task.RefType). + Set("ref_id = ?", task.RefID). + Set("ref_version_id = ?", refVersionID). + Set("payload = ?", payload). + Set("status = ?", model.TaskStatusQueued). + Set("retry_count = 0"). + Set("max_retries = ?", task.MaxRetries). + Set("scheduled_at = ?", now). + Set("claimed_at = NULL"). + Set("lease_until = NULL"). + Set("started_at = NULL"). + Set("completed_at = NULL"). + Set("last_error = NULL"). + Set("wait_reason = NULL"). + Set("status_message = NULL"). + Where("id = ? AND status = ?", existing.ID, existing.Status). + Exec(ctx) + if err != nil { + return err + } + rows, _ := res.RowsAffected() + activated = rows == 1 + return nil + }) if err != nil { return false, fmt.Errorf("%s %q: %w", rule.errorAction, task.IdempotencyKey, err) } - rows, _ := res.RowsAffected() - return rows > 0, nil + return activated, nil } -func (r *BunCacheEvictionRepo) reactivationQuery(task *model.Task, now time.Time) *bun.UpdateQuery { - return r.db.NewUpdate(). - Model((*model.Task)(nil)). - Set("type = ?", task.Type). - Set("stage = ?", task.Stage). - Set("ref_type = ?", task.RefType). - Set("ref_id = ?", task.RefID). - Set("ref_version_id = ?", task.RefVersionID). - Set("payload = ?", task.Payload). - Set("status = ?", model.TaskStatusQueued). - Set("retry_count = 0"). - Set("max_retries = ?", task.MaxRetries). - Set("scheduled_at = ?", now). - Set("claimed_at = NULL"). - Set("lease_until = NULL"). - Set("started_at = NULL"). - Set("completed_at = NULL"). - Set("last_error = NULL"). - Set("wait_reason = NULL"). - Set("status_message = NULL"). - Where("idempotency_key = ?", task.IdempotencyKey) +func taskStatusIn(status model.TaskStatus, statuses []model.TaskStatus) bool { + for _, candidate := range statuses { + if status == candidate { + return true + } + } + return false } func (r *BunCacheEvictionRepo) ActiveLRUBytes(ctx context.Context) (int64, error) { @@ -234,7 +630,9 @@ func (r *BunCacheEvictionRepo) CancelActiveTasksExcept( Set("lease_until = NULL"). Set("started_at = NULL"). Where("type = ?", model.TaskTypeEvictCache). - Where("status IN (?)", bun.List(activeTaskStatuses())) + Where("status IN (?)", bun.List(activeTaskStatuses())). + Where("stage IS NULL OR stage <> ?", cacheeviction.StageReconcileBucketDurability). + Where("NOT (" + cacheDeletionAuthorizedSQL(r.db.Dialect().Name()) + ")") if keepStage != "" { q = q.Where("(stage IS NULL OR stage <> ?)", keepStage) } @@ -250,3 +648,219 @@ func (r *BunCacheEvictionRepo) CancelActiveTasksExcept( rows, _ := res.RowsAffected() return int(rows), nil } + +func (r *BunCacheEvictionRepo) objectVersionByID( + ctx context.Context, + db bun.IDB, + versionID string, +) (*model.ObjectVersion, error) { + version := new(model.ObjectVersion) + if err := db.NewSelect().Model(version).Where("version_id = ?", versionID).Scan(ctx); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + return nil, fmt.Errorf("loading cache eviction object version: %w", err) + } + return version, nil +} + +func lockCacheEvictionContext( + ctx context.Context, + db bun.IDB, + claimedTask *model.Task, + bucketID int64, + uploadID int64, + versionID string, +) (*model.Bucket, *model.StorageUpload, *model.ObjectVersion, *model.Task, error) { + bucket, err := lockBucketByID(ctx, db, bucketID) + if err != nil { + return nil, nil, nil, nil, err + } + if bucket == nil { + return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible + } + uploads, err := lockStorageUploadsByID(ctx, db, []int64{uploadID}) + if err != nil { + return nil, nil, nil, nil, err + } + upload := uploads[uploadID] + if upload == nil || upload.BucketID != bucket.ID { + return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible + } + if err := lockObjectVersionsByID(ctx, db, []string{versionID}); err != nil { + if errors.Is(err, ErrNotFound) { + return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible + } + return nil, nil, nil, nil, err + } + version := new(model.ObjectVersion) + if err := db.NewSelect().Model(version).Where("version_id = ?", versionID).Scan(ctx); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible + } + return nil, nil, nil, nil, fmt.Errorf("loading locked cache eviction object version: %w", err) + } + if version.BucketID != bucket.ID || version.StorageUploadID == nil || *version.StorageUploadID != upload.ID { + return nil, nil, nil, nil, cacheeviction.ErrNoLongerEligible + } + tasks := &BunTaskRepo{db: db} + if err := tasks.LockRunningClaim(ctx, claimedTask); err != nil { + return nil, nil, nil, nil, err + } + lockedTask, err := tasks.GetByID(ctx, claimedTask.ID) + if err != nil { + return nil, nil, nil, nil, err + } + if lockedTask == nil || lockedTask.Type != model.TaskTypeEvictCache { + return nil, nil, nil, nil, fmt.Errorf("cache eviction task changed: %w", ErrConflict) + } + return bucket, upload, version, lockedTask, nil +} + +func requireMinimumDurability( + ctx context.Context, + db bun.IDB, + bucket *model.Bucket, + upload *model.StorageUpload, +) error { + if bucket == nil || upload == nil || upload.BucketID != bucket.ID { + return fmt.Errorf("cache eviction durability context is invalid: %w", ErrInvalidInput) + } + if upload.Status != model.StorageUploadStatusReadable && upload.Status != model.StorageUploadStatusComplete { + return cacheeviction.ErrDurabilityThreshold + } + minimum := minimumDurableCopiesForUpload(bucket, upload.RequestedCopies) + readable, err := countReadableCommittedCopies(ctx, db, upload.ID) + if err != nil { + return err + } + if minimum <= 0 || readable < minimum { + return cacheeviction.ErrDurabilityThreshold + } + return nil +} + +func updateRunningEvictionTask( + ctx context.Context, + db bun.IDB, + claimedTask *model.Task, + refVersionID string, + payload map[string]any, +) error { + taskID, claimedAt, err := runningTaskClaim(claimedTask) + if err != nil { + return err + } + now := time.Now() + res, err := db.NewUpdate(). + Model((*model.Task)(nil)). + Set("ref_version_id = ?", refVersionID). + Set("payload = ?", payload). + Where("id = ? AND status = ?", taskID, model.TaskStatusRunning). + Where("claimed_at = ?", claimedAt). + Where("lease_until IS NOT NULL AND lease_until > ?", now). + Exec(ctx) + if err != nil { + return fmt.Errorf("persisting cache deletion authorization: %w", err) + } + rows, _ := res.RowsAffected() + if rows != 1 { + return fmt.Errorf("persisting cache deletion authorization for task %d: not in active running claim", taskID) + } + return nil +} + +func nextBucketDurabilityCandidate( + ctx context.Context, + db bun.IDB, + bucketID int64, +) (*model.ObjectVersion, error) { + version := new(model.ObjectVersion) + err := db.NewSelect(). + Model(version). + Join("JOIN storage_uploads AS storage_upload ON storage_upload.id = object_version.storage_upload_id"). + Join("JOIN buckets AS durability_bucket ON durability_bucket.id = storage_upload.bucket_id"). + Where("object_version.bucket_id = ?", bucketID). + Where("object_version.state = ?", model.ObjectStateReplicating). + Where("object_version.in_cache = ?", true). + Where("object_version.is_delete_marker = ?", false). + Where("storage_upload.status IN (?)", bun.List([]model.StorageUploadStatus{ + model.StorageUploadStatusReadable, + model.StorageUploadStatusComplete, + })). + Where(minimumDurabilityMetSQL("storage_upload", "durability_bucket")). + OrderExpr("object_version.updated_at ASC"). + OrderExpr("object_version.version_id ASC"). + Limit(1). + Scan(ctx) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return nil, fmt.Errorf("selecting bucket durability candidate: %w", err) + } + return version, nil +} + +func minimumDurabilityMetSQL(uploadAlias, bucketAlias string) string { + return fmt.Sprintf(`( + SELECT COUNT(*) + FROM storage_upload_copies AS durable_copy + JOIN storage_data_sets AS durable_data_set ON durable_data_set.id = durable_copy.storage_data_set_id + WHERE durable_copy.upload_id = %s.id + AND durable_copy.status = '%s' + AND durable_copy.storage_data_set_id IS NOT NULL + AND durable_copy.provider_id IS NOT NULL AND durable_copy.provider_id <> '' + AND durable_data_set.data_set_id IS NOT NULL AND durable_data_set.data_set_id <> '' + AND durable_data_set.status IN (%s) + AND durable_copy.piece_id IS NOT NULL AND durable_copy.piece_id <> '' + AND durable_copy.retrieval_url IS NOT NULL AND durable_copy.retrieval_url <> '' + ) >= CASE + WHEN %s.minimum_durable_copies IS NULL + OR %s.minimum_durable_copies >= %s.requested_copies + THEN %s.requested_copies + ELSE %s.minimum_durable_copies + END`, + uploadAlias, + model.StorageUploadCopyStatusCommitted, + storageHealthReadyDataSetStatusListSQL(), + bucketAlias, + bucketAlias, + uploadAlias, + uploadAlias, + bucketAlias, + ) +} + +func cacheDeletionStateEligible(version *model.ObjectVersion) bool { + if version == nil || version.IsDeleteMarker || !version.InCache || version.StorageUploadID == nil { + return false + } + return version.State == model.ObjectStateStored || version.State == model.ObjectStateCacheEvicted +} + +func cacheAccessTime(version *model.ObjectVersion) time.Time { + if version == nil { + return time.Time{} + } + if version.CacheAccessedAt != nil { + return *version.CacheAccessedAt + } + return version.CreatedAt +} + +func cacheDeletionAuthorizedSQL(dialectName dialect.Name) string { + if dialectName == dialect.PG { + return "COALESCE(CAST(payload ->> 'delete_authorized' AS BOOLEAN), FALSE)" + } + return "COALESCE(CAST(json_extract(payload, '$.delete_authorized') AS INTEGER), 0) = 1" +} + +func (r *BunCacheEvictionRepo) runMaybeTx(ctx context.Context, fn func(bun.IDB) error) error { + if db, ok := r.db.(*bun.DB); ok { + return db.RunInTx(ctx, nil, func(ctx context.Context, tx bun.Tx) error { + return fn(tx) + }) + } + return fn(r.db) +} diff --git a/internal/db/repository/cache_eviction_repo_test.go b/internal/db/repository/cache_eviction_repo_test.go index 86b1310..6659fef 100644 --- a/internal/db/repository/cache_eviction_repo_test.go +++ b/internal/db/repository/cache_eviction_repo_test.go @@ -2,14 +2,226 @@ package repository_test import ( "context" + "errors" "testing" "time" "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" + "github.com/uptrace/bun" ) +func TestCacheEvictionRepo_DurabilityAuthorizationUsesCommitOrder(t *testing.T) { + for _, tc := range []struct { + name string + raiseBeforePromote bool + wantAuthorized bool + }{ + {name: "policy_raise_first_blocks_deletion", raiseBeforePromote: true}, + {name: "authorization_first_survives_policy_raise", wantAuthorized: true}, + } { + t.Run(tc.name, func(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket, version, upload := seedMinimumDurabilityCandidate(t, repos, db, tc.name) + + if _, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 4); err != nil { + t.Fatalf("EnsureBucketDurabilityReconciliation: %v", err) + } + task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) + if err != nil || task == nil { + t.Fatalf("ClaimReady: task=%#v err=%v", task, err) + } + candidate, err := repos.CacheEvictions.NextBucketDurabilityCandidate(ctx, bucket.ID) + if err != nil || candidate == nil || candidate.VersionID != version.VersionID { + t.Fatalf("NextBucketDurabilityCandidate: candidate=%#v err=%v", candidate, err) + } + + minimumThree := 3 + if tc.raiseBeforePromote { + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimumThree, + }); err != nil { + t.Fatalf("raise minimum before authorization: %v", err) + } + } + + deletion, err := repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, version.VersionID, true) + if !tc.wantAuthorized { + if !errors.Is(err, cacheeviction.ErrDurabilityThreshold) || deletion != nil { + t.Fatalf("promotion after policy raise = deletion:%#v err:%v, want durability threshold", deletion, err) + } + got, getErr := repos.Objects.GetVersionByID(ctx, version.VersionID) + if getErr != nil || got == nil || got.State != model.ObjectStateReplicating || !got.InCache { + t.Fatalf("blocked candidate = %#v err=%v, want replicating in cache", got, getErr) + } + return + } + if err != nil || deletion == nil { + t.Fatalf("PromoteBucketDurabilityCandidate: deletion=%#v err=%v", deletion, err) + } + authorized, err := cacheeviction.DeleteAuthorized(task) + if err != nil || !authorized || task.RefVersionID != version.VersionID { + t.Fatalf("task authorization = %t ref=%q err=%v", authorized, task.RefVersionID, err) + } + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimumThree, + }); err != nil { + t.Fatalf("raise minimum after authorization: %v", err) + } + if err := repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); err != nil { + t.Fatalf("RecordAuthorizedDeletion: %v", err) + } + got, err := repos.Objects.GetVersionByID(ctx, version.VersionID) + if err != nil || got == nil || got.State != model.ObjectStateCacheEvicted || got.InCache { + t.Fatalf("authorized candidate = %#v err=%v, want cache evicted", got, err) + } + if task.RefVersionID != "" { + t.Fatalf("coordinator ref after deletion = %q, want cleared", task.RefVersionID) + } + completed, err := repos.CacheEvictions.CompleteBucketDurabilityReconciliation(ctx, task) + if err != nil || !completed { + t.Fatalf("CompleteBucketDurabilityReconciliation = %t err=%v", completed, err) + } + gotUpload, err := repos.Uploads.GetByID(ctx, upload.ID) + if err != nil || gotUpload == nil || gotUpload.Status != model.StorageUploadStatusReadable { + t.Fatalf("upload after cache deletion = %#v err=%v, want readable for repair", gotUpload, err) + } + }) + } +} + +func TestCacheEvictionRepo_RequeuesTerminalCoordinatorWithoutLosingAuthorization(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket, version, _ := seedMinimumDurabilityCandidate(t, repos, db, "preserve-authorization") + if _, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 4); err != nil { + t.Fatalf("EnsureBucketDurabilityReconciliation: %v", err) + } + task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) + if err != nil || task == nil { + t.Fatalf("ClaimReady: task=%#v err=%v", task, err) + } + if _, err := repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, version.VersionID, true); err != nil { + t.Fatalf("PromoteBucketDurabilityCandidate: %v", err) + } + if err := repos.Tasks.FailRunning(ctx, task, "injected cache failure"); err != nil { + t.Fatalf("FailRunning: %v", err) + } + activated, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 7) + if err != nil || !activated { + t.Fatalf("requeue terminal coordinator = %t err=%v", activated, err) + } + got, err := repos.Tasks.GetByID(ctx, task.ID) + if err != nil || got == nil { + t.Fatalf("GetByID: task=%#v err=%v", got, err) + } + authorized, err := cacheeviction.DeleteAuthorized(got) + if err != nil || !authorized || got.RefVersionID != version.VersionID || got.Status != model.TaskStatusQueued || got.MaxRetries != 7 { + t.Fatalf("requeued coordinator = %#v authorized=%t err=%v", got, authorized, err) + } +} + +func TestCacheEvictionRepo_BucketDurabilityCoordinatorPreservesZeroRetries(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "durability-zero-retries") + + created, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 0) + if err != nil || !created { + t.Fatalf("EnsureBucketDurabilityReconciliation = %t err=%v", created, err) + } + tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeEvictCache), cacheeviction.StageReconcileBucketDurability, "", 10, 0) + if err != nil || total != 1 || len(tasks) != 1 { + t.Fatalf("coordinator tasks total=%d tasks=%#v err=%v", total, tasks, err) + } + if tasks[0].MaxRetries != 0 { + t.Fatalf("coordinator max retries = %d, want 0", tasks[0].MaxRetries) + } +} + +func TestCacheEvictionRepo_ClearsCoordinatorAuthorizationWhenVersionWasDeleted(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket, version, _ := seedMinimumDurabilityCandidate(t, repos, db, "deleted-authorized-version") + if _, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 4); err != nil { + t.Fatalf("EnsureBucketDurabilityReconciliation: %v", err) + } + task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) + if err != nil || task == nil { + t.Fatalf("ClaimReady: task=%#v err=%v", task, err) + } + if _, err := repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, version.VersionID, true); err != nil { + t.Fatalf("PromoteBucketDurabilityCandidate: %v", err) + } + if _, err := db.NewDelete().Model((*model.ObjectVersion)(nil)).Where("version_id = ?", version.VersionID).Exec(ctx); err != nil { + t.Fatalf("delete authorized version: %v", err) + } + if err := repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); err != nil { + t.Fatalf("RecordAuthorizedDeletion missing version: %v", err) + } + if task.RefVersionID != "" || task.Payload != nil { + t.Fatalf("cleared coordinator authorization = ref:%q payload:%#v", task.RefVersionID, task.Payload) + } + completed, err := repos.CacheEvictions.CompleteBucketDurabilityReconciliation(ctx, task) + if err != nil || !completed { + t.Fatalf("CompleteBucketDurabilityReconciliation = %t err=%v", completed, err) + } +} + +func TestCacheEvictionRepo_LRUCandidateRequiresCurrentMinimumDurability(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket, version, upload := seedMinimumDurabilityCandidate(t, repos, db, "lru-minimum") + + if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateReplicating, model.ObjectStateStored); err != nil { + t.Fatalf("mark stored: %v", err) + } + if err := repos.Objects.RecordVersionCacheCommit(ctx, version.VersionID, time.Now().Add(-time.Hour)); err != nil { + t.Fatalf("RecordVersionCacheCommit: %v", err) + } + if _, err := db.NewDelete(). + Model((*model.StorageUploadCopy)(nil)). + Where("upload_id = ? AND copy_index = ?", upload.ID, 1). + Exec(ctx); err != nil { + t.Fatalf("remove second readable copy: %v", err) + } + + candidates, err := repos.CacheEvictions.ListLRUCandidates(ctx, time.Now().Add(-time.Hour), 10) + if err != nil { + t.Fatalf("ListLRUCandidates below minimum: %v", err) + } + if len(candidates) != 0 { + t.Fatalf("candidates below minimum = %#v, want none", candidates) + } + + minimumOne := 1 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimumOne, + }); err != nil { + t.Fatalf("lower minimum: %v", err) + } + candidates, err = repos.CacheEvictions.ListLRUCandidates(ctx, time.Now().Add(-time.Hour), 10) + if err != nil { + t.Fatalf("ListLRUCandidates at minimum: %v", err) + } + if len(candidates) != 1 || candidates[0].VersionID != version.VersionID { + t.Fatalf("candidates at minimum = %#v, want %s", candidates, version.VersionID) + } +} + func TestCacheEvictionRepo_EnsureAfterUploadOnlyRequeuesCancelledTask(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) @@ -185,6 +397,7 @@ func TestCacheEvictionRepo_CancelActiveTasksExceptPreservesMatchingStageAndTermi ctx := context.Background() lruStage := cacheeviction.StageLRU afterUploadStage := cacheeviction.StageAfterUpload + reconcileStage := cacheeviction.StageReconcileBucketDurability tasks := []*model.Task{ { Type: model.TaskTypeEvictCache, @@ -217,6 +430,23 @@ func TestCacheEvictionRepo_CancelActiveTasksExceptPreservesMatchingStageAndTermi IdempotencyKey: "cancel-stage-terminal", Status: model.TaskStatusFailed, }, + { + Type: model.TaskTypeEvictCache, + Stage: &reconcileStage, + RefType: "bucket", + RefID: 9, + IdempotencyKey: "cancel-stage-durability-coordinator", + Status: model.TaskStatusQueued, + }, + { + Type: model.TaskTypeEvictCache, + Stage: &afterUploadStage, + RefType: "object", + RefVersionID: "01J0000000000000000000CS05", + IdempotencyKey: "cancel-stage-authorized-after-upload", + Payload: cacheeviction.WithDeleteAuthorization(nil), + Status: model.TaskStatusQueued, + }, } for _, task := range tasks { if err := repos.Tasks.Create(ctx, task); err != nil { @@ -236,6 +466,8 @@ func TestCacheEvictionRepo_CancelActiveTasksExceptPreservesMatchingStageAndTermi model.TaskStatusCancelled, model.TaskStatusCancelled, model.TaskStatusFailed, + model.TaskStatusQueued, + model.TaskStatusQueued, } for index, task := range tasks { got, err := repos.Tasks.GetByID(ctx, task.ID) @@ -248,6 +480,35 @@ func TestCacheEvictionRepo_CancelActiveTasksExceptPreservesMatchingStageAndTermi } } +func seedMinimumDurabilityCandidate( + t *testing.T, + repos *repository.Repositories, + db *bun.DB, + suffix string, +) (*model.Bucket, *model.ObjectVersion, *model.StorageUpload) { + t.Helper() + ctx := context.Background() + bucket := seedBucket(t, db, "durability-candidate-"+suffix) + minimum := 2 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimum, + }); err != nil { + t.Fatalf("UpdateCopyPolicy: %v", err) + } + version := newObjectVersion(bucket.ID, "candidate.txt", model.NewVersionID(), 10) + version.Checksum = "durability-candidate-" + suffix + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 3) + commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") + commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 1, "202", "2002", "2002", "https://two.example/piece") + bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) + return bucket, version, upload +} + func TestCacheEvictionRepo_ListLRUCandidatesOrdersSafeCurrentAndHistoricalVersions(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) diff --git a/internal/db/repository/interfaces.go b/internal/db/repository/interfaces.go index 1f39d07..8a9cac9 100644 --- a/internal/db/repository/interfaces.go +++ b/internal/db/repository/interfaces.go @@ -29,7 +29,9 @@ type BucketRepository interface { SetACL(ctx context.Context, name string, acl []byte) error // SetOwnerAndACL stores both the authoritative owner and compatible ACL. SetOwnerAndACL(ctx context.Context, name string, ownerAccessKey *string, acl []byte) error - // SetDefaultCopies stores the bucket copy policy override. Nil means inherit. + // UpdateCopyPolicy locks and updates the independently optional bucket policy fields. + UpdateCopyPolicy(ctx context.Context, input UpdateBucketCopyPolicyInput) (*model.Bucket, error) + // SetDefaultCopies stores the bucket target override. Nil means inherit. SetDefaultCopies(ctx context.Context, name string, copies *int) error // CountByOwner returns bucket count for the authoritative owner access key. CountByOwner(ctx context.Context, ownerAccessKey string) (int, error) @@ -41,6 +43,15 @@ type BucketRepository interface { CountStorageDataSets(ctx context.Context) (int, error) } +// UpdateBucketCopyPolicyInput distinguishes omitted fields from explicit nulls. +type UpdateBucketCopyPolicyInput struct { + Name string + SetDefaultCopies bool + DefaultCopies *int + SetMinimumDurableCopies bool + MinimumDurableCopies *int +} + // S3AccountRepository defines persistence operations for S3 IAM accounts. type S3AccountRepository interface { Create(ctx context.Context, account *model.S3Account) error @@ -401,6 +412,13 @@ type ReplicaRepairItem struct { Version model.ObjectVersion } +// IncompleteReadableUpload identifies one durable upload that still needs +// work to reach its frozen target copy count. +type IncompleteReadableUpload struct { + Upload model.StorageUpload + Version model.ObjectVersion +} + type BindReadableUploadInput struct { UploadID int64 BucketID int64 @@ -473,6 +491,7 @@ type StorageUploadRepository interface { NextIncompleteCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageUploadCopy, error) NextFinalizableCopyForDataSet(ctx context.Context, storageDataSetID int64) (*model.StorageUploadCopy, error) ListUnavailableDataSetsWithIncompleteCopies(ctx context.Context, afterID int64, limit int) ([]model.StorageDataSet, error) + ListIncompleteReadableUploads(ctx context.Context, afterID int64, limit int) ([]IncompleteReadableUpload, error) ReassignIngressCopy(ctx context.Context, uploadID int64, unavailableCopyIndex int) (*model.StorageUploadCopy, error) MarkUploadCopyPieceReady(ctx context.Context, input MarkUploadCopyPieceReadyInput) error MarkUploadCopyCommitting(ctx context.Context, input MarkUploadCopyCommittingInput) error diff --git a/internal/db/repository/object_repo.go b/internal/db/repository/object_repo.go index 3cb5699..4974a64 100644 --- a/internal/db/repository/object_repo.go +++ b/internal/db/repository/object_repo.go @@ -583,7 +583,10 @@ func (r *BunObjectRepo) FindReusableStoredVersion(ctx context.Context, bucketID err := q.Where("object_version.bucket_id = ? AND object_version.size = ? AND object_version.checksum = ?", bucketID, size, checksum). Where("object_version.is_delete_marker = ?", false). Where("object_version.state IN (?)", bun.List([]model.ObjectState{model.ObjectStateStored, model.ObjectStateCacheEvicted})). - Where("storage_upload.status = ?", model.StorageUploadStatusComplete). + Where("storage_upload.status IN (?)", bun.List([]model.StorageUploadStatus{ + model.StorageUploadStatusReadable, + model.StorageUploadStatusComplete, + })). Where(usableCopyExistsSQL("object_version.storage_upload_id")). OrderExpr("object_version.created_at DESC"). OrderExpr("object_version.version_id DESC"). @@ -916,19 +919,23 @@ func executeVersionCacheAccessUpdate( func (r *BunObjectRepo) SetVersionStorageUploadAndTransition(ctx context.Context, versionID string, storageUploadID int64, from, to model.ObjectState) error { return r.runMaybeTx(ctx, func(db bun.IDB) error { - if _, err := lockStorageUploadForObjectState(ctx, db, storageUploadID, to); err != nil { + upload, err := lockStorageUploadForObjectState(ctx, db, storageUploadID, to) + if err != nil { return fmt.Errorf("locking storage upload for version transition: %w", err) } + if to == model.ObjectStateStored || to == model.ObjectStateCacheEvicted { + if err := requireCurrentMinimumDurableCopies(ctx, db, upload); err != nil { + return fmt.Errorf("checking storage upload durability for version transition: %w", err) + } + } now := time.Now() - query := `UPDATE object_versions - SET storage_upload_id = ?, state = ?, updated_at = ? - WHERE version_id = ? AND state = ? - AND EXISTS ( - SELECT 1 FROM storage_uploads - WHERE id = ? AND status = ? - ) - AND ` + usableCopyExistsSQL("?") - res, err := db.NewRaw(query, storageUploadID, to, now, versionID, from, storageUploadID, model.StorageUploadStatusComplete, storageUploadID).Exec(ctx) + res, err := db.NewUpdate(). + Model((*model.ObjectVersion)(nil)). + Set("storage_upload_id = ?", storageUploadID). + Set("state = ?", to). + Set("updated_at = ?", now). + Where("version_id = ? AND state = ?", versionID, from). + Exec(ctx) if err != nil { return fmt.Errorf("setting version storage upload and transitioning state: %w", err) } diff --git a/internal/db/repository/object_repo_test.go b/internal/db/repository/object_repo_test.go index 5e0bf5b..3f875af 100644 --- a/internal/db/repository/object_repo_test.go +++ b/internal/db/repository/object_repo_test.go @@ -800,6 +800,65 @@ func TestObjectRepo_SetVersionStorageUploadAndTransitionUsesNewUpload(t *testing } } +func TestObjectRepo_NewStoredReferenceRechecksCurrentMinimumDurability(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "reuse-current-minimum-bucket") + minimumOne := 1 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimumOne, + }); err != nil { + t.Fatalf("set minimum one: %v", err) + } + + source := newObjectVersion(bucket.ID, "source.txt", model.NewVersionID(), 10) + source.Checksum = "reuse-current-minimum" + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, source); err != nil { + t.Fatalf("create source: %v", err) + } + upload := startCopyHealthUpload(t, repos, bucket.ID, source.VersionID, source.Size, source.Checksum, 2) + commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") + bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, source) + if complete, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil || complete { + t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err:%v, want stored before target", complete, err) + } + reusable, err := repos.Objects.FindReusableStoredVersion(ctx, bucket.ID, source.Size, source.Checksum) + if err != nil { + t.Fatalf("FindReusableStoredVersion before raising minimum: %v", err) + } + if reusable == nil || reusable.VersionID != source.VersionID { + t.Fatalf("reusable readable upload version = %#v, want %s", reusable, source.VersionID) + } + + minimumTwo := 2 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimumTwo, + }); err != nil { + t.Fatalf("raise minimum: %v", err) + } + follower := newObjectVersion(bucket.ID, "follower.txt", model.NewVersionID(), source.Size) + follower.Checksum = source.Checksum + follower.State = model.ObjectStateStored + follower.InCache = true + follower.StorageUploadID = &upload.ID + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, follower); err != nil { + t.Fatalf("create follower: %v", err) + } + + got, err := repos.Objects.GetVersionByID(ctx, follower.VersionID) + if err != nil || got == nil { + t.Fatalf("get follower: version=%#v err=%v", got, err) + } + if got.State != model.ObjectStateCached || got.StorageUploadID != nil || !got.InCache { + t.Fatalf("follower below current minimum = %#v, want retained cache without reused upload", got) + } +} + func TestObjectRepo_FindReusableActiveUploadVersionRequiresActiveTask(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) diff --git a/internal/db/repository/storage_upload_reference.go b/internal/db/repository/storage_upload_reference.go index 57ea931..72a858a 100644 --- a/internal/db/repository/storage_upload_reference.go +++ b/internal/db/repository/storage_upload_reference.go @@ -58,6 +58,22 @@ func lockStorageUploadForObjectState( uploadID int64, state model.ObjectState, ) (*model.StorageUpload, error) { + var bucketID int64 + if err := db.NewSelect(). + Model((*model.StorageUpload)(nil)). + Column("bucket_id"). + Where("id = ?", uploadID). + Scan(ctx, &bucketID); err != nil { + if err == sql.ErrNoRows { + return nil, fmt.Errorf("loading storage upload %d: %w", uploadID, ErrNotFound) + } + return nil, fmt.Errorf("loading storage upload %d bucket: %w", uploadID, err) + } + if bucket, err := lockBucketByID(ctx, db, bucketID); err != nil { + return nil, err + } else if bucket == nil { + return nil, fmt.Errorf("locking storage upload %d bucket: %w", uploadID, ErrNotFound) + } uploads, err := lockStorageUploadsByID(ctx, db, []int64{uploadID}) if err != nil { return nil, err @@ -87,7 +103,7 @@ func lockStorageUploadForCopyMutation(ctx context.Context, db bun.IDB, uploadID func storageUploadSupportsObjectState(status model.StorageUploadStatus, state model.ObjectState) bool { switch state { case model.ObjectStateStored, model.ObjectStateCacheEvicted: - return status == model.StorageUploadStatusComplete + return status == model.StorageUploadStatusReadable || status == model.StorageUploadStatusComplete case model.ObjectStateReplicating: return status != model.StorageUploadStatusRejected && status != model.StorageUploadStatusSuperseded default: @@ -99,7 +115,10 @@ func prepareNewObjectVersionStorageReference(ctx context.Context, db bun.IDB, ve if version == nil || version.StorageUploadID == nil || *version.StorageUploadID <= 0 { return nil } - _, err := lockStorageUploadForObjectState(ctx, db, *version.StorageUploadID, version.State) + upload, err := lockStorageUploadForObjectState(ctx, db, *version.StorageUploadID, version.State) + if err == nil && (version.State == model.ObjectStateStored || version.State == model.ObjectStateCacheEvicted) { + err = requireCurrentMinimumDurableCopies(ctx, db, upload) + } if err == nil { return nil } diff --git a/internal/db/repository/storage_upload_repo.go b/internal/db/repository/storage_upload_repo.go index 9e2a362..10a90eb 100644 --- a/internal/db/repository/storage_upload_repo.go +++ b/internal/db/repository/storage_upload_repo.go @@ -1242,6 +1242,58 @@ func (r *BunStorageUploadRepo) ListUnavailableDataSetsWithIncompleteCopies(ctx c return bindings, nil } +func (r *BunStorageUploadRepo) ListIncompleteReadableUploads( + ctx context.Context, + afterID int64, + limit int, +) ([]IncompleteReadableUpload, error) { + var uploads []model.StorageUpload + q := r.db.NewSelect(). + Model(&uploads). + Where("status = ?", model.StorageUploadStatusReadable). + Where("id > ?", afterID). + Where(`EXISTS ( + SELECT 1 FROM object_versions AS live_version + WHERE live_version.is_delete_marker = ? + AND live_version.state IN (?, ?) + AND `+objectVersionReferencesStorageUploadSQL("live_version", "storage_upload")+` + )`, false, model.ObjectStateStored, model.ObjectStateCacheEvicted). + OrderExpr("id ASC") + if limit > 0 { + q = q.Limit(limit) + } + if err := q.Scan(ctx); err != nil { + return nil, fmt.Errorf("listing incomplete readable storage uploads: %w", err) + } + + items := make([]IncompleteReadableUpload, 0, len(uploads)) + for i := range uploads { + version := new(model.ObjectVersion) + err := r.db.NewSelect(). + Model(version). + Where("is_delete_marker = ?", false). + Where("state IN (?, ?)", model.ObjectStateStored, model.ObjectStateCacheEvicted). + Where(objectVersionReferencesStorageUploadIDSQL, uploads[i].ID, uploads[i].SourceVersionID). + OrderExpr("in_cache DESC"). + OrderExpr("is_current DESC"). + OrderExpr("created_at DESC"). + OrderExpr("version_id DESC"). + Limit(1). + Scan(ctx) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + continue + } + return nil, fmt.Errorf("selecting durable version for incomplete readable upload %d: %w", uploads[i].ID, err) + } + items = append(items, IncompleteReadableUpload{ + Upload: uploads[i], + Version: *version, + }) + } + return items, nil +} + func (r *BunStorageUploadRepo) ReassignIngressCopy(ctx context.Context, uploadID int64, unavailableCopyIndex int) (*model.StorageUploadCopy, error) { var selected *model.StorageUploadCopy err := r.runMaybeTx(ctx, func(db bun.IDB) error { @@ -1664,6 +1716,24 @@ func (r *BunStorageUploadRepo) FinalizeUploadIfTargetCopiesMet(ctx context.Conte var refs []ObjectVersionRef finalized := false err := r.runMaybeTx(ctx, func(db bun.IDB) error { + var bucketID int64 + if err := db.NewSelect(). + Model((*model.StorageUpload)(nil)). + Column("bucket_id"). + Where("id = ?", input.UploadID). + Scan(ctx, &bucketID); err != nil { + if err == sql.ErrNoRows { + return fmt.Errorf("loading storage upload %d: %w", input.UploadID, ErrNotFound) + } + return fmt.Errorf("loading storage upload bucket: %w", err) + } + bucket, err := lockBucketByID(ctx, db, bucketID) + if err != nil { + return err + } + if bucket == nil { + return fmt.Errorf("storage upload %d bucket not found: %w", input.UploadID, ErrNotFound) + } locked, err := lockStorageUploadsByID(ctx, db, []int64{input.UploadID}) if err != nil { return fmt.Errorf("locking storage upload for finalization: %w", err) @@ -1676,21 +1746,11 @@ func (r *BunStorageUploadRepo) FinalizeUploadIfTargetCopiesMet(ctx context.Conte if err != nil { return err } - if upload.RequestedCopies <= 0 || readable < upload.RequestedCopies { + minimum := minimumDurableCopiesForUpload(bucket, upload.RequestedCopies) + if minimum <= 0 || readable < minimum { return nil } now := time.Now() - _, err = db.NewUpdate(). - Model((*model.StorageUpload)(nil)). - Set("status = ?", model.StorageUploadStatusComplete). - Set("accepted_at = COALESCE(accepted_at, ?)", now). - Set("accept_error = NULL"). - Set("updated_at = ?", now). - Where("id = ?", input.UploadID). - Exec(ctx) - if err != nil { - return fmt.Errorf("marking storage upload complete: %w", err) - } err = db.NewRaw(`UPDATE object_versions SET state = ?, updated_at = ? WHERE storage_upload_id = ? AND state = ? @@ -1698,12 +1758,9 @@ func (r *BunStorageUploadRepo) FinalizeUploadIfTargetCopiesMet(ctx context.Conte model.ObjectStateStored, now, input.UploadID, model.ObjectStateReplicating, ).Scan(ctx, &refs) if err != nil && err != sql.ErrNoRows { - return fmt.Errorf("finalizing object versions for upload: %w", err) + return fmt.Errorf("marking durable object versions stored: %w", err) } for _, ref := range refs { - if err := completeUploadTasksForVersion(ctx, db, ref.VersionID, now, unclaimedTaskStatuses()); err != nil { - return err - } if input.EnqueueAfterUploadEviction { evictions := &BunCacheEvictionRepo{db: db} if _, err := evictions.EnsureAfterUploadTask( @@ -1720,12 +1777,61 @@ func (r *BunStorageUploadRepo) FinalizeUploadIfTargetCopiesMet(ctx context.Conte } } } + if upload.RequestedCopies <= 0 || readable < upload.RequestedCopies { + return nil + } + _, err = db.NewUpdate(). + Model((*model.StorageUpload)(nil)). + Set("status = ?", model.StorageUploadStatusComplete). + Set("accepted_at = COALESCE(accepted_at, ?)", now). + Set("accept_error = NULL"). + Set("updated_at = ?", now). + Where("id = ?", input.UploadID). + Exec(ctx) + if err != nil { + return fmt.Errorf("marking storage upload complete: %w", err) + } + if err := completeUploadTasksForUpload(ctx, db, input.UploadID, now, unclaimedTaskStatuses()); err != nil { + return err + } finalized = true return nil }) return finalized, refs, err } +func minimumDurableCopiesForUpload(bucket *model.Bucket, requestedCopies int) int { + if requestedCopies <= 0 { + return 0 + } + if bucket == nil || bucket.MinimumDurableCopies == nil || *bucket.MinimumDurableCopies >= requestedCopies { + return requestedCopies + } + return *bucket.MinimumDurableCopies +} + +func requireCurrentMinimumDurableCopies(ctx context.Context, db bun.IDB, upload *model.StorageUpload) error { + if upload == nil { + return fmt.Errorf("storage upload is required: %w", ErrInvalidInput) + } + bucket := new(model.Bucket) + if err := db.NewSelect().Model(bucket).Where("id = ?", upload.BucketID).Scan(ctx); err != nil { + if err == sql.ErrNoRows { + return fmt.Errorf("storage upload bucket not found: %w", ErrNotFound) + } + return fmt.Errorf("loading storage upload bucket durability policy: %w", err) + } + minimum := minimumDurableCopiesForUpload(bucket, upload.RequestedCopies) + readable, err := countReadableCommittedCopies(ctx, db, upload.ID) + if err != nil { + return err + } + if minimum <= 0 || readable < minimum { + return fmt.Errorf("storage upload %d has %d of %d required durable copies: %w", upload.ID, readable, minimum, ErrConflict) + } + return nil +} + func (r *BunStorageUploadRepo) FindActiveUploadBySourceVersion(ctx context.Context, versionID string) (*model.StorageUpload, error) { return r.findActiveUploadBySourceVersion(ctx, versionID) } @@ -2074,6 +2180,30 @@ func completeUploadTasksForVersion(ctx context.Context, db bun.IDB, versionID st return nil } +func completeUploadTasksForUpload(ctx context.Context, db bun.IDB, uploadID int64, now time.Time, statuses []model.TaskStatus) error { + if uploadID <= 0 || len(statuses) == 0 { + return fmt.Errorf("completing upload tasks for storage upload: %w", ErrInvalidInput) + } + _, err := db.NewUpdate(). + Model((*model.Task)(nil)). + Set("status = ?", model.TaskStatusCompleted). + Set("completed_at = ?", now). + Set("last_error = NULL"). + Set("wait_reason = NULL"). + Set("status_message = NULL"). + Set("claimed_at = NULL"). + Set("lease_until = NULL"). + Set("started_at = NULL"). + Where("ref_type = ? AND type = ?", "object", model.TaskTypeUpload). + Where("status IN (?)", bun.List(statuses)). + Where("ref_version_id IN (SELECT version_id FROM object_versions WHERE storage_upload_id = ?)", uploadID). + Exec(ctx) + if err != nil { + return fmt.Errorf("completing upload tasks for storage upload: %w", err) + } + return nil +} + func activeUploadStatuses() []model.StorageUploadStatus { return []model.StorageUploadStatus{ model.StorageUploadStatusRunning, diff --git a/internal/db/repository/storage_upload_repo_test.go b/internal/db/repository/storage_upload_repo_test.go index 0da4090..3ce8db2 100644 --- a/internal/db/repository/storage_upload_repo_test.go +++ b/internal/db/repository/storage_upload_repo_test.go @@ -1673,6 +1673,125 @@ func TestStorageUploadRepo_FinalizeUploadIfTargetCopiesMetMovesReplicatingToStor } } +func TestStorageUploadRepo_MinimumDurabilityStoresBeforeTargetAndKeepsRepairWork(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "minimum-durability-finalize-bucket") + minimum := 2 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimum, + }); err != nil { + t.Fatalf("UpdateCopyPolicy: %v", err) + } + + version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000MIN02", 10) + version.Checksum = "minimum-durability-checksum" + objectID, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version) + if err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 3) + commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") + commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 1, "202", "2002", "2002", "https://two.example/piece") + bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) + + stage := "peer_pull" + repairTask := &model.Task{ + Type: model.TaskTypeUpload, + Stage: &stage, + RefType: "object", + RefID: objectID, + RefVersionID: version.VersionID, + IdempotencyKey: "upload:minimum-durability-third-copy", + Payload: map[string]interface{}{"upload_id": upload.ID, "copy_index": 2}, + Status: model.TaskStatusQueued, + MaxRetries: 5, + ScheduledAt: time.Now(), + } + if err := repos.Tasks.Create(ctx, repairTask); err != nil { + t.Fatalf("Create repair task: %v", err) + } + + done, refs, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}) + if err != nil { + t.Fatalf("FinalizeUploadIfTargetCopiesMet minimum: %v", err) + } + if done || len(refs) != 1 || refs[0].VersionID != version.VersionID { + t.Fatalf("minimum finalize = done:%v refs:%#v, want stored without upload completion", done, refs) + } + gotVersion, err := repos.Objects.GetVersionByID(ctx, version.VersionID) + if err != nil || gotVersion == nil || gotVersion.State != model.ObjectStateStored { + t.Fatalf("version after minimum = %#v err=%v, want stored", gotVersion, err) + } + gotUpload, err := repos.Uploads.GetByID(ctx, upload.ID) + if err != nil || gotUpload == nil || gotUpload.Status != model.StorageUploadStatusReadable { + t.Fatalf("upload after minimum = %#v err=%v, want readable", gotUpload, err) + } + gotTask, err := repos.Tasks.GetByID(ctx, repairTask.ID) + if err != nil || gotTask == nil || gotTask.Status != model.TaskStatusQueued { + t.Fatalf("repair task after minimum = %#v err=%v, want queued", gotTask, err) + } + + commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 2, "303", "3003", "2003", "https://three.example/piece") + done, refs, err = repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}) + if err != nil { + t.Fatalf("FinalizeUploadIfTargetCopiesMet target: %v", err) + } + if !done || len(refs) != 0 { + t.Fatalf("target finalize = done:%v refs:%#v, want complete without another state transition", done, refs) + } + gotUpload, err = repos.Uploads.GetByID(ctx, upload.ID) + if err != nil || gotUpload == nil || gotUpload.Status != model.StorageUploadStatusComplete { + t.Fatalf("upload after target = %#v err=%v, want complete", gotUpload, err) + } + gotTask, err = repos.Tasks.GetByID(ctx, repairTask.ID) + if err != nil || gotTask == nil || gotTask.Status != model.TaskStatusCompleted { + t.Fatalf("repair task after target = %#v err=%v, want completed", gotTask, err) + } +} + +func TestStorageUploadRepo_ListIncompleteReadableUploadsIncludesCommittedUnavailableSlot(t *testing.T) { + db := testDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := seedBucket(t, db, "minimum-durability-recovery-bucket") + minimum := 1 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimum, + }); err != nil { + t.Fatalf("UpdateCopyPolicy: %v", err) + } + + version := newObjectVersion(bucket.ID, "file.txt", "01J000000000000000000MIN03", 10) + version.Checksum = "minimum-durability-recovery-checksum" + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent: %v", err) + } + upload := startCopyHealthUpload(t, repos, bucket.ID, version.VersionID, version.Size, version.Checksum, 2) + commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 0, "101", "1001", "2001", "https://one.example/piece") + unavailable := commitStorageHealthCopy(t, repos, bucket.ID, upload.ID, 1, "202", "2002", "2002", "https://two.example/piece") + bindStorageHealthVersion(t, repos, bucket.ID, upload.ID, version) + if err := repos.Uploads.MarkDataSetUnavailable(ctx, unavailable.ID, "temporary outage"); err != nil { + t.Fatalf("MarkDataSetUnavailable: %v", err) + } + if complete, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil || complete { + t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err:%v, want durable before target", complete, err) + } + + items, err := repos.Uploads.ListIncompleteReadableUploads(ctx, 0, 100) + if err != nil { + t.Fatalf("ListIncompleteReadableUploads: %v", err) + } + if len(items) != 1 || items[0].Upload.ID != upload.ID || items[0].Version.VersionID != version.VersionID { + t.Fatalf("incomplete readable uploads = %#v, want upload %d version %s", items, upload.ID, version.VersionID) + } +} + func TestStorageUploadRepo_PrimaryCopyFailureMarksUploadFailed(t *testing.T) { db := testDB(t) repos := repository.NewRepositories(db) diff --git a/internal/model/bucket.go b/internal/model/bucket.go index aacb2ff..994b0e9 100644 --- a/internal/model/bucket.go +++ b/internal/model/bucket.go @@ -26,14 +26,15 @@ func (s BucketStatus) IsWritable() bool { return true } type Bucket struct { bun.BaseModel `bun:"table:buckets"` - ID int64 `bun:",pk,autoincrement"` - Name string `bun:",unique,notnull"` - ACL []byte `bun:",nullzero"` - OwnerAccessKey *string `bun:",nullzero"` - DefaultCopies *int `bun:",nullzero"` - Status BucketStatus `bun:",notnull,default:'active'"` - CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` - UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + ID int64 `bun:",pk,autoincrement"` + Name string `bun:",unique,notnull"` + ACL []byte `bun:",nullzero"` + OwnerAccessKey *string `bun:",nullzero"` + DefaultCopies *int `bun:",nullzero"` + MinimumDurableCopies *int `bun:",nullzero"` + Status BucketStatus `bun:",notnull,default:'active'"` + CreatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` + UpdatedAt time.Time `bun:",nullzero,notnull,default:current_timestamp"` Owner *S3Account `bun:"rel:belongs-to,join:owner_access_key=access_key,on_update:cascade,on_delete:restrict"` } diff --git a/internal/worker/evictor.go b/internal/worker/evictor.go index ecafaa8..f007b05 100644 --- a/internal/worker/evictor.go +++ b/internal/worker/evictor.go @@ -2,7 +2,6 @@ package worker import ( "context" - "fmt" "log/slog" "sync" "time" @@ -203,6 +202,9 @@ func (e *Evictor) processTask(ctx context.Context, task *model.Task) { decision = e.processLRUEviction(ctx, task) case cacheeviction.StageAfterUpload: decision = e.processAfterUploadEviction(ctx, task) + case cacheeviction.StageReconcileBucketDurability: + e.processBucketDurabilityReconciliation(ctx, task) + return default: decision = cancelEviction("Cache eviction task uses an unsupported stage") } @@ -268,7 +270,7 @@ func (e *Evictor) deferReplicatingEviction(ctx context.Context, task *model.Task ctx, task, model.TaskWaitReasonDependency, - "waiting for all copies to commit", + "Waiting for enough durable replicas to release the cache", replicatingEvictDeferDelay, ); err != nil { logger.Error("failed to defer replicating cache eviction", "error", err) @@ -277,40 +279,7 @@ func (e *Evictor) deferReplicatingEviction(ctx context.Context, task *model.Task return } admin.WorkerTasksProcessed.WithLabelValues("evictor", "success").Inc() - logger.Info("cache eviction deferred until replication completes") -} - -func (e *Evictor) recordDeletedCacheState( - ctx context.Context, - task *model.Task, - version *model.ObjectVersion, -) error { - switch version.State { - case model.ObjectStateStored: - if err := state.TransitionState( - ctx, - e.stateMachine, - e.repos.Objects, - task.RefVersionID, - model.ObjectStateStored, - model.ObjectStateCacheEvicted, - ); err != nil { - if latest, latestErr := e.repos.Objects.GetVersionByID(ctx, task.RefVersionID); latestErr == nil && - latest != nil && - latest.State == model.ObjectStateCacheEvicted && - !latest.InCache { - return nil - } - return err - } - case model.ObjectStateCacheEvicted: - if err := e.repos.Objects.SetVersionCachePresence(ctx, task.RefVersionID, false); err != nil { - return err - } - default: - return fmt.Errorf("object is no longer eligible for cache eviction: state %s", version.State) - } - return nil + logger.Info("cache eviction deferred until durability policy is satisfied") } func (e *Evictor) completeTask(ctx context.Context, task *model.Task, logger *slog.Logger, message string) { diff --git a/internal/worker/evictor_after_upload.go b/internal/worker/evictor_after_upload.go index f007174..4afca1b 100644 --- a/internal/worker/evictor_after_upload.go +++ b/internal/worker/evictor_after_upload.go @@ -2,8 +2,11 @@ package worker import ( "context" + "errors" "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/cacheeviction" + "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" ) @@ -11,7 +14,11 @@ func (e *Evictor) processAfterUploadEviction( ctx context.Context, task *model.Task, ) *evictionDecision { - if e.policy != cache.EvictionPolicyAfterUpload { + authorized, err := cacheeviction.DeleteAuthorized(task) + if err != nil { + return cancelEviction("Cache deletion authorization is invalid") + } + if !authorized && e.policy != cache.EvictionPolicyAfterUpload { return cancelEviction("Cache eviction policy no longer removes objects after upload") } @@ -26,45 +33,17 @@ func (e *Evictor) finalizeAfterUploadEviction( ctx context.Context, task *model.Task, ) *evictionDecision { - version, err := e.repos.Objects.GetVersionByID(ctx, task.RefVersionID) - if err != nil { - return retryEviction(err, "loading object version for after-upload cache eviction") - } - if version == nil { - return failEviction("object not found", "object version not found for after-upload cache eviction") - } - switch version.State { - case model.ObjectStateReplicating: + deletion, err := e.repos.CacheEvictions.AuthorizeDeletion(ctx, task, nil) + switch { + case err == nil: + return e.deleteCacheEntry(ctx, task, deletion) + case errors.Is(err, cacheeviction.ErrDurabilityThreshold): return waitForEvictionDependency() - case model.ObjectStateStored: - default: + case errors.Is(err, repository.ErrNotFound): + return failEviction("object not found", "object version not found for after-upload cache eviction") + case errors.Is(err, cacheeviction.ErrNoLongerEligible), errors.Is(err, cacheeviction.ErrAccessChanged): return failEviction("not stored", "object version not in stored state") + default: + return retryEviction(err, "authorizing after-upload cache eviction") } - - bucket, err := e.repos.Buckets.GetByID(ctx, version.BucketID) - if err != nil { - return retryEviction(err, "loading object bucket for after-upload cache eviction") - } - if bucket == nil { - return failEviction("bucket not found", "bucket not found for after-upload cache eviction") - } - - readable, err := e.hasReadableRemoteCopy(ctx, version) - if err != nil { - return retryEviction(err, "checking readable remote copies before after-upload cache eviction") - } - if !readable { - if version.StorageUploadID == nil { - return failEviction( - "no accepted upload", - "object version has no accepted upload, refusing after-upload cache eviction", - ) - } - return failEviction( - "no readable upload copies", - "object version has no readable upload copies, refusing after-upload cache eviction", - ) - } - - return e.deleteCacheEntry(ctx, task, bucket.Name, version) } diff --git a/internal/worker/evictor_after_upload_failure_test.go b/internal/worker/evictor_after_upload_failure_test.go index 05ddee7..bf3f917 100644 --- a/internal/worker/evictor_after_upload_failure_test.go +++ b/internal/worker/evictor_after_upload_failure_test.go @@ -5,11 +5,13 @@ import ( "errors" "io" "strings" + "sync/atomic" "testing" "time" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/cacheeviction" + "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/testutil" ) @@ -54,21 +56,6 @@ func TestEvictor_Preconditions(t *testing.T) { }, wantLastError: "not stored", }, - { - name: "NoReadableCopies", - setup: func(ctx context.Context, t *testing.T, env *testWorkerEnv) *model.Task { - _, objID, versionID := seedStoredObject(t, env) - version, err := env.repos.Objects.GetVersionByID(ctx, versionID) - if err != nil || version == nil || version.StorageUploadID == nil { - t.Fatalf("stored version upload: version=%v err=%v", version, err) - } - if _, err := env.db.NewDelete().Model((*model.StorageUploadCopy)(nil)).Where("upload_id = ?", *version.StorageUploadID).Exec(ctx); err != nil { - t.Fatalf("remove readable copies: %v", err) - } - return seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, 0) - }, - wantLastError: "no readable upload copies", - }, } for _, tt := range tests { @@ -99,6 +86,44 @@ func TestEvictor_Preconditions(t *testing.T) { } } +func TestEvictor_AfterUploadWaitsWhenMinimumIsNoLongerMet(t *testing.T) { + var deleteCalls atomic.Int64 + mc := &testutil.MockCache{DeleteFunc: func(context.Context, string, string) error { + deleteCalls.Add(1) + return nil + }} + env := newTestWorkerEnvWithMockCache(t, mc) + ctx := context.Background() + _, objID, versionID := seedStoredObject(t, env) + version, err := env.repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil || version.StorageUploadID == nil { + t.Fatalf("stored version upload: version=%v err=%v", version, err) + } + if _, err := env.db.NewDelete().Model((*model.StorageUploadCopy)(nil)).Where("upload_id = ?", *version.StorageUploadID).Exec(ctx); err != nil { + t.Fatalf("remove readable copies: %v", err) + } + task := seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, 0) + evictor := newAfterUploadEvictor(env, 1, 10*time.Millisecond) + + runCtx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + _ = evictor.Run(runCtx) + close(done) + }() + waitForTaskStatus(t, env, task.ID, model.TaskStatusWaiting, 3*time.Second) + cancel() + waitForSignal(t, done, time.Second, "after-upload durability wait shutdown") + + got, err := env.repos.Tasks.GetByID(ctx, task.ID) + if err != nil || got == nil || got.RetryCount != 0 || got.WaitReason == nil || *got.WaitReason != model.TaskWaitReasonDependency { + t.Fatalf("waiting task = %#v err=%v", got, err) + } + if deleteCalls.Load() != 0 { + t.Fatalf("cache delete calls = %d, want 0", deleteCalls.Load()) + } +} + func TestEvictor_CacheDeleteFailureLeavesObjectUnchangedAndKeepsTaskRecoverable(t *testing.T) { for _, tc := range []struct { name string @@ -154,3 +179,63 @@ func TestEvictor_CacheDeleteFailureLeavesObjectUnchangedAndKeepsTaskRecoverable( }) } } + +func TestEvictor_DeletionAuthorizationSurvivesStateWriteFailure(t *testing.T) { + var deleteCalls atomic.Int64 + mc := &testutil.MockCache{DeleteFunc: func(context.Context, string, string) error { + deleteCalls.Add(1) + return nil + }} + env := newTestWorkerEnvWithMockCache(t, mc) + ctx := context.Background() + _, objID, versionID := seedStoredObject(t, env) + task := seedTask(t, env, model.TaskTypeEvictCache, objID, versionID, 5, 0) + records := &failFirstDeletionRecordRepo{CacheEvictionRepository: env.repos.CacheEvictions} + env.repos.CacheEvictions = records + evictor := newAfterUploadEvictor(env, 1, 10*time.Millisecond) + + runWorkerUntilTaskRetryCount(t, env, evictor, task.ID, 1, 5*time.Second) + interrupted, err := env.repos.Tasks.GetByID(ctx, task.ID) + if err != nil || interrupted == nil { + t.Fatalf("GetByID after interrupted record: task=%#v err=%v", interrupted, err) + } + authorized, err := cacheeviction.DeleteAuthorized(interrupted) + if err != nil || !authorized || interrupted.RefVersionID != versionID { + t.Fatalf("interrupted authorization = task:%#v authorized:%t err:%v", interrupted, authorized, err) + } + version, err := env.repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil || version.State != model.ObjectStateStored || !version.InCache { + t.Fatalf("version before state convergence = %#v err=%v", version, err) + } + if _, err := env.db.NewUpdate(). + Model((*model.Task)(nil)). + Set("scheduled_at = ?", time.Now().Add(-time.Second)). + Where("id = ?", task.ID). + Exec(ctx); err != nil { + t.Fatalf("make authorized retry ready: %v", err) + } + + completed := runWorkerUntilTask(t, env, evictor, task.ID, 5*time.Second) + if completed.Status != model.TaskStatusCompleted || completed.RetryCount != 1 { + t.Fatalf("completed authorized retry = %#v", completed) + } + version, err = env.repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil || version.State != model.ObjectStateCacheEvicted || version.InCache { + t.Fatalf("version after state convergence = %#v err=%v", version, err) + } + if deleteCalls.Load() != 2 || records.calls.Load() != 2 { + t.Fatalf("recovery calls = delete:%d record:%d, want 2/2", deleteCalls.Load(), records.calls.Load()) + } +} + +type failFirstDeletionRecordRepo struct { + repository.CacheEvictionRepository + calls atomic.Int64 +} + +func (r *failFirstDeletionRecordRepo) RecordAuthorizedDeletion(ctx context.Context, task *model.Task) error { + if r.calls.Add(1) == 1 { + return errors.New("injected state write failure") + } + return r.CacheEvictionRepository.RecordAuthorizedDeletion(ctx, task) +} diff --git a/internal/worker/evictor_after_upload_test.go b/internal/worker/evictor_after_upload_test.go index 24a6efb..ce2dbbd 100644 --- a/internal/worker/evictor_after_upload_test.go +++ b/internal/worker/evictor_after_upload_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" "github.com/strahe/synaps3/internal/testutil" @@ -102,21 +103,22 @@ func TestEvictor_HappyPath(t *testing.T) { } } -type observableReadableCheckRepo struct { - repository.StorageUploadRepository +type observableDeletionAuthorizationRepo struct { + repository.CacheEvictionRepository calls atomic.Int64 started chan struct{} } -func (r *observableReadableCheckRepo) HasReadableCommittedCopy( +func (r *observableDeletionAuthorizationRepo) AuthorizeDeletion( ctx context.Context, - uploadID int64, -) (bool, error) { + task *model.Task, + expectedAccess *time.Time, +) (*cacheeviction.AuthorizedDeletion, error) { if r.calls.Add(1) == 1 { close(r.started) } - return r.StorageUploadRepository.HasReadableCommittedCopy(ctx, uploadID) + return r.CacheEvictionRepository.AuthorizeDeletion(ctx, task, expectedAccess) } func TestEvictor_ChecksRemoteSafetyAfterWaitingForOpenBody(t *testing.T) { @@ -140,11 +142,11 @@ func TestEvictor_ChecksRemoteSafetyAfterWaitingForOpenBody(t *testing.T) { } dataSetID := *copies[0].StorageDataSetID - checks := &observableReadableCheckRepo{ - StorageUploadRepository: env.repos.Uploads, + checks := &observableDeletionAuthorizationRepo{ + CacheEvictionRepository: env.repos.CacheEvictions, started: make(chan struct{}), } - env.repos.Uploads = checks + env.repos.CacheEvictions = checks opened, err := env.cacheGate.Open( versionID, func() (io.ReadCloser, *cache.ObjectInfo, error) { @@ -186,7 +188,7 @@ func TestEvictor_ChecksRemoteSafetyAfterWaitingForOpenBody(t *testing.T) { t.Fatalf("close cache body: %v", err) } waitForSignal(t, checks.started, time.Second, "final remote-safety check") - waitForTaskStatus(t, env, task.ID, model.TaskStatusFailed, 3*time.Second) + waitForTaskStatus(t, env, task.ID, model.TaskStatusWaiting, 3*time.Second) if deleteCalls.Load() != 0 { t.Fatalf("cache Delete calls after remote safety changed = %d, want 0", deleteCalls.Load()) diff --git a/internal/worker/evictor_concurrency_test.go b/internal/worker/evictor_concurrency_test.go index 59dc937..e730726 100644 --- a/internal/worker/evictor_concurrency_test.go +++ b/internal/worker/evictor_concurrency_test.go @@ -239,7 +239,7 @@ func TestEvictor_ReplicatingVersionDefersEvictionAndKeepsCache(t *testing.T) { } continue } - if current.StatusMessage != nil && strings.Contains(*current.StatusMessage, "waiting for all copies") { + if current.StatusMessage != nil && strings.Contains(*current.StatusMessage, "enough durable replicas") { gotTask = current } } @@ -256,7 +256,7 @@ func TestEvictor_ReplicatingVersionDefersEvictionAndKeepsCache(t *testing.T) { if gotTask.WaitReason == nil || *gotTask.WaitReason != model.TaskWaitReasonDependency { t.Fatalf("task wait_reason = %v, want dependency", gotTask.WaitReason) } - if gotTask.StatusMessage == nil || !strings.Contains(*gotTask.StatusMessage, "waiting for all copies") { + if gotTask.StatusMessage == nil || !strings.Contains(*gotTask.StatusMessage, "enough durable replicas") { t.Fatalf("task status_message = %v, want waiting-for-copies reason", gotTask.StatusMessage) } if !gotTask.ScheduledAt.After(originalScheduledAt) { diff --git a/internal/worker/evictor_finalize.go b/internal/worker/evictor_finalize.go index 58e92c7..9352448 100644 --- a/internal/worker/evictor_finalize.go +++ b/internal/worker/evictor_finalize.go @@ -2,7 +2,9 @@ package worker import ( "context" + "fmt" + "github.com/strahe/synaps3/internal/cacheeviction" "github.com/strahe/synaps3/internal/model" ) @@ -26,13 +28,37 @@ type evictionDecision struct { func (e *Evictor) deleteCacheEntry( ctx context.Context, task *model.Task, - bucketName string, - version *model.ObjectVersion, + deletion *cacheeviction.AuthorizedDeletion, ) *evictionDecision { - if err := e.cache.Delete(ctx, bucketName, version.CacheKey); err != nil { - return retryEviction(err, "deleting cache entry") + if deletion == nil { + return failEviction("cache deletion was not authorized", "cache eviction has no authorized target") + } + if deletion.Version.InCache { + if err := e.cache.Delete(ctx, deletion.BucketName, deletion.Version.CacheKey); err != nil { + return retryEviction(err, "deleting cache entry") + } + } + return e.recordCacheEntryDeleted(ctx, task, &deletion.Version) +} + +func (e *Evictor) deleteCoordinatorCacheEntry( + ctx context.Context, + task *model.Task, + deletion *cacheeviction.AuthorizedDeletion, +) error { + if deletion == nil { + return cacheeviction.ErrNoLongerEligible } - return e.recordCacheEntryDeleted(ctx, task, version) + if deletion.Version.InCache { + if err := e.cache.Delete(ctx, deletion.BucketName, deletion.Version.CacheKey); err != nil { + return err + } + } + e.cacheAccessTracker.Forget(deletion.Version.VersionID) + if err := e.repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); err != nil { + return fmt.Errorf("recording cache eviction state: %w", err) + } + return nil } func (e *Evictor) recordCacheEntryDeleted( @@ -41,22 +67,12 @@ func (e *Evictor) recordCacheEntryDeleted( version *model.ObjectVersion, ) *evictionDecision { e.cacheAccessTracker.Forget(version.VersionID) - if err := e.recordDeletedCacheState(ctx, task, version); err != nil { + if err := e.repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); err != nil { return retryEviction(err, "recording cache eviction state") } return completeEviction() } -func (e *Evictor) hasReadableRemoteCopy( - ctx context.Context, - version *model.ObjectVersion, -) (bool, error) { - if version.StorageUploadID == nil { - return false, nil - } - return e.repos.Uploads.HasReadableCommittedCopy(ctx, *version.StorageUploadID) -} - func (e *Evictor) applyEvictionDecision( ctx context.Context, task *model.Task, diff --git a/internal/worker/evictor_lru.go b/internal/worker/evictor_lru.go index c12738c..ebec026 100644 --- a/internal/worker/evictor_lru.go +++ b/internal/worker/evictor_lru.go @@ -2,11 +2,13 @@ package worker import ( "context" + "errors" "fmt" "time" "github.com/strahe/synaps3/internal/cache" "github.com/strahe/synaps3/internal/cacheeviction" + "github.com/strahe/synaps3/internal/db/repository" "github.com/strahe/synaps3/internal/model" ) @@ -14,6 +16,25 @@ func (e *Evictor) processLRUEviction( ctx context.Context, task *model.Task, ) *evictionDecision { + authorized, err := cacheeviction.DeleteAuthorized(task) + if err != nil { + return cancelEviction("LRU cache deletion authorization is invalid") + } + if authorized { + var decision *evictionDecision + e.cacheGate.GuardDeletion(task.RefVersionID, func() { + deletion, authorizeErr := e.repos.CacheEvictions.AuthorizeDeletion(ctx, task, nil) + switch { + case authorizeErr == nil: + decision = e.deleteCacheEntry(ctx, task, deletion) + case errors.Is(authorizeErr, repository.ErrNotFound), errors.Is(authorizeErr, cacheeviction.ErrNoLongerEligible): + decision = cancelEviction("Authorized cache entry no longer exists") + default: + decision = retryEviction(authorizeErr, "resuming authorized LRU cache eviction") + } + }) + return decision + } if e.policy != cache.EvictionPolicyLRU { return cancelEviction("Cache eviction policy no longer uses LRU") } @@ -76,31 +97,29 @@ func (e *Evictor) finalizeLRUEviction( return cancelEviction("Object was accessed after this LRU eviction was planned") } - bucket, err := e.repos.Buckets.GetByID(ctx, version.BucketID) - if err != nil { - return retryEviction(err, "loading object bucket for LRU cache eviction") - } - if bucket == nil { - return cancelEviction("Object bucket no longer exists") + if !e.reserveLRUDeletion(version.Size) { + return cancelEviction("LRU cache usage already reached the low watermark") } - - readable, err := e.hasReadableRemoteCopy(ctx, version) + deletion, err := e.repos.CacheEvictions.AuthorizeDeletion(ctx, task, &accessSnapshot) if err != nil { - return retryEviction(err, "checking readable remote copies before LRU cache eviction") - } - if !readable { - return cancelEviction("Object no longer has a readable committed remote copy") + e.finishLRUDeletion(version.Size, false) + switch { + case errors.Is(err, cacheeviction.ErrDurabilityThreshold), + errors.Is(err, cacheeviction.ErrNoLongerEligible), + errors.Is(err, cacheeviction.ErrAccessChanged): + return cancelEviction("Object is no longer eligible for LRU cache eviction") + default: + return retryEviction(err, "authorizing LRU cache eviction") + } } - - if !e.reserveLRUDeletion(version.Size) { - return cancelEviction("LRU cache usage already reached the low watermark") + if deletion.Version.InCache { + err = e.cache.Delete(ctx, deletion.BucketName, deletion.Version.CacheKey) } - err = e.cache.Delete(ctx, bucket.Name, version.CacheKey) e.finishLRUDeletion(version.Size, err == nil) if err != nil { return retryEviction(err, "deleting cache entry") } - return e.recordCacheEntryDeleted(ctx, task, version) + return e.recordCacheEntryDeleted(ctx, task, &deletion.Version) } func effectiveLRUAccessTime(version *model.ObjectVersion) time.Time { diff --git a/internal/worker/evictor_lru_capacity_test.go b/internal/worker/evictor_lru_capacity_test.go index f856db5..77b83fb 100644 --- a/internal/worker/evictor_lru_capacity_test.go +++ b/internal/worker/evictor_lru_capacity_test.go @@ -17,23 +17,19 @@ import ( "github.com/strahe/synaps3/internal/worker" ) -type blockingVersionStateRepo struct { - repository.ObjectRepository +type blockingDeletionRecordRepo struct { + repository.CacheEvictionRepository versionID string entered chan struct{} release <-chan struct{} once sync.Once } -func (r *blockingVersionStateRepo) UpdateVersionState( +func (r *blockingDeletionRecordRepo) RecordAuthorizedDeletion( ctx context.Context, - versionID string, - from model.ObjectState, - to model.ObjectState, + task *model.Task, ) error { - if versionID == r.versionID && - from == model.ObjectStateStored && - to == model.ObjectStateCacheEvicted { + if task.RefVersionID == r.versionID { r.once.Do(func() { close(r.entered) }) select { case <-r.release: @@ -41,7 +37,7 @@ func (r *blockingVersionStateRepo) UpdateVersionState( return ctx.Err() } } - return r.ObjectRepository.UpdateVersionState(ctx, versionID, from, to) + return r.CacheEvictionRepository.RecordAuthorizedDeletion(ctx, task) } func TestEvictor_LRUCapacityReservationEndsAtPhysicalDelete(t *testing.T) { @@ -89,11 +85,11 @@ func TestEvictor_LRUCapacityReservationEndsAtPhysicalDelete(t *testing.T) { stateUpdateEntered := make(chan struct{}) releaseStateUpdate := make(chan struct{}) var releaseStateOnce sync.Once - env.repos.Objects = &blockingVersionStateRepo{ - ObjectRepository: env.repos.Objects, - versionID: firstVersionID, - entered: stateUpdateEntered, - release: releaseStateUpdate, + env.repos.CacheEvictions = &blockingDeletionRecordRepo{ + CacheEvictionRepository: env.repos.CacheEvictions, + versionID: firstVersionID, + entered: stateUpdateEntered, + release: releaseStateUpdate, } heldSecond, err := env.cacheGate.Open( diff --git a/internal/worker/evictor_lru_finalize_test.go b/internal/worker/evictor_lru_finalize_test.go index b6ad271..9d63924 100644 --- a/internal/worker/evictor_lru_finalize_test.go +++ b/internal/worker/evictor_lru_finalize_test.go @@ -218,11 +218,11 @@ func TestEvictor_LRUChecksRemoteSafetyAfterWaitingForOpenBody(t *testing.T) { t.Fatalf("RecordVersionCacheAccess: %v", err) } task := seedLRUEvictionTask(t, env, objectID, versionID, plannedAt) - checks := &observableReadableCheckRepo{ - StorageUploadRepository: env.repos.Uploads, + checks := &observableDeletionAuthorizationRepo{ + CacheEvictionRepository: env.repos.CacheEvictions, started: make(chan struct{}), } - env.repos.Uploads = checks + env.repos.CacheEvictions = checks opened, err := env.cacheGate.Open( versionID, diff --git a/internal/worker/evictor_reconcile.go b/internal/worker/evictor_reconcile.go new file mode 100644 index 0000000..dc90c3b --- /dev/null +++ b/internal/worker/evictor_reconcile.go @@ -0,0 +1,144 @@ +package worker + +import ( + "context" + "errors" + + "github.com/strahe/synaps3/internal/admin" + "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/cacheeviction" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" +) + +const bucketDurabilityBatchSize = 100 + +func (e *Evictor) processBucketDurabilityReconciliation(ctx context.Context, task *model.Task) { + logger := e.taskLogger(task) + processed := 0 + for processed < bucketDurabilityBatchSize { + authorized, err := cacheeviction.DeleteAuthorized(task) + if err != nil { + e.cancelTask(ctx, task, "Cache deletion authorization is invalid") + return + } + if authorized { + if decision := e.resumeBucketDurabilityDeletion(ctx, task); decision != nil { + e.applyEvictionDecision(ctx, task, decision) + return + } + processed++ + continue + } + + candidate, err := e.repos.CacheEvictions.NextBucketDurabilityCandidate(ctx, task.RefID) + if err != nil { + e.applyEvictionDecision(ctx, task, retryEviction(err, "selecting bucket durability candidate")) + return + } + if candidate == nil { + completed, err := e.repos.CacheEvictions.CompleteBucketDurabilityReconciliation(ctx, task) + switch { + case err == nil && completed: + admin.WorkerTasksProcessed.WithLabelValues("evictor", "success").Inc() + logger.Info("bucket cache policy applied", "versions", processed) + return + case err == nil: + continue + case errors.Is(err, cacheeviction.ErrNoLongerEligible): + e.cancelTask(ctx, task, "Bucket no longer exists") + return + default: + e.applyEvictionDecision(ctx, task, retryEviction(err, "completing bucket durability reconciliation")) + return + } + } + + promoted, decision := e.promoteBucketDurabilityCandidate(ctx, task, candidate.VersionID) + if decision != nil { + e.applyEvictionDecision(ctx, task, decision) + return + } + if promoted { + processed++ + } + } + + if err := e.repos.Tasks.ReleaseRunning(ctx, task); err != nil { + e.applyEvictionDecision(ctx, task, retryEviction(err, "requeueing bucket durability reconciliation")) + return + } + admin.WorkerTasksProcessed.WithLabelValues("evictor", "success").Inc() + logger.Debug("requeued bucket cache policy batch", "versions", processed) +} + +func (e *Evictor) promoteBucketDurabilityCandidate( + ctx context.Context, + task *model.Task, + versionID string, +) (bool, *evictionDecision) { + deleteAfterPromotion := e.policy == cache.EvictionPolicyAfterUpload + if !deleteAfterPromotion { + _, err := e.repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, versionID, false) + switch { + case err == nil: + return true, nil + case errors.Is(err, cacheeviction.ErrNoLongerEligible), + errors.Is(err, cacheeviction.ErrDurabilityThreshold): + return false, nil + default: + return false, retryEviction(err, "promoting bucket durability candidate") + } + } + + var ( + promoted bool + decision *evictionDecision + ) + e.cacheGate.GuardDeletion(versionID, func() { + deletion, err := e.repos.CacheEvictions.PromoteBucketDurabilityCandidate(ctx, task, versionID, true) + switch { + case err == nil: + if err := e.deleteCoordinatorCacheEntry(ctx, task, deletion); err != nil { + decision = retryEviction(err, "deleting reconciled cache entry") + return + } + promoted = true + case errors.Is(err, cacheeviction.ErrNoLongerEligible), + errors.Is(err, cacheeviction.ErrDurabilityThreshold): + return + default: + decision = retryEviction(err, "promoting bucket durability candidate") + } + }) + return promoted, decision +} + +func (e *Evictor) resumeBucketDurabilityDeletion( + ctx context.Context, + task *model.Task, +) *evictionDecision { + var decision *evictionDecision + e.cacheGate.GuardDeletion(task.RefVersionID, func() { + deletion, err := e.repos.CacheEvictions.AuthorizeDeletion(ctx, task, nil) + switch { + case err == nil: + if err := e.deleteCoordinatorCacheEntry(ctx, task, deletion); err != nil { + decision = retryEviction(err, "resuming reconciled cache deletion") + } + case errors.Is(err, repository.ErrNotFound): + switch clearErr := e.repos.CacheEvictions.RecordAuthorizedDeletion(ctx, task); { + case clearErr == nil: + case errors.Is(clearErr, cacheeviction.ErrNoLongerEligible): + decision = cancelEviction("Bucket no longer exists") + default: + decision = retryEviction(clearErr, "clearing completed cache deletion authorization") + } + case errors.Is(err, cacheeviction.ErrNoLongerEligible): + decision = cancelEviction("Authorized cache entry no longer exists") + default: + decision = retryEviction(err, "loading authorized cache deletion") + } + }) + return decision +} diff --git a/internal/worker/evictor_reconcile_internal_test.go b/internal/worker/evictor_reconcile_internal_test.go new file mode 100644 index 0000000..41a7010 --- /dev/null +++ b/internal/worker/evictor_reconcile_internal_test.go @@ -0,0 +1,170 @@ +package worker + +import ( + "context" + "fmt" + "log/slog" + "testing" + "time" + + "github.com/strahe/synaps3/internal/cache" + "github.com/strahe/synaps3/internal/cacheaccess" + "github.com/strahe/synaps3/internal/db/repository" + "github.com/strahe/synaps3/internal/model" + "github.com/strahe/synaps3/internal/state" + "github.com/strahe/synaps3/internal/testutil" + "github.com/uptrace/bun" +) + +func TestEvictorBucketDurabilityReconciliationProcessesOneHundredVersionsPerClaim(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := testutil.SeedBucket(t, db, "durability-batch") + minimum := 1 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimum, + }); err != nil { + t.Fatalf("UpdateCopyPolicy: %v", err) + } + + const versionCount = bucketDurabilityBatchSize + 1 + var sourceVersionID string + for index := range versionCount { + version := &model.ObjectVersion{ + VersionID: model.NewVersionID(), + BucketID: bucket.ID, + Key: fmt.Sprintf("shared-%03d.bin", index), + Size: 10, + ETag: "shared-etag", + Checksum: "shared-durability-checksum", + ContentType: "application/octet-stream", + CacheKey: fmt.Sprintf(".versions/shared-%03d", index), + } + if _, err := repos.Objects.CreateVersionAndSetCurrent(ctx, version); err != nil { + t.Fatalf("CreateVersionAndSetCurrent(%d): %v", index, err) + } + if err := repos.Objects.UpdateVersionState(ctx, version.VersionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { + t.Fatalf("UpdateVersionState(%d): %v", index, err) + } + if index == 0 { + sourceVersionID = version.VersionID + } + } + + upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ + BucketID: bucket.ID, + SourceVersionID: sourceVersionID, + ContentSize: 10, + Checksum: "shared-durability-checksum", + RequestedCopies: 2, + }) + if err != nil { + t.Fatalf("StartObjectUploadAttempt: %v", err) + } + providerID := onChainID(t, "101") + dataSetID := onChainID(t, "1001") + pieceID := onChainID(t, "2001") + binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, + ProviderID: providerID, + CopyIndex: 0, + CreatedByUploadID: upload.ID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, + UploadID: upload.ID, + DataSetID: dataSetID, + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: providerID, + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + UploadID: upload.ID, + CopyIndex: 0, + PieceCID: "bafk2bzacedurabilitybatch", + PieceID: &pieceID, + RetrievalURL: "https://provider.example/piece", + }); err != nil { + t.Fatalf("MarkUploadCopyCommitted: %v", err) + } + refs, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + UploadID: upload.ID, + BucketID: bucket.ID, + ContentSize: 10, + Checksum: "shared-durability-checksum", + }) + if err != nil || len(refs) != versionCount { + t.Fatalf("BindReadableUploadForContent: refs=%d err=%v", len(refs), err) + } + if _, err := repos.CacheEvictions.EnsureBucketDurabilityReconciliation(ctx, bucket.ID, 4); err != nil { + t.Fatalf("EnsureBucketDurabilityReconciliation: %v", err) + } + + evictor := NewEvictor( + repos, + &testutil.MockCache{}, + cacheaccess.NewGate(), + cacheaccess.NewTracker(0, repos.Objects), + state.NewObjectStateMachine(), + 1, + time.Millisecond, + slog.Default(), + WithCacheEvictionPolicy(cache.EvictionPolicyNone, 0, 90, 80, 4), + ) + task, err := repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) + if err != nil || task == nil { + t.Fatalf("ClaimReady first batch: task=%#v err=%v", task, err) + } + evictor.processTask(ctx, task) + assertObjectVersionStateCount(t, db, bucket.ID, model.ObjectStateStored, bucketDurabilityBatchSize) + assertObjectVersionStateCount(t, db, bucket.ID, model.ObjectStateReplicating, 1) + firstBatchTask, err := repos.Tasks.GetByID(ctx, task.ID) + if err != nil || firstBatchTask == nil || firstBatchTask.Status != model.TaskStatusQueued { + t.Fatalf("task after first batch = %#v err=%v, want queued", firstBatchTask, err) + } + + task, err = repos.Tasks.ClaimReady(ctx, model.TaskTypeEvictCache, time.Minute) + if err != nil || task == nil { + t.Fatalf("ClaimReady second batch: task=%#v err=%v", task, err) + } + evictor.processTask(ctx, task) + assertObjectVersionStateCount(t, db, bucket.ID, model.ObjectStateStored, versionCount) + assertObjectVersionStateCount(t, db, bucket.ID, model.ObjectStateReplicating, 0) + completed, err := repos.Tasks.GetByID(ctx, task.ID) + if err != nil || completed == nil || completed.Status != model.TaskStatusCompleted { + t.Fatalf("task after second batch = %#v err=%v, want completed", completed, err) + } +} + +func assertObjectVersionStateCount( + t *testing.T, + db bun.IDB, + bucketID int64, + wantState model.ObjectState, + wantCount int, +) { + t.Helper() + count, err := db.NewSelect(). + Model((*model.ObjectVersion)(nil)). + Where("bucket_id = ? AND state = ?", bucketID, wantState). + Count(context.Background()) + if err != nil { + t.Fatalf("count %s versions: %v", wantState, err) + } + if count != wantCount { + t.Fatalf("%s version count = %d, want %d", wantState, count, wantCount) + } +} diff --git a/internal/worker/manager.go b/internal/worker/manager.go index 3281ee7..3d1a5e5 100644 --- a/internal/worker/manager.go +++ b/internal/worker/manager.go @@ -120,6 +120,7 @@ func (m *Manager) recoverOnStartup(ctx context.Context) { // Reconcile unfinished upload work. m.reconcileTasks(ctx, model.ObjectStateCached, model.TaskTypeUpload, "upload") m.reconcileStagedUploads(ctx) + m.reconcileIncompleteReadableUploads(ctx) m.reconcileUnavailableDataSets(ctx) // Log exhausted task count for operator awareness @@ -131,6 +132,25 @@ func (m *Manager) recoverOnStartup(ctx context.Context) { } } +func (m *Manager) reconcileIncompleteReadableUploads(ctx context.Context) { + afterID := int64(0) + for { + items, err := m.repos.Uploads.ListIncompleteReadableUploads(ctx, afterID, reconcileBatchSize) + if err != nil { + m.logger.Error("failed to list incomplete readable uploads for recovery", "error", err) + return + } + for i := range items { + item := &items[i] + m.enqueueRecoveredUploadRepair(ctx, item.Version, item.Upload.ID) + afterID = item.Upload.ID + } + if len(items) < reconcileBatchSize { + return + } + } +} + func (m *Manager) reconcileUnavailableDataSets(ctx context.Context) { afterID := int64(0) for { @@ -506,7 +526,7 @@ func (m *Manager) enqueueRecoveredUploadRepair(ctx context.Context, version mode MaxRetries: m.uploadMaxRetries, ScheduledAt: time.Now(), } - if err := m.repos.Tasks.Create(ctx, task); err != nil && !errors.Is(err, repository.ErrAlreadyExists) { + if _, err := m.repos.Tasks.EnsureRecurring(ctx, task); err != nil { m.logger.Error("failed to enqueue recovered upload repair", "uploadID", uploadID, "versionID", version.VersionID, "error", err) } } diff --git a/internal/worker/manager_test.go b/internal/worker/manager_test.go index 2e58c8b..e4bfdae 100644 --- a/internal/worker/manager_test.go +++ b/internal/worker/manager_test.go @@ -756,6 +756,93 @@ func TestManager_RecoverOnStartup_DoesNotReplaceAssignedFailedPeer(t *testing.T) } } +func TestManager_RecoverOnStartup_RequeuesIncompleteReadableUploadForStoredVersion(t *testing.T) { + db := testutil.NewTestDB(t) + repos := repository.NewRepositories(db) + ctx := context.Background() + bucket := testutil.SeedBucket(t, db, "mgr-stored-incomplete-recover") + minimum := 1 + if _, err := repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimum, + }); err != nil { + t.Fatalf("UpdateCopyPolicy: %v", err) + } + _, versionID := seedManagerVersion(t, repos, bucket, "stored-incomplete", model.ObjectStateCached) + if err := repos.Objects.UpdateVersionState(ctx, versionID, model.ObjectStateCached, model.ObjectStateUploading); err != nil { + t.Fatalf("uploading: %v", err) + } + version, err := repos.Objects.GetVersionByID(ctx, versionID) + if err != nil || version == nil { + t.Fatalf("GetVersionByID: version=%#v err=%v", version, err) + } + upload, err := repos.Uploads.StartObjectUploadAttempt(ctx, repository.StartObjectUploadAttemptInput{ + BucketID: bucket.ID, + SourceVersionID: versionID, + ContentSize: version.Size, + Checksum: version.Checksum, + RequestedCopies: 2, + }) + if err != nil { + t.Fatalf("StartObjectUploadAttempt: %v", err) + } + binding, err := repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, + ProviderID: onChainID(t, "101"), + CopyIndex: 0, + CreatedByUploadID: upload.ID, + }) + if err != nil { + t.Fatalf("EnsureDataSetBinding: %v", err) + } + if err := repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: binding.ID, + UploadID: upload.ID, + DataSetID: onChainID(t, "1001"), + }); err != nil { + t.Fatalf("MarkDataSetReady: %v", err) + } + if err := repos.Uploads.CreateUploadCopiesForBindings(ctx, upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: binding.ID, + CopyIndex: 0, + TransferMethod: model.StorageCopyTransferMethodIngress, + ProviderID: onChainID(t, "101"), + }}); err != nil { + t.Fatalf("CreateUploadCopiesForBindings: %v", err) + } + if err := repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + UploadID: upload.ID, + CopyIndex: 0, + PieceCID: "bafk2bzacemgrstoredrepair", + PieceID: onChainIDPtr(t, "2001"), + RetrievalURL: "https://provider.example/piece", + }); err != nil { + t.Fatalf("MarkUploadCopyCommitted: %v", err) + } + if _, err := repos.Uploads.BindReadableUploadForContent(ctx, repository.BindReadableUploadInput{ + UploadID: upload.ID, + BucketID: bucket.ID, + ContentSize: version.Size, + Checksum: version.Checksum, + }); err != nil { + t.Fatalf("BindReadableUploadForContent: %v", err) + } + if done, _, err := repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: upload.ID}); err != nil || done { + t.Fatalf("FinalizeUploadIfTargetCopiesMet = done:%t err:%v, want durable but incomplete", done, err) + } + + mgr := worker.NewManager(repos, slog.Default(), cache.EvictionPolicyNone).WithTaskMaxRetries(9, 4) + mgr.Start(ctx) + tasks, total, err := repos.Tasks.List(ctx, string(model.TaskTypeUpload), "prepare_upload", string(model.TaskStatusQueued), 10, 0) + if err != nil { + t.Fatalf("List repair tasks: %v", err) + } + if total != 1 || len(tasks) != 1 || taskPayloadInt64ForTest(tasks[0].Payload, "upload_id") != upload.ID { + t.Fatalf("repair tasks total=%d tasks=%#v, want stored upload %d", total, tasks, upload.ID) + } +} + func TestManager_RecoverOnStartup_ReconcilesAllStagedUploads(t *testing.T) { db := testutil.NewTestDB(t) repos := repository.NewRepositories(db) diff --git a/internal/worker/uploader.go b/internal/worker/uploader.go index 68cfdf8..f298061 100644 --- a/internal/worker/uploader.go +++ b/internal/worker/uploader.go @@ -469,17 +469,78 @@ func (u *Uploader) processTask(ctx context.Context, task *model.Task) { } defer u.publishUploadStateChanged(task, version, bucket) - if version.State == model.ObjectStateStored || version.State == model.ObjectStateCacheEvicted { - if !completeWorkerTask(ctx, u.repos, task, "uploader", logger) { - return - } - logger.Info("upload task already satisfied", "state", version.State) + if durableObjectState(version.State) { + u.processDurableUploadTask(ctx, task, version, bucket, logger) return } u.processStagedTask(ctx, task, version, bucket, uploadTaskStage(task), logger) } +func durableObjectState(state model.ObjectState) bool { + return state == model.ObjectStateStored || state == model.ObjectStateCacheEvicted +} + +func (u *Uploader) processDurableUploadTask( + ctx context.Context, + task *model.Task, + version *model.ObjectVersion, + bucket *model.Bucket, + logger *slog.Logger, +) { + uploadID, ok := taskUploadID(task) + if !ok && version.StorageUploadID != nil { + uploadID = *version.StorageUploadID + ok = uploadID > 0 + } + if !ok { + completeWorkerTask(ctx, u.repos, task, "uploader", logger) + return + } + upload, err := u.repos.Uploads.GetByID(ctx, uploadID) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "load durable upload", err) + return + } + if upload == nil || upload.Status == model.StorageUploadStatusComplete { + completeWorkerTask(ctx, u.repos, task, "uploader", logger) + return + } + + stage := uploadTaskStage(task) + if stage == uploadStagePrepare { + u.prepareReadableUploadRepair(ctx, task, version, bucket, uploadID, logger) + return + } + uploadID, copyIndex, err := uploadStageIDs(task, true) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "parse durable upload task payload", err) + return + } + binding, err := u.repos.Uploads.GetDataSetBindingByCopyIndex(ctx, bucket.ID, copyIndex) + if err != nil { + u.handleTaskFailure(ctx, task, logger, "load durable upload data set", err) + return + } + if binding == nil { + u.prepareReadableUploadRepair(ctx, task, version, bucket, uploadID, logger) + return + } + if binding.Status == model.StorageDataSetStatusFailed && !dataSetBindingEstablished(binding) { + u.prepareReadableUploadRepair(ctx, task, version, bucket, uploadID, logger) + return + } + if binding.Status == model.StorageDataSetStatusPending || binding.Status == model.StorageDataSetStatusCreating { + u.ensureUploadDataSet(ctx, task, version, bucket, uploadID, copyIndex, logger) + return + } + if err := u.ensureReplicaRepairTask(ctx, binding, task.MaxRetries); err != nil { + u.handleTaskFailure(ctx, task, logger, "handoff durable upload repair", err) + return + } + completeWorkerTask(ctx, u.repos, task, "uploader", logger) +} + func (u *Uploader) publishUploadStateChanged(task *model.Task, version *model.ObjectVersion, bucket *model.Bucket) { if u == nil || u.eventPublisher == nil || task == nil { return @@ -742,7 +803,8 @@ func taskUploadID(task *model.Task) (int64, bool) { } func (u *Uploader) prepareReadableUploadRepair(ctx context.Context, task *model.Task, version *model.ObjectVersion, bucket *model.Bucket, uploadID int64, logger *slog.Logger) { - if version.State != model.ObjectStateReplicating || version.StorageUploadID == nil || *version.StorageUploadID != uploadID { + if (version.State != model.ObjectStateReplicating && !durableObjectState(version.State)) || + version.StorageUploadID == nil || *version.StorageUploadID != uploadID { u.handleTaskFailure(ctx, task, logger, "prepare upload repair", fmt.Errorf("object state %s is not repairable for upload %d", version.State, uploadID)) return } @@ -763,8 +825,8 @@ func (u *Uploader) prepareReadableUploadRepair(ctx context.Context, task *model. u.handleTaskFailure(ctx, task, logger, "list readable repair copies", err) return } - if len(readableCopies) == 0 { - u.handleTaskFailure(ctx, task, logger, "prepare upload repair", errors.New("readable source copy not found")) + if len(readableCopies) == 0 && !version.InCache { + u.waitForStorageDependency(ctx, task, logger, "Waiting for a readable replica or retained cache data") return } finalized, _, err := u.repos.Uploads.FinalizeUploadIfTargetCopiesMet( @@ -1251,6 +1313,23 @@ func (u *Uploader) ensureUploadDataSet(ctx context.Context, task *model.Task, ve } } } + if durableObjectState(version.State) { + bindingID := binding.ID + binding, err = u.repos.Uploads.GetDataSetBindingByID(ctx, bindingID) + if err != nil || binding == nil { + if err == nil { + err = fmt.Errorf("dataset binding %d not found", bindingID) + } + u.handleTaskFailure(ctx, task, logger, "reload durable upload data set", err) + return + } + if err := u.ensureReplicaRepairTask(ctx, binding, task.MaxRetries); err != nil { + u.handleTaskFailure(ctx, task, logger, "handoff durable upload repair", err) + return + } + completeWorkerTask(ctx, u.repos, task, "uploader", logger) + return + } nextStage := uploadStagePeerPull if copyRow.TransferMethod == model.StorageCopyTransferMethodIngress { nextStage = uploadStageIngressStore diff --git a/internal/worker/uploader_test.go b/internal/worker/uploader_test.go index 7c4454f..cebad9d 100644 --- a/internal/worker/uploader_test.go +++ b/internal/worker/uploader_test.go @@ -742,6 +742,212 @@ func TestUploader_CompletesRetryWhenObjectIsAlreadyStored(t *testing.T) { } } +func TestUploader_DurableIncompleteUploadKeepsRepairingOriginalReplica(t *testing.T) { + for _, tc := range []struct { + name string + cacheState model.ObjectState + inCache bool + }{ + {name: "stored with retained cache", cacheState: model.ObjectStateStored, inCache: true}, + {name: "cache evicted", cacheState: model.ObjectStateCacheEvicted}, + } { + t.Run(tc.name, func(t *testing.T) { + env := newTestWorkerEnv(t) + ctx := context.Background() + fixture := seedReadableUploadWithPendingPeer(t, env) + bucket, err := env.repos.Buckets.GetByID(ctx, fixture.upload.BucketID) + if err != nil || bucket == nil { + t.Fatalf("GetByID bucket: bucket=%#v err=%v", bucket, err) + } + minimum := 1 + if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimum, + }); err != nil { + t.Fatalf("UpdateCopyPolicy: %v", err) + } + if complete, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: fixture.upload.ID}); err != nil || complete { + t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err:%v, want durable before target", complete, err) + } + if tc.cacheState == model.ObjectStateCacheEvicted { + if err := env.repos.Objects.UpdateVersionState(ctx, fixture.versionID, model.ObjectStateStored, model.ObjectStateCacheEvicted); err != nil { + t.Fatalf("mark cache evicted: %v", err) + } + if err := env.repos.Objects.SetVersionCachePresence(ctx, fixture.versionID, false); err != nil { + t.Fatalf("clear cache presence: %v", err) + } + } + if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.peer.ID, "temporary outage"); err != nil { + t.Fatalf("MarkDataSetUnavailable: %v", err) + } + + var createContextsCalls atomic.Int32 + env.storage.CreateContextsFunc = func(context.Context, *storage.CreateContextsOptions) ([]synapse.UploadContext, error) { + createContextsCalls.Add(1) + return nil, errors.New("durable repair must not select a replacement provider") + } + stage := "prepare_upload" + task := &model.Task{ + Type: model.TaskTypeUpload, + Stage: &stage, + RefType: "object", + RefID: fixture.objID, + RefVersionID: fixture.versionID, + IdempotencyKey: fmt.Sprintf("upload:%s:prepare_upload:%d:durable", fixture.versionID, fixture.upload.ID), + Payload: map[string]interface{}{"upload_id": fixture.upload.ID}, + Status: model.TaskStatusQueued, + MaxRetries: 5, + ScheduledAt: time.Now(), + } + if err := env.repos.Tasks.Create(ctx, task); err != nil { + t.Fatalf("Create durable repair task: %v", err) + } + + uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 2, 1, 10*time.Millisecond, slog.Default()) + gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) + if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { + t.Fatalf("durable repair task = %#v, want completed without retry", gotTask) + } + if createContextsCalls.Load() != 0 { + t.Fatalf("CreateContexts calls = %d, want 0", createContextsCalls.Load()) + } + peerCopy, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, fixture.peer.CopyIndex) + if err != nil || peerCopy == nil { + t.Fatalf("GetUploadCopy peer: copy=%#v err=%v", peerCopy, err) + } + repairTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) + if err != nil || total != 1 || len(repairTasks) != 1 || + taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_data_set_id") != fixture.peer.ID || + taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_upload_copy_id") != peerCopy.ID { + t.Fatalf("repair tasks = %#v total=%d err=%v, want original data set %d copy %d", repairTasks, total, err, fixture.peer.ID, peerCopy.ID) + } + version, err := env.repos.Objects.GetVersionByID(ctx, fixture.versionID) + if err != nil || version == nil || version.State != tc.cacheState || version.InCache != tc.inCache { + t.Fatalf("durable version after repair handoff = %#v err=%v", version, err) + } + }) + } +} + +func TestUploader_DurableEnsureTaskDoesNotReassignIngress(t *testing.T) { + env := newTestWorkerEnv(t) + ctx := context.Background() + fixture := seedReadableUploadWithPendingPeer(t, env) + bucket, err := env.repos.Buckets.GetByID(ctx, fixture.upload.BucketID) + if err != nil || bucket == nil { + t.Fatalf("GetByID bucket: bucket=%#v err=%v", bucket, err) + } + if _, err := env.db.NewUpdate(). + Model((*model.StorageUpload)(nil)). + Set("requested_copies = ?", 3). + Where("id = ?", fixture.upload.ID). + Exec(ctx); err != nil { + t.Fatalf("raise frozen target for fixture: %v", err) + } + if _, err := env.db.NewUpdate(). + Model((*model.StorageUploadCopy)(nil)). + Set("status = ?", model.StorageUploadCopyStatusPending). + Where("upload_id = ? AND copy_index = ?", fixture.upload.ID, fixture.ingress.CopyIndex). + Exec(ctx); err != nil { + t.Fatalf("make ingress copy pending: %v", err) + } + if err := env.repos.Uploads.MarkUploadCopyCommitted(ctx, repository.MarkUploadCopyCommittedInput{ + UploadID: fixture.upload.ID, + CopyIndex: fixture.peer.CopyIndex, + PieceCID: testCID(t).String(), + PieceID: onChainIDPtr(t, "302"), + RetrievalURL: "https://peer.example/piece", + }); err != nil { + t.Fatalf("commit peer copy: %v", err) + } + spare, err := env.repos.Uploads.EnsureDataSetBinding(ctx, repository.EnsureDataSetBindingInput{ + BucketID: bucket.ID, + ProviderID: onChainID(t, "303"), + CopyIndex: 2, + CreatedByUploadID: fixture.upload.ID, + }) + if err != nil { + t.Fatalf("create spare binding: %v", err) + } + if err := env.repos.Uploads.MarkDataSetReady(ctx, repository.MarkDataSetReadyInput{ + ID: spare.ID, + UploadID: fixture.upload.ID, + DataSetID: onChainID(t, "3003"), + }); err != nil { + t.Fatalf("mark spare binding ready: %v", err) + } + if err := env.repos.Uploads.CreateUploadCopiesForBindings(ctx, fixture.upload.ID, []repository.UploadCopyBindingInput{{ + StorageDataSetID: spare.ID, + CopyIndex: spare.CopyIndex, + TransferMethod: model.StorageCopyTransferMethodPeerPull, + ProviderID: spare.ProviderID, + }}); err != nil { + t.Fatalf("create spare copy: %v", err) + } + minimum := 1 + if _, err := env.repos.Buckets.UpdateCopyPolicy(ctx, repository.UpdateBucketCopyPolicyInput{ + Name: bucket.Name, + SetMinimumDurableCopies: true, + MinimumDurableCopies: &minimum, + }); err != nil { + t.Fatalf("set minimum: %v", err) + } + if complete, _, err := env.repos.Uploads.FinalizeUploadIfTargetCopiesMet(ctx, repository.FinalizeUploadInput{UploadID: fixture.upload.ID}); err != nil || complete { + t.Fatalf("FinalizeUploadIfTargetCopiesMet = complete:%t err:%v, want durable before target", complete, err) + } + if err := env.repos.Uploads.MarkDataSetUnavailable(ctx, fixture.ingress.ID, "temporary outage"); err != nil { + t.Fatalf("mark ingress unavailable: %v", err) + } + + stage := "ensure_dataset" + task := &model.Task{ + Type: model.TaskTypeUpload, + Stage: &stage, + RefType: "object", + RefID: fixture.objID, + RefVersionID: fixture.versionID, + IdempotencyKey: fmt.Sprintf("upload:%s:ensure_dataset:%d:0:durable", fixture.versionID, fixture.upload.ID), + Payload: map[string]interface{}{ + "upload_id": fixture.upload.ID, + "copy_index": fixture.ingress.CopyIndex, + "transfer_method": string(model.StorageCopyTransferMethodIngress), + }, + Status: model.TaskStatusQueued, + MaxRetries: 5, + ScheduledAt: time.Now(), + } + if err := env.repos.Tasks.Create(ctx, task); err != nil { + t.Fatalf("create stale ensure task: %v", err) + } + env.storage.CreateContextsFunc = func(context.Context, *storage.CreateContextsOptions) ([]synapse.UploadContext, error) { + return nil, errors.New("durable ensure task must not choose a provider") + } + + uploader := worker.NewUploader(env.repos, env.cache, env.storage, nil, env.sm, cache.EvictionPolicyAfterUpload, 3, 1, 10*time.Millisecond, slog.Default()) + gotTask := runWorkerUntilTask(t, env, uploader, task.ID, 5*time.Second) + if gotTask.Status != model.TaskStatusCompleted || gotTask.RetryCount != 0 { + t.Fatalf("stale ensure task = %#v, want completed without retry", gotTask) + } + spareCopy, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, spare.CopyIndex) + if err != nil || spareCopy == nil { + t.Fatalf("GetUploadCopy spare: copy=%#v err=%v", spareCopy, err) + } + if spareCopy.TransferMethod != model.StorageCopyTransferMethodPeerPull { + t.Fatalf("spare transfer method = %s, want peer pull", spareCopy.TransferMethod) + } + ingressCopy, err := env.repos.Uploads.GetUploadCopy(ctx, fixture.upload.ID, fixture.ingress.CopyIndex) + if err != nil || ingressCopy == nil { + t.Fatalf("GetUploadCopy ingress: copy=%#v err=%v", ingressCopy, err) + } + repairTasks, total, err := env.repos.Tasks.List(ctx, string(model.TaskTypeUpload), "repair_replica", "", 10, 0) + if err != nil || total != 1 || len(repairTasks) != 1 || + taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_data_set_id") != fixture.ingress.ID || + taskPayloadInt64ForTest(repairTasks[0].Payload, "storage_upload_copy_id") != ingressCopy.ID { + t.Fatalf("repair tasks = %#v total=%d err=%v, want original ingress data set %d copy %d", repairTasks, total, err, fixture.ingress.ID, ingressCopy.ID) + } +} + func TestUploader_ClaimsLaterPendingTaskWhileAnotherUploadRuns(t *testing.T) { env := newTestWorkerEnv(t) _, firstObjID, firstVersionID := seedCachedObject(t, env) @@ -3202,6 +3408,7 @@ type readableUploadWithPendingPeerFixture struct { objID int64 versionID string upload *model.StorageUpload + ingress *model.StorageDataSet peer *model.StorageDataSet } @@ -3271,6 +3478,7 @@ func seedReadableUploadWithPendingPeer(t *testing.T, env *testWorkerEnv) readabl objID: objID, versionID: versionID, upload: upload, + ingress: ingress, peer: peer, } } diff --git a/ui/src/api/client.ts b/ui/src/api/client.ts index 6533441..1306baa 100644 --- a/ui/src/api/client.ts +++ b/ui/src/api/client.ts @@ -194,6 +194,8 @@ export interface BucketItem { owner_access_key: string | null default_copies: number | null effective_copies: number + minimum_durable_copies: number | null + effective_minimum_durable_copies: number status: string object_count: number total_size_bytes: number @@ -296,6 +298,8 @@ export interface BucketMutationResponse { owner_access_key: string | null default_copies: number | null effective_copies: number + minimum_durable_copies: number | null + effective_minimum_durable_copies: number status: string } @@ -852,6 +856,7 @@ export interface SettingsData { config_path: string writable: boolean runtime_available?: boolean + runtime_filecoin_default_copies?: number restart_required: boolean s3_users: SettingsS3UsersStatus config: SettingsEditableConfig @@ -1044,7 +1049,12 @@ export const api = { getOverview: () => fetchJSON('/overview'), getBuckets: () => fetchJSON('/buckets'), getBucket: (name: string) => fetchJSON(`/buckets/${encodeURIComponent(name)}`), - createBucket: (payload: { name: string; owner_access_key: string; default_copies?: number | null }) => + createBucket: (payload: { + name: string + owner_access_key: string + default_copies?: number | null + minimum_durable_copies?: number | null + }) => fetchJSON('/buckets', { method: 'POST', body: JSON.stringify(payload), @@ -1054,10 +1064,13 @@ export const api = { method: 'PUT', body: JSON.stringify({ owner_access_key: ownerAccessKey }), }), - updateBucketCopyPolicy: (name: string, defaultCopies: number | null) => + updateBucketCopyPolicy: ( + name: string, + policy: { default_copies?: number | null; minimum_durable_copies?: number | null } + ) => fetchJSON(`/buckets/${encodeURIComponent(name)}/copy-policy`, { method: 'PUT', - body: JSON.stringify({ default_copies: defaultCopies }), + body: JSON.stringify(policy), }), getBucketObjects: (name: string, params: { prefix?: string; delimiter?: string; after?: string; limit?: number }) => { const sp = new URLSearchParams() diff --git a/ui/src/hooks/queries.ts b/ui/src/hooks/queries.ts index 4d20854..604c734 100644 --- a/ui/src/hooks/queries.ts +++ b/ui/src/hooks/queries.ts @@ -105,11 +105,17 @@ export function useCreateBucket() { const qc = useQueryClient() return useMutation({ - mutationFn: (payload: { name: string; ownerAccessKey: string; defaultCopies: number | null }) => + mutationFn: (payload: { + name: string + ownerAccessKey: string + defaultCopies: number | null + minimumDurableCopies: number | null + }) => api.createBucket({ name: payload.name, owner_access_key: payload.ownerAccessKey, default_copies: payload.defaultCopies, + minimum_durable_copies: payload.minimumDurableCopies, }), onSuccess: (bucket) => { qc.invalidateQueries({ queryKey: ['buckets'] }) @@ -137,8 +143,19 @@ export function useUpdateBucketCopyPolicy() { const qc = useQueryClient() return useMutation({ - mutationFn: ({ name, defaultCopies }: { name: string; defaultCopies: number | null }) => - api.updateBucketCopyPolicy(name, defaultCopies), + mutationFn: ({ + name, + defaultCopies, + minimumDurableCopies, + }: { + name: string + defaultCopies?: number | null + minimumDurableCopies?: number | null + }) => + api.updateBucketCopyPolicy(name, { + default_copies: defaultCopies, + minimum_durable_copies: minimumDurableCopies, + }), onSuccess: (bucket) => { qc.invalidateQueries({ queryKey: ['buckets'] }) qc.invalidateQueries({ queryKey: ['bucket', bucket.name] }) diff --git a/ui/src/lib/bucket-copy-policy.ts b/ui/src/lib/bucket-copy-policy.ts index d0f574f..854a025 100644 --- a/ui/src/lib/bucket-copy-policy.ts +++ b/ui/src/lib/bucket-copy-policy.ts @@ -1,8 +1,12 @@ import type { BucketItem } from '@/api/client' -type BucketCopyPolicy = Pick +type BucketCopyPolicy = Pick< + BucketItem, + 'default_copies' | 'effective_copies' | 'minimum_durable_copies' | 'effective_minimum_durable_copies' +> export const inheritedCopyPolicyValue = 'inherit' +export const strictMinimumDurableCopiesValue = 'strict' export const copyPolicyOptions = Array.from({ length: 8 }, (_, index) => index + 1) export function bucketCopyPolicyValue(bucket: Pick) { @@ -14,9 +18,10 @@ export function bucketCopyPolicyLabel(bucket: BucketCopyPolicy) { return bucket.default_copies == null ? `Inherits global default (${copies})` : `Override (${copies})` } -export function bucketCopyPolicyInheritOptionLabel(bucket: BucketCopyPolicy) { - if (bucket.default_copies != null) return 'Inherit global default' - return `Inherit global default (${copyCountLabel(bucket.effective_copies)})` +export function bucketCopyPolicyInheritOptionLabel(bucket: BucketCopyPolicy, runtimeDefaultCopies?: number) { + const copies = bucket.default_copies == null ? bucket.effective_copies : runtimeDefaultCopies + if (copies == null) return 'Inherit current runtime default' + return `Inherit current runtime default (${copyCountLabel(copies)})` } export function bucketCopyPolicySavedMessage() { @@ -24,7 +29,44 @@ export function bucketCopyPolicySavedMessage() { } export function bucketCopyPolicyEffectNote() { - return 'Applies only to uploads started after this change. Existing objects and in-flight uploads keep their current replica plan.' + return 'Target replicas apply to new uploads. Cache release applies to retained cache for current and future uploads.' +} + +export function minimumDurableCopiesValue(bucket: Pick) { + if (bucket.minimum_durable_copies == null || bucket.minimum_durable_copies > bucket.effective_copies) { + return strictMinimumDurableCopiesValue + } + return bucket.minimum_durable_copies.toString() +} + +export function minimumDurableCopiesLabel(bucket: BucketCopyPolicy) { + if (bucket.minimum_durable_copies == null) return 'All replicas (strict)' + return `${bucket.effective_minimum_durable_copies} of ${bucket.effective_copies} ${bucket.effective_copies === 1 ? 'replica' : 'replicas'}` +} + +export function minimumDurableCopiesOptionLabel(copies: number) { + return `${copies} ${copies === 1 ? 'replica' : 'replicas'}` +} + +export function selectedTargetCopies(copyPolicy: string, runtimeDefaultCopies?: number) { + if (copyPolicy === inheritedCopyPolicyValue) return runtimeDefaultCopies ?? null + const copies = Number(copyPolicy) + return Number.isInteger(copies) && copies >= 1 && copies <= 8 ? copies : null +} + +export function minimumDurableCopiesOptions(targetCopies: number | null) { + if (targetCopies == null) return [] + return copyPolicyOptions.filter((copies) => copies <= targetCopies) +} + +export function clampMinimumDurableCopiesValue(value: string, targetCopies: number | null) { + if (value === strictMinimumDurableCopiesValue) return value + if (targetCopies == null || Number(value) > targetCopies) return strictMinimumDurableCopiesValue + return value +} + +export function minimumDurableCopiesWarning() { + return 'Lowering this value can release local cache before every target replica is ready. Raising it cannot restore cache that has already been deleted.' } function copyCountLabel(copies: number) { diff --git a/ui/src/lib/storage-status-labels.ts b/ui/src/lib/storage-status-labels.ts index 299c2a5..cb1f1b3 100644 --- a/ui/src/lib/storage-status-labels.ts +++ b/ui/src/lib/storage-status-labels.ts @@ -11,6 +11,7 @@ export const taskStageOptions = [ 'peer_pull', 'peer_commit', 'repair_replica', + 'reconcile_bucket_durability', ] as const export type TaskStageOption = (typeof taskStageOptions)[number] @@ -23,6 +24,7 @@ const taskStageLabels: Record | '', string> = { peer_pull: 'Sync peer replica', peer_commit: 'Register peer replica on-chain', repair_replica: 'Resume replica upload', + reconcile_bucket_durability: 'Apply cache policy', '': 'Upload', } diff --git a/ui/src/routes/buckets.$name.tsx b/ui/src/routes/buckets.$name.tsx index d85c96a..46f1aae 100644 --- a/ui/src/routes/buckets.$name.tsx +++ b/ui/src/routes/buckets.$name.tsx @@ -109,6 +109,7 @@ import { useRestoreBucketObject, useRestoreBucketObjectVersion, useS3Users, + useSettings, useUpdateBucketCopyPolicy, useUpdateBucketOwner, } from '@/hooks/queries' @@ -118,8 +119,16 @@ import { bucketCopyPolicyLabel, bucketCopyPolicySavedMessage, bucketCopyPolicyValue, + clampMinimumDurableCopiesValue, copyPolicyOptions, inheritedCopyPolicyValue, + minimumDurableCopiesLabel, + minimumDurableCopiesOptionLabel, + minimumDurableCopiesOptions, + minimumDurableCopiesValue, + minimumDurableCopiesWarning, + selectedTargetCopies, + strictMinimumDurableCopiesValue, } from '@/lib/bucket-copy-policy' import { type BucketRouteSearch, normalizeBucketRouteSearch } from '@/lib/bucket-route-search' import { @@ -1836,6 +1845,7 @@ function BucketDetailsOverview({ bucket }: { bucket: NonNullable + void }) { const updateCopyPolicy = useUpdateBucketCopyPolicy() + const { data: settings } = useSettings() const currentCopyPolicy = bucketCopyPolicyValue(bucket) + const currentMinimumDurableCopies = minimumDurableCopiesValue(bucket) const [copyPolicy, setCopyPolicy] = useState(currentCopyPolicy) + const [minimumDurableCopies, setMinimumDurableCopies] = useState(currentMinimumDurableCopies) const [copyPolicyError, setCopyPolicyError] = useState(null) const [copyPolicyNotice, setCopyPolicyNotice] = useState(null) + const runtimeDefaultCopies = settings?.runtime_filecoin_default_copies + const inheritedTargetCopies = + currentCopyPolicy === inheritedCopyPolicyValue ? bucket.effective_copies : runtimeDefaultCopies + const targetCopies = selectedTargetCopies(copyPolicy, inheritedTargetCopies) + const minimumOptions = minimumDurableCopiesOptions(targetCopies) useEffect(() => { setCopyPolicy(currentCopyPolicy) + setMinimumDurableCopies(currentMinimumDurableCopies) setCopyPolicyError(null) - }, [currentCopyPolicy]) + }, [currentCopyPolicy, currentMinimumDurableCopies]) useEffect(() => { if (bucket.name) setCopyPolicyNotice(null) }, [bucket.name]) - const copyPolicyChanged = copyPolicy !== currentCopyPolicy && copyPolicyNotice == null + const copyPolicyChanged = + (copyPolicy !== currentCopyPolicy || minimumDurableCopies !== currentMinimumDurableCopies) && + copyPolicyNotice == null const handleCopyPolicyChange = (next: string) => { + const nextTarget = selectedTargetCopies(next, inheritedTargetCopies) setCopyPolicy(next) + setMinimumDurableCopies((current) => { + if ( + current === strictMinimumDurableCopiesValue && + bucket.minimum_durable_copies != null && + bucket.minimum_durable_copies > bucket.effective_copies && + nextTarget != null && + bucket.minimum_durable_copies <= nextTarget + ) { + return bucket.minimum_durable_copies.toString() + } + return clampMinimumDurableCopiesValue(current, nextTarget) + }) + setCopyPolicyError(null) + setCopyPolicyNotice(null) + } + const handleMinimumDurableCopiesChange = (next: string) => { + setMinimumDurableCopies(next) setCopyPolicyError(null) setCopyPolicyNotice(null) } const saveCopyPolicy = () => { setCopyPolicyError(null) setCopyPolicyNotice(null) + const clearMinimumForTarget = + copyPolicy !== currentCopyPolicy && + minimumDurableCopies === strictMinimumDurableCopiesValue && + bucket.minimum_durable_copies != null && + (targetCopies == null || bucket.minimum_durable_copies > targetCopies) updateCopyPolicy.mutate( { name: bucket.name, - defaultCopies: copyPolicy === inheritedCopyPolicyValue ? null : Number(copyPolicy), + defaultCopies: + copyPolicy === currentCopyPolicy + ? undefined + : copyPolicy === inheritedCopyPolicyValue + ? null + : Number(copyPolicy), + minimumDurableCopies: + minimumDurableCopies === currentMinimumDurableCopies && !clearMinimumForTarget + ? undefined + : minimumDurableCopies === strictMinimumDurableCopiesValue + ? null + : Number(minimumDurableCopies), }, { onSuccess: (savedBucket) => { setCopyPolicy(bucketCopyPolicyValue(savedBucket)) + setMinimumDurableCopies(minimumDurableCopiesValue(savedBucket)) setCopyPolicyNotice(bucketCopyPolicySavedMessage()) }, onError: (mutationError) => { @@ -2241,25 +2297,59 @@ function BucketDetailsSettings({
-

Replicas

-

{bucketCopyPolicyLabel(bucket)}

+

Replica policy

+

+ Target: {bucketCopyPolicyLabel(bucket)} · Release cache: {minimumDurableCopiesLabel(bucket)} +

{bucketCopyPolicyEffectNote()}

-
- + + + + + + + {bucketCopyPolicyInheritOptionLabel(bucket, runtimeDefaultCopies)} - ))} - - - + {copyPolicyOptions.map((copies) => ( + + {copies} {copies === 1 ? 'copy' : 'copies'} + + ))} + + + + + + Release cache after + + + + + {minimumDurableCopiesWarning()} + +

Replica policy

-

- Target: {bucketCopyPolicyLabel(bucket)} · Release cache: {minimumDurableCopiesLabel(bucket)} -

-

{bucketCopyPolicyEffectNote()}

Replicas @@ -2343,7 +2338,9 @@ function BucketDetailsSettings({ - {minimumDurableCopiesChoiceNote()} + {minimumDurableCopies === strictMinimumDurableCopiesValue && ( + {minimumDurableCopiesChoiceNote()} + )} {showsMinimumDurableCopiesWarning(minimumDurableCopies, targetCopies) && ( diff --git a/ui/src/routes/buckets.index.tsx b/ui/src/routes/buckets.index.tsx index 5d7e8d7..41ff6e4 100644 --- a/ui/src/routes/buckets.index.tsx +++ b/ui/src/routes/buckets.index.tsx @@ -26,7 +26,6 @@ import { Select, SelectContent, SelectGroup, SelectItem, SelectTrigger, SelectVa import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' import { useBuckets, useCreateBucket, useS3Users, useSettings, useUpdateBucketOwner } from '@/hooks/queries' import { - bucketCopyPolicyEffectNote, bucketCopyPolicyLabel, clampMinimumDurableCopiesValue, copyPolicyOptions, @@ -213,10 +212,11 @@ function CreateBucketDialog() { - {minimumDurableCopiesChoiceNote()} + {minimumDurableCopies === strictMinimumDurableCopiesValue && ( + {minimumDurableCopiesChoiceNote()} + )} -

{bucketCopyPolicyEffectNote()}

{showsMinimumDurableCopiesWarning(minimumDurableCopies, targetCopies) && ( {minimumDurableCopiesWarning()} diff --git a/ui/test/bucket-copy-policy.test.ts b/ui/test/bucket-copy-policy.test.ts index 5962e38..4bc23bb 100644 --- a/ui/test/bucket-copy-policy.test.ts +++ b/ui/test/bucket-copy-policy.test.ts @@ -2,7 +2,6 @@ import assert from 'node:assert/strict' import test from 'node:test' import { - bucketCopyPolicyEffectNote, bucketCopyPolicyInheritOptionLabel, bucketCopyPolicySavedMessage, clampMinimumDurableCopiesValue, @@ -19,17 +18,10 @@ import { test('bucket copy policy text explains save result and future upload scope', () => { assert.equal(bucketCopyPolicySavedMessage(), 'Replica policy saved.') - assert.equal( - bucketCopyPolicyEffectNote(), - 'New uploads use the Replicas target. Cache can be removed after the Release cache after count is ready, if eviction is enabled. Remaining target replicas keep syncing.' - ) - assert.equal( - minimumDurableCopiesChoiceNote(), - 'All replicas waits for every target replica of that upload. A number stays fixed if the target changes later.' - ) + assert.equal(minimumDurableCopiesChoiceNote(), 'Waits for every target replica of that upload.') assert.equal( minimumDurableCopiesWarning(), - 'Cache may be removed before every target replica is ready, depending on cache eviction settings. Raising this later cannot restore cache that has already been deleted.' + 'Cache may be removed before every target replica is ready. Raising this later cannot restore deleted cache.' ) assert.equal(showsMinimumDurableCopiesWarning(strictMinimumDurableCopiesValue, 3), false) assert.equal(showsMinimumDurableCopiesWarning('3', 3), false) From 1fdf91532051a8df8286b4de90301317503c08b5 Mon Sep 17 00:00:00 2001 From: Lee <7932644+strahe@users.noreply.github.com> Date: Thu, 20 Aug 2026 13:11:48 +0800 Subject: [PATCH 5/5] fix(ui): distinguish strict and explicit cache-release counts --- docs/en/concepts/filecoin-storage-flow.md | 2 +- docs/en/concepts/write-path-cache.md | 2 +- docs/zh/concepts/filecoin-storage-flow.md | 2 +- docs/zh/concepts/write-path-cache.md | 2 +- ui/src/lib/bucket-copy-policy.ts | 20 ++++++++++++++++---- ui/src/routes/buckets.$name.tsx | 9 +++++++-- ui/src/routes/buckets.index.tsx | 12 +++++++++--- ui/test/bucket-copy-policy.test.ts | 21 +++++++++++++++++++-- 8 files changed, 55 insertions(+), 15 deletions(-) diff --git a/docs/en/concepts/filecoin-storage-flow.md b/docs/en/concepts/filecoin-storage-flow.md index 8e4f7a2..73fdaeb 100644 --- a/docs/en/concepts/filecoin-storage-flow.md +++ b/docs/en/concepts/filecoin-storage-flow.md @@ -56,7 +56,7 @@ If an established provider becomes temporarily unavailable while the initial cop ## Target and Minimum Replicas -The target replica count is frozen when an upload starts. By default, Release cache after is All replicas (strict): every target replica frozen for that upload must be readable and committed. A bucket can instead set an explicit count from 1 through the current target. An explicit count stays fixed if the target later changes; All replicas follows each upload's frozen target. Once that threshold is met, the version becomes stored and its cache follows the configured eviction policy, while the upload continues filling its original replica slots until the target is reached. The dashboard keeps showing replica sync progress until every frozen target replica is done. +The target replica count is frozen when an upload starts. By default, **Release cache after** is **All replicas (strict)**: every target replica frozen for that upload must be readable and committed. A bucket can instead set an explicit count from 1 through the current target. An explicit count stays if Replicas later increases; lowering Replicas below that count is rejected until Release cache after is also lowered. **All replicas (strict)** follows each upload's frozen target. Once that threshold is met, the version becomes stored and its cache follows the configured eviction policy, while remaining replicas continue until the upload's target is reached. The dashboard keeps showing replica sync progress until every frozen target replica is done. Changing the target affects new uploads. Changing the minimum also re-evaluates retained cache for current uploads. Increasing the minimum does not move versions that are already stored back to an earlier state and cannot restore cache that has already been deleted. diff --git a/docs/en/concepts/write-path-cache.md b/docs/en/concepts/write-path-cache.md index f9c6d75..1bdbc3a 100644 --- a/docs/en/concepts/write-path-cache.md +++ b/docs/en/concepts/write-path-cache.md @@ -42,7 +42,7 @@ Repeated reads of the same version coalesce access-time updates to at most one d | `after_upload` | Queue each version for removal after its bucket's minimum durable copies commit. | | `none` | Do not create or run automatic cache eviction work. | -Each bucket defaults to strict cache release, so the minimum equals the target replicas frozen for each upload. An operator can set an explicit count from 1 through the current target; that number stays fixed if the target later changes. Lowering the threshold makes retained cache eligible for removal while original replica slots continue syncing. Actual removal still follows `after_upload`, `lru`, or `none`. The minimum is clamped to each upload's target. Raising it affects cache that still exists; it cannot recreate cache that has already been deleted. +Each bucket defaults to strict cache release, so the minimum equals the target replicas frozen for each upload. In the dashboard, set this on the bucket under Settings → Replica policy. An operator can set an explicit count from 1 through the current target. That count stays if Replicas later increases. Lowering Replicas below the stored count is rejected until Release cache after is also lowered. Lowering the threshold makes retained cache eligible for removal while remaining replicas continue syncing. Actual removal still follows `after_upload`, `lru`, or `none`. The minimum is clamped to each upload's target. Raising it affects cache that still exists; it cannot recreate cache that has already been deleted. Only versions that currently meet their minimum and have a readable committed remote copy are eligible. Eviction checks the current minimum again before authorizing deletion and waits for active reads of the same version to close. Because cleanup is asynchronous, writes can still return `507 Insufficient Storage` when cleanup cannot keep pace or no safe candidate exists. diff --git a/docs/zh/concepts/filecoin-storage-flow.md b/docs/zh/concepts/filecoin-storage-flow.md index e5e9a4f..e2d09e2 100644 --- a/docs/zh/concepts/filecoin-storage-flow.md +++ b/docs/zh/concepts/filecoin-storage-flow.md @@ -56,7 +56,7 @@ synaps3 admin task retry 42 ## 目标副本与最低耐久副本 -上传开始时会冻结目标副本数。默认情况下，Release cache after 为 All replicas (strict)：该次上传冻结的全部目标副本都必须可读并完成提交。存储桶也可以设置 1 到当前目标之间的显式数量。显式数量在目标随后提高时保持不变；All replicas 会跟随每次上传冻结的目标。达到该门槛后，版本进入已存储状态，缓存按已配置的淘汰策略处理；上传仍会继续补齐原有副本位，直到达到目标副本数。在冻结的目标副本全部完成前，仪表盘会继续显示副本同步进度。 +上传开始时会冻结目标副本数。默认情况下，**Release cache after** 为 **All replicas (strict)**：该次上传冻结的全部目标副本都必须可读并完成提交。存储桶也可以设置 1 到当前目标之间的显式数量。显式数量在目标随后提高时保持不变；如果把 Replicas 降到低于该数量，请求会被拒绝，必须同时降低 Release cache after。**All replicas (strict)** 会跟随每次上传冻结的目标。达到该门槛后，版本进入已存储状态，缓存按已配置的淘汰策略处理；其余副本会继续补齐，直到达到该次上传的目标副本数。在冻结的目标副本全部完成前，仪表盘会继续显示副本同步进度。 修改目标副本数只影响新上传。修改最低耐久副本数也会重新评估当前上传仍保留的缓存。提高门槛不会让已经进入已存储状态的版本回退，也无法恢复已经删除的缓存。 diff --git a/docs/zh/concepts/write-path-cache.md b/docs/zh/concepts/write-path-cache.md index 9debed3..e9cf4bb 100644 --- a/docs/zh/concepts/write-path-cache.md +++ b/docs/zh/concepts/write-path-cache.md @@ -42,7 +42,7 @@ SynapS3 会校验请求，保存对象及其元数据，再返回 S3 兼容的 E | `after_upload` | 存储桶要求的最低耐久副本提交后，为该版本排队清理。 | | `none` | 不创建或执行自动缓存淘汰任务。 | -每个存储桶默认采用严格缓存释放策略，因此最低耐久副本数等于每次上传冻结的目标副本数。运维人员可以设置 1 到当前目标之间的显式数量；目标随后提高时该数字保持不变。降低门槛只会让仍保留的缓存有资格被删除，原副本位会继续补齐。是否真正删除仍取决于 `after_upload`、`lru` 或 `none`。该门槛不会超过单次上传的目标副本数。提高门槛只影响尚未删除的缓存，无法恢复已经删除的缓存。 +每个存储桶默认采用严格缓存释放策略，因此最低耐久副本数等于每次上传冻结的目标副本数。在仪表盘中，到该存储桶的 Settings → Replica policy 设置。运维人员可以设置 1 到当前目标之间的显式数量；目标随后提高时该数字保持不变。如果把 Replicas 降到低于已保存的显式数量，请求会被拒绝，必须同时降低 Release cache after。降低门槛只会让仍保留的缓存有资格被删除，其余副本会继续补齐。是否真正删除仍取决于 `after_upload`、`lru` 或 `none`。该门槛不会超过单次上传的目标副本数。提高门槛只影响尚未删除的缓存，无法恢复已经删除的缓存。 只有当前满足最低耐久副本数且存在可读已提交远端副本的版本才可淘汰。系统会在授权删除前再次检查当前门槛，并等待同一版本正在进行的读取关闭。清理是异步流程，因此清理追赶不及时或没有安全候选时，写入仍可能返回 `507 Insufficient Storage`。 diff --git a/ui/src/lib/bucket-copy-policy.ts b/ui/src/lib/bucket-copy-policy.ts index ebe7dfb..98080cb 100644 --- a/ui/src/lib/bucket-copy-policy.ts +++ b/ui/src/lib/bucket-copy-policy.ts @@ -24,12 +24,20 @@ export function bucketCopyPolicyInheritOptionLabel(bucket: BucketCopyPolicy, run return `Inherit current runtime default (${copyCountLabel(copies)})` } +export function replicaTargetChoiceNote() { + return 'Applies to new uploads. Existing objects keep the replica target they started with.' +} + export function bucketCopyPolicySavedMessage() { - return 'Replica policy saved.' + return 'Saved. New uploads use this replica target. Cache can be released after the selected count.' } export function minimumDurableCopiesChoiceNote() { - return 'Waits for every target replica of that upload.' + return 'Keeps cache until every replica of that upload is stored, including later Replicas increases.' +} + +export function minimumDurableCopiesFixedCountNote() { + return 'Keeps this count if Replicas later increases.' } export function minimumDurableCopiesValue(bucket: Pick) { @@ -43,8 +51,12 @@ export function minimumDurableCopiesLabel(bucket: BucketCopyPolicy) { return `${bucket.effective_minimum_durable_copies} of ${bucket.effective_copies} ${bucket.effective_copies === 1 ? 'replica' : 'replicas'}` } -export function minimumDurableCopiesOptionLabel(copies: number) { - return `${copies} ${copies === 1 ? 'replica' : 'replicas'}` +export function minimumDurableCopiesOptionLabel(copies: number, targetCopies?: number | null) { + const count = `${copies} ${copies === 1 ? 'replica' : 'replicas'}` + if (targetCopies != null && copies === targetCopies) { + return `${count} (fixed count)` + } + return count } export function selectedTargetCopies(copyPolicy: string, runtimeDefaultCopies?: number) { diff --git a/ui/src/routes/buckets.$name.tsx b/ui/src/routes/buckets.$name.tsx index a6e94db..6d2d5ad 100644 --- a/ui/src/routes/buckets.$name.tsx +++ b/ui/src/routes/buckets.$name.tsx @@ -122,12 +122,14 @@ import { copyPolicyOptions, inheritedCopyPolicyValue, minimumDurableCopiesChoiceNote, + minimumDurableCopiesFixedCountNote, minimumDurableCopiesLabel, minimumDurableCopiesOptionLabel, minimumDurableCopiesOptions, minimumDurableCopiesValue, minimumDurableCopiesWarning, persistMinimumDurableCopies, + replicaTargetChoiceNote, selectedTargetCopies, showsMinimumDurableCopiesWarning, strictMinimumDurableCopiesValue, @@ -2316,6 +2318,7 @@ function BucketDetailsSettings({ + {replicaTargetChoiceNote()} Release cache after @@ -2332,14 +2335,16 @@ function BucketDetailsSettings({ All replicas (strict) {minimumOptions.map((copies) => ( - {minimumDurableCopiesOptionLabel(copies)} + {minimumDurableCopiesOptionLabel(copies, targetCopies)} ))} - {minimumDurableCopies === strictMinimumDurableCopiesValue && ( + {minimumDurableCopies === strictMinimumDurableCopiesValue ? ( {minimumDurableCopiesChoiceNote()} + ) : ( + {minimumDurableCopiesFixedCountNote()} )} diff --git a/ui/src/routes/buckets.index.tsx b/ui/src/routes/buckets.index.tsx index 41ff6e4..ba2a5f0 100644 --- a/ui/src/routes/buckets.index.tsx +++ b/ui/src/routes/buckets.index.tsx @@ -31,10 +31,12 @@ import { copyPolicyOptions, inheritedCopyPolicyValue, minimumDurableCopiesChoiceNote, + minimumDurableCopiesFixedCountNote, minimumDurableCopiesLabel, minimumDurableCopiesOptionLabel, minimumDurableCopiesOptions, minimumDurableCopiesWarning, + replicaTargetChoiceNote, selectedTargetCopies, showsMinimumDurableCopiesWarning, strictMinimumDurableCopiesValue, @@ -184,11 +186,13 @@ function CreateBucketDialog() { - {copyPolicy === inheritedCopyPolicyValue && runtimeDefaultCopies == null && ( + {copyPolicy === inheritedCopyPolicyValue && runtimeDefaultCopies == null ? ( The current runtime default is unavailable. Choose a replica count to configure an explicit cache-release threshold. + ) : ( + {replicaTargetChoiceNote()} )} @@ -206,14 +210,16 @@ function CreateBucketDialog() { All replicas (strict) {minimumOptions.map((copies) => ( - {minimumDurableCopiesOptionLabel(copies)} + {minimumDurableCopiesOptionLabel(copies, targetCopies)} ))} - {minimumDurableCopies === strictMinimumDurableCopiesValue && ( + {minimumDurableCopies === strictMinimumDurableCopiesValue ? ( {minimumDurableCopiesChoiceNote()} + ) : ( + {minimumDurableCopiesFixedCountNote()} )} diff --git a/ui/test/bucket-copy-policy.test.ts b/ui/test/bucket-copy-policy.test.ts index 4bc23bb..7347a54 100644 --- a/ui/test/bucket-copy-policy.test.ts +++ b/ui/test/bucket-copy-policy.test.ts @@ -6,19 +6,36 @@ import { bucketCopyPolicySavedMessage, clampMinimumDurableCopiesValue, minimumDurableCopiesChoiceNote, + minimumDurableCopiesFixedCountNote, minimumDurableCopiesLabel, + minimumDurableCopiesOptionLabel, minimumDurableCopiesOptions, minimumDurableCopiesValue, minimumDurableCopiesWarning, persistMinimumDurableCopies, + replicaTargetChoiceNote, selectedTargetCopies, showsMinimumDurableCopiesWarning, strictMinimumDurableCopiesValue, } from '../src/lib/bucket-copy-policy.ts' test('bucket copy policy text explains save result and future upload scope', () => { - assert.equal(bucketCopyPolicySavedMessage(), 'Replica policy saved.') - assert.equal(minimumDurableCopiesChoiceNote(), 'Waits for every target replica of that upload.') + assert.equal( + replicaTargetChoiceNote(), + 'Applies to new uploads. Existing objects keep the replica target they started with.' + ) + assert.equal( + bucketCopyPolicySavedMessage(), + 'Saved. New uploads use this replica target. Cache can be released after the selected count.' + ) + assert.equal( + minimumDurableCopiesChoiceNote(), + 'Keeps cache until every replica of that upload is stored, including later Replicas increases.' + ) + assert.equal(minimumDurableCopiesFixedCountNote(), 'Keeps this count if Replicas later increases.') + assert.equal(minimumDurableCopiesOptionLabel(3), '3 replicas') + assert.equal(minimumDurableCopiesOptionLabel(3, 3), '3 replicas (fixed count)') + assert.equal(minimumDurableCopiesOptionLabel(2, 3), '2 replicas') assert.equal( minimumDurableCopiesWarning(), 'Cache may be removed before every target replica is ready. Raising this later cannot restore deleted cache.'