diff --git a/blockstore/spool.go b/blockstore/spool.go index a1273c1..3147e95 100644 --- a/blockstore/spool.go +++ b/blockstore/spool.go @@ -50,6 +50,18 @@ func (s *Spool) Path(digest mh.Multihash) string { return filepath.Join(s.dir, hex.EncodeToString(digest)) } +// Remove deletes the blob with the given digest from the spool. Idempotent: +// removing a blob that isn't spooled is not an error. Callers own the +// is-it-safe-to-delete question (shared, content-addressed blobs may be +// referenced by other parts or committed objects). +func (s *Spool) Remove(digest mh.Multihash) error { + err := os.Remove(s.Path(digest)) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("blockstore: spool remove: %w", err) + } + return nil +} + // WriteBlob streams r to the spool, computing its sha256 digest as it writes so // the blob is never held whole in memory (object-body blobs run up to // max_blob_size = 256 MiB; buffering them would put that × concurrency in RAM). diff --git a/config/config.go b/config/config.go index e70ea9a..30aa909 100644 --- a/config/config.go +++ b/config/config.go @@ -76,6 +76,12 @@ type Config struct { // may authorize registered provider DIDs directly). AuthServiceProofs string `mapstructure:"auth_service_proofs" yaml:"auth_service_proofs"` + // MultipartSessionTTL bounds abandoned multipart uploads (Go duration + // string, e.g. "168h"): open sessions older than this are aborted by a + // background sweeper and their spooled parts dropped. Empty → default + // 7 days; a negative duration disables the sweeper. + MultipartSessionTTL string `mapstructure:"multipart_session_ttl" yaml:"multipart_session_ttl"` + // CatalogPlane overrides the catalog logstore pipeline knobs. Any field // left zero/unset falls back to the top-level SealBytes / SealAge / Retain // (and Ship defaults to true) — e.g. to configure the catalog never to ship. @@ -125,6 +131,13 @@ func (c Config) ServerConfig() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } + var mpTTL time.Duration + if c.MultipartSessionTTL != "" { + mpTTL, err = time.ParseDuration(c.MultipartSessionTTL) + if err != nil { + return ServerConfig{}, fmt.Errorf("ingot: parse multipart_session_ttl %q: %w", c.MultipartSessionTTL, err) + } + } // Render the CORS configuration here — the single place it is built — // so a typo fails at startup (via Validate) rather than from New. corsCfg, err := cors.Build(c.CORSAllowedOrigins) @@ -148,6 +161,8 @@ func (c Config) ServerConfig() (ServerConfig, error) { SealAgeCatalog: catAge, ShipCatalog: shipDefault(c.CatalogPlane.Ship), RetainCatalog: firstNonZeroInt(c.CatalogPlane.Retain, c.Retain), + + MultipartSessionTTL: mpTTL, }, nil } diff --git a/config/server.go b/config/server.go index 565f27e..dfe25f6 100644 --- a/config/server.go +++ b/config/server.go @@ -46,6 +46,13 @@ type ServerConfig struct { MaxConnections int MaxRequests int + // MultipartSessionTTL bounds abandoned multipart uploads: open sessions + // older than this are aborted by a background sweeper (dropping their + // spooled parts), and completed-session rows retained for Complete + // idempotency are reaped past the same age. Zero → default 7 days; + // negative → sweeper disabled. + MultipartSessionTTL time.Duration + // CORSConfig is the S3 CORS configuration the backend reports for // every bucket, rendered from Config.CORSAllowedOrigins by // internal/cors. Nil disables CORS entirely (the default). diff --git a/docs/architecture.md b/docs/architecture.md index 2ff2c50..9cef025 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,9 +90,13 @@ The design is shaped by how the Forge upload pipeline works and by a handful of - **The 256 MiB blob ceiling is a knob, not a wall.** Piri currently caps a blob at 256 MiB because it builds the commP Merkle tree in RAM; known improvements (streaming commP) lift this. Ingot treats it as a tunable maximum and splits larger objects. -- **Some primitives are not built yet.** There is no `unallocate` for a parked blob; `blob/remove` - is declared but unhandled; the whole-root on-chain delete is wired but its signature path is - incomplete; the indexer has no retraction. [§9](#9-the-system-contract-piri--sprue--indexer) enumerates the contract Ingot needs. +- **The delete primitives are built.** `/blob/remove` is handled end-to-end (Sprue forwards + `/blob/release` to the nodes and deregisters; Piri releases the space's claim and defers physical + deletion until the aggregate root retires on-chain via the signed `schedulePieceDeletions` + path), and `/blob/abort` — translated by Sprue into `/blob/reject` on the node — retires a + parked (never-accepted) blob — an upload ends in exactly one of accept or reject. The + indexer still has no retraction. [§9](#9-the-system-contract-piri--sprue--indexer) enumerates + the contract Ingot needs. --- @@ -352,7 +356,7 @@ fixed constant (pdp-sim). discarded). This is the sub-`min` tail. **The delete primitives** are deliberately distinct: -- **`unallocate(digest)`** retires a **parked** blob (PUT, never accepted): delete the MinIO bytes +- **`abort(digest)`** retires a **parked** blob (PUT, never accepted): delete the MinIO bytes and the allocation record. No chain involvement. - **`remove(digest)`** releases a **space's claim** on an **accepted** blob. Because dedup is global, Piri deletes the bytes and retires the piece only when the per-`(digest, space)` claim count @@ -425,7 +429,7 @@ CompleteMultipartUpload([parts]) splice; blob_refs += version per digest; guarded root swap AbortMultipartUpload latch session open→aborting - parked blobs → unallocate; already-accepted (deduped) blobs → remove (§6) + parked blobs → abort; already-accepted (deduped) blobs → remove (§6) ``` A completed multipart object is the ordered union of its parts' blobs plus a manifest of byte ranges; @@ -436,13 +440,13 @@ later `GET`/`HEAD ?partNumber=N` can resolve part `N` to its byte span and repor ### 7.3 The session latch (the Abort/Complete race) `Complete` and `Abort` can arrive concurrently for one `uploadId`. Without coordination they collide -on the parts — `Complete` triggering accept while `Abort` unallocates those same parts — leaving the +on the parts — `Complete` triggering accept while `Abort` rejects those same parts — leaving the object half-built or half-deleted. A **single-winner latch** prevents it: an atomic state transition on the session row (`UPDATE … SET state=? WHERE state='open'`). Exactly one of `Complete`→`completing` or `Abort`→`aborting` wins; the loser observes the moved row and returns an -error. Accept is triggered only after the session is latched `completing`, so accept and unallocate +error. Accept is triggered only after the session is latched `completing`, so accept and reject never touch the same part concurrently. A deduped part is already accepted (not parked), so Abort -removes it via the reference path rather than unallocating it. +removes it via the reference path rather than rejecting it. ### 7.4 Read (`GetObject`) @@ -468,7 +472,7 @@ updates are transactional with the commit. | Case | Handling | |---|---| -| Crash after PUT, before accept | Bytes parked; `upload_intents` (parked) drives resume or `unallocate`. No `200` was sent. | +| Crash after PUT, before accept | Bytes parked; `upload_intents` (parked) drives resume or `abort`. No `200` was sent. | | Crash after accept, before commit | Blob durable but unreferenced; `upload_intents` (accepted) drives commit-retry or `remove`. | | Guarded-root-swap mismatch | Reload root, re-splice; blobs already durable, never re-uploaded. | | Concurrent PUT, same key | Distinct versionIds; serialize only on the swap. | @@ -528,11 +532,11 @@ negotiations). | Capability | Service | Status | Notes | |------------------------------------------------------------------------------------------------|---------------------------------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `allocate` / `PUT` / `accept` blob lifecycle | Piri/Sprue | **exists** | The storage primitive Ingot builds on. | -| Ingot-timed accept (PUT a part, defer the conclude until Complete) | Ingot + Sprue | **partial** | Accept already fires on the client's conclude; needs the park-vs-conclude split client-side and a separable Sprue conclude. Ingot does **not** issue `accept` (Piri requires the upload-service DID). | -| `unallocate(digest)` — drop a parked blob | Piri + Sprue + libforge | **to-build** | `blob/remove` arg type exists in libforge; no handler. | -| `remove(digest)` — per-space claim release; physical delete/piece-retire at zero global claims | Piri + Sprue + libforge | **to-build** | Piri keeps per-`(digest,space)` allocation rows to count on. | +| Ingot-timed accept (PUT a part, defer the conclude until Complete) | Ingot + Sprue | **exists** | `forgeclient.BlobAdd` with `WithConclude(false)` stops at the conclude seam (`BlobConclude` finishes it); UploadPart parks (durable, unaggregated), Complete concludes — Sprue's conclude handler already ran accept standalone. Ingot still does **not** issue `accept` (Piri requires the upload-service DID). | +| `abort(digest)` — drop a parked blob | Piri + Sprue + libforge | **exists** | `/blob/abort` on Sprue translates to `/blob/reject` on Piri, which refuses blobs the invoking space has accepted (`BlobAccepted`), deletes the space's allocation, and drops the bytes once no space holds an allocation or acceptance. Sprue recovers the provider from the `cause` receipt chain (a parked blob has no registration). Abort/TTL/supersede unwind through it. | +| `remove(digest)` — per-space claim release; physical delete/piece-retire at zero global claims | Piri + Sprue + libforge | **exists** | `/blob/remove` on Sprue forwards `/blob/release` to Piri, which deletes the space's allocation/acceptance/claim; at zero claims bytes delete immediately (unaggregated) or via the pending-removal sweep once the whole aggregate root is dead (FIL-623/624). Sprue forwards to primary + replicas (FIL-522). | | Configurable, adaptive size policy (`min`/`max`); batch guard for the `extraData` cap | Piri | **partial** | `MinAggregateSize` is hardcoded 128 MiB; lower to ~8 MiB and make configurable. The `addPieces` batch is no longer contract-capped (FWSS v1.3.0 removed the `extraData` cap); size it to the FVM `PiecesAdded` event-size + per-tx gas — a measured ceiling (default `BatchSize=10` is safely within it) (pdp-sim). No contract change. | -| Compaction (Regime B) + complete the on-chain delete signature | Piri | **partial** | Whole-root delete is wired but its `extraData` signature is incomplete; compaction (remove + re-hash survivors + re-add) is new. | +| Compaction (Regime B) + complete the on-chain delete signature | Piri | **partial** | Whole-root delete is signed and wired (`schedulePieceDeletions` with `SignSchedulePieceRemovals` extraData); compaction (remove + re-hash survivors + re-add) is new. | | De-dup at accept (don't re-aggregate a digest already a live piece) | Piri | **to-build** | Backstops one-piece-per-content once accept timing is Ingot-driven. | | Parked-allocation GC + honor `Expires` | Piri | **to-build** | Bounds leakage when an abort never arrives. | | Indexer delete by `(space, digest)` / location-claim CID | Indexer + Sprue | **to-build** | IPNI removal mechanics are the indexer's. | @@ -543,7 +547,7 @@ negotiations). **Determinism / idempotency Ingot must preserve.** The `accept` invocation is built deterministically (stable CID, today via `WithNoNonce` over `{space, digest, size, put-task}`); re-driving accept must -reuse the same put-task link. `remove`/`unallocate` must be idempotent. The forge-root advance must +reuse the same put-task link. `remove`/`abort` must be idempotent. The forge-root advance must happen only after a successful guarded root swap — the catalog log currently advances it before the swap, so a mismatch can leave the forge root pointing at a bucket root that was never adopted. @@ -575,7 +579,7 @@ The first cut keeps digest-before-upload; `allocate-by-size` is a same-rack fast - **`min`/`max` final values** — 8 MiB / 256 MiB are the starting proposals; confirm against the base-fee and transaction-count budget. - **Lifting the 256 MiB ceiling** (streaming commP) versus the simplicity of coarse splitting. -- **Local cache vs near-stateless** — cache sizing/eviction, or commit to near-stateless. +- **Local cache vs near-stateless** — cache sizing/eviction, or commit to near-stateless (#48). - **Catalog GC** — `gc_candidates` is a write-only log this iteration; the catalog CARs on Piri grow with mutation volume until a collector exists. - **Monotonic `nextPieceId` ratchet** under heavy churn (pdp-sim) — possibly periodic dataset @@ -596,7 +600,7 @@ The MVP this supersedes had six structural problems; each is resolved by a layer |-----------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | A per-bucket lock held across the whole write | Ingest and upload run off-lock; only the MST splice + guarded root swap is in the critical section. [§7.1](#71-write-single-shot-putobject) | | The whole object buffered in memory | Hash-while-writing to the local store; stream to Piri; nothing held whole in RAM. [§5](#5-the-data-layer), [§7.1](#71-write-single-shot-putobject) | -| No multipart, no abort | Parked part-blobs, accept-at-Complete, single-winner latch; abort = `unallocate` (parked) / `remove` (deduped). [§7.2](#72-multipart)–7.3, [§9](#9-the-system-contract-piri--sprue--indexer) | +| No multipart, no abort | Parked part-blobs, accept-at-Complete, single-winner latch; abort = `/blob/abort` (parked) / `remove` (deduped). [§7.2](#72-multipart)–7.3, [§9](#9-the-system-contract-piri--sprue--indexer) | | Objects chunked into CARs to obtain a digest | Object body = one blob (≤ `max`) or a coarse `≤ max` split; no fine chunking, no CAR. [§5](#5-the-data-layer), [§10](#10-deployment-topology--the-digest-before-upload-cost) | | No delete of superseded data | Reference index + per-space `remove` + Piri's global claim gate + indexer delete; O(1) for ≥ `min` blobs. [§5](#5-the-data-layer), [§6](#6-the-forgechain-layer), [§9](#9-the-system-contract-piri--sprue--indexer) | | Aggregation blocked partial deletes | A small `min` makes most blobs their own piece (O(1) delete); compaction handles only the sub-`min` tail. [§6](#6-the-forgechain-layer) | @@ -805,23 +809,42 @@ reference index — and is out of scope for this iteration. ### Deferred as forge-mode glue (validated live in smelt, not the in-process harness) -The in-memory harness uses a no-op uploader and serves reads from the spool, so these forge-network -paths are stubbed in-tree and verified against a real sprue+piri+indexer later: - -- **`remove(digest)` / `unallocate(digest)` are logged no-ops.** libforge has the `blob.Remove` - binding; the Piri/Sprue handler is to-build ([§9](#9-the-system-contract-piri--sprue--indexer)). The reference-index bookkeeping (count→0 ⇒ - `RemoveBlob` call site) is fully built and tested; only the network release is stubbed. -- **The local-table `Locator` read tier is not wired.** Body-blob *locations* are recorded at accept, - but the read path that consumes them ([§7.4](#74-read-getobject), [§8](#8-retrieval-addressing-when-bodies-need-a-sharded-dag-index)) is deferred — it is only exercised after spool - eviction (also not built) and is best validated live. -- **Multipart parts upload at Complete, not parked at UploadPart.** The in-process flow spools parts +The in-memory harness uses a no-op uploader and serves reads from the spool; the forge-network +paths below are exercised against the real stack by the smelt-based `itest/` harness in CI: + +- **`remove(digest)` and `abort(digest)` are live.** `RemoveBlob` invokes `/blob/remove` on + sprue, which forwards `/blob/release` to the storage nodes ([§9](#9-the-system-contract-piri--sprue--indexer)); delete finality means claim-release-now, + bytes-at-root-death. `AbortBlob` retires parked part-blobs via `/blob/abort` — sprue translates + it into `/blob/reject` on the node (provider recovered from the `cause` receipt chain); + allocation-expiry GC (FIL-625) remains the backstop when an abort never arrives. +- **The local-table `Locator` read tier is wired and validated.** Body blobs re-resolve after + spool loss from `blob_locations` + `/content/retrieve` (`TestForgeReadAfterEviction`), and + retention-retired catalog blocks resolve via `shard_inclusions` (#44) — the read paths of + [§7.4](#74-read-getobject) / [§8](#8-retrieval-addressing-when-bodies-need-a-sharded-dag-index). + Spool **eviction** itself is still unbuilt: nothing bounds the spool, and `DeleteObject`'s + release is network-side only, so local disk grows with every body byte ever written — the + bounded-cache policy [§5](#5-the-data-layer) specifies is tracked in #48. +- **Multipart parts park at UploadPart, accept at Complete.** (Built: `parkBlobs`/`concludeBlobs` + over the `blob_parks` table.) The in-process harness still spools parts at `UploadPart` and uploads+accepts them at `Complete`; the true forge *parking* (upload early, - accept-at-Complete) and `unallocate`-on-abort from [§7.2](#72-multipart)–[7.3](#73-the-session-latch-the-abortcomplete-race) are forge-mode refinements. + accept-at-Complete) and the `/blob/abort` unwind from [§7.2](#72-multipart)–[7.3](#73-the-session-latch-the-abortcomplete-race) are forge-mode refinements. - **Crash recovery for the spool is not built.** The `upload_intents` × `blob_refs` reconciliation - the failure-mode table in [§7.5](#75-concurrency-durability-and-failure-modes) describes (resume/`unallocate` parked, `remove` accepted-but-unreferenced) + the failure-mode table in [§7.5](#75-concurrency-durability-and-failure-modes) describes (resume/`abort` parked, `remove` accepted-but-unreferenced) is a later phase; a partial post-commit reference-index write currently relies on retry/idempotency. -- **`UploadPartCopy`, `ListParts`, `ListMultipartUploads`, and indexer retraction on delete** are - unimplemented (`ErrNotImplemented` / no-op). +- **`UploadPartCopy` and indexer retraction on delete** are unimplemented + (`ErrNotImplemented` / no-op). `ListParts` and `ListMultipartUploads` are implemented + (paginated, prefix/delimiter/marker semantics; in-flight sessions only). +- **Multipart hygiene (spool-model edition).** Abort and part re-upload delete the + now-unreferenced spooled blobs (guarded against content-addressed sharing with other + sessions and committed objects), and a background sweeper aborts open sessions older + than `multipart_session_ttl` (default 7d) and reaps terminal session rows. A successful + Complete retains its session in state `completed` so a duplicate Complete is idempotent + per S3. `DeleteBucket` implicitly aborts the bucket's in-flight sessions before the space + delete (upstream's conformance teardown never aborts them); because `s3:DeleteBucket` + delegates no blob commands, the abort runs on the space authority captured at `UploadPart`, + and its `/blob/abort` leg is gated on hilt delegating `blob.Abort` with the write set + (fil-forge/hilt#36). The network-side `/blob/abort` unwind remains a parking-flow + concern (above). ### Known correctness boundary diff --git a/forgeclient/blobabort.go b/forgeclient/blobabort.go new file mode 100644 index 0000000..afc338e --- /dev/null +++ b/forgeclient/blobabort.go @@ -0,0 +1,58 @@ +package forgeclient + +import ( + "context" + "fmt" + + blobcmds "github.com/fil-forge/libforge/commands/blob" + ucanlib "github.com/fil-forge/libforge/ucan" + "github.com/fil-forge/ucantone/did" + "github.com/fil-forge/ucantone/execution" + "github.com/fil-forge/ucantone/ucan/invocation" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" +) + +// BlobAbort invokes /blob/abort against the upload service (sprue), +// abandoning the space's in-flight upload of a parked (never-accepted) +// blob. cause is the /blob/add task link (AddedBlob.AddTask) — sprue +// walks its receipt chain to locate the storage node holding the parked +// bytes (which have no registration or acceptance to look up by) and +// forwards a /blob/reject there. The space is the invocation subject. +// Idempotent on the node; a blob the space has accepted is refused with +// BlobAccepted (release it via the reference index / /blob/remove instead). +func (c *Client) BlobAbort(ctx context.Context, space did.DID, digest multihash.Multihash, cause cid.Cid, options ...BlobAddOption) error { + cfg := NewBlobAddConfig(options...) + proofStore := ucanlib.ProofStore(c.tokenStore) + if cfg.ProofStore != nil { + proofStore = cfg.ProofStore + } + + proofs, proofLinks, err := proofStore.ProofChain(ctx, c.signer.DID(), blobcmds.Abort.Command, space) + if err != nil { + return fmt.Errorf("building proof chain: %w", err) + } + inv, err := blobcmds.Abort.Invoke( + c.signer, + space, + // The space is the invocation subject; it is not repeated in the + // arguments. + &blobcmds.AbortArguments{Digest: digest, Cause: cause}, + invocation.WithAudience(c.serviceID), + invocation.WithProofs(proofLinks...), + ) + if err != nil { + return fmt.Errorf("creating invocation: %w", err) + } + + _, _, _, err = Execute[*blobcmds.AbortOK]( + ctx, + c.ucanClient, + inv, + execution.WithDelegations(proofs...), + ) + if err != nil { + return fmt.Errorf("executing invocation: %w", err) + } + return nil +} diff --git a/forgeclient/blobadd.go b/forgeclient/blobadd.go index 2c18f58..d27c76d 100644 --- a/forgeclient/blobadd.go +++ b/forgeclient/blobadd.go @@ -6,6 +6,9 @@ // not just the client's token store. // - No /blob/accept re-delegation: sprue owns accept (as it owns // allocate), so the conclude/put-receipt dance carries no space proof. +// - BlobAdd accepts WithConclude(false) so multipart can defer the +// conclude (park at UploadPart, accept at Complete); BlobConclude +// finishes the parked add later. // // Also dropped from upstream: otel spans, go-log, ctxutil, the progress/stall // readers, and the hard requirement that the accept receipt carry a PDP @@ -55,11 +58,14 @@ type BlobAddConfig struct { // call instead of the client's default token store — used to scope an // invocation to a request's per-access-key proofs. ProofStore ucanlib.ProofStore + // Conclude controls whether BlobAdd concludes the /http/put receipt + // (triggering /blob/accept) before returning. Default true. + Conclude bool } // NewBlobAddConfig builds a BlobAddConfig from options. func NewBlobAddConfig(options ...BlobAddOption) *BlobAddConfig { - cfg := &BlobAddConfig{PutClient: &http.Client{}} + cfg := &BlobAddConfig{PutClient: &http.Client{}, Conclude: true} for _, opt := range options { opt(cfg) } @@ -86,20 +92,65 @@ func WithProofStore(ps ucanlib.ProofStore) BlobAddOption { return func(cfg *BlobAddConfig) { cfg.ProofStore = ps } } -// AddedBlob is the result of a successful BlobAdd. +// WithConclude controls whether BlobAdd concludes the upload before +// returning. WithConclude(false) leaves the blob parked — durable on the +// provider, but piri holds the bytes without aggregating them until +// /blob/accept fires: the returned AddedBlob has a nil Location and carries +// the state [Client.BlobConclude] needs to finish the upload later (or +// [Client.BlobAbort] to abandon it; AddTask is the abort Cause). Moot on +// dedup — when the provider already held accepted bytes for the content, +// accept already ran and the add completes regardless. +func WithConclude(conclude bool) BlobAddOption { + return func(cfg *BlobAddConfig) { cfg.Conclude = conclude } +} + +// AddedBlob is the result of a BlobAdd. Location is set once the blob is +// accepted; with WithConclude(false) it is nil until the deferred +// [Client.BlobConclude] — persist the task links + PutInvocation in between. type AddedBlob struct { - Digest multihash.Multihash - Size uint64 - Location ucan.Invocation // the /assert/location commitment + Digest multihash.Multihash + Size uint64 + // Location is the /assert/location commitment issued at accept; nil + // while the add is unconcluded (parked). + Location ucan.Invocation + // AddTask is the /blob/add task link — the receipt-chain root the + // upload service uses to locate the provider for abort. + AddTask cid.Cid + // AcceptTask is the /blob/accept task link BlobConclude polls. + AcceptTask cid.Cid + // PutInvocation is the issued /http/put invocation, populated only while + // the add is unconcluded. Its metadata embeds the derived signer keys + // needed to synthesize the put receipt at conclude time — treat it as + // sensitive and delete it once concluded or rejected. + PutInvocation []byte } // BlobAdd adds a blob to the upload service (sprue): invoke /blob/add, // PUT the bytes, conclude a synthesized /http/put receipt, then poll the // /blob/accept receipt for the location commitment. The issuer needs a -// /blob/add delegation proof over space. -func (c *Client) BlobAdd(ctx context.Context, space did.DID, content io.Reader, options ...BlobAddOption) (blob AddedBlob, err error) { +// /blob/add delegation proof over space. With WithConclude(false) it stops +// after the PUT — the blob stays parked until [Client.BlobConclude]. +func (c *Client) BlobAdd(ctx context.Context, space did.DID, content io.Reader, options ...BlobAddOption) (AddedBlob, error) { cfg := NewBlobAddConfig(options...) + added, err := c.blobAdd(ctx, space, content, cfg) + if err != nil { + return AddedBlob{}, err + } + // Already accepted (dedup) or deliberately unconcluded — done either way. + if added.Location != nil || !cfg.Conclude { + return added, nil + } + return c.BlobConclude(ctx, space, added) +} +// blobAdd runs the durable half of BlobAdd: /blob/add + PUT the bytes, +// WITHOUT concluding the /http/put receipt — the conclude is what makes the +// upload service trigger /blob/accept on the provider, so the blob stays +// parked (stored, unaggregated) until BlobConclude. The result's Location is +// nil unless the provider already held accepted bytes for this content +// (dedup: allocate returned no upload address and the put receipt was +// pre-issued, so accept already ran). +func (c *Client) blobAdd(ctx context.Context, space did.DID, content io.Reader, cfg *BlobAddConfig) (blob AddedBlob, err error) { putClient := cfg.PutClient contentReader := content contentHash := cfg.PrecomputedDigest @@ -111,7 +162,7 @@ func (c *Client) BlobAdd(ctx context.Context, space did.DID, content io.Reader, if err != nil { c.logger.Error("blob add failed", zap.Stringer("space", space), zap.Error(err), zap.Duration("duration", time.Since(start))) } else { - c.logger.Debug("blob added", zap.Stringer("space", space), zap.Duration("duration", time.Since(start))) + c.logger.Debug("blob added", zap.Stringer("space", space), zap.Bool("parked", blob.Location == nil), zap.Duration("duration", time.Since(start))) } }() @@ -213,31 +264,95 @@ func (c *Client) BlobAdd(ctx context.Context, space did.DID, content io.Reader, } } - // Conclude a synthesized /http/put receipt so /blob/accept can resolve. - // Accept is owned by sprue (like allocate), so no /blob/accept - // re-delegation is attached — the conclude is issued agent→sprue and - // carries no space proof. - if !putSuccess { - if err := c.sendPutReceipt(ctx, putInv); err != nil { - return AddedBlob{}, fmt.Errorf("sending put receipt: %w", err) + // Dedup path: the provider already held accepted bytes for this content, + // so the put receipt was pre-issued and the upload service ran accept + // synchronously — the blob is not parked. Await the accept receipt and + // return the completed AddedBlob. + if putSuccess { + location, aerr := c.awaitAccept(ctx, accInv.Task().Link()) + if aerr != nil { + return AddedBlob{}, aerr } + return AddedBlob{ + Digest: contentHash, + Size: *contentSizePtr, + Location: location, + AddTask: inv.Task().Link(), + AcceptTask: accInv.Task().Link(), + }, nil + } + + // Parked: durable on the provider, conclude deferred to BlobConclude. + return AddedBlob{ + Digest: contentHash, + Size: *contentSizePtr, + AddTask: inv.Task().Link(), + AcceptTask: accInv.Task().Link(), + PutInvocation: putInv.Bytes(), + }, nil +} + +// BlobConclude finishes a parked (unconcluded) BlobAdd: it synthesizes and +// concludes the /http/put receipt (which makes the upload service trigger +// /blob/accept on the provider) and awaits the accept receipt's location +// commitment. Accept is owned by sprue (like allocate), so the conclude +// carries no space proof. Safe to retry — re-concluding an already-concluded +// put is tolerated upstream, and an AddedBlob whose Location is already set +// returns as-is. The result drops PutInvocation (spent — the caller should +// delete its persisted copy too). +func (c *Client) BlobConclude(ctx context.Context, space did.DID, added AddedBlob) (blob AddedBlob, err error) { + if added.Location != nil { + return added, nil + } + start := time.Now() + defer func() { + if err != nil { + c.logger.Error("blob conclude failed", zap.Stringer("space", space), zap.Error(err), zap.Duration("duration", time.Since(start))) + } else { + c.logger.Debug("blob concluded", zap.Stringer("space", space), zap.Duration("duration", time.Since(start))) + } + }() + + putInv := new(invocation.Invocation) + if err := putInv.UnmarshalCBOR(bytes.NewReader(added.PutInvocation)); err != nil { + return AddedBlob{}, fmt.Errorf("decoding parked /http/put invocation: %w", err) + } + + if err := c.sendPutReceipt(ctx, putInv); err != nil { + return AddedBlob{}, fmt.Errorf("sending put receipt: %w", err) } - accRcpt, accMeta, err := c.receiptsClient.Poll(ctx, accInv.Task().Link(), receipt_client.WithRetries(5)) + location, err := c.awaitAccept(ctx, added.AcceptTask) if err != nil { - return AddedBlob{}, fmt.Errorf("polling accept receipt: %w", err) + return AddedBlob{}, err + } + return AddedBlob{ + Digest: added.Digest, + Size: added.Size, + Location: location, + AddTask: added.AddTask, + AcceptTask: added.AcceptTask, + }, nil +} + +// awaitAccept polls the /blob/accept receipt and extracts the +// /assert/location commitment from its metadata. +func (c *Client) awaitAccept(ctx context.Context, acceptTask cid.Cid) (ucan.Invocation, error) { + accRcpt, accMeta, err := c.receiptsClient.Poll(ctx, acceptTask, receipt_client.WithRetries(5)) + if err != nil { + return nil, fmt.Errorf("polling accept receipt: %w", err) } - o, x = accRcpt.Out().Unpack() + o, x := accRcpt.Out().Unpack() if accRcpt.Out().IsErr() { var model edm.ErrorModel if err := model.UnmarshalCBOR(bytes.NewReader(x)); err != nil { - return AddedBlob{}, fmt.Errorf("executing invocation") + return nil, fmt.Errorf("executing invocation") } - return AddedBlob{}, fmt.Errorf("failure in accept receipt: %w", model) + return nil, fmt.Errorf("failure in accept receipt: %w", model) } var accOK blobcmds.AcceptOK if err := accOK.UnmarshalCBOR(bytes.NewReader(o)); err != nil { - return AddedBlob{}, fmt.Errorf("unmarshaling accept receipt output: %w", err) + return nil, fmt.Errorf("unmarshaling accept receipt output: %w", err) } var locationCommitment ucan.Invocation @@ -247,10 +362,9 @@ func (c *Client) BlobAdd(ctx context.Context, space did.DID, content io.Reader, } } if locationCommitment == nil { - return AddedBlob{}, fmt.Errorf("blob accept receipt missing location commitment invocation") + return nil, fmt.Errorf("blob accept receipt missing location commitment invocation") } - - return AddedBlob{Digest: contentHash, Size: *contentSizePtr, Location: locationCommitment}, nil + return locationCommitment, nil } func putBlob(ctx context.Context, client *http.Client, url *url.URL, headers map[string]string, body io.Reader, size int64) error { diff --git a/forgeclient/blobremove.go b/forgeclient/blobremove.go new file mode 100644 index 0000000..a2a05ed --- /dev/null +++ b/forgeclient/blobremove.go @@ -0,0 +1,55 @@ +package forgeclient + +import ( + "context" + "fmt" + + blobcmds "github.com/fil-forge/libforge/commands/blob" + ucanlib "github.com/fil-forge/libforge/ucan" + "github.com/fil-forge/ucantone/did" + "github.com/fil-forge/ucantone/execution" + "github.com/fil-forge/ucantone/ucan/invocation" + "github.com/multiformats/go-multihash" +) + +// BlobRemove invokes /blob/remove against the upload service (sprue), +// releasing the space's claim on the blob. Sprue deregisters the blob and +// forwards a /blob/release to every storage node holding it; piri deletes +// the bytes only when no space claims the digest at all. The space is the +// invocation subject. Idempotent: removing an unknown or already-removed +// blob succeeds. +func (c *Client) BlobRemove(ctx context.Context, space did.DID, digest multihash.Multihash, options ...BlobAddOption) error { + cfg := NewBlobAddConfig(options...) + proofStore := ucanlib.ProofStore(c.tokenStore) + if cfg.ProofStore != nil { + proofStore = cfg.ProofStore + } + + proofs, proofLinks, err := proofStore.ProofChain(ctx, c.signer.DID(), blobcmds.Remove.Command, space) + if err != nil { + return fmt.Errorf("building proof chain: %w", err) + } + inv, err := blobcmds.Remove.Invoke( + c.signer, + space, + // The space is the invocation subject; it is not repeated in the + // arguments. + &blobcmds.RemoveArguments{Digest: digest}, + invocation.WithAudience(c.serviceID), + invocation.WithProofs(proofLinks...), + ) + if err != nil { + return fmt.Errorf("creating invocation: %w", err) + } + + _, _, _, err = Execute[*blobcmds.RemoveOK]( + ctx, + c.ucanClient, + inv, + execution.WithDelegations(proofs...), + ) + if err != nil { + return fmt.Errorf("executing invocation: %w", err) + } + return nil +} diff --git a/go.mod b/go.mod index fb314fb..9baf2f1 100644 --- a/go.mod +++ b/go.mod @@ -9,11 +9,12 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.104.1 github.com/fil-forge/hilt v0.0.1-0.20260724134448-ba71f843f6a4 github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 - github.com/fil-forge/libforge v0.0.0-20260724113901-7fc3b2cec1ef - github.com/fil-forge/smelt v0.0.0-20260720130429-63116166a06c + github.com/fil-forge/libforge v0.0.0-20260727220215-5e299c46f62f + github.com/fil-forge/smelt v0.0.0-20260805233620-f0b69e68aad1 github.com/fil-forge/ucantone v0.0.0-20260727203046-ccb77059de44 github.com/fil-forge/versitygw v0.0.0-20260716095011-7a65883d595a github.com/gofiber/fiber/v3 v3.4.0 + github.com/google/uuid v1.6.0 github.com/ipfs/go-block-format v0.2.4 github.com/ipfs/go-cid v0.6.2 github.com/ipfs/go-ipld-cbor v0.3.0 @@ -109,7 +110,6 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/go.sum b/go.sum index ed1be75..f16426a 100644 --- a/go.sum +++ b/go.sum @@ -183,10 +183,10 @@ github.com/fil-forge/hilt v0.0.1-0.20260724134448-ba71f843f6a4 h1:pDDN87a4dMuH8m github.com/fil-forge/hilt v0.0.1-0.20260724134448-ba71f843f6a4/go.mod h1:AO/+NYsz//BoqdHjVfik0fbxjiq+H+86qzGBt2/2uxQ= github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 h1:Wke8qgaDgy7DGaIS28VpHip+YHSdQP1hGNEZTrXXzb4= github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717/go.mod h1:wFcakLohOqpMRkJzWRdGFGHRFpGB+PpEMCm9wkt2cqU= -github.com/fil-forge/libforge v0.0.0-20260724113901-7fc3b2cec1ef h1:xgkciShyWdCQ2Pyl2qEwP7JMbsdbHdECj/v6E5+fj6U= -github.com/fil-forge/libforge v0.0.0-20260724113901-7fc3b2cec1ef/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= -github.com/fil-forge/smelt v0.0.0-20260720130429-63116166a06c h1:WHvsleEU6ZiNYDFgLx6KtorXulD+IuLiorMgmp4Th8s= -github.com/fil-forge/smelt v0.0.0-20260720130429-63116166a06c/go.mod h1:NM/mk/XiP1Kzsy9HWGQeLsosPUvVY3bIrkUzqswThpU= +github.com/fil-forge/libforge v0.0.0-20260727220215-5e299c46f62f h1:QzgMg8GIE4IhgOE/7DBHFU/4T5oU7J4vAKxbUPKJPGA= +github.com/fil-forge/libforge v0.0.0-20260727220215-5e299c46f62f/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= +github.com/fil-forge/smelt v0.0.0-20260805233620-f0b69e68aad1 h1:0Ia79MRlX3q/wk5/XCvZYou7rkov4xQMpCuNzAb+KJY= +github.com/fil-forge/smelt v0.0.0-20260805233620-f0b69e68aad1/go.mod h1:czwgD0nnuQ8h3GOHdwQ2Guw3C9R3zYP3K/mE+DEyh+Y= github.com/fil-forge/ucantone v0.0.0-20260727203046-ccb77059de44 h1:ofvb2Qq7++VPRelGsLbtnd1ZMKVT4n4QGoa79BhJ6VQ= github.com/fil-forge/ucantone v0.0.0-20260727203046-ccb77059de44/go.mod h1:oFY5BfD0bDeodGlbBHh3/nK99MAS93rGXjoQz7s5qgE= github.com/fil-forge/versitygw v0.0.0-20260716095011-7a65883d595a h1:lDwnNmF4LNbevx/YYNCC0CazMiL4eIe38MvfqbRR3RA= diff --git a/inmem/store.go b/inmem/store.go index f72d98d..c4acdc5 100644 --- a/inmem/store.go +++ b/inmem/store.go @@ -58,6 +58,7 @@ type MemStore struct { intents map[string]registry.UploadIntent // keyed by string(digest) locations map[locKey]registry.BlobLocation // keyed by (space, digest) inclusions map[locKey]registry.BlobInclusion // keyed by (space, digest) + parks map[string]registry.BlobPark // keyed by string(digest) sessions map[string]registry.MultipartSession // keyed by uploadID parts map[string]map[int]registry.MultipartPart // uploadID -> partNumber -> part gcCands map[string]struct{} // keyed by string(cid) @@ -83,6 +84,7 @@ func NewMemStore() *MemStore { intents: map[string]registry.UploadIntent{}, locations: map[locKey]registry.BlobLocation{}, inclusions: map[locKey]registry.BlobInclusion{}, + parks: map[string]registry.BlobPark{}, sessions: map[string]registry.MultipartSession{}, parts: map[string]map[int]registry.MultipartPart{}, gcCands: map[string]struct{}{}, @@ -345,12 +347,23 @@ func (NopUploader) SubmitShard(_ context.Context, _ blockstore.Plane, _ did.DID, return uploader.BlobLocation{}, nil } -func (NopUploader) UploadBlob(_ context.Context, _ did.DID, _ multihash.Multihash, size int64, _ string) (uploader.BlobLocation, error) { - return uploader.BlobLocation{Size: size}, nil +// UploadBlob accepts immediately, even with WithConclude(false) — there is +// no network to park on, so the deferred flow degenerates to the synchronous +// one and reads keep coming from the spool. +func (NopUploader) UploadBlob(_ context.Context, _ did.DID, digest multihash.Multihash, size int64, _ string, _ ...uploader.UploadOption) (uploader.UploadedBlob, error) { + return uploader.UploadedBlob{Digest: digest, Size: size, Location: &uploader.BlobLocation{Size: size}}, nil } func (NopUploader) RemoveBlob(_ context.Context, _ did.DID, _ multihash.Multihash) error { return nil } +func (NopUploader) ConcludeBlob(_ context.Context, _ did.DID, parked uploader.UploadedBlob) (uploader.BlobLocation, error) { + return uploader.BlobLocation{Size: parked.Size}, nil +} + +func (NopUploader) AbortBlob(_ context.Context, _ did.DID, _ multihash.Multihash, _ cid.Cid) error { + return nil +} + // Compile-time guarantees. var ( _ bucketauthority.BucketAuthority = (*MemStore)(nil) @@ -360,5 +373,6 @@ var ( _ blockstore.BlobReader = NopBaseReader{} _ uploader.Uploader = NopUploader{} _ uploader.BodyUploader = NopUploader{} + _ uploader.DeferredBodyUploader = NopUploader{} _ uploader.BlobRemover = NopUploader{} ) diff --git a/inmem/stores.go b/inmem/stores.go index c805487..9d39eb3 100644 --- a/inmem/stores.go +++ b/inmem/stores.go @@ -1,7 +1,10 @@ package inmem import ( + "bytes" "context" + "sort" + "time" "github.com/fil-forge/ingot/registry" "github.com/fil-forge/ucantone/did" @@ -30,7 +33,6 @@ func cloneBytes(b []byte) []byte { } // BlobRefStore =============================================================== - func (m *MemStore) AddBlobClaim(_ context.Context, c registry.BlobClaim) error { m.mu.Lock() defer m.mu.Unlock() @@ -148,6 +150,42 @@ func (m *MemStore) DeleteLocation(_ context.Context, space did.DID, digest []byt return nil } +// ParkStore ================================================================== + +func (m *MemStore) PutPark(_ context.Context, p registry.BlobPark) error { + m.mu.Lock() + defer m.mu.Unlock() + cp := p + cp.Digest = cloneBytes(p.Digest) + cp.AddTask = cloneBytes(p.AddTask) + cp.AcceptTask = cloneBytes(p.AcceptTask) + cp.PutInvocation = cloneBytes(p.PutInvocation) + m.parks[string(p.Digest)] = cp + return nil +} + +func (m *MemStore) GetPark(_ context.Context, digest []byte) (*registry.BlobPark, error) { + m.mu.Lock() + defer m.mu.Unlock() + park, ok := m.parks[string(digest)] + if !ok { + return nil, registry.ErrNotFound + } + cp := park + cp.Digest = cloneBytes(park.Digest) + cp.AddTask = cloneBytes(park.AddTask) + cp.AcceptTask = cloneBytes(park.AcceptTask) + cp.PutInvocation = cloneBytes(park.PutInvocation) + return &cp, nil +} + +func (m *MemStore) DeletePark(_ context.Context, digest []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.parks, string(digest)) + return nil +} + // InclusionStore ============================================================= func (m *MemStore) PutInclusions(_ context.Context, incs []registry.BlobInclusion) error { @@ -186,6 +224,9 @@ func (m *MemStore) CreateSession(_ context.Context, s registry.MultipartSession) if s.State == "" { s.State = registry.SessionOpen } + if s.CreatedAt.IsZero() { + s.CreatedAt = time.Now() + } m.sessions[s.UploadID] = cloneSession(s) return nil } @@ -233,6 +274,9 @@ func (m *MemStore) PutPart(_ context.Context, p registry.MultipartPart) error { if p.State == "" { p.State = registry.PartParked } + if p.CreatedAt.IsZero() { + p.CreatedAt = time.Now() + } byNum := m.parts[p.UploadID] if byNum == nil { byNum = map[int]registry.MultipartPart{} @@ -266,6 +310,60 @@ func (m *MemStore) ListParts(_ context.Context, uploadID string) ([]registry.Mul return out, nil } +func (m *MemStore) ListSessions(_ context.Context, bucket string) ([]registry.MultipartSession, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []registry.MultipartSession + for _, s := range m.sessions { + if s.Bucket == bucket { + out = append(out, cloneSession(s)) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].ObjectKey != out[j].ObjectKey { + return out[i].ObjectKey < out[j].ObjectKey + } + if !out[i].CreatedAt.Equal(out[j].CreatedAt) { + return out[i].CreatedAt.Before(out[j].CreatedAt) + } + return out[i].UploadID < out[j].UploadID + }) + return out, nil +} + +func (m *MemStore) ListStaleSessions(_ context.Context, state string, cutoff time.Time) ([]registry.MultipartSession, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []registry.MultipartSession + for _, s := range m.sessions { + if s.State == state && s.CreatedAt.Before(cutoff) { + out = append(out, cloneSession(s)) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.Before(out[j].CreatedAt) }) + return out, nil +} + +func (m *MemStore) CountPartRefs(_ context.Context, digest []byte, excludeUploadID string) (int, error) { + m.mu.Lock() + defer m.mu.Unlock() + n := 0 + for uploadID, byNum := range m.parts { + if uploadID == excludeUploadID { + continue + } + for _, p := range byNum { + for _, d := range p.BlobDigests { + if bytes.Equal(d, digest) { + n++ + break + } + } + } + } + return n, nil +} + // GCStore ==================================================================== func (m *MemStore) AddGCCandidate(_ context.Context, cidBytes []byte, _ string) error { diff --git a/internal/reqscope/reqscope.go b/internal/reqscope/reqscope.go index 804ba28..c5ea6b1 100644 --- a/internal/reqscope/reqscope.go +++ b/internal/reqscope/reqscope.go @@ -37,6 +37,16 @@ func ProofStore(ctx context.Context) (ucanlib.ProofStore, bool) { return ps, ok } +// WithoutProofStore returns a context whose request-scoped proof store is +// masked. Hilt delegates Forge commands per S3 permission, and some +// permissions (s3:DeleteBucket) carry no blob commands at all — a flow that +// must invoke blob capabilities from such a request (DeleteBucket's implicit +// abort of in-flight multipart uploads) hides the request store so the +// uploader falls back to the space authority captured at UploadPart. +func WithoutProofStore(ctx context.Context) context.Context { + return context.WithValue(ctx, proofStoreKey, nil) +} + type requestContextKey struct{} var requestKey any = requestContextKey{} diff --git a/itest/README.md b/itest/README.md index c3bf607..3f85e6c 100644 --- a/itest/README.md +++ b/itest/README.md @@ -37,14 +37,13 @@ yet — e.g. one built from an unmerged branch — point the stack at it: INGOT_ITEST_UPLOAD_IMAGE= make itest ``` -**Teardown-blocked XFail rows:** a bucket that ever held a non-empty object -body cannot currently be deleted — bodies register blobs in the bucket's -space at PUT, `DeleteObject`'s blob release is a no-op until sprue/piri -implement `/blob/remove`, and hilt's `/s3/bucket/delete` refuses non-empty -spaces. Upstream cases delete their bucket in teardown, so such cases pass -their S3 assertions and fail teardown; they sit in the XFail tables (marked -"teardown-blocked") so the unexpected-pass ratchet flags them for promotion -when `/blob/remove` lands. +**Teardown-blocked XFail rows (historical):** before `DeleteObject` released +network blobs (FIL-588), a bucket that ever held a non-empty object body +could not be deleted — hilt's `/s3/bucket/delete` refuses non-empty spaces — +so dozens of cases passed their S3 assertions and failed their bucket-delete +teardown, and sat in the XFail tables marked "teardown-blocked". The +unexpected-pass ratchet flagged them all when the release path landed; they +now live in the pass tables. ## The conformance partition — `TestForgeVersity` @@ -68,7 +67,7 @@ here and add new cases to the pass table (demote to xfail if they fail). | Test | Covers | ~time | |---|---|---| -| `TestForgeScenarios` | Ingot-unique behaviors upstream can't assert, on a stack whose ingot config lowers `max_blob_size` to 64 KiB (`testdata/config-smallblob.yaml`): coarse blob-split round-trip with spooled-by-digest proof (spool counted inside the container), zero-byte objects, a multipart part spanning multiple internal blobs, and failed-Complete session recovery. | ~3 min | +| `TestForgeScenarios` | Ingot-unique behaviors upstream can't assert, on a stack whose ingot config lowers `max_blob_size` to 64 KiB (`testdata/config-smallblob.yaml`): coarse blob-split round-trip with spooled-by-digest proof (spool counted inside the container), zero-byte objects, a multipart part spanning multiple internal blobs, abort deleting the parts' spooled blobs, and failed-Complete session recovery. | ~3 min | | `TestForgeNativeProvision` | Hilt onboarding end-to-end on a fresh stack: tenant + access key via hilt's Tenant API, then a PUT/GET round-trip over the real ship path. | ~1.7 min | | `TestForgeReadAfterEviction` | The appliance read tier: PUT, wipe `/data/spool`, GET must re-fetch body blobs from piri via the local locator + `/content/retrieve`. | ~1.3 min | diff --git a/itest/forge_delete_test.go b/itest/forge_delete_test.go new file mode 100644 index 0000000..53cdc9e --- /dev/null +++ b/itest/forge_delete_test.go @@ -0,0 +1,94 @@ +//go:build itest + +package itest + +import ( + "context" + "strings" + "testing" + "time" + + ingottest "github.com/fil-forge/ingot/testing" + "github.com/fil-forge/smelt/pkg/stack" +) + +// TestForgeDeleteReleasesNetworkBlob is the delete-finality regression gate +// (FIL-588): DeleteObject must release the blob on the network, not just drop +// registry rows. The chain under test is ingot's reference index (claims→0 ⇒ +// RemoveBlob) → forgeclient /blob/remove → sprue (forward + deregister) → +// piri /blob/release (claim release; deferred physical deletion once the +// PDP aggregate root retires on-chain). +// +// Runs on the stock smelt-SDK images like every other itest. It needs +// piri:main ≥ fil-forge/piri#30 (the /blob/release handler) and sprue:main ≥ +// fil-forge/sprue#33 (the forward); until piri#30 publishes, point +// INGOT_ITEST_PIRI_IMAGE at a branch image (the forgeStack escape hatch). +func TestForgeDeleteReleasesNetworkBlob(t *testing.T) { + ctx := t.Context() + + s, ingotEndpoint := forgeStack(t) + accessKey, secretKey := hiltProvisionTenant(t, ctx, s, "delete") + cfg := forgeConfig(ingotEndpoint, accessKey, secretKey) + + const bucket, key = "delete-bucket", "obj" + + // Unique deterministic body, large enough to be a real blob on piri. + data := make([]byte, 512*1024) + for i := range data { + data[i] = byte(i*13 + 11) + } + + if err := ingottest.CreateBucket(ctx, cfg, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + if err := ingottest.PutBytes(ctx, cfg, bucket, key, data); err != nil { + t.Fatalf("put object: %v", err) + } + + // Precondition: the blob really lives on piri — wipe the spool and prove + // the GET re-fetches from the network (same tier the eviction test pins). + if out, errOut, err := s.Exec(ctx, "ingot", "sh", "-c", "rm -rf /data/spool"); err != nil { + t.Fatalf("evict spool: %v (stdout=%s stderr=%s)", err, out, errOut) + } + if got, err := ingottest.GetBytes(ctx, cfg, bucket, key); err != nil || len(got) != len(data) { + t.Fatalf("read-through from piri before delete: err=%v len=%d", err, len(got)) + } + + if err := ingottest.DeleteObject(ctx, cfg, bucket, key); err != nil { + t.Fatalf("delete object: %v", err) + } + + // The object is gone from the gateway. + if _, err := ingottest.GetBytes(ctx, cfg, bucket, key); err == nil { + t.Fatalf("GET after delete succeeded, want NoSuchKey") + } + + // And the release traversed the network: piri's /blob/release handler ran + // and, with the last claim gone, queued the piece for removal. Byte + // release is fully asynchronous — ingot's releaseBlobs is best-effort + // post-commit, and piri's removal sweep (PDPRemoveSweep, 30s ticks) + // re-verifies claims and pipeline state before finalizing — so poll the + // provider's logs through to the finalization line. + waitForPiriLog(t, ctx, s, "/blob/release", 2*time.Minute) + waitForPiriLog(t, ctx, s, "queueing piece removal", 2*time.Minute) + waitForPiriLog(t, ctx, s, "finalized piece removal", 3*time.Minute) + t.Logf("delete finality OK: /blob/release executed on piri and the sweep finalized the byte release") +} + +// waitForPiriLog polls piri-0's container logs until substr appears. +func waitForPiriLog(t *testing.T, ctx context.Context, s *stack.Stack, substr string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + logs, err := s.Logs(ctx, "piri-0") + if err == nil && strings.Contains(logs, substr) { + return + } + select { + case <-ctx.Done(): + t.Fatalf("context done waiting for piri log %q: %v", substr, ctx.Err()) + case <-time.After(2 * time.Second): + } + } + t.Fatalf("piri-0 logs never contained %q within %s", substr, timeout) +} diff --git a/itest/forge_multipart_deferred_test.go b/itest/forge_multipart_deferred_test.go new file mode 100644 index 0000000..3cbbbe3 --- /dev/null +++ b/itest/forge_multipart_deferred_test.go @@ -0,0 +1,197 @@ +//go:build itest + +package itest + +import ( + "bytes" + "context" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/fil-forge/libforge/digestutil" + "github.com/fil-forge/smelt/pkg/stack" + "github.com/multiformats/go-multihash" +) + +// TestForgeDeferredMultipart is the deferred-accept regression gate (§7.2): +// UploadPart must make each part blob durable on piri WITHOUT accepting it +// (parked — outside the PDP pipeline), Complete must conclude (accept) every +// part, and Abort must unwind parked blobs with /blob/abort → /blob/reject. +// +// Runs on the stock smelt-SDK images like every other itest. It needs +// piri:main ≥ fil-forge/piri#30 (/blob/reject) and sprue:main ≥ +// fil-forge/sprue#33 (/blob/abort forwarding); until piri#30 publishes, +// point INGOT_ITEST_PIRI_IMAGE at a branch image (the forgeStack escape +// hatch). +func TestForgeDeferredMultipart(t *testing.T) { + ctx := t.Context() + s, endpoint := forgeStack(t) + accessKey, secretKey := hiltProvisionTenant(t, ctx, s, "mpdeferred") + cl := sdkClient(forgeS3Conf(endpoint, accessKey, secretKey)) + + // RoundTrip: parts park at UploadPart (durable on piri, NOT accepted), + // Complete concludes them, and the object round-trips. + t.Run("RoundTrip", func(t *testing.T) { + const bucket, key = "mp-deferred", "obj" + if _, err := cl.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { + t.Fatalf("CreateBucket: %v", err) + } + create, err := cl.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), + }) + if err != nil { + t.Fatalf("CreateMultipartUpload: %v", err) + } + + // Unique per-part content (XOR-tagged so nothing dedups against other + // tests); each part is a single blob under the default max_blob_size. + partData := [][]byte{tagged(patternBytes(6<<20), 0x11), tagged(patternBytes(90<<10), 0x22)} + var completed []types.CompletedPart + var whole []byte + var partDigests []string + for i, data := range partData { + pn := int32(i + 1) + up, err := cl.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + PartNumber: aws.Int32(pn), Body: bytes.NewReader(data), + }) + if err != nil { + t.Fatalf("UploadPart %d: %v", pn, err) + } + completed = append(completed, types.CompletedPart{PartNumber: aws.Int32(pn), ETag: up.ETag}) + whole = append(whole, data...) + partDigests = append(partDigests, b58Digest(t, data)) + } + + // The parts are durable on piri (allocated + PUT) but parked: no + // /blob/accept for their digests until Complete. + for _, d := range partDigests { + waitForPiriLogLine(t, ctx, s, 30*time.Second, "/blob/allocate", d) + if piriLogHasLine(t, ctx, s, "/blob/accept", d) { + t.Fatalf("part blob %s was accepted before Complete — parking is broken", d) + } + } + + comp, err := cl.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{Parts: completed}, + }) + if err != nil { + t.Fatalf("CompleteMultipartUpload: %v", err) + } + if et := strings.Trim(aws.ToString(comp.ETag), `"`); !strings.HasSuffix(et, "-2") { + t.Fatalf("complete ETag = %q, want a multipart -2 suffix", et) + } + + // Complete concluded every part: accepts landed on piri, and the + // object round-trips. + for _, d := range partDigests { + waitForPiriLogLine(t, ctx, s, 60*time.Second, "/blob/accept", d) + } + if got := getBody(t, ctx, cl, bucket, key, ""); !bytes.Equal(got, whole) { + t.Fatalf("GET mismatch: got %d bytes, want %d", len(got), len(whole)) + } + }) + + // AbortRejects: an aborted upload's parked blobs are unwound on piri + // via /blob/reject — never accepted, bytes released. + t.Run("AbortRejects", func(t *testing.T) { + const bucket, key = "mp-abort-unalloc", "obj" + if _, err := cl.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { + t.Fatalf("CreateBucket: %v", err) + } + create, err := cl.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), + }) + if err != nil { + t.Fatalf("CreateMultipartUpload: %v", err) + } + data := tagged(patternBytes(512<<10), 0x33) + digest := b58Digest(t, data) + if _, err := cl.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + PartNumber: aws.Int32(1), Body: bytes.NewReader(data), + }); err != nil { + t.Fatalf("UploadPart: %v", err) + } + waitForPiriLogLine(t, ctx, s, 30*time.Second, "/blob/allocate", digest) + + if _, err := cl.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + }); err != nil { + t.Fatalf("AbortMultipartUpload: %v", err) + } + + // The abort traversed ingot → sprue → piri (as /blob/reject), and the blob was + // never accepted. + waitForPiriLogLine(t, ctx, s, 60*time.Second, "/blob/reject", digest) + if piriLogHasLine(t, ctx, s, "/blob/accept", digest) { + t.Fatalf("aborted part blob %s was accepted — abort/accept exclusivity is broken", digest) + } + t.Logf("abort unwound parked blob %s via /blob/reject", digest) + }) +} + +// tagged XORs b with tag so each caller gets globally unique content that +// cannot dedup against other tests' blobs. +func tagged(b []byte, tag byte) []byte { + for i := range b { + b[i] ^= tag + } + return b +} + +// b58Digest returns the base58 sha2-256 multihash of data — the form piri's +// handlers log blob digests in (digestutil.Format). +func b58Digest(t *testing.T, data []byte) string { + t.Helper() + mh, err := multihash.Sum(data, multihash.SHA2_256, -1) + if err != nil { + t.Fatalf("multihash: %v", err) + } + return digestutil.Format(mh) +} + +// piriLogHasLine reports whether any single piri-0 log line contains all +// substrings. +func piriLogHasLine(t *testing.T, ctx context.Context, s *stack.Stack, substrs ...string) bool { + t.Helper() + logs, err := s.Logs(ctx, "piri-0") + if err != nil { + t.Fatalf("piri-0 logs: %v", err) + } + for _, line := range strings.Split(logs, "\n") { + ok := true + for _, sub := range substrs { + if !strings.Contains(line, sub) { + ok = false + break + } + } + if ok { + return true + } + } + return false +} + +// waitForPiriLogLine polls until one piri-0 log line contains all substrings. +func waitForPiriLogLine(t *testing.T, ctx context.Context, s *stack.Stack, timeout time.Duration, substrs ...string) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if piriLogHasLine(t, ctx, s, substrs...) { + return + } + select { + case <-ctx.Done(): + t.Fatalf("context done waiting for piri log %v: %v", substrs, ctx.Err()) + case <-time.After(2 * time.Second): + } + } + t.Fatalf("piri-0 logs never contained a line with all of %v within %s", substrs, timeout) +} diff --git a/itest/scenarios_test.go b/itest/scenarios_test.go index cbae819..6a25ccf 100644 --- a/itest/scenarios_test.go +++ b/itest/scenarios_test.go @@ -154,8 +154,10 @@ func TestForgeScenarios(t *testing.T) { } uploadID := create.UploadId - // Part 1 (150 KiB) spans three 64 KiB blobs; parts 2 and 3 are small. - partData := [][]byte{patternBytes(150 << 10), patternBytes(40 << 10), patternBytes(9 << 10)} + // Non-final parts must meet S3's 5 MiB minimum (enforced at Complete); + // at 64 KiB max_blob_size each spans dozens of internal blobs. The + // final part is small (exempt from the minimum). + partData := [][]byte{patternBytes(6 << 20), patternBytes(5 << 20), patternBytes(9 << 10)} var completed []types.CompletedPart var whole []byte for i, data := range partData { @@ -200,6 +202,47 @@ func TestForgeScenarios(t *testing.T) { } }) + // MultipartAbortCleansSpool: aborting an upload discards its parts — the + // registry rows go (upstream AbortMultipartUpload_success verifies via + // ListMultipartUploads) and, ingot-specifically, the parts' spooled blobs + // are deleted, since under the spool model an abort's cleanup is entirely + // local (nothing shipped to the network before Complete). + t.Run("MultipartAbortCleansSpool", func(t *testing.T) { + const bucket, key = "mpabort-spool", "obj" + if _, err := cl.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { + t.Fatalf("CreateBucket: %v", err) + } + create, err := cl.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{Bucket: aws.String(bucket), Key: aws.String(key)}) + if err != nil { + t.Fatalf("CreateMultipartUpload: %v", err) + } + spoolBefore := spoolBlobCount(t, ctx, s) + // A 150 KiB part spans three 64 KiB blobs. The content must be unique + // to this test — patternBytes at the same offsets would dedup against + // the round-trip subtests' already-spooled blobs and skew the counts. + part := patternBytes(150 << 10) + for i := range part { + part[i] ^= 0xA5 + } + if _, err := cl.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + PartNumber: aws.Int32(1), Body: bytes.NewReader(part), + }); err != nil { + t.Fatalf("UploadPart: %v", err) + } + if got := spoolBlobCount(t, ctx, s) - spoolBefore; got != 3 { + t.Fatalf("UploadPart added %d spool blobs, want 3", got) + } + if _, err := cl.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + }); err != nil { + t.Fatalf("AbortMultipartUpload: %v", err) + } + if got := spoolBlobCount(t, ctx, s) - spoolBefore; got != 0 { + t.Fatalf("abort left %d spooled part blobs behind, want 0", got) + } + }) + // MultipartFailedCompleteStaysAbortable: the zombie-session regression. A // Complete that fails validation (wrong part ETag) must leave the session // open — a retry with the correct ETag succeeds and reads back diff --git a/itest/stack_test.go b/itest/stack_test.go index 3d08cdb..9cc68bb 100644 --- a/itest/stack_test.go +++ b/itest/stack_test.go @@ -124,6 +124,10 @@ func forgeStack(t *testing.T, extra ...stack.Option) (*stack.Stack, string) { t.Logf("using piri image override: %s", img) opts = append(opts, stack.WithPiriImage(img)) } + if img := os.Getenv("INGOT_ITEST_HILT_IMAGE"); img != "" { + t.Logf("using hilt image override: %s", img) + opts = append(opts, stack.WithHiltImage(img)) + } // Same idea one step earlier in the pipeline: mount a locally-built piri // binary (linux, static) over the image's /usr/bin/piri — validates an // unreleased piri/ucantone change with no image build at all. @@ -198,6 +202,9 @@ var hiltAllPermissions = []string{ "s3:CreateBucket", "s3:ListAllMyBuckets", "s3:DeleteBucket", + // Multipart operations, first-class in hilt since fil-forge/hilt#35: + // AbortMultipartUpload carries the blob.Abort + blob.Remove delegations + // the abort leg invokes; the two List ops are catalog reads. "s3:AbortMultipartUpload", "s3:ListMultipartUploadParts", "s3:ListBucketMultipartUploads", diff --git a/itest/versity_multipart_test.go b/itest/versity_multipart_test.go index 0184c2f..db754e3 100644 --- a/itest/versity_multipart_test.go +++ b/itest/versity_multipart_test.go @@ -6,16 +6,16 @@ import ( "github.com/fil-forge/versitygw/tests/integration" ) -// Multipart groups of the S3 conformance partition. These had no prior -// curated lists — the partition below was created empirically against the -// forge-mode stack (see the curation note in README.md). The broad strokes: -// the Create/UploadPart/Complete/Abort happy paths and most input validation -// pass; part-level checksums, tagging/object-lock/ACL surfaces, and the -// ListParts / ListMultipartUploads / UploadPartCopy groups are unimplemented. +// Multipart groups of the S3 conformance partition, partitioned empirically +// against the forge-mode stack (see the curation note in README.md). The +// remaining xfail surface: part-level checksums (FIL-620), tagging/object-lock +// /ACL on create (FIL-534/FIL-525), and the UploadPartCopy group (FIL-586). var createMultipartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.CreateMultipartUpload_non_existing_bucket}, + {name: "dir_obj", fn: integration.CreateMultipartUpload_dir_obj}, {name: "long_metadata", fn: integration.CreateMultipartUpload_long_metadata}, + {name: "with_metadata", fn: integration.CreateMultipartUpload_with_metadata}, {name: "with_object_lock_invalid_retention", fn: integration.CreateMultipartUpload_with_object_lock_invalid_retention}, {name: "past_retain_until_date", fn: integration.CreateMultipartUpload_past_retain_until_date}, {name: "invalid_legal_hold", fn: integration.CreateMultipartUpload_invalid_legal_hold}, @@ -30,8 +30,6 @@ var createMultipartPass = []forgeCase{ } var createMultipartXFail = []forgeCase{ - {name: "dir_obj", fn: integration.CreateMultipartUpload_dir_obj}, - {name: "with_metadata", fn: integration.CreateMultipartUpload_with_metadata}, {name: "with_tagging", fn: integration.CreateMultipartUpload_with_tagging}, {name: "with_object_lock", fn: integration.CreateMultipartUpload_with_object_lock}, {name: "with_object_lock_not_enabled", fn: integration.CreateMultipartUpload_with_object_lock_not_enabled}, @@ -41,6 +39,8 @@ var createMultipartXFail = []forgeCase{ var uploadPartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.UploadPart_non_existing_bucket}, {name: "invalid_part_number", fn: integration.UploadPart_invalid_part_number}, + {name: "non_existing_key", fn: integration.UploadPart_non_existing_key}, + {name: "etag_quoting_consistency", fn: integration.UploadPart_etag_quoting_consistency}, {name: "non_existing_mp_upload", fn: integration.UploadPart_non_existing_mp_upload}, {name: "multiple_checksum_headers", fn: integration.UploadPart_multiple_checksum_headers}, {name: "invalid_checksum_header", fn: integration.UploadPart_invalid_checksum_header}, @@ -49,9 +49,6 @@ var uploadPartPass = []forgeCase{ } var uploadPartXFail = []forgeCase{ - // Verifies part ETags via ListParts, which ingot 501s (NotImplemented). - {name: "etag_quoting_consistency", fn: integration.UploadPart_etag_quoting_consistency}, - {name: "non_existing_key", fn: integration.UploadPart_non_existing_key}, {name: "checksum_algorithm_mistmatch_on_initialization", fn: integration.UploadPart_checksum_algorithm_mistmatch_on_initialization}, {name: "checksum_algorithm_mistmatch_on_initialization_with_value", fn: integration.UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value}, {name: "incorrect_checksums", fn: integration.UploadPart_incorrect_checksums}, @@ -84,28 +81,26 @@ var uploadPartCopyXFail = []forgeCase{ } var listPartsPass = []forgeCase{ - {name: "invalid_max_parts", fn: integration.ListParts_invalid_max_parts}, - {name: "invalid_part_number_marker", fn: integration.ListParts_invalid_part_number_marker}, -} - -var listPartsXFail = []forgeCase{ {name: "incorrect_uploadId", fn: integration.ListParts_incorrect_uploadId}, {name: "incorrect_object_key", fn: integration.ListParts_incorrect_object_key}, + {name: "invalid_max_parts", fn: integration.ListParts_invalid_max_parts}, + {name: "invalid_part_number_marker", fn: integration.ListParts_invalid_part_number_marker}, {name: "default_max_parts", fn: integration.ListParts_default_max_parts}, {name: "exceeding_max_parts", fn: integration.ListParts_exceeding_max_parts}, {name: "truncated", fn: integration.ListParts_truncated}, + {name: "success", fn: integration.ListParts_success}, {name: "with_checksums", fn: integration.ListParts_with_checksums}, +} + +// Explicit null-checksum-type echo is FIL-620. +var listPartsXFail = []forgeCase{ {name: "null_checksums", fn: integration.ListParts_null_checksums}, - {name: "success", fn: integration.ListParts_success}, } var listMultipartUploadsPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.ListMultipartUploads_non_existing_bucket}, - {name: "invalid_max_uploads", fn: integration.ListMultipartUploads_invalid_max_uploads}, -} - -var listMultipartUploadsXFail = []forgeCase{ {name: "empty_result", fn: integration.ListMultipartUploads_empty_result}, + {name: "invalid_max_uploads", fn: integration.ListMultipartUploads_invalid_max_uploads}, {name: "max_uploads", fn: integration.ListMultipartUploads_max_uploads}, {name: "exceeding_max_uploads", fn: integration.ListMultipartUploads_exceeding_max_uploads}, {name: "ignore_upload_id_marker", fn: integration.ListMultipartUploads_ignore_upload_id_marker}, @@ -118,53 +113,42 @@ var listMultipartUploadsXFail = []forgeCase{ {name: "with_checksums", fn: integration.ListMultipartUploads_with_checksums}, } +var listMultipartUploadsXFail = []forgeCase{} + var abortMultipartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.AbortMultipartUpload_non_existing_bucket}, {name: "incorrect_uploadId", fn: integration.AbortMultipartUpload_incorrect_uploadId}, {name: "incorrect_object_key", fn: integration.AbortMultipartUpload_incorrect_object_key}, - {name: "success_status_code", fn: integration.AbortMultipartUpload_success_status_code}, -} - -var abortMultipartXFail = []forgeCase{ - // success asserts the aborted upload disappears from ListMultipartUploads, - // which is unimplemented (see listMultipartUploadsXFail); the abort - // lifecycle itself is proven by TestForgeScenarios. {name: "success", fn: integration.AbortMultipartUpload_success}, + {name: "success_status_code", fn: integration.AbortMultipartUpload_success_status_code}, {name: "if_match_initiated_time", fn: integration.AbortMultipartUpload_if_match_initiated_time}, } +var abortMultipartXFail = []forgeCase{} + var completeMultipartPass = []forgeCase{ // upstream function name carries a typo (CompletedMultipartUpload_...). {name: "non_existing_bucket", fn: integration.CompletedMultipartUpload_non_existing_bucket}, {name: "incorrect_part_number", fn: integration.CompleteMultipartUpload_incorrect_part_number}, - {name: "default_content_type", fn: integration.CompleteMultipartUpload_default_content_type, skip: func() string { - return "timing-flaky under hilt: sporadic missing-proofs 500 on part accept, or teardown 409 pending /blob/remove" - }}, + {name: "missing_part_fields", fn: integration.CompleteMultipartUpload_missing_part_fields}, + {name: "invalid_part_number", fn: integration.CompleteMultipartUpload_invalid_part_number}, + {name: "default_content_type", fn: integration.CompleteMultipartUpload_default_content_type}, {name: "invalid_ETag", fn: integration.CompleteMultipartUpload_invalid_ETag}, + {name: "small_upload_size", fn: integration.CompleteMultipartUpload_small_upload_size}, {name: "empty_parts", fn: integration.CompleteMultipartUpload_empty_parts}, {name: "incorrect_parts_order", fn: integration.CompleteMultipartUpload_incorrect_parts_order}, + {name: "mpu_object_size", fn: integration.CompleteMultipartUpload_mpu_object_size}, {name: "invalid_checksum_type", fn: integration.CompleteMultipartUpload_invalid_checksum_type}, {name: "multiple_final_checksums", fn: integration.CompleteMultipartUpload_multiple_final_checksums}, {name: "invalid_final_checksums", fn: integration.CompleteMultipartUpload_invalid_final_checksums}, {name: "invalid_final_composite_checksum", fn: integration.CompleteMultipartUpload_invalid_final_composite_checksum}, - // The happy-path Complete cases are timing-flaky under the hilt/forge - // stack, failing two independent ways run-to-run: (1) the in-request - // part-blob upload sporadically goes out with no proofs for the bucket - // space and 500s ("is not issued by subject and has no proofs" — the - // same missing-proofs symptom the async catalog ship logs), and (2) - // when it succeeds, teardown's DeleteBucket may 409 (accepted part - // blobs register in the space; /blob/remove is unimplemented — see - // versity_object_test.go's teardown-blocked note). Skip hooks (not - // XFail: an XFail row errors on an unexpected pass, so a flaky case - // cannot live there) until the proof plumbing stabilizes and - // /blob/remove lands. The Complete happy path itself is exercised by - // TestForgeScenarios. - {name: "with_metadata", fn: integration.CompleteMultipartUpload_with_metadata, skip: func() string { - return "timing-flaky under hilt: sporadic missing-proofs 500 on part accept, or teardown 409 pending /blob/remove" - }}, - {name: "success", fn: integration.CompleteMultipartUpload_success, skip: func() string { - return "timing-flaky under hilt: sporadic missing-proofs 500 on part accept, or teardown 409 pending /blob/remove" - }}, + {name: "with_metadata", fn: integration.CompleteMultipartUpload_with_metadata}, + {name: "success", fn: integration.CompleteMultipartUpload_success}, + {name: "already_completed", fn: integration.CompleteMultipartUpload_already_completed}, + // The conditional matrix's overwrite chains release superseded blobs: + // needs hilt ≥ #37 (blob.Remove in the write set) and smelt ≥ #19 (the + // piri blob/release delegation) so the releases carry proofs end-to-end. + {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, // racey_success races ten concurrent 25 MiB multipart uploads of one key // under a 30s client deadline — on a host simultaneously running the // smelt stack its outcome depends on load, not S3 semantics, so it is @@ -174,13 +158,9 @@ var completeMultipartPass = []forgeCase{ }}, } +// Part-level / composite-checksum verification is FIL-620; +// racey_data_integrity additionally leans on atomic concurrent overwrites. var completeMultipartXFail = []forgeCase{ - // The missing-ETag part-field subcheck expects 400; ingot 500s. - {name: "missing_part_fields", fn: integration.CompleteMultipartUpload_missing_part_fields}, - {name: "invalid_part_number", fn: integration.CompleteMultipartUpload_invalid_part_number}, - {name: "small_upload_size", fn: integration.CompleteMultipartUpload_small_upload_size}, - {name: "mpu_object_size", fn: integration.CompleteMultipartUpload_mpu_object_size}, - {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, {name: "invalid_checksum_part", fn: integration.CompleteMultipartUpload_invalid_checksum_part}, {name: "multiple_checksum_part", fn: integration.CompleteMultipartUpload_multiple_checksum_part}, {name: "incorrect_checksum_part", fn: integration.CompleteMultipartUpload_incorrect_checksum_part}, @@ -193,6 +173,5 @@ var completeMultipartXFail = []forgeCase{ {name: "checksum_type_mismatch", fn: integration.CompleteMultipartUpload_checksum_type_mismatch}, {name: "should_ignore_the_final_checksum", fn: integration.CompleteMultipartUpload_should_ignore_the_final_checksum}, {name: "should_succeed_without_final_checksum_type", fn: integration.CompleteMultipartUpload_should_succeed_without_final_checksum_type}, - {name: "already_completed", fn: integration.CompleteMultipartUpload_already_completed}, {name: "racey_data_integrity", fn: integration.CompleteMultipartUpload_racey_data_integrity}, } diff --git a/itest/versity_object_test.go b/itest/versity_object_test.go index 71c4f96..e7ca598 100644 --- a/itest/versity_object_test.go +++ b/itest/versity_object_test.go @@ -8,19 +8,20 @@ import ( // Single-object groups of the S3 conformance partition. // -// TEARDOWN-BLOCKED rows: since the hilt (tenant-management) integration, a -// bucket that ever held a non-empty object body cannot be deleted — object -// bodies register blobs in the bucket's space at PUT, DeleteObject's blob -// release is a no-op until sprue/piri implement /blob/remove (see the TODO -// in uploader/blob.go), and hilt's /s3/bucket/delete refuses non-empty -// spaces (409 BucketNotEmpty). Upstream cases delete their bucket in -// teardown, so every case below marked "teardown-blocked" passes its S3 -// assertions and then fails teardown. They sit in the XFail tables so the -// unexpected-pass ratchet flags them for promotion the moment /blob/remove -// lands. +// Historical note: before DeleteObject released network blobs (FIL-588), +// a bucket that ever held a non-empty object body could not be deleted — +// hilt's /s3/bucket/delete refuses non-empty spaces (409 BucketNotEmpty) — +// so every such case failed its bucket-delete teardown and sat in the +// XFail tables as "teardown-blocked". The unexpected-pass ratchet flagged +// them all when the release path landed on this branch; they now live in +// the pass tables. var putObjectPass = []forgeCase{ {name: "checksum_algorithm_and_header_mismatch", fn: integration.PutObject_checksum_algorithm_and_header_mismatch}, + // Overwrites superseding checksummed bodies release their blobs: needs + // hilt ≥ #37 (blob.Remove in the write set) and smelt ≥ #19 (the piri + // blob/release delegation). + {name: "checksums_success", fn: integration.PutObject_checksums_success}, {name: "dir_object_default_checksum", fn: integration.PutObject_dir_object_default_checksum}, {name: "dir_object_checksums_success", fn: integration.PutObject_dir_object_checksums_success}, {name: "incorrect_checksums", fn: integration.PutObject_incorrect_checksums}, @@ -39,17 +40,14 @@ var putObjectPass = []forgeCase{ {name: "past_retain_until_date", fn: integration.PutObject_past_retain_until_date}, {name: "racey_success", fn: integration.PutObject_racey_success}, {name: "special_chars", fn: integration.PutObject_special_chars}, -} - -var putObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. - {name: "checksums_success", fn: integration.PutObject_checksums_success}, {name: "conditional_writes", fn: integration.PutObject_conditional_writes}, {name: "default_checksum", fn: integration.PutObject_default_checksum}, {name: "default_content_type", fn: integration.PutObject_default_content_type}, {name: "success", fn: integration.PutObject_success}, {name: "with_metadata", fn: integration.PutObject_with_metadata}, +} + +var putObjectXFail = []forgeCase{ // The incorrect_md5 subcheck expects 400 InvalidDigest; ingot 500s. {name: "md5", fn: integration.PutObject_md5}, // A metadata-combining re-PUT is denied (403) under the hilt authorize @@ -69,11 +67,6 @@ var getObjectPass = []forgeCase{ {name: "invalid_part_number", fn: integration.GetObject_invalid_part_number}, {name: "non_existing_key", fn: integration.GetObject_non_existing_key}, {name: "zero_len_with_range", fn: integration.GetObject_zero_len_with_range}, -} - -var getObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. {name: "by_range_resp_status", fn: integration.GetObject_by_range_resp_status}, {name: "checksums", fn: integration.GetObject_checksums}, {name: "conditional_reads", fn: integration.GetObject_conditional_reads}, @@ -91,8 +84,15 @@ var getObjectXFail = []forgeCase{ {name: "range_and_part_number", fn: integration.GetObject_range_and_part_number}, {name: "ranged_with_checksum_mode", fn: integration.GetObject_ranged_with_checksum_mode}, {name: "with_range", fn: integration.GetObject_with_range}, +} + +var getObjectXFail = []forgeCase{ + // Directory objects are served with binary/octet-stream instead of + // application/x-directory. {name: "directory_success", fn: integration.GetObject_directory_success}, + // Requires PutBucketPolicy, which ingot 501s (NotImplemented). {name: "overrides_fail_public", fn: integration.GetObject_overrides_fail_public}, + // Asserts object tagging (TagCount), which is unimplemented. {name: "success", fn: integration.GetObject_success}, } @@ -105,11 +105,6 @@ var headObjectPass = []forgeCase{ {name: "overrides_success", fn: integration.HeadObject_overrides_success}, {name: "dir_with_range", fn: integration.HeadObject_dir_with_range}, {name: "zero_len_with_range", fn: integration.HeadObject_zero_len_with_range}, -} - -var headObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. {name: "checksums", fn: integration.HeadObject_checksums}, {name: "conditional_reads", fn: integration.HeadObject_conditional_reads}, {name: "incidental_dir_object", fn: integration.HeadObject_incidental_dir_object}, @@ -124,7 +119,12 @@ var headObjectXFail = []forgeCase{ {name: "by_range_resp_status", fn: integration.HeadObject_by_range_resp_status}, {name: "ranged_with_checksum_mode", fn: integration.HeadObject_ranged_with_checksum_mode}, {name: "with_range", fn: integration.HeadObject_with_range}, +} + +var headObjectXFail = []forgeCase{ + // Requires PutBucketPolicy, which ingot 501s (NotImplemented). {name: "overrides_fail_public", fn: integration.HeadObject_overrides_fail_public}, + // Asserts object tagging (TagCount), which is unimplemented. {name: "success", fn: integration.HeadObject_success}, } @@ -137,12 +137,10 @@ var deleteObjectPass = []forgeCase{ {name: "non_existing_object", fn: integration.DeleteObject_non_existing_object}, {name: "success", fn: integration.DeleteObject_success}, {name: "success_status_code", fn: integration.DeleteObject_success_status_code}, + {name: "conditional_writes", fn: integration.DeleteObject_conditional_writes}, } var deleteObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. - {name: "conditional_writes", fn: integration.DeleteObject_conditional_writes}, // The ExpectedBucketOwner-matching delete is denied (403) under the // hilt authorize flow (ownership is the tenant's did:plc, not the // account the case expects). @@ -162,13 +160,6 @@ var copyObjectPass = []forgeCase{ {name: "to_itself_with_new_metadata", fn: integration.CopyObject_to_itself_with_new_metadata}, {name: "invalid_tagging_directive", fn: integration.CopyObject_invalid_tagging_directive}, {name: "invalid_checksum_algorithm", fn: integration.CopyObject_invalid_checksum_algorithm}, -} - -// Observed failing against the forge stack: multi-account semantics, tagging, -// object-lock, and checksum-on-copy are unimplemented surface. -var copyObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. {name: "success", fn: integration.CopyObject_success}, {name: "copy_source_starting_with_slash", fn: integration.CopyObject_copy_source_starting_with_slash}, {name: "default_content_type_with_replace_metadata", fn: integration.CopyObject_default_content_type_with_replace_metadata}, @@ -182,6 +173,11 @@ var copyObjectXFail = []forgeCase{ {name: "invalid_legal_hold", fn: integration.CopyObject_invalid_legal_hold}, {name: "invalid_object_lock_mode", fn: integration.CopyObject_invalid_object_lock_mode}, {name: "invalid_website_redirect_location", fn: integration.CopyObject_invalid_website_redirect_location}, +} + +// Observed failing against the forge stack: multi-account semantics, tagging, +// object-lock, and checksum-on-copy are unimplemented surface. +var copyObjectXFail = []forgeCase{ {name: "not_owned_source_bucket", fn: integration.CopyObject_not_owned_source_bucket}, {name: "should_replace_tagging", fn: integration.CopyObject_should_replace_tagging}, {name: "should_copy_tagging", fn: integration.CopyObject_should_copy_tagging}, diff --git a/migrations/sql/00007_multipart_listing.sql b/migrations/sql/00007_multipart_listing.sql new file mode 100644 index 0000000..b79294f --- /dev/null +++ b/migrations/sql/00007_multipart_listing.sql @@ -0,0 +1,48 @@ +-- +goose Up +-- Multipart listing + completion idempotency (FIL-520): +-- * multipart_parts.created_at backs ListParts' per-part LastModified; +-- sessions already carry created_at for ListMultipartUploads' Initiated. +-- * Sessions gain the standard HTTP metadata headers CreateMultipartUpload +-- may carry, so Complete writes them into the manifest exactly like a +-- single-shot PUT does. +-- * A 'completed' session state: Complete now retains the session (and its +-- parts) instead of deleting it, so a duplicate CompleteMultipartUpload +-- with identical parts is idempotent per S3. The abandoned-session +-- sweeper reaps completed rows after the TTL. + +ALTER TABLE ingot.multipart_parts + ADD COLUMN created_at timestamptz NOT NULL DEFAULT now(); + +ALTER TABLE ingot.multipart_sessions + ADD COLUMN content_encoding text, + ADD COLUMN content_disposition text, + ADD COLUMN content_language text, + ADD COLUMN cache_control text, + ADD COLUMN expires text, + ADD COLUMN website_redirect_location text, + ADD COLUMN checksum_algorithm text, + ADD COLUMN checksum_type text; + +ALTER TABLE ingot.multipart_sessions + DROP CONSTRAINT multipart_sessions_state_check, + ADD CONSTRAINT multipart_sessions_state_check + CHECK (state IN ('open','completing','aborting','completed')); + +-- +goose Down +ALTER TABLE ingot.multipart_sessions + DROP CONSTRAINT multipart_sessions_state_check, + ADD CONSTRAINT multipart_sessions_state_check + CHECK (state IN ('open','completing','aborting')); + +ALTER TABLE ingot.multipart_sessions + DROP COLUMN content_encoding, + DROP COLUMN content_disposition, + DROP COLUMN content_language, + DROP COLUMN cache_control, + DROP COLUMN expires, + DROP COLUMN website_redirect_location, + DROP COLUMN checksum_algorithm, + DROP COLUMN checksum_type; + +ALTER TABLE ingot.multipart_parts + DROP COLUMN created_at; diff --git a/migrations/sql/00008_deferred_accept.sql b/migrations/sql/00008_deferred_accept.sql new file mode 100644 index 0000000..98b4d4b --- /dev/null +++ b/migrations/sql/00008_deferred_accept.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- Deferred-accept multipart (§7.2): a part's blobs upload to the provider at +-- UploadPart but the /http/put conclude — which triggers /blob/accept — is +-- deferred to Complete. blob_parks holds the state needed to conclude (or +-- unallocate) later. put_invocation is the sealed /http/put invocation whose +-- metadata embeds the derived signer keys; rows are deleted promptly at +-- conclude/unallocate. Keyed globally by digest, like upload_intents: +-- content-addressed dedup shares a park across sessions and parts. +CREATE TABLE ingot.blob_parks ( + digest bytea PRIMARY KEY, + add_task bytea NOT NULL, -- /space/blob/add task CID (unallocate cause) + accept_task bytea NOT NULL, -- /blob/accept task CID (conclude poll target) + put_invocation bytea NOT NULL, -- sealed /http/put invocation + size bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- +goose Down +DROP TABLE ingot.blob_parks; diff --git a/module.go b/module.go index a563aff..23b97ea 100644 --- a/module.go +++ b/module.go @@ -136,6 +136,7 @@ type serverParams struct { Reader blockstore.BlockReader Uploader uploader.Uploader BodyUploader uploader.BodyUploader + Deferred uploader.DeferredBodyUploader Remover uploader.BlobRemover BucketAuthority bucketauthority.BucketAuthority Registry registry.Registry @@ -145,6 +146,7 @@ type serverParams struct { BlobRefs registry.BlobRefStore GC registry.GCStore Multipart registry.MultipartStore + Parks registry.ParkStore Meta logstore.Meta // IAM authenticates non-root access keys. IAM auth.IAMService `optional:"true"` @@ -177,6 +179,7 @@ func registerServerLifecycle(lc fx.Lifecycle, p serverParams) { BaseBlockReader: p.Reader, Uploader: p.Uploader, BodyUploader: p.BodyUploader, + Deferred: p.Deferred, Remover: p.Remover, Authority: p.BucketAuthority, Registry: p.Registry, @@ -186,6 +189,7 @@ func registerServerLifecycle(lc fx.Lifecycle, p serverParams) { BlobRefs: p.BlobRefs, GC: p.GC, Multipart: p.Multipart, + Parks: p.Parks, Meta: p.Meta, IAM: p.IAM, }) @@ -301,6 +305,7 @@ type registryResult struct { BlobRefs registry.BlobRefStore GC registry.GCStore Multipart registry.MultipartStore + Parks registry.ParkStore Meta logstore.Meta } @@ -313,7 +318,7 @@ type registryResult struct { // needs hilt_url/hilt_did configured. func provideRegistry(pool *pgxpool.Pool) registryResult { pg := registry.NewPostgres(pool) - return registryResult{Registry: pg, Intents: pg, Locations: pg, Inclusions: pg, BlobRefs: pg, GC: pg, Multipart: pg, Meta: pg} + return registryResult{Registry: pg, Intents: pg, Locations: pg, Inclusions: pg, BlobRefs: pg, GC: pg, Multipart: pg, Parks: pg, Meta: pg} } // migrationHookOut feeds the migration PreStartHook into the "ingot_prestart" @@ -367,6 +372,7 @@ type uploaderResult struct { Uploader uploader.Uploader BodyUploader uploader.BodyUploader + Deferred uploader.DeferredBodyUploader Remover uploader.BlobRemover } @@ -379,5 +385,5 @@ func provideUploader(c *forgeclient.Client, logger *zap.Logger) (uploaderResult, if err != nil { return uploaderResult{}, err } - return uploaderResult{Uploader: f, BodyUploader: f, Remover: f}, nil + return uploaderResult{Uploader: f, BodyUploader: f, Deferred: f, Remover: f}, nil } diff --git a/registry/postgres_live_test.go b/registry/postgres_live_test.go index 30825e5..8de1576 100644 --- a/registry/postgres_live_test.go +++ b/registry/postgres_live_test.go @@ -152,6 +152,37 @@ func TestPostgresStores_Live(t *testing.T) { } }) + t.Run("park round trip", func(t *testing.T) { + park := registry.BlobPark{ + Digest: digest, + AddTask: []byte{0x01, 0x02}, + AcceptTask: []byte{0x03, 0x04}, + PutInvocation: []byte("sealed-inv"), + Size: 42, + } + if err := r.PutPark(ctx, park); err != nil { + t.Fatalf("PutPark: %v", err) + } + got, err := r.GetPark(ctx, digest) + if err != nil || string(got.PutInvocation) != "sealed-inv" || got.Size != 42 { + t.Fatalf("GetPark = %+v, err %v", got, err) + } + // Upsert replaces in place. + park.Size = 43 + if err := r.PutPark(ctx, park); err != nil { + t.Fatalf("PutPark (upsert): %v", err) + } + if got, err := r.GetPark(ctx, digest); err != nil || got.Size != 43 { + t.Fatalf("GetPark after upsert = %+v, err %v", got, err) + } + if err := r.DeletePark(ctx, digest); err != nil { + t.Fatalf("DeletePark: %v", err) + } + if _, err := r.GetPark(ctx, digest); err != registry.ErrNotFound { + t.Fatalf("GetPark after delete = %v, want ErrNotFound", err) + } + }) + t.Run("multipart session parts latch metadata", func(t *testing.T) { const id = "upl-1" meta := map[string]string{"x-amz-meta-foo": "bar"} @@ -197,6 +228,71 @@ func TestPostgresStores_Live(t *testing.T) { } }) + t.Run("multipart listing sweeper and part refs", func(t *testing.T) { + mk := func(id, key string) { + t.Helper() + if err := r.CreateSession(ctx, registry.MultipartSession{ + UploadID: id, Bucket: "b", ObjectKey: key, + ContentEncoding: "testenc", ChecksumAlgorithm: "CRC32", ChecksumType: "FULL_OBJECT", + }); err != nil { + t.Fatalf("CreateSession %s: %v", id, err) + } + } + mk("ls-2", "zeta") + mk("ls-1", "alpha") + mk("ls-3", "alpha") // same key, created after ls-1 + + // New session columns round-trip. + s, err := r.GetSession(ctx, "ls-1") + if err != nil || s.ContentEncoding != "testenc" || s.ChecksumAlgorithm != "CRC32" || + s.ChecksumType != "FULL_OBJECT" || s.CreatedAt.IsZero() { + t.Fatalf("GetSession new columns = %+v, err %v", s, err) + } + + // ListSessions: (object_key, created_at, upload_id) order. + sessions, err := r.ListSessions(ctx, "b") + if err != nil || len(sessions) != 3 || + sessions[0].UploadID != "ls-1" || sessions[1].UploadID != "ls-3" || sessions[2].UploadID != "ls-2" { + ids := make([]string, len(sessions)) + for i, x := range sessions { + ids[i] = x.UploadID + } + t.Fatalf("ListSessions order = %v, err %v (want [ls-1 ls-3 ls-2])", ids, err) + } + + // ListStaleSessions: cutoff in the past excludes them, future includes. + if stale, err := r.ListStaleSessions(ctx, registry.SessionOpen, time.Now().Add(-time.Hour)); err != nil || len(stale) != 0 { + t.Fatalf("ListStaleSessions past cutoff = %d, err %v (want 0)", len(stale), err) + } + if stale, err := r.ListStaleSessions(ctx, registry.SessionOpen, time.Now().Add(time.Hour)); err != nil || len(stale) != 3 { + t.Fatalf("ListStaleSessions future cutoff = %d, err %v (want 3)", len(stale), err) + } + + // CountPartRefs: bytea[] ANY-match across sessions, excluding one. + shared := []byte{0xee, 0x01} + if err := r.PutPart(ctx, registry.MultipartPart{UploadID: "ls-1", PartNumber: 1, ETagMD5: []byte{1}, Size: 1, BlobDigests: [][]byte{shared}}); err != nil { + t.Fatalf("PutPart ls-1: %v", err) + } + if err := r.PutPart(ctx, registry.MultipartPart{UploadID: "ls-2", PartNumber: 1, ETagMD5: []byte{2}, Size: 1, BlobDigests: [][]byte{shared, {0xee, 0x02}}}); err != nil { + t.Fatalf("PutPart ls-2: %v", err) + } + if n, err := r.CountPartRefs(ctx, shared, "ls-1"); err != nil || n != 1 { + t.Fatalf("CountPartRefs(shared, exclude ls-1) = %d, err %v (want 1)", n, err) + } + if n, err := r.CountPartRefs(ctx, []byte{0xee, 0x02}, "ls-2"); err != nil || n != 0 { + t.Fatalf("CountPartRefs(unique, exclude owner) = %d, err %v (want 0)", n, err) + } + + // 'completed' passes the widened state CHECK constraint. + if won, err := r.LatchSession(ctx, "ls-1", registry.SessionOpen, registry.SessionCompleted); err != nil || !won { + t.Fatalf("latch to completed won=%v err=%v", won, err) + } + + for _, id := range []string{"ls-1", "ls-2", "ls-3"} { + _ = r.DeleteSession(ctx, id) + } + }) + t.Run("gc candidate idempotent", func(t *testing.T) { if err := r.AddGCCandidate(ctx, digest, "b"); err != nil { t.Fatalf("AddGCCandidate: %v", err) diff --git a/registry/stores.go b/registry/stores.go index c6d861b..60da750 100644 --- a/registry/stores.go +++ b/registry/stores.go @@ -2,6 +2,7 @@ package registry import ( "context" + "time" "github.com/fil-forge/ucantone/did" ) @@ -32,6 +33,10 @@ const ( SessionOpen = "open" SessionCompleting = "completing" SessionAborting = "aborting" + // SessionCompleted: the object committed; the session and its parts are + // retained so a duplicate CompleteMultipartUpload with identical parts is + // idempotent (S3 semantics). Reaped by the abandoned-session sweeper. + SessionCompleted = "completed" ) // multipart_parts.state values (§7.2). @@ -86,14 +91,28 @@ type BlobInclusion struct { RangeEnd int64 // inclusive } -// MultipartSession is one row of ingot.multipart_sessions. +// MultipartSession is one row of ingot.multipart_sessions. The HTTP metadata +// headers (ContentEncoding..Expires) are captured at CreateMultipartUpload so +// Complete can write them into the manifest exactly like a single-shot PUT. type MultipartSession struct { - UploadID string - Bucket string - ObjectKey string - State string - ContentType string - Metadata map[string]string + UploadID string + Bucket string + ObjectKey string + State string + ContentType string + ContentEncoding string + ContentDisposition string + ContentLanguage string + CacheControl string + Expires string + WebsiteRedirectLocation string + // ChecksumAlgorithm/ChecksumType are the x-amz-checksum-* declarations from + // CreateMultipartUpload, echoed by ListMultipartUploads. Per-part checksum + // computation is separate (FIL-620). + ChecksumAlgorithm string + ChecksumType string + Metadata map[string]string + CreatedAt time.Time } // MultipartPart is one row of ingot.multipart_parts. BlobDigests is the @@ -106,6 +125,7 @@ type MultipartPart struct { Size int64 BlobDigests [][]byte State string + CreatedAt time.Time } // BlobRefStore is the reverse reference index (§5, §6). A commit adds a @@ -139,6 +159,31 @@ type LocationStore interface { DeleteLocation(ctx context.Context, space did.DID, digest []byte) error } +// BlobPark is one row of ingot.blob_parks: the persistable state of a blob +// that is durable on its provider but not yet accepted (multipart's deferred +// conclude, §7.2). AddTask/AcceptTask are the /blob/add and +// /blob/accept task CIDs; PutInvocation is the sealed /http/put invocation +// whose metadata carries the derived signer keys needed to conclude — +// sensitive, deleted at conclude/abort. Keyed globally by Digest (like +// upload_intents: content-addressed dedup shares parks across sessions). +type BlobPark struct { + Digest []byte + AddTask []byte // cid bytes + AcceptTask []byte // cid bytes + PutInvocation []byte + Size int64 + CreatedAt time.Time +} + +// ParkStore persists deferred-accept park state between UploadPart and +// Complete/Abort (§7.2). +type ParkStore interface { + PutPark(ctx context.Context, p BlobPark) error + // GetPark returns ErrNotFound when digest has no park row. + GetPark(ctx context.Context, digest []byte) (*BlobPark, error) + DeletePark(ctx context.Context, digest []byte) error +} + // InclusionStore is the local shard-inclusion table (§8): block digest → // (shard digest, byte range) for every block of a shipped catalog segment, // written by the flush path before the segment is marked shipped. Resolved on @@ -163,6 +208,18 @@ type MultipartStore interface { DeleteSession(ctx context.Context, uploadID string) error PutPart(ctx context.Context, p MultipartPart) error ListParts(ctx context.Context, uploadID string) ([]MultipartPart, error) + // ListSessions returns bucket's sessions ordered by (object_key, created_at, + // upload_id) — the S3 ListMultipartUploads presentation order. Filtering + // (prefix/markers/max) happens in the handler; in-flight session counts are + // small. + ListSessions(ctx context.Context, bucket string) ([]MultipartSession, error) + // ListStaleSessions returns sessions in `state` created before cutoff, for + // the abandoned-upload sweeper. + ListStaleSessions(ctx context.Context, state string, cutoff time.Time) ([]MultipartSession, error) + // CountPartRefs returns how many parts OUTSIDE excludeUploadID reference + // digest — the shared-blob guard for abort/supersede spool cleanup + // (content-addressed part blobs may be deduped across sessions). + CountPartRefs(ctx context.Context, digest []byte, excludeUploadID string) (int, error) } // GCStore records superseded MST node CIDs (§4). Write-only this iteration. diff --git a/registry/stores_postgres.go b/registry/stores_postgres.go index a2832b3..3400b94 100644 --- a/registry/stores_postgres.go +++ b/registry/stores_postgres.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "time" "github.com/fil-forge/ucantone/did" "github.com/jackc/pgx/v5" @@ -180,6 +181,46 @@ func (r *Postgres) DeleteLocation(ctx context.Context, space did.DID, digest []b return nil } +// ParkStore ================================================================== + +func (r *Postgres) PutPark(ctx context.Context, p BlobPark) error { + _, err := r.pool.Exec(ctx, + `INSERT INTO ingot.blob_parks (digest, add_task, accept_task, put_invocation, size) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (digest) DO UPDATE + SET add_task = EXCLUDED.add_task, accept_task = EXCLUDED.accept_task, + put_invocation = EXCLUDED.put_invocation, size = EXCLUDED.size`, + p.Digest, p.AddTask, p.AcceptTask, p.PutInvocation, p.Size) + if err != nil { + return fmt.Errorf("registry: put park: %w", err) + } + return nil +} + +func (r *Postgres) GetPark(ctx context.Context, digest []byte) (*BlobPark, error) { + park := &BlobPark{Digest: digest} + err := r.pool.QueryRow(ctx, + `SELECT add_task, accept_task, put_invocation, size, created_at + FROM ingot.blob_parks WHERE digest = $1`, + digest).Scan(&park.AddTask, &park.AcceptTask, &park.PutInvocation, &park.Size, &park.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("registry: get park: %w", err) + } + return park, nil +} + +func (r *Postgres) DeletePark(ctx context.Context, digest []byte) error { + _, err := r.pool.Exec(ctx, + `DELETE FROM ingot.blob_parks WHERE digest = $1`, digest) + if err != nil { + return fmt.Errorf("registry: delete park: %w", err) + } + return nil +} + // InclusionStore ============================================================= func (r *Postgres) PutInclusions(ctx context.Context, incs []BlobInclusion) error { @@ -230,9 +271,15 @@ func (r *Postgres) CreateSession(ctx context.Context, s MultipartSession) error return err } _, err = r.pool.Exec(ctx, - `INSERT INTO ingot.multipart_sessions (upload_id, bucket, object_key, state, content_type, metadata) - VALUES ($1, $2, $3, $4, $5, $6)`, - s.UploadID, s.Bucket, s.ObjectKey, state, nullString(s.ContentType), meta) + `INSERT INTO ingot.multipart_sessions + (upload_id, bucket, object_key, state, content_type, metadata, + content_encoding, content_disposition, content_language, cache_control, expires, + website_redirect_location, checksum_algorithm, checksum_type) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`, + s.UploadID, s.Bucket, s.ObjectKey, state, nullString(s.ContentType), meta, + nullString(s.ContentEncoding), nullString(s.ContentDisposition), + nullString(s.ContentLanguage), nullString(s.CacheControl), nullString(s.Expires), + nullString(s.WebsiteRedirectLocation), nullString(s.ChecksumAlgorithm), nullString(s.ChecksumType)) if err != nil { var pgErr *pgconn.PgError if errors.As(err, &pgErr) && pgErr.Code == uniqueViolation { @@ -244,22 +291,47 @@ func (r *Postgres) CreateSession(ctx context.Context, s MultipartSession) error } func (r *Postgres) GetSession(ctx context.Context, uploadID string) (*MultipartSession, error) { - s := &MultipartSession{UploadID: uploadID} - var contentType *string - var meta []byte - err := r.pool.QueryRow(ctx, - `SELECT bucket, object_key, state, content_type, metadata + row := r.pool.QueryRow(ctx, + `SELECT upload_id, bucket, object_key, state, content_type, metadata, created_at, + content_encoding, content_disposition, content_language, cache_control, expires, + website_redirect_location, checksum_algorithm, checksum_type FROM ingot.multipart_sessions WHERE upload_id = $1`, - uploadID).Scan(&s.Bucket, &s.ObjectKey, &s.State, &contentType, &meta) + uploadID) + s, err := scanSession(row) if errors.Is(err, pgx.ErrNoRows) { return nil, ErrNotFound } if err != nil { return nil, fmt.Errorf("registry: get session: %w", err) } - if contentType != nil { - s.ContentType = *contentType + return s, nil +} + +// scanSession scans one multipart_sessions row in the canonical column order +// (see GetSession/ListSessions selects). +func scanSession(row pgx.Row) (*MultipartSession, error) { + s := &MultipartSession{} + var contentType, ce, cd, cl, cc, exp, wrl, ckAlgo, ckType *string + var meta []byte + err := row.Scan(&s.UploadID, &s.Bucket, &s.ObjectKey, &s.State, &contentType, &meta, &s.CreatedAt, + &ce, &cd, &cl, &cc, &exp, &wrl, &ckAlgo, &ckType) + if err != nil { + return nil, err + } + setIfNotNil := func(dst *string, src *string) { + if src != nil { + *dst = *src + } } + setIfNotNil(&s.ContentType, contentType) + setIfNotNil(&s.ContentEncoding, ce) + setIfNotNil(&s.ContentDisposition, cd) + setIfNotNil(&s.ContentLanguage, cl) + setIfNotNil(&s.CacheControl, cc) + setIfNotNil(&s.Expires, exp) + setIfNotNil(&s.WebsiteRedirectLocation, wrl) + setIfNotNil(&s.ChecksumAlgorithm, ckAlgo) + setIfNotNil(&s.ChecksumType, ckType) if s.Metadata, err = unmarshalMetadata(meta); err != nil { return nil, err } @@ -304,7 +376,7 @@ func (r *Postgres) PutPart(ctx context.Context, p MultipartPart) error { func (r *Postgres) ListParts(ctx context.Context, uploadID string) ([]MultipartPart, error) { rows, err := r.pool.Query(ctx, - `SELECT part_number, etag_md5, size, blob_digests, state + `SELECT part_number, etag_md5, size, blob_digests, state, created_at FROM ingot.multipart_parts WHERE upload_id = $1 ORDER BY part_number ASC`, uploadID) if err != nil { @@ -315,7 +387,7 @@ func (r *Postgres) ListParts(ctx context.Context, uploadID string) ([]MultipartP var out []MultipartPart for rows.Next() { p := MultipartPart{UploadID: uploadID} - if err := rows.Scan(&p.PartNumber, &p.ETagMD5, &p.Size, &p.BlobDigests, &p.State); err != nil { + if err := rows.Scan(&p.PartNumber, &p.ETagMD5, &p.Size, &p.BlobDigests, &p.State, &p.CreatedAt); err != nil { return nil, fmt.Errorf("registry: list parts scan: %w", err) } out = append(out, p) @@ -326,6 +398,72 @@ func (r *Postgres) ListParts(ctx context.Context, uploadID string) ([]MultipartP return out, nil } +func (r *Postgres) ListSessions(ctx context.Context, bucket string) ([]MultipartSession, error) { + rows, err := r.pool.Query(ctx, + `SELECT upload_id, bucket, object_key, state, content_type, metadata, created_at, + content_encoding, content_disposition, content_language, cache_control, expires, + website_redirect_location, checksum_algorithm, checksum_type + FROM ingot.multipart_sessions WHERE bucket = $1 + ORDER BY object_key ASC, created_at ASC, upload_id ASC`, + bucket) + if err != nil { + return nil, fmt.Errorf("registry: list sessions: %w", err) + } + defer rows.Close() + + var out []MultipartSession + for rows.Next() { + s, err := scanSession(rows) + if err != nil { + return nil, fmt.Errorf("registry: list sessions scan: %w", err) + } + out = append(out, *s) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("registry: list sessions rows: %w", err) + } + return out, nil +} + +func (r *Postgres) ListStaleSessions(ctx context.Context, state string, cutoff time.Time) ([]MultipartSession, error) { + rows, err := r.pool.Query(ctx, + `SELECT upload_id, bucket, object_key, state, content_type, metadata, created_at, + content_encoding, content_disposition, content_language, cache_control, expires, + website_redirect_location, checksum_algorithm, checksum_type + FROM ingot.multipart_sessions WHERE state = $1 AND created_at < $2 + ORDER BY created_at ASC`, + state, cutoff) + if err != nil { + return nil, fmt.Errorf("registry: list stale sessions: %w", err) + } + defer rows.Close() + + var out []MultipartSession + for rows.Next() { + s, err := scanSession(rows) + if err != nil { + return nil, fmt.Errorf("registry: list stale sessions scan: %w", err) + } + out = append(out, *s) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("registry: list stale sessions rows: %w", err) + } + return out, nil +} + +func (r *Postgres) CountPartRefs(ctx context.Context, digest []byte, excludeUploadID string) (int, error) { + var n int + err := r.pool.QueryRow(ctx, + `SELECT count(*) FROM ingot.multipart_parts + WHERE $1 = ANY(blob_digests) AND upload_id <> $2`, + digest, excludeUploadID).Scan(&n) + if err != nil { + return 0, fmt.Errorf("registry: count part refs: %w", err) + } + return n, nil +} + // GCStore ==================================================================== func (r *Postgres) AddGCCandidate(ctx context.Context, cidBytes []byte, bucket string) error { diff --git a/s3frontend/backend.go b/s3frontend/backend.go index 14e894e..3e6abf0 100644 --- a/s3frontend/backend.go +++ b/s3frontend/backend.go @@ -54,6 +54,8 @@ type Backend struct { log blockstore.Log spool *blockstore.Spool uploader uploader.BodyUploader + deferred uploader.DeferredBodyUploader + parks registry.ParkStore remover uploader.BlobRemover logger *zap.Logger @@ -96,6 +98,11 @@ type Deps struct { // accept) synchronously, before the manifest commits. Remover releases a // space's claim on a blob when its last reference is dropped. Uploader uploader.BodyUploader + // Deferred extends Uploader for multipart's deferred accept + // (WithConclude(false), then ConcludeBlob/AbortBlob); Parks persists + // park state between UploadPart and Complete/Abort. + Deferred uploader.DeferredBodyUploader + Parks registry.ParkStore Remover uploader.BlobRemover // MaxBlobSize is the coarse-split blob ceiling (0 → bucket default). @@ -144,6 +151,8 @@ func New(d Deps) *Backend { log: d.Log, spool: d.Spool, uploader: d.Uploader, + deferred: d.Deferred, + parks: d.Parks, remover: d.Remover, logger: logger, maxBlobSize: d.MaxBlobSize, diff --git a/s3frontend/bucket.go b/s3frontend/bucket.go index c6326ed..2eae185 100644 --- a/s3frontend/bucket.go +++ b/s3frontend/bucket.go @@ -222,6 +222,27 @@ func (b *Backend) DeleteBucket(ctx context.Context, name string) error { } } + // In-flight multipart uploads do not block deletion (the upstream + // conformance contract's teardown deletes buckets without aborting + // them): abort any open sessions, releasing their parked part blobs + // from the space, before asking hilt to delete the space — which + // refuses while the space still holds blob registrations. + sessions, err := b.multipart.ListSessions(ctx, name) + if err != nil { + return fmt.Errorf("s3frontend: delete bucket: list mp sessions: %w", err) + } + aborted := 0 + for _, s := range sessions { + if s.State == registry.SessionOpen { + b.abortOpenSession(ctx, st.Space, s) + aborted++ + } + } + if aborted > 0 { + b.logger.Info("delete bucket: aborted in-flight multipart sessions", + zap.String("bucket", name), zap.Int("aborted", aborted), zap.Int("total", len(sessions))) + } + req, ok := reqscope.Request(ctx) if !ok { return errors.New("s3frontend: delete bucket: no request in context") diff --git a/s3frontend/multipart.go b/s3frontend/multipart.go index 3722487..2263155 100644 --- a/s3frontend/multipart.go +++ b/s3frontend/multipart.go @@ -12,17 +12,27 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/fil-forge/ucantone/did" "github.com/fil-forge/versitygw/backend" "github.com/fil-forge/versitygw/s3err" "github.com/fil-forge/versitygw/s3response" + "github.com/google/uuid" "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" + "go.uber.org/zap" msbucket "github.com/fil-forge/ingot/bucket" "github.com/fil-forge/ingot/bucketop" + "github.com/fil-forge/ingot/internal/reqscope" "github.com/fil-forge/ingot/mst" "github.com/fil-forge/ingot/registry" + "github.com/fil-forge/ingot/uploader" ) +// defaultMaxListing is the S3 default and cap for max-parts / max-uploads. +const defaultMaxListing = 1000 + // newUploadID returns a random 128-bit hex upload id. func newUploadID() (string, error) { var b [16]byte @@ -33,8 +43,9 @@ func newUploadID() (string, error) { } // CreateMultipartUpload opens a multipart session: it records the destination -// bucket/key plus the content-type and user metadata so Complete can write the -// manifest without the client resupplying them, and returns the upload id. +// bucket/key plus the content-type, the passthrough HTTP metadata headers, and +// user metadata so Complete can write the manifest without the client +// resupplying them, and returns the upload id. func (b *Backend) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { if input.Bucket == nil || input.Key == nil { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -43,6 +54,11 @@ func (b *Backend) CreateMultipartUpload(ctx context.Context, input s3response.Cr if !mst.IsValidKey(key) { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidRequest) } + // A directory object (trailing "/") is zero-length by definition; a + // multipart upload to one necessarily carries data. + if strings.HasSuffix(key, "/") { + return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData) + } if _, err := b.reg.Get(ctx, bucket); err != nil { if errors.Is(err, registry.ErrNotFound) { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -58,36 +74,97 @@ func (b *Backend) CreateMultipartUpload(ctx context.Context, input s3response.Cr ct = "application/octet-stream" } if err := b.multipart.CreateSession(ctx, registry.MultipartSession{ - UploadID: uploadID, - Bucket: bucket, - ObjectKey: key, - State: registry.SessionOpen, - ContentType: ct, - Metadata: input.Metadata, + UploadID: uploadID, + Bucket: bucket, + ObjectKey: key, + State: registry.SessionOpen, + ContentType: ct, + ContentEncoding: backend.GetStringFromPtr(input.ContentEncoding), + ContentDisposition: backend.GetStringFromPtr(input.ContentDisposition), + ContentLanguage: backend.GetStringFromPtr(input.ContentLanguage), + CacheControl: backend.GetStringFromPtr(input.CacheControl), + Expires: backend.GetStringFromPtr(input.Expires), + WebsiteRedirectLocation: backend.GetStringFromPtr(input.WebsiteRedirectLocation), + ChecksumAlgorithm: string(input.ChecksumAlgorithm), + ChecksumType: string(input.ChecksumType), + Metadata: input.Metadata, }); err != nil { return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("s3frontend: create session: %w", err) } return s3response.InitiateMultipartUploadResult{Bucket: bucket, Key: key, UploadId: uploadID}, nil } -// UploadPart ingests one part: it coarse-splits the part body into blobs and -// spools each to local disk (recording upload_intents), then records the part -// (its ordered blob digests, md5, size). The part's blobs are NOT uploaded to -// Forge yet — that is deferred to Complete (so an Abort only ever cleans up -// local state). The part ETag is the hex md5 of the part bytes. +// openSession fetches uploadID's session and maps anything that is not an +// in-flight upload for (bucket, key) to NoSuchUpload: unknown id, a key that +// doesn't match the session's, or a session no longer open (completed uploads +// are retained for Complete idempotency but are gone as far as the other +// multipart operations are concerned). +func (b *Backend) openSession(ctx context.Context, uploadID string, key *string) (*registry.MultipartSession, error) { + sess, err := b.multipart.GetSession(ctx, uploadID) + if err != nil { + if errors.Is(err, registry.ErrNotFound) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + return nil, fmt.Errorf("s3frontend: get session: %w", err) + } + if key != nil && *key != sess.ObjectKey { + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + if sess.State != registry.SessionOpen { + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + return sess, nil +} + +// bucketSpace resolves the Forge space owning bucketName. Every network-side +// blob action is space-scoped (the space is the UCAN subject), so the +// multipart paths resolve it once per operation from the bucket registry. +func (b *Backend) bucketSpace(ctx context.Context, bucketName string) (did.DID, error) { + st, err := b.reg.Get(ctx, bucketName) + if err != nil { + if errors.Is(err, registry.ErrNotFound) { + return did.Undef, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + return did.Undef, fmt.Errorf("s3frontend: resolve bucket space: %w", err) + } + return st.Space, nil +} + +// UploadPart ingests one part: it coarse-splits the part body into blobs, +// spools each to local disk (recording upload_intents), records the part +// (its ordered blob digests, md5, size), and uploads each blob to its +// provider — PARKED, not accepted: the /http/put conclude that triggers +// /blob/accept is deferred to Complete, so the bytes are durable but stay +// out of the PDP pipeline, and an Abort unwinds them with /blob/abort +// (§7.2). Re-uploading a part number supersedes the prior part; the +// superseded part's now-unreferenced blobs are dropped from the spool and +// rejected. The part ETag is the hex md5 of the part bytes. func (b *Backend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { if input.Bucket == nil || input.Key == nil || input.UploadId == nil || input.PartNumber == nil { return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) } uploadID := *input.UploadId - sess, err := b.multipart.GetSession(ctx, uploadID) + sess, err := b.openSession(ctx, uploadID, input.Key) if err != nil { - if errors.Is(err, registry.ErrNotFound) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + return nil, err + } + + // Capture the superseded part's blobs (if any) before overwriting, so + // last-write-wins doesn't strand its spool files. + var superseded [][]byte + if prior, err := b.multipart.ListParts(ctx, uploadID); err == nil { + for _, p := range prior { + if p.PartNumber == int(*input.PartNumber) { + superseded = p.BlobDigests + break + } } - return nil, fmt.Errorf("s3frontend: upload part: %w", err) } + space, err := b.bucketSpace(ctx, sess.Bucket) + if err != nil { + return nil, err + } body, err := b.splitSpool(ctx, sess.Bucket, input.Body) if err != nil { return nil, fmt.Errorf("s3frontend: upload part ingest: %w", err) @@ -102,6 +179,16 @@ func (b *Backend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s }); err != nil { return nil, fmt.Errorf("s3frontend: record part: %w", err) } + // Park the part's blobs on their providers before returning 200 — the + // part is durable on the network as soon as the client sees success. + // The part row is recorded first so a crash mid-park leaves re-drivable + // spooled intents. + if err := b.parkBlobs(ctx, space, body.Blobs); err != nil { + return nil, fmt.Errorf("s3frontend: park part blobs: %w", err) + } + if len(superseded) > 0 { + b.cleanupPartBlobs(ctx, space, uploadID, superseded) + } etag := `"` + hex.EncodeToString(body.MD5) + `"` return &s3.UploadPartOutput{ETag: &etag}, nil } @@ -111,6 +198,10 @@ func (b *Backend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s // list against the recorded parts, accepts every part's blobs on Forge, and // commits a manifest whose Body is the ordered union of the parts' blobs. The // object ETag is hex(md5(concat of part md5s)) + "-N". +// +// A successful Complete retains the session in state 'completed' (with its +// parts), so a duplicate Complete with an identical part list is idempotent +// per S3; the abandoned-session sweeper reaps the row later. func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) { if input.Bucket == nil || input.Key == nil || input.UploadId == nil { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -130,25 +221,32 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet } return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("s3frontend: complete: %w", err) } - // Single-winner latch vs a racing Abort: only the writer that moves the - // session off 'open' proceeds (§7.3). - won, err := b.multipart.LatchSession(ctx, uploadID, registry.SessionOpen, registry.SessionCompleting) - if err != nil { - return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("s3frontend: latch: %w", err) - } - if !won { - return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNoSuchUpload) + + // Conditional writes: S3 documents only If-None-Match: * (don't overwrite) + // and If-Match (require current ETag) for Complete; a concrete If-None-Match + // value — or combining If-None-Match with If-Match — is NotImplemented. + ifMatch, ifNoneMatch := input.IfMatch, input.IfNoneMatch + if ifNoneMatch != nil && (*ifNoneMatch != "*" || ifMatch != nil) { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNotImplemented) } - // If anything below fails before the object is committed, revert the session - // to 'open' so the upload stays abortable / retriable rather than zombied in - // 'completing'. committed is set once the manifest is durable (the point of - // no return). - committed := false - defer func() { - if !committed { - _, _ = b.multipart.LatchSession(ctx, uploadID, registry.SessionCompleting, registry.SessionOpen) + if ifMatch != nil || ifNoneMatch != nil { + current, _, lerr := b.lookupManifest(ctx, bucket, key) + exists := lerr == nil + if lerr != nil && !isNoSuchKey(lerr) { + return s3response.CompleteMultipartUploadResult{}, "", lerr } - }() + if ifMatch != nil { + if !exists { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if !etagsEqual(*ifMatch, current.ETag) { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + } + if ifNoneMatch != nil && exists { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + } if input.MultipartUpload == nil || len(input.MultipartUpload.Parts) == 0 { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) @@ -162,18 +260,24 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet byNum[p.PartNumber] = p } - // Validate the requested parts (ascending, each matching a recorded part by - // number + ETag) and assemble the ordered body + the multipart ETag. - var blobs []msbucket.BlobRef - var partSizes []int64 - var offset int64 + // Validate the requested parts (in-range and ascending, each matching a + // recorded part by number + ETag) and compute the multipart ETag. + requested := make([]registry.MultipartPart, 0, len(input.MultipartUpload.Parts)) etagHasher := md5.New() prev := 0 for _, rp := range input.MultipartUpload.Parts { - if rp.PartNumber == nil { - return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) + // A part entry missing either field is malformed XML; a part number + // below 1 is an InvalidArgument (both per the upstream posix + // backend). Out-of-range numbers fall through to the membership + // check (no stored part can match) and report InvalidPart. + if rp.PartNumber == nil || rp.ETag == nil { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrMalformedXML) } num := int(*rp.PartNumber) + if num < 1 { + return s3response.CompleteMultipartUploadResult{}, "", + s3err.GetInvalidArgumentErr(s3err.InvalidArgCompleteMpPartNumber, strconv.Itoa(num)) + } if num <= prev { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPartOrder) } @@ -182,12 +286,66 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet if !ok { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) } - if rp.ETag != nil && !etagsEqual(*rp.ETag, hex.EncodeToString(sp.ETagMD5)) { + if !etagsEqual(*rp.ETag, hex.EncodeToString(sp.ETagMD5)) { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) } + requested = append(requested, sp) etagHasher.Write(sp.ETagMD5) - // Record this part's byte span (it may span several blobs) so a later - // GET/HEAD ?partNumber=N can address it (§7.2). + } + // Every part but the last must meet S3's protocol-level 5 MiB minimum + // (backend.MinPartSize — an S3 constant clients and SDKs assume, not an + // operator knob). + var total int64 + for i, sp := range requested { + if i < len(requested)-1 && sp.Size < backend.MinPartSize { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrEntityTooSmall) + } + total += sp.Size + } + if input.MpuObjectSize != nil { + if *input.MpuObjectSize < 0 { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetNegatvieMpObjectSizeErr(*input.MpuObjectSize) + } + if *input.MpuObjectSize != total { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetIncorrectMpObjectSizeErr(total, *input.MpuObjectSize) + } + } + etag := hex.EncodeToString(etagHasher.Sum(nil)) + "-" + strconv.Itoa(len(requested)) + + // Idempotent re-Complete: the prior Complete committed the object; the + // validation above already proved the client's part list matches the + // retained parts, so return the same result without recommitting. + if sess.State == registry.SessionCompleted { + etagQ := `"` + etag + `"` + return s3response.CompleteMultipartUploadResult{Bucket: &bucket, Key: &key, ETag: &etagQ}, "", nil + } + + // Single-winner latch vs a racing Abort: only the writer that moves the + // session off 'open' proceeds (§7.3). + won, err := b.multipart.LatchSession(ctx, uploadID, registry.SessionOpen, registry.SessionCompleting) + if err != nil { + return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("s3frontend: latch: %w", err) + } + if !won { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + // If anything below fails before the object is committed, revert the session + // to 'open' so the upload stays abortable / retriable rather than zombied in + // 'completing'. committed is set once the manifest is durable (the point of + // no return). + committed := false + defer func() { + if !committed { + _, _ = b.multipart.LatchSession(ctx, uploadID, registry.SessionCompleting, registry.SessionOpen) + } + }() + + // Assemble the ordered body: each part's byte span (it may span several + // blobs) is recorded so a later GET/HEAD ?partNumber=N can address it (§7.2). + var blobs []msbucket.BlobRef + var partSizes []int64 + var offset int64 + for _, sp := range requested { partStart := offset for _, d := range sp.BlobDigests { in, err := b.intents.GetIntent(ctx, d) @@ -200,49 +358,67 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet partSizes = append(partSizes, offset-partStart) } - // Accept every part's blobs on Forge (no-op in the harness), then commit. - if err := b.uploadBlobs(ctx, bucketState.Space, blobs); err != nil { + // Accept every part's blobs on Forge: parked blobs conclude (the deferred + // /http/put receipt fires /blob/accept), stragglers that never parked + // (crash between spool and park) fall back to the whole synchronous + // upload. Then commit. + if err := b.concludeBlobs(ctx, bucketState.Space, blobs); err != nil { return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("s3frontend: accept parts: %w", err) } - etag := hex.EncodeToString(etagHasher.Sum(nil)) + "-" + strconv.Itoa(len(input.MultipartUpload.Parts)) mf := &msbucket.ObjectManifest{ - Key: key, - ContentType: sess.ContentType, - Created: time.Now().Unix(), - Body: msbucket.Body{Size: offset, Blobs: blobs, PartSizes: partSizes}, - ETag: etag, - Metadata: sess.Metadata, + Key: key, + ContentType: sess.ContentType, + Created: time.Now().Unix(), + Body: msbucket.Body{Size: offset, Blobs: blobs, PartSizes: partSizes}, + ETag: etag, + ContentEncoding: sess.ContentEncoding, + ContentDisposition: sess.ContentDisposition, + ContentLanguage: sess.ContentLanguage, + CacheControl: sess.CacheControl, + Expires: sess.Expires, + WebsiteRedirectLocation: sess.WebsiteRedirectLocation, + Metadata: sess.Metadata, } if err := b.commitManifest(ctx, bucketState, key, mf, bodyDigests(mf.Body)); err != nil { return s3response.CompleteMultipartUploadResult{}, "", err } committed = true - // The object is durable; session cleanup is best-effort (a lingering session - // is harmless and reapable later, and must not fail an otherwise-good - // Complete). - _ = b.multipart.DeleteSession(ctx, uploadID) + // The object is durable. Retain the session (state 'completed') and its + // parts so a duplicate Complete is idempotent; the sweeper reaps it later. + // Best-effort: a failed latch leaves the row in 'completing', which the + // sweeper also treats as terminal after the TTL. + if _, err := b.multipart.LatchSession(ctx, uploadID, registry.SessionCompleting, registry.SessionCompleted); err != nil { + b.logger.Warn("latch session to completed failed; sweeper reaps the completing row after the TTL", + zap.String("uploadID", uploadID), zap.Error(err)) + } etagQ := `"` + etag + `"` return s3response.CompleteMultipartUploadResult{Bucket: &bucket, Key: &key, ETag: &etagQ}, "", nil } // AbortMultipartUpload cancels a multipart upload: it latches the session -// (single-winner vs Complete) and drops it (cascading its parts). The spooled -// part blobs are content-addressed and may be shared, so they are left for GC -// rather than deleted here; no reference claims were taken (those happen only at -// Complete), so nothing else needs unwinding. +// (single-winner vs Complete), drops it (cascading its parts), and removes the +// parts' now-unreferenced blobs from the spool — unallocating any that were +// parked on a provider (an upload ends in exactly one of accept or +// abort). No reference claims were taken (those happen only at +// Complete). func (b *Backend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error { if input.UploadId == nil { return s3err.GetAPIError(s3err.ErrInvalidRequest) } uploadID := *input.UploadId - if _, err := b.multipart.GetSession(ctx, uploadID); err != nil { - if errors.Is(err, registry.ErrNotFound) { - return s3err.GetAPIError(s3err.ErrNoSuchUpload) - } - return fmt.Errorf("s3frontend: abort: %w", err) + sess, err := b.openSession(ctx, uploadID, input.Key) + if err != nil { + return err + } + // If-Match-Initiated-Time: reject when the provided timestamp predates the + // upload's initiation (compared at second precision — Initiated is + // presented via RFC3339); future timestamps are ignored per S3. + if input.IfMatchInitiatedTime != nil && + input.IfMatchInitiatedTime.Truncate(time.Second).Before(sess.CreatedAt.Truncate(time.Second)) { + return s3err.GetAPIError(s3err.ErrPreconditionFailed) } won, err := b.multipart.LatchSession(ctx, uploadID, registry.SessionOpen, registry.SessionAborting) if err != nil { @@ -251,12 +427,509 @@ func (b *Backend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMulti if !won { return s3err.GetAPIError(s3err.ErrNoSuchUpload) } + // Snapshot the parts' blob digests before the cascade delete, then drop + // the session and clean the spool. + var digests [][]byte + if parts, err := b.multipart.ListParts(ctx, uploadID); err == nil { + for _, p := range parts { + digests = append(digests, p.BlobDigests...) + } + } if err := b.multipart.DeleteSession(ctx, uploadID); err != nil { return fmt.Errorf("s3frontend: delete session: %w", err) } + space, err := b.bucketSpace(ctx, sess.Bucket) + if err != nil { + return err + } + b.cleanupPartBlobs(ctx, space, uploadID, digests) + return nil +} + +// abortOpenSession force-aborts an open multipart session exactly like a +// client Abort: latch (losing gracefully to a concurrent Complete/Abort), +// drop the session, release its parts' now-unreferenced blobs. Used by +// DeleteBucket's implicit abort of in-flight uploads. +func (b *Backend) abortOpenSession(ctx context.Context, space did.DID, sess registry.MultipartSession) { + // s3:DeleteBucket delegates no blob commands (hilt's s3perm maps it to + // nil), so the surrounding request's proofs cannot authorize + // /blob/abort; mask them so the uploader falls back to the blob + // authority captured at UploadPart — the same resolution the + // session-expiry sweeper uses. blob.Abort rides the write set as of + // fil-forge/hilt#36. + ctx = reqscope.WithoutProofStore(ctx) + won, err := b.multipart.LatchSession(ctx, sess.UploadID, registry.SessionOpen, registry.SessionAborting) + if err != nil || !won { + return + } + var digests [][]byte + if parts, err := b.multipart.ListParts(ctx, sess.UploadID); err == nil { + for _, p := range parts { + digests = append(digests, p.BlobDigests...) + } + } + if err := b.multipart.DeleteSession(ctx, sess.UploadID); err != nil { + return + } + b.cleanupPartBlobs(ctx, space, sess.UploadID, digests) +} + +// cleanupPartBlobs removes spooled blobs that belonged to aborted, expired, or +// superseded parts of uploadID — unless the blob is still referenced: by a +// part of another in-flight session (content-addressed dedup), by a part still +// live in THIS session (a re-uploaded part may share blobs with its +// replacement or a sibling part), or by a committed object (reference claims / +// non-spooled intent state). Best-effort: cleanup failure never fails the S3 +// operation; a stranded spool file is reapable later. +func (b *Backend) cleanupPartBlobs(ctx context.Context, space did.DID, uploadID string, digests [][]byte) { + if len(digests) == 0 { + return + } + // Digests still referenced by this session's live parts (after the + // abort/supersede that triggered this cleanup). + live := map[string]bool{} + if parts, err := b.multipart.ListParts(ctx, uploadID); err == nil { + for _, p := range parts { + for _, d := range p.BlobDigests { + live[string(d)] = true + } + } + } + seen := map[string]bool{} + for _, d := range digests { + k := string(d) + if seen[k] || live[k] { + continue + } + seen[k] = true + if n, err := b.multipart.CountPartRefs(ctx, d, uploadID); err != nil || n > 0 { + continue + } + if n, err := b.blobRefs.CountClaims(ctx, space, d); err != nil || n > 0 { + continue + } + state := registry.IntentSpooled + if in, err := b.intents.GetIntent(ctx, d); err == nil { + state = in.State + } + if state != registry.IntentSpooled && state != registry.IntentParked { + // Accepted/published blobs are the reference index's to manage. + continue + } + // A parked blob is durable on its provider — release it there too + // (best-effort; the reject on piri is idempotent, a straggler is + // FIL-625's to reap). Cause is the /blob/add task link the + // upload service needs to locate the provider. A BlobAccepted + // refusal is benign — a concurrent session in this space accepted + // the same content, so the reference index owns the blob now — and + // the park row is obsolete either way. + if state == registry.IntentParked { + if park, err := b.parks.GetPark(ctx, d); err == nil { + if cause, err := cid.Cast(park.AddTask); err == nil { + if aerr := b.deferred.AbortBlob(ctx, space, mh.Multihash(d), cause); aerr != nil { + b.logger.Warn("abort parked blob failed; provider-side release deferred", + zap.String("digest", hex.EncodeToString(d)), zap.Error(aerr)) + } + } + if derr := b.parks.DeletePark(ctx, d); derr != nil { + b.logger.Warn("delete park row failed", + zap.String("digest", hex.EncodeToString(d)), zap.Error(derr)) + } + } + } + if rerr := b.spool.Remove(mh.Multihash(d)); rerr != nil { + b.logger.Warn("remove spooled blob failed", + zap.String("digest", hex.EncodeToString(d)), zap.Error(rerr)) + } + if derr := b.intents.DeleteIntent(ctx, d); derr != nil { + b.logger.Warn("delete upload intent failed", + zap.String("digest", hex.EncodeToString(d)), zap.Error(derr)) + } + } +} + +// parkBlobs makes each blob durable on its provider without accepting it: +// already-located blobs are marked accepted (dedup), already-parked blobs are +// skipped (another part or session parked the same content), the rest upload +// with the conclude deferred (WithConclude(false)) and persist their park +// state for Complete/Abort. +func (b *Backend) parkBlobs(ctx context.Context, space did.DID, blobs []msbucket.BlobRef) error { + for _, blob := range blobs { + digest := mh.Multihash(blob.Digest) + if existing, err := b.locations.GetLocation(ctx, space, blob.Digest); err == nil && existing != nil { + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { + return fmt.Errorf("mark accepted (dedup): %w", err) + } + continue + } else if err != nil && !errors.Is(err, registry.ErrNotFound) { + return fmt.Errorf("lookup location: %w", err) + } + if _, err := b.parks.GetPark(ctx, blob.Digest); err == nil { + continue // already parked by a sibling part or session + } else if !errors.Is(err, registry.ErrNotFound) { + return fmt.Errorf("lookup park: %w", err) + } + + res, err := b.deferred.UploadBlob(ctx, space, digest, blob.Length, b.spool.Path(digest), uploader.WithConclude(false)) + if err != nil { + return fmt.Errorf("park blob: %w", err) + } + if res.Location != nil { + // The provider already held accepted bytes for this content — + // accept ran despite the deferred conclude (dedup), record the + // location like the synchronous path. + if err := b.locations.PutLocation(ctx, registry.BlobLocation{ + Space: space, + Digest: blob.Digest, + Provider: res.Location.Provider, + URL: res.Location.URL, + Size: res.Location.Size, + }); err != nil { + return fmt.Errorf("record location: %w", err) + } + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { + return fmt.Errorf("mark accepted: %w", err) + } + continue + } + // Location == nil ⇔ parked: durable on the provider with accept + // deferred — persist the conclude state for Complete/Abort. + if err := b.parks.PutPark(ctx, registry.BlobPark{ + Digest: blob.Digest, + AddTask: res.AddTask.Bytes(), + AcceptTask: res.AcceptTask.Bytes(), + PutInvocation: res.PutInvocation, + Size: blob.Length, + }); err != nil { + return fmt.Errorf("record park: %w", err) + } + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentParked); err != nil { + return fmt.Errorf("mark parked: %w", err) + } + } + return nil +} + +// concludeBlobs is Complete's park-aware counterpart to uploadBlobs: located +// blobs are already accepted (dedup); parked blobs conclude their deferred +// /http/put receipt — firing /blob/accept — and record their location; +// blobs that never parked (crash between spool and park) fall back to the +// whole synchronous upload. +func (b *Backend) concludeBlobs(ctx context.Context, space did.DID, blobs []msbucket.BlobRef) error { + for _, blob := range blobs { + digest := mh.Multihash(blob.Digest) + if existing, err := b.locations.GetLocation(ctx, space, blob.Digest); err == nil && existing != nil { + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { + return fmt.Errorf("mark accepted (dedup): %w", err) + } + continue + } else if err != nil && !errors.Is(err, registry.ErrNotFound) { + return fmt.Errorf("lookup location: %w", err) + } + + park, err := b.parks.GetPark(ctx, blob.Digest) + if err != nil && !errors.Is(err, registry.ErrNotFound) { + return fmt.Errorf("lookup park: %w", err) + } + var loc uploader.BlobLocation + if park != nil { + addTask, err := cid.Cast(park.AddTask) + if err != nil { + return fmt.Errorf("decode park add task: %w", err) + } + acceptTask, err := cid.Cast(park.AcceptTask) + if err != nil { + return fmt.Errorf("decode park accept task: %w", err) + } + loc, err = b.deferred.ConcludeBlob(ctx, space, uploader.UploadedBlob{ + Digest: digest, + Size: park.Size, + AddTask: addTask, + AcceptTask: acceptTask, + PutInvocation: park.PutInvocation, + }) + if err != nil { + return fmt.Errorf("conclude blob: %w", err) + } + } else { + // Never parked (crash between spool and park): the spooled copy + // drives the whole synchronous upload. + res, uerr := b.uploader.UploadBlob(ctx, space, digest, blob.Length, b.spool.Path(digest)) + if uerr != nil { + return fmt.Errorf("upload blob: %w", uerr) + } + if res.Location == nil { + return fmt.Errorf("upload blob %x: concluding upload returned no location", blob.Digest) + } + loc = *res.Location + } + + if err := b.locations.PutLocation(ctx, registry.BlobLocation{ + Space: space, + Digest: blob.Digest, + Provider: loc.Provider, + URL: loc.URL, + Size: loc.Size, + }); err != nil { + return fmt.Errorf("record location: %w", err) + } + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { + return fmt.Errorf("mark accepted: %w", err) + } + if park != nil { + // The sealed put invocation is spent — drop it promptly. + if err := b.parks.DeletePark(ctx, blob.Digest); err != nil { + return fmt.Errorf("drop park: %w", err) + } + } + } return nil } +// ListParts returns the recorded parts of an in-flight upload, paginated by +// part number. +func (b *Backend) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) { + if input.Bucket == nil || input.Key == nil || input.UploadId == nil { + return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + uploadID := *input.UploadId + if _, err := b.openSession(ctx, uploadID, input.Key); err != nil { + return s3response.ListPartsResult{}, err + } + + marker := 0 + if input.PartNumberMarker != nil && *input.PartNumberMarker != "" { + m, err := strconv.Atoi(*input.PartNumberMarker) + if err != nil { + return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + marker = m + } + maxParts := defaultMaxListing + if input.MaxParts != nil && *input.MaxParts > 0 && int(*input.MaxParts) < defaultMaxListing { + maxParts = int(*input.MaxParts) + } + + stored, err := b.multipart.ListParts(ctx, uploadID) + if err != nil { + return s3response.ListPartsResult{}, fmt.Errorf("s3frontend: list parts: %w", err) + } + var parts []s3response.Part + truncated := false + next := 0 + for _, p := range stored { + if p.PartNumber <= marker { + continue + } + if len(parts) == maxParts { + truncated = true + break + } + parts = append(parts, s3response.Part{ + PartNumber: p.PartNumber, + ETag: `"` + hex.EncodeToString(p.ETagMD5) + `"`, + Size: p.Size, + LastModified: p.CreatedAt.UTC(), + }) + next = p.PartNumber + } + res := s3response.ListPartsResult{ + Bucket: *input.Bucket, + Key: *input.Key, + UploadID: uploadID, + StorageClass: types.StorageClassStandard, + PartNumberMarker: marker, + MaxParts: maxParts, + IsTruncated: truncated, + Parts: parts, + } + if truncated { + res.NextPartNumberMarker = next + } + return res, nil +} + +// ListMultipartUploads lists the bucket's in-flight (open) multipart uploads +// in (key, initiation) order, with S3's prefix/delimiter/marker pagination. +func (b *Backend) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { + if input.Bucket == nil { + return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + bucket := *input.Bucket + if _, err := b.reg.Get(ctx, bucket); err != nil { + if errors.Is(err, registry.ErrNotFound) { + return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + return s3response.ListMultipartUploadsResult{}, fmt.Errorf("s3frontend: list mpu: %w", err) + } + + prefix := backend.GetStringFromPtr(input.Prefix) + delimiter := backend.GetStringFromPtr(input.Delimiter) + keyMarker := backend.GetStringFromPtr(input.KeyMarker) + uploadIDMarker := backend.GetStringFromPtr(input.UploadIdMarker) + maxUploads := defaultMaxListing + if input.MaxUploads != nil && *input.MaxUploads > 0 && int(*input.MaxUploads) < defaultMaxListing { + maxUploads = int(*input.MaxUploads) + } + + all, err := b.multipart.ListSessions(ctx, bucket) + if err != nil { + return s3response.ListMultipartUploadsResult{}, fmt.Errorf("s3frontend: list sessions: %w", err) + } + // In-flight uploads only, honoring the prefix. + inflight := all[:0] + for _, s := range all { + if s.State == registry.SessionOpen && strings.HasPrefix(s.ObjectKey, prefix) { + inflight = append(inflight, s) + } + } + + // Marker positioning, mirroring upstream's MultipartUploadLister: an + // upload-id marker is meaningful only alongside a key marker; it must be + // a valid UUID and must name an upload of the FIRST key group at or + // after the key marker (else InvalidArgument), and the listing resumes + // just past it. + start := 0 + if keyMarker != "" { + if uploadIDMarker != "" { + if _, err := uuid.Parse(uploadIDMarker); err != nil { + return s3response.ListMultipartUploadsResult{}, + s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, uploadIDMarker) + } + i := 0 + for i < len(inflight) && inflight[i].ObjectKey < keyMarker { + i++ + } + pos := -1 + if i < len(inflight) { + firstKey := inflight[i].ObjectKey + for j := i; j < len(inflight) && inflight[j].ObjectKey == firstKey; j++ { + if inflight[j].UploadID == uploadIDMarker { + pos = j + break + } + } + } + if pos < 0 { + return s3response.ListMultipartUploadsResult{}, + s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, uploadIDMarker) + } + start = pos + 1 + } else { + for start < len(inflight) && inflight[start].ObjectKey <= keyMarker { + start++ + } + } + } + + // Walk the ordered tail, collapsing keys under the delimiter into common + // prefixes; uploads and prefixes count toward max-uploads together. + var uploads []s3response.Upload + var prefixes []s3response.CommonPrefix + emittedPrefix := map[string]bool{} + count := 0 + truncated := false + nextKeyMarker, nextUploadIDMarker := "", "" + for _, s := range inflight[start:] { + itemKey := s.ObjectKey + isPrefix := false + if delimiter != "" { + if i := strings.Index(s.ObjectKey[len(prefix):], delimiter); i >= 0 { + itemKey = s.ObjectKey[:len(prefix)+i+len(delimiter)] + isPrefix = true + } + } + if isPrefix && emittedPrefix[itemKey] { + continue + } + if count == maxUploads { + truncated = true + break + } + if isPrefix { + emittedPrefix[itemKey] = true + prefixes = append(prefixes, s3response.CommonPrefix{Prefix: itemKey}) + } else { + uploads = append(uploads, s3response.Upload{ + Key: s.ObjectKey, + UploadID: s.UploadID, + StorageClass: types.StorageClassStandard, + Initiated: s.CreatedAt.UTC(), + ChecksumAlgorithm: types.ChecksumAlgorithm(s.ChecksumAlgorithm), + ChecksumType: types.ChecksumType(s.ChecksumType), + }) + } + count++ + nextKeyMarker, nextUploadIDMarker = itemKey, s.UploadID + } + res := s3response.ListMultipartUploadsResult{ + Bucket: bucket, + KeyMarker: keyMarker, + UploadIDMarker: uploadIDMarker, + Delimiter: delimiter, + Prefix: prefix, + MaxUploads: maxUploads, + IsTruncated: truncated, + Uploads: uploads, + CommonPrefixes: prefixes, + } + if truncated { + res.NextKeyMarker = nextKeyMarker + res.NextUploadIDMarker = nextUploadIDMarker + } + return res, nil +} + +// SweepStaleMultipartSessions aborts in-flight multipart sessions older than +// ttl (dropping their spooled parts, exactly like a client Abort) and reaps +// completed/aborting leftovers past the same age. Returns how many sessions +// were cleaned. Called periodically by the daemon's sweeper loop. +func (b *Backend) SweepStaleMultipartSessions(ctx context.Context, ttl time.Duration) (int, error) { + cutoff := time.Now().Add(-ttl) + cleaned := 0 + // Stale open sessions: latch (losing gracefully to a concurrent + // Complete/Abort) and clean up like an abort. + stale, err := b.multipart.ListStaleSessions(ctx, registry.SessionOpen, cutoff) + if err != nil { + return 0, fmt.Errorf("s3frontend: sweep list: %w", err) + } + for _, s := range stale { + won, err := b.multipart.LatchSession(ctx, s.UploadID, registry.SessionOpen, registry.SessionAborting) + if err != nil || !won { + continue + } + var digests [][]byte + if parts, err := b.multipart.ListParts(ctx, s.UploadID); err == nil { + for _, p := range parts { + digests = append(digests, p.BlobDigests...) + } + } + if err := b.multipart.DeleteSession(ctx, s.UploadID); err != nil { + continue + } + space, serr := b.bucketSpace(ctx, s.Bucket) + if serr != nil { + continue // bucket gone; spool rows are reapable later + } + b.cleanupPartBlobs(ctx, space, s.UploadID, digests) + cleaned++ + } + // Terminal leftovers: completed sessions retained for Complete idempotency, + // and any 'completing'/'aborting' rows stranded by a crash mid-transition. + for _, state := range []string{registry.SessionCompleted, registry.SessionCompleting, registry.SessionAborting} { + leftovers, err := b.multipart.ListStaleSessions(ctx, state, cutoff) + if err != nil { + continue + } + for _, s := range leftovers { + if err := b.multipart.DeleteSession(ctx, s.UploadID); err == nil { + cleaned++ + } + } + } + return cleaned, nil +} + // commitManifest splices mf into (bucket, key) and reconciles the reference // index against the prior version's digests, releasing dropped blobs after the // commit. Shared by CopyObject and CompleteMultipartUpload (a plain commit with diff --git a/s3frontend/object.go b/s3frontend/object.go index 0ebcb6e..dcd885c 100644 --- a/s3frontend/object.go +++ b/s3frontend/object.go @@ -282,10 +282,16 @@ func (b *Backend) uploadBlobs(ctx context.Context, space did.DID, blobs []msbuck } else if err != nil && !errors.Is(err, registry.ErrNotFound) { return fmt.Errorf("lookup location: %w", err) } - loc, err := b.uploader.UploadBlob(ctx, space, digest, blob.Length, b.spool.Path(digest)) + res, err := b.uploader.UploadBlob(ctx, space, digest, blob.Length, b.spool.Path(digest)) if err != nil { return fmt.Errorf("upload blob: %w", err) } + // A concluding UploadBlob (the default) returns an accepted location + // or errors; guard the contract rather than deref-panic on a bad impl. + if res.Location == nil { + return fmt.Errorf("upload blob %x: concluding upload returned no location", blob.Digest) + } + loc := res.Location if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { return fmt.Errorf("mark accepted: %w", err) } @@ -478,15 +484,14 @@ func (b *Backend) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s contentType := mf.ContentType out := &s3.HeadObjectOutput{ - AcceptRanges: backend.GetPtrFromString("bytes"), - ContentLength: &length, - ContentType: &contentType, - ContentEncoding: strPtrOrNil(mf.ContentEncoding), - ContentDisposition: strPtrOrNil(mf.ContentDisposition), - ContentLanguage: strPtrOrNil(mf.ContentLanguage), - CacheControl: strPtrOrNil(mf.CacheControl), - ExpiresString: strPtrOrNil(mf.Expires), - + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + ContentType: &contentType, + ContentEncoding: strPtrOrNil(mf.ContentEncoding), + ContentDisposition: strPtrOrNil(mf.ContentDisposition), + ContentLanguage: strPtrOrNil(mf.ContentLanguage), + CacheControl: strPtrOrNil(mf.CacheControl), + ExpiresString: strPtrOrNil(mf.Expires), WebsiteRedirectLocation: strPtrOrNil(mf.WebsiteRedirectLocation), Metadata: mf.Metadata, ContentRange: contentRange, @@ -546,16 +551,15 @@ func (b *Backend) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3. lastModified := time.Unix(mf.Created, 0) contentType := mf.ContentType out := &s3.GetObjectOutput{ - AcceptRanges: backend.GetPtrFromString("bytes"), - Body: body, - ContentLength: &length, - ContentType: &contentType, - ContentEncoding: strPtrOrNil(mf.ContentEncoding), - ContentDisposition: strPtrOrNil(mf.ContentDisposition), - ContentLanguage: strPtrOrNil(mf.ContentLanguage), - CacheControl: strPtrOrNil(mf.CacheControl), - ExpiresString: strPtrOrNil(mf.Expires), - + AcceptRanges: backend.GetPtrFromString("bytes"), + Body: body, + ContentLength: &length, + ContentType: &contentType, + ContentEncoding: strPtrOrNil(mf.ContentEncoding), + ContentDisposition: strPtrOrNil(mf.ContentDisposition), + ContentLanguage: strPtrOrNil(mf.ContentLanguage), + CacheControl: strPtrOrNil(mf.CacheControl), + ExpiresString: strPtrOrNil(mf.Expires), WebsiteRedirectLocation: strPtrOrNil(mf.WebsiteRedirectLocation), Metadata: mf.Metadata, ContentRange: contentRange, diff --git a/s3frontend/upload_dedup_test.go b/s3frontend/upload_dedup_test.go index d3d202f..64cbf41 100644 --- a/s3frontend/upload_dedup_test.go +++ b/s3frontend/upload_dedup_test.go @@ -26,11 +26,15 @@ type countingUploader struct { calls map[string]int } -func (u *countingUploader) UploadBlob(_ context.Context, space did.DID, digest multihash.Multihash, size int64, _ string) (uploader.BlobLocation, error) { +func (u *countingUploader) UploadBlob(_ context.Context, _ did.DID, digest multihash.Multihash, size int64, _ string, _ ...uploader.UploadOption) (uploader.UploadedBlob, error) { u.mu.Lock() u.calls[string(digest)]++ u.mu.Unlock() - return uploader.BlobLocation{Provider: "did:test:piri", URL: "http://piri/blob", Size: size}, nil + return uploader.UploadedBlob{ + Digest: digest, + Size: size, + Location: &uploader.BlobLocation{Provider: "did:test:piri", URL: "http://piri/blob", Size: size}, + }, nil } func (u *countingUploader) count(digest []byte) int { diff --git a/server.go b/server.go index fc1e274..5ed4b35 100644 --- a/server.go +++ b/server.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "time" "github.com/fil-forge/versitygw/auth" "github.com/fil-forge/versitygw/metrics" @@ -50,7 +51,11 @@ type ServerDeps struct { // space's claim on a blob when its last reference is dropped. In tests both // are no-ops and reads are served from the local spool. BodyUploader uploader.BodyUploader - Remover uploader.BlobRemover + // Deferred extends BodyUploader for multipart's deferred accept: + // park at UploadPart (WithConclude(false)), conclude at Complete, + // abort at Abort. + Deferred uploader.DeferredBodyUploader + Remover uploader.BlobRemover // Authority is the service that authorizes bucket creation and deletion. Authority bucketauthority.BucketAuthority @@ -72,6 +77,9 @@ type ServerDeps struct { BlobRefs registry.BlobRefStore GC registry.GCStore Multipart registry.MultipartStore + // Parks persists deferred-accept park state between UploadPart and + // Complete/Abort. + Parks registry.ParkStore // Meta is the persistence backing for log-segment metadata. // Typically the same instance as Registry. @@ -88,11 +96,12 @@ type ServerDeps struct { // lifecycle. fx callers wrap these in OnStart/OnStop hooks; tests // call them directly. type Server struct { - cfg config.ServerConfig - logger *zap.Logger - log blockstore.Log - backend *s3frontend.Backend - api *s3api.S3ApiServer + cfg config.ServerConfig + logger *zap.Logger + log blockstore.Log + backend *s3frontend.Backend + api *s3api.S3ApiServer + sweepStop chan struct{} } // New wires a ServerDeps + ServerConfig into a runnable Server. The @@ -145,10 +154,12 @@ func New(ctx context.Context, cfg config.ServerConfig, deps ServerDeps) (*Server BlobRefs: deps.BlobRefs, GC: deps.GC, Multipart: deps.Multipart, + Parks: deps.Parks, Reads: bs, Log: log, Spool: spool, Uploader: deps.BodyUploader, + Deferred: deps.Deferred, Remover: deps.Remover, MaxBlobSize: cfg.MaxBlobSize, CORS: cfg.CORSConfig, @@ -190,15 +201,58 @@ func (s *Server) Start(ctx context.Context) error { s.logger.Error("ingot listener error", zap.Error(err)) } }() + s.startMultipartSweeper() return nil } +// startMultipartSweeper spawns the abandoned-multipart-session sweeper: open +// sessions older than MultipartSessionTTL are aborted (their spooled parts +// dropped) and terminal session rows reaped. Zero TTL → 7-day default; +// negative → disabled. +func (s *Server) startMultipartSweeper() { + ttl := s.cfg.MultipartSessionTTL + if ttl == 0 { + ttl = 7 * 24 * time.Hour + } + if ttl < 0 { + return + } + interval := ttl / 2 + if interval > 10*time.Minute { + interval = 10 * time.Minute + } + s.sweepStop = make(chan struct{}) + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-s.sweepStop: + return + case <-ticker.C: + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + n, err := s.backend.SweepStaleMultipartSessions(ctx, ttl) + cancel() + if err != nil { + s.logger.Warn("multipart sweep", zap.Error(err)) + } else if n > 0 { + s.logger.Info("multipart sweep reaped stale sessions", zap.Int("count", n)) + } + } + } + }() +} + // Stop shuts the listener down and drains the log. Always returns // the combined error of the two operations so callers see all // failure modes; either alone is non-fatal to the other. func (s *Server) Stop(ctx context.Context) error { s.logger.Info("shutting down ingot S3 listener") + if s.sweepStop != nil { + close(s.sweepStop) + s.sweepStop = nil + } var errs []error if err := s.api.ShutDown(); err != nil { errs = append(errs, fmt.Errorf("s3api shutdown: %w", err)) @@ -370,6 +424,12 @@ func validateServerInputs(cfg config.ServerConfig, deps ServerDeps) error { if deps.BodyUploader == nil { return errors.New("ingot: ServerDeps.BodyUploader is required") } + if deps.Deferred == nil { + return errors.New("ingot: ServerDeps.Deferred is required") + } + if deps.Parks == nil { + return errors.New("ingot: ServerDeps.Parks is required") + } if deps.Registry == nil { return errors.New("ingot: ServerDeps.Registry is required") } diff --git a/testing/roundtrip.go b/testing/roundtrip.go index 1e58c1e..3d59b33 100644 --- a/testing/roundtrip.go +++ b/testing/roundtrip.go @@ -60,6 +60,16 @@ func PutBytes(ctx context.Context, c Config, bucket, key string, body []byte) er return err } +// DeleteObject deletes object key from bucket. +func DeleteObject(ctx context.Context, c Config, bucket, key string) error { + cl, err := s3Client(ctx, c) + if err != nil { + return err + } + _, err = cl.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &key}) + return err +} + // ListKeys returns every object key in bucket via undelimited ListObjectsV2 // pages — the listing shape that fetches every leaf's manifest (no // common-prefix collapsing). diff --git a/uploader/blob.go b/uploader/blob.go index 94b9d43..ac49c5f 100644 --- a/uploader/blob.go +++ b/uploader/blob.go @@ -7,8 +7,11 @@ import ( "os" assertcmds "github.com/fil-forge/libforge/commands/assert" + blobcmds "github.com/fil-forge/libforge/commands/blob" "github.com/fil-forge/libforge/digestutil" "github.com/fil-forge/ucantone/did" + ucanerrors "github.com/fil-forge/ucantone/errors" + "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" "go.uber.org/zap" @@ -26,27 +29,72 @@ type BlobLocation struct { Size int64 // whole-blob byte length } +// UploadOption configures BodyUploader.UploadBlob. +type UploadOption func(*uploadConfig) + +type uploadConfig struct { + conclude bool +} + +func newUploadConfig(options ...UploadOption) *uploadConfig { + cfg := &uploadConfig{conclude: true} + for _, opt := range options { + opt(cfg) + } + return cfg +} + +// WithConclude controls whether UploadBlob triggers accept before returning. +// WithConclude(false) leaves the blob parked — durable on the provider, +// accept deferred (multipart's UploadPart): the returned UploadedBlob has a +// nil Location and carries the state ConcludeBlob needs. Moot on dedup — +// when the provider already held accepted bytes for the content, accept +// already ran and the upload completes regardless. +func WithConclude(conclude bool) UploadOption { + return func(cfg *uploadConfig) { cfg.conclude = conclude } +} + +// UploadedBlob is the result of UploadBlob. Location == nil ⇔ the blob is +// parked (uploaded with WithConclude(false), no dedup hit): persist the task +// links + PutInvocation (the blob_parks row) and finish with ConcludeBlob at +// Complete, or abandon with AbortBlob (AddTask is the Cause). A concluding +// upload — the default — always returns a non-nil Location or errors. +type UploadedBlob struct { + Digest multihash.Multihash + Size int64 + // Location of the accepted blob; nil while parked. + Location *BlobLocation + // AddTask is the /blob/add task CID (the abort Cause); AcceptTask is the + // /blob/accept task CID the conclude polls. + AddTask cid.Cid + AcceptTask cid.Cid + // PutInvocation is the issued /http/put invocation, populated only while + // parked. Its metadata embeds derived signer keys — sensitive; delete it + // once concluded or rejected. + PutInvocation []byte +} + // BodyUploader makes one object-body blob durable on Forge by digest: allocate -// → PUT (skipped on dedup) → accept, returning its published location. It is the +// → PUT (skipped on dedup) → accept, returning its published location. +// WithConclude(false) stops before accept (see UploadedBlob). It is the // data-plane counterpart to Uploader (which ships catalog CAR segments). Unlike -// the old data-plane pipeline, this is synchronous: a blob is durable and -// accepted on Piri before the write path commits the manifest that references -// it (docs/architecture.md §5, §7.1). -// -// The space is owned by the implementation (Forge is constructed with one), so -// callers need not thread it. +// the old data-plane pipeline, this is synchronous: a blob is durable — and, +// when concluding, accepted — on Piri before the write path commits the +// manifest that references it (docs/architecture.md §5, §7.1). type BodyUploader interface { - UploadBlob(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string) (BlobLocation, error) + UploadBlob(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string, opts ...UploadOption) (UploadedBlob, error) } // UploadBlob uploads one spooled blob to Forge. For a single-shot PutObject the // allocate→PUT→accept happens in one call (forgeclient.BlobAdd already drives // the whole flow and returns the location commitment); multipart's deferred -// accept will need a decomposed path (a later phase). -func (u *Forge) UploadBlob(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string) (BlobLocation, error) { +// accept passes WithConclude(false) and finishes with ConcludeBlob at +// Complete (or AbortBlob at Abort). +func (u *Forge) UploadBlob(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string, opts ...UploadOption) (UploadedBlob, error) { + cfg := newUploadConfig(opts...) f, err := os.Open(localPath) if err != nil { - return BlobLocation{}, fmt.Errorf("uploader: open spooled blob %s: %w", localPath, err) + return UploadedBlob{}, fmt.Errorf("uploader: open spooled blob %s: %w", localPath, err) } defer f.Close() @@ -61,7 +109,7 @@ func (u *Forge) UploadBlob(ctx context.Context, space did.DID, digest multihash. // attributable. store, ok := reqscope.ProofStore(ctx) if !ok { - return BlobLocation{}, fmt.Errorf("uploader: no request-scoped proof store for space %s (IAM layer did not attach one)", space) + return UploadedBlob{}, fmt.Errorf("uploader: no request-scoped proof store for space %s (IAM layer did not attach one)", space) } u.captureShipProofs(space, store) @@ -69,12 +117,34 @@ func (u *Forge) UploadBlob(ctx context.Context, space did.DID, digest multihash. forgeclient.WithPrecomputedDigest(digest, uint64(size)), forgeclient.WithPutClient(u.putClient), forgeclient.WithProofStore(store), + forgeclient.WithConclude(cfg.conclude), ) if err != nil { - return BlobLocation{}, fmt.Errorf("uploader: upload blob: %w", err) + return UploadedBlob{}, fmt.Errorf("uploader: upload blob: %w", err) } + return uploadedFromAdded(added) +} - return locationFromAdded(added) +// uploadedFromAdded maps a forgeclient result into the uploader's shape: an +// accepted blob's location commitment parses into a BlobLocation; a parked +// (unconcluded) blob carries its pending-accept state through. +func uploadedFromAdded(added forgeclient.AddedBlob) (UploadedBlob, error) { + ub := UploadedBlob{ + Digest: added.Digest, + Size: int64(added.Size), + AddTask: added.AddTask, + AcceptTask: added.AcceptTask, + } + if added.Location == nil { + ub.PutInvocation = added.PutInvocation + return ub, nil + } + loc, err := locationFromAdded(added) + if err != nil { + return UploadedBlob{}, err + } + ub.Location = &loc + return ub, nil } // locationFromAdded parses the /assert/location commitment piri issued at @@ -98,27 +168,110 @@ func locationFromAdded(added forgeclient.AddedBlob) (BlobLocation, error) { var _ BodyUploader = (*Forge)(nil) -// BlobRemover releases this space's claim on an accepted blob. Because dedup is +// DeferredBodyUploader extends BodyUploader for multipart's deferred accept: +// UploadBlob with WithConclude(false) makes the bytes durable (parked) at +// UploadPart; ConcludeBlob triggers accept at Complete; AbortBlob abandons a +// parked blob at Abort. +type DeferredBodyUploader interface { + BodyUploader + ConcludeBlob(ctx context.Context, space did.DID, parked UploadedBlob) (BlobLocation, error) + AbortBlob(ctx context.Context, space did.DID, digest multihash.Multihash, cause cid.Cid) error +} + +// ConcludeBlob finishes a parked upload: it concludes the deferred /http/put +// receipt (triggering /blob/accept on the provider) and returns the published +// location. parked is the UploadedBlob a WithConclude(false) upload returned +// (rehydrated from its blob_parks row). The conclude carries no space proof +// (accept is owned by sprue), so no proof store is required. Safe to retry. +func (u *Forge) ConcludeBlob(ctx context.Context, space did.DID, parked UploadedBlob) (BlobLocation, error) { + added, err := u.client.BlobConclude(ctx, space, forgeclient.AddedBlob{ + Digest: parked.Digest, + Size: uint64(parked.Size), + AddTask: parked.AddTask, + AcceptTask: parked.AcceptTask, + PutInvocation: parked.PutInvocation, + }) + if err != nil { + return BlobLocation{}, fmt.Errorf("uploader: conclude blob: %w", err) + } + return locationFromAdded(added) +} + +// AbortBlob abandons a parked blob via /blob/abort on the upload +// service: sprue recovers the provider from the cause receipt chain and the +// node releases the allocation + parked bytes. cause is the parked blob's +// AddTask. The proof store is request-scoped when present (an S3 Abort) and +// otherwise the store captured at park time (the session-expiry sweeper). +// Errors are logged here (callers treat abort cleanup as best-effort and may +// discard them). +func (u *Forge) AbortBlob(ctx context.Context, space did.DID, digest multihash.Multihash, cause cid.Cid) error { + u.logger.Info("blob abort", + zap.Stringer("space", space), + zap.String("digest", digestutil.Format(digest)), + ) + store, ok := u.shipProofStore(ctx, space) + if !ok { + return fmt.Errorf("uploader: no proof store for space %s (no request scope and no captured write authority)", space) + } + if err := u.client.BlobAbort(ctx, space, digest, cause, forgeclient.WithProofStore(store)); err != nil { + // A BlobAccepted refusal is final, not a fault: the space accepted + // this content (e.g. a concurrent session completed with the same + // content-addressed part), so the blob now belongs to the reference + // index and is released via /blob/remove when its last claim drops. + var named ucanerrors.Named + if ucanerrors.As(err, &named) && named.Name() == blobcmds.BlobAcceptedErrorName { + u.logger.Info("blob abort refused: blob accepted by the space; reference accounting owns it", + zap.Stringer("space", space), + zap.String("digest", digestutil.Format(digest)), + ) + } else { + u.logger.Error("blob abort failed", + zap.Stringer("space", space), + zap.String("digest", digestutil.Format(digest)), + zap.Error(err), + ) + } + return fmt.Errorf("uploader: aborting blob: %w", err) + } + return nil +} + +var _ DeferredBodyUploader = (*Forge)(nil) + +// BlobRemover releases a space's claim on an accepted blob. Because dedup is // global, Piri deletes the bytes and retires the piece only when no space -// claims the digest at all (docs/architecture.md §6). The space is owned by the -// implementation (like UploadBlob). +// claims the digest at all (docs/architecture.md §6). type BlobRemover interface { RemoveBlob(ctx context.Context, space did.DID, digest multihash.Multihash) error } -// RemoveBlob releases the space's claim on digest. -// -// TODO(phase 7 / smelt): wire forgeclient.BlobRemove against the upload service -// — the libforge blob.Remove binding exists, but the Piri/Sprue handler is -// to-build (docs/architecture.md §9). Until it lands this is a logged no-op, so -// the reference-index bookkeeping (blob_refs count → 0 → RemoveBlob) is -// exercised end-to-end without a working network primitive; bytes accumulate on -// Piri until the handler exists. -func (u *Forge) RemoveBlob(_ context.Context, space did.DID, digest multihash.Multihash) error { - u.logger.Info("blob remove (no-op; Piri handler to-build)", +// RemoveBlob releases the space's claim on digest via /blob/remove on the +// upload service: sprue deregisters the blob and forwards a /blob/release to +// the storage nodes holding it; piri deletes the bytes only once no space +// claims the digest (and, for aggregated pieces, once the PDP root retires +// on-chain). The proof store is request-scoped when present (a DeleteObject) +// and otherwise the store captured from a recent write to the space. +// Idempotent. +func (u *Forge) RemoveBlob(ctx context.Context, space did.DID, digest multihash.Multihash) error { + u.logger.Info("blob remove", zap.Stringer("space", space), zap.String("digest", digestutil.Format(digest)), ) + store, ok := u.shipProofStore(ctx, space) + if !ok { + return fmt.Errorf("uploader: no proof store for space %s (no request scope and no captured write authority)", space) + } + if err := u.client.BlobRemove(ctx, space, digest, forgeclient.WithProofStore(store)); err != nil { + // Callers treat removal as best-effort and may discard the error, so + // log it here — a silent failure leaks bytes on the network with no + // trace. + u.logger.Error("blob remove failed", + zap.Stringer("space", space), + zap.String("digest", digestutil.Format(digest)), + zap.Error(err), + ) + return fmt.Errorf("uploader: removing blob: %w", err) + } return nil }