From 42a272be2b03c3803fa9b5230378a239e7cf3fad Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 14 Jul 2026 17:22:19 -0700 Subject: [PATCH 01/21] =?UTF-8?q?feat(multipart):=20complete=20FIL-520=20?= =?UTF-8?q?=E2=80=94=20ListParts,=20ListMultipartUploads,=20S3=20validatio?= =?UTF-8?q?n,=20abort/expiry=20hygiene?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the S3 multipart surface on the spool model (parts spool locally at UploadPart; ship+accept at Complete — the R1 architecture decision recorded on FIL-520; network unallocate-on-abort stays with the parking refinement, FIL-589/623/624). New surface: - ListParts: pagination (part-number-marker / max-parts, default+cap 1000), NoSuchUpload for unknown/mismatched/closed uploads, per-part LastModified (new multipart_parts.created_at). - ListMultipartUploads: (key, initiated) ordering, prefix/delimiter with CommonPrefixes, key/upload-id marker semantics incl. the InvalidUploadIdMarker validation, in-flight sessions only, session checksum algorithm/type echo. Conformance fixes: - UploadPart against a mismatched key -> NoSuchUpload (was silently accepted — a real bug: any valid uploadId accepted parts for any key). - Complete: part-number range check (InvalidArgCompleteMpPartNumber) before order check; EntityTooSmall for sub-5MiB non-final parts; x-amz-mp-object-size validation; conditional writes per S3 (If-Match / If-None-Match:* honored, concrete If-None-Match -> NotImplemented); duplicate Complete is idempotent — sessions are retained in a new 'completed' state instead of deleted. - CreateMultipartUpload: trailing-slash keys -> DirectoryObjectContainsData; content-encoding/disposition/language, cache-control, expires, and website-redirect-location persist through the session into the manifest (manifest gains WebsiteRedirectLocation; PutObject/Head/Get wire it too). Hygiene ('remove uploaded parts on abort/expiry', spool-model edition): - Abort and part re-upload delete the parts' now-unreferenced spooled blobs + intents, guarded against content-addressed sharing (other sessions via new CountPartRefs, committed objects via blob_refs claims, shipped blobs via intent state). - Abort honors If-Match-Initiated-Time. - Background sweeper (multipart_session_ttl, default 7d; negative disables) aborts stale open sessions and reaps terminal rows. Registry: multipart_sessions gains header columns + completed state (migration 00004); MultipartStore gains ListSessions / ListStaleSessions / CountPartRefs (postgres + inmem; live-test coverage for ordering, cutoff, bytea[] ANY-match, and the widened CHECK constraint). itest: 28 cases promoted xfail->pass (217 pass / 76 known-fail+skips, was 189/104). TestForgeScenarios gains MultipartAbortCleansSpool and uses >=5MiB non-final parts per the new EntityTooSmall enforcement. Remaining multipart xfail is FIL-620 (checksums), FIL-534/525 (tagging/lock/ACL), FIL-586 (UploadPartCopy). Refs: FIL-520 Co-Authored-By: Claude Fable 5 --- blockstore/spool.go | 12 + bucket/cbor_gen.go | 36 +- bucket/manifest.go | 5 + config.go | 15 + docs/architecture.md | 11 +- inmem/stores.go | 63 +++ itest/README.md | 2 +- itest/scenarios_test.go | 47 +- itest/versity_multipart_test.go | 58 +-- migrations/sql/00004_multipart_listing.sql | 48 ++ registry/postgres_live_test.go | 65 +++ registry/stores.go | 50 +- registry/stores_postgres.go | 124 ++++- s3frontend/multipart.go | 548 ++++++++++++++++++--- s3frontend/object.go | 87 ++-- server.go | 61 ++- 16 files changed, 1069 insertions(+), 163 deletions(-) create mode 100644 migrations/sql/00004_multipart_listing.sql diff --git a/blockstore/spool.go b/blockstore/spool.go index efc92d6..4ba1755 100644 --- a/blockstore/spool.go +++ b/blockstore/spool.go @@ -49,6 +49,18 @@ func (s *Spool) Path(digest mh.Multihash) string { return filepath.Join(s.dir, hex.EncodeToString(digest)) } +// Remove deletes the blob with the given digest from the spool. Idempotent: +// removing a blob that isn't spooled is not an error. Callers own the +// is-it-safe-to-delete question (shared, content-addressed blobs may be +// referenced by other parts or committed objects). +func (s *Spool) Remove(digest mh.Multihash) error { + err := os.Remove(s.Path(digest)) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("blockstore: spool remove: %w", err) + } + return nil +} + // WriteBlob streams r to the spool, computing its sha256 digest as it writes so // the blob is never held whole in memory (object-body blobs run up to // max_blob_size = 256 MiB; buffering them would put that × concurrency in RAM). diff --git a/bucket/cbor_gen.go b/bucket/cbor_gen.go index 351c892..52892a4 100644 --- a/bucket/cbor_gen.go +++ b/bucket/cbor_gen.go @@ -26,7 +26,7 @@ func (t *ObjectManifest) MarshalCBOR(w io.Writer) error { cw := cbg.NewCborWriter(w) - if _, err := cw.Write([]byte{174}); err != nil { + if _, err := cw.Write([]byte{175}); err != nil { return err } @@ -367,6 +367,29 @@ func (t *ObjectManifest) MarshalCBOR(w io.Writer) error { } } + + // t.WebsiteRedirectLocation (string) (string) + if len("wr") > 1000000 { + return xerrors.Errorf("Value in field \"wr\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("wr"))); err != nil { + return err + } + if _, err := cw.WriteString(string("wr")); err != nil { + return err + } + + if len(t.WebsiteRedirectLocation) > 1000000 { + return xerrors.Errorf("Value in field t.WebsiteRedirectLocation was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.WebsiteRedirectLocation))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.WebsiteRedirectLocation)); err != nil { + return err + } return nil } @@ -618,6 +641,17 @@ func (t *ObjectManifest) UnmarshalCBOR(r io.Reader) (err error) { t.Metadata[k] = v } + // t.WebsiteRedirectLocation (string) (string) + case "wr": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.WebsiteRedirectLocation = string(sval) + } default: // Field doesn't exist on this type, so ignore it diff --git a/bucket/manifest.go b/bucket/manifest.go index 5a2a998..007fc5d 100644 --- a/bucket/manifest.go +++ b/bucket/manifest.go @@ -46,6 +46,11 @@ type ObjectManifest struct { // unimplemented feature — see docs/architecture.md §12.) Expires string `cborgen:"ex"` + // WebsiteRedirectLocation is the x-amz-website-redirect-location system + // header, carried through PUT/CreateMultipartUpload and replayed on + // HEAD/GET like the other passthrough headers. + WebsiteRedirectLocation string `cborgen:"wr"` + // Metadata is the user metadata map (the x-amz-meta-* headers, with // the prefix stripped and keys lower-cased by the S3 layer). Nil when // the object carries no user metadata. diff --git a/config.go b/config.go index 2f9ae0f..12bfcdf 100644 --- a/config.go +++ b/config.go @@ -44,6 +44,12 @@ type Config struct { // Optional; defaults to DataDir. TokenStoreDir string `mapstructure:"token_store_dir" yaml:"token_store_dir"` + // MultipartSessionTTL bounds abandoned multipart uploads (Go duration + // string, e.g. "168h"): open sessions older than this are aborted by a + // background sweeper and their spooled parts dropped. Empty → default + // 7 days; a negative duration disables the sweeper. + MultipartSessionTTL string `mapstructure:"multipart_session_ttl" yaml:"multipart_session_ttl"` + // CatalogPlane overrides the catalog logstore pipeline knobs. Any field // left zero/unset falls back to the top-level SealBytes / SealAge / Retain // (and Ship defaults to true) — e.g. to configure the catalog never to ship. @@ -86,6 +92,13 @@ func (c Config) ServerConfig() (ServerConfig, error) { if err != nil { return ServerConfig{}, err } + var mpTTL time.Duration + if c.MultipartSessionTTL != "" { + mpTTL, err = time.ParseDuration(c.MultipartSessionTTL) + if err != nil { + return ServerConfig{}, fmt.Errorf("ingot: parse multipart_session_ttl %q: %w", c.MultipartSessionTTL, err) + } + } return ServerConfig{ Addr: c.Addr, DataDir: c.DataDir, @@ -101,6 +114,8 @@ func (c Config) ServerConfig() (ServerConfig, error) { SealAgeCatalog: catAge, ShipCatalog: shipDefault(c.CatalogPlane.Ship), RetainCatalog: firstNonZeroInt(c.CatalogPlane.Retain, c.Retain), + + MultipartSessionTTL: mpTTL, }, nil } diff --git a/docs/architecture.md b/docs/architecture.md index ebb8b31..7b28969 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -810,8 +810,15 @@ paths are stubbed in-tree and verified against a real sprue+piri+indexer later: - **Crash recovery for the spool is not built.** The `upload_intents` × `blob_refs` reconciliation the failure-mode table in [§7.5](#75-concurrency-durability-and-failure-modes) describes (resume/`unallocate` parked, `remove` accepted-but-unreferenced) is a later phase; a partial post-commit reference-index write currently relies on retry/idempotency. -- **`UploadPartCopy`, `ListParts`, `ListMultipartUploads`, and indexer retraction on delete** are - unimplemented (`ErrNotImplemented` / no-op). +- **`UploadPartCopy` and indexer retraction on delete** are unimplemented + (`ErrNotImplemented` / no-op). `ListParts` and `ListMultipartUploads` are implemented + (paginated, prefix/delimiter/marker semantics; in-flight sessions only). +- **Multipart hygiene (spool-model edition).** Abort and part re-upload delete the + now-unreferenced spooled blobs (guarded against content-addressed sharing with other + sessions and committed objects), and a background sweeper aborts open sessions older + than `multipart_session_ttl` (default 7d) and reaps terminal session rows. A successful + Complete retains its session in state `completed` so a duplicate Complete is idempotent + per S3. The network-side `unallocate`-on-abort remains a parking-flow concern (above). ### Known correctness boundary diff --git a/inmem/stores.go b/inmem/stores.go index 66114c7..e01bdc9 100644 --- a/inmem/stores.go +++ b/inmem/stores.go @@ -1,7 +1,10 @@ package inmem import ( + "bytes" "context" + "sort" + "time" "github.com/fil-forge/ingot/registry" ) @@ -157,6 +160,9 @@ func (m *MemStore) CreateSession(_ context.Context, s registry.MultipartSession) if s.State == "" { s.State = registry.SessionOpen } + if s.CreatedAt.IsZero() { + s.CreatedAt = time.Now() + } m.sessions[s.UploadID] = cloneSession(s) return nil } @@ -204,6 +210,9 @@ func (m *MemStore) PutPart(_ context.Context, p registry.MultipartPart) error { if p.State == "" { p.State = registry.PartParked } + if p.CreatedAt.IsZero() { + p.CreatedAt = time.Now() + } byNum := m.parts[p.UploadID] if byNum == nil { byNum = map[int]registry.MultipartPart{} @@ -237,6 +246,60 @@ func (m *MemStore) ListParts(_ context.Context, uploadID string) ([]registry.Mul return out, nil } +func (m *MemStore) ListSessions(_ context.Context, bucket string) ([]registry.MultipartSession, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []registry.MultipartSession + for _, s := range m.sessions { + if s.Bucket == bucket { + out = append(out, cloneSession(s)) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].ObjectKey != out[j].ObjectKey { + return out[i].ObjectKey < out[j].ObjectKey + } + if !out[i].CreatedAt.Equal(out[j].CreatedAt) { + return out[i].CreatedAt.Before(out[j].CreatedAt) + } + return out[i].UploadID < out[j].UploadID + }) + return out, nil +} + +func (m *MemStore) ListStaleSessions(_ context.Context, state string, cutoff time.Time) ([]registry.MultipartSession, error) { + m.mu.Lock() + defer m.mu.Unlock() + var out []registry.MultipartSession + for _, s := range m.sessions { + if s.State == state && s.CreatedAt.Before(cutoff) { + out = append(out, cloneSession(s)) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].CreatedAt.Before(out[j].CreatedAt) }) + return out, nil +} + +func (m *MemStore) CountPartRefs(_ context.Context, digest []byte, excludeUploadID string) (int, error) { + m.mu.Lock() + defer m.mu.Unlock() + n := 0 + for uploadID, byNum := range m.parts { + if uploadID == excludeUploadID { + continue + } + for _, p := range byNum { + for _, d := range p.BlobDigests { + if bytes.Equal(d, digest) { + n++ + break + } + } + } + } + return n, nil +} + // GCStore ==================================================================== func (m *MemStore) AddGCCandidate(_ context.Context, cidBytes []byte, _ string) error { diff --git a/itest/README.md b/itest/README.md index df06da0..8ceba93 100644 --- a/itest/README.md +++ b/itest/README.md @@ -39,7 +39,7 @@ here and add new cases to the pass table (demote to xfail if they fail). | Test | Covers | ~time | |---|---|---| -| `TestForgeScenarios` | Ingot-unique behaviors upstream can't assert, on a stack whose ingot config lowers `max_blob_size` to 64 KiB (`testdata/config-smallblob.yaml`): coarse blob-split round-trip with spooled-by-digest proof (spool counted inside the container), zero-byte objects, a multipart part spanning multiple internal blobs, and failed-Complete session recovery. | ~3 min | +| `TestForgeScenarios` | Ingot-unique behaviors upstream can't assert, on a stack whose ingot config lowers `max_blob_size` to 64 KiB (`testdata/config-smallblob.yaml`): coarse blob-split round-trip with spooled-by-digest proof (spool counted inside the container), zero-byte objects, a multipart part spanning multiple internal blobs, abort deleting the parts' spooled blobs, and failed-Complete session recovery. | ~3 min | | `TestForgeNativeProvision` | Guppy-free onboarding: `ingot login` + `space generate --provision-to` in the container, then a PUT/GET round-trip over the real ship path. | ~1.7 min | | `TestForgeReadAfterEviction` | The appliance read tier: PUT, wipe `/data/spool`, GET must re-fetch body blobs from piri via the local locator + `/content/retrieve`. | ~1.3 min | diff --git a/itest/scenarios_test.go b/itest/scenarios_test.go index a782b4f..3daee46 100644 --- a/itest/scenarios_test.go +++ b/itest/scenarios_test.go @@ -153,8 +153,10 @@ func TestForgeScenarios(t *testing.T) { } uploadID := create.UploadId - // Part 1 (150 KiB) spans three 64 KiB blobs; parts 2 and 3 are small. - partData := [][]byte{patternBytes(150 << 10), patternBytes(40 << 10), patternBytes(9 << 10)} + // Non-final parts must meet S3's 5 MiB minimum (enforced at Complete); + // at 64 KiB max_blob_size each spans dozens of internal blobs. The + // final part is small (exempt from the minimum). + partData := [][]byte{patternBytes(6 << 20), patternBytes(5 << 20), patternBytes(9 << 10)} var completed []types.CompletedPart var whole []byte for i, data := range partData { @@ -199,6 +201,47 @@ func TestForgeScenarios(t *testing.T) { } }) + // MultipartAbortCleansSpool: aborting an upload discards its parts — the + // registry rows go (upstream AbortMultipartUpload_success verifies via + // ListMultipartUploads) and, ingot-specifically, the parts' spooled blobs + // are deleted, since under the spool model an abort's cleanup is entirely + // local (nothing shipped to the network before Complete). + t.Run("MultipartAbortCleansSpool", func(t *testing.T) { + const bucket, key = "mpabort-spool", "obj" + if _, err := cl.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { + t.Fatalf("CreateBucket: %v", err) + } + create, err := cl.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{Bucket: aws.String(bucket), Key: aws.String(key)}) + if err != nil { + t.Fatalf("CreateMultipartUpload: %v", err) + } + spoolBefore := spoolBlobCount(t, ctx, s) + // A 150 KiB part spans three 64 KiB blobs. The content must be unique + // to this test — patternBytes at the same offsets would dedup against + // the round-trip subtests' already-spooled blobs and skew the counts. + part := patternBytes(150 << 10) + for i := range part { + part[i] ^= 0xA5 + } + if _, err := cl.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + PartNumber: aws.Int32(1), Body: bytes.NewReader(part), + }); err != nil { + t.Fatalf("UploadPart: %v", err) + } + if got := spoolBlobCount(t, ctx, s) - spoolBefore; got != 3 { + t.Fatalf("UploadPart added %d spool blobs, want 3", got) + } + if _, err := cl.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + }); err != nil { + t.Fatalf("AbortMultipartUpload: %v", err) + } + if got := spoolBlobCount(t, ctx, s) - spoolBefore; got != 0 { + t.Fatalf("abort left %d spooled part blobs behind, want 0", got) + } + }) + // MultipartFailedCompleteStaysAbortable: the zombie-session regression. A // Complete that fails validation (wrong part ETag) must leave the session // open — a retry with the correct ETag succeeds and reads back diff --git a/itest/versity_multipart_test.go b/itest/versity_multipart_test.go index dae7ce9..68c2ec8 100644 --- a/itest/versity_multipart_test.go +++ b/itest/versity_multipart_test.go @@ -6,16 +6,16 @@ import ( "github.com/versity/versitygw/tests/integration" ) -// Multipart groups of the S3 conformance partition. These had no prior -// curated lists — the partition below was created empirically against the -// forge-mode stack (see the curation note in README.md). The broad strokes: -// the Create/UploadPart/Complete/Abort happy paths and most input validation -// pass; part-level checksums, tagging/object-lock/ACL surfaces, and the -// ListParts / ListMultipartUploads / UploadPartCopy groups are unimplemented. +// Multipart groups of the S3 conformance partition, partitioned empirically +// against the forge-mode stack (see the curation note in README.md). The +// remaining xfail surface: part-level checksums (FIL-620), tagging/object-lock +// /ACL on create (FIL-534/FIL-525), and the UploadPartCopy group (FIL-586). var createMultipartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.CreateMultipartUpload_non_existing_bucket}, + {name: "dir_obj", fn: integration.CreateMultipartUpload_dir_obj}, {name: "long_metadata", fn: integration.CreateMultipartUpload_long_metadata}, + {name: "with_metadata", fn: integration.CreateMultipartUpload_with_metadata}, {name: "with_object_lock_invalid_retention", fn: integration.CreateMultipartUpload_with_object_lock_invalid_retention}, {name: "past_retain_until_date", fn: integration.CreateMultipartUpload_past_retain_until_date}, {name: "invalid_legal_hold", fn: integration.CreateMultipartUpload_invalid_legal_hold}, @@ -29,8 +29,6 @@ var createMultipartPass = []forgeCase{ } var createMultipartXFail = []forgeCase{ - {name: "dir_obj", fn: integration.CreateMultipartUpload_dir_obj}, - {name: "with_metadata", fn: integration.CreateMultipartUpload_with_metadata}, {name: "with_tagging", fn: integration.CreateMultipartUpload_with_tagging}, {name: "with_object_lock", fn: integration.CreateMultipartUpload_with_object_lock}, {name: "with_object_lock_not_enabled", fn: integration.CreateMultipartUpload_with_object_lock_not_enabled}, @@ -40,6 +38,7 @@ var createMultipartXFail = []forgeCase{ var uploadPartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.UploadPart_non_existing_bucket}, {name: "invalid_part_number", fn: integration.UploadPart_invalid_part_number}, + {name: "non_existing_key", fn: integration.UploadPart_non_existing_key}, {name: "non_existing_mp_upload", fn: integration.UploadPart_non_existing_mp_upload}, {name: "multiple_checksum_headers", fn: integration.UploadPart_multiple_checksum_headers}, {name: "invalid_checksum_header", fn: integration.UploadPart_invalid_checksum_header}, @@ -48,7 +47,6 @@ var uploadPartPass = []forgeCase{ } var uploadPartXFail = []forgeCase{ - {name: "non_existing_key", fn: integration.UploadPart_non_existing_key}, {name: "checksum_algorithm_mistmatch_on_initialization", fn: integration.UploadPart_checksum_algorithm_mistmatch_on_initialization}, {name: "checksum_algorithm_mistmatch_on_initialization_with_value", fn: integration.UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value}, {name: "incorrect_checksums", fn: integration.UploadPart_incorrect_checksums}, @@ -82,28 +80,26 @@ var uploadPartCopyXFail = []forgeCase{ } var listPartsPass = []forgeCase{ - {name: "invalid_max_parts", fn: integration.ListParts_invalid_max_parts}, - {name: "invalid_part_number_marker", fn: integration.ListParts_invalid_part_number_marker}, -} - -var listPartsXFail = []forgeCase{ {name: "incorrect_uploadId", fn: integration.ListParts_incorrect_uploadId}, {name: "incorrect_object_key", fn: integration.ListParts_incorrect_object_key}, + {name: "invalid_max_parts", fn: integration.ListParts_invalid_max_parts}, + {name: "invalid_part_number_marker", fn: integration.ListParts_invalid_part_number_marker}, {name: "default_max_parts", fn: integration.ListParts_default_max_parts}, {name: "exceeding_max_parts", fn: integration.ListParts_exceeding_max_parts}, {name: "truncated", fn: integration.ListParts_truncated}, + {name: "success", fn: integration.ListParts_success}, {name: "with_checksums", fn: integration.ListParts_with_checksums}, +} + +// Explicit null-checksum-type echo is FIL-620. +var listPartsXFail = []forgeCase{ {name: "null_checksums", fn: integration.ListParts_null_checksums}, - {name: "success", fn: integration.ListParts_success}, } var listMultipartUploadsPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.ListMultipartUploads_non_existing_bucket}, - {name: "invalid_max_uploads", fn: integration.ListMultipartUploads_invalid_max_uploads}, -} - -var listMultipartUploadsXFail = []forgeCase{ {name: "empty_result", fn: integration.ListMultipartUploads_empty_result}, + {name: "invalid_max_uploads", fn: integration.ListMultipartUploads_invalid_max_uploads}, {name: "max_uploads", fn: integration.ListMultipartUploads_max_uploads}, {name: "exceeding_max_uploads", fn: integration.ListMultipartUploads_exceeding_max_uploads}, {name: "ignore_upload_id_marker", fn: integration.ListMultipartUploads_ignore_upload_id_marker}, @@ -116,35 +112,38 @@ var listMultipartUploadsXFail = []forgeCase{ {name: "with_checksums", fn: integration.ListMultipartUploads_with_checksums}, } +var listMultipartUploadsXFail = []forgeCase{} + var abortMultipartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.AbortMultipartUpload_non_existing_bucket}, {name: "incorrect_uploadId", fn: integration.AbortMultipartUpload_incorrect_uploadId}, {name: "incorrect_object_key", fn: integration.AbortMultipartUpload_incorrect_object_key}, - {name: "success_status_code", fn: integration.AbortMultipartUpload_success_status_code}, -} - -var abortMultipartXFail = []forgeCase{ - // success asserts the aborted upload disappears from ListMultipartUploads, - // which is unimplemented (see listMultipartUploadsXFail); the abort - // lifecycle itself is proven by TestForgeScenarios. {name: "success", fn: integration.AbortMultipartUpload_success}, + {name: "success_status_code", fn: integration.AbortMultipartUpload_success_status_code}, {name: "if_match_initiated_time", fn: integration.AbortMultipartUpload_if_match_initiated_time}, } +var abortMultipartXFail = []forgeCase{} + var completeMultipartPass = []forgeCase{ // upstream function name carries a typo (CompletedMultipartUpload_...). {name: "non_existing_bucket", fn: integration.CompletedMultipartUpload_non_existing_bucket}, {name: "incorrect_part_number", fn: integration.CompleteMultipartUpload_incorrect_part_number}, + {name: "invalid_part_number", fn: integration.CompleteMultipartUpload_invalid_part_number}, {name: "default_content_type", fn: integration.CompleteMultipartUpload_default_content_type}, {name: "invalid_ETag", fn: integration.CompleteMultipartUpload_invalid_ETag}, + {name: "small_upload_size", fn: integration.CompleteMultipartUpload_small_upload_size}, {name: "empty_parts", fn: integration.CompleteMultipartUpload_empty_parts}, {name: "incorrect_parts_order", fn: integration.CompleteMultipartUpload_incorrect_parts_order}, + {name: "mpu_object_size", fn: integration.CompleteMultipartUpload_mpu_object_size}, + {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, {name: "invalid_checksum_type", fn: integration.CompleteMultipartUpload_invalid_checksum_type}, {name: "multiple_final_checksums", fn: integration.CompleteMultipartUpload_multiple_final_checksums}, {name: "invalid_final_checksums", fn: integration.CompleteMultipartUpload_invalid_final_checksums}, {name: "invalid_final_composite_checksum", fn: integration.CompleteMultipartUpload_invalid_final_composite_checksum}, {name: "with_metadata", fn: integration.CompleteMultipartUpload_with_metadata}, {name: "success", fn: integration.CompleteMultipartUpload_success}, + {name: "already_completed", fn: integration.CompleteMultipartUpload_already_completed}, // racey_success races ten concurrent 25 MiB multipart uploads of one key // under a 30s client deadline — on a host simultaneously running the // smelt stack its outcome depends on load, not S3 semantics, so it is @@ -154,11 +153,9 @@ var completeMultipartPass = []forgeCase{ }}, } +// Part-level / composite-checksum verification is FIL-620; +// racey_data_integrity additionally leans on atomic concurrent overwrites. var completeMultipartXFail = []forgeCase{ - {name: "invalid_part_number", fn: integration.CompleteMultipartUpload_invalid_part_number}, - {name: "small_upload_size", fn: integration.CompleteMultipartUpload_small_upload_size}, - {name: "mpu_object_size", fn: integration.CompleteMultipartUpload_mpu_object_size}, - {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, {name: "invalid_checksum_part", fn: integration.CompleteMultipartUpload_invalid_checksum_part}, {name: "multiple_checksum_part", fn: integration.CompleteMultipartUpload_multiple_checksum_part}, {name: "incorrect_checksum_part", fn: integration.CompleteMultipartUpload_incorrect_checksum_part}, @@ -171,6 +168,5 @@ var completeMultipartXFail = []forgeCase{ {name: "checksum_type_mismatch", fn: integration.CompleteMultipartUpload_checksum_type_mismatch}, {name: "should_ignore_the_final_checksum", fn: integration.CompleteMultipartUpload_should_ignore_the_final_checksum}, {name: "should_succeed_without_final_checksum_type", fn: integration.CompleteMultipartUpload_should_succeed_without_final_checksum_type}, - {name: "already_completed", fn: integration.CompleteMultipartUpload_already_completed}, {name: "racey_data_integrity", fn: integration.CompleteMultipartUpload_racey_data_integrity}, } diff --git a/migrations/sql/00004_multipart_listing.sql b/migrations/sql/00004_multipart_listing.sql new file mode 100644 index 0000000..b79294f --- /dev/null +++ b/migrations/sql/00004_multipart_listing.sql @@ -0,0 +1,48 @@ +-- +goose Up +-- Multipart listing + completion idempotency (FIL-520): +-- * multipart_parts.created_at backs ListParts' per-part LastModified; +-- sessions already carry created_at for ListMultipartUploads' Initiated. +-- * Sessions gain the standard HTTP metadata headers CreateMultipartUpload +-- may carry, so Complete writes them into the manifest exactly like a +-- single-shot PUT does. +-- * A 'completed' session state: Complete now retains the session (and its +-- parts) instead of deleting it, so a duplicate CompleteMultipartUpload +-- with identical parts is idempotent per S3. The abandoned-session +-- sweeper reaps completed rows after the TTL. + +ALTER TABLE ingot.multipart_parts + ADD COLUMN created_at timestamptz NOT NULL DEFAULT now(); + +ALTER TABLE ingot.multipart_sessions + ADD COLUMN content_encoding text, + ADD COLUMN content_disposition text, + ADD COLUMN content_language text, + ADD COLUMN cache_control text, + ADD COLUMN expires text, + ADD COLUMN website_redirect_location text, + ADD COLUMN checksum_algorithm text, + ADD COLUMN checksum_type text; + +ALTER TABLE ingot.multipart_sessions + DROP CONSTRAINT multipart_sessions_state_check, + ADD CONSTRAINT multipart_sessions_state_check + CHECK (state IN ('open','completing','aborting','completed')); + +-- +goose Down +ALTER TABLE ingot.multipart_sessions + DROP CONSTRAINT multipart_sessions_state_check, + ADD CONSTRAINT multipart_sessions_state_check + CHECK (state IN ('open','completing','aborting')); + +ALTER TABLE ingot.multipart_sessions + DROP COLUMN content_encoding, + DROP COLUMN content_disposition, + DROP COLUMN content_language, + DROP COLUMN cache_control, + DROP COLUMN expires, + DROP COLUMN website_redirect_location, + DROP COLUMN checksum_algorithm, + DROP COLUMN checksum_type; + +ALTER TABLE ingot.multipart_parts + DROP COLUMN created_at; diff --git a/registry/postgres_live_test.go b/registry/postgres_live_test.go index e3d76af..8901843 100644 --- a/registry/postgres_live_test.go +++ b/registry/postgres_live_test.go @@ -187,6 +187,71 @@ func TestPostgresStores_Live(t *testing.T) { } }) + t.Run("multipart listing sweeper and part refs", func(t *testing.T) { + mk := func(id, key string) { + t.Helper() + if err := r.CreateSession(ctx, registry.MultipartSession{ + UploadID: id, Bucket: "b", ObjectKey: key, + ContentEncoding: "testenc", ChecksumAlgorithm: "CRC32", ChecksumType: "FULL_OBJECT", + }); err != nil { + t.Fatalf("CreateSession %s: %v", id, err) + } + } + mk("ls-2", "zeta") + mk("ls-1", "alpha") + mk("ls-3", "alpha") // same key, created after ls-1 + + // New session columns round-trip. + s, err := r.GetSession(ctx, "ls-1") + if err != nil || s.ContentEncoding != "testenc" || s.ChecksumAlgorithm != "CRC32" || + s.ChecksumType != "FULL_OBJECT" || s.CreatedAt.IsZero() { + t.Fatalf("GetSession new columns = %+v, err %v", s, err) + } + + // ListSessions: (object_key, created_at, upload_id) order. + sessions, err := r.ListSessions(ctx, "b") + if err != nil || len(sessions) != 3 || + sessions[0].UploadID != "ls-1" || sessions[1].UploadID != "ls-3" || sessions[2].UploadID != "ls-2" { + ids := make([]string, len(sessions)) + for i, x := range sessions { + ids[i] = x.UploadID + } + t.Fatalf("ListSessions order = %v, err %v (want [ls-1 ls-3 ls-2])", ids, err) + } + + // ListStaleSessions: cutoff in the past excludes them, future includes. + if stale, err := r.ListStaleSessions(ctx, registry.SessionOpen, time.Now().Add(-time.Hour)); err != nil || len(stale) != 0 { + t.Fatalf("ListStaleSessions past cutoff = %d, err %v (want 0)", len(stale), err) + } + if stale, err := r.ListStaleSessions(ctx, registry.SessionOpen, time.Now().Add(time.Hour)); err != nil || len(stale) != 3 { + t.Fatalf("ListStaleSessions future cutoff = %d, err %v (want 3)", len(stale), err) + } + + // CountPartRefs: bytea[] ANY-match across sessions, excluding one. + shared := []byte{0xee, 0x01} + if err := r.PutPart(ctx, registry.MultipartPart{UploadID: "ls-1", PartNumber: 1, ETagMD5: []byte{1}, Size: 1, BlobDigests: [][]byte{shared}}); err != nil { + t.Fatalf("PutPart ls-1: %v", err) + } + if err := r.PutPart(ctx, registry.MultipartPart{UploadID: "ls-2", PartNumber: 1, ETagMD5: []byte{2}, Size: 1, BlobDigests: [][]byte{shared, {0xee, 0x02}}}); err != nil { + t.Fatalf("PutPart ls-2: %v", err) + } + if n, err := r.CountPartRefs(ctx, shared, "ls-1"); err != nil || n != 1 { + t.Fatalf("CountPartRefs(shared, exclude ls-1) = %d, err %v (want 1)", n, err) + } + if n, err := r.CountPartRefs(ctx, []byte{0xee, 0x02}, "ls-2"); err != nil || n != 0 { + t.Fatalf("CountPartRefs(unique, exclude owner) = %d, err %v (want 0)", n, err) + } + + // 'completed' passes the widened state CHECK constraint. + if won, err := r.LatchSession(ctx, "ls-1", registry.SessionOpen, registry.SessionCompleted); err != nil || !won { + t.Fatalf("latch to completed won=%v err=%v", won, err) + } + + for _, id := range []string{"ls-1", "ls-2", "ls-3"} { + _ = r.DeleteSession(ctx, id) + } + }) + t.Run("gc candidate idempotent", func(t *testing.T) { if err := r.AddGCCandidate(ctx, digest, "b"); err != nil { t.Fatalf("AddGCCandidate: %v", err) diff --git a/registry/stores.go b/registry/stores.go index 0deba49..167b0ab 100644 --- a/registry/stores.go +++ b/registry/stores.go @@ -1,6 +1,9 @@ package registry -import "context" +import ( + "context" + "time" +) // This file defines the relational surface the upload/storage/delete // architecture relies on (docs/architecture.md §5–§7, Appendix C): @@ -28,6 +31,10 @@ const ( SessionOpen = "open" SessionCompleting = "completing" SessionAborting = "aborting" + // SessionCompleted: the object committed; the session and its parts are + // retained so a duplicate CompleteMultipartUpload with identical parts is + // idempotent (S3 semantics). Reaped by the abandoned-session sweeper. + SessionCompleted = "completed" ) // multipart_parts.state values (§7.2). @@ -68,14 +75,28 @@ type BlobLocation struct { Size int64 } -// MultipartSession is one row of ingot.multipart_sessions. +// MultipartSession is one row of ingot.multipart_sessions. The HTTP metadata +// headers (ContentEncoding..Expires) are captured at CreateMultipartUpload so +// Complete can write them into the manifest exactly like a single-shot PUT. type MultipartSession struct { - UploadID string - Bucket string - ObjectKey string - State string - ContentType string - Metadata map[string]string + UploadID string + Bucket string + ObjectKey string + State string + ContentType string + ContentEncoding string + ContentDisposition string + ContentLanguage string + CacheControl string + Expires string + WebsiteRedirectLocation string + // ChecksumAlgorithm/ChecksumType are the x-amz-checksum-* declarations from + // CreateMultipartUpload, echoed by ListMultipartUploads. Per-part checksum + // computation is separate (FIL-620). + ChecksumAlgorithm string + ChecksumType string + Metadata map[string]string + CreatedAt time.Time } // MultipartPart is one row of ingot.multipart_parts. BlobDigests is the @@ -88,6 +109,7 @@ type MultipartPart struct { Size int64 BlobDigests [][]byte State string + CreatedAt time.Time } // BlobRefStore is the reverse reference index (§5, §6). A commit adds a @@ -133,6 +155,18 @@ type MultipartStore interface { DeleteSession(ctx context.Context, uploadID string) error PutPart(ctx context.Context, p MultipartPart) error ListParts(ctx context.Context, uploadID string) ([]MultipartPart, error) + // ListSessions returns bucket's sessions ordered by (object_key, created_at, + // upload_id) — the S3 ListMultipartUploads presentation order. Filtering + // (prefix/markers/max) happens in the handler; in-flight session counts are + // small. + ListSessions(ctx context.Context, bucket string) ([]MultipartSession, error) + // ListStaleSessions returns sessions in `state` created before cutoff, for + // the abandoned-upload sweeper. + ListStaleSessions(ctx context.Context, state string, cutoff time.Time) ([]MultipartSession, error) + // CountPartRefs returns how many parts OUTSIDE excludeUploadID reference + // digest — the shared-blob guard for abort/supersede spool cleanup + // (content-addressed part blobs may be deduped across sessions). + CountPartRefs(ctx context.Context, digest []byte, excludeUploadID string) (int, error) } // GCStore records superseded MST node CIDs (§4). Write-only this iteration. diff --git a/registry/stores_postgres.go b/registry/stores_postgres.go index bbc3c9e..8470c7e 100644 --- a/registry/stores_postgres.go +++ b/registry/stores_postgres.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "time" "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" @@ -190,9 +191,15 @@ func (r *Postgres) CreateSession(ctx context.Context, s MultipartSession) error return err } _, err = r.pool.Exec(ctx, - `INSERT INTO ingot.multipart_sessions (upload_id, bucket, object_key, state, content_type, metadata) - VALUES ($1, $2, $3, $4, $5, $6)`, - s.UploadID, s.Bucket, s.ObjectKey, state, nullString(s.ContentType), meta) + `INSERT INTO ingot.multipart_sessions + (upload_id, bucket, object_key, state, content_type, metadata, + content_encoding, content_disposition, content_language, cache_control, expires, + website_redirect_location, checksum_algorithm, checksum_type) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`, + s.UploadID, s.Bucket, s.ObjectKey, state, nullString(s.ContentType), meta, + nullString(s.ContentEncoding), nullString(s.ContentDisposition), + nullString(s.ContentLanguage), nullString(s.CacheControl), nullString(s.Expires), + nullString(s.WebsiteRedirectLocation), nullString(s.ChecksumAlgorithm), nullString(s.ChecksumType)) if err != nil { var pgErr *pgconn.PgError if errors.As(err, &pgErr) && pgErr.Code == uniqueViolation { @@ -204,22 +211,47 @@ func (r *Postgres) CreateSession(ctx context.Context, s MultipartSession) error } func (r *Postgres) GetSession(ctx context.Context, uploadID string) (*MultipartSession, error) { - s := &MultipartSession{UploadID: uploadID} - var contentType *string - var meta []byte - err := r.pool.QueryRow(ctx, - `SELECT bucket, object_key, state, content_type, metadata + row := r.pool.QueryRow(ctx, + `SELECT upload_id, bucket, object_key, state, content_type, metadata, created_at, + content_encoding, content_disposition, content_language, cache_control, expires, + website_redirect_location, checksum_algorithm, checksum_type FROM ingot.multipart_sessions WHERE upload_id = $1`, - uploadID).Scan(&s.Bucket, &s.ObjectKey, &s.State, &contentType, &meta) + uploadID) + s, err := scanSession(row) if errors.Is(err, pgx.ErrNoRows) { return nil, ErrNotFound } if err != nil { return nil, fmt.Errorf("registry: get session: %w", err) } - if contentType != nil { - s.ContentType = *contentType + return s, nil +} + +// scanSession scans one multipart_sessions row in the canonical column order +// (see GetSession/ListSessions selects). +func scanSession(row pgx.Row) (*MultipartSession, error) { + s := &MultipartSession{} + var contentType, ce, cd, cl, cc, exp, wrl, ckAlgo, ckType *string + var meta []byte + err := row.Scan(&s.UploadID, &s.Bucket, &s.ObjectKey, &s.State, &contentType, &meta, &s.CreatedAt, + &ce, &cd, &cl, &cc, &exp, &wrl, &ckAlgo, &ckType) + if err != nil { + return nil, err + } + setIfNotNil := func(dst *string, src *string) { + if src != nil { + *dst = *src + } } + setIfNotNil(&s.ContentType, contentType) + setIfNotNil(&s.ContentEncoding, ce) + setIfNotNil(&s.ContentDisposition, cd) + setIfNotNil(&s.ContentLanguage, cl) + setIfNotNil(&s.CacheControl, cc) + setIfNotNil(&s.Expires, exp) + setIfNotNil(&s.WebsiteRedirectLocation, wrl) + setIfNotNil(&s.ChecksumAlgorithm, ckAlgo) + setIfNotNil(&s.ChecksumType, ckType) if s.Metadata, err = unmarshalMetadata(meta); err != nil { return nil, err } @@ -264,7 +296,7 @@ func (r *Postgres) PutPart(ctx context.Context, p MultipartPart) error { func (r *Postgres) ListParts(ctx context.Context, uploadID string) ([]MultipartPart, error) { rows, err := r.pool.Query(ctx, - `SELECT part_number, etag_md5, size, blob_digests, state + `SELECT part_number, etag_md5, size, blob_digests, state, created_at FROM ingot.multipart_parts WHERE upload_id = $1 ORDER BY part_number ASC`, uploadID) if err != nil { @@ -275,7 +307,7 @@ func (r *Postgres) ListParts(ctx context.Context, uploadID string) ([]MultipartP var out []MultipartPart for rows.Next() { p := MultipartPart{UploadID: uploadID} - if err := rows.Scan(&p.PartNumber, &p.ETagMD5, &p.Size, &p.BlobDigests, &p.State); err != nil { + if err := rows.Scan(&p.PartNumber, &p.ETagMD5, &p.Size, &p.BlobDigests, &p.State, &p.CreatedAt); err != nil { return nil, fmt.Errorf("registry: list parts scan: %w", err) } out = append(out, p) @@ -286,6 +318,72 @@ func (r *Postgres) ListParts(ctx context.Context, uploadID string) ([]MultipartP return out, nil } +func (r *Postgres) ListSessions(ctx context.Context, bucket string) ([]MultipartSession, error) { + rows, err := r.pool.Query(ctx, + `SELECT upload_id, bucket, object_key, state, content_type, metadata, created_at, + content_encoding, content_disposition, content_language, cache_control, expires, + website_redirect_location, checksum_algorithm, checksum_type + FROM ingot.multipart_sessions WHERE bucket = $1 + ORDER BY object_key ASC, created_at ASC, upload_id ASC`, + bucket) + if err != nil { + return nil, fmt.Errorf("registry: list sessions: %w", err) + } + defer rows.Close() + + var out []MultipartSession + for rows.Next() { + s, err := scanSession(rows) + if err != nil { + return nil, fmt.Errorf("registry: list sessions scan: %w", err) + } + out = append(out, *s) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("registry: list sessions rows: %w", err) + } + return out, nil +} + +func (r *Postgres) ListStaleSessions(ctx context.Context, state string, cutoff time.Time) ([]MultipartSession, error) { + rows, err := r.pool.Query(ctx, + `SELECT upload_id, bucket, object_key, state, content_type, metadata, created_at, + content_encoding, content_disposition, content_language, cache_control, expires, + website_redirect_location, checksum_algorithm, checksum_type + FROM ingot.multipart_sessions WHERE state = $1 AND created_at < $2 + ORDER BY created_at ASC`, + state, cutoff) + if err != nil { + return nil, fmt.Errorf("registry: list stale sessions: %w", err) + } + defer rows.Close() + + var out []MultipartSession + for rows.Next() { + s, err := scanSession(rows) + if err != nil { + return nil, fmt.Errorf("registry: list stale sessions scan: %w", err) + } + out = append(out, *s) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("registry: list stale sessions rows: %w", err) + } + return out, nil +} + +func (r *Postgres) CountPartRefs(ctx context.Context, digest []byte, excludeUploadID string) (int, error) { + var n int + err := r.pool.QueryRow(ctx, + `SELECT count(*) FROM ingot.multipart_parts + WHERE $1 = ANY(blob_digests) AND upload_id <> $2`, + digest, excludeUploadID).Scan(&n) + if err != nil { + return 0, fmt.Errorf("registry: count part refs: %w", err) + } + return n, nil +} + // GCStore ==================================================================== func (r *Postgres) AddGCCandidate(ctx context.Context, cidBytes []byte, bucket string) error { diff --git a/s3frontend/multipart.go b/s3frontend/multipart.go index 69920b4..e4a207a 100644 --- a/s3frontend/multipart.go +++ b/s3frontend/multipart.go @@ -12,7 +12,9 @@ import ( "time" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/ipfs/go-cid" + mh "github.com/multiformats/go-multihash" "github.com/versity/versitygw/backend" "github.com/versity/versitygw/s3err" "github.com/versity/versitygw/s3response" @@ -23,6 +25,13 @@ import ( "github.com/fil-forge/ingot/registry" ) +// minPartSize is S3's minimum size for every part except the last one, +// enforced at CompleteMultipartUpload (EntityTooSmall). +const minPartSize = 5 << 20 + +// defaultMaxListing is the S3 default and cap for max-parts / max-uploads. +const defaultMaxListing = 1000 + // newUploadID returns a random 128-bit hex upload id. func newUploadID() (string, error) { var b [16]byte @@ -33,8 +42,9 @@ func newUploadID() (string, error) { } // CreateMultipartUpload opens a multipart session: it records the destination -// bucket/key plus the content-type and user metadata so Complete can write the -// manifest without the client resupplying them, and returns the upload id. +// bucket/key plus the content-type, the passthrough HTTP metadata headers, and +// user metadata so Complete can write the manifest without the client +// resupplying them, and returns the upload id. func (b *Backend) CreateMultipartUpload(ctx context.Context, input s3response.CreateMultipartUploadInput) (s3response.InitiateMultipartUploadResult, error) { if input.Bucket == nil || input.Key == nil { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -43,6 +53,11 @@ func (b *Backend) CreateMultipartUpload(ctx context.Context, input s3response.Cr if !mst.IsValidKey(key) { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrInvalidArgument) } + // A directory object (trailing "/") is zero-length by definition; a + // multipart upload to one necessarily carries data. + if strings.HasSuffix(key, "/") { + return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrDirectoryObjectContainsData) + } if _, err := b.reg.Get(ctx, bucket); err != nil { if errors.Is(err, registry.ErrNotFound) { return s3response.InitiateMultipartUploadResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) @@ -58,34 +73,75 @@ func (b *Backend) CreateMultipartUpload(ctx context.Context, input s3response.Cr ct = "application/octet-stream" } if err := b.multipart.CreateSession(ctx, registry.MultipartSession{ - UploadID: uploadID, - Bucket: bucket, - ObjectKey: key, - State: registry.SessionOpen, - ContentType: ct, - Metadata: input.Metadata, + UploadID: uploadID, + Bucket: bucket, + ObjectKey: key, + State: registry.SessionOpen, + ContentType: ct, + ContentEncoding: backend.GetStringFromPtr(input.ContentEncoding), + ContentDisposition: backend.GetStringFromPtr(input.ContentDisposition), + ContentLanguage: backend.GetStringFromPtr(input.ContentLanguage), + CacheControl: backend.GetStringFromPtr(input.CacheControl), + Expires: backend.GetStringFromPtr(input.Expires), + WebsiteRedirectLocation: backend.GetStringFromPtr(input.WebsiteRedirectLocation), + ChecksumAlgorithm: string(input.ChecksumAlgorithm), + ChecksumType: string(input.ChecksumType), + Metadata: input.Metadata, }); err != nil { return s3response.InitiateMultipartUploadResult{}, fmt.Errorf("s3frontend: create session: %w", err) } return s3response.InitiateMultipartUploadResult{Bucket: bucket, Key: key, UploadId: uploadID}, nil } +// openSession fetches uploadID's session and maps anything that is not an +// in-flight upload for (bucket, key) to NoSuchUpload: unknown id, a key that +// doesn't match the session's, or a session no longer open (completed uploads +// are retained for Complete idempotency but are gone as far as the other +// multipart operations are concerned). +func (b *Backend) openSession(ctx context.Context, uploadID string, key *string) (*registry.MultipartSession, error) { + sess, err := b.multipart.GetSession(ctx, uploadID) + if err != nil { + if errors.Is(err, registry.ErrNotFound) { + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + return nil, fmt.Errorf("s3frontend: get session: %w", err) + } + if key != nil && *key != sess.ObjectKey { + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + if sess.State != registry.SessionOpen { + return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + return sess, nil +} + // UploadPart ingests one part: it coarse-splits the part body into blobs and // spools each to local disk (recording upload_intents), then records the part // (its ordered blob digests, md5, size). The part's blobs are NOT uploaded to // Forge yet — that is deferred to Complete (so an Abort only ever cleans up -// local state). The part ETag is the hex md5 of the part bytes. +// local state). Re-uploading a part number supersedes the prior part; the +// superseded part's now-unreferenced blobs are dropped from the spool. The +// part ETag is the hex md5 of the part bytes. func (b *Backend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { if input.Bucket == nil || input.Key == nil || input.UploadId == nil || input.PartNumber == nil { return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) } uploadID := *input.UploadId - sess, err := b.multipart.GetSession(ctx, uploadID) + sess, err := b.openSession(ctx, uploadID, input.Key) if err != nil { - if errors.Is(err, registry.ErrNotFound) { - return nil, s3err.GetAPIError(s3err.ErrNoSuchUpload) + return nil, err + } + + // Capture the superseded part's blobs (if any) before overwriting, so + // last-write-wins doesn't strand its spool files. + var superseded [][]byte + if prior, err := b.multipart.ListParts(ctx, uploadID); err == nil { + for _, p := range prior { + if p.PartNumber == int(*input.PartNumber) { + superseded = p.BlobDigests + break + } } - return nil, fmt.Errorf("s3frontend: upload part: %w", err) } body, err := b.splitSpool(ctx, sess.Bucket, input.Body) @@ -102,6 +158,9 @@ func (b *Backend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s }); err != nil { return nil, fmt.Errorf("s3frontend: record part: %w", err) } + if len(superseded) > 0 { + b.cleanupPartBlobs(ctx, uploadID, superseded) + } etag := `"` + hex.EncodeToString(body.MD5) + `"` return &s3.UploadPartOutput{ETag: &etag}, nil } @@ -111,6 +170,10 @@ func (b *Backend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s // list against the recorded parts, accepts every part's blobs on Forge, and // commits a manifest whose Body is the ordered union of the parts' blobs. The // object ETag is hex(md5(concat of part md5s)) + "-N". +// +// A successful Complete retains the session in state 'completed' (with its +// parts), so a duplicate Complete with an identical part list is idempotent +// per S3; the abandoned-session sweeper reaps the row later. func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (s3response.CompleteMultipartUploadResult, string, error) { if input.Bucket == nil || input.Key == nil || input.UploadId == nil { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -123,25 +186,32 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet } return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("s3frontend: complete: %w", err) } - // Single-winner latch vs a racing Abort: only the writer that moves the - // session off 'open' proceeds (§7.3). - won, err := b.multipart.LatchSession(ctx, uploadID, registry.SessionOpen, registry.SessionCompleting) - if err != nil { - return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("s3frontend: latch: %w", err) - } - if !won { - return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNoSuchUpload) + + // Conditional writes: S3 documents only If-None-Match: * (don't overwrite) + // and If-Match (require current ETag) for Complete; a concrete If-None-Match + // value — or combining If-None-Match with If-Match — is NotImplemented. + ifMatch, ifNoneMatch := input.IfMatch, input.IfNoneMatch + if ifNoneMatch != nil && (*ifNoneMatch != "*" || ifMatch != nil) { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNotImplemented) } - // If anything below fails before the object is committed, revert the session - // to 'open' so the upload stays abortable / retriable rather than zombied in - // 'completing'. committed is set once the manifest is durable (the point of - // no return). - committed := false - defer func() { - if !committed { - _, _ = b.multipart.LatchSession(ctx, uploadID, registry.SessionCompleting, registry.SessionOpen) + if ifMatch != nil || ifNoneMatch != nil { + current, lerr := b.lookupManifest(ctx, bucket, key) + exists := lerr == nil + if lerr != nil && !isNoSuchKey(lerr) { + return s3response.CompleteMultipartUploadResult{}, "", lerr } - }() + if ifMatch != nil { + if !exists { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNoSuchKey) + } + if !etagsEqual(*ifMatch, current.ETag) { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + } + if ifNoneMatch != nil && exists { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrPreconditionFailed) + } + } if input.MultipartUpload == nil || len(input.MultipartUpload.Parts) == 0 { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) @@ -155,11 +225,9 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet byNum[p.PartNumber] = p } - // Validate the requested parts (ascending, each matching a recorded part by - // number + ETag) and assemble the ordered body + the multipart ETag. - var blobs []msbucket.BlobRef - var partSizes []int64 - var offset int64 + // Validate the requested parts (in-range and ascending, each matching a + // recorded part by number + ETag) and compute the multipart ETag. + requested := make([]registry.MultipartPart, 0, len(input.MultipartUpload.Parts)) etagHasher := md5.New() prev := 0 for _, rp := range input.MultipartUpload.Parts { @@ -167,6 +235,10 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) } num := int(*rp.PartNumber) + if num < 1 || num > 10000 { + return s3response.CompleteMultipartUploadResult{}, "", + s3err.GetAPIError(s3err.ErrInvalidCompleteMpPartNumber) + } if num <= prev { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPartOrder) } @@ -178,9 +250,61 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet if rp.ETag != nil && !etagsEqual(*rp.ETag, hex.EncodeToString(sp.ETagMD5)) { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) } + requested = append(requested, sp) etagHasher.Write(sp.ETagMD5) - // Record this part's byte span (it may span several blobs) so a later - // GET/HEAD ?partNumber=N can address it (§7.2). + } + // Every part but the last must meet S3's 5 MiB minimum. + var total int64 + for i, sp := range requested { + if i < len(requested)-1 && sp.Size < minPartSize { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrEntityTooSmall) + } + total += sp.Size + } + if input.MpuObjectSize != nil { + if *input.MpuObjectSize < 0 { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetNegatvieMpObjectSizeErr(*input.MpuObjectSize) + } + if *input.MpuObjectSize != total { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetIncorrectMpObjectSizeErr(total, *input.MpuObjectSize) + } + } + etag := hex.EncodeToString(etagHasher.Sum(nil)) + "-" + strconv.Itoa(len(requested)) + + // Idempotent re-Complete: the prior Complete committed the object; the + // validation above already proved the client's part list matches the + // retained parts, so return the same result without recommitting. + if sess.State == registry.SessionCompleted { + etagQ := `"` + etag + `"` + return s3response.CompleteMultipartUploadResult{Bucket: &bucket, Key: &key, ETag: &etagQ}, "", nil + } + + // Single-winner latch vs a racing Abort: only the writer that moves the + // session off 'open' proceeds (§7.3). + won, err := b.multipart.LatchSession(ctx, uploadID, registry.SessionOpen, registry.SessionCompleting) + if err != nil { + return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("s3frontend: latch: %w", err) + } + if !won { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrNoSuchUpload) + } + // If anything below fails before the object is committed, revert the session + // to 'open' so the upload stays abortable / retriable rather than zombied in + // 'completing'. committed is set once the manifest is durable (the point of + // no return). + committed := false + defer func() { + if !committed { + _, _ = b.multipart.LatchSession(ctx, uploadID, registry.SessionCompleting, registry.SessionOpen) + } + }() + + // Assemble the ordered body: each part's byte span (it may span several + // blobs) is recorded so a later GET/HEAD ?partNumber=N can address it (§7.2). + var blobs []msbucket.BlobRef + var partSizes []int64 + var offset int64 + for _, sp := range requested { partStart := offset for _, d := range sp.BlobDigests { in, err := b.intents.GetIntent(ctx, d) @@ -193,49 +317,59 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet partSizes = append(partSizes, offset-partStart) } - // Accept every part's blobs on Forge (no-op in the harness), then commit. + // Accept every part's blobs on Forge (no-op in standalone), then commit. if err := b.uploadBlobs(ctx, blobs); err != nil { return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("s3frontend: accept parts: %w", err) } - etag := hex.EncodeToString(etagHasher.Sum(nil)) + "-" + strconv.Itoa(len(input.MultipartUpload.Parts)) mf := &msbucket.ObjectManifest{ - Key: key, - ContentType: sess.ContentType, - Created: time.Now().Unix(), - Body: msbucket.Body{Size: offset, Blobs: blobs, PartSizes: partSizes}, - ETag: etag, - Metadata: sess.Metadata, + Key: key, + ContentType: sess.ContentType, + Created: time.Now().Unix(), + Body: msbucket.Body{Size: offset, Blobs: blobs, PartSizes: partSizes}, + ETag: etag, + ContentEncoding: sess.ContentEncoding, + ContentDisposition: sess.ContentDisposition, + ContentLanguage: sess.ContentLanguage, + CacheControl: sess.CacheControl, + Expires: sess.Expires, + WebsiteRedirectLocation: sess.WebsiteRedirectLocation, + Metadata: sess.Metadata, } if err := b.commitManifest(ctx, bucket, key, mf, bodyDigests(mf.Body)); err != nil { return s3response.CompleteMultipartUploadResult{}, "", err } committed = true - // The object is durable; session cleanup is best-effort (a lingering session - // is harmless and reapable later, and must not fail an otherwise-good - // Complete). - _ = b.multipart.DeleteSession(ctx, uploadID) + // The object is durable. Retain the session (state 'completed') and its + // parts so a duplicate Complete is idempotent; the sweeper reaps it later. + // Best-effort: a failed latch leaves the row in 'completing', which the + // sweeper also treats as terminal after the TTL. + _, _ = b.multipart.LatchSession(ctx, uploadID, registry.SessionCompleting, registry.SessionCompleted) etagQ := `"` + etag + `"` return s3response.CompleteMultipartUploadResult{Bucket: &bucket, Key: &key, ETag: &etagQ}, "", nil } // AbortMultipartUpload cancels a multipart upload: it latches the session -// (single-winner vs Complete) and drops it (cascading its parts). The spooled -// part blobs are content-addressed and may be shared, so they are left for GC -// rather than deleted here; no reference claims were taken (those happen only at -// Complete), so nothing else needs unwinding. +// (single-winner vs Complete), drops it (cascading its parts), and removes the +// parts' now-unreferenced blobs from the spool. No reference claims were taken +// (those happen only at Complete), so nothing network-side needs unwinding. func (b *Backend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error { if input.UploadId == nil { return s3err.GetAPIError(s3err.ErrInvalidRequest) } uploadID := *input.UploadId - if _, err := b.multipart.GetSession(ctx, uploadID); err != nil { - if errors.Is(err, registry.ErrNotFound) { - return s3err.GetAPIError(s3err.ErrNoSuchUpload) - } - return fmt.Errorf("s3frontend: abort: %w", err) + sess, err := b.openSession(ctx, uploadID, input.Key) + if err != nil { + return err + } + // If-Match-Initiated-Time: reject when the provided timestamp predates the + // upload's initiation (compared at second precision — Initiated is + // presented via RFC3339); future timestamps are ignored per S3. + if input.IfMatchInitiatedTime != nil && + input.IfMatchInitiatedTime.Truncate(time.Second).Before(sess.CreatedAt.Truncate(time.Second)) { + return s3err.GetAPIError(s3err.ErrPreconditionFailed) } won, err := b.multipart.LatchSession(ctx, uploadID, registry.SessionOpen, registry.SessionAborting) if err != nil { @@ -244,12 +378,310 @@ func (b *Backend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMulti if !won { return s3err.GetAPIError(s3err.ErrNoSuchUpload) } + // Snapshot the parts' blob digests before the cascade delete, then drop + // the session and clean the spool. + var digests [][]byte + if parts, err := b.multipart.ListParts(ctx, uploadID); err == nil { + for _, p := range parts { + digests = append(digests, p.BlobDigests...) + } + } if err := b.multipart.DeleteSession(ctx, uploadID); err != nil { return fmt.Errorf("s3frontend: delete session: %w", err) } + b.cleanupPartBlobs(ctx, uploadID, digests) return nil } +// cleanupPartBlobs removes spooled blobs that belonged to aborted, expired, or +// superseded parts of uploadID — unless the blob is still referenced: by a +// part of another in-flight session (content-addressed dedup), by a part still +// live in THIS session (a re-uploaded part may share blobs with its +// replacement or a sibling part), or by a committed object (reference claims / +// non-spooled intent state). Best-effort: cleanup failure never fails the S3 +// operation; a stranded spool file is reapable later. +func (b *Backend) cleanupPartBlobs(ctx context.Context, uploadID string, digests [][]byte) { + if len(digests) == 0 { + return + } + // Digests still referenced by this session's live parts (after the + // abort/supersede that triggered this cleanup). + live := map[string]bool{} + if parts, err := b.multipart.ListParts(ctx, uploadID); err == nil { + for _, p := range parts { + for _, d := range p.BlobDigests { + live[string(d)] = true + } + } + } + seen := map[string]bool{} + for _, d := range digests { + k := string(d) + if seen[k] || live[k] { + continue + } + seen[k] = true + if n, err := b.multipart.CountPartRefs(ctx, d, uploadID); err != nil || n > 0 { + continue + } + if n, err := b.blobRefs.CountClaims(ctx, b.space, d); err != nil || n > 0 { + continue + } + if in, err := b.intents.GetIntent(ctx, d); err == nil && in.State != registry.IntentSpooled { + // Shipped/accepted blobs are the reference index's to manage. + continue + } + _ = b.spool.Remove(mh.Multihash(d)) + _ = b.intents.DeleteIntent(ctx, d) + } +} + +// ListParts returns the recorded parts of an in-flight upload, paginated by +// part number. +func (b *Backend) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) { + if input.Bucket == nil || input.Key == nil || input.UploadId == nil { + return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrInvalidRequest) + } + uploadID := *input.UploadId + if _, err := b.openSession(ctx, uploadID, input.Key); err != nil { + return s3response.ListPartsResult{}, err + } + + marker := 0 + if input.PartNumberMarker != nil && *input.PartNumberMarker != "" { + m, err := strconv.Atoi(*input.PartNumberMarker) + if err != nil { + return s3response.ListPartsResult{}, s3err.GetAPIError(s3err.ErrInvalidArgument) + } + marker = m + } + maxParts := defaultMaxListing + if input.MaxParts != nil && *input.MaxParts > 0 && int(*input.MaxParts) < defaultMaxListing { + maxParts = int(*input.MaxParts) + } + + stored, err := b.multipart.ListParts(ctx, uploadID) + if err != nil { + return s3response.ListPartsResult{}, fmt.Errorf("s3frontend: list parts: %w", err) + } + var parts []s3response.Part + truncated := false + next := 0 + for _, p := range stored { + if p.PartNumber <= marker { + continue + } + if len(parts) == maxParts { + truncated = true + break + } + parts = append(parts, s3response.Part{ + PartNumber: p.PartNumber, + ETag: `"` + hex.EncodeToString(p.ETagMD5) + `"`, + Size: p.Size, + LastModified: p.CreatedAt.UTC(), + }) + next = p.PartNumber + } + res := s3response.ListPartsResult{ + Bucket: *input.Bucket, + Key: *input.Key, + UploadID: uploadID, + StorageClass: types.StorageClassStandard, + PartNumberMarker: marker, + MaxParts: maxParts, + IsTruncated: truncated, + Parts: parts, + } + if truncated { + res.NextPartNumberMarker = next + } + return res, nil +} + +// ListMultipartUploads lists the bucket's in-flight (open) multipart uploads +// in (key, initiation) order, with S3's prefix/delimiter/marker pagination. +func (b *Backend) ListMultipartUploads(ctx context.Context, input *s3.ListMultipartUploadsInput) (s3response.ListMultipartUploadsResult, error) { + if input.Bucket == nil { + return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrInvalidBucketName) + } + bucket := *input.Bucket + if _, err := b.reg.Get(ctx, bucket); err != nil { + if errors.Is(err, registry.ErrNotFound) { + return s3response.ListMultipartUploadsResult{}, s3err.GetAPIError(s3err.ErrNoSuchBucket) + } + return s3response.ListMultipartUploadsResult{}, fmt.Errorf("s3frontend: list mpu: %w", err) + } + + prefix := backend.GetStringFromPtr(input.Prefix) + delimiter := backend.GetStringFromPtr(input.Delimiter) + keyMarker := backend.GetStringFromPtr(input.KeyMarker) + uploadIDMarker := backend.GetStringFromPtr(input.UploadIdMarker) + maxUploads := defaultMaxListing + if input.MaxUploads != nil && *input.MaxUploads > 0 && int(*input.MaxUploads) < defaultMaxListing { + maxUploads = int(*input.MaxUploads) + } + + all, err := b.multipart.ListSessions(ctx, bucket) + if err != nil { + return s3response.ListMultipartUploadsResult{}, fmt.Errorf("s3frontend: list sessions: %w", err) + } + // In-flight uploads only, honoring the prefix. + inflight := all[:0] + for _, s := range all { + if s.State == registry.SessionOpen && strings.HasPrefix(s.ObjectKey, prefix) { + inflight = append(inflight, s) + } + } + + // Marker positioning. An upload-id marker is meaningful only alongside a + // key marker: when the key marker names a live key, the id must belong to + // one of that key's uploads (else InvalidArgument); when it doesn't (the + // marker key was completed/aborted meanwhile), the id positions within the + // keys ordered after it. + start := 0 + if keyMarker != "" { + if uploadIDMarker != "" { + keyLive := false + pos := -1 + for i, s := range inflight { + if s.ObjectKey == keyMarker { + keyLive = true + if s.UploadID == uploadIDMarker { + pos = i + } + } + } + if keyLive && pos < 0 { + return s3response.ListMultipartUploadsResult{}, + s3err.GetAPIError(s3err.ErrInvalidUploadIdMarker) + } + if pos < 0 { + for i, s := range inflight { + if s.ObjectKey >= keyMarker && s.UploadID == uploadIDMarker { + pos = i + break + } + } + } + if pos >= 0 { + start = pos + 1 + } else { + for start < len(inflight) && inflight[start].ObjectKey <= keyMarker { + start++ + } + } + } else { + for start < len(inflight) && inflight[start].ObjectKey <= keyMarker { + start++ + } + } + } + + // Walk the ordered tail, collapsing keys under the delimiter into common + // prefixes; uploads and prefixes count toward max-uploads together. + var uploads []s3response.Upload + var prefixes []s3response.CommonPrefix + emittedPrefix := map[string]bool{} + count := 0 + truncated := false + nextKeyMarker, nextUploadIDMarker := "", "" + for _, s := range inflight[start:] { + itemKey := s.ObjectKey + isPrefix := false + if delimiter != "" { + if i := strings.Index(s.ObjectKey[len(prefix):], delimiter); i >= 0 { + itemKey = s.ObjectKey[:len(prefix)+i+len(delimiter)] + isPrefix = true + } + } + if isPrefix && emittedPrefix[itemKey] { + continue + } + if count == maxUploads { + truncated = true + break + } + if isPrefix { + emittedPrefix[itemKey] = true + prefixes = append(prefixes, s3response.CommonPrefix{Prefix: itemKey}) + } else { + uploads = append(uploads, s3response.Upload{ + Key: s.ObjectKey, + UploadID: s.UploadID, + StorageClass: types.StorageClassStandard, + Initiated: s.CreatedAt.UTC(), + ChecksumAlgorithm: types.ChecksumAlgorithm(s.ChecksumAlgorithm), + ChecksumType: types.ChecksumType(s.ChecksumType), + }) + } + count++ + nextKeyMarker, nextUploadIDMarker = itemKey, s.UploadID + } + res := s3response.ListMultipartUploadsResult{ + Bucket: bucket, + KeyMarker: keyMarker, + UploadIDMarker: uploadIDMarker, + Delimiter: delimiter, + Prefix: prefix, + MaxUploads: maxUploads, + IsTruncated: truncated, + Uploads: uploads, + CommonPrefixes: prefixes, + } + if truncated { + res.NextKeyMarker = nextKeyMarker + res.NextUploadIDMarker = nextUploadIDMarker + } + return res, nil +} + +// SweepStaleMultipartSessions aborts in-flight multipart sessions older than +// ttl (dropping their spooled parts, exactly like a client Abort) and reaps +// completed/aborting leftovers past the same age. Returns how many sessions +// were cleaned. Called periodically by the daemon's sweeper loop. +func (b *Backend) SweepStaleMultipartSessions(ctx context.Context, ttl time.Duration) (int, error) { + cutoff := time.Now().Add(-ttl) + cleaned := 0 + // Stale open sessions: latch (losing gracefully to a concurrent + // Complete/Abort) and clean up like an abort. + stale, err := b.multipart.ListStaleSessions(ctx, registry.SessionOpen, cutoff) + if err != nil { + return 0, fmt.Errorf("s3frontend: sweep list: %w", err) + } + for _, s := range stale { + won, err := b.multipart.LatchSession(ctx, s.UploadID, registry.SessionOpen, registry.SessionAborting) + if err != nil || !won { + continue + } + var digests [][]byte + if parts, err := b.multipart.ListParts(ctx, s.UploadID); err == nil { + for _, p := range parts { + digests = append(digests, p.BlobDigests...) + } + } + if err := b.multipart.DeleteSession(ctx, s.UploadID); err != nil { + continue + } + b.cleanupPartBlobs(ctx, s.UploadID, digests) + cleaned++ + } + // Terminal leftovers: completed sessions retained for Complete idempotency, + // and any 'completing'/'aborting' rows stranded by a crash mid-transition. + for _, state := range []string{registry.SessionCompleted, registry.SessionCompleting, registry.SessionAborting} { + leftovers, err := b.multipart.ListStaleSessions(ctx, state, cutoff) + if err != nil { + continue + } + for _, s := range leftovers { + if err := b.multipart.DeleteSession(ctx, s.UploadID); err == nil { + cleaned++ + } + } + } + return cleaned, nil +} + // commitManifest splices mf into (bucket, key) and reconciles the reference // index against the prior version's digests, releasing dropped blobs after the // commit. Shared by CopyObject and CompleteMultipartUpload (a plain commit with diff --git a/s3frontend/object.go b/s3frontend/object.go index 11469cf..21b9a4e 100644 --- a/s3frontend/object.go +++ b/s3frontend/object.go @@ -124,19 +124,20 @@ func (b *Backend) PutObject(ctx context.Context, input s3response.PutObjectInput // blob_refs from the committed catalog. err = b.txns.WithTx(ctx, bucketName, func(ctx context.Context, tx *bucketop.Tx) (cid.Cid, error) { mf = &msbucket.ObjectManifest{ - Key: key, - ContentType: contentType, - Created: time.Now().Unix(), - Body: bodyRec, - ETag: hex.EncodeToString(bodyRec.MD5), - ChecksumAlgorithm: ckAlgo, - Checksum: ckVal, - ContentEncoding: backend.GetStringFromPtr(input.ContentEncoding), - ContentDisposition: backend.GetStringFromPtr(input.ContentDisposition), - ContentLanguage: backend.GetStringFromPtr(input.ContentLanguage), - CacheControl: backend.GetStringFromPtr(input.CacheControl), - Expires: backend.GetStringFromPtr(input.Expires), - Metadata: input.Metadata, + Key: key, + ContentType: contentType, + Created: time.Now().Unix(), + Body: bodyRec, + ETag: hex.EncodeToString(bodyRec.MD5), + ChecksumAlgorithm: ckAlgo, + Checksum: ckVal, + ContentEncoding: backend.GetStringFromPtr(input.ContentEncoding), + ContentDisposition: backend.GetStringFromPtr(input.ContentDisposition), + ContentLanguage: backend.GetStringFromPtr(input.ContentLanguage), + CacheControl: backend.GetStringFromPtr(input.CacheControl), + Expires: backend.GetStringFromPtr(input.Expires), + WebsiteRedirectLocation: backend.GetStringFromPtr(input.WebsiteRedirectLocation), + Metadata: input.Metadata, } mfCid, err := tx.Put(ctx, mf) if err != nil { @@ -475,20 +476,21 @@ func (b *Backend) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s contentType := mf.ContentType out := &s3.HeadObjectOutput{ - AcceptRanges: backend.GetPtrFromString("bytes"), - ContentLength: &length, - ContentType: &contentType, - ContentEncoding: strPtrOrNil(mf.ContentEncoding), - ContentDisposition: strPtrOrNil(mf.ContentDisposition), - ContentLanguage: strPtrOrNil(mf.ContentLanguage), - CacheControl: strPtrOrNil(mf.CacheControl), - ExpiresString: strPtrOrNil(mf.Expires), - Metadata: mf.Metadata, - ContentRange: contentRange, - PartsCount: partsCount, - ETag: &etag, - LastModified: &lastModified, - StorageClass: types.StorageClassStandard, + AcceptRanges: backend.GetPtrFromString("bytes"), + ContentLength: &length, + ContentType: &contentType, + ContentEncoding: strPtrOrNil(mf.ContentEncoding), + ContentDisposition: strPtrOrNil(mf.ContentDisposition), + ContentLanguage: strPtrOrNil(mf.ContentLanguage), + CacheControl: strPtrOrNil(mf.CacheControl), + ExpiresString: strPtrOrNil(mf.Expires), + WebsiteRedirectLocation: strPtrOrNil(mf.WebsiteRedirectLocation), + Metadata: mf.Metadata, + ContentRange: contentRange, + PartsCount: partsCount, + ETag: &etag, + LastModified: &lastModified, + StorageClass: types.StorageClassStandard, } // Echo the stored checksum only for a whole-object HEAD with checksum mode on // (a ranged HEAD's checksum would not match the full object). @@ -541,21 +543,22 @@ func (b *Backend) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3. lastModified := time.Unix(mf.Created, 0) contentType := mf.ContentType out := &s3.GetObjectOutput{ - AcceptRanges: backend.GetPtrFromString("bytes"), - Body: body, - ContentLength: &length, - ContentType: &contentType, - ContentEncoding: strPtrOrNil(mf.ContentEncoding), - ContentDisposition: strPtrOrNil(mf.ContentDisposition), - ContentLanguage: strPtrOrNil(mf.ContentLanguage), - CacheControl: strPtrOrNil(mf.CacheControl), - ExpiresString: strPtrOrNil(mf.Expires), - Metadata: mf.Metadata, - ContentRange: contentRange, - PartsCount: partsCount, - ETag: &etag, - LastModified: &lastModified, - StorageClass: types.StorageClassStandard, + AcceptRanges: backend.GetPtrFromString("bytes"), + Body: body, + ContentLength: &length, + ContentType: &contentType, + ContentEncoding: strPtrOrNil(mf.ContentEncoding), + ContentDisposition: strPtrOrNil(mf.ContentDisposition), + ContentLanguage: strPtrOrNil(mf.ContentLanguage), + CacheControl: strPtrOrNil(mf.CacheControl), + ExpiresString: strPtrOrNil(mf.Expires), + WebsiteRedirectLocation: strPtrOrNil(mf.WebsiteRedirectLocation), + Metadata: mf.Metadata, + ContentRange: contentRange, + PartsCount: partsCount, + ETag: &etag, + LastModified: &lastModified, + StorageClass: types.StorageClassStandard, } // Echo the stored checksum only for a whole-object GET with checksum mode on // (a ranged GET's checksum would not match the full object). diff --git a/server.go b/server.go index 1baafbb..e237117 100644 --- a/server.go +++ b/server.go @@ -63,6 +63,13 @@ type ServerConfig struct { // request), so New substitutes a sensible default. MaxConnections int MaxRequests int + + // MultipartSessionTTL bounds abandoned multipart uploads: open sessions + // older than this are aborted by a background sweeper (dropping their + // spooled parts), and completed-session rows retained for Complete + // idempotency are reaped past the same age. Zero → default 7 days; + // negative → sweeper disabled. + MultipartSessionTTL time.Duration } // ServerDeps bundles the runtime collaborators of an ingot Server @@ -119,11 +126,12 @@ type ServerDeps struct { // lifecycle. fx callers wrap these in OnStart/OnStop hooks; tests // call them directly. type Server struct { - cfg ServerConfig - logger *zap.Logger - log blockstore.Log - backend *s3frontend.Backend - api *s3api.S3ApiServer + cfg ServerConfig + logger *zap.Logger + log blockstore.Log + backend *s3frontend.Backend + api *s3api.S3ApiServer + sweepStop chan struct{} } // New wires a ServerDeps + ServerConfig into a runnable Server. The @@ -214,15 +222,58 @@ func (s *Server) Start(ctx context.Context) error { s.logger.Error("ingot listener error", zap.Error(err)) } }() + s.startMultipartSweeper() return nil } +// startMultipartSweeper spawns the abandoned-multipart-session sweeper: open +// sessions older than MultipartSessionTTL are aborted (their spooled parts +// dropped) and terminal session rows reaped. Zero TTL → 7-day default; +// negative → disabled. +func (s *Server) startMultipartSweeper() { + ttl := s.cfg.MultipartSessionTTL + if ttl == 0 { + ttl = 7 * 24 * time.Hour + } + if ttl < 0 { + return + } + interval := ttl / 2 + if interval > 10*time.Minute { + interval = 10 * time.Minute + } + s.sweepStop = make(chan struct{}) + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-s.sweepStop: + return + case <-ticker.C: + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + n, err := s.backend.SweepStaleMultipartSessions(ctx, ttl) + cancel() + if err != nil { + s.logger.Warn("multipart sweep", zap.Error(err)) + } else if n > 0 { + s.logger.Info("multipart sweep reaped stale sessions", zap.Int("count", n)) + } + } + } + }() +} + // Stop shuts the listener down and drains the log. Always returns // the combined error of the two operations so callers see all // failure modes; either alone is non-fatal to the other. func (s *Server) Stop(ctx context.Context) error { s.logger.Info("shutting down ingot S3 listener") + if s.sweepStop != nil { + close(s.sweepStop) + s.sweepStop = nil + } var errs []error if err := s.api.ShutDown(); err != nil { errs = append(errs, fmt.Errorf("s3api shutdown: %w", err)) From 8159cbb0b437e0822e0692a293fab049f67d1376 Mon Sep 17 00:00:00 2001 From: frrist Date: Fri, 17 Jul 2026 12:57:53 -0700 Subject: [PATCH 02/21] deps: bump libforge to main (blob remove/abort/reject bindings) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit github.com/fil-forge/libforge aac837a — Space on RemoveArguments plus the /blob/abort binding the deferred-multipart abort path invokes. Co-Authored-By: Claude Fable 5 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 74806ba..be1dbc5 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.15 github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1 github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 - github.com/fil-forge/libforge v0.0.0-20260701162346-f0706e1641a3 + github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2 github.com/fil-forge/smelt v0.0.0-20260714201619-f2ecac50394c github.com/fil-forge/ucantone v0.0.0-20260706102443-79141c5cc52e github.com/fxamacker/cbor/v2 v2.9.2 diff --git a/go.sum b/go.sum index d061849..3fd68be 100644 --- a/go.sum +++ b/go.sum @@ -248,8 +248,8 @@ github.com/fil-forge/go-ucanto v0.0.0-20260507172450-5cb5d073f8ab h1:2J2cDThqTKP github.com/fil-forge/go-ucanto v0.0.0-20260507172450-5cb5d073f8ab/go.mod h1:lZF3UXZ2hGLKYmXdquG50JqI9pRlUrV6lubGtgOYfwc= github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 h1:Wke8qgaDgy7DGaIS28VpHip+YHSdQP1hGNEZTrXXzb4= github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717/go.mod h1:wFcakLohOqpMRkJzWRdGFGHRFpGB+PpEMCm9wkt2cqU= -github.com/fil-forge/libforge v0.0.0-20260701162346-f0706e1641a3 h1:/EDxpbuVeSXH3FLF7MK4G4wrDUFmCO9fLQHL23RZ2uU= -github.com/fil-forge/libforge v0.0.0-20260701162346-f0706e1641a3/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= +github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2 h1:mwb14/aanwFBGdQJ/0WCkNX/AzUXF/iRKMVjhXtYd6Q= +github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= github.com/fil-forge/smelt v0.0.0-20260714201619-f2ecac50394c h1:LqWeuCe44vpkqz/F5kqMjK380FOzvLvYrFlR/I9GVUo= github.com/fil-forge/smelt v0.0.0-20260714201619-f2ecac50394c/go.mod h1:W6aiYnuM+n0Q7G6fH3Ok4EE2iMfUSpA6f8r3i/+moBs= github.com/fil-forge/ucantone v0.0.0-20260706102443-79141c5cc52e h1:di/SseJVEO6bznSH/UEnAE2nSwwp/gqgnPLgE9/x2Zg= From f530681b034ba1118888ac84c07ec2d6f4ff4f86 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 14 Jul 2026 18:47:59 -0700 Subject: [PATCH 03/21] =?UTF-8?q?feat:=20wire=20/blob/remove=20=E2=80=94?= =?UTF-8?q?=20DeleteObject=20releases=20blobs=20on=20the=20network=20(FIL-?= =?UTF-8?q?588)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit forgeclient gains BlobRemove (subject = space, digest in args); the uploader's RemoveBlob no-op becomes the real call, so the reference-index bookkeeping (claims→0) now releases the space's claim through sprue → piri. Delete finality is claim-release-now, bytes-at-root-death: piri deletes unaggregated bytes immediately and defers aggregated ones until the PDP root retires on-chain. itest: TestForgeDeleteReleasesNetworkBlob is the regression gate — PUT, prove the blob serves from piri (spool wipe + read-through), DeleteObject, then assert piri executed /blob/remove and queued the piece for removal. Gated behind ITEST_PIRI_BIN/ITEST_SPRUE_BIN until published piri/sprue images carry the handlers. docs: architecture.md §2/§9 status rows flip remove(digest) to exists; deferred-glue note updated. Co-Authored-By: Claude Fable 5 --- docs/architecture.md | 21 ++++--- forgeclient/blobremove.go | 45 +++++++++++++++ itest/forge_delete_test.go | 109 +++++++++++++++++++++++++++++++++++++ testing/roundtrip.go | 10 ++++ uploader/blob.go | 20 +++---- 5 files changed, 186 insertions(+), 19 deletions(-) create mode 100644 forgeclient/blobremove.go create mode 100644 itest/forge_delete_test.go diff --git a/docs/architecture.md b/docs/architecture.md index 7b28969..0bc59b8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,9 +90,11 @@ The design is shaped by how the Forge upload pipeline works and by a handful of - **The 256 MiB blob ceiling is a knob, not a wall.** Piri currently caps a blob at 256 MiB because it builds the commP Merkle tree in RAM; known improvements (streaming commP) lift this. Ingot treats it as a tunable maximum and splits larger objects. -- **Some primitives are not built yet.** There is no `unallocate` for a parked blob; `blob/remove` - is declared but unhandled; the whole-root on-chain delete is wired but its signature path is - incomplete; the indexer has no retraction. [§9](#9-the-system-contract-piri--sprue--indexer) enumerates the contract Ingot needs. +- **Some primitives are not built yet.** `blob/remove` is now handled end-to-end (Sprue + deregisters + forwards; Piri releases the space's claim and defers physical deletion until the + aggregate root retires on-chain via the signed `schedulePieceDeletions` path), but there is + still no `unallocate` for a parked blob and the indexer has no retraction. + [§9](#9-the-system-contract-piri--sprue--indexer) enumerates the contract Ingot needs. --- @@ -522,10 +524,10 @@ negotiations). |------------------------------------------------------------------------------------------------|---------------------------------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `allocate` / `PUT` / `accept` blob lifecycle | Piri/Sprue | **exists** | The storage primitive Ingot builds on. | | Ingot-timed accept (PUT a part, defer the conclude until Complete) | Ingot + Sprue | **partial** | Accept already fires on the client's conclude; needs the park-vs-conclude split client-side and a separable Sprue conclude. Ingot does **not** issue `accept` (Piri requires the upload-service DID). | -| `unallocate(digest)` — drop a parked blob | Piri + Sprue + libforge | **to-build** | `blob/remove` arg type exists in libforge; no handler. | -| `remove(digest)` — per-space claim release; physical delete/piece-retire at zero global claims | Piri + Sprue + libforge | **to-build** | Piri keeps per-`(digest,space)` allocation rows to count on. | +| `unallocate(digest)` — drop a parked blob | Piri + Sprue + libforge | **to-build** | `blob/remove` releases claims on accepted blobs; a dedicated parked-blob unallocate (pre-accept) is still open (allocation expiry is the backstop). | +| `remove(digest)` — per-space claim release; physical delete/piece-retire at zero global claims | Piri + Sprue + libforge | **exists** | `/blob/remove` on Piri deletes the space's allocation/acceptance/claim; at zero claims bytes delete immediately (unaggregated) or via the pending-removal sweep once the whole aggregate root is dead (FIL-623/624). Sprue forwards to primary + replicas (FIL-522). | | Configurable, adaptive size policy (`min`/`max`); batch guard for the `extraData` cap | Piri | **partial** | `MinAggregateSize` is hardcoded 128 MiB; lower to ~8 MiB and make configurable. The `addPieces` batch is no longer contract-capped (FWSS v1.3.0 removed the `extraData` cap); size it to the FVM `PiecesAdded` event-size + per-tx gas — a measured ceiling (default `BatchSize=10` is safely within it) (pdp-sim). No contract change. | -| Compaction (Regime B) + complete the on-chain delete signature | Piri | **partial** | Whole-root delete is wired but its `extraData` signature is incomplete; compaction (remove + re-hash survivors + re-add) is new. | +| Compaction (Regime B) + complete the on-chain delete signature | Piri | **partial** | Whole-root delete is signed and wired (`schedulePieceDeletions` with `SignSchedulePieceRemovals` extraData); compaction (remove + re-hash survivors + re-add) is new. | | De-dup at accept (don't re-aggregate a digest already a live piece) | Piri | **to-build** | Backstops one-piece-per-content once accept timing is Ingot-driven. | | Parked-allocation GC + honor `Expires` | Piri | **to-build** | Bounds leakage when an abort never arrives. | | Indexer delete by `(space, digest)` / location-claim CID | Indexer + Sprue | **to-build** | IPNI removal mechanics are the indexer's. | @@ -798,9 +800,10 @@ reference index — and is out of scope for this iteration. The in-memory harness uses a no-op uploader and serves reads from the spool, so these forge-network paths are stubbed in-tree and verified against a real sprue+piri+indexer later: -- **`remove(digest)` / `unallocate(digest)` are logged no-ops.** libforge has the `blob.Remove` - binding; the Piri/Sprue handler is to-build ([§9](#9-the-system-contract-piri--sprue--indexer)). The reference-index bookkeeping (count→0 ⇒ - `RemoveBlob` call site) is fully built and tested; only the network release is stubbed. +- **`remove(digest)` is live; `unallocate(digest)` is still open.** `RemoveBlob` invokes + `/blob/remove` on sprue, which forwards to the storage nodes ([§9](#9-the-system-contract-piri--sprue--indexer)); delete finality means + claim-release-now, bytes-at-root-death. A dedicated pre-accept unallocate remains to-build + (allocation expiry is the backstop). - **The local-table `Locator` read tier is not wired.** Body-blob *locations* are recorded at accept, but the read path that consumes them ([§7.4](#74-read-getobject), [§8](#8-retrieval-addressing-when-bodies-need-a-sharded-dag-index)) is deferred — it is only exercised after spool eviction (also not built) and is best validated live. diff --git a/forgeclient/blobremove.go b/forgeclient/blobremove.go new file mode 100644 index 0000000..94e122e --- /dev/null +++ b/forgeclient/blobremove.go @@ -0,0 +1,45 @@ +package forgeclient + +import ( + "context" + "fmt" + + blobcmds "github.com/fil-forge/libforge/commands/blob" + "github.com/fil-forge/ucantone/did" + "github.com/fil-forge/ucantone/execution" + "github.com/fil-forge/ucantone/ucan/invocation" + "github.com/multiformats/go-multihash" +) + +// BlobRemove invokes /blob/remove against the upload service (sprue), +// releasing the space's claim on the blob. Sprue deregisters the blob and +// forwards the removal to every storage node holding it; piri deletes the +// bytes only when no space claims the digest at all. Idempotent: removing an +// unknown or already-removed blob succeeds. +func (c *Client) BlobRemove(ctx context.Context, digest multihash.Multihash, space did.DID) error { + proofs, proofLinks, err := c.ProofChain(ctx, c.signer.DID(), blobcmds.Remove.Command, space) + if err != nil { + return fmt.Errorf("building proof chain: %w", err) + } + inv, err := blobcmds.Remove.Invoke( + c.signer, + space, + &blobcmds.RemoveArguments{Space: space, Digest: digest}, + invocation.WithAudience(c.serviceID), + invocation.WithProofs(proofLinks...), + ) + if err != nil { + return fmt.Errorf("creating invocation: %w", err) + } + + _, _, _, err = Execute[*blobcmds.RemoveOK]( + ctx, + c.ucanClient, + inv, + execution.WithDelegations(proofs...), + ) + if err != nil { + return fmt.Errorf("executing invocation: %w", err) + } + return nil +} diff --git a/itest/forge_delete_test.go b/itest/forge_delete_test.go new file mode 100644 index 0000000..475094e --- /dev/null +++ b/itest/forge_delete_test.go @@ -0,0 +1,109 @@ +//go:build itest + +package itest + +import ( + "context" + "os" + "strings" + "testing" + "time" + + ingottest "github.com/fil-forge/ingot/testing" + "github.com/fil-forge/smelt/pkg/stack" +) + +// TestForgeDeleteReleasesNetworkBlob is the delete-finality regression gate +// (FIL-588): DeleteObject must release the blob on the network, not just drop +// registry rows. The chain under test is ingot's reference index (claims→0 ⇒ +// RemoveBlob) → forgeclient /blob/remove → sprue (deregister + forward) → +// piri (claim release; deferred physical deletion once the PDP aggregate +// root retires on-chain). +// +// The published piri/sprue images predate the /blob/remove handlers, so this +// test injects working-tree builds of both and skips when they aren't +// provided: +// +// (cd ../../piri && CGO_ENABLED=0 GOOS=linux go build -o /tmp/piri ./cmd) +// (cd ../../sprue && CGO_ENABLED=0 GOOS=linux go build -o /tmp/sprue ./cmd/main.go) +// ITEST_PIRI_BIN=/tmp/piri ITEST_SPRUE_BIN=/tmp/sprue \ +// go test -tags itest ./itest -run TestForgeDeleteReleasesNetworkBlob -v -timeout 900s +// +// Once images with the handlers publish, drop the env gate and the binary +// injection. +func TestForgeDeleteReleasesNetworkBlob(t *testing.T) { + piriBin := os.Getenv("ITEST_PIRI_BIN") + sprueBin := os.Getenv("ITEST_SPRUE_BIN") + if piriBin == "" || sprueBin == "" { + t.Skip("requires piri+sprue builds with the /blob/remove chain: set ITEST_PIRI_BIN and ITEST_SPRUE_BIN (see test doc comment)") + } + + ctx := t.Context() + const email = "test@example.com" + + s, ingotEndpoint := forgeStack(t, + stack.WithServiceBinary("piri", piriBin), + stack.WithServiceBinary("upload", sprueBin), + ) + ingotSelfProvision(t, ctx, s, email) + cfg := forgeConfig(ingotEndpoint) + + const bucket, key = "delete-bucket", "obj" + + // Unique deterministic body, large enough to be a real blob on piri. + data := make([]byte, 512*1024) + for i := range data { + data[i] = byte(i*13 + 11) + } + + if err := ingottest.CreateBucket(ctx, cfg, bucket); err != nil { + t.Fatalf("create bucket: %v", err) + } + if err := ingottest.PutBytes(ctx, cfg, bucket, key, data); err != nil { + t.Fatalf("put object: %v", err) + } + + // Precondition: the blob really lives on piri — wipe the spool and prove + // the GET re-fetches from the network (same tier the eviction test pins). + if out, errOut, err := s.Exec(ctx, "ingot", "sh", "-c", "rm -rf /data/spool"); err != nil { + t.Fatalf("evict spool: %v (stdout=%s stderr=%s)", err, out, errOut) + } + if got, err := ingottest.GetBytes(ctx, cfg, bucket, key); err != nil || len(got) != len(data) { + t.Fatalf("read-through from piri before delete: err=%v len=%d", err, len(got)) + } + + if err := ingottest.DeleteObject(ctx, cfg, bucket, key); err != nil { + t.Fatalf("delete object: %v", err) + } + + // The object is gone from the gateway. + if _, err := ingottest.GetBytes(ctx, cfg, bucket, key); err == nil { + t.Fatalf("GET after delete succeeded, want NoSuchKey") + } + + // And the release traversed the network: piri's /blob/remove handler ran + // and, with the last claim gone, queued the piece for removal. The + // removal is asynchronous on the ingot side (releaseBlobs is best-effort, + // post-commit), so poll the provider's logs. + waitForPiriLog(t, ctx, s, "/blob/remove", 2*time.Minute) + waitForPiriLog(t, ctx, s, "removing piece", 2*time.Minute) + t.Logf("delete finality OK: /blob/remove executed on piri and the piece was queued for removal") +} + +// waitForPiriLog polls piri-0's container logs until substr appears. +func waitForPiriLog(t *testing.T, ctx context.Context, s *stack.Stack, substr string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + logs, err := s.Logs(ctx, "piri-0") + if err == nil && strings.Contains(logs, substr) { + return + } + select { + case <-ctx.Done(): + t.Fatalf("context done waiting for piri log %q: %v", substr, ctx.Err()) + case <-time.After(2 * time.Second): + } + } + t.Fatalf("piri-0 logs never contained %q within %s", substr, timeout) +} diff --git a/testing/roundtrip.go b/testing/roundtrip.go index de5aecb..626b30c 100644 --- a/testing/roundtrip.go +++ b/testing/roundtrip.go @@ -60,6 +60,16 @@ func PutBytes(ctx context.Context, c Config, bucket, key string, body []byte) er return err } +// DeleteObject deletes object key from bucket. +func DeleteObject(ctx context.Context, c Config, bucket, key string) error { + cl, err := s3Client(ctx, c) + if err != nil { + return err + } + _, err = cl.DeleteObject(ctx, &s3.DeleteObjectInput{Bucket: &bucket, Key: &key}) + return err +} + // GetBytes downloads object key from bucket. func GetBytes(ctx context.Context, c Config, bucket, key string) ([]byte, error) { cl, err := s3Client(ctx, c) diff --git a/uploader/blob.go b/uploader/blob.go index 6d9ede3..994c880 100644 --- a/uploader/blob.go +++ b/uploader/blob.go @@ -84,19 +84,19 @@ type BlobRemover interface { RemoveBlob(ctx context.Context, digest multihash.Multihash) error } -// RemoveBlob releases the space's claim on digest. -// -// TODO(phase 7 / smelt): wire forgeclient.BlobRemove against the upload service -// — the libforge blob.Remove binding exists, but the Piri/Sprue handler is -// to-build (docs/architecture.md §9). Until it lands this is a logged no-op, so -// the reference-index bookkeeping (blob_refs count → 0 → RemoveBlob) is -// exercised end-to-end without a working network primitive; bytes accumulate on -// Piri until the handler exists. -func (u *Forge) RemoveBlob(_ context.Context, digest multihash.Multihash) error { - u.logger.Info("blob remove (no-op; Piri handler to-build)", +// RemoveBlob releases the space's claim on digest via /blob/remove on the +// upload service: sprue deregisters the blob and forwards the removal to the +// storage nodes holding it; piri deletes the bytes only once no space claims +// the digest (and, for aggregated pieces, once the PDP root retires +// on-chain). Idempotent. +func (u *Forge) RemoveBlob(ctx context.Context, digest multihash.Multihash) error { + u.logger.Info("blob remove", zap.Stringer("space", u.space), zap.String("digest", digestutil.Format(digest)), ) + if err := u.client.BlobRemove(ctx, digest, u.space); err != nil { + return fmt.Errorf("uploader: removing blob: %w", err) + } return nil } From 4125b97f0e569fd81ddd8ee71eca09b20bcf09fe Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 14 Jul 2026 19:12:29 -0700 Subject: [PATCH 04/21] fix: seed /blob/remove space delegation; log failed blob removals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The e2e flushed out two gaps: the agent's seeded space delegations did not cover /blob/remove, so the removal invocation shipped proofless and sprue's validator rejected it; and releaseBlobs discards errors by design, which made that failure invisible — RemoveBlob now logs its own errors. The seeding sentinel moves to /blob/remove so stores seeded by older builds re-seed the new cap. Co-Authored-By: Claude Fable 5 --- module.go | 16 ++++++++++------ uploader/blob.go | 8 ++++++++ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/module.go b/module.go index 3273d57..c3f1196 100644 --- a/module.go +++ b/module.go @@ -384,15 +384,16 @@ func provideUploader(c *forgeclient.Client, space spaceIssuer, logger *zap.Logge // holds a /blob/add chain for the agent (idempotent across restarts). func seedSpaceDelegations(ctx context.Context, spaceIssuer ucan.Issuer, agent did.DID, store tokenstore.Store, logger *zap.Logger) error { space := spaceIssuer.DID() - // Sentinel on /blob/allocate: it is the cap the upload service re-invokes - // against piri on the space's behalf, so a store missing it cannot ship. - // (An older store seeded before allocate/accept were added re-seeds here.) - if proofs, _, err := store.ProofChain(ctx, agent, blobcmds.Allocate.Command, space); err == nil && len(proofs) > 0 { + // Sentinel on /blob/remove: the newest cap in the list — a store missing + // it was seeded by an older build and re-seeds here (AddDelegations is + // additive, so re-seeding the earlier caps is harmless). + if proofs, _, err := store.ProofChain(ctx, agent, blobcmds.Remove.Command, space); err == nil && len(proofs) > 0 { return nil // already seeded (or login-provisioned) } // The edge-client flow needs space->agent authority over: /blob/add (the - // add invocation), /blob/allocate + /blob/accept (re-delegated to sprue so - // it can allocate/accept against piri), /index/add (the index publish), and + // add invocation), /blob/remove (the delete-finality release), + // /blob/allocate + /blob/accept (re-delegated to sprue so it can + // allocate/accept against piri), /index/add (the index publish), and // /content/retrieve (the read path + the index retrieval-auth). type cap struct { name string @@ -402,6 +403,9 @@ func seedSpaceDelegations(ctx context.Context, spaceIssuer ucan.Issuer, agent di {"/blob/add", func() (ucan.Delegation, error) { return blobcmds.Add.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) }}, + {"/blob/remove", func() (ucan.Delegation, error) { + return blobcmds.Remove.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) + }}, {"/blob/allocate", func() (ucan.Delegation, error) { return blobcmds.Allocate.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) }}, diff --git a/uploader/blob.go b/uploader/blob.go index 994c880..505246d 100644 --- a/uploader/blob.go +++ b/uploader/blob.go @@ -95,6 +95,14 @@ func (u *Forge) RemoveBlob(ctx context.Context, digest multihash.Multihash) erro zap.String("digest", digestutil.Format(digest)), ) if err := u.client.BlobRemove(ctx, digest, u.space); err != nil { + // Callers treat removal as best-effort and may discard the error, so + // log it here — a silent failure leaks bytes on the network with no + // trace. + u.logger.Error("blob remove failed", + zap.Stringer("space", u.space), + zap.String("digest", digestutil.Format(digest)), + zap.Error(err), + ) return fmt.Errorf("uploader: removing blob: %w", err) } return nil From 9574063c56df6c2c53db5cf37ef281daf07b319d Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 14 Jul 2026 19:12:29 -0700 Subject: [PATCH 05/21] temp: replace smelt with local checkout The delete-finality e2e needs smelt's piri registration proofs to include blob/remove (fil-forge/smelt frrist/fil-588-blob-remove-v2). Drop this commit and repin once that smelt PR merges. Co-Authored-By: Claude Fable 5 --- go.mod | 2 ++ go.sum | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index be1dbc5..493355e 100644 --- a/go.mod +++ b/go.mod @@ -252,3 +252,5 @@ require ( pitr.ca/jsontokenizer v0.3.2 // indirect tags.cncf.io/container-device-interface v1.1.0 // indirect ) + +replace github.com/fil-forge/smelt => ../smelt diff --git a/go.sum b/go.sum index 3fd68be..247360d 100644 --- a/go.sum +++ b/go.sum @@ -250,8 +250,6 @@ github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 h1:W github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717/go.mod h1:wFcakLohOqpMRkJzWRdGFGHRFpGB+PpEMCm9wkt2cqU= github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2 h1:mwb14/aanwFBGdQJ/0WCkNX/AzUXF/iRKMVjhXtYd6Q= github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= -github.com/fil-forge/smelt v0.0.0-20260714201619-f2ecac50394c h1:LqWeuCe44vpkqz/F5kqMjK380FOzvLvYrFlR/I9GVUo= -github.com/fil-forge/smelt v0.0.0-20260714201619-f2ecac50394c/go.mod h1:W6aiYnuM+n0Q7G6fH3Ok4EE2iMfUSpA6f8r3i/+moBs= github.com/fil-forge/ucantone v0.0.0-20260706102443-79141c5cc52e h1:di/SseJVEO6bznSH/UEnAE2nSwwp/gqgnPLgE9/x2Zg= github.com/fil-forge/ucantone v0.0.0-20260706102443-79141c5cc52e/go.mod h1:oFY5BfD0bDeodGlbBHh3/nK99MAS93rGXjoQz7s5qgE= github.com/filecoin-project/go-data-segment v0.0.1 h1:1wmDxOG4ubWQm3ZC1XI5nCon5qgSq7Ra3Rb6Dbu10Gs= From ea773a5c52f5aca2e5dfa31e31b54280f302087c Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 15 Jul 2026 11:23:42 -0700 Subject: [PATCH 06/21] =?UTF-8?q?feat:=20deferred-accept=20multipart=20?= =?UTF-8?q?=E2=80=94=20park=20at=20UploadPart,=20accept=20at=20Complete,?= =?UTF-8?q?=20unallocate=20on=20abort=20(=C2=A77.2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UploadPart now uploads each part blob to its provider before returning 200: forgeclient.BlobAddParked runs /blob/add + the HTTP PUT but postpones the /http/put conclude — the conclude is what makes sprue fire /blob/accept, so the bytes are durable but stay out of the PDP pipeline. The park state (add/accept task links + the sealed put invocation) persists in the new blob_parks table (migration 00005). CompleteMultipartUpload concludes each parked blob (BlobConclude: send the put receipt, await the accept receipt's location commitment); blobs that never parked fall back to the whole synchronous upload. Abort, the TTL sweeper, and part-supersede unwind parked blobs with /blob/unallocate (cause = the add task link), behind the existing live-part/CountPartRefs/CountClaims guards — an upload now ends in exactly one of accept or unallocate. BlobAdd is unchanged behavior: it is BlobAddParked + BlobConclude composed. The harness NopUploader accepts immediately, so the in-process flow is unchanged. Architecture §2/§7.2/§9 rows flipped. Co-Authored-By: Claude Fable 5 --- cmd/serve.go | 2 + docs/architecture.md | 27 ++-- forgeclient/blobadd.go | 183 +++++++++++++++++----- forgeclient/blobunallocate.go | 48 ++++++ inmem/store.go | 32 +++- inmem/stores.go | 36 +++++ migrations/sql/00005_deferred_accept.sql | 19 +++ module.go | 21 ++- registry/stores.go | 35 ++++- registry/stores_postgres.go | 40 +++++ s3frontend/backend.go | 8 + s3frontend/multipart.go | 187 +++++++++++++++++++++-- server.go | 16 +- uploader/blob.go | 86 ++++++++++- 14 files changed, 653 insertions(+), 87 deletions(-) create mode 100644 forgeclient/blobunallocate.go create mode 100644 migrations/sql/00005_deferred_accept.sql diff --git a/cmd/serve.go b/cmd/serve.go index 01b7423..2221f92 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -98,10 +98,12 @@ func standaloneApp(cfg *DaemonConfig, logger *zap.Logger) (*fx.App, error) { fx.Annotate(func() *inmem.MemStore { return mem }, fx.As(new(registry.BlobRefStore))), fx.Annotate(func() *inmem.MemStore { return mem }, fx.As(new(registry.GCStore))), fx.Annotate(func() *inmem.MemStore { return mem }, fx.As(new(registry.MultipartStore))), + fx.Annotate(func() *inmem.MemStore { return mem }, fx.As(new(registry.ParkStore))), fx.Annotate(func() *inmem.MemStore { return mem }, fx.As(new(logstore.Meta))), fx.Annotate(func() inmem.NopBaseReader { return inmem.NopBaseReader{} }, fx.As(new(blockstore.BlockReader))), fx.Annotate(func() inmem.NopUploader { return inmem.NopUploader{} }, fx.As(new(uploader.Uploader))), fx.Annotate(func() inmem.NopUploader { return inmem.NopUploader{} }, fx.As(new(uploader.BodyUploader))), + fx.Annotate(func() inmem.NopUploader { return inmem.NopUploader{} }, fx.As(new(uploader.DeferredBodyUploader))), fx.Annotate(func() inmem.NopUploader { return inmem.NopUploader{} }, fx.As(new(uploader.BlobRemover))), ), ingot.ServerModule, diff --git a/docs/architecture.md b/docs/architecture.md index 0bc59b8..f477bfa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,11 +90,12 @@ The design is shaped by how the Forge upload pipeline works and by a handful of - **The 256 MiB blob ceiling is a knob, not a wall.** Piri currently caps a blob at 256 MiB because it builds the commP Merkle tree in RAM; known improvements (streaming commP) lift this. Ingot treats it as a tunable maximum and splits larger objects. -- **Some primitives are not built yet.** `blob/remove` is now handled end-to-end (Sprue - deregisters + forwards; Piri releases the space's claim and defers physical deletion until the - aggregate root retires on-chain via the signed `schedulePieceDeletions` path), but there is - still no `unallocate` for a parked blob and the indexer has no retraction. - [§9](#9-the-system-contract-piri--sprue--indexer) enumerates the contract Ingot needs. +- **The delete primitives are built.** `blob/remove` is handled end-to-end (Sprue deregisters + + forwards; Piri releases the space's claim and defers physical deletion until the aggregate root + retires on-chain via the signed `schedulePieceDeletions` path), and `blob/unallocate` retires a + parked (never-accepted) blob — an upload ends in exactly one of accept or unallocate. The + indexer still has no retraction. [§9](#9-the-system-contract-piri--sprue--indexer) enumerates + the contract Ingot needs. --- @@ -523,8 +524,8 @@ negotiations). | Capability | Service | Status | Notes | |------------------------------------------------------------------------------------------------|---------------------------------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `allocate` / `PUT` / `accept` blob lifecycle | Piri/Sprue | **exists** | The storage primitive Ingot builds on. | -| Ingot-timed accept (PUT a part, defer the conclude until Complete) | Ingot + Sprue | **partial** | Accept already fires on the client's conclude; needs the park-vs-conclude split client-side and a separable Sprue conclude. Ingot does **not** issue `accept` (Piri requires the upload-service DID). | -| `unallocate(digest)` — drop a parked blob | Piri + Sprue + libforge | **to-build** | `blob/remove` releases claims on accepted blobs; a dedicated parked-blob unallocate (pre-accept) is still open (allocation expiry is the backstop). | +| Ingot-timed accept (PUT a part, defer the conclude until Complete) | Ingot + Sprue | **exists** | `forgeclient.BlobAddParked`/`BlobConclude` split the flow at the conclude seam; UploadPart parks (durable, unaggregated), Complete concludes — Sprue's conclude handler already ran accept standalone. Ingot still does **not** issue `accept` (Piri requires the upload-service DID). | +| `unallocate(digest)` — drop a parked blob | Piri + Sprue + libforge | **exists** | `/blob/unallocate` on Piri refuses accepted blobs (`BlobAccepted`), deletes the space's allocation, and drops the bytes at zero allocations. Sprue recovers the provider from the `cause` receipt chain (a parked blob has no registration). Abort/TTL/supersede unwind through it. | | `remove(digest)` — per-space claim release; physical delete/piece-retire at zero global claims | Piri + Sprue + libforge | **exists** | `/blob/remove` on Piri deletes the space's allocation/acceptance/claim; at zero claims bytes delete immediately (unaggregated) or via the pending-removal sweep once the whole aggregate root is dead (FIL-623/624). Sprue forwards to primary + replicas (FIL-522). | | Configurable, adaptive size policy (`min`/`max`); batch guard for the `extraData` cap | Piri | **partial** | `MinAggregateSize` is hardcoded 128 MiB; lower to ~8 MiB and make configurable. The `addPieces` batch is no longer contract-capped (FWSS v1.3.0 removed the `extraData` cap); size it to the FVM `PiecesAdded` event-size + per-tx gas — a measured ceiling (default `BatchSize=10` is safely within it) (pdp-sim). No contract change. | | Compaction (Regime B) + complete the on-chain delete signature | Piri | **partial** | Whole-root delete is signed and wired (`schedulePieceDeletions` with `SignSchedulePieceRemovals` extraData); compaction (remove + re-hash survivors + re-add) is new. | @@ -800,14 +801,16 @@ reference index — and is out of scope for this iteration. The in-memory harness uses a no-op uploader and serves reads from the spool, so these forge-network paths are stubbed in-tree and verified against a real sprue+piri+indexer later: -- **`remove(digest)` is live; `unallocate(digest)` is still open.** `RemoveBlob` invokes - `/blob/remove` on sprue, which forwards to the storage nodes ([§9](#9-the-system-contract-piri--sprue--indexer)); delete finality means - claim-release-now, bytes-at-root-death. A dedicated pre-accept unallocate remains to-build - (allocation expiry is the backstop). +- **`remove(digest)` and `unallocate(digest)` are live.** `RemoveBlob` invokes `/blob/remove` on + sprue, which forwards to the storage nodes ([§9](#9-the-system-contract-piri--sprue--indexer)); delete finality means claim-release-now, + bytes-at-root-death. `UnallocateBlob` retires parked part-blobs on abort via `/blob/unallocate` + (provider recovered from the `cause` receipt chain); allocation-expiry GC (FIL-625) remains the + backstop when an abort never arrives. - **The local-table `Locator` read tier is not wired.** Body-blob *locations* are recorded at accept, but the read path that consumes them ([§7.4](#74-read-getobject), [§8](#8-retrieval-addressing-when-bodies-need-a-sharded-dag-index)) is deferred — it is only exercised after spool eviction (also not built) and is best validated live. -- **Multipart parts upload at Complete, not parked at UploadPart.** The in-process flow spools parts +- **Multipart parts park at UploadPart, accept at Complete.** (Built: `parkBlobs`/`concludeBlobs` + over the `blob_parks` table.) The in-process harness still spools parts at `UploadPart` and uploads+accepts them at `Complete`; the true forge *parking* (upload early, accept-at-Complete) and `unallocate`-on-abort from [§7.2](#72-multipart)–[7.3](#73-the-session-latch-the-abortcomplete-race) are forge-mode refinements. - **Crash recovery for the spool is not built.** The `upload_intents` × `blob_refs` reconciliation diff --git a/forgeclient/blobadd.go b/forgeclient/blobadd.go index 238371d..1e289fa 100644 --- a/forgeclient/blobadd.go +++ b/forgeclient/blobadd.go @@ -80,11 +80,53 @@ type AddedBlob struct { Location ucan.Invocation // the /assert/location commitment } +// ParkedBlob is the persistable state of a blob that has been added and +// uploaded (durable on the provider) but not yet concluded — piri holds the +// bytes without aggregating them until /blob/accept fires. Persist it and +// finish the upload later with [Client.BlobConclude], or abandon it with +// [Client.BlobUnallocate] (AddTask is the unallocate Cause). +type ParkedBlob struct { + Digest multihash.Multihash + Size uint64 + // AddTask is the /space/blob/add task link — the receipt-chain root the + // upload service uses to locate the provider for unallocate. + AddTask cid.Cid + // AcceptTask is the /blob/accept task link BlobConclude polls. + AcceptTask cid.Cid + // PutInvocation is the sealed /http/put invocation. Its metadata embeds + // the derived signer keys needed to synthesize the put receipt at + // conclude time — treat it as sensitive and delete it once concluded or + // unallocated. + PutInvocation []byte +} + // BlobAdd adds a blob to the upload service (sprue): invoke /blob/add, // PUT the bytes, conclude a synthesized /http/put receipt, then poll the // /blob/accept receipt for the location commitment. The issuer needs a // /blob/add delegation proof over space. -func (c *Client) BlobAdd(ctx context.Context, content io.Reader, space did.DID, options ...BlobAddOption) (blob AddedBlob, err error) { +// +// It is [Client.BlobAddParked] + [Client.BlobConclude] in one shot. +func (c *Client) BlobAdd(ctx context.Context, content io.Reader, space did.DID, options ...BlobAddOption) (AddedBlob, error) { + parked, added, err := c.BlobAddParked(ctx, content, space, options...) + if err != nil { + return AddedBlob{}, err + } + if added != nil { + return *added, nil + } + return c.BlobConclude(ctx, *parked, space) +} + +// BlobAddParked runs the durable half of BlobAdd: /blob/add + PUT the bytes, +// WITHOUT concluding the /http/put receipt — the conclude is what makes the +// upload service trigger /blob/accept on the provider, so the blob stays +// parked (stored, unaggregated) until BlobConclude. +// +// Exactly one of the returns is non-nil: a ParkedBlob awaiting conclude, or +// — when the provider already held accepted bytes for this content (dedup: +// allocate returned no upload address and the put receipt was pre-issued, so +// accept already ran) — the completed AddedBlob. +func (c *Client) BlobAddParked(ctx context.Context, content io.Reader, space did.DID, options ...BlobAddOption) (parked *ParkedBlob, added *AddedBlob, err error) { cfg := NewBlobAddConfig(options...) putClient := cfg.PutClient @@ -96,20 +138,20 @@ func (c *Client) BlobAdd(ctx context.Context, content io.Reader, space did.DID, start := time.Now() defer func() { if err != nil { - c.logger.Error("blob add failed", zap.Stringer("space", space), zap.Error(err), zap.Duration("duration", time.Since(start))) + c.logger.Error("blob add (parked) failed", zap.Stringer("space", space), zap.Error(err), zap.Duration("duration", time.Since(start))) } else { - c.logger.Debug("blob added", zap.Stringer("space", space), zap.Duration("duration", time.Since(start))) + c.logger.Debug("blob added (parked)", zap.Stringer("space", space), zap.Duration("duration", time.Since(start))) } }() if needsHash { contentBytes, rerr := io.ReadAll(content) if rerr != nil { - return AddedBlob{}, fmt.Errorf("reading content: %w", rerr) + return nil, nil, fmt.Errorf("reading content: %w", rerr) } contentHash, err = multihash.Sum(contentBytes, multihash.SHA2_256, -1) if err != nil { - return AddedBlob{}, fmt.Errorf("computing content multihash: %w", err) + return nil, nil, fmt.Errorf("computing content multihash: %w", err) } contentReader = bytes.NewReader(contentBytes) contentSize := uint64(len(contentBytes)) @@ -118,7 +160,7 @@ func (c *Client) BlobAdd(ctx context.Context, content io.Reader, space did.DID, proofs, proofLinks, err := c.ProofChain(ctx, c.signer.DID(), blobcmds.Add.Command, space) if err != nil { - return AddedBlob{}, fmt.Errorf("building proof chain: %w", err) + return nil, nil, fmt.Errorf("building proof chain: %w", err) } dlgPolicy, err := policy.Build( @@ -126,19 +168,19 @@ func (c *Client) BlobAdd(ctx context.Context, content io.Reader, space did.DID, policy.Equal(".blob.size", int64(*contentSizePtr)), ) if err != nil { - return AddedBlob{}, fmt.Errorf("building delegation policy: %w", err) + return nil, nil, fmt.Errorf("building delegation policy: %w", err) } allocDlg, allocProofs, err := delegateWithProofs( ctx, c.signer, c.serviceID, space, blobcmds.Allocate.Command, dlgPolicy, c) if err != nil { - return AddedBlob{}, fmt.Errorf("delegating /blob/allocate: %w", err) + return nil, nil, fmt.Errorf("delegating /blob/allocate: %w", err) } accDlg, accProofs, err := delegateWithProofs( ctx, c.signer, c.serviceID, space, blobcmds.Accept.Command, dlgPolicy, c) if err != nil { - return AddedBlob{}, fmt.Errorf("delegating /blob/accept: %w", err) + return nil, nil, fmt.Errorf("delegating /blob/accept: %w", err) } inv, err := blobcmds.Add.Invoke( @@ -148,7 +190,7 @@ func (c *Client) BlobAdd(ctx context.Context, content io.Reader, space did.DID, invocation.WithProofs(proofLinks...), ) if err != nil { - return AddedBlob{}, fmt.Errorf("generating invocation: %w", err) + return nil, nil, fmt.Errorf("generating invocation: %w", err) } addOK, _, meta, err := Execute[*blobcmds.AddOK]( @@ -159,51 +201,51 @@ func (c *Client) BlobAdd(ctx context.Context, content io.Reader, space did.DID, execution.WithDelegations(accProofs...), ) if err != nil { - return AddedBlob{}, fmt.Errorf("executing blob add: %w", err) + return nil, nil, fmt.Errorf("executing blob add: %w", err) } accInv, err := findInvocation(addOK.Site.Task, meta.Invocations()) if err != nil { - return AddedBlob{}, fmt.Errorf("finding /blob/accept invocation: %w", err) + return nil, nil, fmt.Errorf("finding /blob/accept invocation: %w", err) } var accArgs blobcmds.AcceptArguments if err := accArgs.UnmarshalCBOR(bytes.NewReader(accInv.ArgumentsBytes())); err != nil { - return AddedBlob{}, fmt.Errorf("unmarshaling /blob/accept arguments: %w", err) + return nil, nil, fmt.Errorf("unmarshaling /blob/accept arguments: %w", err) } putInv, err := findInvocation(accArgs.Put.Task, meta.Invocations()) if err != nil { - return AddedBlob{}, fmt.Errorf("finding /http/put invocation: %w", err) + return nil, nil, fmt.Errorf("finding /http/put invocation: %w", err) } var putArgs httpcmds.PutArguments if err := putArgs.UnmarshalCBOR(bytes.NewReader(putInv.ArgumentsBytes())); err != nil { - return AddedBlob{}, fmt.Errorf("unmarshaling /http/put arguments: %w", err) + return nil, nil, fmt.Errorf("unmarshaling /http/put arguments: %w", err) } putRcpt := maybeFindReceipt(accArgs.Put.Task, meta.Receipts()) allocInv, err := findInvocation(putArgs.Destination.Task, meta.Invocations()) if err != nil { - return AddedBlob{}, fmt.Errorf("finding /blob/allocate invocation: %w", err) + return nil, nil, fmt.Errorf("finding /blob/allocate invocation: %w", err) } var allocArgs blobcmds.AllocateArguments if err := allocArgs.UnmarshalCBOR(bytes.NewReader(allocInv.ArgumentsBytes())); err != nil { - return AddedBlob{}, fmt.Errorf("unmarshaling /blob/allocate arguments: %w", err) + return nil, nil, fmt.Errorf("unmarshaling /blob/allocate arguments: %w", err) } allocRcpt, err := findReceipt(putArgs.Destination.Task, meta.Receipts()) if err != nil { - return AddedBlob{}, fmt.Errorf("finding /blob/allocate receipt: %w", err) + return nil, nil, fmt.Errorf("finding /blob/allocate receipt: %w", err) } o, x := allocRcpt.Out().Unpack() if allocRcpt.Out().IsErr() { var model edm.ErrorModel if err := model.UnmarshalCBOR(bytes.NewReader(x)); err != nil { - return AddedBlob{}, fmt.Errorf("executing invocation") + return nil, nil, fmt.Errorf("executing invocation") } - return AddedBlob{}, fmt.Errorf("failure in allocation receipt: %w", model) + return nil, nil, fmt.Errorf("failure in allocation receipt: %w", model) } var allocOK blobcmds.AllocateOK if err := allocOK.UnmarshalCBOR(bytes.NewReader(o)); err != nil { - return AddedBlob{}, fmt.Errorf("unmarshaling allocation receipt output: %w", err) + return nil, nil, fmt.Errorf("unmarshaling allocation receipt output: %w", err) } putSuccess := putRcpt != nil && putRcpt.Out().IsOK() @@ -214,40 +256,98 @@ func (c *Client) BlobAdd(ctx context.Context, content io.Reader, space did.DID, // piri's PUT endpoint requires it. if allocOK.Address != nil && !putSuccess { if err := putBlob(ctx, putClient, allocOK.Address.URL.URL(), allocOK.Address.Headers, contentReader, int64(*contentSizePtr)); err != nil { - return AddedBlob{}, fmt.Errorf("putting blob: %w", err) + return nil, nil, fmt.Errorf("putting blob: %w", err) } } - // Conclude a synthesized /http/put receipt so /blob/accept can resolve. - if !putSuccess { - accDlg, accProofs, derr := delegateWithProofs( - ctx, c.signer, c.serviceID, space, blobcmds.Accept.Command, dlgPolicy, c) - if derr != nil { - return AddedBlob{}, fmt.Errorf("delegating /blob/accept: %w", derr) + // Dedup path: the provider already held accepted bytes for this content, + // so the put receipt was pre-issued and the upload service ran accept + // synchronously — the blob is not parked. Await the accept receipt and + // return the completed AddedBlob. + if putSuccess { + location, aerr := c.awaitAccept(ctx, accInv.Task().Link()) + if aerr != nil { + err = aerr + return nil, nil, err } - if err := c.sendPutReceipt(ctx, putInv, - execution.WithDelegations(accDlg), - execution.WithDelegations(accProofs...), - ); err != nil { - return AddedBlob{}, fmt.Errorf("sending put receipt: %w", err) + return nil, &AddedBlob{Digest: contentHash, Size: *contentSizePtr, Location: location}, nil + } + + // Parked: durable on the provider, conclude deferred to BlobConclude. + return &ParkedBlob{ + Digest: contentHash, + Size: *contentSizePtr, + AddTask: inv.Task().Link(), + AcceptTask: accInv.Task().Link(), + PutInvocation: putInv.Bytes(), + }, nil, nil +} + +// BlobConclude finishes a parked upload: it synthesizes and concludes the +// /http/put receipt (which makes the upload service trigger /blob/accept on +// the provider) and awaits the accept receipt's location commitment. Safe to +// retry — re-concluding an already-concluded put is tolerated upstream. +func (c *Client) BlobConclude(ctx context.Context, parked ParkedBlob, space did.DID) (blob AddedBlob, err error) { + start := time.Now() + defer func() { + if err != nil { + c.logger.Error("blob conclude failed", zap.Stringer("space", space), zap.Error(err), zap.Duration("duration", time.Since(start))) + } else { + c.logger.Debug("blob concluded", zap.Stringer("space", space), zap.Duration("duration", time.Since(start))) } + }() + + putInv := new(invocation.Invocation) + if err := putInv.UnmarshalCBOR(bytes.NewReader(parked.PutInvocation)); err != nil { + return AddedBlob{}, fmt.Errorf("decoding parked /http/put invocation: %w", err) + } + + // The accept delegation is re-derived rather than persisted: it is a + // function of the signer, space, and the digest/size policy. + dlgPolicy, err := policy.Build( + policy.Equal(".blob.digest", []byte(parked.Digest)), + policy.Equal(".blob.size", int64(parked.Size)), + ) + if err != nil { + return AddedBlob{}, fmt.Errorf("building delegation policy: %w", err) + } + accDlg, accProofs, err := delegateWithProofs( + ctx, c.signer, c.serviceID, space, blobcmds.Accept.Command, dlgPolicy, c) + if err != nil { + return AddedBlob{}, fmt.Errorf("delegating /blob/accept: %w", err) + } + if err := c.sendPutReceipt(ctx, putInv, + execution.WithDelegations(accDlg), + execution.WithDelegations(accProofs...), + ); err != nil { + return AddedBlob{}, fmt.Errorf("sending put receipt: %w", err) } - accRcpt, accMeta, err := c.receiptsClient.Poll(ctx, accInv.Task().Link(), receipt_client.WithRetries(5)) + location, err := c.awaitAccept(ctx, parked.AcceptTask) if err != nil { - return AddedBlob{}, fmt.Errorf("polling accept receipt: %w", err) + return AddedBlob{}, err } - o, x = accRcpt.Out().Unpack() + return AddedBlob{Digest: parked.Digest, Size: parked.Size, Location: location}, nil +} + +// awaitAccept polls the /blob/accept receipt and extracts the +// /assert/location commitment from its metadata. +func (c *Client) awaitAccept(ctx context.Context, acceptTask cid.Cid) (ucan.Invocation, error) { + accRcpt, accMeta, err := c.receiptsClient.Poll(ctx, acceptTask, receipt_client.WithRetries(5)) + if err != nil { + return nil, fmt.Errorf("polling accept receipt: %w", err) + } + o, x := accRcpt.Out().Unpack() if accRcpt.Out().IsErr() { var model edm.ErrorModel if err := model.UnmarshalCBOR(bytes.NewReader(x)); err != nil { - return AddedBlob{}, fmt.Errorf("executing invocation") + return nil, fmt.Errorf("executing invocation") } - return AddedBlob{}, fmt.Errorf("failure in accept receipt: %w", model) + return nil, fmt.Errorf("failure in accept receipt: %w", model) } var accOK blobcmds.AcceptOK if err := accOK.UnmarshalCBOR(bytes.NewReader(o)); err != nil { - return AddedBlob{}, fmt.Errorf("unmarshaling accept receipt output: %w", err) + return nil, fmt.Errorf("unmarshaling accept receipt output: %w", err) } var locationCommitment ucan.Invocation @@ -257,10 +357,9 @@ func (c *Client) BlobAdd(ctx context.Context, content io.Reader, space did.DID, } } if locationCommitment == nil { - return AddedBlob{}, fmt.Errorf("blob accept receipt missing location commitment invocation") + return nil, fmt.Errorf("blob accept receipt missing location commitment invocation") } - - return AddedBlob{Digest: contentHash, Size: *contentSizePtr, Location: locationCommitment}, nil + return locationCommitment, nil } func putBlob(ctx context.Context, client *http.Client, url *url.URL, headers map[string]string, body io.Reader, size int64) error { diff --git a/forgeclient/blobunallocate.go b/forgeclient/blobunallocate.go new file mode 100644 index 0000000..4e706c2 --- /dev/null +++ b/forgeclient/blobunallocate.go @@ -0,0 +1,48 @@ +package forgeclient + +import ( + "context" + "fmt" + + blobcmds "github.com/fil-forge/libforge/commands/blob" + "github.com/fil-forge/ucantone/did" + "github.com/fil-forge/ucantone/execution" + "github.com/fil-forge/ucantone/ucan/invocation" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" +) + +// BlobUnallocate invokes /blob/unallocate against the upload service (sprue), +// retiring the space's parked (never-accepted) blob. cause is the +// /space/blob/add task link (ParkedBlob.AddTask) — sprue walks its receipt +// chain to locate the storage node holding the parked bytes, which have no +// registration or acceptance to look up by. Idempotent on the node; a blob +// that has been accepted is refused (release it via the reference index / +// /blob/remove instead). +func (c *Client) BlobUnallocate(ctx context.Context, digest multihash.Multihash, space did.DID, cause cid.Cid) error { + proofs, proofLinks, err := c.ProofChain(ctx, c.signer.DID(), blobcmds.Unallocate.Command, space) + if err != nil { + return fmt.Errorf("building proof chain: %w", err) + } + inv, err := blobcmds.Unallocate.Invoke( + c.signer, + space, + &blobcmds.UnallocateArguments{Space: space, Digest: digest, Cause: &cause}, + invocation.WithAudience(c.serviceID), + invocation.WithProofs(proofLinks...), + ) + if err != nil { + return fmt.Errorf("creating invocation: %w", err) + } + + _, _, _, err = Execute[*blobcmds.UnallocateOK]( + ctx, + c.ucanClient, + inv, + execution.WithDelegations(proofs...), + ) + if err != nil { + return fmt.Errorf("executing invocation: %w", err) + } + return nil +} diff --git a/inmem/store.go b/inmem/store.go index f2bf970..d322c5b 100644 --- a/inmem/store.go +++ b/inmem/store.go @@ -42,6 +42,7 @@ type MemStore struct { blobRefs map[claimKey]registry.BlobClaim intents map[string]registry.UploadIntent // keyed by string(digest) locations map[locKey]registry.BlobLocation // keyed by (space, digest) + parks map[string]registry.BlobPark // keyed by string(digest) sessions map[string]registry.MultipartSession // keyed by uploadID parts map[string]map[int]registry.MultipartPart // uploadID -> partNumber -> part gcCands map[string]struct{} // keyed by string(cid) @@ -65,6 +66,7 @@ func NewMemStore() *MemStore { blobRefs: map[claimKey]registry.BlobClaim{}, intents: map[string]registry.UploadIntent{}, locations: map[locKey]registry.BlobLocation{}, + parks: map[string]registry.BlobPark{}, sessions: map[string]registry.MultipartSession{}, parts: map[string]map[int]registry.MultipartPart{}, gcCands: map[string]struct{}{}, @@ -258,13 +260,29 @@ func (NopUploader) UploadBlob(_ context.Context, _ multihash.Multihash, size int func (NopUploader) RemoveBlob(_ context.Context, _ multihash.Multihash) error { return nil } +// UploadBlobParked accepts immediately in the harness — there is no network +// to park on, so the deferred flow degenerates to the synchronous one and +// reads keep coming from the spool. +func (NopUploader) UploadBlobParked(_ context.Context, _ multihash.Multihash, size int64, _ string) (*uploader.ParkedBlobState, *uploader.BlobLocation, error) { + return nil, &uploader.BlobLocation{Size: size}, nil +} + +func (NopUploader) ConcludeBlob(_ context.Context, parked uploader.ParkedBlobState) (uploader.BlobLocation, error) { + return uploader.BlobLocation{Size: int64(parked.Size)}, nil +} + +func (NopUploader) UnallocateBlob(_ context.Context, _ multihash.Multihash, _ cid.Cid) error { + return nil +} + // Compile-time guarantees. var ( - _ registry.Registry = (*MemStore)(nil) - _ logstore.Meta = (*MemStore)(nil) - _ blockstore.BlockReader = NopBaseReader{} - _ blockstore.BlobReader = NopBaseReader{} - _ uploader.Uploader = NopUploader{} - _ uploader.BodyUploader = NopUploader{} - _ uploader.BlobRemover = NopUploader{} + _ registry.Registry = (*MemStore)(nil) + _ logstore.Meta = (*MemStore)(nil) + _ blockstore.BlockReader = NopBaseReader{} + _ blockstore.BlobReader = NopBaseReader{} + _ uploader.Uploader = NopUploader{} + _ uploader.BodyUploader = NopUploader{} + _ uploader.DeferredBodyUploader = NopUploader{} + _ uploader.BlobRemover = NopUploader{} ) diff --git a/inmem/stores.go b/inmem/stores.go index e01bdc9..4b56f7d 100644 --- a/inmem/stores.go +++ b/inmem/stores.go @@ -149,6 +149,42 @@ func (m *MemStore) DeleteLocation(_ context.Context, space string, digest []byte return nil } +// ParkStore ================================================================== + +func (m *MemStore) PutPark(_ context.Context, p registry.BlobPark) error { + m.mu.Lock() + defer m.mu.Unlock() + cp := p + cp.Digest = cloneBytes(p.Digest) + cp.AddTask = cloneBytes(p.AddTask) + cp.AcceptTask = cloneBytes(p.AcceptTask) + cp.PutInvocation = cloneBytes(p.PutInvocation) + m.parks[string(p.Digest)] = cp + return nil +} + +func (m *MemStore) GetPark(_ context.Context, digest []byte) (*registry.BlobPark, error) { + m.mu.Lock() + defer m.mu.Unlock() + park, ok := m.parks[string(digest)] + if !ok { + return nil, registry.ErrNotFound + } + cp := park + cp.Digest = cloneBytes(park.Digest) + cp.AddTask = cloneBytes(park.AddTask) + cp.AcceptTask = cloneBytes(park.AcceptTask) + cp.PutInvocation = cloneBytes(park.PutInvocation) + return &cp, nil +} + +func (m *MemStore) DeletePark(_ context.Context, digest []byte) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.parks, string(digest)) + return nil +} + // MultipartStore ============================================================= func (m *MemStore) CreateSession(_ context.Context, s registry.MultipartSession) error { diff --git a/migrations/sql/00005_deferred_accept.sql b/migrations/sql/00005_deferred_accept.sql new file mode 100644 index 0000000..98b4d4b --- /dev/null +++ b/migrations/sql/00005_deferred_accept.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- Deferred-accept multipart (§7.2): a part's blobs upload to the provider at +-- UploadPart but the /http/put conclude — which triggers /blob/accept — is +-- deferred to Complete. blob_parks holds the state needed to conclude (or +-- unallocate) later. put_invocation is the sealed /http/put invocation whose +-- metadata embeds the derived signer keys; rows are deleted promptly at +-- conclude/unallocate. Keyed globally by digest, like upload_intents: +-- content-addressed dedup shares a park across sessions and parts. +CREATE TABLE ingot.blob_parks ( + digest bytea PRIMARY KEY, + add_task bytea NOT NULL, -- /space/blob/add task CID (unallocate cause) + accept_task bytea NOT NULL, -- /blob/accept task CID (conclude poll target) + put_invocation bytea NOT NULL, -- sealed /http/put invocation + size bigint NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() +); + +-- +goose Down +DROP TABLE ingot.blob_parks; diff --git a/module.go b/module.go index c3f1196..7b7f30f 100644 --- a/module.go +++ b/module.go @@ -135,6 +135,7 @@ type serverParams struct { Reader blockstore.BlockReader Uploader uploader.Uploader BodyUploader uploader.BodyUploader + Deferred uploader.DeferredBodyUploader Remover uploader.BlobRemover Registry registry.Registry Intents registry.IntentStore @@ -142,6 +143,7 @@ type serverParams struct { BlobRefs registry.BlobRefStore GC registry.GCStore Multipart registry.MultipartStore + Parks registry.ParkStore Meta logstore.Meta // Space is the Forge space this instance owns. Optional: standalone / the // harness don't provide it (reads come from the spool), so it defaults to "". @@ -179,6 +181,7 @@ func registerServerLifecycle(lc fx.Lifecycle, p serverParams) { BaseBlockReader: p.Reader, Uploader: p.Uploader, BodyUploader: p.BodyUploader, + Deferred: p.Deferred, Remover: p.Remover, Registry: p.Registry, Intents: p.Intents, @@ -186,6 +189,7 @@ func registerServerLifecycle(lc fx.Lifecycle, p serverParams) { BlobRefs: p.BlobRefs, GC: p.GC, Multipart: p.Multipart, + Parks: p.Parks, Meta: p.Meta, Space: string(p.Space), }) @@ -294,6 +298,7 @@ type registryResult struct { BlobRefs registry.BlobRefStore GC registry.GCStore Multipart registry.MultipartStore + Parks registry.ParkStore Meta logstore.Meta } @@ -304,7 +309,7 @@ type registryResult struct { // candidate log (GCStore), and segment metadata (Meta). func provideRegistry(pool *pgxpool.Pool) registryResult { pg := registry.NewPostgres(pool) - return registryResult{Registry: pg, Intents: pg, Locations: pg, BlobRefs: pg, GC: pg, Multipart: pg, Meta: pg} + return registryResult{Registry: pg, Intents: pg, Locations: pg, BlobRefs: pg, GC: pg, Multipart: pg, Parks: pg, Meta: pg} } // provideServerSpace exposes the owned space DID (as a string) to the server. @@ -360,6 +365,7 @@ type uploaderResult struct { Uploader uploader.Uploader BodyUploader uploader.BodyUploader + Deferred uploader.DeferredBodyUploader Remover uploader.BlobRemover } @@ -376,7 +382,7 @@ func provideUploader(c *forgeclient.Client, space spaceIssuer, logger *zap.Logge if err != nil { return uploaderResult{}, err } - return uploaderResult{Uploader: f, BodyUploader: f, Remover: f}, nil + return uploaderResult{Uploader: f, BodyUploader: f, Deferred: f, Remover: f}, nil } // seedSpaceDelegations self-issues no-expiry space→agent delegations for @@ -384,10 +390,10 @@ func provideUploader(c *forgeclient.Client, space spaceIssuer, logger *zap.Logge // holds a /blob/add chain for the agent (idempotent across restarts). func seedSpaceDelegations(ctx context.Context, spaceIssuer ucan.Issuer, agent did.DID, store tokenstore.Store, logger *zap.Logger) error { space := spaceIssuer.DID() - // Sentinel on /blob/remove: the newest cap in the list — a store missing - // it was seeded by an older build and re-seeds here (AddDelegations is - // additive, so re-seeding the earlier caps is harmless). - if proofs, _, err := store.ProofChain(ctx, agent, blobcmds.Remove.Command, space); err == nil && len(proofs) > 0 { + // Sentinel on /blob/unallocate: the newest cap in the list — a store + // missing it was seeded by an older build and re-seeds here + // (AddDelegations is additive, so re-seeding the earlier caps is harmless). + if proofs, _, err := store.ProofChain(ctx, agent, blobcmds.Unallocate.Command, space); err == nil && len(proofs) > 0 { return nil // already seeded (or login-provisioned) } // The edge-client flow needs space->agent authority over: /blob/add (the @@ -406,6 +412,9 @@ func seedSpaceDelegations(ctx context.Context, spaceIssuer ucan.Issuer, agent di {"/blob/remove", func() (ucan.Delegation, error) { return blobcmds.Remove.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) }}, + {"/blob/unallocate", func() (ucan.Delegation, error) { + return blobcmds.Unallocate.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) + }}, {"/blob/allocate", func() (ucan.Delegation, error) { return blobcmds.Allocate.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) }}, diff --git a/registry/stores.go b/registry/stores.go index 167b0ab..c218856 100644 --- a/registry/stores.go +++ b/registry/stores.go @@ -79,11 +79,11 @@ type BlobLocation struct { // headers (ContentEncoding..Expires) are captured at CreateMultipartUpload so // Complete can write them into the manifest exactly like a single-shot PUT. type MultipartSession struct { - UploadID string - Bucket string - ObjectKey string - State string - ContentType string + UploadID string + Bucket string + ObjectKey string + State string + ContentType string ContentEncoding string ContentDisposition string ContentLanguage string @@ -143,6 +143,31 @@ type LocationStore interface { DeleteLocation(ctx context.Context, space string, digest []byte) error } +// BlobPark is one row of ingot.blob_parks: the persistable state of a blob +// that is durable on its provider but not yet accepted (multipart's deferred +// conclude, §7.2). AddTask/AcceptTask are the /space/blob/add and +// /blob/accept task CIDs; PutInvocation is the sealed /http/put invocation +// whose metadata carries the derived signer keys needed to conclude — +// sensitive, deleted at conclude/unallocate. Keyed globally by Digest (like +// upload_intents: content-addressed dedup shares parks across sessions). +type BlobPark struct { + Digest []byte + AddTask []byte // cid bytes + AcceptTask []byte // cid bytes + PutInvocation []byte + Size int64 + CreatedAt time.Time +} + +// ParkStore persists deferred-accept park state between UploadPart and +// Complete/Abort (§7.2). +type ParkStore interface { + PutPark(ctx context.Context, p BlobPark) error + // GetPark returns ErrNotFound when digest has no park row. + GetPark(ctx context.Context, digest []byte) (*BlobPark, error) + DeletePark(ctx context.Context, digest []byte) error +} + // MultipartStore tracks in-flight multipart uploads (§7.2). LatchSession is // the single-winner transition (§7.3): exactly one of Complete/Abort moves // the session off 'open'. diff --git a/registry/stores_postgres.go b/registry/stores_postgres.go index 8470c7e..e87b99b 100644 --- a/registry/stores_postgres.go +++ b/registry/stores_postgres.go @@ -179,6 +179,46 @@ func (r *Postgres) DeleteLocation(ctx context.Context, space string, digest []by return nil } +// ParkStore ================================================================== + +func (r *Postgres) PutPark(ctx context.Context, p BlobPark) error { + _, err := r.pool.Exec(ctx, + `INSERT INTO ingot.blob_parks (digest, add_task, accept_task, put_invocation, size) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT (digest) DO UPDATE + SET add_task = EXCLUDED.add_task, accept_task = EXCLUDED.accept_task, + put_invocation = EXCLUDED.put_invocation, size = EXCLUDED.size`, + p.Digest, p.AddTask, p.AcceptTask, p.PutInvocation, p.Size) + if err != nil { + return fmt.Errorf("registry: put park: %w", err) + } + return nil +} + +func (r *Postgres) GetPark(ctx context.Context, digest []byte) (*BlobPark, error) { + park := &BlobPark{Digest: digest} + err := r.pool.QueryRow(ctx, + `SELECT add_task, accept_task, put_invocation, size, created_at + FROM ingot.blob_parks WHERE digest = $1`, + digest).Scan(&park.AddTask, &park.AcceptTask, &park.PutInvocation, &park.Size, &park.CreatedAt) + if errors.Is(err, pgx.ErrNoRows) { + return nil, ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("registry: get park: %w", err) + } + return park, nil +} + +func (r *Postgres) DeletePark(ctx context.Context, digest []byte) error { + _, err := r.pool.Exec(ctx, + `DELETE FROM ingot.blob_parks WHERE digest = $1`, digest) + if err != nil { + return fmt.Errorf("registry: delete park: %w", err) + } + return nil +} + // MultipartStore ============================================================= func (r *Postgres) CreateSession(ctx context.Context, s MultipartSession) error { diff --git a/s3frontend/backend.go b/s3frontend/backend.go index 200e7cd..0039a09 100644 --- a/s3frontend/backend.go +++ b/s3frontend/backend.go @@ -49,6 +49,8 @@ type Backend struct { txns *bucketop.Coordinator spool *blockstore.Spool uploader uploader.BodyUploader + deferred uploader.DeferredBodyUploader + parks registry.ParkStore remover uploader.BlobRemover // space is the Forge space this instance owns; the key under which body @@ -87,6 +89,10 @@ type Deps struct { // accept) synchronously, before the manifest commits. Remover releases a // space's claim on a blob when its last reference is dropped. Uploader uploader.BodyUploader + // Deferred decomposes Uploader for multipart's deferred accept; Parks + // persists park state between UploadPart and Complete/Abort. + Deferred uploader.DeferredBodyUploader + Parks registry.ParkStore Remover uploader.BlobRemover // Space is the Forge space this instance owns (empty in the harness). @@ -112,6 +118,8 @@ func New(d Deps) *Backend { txns: bucketop.NewCoordinator(bucketop.Deps{Reg: d.Registry, Log: d.Log, Reads: d.Reads}), spool: d.Spool, uploader: d.Uploader, + deferred: d.Deferred, + parks: d.Parks, remover: d.Remover, space: d.Space, maxBlobSize: d.MaxBlobSize, diff --git a/s3frontend/multipart.go b/s3frontend/multipart.go index e4a207a..9f4d6f3 100644 --- a/s3frontend/multipart.go +++ b/s3frontend/multipart.go @@ -23,6 +23,7 @@ import ( "github.com/fil-forge/ingot/bucketop" "github.com/fil-forge/ingot/mst" "github.com/fil-forge/ingot/registry" + "github.com/fil-forge/ingot/uploader" ) // minPartSize is S3's minimum size for every part except the last one, @@ -115,13 +116,15 @@ func (b *Backend) openSession(ctx context.Context, uploadID string, key *string) return sess, nil } -// UploadPart ingests one part: it coarse-splits the part body into blobs and -// spools each to local disk (recording upload_intents), then records the part -// (its ordered blob digests, md5, size). The part's blobs are NOT uploaded to -// Forge yet — that is deferred to Complete (so an Abort only ever cleans up -// local state). Re-uploading a part number supersedes the prior part; the -// superseded part's now-unreferenced blobs are dropped from the spool. The -// part ETag is the hex md5 of the part bytes. +// UploadPart ingests one part: it coarse-splits the part body into blobs, +// spools each to local disk (recording upload_intents), records the part +// (its ordered blob digests, md5, size), and uploads each blob to its +// provider — PARKED, not accepted: the /http/put conclude that triggers +// /blob/accept is deferred to Complete, so the bytes are durable but stay +// out of the PDP pipeline, and an Abort unwinds them with /blob/unallocate +// (§7.2). Re-uploading a part number supersedes the prior part; the +// superseded part's now-unreferenced blobs are dropped from the spool and +// unallocated. The part ETag is the hex md5 of the part bytes. func (b *Backend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { if input.Bucket == nil || input.Key == nil || input.UploadId == nil || input.PartNumber == nil { return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -158,6 +161,13 @@ func (b *Backend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s }); err != nil { return nil, fmt.Errorf("s3frontend: record part: %w", err) } + // Park the part's blobs on their providers before returning 200 — the + // part is durable on the network as soon as the client sees success. + // The part row is recorded first so a crash mid-park leaves re-drivable + // spooled intents. + if err := b.parkBlobs(ctx, body.Blobs); err != nil { + return nil, fmt.Errorf("s3frontend: park part blobs: %w", err) + } if len(superseded) > 0 { b.cleanupPartBlobs(ctx, uploadID, superseded) } @@ -317,8 +327,11 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet partSizes = append(partSizes, offset-partStart) } - // Accept every part's blobs on Forge (no-op in standalone), then commit. - if err := b.uploadBlobs(ctx, blobs); err != nil { + // Accept every part's blobs on Forge: parked blobs conclude (the deferred + // /http/put receipt fires /blob/accept), stragglers that never parked + // (crash between spool and park) fall back to the whole synchronous + // upload. No-op in standalone. Then commit. + if err := b.concludeBlobs(ctx, blobs); err != nil { return s3response.CompleteMultipartUploadResult{}, "", fmt.Errorf("s3frontend: accept parts: %w", err) } @@ -353,8 +366,10 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet // AbortMultipartUpload cancels a multipart upload: it latches the session // (single-winner vs Complete), drops it (cascading its parts), and removes the -// parts' now-unreferenced blobs from the spool. No reference claims were taken -// (those happen only at Complete), so nothing network-side needs unwinding. +// parts' now-unreferenced blobs from the spool — unallocating any that were +// parked on a provider (an upload ends in exactly one of accept or +// unallocate). No reference claims were taken (those happen only at +// Complete). func (b *Backend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error { if input.UploadId == nil { return s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -427,15 +442,161 @@ func (b *Backend) cleanupPartBlobs(ctx context.Context, uploadID string, digests if n, err := b.blobRefs.CountClaims(ctx, b.space, d); err != nil || n > 0 { continue } - if in, err := b.intents.GetIntent(ctx, d); err == nil && in.State != registry.IntentSpooled { - // Shipped/accepted blobs are the reference index's to manage. + state := registry.IntentSpooled + if in, err := b.intents.GetIntent(ctx, d); err == nil { + state = in.State + } + if state != registry.IntentSpooled && state != registry.IntentParked { + // Accepted/published blobs are the reference index's to manage. continue } + // A parked blob is durable on its provider — release it there too + // (best-effort; piri's unallocate is idempotent, a straggler is + // FIL-625's to reap). Cause is the /space/blob/add task link the + // upload service needs to locate the provider. + if state == registry.IntentParked { + if park, err := b.parks.GetPark(ctx, d); err == nil { + if cause, err := cid.Cast(park.AddTask); err == nil { + _ = b.deferred.UnallocateBlob(ctx, mh.Multihash(d), cause) + } + _ = b.parks.DeletePark(ctx, d) + } + } _ = b.spool.Remove(mh.Multihash(d)) _ = b.intents.DeleteIntent(ctx, d) } } +// parkBlobs makes each blob durable on its provider without accepting it: +// already-located blobs are marked accepted (dedup), already-parked blobs are +// skipped (another part or session parked the same content), the rest run +// the parked upload and persist their park state for Complete/Abort. +func (b *Backend) parkBlobs(ctx context.Context, blobs []msbucket.BlobRef) error { + for _, blob := range blobs { + digest := mh.Multihash(blob.Digest) + if existing, err := b.locations.GetLocation(ctx, b.space, blob.Digest); err == nil && existing != nil { + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { + return fmt.Errorf("mark accepted (dedup): %w", err) + } + continue + } else if err != nil && !errors.Is(err, registry.ErrNotFound) { + return fmt.Errorf("lookup location: %w", err) + } + if _, err := b.parks.GetPark(ctx, blob.Digest); err == nil { + continue // already parked by a sibling part or session + } else if !errors.Is(err, registry.ErrNotFound) { + return fmt.Errorf("lookup park: %w", err) + } + + parked, located, err := b.deferred.UploadBlobParked(ctx, digest, blob.Length, b.spool.Path(digest)) + if err != nil { + return fmt.Errorf("park blob: %w", err) + } + if located != nil { + // The provider already held accepted bytes for this content — + // accept ran, record the location like the synchronous path. + if err := b.locations.PutLocation(ctx, registry.BlobLocation{ + Space: b.space, + Digest: blob.Digest, + Provider: located.Provider, + URL: located.URL, + Size: located.Size, + }); err != nil { + return fmt.Errorf("record location: %w", err) + } + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { + return fmt.Errorf("mark accepted: %w", err) + } + continue + } + if err := b.parks.PutPark(ctx, registry.BlobPark{ + Digest: blob.Digest, + AddTask: parked.AddTask.Bytes(), + AcceptTask: parked.AcceptTask.Bytes(), + PutInvocation: parked.PutInvocation, + Size: blob.Length, + }); err != nil { + return fmt.Errorf("record park: %w", err) + } + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentParked); err != nil { + return fmt.Errorf("mark parked: %w", err) + } + } + return nil +} + +// concludeBlobs is Complete's park-aware counterpart to uploadBlobs: located +// blobs are already accepted (dedup); parked blobs conclude their deferred +// /http/put receipt — firing /blob/accept — and record their location; +// blobs that never parked (crash between spool and park) fall back to the +// whole synchronous upload. +func (b *Backend) concludeBlobs(ctx context.Context, blobs []msbucket.BlobRef) error { + for _, blob := range blobs { + digest := mh.Multihash(blob.Digest) + if existing, err := b.locations.GetLocation(ctx, b.space, blob.Digest); err == nil && existing != nil { + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { + return fmt.Errorf("mark accepted (dedup): %w", err) + } + continue + } else if err != nil && !errors.Is(err, registry.ErrNotFound) { + return fmt.Errorf("lookup location: %w", err) + } + + park, err := b.parks.GetPark(ctx, blob.Digest) + if err != nil && !errors.Is(err, registry.ErrNotFound) { + return fmt.Errorf("lookup park: %w", err) + } + var loc uploader.BlobLocation + if park != nil { + addTask, err := cid.Cast(park.AddTask) + if err != nil { + return fmt.Errorf("decode park add task: %w", err) + } + acceptTask, err := cid.Cast(park.AcceptTask) + if err != nil { + return fmt.Errorf("decode park accept task: %w", err) + } + loc, err = b.deferred.ConcludeBlob(ctx, uploader.ParkedBlobState{ + Digest: digest, + Size: uint64(park.Size), + AddTask: addTask, + AcceptTask: acceptTask, + PutInvocation: park.PutInvocation, + }) + if err != nil { + return fmt.Errorf("conclude blob: %w", err) + } + } else { + // Never parked (crash between spool and park): the spooled copy + // drives the whole synchronous upload. + loc, err = b.uploader.UploadBlob(ctx, digest, blob.Length, b.spool.Path(digest)) + if err != nil { + return fmt.Errorf("upload blob: %w", err) + } + } + + if err := b.locations.PutLocation(ctx, registry.BlobLocation{ + Space: b.space, + Digest: blob.Digest, + Provider: loc.Provider, + URL: loc.URL, + Size: loc.Size, + }); err != nil { + return fmt.Errorf("record location: %w", err) + } + if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { + return fmt.Errorf("mark accepted: %w", err) + } + if park != nil { + // The sealed put invocation is spent — drop it promptly. + if err := b.parks.DeletePark(ctx, blob.Digest); err != nil { + return fmt.Errorf("drop park: %w", err) + } + } + } + return nil +} + // ListParts returns the recorded parts of an in-flight upload, paginated by // part number. func (b *Backend) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3response.ListPartsResult, error) { diff --git a/server.go b/server.go index e237117..274cac6 100644 --- a/server.go +++ b/server.go @@ -94,7 +94,10 @@ type ServerDeps struct { // space's claim on a blob when its last reference is dropped. In tests both // are no-ops and reads are served from the local spool. BodyUploader uploader.BodyUploader - Remover uploader.BlobRemover + // Deferred is BodyUploader decomposed for multipart's deferred accept: + // park at UploadPart, conclude at Complete, unallocate at Abort. + Deferred uploader.DeferredBodyUploader + Remover uploader.BlobRemover // Registry tracks per-bucket roots. *registry.Postgres satisfies // both Registry and Meta in production; tests can supply two @@ -111,6 +114,9 @@ type ServerDeps struct { BlobRefs registry.BlobRefStore GC registry.GCStore Multipart registry.MultipartStore + // Parks persists deferred-accept park state between UploadPart and + // Complete/Abort. + Parks registry.ParkStore // Space is the Forge space this instance owns — the key body-blob locations // (and, later, reference claims) are recorded under. Empty in standalone / @@ -178,10 +184,12 @@ func New(ctx context.Context, cfg ServerConfig, deps ServerDeps) (*Server, error BlobRefs: deps.BlobRefs, GC: deps.GC, Multipart: deps.Multipart, + Parks: deps.Parks, Reads: bs, Log: log, Spool: spool, Uploader: deps.BodyUploader, + Deferred: deps.Deferred, Remover: deps.Remover, Space: deps.Space, MaxBlobSize: cfg.MaxBlobSize, @@ -383,6 +391,12 @@ func validateServerInputs(cfg ServerConfig, deps ServerDeps) error { if deps.BodyUploader == nil { return errors.New("ingot: ServerDeps.BodyUploader is required") } + if deps.Deferred == nil { + return errors.New("ingot: ServerDeps.Deferred is required") + } + if deps.Parks == nil { + return errors.New("ingot: ServerDeps.Parks is required") + } if deps.Registry == nil { return errors.New("ingot: ServerDeps.Registry is required") } diff --git a/uploader/blob.go b/uploader/blob.go index 505246d..ddc25cc 100644 --- a/uploader/blob.go +++ b/uploader/blob.go @@ -8,6 +8,7 @@ import ( assertcmds "github.com/fil-forge/libforge/commands/assert" "github.com/fil-forge/libforge/digestutil" + "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" "go.uber.org/zap" @@ -40,7 +41,7 @@ type BodyUploader interface { // UploadBlob uploads one spooled blob to Forge. For a single-shot PutObject the // allocate→PUT→accept happens in one call (forgeclient.BlobAdd already drives // the whole flow and returns the location commitment); multipart's deferred -// accept will need a decomposed path (a later phase). +// accept uses the decomposed UploadBlobParked/ConcludeBlob pair instead. func (u *Forge) UploadBlob(ctx context.Context, digest multihash.Multihash, size int64, localPath string) (BlobLocation, error) { f, err := os.Open(localPath) if err != nil { @@ -55,7 +56,12 @@ func (u *Forge) UploadBlob(ctx context.Context, digest multihash.Multihash, size if err != nil { return BlobLocation{}, fmt.Errorf("uploader: upload blob: %w", err) } + return locationOf(added) +} +// locationOf extracts the local blob-location record from a completed add's +// /assert/location commitment. +func locationOf(added forgeclient.AddedBlob) (BlobLocation, error) { loc := BlobLocation{Size: int64(added.Size)} // added.Location is the /assert/location commitment piri issued at accept; // parse its arguments for the provider DID + retrieval URL so a later read @@ -76,6 +82,84 @@ func (u *Forge) UploadBlob(ctx context.Context, digest multihash.Multihash, size var _ BodyUploader = (*Forge)(nil) +// ParkedBlobState is the persistable state of a blob that is durable on the +// provider but not yet accepted (the /http/put conclude is deferred). It is +// stored in the blob_parks table between UploadPart and Complete/Abort. +type ParkedBlobState = forgeclient.ParkedBlob + +// DeferredBodyUploader is the decomposed BodyUploader for multipart's +// deferred accept: UploadBlobParked makes the bytes durable (parked) at +// UploadPart; ConcludeBlob triggers accept at Complete; UnallocateBlob +// abandons a parked blob at Abort. Exactly one of UploadBlobParked's returns +// is non-nil — an already-accepted (deduped) blob completes immediately. +type DeferredBodyUploader interface { + UploadBlobParked(ctx context.Context, digest multihash.Multihash, size int64, localPath string) (*ParkedBlobState, *BlobLocation, error) + ConcludeBlob(ctx context.Context, parked ParkedBlobState) (BlobLocation, error) + UnallocateBlob(ctx context.Context, digest multihash.Multihash, cause cid.Cid) error +} + +// UploadBlobParked makes one spooled blob durable on the provider WITHOUT +// accepting it: /blob/add + PUT, conclude deferred. Returns the persistable +// park state, or the completed location when the provider already held +// accepted bytes for this content (dedup). +func (u *Forge) UploadBlobParked(ctx context.Context, digest multihash.Multihash, size int64, localPath string) (*ParkedBlobState, *BlobLocation, error) { + f, err := os.Open(localPath) + if err != nil { + return nil, nil, fmt.Errorf("uploader: open spooled blob %s: %w", localPath, err) + } + defer f.Close() + + parked, added, err := u.client.BlobAddParked(ctx, f, u.space, + forgeclient.WithPrecomputedDigest(digest, uint64(size)), + forgeclient.WithPutClient(u.putClient), + ) + if err != nil { + return nil, nil, fmt.Errorf("uploader: park blob: %w", err) + } + if added != nil { + loc, err := locationOf(*added) + if err != nil { + return nil, nil, err + } + return nil, &loc, nil + } + return parked, nil, nil +} + +// ConcludeBlob finishes a parked upload: it concludes the deferred /http/put +// receipt (triggering /blob/accept on the provider) and returns the published +// location. Safe to retry. +func (u *Forge) ConcludeBlob(ctx context.Context, parked ParkedBlobState) (BlobLocation, error) { + added, err := u.client.BlobConclude(ctx, parked, u.space) + if err != nil { + return BlobLocation{}, fmt.Errorf("uploader: conclude blob: %w", err) + } + return locationOf(added) +} + +// UnallocateBlob abandons a parked blob via /blob/unallocate on the upload +// service: sprue recovers the provider from the cause receipt chain and the +// node releases the allocation + parked bytes. cause is the parked blob's +// AddTask. Errors are logged here (callers treat abort cleanup as +// best-effort and may discard them). +func (u *Forge) UnallocateBlob(ctx context.Context, digest multihash.Multihash, cause cid.Cid) error { + u.logger.Info("blob unallocate", + zap.Stringer("space", u.space), + zap.String("digest", digestutil.Format(digest)), + ) + if err := u.client.BlobUnallocate(ctx, digest, u.space, cause); err != nil { + u.logger.Error("blob unallocate failed", + zap.Stringer("space", u.space), + zap.String("digest", digestutil.Format(digest)), + zap.Error(err), + ) + return fmt.Errorf("uploader: unallocating blob: %w", err) + } + return nil +} + +var _ DeferredBodyUploader = (*Forge)(nil) + // BlobRemover releases this space's claim on an accepted blob. Because dedup is // global, Piri deletes the bytes and retires the piece only when no space // claims the digest at all (docs/architecture.md §6). The space is owned by the From 09cbe0dfa723f1440709bfb56e74cc7112968c15 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 15 Jul 2026 11:32:04 -0700 Subject: [PATCH 07/21] test: deferred-multipart e2e gate + blob_parks live-store coverage TestForgeDeferredMultipart (env-gated like the delete-finality test): RoundTrip proves parts are durable-but-parked at UploadPart (piri allocates, no accept), concluded at Complete (accepts land, GET round-trips); AbortUnallocates proves abort unwinds a parked blob end-to-end via /blob/unallocate. Plus a park round-trip in the postgres live test. Co-Authored-By: Claude Fable 5 --- itest/forge_multipart_deferred_test.go | 210 +++++++++++++++++++++++++ registry/postgres_live_test.go | 31 ++++ 2 files changed, 241 insertions(+) create mode 100644 itest/forge_multipart_deferred_test.go diff --git a/itest/forge_multipart_deferred_test.go b/itest/forge_multipart_deferred_test.go new file mode 100644 index 0000000..3e287fb --- /dev/null +++ b/itest/forge_multipart_deferred_test.go @@ -0,0 +1,210 @@ +//go:build itest + +package itest + +import ( + "bytes" + "context" + "os" + "strings" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/fil-forge/libforge/digestutil" + "github.com/fil-forge/smelt/pkg/stack" + "github.com/multiformats/go-multihash" +) + +// TestForgeDeferredMultipart is the deferred-accept regression gate (§7.2): +// UploadPart must make each part blob durable on piri WITHOUT accepting it +// (parked — outside the PDP pipeline), Complete must conclude (accept) every +// part, and Abort must unwind parked blobs with /blob/unallocate. +// +// Requires piri + sprue builds with /blob/remove + /blob/unallocate; the +// published images predate them, so inject working-tree binaries and skip +// otherwise (same gate as TestForgeDeleteReleasesNetworkBlob): +// +// (cd ../../piri && CGO_ENABLED=0 GOOS=linux go build -o /tmp/piri ./cmd) +// (cd ../../sprue && CGO_ENABLED=0 GOOS=linux go build -o /tmp/sprue ./cmd/main.go) +// ITEST_PIRI_BIN=/tmp/piri ITEST_SPRUE_BIN=/tmp/sprue \ +// go test -tags itest ./itest -run TestForgeDeferredMultipart -v -timeout 1200s +func TestForgeDeferredMultipart(t *testing.T) { + piriBin := os.Getenv("ITEST_PIRI_BIN") + sprueBin := os.Getenv("ITEST_SPRUE_BIN") + if piriBin == "" || sprueBin == "" { + t.Skip("requires piri+sprue builds with the /blob/unallocate chain: set ITEST_PIRI_BIN and ITEST_SPRUE_BIN (see test doc comment)") + } + + ctx := t.Context() + s, endpoint := forgeStack(t, + stack.WithServiceBinary("piri", piriBin), + stack.WithServiceBinary("upload", sprueBin), + ) + ingotSelfProvision(t, ctx, s, "test@example.com") + cl := sdkClient(forgeS3Conf(endpoint)) + + // RoundTrip: parts park at UploadPart (durable on piri, NOT accepted), + // Complete concludes them, and the object round-trips. + t.Run("RoundTrip", func(t *testing.T) { + const bucket, key = "mp-deferred", "obj" + if _, err := cl.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { + t.Fatalf("CreateBucket: %v", err) + } + create, err := cl.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), + }) + if err != nil { + t.Fatalf("CreateMultipartUpload: %v", err) + } + + // Unique per-part content (XOR-tagged so nothing dedups against other + // tests); each part is a single blob under the default max_blob_size. + partData := [][]byte{tagged(patternBytes(6<<20), 0x11), tagged(patternBytes(90<<10), 0x22)} + var completed []types.CompletedPart + var whole []byte + var partDigests []string + for i, data := range partData { + pn := int32(i + 1) + up, err := cl.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + PartNumber: aws.Int32(pn), Body: bytes.NewReader(data), + }) + if err != nil { + t.Fatalf("UploadPart %d: %v", pn, err) + } + completed = append(completed, types.CompletedPart{PartNumber: aws.Int32(pn), ETag: up.ETag}) + whole = append(whole, data...) + partDigests = append(partDigests, b58Digest(t, data)) + } + + // The parts are durable on piri (allocated + PUT) but parked: no + // /blob/accept for their digests until Complete. + for _, d := range partDigests { + waitForPiriLogLine(t, ctx, s, 30*time.Second, "/blob/allocate", d) + if piriLogHasLine(t, ctx, s, "/blob/accept", d) { + t.Fatalf("part blob %s was accepted before Complete — parking is broken", d) + } + } + + comp, err := cl.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{Parts: completed}, + }) + if err != nil { + t.Fatalf("CompleteMultipartUpload: %v", err) + } + if et := strings.Trim(aws.ToString(comp.ETag), `"`); !strings.HasSuffix(et, "-2") { + t.Fatalf("complete ETag = %q, want a multipart -2 suffix", et) + } + + // Complete concluded every part: accepts landed on piri, and the + // object round-trips. + for _, d := range partDigests { + waitForPiriLogLine(t, ctx, s, 60*time.Second, "/blob/accept", d) + } + if got := getBody(t, ctx, cl, bucket, key, ""); !bytes.Equal(got, whole) { + t.Fatalf("GET mismatch: got %d bytes, want %d", len(got), len(whole)) + } + }) + + // AbortUnallocates: an aborted upload's parked blobs are unwound on piri + // via /blob/unallocate — never accepted, bytes released. + t.Run("AbortUnallocates", func(t *testing.T) { + const bucket, key = "mp-abort-unalloc", "obj" + if _, err := cl.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { + t.Fatalf("CreateBucket: %v", err) + } + create, err := cl.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), + }) + if err != nil { + t.Fatalf("CreateMultipartUpload: %v", err) + } + data := tagged(patternBytes(512<<10), 0x33) + digest := b58Digest(t, data) + if _, err := cl.UploadPart(ctx, &s3.UploadPartInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + PartNumber: aws.Int32(1), Body: bytes.NewReader(data), + }); err != nil { + t.Fatalf("UploadPart: %v", err) + } + waitForPiriLogLine(t, ctx, s, 30*time.Second, "/blob/allocate", digest) + + if _, err := cl.AbortMultipartUpload(ctx, &s3.AbortMultipartUploadInput{ + Bucket: aws.String(bucket), Key: aws.String(key), UploadId: create.UploadId, + }); err != nil { + t.Fatalf("AbortMultipartUpload: %v", err) + } + + // The unallocate traversed ingot → sprue → piri, and the blob was + // never accepted. + waitForPiriLogLine(t, ctx, s, 60*time.Second, "/blob/unallocate", digest) + if piriLogHasLine(t, ctx, s, "/blob/accept", digest) { + t.Fatalf("aborted part blob %s was accepted — abort/accept exclusivity is broken", digest) + } + t.Logf("abort unwound parked blob %s via /blob/unallocate", digest) + }) +} + +// tagged XORs b with tag so each caller gets globally unique content that +// cannot dedup against other tests' blobs. +func tagged(b []byte, tag byte) []byte { + for i := range b { + b[i] ^= tag + } + return b +} + +// b58Digest returns the base58 sha2-256 multihash of data — the form piri's +// handlers log blob digests in (digestutil.Format). +func b58Digest(t *testing.T, data []byte) string { + t.Helper() + mh, err := multihash.Sum(data, multihash.SHA2_256, -1) + if err != nil { + t.Fatalf("multihash: %v", err) + } + return digestutil.Format(mh) +} + +// piriLogHasLine reports whether any single piri-0 log line contains all +// substrings. +func piriLogHasLine(t *testing.T, ctx context.Context, s *stack.Stack, substrs ...string) bool { + t.Helper() + logs, err := s.Logs(ctx, "piri-0") + if err != nil { + t.Fatalf("piri-0 logs: %v", err) + } + for _, line := range strings.Split(logs, "\n") { + ok := true + for _, sub := range substrs { + if !strings.Contains(line, sub) { + ok = false + break + } + } + if ok { + return true + } + } + return false +} + +// waitForPiriLogLine polls until one piri-0 log line contains all substrings. +func waitForPiriLogLine(t *testing.T, ctx context.Context, s *stack.Stack, timeout time.Duration, substrs ...string) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if piriLogHasLine(t, ctx, s, substrs...) { + return + } + select { + case <-ctx.Done(): + t.Fatalf("context done waiting for piri log %v: %v", substrs, ctx.Err()) + case <-time.After(2 * time.Second): + } + } + t.Fatalf("piri-0 logs never contained a line with all of %v within %s", substrs, timeout) +} diff --git a/registry/postgres_live_test.go b/registry/postgres_live_test.go index 8901843..cdbf873 100644 --- a/registry/postgres_live_test.go +++ b/registry/postgres_live_test.go @@ -142,6 +142,37 @@ func TestPostgresStores_Live(t *testing.T) { } }) + t.Run("park round trip", func(t *testing.T) { + park := registry.BlobPark{ + Digest: digest, + AddTask: []byte{0x01, 0x02}, + AcceptTask: []byte{0x03, 0x04}, + PutInvocation: []byte("sealed-inv"), + Size: 42, + } + if err := r.PutPark(ctx, park); err != nil { + t.Fatalf("PutPark: %v", err) + } + got, err := r.GetPark(ctx, digest) + if err != nil || string(got.PutInvocation) != "sealed-inv" || got.Size != 42 { + t.Fatalf("GetPark = %+v, err %v", got, err) + } + // Upsert replaces in place. + park.Size = 43 + if err := r.PutPark(ctx, park); err != nil { + t.Fatalf("PutPark (upsert): %v", err) + } + if got, err := r.GetPark(ctx, digest); err != nil || got.Size != 43 { + t.Fatalf("GetPark after upsert = %+v, err %v", got, err) + } + if err := r.DeletePark(ctx, digest); err != nil { + t.Fatalf("DeletePark: %v", err) + } + if _, err := r.GetPark(ctx, digest); err != registry.ErrNotFound { + t.Fatalf("GetPark after delete = %v, want ErrNotFound", err) + } + }) + t.Run("multipart session parts latch metadata", func(t *testing.T) { const id = "upl-1" meta := map[string]string{"x-amz-meta-foo": "bar"} From 526cc92e06e709a3aa7ca5deb8ff43f63deee926 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 15 Jul 2026 14:15:30 -0700 Subject: [PATCH 08/21] test(itest): delete-finality asserts through piri's async removal sweep MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Piri's byte release is now fully asynchronous (defer-by-default RemovePiece + PDPRemoveSweep periodic task): the sweep re-verifies claims and pipeline state before deleting. Follow the renamed 'queueing piece removal' log line and additionally wait for 'finalized piece removal' — proving end-to-end that the sweep actually released the bytes, not merely queued the request. Co-Authored-By: Claude Fable 5 --- itest/forge_delete_test.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/itest/forge_delete_test.go b/itest/forge_delete_test.go index 475094e..c3de13b 100644 --- a/itest/forge_delete_test.go +++ b/itest/forge_delete_test.go @@ -82,12 +82,15 @@ func TestForgeDeleteReleasesNetworkBlob(t *testing.T) { } // And the release traversed the network: piri's /blob/remove handler ran - // and, with the last claim gone, queued the piece for removal. The - // removal is asynchronous on the ingot side (releaseBlobs is best-effort, - // post-commit), so poll the provider's logs. + // and, with the last claim gone, queued the piece for removal. Byte + // release is fully asynchronous — ingot's releaseBlobs is best-effort + // post-commit, and piri's removal sweep (PDPRemoveSweep, 30s ticks) + // re-verifies claims and pipeline state before finalizing — so poll the + // provider's logs through to the finalization line. waitForPiriLog(t, ctx, s, "/blob/remove", 2*time.Minute) - waitForPiriLog(t, ctx, s, "removing piece", 2*time.Minute) - t.Logf("delete finality OK: /blob/remove executed on piri and the piece was queued for removal") + waitForPiriLog(t, ctx, s, "queueing piece removal", 2*time.Minute) + waitForPiriLog(t, ctx, s, "finalized piece removal", 3*time.Minute) + t.Logf("delete finality OK: /blob/remove executed on piri and the sweep finalized the byte release") } // waitForPiriLog polls piri-0's container logs until substr appears. From 391df326e26677d0c476c611b44be23c7984f43f Mon Sep 17 00:00:00 2001 From: frrist Date: Fri, 17 Jul 2026 12:59:43 -0700 Subject: [PATCH 09/21] =?UTF-8?q?rename:=20/blob/unallocate=20=E2=86=92=20?= =?UTF-8?q?/blob/abort=20+=20/blob/reject=20(libforge=20PR=20#46)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The abandon verb split per hop in review: clients (ingot) invoke /blob/abort on the upload service, which forwards /blob/reject to the storage node. BlobUnallocate → BlobAbort (Cause now required by type), uploader iface UnallocateBlob → AbortBlob, seeded space delegation + itest log-greps renamed. Co-Authored-By: Claude Fable 5 --- .../{blobunallocate.go => blobabort.go} | 23 ++++++++++--------- forgeclient/blobadd.go | 6 ++--- inmem/store.go | 2 +- itest/forge_multipart_deferred_test.go | 18 +++++++-------- module.go | 8 +++---- registry/stores.go | 2 +- s3frontend/multipart.go | 10 ++++---- server.go | 2 +- uploader/blob.go | 14 +++++------ 9 files changed, 43 insertions(+), 42 deletions(-) rename forgeclient/{blobunallocate.go => blobabort.go} (50%) diff --git a/forgeclient/blobunallocate.go b/forgeclient/blobabort.go similarity index 50% rename from forgeclient/blobunallocate.go rename to forgeclient/blobabort.go index 4e706c2..e258d5c 100644 --- a/forgeclient/blobunallocate.go +++ b/forgeclient/blobabort.go @@ -12,22 +12,23 @@ import ( "github.com/multiformats/go-multihash" ) -// BlobUnallocate invokes /blob/unallocate against the upload service (sprue), -// retiring the space's parked (never-accepted) blob. cause is the -// /space/blob/add task link (ParkedBlob.AddTask) — sprue walks its receipt -// chain to locate the storage node holding the parked bytes, which have no -// registration or acceptance to look up by. Idempotent on the node; a blob -// that has been accepted is refused (release it via the reference index / +// BlobAbort invokes /blob/abort against the upload service (sprue), +// abandoning the space's in-flight upload of a parked (never-accepted) +// blob. cause is the /space/blob/add task link (ParkedBlob.AddTask) — sprue +// walks its receipt chain to locate the storage node holding the parked +// bytes (which have no registration or acceptance to look up by) and +// forwards a /blob/reject there. Idempotent on the node; a blob that has +// been accepted is refused (release it via the reference index / // /blob/remove instead). -func (c *Client) BlobUnallocate(ctx context.Context, digest multihash.Multihash, space did.DID, cause cid.Cid) error { - proofs, proofLinks, err := c.ProofChain(ctx, c.signer.DID(), blobcmds.Unallocate.Command, space) +func (c *Client) BlobAbort(ctx context.Context, digest multihash.Multihash, space did.DID, cause cid.Cid) error { + proofs, proofLinks, err := c.ProofChain(ctx, c.signer.DID(), blobcmds.Abort.Command, space) if err != nil { return fmt.Errorf("building proof chain: %w", err) } - inv, err := blobcmds.Unallocate.Invoke( + inv, err := blobcmds.Abort.Invoke( c.signer, space, - &blobcmds.UnallocateArguments{Space: space, Digest: digest, Cause: &cause}, + &blobcmds.AbortArguments{Space: space, Digest: digest, Cause: cause}, invocation.WithAudience(c.serviceID), invocation.WithProofs(proofLinks...), ) @@ -35,7 +36,7 @@ func (c *Client) BlobUnallocate(ctx context.Context, digest multihash.Multihash, return fmt.Errorf("creating invocation: %w", err) } - _, _, _, err = Execute[*blobcmds.UnallocateOK]( + _, _, _, err = Execute[*blobcmds.AbortOK]( ctx, c.ucanClient, inv, diff --git a/forgeclient/blobadd.go b/forgeclient/blobadd.go index 1e289fa..4d17e5e 100644 --- a/forgeclient/blobadd.go +++ b/forgeclient/blobadd.go @@ -84,19 +84,19 @@ type AddedBlob struct { // uploaded (durable on the provider) but not yet concluded — piri holds the // bytes without aggregating them until /blob/accept fires. Persist it and // finish the upload later with [Client.BlobConclude], or abandon it with -// [Client.BlobUnallocate] (AddTask is the unallocate Cause). +// [Client.BlobAbort] (AddTask is the abort Cause). type ParkedBlob struct { Digest multihash.Multihash Size uint64 // AddTask is the /space/blob/add task link — the receipt-chain root the - // upload service uses to locate the provider for unallocate. + // upload service uses to locate the provider for abort. AddTask cid.Cid // AcceptTask is the /blob/accept task link BlobConclude polls. AcceptTask cid.Cid // PutInvocation is the sealed /http/put invocation. Its metadata embeds // the derived signer keys needed to synthesize the put receipt at // conclude time — treat it as sensitive and delete it once concluded or - // unallocated. + // rejected. PutInvocation []byte } diff --git a/inmem/store.go b/inmem/store.go index d322c5b..35e0265 100644 --- a/inmem/store.go +++ b/inmem/store.go @@ -271,7 +271,7 @@ func (NopUploader) ConcludeBlob(_ context.Context, parked uploader.ParkedBlobSta return uploader.BlobLocation{Size: int64(parked.Size)}, nil } -func (NopUploader) UnallocateBlob(_ context.Context, _ multihash.Multihash, _ cid.Cid) error { +func (NopUploader) AbortBlob(_ context.Context, _ multihash.Multihash, _ cid.Cid) error { return nil } diff --git a/itest/forge_multipart_deferred_test.go b/itest/forge_multipart_deferred_test.go index 3e287fb..92a2525 100644 --- a/itest/forge_multipart_deferred_test.go +++ b/itest/forge_multipart_deferred_test.go @@ -21,9 +21,9 @@ import ( // TestForgeDeferredMultipart is the deferred-accept regression gate (§7.2): // UploadPart must make each part blob durable on piri WITHOUT accepting it // (parked — outside the PDP pipeline), Complete must conclude (accept) every -// part, and Abort must unwind parked blobs with /blob/unallocate. +// part, and Abort must unwind parked blobs with /blob/abort → /blob/reject. // -// Requires piri + sprue builds with /blob/remove + /blob/unallocate; the +// Requires piri + sprue builds with /blob/remove + /blob/abort/reject; the // published images predate them, so inject working-tree binaries and skip // otherwise (same gate as TestForgeDeleteReleasesNetworkBlob): // @@ -35,7 +35,7 @@ func TestForgeDeferredMultipart(t *testing.T) { piriBin := os.Getenv("ITEST_PIRI_BIN") sprueBin := os.Getenv("ITEST_SPRUE_BIN") if piriBin == "" || sprueBin == "" { - t.Skip("requires piri+sprue builds with the /blob/unallocate chain: set ITEST_PIRI_BIN and ITEST_SPRUE_BIN (see test doc comment)") + t.Skip("requires piri+sprue builds with the /blob/abort chain: set ITEST_PIRI_BIN and ITEST_SPRUE_BIN (see test doc comment)") } ctx := t.Context() @@ -110,9 +110,9 @@ func TestForgeDeferredMultipart(t *testing.T) { } }) - // AbortUnallocates: an aborted upload's parked blobs are unwound on piri - // via /blob/unallocate — never accepted, bytes released. - t.Run("AbortUnallocates", func(t *testing.T) { + // AbortRejects: an aborted upload's parked blobs are unwound on piri + // via /blob/reject — never accepted, bytes released. + t.Run("AbortRejects", func(t *testing.T) { const bucket, key = "mp-abort-unalloc", "obj" if _, err := cl.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(bucket)}); err != nil { t.Fatalf("CreateBucket: %v", err) @@ -139,13 +139,13 @@ func TestForgeDeferredMultipart(t *testing.T) { t.Fatalf("AbortMultipartUpload: %v", err) } - // The unallocate traversed ingot → sprue → piri, and the blob was + // The abort traversed ingot → sprue → piri (as /blob/reject), and the blob was // never accepted. - waitForPiriLogLine(t, ctx, s, 60*time.Second, "/blob/unallocate", digest) + waitForPiriLogLine(t, ctx, s, 60*time.Second, "/blob/reject", digest) if piriLogHasLine(t, ctx, s, "/blob/accept", digest) { t.Fatalf("aborted part blob %s was accepted — abort/accept exclusivity is broken", digest) } - t.Logf("abort unwound parked blob %s via /blob/unallocate", digest) + t.Logf("abort unwound parked blob %s via /blob/reject", digest) }) } diff --git a/module.go b/module.go index 7b7f30f..4ecf4d5 100644 --- a/module.go +++ b/module.go @@ -390,10 +390,10 @@ func provideUploader(c *forgeclient.Client, space spaceIssuer, logger *zap.Logge // holds a /blob/add chain for the agent (idempotent across restarts). func seedSpaceDelegations(ctx context.Context, spaceIssuer ucan.Issuer, agent did.DID, store tokenstore.Store, logger *zap.Logger) error { space := spaceIssuer.DID() - // Sentinel on /blob/unallocate: the newest cap in the list — a store + // Sentinel on /blob/abort: the newest cap in the list — a store // missing it was seeded by an older build and re-seeds here // (AddDelegations is additive, so re-seeding the earlier caps is harmless). - if proofs, _, err := store.ProofChain(ctx, agent, blobcmds.Unallocate.Command, space); err == nil && len(proofs) > 0 { + if proofs, _, err := store.ProofChain(ctx, agent, blobcmds.Abort.Command, space); err == nil && len(proofs) > 0 { return nil // already seeded (or login-provisioned) } // The edge-client flow needs space->agent authority over: /blob/add (the @@ -412,8 +412,8 @@ func seedSpaceDelegations(ctx context.Context, spaceIssuer ucan.Issuer, agent di {"/blob/remove", func() (ucan.Delegation, error) { return blobcmds.Remove.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) }}, - {"/blob/unallocate", func() (ucan.Delegation, error) { - return blobcmds.Unallocate.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) + {"/blob/abort", func() (ucan.Delegation, error) { + return blobcmds.Abort.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) }}, {"/blob/allocate", func() (ucan.Delegation, error) { return blobcmds.Allocate.Delegate(spaceIssuer, agent, space, delegation.WithNoExpiration()) diff --git a/registry/stores.go b/registry/stores.go index c218856..e35b880 100644 --- a/registry/stores.go +++ b/registry/stores.go @@ -148,7 +148,7 @@ type LocationStore interface { // conclude, §7.2). AddTask/AcceptTask are the /space/blob/add and // /blob/accept task CIDs; PutInvocation is the sealed /http/put invocation // whose metadata carries the derived signer keys needed to conclude — -// sensitive, deleted at conclude/unallocate. Keyed globally by Digest (like +// sensitive, deleted at conclude/abort. Keyed globally by Digest (like // upload_intents: content-addressed dedup shares parks across sessions). type BlobPark struct { Digest []byte diff --git a/s3frontend/multipart.go b/s3frontend/multipart.go index 9f4d6f3..fec5814 100644 --- a/s3frontend/multipart.go +++ b/s3frontend/multipart.go @@ -121,10 +121,10 @@ func (b *Backend) openSession(ctx context.Context, uploadID string, key *string) // (its ordered blob digests, md5, size), and uploads each blob to its // provider — PARKED, not accepted: the /http/put conclude that triggers // /blob/accept is deferred to Complete, so the bytes are durable but stay -// out of the PDP pipeline, and an Abort unwinds them with /blob/unallocate +// out of the PDP pipeline, and an Abort unwinds them with /blob/abort // (§7.2). Re-uploading a part number supersedes the prior part; the // superseded part's now-unreferenced blobs are dropped from the spool and -// unallocated. The part ETag is the hex md5 of the part bytes. +// rejected. The part ETag is the hex md5 of the part bytes. func (b *Backend) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3.UploadPartOutput, error) { if input.Bucket == nil || input.Key == nil || input.UploadId == nil || input.PartNumber == nil { return nil, s3err.GetAPIError(s3err.ErrInvalidRequest) @@ -368,7 +368,7 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet // (single-winner vs Complete), drops it (cascading its parts), and removes the // parts' now-unreferenced blobs from the spool — unallocating any that were // parked on a provider (an upload ends in exactly one of accept or -// unallocate). No reference claims were taken (those happen only at +// abort). No reference claims were taken (those happen only at // Complete). func (b *Backend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error { if input.UploadId == nil { @@ -451,13 +451,13 @@ func (b *Backend) cleanupPartBlobs(ctx context.Context, uploadID string, digests continue } // A parked blob is durable on its provider — release it there too - // (best-effort; piri's unallocate is idempotent, a straggler is + // (best-effort; the reject on piri is idempotent, a straggler is // FIL-625's to reap). Cause is the /space/blob/add task link the // upload service needs to locate the provider. if state == registry.IntentParked { if park, err := b.parks.GetPark(ctx, d); err == nil { if cause, err := cid.Cast(park.AddTask); err == nil { - _ = b.deferred.UnallocateBlob(ctx, mh.Multihash(d), cause) + _ = b.deferred.AbortBlob(ctx, mh.Multihash(d), cause) } _ = b.parks.DeletePark(ctx, d) } diff --git a/server.go b/server.go index 274cac6..39dabbc 100644 --- a/server.go +++ b/server.go @@ -95,7 +95,7 @@ type ServerDeps struct { // are no-ops and reads are served from the local spool. BodyUploader uploader.BodyUploader // Deferred is BodyUploader decomposed for multipart's deferred accept: - // park at UploadPart, conclude at Complete, unallocate at Abort. + // park at UploadPart, conclude at Complete, abort at Abort. Deferred uploader.DeferredBodyUploader Remover uploader.BlobRemover diff --git a/uploader/blob.go b/uploader/blob.go index ddc25cc..d49d01b 100644 --- a/uploader/blob.go +++ b/uploader/blob.go @@ -89,13 +89,13 @@ type ParkedBlobState = forgeclient.ParkedBlob // DeferredBodyUploader is the decomposed BodyUploader for multipart's // deferred accept: UploadBlobParked makes the bytes durable (parked) at -// UploadPart; ConcludeBlob triggers accept at Complete; UnallocateBlob +// UploadPart; ConcludeBlob triggers accept at Complete; AbortBlob // abandons a parked blob at Abort. Exactly one of UploadBlobParked's returns // is non-nil — an already-accepted (deduped) blob completes immediately. type DeferredBodyUploader interface { UploadBlobParked(ctx context.Context, digest multihash.Multihash, size int64, localPath string) (*ParkedBlobState, *BlobLocation, error) ConcludeBlob(ctx context.Context, parked ParkedBlobState) (BlobLocation, error) - UnallocateBlob(ctx context.Context, digest multihash.Multihash, cause cid.Cid) error + AbortBlob(ctx context.Context, digest multihash.Multihash, cause cid.Cid) error } // UploadBlobParked makes one spooled blob durable on the provider WITHOUT @@ -137,18 +137,18 @@ func (u *Forge) ConcludeBlob(ctx context.Context, parked ParkedBlobState) (BlobL return locationOf(added) } -// UnallocateBlob abandons a parked blob via /blob/unallocate on the upload +// AbortBlob abandons a parked blob via /blob/abort on the upload // service: sprue recovers the provider from the cause receipt chain and the // node releases the allocation + parked bytes. cause is the parked blob's // AddTask. Errors are logged here (callers treat abort cleanup as // best-effort and may discard them). -func (u *Forge) UnallocateBlob(ctx context.Context, digest multihash.Multihash, cause cid.Cid) error { - u.logger.Info("blob unallocate", +func (u *Forge) AbortBlob(ctx context.Context, digest multihash.Multihash, cause cid.Cid) error { + u.logger.Info("blob abort", zap.Stringer("space", u.space), zap.String("digest", digestutil.Format(digest)), ) - if err := u.client.BlobUnallocate(ctx, digest, u.space, cause); err != nil { - u.logger.Error("blob unallocate failed", + if err := u.client.BlobAbort(ctx, digest, u.space, cause); err != nil { + u.logger.Error("blob abort failed", zap.Stringer("space", u.space), zap.String("digest", digestutil.Format(digest)), zap.Error(err), From af4f92e38a0721b63d7f3565abefd2a19b0a20c3 Mon Sep 17 00:00:00 2001 From: frrist Date: Fri, 17 Jul 2026 13:00:58 -0700 Subject: [PATCH 10/21] test(itest): boot the removal-gated tests on Curio-piri topology The Curio-based piri requires a Postgres piri node (harmonydb); the gated delete-finality and deferred-multipart tests now request one and honor ITEST_BLOCKCHAIN_IMAGE until the published localdev image carries the mockrpc Ticket fix. Co-Authored-By: Claude Fable 5 --- itest/forge_delete_test.go | 12 ++++++++++-- itest/forge_multipart_deferred_test.go | 12 ++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/itest/forge_delete_test.go b/itest/forge_delete_test.go index c3de13b..4b70c81 100644 --- a/itest/forge_delete_test.go +++ b/itest/forge_delete_test.go @@ -41,10 +41,18 @@ func TestForgeDeleteReleasesNetworkBlob(t *testing.T) { ctx := t.Context() const email = "test@example.com" - s, ingotEndpoint := forgeStack(t, + // The Curio-based piri requires a Postgres node (harmonydb) and, until + // the published localdev image carries the mockrpc Ticket fix, an + // overridable blockchain image. + stackOpts := []stack.Option{ + stack.WithPiriNodes(stack.PiriNodeConfig{Postgres: true}), stack.WithServiceBinary("piri", piriBin), stack.WithServiceBinary("upload", sprueBin), - ) + } + if img := os.Getenv("ITEST_BLOCKCHAIN_IMAGE"); img != "" { + stackOpts = append(stackOpts, stack.WithBlockchainImage(img)) + } + s, ingotEndpoint := forgeStack(t, stackOpts...) ingotSelfProvision(t, ctx, s, email) cfg := forgeConfig(ingotEndpoint) diff --git a/itest/forge_multipart_deferred_test.go b/itest/forge_multipart_deferred_test.go index 92a2525..ed8e888 100644 --- a/itest/forge_multipart_deferred_test.go +++ b/itest/forge_multipart_deferred_test.go @@ -39,10 +39,18 @@ func TestForgeDeferredMultipart(t *testing.T) { } ctx := t.Context() - s, endpoint := forgeStack(t, + // The Curio-based piri requires a Postgres node (harmonydb) and, until + // the published localdev image carries the mockrpc Ticket fix, an + // overridable blockchain image. + stackOpts := []stack.Option{ + stack.WithPiriNodes(stack.PiriNodeConfig{Postgres: true}), stack.WithServiceBinary("piri", piriBin), stack.WithServiceBinary("upload", sprueBin), - ) + } + if img := os.Getenv("ITEST_BLOCKCHAIN_IMAGE"); img != "" { + stackOpts = append(stackOpts, stack.WithBlockchainImage(img)) + } + s, endpoint := forgeStack(t, stackOpts...) ingotSelfProvision(t, ctx, s, "test@example.com") cl := sdkClient(forgeS3Conf(endpoint)) From 086a2a8b3e46bcea5a603d19c64b3ee259e12d8a Mon Sep 17 00:00:00 2001 From: frrist Date: Fri, 17 Jul 2026 15:06:58 -0700 Subject: [PATCH 11/21] deps: pin smelt to the fil-588 branch, drop the local replace CI has no sibling smelt checkout; pin the pushed branch commit (fil-forge/smelt#19, 2718317) instead. Bump to the merge commit when that PR lands. Co-Authored-By: Claude Fable 5 --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 493355e..d66a192 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1 github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2 - github.com/fil-forge/smelt v0.0.0-20260714201619-f2ecac50394c + github.com/fil-forge/smelt v0.0.0-20260717195554-271831781c1d github.com/fil-forge/ucantone v0.0.0-20260706102443-79141c5cc52e github.com/fxamacker/cbor/v2 v2.9.2 github.com/go-jose/go-jose/v4 v4.1.4 @@ -252,5 +252,3 @@ require ( pitr.ca/jsontokenizer v0.3.2 // indirect tags.cncf.io/container-device-interface v1.1.0 // indirect ) - -replace github.com/fil-forge/smelt => ../smelt diff --git a/go.sum b/go.sum index 247360d..58ff2e5 100644 --- a/go.sum +++ b/go.sum @@ -250,6 +250,8 @@ github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 h1:W github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717/go.mod h1:wFcakLohOqpMRkJzWRdGFGHRFpGB+PpEMCm9wkt2cqU= github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2 h1:mwb14/aanwFBGdQJ/0WCkNX/AzUXF/iRKMVjhXtYd6Q= github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= +github.com/fil-forge/smelt v0.0.0-20260717195554-271831781c1d h1:hRW4J1J1qZVKl+IBMIPwyHkjztoE37wAlBGjjJLkGgk= +github.com/fil-forge/smelt v0.0.0-20260717195554-271831781c1d/go.mod h1:3UpZd5C7w59IDICASaYqIF6Uu/0uZkp7tj9Ivg0ZmRU= github.com/fil-forge/ucantone v0.0.0-20260706102443-79141c5cc52e h1:di/SseJVEO6bznSH/UEnAE2nSwwp/gqgnPLgE9/x2Zg= github.com/fil-forge/ucantone v0.0.0-20260706102443-79141c5cc52e/go.mod h1:oFY5BfD0bDeodGlbBHh3/nK99MAS93rGXjoQz7s5qgE= github.com/filecoin-project/go-data-segment v0.0.1 h1:1wmDxOG4ubWQm3ZC1XI5nCon5qgSq7Ra3Rb6Dbu10Gs= From 43d6482d9b5ef9f366aa2eb211ce9de16fc83f5e Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 22 Jul 2026 16:59:47 -0700 Subject: [PATCH 12/21] feat(blob)!: RFC-shaped remove/abort args; recognize BlobAccepted refusals MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The blob-removal RFC drops the space from the client-leg arguments — /blob/remove is {digest} and /blob/abort is {digest, cause}, with the space as the invocation subject. Ingot already invoked with the space as subject, so only the argument literals change. Treat the node's BlobAccepted refusal on abort as final rather than a fault: it means this space accepted the same content concurrently, so the blob belongs to the reference index and is released via /blob/remove when its last claim drops. The uploader logs it distinctly instead of as an error; the multipart cleanup path already discards the park row, which is obsolete either way. Co-Authored-By: Claude Fable 5 --- docs/architecture.md | 45 +++++++++++++------------- forgeclient/blobabort.go | 6 ++-- forgeclient/blobadd.go | 2 +- forgeclient/blobremove.go | 4 ++- go.mod | 2 +- go.sum | 4 +-- itest/forge_delete_test.go | 16 ++++----- itest/forge_multipart_deferred_test.go | 7 ++-- registry/stores.go | 2 +- s3frontend/multipart.go | 7 ++-- uploader/blob.go | 26 +++++++++++---- 11 files changed, 72 insertions(+), 49 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index f477bfa..1c03831 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,10 +90,11 @@ The design is shaped by how the Forge upload pipeline works and by a handful of - **The 256 MiB blob ceiling is a knob, not a wall.** Piri currently caps a blob at 256 MiB because it builds the commP Merkle tree in RAM; known improvements (streaming commP) lift this. Ingot treats it as a tunable maximum and splits larger objects. -- **The delete primitives are built.** `blob/remove` is handled end-to-end (Sprue deregisters + - forwards; Piri releases the space's claim and defers physical deletion until the aggregate root - retires on-chain via the signed `schedulePieceDeletions` path), and `blob/unallocate` retires a - parked (never-accepted) blob — an upload ends in exactly one of accept or unallocate. The +- **The delete primitives are built.** `/blob/remove` is handled end-to-end (Sprue forwards + `/blob/release` to the nodes and deregisters; Piri releases the space's claim and defers physical + deletion until the aggregate root retires on-chain via the signed `schedulePieceDeletions` + path), and `/blob/abort` — translated by Sprue into `/blob/reject` on the node — retires a + parked (never-accepted) blob — an upload ends in exactly one of accept or reject. The indexer still has no retraction. [§9](#9-the-system-contract-piri--sprue--indexer) enumerates the contract Ingot needs. @@ -355,7 +356,7 @@ fixed constant (pdp-sim). discarded). This is the sub-`min` tail. **The delete primitives** are deliberately distinct: -- **`unallocate(digest)`** retires a **parked** blob (PUT, never accepted): delete the MinIO bytes +- **`abort(digest)`** retires a **parked** blob (PUT, never accepted): delete the MinIO bytes and the allocation record. No chain involvement. - **`remove(digest)`** releases a **space's claim** on an **accepted** blob. Because dedup is global, Piri deletes the bytes and retires the piece only when the per-`(digest, space)` claim count @@ -428,7 +429,7 @@ CompleteMultipartUpload([parts]) splice; blob_refs += version per digest; guarded root swap AbortMultipartUpload latch session open→aborting - parked blobs → unallocate; already-accepted (deduped) blobs → remove (§6) + parked blobs → abort; already-accepted (deduped) blobs → remove (§6) ``` A completed multipart object is the ordered union of its parts' blobs plus a manifest of byte ranges; @@ -439,13 +440,13 @@ later `GET`/`HEAD ?partNumber=N` can resolve part `N` to its byte span and repor ### 7.3 The session latch (the Abort/Complete race) `Complete` and `Abort` can arrive concurrently for one `uploadId`. Without coordination they collide -on the parts — `Complete` triggering accept while `Abort` unallocates those same parts — leaving the +on the parts — `Complete` triggering accept while `Abort` rejects those same parts — leaving the object half-built or half-deleted. A **single-winner latch** prevents it: an atomic state transition on the session row (`UPDATE … SET state=? WHERE state='open'`). Exactly one of `Complete`→`completing` or `Abort`→`aborting` wins; the loser observes the moved row and returns an -error. Accept is triggered only after the session is latched `completing`, so accept and unallocate +error. Accept is triggered only after the session is latched `completing`, so accept and reject never touch the same part concurrently. A deduped part is already accepted (not parked), so Abort -removes it via the reference path rather than unallocating it. +removes it via the reference path rather than rejecting it. ### 7.4 Read (`GetObject`) @@ -471,7 +472,7 @@ updates are transactional with the commit. | Case | Handling | |---|---| -| Crash after PUT, before accept | Bytes parked; `upload_intents` (parked) drives resume or `unallocate`. No `200` was sent. | +| Crash after PUT, before accept | Bytes parked; `upload_intents` (parked) drives resume or `abort`. No `200` was sent. | | Crash after accept, before commit | Blob durable but unreferenced; `upload_intents` (accepted) drives commit-retry or `remove`. | | Guarded-root-swap mismatch | Reload root, re-splice; blobs already durable, never re-uploaded. | | Concurrent PUT, same key | Distinct versionIds; serialize only on the swap. | @@ -525,8 +526,8 @@ negotiations). |------------------------------------------------------------------------------------------------|---------------------------------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `allocate` / `PUT` / `accept` blob lifecycle | Piri/Sprue | **exists** | The storage primitive Ingot builds on. | | Ingot-timed accept (PUT a part, defer the conclude until Complete) | Ingot + Sprue | **exists** | `forgeclient.BlobAddParked`/`BlobConclude` split the flow at the conclude seam; UploadPart parks (durable, unaggregated), Complete concludes — Sprue's conclude handler already ran accept standalone. Ingot still does **not** issue `accept` (Piri requires the upload-service DID). | -| `unallocate(digest)` — drop a parked blob | Piri + Sprue + libforge | **exists** | `/blob/unallocate` on Piri refuses accepted blobs (`BlobAccepted`), deletes the space's allocation, and drops the bytes at zero allocations. Sprue recovers the provider from the `cause` receipt chain (a parked blob has no registration). Abort/TTL/supersede unwind through it. | -| `remove(digest)` — per-space claim release; physical delete/piece-retire at zero global claims | Piri + Sprue + libforge | **exists** | `/blob/remove` on Piri deletes the space's allocation/acceptance/claim; at zero claims bytes delete immediately (unaggregated) or via the pending-removal sweep once the whole aggregate root is dead (FIL-623/624). Sprue forwards to primary + replicas (FIL-522). | +| `abort(digest)` — drop a parked blob | Piri + Sprue + libforge | **exists** | `/blob/abort` on Sprue translates to `/blob/reject` on Piri, which refuses blobs the invoking space has accepted (`BlobAccepted`), deletes the space's allocation, and drops the bytes once no space holds an allocation or acceptance. Sprue recovers the provider from the `cause` receipt chain (a parked blob has no registration). Abort/TTL/supersede unwind through it. | +| `remove(digest)` — per-space claim release; physical delete/piece-retire at zero global claims | Piri + Sprue + libforge | **exists** | `/blob/remove` on Sprue forwards `/blob/release` to Piri, which deletes the space's allocation/acceptance/claim; at zero claims bytes delete immediately (unaggregated) or via the pending-removal sweep once the whole aggregate root is dead (FIL-623/624). Sprue forwards to primary + replicas (FIL-522). | | Configurable, adaptive size policy (`min`/`max`); batch guard for the `extraData` cap | Piri | **partial** | `MinAggregateSize` is hardcoded 128 MiB; lower to ~8 MiB and make configurable. The `addPieces` batch is no longer contract-capped (FWSS v1.3.0 removed the `extraData` cap); size it to the FVM `PiecesAdded` event-size + per-tx gas — a measured ceiling (default `BatchSize=10` is safely within it) (pdp-sim). No contract change. | | Compaction (Regime B) + complete the on-chain delete signature | Piri | **partial** | Whole-root delete is signed and wired (`schedulePieceDeletions` with `SignSchedulePieceRemovals` extraData); compaction (remove + re-hash survivors + re-add) is new. | | De-dup at accept (don't re-aggregate a digest already a live piece) | Piri | **to-build** | Backstops one-piece-per-content once accept timing is Ingot-driven. | @@ -539,7 +540,7 @@ negotiations). **Determinism / idempotency Ingot must preserve.** The `accept` invocation is built deterministically (stable CID, today via `WithNoNonce` over `{space, digest, size, put-task}`); re-driving accept must -reuse the same put-task link. `remove`/`unallocate` must be idempotent. The forge-root advance must +reuse the same put-task link. `remove`/`abort` must be idempotent. The forge-root advance must happen only after a successful guarded root swap — the catalog log currently advances it before the swap, so a mismatch can leave the forge root pointing at a bucket root that was never adopted. @@ -592,7 +593,7 @@ The MVP this supersedes had six structural problems; each is resolved by a layer |-----------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | A per-bucket lock held across the whole write | Ingest and upload run off-lock; only the MST splice + guarded root swap is in the critical section. [§7.1](#71-write-single-shot-putobject) | | The whole object buffered in memory | Hash-while-writing to the local store; stream to Piri; nothing held whole in RAM. [§5](#5-the-data-layer), [§7.1](#71-write-single-shot-putobject) | -| No multipart, no abort | Parked part-blobs, accept-at-Complete, single-winner latch; abort = `unallocate` (parked) / `remove` (deduped). [§7.2](#72-multipart)–7.3, [§9](#9-the-system-contract-piri--sprue--indexer) | +| No multipart, no abort | Parked part-blobs, accept-at-Complete, single-winner latch; abort = `/blob/abort` (parked) / `remove` (deduped). [§7.2](#72-multipart)–7.3, [§9](#9-the-system-contract-piri--sprue--indexer) | | Objects chunked into CARs to obtain a digest | Object body = one blob (≤ `max`) or a coarse `≤ max` split; no fine chunking, no CAR. [§5](#5-the-data-layer), [§10](#10-deployment-topology--the-digest-before-upload-cost) | | No delete of superseded data | Reference index + per-space `remove` + Piri's global claim gate + indexer delete; O(1) for ≥ `min` blobs. [§5](#5-the-data-layer), [§6](#6-the-forgechain-layer), [§9](#9-the-system-contract-piri--sprue--indexer) | | Aggregation blocked partial deletes | A small `min` makes most blobs their own piece (O(1) delete); compaction handles only the sub-`min` tail. [§6](#6-the-forgechain-layer) | @@ -801,20 +802,20 @@ reference index — and is out of scope for this iteration. The in-memory harness uses a no-op uploader and serves reads from the spool, so these forge-network paths are stubbed in-tree and verified against a real sprue+piri+indexer later: -- **`remove(digest)` and `unallocate(digest)` are live.** `RemoveBlob` invokes `/blob/remove` on - sprue, which forwards to the storage nodes ([§9](#9-the-system-contract-piri--sprue--indexer)); delete finality means claim-release-now, - bytes-at-root-death. `UnallocateBlob` retires parked part-blobs on abort via `/blob/unallocate` - (provider recovered from the `cause` receipt chain); allocation-expiry GC (FIL-625) remains the - backstop when an abort never arrives. +- **`remove(digest)` and `abort(digest)` are live.** `RemoveBlob` invokes `/blob/remove` on + sprue, which forwards `/blob/release` to the storage nodes ([§9](#9-the-system-contract-piri--sprue--indexer)); delete finality means claim-release-now, + bytes-at-root-death. `AbortBlob` retires parked part-blobs via `/blob/abort` — sprue translates + it into `/blob/reject` on the node (provider recovered from the `cause` receipt chain); + allocation-expiry GC (FIL-625) remains the backstop when an abort never arrives. - **The local-table `Locator` read tier is not wired.** Body-blob *locations* are recorded at accept, but the read path that consumes them ([§7.4](#74-read-getobject), [§8](#8-retrieval-addressing-when-bodies-need-a-sharded-dag-index)) is deferred — it is only exercised after spool eviction (also not built) and is best validated live. - **Multipart parts park at UploadPart, accept at Complete.** (Built: `parkBlobs`/`concludeBlobs` over the `blob_parks` table.) The in-process harness still spools parts at `UploadPart` and uploads+accepts them at `Complete`; the true forge *parking* (upload early, - accept-at-Complete) and `unallocate`-on-abort from [§7.2](#72-multipart)–[7.3](#73-the-session-latch-the-abortcomplete-race) are forge-mode refinements. + accept-at-Complete) and the `/blob/abort` unwind from [§7.2](#72-multipart)–[7.3](#73-the-session-latch-the-abortcomplete-race) are forge-mode refinements. - **Crash recovery for the spool is not built.** The `upload_intents` × `blob_refs` reconciliation - the failure-mode table in [§7.5](#75-concurrency-durability-and-failure-modes) describes (resume/`unallocate` parked, `remove` accepted-but-unreferenced) + the failure-mode table in [§7.5](#75-concurrency-durability-and-failure-modes) describes (resume/`abort` parked, `remove` accepted-but-unreferenced) is a later phase; a partial post-commit reference-index write currently relies on retry/idempotency. - **`UploadPartCopy` and indexer retraction on delete** are unimplemented (`ErrNotImplemented` / no-op). `ListParts` and `ListMultipartUploads` are implemented @@ -824,7 +825,7 @@ paths are stubbed in-tree and verified against a real sprue+piri+indexer later: sessions and committed objects), and a background sweeper aborts open sessions older than `multipart_session_ttl` (default 7d) and reaps terminal session rows. A successful Complete retains its session in state `completed` so a duplicate Complete is idempotent - per S3. The network-side `unallocate`-on-abort remains a parking-flow concern (above). + per S3. The network-side `/blob/abort` unwind remains a parking-flow concern (above). ### Known correctness boundary diff --git a/forgeclient/blobabort.go b/forgeclient/blobabort.go index e258d5c..0e9939b 100644 --- a/forgeclient/blobabort.go +++ b/forgeclient/blobabort.go @@ -14,7 +14,7 @@ import ( // BlobAbort invokes /blob/abort against the upload service (sprue), // abandoning the space's in-flight upload of a parked (never-accepted) -// blob. cause is the /space/blob/add task link (ParkedBlob.AddTask) — sprue +// blob. cause is the /blob/add task link (ParkedBlob.AddTask) — sprue // walks its receipt chain to locate the storage node holding the parked // bytes (which have no registration or acceptance to look up by) and // forwards a /blob/reject there. Idempotent on the node; a blob that has @@ -28,7 +28,9 @@ func (c *Client) BlobAbort(ctx context.Context, digest multihash.Multihash, spac inv, err := blobcmds.Abort.Invoke( c.signer, space, - &blobcmds.AbortArguments{Space: space, Digest: digest, Cause: cause}, + // The space is the invocation subject; it is not repeated in the + // arguments. + &blobcmds.AbortArguments{Digest: digest, Cause: cause}, invocation.WithAudience(c.serviceID), invocation.WithProofs(proofLinks...), ) diff --git a/forgeclient/blobadd.go b/forgeclient/blobadd.go index 4d17e5e..7934461 100644 --- a/forgeclient/blobadd.go +++ b/forgeclient/blobadd.go @@ -88,7 +88,7 @@ type AddedBlob struct { type ParkedBlob struct { Digest multihash.Multihash Size uint64 - // AddTask is the /space/blob/add task link — the receipt-chain root the + // AddTask is the /blob/add task link — the receipt-chain root the // upload service uses to locate the provider for abort. AddTask cid.Cid // AcceptTask is the /blob/accept task link BlobConclude polls. diff --git a/forgeclient/blobremove.go b/forgeclient/blobremove.go index 94e122e..22d19b0 100644 --- a/forgeclient/blobremove.go +++ b/forgeclient/blobremove.go @@ -24,7 +24,9 @@ func (c *Client) BlobRemove(ctx context.Context, digest multihash.Multihash, spa inv, err := blobcmds.Remove.Invoke( c.signer, space, - &blobcmds.RemoveArguments{Space: space, Digest: digest}, + // The space is the invocation subject; it is not repeated in the + // arguments. + &blobcmds.RemoveArguments{Digest: digest}, invocation.WithAudience(c.serviceID), invocation.WithProofs(proofLinks...), ) diff --git a/go.mod b/go.mod index d66a192..23f81bb 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.19.15 github.com/aws/aws-sdk-go-v2/service/s3 v1.99.1 github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 - github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2 + github.com/fil-forge/libforge v0.0.0-20260723212548-3e5e6ba95711 github.com/fil-forge/smelt v0.0.0-20260717195554-271831781c1d github.com/fil-forge/ucantone v0.0.0-20260706102443-79141c5cc52e github.com/fxamacker/cbor/v2 v2.9.2 diff --git a/go.sum b/go.sum index 58ff2e5..58203d8 100644 --- a/go.sum +++ b/go.sum @@ -248,8 +248,8 @@ github.com/fil-forge/go-ucanto v0.0.0-20260507172450-5cb5d073f8ab h1:2J2cDThqTKP github.com/fil-forge/go-ucanto v0.0.0-20260507172450-5cb5d073f8ab/go.mod h1:lZF3UXZ2hGLKYmXdquG50JqI9pRlUrV6lubGtgOYfwc= github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 h1:Wke8qgaDgy7DGaIS28VpHip+YHSdQP1hGNEZTrXXzb4= github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717/go.mod h1:wFcakLohOqpMRkJzWRdGFGHRFpGB+PpEMCm9wkt2cqU= -github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2 h1:mwb14/aanwFBGdQJ/0WCkNX/AzUXF/iRKMVjhXtYd6Q= -github.com/fil-forge/libforge v0.0.0-20260717174550-aac837a730c2/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= +github.com/fil-forge/libforge v0.0.0-20260723212548-3e5e6ba95711 h1:xO5gwfL3W2wqaLvwFUDj4MuWX9wHK9TGV8F3jFe/mZM= +github.com/fil-forge/libforge v0.0.0-20260723212548-3e5e6ba95711/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= github.com/fil-forge/smelt v0.0.0-20260717195554-271831781c1d h1:hRW4J1J1qZVKl+IBMIPwyHkjztoE37wAlBGjjJLkGgk= github.com/fil-forge/smelt v0.0.0-20260717195554-271831781c1d/go.mod h1:3UpZd5C7w59IDICASaYqIF6Uu/0uZkp7tj9Ivg0ZmRU= github.com/fil-forge/ucantone v0.0.0-20260706102443-79141c5cc52e h1:di/SseJVEO6bznSH/UEnAE2nSwwp/gqgnPLgE9/x2Zg= diff --git a/itest/forge_delete_test.go b/itest/forge_delete_test.go index 4b70c81..8f9b431 100644 --- a/itest/forge_delete_test.go +++ b/itest/forge_delete_test.go @@ -16,11 +16,11 @@ import ( // TestForgeDeleteReleasesNetworkBlob is the delete-finality regression gate // (FIL-588): DeleteObject must release the blob on the network, not just drop // registry rows. The chain under test is ingot's reference index (claims→0 ⇒ -// RemoveBlob) → forgeclient /blob/remove → sprue (deregister + forward) → -// piri (claim release; deferred physical deletion once the PDP aggregate -// root retires on-chain). +// RemoveBlob) → forgeclient /blob/remove → sprue (forward + deregister) → +// piri /blob/release (claim release; deferred physical deletion once the +// PDP aggregate root retires on-chain). // -// The published piri/sprue images predate the /blob/remove handlers, so this +// The published piri/sprue images predate the removal handlers, so this // test injects working-tree builds of both and skips when they aren't // provided: // @@ -35,7 +35,7 @@ func TestForgeDeleteReleasesNetworkBlob(t *testing.T) { piriBin := os.Getenv("ITEST_PIRI_BIN") sprueBin := os.Getenv("ITEST_SPRUE_BIN") if piriBin == "" || sprueBin == "" { - t.Skip("requires piri+sprue builds with the /blob/remove chain: set ITEST_PIRI_BIN and ITEST_SPRUE_BIN (see test doc comment)") + t.Skip("requires piri+sprue builds with the blob-removal chain: set ITEST_PIRI_BIN and ITEST_SPRUE_BIN (see test doc comment)") } ctx := t.Context() @@ -89,16 +89,16 @@ func TestForgeDeleteReleasesNetworkBlob(t *testing.T) { t.Fatalf("GET after delete succeeded, want NoSuchKey") } - // And the release traversed the network: piri's /blob/remove handler ran + // And the release traversed the network: piri's /blob/release handler ran // and, with the last claim gone, queued the piece for removal. Byte // release is fully asynchronous — ingot's releaseBlobs is best-effort // post-commit, and piri's removal sweep (PDPRemoveSweep, 30s ticks) // re-verifies claims and pipeline state before finalizing — so poll the // provider's logs through to the finalization line. - waitForPiriLog(t, ctx, s, "/blob/remove", 2*time.Minute) + waitForPiriLog(t, ctx, s, "/blob/release", 2*time.Minute) waitForPiriLog(t, ctx, s, "queueing piece removal", 2*time.Minute) waitForPiriLog(t, ctx, s, "finalized piece removal", 3*time.Minute) - t.Logf("delete finality OK: /blob/remove executed on piri and the sweep finalized the byte release") + t.Logf("delete finality OK: /blob/release executed on piri and the sweep finalized the byte release") } // waitForPiriLog polls piri-0's container logs until substr appears. diff --git a/itest/forge_multipart_deferred_test.go b/itest/forge_multipart_deferred_test.go index ed8e888..d21a84c 100644 --- a/itest/forge_multipart_deferred_test.go +++ b/itest/forge_multipart_deferred_test.go @@ -23,9 +23,10 @@ import ( // (parked — outside the PDP pipeline), Complete must conclude (accept) every // part, and Abort must unwind parked blobs with /blob/abort → /blob/reject. // -// Requires piri + sprue builds with /blob/remove + /blob/abort/reject; the -// published images predate them, so inject working-tree binaries and skip -// otherwise (same gate as TestForgeDeleteReleasesNetworkBlob): +// Requires piri + sprue builds with the blob-removal chain (/blob/remove → +// /blob/release, /blob/abort → /blob/reject); the published images predate +// them, so inject working-tree binaries and skip otherwise (same gate as +// TestForgeDeleteReleasesNetworkBlob): // // (cd ../../piri && CGO_ENABLED=0 GOOS=linux go build -o /tmp/piri ./cmd) // (cd ../../sprue && CGO_ENABLED=0 GOOS=linux go build -o /tmp/sprue ./cmd/main.go) diff --git a/registry/stores.go b/registry/stores.go index e35b880..1ac7d11 100644 --- a/registry/stores.go +++ b/registry/stores.go @@ -145,7 +145,7 @@ type LocationStore interface { // BlobPark is one row of ingot.blob_parks: the persistable state of a blob // that is durable on its provider but not yet accepted (multipart's deferred -// conclude, §7.2). AddTask/AcceptTask are the /space/blob/add and +// conclude, §7.2). AddTask/AcceptTask are the /blob/add and // /blob/accept task CIDs; PutInvocation is the sealed /http/put invocation // whose metadata carries the derived signer keys needed to conclude — // sensitive, deleted at conclude/abort. Keyed globally by Digest (like diff --git a/s3frontend/multipart.go b/s3frontend/multipart.go index fec5814..bf82be7 100644 --- a/s3frontend/multipart.go +++ b/s3frontend/multipart.go @@ -452,8 +452,11 @@ func (b *Backend) cleanupPartBlobs(ctx context.Context, uploadID string, digests } // A parked blob is durable on its provider — release it there too // (best-effort; the reject on piri is idempotent, a straggler is - // FIL-625's to reap). Cause is the /space/blob/add task link the - // upload service needs to locate the provider. + // FIL-625's to reap). Cause is the /blob/add task link the + // upload service needs to locate the provider. A BlobAccepted + // refusal is benign — a concurrent session in this space accepted + // the same content, so the reference index owns the blob now — and + // the park row is obsolete either way. if state == registry.IntentParked { if park, err := b.parks.GetPark(ctx, d); err == nil { if cause, err := cid.Cast(park.AddTask); err == nil { diff --git a/uploader/blob.go b/uploader/blob.go index d49d01b..e432741 100644 --- a/uploader/blob.go +++ b/uploader/blob.go @@ -7,7 +7,9 @@ import ( "os" assertcmds "github.com/fil-forge/libforge/commands/assert" + blobcmds "github.com/fil-forge/libforge/commands/blob" "github.com/fil-forge/libforge/digestutil" + ucanerrors "github.com/fil-forge/ucantone/errors" "github.com/ipfs/go-cid" "github.com/multiformats/go-multihash" "go.uber.org/zap" @@ -148,12 +150,24 @@ func (u *Forge) AbortBlob(ctx context.Context, digest multihash.Multihash, cause zap.String("digest", digestutil.Format(digest)), ) if err := u.client.BlobAbort(ctx, digest, u.space, cause); err != nil { - u.logger.Error("blob abort failed", - zap.Stringer("space", u.space), - zap.String("digest", digestutil.Format(digest)), - zap.Error(err), - ) - return fmt.Errorf("uploader: unallocating blob: %w", err) + // A BlobAccepted refusal is final, not a fault: the space accepted + // this content (e.g. a concurrent session completed with the same + // content-addressed part), so the blob now belongs to the reference + // index and is released via /blob/remove when its last claim drops. + var named ucanerrors.Named + if ucanerrors.As(err, &named) && named.Name() == blobcmds.BlobAcceptedErrorName { + u.logger.Info("blob abort refused: blob accepted by the space; reference accounting owns it", + zap.Stringer("space", u.space), + zap.String("digest", digestutil.Format(digest)), + ) + } else { + u.logger.Error("blob abort failed", + zap.Stringer("space", u.space), + zap.String("digest", digestutil.Format(digest)), + zap.Error(err), + ) + } + return fmt.Errorf("uploader: aborting blob: %w", err) } return nil } From 3d6516f7e4138606b92e63ad770c6f919524987f Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 28 Jul 2026 13:45:52 -0700 Subject: [PATCH 13/21] fix(migrations): renumber multipart migrations past main's versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit goose refuses duplicate version numbers, and this branch carried 00004/00005 twice (its multipart_listing/deferred_accept beside main's bucket_created_at_space/segment_bucket) since the Hilt merge — breaking forge-mode startup. Renumber the unreleased branch migrations to 00007/00008 so a fresh database applies main's sequence first. Verified with TestUp_Live against a throwaway Postgres (migrates to version 8, second Up is a no-op). Co-Authored-By: Claude Fable 5 --- .../{00004_multipart_listing.sql => 00007_multipart_listing.sql} | 0 .../sql/{00005_deferred_accept.sql => 00008_deferred_accept.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename migrations/sql/{00004_multipart_listing.sql => 00007_multipart_listing.sql} (100%) rename migrations/sql/{00005_deferred_accept.sql => 00008_deferred_accept.sql} (100%) diff --git a/migrations/sql/00004_multipart_listing.sql b/migrations/sql/00007_multipart_listing.sql similarity index 100% rename from migrations/sql/00004_multipart_listing.sql rename to migrations/sql/00007_multipart_listing.sql diff --git a/migrations/sql/00005_deferred_accept.sql b/migrations/sql/00008_deferred_accept.sql similarity index 100% rename from migrations/sql/00005_deferred_accept.sql rename to migrations/sql/00008_deferred_accept.sql From e5e04c0582f1c129549bf163f8c7073c99a06667 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 28 Jul 2026 14:35:27 -0700 Subject: [PATCH 14/21] test(itest): re-curate the versity partition for the delete-release era MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The blob-removal chain landing across the stack (this branch's DeleteObject → /blob/remove, sprue#33's forwarding, piri's release sweep) unblocked bucket-delete teardown, flipping every "teardown-blocked" xfail row green: promote all 50 (PutObject 5, GetObject 17, HeadObject 14, DeleteObject 1, CopyObject 13) to the pass tables. Demote four multipart cases that genuinely fail — the upstream error-code mismatches (ListMultipartUploads invalid_uploadId_marker, CompleteMultipartUpload invalid_part_number), missing-ETag validation on Complete (missing_part_fields), and conditional writes on Complete (conditional_writes) — with reasons in the tables. Validated with a full local TestForgeVersity run against fresh :main images: 216 pass, 0 fail. Co-Authored-By: Claude Fable 5 --- itest/README.md | 15 ++++---- itest/versity_multipart_test.go | 23 ++++++++--- itest/versity_object_test.go | 68 ++++++++++++++++----------------- 3 files changed, 56 insertions(+), 50 deletions(-) diff --git a/itest/README.md b/itest/README.md index d63972d..3f85e6c 100644 --- a/itest/README.md +++ b/itest/README.md @@ -37,14 +37,13 @@ yet — e.g. one built from an unmerged branch — point the stack at it: INGOT_ITEST_UPLOAD_IMAGE= make itest ``` -**Teardown-blocked XFail rows:** a bucket that ever held a non-empty object -body cannot currently be deleted — bodies register blobs in the bucket's -space at PUT, `DeleteObject`'s blob release is a no-op until sprue/piri -implement `/blob/remove`, and hilt's `/s3/bucket/delete` refuses non-empty -spaces. Upstream cases delete their bucket in teardown, so such cases pass -their S3 assertions and fail teardown; they sit in the XFail tables (marked -"teardown-blocked") so the unexpected-pass ratchet flags them for promotion -when `/blob/remove` lands. +**Teardown-blocked XFail rows (historical):** before `DeleteObject` released +network blobs (FIL-588), a bucket that ever held a non-empty object body +could not be deleted — hilt's `/s3/bucket/delete` refuses non-empty spaces — +so dozens of cases passed their S3 assertions and failed their bucket-delete +teardown, and sat in the XFail tables marked "teardown-blocked". The +unexpected-pass ratchet flagged them all when the release path landed; they +now live in the pass tables. ## The conformance partition — `TestForgeVersity` diff --git a/itest/versity_multipart_test.go b/itest/versity_multipart_test.go index ff81cd3..43c62ea 100644 --- a/itest/versity_multipart_test.go +++ b/itest/versity_multipart_test.go @@ -9,7 +9,9 @@ import ( // Multipart groups of the S3 conformance partition, partitioned empirically // against the forge-mode stack (see the curation note in README.md). The // remaining xfail surface: part-level checksums (FIL-620), tagging/object-lock -// /ACL on create (FIL-534/FIL-525), and the UploadPartCopy group (FIL-586). +// /ACL on create (FIL-534/FIL-525), the UploadPartCopy group (FIL-586), +// upstream error-code alignment (InvalidArgument where ingot returns a more +// specific code), and conditional writes on Complete. var createMultipartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.CreateMultipartUpload_non_existing_bucket}, @@ -104,7 +106,6 @@ var listMultipartUploadsPass = []forgeCase{ {name: "max_uploads", fn: integration.ListMultipartUploads_max_uploads}, {name: "exceeding_max_uploads", fn: integration.ListMultipartUploads_exceeding_max_uploads}, {name: "ignore_upload_id_marker", fn: integration.ListMultipartUploads_ignore_upload_id_marker}, - {name: "invalid_uploadId_marker", fn: integration.ListMultipartUploads_invalid_uploadId_marker}, {name: "keyMarker_not_from_list", fn: integration.ListMultipartUploads_keyMarker_not_from_list}, {name: "delimiter_truncated", fn: integration.ListMultipartUploads_delimiter_truncated}, {name: "prefix", fn: integration.ListMultipartUploads_prefix}, @@ -113,7 +114,11 @@ var listMultipartUploadsPass = []forgeCase{ {name: "with_checksums", fn: integration.ListMultipartUploads_with_checksums}, } -var listMultipartUploadsXFail = []forgeCase{} +var listMultipartUploadsXFail = []forgeCase{ + // Upstream expects InvalidArgument for a malformed upload-id-marker; + // ingot returns InvalidRequest. + {name: "invalid_uploadId_marker", fn: integration.ListMultipartUploads_invalid_uploadId_marker}, +} var abortMultipartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.AbortMultipartUpload_non_existing_bucket}, @@ -130,15 +135,12 @@ var completeMultipartPass = []forgeCase{ // upstream function name carries a typo (CompletedMultipartUpload_...). {name: "non_existing_bucket", fn: integration.CompletedMultipartUpload_non_existing_bucket}, {name: "incorrect_part_number", fn: integration.CompleteMultipartUpload_incorrect_part_number}, - {name: "missing_part_fields", fn: integration.CompleteMultipartUpload_missing_part_fields}, - {name: "invalid_part_number", fn: integration.CompleteMultipartUpload_invalid_part_number}, {name: "default_content_type", fn: integration.CompleteMultipartUpload_default_content_type}, {name: "invalid_ETag", fn: integration.CompleteMultipartUpload_invalid_ETag}, {name: "small_upload_size", fn: integration.CompleteMultipartUpload_small_upload_size}, {name: "empty_parts", fn: integration.CompleteMultipartUpload_empty_parts}, {name: "incorrect_parts_order", fn: integration.CompleteMultipartUpload_incorrect_parts_order}, {name: "mpu_object_size", fn: integration.CompleteMultipartUpload_mpu_object_size}, - {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, {name: "invalid_checksum_type", fn: integration.CompleteMultipartUpload_invalid_checksum_type}, {name: "multiple_final_checksums", fn: integration.CompleteMultipartUpload_multiple_final_checksums}, {name: "invalid_final_checksums", fn: integration.CompleteMultipartUpload_invalid_final_checksums}, @@ -158,6 +160,15 @@ var completeMultipartPass = []forgeCase{ // Part-level / composite-checksum verification is FIL-620; // racey_data_integrity additionally leans on atomic concurrent overwrites. var completeMultipartXFail = []forgeCase{ + // A part entry missing its ETag is accepted (200 with a result body) + // where upstream expects an Error response. + {name: "missing_part_fields", fn: integration.CompleteMultipartUpload_missing_part_fields}, + // Upstream expects InvalidArgument; ingot returns InvalidPartNumber. + {name: "invalid_part_number", fn: integration.CompleteMultipartUpload_invalid_part_number}, + // Conditional headers on Complete are unenforced; the object written + // past the precondition then fails the case's bucket teardown (409 + // BucketNotEmpty). + {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, {name: "invalid_checksum_part", fn: integration.CompleteMultipartUpload_invalid_checksum_part}, {name: "multiple_checksum_part", fn: integration.CompleteMultipartUpload_multiple_checksum_part}, {name: "incorrect_checksum_part", fn: integration.CompleteMultipartUpload_incorrect_checksum_part}, diff --git a/itest/versity_object_test.go b/itest/versity_object_test.go index 71c4f96..5333010 100644 --- a/itest/versity_object_test.go +++ b/itest/versity_object_test.go @@ -8,16 +8,13 @@ import ( // Single-object groups of the S3 conformance partition. // -// TEARDOWN-BLOCKED rows: since the hilt (tenant-management) integration, a -// bucket that ever held a non-empty object body cannot be deleted — object -// bodies register blobs in the bucket's space at PUT, DeleteObject's blob -// release is a no-op until sprue/piri implement /blob/remove (see the TODO -// in uploader/blob.go), and hilt's /s3/bucket/delete refuses non-empty -// spaces (409 BucketNotEmpty). Upstream cases delete their bucket in -// teardown, so every case below marked "teardown-blocked" passes its S3 -// assertions and then fails teardown. They sit in the XFail tables so the -// unexpected-pass ratchet flags them for promotion the moment /blob/remove -// lands. +// Historical note: before DeleteObject released network blobs (FIL-588), +// a bucket that ever held a non-empty object body could not be deleted — +// hilt's /s3/bucket/delete refuses non-empty spaces (409 BucketNotEmpty) — +// so every such case failed its bucket-delete teardown and sat in the +// XFail tables as "teardown-blocked". The unexpected-pass ratchet flagged +// them all when the release path landed on this branch; they now live in +// the pass tables. var putObjectPass = []forgeCase{ {name: "checksum_algorithm_and_header_mismatch", fn: integration.PutObject_checksum_algorithm_and_header_mismatch}, @@ -39,17 +36,18 @@ var putObjectPass = []forgeCase{ {name: "past_retain_until_date", fn: integration.PutObject_past_retain_until_date}, {name: "racey_success", fn: integration.PutObject_racey_success}, {name: "special_chars", fn: integration.PutObject_special_chars}, -} - -var putObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. - {name: "checksums_success", fn: integration.PutObject_checksums_success}, {name: "conditional_writes", fn: integration.PutObject_conditional_writes}, {name: "default_checksum", fn: integration.PutObject_default_checksum}, {name: "default_content_type", fn: integration.PutObject_default_content_type}, {name: "success", fn: integration.PutObject_success}, {name: "with_metadata", fn: integration.PutObject_with_metadata}, +} + +var putObjectXFail = []forgeCase{ + // Still teardown-blocked even with DeleteObject's blob release: the + // case's checksummed bodies survive deletion and the bucket delete + // 409s (BucketNotEmpty). + {name: "checksums_success", fn: integration.PutObject_checksums_success}, // The incorrect_md5 subcheck expects 400 InvalidDigest; ingot 500s. {name: "md5", fn: integration.PutObject_md5}, // A metadata-combining re-PUT is denied (403) under the hilt authorize @@ -69,11 +67,6 @@ var getObjectPass = []forgeCase{ {name: "invalid_part_number", fn: integration.GetObject_invalid_part_number}, {name: "non_existing_key", fn: integration.GetObject_non_existing_key}, {name: "zero_len_with_range", fn: integration.GetObject_zero_len_with_range}, -} - -var getObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. {name: "by_range_resp_status", fn: integration.GetObject_by_range_resp_status}, {name: "checksums", fn: integration.GetObject_checksums}, {name: "conditional_reads", fn: integration.GetObject_conditional_reads}, @@ -91,8 +84,15 @@ var getObjectXFail = []forgeCase{ {name: "range_and_part_number", fn: integration.GetObject_range_and_part_number}, {name: "ranged_with_checksum_mode", fn: integration.GetObject_ranged_with_checksum_mode}, {name: "with_range", fn: integration.GetObject_with_range}, +} + +var getObjectXFail = []forgeCase{ + // Directory objects are served with binary/octet-stream instead of + // application/x-directory. {name: "directory_success", fn: integration.GetObject_directory_success}, + // Requires PutBucketPolicy, which ingot 501s (NotImplemented). {name: "overrides_fail_public", fn: integration.GetObject_overrides_fail_public}, + // Asserts object tagging (TagCount), which is unimplemented. {name: "success", fn: integration.GetObject_success}, } @@ -105,11 +105,6 @@ var headObjectPass = []forgeCase{ {name: "overrides_success", fn: integration.HeadObject_overrides_success}, {name: "dir_with_range", fn: integration.HeadObject_dir_with_range}, {name: "zero_len_with_range", fn: integration.HeadObject_zero_len_with_range}, -} - -var headObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. {name: "checksums", fn: integration.HeadObject_checksums}, {name: "conditional_reads", fn: integration.HeadObject_conditional_reads}, {name: "incidental_dir_object", fn: integration.HeadObject_incidental_dir_object}, @@ -124,7 +119,12 @@ var headObjectXFail = []forgeCase{ {name: "by_range_resp_status", fn: integration.HeadObject_by_range_resp_status}, {name: "ranged_with_checksum_mode", fn: integration.HeadObject_ranged_with_checksum_mode}, {name: "with_range", fn: integration.HeadObject_with_range}, +} + +var headObjectXFail = []forgeCase{ + // Requires PutBucketPolicy, which ingot 501s (NotImplemented). {name: "overrides_fail_public", fn: integration.HeadObject_overrides_fail_public}, + // Asserts object tagging (TagCount), which is unimplemented. {name: "success", fn: integration.HeadObject_success}, } @@ -137,12 +137,10 @@ var deleteObjectPass = []forgeCase{ {name: "non_existing_object", fn: integration.DeleteObject_non_existing_object}, {name: "success", fn: integration.DeleteObject_success}, {name: "success_status_code", fn: integration.DeleteObject_success_status_code}, + {name: "conditional_writes", fn: integration.DeleteObject_conditional_writes}, } var deleteObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. - {name: "conditional_writes", fn: integration.DeleteObject_conditional_writes}, // The ExpectedBucketOwner-matching delete is denied (403) under the // hilt authorize flow (ownership is the tenant's did:plc, not the // account the case expects). @@ -162,13 +160,6 @@ var copyObjectPass = []forgeCase{ {name: "to_itself_with_new_metadata", fn: integration.CopyObject_to_itself_with_new_metadata}, {name: "invalid_tagging_directive", fn: integration.CopyObject_invalid_tagging_directive}, {name: "invalid_checksum_algorithm", fn: integration.CopyObject_invalid_checksum_algorithm}, -} - -// Observed failing against the forge stack: multi-account semantics, tagging, -// object-lock, and checksum-on-copy are unimplemented surface. -var copyObjectXFail = []forgeCase{ - // Teardown-blocked (see the header comment): S3 assertions pass, the - // bucket delete 409s. {name: "success", fn: integration.CopyObject_success}, {name: "copy_source_starting_with_slash", fn: integration.CopyObject_copy_source_starting_with_slash}, {name: "default_content_type_with_replace_metadata", fn: integration.CopyObject_default_content_type_with_replace_metadata}, @@ -182,6 +173,11 @@ var copyObjectXFail = []forgeCase{ {name: "invalid_legal_hold", fn: integration.CopyObject_invalid_legal_hold}, {name: "invalid_object_lock_mode", fn: integration.CopyObject_invalid_object_lock_mode}, {name: "invalid_website_redirect_location", fn: integration.CopyObject_invalid_website_redirect_location}, +} + +// Observed failing against the forge stack: multi-account semantics, tagging, +// object-lock, and checksum-on-copy are unimplemented surface. +var copyObjectXFail = []forgeCase{ {name: "not_owned_source_bucket", fn: integration.CopyObject_not_owned_source_bucket}, {name: "should_replace_tagging", fn: integration.CopyObject_should_replace_tagging}, {name: "should_copy_tagging", fn: integration.CopyObject_should_copy_tagging}, From 36c34907f65a7d3f02e8a273bb99774b0633b367 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 28 Jul 2026 15:03:46 -0700 Subject: [PATCH 15/21] fix(s3frontend): conformance-align multipart validation; abort in-flight uploads on DeleteBucket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three upstream conformance cases return to the pass tables with behavior fixes instead of xfail rows: - CompleteMultipartUpload: a part entry missing PartNumber or ETag is MalformedXML, and a part number below 1 is InvalidArgument (PartNumber), mirroring versitygw's posix backend; the ETag match is now mandatory and the >10000 special case folds into the membership check (InvalidPart). - ListMultipartUploads: upload-id-marker validation mirrors upstream's MultipartUploadLister — the marker must be a valid UUID naming an upload of the first key group at/after the key marker (else InvalidArgument, upload-id-marker), and listing resumes past it. DeleteBucket now implicitly aborts the bucket's open multipart sessions (upstream's teardown never aborts them and expects the delete to succeed), releasing their parked part blobs before the hilt space delete. CompleteMultipartUpload_conditional_writes stays xfail: its conditional matrix passes, but the implicit abort's /blob/abort goes out proofless — hilt's per-operation grants carry blob.Abort for the S3 Abort operation only. Promote once hilt's s3perm map grants blob.Abort on bucket delete. Validated: full local TestForgeVersity, 219 pass / 0 fail. Co-Authored-By: Claude Fable 5 --- go.mod | 2 +- itest/versity_multipart_test.go | 28 +++++----- s3frontend/bucket.go | 21 ++++++++ s3frontend/multipart.go | 90 +++++++++++++++++++++------------ 4 files changed, 91 insertions(+), 50 deletions(-) diff --git a/go.mod b/go.mod index cf12113..e57668b 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/fxamacker/cbor/v2 v2.9.2 github.com/go-jose/go-jose/v4 v4.1.4 github.com/gofiber/fiber/v3 v3.3.0 + github.com/google/uuid v1.6.0 github.com/ipfs/go-block-format v0.2.3 github.com/ipfs/go-cid v0.6.1 github.com/ipfs/go-ipld-cbor v0.2.1 @@ -111,7 +112,6 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect - github.com/google/uuid v1.6.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect diff --git a/itest/versity_multipart_test.go b/itest/versity_multipart_test.go index 43c62ea..1f7932f 100644 --- a/itest/versity_multipart_test.go +++ b/itest/versity_multipart_test.go @@ -9,9 +9,7 @@ import ( // Multipart groups of the S3 conformance partition, partitioned empirically // against the forge-mode stack (see the curation note in README.md). The // remaining xfail surface: part-level checksums (FIL-620), tagging/object-lock -// /ACL on create (FIL-534/FIL-525), the UploadPartCopy group (FIL-586), -// upstream error-code alignment (InvalidArgument where ingot returns a more -// specific code), and conditional writes on Complete. +// /ACL on create (FIL-534/FIL-525), and the UploadPartCopy group (FIL-586). var createMultipartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.CreateMultipartUpload_non_existing_bucket}, @@ -106,6 +104,7 @@ var listMultipartUploadsPass = []forgeCase{ {name: "max_uploads", fn: integration.ListMultipartUploads_max_uploads}, {name: "exceeding_max_uploads", fn: integration.ListMultipartUploads_exceeding_max_uploads}, {name: "ignore_upload_id_marker", fn: integration.ListMultipartUploads_ignore_upload_id_marker}, + {name: "invalid_uploadId_marker", fn: integration.ListMultipartUploads_invalid_uploadId_marker}, {name: "keyMarker_not_from_list", fn: integration.ListMultipartUploads_keyMarker_not_from_list}, {name: "delimiter_truncated", fn: integration.ListMultipartUploads_delimiter_truncated}, {name: "prefix", fn: integration.ListMultipartUploads_prefix}, @@ -114,11 +113,7 @@ var listMultipartUploadsPass = []forgeCase{ {name: "with_checksums", fn: integration.ListMultipartUploads_with_checksums}, } -var listMultipartUploadsXFail = []forgeCase{ - // Upstream expects InvalidArgument for a malformed upload-id-marker; - // ingot returns InvalidRequest. - {name: "invalid_uploadId_marker", fn: integration.ListMultipartUploads_invalid_uploadId_marker}, -} +var listMultipartUploadsXFail = []forgeCase{} var abortMultipartPass = []forgeCase{ {name: "non_existing_bucket", fn: integration.AbortMultipartUpload_non_existing_bucket}, @@ -135,6 +130,8 @@ var completeMultipartPass = []forgeCase{ // upstream function name carries a typo (CompletedMultipartUpload_...). {name: "non_existing_bucket", fn: integration.CompletedMultipartUpload_non_existing_bucket}, {name: "incorrect_part_number", fn: integration.CompleteMultipartUpload_incorrect_part_number}, + {name: "missing_part_fields", fn: integration.CompleteMultipartUpload_missing_part_fields}, + {name: "invalid_part_number", fn: integration.CompleteMultipartUpload_invalid_part_number}, {name: "default_content_type", fn: integration.CompleteMultipartUpload_default_content_type}, {name: "invalid_ETag", fn: integration.CompleteMultipartUpload_invalid_ETag}, {name: "small_upload_size", fn: integration.CompleteMultipartUpload_small_upload_size}, @@ -160,14 +157,13 @@ var completeMultipartPass = []forgeCase{ // Part-level / composite-checksum verification is FIL-620; // racey_data_integrity additionally leans on atomic concurrent overwrites. var completeMultipartXFail = []forgeCase{ - // A part entry missing its ETag is accepted (200 with a result body) - // where upstream expects an Error response. - {name: "missing_part_fields", fn: integration.CompleteMultipartUpload_missing_part_fields}, - // Upstream expects InvalidArgument; ingot returns InvalidPartNumber. - {name: "invalid_part_number", fn: integration.CompleteMultipartUpload_invalid_part_number}, - // Conditional headers on Complete are unenforced; the object written - // past the precondition then fails the case's bucket teardown (409 - // BucketNotEmpty). + // The conditional matrix itself passes; the case then fails its bucket + // teardown: upstream's teardown never aborts in-flight uploads, so + // DeleteBucket implicitly aborts them (s3frontend/bucket.go), but the + // /blob/abort goes out proofless — hilt's per-operation proof grants + // carry blob.Abort for the S3 Abort operation only, not for + // DeleteBucket/UploadPart. Promote once hilt's s3perm map grants + // blob.Abort on bucket delete. {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, {name: "invalid_checksum_part", fn: integration.CompleteMultipartUpload_invalid_checksum_part}, {name: "multiple_checksum_part", fn: integration.CompleteMultipartUpload_multiple_checksum_part}, diff --git a/s3frontend/bucket.go b/s3frontend/bucket.go index c6326ed..2eae185 100644 --- a/s3frontend/bucket.go +++ b/s3frontend/bucket.go @@ -222,6 +222,27 @@ func (b *Backend) DeleteBucket(ctx context.Context, name string) error { } } + // In-flight multipart uploads do not block deletion (the upstream + // conformance contract's teardown deletes buckets without aborting + // them): abort any open sessions, releasing their parked part blobs + // from the space, before asking hilt to delete the space — which + // refuses while the space still holds blob registrations. + sessions, err := b.multipart.ListSessions(ctx, name) + if err != nil { + return fmt.Errorf("s3frontend: delete bucket: list mp sessions: %w", err) + } + aborted := 0 + for _, s := range sessions { + if s.State == registry.SessionOpen { + b.abortOpenSession(ctx, st.Space, s) + aborted++ + } + } + if aborted > 0 { + b.logger.Info("delete bucket: aborted in-flight multipart sessions", + zap.String("bucket", name), zap.Int("aborted", aborted), zap.Int("total", len(sessions))) + } + req, ok := reqscope.Request(ctx) if !ok { return errors.New("s3frontend: delete bucket: no request in context") diff --git a/s3frontend/multipart.go b/s3frontend/multipart.go index 04a39ac..ea5a8ff 100644 --- a/s3frontend/multipart.go +++ b/s3frontend/multipart.go @@ -17,8 +17,10 @@ import ( "github.com/fil-forge/versitygw/backend" "github.com/fil-forge/versitygw/s3err" "github.com/fil-forge/versitygw/s3response" + "github.com/google/uuid" "github.com/ipfs/go-cid" mh "github.com/multiformats/go-multihash" + "go.uber.org/zap" msbucket "github.com/fil-forge/ingot/bucket" "github.com/fil-forge/ingot/bucketop" @@ -267,13 +269,17 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet etagHasher := md5.New() prev := 0 for _, rp := range input.MultipartUpload.Parts { - if rp.PartNumber == nil { - return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) + // A part entry missing either field is malformed XML; a part number + // below 1 is an InvalidArgument (both per the upstream posix + // backend). Out-of-range numbers fall through to the membership + // check (no stored part can match) and report InvalidPart. + if rp.PartNumber == nil || rp.ETag == nil { + return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrMalformedXML) } num := int(*rp.PartNumber) - if num < 1 || num > 10000 { + if num < 1 { return s3response.CompleteMultipartUploadResult{}, "", - s3err.GetAPIError(s3err.ErrInvalidPartNumberRange) + s3err.GetInvalidArgumentErr(s3err.InvalidArgCompleteMpPartNumber, strconv.Itoa(num)) } if num <= prev { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPartOrder) @@ -283,7 +289,7 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet if !ok { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) } - if rp.ETag != nil && !etagsEqual(*rp.ETag, hex.EncodeToString(sp.ETagMD5)) { + if !etagsEqual(*rp.ETag, hex.EncodeToString(sp.ETagMD5)) { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrInvalidPart) } requested = append(requested, sp) @@ -438,6 +444,27 @@ func (b *Backend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMulti return nil } +// abortOpenSession force-aborts an open multipart session exactly like a +// client Abort: latch (losing gracefully to a concurrent Complete/Abort), +// drop the session, release its parts' now-unreferenced blobs. Used by +// DeleteBucket's implicit abort of in-flight uploads. +func (b *Backend) abortOpenSession(ctx context.Context, space did.DID, sess registry.MultipartSession) { + won, err := b.multipart.LatchSession(ctx, sess.UploadID, registry.SessionOpen, registry.SessionAborting) + if err != nil || !won { + return + } + var digests [][]byte + if parts, err := b.multipart.ListParts(ctx, sess.UploadID); err == nil { + for _, p := range parts { + digests = append(digests, p.BlobDigests...) + } + } + if err := b.multipart.DeleteSession(ctx, sess.UploadID); err != nil { + return + } + b.cleanupPartBlobs(ctx, space, sess.UploadID, digests) +} + // cleanupPartBlobs removes spooled blobs that belonged to aborted, expired, or // superseded parts of uploadID — unless the blob is still referenced: by a // part of another in-flight session (content-addressed dedup), by a part still @@ -490,7 +517,10 @@ func (b *Backend) cleanupPartBlobs(ctx context.Context, space did.DID, uploadID if state == registry.IntentParked { if park, err := b.parks.GetPark(ctx, d); err == nil { if cause, err := cid.Cast(park.AddTask); err == nil { - _ = b.deferred.AbortBlob(ctx, space, mh.Multihash(d), cause) + if aerr := b.deferred.AbortBlob(ctx, space, mh.Multihash(d), cause); aerr != nil { + b.logger.Warn("abort parked blob failed; provider-side release deferred", + zap.String("digest", hex.EncodeToString(d)), zap.Error(aerr)) + } } _ = b.parks.DeletePark(ctx, d) } @@ -728,43 +758,37 @@ func (b *Backend) ListMultipartUploads(ctx context.Context, input *s3.ListMultip } } - // Marker positioning. An upload-id marker is meaningful only alongside a - // key marker: when the key marker names a live key, the id must belong to - // one of that key's uploads (else InvalidArgument); when it doesn't (the - // marker key was completed/aborted meanwhile), the id positions within the - // keys ordered after it. + // Marker positioning, mirroring upstream's MultipartUploadLister: an + // upload-id marker is meaningful only alongside a key marker; it must be + // a valid UUID and must name an upload of the FIRST key group at or + // after the key marker (else InvalidArgument), and the listing resumes + // just past it. start := 0 if keyMarker != "" { if uploadIDMarker != "" { - keyLive := false - pos := -1 - for i, s := range inflight { - if s.ObjectKey == keyMarker { - keyLive = true - if s.UploadID == uploadIDMarker { - pos = i - } - } - } - if keyLive && pos < 0 { + if _, err := uuid.Parse(uploadIDMarker); err != nil { return s3response.ListMultipartUploadsResult{}, - s3err.GetAPIError(s3err.ErrInvalidRequest) + s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, uploadIDMarker) } - if pos < 0 { - for i, s := range inflight { - if s.ObjectKey >= keyMarker && s.UploadID == uploadIDMarker { - pos = i + i := 0 + for i < len(inflight) && inflight[i].ObjectKey < keyMarker { + i++ + } + pos := -1 + if i < len(inflight) { + firstKey := inflight[i].ObjectKey + for j := i; j < len(inflight) && inflight[j].ObjectKey == firstKey; j++ { + if inflight[j].UploadID == uploadIDMarker { + pos = j break } } } - if pos >= 0 { - start = pos + 1 - } else { - for start < len(inflight) && inflight[start].ObjectKey <= keyMarker { - start++ - } + if pos < 0 { + return s3response.ListMultipartUploadsResult{}, + s3err.GetInvalidArgumentErr(s3err.InvalidArgUploadIdMarker, uploadIDMarker) } + start = pos + 1 } else { for start < len(inflight) && inflight[start].ObjectKey <= keyMarker { start++ From abe6e038b03ba51c4f9682c400f86e3daa806c52 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 28 Jul 2026 15:45:17 -0700 Subject: [PATCH 16/21] =?UTF-8?q?docs(architecture):=20refresh=20=C2=A711/?= =?UTF-8?q?=C2=A712=20status=20for=20the=20read=20tier,=20DeleteBucket=20a?= =?UTF-8?q?bort,=20and=20spool=20eviction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The local-table Locator read tier is wired and validated (TestForgeReadAfterEviction; #44 for retention-retired catalog blocks), so the "not wired" bullet inverts: the remaining gap is spool eviction itself — unbounded today, DeleteObject releases network-side only — tracked in #48 alongside §5's bounded-cache spec. Record DeleteBucket's implicit abort of in-flight multipart sessions (and its hilt blob.Abort gate), point §11's cache-eviction open question at #48, and note the forge paths are now exercised by the in-repo smelt itest harness. Co-Authored-By: Claude Fable 5 --- docs/architecture.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 7b91cb0..a63db51 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -579,7 +579,7 @@ The first cut keeps digest-before-upload; `allocate-by-size` is a same-rack fast - **`min`/`max` final values** — 8 MiB / 256 MiB are the starting proposals; confirm against the base-fee and transaction-count budget. - **Lifting the 256 MiB ceiling** (streaming commP) versus the simplicity of coarse splitting. -- **Local cache vs near-stateless** — cache sizing/eviction, or commit to near-stateless. +- **Local cache vs near-stateless** — cache sizing/eviction, or commit to near-stateless (#48). - **Catalog GC** — `gc_candidates` is a write-only log this iteration; the catalog CARs on Piri grow with mutation volume until a collector exists. - **Monotonic `nextPieceId` ratchet** under heavy churn (pdp-sim) — possibly periodic dataset @@ -809,17 +809,21 @@ reference index — and is out of scope for this iteration. ### Deferred as forge-mode glue (validated live in smelt, not the in-process harness) -The in-memory harness uses a no-op uploader and serves reads from the spool, so these forge-network -paths are stubbed in-tree and verified against a real sprue+piri+indexer later: +The in-memory harness uses a no-op uploader and serves reads from the spool; the forge-network +paths below are exercised against the real stack by the smelt-based `itest/` harness in CI: - **`remove(digest)` and `abort(digest)` are live.** `RemoveBlob` invokes `/blob/remove` on sprue, which forwards `/blob/release` to the storage nodes ([§9](#9-the-system-contract-piri--sprue--indexer)); delete finality means claim-release-now, bytes-at-root-death. `AbortBlob` retires parked part-blobs via `/blob/abort` — sprue translates it into `/blob/reject` on the node (provider recovered from the `cause` receipt chain); allocation-expiry GC (FIL-625) remains the backstop when an abort never arrives. -- **The local-table `Locator` read tier is not wired.** Body-blob *locations* are recorded at accept, - but the read path that consumes them ([§7.4](#74-read-getobject), [§8](#8-retrieval-addressing-when-bodies-need-a-sharded-dag-index)) is deferred — it is only exercised after spool - eviction (also not built) and is best validated live. +- **The local-table `Locator` read tier is wired and validated.** Body blobs re-resolve after + spool loss from `blob_locations` + `/content/retrieve` (`TestForgeReadAfterEviction`), and + retention-retired catalog blocks resolve via `shard_inclusions` (#44) — the read paths of + [§7.4](#74-read-getobject) / [§8](#8-retrieval-addressing-when-bodies-need-a-sharded-dag-index). + Spool **eviction** itself is still unbuilt: nothing bounds the spool, and `DeleteObject`'s + release is network-side only, so local disk grows with every body byte ever written — the + bounded-cache policy [§5](#5-the-data-layer) specifies is tracked in #48. - **Multipart parts park at UploadPart, accept at Complete.** (Built: `parkBlobs`/`concludeBlobs` over the `blob_parks` table.) The in-process harness still spools parts at `UploadPart` and uploads+accepts them at `Complete`; the true forge *parking* (upload early, @@ -835,7 +839,10 @@ paths are stubbed in-tree and verified against a real sprue+piri+indexer later: sessions and committed objects), and a background sweeper aborts open sessions older than `multipart_session_ttl` (default 7d) and reaps terminal session rows. A successful Complete retains its session in state `completed` so a duplicate Complete is idempotent - per S3. The network-side `/blob/abort` unwind remains a parking-flow concern (above). + per S3. `DeleteBucket` implicitly aborts the bucket's in-flight sessions before the space + delete (upstream's conformance teardown never aborts them); its `/blob/abort` leg is gated + on hilt granting `blob.Abort` for the bucket-delete operation. The network-side `/blob/abort` + unwind remains a parking-flow concern (above). ### Known correctness boundary From fe6220a79d8d5f4dbe307e794bb52ca70180d703 Mon Sep 17 00:00:00 2001 From: frrist Date: Tue, 28 Jul 2026 16:26:08 -0700 Subject: [PATCH 17/21] review: stock-image gates, MinPartSize provenance, cleanup logging, drop ci-error.log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The delete-finality and deferred-multipart gates run on the stock smelt-SDK images like every other itest: env gate and binary injection dropped. They need piri:main ≥ piri#30 and a hilt with hilt#36's blob.Abort grant; until those publish, the standard INGOT_ITEST_PIRI_IMAGE / (new) INGOT_ITEST_HILT_IMAGE overrides cover local runs. Validated end-to-end with branch-built piri+hilt images: both gates green, AbortRejects 0.12s. - Pin smelt at the smelt#19 branch head — its generated stack proofs carry the /blob/release + /blob/reject node delegations the gates exercise; re-pin on merge. - Complete's 5 MiB minimum-part floor now reads backend.MinPartSize: it is S3's protocol constant, not an operator knob. - Log the previously discarded errors: the post-commit latch to 'completed', and DeletePark / spool.Remove / DeleteIntent in cleanupPartBlobs. - Restore the captured-store mask in abortOpenSession with the corrected rationale: s3:DeleteBucket delegates no blob commands, so DeleteBucket's implicit abort must run on the authority captured at UploadPart (blob.Abort rides the write set per hilt#36); fix the xfail and architecture-doc comments that misattributed the grant. - Remove the stray ci-error.log. Co-Authored-By: Claude Fable 5 --- ci-error.log | 2494 ------------------------ docs/architecture.md | 8 +- go.mod | 2 +- go.sum | 4 +- internal/reqscope/reqscope.go | 10 + itest/forge_delete_test.go | 35 +- itest/forge_multipart_deferred_test.go | 34 +- itest/stack_test.go | 4 + itest/versity_multipart_test.go | 7 +- s3frontend/multipart.go | 38 +- 10 files changed, 64 insertions(+), 2572 deletions(-) delete mode 100644 ci-error.log diff --git a/ci-error.log b/ci-error.log deleted file mode 100644 index 7a797d2..0000000 --- a/ci-error.log +++ /dev/null @@ -1,2494 +0,0 @@ -Run protocol/multiple-go-modules@v1.4 -Run status=0 --test.shuffle 1783018448984240801 -=== RUN TestModuleValidate_Disabled ---- PASS: TestModuleValidate_Disabled (0.00s) -=== RUN TestModuleValidate_Enabled ---- PASS: TestModuleValidate_Enabled (0.01s) -PASS -ok github.com/fil-forge/ingot 1.080s --test.shuffle 1783018448950969087 -=== RUN TestCached_OpenBlob_BypassesCache ---- PASS: TestCached_OpenBlob_BypassesCache (0.00s) -=== RUN TestCached_Disabled ---- PASS: TestCached_Disabled (0.00s) -=== RUN TestLayered_OpenBlob_Tiering ---- PASS: TestLayered_OpenBlob_Tiering (0.00s) -=== RUN TestCached_Hit ---- PASS: TestCached_Hit (0.00s) -=== RUN TestCached_EvictsByBytes ---- PASS: TestCached_EvictsByBytes (0.00s) -=== RUN TestStagingDiscardLeavesLogUntouched ---- PASS: TestStagingDiscardLeavesLogUntouched (0.00s) -=== RUN TestLayeredAndStagingHappyPath ---- PASS: TestLayeredAndStagingHappyPath (0.02s) -=== RUN TestLayeredFallsThroughToBaseOnMiss ---- PASS: TestLayeredFallsThroughToBaseOnMiss (0.00s) -PASS -ok github.com/fil-forge/ingot/blockstore 1.064s -? github.com/fil-forge/ingot/blockstore/locator [no test files] --test.shuffle 1783018448955249572 -=== RUN TestSplitBody_StreamingRoundTrip ---- PASS: TestSplitBody_StreamingRoundTrip (0.00s) -=== RUN TestSplitBody_Empty ---- PASS: TestSplitBody_Empty (0.00s) -=== RUN TestOpenBodyRange ---- PASS: TestOpenBodyRange (0.00s) -PASS -ok github.com/fil-forge/ingot/bucket 1.047s -? github.com/fil-forge/ingot/bucketop [no test files] -? github.com/fil-forge/ingot/cars [no test files] --test.shuffle 1783018454133616086 -=== RUN TestParseAccount -=== RUN TestParseAccount/plain_email -=== RUN TestParseAccount/did:mailto_passthrough -=== RUN TestParseAccount/garbage_errors ---- PASS: TestParseAccount (0.00s) - --- PASS: TestParseAccount/plain_email (0.00s) - --- PASS: TestParseAccount/did:mailto_passthrough (0.00s) - --- PASS: TestParseAccount/garbage_errors (0.00s) -=== RUN TestStandaloneApp_GraphValidates - logger.go:146: 2026-07-02T18:54:14.141Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:14.141Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:14.141Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:14.141Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:92)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:92)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"]} - logger.go:146: 2026-07-02T18:54:14.141Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:93)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:93)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"]} - logger.go:146: 2026-07-02T18:54:14.141Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func1(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:14.141Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func2(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:14.141Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func3(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:14.141Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func4(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:14.142Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func5(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:14.142Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func6(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:14.142Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func7(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:14.142Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func8(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:14.142Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func9(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:14.142Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func10(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:14.142Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func11(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.TestStandaloneApp_GraphValidates (/home/runner/work/ingot/ingot/cmd/serve_test.go:23)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:94)", "github.com/fil-forge/ingot/cmd.standaloneApp (/home/runner/work/ingot/ingot/cmd/serve.go:90)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:14.142Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/cmd.fxLogger.func1()"} - logger.go:146: 2026-07-02T18:54:14.142Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "23.033µs"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "14.006µs"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "16.5µs"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func8(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func8(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "126.506µs"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func9(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.143Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func9(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "52.608µs"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func10(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func10(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "35.496µs"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func11(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func11(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "31.328µs"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func1(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func1(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "54.371µs"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func2(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func2(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "65.532µs"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func3(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func3(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "48.18µs"} - logger.go:146: 2026-07-02T18:54:14.144Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func4(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.145Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func4(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "35.216µs"} - logger.go:146: 2026-07-02T18:54:14.145Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func5(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.145Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func5(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "34.504µs"} - logger.go:146: 2026-07-02T18:54:14.145Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func6(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.145Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func6(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "47.028µs"} - logger.go:146: 2026-07-02T18:54:14.145Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func7(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:14.145Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/cmd.standaloneApp.func7(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "40.356µs"} ---- PASS: TestStandaloneApp_GraphValidates (0.01s) -=== RUN TestPickAccount -=== RUN TestPickAccount/no_flag,_zero_accounts_->_error -=== RUN TestPickAccount/no_flag,_one_account_->_that_account -=== RUN TestPickAccount/no_flag,_many_accounts_->_error -=== RUN TestPickAccount/explicit_flag_in_the_set_->_that_account -=== RUN TestPickAccount/explicit_flag_(did:mailto)_in_the_set_->_that_account -=== RUN TestPickAccount/explicit_flag_not_in_the_set_->_error -=== RUN TestPickAccount/explicit_garbage_flag_->_error ---- PASS: TestPickAccount (0.00s) - --- PASS: TestPickAccount/no_flag,_zero_accounts_->_error (0.00s) - --- PASS: TestPickAccount/no_flag,_one_account_->_that_account (0.00s) - --- PASS: TestPickAccount/no_flag,_many_accounts_->_error (0.00s) - --- PASS: TestPickAccount/explicit_flag_in_the_set_->_that_account (0.00s) - --- PASS: TestPickAccount/explicit_flag_(did:mailto)_in_the_set_->_that_account (0.00s) - --- PASS: TestPickAccount/explicit_flag_not_in_the_set_->_error (0.00s) - --- PASS: TestPickAccount/explicit_garbage_flag_->_error (0.00s) -PASS -ok github.com/fil-forge/ingot/cmd 1.074s -? github.com/fil-forge/ingot/cmd/ingot [no test files] --test.shuffle 1783018454085909383 -=== RUN TestUnwrapInputValidation -=== RUN TestUnwrapInputValidation/bad_KEK -=== RUN TestUnwrapInputValidation/wrapped_too_short -=== RUN TestUnwrapInputValidation/wrapped_not_block-aligned ---- PASS: TestUnwrapInputValidation (0.00s) - --- PASS: TestUnwrapInputValidation/bad_KEK (0.00s) - --- PASS: TestUnwrapInputValidation/wrapped_too_short (0.00s) - --- PASS: TestUnwrapInputValidation/wrapped_not_block-aligned (0.00s) -=== RUN TestInputsNotMutated ---- PASS: TestInputsNotMutated (0.00s) -=== RUN TestUnwrapTampered ---- PASS: TestUnwrapTampered (0.00s) -=== RUN TestRoundTrip256 ---- PASS: TestRoundTrip256 (0.00s) -=== RUN TestWrapInputValidation -=== RUN TestWrapInputValidation/short_KEK -=== RUN TestWrapInputValidation/odd_KEK -=== RUN TestWrapInputValidation/key_data_too_short -=== RUN TestWrapInputValidation/key_data_not_block-aligned -=== RUN TestWrapInputValidation/empty_key_data ---- PASS: TestWrapInputValidation (0.00s) - --- PASS: TestWrapInputValidation/short_KEK (0.00s) - --- PASS: TestWrapInputValidation/odd_KEK (0.00s) - --- PASS: TestWrapInputValidation/key_data_too_short (0.00s) - --- PASS: TestWrapInputValidation/key_data_not_block-aligned (0.00s) - --- PASS: TestWrapInputValidation/empty_key_data (0.00s) -=== RUN TestUnwrapWrongKEK ---- PASS: TestUnwrapWrongKEK (0.00s) -=== RUN TestRFC3394Vectors -=== RUN TestRFC3394Vectors/4.1_wrap_128-bit_data_with_128-bit_KEK -=== RUN TestRFC3394Vectors/4.2_wrap_128-bit_data_with_192-bit_KEK -=== RUN TestRFC3394Vectors/4.3_wrap_128-bit_data_with_256-bit_KEK -=== RUN TestRFC3394Vectors/4.4_wrap_192-bit_data_with_192-bit_KEK -=== RUN TestRFC3394Vectors/4.5_wrap_192-bit_data_with_256-bit_KEK -=== RUN TestRFC3394Vectors/4.6_wrap_256-bit_data_with_256-bit_KEK ---- PASS: TestRFC3394Vectors (0.00s) - --- PASS: TestRFC3394Vectors/4.1_wrap_128-bit_data_with_128-bit_KEK (0.00s) - --- PASS: TestRFC3394Vectors/4.2_wrap_128-bit_data_with_192-bit_KEK (0.00s) - --- PASS: TestRFC3394Vectors/4.3_wrap_128-bit_data_with_256-bit_KEK (0.00s) - --- PASS: TestRFC3394Vectors/4.4_wrap_192-bit_data_with_192-bit_KEK (0.00s) - --- PASS: TestRFC3394Vectors/4.5_wrap_192-bit_data_with_256-bit_KEK (0.00s) - --- PASS: TestRFC3394Vectors/4.6_wrap_256-bit_data_with_256-bit_KEK (0.00s) -PASS -ok github.com/fil-forge/ingot/fee/aeskw 1.018s --test.shuffle 1783018454090957664 -=== RUN TestWriteErrorPropagates ---- PASS: TestWriteErrorPropagates (0.00s) -=== RUN TestChunkSizeAccessorAndDefault ---- PASS: TestChunkSizeAccessorAndDefault (0.00s) -=== RUN TestReaderSteadyStateAllocs ---- PASS: TestReaderSteadyStateAllocs (0.00s) -=== RUN TestWriterChunkCountGuard ---- PASS: TestWriterChunkCountGuard (0.00s) -=== RUN TestDeterministic ---- PASS: TestDeterministic (0.00s) -=== RUN TestBoundedMemory ---- PASS: TestBoundedMemory (4.84s) -=== RUN TestWriteAfterCloseFails ---- PASS: TestWriteAfterCloseFails (0.00s) -=== RUN TestReadErrorPropagates ---- PASS: TestReadErrorPropagates (0.00s) -=== RUN TestRoundTrip -=== RUN TestRoundTrip/min_chunk_size -=== RUN TestRoundTrip/default_chunk_size ---- PASS: TestRoundTrip (0.11s) - --- PASS: TestRoundTrip/min_chunk_size (0.00s) - --- PASS: TestRoundTrip/default_chunk_size (0.11s) -=== RUN TestAADIsCopiedOnConstruction ---- PASS: TestAADIsCopiedOnConstruction (0.00s) -=== RUN TestWriteAllShortWriteGuard ---- PASS: TestWriteAllShortWriteGuard (0.00s) -=== RUN TestWriterSteadyStateAllocs ---- PASS: TestWriterSteadyStateAllocs (0.02s) -=== RUN TestStreamingOddBoundaries ---- PASS: TestStreamingOddBoundaries (0.19s) -=== RUN TestNewBaseNonce ---- PASS: TestNewBaseNonce (0.00s) -=== RUN TestStreamNonceLayout ---- PASS: TestStreamNonceLayout (0.00s) -=== RUN TestConfigValidation -=== RUN TestConfigValidation/short_key -=== RUN TestConfigValidation/long_key -=== RUN TestConfigValidation/short_nonce -=== RUN TestConfigValidation/long_nonce -=== RUN TestConfigValidation/chunk_too_small -=== RUN TestConfigValidation/chunk_too_large ---- PASS: TestConfigValidation (0.00s) - --- PASS: TestConfigValidation/short_key (0.00s) - --- PASS: TestConfigValidation/long_key (0.00s) - --- PASS: TestConfigValidation/short_nonce (0.00s) - --- PASS: TestConfigValidation/long_nonce (0.00s) - --- PASS: TestConfigValidation/chunk_too_small (0.00s) - --- PASS: TestConfigValidation/chunk_too_large (0.00s) -=== RUN TestReaderChunkCountGuard ---- PASS: TestReaderChunkCountGuard (0.00s) -=== RUN TestEncryptedSize ---- PASS: TestEncryptedSize (0.00s) -=== RUN TestNilAADRoundTrips ---- PASS: TestNilAADRoundTrips (0.00s) -=== RUN TestDecryptFailures -=== RUN TestDecryptFailures/drop_final_chunk -=== RUN TestDecryptFailures/cut_final_chunk_tag -=== RUN TestDecryptFailures/cut_mid_non-final_chunk -=== RUN TestDecryptFailures/empty_ciphertext -=== RUN TestDecryptFailures/single_byte -=== RUN TestDecryptFailures/swapped_chunks -=== RUN TestDecryptFailures/flip_first_chunk_body -=== RUN TestDecryptFailures/flip_first_chunk_tag -=== RUN TestDecryptFailures/flip_second_chunk -=== RUN TestDecryptFailures/flip_final_chunk -=== RUN TestDecryptFailures/trailing_after_full-size_final_chunk -=== RUN TestDecryptFailures/trailing_after_short_final_chunk -=== RUN TestDecryptFailures/wrong_key -=== RUN TestDecryptFailures/wrong_base_nonce -=== RUN TestDecryptFailures/wrong_AAD ---- PASS: TestDecryptFailures (0.00s) - --- PASS: TestDecryptFailures/drop_final_chunk (0.00s) - --- PASS: TestDecryptFailures/cut_final_chunk_tag (0.00s) - --- PASS: TestDecryptFailures/cut_mid_non-final_chunk (0.00s) - --- PASS: TestDecryptFailures/empty_ciphertext (0.00s) - --- PASS: TestDecryptFailures/single_byte (0.00s) - --- PASS: TestDecryptFailures/swapped_chunks (0.00s) - --- PASS: TestDecryptFailures/flip_first_chunk_body (0.00s) - --- PASS: TestDecryptFailures/flip_first_chunk_tag (0.00s) - --- PASS: TestDecryptFailures/flip_second_chunk (0.00s) - --- PASS: TestDecryptFailures/flip_final_chunk (0.00s) - --- PASS: TestDecryptFailures/trailing_after_full-size_final_chunk (0.00s) - --- PASS: TestDecryptFailures/trailing_after_short_final_chunk (0.00s) - --- PASS: TestDecryptFailures/wrong_key (0.00s) - --- PASS: TestDecryptFailures/wrong_base_nonce (0.00s) - --- PASS: TestDecryptFailures/wrong_AAD (0.00s) -PASS -ok github.com/fil-forge/ingot/fee/aesstream 6.192s --test.shuffle 1783018454505822173 -=== RUN TestEncodeRejectsInvalidLabel ---- PASS: TestEncodeRejectsInvalidLabel (0.00s) -=== RUN TestEncode -=== RUN TestEncode/deterministic -=== RUN TestEncode/requires_recipient -=== RUN TestEncode/empty_protected_header ---- PASS: TestEncode (0.00s) - --- PASS: TestEncode/deterministic (0.00s) - --- PASS: TestEncode/requires_recipient (0.00s) - --- PASS: TestEncode/empty_protected_header (0.00s) -=== RUN TestHeader -=== RUN TestHeader/accessors -=== RUN TestHeader/label_normalization -=== RUN TestHeader/int_uint_boundaries -=== RUN TestHeader/integer_width_normalization ---- PASS: TestHeader (0.00s) - --- PASS: TestHeader/accessors (0.00s) - --- PASS: TestHeader/label_normalization (0.00s) - --- PASS: TestHeader/int_uint_boundaries (0.00s) - --- PASS: TestHeader/integer_width_normalization (0.00s) -=== RUN TestRoundTrip -=== RUN TestRoundTrip/preserves_all_fields -=== RUN TestRoundTrip/multi_recipient -=== RUN TestRoundTrip/detached_payload -=== RUN TestRoundTrip/large_unsigned_value ---- PASS: TestRoundTrip (0.00s) - --- PASS: TestRoundTrip/preserves_all_fields (0.00s) - --- PASS: TestRoundTrip/multi_recipient (0.00s) - --- PASS: TestRoundTrip/detached_payload (0.00s) - --- PASS: TestRoundTrip/large_unsigned_value (0.00s) -=== RUN TestEncStructure -=== RUN TestEncStructure/binds_protected_not_unprotected -=== RUN TestEncStructure/stable_across_round_trip ---- PASS: TestEncStructure (0.00s) - --- PASS: TestEncStructure/binds_protected_not_unprotected (0.00s) - --- PASS: TestEncStructure/stable_across_round_trip (0.00s) -=== RUN TestHeaderSetPanics -=== RUN TestHeaderSetPanics/overflow_label -=== RUN TestHeaderSetPanics/invalid_label ---- PASS: TestHeaderSetPanics (0.00s) - --- PASS: TestHeaderSetPanics/overflow_label (0.00s) - --- PASS: TestHeaderSetPanics/invalid_label (0.00s) -=== RUN TestVectors -=== RUN TestVectors/encode_minimal -=== RUN TestVectors/decode_minimal -=== RUN TestVectors/decode_rich -=== RUN TestVectors/enc_structure_empty_protected ---- PASS: TestVectors (0.00s) - --- PASS: TestVectors/encode_minimal (0.00s) - --- PASS: TestVectors/decode_minimal (0.00s) - --- PASS: TestVectors/decode_rich (0.00s) - --- PASS: TestVectors/enc_structure_empty_protected (0.00s) -=== RUN TestDecodeMalformed -=== RUN TestDecodeMalformed/empty_input -=== RUN TestDecodeMalformed/truncated_array -=== RUN TestDecodeMalformed/truncated_mid-item -=== RUN TestDecodeMalformed/not_a_tag_(bare_array) -=== RUN TestDecodeMalformed/bare_integer,_not_a_tag -=== RUN TestDecodeMalformed/wrong_tag_(16,_Encrypt0) -=== RUN TestDecodeMalformed/array_too_short_(3_elems) -=== RUN TestDecodeMalformed/array_too_long_(5_elems) -=== RUN TestDecodeMalformed/protected_not_a_byte_string -=== RUN TestDecodeMalformed/protected_content_not_a_map -=== RUN TestDecodeMalformed/unprotected_not_a_map -=== RUN TestDecodeMalformed/body_ciphertext_not_null -=== RUN TestDecodeMalformed/recipients_not_an_array -=== RUN TestDecodeMalformed/recipients_empty -=== RUN TestDecodeMalformed/recipient_not_an_array -=== RUN TestDecodeMalformed/recipient_array_too_short_(2) -=== RUN TestDecodeMalformed/recipient_nested_(4_elems) -=== RUN TestDecodeMalformed/recipient_ciphertext_wrong_type -=== RUN TestDecodeMalformed/duplicate_protected_label -=== RUN TestDecodeMalformed/duplicate_unprotected_label ---- PASS: TestDecodeMalformed (0.00s) - --- PASS: TestDecodeMalformed/empty_input (0.00s) - --- PASS: TestDecodeMalformed/truncated_array (0.00s) - --- PASS: TestDecodeMalformed/truncated_mid-item (0.00s) - --- PASS: TestDecodeMalformed/not_a_tag_(bare_array) (0.00s) - --- PASS: TestDecodeMalformed/bare_integer,_not_a_tag (0.00s) - --- PASS: TestDecodeMalformed/wrong_tag_(16,_Encrypt0) (0.00s) - --- PASS: TestDecodeMalformed/array_too_short_(3_elems) (0.00s) - --- PASS: TestDecodeMalformed/array_too_long_(5_elems) (0.00s) - --- PASS: TestDecodeMalformed/protected_not_a_byte_string (0.00s) - --- PASS: TestDecodeMalformed/protected_content_not_a_map (0.00s) - --- PASS: TestDecodeMalformed/unprotected_not_a_map (0.00s) - --- PASS: TestDecodeMalformed/body_ciphertext_not_null (0.00s) - --- PASS: TestDecodeMalformed/recipients_not_an_array (0.00s) - --- PASS: TestDecodeMalformed/recipients_empty (0.00s) - --- PASS: TestDecodeMalformed/recipient_not_an_array (0.00s) - --- PASS: TestDecodeMalformed/recipient_array_too_short_(2) (0.00s) - --- PASS: TestDecodeMalformed/recipient_nested_(4_elems) (0.00s) - --- PASS: TestDecodeMalformed/recipient_ciphertext_wrong_type (0.00s) - --- PASS: TestDecodeMalformed/duplicate_protected_label (0.00s) - --- PASS: TestDecodeMalformed/duplicate_unprotected_label (0.00s) -=== RUN TestDecodeExpectedType -=== RUN TestDecodeExpectedType/matching_type -=== RUN TestDecodeExpectedType/wrong_type -=== RUN TestDecodeExpectedType/missing_type -=== RUN TestDecodeExpectedType/type_present_but_not_a_string -=== RUN TestDecodeExpectedType/no_check_when_option_omitted ---- PASS: TestDecodeExpectedType (0.00s) - --- PASS: TestDecodeExpectedType/matching_type (0.00s) - --- PASS: TestDecodeExpectedType/wrong_type (0.00s) - --- PASS: TestDecodeExpectedType/missing_type (0.00s) - --- PASS: TestDecodeExpectedType/type_present_but_not_a_string (0.00s) - --- PASS: TestDecodeExpectedType/no_check_when_option_omitted (0.00s) -=== RUN Example ---- PASS: Example (0.00s) -PASS -ok github.com/fil-forge/ingot/fee/cose 1.029s --test.shuffle 1783018455556095894 -=== RUN TestKDFContextCanonical ---- PASS: TestKDFContextCanonical (0.00s) -=== RUN TestConcatKDFContextSensitivity ---- PASS: TestConcatKDFContextSensitivity (0.00s) -=== RUN TestWrapUnwrapRoundTrip ---- PASS: TestWrapUnwrapRoundTrip (0.01s) -=== RUN TestUnwrapInputValidation -=== RUN TestUnwrapInputValidation/nil_private_key -=== RUN TestUnwrapInputValidation/nil_wrapped -=== RUN TestUnwrapInputValidation/nil_ephemeral_key -=== RUN TestUnwrapInputValidation/ephemeral_wrong_curve ---- PASS: TestUnwrapInputValidation (0.00s) - --- PASS: TestUnwrapInputValidation/nil_private_key (0.00s) - --- PASS: TestUnwrapInputValidation/nil_wrapped (0.00s) - --- PASS: TestUnwrapInputValidation/nil_ephemeral_key (0.00s) - --- PASS: TestUnwrapInputValidation/ephemeral_wrong_curve (0.00s) -=== RUN TestWrapInputValidation -=== RUN TestWrapInputValidation/nil_public_key -=== RUN TestWrapInputValidation/wrong_curve -=== RUN TestWrapInputValidation/cek_too_short -=== RUN TestWrapInputValidation/cek_not_block-aligned -=== RUN TestWrapInputValidation/nil_cek ---- PASS: TestWrapInputValidation (0.00s) - --- PASS: TestWrapInputValidation/nil_public_key (0.00s) - --- PASS: TestWrapInputValidation/wrong_curve (0.00s) - --- PASS: TestWrapInputValidation/cek_too_short (0.00s) - --- PASS: TestWrapInputValidation/cek_not_block-aligned (0.00s) - --- PASS: TestWrapInputValidation/nil_cek (0.00s) -=== RUN TestUnwrapWrongPrivateKey ---- PASS: TestUnwrapWrongPrivateKey (0.00s) -=== RUN TestConcatKDFSingleBlock ---- PASS: TestConcatKDFSingleBlock (0.00s) -=== RUN TestUnwrapTamperedEphemeral ---- PASS: TestUnwrapTamperedEphemeral (0.00s) -=== RUN TestUnwrapTamperedCEK ---- PASS: TestUnwrapTamperedCEK (0.04s) -=== RUN TestCBORIntEncoding ---- PASS: TestCBORIntEncoding (0.00s) -=== RUN TestWrapFreshEphemeralPerCall ---- PASS: TestWrapFreshEphemeralPerCall (0.00s) -=== RUN TestUnwrapLowOrderEphemeral ---- PASS: TestUnwrapLowOrderEphemeral (0.00s) -=== RUN TestConcatKDFMultiBlock ---- PASS: TestConcatKDFMultiBlock (0.00s) -=== RUN TestKnownAnswerVector ---- PASS: TestKnownAnswerVector (0.00s) -=== RUN TestKDFContextProtected ---- PASS: TestKDFContextProtected (0.00s) -=== RUN TestCBORHeadEncoding -=== RUN TestCBORHeadEncoding/uint_tiny -=== RUN TestCBORHeadEncoding/uint_max_inline -=== RUN TestCBORHeadEncoding/uint_1-byte -=== RUN TestCBORHeadEncoding/uint_1-byte_max -=== RUN TestCBORHeadEncoding/uint_2-byte -=== RUN TestCBORHeadEncoding/uint_2-byte_max -=== RUN TestCBORHeadEncoding/uint_4-byte -=== RUN TestCBORHeadEncoding/uint_4-byte_max -=== RUN TestCBORHeadEncoding/uint_8-byte -=== RUN TestCBORHeadEncoding/array(4) -=== RUN TestCBORHeadEncoding/empty_bstr -=== RUN TestCBORHeadEncoding/4-byte_bstr ---- PASS: TestCBORHeadEncoding (0.00s) - --- PASS: TestCBORHeadEncoding/uint_tiny (0.00s) - --- PASS: TestCBORHeadEncoding/uint_max_inline (0.00s) - --- PASS: TestCBORHeadEncoding/uint_1-byte (0.00s) - --- PASS: TestCBORHeadEncoding/uint_1-byte_max (0.00s) - --- PASS: TestCBORHeadEncoding/uint_2-byte (0.00s) - --- PASS: TestCBORHeadEncoding/uint_2-byte_max (0.00s) - --- PASS: TestCBORHeadEncoding/uint_4-byte (0.00s) - --- PASS: TestCBORHeadEncoding/uint_4-byte_max (0.00s) - --- PASS: TestCBORHeadEncoding/uint_8-byte (0.00s) - --- PASS: TestCBORHeadEncoding/array(4) (0.00s) - --- PASS: TestCBORHeadEncoding/empty_bstr (0.00s) - --- PASS: TestCBORHeadEncoding/4-byte_bstr (0.00s) -PASS -ok github.com/fil-forge/ingot/fee/ecdhkw 1.084s --test.shuffle 1783018455708793781 -=== RUN TestAccounts -=== RUN TestAccounts/empty_store_returns_no_accounts -=== RUN TestAccounts/non-mailto_issuer_is_filtered_out -=== RUN TestAccounts/returns_distinct_mailto_accounts,_sorted ---- PASS: TestAccounts (0.01s) - --- PASS: TestAccounts/empty_store_returns_no_accounts (0.00s) - --- PASS: TestAccounts/non-mailto_issuer_is_filtered_out (0.00s) - --- PASS: TestAccounts/returns_distinct_mailto_accounts,_sorted (0.00s) -PASS -ok github.com/fil-forge/ingot/forgeclient 1.041s -? github.com/fil-forge/ingot/gen [no test files] --test.shuffle 1783018456454161546 -=== RUN TestParts_OrderedAndCascade ---- PASS: TestParts_OrderedAndCascade (0.00s) -=== RUN TestIntents_StateMachine ---- PASS: TestIntents_StateMachine (0.00s) -=== RUN TestMarkSegmentShipped_GuardsForgeRootOnRoot ---- PASS: TestMarkSegmentShipped_GuardsForgeRootOnRoot (0.00s) -=== RUN TestMultipartLatch_SingleWinner_Concurrent ---- PASS: TestMultipartLatch_SingleWinner_Concurrent (0.00s) -=== RUN TestMultipartLatch_SingleWinner_Sequential ---- PASS: TestMultipartLatch_SingleWinner_Sequential (0.00s) -=== RUN TestLocations_RoundTrip ---- PASS: TestLocations_RoundTrip (0.00s) -=== RUN TestBlobRefs_CountToZero ---- PASS: TestBlobRefs_CountToZero (0.00s) -PASS -ok github.com/fil-forge/ingot/inmem 1.031s -? github.com/fil-forge/ingot/internal/build [no test files] -? github.com/fil-forge/ingot/internal/ucanexec [no test files] --test.shuffle 1783018457520307022 -=== RUN TestAppendBatchDedupesAcrossOps ---- PASS: TestAppendBatchDedupesAcrossOps (0.01s) -=== RUN TestSealByAge ---- PASS: TestSealByAge (0.10s) -=== RUN TestRetentionDropsOldFlushed ---- PASS: TestRetentionDropsOldFlushed (0.00s) -=== RUN TestAppendBatchAllDuplicatesStillRecordsOpRoot ---- PASS: TestAppendBatchAllDuplicatesStillRecordsOpRoot (0.00s) -=== RUN TestSealBySize ---- PASS: TestSealBySize (0.01s) -=== RUN TestAppendBatchEmptyBlocksAccepted ---- PASS: TestAppendBatchEmptyBlocksAccepted (0.00s) -=== RUN TestGetMissReturnsErrNotFound ---- PASS: TestGetMissReturnsErrNotFound (0.00s) -=== RUN TestAppendThenGetSameProcess ---- PASS: TestAppendThenGetSameProcess (0.00s) -=== RUN TestForceSealRecoveredOpenOnRestart ---- PASS: TestForceSealRecoveredOpenOnRestart (0.00s) -=== RUN TestCatalogNeverShips ---- PASS: TestCatalogNeverShips (0.33s) -PASS -ok github.com/fil-forge/ingot/logstore 1.496s --test.shuffle 1783018457504474164 -=== RUN TestUp_Live - up_live_test.go:26: set INGOT_TEST_DSN to run the live migration test ---- SKIP: TestUp_Live (0.00s) -PASS -ok github.com/fil-forge/ingot/migrations 1.013s -? github.com/fil-forge/ingot/mst [no test files] --test.shuffle 1783018458726225614 -=== RUN TestLocalLocator_NotFound ---- PASS: TestLocalLocator_NotFound (0.00s) -=== RUN TestPostgresStores_Live - postgres_live_test.go:38: set INGOT_TEST_DSN to run the live Postgres store test ---- SKIP: TestPostgresStores_Live (0.00s) -=== RUN TestLocalLocator_Locate ---- PASS: TestLocalLocator_Locate (0.00s) -PASS -ok github.com/fil-forge/ingot/registry 1.043s --test.shuffle 1783018459879059818 -=== RUN TestSelectBytes_Dispatch ---- PASS: TestSelectBytes_Dispatch (0.00s) -=== RUN TestRefIndex_DuplicateBlobInManifestReleasedOnce ---- PASS: TestRefIndex_DuplicateBlobInManifestReleasedOnce (0.01s) -=== RUN TestRefIndex_DedupAcrossKeys ---- PASS: TestRefIndex_DedupAcrossKeys (0.00s) -=== RUN TestRefIndex_OverwriteSameContentKeepsClaim ---- PASS: TestRefIndex_OverwriteSameContentKeepsClaim (0.00s) -=== RUN TestUploadDedup_SkipsAlreadyLocatedBlob ---- PASS: TestUploadDedup_SkipsAlreadyLocatedBlob (0.00s) -=== RUN TestPartRange -=== RUN TestPartRange/mp_part_1 -=== RUN TestPartRange/mp_part_2 -=== RUN TestPartRange/mp_part_3 -=== RUN TestPartRange/mp_exceeds -=== RUN TestPartRange/mp_zero-length_part -=== RUN TestPartRange/single_part_1_=_whole_object -=== RUN TestPartRange/single_part_2_exceeds -=== RUN TestPartRange/empty_part_1_=_no_range -=== RUN TestPartRange/empty_part_2_exceeds ---- PASS: TestPartRange (0.00s) - --- PASS: TestPartRange/mp_part_1 (0.00s) - --- PASS: TestPartRange/mp_part_2 (0.00s) - --- PASS: TestPartRange/mp_part_3 (0.00s) - --- PASS: TestPartRange/mp_exceeds (0.00s) - --- PASS: TestPartRange/mp_zero-length_part (0.00s) - --- PASS: TestPartRange/single_part_1_=_whole_object (0.00s) - --- PASS: TestPartRange/single_part_2_exceeds (0.00s) - --- PASS: TestPartRange/empty_part_1_=_no_range (0.00s) - --- PASS: TestPartRange/empty_part_2_exceeds (0.00s) -=== RUN TestRefIndex_DeleteOneOfTwoKeepsBlob ---- PASS: TestRefIndex_DeleteOneOfTwoKeepsBlob (0.01s) -=== RUN TestChecksumFromInput_DefaultsToCRC64NVME -=== RUN TestChecksumFromInput_DefaultsToCRC64NVME/no_checksum_→_default_crc64nvme_(compute,_no_validation) -=== RUN TestChecksumFromInput_DefaultsToCRC64NVME/explicit_value_wins_over_the_default -=== RUN TestChecksumFromInput_DefaultsToCRC64NVME/named_algorithm_(no_value)_wins_over_the_default ---- PASS: TestChecksumFromInput_DefaultsToCRC64NVME (0.00s) - --- PASS: TestChecksumFromInput_DefaultsToCRC64NVME/no_checksum_→_default_crc64nvme_(compute,_no_validation) (0.00s) - --- PASS: TestChecksumFromInput_DefaultsToCRC64NVME/explicit_value_wins_over_the_default (0.00s) - --- PASS: TestChecksumFromInput_DefaultsToCRC64NVME/named_algorithm_(no_value)_wins_over_the_default (0.00s) -=== RUN TestRefIndex_DeleteReleasesAtZero ---- PASS: TestRefIndex_DeleteReleasesAtZero (0.00s) -=== RUN TestRefIndex_OverwriteDifferentContentReleasesOld ---- PASS: TestRefIndex_OverwriteDifferentContentReleasesOld (0.00s) -PASS -ok github.com/fil-forge/ingot/s3frontend 1.072s --test.shuffle 1783018464376373102 -=== RUN TestMultipart_Abort - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:24.380Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:24.380Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:24.380Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_Abort (/home/runner/work/ingot/ingot/testing/multipart_test.go:156)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "15.108µs"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "13.795µs"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "11.301µs"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.381Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "96.35µs"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "24.896µs"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "23.223µs"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "21.15µs"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "34.014µs"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "44.784µs"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "32.019µs"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "31.298µs"} - logger.go:146: 2026-07-02T18:54:24.382Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.383Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "29.114µs"} - logger.go:146: 2026-07-02T18:54:24.383Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.383Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "28.824µs"} - logger.go:146: 2026-07-02T18:54:24.383Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.383Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "27.962µs"} - logger.go:146: 2026-07-02T18:54:24.383Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:24.385Z INFO starting ingot S3 listener {"addr": "127.0.0.1:37433", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-2486866961", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:24.385Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.806414ms"} - logger.go:146: 2026-07-02T18:54:24.385Z INFO started - logger.go:146: 2026-07-02T18:54:24.396Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:24.396Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:24.396Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:24.396Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "154.344µs"} ---- PASS: TestMultipart_Abort (0.02s) -=== RUN TestMultipart_FailedCompleteStaysAbortable - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:24.400Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:24.400Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:24.400Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_FailedCompleteStaysAbortable (/home/runner/work/ingot/ingot/testing/multipart_test.go:102)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "15.389µs"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "17.733µs"} - logger.go:146: 2026-07-02T18:54:24.401Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:24.402Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "21.99µs"} - logger.go:146: 2026-07-02T18:54:24.402Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.402Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "65.532µs"} - logger.go:146: 2026-07-02T18:54:24.402Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.402Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "51.857µs"} - logger.go:146: 2026-07-02T18:54:24.402Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "33.833µs"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "28.312µs"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "34.484µs"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "34.805µs"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "33.673µs"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.403Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "32.931µs"} - logger.go:146: 2026-07-02T18:54:24.404Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.404Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "31.038µs"} - logger.go:146: 2026-07-02T18:54:24.404Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.404Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "33.873µs"} - logger.go:146: 2026-07-02T18:54:24.404Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.404Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "40.305µs"} - logger.go:146: 2026-07-02T18:54:24.404Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:24.406Z INFO starting ingot S3 listener {"addr": "127.0.0.1:45175", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-1080784318", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:24.406Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.64353ms"} - logger.go:146: 2026-07-02T18:54:24.406Z INFO started -SDK 2026/07/02 18:54:24 WARN Response has no supported checksum. Not validating response payload. - logger.go:146: 2026-07-02T18:54:24.417Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:24.417Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:24.418Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:24.419Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.704985ms"} ---- PASS: TestMultipart_FailedCompleteStaysAbortable (0.02s) -=== RUN TestSmokeXFail_GetObject - logger.go:146: 2026-07-02T18:54:24.422Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:24.422Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:24.422Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:24.422Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:24.422Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:24.422Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:24.422Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:329)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "13.254µs"} - logger.go:146: 2026-07-02T18:54:24.423Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "24.004µs"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "10.239µs"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "58.489µs"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "31.95µs"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "25.828µs"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "27.561µs"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "30.396µs"} - logger.go:146: 2026-07-02T18:54:24.424Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "33.141µs"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "34.705µs"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "20.127µs"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "29.575µs"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "34.043µs"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.425Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "45.294µs"} - logger.go:146: 2026-07-02T18:54:24.426Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:24.427Z INFO starting ingot S3 listener {"addr": "127.0.0.1:33189", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-1820238032", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:24.427Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.585542ms"} - logger.go:146: 2026-07-02T18:54:24.427Z INFO started -=== RUN TestSmokeXFail_GetObject/directory_success -RUN GetObject_directory_success -FAIL GetObject_directory_success: expected content type application/x-directory, instead got binary/octet-stream - smoke_test.go:337: known-failing: expected content type application/x-directory, instead got binary/octet-stream -=== RUN TestSmokeXFail_GetObject/overrides_fail_public -RUN GetObject_overrides_fail_public -FAIL GetObject_overrides_fail_public: operation error S3: PutBucketPolicy, https response error StatusCode: 501, RequestID: , HostID: , api error NotImplemented: A header you provided implies functionality that is not implemented. - smoke_test.go:337: known-failing: operation error S3: PutBucketPolicy, https response error StatusCode: 501, RequestID: , HostID: , api error NotImplemented: A header you provided implies functionality that is not implemented. -=== RUN TestSmokeXFail_GetObject/success -RUN GetObject_success -FAIL GetObject_success: expected tag count to be 2, instead got 0 - smoke_test.go:337: known-failing: expected tag count to be 2, instead got 0 -=== NAME TestSmokeXFail_GetObject - logger.go:146: 2026-07-02T18:54:24.468Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:24.468Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:24.468Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:24.469Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "487.942µs"} ---- PASS: TestSmokeXFail_GetObject (0.05s) - --- SKIP: TestSmokeXFail_GetObject/directory_success (0.01s) - --- SKIP: TestSmokeXFail_GetObject/overrides_fail_public (0.01s) - --- SKIP: TestSmokeXFail_GetObject/success (0.02s) -=== RUN TestSmoke_HeadObject - logger.go:146: 2026-07-02T18:54:24.472Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:24.472Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:24.472Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:24.472Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:24.472Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:24.472Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:24.472Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:24.472Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:220)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.473Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "15.188µs"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "17.382µs"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "15.709µs"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "55.142µs"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "26.039µs"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "31.128µs"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "30.627µs"} - logger.go:146: 2026-07-02T18:54:24.474Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "41.697µs"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "40.616µs"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "39.223µs"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "39.273µs"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "39.183µs"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "37.841µs"} - logger.go:146: 2026-07-02T18:54:24.475Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:24.476Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "41.027µs"} - logger.go:146: 2026-07-02T18:54:24.476Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:24.478Z INFO starting ingot S3 listener {"addr": "127.0.0.1:35311", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-4251082191", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:24.478Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.637608ms"} - logger.go:146: 2026-07-02T18:54:24.478Z INFO started -=== RUN TestSmoke_HeadObject/checksums -RUN HeadObject_checksums -PASS HeadObject_checksums -=== RUN TestSmoke_HeadObject/conditional_reads -RUN HeadObject_conditional_reads -PASS HeadObject_conditional_reads -=== RUN TestSmoke_HeadObject/directory_object_noslash -RUN HeadObject_directory_object_noslash -PASS HeadObject_directory_object_noslash -=== RUN TestSmoke_HeadObject/empty_object_part_number_1 -RUN HeadObject_empty_object_part_number_1 -PASS HeadObject_empty_object_part_number_1 -=== RUN TestSmoke_HeadObject/invalid_parent_dir -RUN HeadObject_invalid_parent_dir -PASS HeadObject_invalid_parent_dir -=== RUN TestSmoke_HeadObject/invalid_part_number -RUN HeadObject_invalid_part_number -PASS HeadObject_invalid_part_number -=== RUN TestSmoke_HeadObject/mp_part_number_exceeds_parts_count -RUN HeadObject_mp_part_number_exceeds_parts_count -PASS HeadObject_mp_part_number_exceeds_parts_count -=== RUN TestSmoke_HeadObject/mp_part_number_resp_status -RUN HeadObject_mp_part_number_resp_status -PASS HeadObject_mp_part_number_resp_status -=== RUN TestSmoke_HeadObject/mp_part_number_success -RUN HeadObject_mp_part_number_success -PASS HeadObject_mp_part_number_success -=== RUN TestSmoke_HeadObject/non_mp_part_number_1_success -RUN HeadObject_non_mp_part_number_1_success -PASS HeadObject_non_mp_part_number_1_success -=== RUN TestSmoke_HeadObject/non_existing_dir_object -RUN HeadObject_non_existing_dir_object -PASS HeadObject_non_existing_dir_object -=== RUN TestSmoke_HeadObject/non_existing_object -RUN HeadObject_non_existing_object -PASS HeadObject_non_existing_object -=== RUN TestSmoke_HeadObject/not_enabled_checksum_mode -RUN HeadObject_not_enabled_checksum_mode -PASS HeadObject_not_enabled_checksum_mode -=== RUN TestSmoke_HeadObject/overrides_presign_success -RUN HeadObject_overrides_presign_success -PASS HeadObject_overrides_presign_success -=== RUN TestSmoke_HeadObject/overrides_success -RUN HeadObject_overrides_success -PASS HeadObject_overrides_success -=== RUN TestSmoke_HeadObject/range_and_part_number -RUN HeadObject_range_and_part_number -PASS HeadObject_range_and_part_number -=== RUN TestSmoke_HeadObject/by_range_resp_status -RUN HeadObject_by_range_resp_status -PASS HeadObject_by_range_resp_status -=== RUN TestSmoke_HeadObject/dir_with_range -RUN HeadObject_dir_with_range -PASS HeadObject_dir_with_range -=== RUN TestSmoke_HeadObject/ranged_with_checksum_mode -RUN HeadObject_ranged_with_checksum_mode -PASS HeadObject_ranged_with_checksum_mode -=== RUN TestSmoke_HeadObject/with_range -RUN HeadObject_with_range -PASS HeadObject_with_range -=== RUN TestSmoke_HeadObject/zero_len_with_range -RUN HeadObject_zero_len_with_range -PASS HeadObject_zero_len_with_range -=== NAME TestSmoke_HeadObject - logger.go:146: 2026-07-02T18:54:26.634Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.634Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:26.634Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:26.636Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "2.305297ms"} ---- PASS: TestSmoke_HeadObject (2.17s) - --- PASS: TestSmoke_HeadObject/checksums (0.03s) - --- PASS: TestSmoke_HeadObject/conditional_reads (1.16s) - --- PASS: TestSmoke_HeadObject/directory_object_noslash (0.01s) - --- PASS: TestSmoke_HeadObject/empty_object_part_number_1 (0.01s) - --- PASS: TestSmoke_HeadObject/invalid_parent_dir (0.01s) - --- PASS: TestSmoke_HeadObject/invalid_part_number (0.00s) - --- PASS: TestSmoke_HeadObject/mp_part_number_exceeds_parts_count (0.29s) - --- PASS: TestSmoke_HeadObject/mp_part_number_resp_status (0.12s) - --- PASS: TestSmoke_HeadObject/mp_part_number_success (0.18s) - --- PASS: TestSmoke_HeadObject/non_mp_part_number_1_success (0.01s) - --- PASS: TestSmoke_HeadObject/non_existing_dir_object (0.02s) - --- PASS: TestSmoke_HeadObject/non_existing_object (0.01s) - --- PASS: TestSmoke_HeadObject/not_enabled_checksum_mode (0.01s) - --- PASS: TestSmoke_HeadObject/overrides_presign_success (0.02s) - --- PASS: TestSmoke_HeadObject/overrides_success (0.01s) - --- PASS: TestSmoke_HeadObject/range_and_part_number (0.01s) - --- PASS: TestSmoke_HeadObject/by_range_resp_status (0.04s) - --- PASS: TestSmoke_HeadObject/dir_with_range (0.02s) - --- PASS: TestSmoke_HeadObject/ranged_with_checksum_mode (0.13s) - --- PASS: TestSmoke_HeadObject/with_range (0.04s) - --- PASS: TestSmoke_HeadObject/zero_len_with_range (0.02s) -=== RUN TestSmokeXFail_ListBuckets - logger.go:146: 2026-07-02T18:54:26.647Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:26.647Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:26.647Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:26.647Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.647Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.647Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:26.647Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:26.647Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:26.647Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:26.647Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:290)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "15.66µs"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.648Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "18.204µs"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "16.771µs"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "61.274µs"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "31.498µs"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "46.737µs"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "45.034µs"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.649Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "27.962µs"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "23.584µs"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "31.869µs"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "31.278µs"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "20.218µs"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "33.673µs"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.650Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "42.179µs"} - logger.go:146: 2026-07-02T18:54:26.651Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.652Z INFO starting ingot S3 listener {"addr": "127.0.0.1:45695", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-1902393429", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:26.652Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.512927ms"} - logger.go:146: 2026-07-02T18:54:26.652Z INFO started -=== RUN TestSmokeXFail_ListBuckets/as_admin -RUN ListBuckets_as_admin -FAIL ListBuckets_as_admin: fork/exec ./versitygw: no such file or directory - smoke_test.go:298: known-failing: fork/exec ./versitygw: no such file or directory -=== RUN TestSmokeXFail_ListBuckets/as_user -RUN ListBuckets_as_user -FAIL ListBuckets_as_user: fork/exec ./versitygw: no such file or directory - smoke_test.go:298: known-failing: fork/exec ./versitygw: no such file or directory -=== NAME TestSmokeXFail_ListBuckets - logger.go:146: 2026-07-02T18:54:26.677Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.677Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:26.677Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:26.677Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "129.01µs"} ---- PASS: TestSmokeXFail_ListBuckets (0.03s) - --- SKIP: TestSmokeXFail_ListBuckets/as_admin (0.01s) - --- SKIP: TestSmokeXFail_ListBuckets/as_user (0.01s) -=== RUN TestBlobSplit_MultiBlobRoundTrip - logger.go:146: 2026-07-02T18:54:26.680Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:26.680Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:26.680Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:26.680Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.680Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.680Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:26.680Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_MultiBlobRoundTrip (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:64)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "15.409µs"} - logger.go:146: 2026-07-02T18:54:26.681Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "17.443µs"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "10.019µs"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "57.848µs"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "29.856µs"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "46.877µs"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "29.545µs"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "36.537µs"} - logger.go:146: 2026-07-02T18:54:26.682Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "36.859µs"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "21.84µs"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "30.637µs"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "30.546µs"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "43.791µs"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "23.233µs"} - logger.go:146: 2026-07-02T18:54:26.683Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.686Z INFO starting ingot S3 listener {"addr": "127.0.0.1:44217", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-45819917", "max_blob_size": 65536} - logger.go:146: 2026-07-02T18:54:26.686Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "2.410227ms"} - logger.go:146: 2026-07-02T18:54:26.686Z INFO started - logger.go:146: 2026-07-02T18:54:26.707Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.707Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:26.707Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:26.707Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "600.503µs"} ---- PASS: TestBlobSplit_MultiBlobRoundTrip (0.03s) -=== RUN TestSmoke_HeadBucket - logger.go:146: 2026-07-02T18:54:26.710Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:26.710Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:26.710Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:26.710Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.710Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_HeadBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:77)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:26.711Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "15.899µs"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "17.933µs"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "15.789µs"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "57.147µs"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "33.612µs"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "31.038µs"} - logger.go:146: 2026-07-02T18:54:26.712Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "30.888µs"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "39.293µs"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "36.278µs"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "34.945µs"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "34.785µs"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "31.509µs"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.713Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "37.139µs"} - logger.go:146: 2026-07-02T18:54:26.714Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.714Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "37.78µs"} - logger.go:146: 2026-07-02T18:54:26.714Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.715Z INFO starting ingot S3 listener {"addr": "127.0.0.1:40379", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-2351865394", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:26.715Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.476349ms"} - logger.go:146: 2026-07-02T18:54:26.716Z INFO started -=== RUN TestSmoke_HeadBucket/non_existing_bucket -RUN HeadBucket_non_existing_bucket -PASS HeadBucket_non_existing_bucket -=== RUN TestSmoke_HeadBucket/success -RUN HeadBucket_success -PASS HeadBucket_success -=== NAME TestSmoke_HeadBucket - logger.go:146: 2026-07-02T18:54:26.727Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.727Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:26.728Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:26.728Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "119.621µs"} ---- PASS: TestSmoke_HeadBucket (0.02s) - --- PASS: TestSmoke_HeadBucket/non_existing_bucket (0.01s) - --- PASS: TestSmoke_HeadBucket/success (0.01s) -=== RUN TestSmoke_CreateBucket - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.731Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:62)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:26.731Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "17.903µs"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "16.912µs"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "17.001µs"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "58.148µs"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "28.503µs"} - logger.go:146: 2026-07-02T18:54:26.732Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "28.513µs"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "29.505µs"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "35.857µs"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "30.657µs"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "35.085µs"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "29.855µs"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "28.714µs"} - logger.go:146: 2026-07-02T18:54:26.733Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.734Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "32.801µs"} - logger.go:146: 2026-07-02T18:54:26.734Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.734Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "32.04µs"} - logger.go:146: 2026-07-02T18:54:26.734Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.735Z INFO starting ingot S3 listener {"addr": "127.0.0.1:42191", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-2652428260", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:26.735Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.478883ms"} - logger.go:146: 2026-07-02T18:54:26.736Z INFO started -=== RUN TestSmoke_CreateBucket/invalid_bucket_name -RUN CreateBucket_invalid_bucket_name -PASS CreateBucket_invalid_bucket_name -=== RUN TestSmoke_CreateBucket/invalid_canned_acl -RUN CreateBucket_invalid_canned_acl -PASS CreateBucket_invalid_canned_acl -=== RUN TestSmoke_CreateBucket/invalid_location_constraint -RUN CreateBucket_invalid_location_constraint -PASS CreateBucket_invalid_location_constraint -=== RUN TestSmoke_CreateBucket/invalid_ownership -RUN CreateBucket_invalid_ownership -PASS CreateBucket_invalid_ownership -=== RUN TestSmoke_CreateBucket/ownership_with_acl -RUN CreateBucket_ownership_with_acl -PASS CreateBucket_ownership_with_acl -=== RUN TestSmoke_CreateBucket/success -RUN CreateBucket_success -PASS CreateBucket_success -=== NAME TestSmoke_CreateBucket - logger.go:146: 2026-07-02T18:54:26.754Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.754Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:26.754Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:26.754Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "118.31µs"} ---- PASS: TestSmoke_CreateBucket (0.03s) - --- PASS: TestSmoke_CreateBucket/invalid_bucket_name (0.00s) - --- PASS: TestSmoke_CreateBucket/invalid_canned_acl (0.00s) - --- PASS: TestSmoke_CreateBucket/invalid_location_constraint (0.00s) - --- PASS: TestSmoke_CreateBucket/invalid_ownership (0.00s) - --- PASS: TestSmoke_CreateBucket/ownership_with_acl (0.00s) - --- PASS: TestSmoke_CreateBucket/success (0.00s) -=== RUN TestMultipart_RoundTrip - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.757Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:26.757Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestMultipart_RoundTrip (/home/runner/work/ingot/ingot/testing/multipart_test.go:26)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "16.34µs"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "18.053µs"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "16.982µs"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "60.994µs"} - logger.go:146: 2026-07-02T18:54:26.758Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "32.831µs"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "31.248µs"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "31.478µs"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "42.068µs"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "38.973µs"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "35.676µs"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.759Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "35.756µs"} - logger.go:146: 2026-07-02T18:54:26.760Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.760Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "33.192µs"} - logger.go:146: 2026-07-02T18:54:26.760Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.760Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "37.49µs"} - logger.go:146: 2026-07-02T18:54:26.760Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.760Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "39.704µs"} - logger.go:146: 2026-07-02T18:54:26.760Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.762Z INFO starting ingot S3 listener {"addr": "127.0.0.1:43371", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-1927865392", "max_blob_size": 32768} - logger.go:146: 2026-07-02T18:54:26.762Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.486687ms"} - logger.go:146: 2026-07-02T18:54:26.762Z INFO started -SDK 2026/07/02 18:54:26 WARN Response has no supported checksum. Not validating response payload. - logger.go:146: 2026-07-02T18:54:26.782Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.782Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:26.782Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:26.783Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "588.132µs"} ---- PASS: TestMultipart_RoundTrip (0.03s) -=== RUN TestSmoke_PutObject - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.786Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:26.786Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:150)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "14.968µs"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "40.977µs"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "16.2µs"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.787Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "70.03µs"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "28.804µs"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "26.8µs"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "25.447µs"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "34.825µs"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "30.777µs"} - logger.go:146: 2026-07-02T18:54:26.788Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.789Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "39.663µs"} - logger.go:146: 2026-07-02T18:54:26.789Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.789Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "31.529µs"} - logger.go:146: 2026-07-02T18:54:26.789Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.789Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "27.702µs"} - logger.go:146: 2026-07-02T18:54:26.789Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.790Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "31.749µs"} - logger.go:146: 2026-07-02T18:54:26.790Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:26.790Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "33.262µs"} - logger.go:146: 2026-07-02T18:54:26.790Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:26.791Z INFO starting ingot S3 listener {"addr": "127.0.0.1:41745", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-967706686", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:26.792Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.441474ms"} - logger.go:146: 2026-07-02T18:54:26.792Z INFO started -=== RUN TestSmoke_PutObject/checksum_algorithm_and_header_mismatch -RUN PutObject_checksum_algorithm_and_header_mismatch -PASS PutObject_checksum_algorithm_and_header_mismatch -=== RUN TestSmoke_PutObject/checksums_success -RUN PutObject_checksums_success -PASS PutObject_checksums_success -=== RUN TestSmoke_PutObject/conditional_writes -RUN PutObject_conditional_writes -PASS PutObject_conditional_writes -=== RUN TestSmoke_PutObject/default_checksum -RUN PutObject_default_checksum -PASS PutObject_default_checksum -=== RUN TestSmoke_PutObject/default_content_type -RUN PutObject_default_content_type -PASS PutObject_default_content_type -=== RUN TestSmoke_PutObject/dir_object_default_checksum -RUN PutObject_dir_object_default_checksum -PASS PutObject_dir_object_default_checksum -=== RUN TestSmoke_PutObject/dir_object_checksums_success -RUN PutObject_dir_object_checksums_success -PASS PutObject_dir_object_checksums_success -=== RUN TestSmoke_PutObject/incorrect_checksums -RUN PutObject_incorrect_checksums -PASS PutObject_incorrect_checksums -=== RUN TestSmoke_PutObject/invalid_credentials -RUN PutObject_invalid_credentials -PASS PutObject_invalid_credentials -=== RUN TestSmoke_PutObject/false_negative_object_names -RUN PutObject_false_negative_object_names -PASS PutObject_false_negative_object_names -=== RUN TestSmoke_PutObject/invalid_checksum_header -RUN PutObject_invalid_checksum_header -PASS PutObject_invalid_checksum_header -=== RUN TestSmoke_PutObject/invalid_legal_hold -RUN PutObject_invalid_legal_hold -PASS PutObject_invalid_legal_hold -=== RUN TestSmoke_PutObject/invalid_object_lock_mode -RUN PutObject_invalid_object_lock_mode -PASS PutObject_invalid_object_lock_mode -=== RUN TestSmoke_PutObject/invalid_object_names -RUN PutObject_invalid_object_names -PASS PutObject_invalid_object_names -=== RUN TestSmoke_PutObject/invalid_retain_until_date -RUN PutObject_invalid_retain_until_date -PASS PutObject_invalid_retain_until_date -=== RUN TestSmoke_PutObject/long_metadata -RUN PutObject_long_metadata -PASS PutObject_long_metadata -=== RUN TestSmoke_PutObject/missing_object_lock_retention_config -RUN PutObject_missing_object_lock_retention_config -PASS PutObject_missing_object_lock_retention_config -=== RUN TestSmoke_PutObject/multiple_checksum_headers -RUN PutObject_multiple_checksum_headers -PASS PutObject_multiple_checksum_headers -=== RUN TestSmoke_PutObject/non_existing_bucket -RUN PutObject_non_existing_bucket -PASS PutObject_non_existing_bucket -=== RUN TestSmoke_PutObject/past_retain_until_date -RUN PutObject_past_retain_until_date -PASS PutObject_past_retain_until_date -=== RUN TestSmoke_PutObject/racey_success -RUN PutObject_racey_success -PASS PutObject_racey_success -=== RUN TestSmoke_PutObject/special_chars -RUN PutObject_special_chars -PASS PutObject_special_chars -=== RUN TestSmoke_PutObject/success -RUN PutObject_success -PASS PutObject_success -=== RUN TestSmoke_PutObject/should_combine_metadata -RUN PutObject_should_combine_metadata -PASS PutObject_should_combine_metadata -=== RUN TestSmoke_PutObject/with_metadata -RUN PutObject_with_metadata -PASS PutObject_with_metadata -=== NAME TestSmoke_PutObject - logger.go:146: 2026-07-02T18:54:27.259Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:27.259Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:27.259Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:27.263Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "4.15206ms"} ---- PASS: TestSmoke_PutObject (0.48s) - --- PASS: TestSmoke_PutObject/checksum_algorithm_and_header_mismatch (0.01s) - --- PASS: TestSmoke_PutObject/checksums_success (0.02s) - --- PASS: TestSmoke_PutObject/conditional_writes (0.05s) - --- PASS: TestSmoke_PutObject/default_checksum (0.01s) - --- PASS: TestSmoke_PutObject/default_content_type (0.01s) - --- PASS: TestSmoke_PutObject/dir_object_default_checksum (0.01s) - --- PASS: TestSmoke_PutObject/dir_object_checksums_success (0.02s) - --- PASS: TestSmoke_PutObject/incorrect_checksums (0.02s) - --- PASS: TestSmoke_PutObject/invalid_credentials (0.01s) - --- PASS: TestSmoke_PutObject/false_negative_object_names (0.05s) - --- PASS: TestSmoke_PutObject/invalid_checksum_header (0.02s) - --- PASS: TestSmoke_PutObject/invalid_legal_hold (0.00s) - --- PASS: TestSmoke_PutObject/invalid_object_lock_mode (0.01s) - --- PASS: TestSmoke_PutObject/invalid_object_names (0.02s) - --- PASS: TestSmoke_PutObject/invalid_retain_until_date (0.00s) - --- PASS: TestSmoke_PutObject/long_metadata (0.01s) - --- PASS: TestSmoke_PutObject/missing_object_lock_retention_config (0.01s) - --- PASS: TestSmoke_PutObject/multiple_checksum_headers (0.01s) - --- PASS: TestSmoke_PutObject/non_existing_bucket (0.01s) - --- PASS: TestSmoke_PutObject/past_retain_until_date (0.01s) - --- PASS: TestSmoke_PutObject/racey_success (0.02s) - --- PASS: TestSmoke_PutObject/special_chars (0.12s) - --- PASS: TestSmoke_PutObject/success (0.01s) - --- PASS: TestSmoke_PutObject/should_combine_metadata (0.01s) - --- PASS: TestSmoke_PutObject/with_metadata (0.03s) -=== RUN TestSmoke_CopyObject - logger.go:146: 2026-07-02T18:54:27.267Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:27.267Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:27.267Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:27.268Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:393)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "10.299µs"} - logger.go:146: 2026-07-02T18:54:27.268Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "10.69µs"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "9.407µs"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "50.064µs"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "18.595µs"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "17.934µs"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "17.052µs"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "27.331µs"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "20.728µs"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.269Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "21.42µs"} - logger.go:146: 2026-07-02T18:54:27.270Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.270Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "19.667µs"} - logger.go:146: 2026-07-02T18:54:27.270Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.270Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "19.175µs"} - logger.go:146: 2026-07-02T18:54:27.270Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.270Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "20.659µs"} - logger.go:146: 2026-07-02T18:54:27.270Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:27.270Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "22.361µs"} - logger.go:146: 2026-07-02T18:54:27.270Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:27.272Z INFO starting ingot S3 listener {"addr": "127.0.0.1:45113", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-1606300004", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:27.272Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.496636ms"} - logger.go:146: 2026-07-02T18:54:27.272Z INFO started -=== RUN TestSmoke_CopyObject/success -RUN CopyObject_success -PASS CopyObject_success -=== RUN TestSmoke_CopyObject/copy_to_itself -RUN CopyObject_copy_to_itself -PASS CopyObject_copy_to_itself -=== RUN TestSmoke_CopyObject/copy_to_itself_invalid_directive -RUN CopyObject_copy_to_itself_invalid_directive -PASS CopyObject_copy_to_itself_invalid_directive -=== RUN TestSmoke_CopyObject/copy_source_starting_with_slash -RUN CopyObject_CopySource_starting_with_slash -PASS CopyObject_CopySource_starting_with_slash -=== RUN TestSmoke_CopyObject/default_content_type_with_replace_metadata -RUN CopyObject_default_content_type_with_replace_metadata -PASS CopyObject_default_content_type_with_replace_metadata -=== RUN TestSmoke_CopyObject/invalid_copy_source -RUN CopyObject_invalid_copy_source -PASS CopyObject_invalid_copy_source -=== RUN TestSmoke_CopyObject/non_existing_dst_bucket -RUN CopyObject_non_existing_dst_bucket -PASS CopyObject_non_existing_dst_bucket -=== RUN TestSmoke_CopyObject/non_existing_dir_object -RUN CopyObject_non_existing_dir_object -PASS CopyObject_non_existing_dir_object -=== RUN TestSmoke_CopyObject/with_metadata -RUN CopyObject_with_metadata -PASS CopyObject_with_metadata -=== RUN TestSmoke_CopyObject/to_itself_with_new_metadata -RUN CopyObject_to_itself_with_new_metadata -PASS CopyObject_to_itself_with_new_metadata -=== RUN TestSmoke_CopyObject/conditional_reads -RUN CopyObject_conditional_reads -PASS CopyObject_conditional_reads -=== RUN TestSmoke_CopyObject/with_special_characters -RUN CopyObject_with_special_characters -PASS CopyObject_with_special_characters -=== RUN TestSmoke_CopyObject/long_metadata -RUN CopyObject_long_metadata -PASS CopyObject_long_metadata -=== RUN TestSmoke_CopyObject/should_copy_meta_props -RUN CopyObject_should_copy_meta_props -PASS CopyObject_should_copy_meta_props -=== RUN TestSmoke_CopyObject/should_replace_meta_props -RUN CopyObject_should_replace_meta_props -PASS CopyObject_should_replace_meta_props -=== NAME TestSmoke_CopyObject - logger.go:146: 2026-07-02T18:54:28.834Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:28.834Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:28.834Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:28.836Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "2.180769ms"} ---- PASS: TestSmoke_CopyObject (1.57s) - --- PASS: TestSmoke_CopyObject/success (0.05s) - --- PASS: TestSmoke_CopyObject/copy_to_itself (0.01s) - --- PASS: TestSmoke_CopyObject/copy_to_itself_invalid_directive (0.01s) - --- PASS: TestSmoke_CopyObject/copy_source_starting_with_slash (0.05s) - --- PASS: TestSmoke_CopyObject/default_content_type_with_replace_metadata (0.01s) - --- PASS: TestSmoke_CopyObject/invalid_copy_source (0.03s) - --- PASS: TestSmoke_CopyObject/non_existing_dst_bucket (0.01s) - --- PASS: TestSmoke_CopyObject/non_existing_dir_object (0.03s) - --- PASS: TestSmoke_CopyObject/with_metadata (0.01s) - --- PASS: TestSmoke_CopyObject/to_itself_with_new_metadata (0.01s) - --- PASS: TestSmoke_CopyObject/conditional_reads (1.24s) - --- PASS: TestSmoke_CopyObject/with_special_characters (0.05s) - --- PASS: TestSmoke_CopyObject/long_metadata (0.01s) - --- PASS: TestSmoke_CopyObject/should_copy_meta_props (0.02s) - --- PASS: TestSmoke_CopyObject/should_replace_meta_props (0.01s) -=== RUN TestSmoke_ListBuckets - logger.go:146: 2026-07-02T18:54:28.840Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:28.840Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:28.840Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:28.840Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:28.840Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:28.840Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:28.840Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:28.840Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_ListBuckets (/home/runner/work/ingot/ingot/testing/smoke_test.go:95)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "13.906µs"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:28.841Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "16.861µs"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "10.429µs"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "65.872µs"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "29.504µs"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "44.022µs"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "19.937µs"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "35.166µs"} - logger.go:146: 2026-07-02T18:54:28.842Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "34.313µs"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "22.251µs"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "29.635µs"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "32.45µs"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "37.43µs"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.843Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "23.784µs"} - logger.go:146: 2026-07-02T18:54:28.844Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:28.847Z INFO starting ingot S3 listener {"addr": "127.0.0.1:35219", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-3855166526", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:28.847Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "3.594173ms"} - logger.go:146: 2026-07-02T18:54:28.847Z INFO started -=== RUN TestSmoke_ListBuckets/empty_success -RUN ListBuckets_empty_success -PASS ListBuckets_empty_success -=== RUN TestSmoke_ListBuckets/invalid_max_buckets -RUN ListBuckets_invalid_max_buckets -PASS ListBuckets_invalid_max_buckets -=== RUN TestSmoke_ListBuckets/success -RUN ListBuckets_success -PASS ListBuckets_success -=== RUN TestSmoke_ListBuckets/truncated -RUN ListBuckets_truncated -bucket names are not equal: test-bucket-100 != test-bucket-98 -FAIL ListBuckets_truncated: expected list buckets result to be test-bucket-98,test-bucket-99,test-bucket-100, instead got test-bucket-100,test-bucket-101,test-bucket-102 - smoke_test.go:99: expected list buckets result to be test-bucket-98,test-bucket-99,test-bucket-100, instead got test-bucket-100,test-bucket-101,test-bucket-102 -=== RUN TestSmoke_ListBuckets/with_prefix -RUN ListBuckets_with_prefix -PASS ListBuckets_with_prefix -=== NAME TestSmoke_ListBuckets - logger.go:146: 2026-07-02T18:54:28.917Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:28.917Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:28.917Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:28.917Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "133.99µs"} ---- FAIL: TestSmoke_ListBuckets (0.08s) - --- PASS: TestSmoke_ListBuckets/empty_success (0.00s) - --- PASS: TestSmoke_ListBuckets/invalid_max_buckets (0.01s) - --- PASS: TestSmoke_ListBuckets/success (0.03s) - --- FAIL: TestSmoke_ListBuckets/truncated (0.01s) - --- PASS: TestSmoke_ListBuckets/with_prefix (0.02s) -=== RUN TestSmokeXFail_CopyObject - logger.go:146: 2026-07-02T18:54:28.920Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:28.920Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:28.920Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:28.921Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CopyObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:425)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.921Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "10.009µs"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "12.634µs"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "10.079µs"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "59.15µs"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "18.995µs"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "18.494µs"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "19.507µs"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "23.764µs"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "21.64µs"} - logger.go:146: 2026-07-02T18:54:28.922Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.923Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "30.707µs"} - logger.go:146: 2026-07-02T18:54:28.923Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.923Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "35.376µs"} - logger.go:146: 2026-07-02T18:54:28.923Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.923Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "19.716µs"} - logger.go:146: 2026-07-02T18:54:28.923Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.923Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "187.479µs"} - logger.go:146: 2026-07-02T18:54:28.923Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.923Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "46.376µs"} - logger.go:146: 2026-07-02T18:54:28.924Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:28.926Z INFO starting ingot S3 listener {"addr": "127.0.0.1:41389", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-2274989017", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:28.926Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "2.050006ms"} - logger.go:146: 2026-07-02T18:54:28.926Z INFO started -=== RUN TestSmokeXFail_CopyObject/overwrite_same_file_object -RUN CopyObject_overwrite_same_file_object -FAIL CopyObject_overwrite_same_file_object: expected ObjectParentIsFile, instead got nil - smoke_test.go:433: known-failing: expected ObjectParentIsFile, instead got nil -=== NAME TestSmokeXFail_CopyObject - logger.go:146: 2026-07-02T18:54:28.939Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:28.939Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:28.939Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:28.939Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "673.078µs"} ---- PASS: TestSmokeXFail_CopyObject (0.02s) - --- SKIP: TestSmokeXFail_CopyObject/overwrite_same_file_object (0.01s) -=== RUN TestSmoke_DeleteObject - logger.go:146: 2026-07-02T18:54:28.942Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:28.942Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:28.942Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:28.942Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:28.942Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:243)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:28.943Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "12.854µs"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "19.917µs"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "10.74µs"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "59.38µs"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "27.872µs"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "31.278µs"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.944Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "29.125µs"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "25.728µs"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "33.052µs"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "35.116µs"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "21.82µs"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "26.78µs"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "34.333µs"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:28.945Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "34.584µs"} - logger.go:146: 2026-07-02T18:54:28.946Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:28.947Z INFO starting ingot S3 listener {"addr": "127.0.0.1:33311", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-1765402998", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:28.947Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.442857ms"} - logger.go:146: 2026-07-02T18:54:28.947Z INFO started -=== RUN TestSmoke_DeleteObject/conditional_writes -RUN DeleteObject_conditional_writes -PASS DeleteObject_conditional_writes -=== RUN TestSmoke_DeleteObject/directory_object -RUN DeleteObject_directory_object -PASS DeleteObject_directory_object -=== RUN TestSmoke_DeleteObject/directory_object_noslash -RUN DeleteObject_directory_object_noslash -PASS DeleteObject_directory_object_noslash -=== RUN TestSmoke_DeleteObject/expected_bucket_owner -RUN DeleteObject_expected_bucket_owner -PASS DeleteObject_expected_bucket_owner -=== RUN TestSmoke_DeleteObject/incorrect_expected_bucket_owner -RUN DeleteObject_incorrect_expected_bucket_owner -PASS DeleteObject_incorrect_expected_bucket_owner -=== RUN TestSmoke_DeleteObject/non_empty_dir_obj -RUN DeleteObject_non_empty_dir_obj -PASS DeleteObject_non_empty_dir_obj -=== RUN TestSmoke_DeleteObject/non_existing_dir_object -RUN DeleteObject_non_existing_dir_object -PASS DeleteObject_non_existing_dir_object -=== RUN TestSmoke_DeleteObject/non_existing_object -RUN DeleteObject_non_existing_object -PASS DeleteObject_non_existing_object -=== RUN TestSmoke_DeleteObject/success -RUN DeleteObject_success -PASS DeleteObject_success -=== RUN TestSmoke_DeleteObject/success_status_code -RUN DeleteObject_success_status_code -PASS DeleteObject_success_status_code -=== NAME TestSmoke_DeleteObject - logger.go:146: 2026-07-02T18:54:29.097Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.097Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:29.097Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:29.098Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.056848ms"} ---- PASS: TestSmoke_DeleteObject (0.16s) - --- PASS: TestSmoke_DeleteObject/conditional_writes (0.08s) - --- PASS: TestSmoke_DeleteObject/directory_object (0.01s) - --- PASS: TestSmoke_DeleteObject/directory_object_noslash (0.01s) - --- PASS: TestSmoke_DeleteObject/expected_bucket_owner (0.00s) - --- PASS: TestSmoke_DeleteObject/incorrect_expected_bucket_owner (0.01s) - --- PASS: TestSmoke_DeleteObject/non_empty_dir_obj (0.01s) - --- PASS: TestSmoke_DeleteObject/non_existing_dir_object (0.01s) - --- PASS: TestSmoke_DeleteObject/non_existing_object (0.00s) - --- PASS: TestSmoke_DeleteObject/success (0.01s) - --- PASS: TestSmoke_DeleteObject/success_status_code (0.01s) -=== RUN TestSmoke_DeleteBucket - logger.go:146: 2026-07-02T18:54:29.101Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:29.101Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:29.101Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:29.101Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.101Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.101Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:29.101Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:29.101Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:112)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.102Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "16.51µs"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "20.838µs"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "16.952µs"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "66.624µs"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "33.843µs"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "32.861µs"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.103Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "31.378µs"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "40.385µs"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "37.389µs"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "37.98µs"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "50.755µs"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "34.334µs"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "35.757µs"} - logger.go:146: 2026-07-02T18:54:29.104Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.105Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "38.572µs"} - logger.go:146: 2026-07-02T18:54:29.105Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.106Z INFO starting ingot S3 listener {"addr": "127.0.0.1:42583", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-1547585178", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:29.107Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.597394ms"} - logger.go:146: 2026-07-02T18:54:29.107Z INFO started -=== RUN TestSmoke_DeleteBucket/incorrect_expected_bucket_owner -RUN DeleteBucket_incorrect_expected_bucket_owner -PASS DeleteBucket_incorrect_expected_bucket_owner -=== RUN TestSmoke_DeleteBucket/non_empty_bucket -RUN DeleteBucket_non_empty_bucket -PASS DeleteBucket_non_empty_bucket -=== RUN TestSmoke_DeleteBucket/non_existing_bucket -RUN DeleteBucket_non_existing_bucket -PASS DeleteBucket_non_existing_bucket -=== RUN TestSmoke_DeleteBucket/success_status_code -RUN DeleteBucket_success_status_code -PASS DeleteBucket_success_status_code -=== NAME TestSmoke_DeleteBucket - logger.go:146: 2026-07-02T18:54:29.126Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.126Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:29.126Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:29.127Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "632.235µs"} ---- PASS: TestSmoke_DeleteBucket (0.03s) - --- PASS: TestSmoke_DeleteBucket/incorrect_expected_bucket_owner (0.01s) - --- PASS: TestSmoke_DeleteBucket/non_empty_bucket (0.01s) - --- PASS: TestSmoke_DeleteBucket/non_existing_bucket (0.00s) - --- PASS: TestSmoke_DeleteBucket/success_status_code (0.00s) -=== RUN TestRun_CapturesCaseNames -RUN CreateBucket_invalid_bucket_name -PASS CreateBucket_invalid_bucket_name -RUN CreateBucket_existing_bucket -FAIL CreateBucket_existing_bucket: fork/exec ./versitygw: no such file or directory -RUN CreateBucket_owned_by_you -FAIL CreateBucket_owned_by_you: expected error to be -BucketAlreadyOwnedByYouYour previous request to create the named bucket succeeded and you already own it.409, instead got operation error S3: CreateBucket, https response error StatusCode: 409, RequestID: , HostID: , BucketAlreadyExists: The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again. -RUN CreateBucket_invalid_ownership -PASS CreateBucket_invalid_ownership -RUN CreateBucket_ownership_with_acl -PASS CreateBucket_ownership_with_acl -RUN CreateBucket_as_user -FAIL CreateBucket_as_user: fork/exec ./versitygw: no such file or directory -RUN CreateBucket_default_acl -FAIL CreateBucket_default_acl: expected grants length to be 1, instead got 0 -RUN CreateBucket_non_default_acl -FAIL CreateBucket_non_default_acl: fork/exec ./versitygw: no such file or directory -RUN CreateBucket_private_canned_acl -FAIL CreateBucket_private_canned_acl: expected grants length to be 1, instead got 0 -RUN CreateBucket_private_canned_acl_bucket_owner_enforced_ownership -FAIL CreateBucket_private_canned_acl_bucket_owner_enforced_ownership: expected grants length to be 1, instead got 0 -RUN CreateBucket_success -PASS CreateBucket_success -RUN CreateBucket_default_object_lock -FAIL CreateBucket_default_object_lock: operation error S3: GetObjectLockConfiguration, https response error StatusCode: 404, RequestID: , HostID: , api error ObjectLockConfigurationNotFoundError: Object Lock configuration does not exist for this bucket. -RUN CreateBucket_invalid_location_constraint -PASS CreateBucket_invalid_location_constraint -RUN CreateBucket_long_tags -FAIL CreateBucket_long_tags: expected InvalidTag, instead got nil -RUN CreateBucket_invalid_tags -FAIL CreateBucket_invalid_tags: test 1 failed: expected err -InvalidTagThe TagKey you have provided is invalid400, instead got nil -RUN CreateBucket_duplicate_keys -FAIL CreateBucket_duplicate_keys: expected InvalidTag, instead got nil -RUN CreateBucket_tag_count_limit -FAIL CreateBucket_tag_count_limit: expected BadRequest, instead got nil -RUN CreateBucket_invalid_canned_acl -PASS CreateBucket_invalid_canned_acl ---- PASS: TestRun_CapturesCaseNames (0.05s) -=== RUN TestSmoke_DeleteObjects - logger.go:146: 2026-07-02T18:54:29.181Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.182Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_DeleteObjects (/home/runner/work/ingot/ingot/testing/smoke_test.go:409)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:29.182Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "10.679µs"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "10.299µs"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "8.936µs"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "57.347µs"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "18.004µs"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "50.063µs"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "19.246µs"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "37.79µs"} - logger.go:146: 2026-07-02T18:54:29.183Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "30.847µs"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "32.21µs"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "28.944µs"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "19.777µs"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "31.288µs"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.184Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "24.425µs"} - logger.go:146: 2026-07-02T18:54:29.185Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.186Z INFO starting ingot S3 listener {"addr": "127.0.0.1:38441", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-2941464538", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:29.186Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.599117ms"} - logger.go:146: 2026-07-02T18:54:29.186Z INFO started -=== RUN TestSmoke_DeleteObjects/success -RUN DeleteObjects_success -PASS DeleteObjects_success -=== RUN TestSmoke_DeleteObjects/empty_input -RUN DeleteObjects_empty_input -PASS DeleteObjects_empty_input -=== RUN TestSmoke_DeleteObjects/non_existing_objects -RUN DeleteObjects_empty_input -PASS DeleteObjects_empty_input -=== NAME TestSmoke_DeleteObjects - logger.go:146: 2026-07-02T18:54:29.251Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.251Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:29.251Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:29.252Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.744365ms"} ---- PASS: TestSmoke_DeleteObjects (0.07s) - --- PASS: TestSmoke_DeleteObjects/success (0.04s) - --- PASS: TestSmoke_DeleteObjects/empty_input (0.02s) - --- PASS: TestSmoke_DeleteObjects/non_existing_objects (0.01s) -=== RUN TestSmokeXFail_CreateBucket - logger.go:146: 2026-07-02T18:54:29.255Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.256Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:29.256Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_CreateBucket (/home/runner/work/ingot/ingot/testing/smoke_test.go:272)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "14.026µs"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "18.754µs"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "16.451µs"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "62.056µs"} - logger.go:146: 2026-07-02T18:54:29.257Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "32.17µs"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "33.192µs"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "30.347µs"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "39.895µs"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "37.2µs"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "37.41µs"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "33.753µs"} - logger.go:146: 2026-07-02T18:54:29.258Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.259Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "31.699µs"} - logger.go:146: 2026-07-02T18:54:29.259Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.259Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "36.888µs"} - logger.go:146: 2026-07-02T18:54:29.259Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.259Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "39.654µs"} - logger.go:146: 2026-07-02T18:54:29.259Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.261Z INFO starting ingot S3 listener {"addr": "127.0.0.1:32785", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-2850555512", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:29.261Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.487389ms"} - logger.go:146: 2026-07-02T18:54:29.261Z INFO started -=== RUN TestSmokeXFail_CreateBucket/as_user -RUN CreateBucket_as_user -FAIL CreateBucket_as_user: fork/exec ./versitygw: no such file or directory - smoke_test.go:280: known-failing: CreateBucket_as_user: fork/exec ./versitygw: no such file or directory -=== RUN TestSmokeXFail_CreateBucket/default_acl -RUN CreateBucket_default_acl -FAIL CreateBucket_default_acl: expected grants length to be 1, instead got 0 - smoke_test.go:280: known-failing: expected grants length to be 1, instead got 0 -=== RUN TestSmokeXFail_CreateBucket/default_object_lock -RUN CreateBucket_default_object_lock -FAIL CreateBucket_default_object_lock: operation error S3: GetObjectLockConfiguration, https response error StatusCode: 404, RequestID: , HostID: , api error ObjectLockConfigurationNotFoundError: Object Lock configuration does not exist for this bucket. - smoke_test.go:280: known-failing: CreateBucket_default_object_lock: operation error S3: GetObjectLockConfiguration, https response error StatusCode: 404, RequestID: , HostID: , api error ObjectLockConfigurationNotFoundError: Object Lock configuration does not exist for this bucket. -=== RUN TestSmokeXFail_CreateBucket/duplicate_keys -RUN CreateBucket_duplicate_keys -FAIL CreateBucket_duplicate_keys: expected InvalidTag, instead got nil - smoke_test.go:280: known-failing: expected InvalidTag, instead got nil -=== RUN TestSmokeXFail_CreateBucket/existing_bucket -RUN CreateBucket_existing_bucket -FAIL CreateBucket_existing_bucket: fork/exec ./versitygw: no such file or directory - smoke_test.go:280: known-failing: CreateBucket_existing_bucket: fork/exec ./versitygw: no such file or directory -=== RUN TestSmokeXFail_CreateBucket/invalid_tags -RUN CreateBucket_invalid_tags -FAIL CreateBucket_invalid_tags: test 1 failed: expected err -InvalidTagThe TagKey you have provided is invalid400, instead got nil - smoke_test.go:280: known-failing: test 1 failed: expected err - InvalidTagThe TagKey you have provided is invalid400, instead got nil -=== RUN TestSmokeXFail_CreateBucket/long_tags -RUN CreateBucket_long_tags -FAIL CreateBucket_long_tags: expected InvalidTag, instead got nil - smoke_test.go:280: known-failing: expected InvalidTag, instead got nil -=== RUN TestSmokeXFail_CreateBucket/non_default_acl -RUN CreateBucket_non_default_acl -FAIL CreateBucket_non_default_acl: fork/exec ./versitygw: no such file or directory - smoke_test.go:280: known-failing: CreateBucket_non_default_acl: fork/exec ./versitygw: no such file or directory -=== RUN TestSmokeXFail_CreateBucket/owned_by_you -RUN CreateBucket_owned_by_you -FAIL CreateBucket_owned_by_you: expected error to be -BucketAlreadyOwnedByYouYour previous request to create the named bucket succeeded and you already own it.409, instead got operation error S3: CreateBucket, https response error StatusCode: 409, RequestID: , HostID: , BucketAlreadyExists: The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again. - smoke_test.go:280: known-failing: expected error to be - BucketAlreadyOwnedByYouYour previous request to create the named bucket succeeded and you already own it.409, instead got operation error S3: CreateBucket, https response error StatusCode: 409, RequestID: , HostID: , BucketAlreadyExists: The requested bucket name is not available. The bucket name can not be an existing collection, and the bucket namespace is shared by all users of the system. Please select a different name and try again. -=== RUN TestSmokeXFail_CreateBucket/private_canned_acl -RUN CreateBucket_private_canned_acl -FAIL CreateBucket_private_canned_acl: expected grants length to be 1, instead got 0 - smoke_test.go:280: known-failing: expected grants length to be 1, instead got 0 -=== RUN TestSmokeXFail_CreateBucket/private_canned_acl_bucket_owner_enforced_ownership -RUN CreateBucket_private_canned_acl_bucket_owner_enforced_ownership -FAIL CreateBucket_private_canned_acl_bucket_owner_enforced_ownership: expected grants length to be 1, instead got 0 - smoke_test.go:280: known-failing: expected grants length to be 1, instead got 0 -=== RUN TestSmokeXFail_CreateBucket/tag_count_limit -RUN CreateBucket_tag_count_limit -FAIL CreateBucket_tag_count_limit: expected BadRequest, instead got nil - smoke_test.go:280: known-failing: expected BadRequest, instead got nil -=== NAME TestSmokeXFail_CreateBucket - logger.go:146: 2026-07-02T18:54:29.292Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.292Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:29.292Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:29.293Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "169.327µs"} ---- PASS: TestSmokeXFail_CreateBucket (0.04s) - --- SKIP: TestSmokeXFail_CreateBucket/as_user (0.00s) - --- SKIP: TestSmokeXFail_CreateBucket/default_acl (0.01s) - --- SKIP: TestSmokeXFail_CreateBucket/default_object_lock (0.00s) - --- SKIP: TestSmokeXFail_CreateBucket/duplicate_keys (0.00s) - --- SKIP: TestSmokeXFail_CreateBucket/existing_bucket (0.00s) - --- SKIP: TestSmokeXFail_CreateBucket/invalid_tags (0.00s) - --- SKIP: TestSmokeXFail_CreateBucket/long_tags (0.00s) - --- SKIP: TestSmokeXFail_CreateBucket/non_default_acl (0.00s) - --- SKIP: TestSmokeXFail_CreateBucket/owned_by_you (0.01s) - --- SKIP: TestSmokeXFail_CreateBucket/private_canned_acl (0.00s) - --- SKIP: TestSmokeXFail_CreateBucket/private_canned_acl_bucket_owner_enforced_ownership (0.00s) - --- SKIP: TestSmokeXFail_CreateBucket/tag_count_limit (0.00s) -=== RUN TestHarnessLifecycle - logger.go:146: 2026-07-02T18:54:29.295Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:29.295Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:29.295Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:29.295Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.295Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.295Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestHarnessLifecycle (/home/runner/work/ingot/ingot/testing/harness_test.go:19)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "10.019µs"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "10.7µs"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "9.488µs"} - logger.go:146: 2026-07-02T18:54:29.296Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "46.597µs"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "17.983µs"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "17.883µs"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "16.53µs"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "23.434µs"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "44.843µs"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "21.7µs"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "20.939µs"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "18.184µs"} - logger.go:146: 2026-07-02T18:54:29.297Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.298Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "21.24µs"} - logger.go:146: 2026-07-02T18:54:29.298Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.298Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "21.981µs"} - logger.go:146: 2026-07-02T18:54:29.298Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.299Z INFO starting ingot S3 listener {"addr": "127.0.0.1:46705", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-2540827856", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:29.299Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.481849ms"} - logger.go:146: 2026-07-02T18:54:29.300Z INFO started - logger.go:146: 2026-07-02T18:54:29.300Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.300Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:29.301Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:29.301Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "123.264µs"} ---- PASS: TestHarnessLifecycle (0.01s) -=== RUN TestSmokeXFail_HeadObject - logger.go:146: 2026-07-02T18:54:29.303Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:29.303Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.304Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_HeadObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:347)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:29.304Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "12.363µs"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "16.511µs"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "10.219µs"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "62.786µs"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "31.398µs"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "26.209µs"} - logger.go:146: 2026-07-02T18:54:29.305Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "29.054µs"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "33.092µs"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "23.133µs"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "21.8µs"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "29.435µs"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "31.038µs"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "21.801µs"} - logger.go:146: 2026-07-02T18:54:29.306Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.307Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "33.132µs"} - logger.go:146: 2026-07-02T18:54:29.307Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.308Z INFO starting ingot S3 listener {"addr": "127.0.0.1:38101", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-3636580989", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:29.308Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.453416ms"} - logger.go:146: 2026-07-02T18:54:29.308Z INFO started -=== RUN TestSmokeXFail_HeadObject/overrides_fail_public -RUN HeadObject_overrides_fail_public -FAIL HeadObject_overrides_fail_public: operation error S3: PutBucketPolicy, https response error StatusCode: 501, RequestID: , HostID: , api error NotImplemented: A header you provided implies functionality that is not implemented. - smoke_test.go:355: known-failing: operation error S3: PutBucketPolicy, https response error StatusCode: 501, RequestID: , HostID: , api error NotImplemented: A header you provided implies functionality that is not implemented. -=== RUN TestSmokeXFail_HeadObject/success -RUN HeadObject_success -FAIL HeadObject_success: expected the tagcount to be 1, instead got 0 - smoke_test.go:355: known-failing: expected the tagcount to be 1, instead got 0 -=== NAME TestSmokeXFail_HeadObject - logger.go:146: 2026-07-02T18:54:29.339Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.339Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:29.339Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:29.339Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "641.462µs"} ---- PASS: TestSmokeXFail_HeadObject (0.04s) - --- SKIP: TestSmokeXFail_HeadObject/overrides_fail_public (0.01s) - --- SKIP: TestSmokeXFail_HeadObject/success (0.02s) -=== RUN TestSmoke_GetObject - logger.go:146: 2026-07-02T18:54:29.342Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:29.342Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:29.342Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.343Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmoke_GetObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:186)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "9.959µs"} - logger.go:146: 2026-07-02T18:54:29.343Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "10.78µs"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "9.107µs"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "51.806µs"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "18.795µs"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "17.943µs"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "44.583µs"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "25.016µs"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "20.268µs"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "19.646µs"} - logger.go:146: 2026-07-02T18:54:29.344Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.345Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "19.376µs"} - logger.go:146: 2026-07-02T18:54:29.345Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.345Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "17.963µs"} - logger.go:146: 2026-07-02T18:54:29.345Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.345Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "20.999µs"} - logger.go:146: 2026-07-02T18:54:29.345Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:29.345Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "22.402µs"} - logger.go:146: 2026-07-02T18:54:29.345Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:29.347Z INFO starting ingot S3 listener {"addr": "127.0.0.1:41863", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-968508425", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:29.347Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.531922ms"} - logger.go:146: 2026-07-02T18:54:29.347Z INFO started -=== RUN TestSmoke_GetObject/by_range_resp_status -RUN GetObject_by_range_resp_status -PASS GetObject_by_range_resp_status -=== RUN TestSmoke_GetObject/checksums -RUN GetObject_checksums -PASS GetObject_checksums -=== RUN TestSmoke_GetObject/conditional_reads -RUN GetObject_conditional_reads -PASS GetObject_conditional_reads -=== RUN TestSmoke_GetObject/dir_object_checksum -RUN GetObject_dir_object_checksum -PASS GetObject_dir_object_checksum -=== RUN TestSmoke_GetObject/dir_with_range -RUN GetObject_dir_with_range -PASS GetObject_dir_with_range -=== RUN TestSmoke_GetObject/directory_object_noslash -RUN GetObject_directory_object_noslash -PASS GetObject_directory_object_noslash -=== RUN TestSmoke_GetObject/empty_object_part_number_1 -RUN GetObject_empty_object_part_number_1 -PASS GetObject_empty_object_part_number_1 -=== RUN TestSmoke_GetObject/invalid_parent -RUN GetObject_invalid_parent -PASS GetObject_invalid_parent -=== RUN TestSmoke_GetObject/invalid_part_number -RUN GetObject_invalid_part_number -PASS GetObject_invalid_part_number -=== RUN TestSmoke_GetObject/mp_part_number_exceeds_parts_count -RUN GetObject_mp_part_number_exceeds_parts_count -PASS GetObject_mp_part_number_exceeds_parts_count -=== RUN TestSmoke_GetObject/mp_part_number_resp_status -RUN GetObject_mp_part_number_resp_status -PASS GetObject_mp_part_number_resp_status -=== RUN TestSmoke_GetObject/mp_part_number_success -RUN GetObject_mp_part_number_success -PASS GetObject_mp_part_number_success -=== RUN TestSmoke_GetObject/non_mp_part_number_1_success -RUN GetObject_non_mp_part_number_1_success -PASS GetObject_non_mp_part_number_1_success -=== RUN TestSmoke_GetObject/large_object -RUN GetObject_large_object -PASS GetObject_large_object -=== RUN TestSmoke_GetObject/non_existing_dir_object -RUN GetObject_non_existing_dir_object -PASS GetObject_non_existing_dir_object -=== RUN TestSmoke_GetObject/non_existing_key -RUN GetObject_non_existing_key -PASS GetObject_non_existing_key -=== RUN TestSmoke_GetObject/not_enabled_checksum_mode -RUN GetObject_not_enabled_checksum_mode -PASS GetObject_not_enabled_checksum_mode -=== RUN TestSmoke_GetObject/overrides_presign_success -RUN GetObject_overrides_presign_success -PASS GetObject_overrides_presign_success -=== RUN TestSmoke_GetObject/overrides_success -RUN GetObject_overrides_success -PASS GetObject_overrides_success -=== RUN TestSmoke_GetObject/range_and_part_number -RUN GetObject_range_and_part_number -PASS GetObject_range_and_part_number -=== RUN TestSmoke_GetObject/ranged_with_checksum_mode -RUN GetObject_ranged_with_checksum_mode -PASS GetObject_ranged_with_checksum_mode -=== RUN TestSmoke_GetObject/with_range -RUN GetObject_with_range -PASS GetObject_with_range -=== RUN TestSmoke_GetObject/zero_len_with_range -RUN GetObject_zero_len_with_range -PASS GetObject_zero_len_with_range -=== NAME TestSmoke_GetObject - logger.go:146: 2026-07-02T18:54:34.362Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:34.362Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:34.362Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:44.362Z ERROR OnStop hook failed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "error": "ingot shutdown: [s3api shutdown: context deadline exceeded]"} - logger.go:146: 2026-07-02T18:54:44.362Z ERROR stop failed {"error": "ingot shutdown: [s3api shutdown: context deadline exceeded]"} ---- PASS: TestSmoke_GetObject (15.04s) - --- PASS: TestSmoke_GetObject/by_range_resp_status (0.04s) - --- PASS: TestSmoke_GetObject/checksums (0.03s) - --- PASS: TestSmoke_GetObject/conditional_reads (1.17s) - --- PASS: TestSmoke_GetObject/dir_object_checksum (0.03s) - --- PASS: TestSmoke_GetObject/dir_with_range (0.03s) - --- PASS: TestSmoke_GetObject/directory_object_noslash (0.01s) - --- PASS: TestSmoke_GetObject/empty_object_part_number_1 (0.01s) - --- PASS: TestSmoke_GetObject/invalid_parent (0.02s) - --- PASS: TestSmoke_GetObject/invalid_part_number (0.01s) - --- PASS: TestSmoke_GetObject/mp_part_number_exceeds_parts_count (0.28s) - --- PASS: TestSmoke_GetObject/mp_part_number_resp_status (0.13s) - --- PASS: TestSmoke_GetObject/mp_part_number_success (0.50s) - --- PASS: TestSmoke_GetObject/non_mp_part_number_1_success (0.01s) - --- PASS: TestSmoke_GetObject/large_object (2.47s) - --- PASS: TestSmoke_GetObject/non_existing_dir_object (0.02s) - --- PASS: TestSmoke_GetObject/non_existing_key (0.01s) - --- PASS: TestSmoke_GetObject/not_enabled_checksum_mode (0.01s) - --- PASS: TestSmoke_GetObject/overrides_presign_success (0.02s) - --- PASS: TestSmoke_GetObject/overrides_success (0.02s) - --- PASS: TestSmoke_GetObject/range_and_part_number (0.01s) - --- PASS: TestSmoke_GetObject/ranged_with_checksum_mode (0.13s) - --- PASS: TestSmoke_GetObject/with_range (0.05s) - --- PASS: TestSmoke_GetObject/zero_len_with_range (0.03s) -=== RUN TestParseCaseLines ---- PASS: TestParseCaseLines (0.00s) -=== RUN TestSmokeXFail_PutObject - logger.go:146: 2026-07-02T18:54:44.385Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:44.385Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:44.385Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:44.385Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:44.385Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:44.385Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:44.385Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:44.385Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:44.385Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:44.385Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:44.385Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_PutObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:310)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "16.421µs"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "17.163µs"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "10.86µs"} - logger.go:146: 2026-07-02T18:54:44.386Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "61.214µs"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "32µs"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "30.176µs"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "18.163µs"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "38.852µs"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "35.796µs"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "21.63µs"} - logger.go:146: 2026-07-02T18:54:44.387Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.388Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "30.567µs"} - logger.go:146: 2026-07-02T18:54:44.388Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.388Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "31.128µs"} - logger.go:146: 2026-07-02T18:54:44.388Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.388Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "21.921µs"} - logger.go:146: 2026-07-02T18:54:44.388Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.388Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "35.587µs"} - logger.go:146: 2026-07-02T18:54:44.388Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:44.390Z INFO starting ingot S3 listener {"addr": "127.0.0.1:34727", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-2692461094", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:44.390Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.438097ms"} - logger.go:146: 2026-07-02T18:54:44.390Z INFO started -=== RUN TestSmokeXFail_PutObject/missing_bucket_lock -RUN PutObject_missing_bucket_lock -FAIL PutObject_missing_bucket_lock: expected InvalidRequest, instead got nil - smoke_test.go:318: known-failing: expected InvalidRequest, instead got nil -=== RUN TestSmokeXFail_PutObject/object_acl_not_supported -RUN PutObject_object_acl_not_supported -FAIL PutObject_object_acl_not_supported: fork/exec ./versitygw: no such file or directory - smoke_test.go:318: known-failing: fork/exec ./versitygw: no such file or directory -=== RUN TestSmokeXFail_PutObject/tagging -RUN PutObject_tagging -FAIL PutObject_tagging: test case 1 failed for file object: operation error S3: GetObjectTagging, https response error StatusCode: 501, RequestID: , HostID: , api error NotImplemented: A header you provided implies functionality that is not implemented. - smoke_test.go:318: known-failing: test case 1 failed for file object: operation error S3: GetObjectTagging, https response error StatusCode: 501, RequestID: , HostID: , api error NotImplemented: A header you provided implies functionality that is not implemented. -=== RUN TestSmokeXFail_PutObject/with_object_lock -RUN PutObject_with_object_lock -FAIL PutObject_with_object_lock: expected object lock mode to be COMPLIANCE, instead got - smoke_test.go:318: known-failing: expected object lock mode to be COMPLIANCE, instead got -=== NAME TestSmokeXFail_PutObject - logger.go:146: 2026-07-02T18:54:44.423Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:44.423Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:44.423Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:44.424Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "561.476µs"} ---- PASS: TestSmokeXFail_PutObject (0.04s) - --- SKIP: TestSmokeXFail_PutObject/missing_bucket_lock (0.01s) - --- SKIP: TestSmokeXFail_PutObject/object_acl_not_supported (0.00s) - --- SKIP: TestSmokeXFail_PutObject/tagging (0.01s) - --- SKIP: TestSmokeXFail_PutObject/with_object_lock (0.01s) -=== RUN TestSmokeXFail_DeleteObject - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:44.427Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:44.427Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:44.427Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.smokeHarness (/home/runner/work/ingot/ingot/testing/smoke_test.go:41)", "github.com/fil-forge/ingot/testing.TestSmokeXFail_DeleteObject (/home/runner/work/ingot/ingot/testing/smoke_test.go:362)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "14.046µs"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "19.597µs"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "11.261µs"} - logger.go:146: 2026-07-02T18:54:44.428Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "60.973µs"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "29.835µs"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "31.308µs"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "18.064µs"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "39.754µs"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "35.907µs"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.429Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "32.821µs"} - logger.go:146: 2026-07-02T18:54:44.430Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.430Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "24.436µs"} - logger.go:146: 2026-07-02T18:54:44.430Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.430Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "32.01µs"} - logger.go:146: 2026-07-02T18:54:44.430Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.430Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "37.069µs"} - logger.go:146: 2026-07-02T18:54:44.430Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.430Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "26.019µs"} - logger.go:146: 2026-07-02T18:54:44.430Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:44.432Z INFO starting ingot S3 listener {"addr": "127.0.0.1:32831", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-2056498819", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:44.432Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.453496ms"} - logger.go:146: 2026-07-02T18:54:44.432Z INFO started - logger.go:146: 2026-07-02T18:54:44.432Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:44.432Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:44.432Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:44.433Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "128.798µs"} ---- PASS: TestSmokeXFail_DeleteObject (0.01s) -=== RUN TestBlobSplit_ZeroByteObject - logger.go:146: 2026-07-02T18:54:44.435Z INFO provided {"constructor": "go.uber.org/fx.New.func1()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Lifecycle"} - logger.go:146: 2026-07-02T18:54:44.435Z INFO provided {"constructor": "go.uber.org/fx.(*App).shutdowner-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.Shutdowner"} - logger.go:146: 2026-07-02T18:54:44.435Z INFO provided {"constructor": "go.uber.org/fx.(*App).dotGraph-fm()", "stacktrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["go.uber.org/fx.New (/home/runner/go/pkg/mod/go.uber.org/fx@v1.24.0/app.go:481)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "fx.DotGraph"} - logger.go:146: 2026-07-02T18:54:44.435Z INFO supplied {"type": "*zap.Logger", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:146)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:44.435Z INFO supplied {"type": "ingot.ServerConfig", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:147)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"]} - logger.go:146: 2026-07-02T18:54:44.435Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.Registry"} - logger.go:146: 2026-07-02T18:54:44.435Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.IntentStore"} - logger.go:146: 2026-07-02T18:54:44.435Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.LocationStore"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.BlobRefStore"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.GCStore"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "registry.MultipartStore"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "logstore.Meta"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "blockstore.BlockReader"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.Uploader"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BodyUploader"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO provided {"constructor": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "stacktrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing_test.TestBlobSplit_ZeroByteObject (/home/runner/work/ingot/ingot/testing/blobsplit_test.go:141)", "testing.tRunner (/opt/hostedtoolcache/go/1.25.7/x64/src/testing/testing.go:1934)"], "moduletrace": ["github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:164)", "github.com/fil-forge/ingot/testing.StartHarness (/home/runner/work/ingot/ingot/testing/harness.go:142)"], "type": "uploader.BlobRemover"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO initialized custom fxevent.Logger {"function": "github.com/fil-forge/ingot/testing.StartHarness.func1()"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO invoking {"function": "github.com/fil-forge/ingot.registerServerLifecycle()", "module": "ingot-server"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO before run {"name": "go.uber.org/fx.New.func1()", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO run {"name": "go.uber.org/fx.New.func1()", "kind": "provide", "runtime": "10.139µs"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO before run {"name": "stub(ingot.ServerConfig)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO run {"name": "stub(ingot.ServerConfig)", "kind": "supply", "runtime": "10.059µs"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO before run {"name": "stub(*zap.Logger)", "kind": "supply"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO run {"name": "stub(*zap.Logger)", "kind": "supply", "runtime": "9.107µs"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.436Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func9(), fx.As([[blockstore.BlockReader]])", "kind": "provide", "runtime": "44.613µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func10(), fx.As([[uploader.Uploader]])", "kind": "provide", "runtime": "17.152µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func11(), fx.As([[uploader.BodyUploader]])", "kind": "provide", "runtime": "17.453µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func12(), fx.As([[uploader.BlobRemover]])", "kind": "provide", "runtime": "15.929µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func2(), fx.As([[registry.Registry]])", "kind": "provide", "runtime": "21.379µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func3(), fx.As([[registry.IntentStore]])", "kind": "provide", "runtime": "19.707µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func4(), fx.As([[registry.LocationStore]])", "kind": "provide", "runtime": "23.483µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func5(), fx.As([[registry.BlobRefStore]])", "kind": "provide", "runtime": "19.346µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func6(), fx.As([[registry.GCStore]])", "kind": "provide", "runtime": "17.823µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func7(), fx.As([[registry.MultipartStore]])", "kind": "provide", "runtime": "18.665µs"} - logger.go:146: 2026-07-02T18:54:44.437Z INFO before run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide"} - logger.go:146: 2026-07-02T18:54:44.438Z INFO run {"name": "fx.Annotate(github.com/fil-forge/ingot/testing.StartHarness.func8(), fx.As([[logstore.Meta]])", "kind": "provide", "runtime": "20.719µs"} - logger.go:146: 2026-07-02T18:54:44.438Z INFO OnStart hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:44.439Z INFO starting ingot S3 listener {"addr": "127.0.0.1:33613", "region": "us-east-1", "data_dir": "/tmp/ingot-harness-3461269692", "max_blob_size": 268435456} - logger.go:146: 2026-07-02T18:54:44.439Z INFO OnStart hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func1()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "1.488732ms"} - logger.go:146: 2026-07-02T18:54:44.439Z INFO started - logger.go:146: 2026-07-02T18:54:44.446Z INFO OnStop hook executing {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle"} - logger.go:146: 2026-07-02T18:54:44.447Z INFO shutting down ingot S3 listener - logger.go:146: 2026-07-02T18:54:44.447Z ERROR ingot listener error {"error": "listener closed"} - logger.go:146: 2026-07-02T18:54:44.447Z INFO OnStop hook executed {"callee": "github.com/fil-forge/ingot.registerServerLifecycle.func2()", "caller": "github.com/fil-forge/ingot.registerServerLifecycle", "runtime": "620.585µs"} ---- PASS: TestBlobSplit_ZeroByteObject (0.01s) -FAIL -FAIL github.com/fil-forge/ingot/testing 20.164s -? github.com/fil-forge/ingot/tokenstore [no test files] -? github.com/fil-forge/ingot/uploader [no test files] -FAIL -Error: Process completed with exit code 1. diff --git a/docs/architecture.md b/docs/architecture.md index a63db51..cf3d93b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -840,9 +840,11 @@ paths below are exercised against the real stack by the smelt-based `itest/` har than `multipart_session_ttl` (default 7d) and reaps terminal session rows. A successful Complete retains its session in state `completed` so a duplicate Complete is idempotent per S3. `DeleteBucket` implicitly aborts the bucket's in-flight sessions before the space - delete (upstream's conformance teardown never aborts them); its `/blob/abort` leg is gated - on hilt granting `blob.Abort` for the bucket-delete operation. The network-side `/blob/abort` - unwind remains a parking-flow concern (above). + delete (upstream's conformance teardown never aborts them); because `s3:DeleteBucket` + delegates no blob commands, the abort runs on the space authority captured at `UploadPart`, + and its `/blob/abort` leg is gated on hilt delegating `blob.Abort` with the write set + (fil-forge/hilt#36). The network-side `/blob/abort` unwind remains a parking-flow + concern (above). ### Known correctness boundary diff --git a/go.mod b/go.mod index e57668b..c75e513 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/fil-forge/hilt v0.0.1-0.20260724134448-ba71f843f6a4 github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 github.com/fil-forge/libforge v0.0.0-20260727220215-5e299c46f62f - github.com/fil-forge/smelt v0.0.0-20260720130429-63116166a06c + github.com/fil-forge/smelt v0.0.0-20260728210138-0a0154e6e6d0 github.com/fil-forge/ucantone v0.0.0-20260727203046-ccb77059de44 github.com/fil-forge/versitygw v0.0.0-20260716095011-7a65883d595a github.com/fxamacker/cbor/v2 v2.9.2 diff --git a/go.sum b/go.sum index b44ca59..24ce439 100644 --- a/go.sum +++ b/go.sum @@ -252,8 +252,8 @@ github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 h1:W github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717/go.mod h1:wFcakLohOqpMRkJzWRdGFGHRFpGB+PpEMCm9wkt2cqU= github.com/fil-forge/libforge v0.0.0-20260727220215-5e299c46f62f h1:QzgMg8GIE4IhgOE/7DBHFU/4T5oU7J4vAKxbUPKJPGA= github.com/fil-forge/libforge v0.0.0-20260727220215-5e299c46f62f/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= -github.com/fil-forge/smelt v0.0.0-20260720130429-63116166a06c h1:WHvsleEU6ZiNYDFgLx6KtorXulD+IuLiorMgmp4Th8s= -github.com/fil-forge/smelt v0.0.0-20260720130429-63116166a06c/go.mod h1:NM/mk/XiP1Kzsy9HWGQeLsosPUvVY3bIrkUzqswThpU= +github.com/fil-forge/smelt v0.0.0-20260728210138-0a0154e6e6d0 h1:QmwxgO2W6bL6B9Ce2E2nIvxc8EyIdS5hkrDvSlsTQE0= +github.com/fil-forge/smelt v0.0.0-20260728210138-0a0154e6e6d0/go.mod h1:czwgD0nnuQ8h3GOHdwQ2Guw3C9R3zYP3K/mE+DEyh+Y= github.com/fil-forge/ucantone v0.0.0-20260727203046-ccb77059de44 h1:ofvb2Qq7++VPRelGsLbtnd1ZMKVT4n4QGoa79BhJ6VQ= github.com/fil-forge/ucantone v0.0.0-20260727203046-ccb77059de44/go.mod h1:oFY5BfD0bDeodGlbBHh3/nK99MAS93rGXjoQz7s5qgE= github.com/fil-forge/versitygw v0.0.0-20260716095011-7a65883d595a h1:lDwnNmF4LNbevx/YYNCC0CazMiL4eIe38MvfqbRR3RA= diff --git a/internal/reqscope/reqscope.go b/internal/reqscope/reqscope.go index 804ba28..c5ea6b1 100644 --- a/internal/reqscope/reqscope.go +++ b/internal/reqscope/reqscope.go @@ -37,6 +37,16 @@ func ProofStore(ctx context.Context) (ucanlib.ProofStore, bool) { return ps, ok } +// WithoutProofStore returns a context whose request-scoped proof store is +// masked. Hilt delegates Forge commands per S3 permission, and some +// permissions (s3:DeleteBucket) carry no blob commands at all — a flow that +// must invoke blob capabilities from such a request (DeleteBucket's implicit +// abort of in-flight multipart uploads) hides the request store so the +// uploader falls back to the space authority captured at UploadPart. +func WithoutProofStore(ctx context.Context) context.Context { + return context.WithValue(ctx, proofStoreKey, nil) +} + type requestContextKey struct{} var requestKey any = requestContextKey{} diff --git a/itest/forge_delete_test.go b/itest/forge_delete_test.go index d928adb..53cdc9e 100644 --- a/itest/forge_delete_test.go +++ b/itest/forge_delete_test.go @@ -4,7 +4,6 @@ package itest import ( "context" - "os" "strings" "testing" "time" @@ -20,38 +19,14 @@ import ( // piri /blob/release (claim release; deferred physical deletion once the // PDP aggregate root retires on-chain). // -// The published piri/sprue images predate the removal handlers, so this -// test injects working-tree builds of both and skips when they aren't -// provided: -// -// (cd ../../piri && CGO_ENABLED=0 GOOS=linux go build -o /tmp/piri ./cmd) -// (cd ../../sprue && CGO_ENABLED=0 GOOS=linux go build -o /tmp/sprue ./cmd/main.go) -// ITEST_PIRI_BIN=/tmp/piri ITEST_SPRUE_BIN=/tmp/sprue \ -// go test -tags itest ./itest -run TestForgeDeleteReleasesNetworkBlob -v -timeout 900s -// -// Once images with the handlers publish, drop the env gate and the binary -// injection. +// Runs on the stock smelt-SDK images like every other itest. It needs +// piri:main ≥ fil-forge/piri#30 (the /blob/release handler) and sprue:main ≥ +// fil-forge/sprue#33 (the forward); until piri#30 publishes, point +// INGOT_ITEST_PIRI_IMAGE at a branch image (the forgeStack escape hatch). func TestForgeDeleteReleasesNetworkBlob(t *testing.T) { - piriBin := os.Getenv("ITEST_PIRI_BIN") - sprueBin := os.Getenv("ITEST_SPRUE_BIN") - if piriBin == "" || sprueBin == "" { - t.Skip("requires piri+sprue builds with the blob-removal chain: set ITEST_PIRI_BIN and ITEST_SPRUE_BIN (see test doc comment)") - } - ctx := t.Context() - // The Curio-based piri requires a Postgres node (harmonydb) and, until - // the published localdev image carries the mockrpc Ticket fix, an - // overridable blockchain image. - stackOpts := []stack.Option{ - stack.WithPiriNodes(stack.PiriNodeConfig{Postgres: true}), - stack.WithServiceBinary("piri", piriBin), - stack.WithServiceBinary("upload", sprueBin), - } - if img := os.Getenv("ITEST_BLOCKCHAIN_IMAGE"); img != "" { - stackOpts = append(stackOpts, stack.WithBlockchainImage(img)) - } - s, ingotEndpoint := forgeStack(t, stackOpts...) + s, ingotEndpoint := forgeStack(t) accessKey, secretKey := hiltProvisionTenant(t, ctx, s, "delete") cfg := forgeConfig(ingotEndpoint, accessKey, secretKey) diff --git a/itest/forge_multipart_deferred_test.go b/itest/forge_multipart_deferred_test.go index 06ec730..3cbbbe3 100644 --- a/itest/forge_multipart_deferred_test.go +++ b/itest/forge_multipart_deferred_test.go @@ -5,7 +5,6 @@ package itest import ( "bytes" "context" - "os" "strings" "testing" "time" @@ -23,35 +22,14 @@ import ( // (parked — outside the PDP pipeline), Complete must conclude (accept) every // part, and Abort must unwind parked blobs with /blob/abort → /blob/reject. // -// Requires piri + sprue builds with the blob-removal chain (/blob/remove → -// /blob/release, /blob/abort → /blob/reject); the published images predate -// them, so inject working-tree binaries and skip otherwise (same gate as -// TestForgeDeleteReleasesNetworkBlob): -// -// (cd ../../piri && CGO_ENABLED=0 GOOS=linux go build -o /tmp/piri ./cmd) -// (cd ../../sprue && CGO_ENABLED=0 GOOS=linux go build -o /tmp/sprue ./cmd/main.go) -// ITEST_PIRI_BIN=/tmp/piri ITEST_SPRUE_BIN=/tmp/sprue \ -// go test -tags itest ./itest -run TestForgeDeferredMultipart -v -timeout 1200s +// Runs on the stock smelt-SDK images like every other itest. It needs +// piri:main ≥ fil-forge/piri#30 (/blob/reject) and sprue:main ≥ +// fil-forge/sprue#33 (/blob/abort forwarding); until piri#30 publishes, +// point INGOT_ITEST_PIRI_IMAGE at a branch image (the forgeStack escape +// hatch). func TestForgeDeferredMultipart(t *testing.T) { - piriBin := os.Getenv("ITEST_PIRI_BIN") - sprueBin := os.Getenv("ITEST_SPRUE_BIN") - if piriBin == "" || sprueBin == "" { - t.Skip("requires piri+sprue builds with the /blob/abort chain: set ITEST_PIRI_BIN and ITEST_SPRUE_BIN (see test doc comment)") - } - ctx := t.Context() - // The Curio-based piri requires a Postgres node (harmonydb) and, until - // the published localdev image carries the mockrpc Ticket fix, an - // overridable blockchain image. - stackOpts := []stack.Option{ - stack.WithPiriNodes(stack.PiriNodeConfig{Postgres: true}), - stack.WithServiceBinary("piri", piriBin), - stack.WithServiceBinary("upload", sprueBin), - } - if img := os.Getenv("ITEST_BLOCKCHAIN_IMAGE"); img != "" { - stackOpts = append(stackOpts, stack.WithBlockchainImage(img)) - } - s, endpoint := forgeStack(t, stackOpts...) + s, endpoint := forgeStack(t) accessKey, secretKey := hiltProvisionTenant(t, ctx, s, "mpdeferred") cl := sdkClient(forgeS3Conf(endpoint, accessKey, secretKey)) diff --git a/itest/stack_test.go b/itest/stack_test.go index f03dbf1..ca1ae61 100644 --- a/itest/stack_test.go +++ b/itest/stack_test.go @@ -124,6 +124,10 @@ func forgeStack(t *testing.T, extra ...stack.Option) (*stack.Stack, string) { t.Logf("using piri image override: %s", img) opts = append(opts, stack.WithPiriImage(img)) } + if img := os.Getenv("INGOT_ITEST_HILT_IMAGE"); img != "" { + t.Logf("using hilt image override: %s", img) + opts = append(opts, stack.WithHiltImage(img)) + } // Same idea one step earlier in the pipeline: mount a locally-built piri // binary (linux, static) over the image's /usr/bin/piri — validates an // unreleased piri/ucantone change with no image build at all. diff --git a/itest/versity_multipart_test.go b/itest/versity_multipart_test.go index 1f7932f..f1d2de6 100644 --- a/itest/versity_multipart_test.go +++ b/itest/versity_multipart_test.go @@ -160,10 +160,9 @@ var completeMultipartXFail = []forgeCase{ // The conditional matrix itself passes; the case then fails its bucket // teardown: upstream's teardown never aborts in-flight uploads, so // DeleteBucket implicitly aborts them (s3frontend/bucket.go), but the - // /blob/abort goes out proofless — hilt's per-operation proof grants - // carry blob.Abort for the S3 Abort operation only, not for - // DeleteBucket/UploadPart. Promote once hilt's s3perm map grants - // blob.Abort on bucket delete. + // /blob/abort goes out proofless — hilt's s3perm map delegates no + // blob.Abort today. fil-forge/hilt#36 adds it to the write set; + // promote once a hilt image with it publishes. {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, {name: "invalid_checksum_part", fn: integration.CompleteMultipartUpload_invalid_checksum_part}, {name: "multiple_checksum_part", fn: integration.CompleteMultipartUpload_multiple_checksum_part}, diff --git a/s3frontend/multipart.go b/s3frontend/multipart.go index ea5a8ff..da0cb30 100644 --- a/s3frontend/multipart.go +++ b/s3frontend/multipart.go @@ -24,15 +24,12 @@ import ( msbucket "github.com/fil-forge/ingot/bucket" "github.com/fil-forge/ingot/bucketop" + "github.com/fil-forge/ingot/internal/reqscope" "github.com/fil-forge/ingot/mst" "github.com/fil-forge/ingot/registry" "github.com/fil-forge/ingot/uploader" ) -// minPartSize is S3's minimum size for every part except the last one, -// enforced at CompleteMultipartUpload (EntityTooSmall). -const minPartSize = 5 << 20 - // defaultMaxListing is the S3 default and cap for max-parts / max-uploads. const defaultMaxListing = 1000 @@ -295,10 +292,12 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet requested = append(requested, sp) etagHasher.Write(sp.ETagMD5) } - // Every part but the last must meet S3's 5 MiB minimum. + // Every part but the last must meet S3's protocol-level 5 MiB minimum + // (backend.MinPartSize — an S3 constant clients and SDKs assume, not an + // operator knob). var total int64 for i, sp := range requested { - if i < len(requested)-1 && sp.Size < minPartSize { + if i < len(requested)-1 && sp.Size < backend.MinPartSize { return s3response.CompleteMultipartUploadResult{}, "", s3err.GetAPIError(s3err.ErrEntityTooSmall) } total += sp.Size @@ -390,7 +389,10 @@ func (b *Backend) CompleteMultipartUpload(ctx context.Context, input *s3.Complet // parts so a duplicate Complete is idempotent; the sweeper reaps it later. // Best-effort: a failed latch leaves the row in 'completing', which the // sweeper also treats as terminal after the TTL. - _, _ = b.multipart.LatchSession(ctx, uploadID, registry.SessionCompleting, registry.SessionCompleted) + if _, err := b.multipart.LatchSession(ctx, uploadID, registry.SessionCompleting, registry.SessionCompleted); err != nil { + b.logger.Warn("latch session to completed failed; sweeper reaps the completing row after the TTL", + zap.String("uploadID", uploadID), zap.Error(err)) + } etagQ := `"` + etag + `"` return s3response.CompleteMultipartUploadResult{Bucket: &bucket, Key: &key, ETag: &etagQ}, "", nil @@ -449,6 +451,13 @@ func (b *Backend) AbortMultipartUpload(ctx context.Context, input *s3.AbortMulti // drop the session, release its parts' now-unreferenced blobs. Used by // DeleteBucket's implicit abort of in-flight uploads. func (b *Backend) abortOpenSession(ctx context.Context, space did.DID, sess registry.MultipartSession) { + // s3:DeleteBucket delegates no blob commands (hilt's s3perm maps it to + // nil), so the surrounding request's proofs cannot authorize + // /blob/abort; mask them so the uploader falls back to the blob + // authority captured at UploadPart — the same resolution the + // session-expiry sweeper uses. blob.Abort rides the write set as of + // fil-forge/hilt#36. + ctx = reqscope.WithoutProofStore(ctx) won, err := b.multipart.LatchSession(ctx, sess.UploadID, registry.SessionOpen, registry.SessionAborting) if err != nil || !won { return @@ -522,11 +531,20 @@ func (b *Backend) cleanupPartBlobs(ctx context.Context, space did.DID, uploadID zap.String("digest", hex.EncodeToString(d)), zap.Error(aerr)) } } - _ = b.parks.DeletePark(ctx, d) + if derr := b.parks.DeletePark(ctx, d); derr != nil { + b.logger.Warn("delete park row failed", + zap.String("digest", hex.EncodeToString(d)), zap.Error(derr)) + } } } - _ = b.spool.Remove(mh.Multihash(d)) - _ = b.intents.DeleteIntent(ctx, d) + if rerr := b.spool.Remove(mh.Multihash(d)); rerr != nil { + b.logger.Warn("remove spooled blob failed", + zap.String("digest", hex.EncodeToString(d)), zap.Error(rerr)) + } + if derr := b.intents.DeleteIntent(ctx, d); derr != nil { + b.logger.Warn("delete upload intent failed", + zap.String("digest", hex.EncodeToString(d)), zap.Error(derr)) + } } } From ac2bd3cc541eb6bdee00eae066aee76f6f1ff855 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 30 Jul 2026 11:11:00 -0700 Subject: [PATCH 18/21] review(uploader,forgeclient): WithConclude(false) replaces the parked add/upload split MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fold the PR #40 API-shape review (r3674608706, r3674935223): the deferred accept is an option on the one entry point, not a second entry point. - forgeclient: ParkedBlob's fields (AddTask, AcceptTask, PutInvocation) fold into AddedBlob; Location == nil marks an unconcluded add. BlobAdd takes WithConclude (default true); BlobAddParked becomes the private durable half. BlobConclude takes the AddedBlob, no-ops when already concluded, and drops the spent PutInvocation from its result. - uploader: UploadBlob gains UploadOption and returns UploadedBlob (nil-able *BlobLocation + the pending-accept state, populated only while parked); DeferredBodyUploader embeds BodyUploader and keeps ConcludeBlob/AbortBlob. Callers guard the "concluding upload always returns a location" contract explicitly instead of deref-panicking on a misbehaving impl. - s3frontend: parkBlobs passes WithConclude(false); the dedup-vs-parked branch is a documented Location nil-check. The park vocabulary stays in the registry layer (blob_parks, BlobPark, ParkStore) and docs — it is the blob-removal RFC's lifecycle term for the allocated-but-unaccepted state; only the redundant API split is gone. Validated: unit suite; full itest partition green against piri#30 head (fil-forge/piri@b5208a3) + hilt#36 images. Co-Authored-By: Claude Fable 5 --- docs/architecture.md | 2 +- forgeclient/blobabort.go | 2 +- forgeclient/blobadd.go | 159 ++++++++++++++++++------------- inmem/store.go | 18 ++-- s3frontend/backend.go | 5 +- s3frontend/multipart.go | 40 ++++---- s3frontend/object.go | 8 +- s3frontend/upload_dedup_test.go | 8 +- server.go | 5 +- uploader/blob.go | 163 +++++++++++++++++++------------- 10 files changed, 242 insertions(+), 168 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index cf3d93b..9cef025 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -532,7 +532,7 @@ negotiations). | Capability | Service | Status | Notes | |------------------------------------------------------------------------------------------------|---------------------------------|--------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| | `allocate` / `PUT` / `accept` blob lifecycle | Piri/Sprue | **exists** | The storage primitive Ingot builds on. | -| Ingot-timed accept (PUT a part, defer the conclude until Complete) | Ingot + Sprue | **exists** | `forgeclient.BlobAddParked`/`BlobConclude` split the flow at the conclude seam; UploadPart parks (durable, unaggregated), Complete concludes — Sprue's conclude handler already ran accept standalone. Ingot still does **not** issue `accept` (Piri requires the upload-service DID). | +| Ingot-timed accept (PUT a part, defer the conclude until Complete) | Ingot + Sprue | **exists** | `forgeclient.BlobAdd` with `WithConclude(false)` stops at the conclude seam (`BlobConclude` finishes it); UploadPart parks (durable, unaggregated), Complete concludes — Sprue's conclude handler already ran accept standalone. Ingot still does **not** issue `accept` (Piri requires the upload-service DID). | | `abort(digest)` — drop a parked blob | Piri + Sprue + libforge | **exists** | `/blob/abort` on Sprue translates to `/blob/reject` on Piri, which refuses blobs the invoking space has accepted (`BlobAccepted`), deletes the space's allocation, and drops the bytes once no space holds an allocation or acceptance. Sprue recovers the provider from the `cause` receipt chain (a parked blob has no registration). Abort/TTL/supersede unwind through it. | | `remove(digest)` — per-space claim release; physical delete/piece-retire at zero global claims | Piri + Sprue + libforge | **exists** | `/blob/remove` on Sprue forwards `/blob/release` to Piri, which deletes the space's allocation/acceptance/claim; at zero claims bytes delete immediately (unaggregated) or via the pending-removal sweep once the whole aggregate root is dead (FIL-623/624). Sprue forwards to primary + replicas (FIL-522). | | Configurable, adaptive size policy (`min`/`max`); batch guard for the `extraData` cap | Piri | **partial** | `MinAggregateSize` is hardcoded 128 MiB; lower to ~8 MiB and make configurable. The `addPieces` batch is no longer contract-capped (FWSS v1.3.0 removed the `extraData` cap); size it to the FVM `PiecesAdded` event-size + per-tx gas — a measured ceiling (default `BatchSize=10` is safely within it) (pdp-sim). No contract change. | diff --git a/forgeclient/blobabort.go b/forgeclient/blobabort.go index e8095f8..afc338e 100644 --- a/forgeclient/blobabort.go +++ b/forgeclient/blobabort.go @@ -15,7 +15,7 @@ import ( // BlobAbort invokes /blob/abort against the upload service (sprue), // abandoning the space's in-flight upload of a parked (never-accepted) -// blob. cause is the /blob/add task link (ParkedBlob.AddTask) — sprue +// blob. cause is the /blob/add task link (AddedBlob.AddTask) — sprue // walks its receipt chain to locate the storage node holding the parked // bytes (which have no registration or acceptance to look up by) and // forwards a /blob/reject there. The space is the invocation subject. diff --git a/forgeclient/blobadd.go b/forgeclient/blobadd.go index bb0372a..d27c76d 100644 --- a/forgeclient/blobadd.go +++ b/forgeclient/blobadd.go @@ -6,8 +6,9 @@ // not just the client's token store. // - No /blob/accept re-delegation: sprue owns accept (as it owns // allocate), so the conclude/put-receipt dance carries no space proof. -// - BlobAdd decomposes into BlobAddParked + BlobConclude so multipart can -// defer the conclude (park at UploadPart, accept at Complete). +// - BlobAdd accepts WithConclude(false) so multipart can defer the +// conclude (park at UploadPart, accept at Complete); BlobConclude +// finishes the parked add later. // // Also dropped from upstream: otel spans, go-log, ctxutil, the progress/stall // readers, and the hard requirement that the accept receipt carry a PDP @@ -57,11 +58,14 @@ type BlobAddConfig struct { // call instead of the client's default token store — used to scope an // invocation to a request's per-access-key proofs. ProofStore ucanlib.ProofStore + // Conclude controls whether BlobAdd concludes the /http/put receipt + // (triggering /blob/accept) before returning. Default true. + Conclude bool } // NewBlobAddConfig builds a BlobAddConfig from options. func NewBlobAddConfig(options ...BlobAddOption) *BlobAddConfig { - cfg := &BlobAddConfig{PutClient: &http.Client{}} + cfg := &BlobAddConfig{PutClient: &http.Client{}, Conclude: true} for _, opt := range options { opt(cfg) } @@ -88,62 +92,65 @@ func WithProofStore(ps ucanlib.ProofStore) BlobAddOption { return func(cfg *BlobAddConfig) { cfg.ProofStore = ps } } -// AddedBlob is the result of a successful BlobAdd. -type AddedBlob struct { - Digest multihash.Multihash - Size uint64 - Location ucan.Invocation // the /assert/location commitment +// WithConclude controls whether BlobAdd concludes the upload before +// returning. WithConclude(false) leaves the blob parked — durable on the +// provider, but piri holds the bytes without aggregating them until +// /blob/accept fires: the returned AddedBlob has a nil Location and carries +// the state [Client.BlobConclude] needs to finish the upload later (or +// [Client.BlobAbort] to abandon it; AddTask is the abort Cause). Moot on +// dedup — when the provider already held accepted bytes for the content, +// accept already ran and the add completes regardless. +func WithConclude(conclude bool) BlobAddOption { + return func(cfg *BlobAddConfig) { cfg.Conclude = conclude } } -// ParkedBlob is the persistable state of a blob that has been added and -// uploaded (durable on the provider) but not yet concluded — piri holds the -// bytes without aggregating them until /blob/accept fires. Persist it and -// finish the upload later with [Client.BlobConclude], or abandon it with -// [Client.BlobAbort] (AddTask is the abort Cause). -type ParkedBlob struct { +// AddedBlob is the result of a BlobAdd. Location is set once the blob is +// accepted; with WithConclude(false) it is nil until the deferred +// [Client.BlobConclude] — persist the task links + PutInvocation in between. +type AddedBlob struct { Digest multihash.Multihash Size uint64 + // Location is the /assert/location commitment issued at accept; nil + // while the add is unconcluded (parked). + Location ucan.Invocation // AddTask is the /blob/add task link — the receipt-chain root the // upload service uses to locate the provider for abort. AddTask cid.Cid // AcceptTask is the /blob/accept task link BlobConclude polls. AcceptTask cid.Cid - // PutInvocation is the sealed /http/put invocation. Its metadata embeds - // the derived signer keys needed to synthesize the put receipt at - // conclude time — treat it as sensitive and delete it once concluded or - // rejected. + // PutInvocation is the issued /http/put invocation, populated only while + // the add is unconcluded. Its metadata embeds the derived signer keys + // needed to synthesize the put receipt at conclude time — treat it as + // sensitive and delete it once concluded or rejected. PutInvocation []byte } // BlobAdd adds a blob to the upload service (sprue): invoke /blob/add, // PUT the bytes, conclude a synthesized /http/put receipt, then poll the // /blob/accept receipt for the location commitment. The issuer needs a -// /blob/add delegation proof over space. -// -// It is [Client.BlobAddParked] + [Client.BlobConclude] in one shot. +// /blob/add delegation proof over space. With WithConclude(false) it stops +// after the PUT — the blob stays parked until [Client.BlobConclude]. func (c *Client) BlobAdd(ctx context.Context, space did.DID, content io.Reader, options ...BlobAddOption) (AddedBlob, error) { - parked, added, err := c.BlobAddParked(ctx, space, content, options...) + cfg := NewBlobAddConfig(options...) + added, err := c.blobAdd(ctx, space, content, cfg) if err != nil { return AddedBlob{}, err } - if added != nil { - return *added, nil + // Already accepted (dedup) or deliberately unconcluded — done either way. + if added.Location != nil || !cfg.Conclude { + return added, nil } - return c.BlobConclude(ctx, space, *parked) + return c.BlobConclude(ctx, space, added) } -// BlobAddParked runs the durable half of BlobAdd: /blob/add + PUT the bytes, +// blobAdd runs the durable half of BlobAdd: /blob/add + PUT the bytes, // WITHOUT concluding the /http/put receipt — the conclude is what makes the // upload service trigger /blob/accept on the provider, so the blob stays -// parked (stored, unaggregated) until BlobConclude. -// -// Exactly one of the returns is non-nil: a ParkedBlob awaiting conclude, or -// — when the provider already held accepted bytes for this content (dedup: -// allocate returned no upload address and the put receipt was pre-issued, so -// accept already ran) — the completed AddedBlob. -func (c *Client) BlobAddParked(ctx context.Context, space did.DID, content io.Reader, options ...BlobAddOption) (parked *ParkedBlob, added *AddedBlob, err error) { - cfg := NewBlobAddConfig(options...) - +// parked (stored, unaggregated) until BlobConclude. The result's Location is +// nil unless the provider already held accepted bytes for this content +// (dedup: allocate returned no upload address and the put receipt was +// pre-issued, so accept already ran). +func (c *Client) blobAdd(ctx context.Context, space did.DID, content io.Reader, cfg *BlobAddConfig) (blob AddedBlob, err error) { putClient := cfg.PutClient contentReader := content contentHash := cfg.PrecomputedDigest @@ -153,20 +160,20 @@ func (c *Client) BlobAddParked(ctx context.Context, space did.DID, content io.Re start := time.Now() defer func() { if err != nil { - c.logger.Error("blob add (parked) failed", zap.Stringer("space", space), zap.Error(err), zap.Duration("duration", time.Since(start))) + c.logger.Error("blob add failed", zap.Stringer("space", space), zap.Error(err), zap.Duration("duration", time.Since(start))) } else { - c.logger.Debug("blob added (parked)", zap.Stringer("space", space), zap.Duration("duration", time.Since(start))) + c.logger.Debug("blob added", zap.Stringer("space", space), zap.Bool("parked", blob.Location == nil), zap.Duration("duration", time.Since(start))) } }() if needsHash { contentBytes, rerr := io.ReadAll(content) if rerr != nil { - return nil, nil, fmt.Errorf("reading content: %w", rerr) + return AddedBlob{}, fmt.Errorf("reading content: %w", rerr) } contentHash, err = multihash.Sum(contentBytes, multihash.SHA2_256, -1) if err != nil { - return nil, nil, fmt.Errorf("computing content multihash: %w", err) + return AddedBlob{}, fmt.Errorf("computing content multihash: %w", err) } contentReader = bytes.NewReader(contentBytes) contentSize := uint64(len(contentBytes)) @@ -180,7 +187,7 @@ func (c *Client) BlobAddParked(ctx context.Context, space did.DID, content io.Re proofs, proofLinks, err := proofStore.ProofChain(ctx, c.signer.DID(), blobcmds.Add.Command, space) if err != nil { - return nil, nil, fmt.Errorf("building proof chain: %w", err) + return AddedBlob{}, fmt.Errorf("building proof chain: %w", err) } inv, err := blobcmds.Add.Invoke( @@ -190,7 +197,7 @@ func (c *Client) BlobAddParked(ctx context.Context, space did.DID, content io.Re invocation.WithProofs(proofLinks...), ) if err != nil { - return nil, nil, fmt.Errorf("generating invocation: %w", err) + return AddedBlob{}, fmt.Errorf("generating invocation: %w", err) } addOK, _, meta, err := Execute[*blobcmds.AddOK]( @@ -198,51 +205,51 @@ func (c *Client) BlobAddParked(ctx context.Context, space did.DID, content io.Re execution.WithDelegations(proofs...), ) if err != nil { - return nil, nil, fmt.Errorf("executing blob add: %w", err) + return AddedBlob{}, fmt.Errorf("executing blob add: %w", err) } accInv, err := findInvocation(addOK.Site.Task, meta.Invocations()) if err != nil { - return nil, nil, fmt.Errorf("finding /blob/accept invocation: %w", err) + return AddedBlob{}, fmt.Errorf("finding /blob/accept invocation: %w", err) } var accArgs blobcmds.AcceptArguments if err := accArgs.UnmarshalCBOR(bytes.NewReader(accInv.ArgumentsBytes())); err != nil { - return nil, nil, fmt.Errorf("unmarshaling /blob/accept arguments: %w", err) + return AddedBlob{}, fmt.Errorf("unmarshaling /blob/accept arguments: %w", err) } putInv, err := findInvocation(accArgs.Put.Task, meta.Invocations()) if err != nil { - return nil, nil, fmt.Errorf("finding /http/put invocation: %w", err) + return AddedBlob{}, fmt.Errorf("finding /http/put invocation: %w", err) } var putArgs httpcmds.PutArguments if err := putArgs.UnmarshalCBOR(bytes.NewReader(putInv.ArgumentsBytes())); err != nil { - return nil, nil, fmt.Errorf("unmarshaling /http/put arguments: %w", err) + return AddedBlob{}, fmt.Errorf("unmarshaling /http/put arguments: %w", err) } putRcpt := maybeFindReceipt(accArgs.Put.Task, meta.Receipts()) allocInv, err := findInvocation(putArgs.Destination.Task, meta.Invocations()) if err != nil { - return nil, nil, fmt.Errorf("finding /blob/allocate invocation: %w", err) + return AddedBlob{}, fmt.Errorf("finding /blob/allocate invocation: %w", err) } var allocArgs blobcmds.AllocateArguments if err := allocArgs.UnmarshalCBOR(bytes.NewReader(allocInv.ArgumentsBytes())); err != nil { - return nil, nil, fmt.Errorf("unmarshaling /blob/allocate arguments: %w", err) + return AddedBlob{}, fmt.Errorf("unmarshaling /blob/allocate arguments: %w", err) } allocRcpt, err := findReceipt(putArgs.Destination.Task, meta.Receipts()) if err != nil { - return nil, nil, fmt.Errorf("finding /blob/allocate receipt: %w", err) + return AddedBlob{}, fmt.Errorf("finding /blob/allocate receipt: %w", err) } o, x := allocRcpt.Out().Unpack() if allocRcpt.Out().IsErr() { var model edm.ErrorModel if err := model.UnmarshalCBOR(bytes.NewReader(x)); err != nil { - return nil, nil, fmt.Errorf("executing invocation") + return AddedBlob{}, fmt.Errorf("executing invocation") } - return nil, nil, fmt.Errorf("failure in allocation receipt: %w", model) + return AddedBlob{}, fmt.Errorf("failure in allocation receipt: %w", model) } var allocOK blobcmds.AllocateOK if err := allocOK.UnmarshalCBOR(bytes.NewReader(o)); err != nil { - return nil, nil, fmt.Errorf("unmarshaling allocation receipt output: %w", err) + return AddedBlob{}, fmt.Errorf("unmarshaling allocation receipt output: %w", err) } putSuccess := putRcpt != nil && putRcpt.Out().IsOK() @@ -253,7 +260,7 @@ func (c *Client) BlobAddParked(ctx context.Context, space did.DID, content io.Re // piri's PUT endpoint requires it. if allocOK.Address != nil && !putSuccess { if err := putBlob(ctx, putClient, allocOK.Address.URL.URL(), allocOK.Address.Headers, contentReader, int64(*contentSizePtr)); err != nil { - return nil, nil, fmt.Errorf("putting blob: %w", err) + return AddedBlob{}, fmt.Errorf("putting blob: %w", err) } } @@ -264,29 +271,39 @@ func (c *Client) BlobAddParked(ctx context.Context, space did.DID, content io.Re if putSuccess { location, aerr := c.awaitAccept(ctx, accInv.Task().Link()) if aerr != nil { - err = aerr - return nil, nil, err + return AddedBlob{}, aerr } - return nil, &AddedBlob{Digest: contentHash, Size: *contentSizePtr, Location: location}, nil + return AddedBlob{ + Digest: contentHash, + Size: *contentSizePtr, + Location: location, + AddTask: inv.Task().Link(), + AcceptTask: accInv.Task().Link(), + }, nil } // Parked: durable on the provider, conclude deferred to BlobConclude. - return &ParkedBlob{ + return AddedBlob{ Digest: contentHash, Size: *contentSizePtr, AddTask: inv.Task().Link(), AcceptTask: accInv.Task().Link(), PutInvocation: putInv.Bytes(), - }, nil, nil + }, nil } -// BlobConclude finishes a parked upload: it synthesizes and concludes the -// /http/put receipt (which makes the upload service trigger /blob/accept on -// the provider) and awaits the accept receipt's location commitment. Accept -// is owned by sprue (like allocate), so the conclude carries no space proof. -// Safe to retry — re-concluding an already-concluded put is tolerated -// upstream. -func (c *Client) BlobConclude(ctx context.Context, space did.DID, parked ParkedBlob) (blob AddedBlob, err error) { +// BlobConclude finishes a parked (unconcluded) BlobAdd: it synthesizes and +// concludes the /http/put receipt (which makes the upload service trigger +// /blob/accept on the provider) and awaits the accept receipt's location +// commitment. Accept is owned by sprue (like allocate), so the conclude +// carries no space proof. Safe to retry — re-concluding an already-concluded +// put is tolerated upstream, and an AddedBlob whose Location is already set +// returns as-is. The result drops PutInvocation (spent — the caller should +// delete its persisted copy too). +func (c *Client) BlobConclude(ctx context.Context, space did.DID, added AddedBlob) (blob AddedBlob, err error) { + if added.Location != nil { + return added, nil + } start := time.Now() defer func() { if err != nil { @@ -297,7 +314,7 @@ func (c *Client) BlobConclude(ctx context.Context, space did.DID, parked ParkedB }() putInv := new(invocation.Invocation) - if err := putInv.UnmarshalCBOR(bytes.NewReader(parked.PutInvocation)); err != nil { + if err := putInv.UnmarshalCBOR(bytes.NewReader(added.PutInvocation)); err != nil { return AddedBlob{}, fmt.Errorf("decoding parked /http/put invocation: %w", err) } @@ -305,11 +322,17 @@ func (c *Client) BlobConclude(ctx context.Context, space did.DID, parked ParkedB return AddedBlob{}, fmt.Errorf("sending put receipt: %w", err) } - location, err := c.awaitAccept(ctx, parked.AcceptTask) + location, err := c.awaitAccept(ctx, added.AcceptTask) if err != nil { return AddedBlob{}, err } - return AddedBlob{Digest: parked.Digest, Size: parked.Size, Location: location}, nil + return AddedBlob{ + Digest: added.Digest, + Size: added.Size, + Location: location, + AddTask: added.AddTask, + AcceptTask: added.AcceptTask, + }, nil } // awaitAccept polls the /blob/accept receipt and extracts the diff --git a/inmem/store.go b/inmem/store.go index aa9a509..c4acdc5 100644 --- a/inmem/store.go +++ b/inmem/store.go @@ -347,21 +347,17 @@ func (NopUploader) SubmitShard(_ context.Context, _ blockstore.Plane, _ did.DID, return uploader.BlobLocation{}, nil } -func (NopUploader) UploadBlob(_ context.Context, _ did.DID, _ multihash.Multihash, size int64, _ string) (uploader.BlobLocation, error) { - return uploader.BlobLocation{Size: size}, nil +// UploadBlob accepts immediately, even with WithConclude(false) — there is +// no network to park on, so the deferred flow degenerates to the synchronous +// one and reads keep coming from the spool. +func (NopUploader) UploadBlob(_ context.Context, _ did.DID, digest multihash.Multihash, size int64, _ string, _ ...uploader.UploadOption) (uploader.UploadedBlob, error) { + return uploader.UploadedBlob{Digest: digest, Size: size, Location: &uploader.BlobLocation{Size: size}}, nil } func (NopUploader) RemoveBlob(_ context.Context, _ did.DID, _ multihash.Multihash) error { return nil } -// UploadBlobParked accepts immediately in the harness — there is no network -// to park on, so the deferred flow degenerates to the synchronous one and -// reads keep coming from the spool. -func (NopUploader) UploadBlobParked(_ context.Context, _ did.DID, _ multihash.Multihash, size int64, _ string) (*uploader.ParkedBlobState, *uploader.BlobLocation, error) { - return nil, &uploader.BlobLocation{Size: size}, nil -} - -func (NopUploader) ConcludeBlob(_ context.Context, _ did.DID, parked uploader.ParkedBlobState) (uploader.BlobLocation, error) { - return uploader.BlobLocation{Size: int64(parked.Size)}, nil +func (NopUploader) ConcludeBlob(_ context.Context, _ did.DID, parked uploader.UploadedBlob) (uploader.BlobLocation, error) { + return uploader.BlobLocation{Size: parked.Size}, nil } func (NopUploader) AbortBlob(_ context.Context, _ did.DID, _ multihash.Multihash, _ cid.Cid) error { diff --git a/s3frontend/backend.go b/s3frontend/backend.go index b613702..3e6abf0 100644 --- a/s3frontend/backend.go +++ b/s3frontend/backend.go @@ -98,8 +98,9 @@ type Deps struct { // accept) synchronously, before the manifest commits. Remover releases a // space's claim on a blob when its last reference is dropped. Uploader uploader.BodyUploader - // Deferred decomposes Uploader for multipart's deferred accept; Parks - // persists park state between UploadPart and Complete/Abort. + // Deferred extends Uploader for multipart's deferred accept + // (WithConclude(false), then ConcludeBlob/AbortBlob); Parks persists + // park state between UploadPart and Complete/Abort. Deferred uploader.DeferredBodyUploader Parks registry.ParkStore Remover uploader.BlobRemover diff --git a/s3frontend/multipart.go b/s3frontend/multipart.go index da0cb30..2263155 100644 --- a/s3frontend/multipart.go +++ b/s3frontend/multipart.go @@ -550,8 +550,9 @@ func (b *Backend) cleanupPartBlobs(ctx context.Context, space did.DID, uploadID // parkBlobs makes each blob durable on its provider without accepting it: // already-located blobs are marked accepted (dedup), already-parked blobs are -// skipped (another part or session parked the same content), the rest run -// the parked upload and persist their park state for Complete/Abort. +// skipped (another part or session parked the same content), the rest upload +// with the conclude deferred (WithConclude(false)) and persist their park +// state for Complete/Abort. func (b *Backend) parkBlobs(ctx context.Context, space did.DID, blobs []msbucket.BlobRef) error { for _, blob := range blobs { digest := mh.Multihash(blob.Digest) @@ -569,19 +570,20 @@ func (b *Backend) parkBlobs(ctx context.Context, space did.DID, blobs []msbucket return fmt.Errorf("lookup park: %w", err) } - parked, located, err := b.deferred.UploadBlobParked(ctx, space, digest, blob.Length, b.spool.Path(digest)) + res, err := b.deferred.UploadBlob(ctx, space, digest, blob.Length, b.spool.Path(digest), uploader.WithConclude(false)) if err != nil { return fmt.Errorf("park blob: %w", err) } - if located != nil { + if res.Location != nil { // The provider already held accepted bytes for this content — - // accept ran, record the location like the synchronous path. + // accept ran despite the deferred conclude (dedup), record the + // location like the synchronous path. if err := b.locations.PutLocation(ctx, registry.BlobLocation{ Space: space, Digest: blob.Digest, - Provider: located.Provider, - URL: located.URL, - Size: located.Size, + Provider: res.Location.Provider, + URL: res.Location.URL, + Size: res.Location.Size, }); err != nil { return fmt.Errorf("record location: %w", err) } @@ -590,11 +592,13 @@ func (b *Backend) parkBlobs(ctx context.Context, space did.DID, blobs []msbucket } continue } + // Location == nil ⇔ parked: durable on the provider with accept + // deferred — persist the conclude state for Complete/Abort. if err := b.parks.PutPark(ctx, registry.BlobPark{ Digest: blob.Digest, - AddTask: parked.AddTask.Bytes(), - AcceptTask: parked.AcceptTask.Bytes(), - PutInvocation: parked.PutInvocation, + AddTask: res.AddTask.Bytes(), + AcceptTask: res.AcceptTask.Bytes(), + PutInvocation: res.PutInvocation, Size: blob.Length, }); err != nil { return fmt.Errorf("record park: %w", err) @@ -637,9 +641,9 @@ func (b *Backend) concludeBlobs(ctx context.Context, space did.DID, blobs []msbu if err != nil { return fmt.Errorf("decode park accept task: %w", err) } - loc, err = b.deferred.ConcludeBlob(ctx, space, uploader.ParkedBlobState{ + loc, err = b.deferred.ConcludeBlob(ctx, space, uploader.UploadedBlob{ Digest: digest, - Size: uint64(park.Size), + Size: park.Size, AddTask: addTask, AcceptTask: acceptTask, PutInvocation: park.PutInvocation, @@ -650,10 +654,14 @@ func (b *Backend) concludeBlobs(ctx context.Context, space did.DID, blobs []msbu } else { // Never parked (crash between spool and park): the spooled copy // drives the whole synchronous upload. - loc, err = b.uploader.UploadBlob(ctx, space, digest, blob.Length, b.spool.Path(digest)) - if err != nil { - return fmt.Errorf("upload blob: %w", err) + res, uerr := b.uploader.UploadBlob(ctx, space, digest, blob.Length, b.spool.Path(digest)) + if uerr != nil { + return fmt.Errorf("upload blob: %w", uerr) + } + if res.Location == nil { + return fmt.Errorf("upload blob %x: concluding upload returned no location", blob.Digest) } + loc = *res.Location } if err := b.locations.PutLocation(ctx, registry.BlobLocation{ diff --git a/s3frontend/object.go b/s3frontend/object.go index 32c3c6e..dcd885c 100644 --- a/s3frontend/object.go +++ b/s3frontend/object.go @@ -282,10 +282,16 @@ func (b *Backend) uploadBlobs(ctx context.Context, space did.DID, blobs []msbuck } else if err != nil && !errors.Is(err, registry.ErrNotFound) { return fmt.Errorf("lookup location: %w", err) } - loc, err := b.uploader.UploadBlob(ctx, space, digest, blob.Length, b.spool.Path(digest)) + res, err := b.uploader.UploadBlob(ctx, space, digest, blob.Length, b.spool.Path(digest)) if err != nil { return fmt.Errorf("upload blob: %w", err) } + // A concluding UploadBlob (the default) returns an accepted location + // or errors; guard the contract rather than deref-panic on a bad impl. + if res.Location == nil { + return fmt.Errorf("upload blob %x: concluding upload returned no location", blob.Digest) + } + loc := res.Location if err := b.intents.SetIntentState(ctx, blob.Digest, registry.IntentAccepted); err != nil { return fmt.Errorf("mark accepted: %w", err) } diff --git a/s3frontend/upload_dedup_test.go b/s3frontend/upload_dedup_test.go index d3d202f..64cbf41 100644 --- a/s3frontend/upload_dedup_test.go +++ b/s3frontend/upload_dedup_test.go @@ -26,11 +26,15 @@ type countingUploader struct { calls map[string]int } -func (u *countingUploader) UploadBlob(_ context.Context, space did.DID, digest multihash.Multihash, size int64, _ string) (uploader.BlobLocation, error) { +func (u *countingUploader) UploadBlob(_ context.Context, _ did.DID, digest multihash.Multihash, size int64, _ string, _ ...uploader.UploadOption) (uploader.UploadedBlob, error) { u.mu.Lock() u.calls[string(digest)]++ u.mu.Unlock() - return uploader.BlobLocation{Provider: "did:test:piri", URL: "http://piri/blob", Size: size}, nil + return uploader.UploadedBlob{ + Digest: digest, + Size: size, + Location: &uploader.BlobLocation{Provider: "did:test:piri", URL: "http://piri/blob", Size: size}, + }, nil } func (u *countingUploader) count(digest []byte) int { diff --git a/server.go b/server.go index e686c9d..5ed4b35 100644 --- a/server.go +++ b/server.go @@ -51,8 +51,9 @@ type ServerDeps struct { // space's claim on a blob when its last reference is dropped. In tests both // are no-ops and reads are served from the local spool. BodyUploader uploader.BodyUploader - // Deferred is BodyUploader decomposed for multipart's deferred accept: - // park at UploadPart, conclude at Complete, abort at Abort. + // Deferred extends BodyUploader for multipart's deferred accept: + // park at UploadPart (WithConclude(false)), conclude at Complete, + // abort at Abort. Deferred uploader.DeferredBodyUploader Remover uploader.BlobRemover diff --git a/uploader/blob.go b/uploader/blob.go index 0207319..ac49c5f 100644 --- a/uploader/blob.go +++ b/uploader/blob.go @@ -29,24 +29,72 @@ type BlobLocation struct { Size int64 // whole-blob byte length } +// UploadOption configures BodyUploader.UploadBlob. +type UploadOption func(*uploadConfig) + +type uploadConfig struct { + conclude bool +} + +func newUploadConfig(options ...UploadOption) *uploadConfig { + cfg := &uploadConfig{conclude: true} + for _, opt := range options { + opt(cfg) + } + return cfg +} + +// WithConclude controls whether UploadBlob triggers accept before returning. +// WithConclude(false) leaves the blob parked — durable on the provider, +// accept deferred (multipart's UploadPart): the returned UploadedBlob has a +// nil Location and carries the state ConcludeBlob needs. Moot on dedup — +// when the provider already held accepted bytes for the content, accept +// already ran and the upload completes regardless. +func WithConclude(conclude bool) UploadOption { + return func(cfg *uploadConfig) { cfg.conclude = conclude } +} + +// UploadedBlob is the result of UploadBlob. Location == nil ⇔ the blob is +// parked (uploaded with WithConclude(false), no dedup hit): persist the task +// links + PutInvocation (the blob_parks row) and finish with ConcludeBlob at +// Complete, or abandon with AbortBlob (AddTask is the Cause). A concluding +// upload — the default — always returns a non-nil Location or errors. +type UploadedBlob struct { + Digest multihash.Multihash + Size int64 + // Location of the accepted blob; nil while parked. + Location *BlobLocation + // AddTask is the /blob/add task CID (the abort Cause); AcceptTask is the + // /blob/accept task CID the conclude polls. + AddTask cid.Cid + AcceptTask cid.Cid + // PutInvocation is the issued /http/put invocation, populated only while + // parked. Its metadata embeds derived signer keys — sensitive; delete it + // once concluded or rejected. + PutInvocation []byte +} + // BodyUploader makes one object-body blob durable on Forge by digest: allocate -// → PUT (skipped on dedup) → accept, returning its published location. It is the +// → PUT (skipped on dedup) → accept, returning its published location. +// WithConclude(false) stops before accept (see UploadedBlob). It is the // data-plane counterpart to Uploader (which ships catalog CAR segments). Unlike -// the old data-plane pipeline, this is synchronous: a blob is durable and -// accepted on Piri before the write path commits the manifest that references -// it (docs/architecture.md §5, §7.1). +// the old data-plane pipeline, this is synchronous: a blob is durable — and, +// when concluding, accepted — on Piri before the write path commits the +// manifest that references it (docs/architecture.md §5, §7.1). type BodyUploader interface { - UploadBlob(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string) (BlobLocation, error) + UploadBlob(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string, opts ...UploadOption) (UploadedBlob, error) } // UploadBlob uploads one spooled blob to Forge. For a single-shot PutObject the // allocate→PUT→accept happens in one call (forgeclient.BlobAdd already drives // the whole flow and returns the location commitment); multipart's deferred -// accept uses the decomposed UploadBlobParked/ConcludeBlob pair instead. -func (u *Forge) UploadBlob(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string) (BlobLocation, error) { +// accept passes WithConclude(false) and finishes with ConcludeBlob at +// Complete (or AbortBlob at Abort). +func (u *Forge) UploadBlob(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string, opts ...UploadOption) (UploadedBlob, error) { + cfg := newUploadConfig(opts...) f, err := os.Open(localPath) if err != nil { - return BlobLocation{}, fmt.Errorf("uploader: open spooled blob %s: %w", localPath, err) + return UploadedBlob{}, fmt.Errorf("uploader: open spooled blob %s: %w", localPath, err) } defer f.Close() @@ -61,7 +109,7 @@ func (u *Forge) UploadBlob(ctx context.Context, space did.DID, digest multihash. // attributable. store, ok := reqscope.ProofStore(ctx) if !ok { - return BlobLocation{}, fmt.Errorf("uploader: no request-scoped proof store for space %s (IAM layer did not attach one)", space) + return UploadedBlob{}, fmt.Errorf("uploader: no request-scoped proof store for space %s (IAM layer did not attach one)", space) } u.captureShipProofs(space, store) @@ -69,11 +117,34 @@ func (u *Forge) UploadBlob(ctx context.Context, space did.DID, digest multihash. forgeclient.WithPrecomputedDigest(digest, uint64(size)), forgeclient.WithPutClient(u.putClient), forgeclient.WithProofStore(store), + forgeclient.WithConclude(cfg.conclude), ) if err != nil { - return BlobLocation{}, fmt.Errorf("uploader: upload blob: %w", err) + return UploadedBlob{}, fmt.Errorf("uploader: upload blob: %w", err) } - return locationFromAdded(added) + return uploadedFromAdded(added) +} + +// uploadedFromAdded maps a forgeclient result into the uploader's shape: an +// accepted blob's location commitment parses into a BlobLocation; a parked +// (unconcluded) blob carries its pending-accept state through. +func uploadedFromAdded(added forgeclient.AddedBlob) (UploadedBlob, error) { + ub := UploadedBlob{ + Digest: added.Digest, + Size: int64(added.Size), + AddTask: added.AddTask, + AcceptTask: added.AcceptTask, + } + if added.Location == nil { + ub.PutInvocation = added.PutInvocation + return ub, nil + } + loc, err := locationFromAdded(added) + if err != nil { + return UploadedBlob{}, err + } + ub.Location = &loc + return ub, nil } // locationFromAdded parses the /assert/location commitment piri issued at @@ -97,65 +168,29 @@ func locationFromAdded(added forgeclient.AddedBlob) (BlobLocation, error) { var _ BodyUploader = (*Forge)(nil) -// ParkedBlobState is the persistable state of a blob that is durable on the -// provider but not yet accepted (the /http/put conclude is deferred). It is -// stored in the blob_parks table between UploadPart and Complete/Abort. -type ParkedBlobState = forgeclient.ParkedBlob - -// DeferredBodyUploader is the decomposed BodyUploader for multipart's -// deferred accept: UploadBlobParked makes the bytes durable (parked) at -// UploadPart; ConcludeBlob triggers accept at Complete; AbortBlob -// abandons a parked blob at Abort. Exactly one of UploadBlobParked's returns -// is non-nil — an already-accepted (deduped) blob completes immediately. +// DeferredBodyUploader extends BodyUploader for multipart's deferred accept: +// UploadBlob with WithConclude(false) makes the bytes durable (parked) at +// UploadPart; ConcludeBlob triggers accept at Complete; AbortBlob abandons a +// parked blob at Abort. type DeferredBodyUploader interface { - UploadBlobParked(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string) (*ParkedBlobState, *BlobLocation, error) - ConcludeBlob(ctx context.Context, space did.DID, parked ParkedBlobState) (BlobLocation, error) + BodyUploader + ConcludeBlob(ctx context.Context, space did.DID, parked UploadedBlob) (BlobLocation, error) AbortBlob(ctx context.Context, space did.DID, digest multihash.Multihash, cause cid.Cid) error } -// UploadBlobParked makes one spooled blob durable on the provider WITHOUT -// accepting it: /blob/add + PUT, conclude deferred. Returns the persistable -// park state, or the completed location when the provider already held -// accepted bytes for this content (dedup). Like UploadBlob it runs on the -// request ctx and requires the request-scoped proof store, which it captures -// for the off-request abort path (the session-expiry sweeper). -func (u *Forge) UploadBlobParked(ctx context.Context, space did.DID, digest multihash.Multihash, size int64, localPath string) (*ParkedBlobState, *BlobLocation, error) { - f, err := os.Open(localPath) - if err != nil { - return nil, nil, fmt.Errorf("uploader: open spooled blob %s: %w", localPath, err) - } - defer f.Close() - - store, ok := reqscope.ProofStore(ctx) - if !ok { - return nil, nil, fmt.Errorf("uploader: no request-scoped proof store for space %s (IAM layer did not attach one)", space) - } - u.captureShipProofs(space, store) - - parked, added, err := u.client.BlobAddParked(ctx, space, f, - forgeclient.WithPrecomputedDigest(digest, uint64(size)), - forgeclient.WithPutClient(u.putClient), - forgeclient.WithProofStore(store), - ) - if err != nil { - return nil, nil, fmt.Errorf("uploader: park blob: %w", err) - } - if added != nil { - loc, err := locationFromAdded(*added) - if err != nil { - return nil, nil, err - } - return nil, &loc, nil - } - return parked, nil, nil -} - // ConcludeBlob finishes a parked upload: it concludes the deferred /http/put // receipt (triggering /blob/accept on the provider) and returns the published -// location. The conclude carries no space proof (accept is owned by sprue), -// so no proof store is required. Safe to retry. -func (u *Forge) ConcludeBlob(ctx context.Context, space did.DID, parked ParkedBlobState) (BlobLocation, error) { - added, err := u.client.BlobConclude(ctx, space, parked) +// location. parked is the UploadedBlob a WithConclude(false) upload returned +// (rehydrated from its blob_parks row). The conclude carries no space proof +// (accept is owned by sprue), so no proof store is required. Safe to retry. +func (u *Forge) ConcludeBlob(ctx context.Context, space did.DID, parked UploadedBlob) (BlobLocation, error) { + added, err := u.client.BlobConclude(ctx, space, forgeclient.AddedBlob{ + Digest: parked.Digest, + Size: uint64(parked.Size), + AddTask: parked.AddTask, + AcceptTask: parked.AcceptTask, + PutInvocation: parked.PutInvocation, + }) if err != nil { return BlobLocation{}, fmt.Errorf("uploader: conclude blob: %w", err) } From c4bfe221a382d361e4ad9b9fb33df5a5603eb9a0 Mon Sep 17 00:00:00 2001 From: frrist Date: Thu, 30 Jul 2026 12:38:14 -0700 Subject: [PATCH 19/21] itest: grant the hilt#35 multipart permissions to the test tenant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hilt#35 (which superseded hilt#36's bare blob.Abort grant) makes the multipart operations first-class in the authorizer: DELETE ?uploadId now classifies as s3:AbortMultipartUpload instead of falling through to s3:DeleteObject. The harness's hiltAllPermissions list predates that, so once hilt:main published, every client-issued S3 abort was denied at request authorization — the whole AbortMultipartUpload versity group and TestForgeDeferredMultipart/AbortRejects failed CI with 403 AccessDenied. Add s3:AbortMultipartUpload plus the two multipart List permissions, keeping the list's contract ("every S3 permission hilt recognizes"). Also refresh the conditional_writes xfail rationale: with blob.Abort now granted its failure moved — the conditional matrix passes and teardown fails with BucketNotEmpty on DeleteBucket, which needs its own diagnosis before the row can promote. Validated against the published piri/hilt :main images (post piri#30/hilt#35): deferred gate + all versity multipart groups pass; conditional_writes still xfails (no unexpected pass). Co-Authored-By: Claude Fable 5 --- itest/stack_test.go | 6 ++++++ itest/versity_multipart_test.go | 10 +++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/itest/stack_test.go b/itest/stack_test.go index ca1ae61..9cc68bb 100644 --- a/itest/stack_test.go +++ b/itest/stack_test.go @@ -202,6 +202,12 @@ var hiltAllPermissions = []string{ "s3:CreateBucket", "s3:ListAllMyBuckets", "s3:DeleteBucket", + // Multipart operations, first-class in hilt since fil-forge/hilt#35: + // AbortMultipartUpload carries the blob.Abort + blob.Remove delegations + // the abort leg invokes; the two List ops are catalog reads. + "s3:AbortMultipartUpload", + "s3:ListMultipartUploadParts", + "s3:ListBucketMultipartUploads", } // hiltProvisionTenant provisions tenantID in hilt with an all-permission diff --git a/itest/versity_multipart_test.go b/itest/versity_multipart_test.go index f1d2de6..01201a2 100644 --- a/itest/versity_multipart_test.go +++ b/itest/versity_multipart_test.go @@ -158,11 +158,11 @@ var completeMultipartPass = []forgeCase{ // racey_data_integrity additionally leans on atomic concurrent overwrites. var completeMultipartXFail = []forgeCase{ // The conditional matrix itself passes; the case then fails its bucket - // teardown: upstream's teardown never aborts in-flight uploads, so - // DeleteBucket implicitly aborts them (s3frontend/bucket.go), but the - // /blob/abort goes out proofless — hilt's s3perm map delegates no - // blob.Abort today. fil-forge/hilt#36 adds it to the write set; - // promote once a hilt image with it publishes. + // teardown, now with BucketNotEmpty from DeleteBucket (an object the + // upstream teardown's listing misses survives into the delete). The + // original blocker — /blob/abort going out proofless — is gone since + // fil-forge/hilt#35 granted blob.Abort with the write set; the teardown + // residue needs its own diagnosis before this row can promote. {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, {name: "invalid_checksum_part", fn: integration.CompleteMultipartUpload_invalid_checksum_part}, {name: "multiple_checksum_part", fn: integration.CompleteMultipartUpload_multiple_checksum_part}, From 4ec7100758641136098a510751fcd99d27f5e530 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 5 Aug 2026 16:37:17 -0700 Subject: [PATCH 20/21] build(itest): bump smelt for the piri blob/release + blob/reject delegations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit smelt#19 seeds the piri→upload delegation with blob/release and blob/reject, so sprue's forwards carry proofs and piri accepts them — TestForgeDeleteReleasesNetworkBlob and TestForgeDeferredMultipart pass against this pin (validated pre-merge via a local replace). Co-Authored-By: Claude Fable 5 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8b9ab5b..9baf2f1 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/fil-forge/hilt v0.0.1-0.20260724134448-ba71f843f6a4 github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 github.com/fil-forge/libforge v0.0.0-20260727220215-5e299c46f62f - github.com/fil-forge/smelt v0.0.0-20260720130429-63116166a06c + github.com/fil-forge/smelt v0.0.0-20260805233620-f0b69e68aad1 github.com/fil-forge/ucantone v0.0.0-20260727203046-ccb77059de44 github.com/fil-forge/versitygw v0.0.0-20260716095011-7a65883d595a github.com/gofiber/fiber/v3 v3.4.0 diff --git a/go.sum b/go.sum index 4006147..f16426a 100644 --- a/go.sum +++ b/go.sum @@ -185,8 +185,8 @@ github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717 h1:W github.com/fil-forge/indexing-service v1.13.5-0.20260619142411-efe3f5fab717/go.mod h1:wFcakLohOqpMRkJzWRdGFGHRFpGB+PpEMCm9wkt2cqU= github.com/fil-forge/libforge v0.0.0-20260727220215-5e299c46f62f h1:QzgMg8GIE4IhgOE/7DBHFU/4T5oU7J4vAKxbUPKJPGA= github.com/fil-forge/libforge v0.0.0-20260727220215-5e299c46f62f/go.mod h1:0kXihIQ4L2uZ00nR5XrZ/Y8Db7Ht/qQNuiWslwMJ95M= -github.com/fil-forge/smelt v0.0.0-20260720130429-63116166a06c h1:WHvsleEU6ZiNYDFgLx6KtorXulD+IuLiorMgmp4Th8s= -github.com/fil-forge/smelt v0.0.0-20260720130429-63116166a06c/go.mod h1:NM/mk/XiP1Kzsy9HWGQeLsosPUvVY3bIrkUzqswThpU= +github.com/fil-forge/smelt v0.0.0-20260805233620-f0b69e68aad1 h1:0Ia79MRlX3q/wk5/XCvZYou7rkov4xQMpCuNzAb+KJY= +github.com/fil-forge/smelt v0.0.0-20260805233620-f0b69e68aad1/go.mod h1:czwgD0nnuQ8h3GOHdwQ2Guw3C9R3zYP3K/mE+DEyh+Y= github.com/fil-forge/ucantone v0.0.0-20260727203046-ccb77059de44 h1:ofvb2Qq7++VPRelGsLbtnd1ZMKVT4n4QGoa79BhJ6VQ= github.com/fil-forge/ucantone v0.0.0-20260727203046-ccb77059de44/go.mod h1:oFY5BfD0bDeodGlbBHh3/nK99MAS93rGXjoQz7s5qgE= github.com/fil-forge/versitygw v0.0.0-20260716095011-7a65883d595a h1:lDwnNmF4LNbevx/YYNCC0CazMiL4eIe38MvfqbRR3RA= From d55ab0800ef7e56932353172b347d354da024de0 Mon Sep 17 00:00:00 2001 From: frrist Date: Wed, 5 Aug 2026 16:52:42 -0700 Subject: [PATCH 21/21] itest: promote conditional_writes + checksums_success (delete-release chain complete) Both rows were teardown-blocked on superseded blobs leaking in the space: the write-path release went out proofless. hilt#37 grants blob.Remove with the write set and smelt#19 seeds the piri blob/release delegation, so the releases now carry proofs end-to-end and CI's unexpected-pass ratchet flagged both rows for promotion. Co-Authored-By: Claude Fable 5 --- itest/versity_multipart_test.go | 11 ++++------- itest/versity_object_test.go | 8 ++++---- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/itest/versity_multipart_test.go b/itest/versity_multipart_test.go index 01201a2..db754e3 100644 --- a/itest/versity_multipart_test.go +++ b/itest/versity_multipart_test.go @@ -145,6 +145,10 @@ var completeMultipartPass = []forgeCase{ {name: "with_metadata", fn: integration.CompleteMultipartUpload_with_metadata}, {name: "success", fn: integration.CompleteMultipartUpload_success}, {name: "already_completed", fn: integration.CompleteMultipartUpload_already_completed}, + // The conditional matrix's overwrite chains release superseded blobs: + // needs hilt ≥ #37 (blob.Remove in the write set) and smelt ≥ #19 (the + // piri blob/release delegation) so the releases carry proofs end-to-end. + {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, // racey_success races ten concurrent 25 MiB multipart uploads of one key // under a 30s client deadline — on a host simultaneously running the // smelt stack its outcome depends on load, not S3 semantics, so it is @@ -157,13 +161,6 @@ var completeMultipartPass = []forgeCase{ // Part-level / composite-checksum verification is FIL-620; // racey_data_integrity additionally leans on atomic concurrent overwrites. var completeMultipartXFail = []forgeCase{ - // The conditional matrix itself passes; the case then fails its bucket - // teardown, now with BucketNotEmpty from DeleteBucket (an object the - // upstream teardown's listing misses survives into the delete). The - // original blocker — /blob/abort going out proofless — is gone since - // fil-forge/hilt#35 granted blob.Abort with the write set; the teardown - // residue needs its own diagnosis before this row can promote. - {name: "conditional_writes", fn: integration.CompleteMultipartUpload_conditional_writes}, {name: "invalid_checksum_part", fn: integration.CompleteMultipartUpload_invalid_checksum_part}, {name: "multiple_checksum_part", fn: integration.CompleteMultipartUpload_multiple_checksum_part}, {name: "incorrect_checksum_part", fn: integration.CompleteMultipartUpload_incorrect_checksum_part}, diff --git a/itest/versity_object_test.go b/itest/versity_object_test.go index 5333010..e7ca598 100644 --- a/itest/versity_object_test.go +++ b/itest/versity_object_test.go @@ -18,6 +18,10 @@ import ( var putObjectPass = []forgeCase{ {name: "checksum_algorithm_and_header_mismatch", fn: integration.PutObject_checksum_algorithm_and_header_mismatch}, + // Overwrites superseding checksummed bodies release their blobs: needs + // hilt ≥ #37 (blob.Remove in the write set) and smelt ≥ #19 (the piri + // blob/release delegation). + {name: "checksums_success", fn: integration.PutObject_checksums_success}, {name: "dir_object_default_checksum", fn: integration.PutObject_dir_object_default_checksum}, {name: "dir_object_checksums_success", fn: integration.PutObject_dir_object_checksums_success}, {name: "incorrect_checksums", fn: integration.PutObject_incorrect_checksums}, @@ -44,10 +48,6 @@ var putObjectPass = []forgeCase{ } var putObjectXFail = []forgeCase{ - // Still teardown-blocked even with DeleteObject's blob release: the - // case's checksummed bodies survive deletion and the bucket delete - // 409s (BucketNotEmpty). - {name: "checksums_success", fn: integration.PutObject_checksums_success}, // The incorrect_md5 subcheck expects 400 InvalidDigest; ingot 500s. {name: "md5", fn: integration.PutObject_md5}, // A metadata-combining re-PUT is denied (403) under the hilt authorize