FEE wrap material on blob_encryption_params (FIL-480)

.claude

@@ -0,0 +1 @@
+/Users/bajtos/src/fil-forge/ingot/.claude

inmem/store.go

@@ -59,6 +59,7 @@ type MemStore struct {
 	blobRefs   map[claimKey]registry.BlobClaim
 	intents    map[string]registry.UploadIntent          // keyed by string(digest)
 	locations  map[locKey]registry.BlobLocation          // keyed by (space, digest)
+	encParams  map[locKey]registry.BlobEncryptionParams  // keyed by (space, digest)
 	inclusions map[locKey]registry.BlobInclusion         // keyed by (space, digest)
 	parks      map[string]registry.BlobPark              // keyed by string(digest)
 	sessions   map[string]registry.MultipartSession      // keyed by uploadID
@@ -67,8 +68,9 @@ type MemStore struct {
 	revCursor  *registry.RevocationCursor                // the single revocation_cursor row
 }
 
-// claimKey / locKey are the composite map keys for the blob_refs and
-// blob_locations tables (digest bytes carried as a string for comparability).
+// claimKey / locKey are the composite map keys for the blob_refs and the
+// (space, digest)-keyed tables — blob_locations, blob_encryption_params and
+// shard_inclusions (digest bytes carried as a string for comparability).
 type claimKey struct {
 	digest, bucket, objectKey, versionID string
 }
@@ -87,6 +89,7 @@ func NewMemStore() *MemStore {
 		blobRefs:   map[claimKey]registry.BlobClaim{},
 		intents:    map[string]registry.UploadIntent{},
 		locations:  map[locKey]registry.BlobLocation{},
+		encParams:  map[locKey]registry.BlobEncryptionParams{},
 		inclusions: map[locKey]registry.BlobInclusion{},
 		parks:      map[string]registry.BlobPark{},
 		sessions:   map[string]registry.MultipartSession{},

inmem/stores.go

@@ -11,15 +11,16 @@ import (
 )
 
 // In-memory implementations of the architecture's relational stores
-// (registry.BlobRefStore / IntentStore / LocationStore / MultipartStore /
-// GCStore), mirroring the Postgres tables so the in-process suite and
-// standalone mode exercise the same write/read/delete code paths.
+// (registry.BlobRefStore / IntentStore / LocationStore / EncryptionParamsStore /
+// MultipartStore / GCStore), mirroring the Postgres tables so the in-process
+// suite and standalone mode exercise the same write/read/delete code paths.
 
 // Compile-time assertions: *MemStore satisfies every store interface.
 var (
 	_ registry.BlobRefStore          = (*MemStore)(nil)
 	_ registry.IntentStore           = (*MemStore)(nil)
 	_ registry.LocationStore         = (*MemStore)(nil)
+	_ registry.EncryptionParamsStore = (*MemStore)(nil)
 	_ registry.InclusionStore        = (*MemStore)(nil)
 	_ registry.MultipartStore        = (*MemStore)(nil)
 	_ registry.GCStore               = (*MemStore)(nil)
@@ -126,9 +127,7 @@ func (m *MemStore) DeleteIntent(_ context.Context, digest []byte) error {
 func (m *MemStore) PutLocation(_ context.Context, loc registry.BlobLocation) error {
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	cp := loc
-	cp.Digest = cloneBytes(loc.Digest)
-	m.locations[locKey{loc.Space, string(loc.Digest)}] = cp
+	m.locations[locKey{loc.Space, string(loc.Digest)}] = cloneLocation(loc)
 	return nil
 }
 
@@ -139,8 +138,7 @@ func (m *MemStore) GetLocation(_ context.Context, space did.DID, digest []byte)
 	if !ok {
 		return nil, registry.ErrNotFound
 	}
-	cp := loc
-	cp.Digest = cloneBytes(loc.Digest)
+	cp := cloneLocation(loc)
 	return &cp, nil
 }
 
@@ -151,6 +149,37 @@ func (m *MemStore) DeleteLocation(_ context.Context, space did.DID, digest []byt
 	return nil
 }
 
+// EncryptionParamsStore ======================================================
+
+func (m *MemStore) PutEncryptionParams(_ context.Context, params registry.BlobEncryptionParams) error {
+	// Match the Postgres store, whose columns are all NOT NULL.
+	if err := params.Validate(); err != nil {
+		return err
+	}
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.encParams[locKey{params.Space, string(params.Digest)}] = cloneEncryptionParams(params)
+	return nil
+}
+
+func (m *MemStore) GetEncryptionParams(_ context.Context, space did.DID, digest []byte) (*registry.BlobEncryptionParams, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	params, ok := m.encParams[locKey{space, string(digest)}]
+	if !ok {
+		return nil, registry.ErrNotFound
+	}
+	cp := cloneEncryptionParams(params)
+	return &cp, nil
+}
+
+func (m *MemStore) DeleteEncryptionParams(_ context.Context, space did.DID, digest []byte) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	delete(m.encParams, locKey{space, string(digest)})
+	return nil
+}
+
 // ParkStore ==================================================================
 
 func (m *MemStore) PutPark(_ context.Context, p registry.BlobPark) error {
@@ -406,6 +435,22 @@ func cloneSession(s registry.MultipartSession) registry.MultipartSession {
 	return s
 }
 
+// cloneLocation deep-copies a BlobLocation's digest so the stored copy and any
+// returned copy never alias the caller's slice.
+func cloneLocation(loc registry.BlobLocation) registry.BlobLocation {
+	loc.Digest = cloneBytes(loc.Digest)
+	return loc
+}
+
+// cloneEncryptionParams deep-copies a BlobEncryptionParams' byte-slice fields
+// so the stored copy and any returned copy never alias the caller's slices.
+func cloneEncryptionParams(p registry.BlobEncryptionParams) registry.BlobEncryptionParams {
+	p.Digest = cloneBytes(p.Digest)
+	p.BaseNonce = cloneBytes(p.BaseNonce)
+	p.AAD = cloneBytes(p.AAD)
+	return p
+}
+
 func clonePart(p registry.MultipartPart) registry.MultipartPart {
 	p.ETagMD5 = cloneBytes(p.ETagMD5)
 	if p.BlobDigests != nil {

inmem/stores_test.go

@@ -2,6 +2,8 @@ package inmem
 
 import (
 	"context"
+	"errors"
+	"reflect"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -254,6 +256,166 @@ func TestLocations_RoundTrip(t *testing.T) {
 	}
 }
 
+// feeParams returns a complete parameter set for space/digest, the shape a FEE
+// envelope's row takes.
+func feeParams(space did.DID, digest []byte) registry.BlobEncryptionParams {
+	return registry.BlobEncryptionParams{
+		Space:              space,
+		Digest:             digest,
+		TenantRecipientKID: "did:key:tenant#wrap",
+		HeaderLen:          212,
+		BaseNonce:          []byte("nonce07"),
+		ChunkSize:          65536,
+		AAD:                []byte("cose-enc-structure"),
+	}
+}
+
+func TestEncryptionParams_RoundTrip(t *testing.T) {
+	ctx := context.Background()
+	m := NewMemStore()
+	space := testutil.RandomDID(t)
+	digest := []byte("enc-digest")
+	want := feeParams(space, digest)
+
+	if err := m.PutEncryptionParams(ctx, want); err != nil {
+		t.Fatalf("PutEncryptionParams: %v", err)
+	}
+	got, err := m.GetEncryptionParams(ctx, space, digest)
+	if err != nil {
+		t.Fatalf("GetEncryptionParams: %v", err)
+	}
+	if !reflect.DeepEqual(*got, want) {
+		t.Fatalf("GetEncryptionParams = %+v, want %+v", *got, want)
+	}
+}
+
+// A blob with no row is a plaintext blob, which is how the read path learns not
+// to decrypt.
+func TestEncryptionParams_MissingIsNotFound(t *testing.T) {
+	ctx := context.Background()
+	m := NewMemStore()
+
+	_, err := m.GetEncryptionParams(ctx, testutil.RandomDID(t), []byte("absent"))
+	if !errors.Is(err, registry.ErrNotFound) {
+		t.Fatalf("GetEncryptionParams err = %v, want ErrNotFound", err)
+	}
+}
+
+func TestEncryptionParams_DeleteRemovesRow(t *testing.T) {
+	ctx := context.Background()
+	m := NewMemStore()
+	space := testutil.RandomDID(t)
+	digest := []byte("enc-digest")
+	if err := m.PutEncryptionParams(ctx, feeParams(space, digest)); err != nil {
+		t.Fatalf("PutEncryptionParams: %v", err)
+	}
+
+	if err := m.DeleteEncryptionParams(ctx, space, digest); err != nil {
+		t.Fatalf("DeleteEncryptionParams: %v", err)
+	}
+	if _, err := m.GetEncryptionParams(ctx, space, digest); !errors.Is(err, registry.ErrNotFound) {
+		t.Fatalf("GetEncryptionParams after delete err = %v, want ErrNotFound", err)
+	}
+}
+
+func TestEncryptionParams_DeleteIsIdempotent(t *testing.T) {
+	ctx := context.Background()
+	m := NewMemStore()
+
+	if err := m.DeleteEncryptionParams(ctx, testutil.RandomDID(t), []byte("absent")); err != nil {
+		t.Fatalf("DeleteEncryptionParams(absent): %v", err)
+	}
+}
+
+// The store must not alias the caller's byte slices, nor let a caller reach
+// back into it through a returned copy.
+func TestEncryptionParams_NoSliceAliasing(t *testing.T) {
+	ctx := context.Background()
+	m := NewMemStore()
+	space := testutil.RandomDID(t)
+	digest := []byte("enc-digest")
+	params := feeParams(space, digest)
+
+	if err := m.PutEncryptionParams(ctx, params); err != nil {
+		t.Fatalf("PutEncryptionParams: %v", err)
+	}
+	params.BaseNonce[0] = 'X'
+	params.AAD[0] = 'X'
+
+	got, err := m.GetEncryptionParams(ctx, space, digest)
+	if err != nil {
+		t.Fatalf("GetEncryptionParams: %v", err)
+	}
+	got.BaseNonce[0] = 'Y'
+
+	again, err := m.GetEncryptionParams(ctx, space, digest)
+	if err != nil {
+		t.Fatalf("GetEncryptionParams (again): %v", err)
+	}
+	if !reflect.DeepEqual(*again, feeParams(space, digest)) {
+		t.Fatalf("stored params were aliased: %+v", *again)
+	}
+}
+
+// Every parameter is required: a row missing any one of them could not be
+// decrypted with, so PutEncryptionParams rejects it and stores nothing.
+func TestEncryptionParams_IncompleteRejected(t *testing.T) {
+	space := testutil.RandomDID(t)
+	digest := []byte("enc-digest")
+
+	cases := []struct {
+		name   string
+		mutate func(*registry.BlobEncryptionParams)
+	}{
+		{"no space", func(p *registry.BlobEncryptionParams) { p.Space = did.Undef }},
+		{"no digest", func(p *registry.BlobEncryptionParams) { p.Digest = nil }},
+		{"no recipient kid", func(p *registry.BlobEncryptionParams) { p.TenantRecipientKID = "" }},
+		{"no header length", func(p *registry.BlobEncryptionParams) { p.HeaderLen = 0 }},
+		{"no base nonce", func(p *registry.BlobEncryptionParams) { p.BaseNonce = nil }},
+		{"no chunk size", func(p *registry.BlobEncryptionParams) { p.ChunkSize = 0 }},
+		{"no AAD", func(p *registry.BlobEncryptionParams) { p.AAD = nil }},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx := context.Background()
+			m := NewMemStore()
+			params := feeParams(space, digest)
+			tc.mutate(&params)
+
+			if err := m.PutEncryptionParams(ctx, params); !errors.Is(err, registry.ErrInvalidEncryptionParams) {
+				t.Fatalf("PutEncryptionParams err = %v, want ErrInvalidEncryptionParams", err)
+			}
+			if _, err := m.GetEncryptionParams(ctx, space, digest); !errors.Is(err, registry.ErrNotFound) {
+				t.Fatalf("incomplete params leaked a row")
+			}
+		})
+	}
+}
+
+// The two tables have independent lifecycles and no cascade between them:
+// deleting a location leaves the encryption parameters in place, which is why a
+// caller removing a blob must delete both.
+func TestEncryptionParams_IndependentOfLocation(t *testing.T) {
+	ctx := context.Background()
+	m := NewMemStore()
+	space := testutil.RandomDID(t)
+	digest := []byte("enc-digest")
+	if err := m.PutLocation(ctx, registry.BlobLocation{Space: space, Digest: digest, Provider: "did:piri:1", URL: "http://piri/enc", Size: 4096}); err != nil {
+		t.Fatalf("PutLocation: %v", err)
+	}
+	if err := m.PutEncryptionParams(ctx, feeParams(space, digest)); err != nil {
+		t.Fatalf("PutEncryptionParams: %v", err)
+	}
+
+	if err := m.DeleteLocation(ctx, space, digest); err != nil {
+		t.Fatalf("DeleteLocation: %v", err)
+	}
+
+	if _, err := m.GetEncryptionParams(ctx, space, digest); err != nil {
+		t.Fatalf("DeleteLocation shredded the encryption params: %v", err)
+	}
+}
+
 func TestRevocationCursor_UpsertRoundTrip(t *testing.T) {
 	ctx := context.Background()
 	m := NewMemStore()

migrations/sql/00014_blob_encryption.sql

@@ -0,0 +1,49 @@
+-- +goose Up
+-- FEE (FilOne encryption envelope) per-blob encryption parameters. When Ingot
+-- encrypts an object's body, each body blob is stored as an independent
+-- COSE/STREAM ciphertext envelope; a range GET must be able to decrypt any byte
+-- span of that envelope WITHOUT first fetching and parsing its header. A row
+-- here caches exactly the inputs the read path's decryptor needs, so a read
+-- unwraps the CEK (held in OpenBao, under the region KEK) and goes straight to
+-- a body-range fetch — no envelope-header round-trip.
+--
+-- The existence of a row is what marks a blob as encrypted, so every column is
+-- NOT NULL: there is no such thing as a half-populated parameter set the
+-- decrypt path could not use. An unencrypted blob simply has no row.
+--
+-- Deliberately a separate table from ingot.blob_locations, and deliberately
+-- WITHOUT a foreign key to it. blob_locations is a reconstructible cache of the
+-- indexing-service contract — every row can be re-derived from the indexer or
+-- the accept receipt, and the table goes away when the topology moves to a real
+-- indexer. A row here is instead the marker that a blob is encrypted, so the
+-- two have independent lifecycles, and an FK would additionally force
+-- location-before-parameters write ordering.
+--
+-- aad holds the whole COSE Enc_structure rather than just the protected header,
+-- because the structure's context string differs between a COSE_Encrypt and a
+-- COSE_Encrypt0 and a bare row cannot record which form was used. The protected
+-- header stays recoverable from it as element 1.
+--
+-- No key material is stored here: the region-KEK-wrapped CEK and its key
+-- version live in OpenBao. Per-blob crypto-shred is deleting the key there;
+-- deleting this row only drops the cached decrypt parameters. Because there is no cascade, a caller removing a
+-- blob must delete here as well as from blob_locations.
+CREATE TABLE ingot.blob_encryption_params (
+    space                text   NOT NULL,
+    digest               bytea  NOT NULL,                 -- ciphertext blob multihash, as in blob_locations
+    tenant_recipient_kid text   NOT NULL                  -- opaque id of the Hilt wrap key (insurance-recovery unwrap)
+                             CHECK (tenant_recipient_kid <> ''),
+    header_len           bigint NOT NULL                  -- encoded envelope length; the ciphertext starts at this offset
+                             CHECK (header_len > 0),
+    base_nonce           bytea  NOT NULL                  -- COSE iv: the STREAM nonce seed for this blob's ciphertext
+                             CHECK (octet_length(base_nonce) > 0),
+    chunk_size           bigint NOT NULL                  -- FEE chunk size from the COSE protected header
+                             CHECK (chunk_size > 0),
+    aad                  bytea  NOT NULL                  -- COSE Enc_structure, bound into every chunk's GCM tag
+                             CHECK (octet_length(aad) > 0),
+    created_at           timestamptz NOT NULL DEFAULT now(),
+    PRIMARY KEY (space, digest)
+);
+
+-- +goose Down
+DROP TABLE ingot.blob_encryption_params;

migrations/up_live_test.go

@@ -42,7 +42,7 @@ func TestUp_Live(t *testing.T) {
 
 	want := []string{
 		"buckets", "segments", "segment_op_roots",
-		"blob_refs", "upload_intents", "blob_locations",
+		"blob_refs", "upload_intents", "blob_locations", "blob_encryption_params",
 		"multipart_sessions", "multipart_parts", "gc_candidates",
 	}
 	for _, tbl := range want {
@@ -71,4 +71,24 @@ func TestUp_Live(t *testing.T) {
 			t.Errorf("column ingot.buckets.%s does not exist after migration", col)
 		}
 	}
+
+	// blob_encryption_params (00014): every column exists and is NOT NULL — the
+	// presence of a row is what marks a blob as encrypted, so there is no such
+	// thing as a half-populated parameter set.
+	for _, col := range []string{
+		"space", "digest", "tenant_recipient_kid", "header_len", "base_nonce",
+		"chunk_size", "aad", "created_at",
+	} {
+		var nullable string
+		err := pool.QueryRow(ctx,
+			`SELECT is_nullable FROM information_schema.columns
+			 WHERE table_schema = 'ingot' AND table_name = 'blob_encryption_params' AND column_name = $1`, col).Scan(&nullable)
+		if err != nil {
+			t.Errorf("column ingot.blob_encryption_params.%s missing after migration: %v", col, err)
+			continue
+		}
+		if nullable != "NO" {
+			t.Errorf("column ingot.blob_encryption_params.%s is_nullable = %q, want NO", col, nullable)
+		}
+	}
 }