FEE wrap material on blob_encryption_params (FIL-480)

inmem/stores.go

@@ -119,11 +119,13 @@ func (m *MemStore) DeleteIntent(_ context.Context, digest []byte) error {
 // LocationStore ==============================================================
 
 func (m *MemStore) PutLocation(_ context.Context, loc registry.BlobLocation) error {
+	// Match the Postgres store: reject a partial FEE set (see ValidateFEE).
+	if err := loc.ValidateFEE(); err != nil {
+		return err
+	}
 	m.mu.Lock()
 	defer m.mu.Unlock()
-	cp := loc
-	cp.Digest = cloneBytes(loc.Digest)
-	m.locations[locKey{loc.Space, string(loc.Digest)}] = cp
+	m.locations[locKey{loc.Space, string(loc.Digest)}] = cloneLocation(loc)
 	return nil
 }
 
@@ -134,8 +136,7 @@ func (m *MemStore) GetLocation(_ context.Context, space string, digest []byte) (
 	if !ok {
 		return nil, registry.ErrNotFound
 	}
-	cp := loc
-	cp.Digest = cloneBytes(loc.Digest)
+	cp := cloneLocation(loc)
 	return &cp, nil
 }
 
@@ -259,6 +260,17 @@ func cloneSession(s registry.MultipartSession) registry.MultipartSession {
 	return s
 }
 
+// cloneLocation deep-copies a BlobLocation's byte-slice fields (the digest and
+// the FEE wrap material) so the stored copy and any returned copy never alias
+// the caller's slices. Nil slices stay nil, preserving "unencrypted blob".
+func cloneLocation(loc registry.BlobLocation) registry.BlobLocation {
+	loc.Digest = cloneBytes(loc.Digest)
+	loc.RegionWrappedCEK = cloneBytes(loc.RegionWrappedCEK)
+	loc.BaseNonce = cloneBytes(loc.BaseNonce)
+	loc.ProtectedHeader = cloneBytes(loc.ProtectedHeader)
+	return loc
+}
+
 func clonePart(p registry.MultipartPart) registry.MultipartPart {
 	p.ETagMD5 = cloneBytes(p.ETagMD5)
 	if p.BlobDigests != nil {

inmem/stores_test.go

@@ -2,6 +2,7 @@ package inmem
 
 import (
 	"context"
+	"errors"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -239,6 +240,11 @@ func TestLocations_RoundTrip(t *testing.T) {
 	if loc.URL != "http://piri/blob" || loc.Size != 100 || loc.Provider != "did:piri:1" {
 		t.Fatalf("GetLocation = %+v", loc)
 	}
+	// An unencrypted blob carries no FEE wrap material.
+	if loc.RegionWrappedCEK != nil || loc.BaseNonce != nil || loc.ProtectedHeader != nil ||
+		loc.RegionKeyVersion != "" || loc.TenantRecipientKID != "" || loc.ChunkSize != 0 {
+		t.Fatalf("unencrypted location carries wrap material: %+v", loc)
+	}
 	if _, err := m.GetLocation(ctx, "did:other", digest); err != registry.ErrNotFound {
 		t.Fatalf("GetLocation wrong space err = %v, want ErrNotFound", err)
 	}
@@ -250,6 +256,107 @@ func TestLocations_RoundTrip(t *testing.T) {
 	}
 }
 
+// TestLocations_FEEWrapMaterial round-trips the FEE wrap columns and verifies
+// that (a) the stored copy does not alias the caller's byte slices, (b) a
+// returned copy does not either, and (c) a re-wrap in place (a PutLocation with
+// a new RegionWrappedCEK/RegionKeyVersion) updates only the wrap material.
+func TestLocations_FEEWrapMaterial(t *testing.T) {
+	ctx := context.Background()
+	m := NewMemStore()
+	const space = "did:space:enc"
+	digest := []byte("enc-digest")
+
+	enc := registry.BlobLocation{
+		Space: space, Digest: digest, Provider: "did:piri:1", URL: "http://piri/enc", Size: 4096,
+		RegionWrappedCEK:   []byte("wrapped-cek"),
+		RegionKeyVersion:   "region-v1",
+		TenantRecipientKID: "did:key:tenant#wrap",
+		BaseNonce:          []byte("nonce07"),
+		ChunkSize:          65536,
+		ProtectedHeader:    []byte("cose-protected"),
+	}
+	if err := m.PutLocation(ctx, enc); err != nil {
+		t.Fatalf("PutLocation: %v", err)
+	}
+
+	// Mutating the caller's slices after Put must not corrupt the store.
+	enc.RegionWrappedCEK[0] = 'X'
+	enc.BaseNonce[0] = 'X'
+	enc.ProtectedHeader[0] = 'X'
+
+	got, err := m.GetLocation(ctx, space, digest)
+	if err != nil {
+		t.Fatalf("GetLocation: %v", err)
+	}
+	if string(got.RegionWrappedCEK) != "wrapped-cek" || string(got.BaseNonce) != "nonce07" ||
+		string(got.ProtectedHeader) != "cose-protected" {
+		t.Fatalf("store aliased caller slices: %+v", got)
+	}
+	if got.RegionKeyVersion != "region-v1" || got.TenantRecipientKID != "did:key:tenant#wrap" || got.ChunkSize != 65536 {
+		t.Fatalf("GetLocation wrap scalars = %+v", got)
+	}
+
+	// Mutating a returned copy must not corrupt the store either.
+	got.RegionWrappedCEK[0] = 'Y'
+	again, _ := m.GetLocation(ctx, space, digest)
+	if string(again.RegionWrappedCEK) != "wrapped-cek" {
+		t.Fatalf("store mutated through returned slice: %q", again.RegionWrappedCEK)
+	}
+
+	// Re-wrap in place: new CEK + key version, same location, other fields kept.
+	rewrapped := *again
+	rewrapped.RegionWrappedCEK = []byte("wrapped-cek-v2")
+	rewrapped.RegionKeyVersion = "region-v2"
+	if err := m.PutLocation(ctx, rewrapped); err != nil {
+		t.Fatalf("PutLocation (re-wrap): %v", err)
+	}
+	after, err := m.GetLocation(ctx, space, digest)
+	if err != nil {
+		t.Fatalf("GetLocation after re-wrap: %v", err)
+	}
+	if string(after.RegionWrappedCEK) != "wrapped-cek-v2" || after.RegionKeyVersion != "region-v2" {
+		t.Fatalf("re-wrap did not take: %+v", after)
+	}
+	if after.URL != "http://piri/enc" || string(after.BaseNonce) != "nonce07" || after.ChunkSize != 65536 {
+		t.Fatalf("re-wrap disturbed non-wrap fields: %+v", after)
+	}
+}
+
+// TestLocations_PartialFEE_Rejected verifies the all-or-nothing FEE invariant:
+// a location with some but not all wrap material is rejected with ErrPartialFEE
+// and never persisted, while a fully-absent (unencrypted) row is accepted.
+func TestLocations_PartialFEE_Rejected(t *testing.T) {
+	ctx := context.Background()
+	m := NewMemStore()
+	base := registry.BlobLocation{Space: "s", Digest: []byte("d"), Provider: "did:piri", URL: "u", Size: 10}
+
+	partials := map[string]func(*registry.BlobLocation){
+		"only wrapped CEK": func(l *registry.BlobLocation) { l.RegionWrappedCEK = []byte("cek") },
+		"only key version": func(l *registry.BlobLocation) { l.RegionKeyVersion = "v1" },
+		"only chunk size":  func(l *registry.BlobLocation) { l.ChunkSize = 4096 },
+		"missing protected": func(l *registry.BlobLocation) {
+			l.RegionWrappedCEK, l.RegionKeyVersion, l.TenantRecipientKID = []byte("cek"), "v1", "kid"
+			l.BaseNonce, l.ChunkSize = []byte("nonce07"), 4096 // ProtectedHeader left empty
+		},
+	}
+	for name, mutate := range partials {
+		loc := base
+		mutate(&loc)
+		if err := m.PutLocation(ctx, loc); !errors.Is(err, registry.ErrPartialFEE) {
+			t.Fatalf("PutLocation(%s) err = %v, want ErrPartialFEE", name, err)
+		}
+		// Nothing should have been stored.
+		if _, err := m.GetLocation(ctx, "s", []byte("d")); !errors.Is(err, registry.ErrNotFound) {
+			t.Fatalf("partial FEE (%s) leaked a row", name)
+		}
+	}
+
+	// Fully absent (unencrypted) is accepted.
+	if err := m.PutLocation(ctx, base); err != nil {
+		t.Fatalf("PutLocation(unencrypted): %v", err)
+	}
+}
+
 // helpers
 
 func mustAdd(t *testing.T, m *MemStore, c registry.BlobClaim) {

migrations/sql/00004_blob_encryption.sql

@@ -0,0 +1,32 @@
+-- +goose Up
+-- FEE (FilOne encryption envelope) per-blob wrap material, added to
+-- ingot.blob_locations. When Ingot encrypts an object's body, each body blob is
+-- stored as an independent COSE/STREAM ciphertext envelope; a range GET must be
+-- able to decrypt any byte span of that envelope WITHOUT first fetching and
+-- parsing its header. These columns cache exactly the inputs the read path's
+-- aesstream decryptor needs, so a read unwraps the CEK (under the region KEK)
+-- and goes straight to a body-range fetch — no envelope-header round-trip.
+--
+-- All columns are nullable. An unencrypted blob carries a location row with
+-- these columns NULL. Raw CEK bytes are never stored — only the CEK wrapped
+-- under the region KEK (region_wrapped_cek) plus the key-version / recipient
+-- identifiers needed to unwrap it. Per-blob crypto-shred is nulling these
+-- columns (or deleting the row) — no separate mechanism. Re-wrap under a
+-- rotated region key updates region_wrapped_cek and region_key_version in
+-- place.
+ALTER TABLE ingot.blob_locations
+    ADD COLUMN region_wrapped_cek   bytea,   -- CEK wrapped under the region KEK (A256KW)
+    ADD COLUMN region_key_version   text,    -- opaque id of the region KEK version used (rotation re-wrap)
+    ADD COLUMN tenant_recipient_kid text,    -- opaque id of the Hilt wrap key (insurance-recovery unwrap)
+    ADD COLUMN base_nonce           bytea,   -- COSE iv: the STREAM nonce seed for this blob's ciphertext
+    ADD COLUMN chunk_size           bigint,  -- FEE chunk size from the COSE protected header
+    ADD COLUMN protected_header     bytea;   -- raw COSE protected header bytes (Enc_structure/AAD reconstruction)
+
+-- +goose Down
+ALTER TABLE ingot.blob_locations
+    DROP COLUMN protected_header,
+    DROP COLUMN chunk_size,
+    DROP COLUMN base_nonce,
+    DROP COLUMN tenant_recipient_kid,
+    DROP COLUMN region_key_version,
+    DROP COLUMN region_wrapped_cek;

migrations/up_live_test.go

@@ -71,4 +71,22 @@ func TestUp_Live(t *testing.T) {
 			t.Errorf("column ingot.buckets.%s does not exist after migration", col)
 		}
 	}
+
+	// The FEE wrap columns added to blob_locations (00004) exist and are nullable.
+	for _, col := range []string{
+		"region_wrapped_cek", "region_key_version", "tenant_recipient_kid",
+		"base_nonce", "chunk_size", "protected_header",
+	} {
+		var nullable string
+		err := pool.QueryRow(ctx,
+			`SELECT is_nullable FROM information_schema.columns
+			 WHERE table_schema = 'ingot' AND table_name = 'blob_locations' AND column_name = $1`, col).Scan(&nullable)
+		if err != nil {
+			t.Errorf("column ingot.blob_locations.%s missing after migration: %v", col, err)
+			continue
+		}
+		if nullable != "YES" {
+			t.Errorf("column ingot.blob_locations.%s is_nullable = %q, want YES", col, nullable)
+		}
+	}
 }

registry/postgres_live_test.go

@@ -1,7 +1,9 @@
 package registry_test
 
 import (
+	"bytes"
 	"context"
+	"errors"
 	"os"
 	"testing"
 	"time"
@@ -127,13 +129,18 @@ func TestPostgresStores_Live(t *testing.T) {
 	})
 
 	t.Run("location round trip", func(t *testing.T) {
+		// Unencrypted blob: the FEE wrap columns store as NULL and read back zero.
 		if err := r.PutLocation(ctx, registry.BlobLocation{Space: "s", Digest: digest, Provider: "did:piri", URL: "http://piri/b", Size: 100}); err != nil {
 			t.Fatalf("PutLocation: %v", err)
 		}
 		loc, err := r.GetLocation(ctx, "s", digest)
 		if err != nil || loc.URL != "http://piri/b" || loc.Size != 100 {
 			t.Fatalf("GetLocation = %+v, err %v", loc, err)
 		}
+		if loc.RegionWrappedCEK != nil || loc.BaseNonce != nil || loc.ProtectedHeader != nil ||
+			loc.RegionKeyVersion != "" || loc.TenantRecipientKID != "" || loc.ChunkSize != 0 {
+			t.Fatalf("unencrypted location read back wrap material (NULL round-trip): %+v", loc)
+		}
 		if err := r.DeleteLocation(ctx, "s", digest); err != nil {
 			t.Fatalf("DeleteLocation: %v", err)
 		}
@@ -142,6 +149,67 @@ func TestPostgresStores_Live(t *testing.T) {
 		}
 	})
 
+	t.Run("location FEE wrap material", func(t *testing.T) {
+		// Binary bytea (with an embedded NUL) exercises real byte round-trips.
+		encDigest := []byte{0x00, 0x01, 0x02, 0xff}
+		enc := registry.BlobLocation{
+			Space: "s", Digest: encDigest, Provider: "did:piri", URL: "http://piri/enc", Size: 4096,
+			RegionWrappedCEK:   []byte{0x00, 0xde, 0xad, 0xbe, 0xef},
+			RegionKeyVersion:   "region-v1",
+			TenantRecipientKID: "did:key:tenant#wrap",
+			BaseNonce:          []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07},
+			ChunkSize:          65536,
+			ProtectedHeader:    []byte{0xa1, 0x00, 0x18, 0x20},
+		}
+		if err := r.PutLocation(ctx, enc); err != nil {
+			t.Fatalf("PutLocation (encrypted): %v", err)
+		}
+		got, err := r.GetLocation(ctx, "s", encDigest)
+		if err != nil {
+			t.Fatalf("GetLocation (encrypted): %v", err)
+		}
+		if !bytes.Equal(got.RegionWrappedCEK, enc.RegionWrappedCEK) ||
+			!bytes.Equal(got.BaseNonce, enc.BaseNonce) ||
+			!bytes.Equal(got.ProtectedHeader, enc.ProtectedHeader) {
+			t.Fatalf("bytea wrap material round-trip mismatch: %+v", got)
+		}
+		if got.RegionKeyVersion != "region-v1" || got.TenantRecipientKID != "did:key:tenant#wrap" || got.ChunkSize != 65536 {
+			t.Fatalf("wrap scalar round-trip mismatch: %+v", got)
+		}
+
+		// Re-wrap in place: a PutLocation upsert swaps the CEK + key version.
+		got.RegionWrappedCEK = []byte{0x11, 0x22, 0x33}
+		got.RegionKeyVersion = "region-v2"
+		if err := r.PutLocation(ctx, *got); err != nil {
+			t.Fatalf("PutLocation (re-wrap): %v", err)
+		}
+		after, err := r.GetLocation(ctx, "s", encDigest)
+		if err != nil || !bytes.Equal(after.RegionWrappedCEK, []byte{0x11, 0x22, 0x33}) || after.RegionKeyVersion != "region-v2" {
+			t.Fatalf("re-wrap round-trip = %+v, err %v", after, err)
+		}
+		if after.ChunkSize != 65536 || !bytes.Equal(after.BaseNonce, enc.BaseNonce) {
+			t.Fatalf("re-wrap disturbed non-wrap fields: %+v", after)
+		}
+		if err := r.DeleteLocation(ctx, "s", encDigest); err != nil {
+			t.Fatalf("DeleteLocation: %v", err)
+		}
+	})
+
+	t.Run("location partial FEE rejected", func(t *testing.T) {
+		// A partial wrap set is rejected before it reaches SQL and leaves no row.
+		d := []byte{0x77}
+		partial := registry.BlobLocation{
+			Space: "s", Digest: d, Provider: "did:piri", URL: "u", Size: 10,
+			RegionWrappedCEK: []byte{0x01}, // the rest deliberately absent
+		}
+		if err := r.PutLocation(ctx, partial); !errors.Is(err, registry.ErrPartialFEE) {
+			t.Fatalf("PutLocation(partial) = %v, want ErrPartialFEE", err)
+		}
+		if _, err := r.GetLocation(ctx, "s", d); !errors.Is(err, registry.ErrNotFound) {
+			t.Fatalf("partial FEE leaked a row: %v", err)
+		}
+	})
+
 	t.Run("multipart session parts latch metadata", func(t *testing.T) {
 		const id = "upl-1"
 		meta := map[string]string{"x-amz-meta-foo": "bar"}