feat: feed legacy payload header (#5029)

Co-authored-by: nugaon <[email protected]>

Author: nugaon

openapi/Swarm.yaml

@@ -1036,6 +1036,12 @@ paths:
             $ref: "SwarmCommon.yaml#/components/schemas/FeedType"
           required: false
           description: "Feed indexing scheme (default: sequence)"
+        - in: header
+          name: legacy-feed-resolution
+          schema:
+            type: boolean
+          required: false
+          description: "Resolves feed payloads in legacy structure (timestamp, content address)."
         - $ref: "SwarmCommon.yaml#/components/parameters/SwarmOnlyRootChunkParameter"
         - $ref: "SwarmCommon.yaml#/components/parameters/SwarmCache"
         - $ref: "SwarmCommon.yaml#/components/parameters/SwarmRedundancyStrategyParameter"

pkg/api/api.go

@@ -78,6 +78,7 @@ const (
 	SwarmSocSignatureHeader           = "Swarm-Soc-Signature"
 	SwarmFeedIndexHeader              = "Swarm-Feed-Index"
 	SwarmFeedIndexNextHeader          = "Swarm-Feed-Index-Next"
+	SwarmLegacyFeedResolve            = "Swarm-Feed-Legacy-Resolve"
 	SwarmOnlyRootChunk                = "Swarm-Only-Root-Chunk"
 	SwarmCollectionHeader             = "Swarm-Collection"
 	SwarmPostageBatchIdHeader         = "Swarm-Postage-Batch-Id"

pkg/api/bzz.go

@@ -431,7 +431,7 @@ FETCH:
 				jsonhttp.NotFound(w, "no update found")
 				return
 			}
-			wc, err := feeds.GetWrappedChunk(ctx, s.storer.Download(cache), ch)
+			wc, err := feeds.GetWrappedChunk(ctx, s.storer.Download(cache), ch, false)
 			if err != nil {
 				logger.Debug("bzz download: mapStructure feed update failed", "error", err)
 				logger.Error(nil, "bzz download: mapStructure feed update failed")

pkg/api/bzz_test.go

@@ -26,6 +26,7 @@ import (
 	"github.com/ethersphere/bee/v2/pkg/manifest"
 	mockbatchstore "github.com/ethersphere/bee/v2/pkg/postage/batchstore/mock"
 	mockpost "github.com/ethersphere/bee/v2/pkg/postage/mock"
+	testingsoc "github.com/ethersphere/bee/v2/pkg/soc/testing"
 	"github.com/ethersphere/bee/v2/pkg/storage/inmemchunkstore"
 	mockstorer "github.com/ethersphere/bee/v2/pkg/storer/mock"
 	"github.com/ethersphere/bee/v2/pkg/swarm"
@@ -794,16 +795,22 @@ func TestFeedIndirection(t *testing.T) {
 		t.Fatalf("expected file reference, did not got any")
 	}
 
-	// now use the "content" to mock the feed lookup
+	// get root chunk of data
+	// and wrap it in a feed
+	rootCh, err := storer.ChunkStore().Get(context.Background(), resp.Reference)
+	if err != nil {
+		t.Fatal(err)
+	}
+	socRootCh := testingsoc.GenerateMockSOC(t, rootCh.Data()[swarm.SpanSize:]).Chunk()
+
+	// now use the "content" root chunk to mock the feed lookup
 	// also, use the mocked mantaray chunks that unmarshal
 	// into a real manifest with the mocked feed values when
 	// called from the bzz endpoint. then call the bzz endpoint with
 	// the pregenerated feed root manifest hash
 
-	feedUpdate := toChunk(t, 121212, resp.Reference.Bytes())
-
 	var (
-		look                = newMockLookup(-1, 0, feedUpdate, nil, &id{}, nil)
+		look                = newMockLookup(-1, 0, socRootCh, nil, &id{}, nil)
 		factory             = newMockFactory(look)
 		bzzDownloadResource = func(addr, path string) string { return "/bzz/" + addr + "/" + path }
 		ctx                 = context.Background()
@@ -813,7 +820,6 @@ func TestFeedIndirection(t *testing.T) {
 		Logger: logger,
 		Feeds:  factory,
 	})
-	err := storer.Cache().Put(ctx, feedUpdate)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/api/feed.go

@@ -65,7 +65,8 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	headers := struct {
-		OnlyRootChunk bool `map:"Swarm-Only-Root-Chunk"`
+		OnlyRootChunk     bool `map:"Swarm-Only-Root-Chunk"`
+		LegacyFeedResolve bool `map:"Swarm-Feed-Legacy-Resolve"`
 	}{}
 	if response := s.mapStructure(r.Header, &headers); response != nil {
 		response("invalid header params", logger, w)
@@ -102,7 +103,7 @@ func (s *Service) feedGetHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	wc, err := feeds.GetWrappedChunk(r.Context(), s.storer.Download(false), ch)
+	wc, err := feeds.GetWrappedChunk(r.Context(), s.storer.Download(false), ch, headers.LegacyFeedResolve)
 	if err != nil {
 		logger.Error(nil, "wrapped chunk cannot be retrieved")
 		jsonhttp.NotFound(w, "wrapped chunk cannot be retrieved")

pkg/api/feed_test.go

@@ -77,6 +77,7 @@ func TestFeed_Get(t *testing.T) {
 
 		jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", "12"), http.StatusOK,
 			jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()[swarm.SpanSize:]),
+			jsonhttptest.WithRequestHeader(api.SwarmLegacyFeedResolve, "true"),
 			jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)),
 			jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexHeader),
 			jsonhttptest.WithExpectedResponseHeader(api.AccessControlExposeHeaders, api.SwarmFeedIndexNextHeader),
@@ -86,7 +87,7 @@ func TestFeed_Get(t *testing.T) {
 		)
 	})
 
-	t.Run("latest", func(t *testing.T) {
+	t.Run("latest with legacy payload", func(t *testing.T) {
 		t.Parallel()
 
 		var (
@@ -103,6 +104,7 @@ func TestFeed_Get(t *testing.T) {
 		)
 
 		jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusOK,
+			jsonhttptest.WithRequestHeader(api.SwarmLegacyFeedResolve, "true"),
 			jsonhttptest.WithExpectedResponse(mockWrappedCh.Data()[swarm.SpanSize:]),
 			jsonhttptest.WithExpectedContentLength(len(mockWrappedCh.Data()[swarm.SpanSize:])),
 			jsonhttptest.WithExpectedResponseHeader(api.SwarmFeedIndexHeader, hex.EncodeToString(idBytes)),
@@ -161,7 +163,9 @@ func TestFeed_Get(t *testing.T) {
 			})
 		)
 
-		jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusNotFound)
+		jsonhttptest.Request(t, client, http.MethodGet, feedResource(ownerString, "aabbcc", ""), http.StatusNotFound,
+			jsonhttptest.WithRequestHeader(api.SwarmLegacyFeedResolve, "true"),
+		)
 	})
 
 	t.Run("bigger payload than one chunk", func(t *testing.T) {

pkg/feeds/getter.go

@@ -50,7 +50,7 @@ func (f *Getter) Get(ctx context.Context, i Index) (swarm.Chunk, error) {
 	return f.getter.Get(ctx, addr)
 }
 
-func GetWrappedChunk(ctx context.Context, getter storage.Getter, ch swarm.Chunk) (swarm.Chunk, error) {
+func GetWrappedChunk(ctx context.Context, getter storage.Getter, ch swarm.Chunk, legacyResolve bool) (swarm.Chunk, error) {
 	wc, err := FromChunk(ch)
 	if err != nil {
 		return nil, err
@@ -59,16 +59,15 @@ func GetWrappedChunk(ctx context.Context, getter storage.Getter, ch swarm.Chunk)
 	// possible values right now:
 	// unencrypted ref: span+timestamp+ref => 8+8+32=48
 	// encrypted ref: span+timestamp+ref+decryptKey => 8+8+64=80
-	ref, err := legacyPayload(wc)
-	if err != nil {
-		if errors.Is(err, errNotLegacyPayload) {
-			return wc, nil
+	if legacyResolve {
+		ref, err := legacyPayload(wc)
+		if err != nil {
+			return nil, err
+		}
+		wc, err = getter.Get(ctx, ref)
+		if err != nil {
+			return nil, err
 		}
-		return nil, err
-	}
-	wc, err = getter.Get(ctx, ref)
-	if err != nil {
-		return nil, err
 	}
 
 	return wc, nil

pkg/feeds/getter_test.go

@@ -17,18 +17,24 @@ import (
 func TestGetWrappedChunk(t *testing.T) {
 	storer := mockstorer.New()
 
+	data := []byte("data")
 	// new format (wraps chunk)
-	ch := soctesting.GenerateMockSOC(t, []byte("data")).Chunk()
-	wch, err := GetWrappedChunk(context.Background(), storer.ChunkStore(), ch)
+	ch := soctesting.GenerateMockSOC(t, data).Chunk()
+	wch, err := GetWrappedChunk(context.Background(), storer.ChunkStore(), ch, false)
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if !bytes.Equal(wch.Data()[8:], []byte("data")) {
+	if !bytes.Equal(wch.Data()[8:], data) {
 		t.Fatal("data mismatch")
 	}
 
 	// old format
+	err = storer.Put(context.Background(), wch)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	tt := []struct {
 		name string
 		addr []byte
@@ -49,19 +55,30 @@ func TestGetWrappedChunk(t *testing.T) {
 			binary.BigEndian.PutUint64(timestamp, 1)
 			ch = soctesting.GenerateMockSOC(t, append(timestamp, tc.addr...)).Chunk()
 
-			err = storer.Put(context.Background(), wch)
+			wch, err = GetWrappedChunk(context.Background(), storer.ChunkStore(), ch, true)
 			if err != nil {
 				t.Fatal(err)
 			}
 
-			wch, err = GetWrappedChunk(context.Background(), storer.ChunkStore(), ch)
-			if err != nil {
-				t.Fatal(err)
-			}
-
-			if !bytes.Equal(wch.Data()[8:], []byte("data")) {
+			if !bytes.Equal(wch.Data()[8:], data) {
 				t.Fatal("data mismatch")
 			}
 		})
 	}
+
+	t.Run("returns feed legacy payload", func(t *testing.T) {
+		timestamp := make([]byte, 8)
+		binary.BigEndian.PutUint64(timestamp, 1)
+		feedChData := append(timestamp, wch.Address().Bytes()...)
+		ch = soctesting.GenerateMockSOC(t, feedChData).Chunk()
+
+		wch, err = GetWrappedChunk(context.Background(), storer.ChunkStore(), ch, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if !bytes.Equal(wch.Data()[8:], feedChData) {
+			t.Fatal("data should be similar as old legacy feed payload format.")
+		}
+	})
 }

pkg/node/bootstrap.go

@@ -335,7 +335,7 @@ func getLatestSnapshot(
 		return nil, err
 	}
 
-	return feeds.GetWrappedChunk(ctx, st, u)
+	return feeds.GetWrappedChunk(ctx, st, u, false)
 }
 
 func batchStoreExists(s storage.StateStorer) (bool, error) {