Skip to content

Commit c8cc1b0

Browse files
committed
feat: meta=eof for IPIP-431; ask for and expect (but not require) from http fetches
Ref: ipfs/specs#431 Ref: ipld/frisbii#15
1 parent 26c5ca7 commit c8cc1b0

File tree

13 files changed

+340
-63
lines changed

13 files changed

+340
-63
lines changed

cmd/lassie/fetch.go

+3
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,9 @@ func defaultFetchRun(
291291
blockCount,
292292
humanize.IBytes(stats.Size),
293293
)
294+
if stats.CarProperties != nil {
295+
fmt.Fprintf(msgWriter, "\t CAR Metadata: %s\n", stats.CarProperties.String())
296+
}
294297

295298
return nil
296299
}

go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ require (
2525
github.com/ipfs/go-unixfsnode v1.7.1
2626
github.com/ipld/go-car/v2 v2.10.1
2727
github.com/ipld/go-codec-dagpb v1.6.0
28-
github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0
28+
github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de
2929
github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff
3030
github.com/libp2p/go-libp2p v0.27.8
3131
github.com/libp2p/go-libp2p-routing-helpers v0.7.0

go.sum

+4-4
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,7 @@ github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ=
126126
github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
127127
github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk=
128128
github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY=
129-
github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY=
129+
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
130130
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
131131
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
132132
github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
@@ -336,8 +336,8 @@ github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI
336336
github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg=
337337
github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc=
338338
github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s=
339-
github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0 h1:iJTl9tx5DEsnKpppX5PmfdoQ3ITuBmkh3yyEpHWY2SI=
340-
github.com/ipld/go-ipld-prime v0.20.1-0.20230329011551-5056175565b0/go.mod h1:wmOtdy70ajP48iZITH8uLsGJVMqA4EJM61/bSfYYGhs=
339+
github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de h1:N6Wfk6dvcBjF4AJJDSmti6CkgHWZPDZ0fuqSQL+kKnU=
340+
github.com/ipld/go-ipld-prime v0.21.1-0.20230811030745-6e31cea491de/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ=
341341
github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
342342
github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff h1:xbKrIvnpQkbF8iHPk/HGcegsypCDpcXWHhzBCLyCWf8=
343343
github.com/ipni/go-libipni v0.0.8-0.20230425184153-86a1fcb7f7ff/go.mod h1:paYP9U4N3/vOzGCuN9kU972vtvw9JUcQjOKyiCFGwRk=
@@ -616,7 +616,7 @@ github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU=
616616
github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc=
617617
github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU=
618618
github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM=
619-
github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U=
619+
github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s=
620620
github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
621621
github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw=
622622
github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ=

pkg/httputil/constants.go

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
package httputil
2+
3+
import "fmt"
4+
5+
const (
6+
MimeTypeCar = "application/vnd.ipld.car" // The only accepted MIME type
7+
MimeTypeCarVersion = "1" // We only accept version 1 of the MIME type
8+
ResponseAcceptRangesHeader = "none" // We currently don't accept range requests
9+
ResponseCacheControlHeader = "public, max-age=29030400, immutable" // Magic cache control values
10+
FilenameExtCar = ".car" // The only valid filename extension
11+
FormatParameterCar = "car" // The only valid format parameter value
12+
DefaultIncludeDupes = true // The default value for an unspecified "dups" parameter. See https://github.com/ipfs/specs/pull/412.
13+
)
14+
15+
var (
16+
ResponseChunkDelimeter = []byte("0\r\n") // An http/1.1 chunk delimeter, used for specifying an early end to the response
17+
ResponseContentTypeHeader = fmt.Sprintf("%s; version=%s", MimeTypeCar, MimeTypeCarVersion)
18+
RequestAcceptHeader = fmt.Sprintf("%s; version=%s; order=dfs; dups=y; meta=eof", MimeTypeCar, MimeTypeCarVersion)
19+
)

pkg/httputil/metadata/metadata.go

+70
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package metadata
2+
3+
import (
4+
"fmt"
5+
"io"
6+
7+
"github.com/filecoin-project/lassie/pkg/types"
8+
"github.com/ipfs/go-cid"
9+
"github.com/ipld/go-ipld-prime/codec/dagjson"
10+
bindnoderegistry "github.com/ipld/go-ipld-prime/node/bindnode/registry"
11+
mh "github.com/multiformats/go-multihash"
12+
13+
_ "embed"
14+
)
15+
16+
//go:embed metadata.ipldsch
17+
var schema []byte
18+
19+
var BindnodeRegistry = bindnoderegistry.NewRegistry()
20+
21+
type CarMetadata struct {
22+
Metadata *Metadata
23+
}
24+
25+
func (cm CarMetadata) Serialize(w io.Writer) error {
26+
// TODO: do the same checks we do on Deserialize()
27+
return BindnodeRegistry.TypeToWriter(&cm, w, dagjson.Encode)
28+
}
29+
30+
func (cm *CarMetadata) Deserialize(r io.Reader) error {
31+
cmIface, err := BindnodeRegistry.TypeFromReader(r, &CarMetadata{}, dagjson.Decode)
32+
if err != nil {
33+
return fmt.Errorf("invalid CarMetadata: %w", err)
34+
}
35+
cmm := cmIface.(*CarMetadata) // safe to assume type
36+
if cmm.Metadata.Properties == nil && cmm.Metadata.Error == nil {
37+
return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields")
38+
}
39+
if (cmm.Metadata.Properties == nil) == (cmm.Metadata.Error == nil) {
40+
return fmt.Errorf("invalid CarMetadata: must contain either properties or error fields, not both")
41+
}
42+
if cmm.Metadata.Properties != nil {
43+
if _, err := mh.Decode(cmm.Metadata.Properties.ChecksumMultihash); err != nil {
44+
return fmt.Errorf("invalid CarMetadata: checksum multihash: %w", err)
45+
}
46+
}
47+
// TODO: parse and check EntityBytes format
48+
*cm = *cmm
49+
return nil
50+
}
51+
52+
type Metadata struct {
53+
Request Request
54+
Properties *types.CarProperties
55+
Error *string
56+
}
57+
58+
type Request struct {
59+
Root cid.Cid
60+
Path *string
61+
Scope types.DagScope
62+
Duplicates bool
63+
EntityBytes *string
64+
}
65+
66+
func init() {
67+
if err := BindnodeRegistry.RegisterType((*CarMetadata)(nil), string(schema), "CarMetadata"); err != nil {
68+
panic(err.Error())
69+
}
70+
}
+31
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
type CarMetadata union {
2+
| Metadata "car-metadata/v1"
3+
} representation keyed
4+
5+
type Metadata struct {
6+
request Request
7+
# must contain either a properties or an error
8+
properties optional CarProperties
9+
error optional String
10+
}
11+
12+
type Request struct {
13+
root &Any
14+
path optional String
15+
scope DagScope
16+
duplicates Bool (rename "dups")
17+
entityBytes optional String (rename "entity-bytes") # Must be a valid entity-bytes param: "from:to"
18+
}
19+
20+
type DagScope enum {
21+
| all
22+
| entity
23+
| block
24+
}
25+
26+
type CarProperties struct {
27+
carBytes Int (rename "car_bytes")
28+
dataBytes Int (rename "data_bytes")
29+
blockCount Int (rename "block_count")
30+
checksumMultihash optional Bytes (rename "checksum") # Must be a valid multihash
31+
}
+118
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,118 @@
1+
package metadata_test
2+
3+
import (
4+
"bytes"
5+
"testing"
6+
7+
"github.com/filecoin-project/lassie/pkg/httputil/metadata"
8+
"github.com/filecoin-project/lassie/pkg/types"
9+
"github.com/ipfs/go-cid"
10+
"github.com/stretchr/testify/require"
11+
)
12+
13+
var testCid = cid.MustParse("bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4")
14+
15+
func TestCarMetadataRoundtrip(t *testing.T) {
16+
path := "/birb.mp4"
17+
orig := metadata.CarMetadata{
18+
Metadata: &metadata.Metadata{
19+
Request: metadata.Request{
20+
Root: testCid,
21+
Path: &path,
22+
Scope: types.DagScopeAll,
23+
Duplicates: true,
24+
},
25+
Properties: &types.CarProperties{
26+
CarBytes: 202020,
27+
DataBytes: 101010,
28+
BlockCount: 303,
29+
ChecksumMultihash: testCid.Hash(),
30+
},
31+
},
32+
}
33+
var buf bytes.Buffer
34+
require.NoError(t, orig.Serialize(&buf))
35+
36+
t.Log("metadata dag-json:", buf.String())
37+
38+
var roundtrip metadata.CarMetadata
39+
require.NoError(t, roundtrip.Deserialize(&buf))
40+
require.Equal(t, orig, roundtrip)
41+
require.NotNil(t, roundtrip.Metadata)
42+
require.Equal(t, testCid, roundtrip.Metadata.Request.Root)
43+
require.NotNil(t, roundtrip.Metadata.Request.Path)
44+
require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path)
45+
require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope)
46+
require.True(t, roundtrip.Metadata.Request.Duplicates)
47+
require.NotNil(t, roundtrip.Metadata.Properties)
48+
require.Nil(t, roundtrip.Metadata.Error)
49+
require.Equal(t, int64(202020), roundtrip.Metadata.Properties.CarBytes)
50+
require.Equal(t, int64(101010), roundtrip.Metadata.Properties.DataBytes)
51+
require.Equal(t, int64(303), roundtrip.Metadata.Properties.BlockCount)
52+
require.Equal(t, []byte(testCid.Hash()), roundtrip.Metadata.Properties.ChecksumMultihash)
53+
}
54+
55+
func TestCarMetadataErrorRoundtrip(t *testing.T) {
56+
path := "/birb.mp4"
57+
msg := "something bad happened"
58+
orig := metadata.CarMetadata{
59+
Metadata: &metadata.Metadata{
60+
Request: metadata.Request{
61+
Root: testCid,
62+
Path: &path,
63+
Scope: types.DagScopeAll,
64+
Duplicates: true,
65+
},
66+
Error: &msg,
67+
},
68+
}
69+
var buf bytes.Buffer
70+
require.NoError(t, orig.Serialize(&buf))
71+
72+
t.Log("metadata dag-json:", buf.String())
73+
74+
var roundtrip metadata.CarMetadata
75+
require.NoError(t, roundtrip.Deserialize(&buf))
76+
require.Equal(t, orig, roundtrip)
77+
require.NotNil(t, roundtrip.Metadata)
78+
require.Equal(t, testCid, roundtrip.Metadata.Request.Root)
79+
require.NotNil(t, roundtrip.Metadata.Request.Path)
80+
require.Equal(t, "/birb.mp4", *roundtrip.Metadata.Request.Path)
81+
require.Equal(t, types.DagScopeAll, roundtrip.Metadata.Request.Scope)
82+
require.True(t, roundtrip.Metadata.Request.Duplicates)
83+
require.Nil(t, roundtrip.Metadata.Properties)
84+
require.NotNil(t, roundtrip.Metadata.Error)
85+
require.Equal(t, "something bad happened", *roundtrip.Metadata.Error)
86+
}
87+
88+
func TestBadMetadata(t *testing.T) {
89+
testCases := []struct {
90+
name string
91+
byts string
92+
err string
93+
}{
94+
{"empty", `{}`, `union structure constraints for CarMetadata caused rejection: a union must have exactly one entry`},
95+
{"bad key", `{"not metadata":true}`, `union structure constraints for CarMetadata caused rejection: no member named "not metadata"`},
96+
{
97+
"bad multihash",
98+
`{"car-metadata/v1":{"properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"bm90IGEgbXVsdGloYXNo"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`,
99+
`invalid CarMetadata: checksum multihash:`,
100+
},
101+
{
102+
"no properties or error",
103+
`{"car-metadata/v1":{"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`,
104+
`invalid CarMetadata: must contain either properties or error fields`,
105+
},
106+
{
107+
"both properties and error",
108+
`{"car-metadata/v1":{"error":"something bad happened","properties":{"block_count":303,"car_bytes":202020,"checksum":{"/":{"bytes":"EiBd9neBCasGxUmysJN7nGza4ylHikmbsP2+nXs6BlIpvw"}},"data_bytes":101010},"request":{"dups":true,"path":"/birb.mp4","root":{"/":"bafybeic56z3yccnla3cutmvqsn5zy3g24muupcsjtoyp3pu5pm5amurjx4"},"scope":"all"}}}`,
109+
`invalid CarMetadata: must contain either properties or error fields, not both`,
110+
},
111+
}
112+
for _, tc := range testCases {
113+
t.Run(tc.name, func(t *testing.T) {
114+
var roundtrip metadata.CarMetadata
115+
require.ErrorContains(t, roundtrip.Deserialize(bytes.NewBuffer([]byte(tc.byts))), tc.err)
116+
})
117+
}
118+
}

pkg/server/http/util.go renamed to pkg/httputil/server.go

+18-9
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package httpserver
1+
package httputil
22

33
import (
44
"errors"
@@ -74,33 +74,34 @@ func ParseFilename(req *http.Request) (string, error) {
7474
//
7575
// Lassie only allows the "car" format query parameter
7676
// https://specs.ipfs.tech/http-gateways/path-gateway/#format-request-query-parameter
77-
func CheckFormat(req *http.Request) (bool, error) {
77+
func CheckFormat(req *http.Request) (bool, bool, error) {
7878
hasAccept := req.Header.Get("Accept") != ""
7979
// check if Accept header includes application/vnd.ipld.car
80-
validAccept, includeDupes := ParseAccept(req.Header.Get("Accept"))
80+
validAccept, includeDupes, includeMeta := ParseAccept(req.Header.Get("Accept"))
8181
if hasAccept && !validAccept {
82-
return false, fmt.Errorf("no acceptable content type")
82+
return false, false, fmt.Errorf("no acceptable content type")
8383
}
8484

8585
// check if format is "car"
8686
hasFormat := req.URL.Query().Has("format")
8787
if hasFormat && req.URL.Query().Get("format") != FormatParameterCar {
88-
return false, fmt.Errorf("requested non-supported format %s", req.URL.Query().Get("format"))
88+
return false, false, fmt.Errorf("requested non-supported format %s", req.URL.Query().Get("format"))
8989
}
9090

9191
// if neither are provided return
9292
// one of them has to be given with a CAR type since we only return CAR data
9393
if !validAccept && !hasFormat {
94-
return false, fmt.Errorf("neither a valid accept header or format parameter were provided")
94+
return false, false, fmt.Errorf("neither a valid accept header or format parameter were provided")
9595
}
9696

97-
return includeDupes, nil
97+
return includeDupes, includeMeta, nil
9898
}
9999

100100
// ParseAccept validates that the request Accept header is of the type CAR and
101101
// returns whether or not duplicate blocks are allowed in the response via
102-
// IPIP-412: https://github.com/ipfs/specs/pull/412.
103-
func ParseAccept(acceptHeader string) (validAccept bool, includeDupes bool) {
102+
// IPIP-412: https://github.com/ipfs/specs/pull/412, and whether or not
103+
// metadata is requested via IPIP-431: https://github.com/ipfs/specs/pull/431.
104+
func ParseAccept(acceptHeader string) (validAccept bool, includeDupes bool, includeMeta bool) {
104105
acceptTypes := strings.Split(acceptHeader, ",")
105106
validAccept = false
106107
includeDupes = DefaultIncludeDupes
@@ -140,6 +141,14 @@ func ParseAccept(acceptHeader string) (validAccept bool, includeDupes bool) {
140141
// we only do dfs, which also satisfies unk, future extensions are not yet supported
141142
validAccept = false
142143
}
144+
case "meta":
145+
switch value {
146+
case "eof":
147+
includeMeta = true
148+
default:
149+
// we only support eof, future extensions are not yet supported
150+
validAccept = false
151+
}
143152
default:
144153
// ignore others
145154
}

0 commit comments

Comments
 (0)