Skip to content
This repository was archived by the owner on Jun 20, 2023. It is now read-only.

Commit 397f536

Browse files
committed
Add various sanity checks for size specifications
1 parent f076b1e commit 397f536

File tree

2 files changed

+29
-4
lines changed

2 files changed

+29
-4
lines changed

parse.go

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,25 @@ import (
88
"strings"
99
)
1010

11+
const (
12+
// DefaultBlockSize is the chunk size that splitters produce (or aim to).
13+
DefaultBlockSize int64 = 1024 * 256
14+
15+
// 1 MB, on-wire block size for "datablocks ( unixfs, etc )"
16+
// copy of https://github.com/ipfs/go-unixfs/blob/v0.2.3/importer/helpers/helpers.go#L8
17+
BlockSizeLimit int = 1048576
18+
19+
// in case we are using raw-leaves: this would match BlockSizeLimit, but we can't assume that
20+
// be conservative and substract the PB wraping size of a full DAG-PB+UnixFS node describing 1M
21+
// (2b(type2/file)+4b(data-field:3-byte-len-delimited)+4b(size-field:3-byte-varint))+(4b(DAG-type-1:3-byte-len-delimited))
22+
// FIXME - this calculation will need an update for CBOR
23+
BlockPayloadLimit int = (BlockSizeLimit - (2 + 4 + 4 + 4))
24+
)
25+
1126
var (
1227
ErrRabinMin = errors.New("rabin min must be greater than 16")
13-
ErrSize = errors.New("chunker size muster greater than 0")
28+
ErrSize = errors.New("chunker size must be greater than 0")
29+
ErrSizeMax = fmt.Errorf("chunker parameters may not exceed the maximum block payload size of %d", BlockPayloadLimit)
1430
)
1531

1632
// FromString returns a Splitter depending on the given string:
@@ -28,6 +44,8 @@ func FromString(r io.Reader, chunker string) (Splitter, error) {
2844
return nil, err
2945
} else if size <= 0 {
3046
return nil, ErrSize
47+
} else if size > BlockPayloadLimit {
48+
return nil, ErrSizeMax
3149
}
3250
return NewSizeSplitter(r, int64(size)), nil
3351

@@ -51,6 +69,8 @@ func parseRabinString(r io.Reader, chunker string) (Splitter, error) {
5169
size, err := strconv.Atoi(parts[1])
5270
if err != nil {
5371
return nil, err
72+
} else if int(float32(size)*1.5) > BlockPayloadLimit { // FIXME - there is probably a better way to bubble up this calculation from NewRabin()
73+
return nil, ErrSizeMax
5474
}
5575
return NewRabin(r, uint64(size)), nil
5676
case 4:
@@ -84,6 +104,14 @@ func parseRabinString(r io.Reader, chunker string) (Splitter, error) {
84104
return nil, err
85105
}
86106

107+
if min >= avg {
108+
return nil, errors.New("incorrect format: rabin-min must be smaller than rabin-avg")
109+
} else if avg >= max {
110+
return nil, errors.New("incorrect format: rabin-avg must be smaller than rabin-max")
111+
} else if max > BlockPayloadLimit {
112+
return nil, ErrSizeMax
113+
}
114+
87115
return NewRabinMinMax(r, uint64(min), uint64(avg), uint64(max)), nil
88116
default:
89117
return nil, errors.New("incorrect format (expected 'rabin' 'rabin-[avg]' or 'rabin-[min]-[avg]-[max]'")

splitting.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@ import (
1313

1414
var log = logging.Logger("chunk")
1515

16-
// DefaultBlockSize is the chunk size that splitters produce (or aim to).
17-
var DefaultBlockSize int64 = 1024 * 256
18-
1916
// A Splitter reads bytes from a Reader and creates "chunks" (byte slices)
2017
// that can be used to build DAG nodes.
2118
type Splitter interface {

0 commit comments

Comments
 (0)