@@ -8,9 +8,25 @@ import (
8
8
"strings"
9
9
)
10
10
11
+ const (
12
+ // DefaultBlockSize is the chunk size that splitters produce (or aim to).
13
+ DefaultBlockSize int64 = 1024 * 256
14
+
15
+ // 1 MB, on-wire block size for "datablocks ( unixfs, etc )"
16
+ // copy of https://github.com/ipfs/go-unixfs/blob/v0.2.3/importer/helpers/helpers.go#L8
17
+ BlockSizeLimit int = 1048576
18
+
19
+ // in case we are using raw-leaves: this would match BlockSizeLimit, but we can't assume that
20
+ // be conservative and substract the PB wraping size of a full DAG-PB+UnixFS node describing 1M
21
+ // (2b(type2/file)+4b(data-field:3-byte-len-delimited)+4b(size-field:3-byte-varint))+(4b(DAG-type-1:3-byte-len-delimited))
22
+ // FIXME - this calculation will need an update for CBOR
23
+ BlockPayloadLimit int = (BlockSizeLimit - (2 + 4 + 4 + 4 ))
24
+ )
25
+
11
26
var (
12
27
ErrRabinMin = errors .New ("rabin min must be greater than 16" )
13
- ErrSize = errors .New ("chunker size muster greater than 0" )
28
+ ErrSize = errors .New ("chunker size must be greater than 0" )
29
+ ErrSizeMax = fmt .Errorf ("chunker parameters may not exceed the maximum block payload size of %d" , BlockPayloadLimit )
14
30
)
15
31
16
32
// FromString returns a Splitter depending on the given string:
@@ -28,6 +44,8 @@ func FromString(r io.Reader, chunker string) (Splitter, error) {
28
44
return nil , err
29
45
} else if size <= 0 {
30
46
return nil , ErrSize
47
+ } else if size > BlockPayloadLimit {
48
+ return nil , ErrSizeMax
31
49
}
32
50
return NewSizeSplitter (r , int64 (size )), nil
33
51
@@ -51,6 +69,8 @@ func parseRabinString(r io.Reader, chunker string) (Splitter, error) {
51
69
size , err := strconv .Atoi (parts [1 ])
52
70
if err != nil {
53
71
return nil , err
72
+ } else if int (float32 (size )* 1.5 ) > BlockPayloadLimit { // FIXME - there is probably a better way to bubble up this calculation from NewRabin()
73
+ return nil , ErrSizeMax
54
74
}
55
75
return NewRabin (r , uint64 (size )), nil
56
76
case 4 :
@@ -84,6 +104,14 @@ func parseRabinString(r io.Reader, chunker string) (Splitter, error) {
84
104
return nil , err
85
105
}
86
106
107
+ if min >= avg {
108
+ return nil , errors .New ("incorrect format: rabin-min must be smaller than rabin-avg" )
109
+ } else if avg >= max {
110
+ return nil , errors .New ("incorrect format: rabin-avg must be smaller than rabin-max" )
111
+ } else if max > BlockPayloadLimit {
112
+ return nil , ErrSizeMax
113
+ }
114
+
87
115
return NewRabinMinMax (r , uint64 (min ), uint64 (avg ), uint64 (max )), nil
88
116
default :
89
117
return nil , errors .New ("incorrect format (expected 'rabin' 'rabin-[avg]' or 'rabin-[min]-[avg]-[max]'" )
0 commit comments