Skip to content
This repository was archived by the owner on Feb 21, 2024. It is now read-only.

Commit 5ebd7e8

Browse files
committed
move requiredDepth calculation after min/max ranges are checked
Without this, a data set with a ludicrously large value in it could break a BSI field's depth even though the import would then reject it. always treat BSI fields as having at least their depth: If you imported only small values, BSI fields could end up not bothering to clear higher bits in existing values, which produced strange behaviors.
1 parent 5d71f04 commit 5ebd7e8

File tree

1 file changed

+30
-26
lines changed

1 file changed

+30
-26
lines changed

field.go

+30-26
Original file line numberDiff line numberDiff line change
@@ -1269,23 +1269,45 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
12691269
return errors.Wrap(ErrBSIGroupNotFound, f.name)
12701270
}
12711271

1272-
// Find the lowest/highest values.
1272+
// We want to determine the required bit depth, in case the field doesn't
1273+
// have as many bits currently as would be needed to represent these values,
1274+
// but only if the values are in-range for the field.
12731275
var min, max int64
1274-
for i, value := range values {
1275-
if i == 0 || value < min {
1276-
min = value
1276+
if len(values) > 0 {
1277+
min, max = values[0], values[0]
1278+
}
1279+
1280+
// Split import data by fragment.
1281+
dataByFragment := make(map[importKey]importValueData)
1282+
for i := range columnIDs {
1283+
columnID, value := columnIDs[i], values[i]
1284+
if value > bsig.Max {
1285+
return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooHigh, columnID, value)
1286+
} else if value < bsig.Min {
1287+
return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value)
12771288
}
1278-
if i == 0 || value > max {
1289+
if value > max {
12791290
max = value
12801291
}
1292+
if value < min {
1293+
min = value
1294+
}
1295+
1296+
// Attach value to each bsiGroup view.
1297+
for _, name := range []string{viewName} {
1298+
key := importKey{View: name, Shard: columnID / ShardWidth}
1299+
data := dataByFragment[key]
1300+
data.ColumnIDs = append(data.ColumnIDs, columnID)
1301+
data.Values = append(data.Values, value)
1302+
dataByFragment[key] = data
1303+
}
12811304
}
12821305

12831306
// Determine the highest bit depth required by the min & max.
12841307
requiredDepth := bitDepthInt64(min - bsig.Base)
12851308
if v := bitDepthInt64(max - bsig.Base); v > requiredDepth {
12861309
requiredDepth = v
12871310
}
1288-
12891311
// Increase bit depth if required.
12901312
if requiredDepth > bsig.BitDepth {
12911313
if err := func() error {
@@ -1297,26 +1319,8 @@ func (f *Field) importValue(columnIDs []uint64, values []int64, options *ImportO
12971319
}(); err != nil {
12981320
return errors.Wrap(err, "increasing bsi bit depth")
12991321
}
1300-
}
1301-
1302-
// Split import data by fragment.
1303-
dataByFragment := make(map[importKey]importValueData)
1304-
for i := range columnIDs {
1305-
columnID, value := columnIDs[i], values[i]
1306-
if value > bsig.Max {
1307-
return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooHigh, columnID, value)
1308-
} else if value < bsig.Min {
1309-
return fmt.Errorf("%v, columnID=%v, value=%v", ErrBSIGroupValueTooLow, columnID, value)
1310-
}
1311-
1312-
// Attach value to each bsiGroup view.
1313-
for _, name := range []string{viewName} {
1314-
key := importKey{View: name, Shard: columnID / ShardWidth}
1315-
data := dataByFragment[key]
1316-
data.ColumnIDs = append(data.ColumnIDs, columnID)
1317-
data.Values = append(data.Values, value)
1318-
dataByFragment[key] = data
1319-
}
1322+
} else {
1323+
requiredDepth = bsig.BitDepth
13201324
}
13211325

13221326
// Import into each fragment.

0 commit comments

Comments
 (0)