Skip to content

Commit 070f065

Browse files
CommanderStormCommanderStorm
committed
wip
1 parent 4eefb9d commit 070f065

53 files changed

Lines changed: 580 additions & 4 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cpp/src/mlt/decode/int.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ class IntegerDecoder : public util::noncopyable {
2626
// We could create new ones, if really necessary.
2727
IntegerDecoder& operator=(IntegerDecoder&&) = delete;
2828

29+
/// Get the number of decoded values a stream yields (the buffer size for `decodeIntArray`).
30+
static std::size_t getIntArrayBufferSize(std::size_t count, const StreamMetadata&);
31+
2932
/// Decode a buffer of integers into another, according to the encoding scheme specified by the metadata
3033
/// @param values Input values
3134
/// @param out Output values (should be sized according to `getIntArrayBufferSize`)
@@ -132,9 +135,6 @@ class IntegerDecoder : public util::noncopyable {
132135
return {*this, buffer[bufferIndex].data()};
133136
}
134137

135-
/// Get the size of the buffer necessary for `decodeIntArray`
136-
static std::size_t getIntArrayBufferSize(std::size_t count, const StreamMetadata&);
137-
138138
template <typename TDecode, typename TTarget = TDecode>
139139
requires(std::is_integral_v<TDecode> && (std::is_integral_v<TTarget> || std::is_enum_v<TTarget>))
140140
void decodeStream(BufferStream& tileData, TTarget* out, std::size_t outSize, const StreamMetadata&);

‎cpp/src/mlt/decode/property.hpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,12 @@ class PropertyDecoder {
9393
streamMetadata = StreamMetadata::decode(tileData);
9494
}
9595

96-
if (column.nullable && streamMetadata && presentValueCount < streamMetadata->getNumValues()) {
96+
// Compare against the logical value count, not the raw stream length: an RLE/Delta-RLE
97+
// data stream reports `getNumValues()` as the count of encoded ints (e.g. a run-length +
98+
// value pair), which exceeds the number of decoded values for a single-value run.
99+
if (column.nullable && streamMetadata &&
100+
presentValueCount <
101+
IntegerDecoder::getIntArrayBufferSize(streamMetadata->getNumValues(), *streamMetadata)) {
97102
throw std::runtime_error("Unexpected present value column");
98103
}
99104

‎java/mlt-tools/src/main/java/org/maplibre/mlt/tools/SyntheticMltGenerator.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,20 +351,27 @@ private static void generateProperties() throws IOException {
351351
// write(layer("prop_i16_null_val", feat(p0), feat(p0, prop("val", (short) 42))), cfg());
352352

353353
write("prop_i32", feat(p0, prop("val", (int) 42)), cfg());
354+
// Single-value DELTA: numValues==1 takes the CONST path, which the multi-feature props_*
355+
// tests (FLAT path) never exercise. This is the quadrant that hid the const un-ZigZag bug.
356+
// RLE/DELTA_RLE are omitted here because for a single value the encoder collapses them to
357+
// the plain NONE encoding (identical bytes); the Rust generator forces those distinct.
358+
write("prop_i32_delta", feat(p0, prop("val", (int) 42)), cfg(DELTA));
354359
write("prop_i32_neg", feat(p0, prop("val", (int) -42)), cfg());
355360
write("prop_i32_min", feat(p0, prop("val", Integer.MIN_VALUE)), cfg());
356361
write("prop_i32_max", feat(p0, prop("val", Integer.MAX_VALUE)), cfg());
357362
write(layer("prop_i32_val_null", feat(p0, prop("val", (int) 42)), feat(p0)), cfg());
358363
write(layer("prop_i32_null_val", feat(p0), feat(p0, prop("val", (int) 42))), cfg());
359364

360365
write("prop_u32", feat(p0, prop("val", U32.of(42L))), cfg());
366+
write("prop_u32_delta", feat(p0, prop("val", U32.of(42L))), cfg(DELTA));
361367
write("prop_u32_min", feat(p0, prop("val", U32.of(0L))), cfg());
362368
write("prop_u32_max", feat(p0, prop("val", U32.of(0xFFFFFFFFL))), cfg());
363369
write(layer("prop_u32_val_null", feat(p0, prop("val", U32.of(42L))), feat(p0)), cfg());
364370
write(layer("prop_u32_null_val", feat(p0), feat(p0, prop("val", U32.of(42L)))), cfg());
365371

366372
long i64_value = 9_876_543_210L;
367373
write("prop_i64", feat(p0, prop("val", i64_value)), cfg());
374+
write("prop_i64_delta", feat(p0, prop("val", i64_value)), cfg(DELTA));
368375
write("prop_i64_neg", feat(p0, prop("val", (long) -9_876_543_210L)), cfg());
369376
write("prop_i64_min", feat(p0, prop("val", Long.MIN_VALUE)), cfg());
370377
write("prop_i64_max", feat(p0, prop("val", Long.MAX_VALUE)), cfg());
@@ -374,6 +381,8 @@ private static void generateProperties() throws IOException {
374381
U64 u64_value = U64.of(BigInteger.valueOf(1234567890123456789L));
375382
U64 u64_max = U64.of(new BigInteger("18446744073709551615"));
376383
write("prop_u64", feat(p0, prop("bignum", u64_value)), cfg());
384+
// u64 single-value DELTA: the CONST+ZigZag shape that hid the un-ZigZag bug (from real ev.mlt).
385+
write("prop_u64_delta", feat(p0, prop("bignum", u64_value)), cfg(DELTA));
377386
write("prop_u64_min", feat(p0, prop("bignum", U64.of(BigInteger.ZERO))), cfg());
378387
write("prop_u64_max", feat(p0, prop("bignum", u64_max)), cfg());
379388
write(layer("prop_u64_val_null", feat(p0, prop("val", u64_value)), feat(p0)), cfg());

‎rust/mlt-synthetics/src/main.rs‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,20 @@ fn generate_properties(w: &mut SynthWriter) {
553553
.write(w, "prop_i32_np");
554554
p0().add_prop(e_int, P::opt_i32("val", vec![Some(42)]))
555555
.write(w, "prop_i32");
556+
// Single-value variants for every encoding: numValues==1 takes the CONST/SEQUENCE path,
557+
// which the multi-feature props_* tests (FLAT path) never exercise.
558+
p0().add_prop(E::delta_varint(), P::i32("val", vec![42]))
559+
.write(w, "prop_i32_delta_np");
560+
p0().add_prop(E::delta_varint(), P::opt_i32("val", vec![Some(42)]))
561+
.write(w, "prop_i32_delta");
562+
p0().add_prop(E::rle_varint(), P::i32("val", vec![42]))
563+
.write(w, "prop_i32_rle_np");
564+
p0().add_prop(E::rle_varint(), P::opt_i32("val", vec![Some(42)]))
565+
.write(w, "prop_i32_rle-rust");
566+
p0().add_prop(E::delta_rle_varint(), P::i32("val", vec![42]))
567+
.write(w, "prop_i32_delta_rle_np");
568+
p0().add_prop(E::delta_rle_varint(), P::opt_i32("val", vec![Some(42)]))
569+
.write(w, "prop_i32_delta_rle-rust");
556570
p0().add_prop(e_int, P::i32("val", vec![-42]))
557571
.write(w, "prop_i32_neg_np");
558572
p0().add_prop(e_int, P::opt_i32("val", vec![Some(-42)]))
@@ -579,6 +593,18 @@ fn generate_properties(w: &mut SynthWriter) {
579593
.write(w, "prop_u32_np");
580594
p0().add_prop(e_int, P::opt_u32("val", vec![Some(42)]))
581595
.write(w, "prop_u32");
596+
p0().add_prop(E::delta_varint(), P::u32("val", vec![42]))
597+
.write(w, "prop_u32_delta_np");
598+
p0().add_prop(E::delta_varint(), P::opt_u32("val", vec![Some(42)]))
599+
.write(w, "prop_u32_delta");
600+
p0().add_prop(E::rle_varint(), P::u32("val", vec![42]))
601+
.write(w, "prop_u32_rle_np");
602+
p0().add_prop(E::rle_varint(), P::opt_u32("val", vec![Some(42)]))
603+
.write(w, "prop_u32_rle-rust");
604+
p0().add_prop(E::delta_rle_varint(), P::u32("val", vec![42]))
605+
.write(w, "prop_u32_delta_rle_np");
606+
p0().add_prop(E::delta_rle_varint(), P::opt_u32("val", vec![Some(42)]))
607+
.write(w, "prop_u32_delta_rle-rust");
582608
p0().add_prop(e_int, P::u32("val", vec![0]))
583609
.write(w, "prop_u32_min_np");
584610
p0().add_prop(e_int, P::opt_u32("val", vec![Some(0)]))
@@ -601,6 +627,27 @@ fn generate_properties(w: &mut SynthWriter) {
601627
.write(w, "prop_i64_np");
602628
p0().add_prop(e_int, P::opt_i64("val", vec![Some(9_876_543_210)]))
603629
.write(w, "prop_i64");
630+
p0().add_prop(E::delta_varint(), P::i64("val", vec![9_876_543_210]))
631+
.write(w, "prop_i64_delta_np");
632+
p0().add_prop(
633+
E::delta_varint(),
634+
P::opt_i64("val", vec![Some(9_876_543_210)]),
635+
)
636+
.write(w, "prop_i64_delta");
637+
p0().add_prop(E::rle_varint(), P::i64("val", vec![9_876_543_210]))
638+
.write(w, "prop_i64_rle_np");
639+
p0().add_prop(
640+
E::rle_varint(),
641+
P::opt_i64("val", vec![Some(9_876_543_210)]),
642+
)
643+
.write(w, "prop_i64_rle-rust");
644+
p0().add_prop(E::delta_rle_varint(), P::i64("val", vec![9_876_543_210]))
645+
.write(w, "prop_i64_delta_rle_np");
646+
p0().add_prop(
647+
E::delta_rle_varint(),
648+
P::opt_i64("val", vec![Some(9_876_543_210)]),
649+
)
650+
.write(w, "prop_i64_delta_rle-rust");
604651
p0().add_prop(e_int, P::i64("val", vec![-9_876_543_210]))
605652
.write(w, "prop_i64_neg_np");
606653
p0().add_prop(e_int, P::opt_i64("val", vec![Some(-9_876_543_210)]))
@@ -638,6 +685,38 @@ fn generate_properties(w: &mut SynthWriter) {
638685
.write(w, "prop_u64_max_np");
639686
p0().add_prop(e_int, P::opt_u64("bignum", vec![Some(u64::MAX)]))
640687
.write(w, "prop_u64_max");
688+
// u64 single-value DELTA: the CONST+ZigZag shape that hid a TS decoder bug
689+
// (decodeUnsignedConstInt64Stream skipped the un-ZigZag, returning 2x). From real ev.mlt.
690+
p0().add_prop(
691+
E::delta_varint(),
692+
P::u64("bignum", vec![1_234_567_890_123_456_789]),
693+
)
694+
.write(w, "prop_u64_delta_np");
695+
p0().add_prop(
696+
E::delta_varint(),
697+
P::opt_u64("bignum", vec![Some(1_234_567_890_123_456_789)]),
698+
)
699+
.write(w, "prop_u64_delta");
700+
p0().add_prop(
701+
E::rle_varint(),
702+
P::u64("bignum", vec![1_234_567_890_123_456_789]),
703+
)
704+
.write(w, "prop_u64_rle_np");
705+
p0().add_prop(
706+
E::rle_varint(),
707+
P::opt_u64("bignum", vec![Some(1_234_567_890_123_456_789)]),
708+
)
709+
.write(w, "prop_u64_rle-rust");
710+
p0().add_prop(
711+
E::delta_rle_varint(),
712+
P::u64("bignum", vec![1_234_567_890_123_456_789]),
713+
)
714+
.write(w, "prop_u64_delta_rle_np");
715+
p0().add_prop(
716+
E::delta_rle_varint(),
717+
P::opt_u64("bignum", vec![Some(1_234_567_890_123_456_789)]),
718+
)
719+
.write(w, "prop_u64_delta_rle-rust");
641720
// Two-feature optional u64 variants (key is "val" to match Java)
642721
geo_varint_with_rle()
643722
.geos([P0, P0])

‎test/synthetic/0x01-rust/prop_i32_delta_np.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"type": "FeatureCollection",
3+
"features": [
4+
{
5+
"type": "Feature",
6+
"properties": {
7+
"_extent": 80,
8+
"_layer": "layer1",
9+
"val": 42
10+
},
11+
"geometry": {
12+
"type": "Point",
13+
"coordinates": [
14+
13,
15+
42
16+
]
17+
}
18+
}
19+
]
20+
}

‎test/synthetic/0x01-rust/prop_i32_delta_np.mlt‎

34 Bytes
Binary file not shown.

‎test/synthetic/0x01-rust/prop_i32_delta_rle.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"type": "FeatureCollection",
3+
"features": [
4+
{
5+
"type": "Feature",
6+
"properties": {
7+
"_extent": 80,
8+
"_layer": "layer1",
9+
"val": 42
10+
},
11+
"geometry": {
12+
"type": "Point",
13+
"coordinates": [
14+
13,
15+
42
16+
]
17+
}
18+
}
19+
]
20+
}

‎test/synthetic/0x01-rust/prop_i32_delta_rle.mlt‎

43 Bytes
Binary file not shown.

‎test/synthetic/0x01-rust/prop_i32_delta_rle_np.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"type": "FeatureCollection",
3+
"features": [
4+
{
5+
"type": "Feature",
6+
"properties": {
7+
"_extent": 80,
8+
"_layer": "layer1",
9+
"val": 42
10+
},
11+
"geometry": {
12+
"type": "Point",
13+
"coordinates": [
14+
13,
15+
42
16+
]
17+
}
18+
}
19+
]
20+
}

‎test/synthetic/0x01-rust/prop_i32_delta_rle_np.mlt‎

37 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)