Skip to content

Commit 0663b40

Browse files
CommanderStormCommanderStorm
committed
wip
1 parent 4eefb9d commit 0663b40

53 files changed

Lines changed: 577 additions & 4 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎cpp/src/mlt/decode/int.hpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ class IntegerDecoder : public util::noncopyable {
2626
// We could create new ones, if really necessary.
2727
IntegerDecoder& operator=(IntegerDecoder&&) = delete;
2828

29+
/// Number of decoded values a stream yields (the buffer size for `decodeIntArray`).
30+
static std::size_t getIntArrayBufferSize(std::size_t count, const StreamMetadata&);
31+
2932
/// Decode a buffer of integers into another, according to the encoding scheme specified by the metadata
3033
/// @param values Input values
3134
/// @param out Output values (should be sized according to `getIntArrayBufferSize`)
@@ -132,9 +135,6 @@ class IntegerDecoder : public util::noncopyable {
132135
return {*this, buffer[bufferIndex].data()};
133136
}
134137

135-
/// Get the size of the buffer necessary for `decodeIntArray`
136-
static std::size_t getIntArrayBufferSize(std::size_t count, const StreamMetadata&);
137-
138138
template <typename TDecode, typename TTarget = TDecode>
139139
requires(std::is_integral_v<TDecode> && (std::is_integral_v<TTarget> || std::is_enum_v<TTarget>))
140140
void decodeStream(BufferStream& tileData, TTarget* out, std::size_t outSize, const StreamMetadata&);

‎cpp/src/mlt/decode/property.hpp‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -93,7 +93,11 @@ class PropertyDecoder {
9393
streamMetadata = StreamMetadata::decode(tileData);
9494
}
9595

96-
if (column.nullable && streamMetadata && presentValueCount < streamMetadata->getNumValues()) {
96+
// A single-value RLE run encodes as more ints than it decodes to.
97+
// Compare the logical (decoded) count, not the raw stream length, to avoid a spurious throw.
98+
if (column.nullable && streamMetadata &&
99+
presentValueCount <
100+
IntegerDecoder::getIntArrayBufferSize(streamMetadata->getNumValues(), *streamMetadata)) {
97101
throw std::runtime_error("Unexpected present value column");
98102
}
99103

‎java/mlt-tools/src/main/java/org/maplibre/mlt/tools/SyntheticMltGenerator.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,20 +351,27 @@ private static void generateProperties() throws IOException {
351351
// write(layer("prop_i16_null_val", feat(p0), feat(p0, prop("val", (short) 42))), cfg());
352352

353353
write("prop_i32", feat(p0, prop("val", (int) 42)), cfg());
354+
// Single-value DELTA hits the CONST path the multi-feature props_* miss.
355+
// This is the case that hid the const un-ZigZag bug.
356+
// RLE/DELTA_RLE collapse to plain NONE for one value, so Java can't emit them distinctly.
357+
// They live only in the Rust generator.
358+
write("prop_i32_delta", feat(p0, prop("val", (int) 42)), cfg(DELTA));
354359
write("prop_i32_neg", feat(p0, prop("val", (int) -42)), cfg());
355360
write("prop_i32_min", feat(p0, prop("val", Integer.MIN_VALUE)), cfg());
356361
write("prop_i32_max", feat(p0, prop("val", Integer.MAX_VALUE)), cfg());
357362
write(layer("prop_i32_val_null", feat(p0, prop("val", (int) 42)), feat(p0)), cfg());
358363
write(layer("prop_i32_null_val", feat(p0), feat(p0, prop("val", (int) 42))), cfg());
359364

360365
write("prop_u32", feat(p0, prop("val", U32.of(42L))), cfg());
366+
write("prop_u32_delta", feat(p0, prop("val", U32.of(42L))), cfg(DELTA));
361367
write("prop_u32_min", feat(p0, prop("val", U32.of(0L))), cfg());
362368
write("prop_u32_max", feat(p0, prop("val", U32.of(0xFFFFFFFFL))), cfg());
363369
write(layer("prop_u32_val_null", feat(p0, prop("val", U32.of(42L))), feat(p0)), cfg());
364370
write(layer("prop_u32_null_val", feat(p0), feat(p0, prop("val", U32.of(42L)))), cfg());
365371

366372
long i64_value = 9_876_543_210L;
367373
write("prop_i64", feat(p0, prop("val", i64_value)), cfg());
374+
write("prop_i64_delta", feat(p0, prop("val", i64_value)), cfg(DELTA));
368375
write("prop_i64_neg", feat(p0, prop("val", (long) -9_876_543_210L)), cfg());
369376
write("prop_i64_min", feat(p0, prop("val", Long.MIN_VALUE)), cfg());
370377
write("prop_i64_max", feat(p0, prop("val", Long.MAX_VALUE)), cfg());
@@ -374,6 +381,8 @@ private static void generateProperties() throws IOException {
374381
U64 u64_value = U64.of(BigInteger.valueOf(1234567890123456789L));
375382
U64 u64_max = U64.of(new BigInteger("18446744073709551615"));
376383
write("prop_u64", feat(p0, prop("bignum", u64_value)), cfg());
384+
// The exact shape that hid the const un-ZigZag bug, from real ev.mlt tiles.
385+
write("prop_u64_delta", feat(p0, prop("bignum", u64_value)), cfg(DELTA));
377386
write("prop_u64_min", feat(p0, prop("bignum", U64.of(BigInteger.ZERO))), cfg());
378387
write("prop_u64_max", feat(p0, prop("bignum", u64_max)), cfg());
379388
write(layer("prop_u64_val_null", feat(p0, prop("val", u64_value)), feat(p0)), cfg());

‎rust/mlt-synthetics/src/main.rs‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -553,6 +553,19 @@ fn generate_properties(w: &mut SynthWriter) {
553553
.write(w, "prop_i32_np");
554554
p0().add_prop(e_int, P::opt_i32("val", vec![Some(42)]))
555555
.write(w, "prop_i32");
556+
// Single-value variants hit the CONST/SEQUENCE path the multi-feature props_* tests miss.
557+
p0().add_prop(E::delta_varint(), P::i32("val", vec![42]))
558+
.write(w, "prop_i32_delta_np");
559+
p0().add_prop(E::delta_varint(), P::opt_i32("val", vec![Some(42)]))
560+
.write(w, "prop_i32_delta");
561+
p0().add_prop(E::rle_varint(), P::i32("val", vec![42]))
562+
.write(w, "prop_i32_rle_np");
563+
p0().add_prop(E::rle_varint(), P::opt_i32("val", vec![Some(42)]))
564+
.write(w, "prop_i32_rle-rust");
565+
p0().add_prop(E::delta_rle_varint(), P::i32("val", vec![42]))
566+
.write(w, "prop_i32_delta_rle_np");
567+
p0().add_prop(E::delta_rle_varint(), P::opt_i32("val", vec![Some(42)]))
568+
.write(w, "prop_i32_delta_rle-rust");
556569
p0().add_prop(e_int, P::i32("val", vec![-42]))
557570
.write(w, "prop_i32_neg_np");
558571
p0().add_prop(e_int, P::opt_i32("val", vec![Some(-42)]))
@@ -579,6 +592,18 @@ fn generate_properties(w: &mut SynthWriter) {
579592
.write(w, "prop_u32_np");
580593
p0().add_prop(e_int, P::opt_u32("val", vec![Some(42)]))
581594
.write(w, "prop_u32");
595+
p0().add_prop(E::delta_varint(), P::u32("val", vec![42]))
596+
.write(w, "prop_u32_delta_np");
597+
p0().add_prop(E::delta_varint(), P::opt_u32("val", vec![Some(42)]))
598+
.write(w, "prop_u32_delta");
599+
p0().add_prop(E::rle_varint(), P::u32("val", vec![42]))
600+
.write(w, "prop_u32_rle_np");
601+
p0().add_prop(E::rle_varint(), P::opt_u32("val", vec![Some(42)]))
602+
.write(w, "prop_u32_rle-rust");
603+
p0().add_prop(E::delta_rle_varint(), P::u32("val", vec![42]))
604+
.write(w, "prop_u32_delta_rle_np");
605+
p0().add_prop(E::delta_rle_varint(), P::opt_u32("val", vec![Some(42)]))
606+
.write(w, "prop_u32_delta_rle-rust");
582607
p0().add_prop(e_int, P::u32("val", vec![0]))
583608
.write(w, "prop_u32_min_np");
584609
p0().add_prop(e_int, P::opt_u32("val", vec![Some(0)]))
@@ -601,6 +626,27 @@ fn generate_properties(w: &mut SynthWriter) {
601626
.write(w, "prop_i64_np");
602627
p0().add_prop(e_int, P::opt_i64("val", vec![Some(9_876_543_210)]))
603628
.write(w, "prop_i64");
629+
p0().add_prop(E::delta_varint(), P::i64("val", vec![9_876_543_210]))
630+
.write(w, "prop_i64_delta_np");
631+
p0().add_prop(
632+
E::delta_varint(),
633+
P::opt_i64("val", vec![Some(9_876_543_210)]),
634+
)
635+
.write(w, "prop_i64_delta");
636+
p0().add_prop(E::rle_varint(), P::i64("val", vec![9_876_543_210]))
637+
.write(w, "prop_i64_rle_np");
638+
p0().add_prop(
639+
E::rle_varint(),
640+
P::opt_i64("val", vec![Some(9_876_543_210)]),
641+
)
642+
.write(w, "prop_i64_rle-rust");
643+
p0().add_prop(E::delta_rle_varint(), P::i64("val", vec![9_876_543_210]))
644+
.write(w, "prop_i64_delta_rle_np");
645+
p0().add_prop(
646+
E::delta_rle_varint(),
647+
P::opt_i64("val", vec![Some(9_876_543_210)]),
648+
)
649+
.write(w, "prop_i64_delta_rle-rust");
604650
p0().add_prop(e_int, P::i64("val", vec![-9_876_543_210]))
605651
.write(w, "prop_i64_neg_np");
606652
p0().add_prop(e_int, P::opt_i64("val", vec![Some(-9_876_543_210)]))
@@ -638,6 +684,37 @@ fn generate_properties(w: &mut SynthWriter) {
638684
.write(w, "prop_u64_max_np");
639685
p0().add_prop(e_int, P::opt_u64("bignum", vec![Some(u64::MAX)]))
640686
.write(w, "prop_u64_max");
687+
// The exact shape that hid the const un-ZigZag bug, from real ev.mlt tiles.
688+
p0().add_prop(
689+
E::delta_varint(),
690+
P::u64("bignum", vec![1_234_567_890_123_456_789]),
691+
)
692+
.write(w, "prop_u64_delta_np");
693+
p0().add_prop(
694+
E::delta_varint(),
695+
P::opt_u64("bignum", vec![Some(1_234_567_890_123_456_789)]),
696+
)
697+
.write(w, "prop_u64_delta");
698+
p0().add_prop(
699+
E::rle_varint(),
700+
P::u64("bignum", vec![1_234_567_890_123_456_789]),
701+
)
702+
.write(w, "prop_u64_rle_np");
703+
p0().add_prop(
704+
E::rle_varint(),
705+
P::opt_u64("bignum", vec![Some(1_234_567_890_123_456_789)]),
706+
)
707+
.write(w, "prop_u64_rle-rust");
708+
p0().add_prop(
709+
E::delta_rle_varint(),
710+
P::u64("bignum", vec![1_234_567_890_123_456_789]),
711+
)
712+
.write(w, "prop_u64_delta_rle_np");
713+
p0().add_prop(
714+
E::delta_rle_varint(),
715+
P::opt_u64("bignum", vec![Some(1_234_567_890_123_456_789)]),
716+
)
717+
.write(w, "prop_u64_delta_rle-rust");
641718
// Two-feature optional u64 variants (key is "val" to match Java)
642719
geo_varint_with_rle()
643720
.geos([P0, P0])

‎test/synthetic/0x01-rust/prop_i32_delta_np.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"type": "FeatureCollection",
3+
"features": [
4+
{
5+
"type": "Feature",
6+
"properties": {
7+
"_extent": 80,
8+
"_layer": "layer1",
9+
"val": 42
10+
},
11+
"geometry": {
12+
"type": "Point",
13+
"coordinates": [
14+
13,
15+
42
16+
]
17+
}
18+
}
19+
]
20+
}

‎test/synthetic/0x01-rust/prop_i32_delta_np.mlt‎

34 Bytes
Binary file not shown.

‎test/synthetic/0x01-rust/prop_i32_delta_rle.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"type": "FeatureCollection",
3+
"features": [
4+
{
5+
"type": "Feature",
6+
"properties": {
7+
"_extent": 80,
8+
"_layer": "layer1",
9+
"val": 42
10+
},
11+
"geometry": {
12+
"type": "Point",
13+
"coordinates": [
14+
13,
15+
42
16+
]
17+
}
18+
}
19+
]
20+
}

‎test/synthetic/0x01-rust/prop_i32_delta_rle.mlt‎

43 Bytes
Binary file not shown.

‎test/synthetic/0x01-rust/prop_i32_delta_rle_np.json‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
{
2+
"type": "FeatureCollection",
3+
"features": [
4+
{
5+
"type": "Feature",
6+
"properties": {
7+
"_extent": 80,
8+
"_layer": "layer1",
9+
"val": 42
10+
},
11+
"geometry": {
12+
"type": "Point",
13+
"coordinates": [
14+
13,
15+
42
16+
]
17+
}
18+
}
19+
]
20+
}

‎test/synthetic/0x01-rust/prop_i32_delta_rle_np.mlt‎

37 Bytes
Binary file not shown.

0 commit comments

Comments
 (0)