test: make sure that marked-failing synthetics actually fail (#1451)

Currently, when a synthetic test passes we would not know.
This happened to MLT-Wasm, so lets assert that this does not happen.

Author: CommanderStorm

cpp/tool/synthetic.test.ts

@@ -3,6 +3,7 @@ import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
 import {
   compareWithTolerance,
+  expectUnsupported,
   getTestCases,
   writeActualOutput,
 } from "synthetic-test-utils";
@@ -11,7 +12,7 @@ import { describe, expect, it } from "vitest";
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const binary = resolve(__dirname, "../build/tool/mlt-cpp-json");
 
-const SKIPPED_TESTS = [];
+const SKIPPED_TESTS: string[] = [];
 
 describe("MLT Decoder - Synthetic tests", () => {
   expect.addEqualityTesters([compareWithTolerance]);
@@ -28,10 +29,9 @@ describe("MLT Decoder - Synthetic tests", () => {
     });
   }
 
-  for (const skippedTest of testCases.skipped) {
-    it.skip(skippedTest, () => {
-      // Test is skipped since it is not supported yet
-    });
+  for (const { name, content, fileName } of testCases.skipped) {
+    it(`${name} (unsupported)`, () =>
+      expectUnsupported(() => decodeMLT(fileName), content));
   }
 });
 

rust/mlt-wasm/js/synthetic.spec.ts

@@ -3,6 +3,7 @@ import type { VectorTileLike } from "@maplibre/vt-pbf";
 import { describe, expect, it } from "vitest";
 import {
   compareWithTolerance,
+  expectUnsupported,
   getTestCases,
   writeActualOutput,
 } from "../../../test/synthetic/synthetic-test-utils";
@@ -13,21 +14,7 @@ import {
   type MltLayer,
 } from "./vectorTile";
 
-const UNIMPLEMENTED_SYNTHETICS = new Map([
-  ["poly_collinear_fpf", "FastPFor not supported"],
-  ["poly_collinear_fpf_tes", "FastPFor not supported"],
-  ["poly_fpf", "FastPFor not supported"],
-  ["poly_fpf_tes", "FastPFor not supported"],
-  ["poly_hole_fpf", "FastPFor not supported"],
-  ["poly_hole_fpf_tes", "FastPFor not supported"],
-  ["poly_hole_touching_fpf", "FastPFor not supported"],
-  ["poly_hole_touching_fpf_tes", "FastPFor not supported"],
-  ["poly_multi_fpf", "FastPFor not supported"],
-  ["poly_multi_fpf_tes", "FastPFor not supported"],
-  ["poly_self_intersect_fpf", "FastPFor not supported"],
-  ["poly_self_intersect_fpf_tes", "FastPFor not supported"],
-  ["poly_multi_morton_hole_morton", "Pending investigation"],
-]);
+const UNIMPLEMENTED_SYNTHETICS = new Map<string, string>([]);
 
 describe("MLT WASM Decoder - Synthetic tests", () => {
   expect.addEqualityTesters([compareWithTolerance]);
@@ -41,10 +28,9 @@ describe("MLT WASM Decoder - Synthetic tests", () => {
     });
   }
 
-  for (const skippedTest of testCases.skipped) {
-    it.skip(skippedTest, () => {
-      // Test is skipped since it is not supported yet
-    });
+  for (const { name, content, fileName } of testCases.skipped) {
+    it(`${name} (unsupported: ${UNIMPLEMENTED_SYNTHETICS.get(name)})`, () =>
+      expectUnsupported(() => decodeMLT(fileName), content));
   }
 });
 

test/synthetic/synthetic-test-utils/index.ts

@@ -2,6 +2,7 @@ import { globSync, readFileSync, writeFileSync } from "node:fs";
 import * as path from "node:path";
 import { dirname, resolve } from "node:path";
 import { fileURLToPath } from "node:url";
+import { expect } from "vitest";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const RELATIVE_FLOAT_TOLERANCE = 0.0001 / 100;
@@ -44,29 +45,57 @@ export function writeActualOutput(
   return actualFile;
 }
 
+export type TestCase = { name: string; content: object; fileName: string };
+
 export function getTestCases(skipList: string[]): {
-  active: { name: string; content: object; fileName: string }[];
-  skipped: string[];
+  active: TestCase[];
+  skipped: TestCase[];
 } {
   const syntheticDir = resolve(__dirname, "..");
   const mltFiles = globSync(`**/*.mlt`, {
     cwd: syntheticDir,
   }).map((mltFile: string) => path.join(syntheticDir, mltFile));
 
-  const active: { name: string; content: object; fileName: string }[] = [];
-  const skipped: string[] = [];
+  const active: TestCase[] = [];
+  const skipped: TestCase[] = [];
+  const matched = new Set<string>();
 
   for (const mltFile of mltFiles) {
     const testName = path.relative(syntheticDir, mltFile).replace(/\.mlt$/, "");
+    const jsonFile = mltFile.replace(/\.mlt$/, ".json");
+    const expected = JSON.parse(readFileSync(jsonFile, "utf-8"));
+    const testCase = { name: testName, fileName: mltFile, content: expected };
     if (skipList.includes(testName)) {
-      skipped.push(testName);
+      matched.add(testName);
+      skipped.push(testCase);
     } else {
-      const jsonFile = mltFile.replace(/\.mlt$/, ".json");
-      const expectedRaw = readFileSync(jsonFile, "utf-8");
-      const expected = JSON.parse(expectedRaw);
-      active.push({ name: testName, fileName: mltFile, content: expected });
+      active.push(testCase);
     }
   }
 
+  const unmatched = skipList.filter((name) => !matched.has(name));
+  if (unmatched.length > 0) {
+    throw new Error(
+      `Exclusion list references unknown synthetic test(s): ${unmatched.join(", ")}. ` +
+        `Use the full path-style name, e.g. "0x01/poly_fpf".`,
+    );
+  }
+
   return { active, skipped };
 }
+
+export async function expectUnsupported(
+  decode: () => Promise<unknown>,
+  content: object,
+): Promise<void> {
+  let actual: unknown;
+  try {
+    actual = await decode();
+  } catch {
+    return;
+  }
+  expect(
+    actual,
+    "decoded and matched the expected output — remove it from the exclusion list",
+  ).not.toEqual(content);
+}

ts/src/synthetic.spec.ts

@@ -2,16 +2,19 @@ import { readFile } from "node:fs/promises";
 import { describe, expect, it } from "vitest";
 import { classifyRings } from "@maplibre/maplibre-gl-style-spec";
 import { GEOMETRY_TYPE } from "./vector/geometry/geometryType";
-import { compareWithTolerance, getTestCases, writeActualOutput } from "../../test/synthetic/synthetic-test-utils";
+import {
+    compareWithTolerance,
+    expectUnsupported,
+    getTestCases,
+    writeActualOutput,
+} from "../../test/synthetic/synthetic-test-utils";
 import decodeTile from "./mltDecoder";
 import type { Geometry } from "./vector/geometry/geometryVector";
 import type FeatureTable from "./vector/featureTable";
 
 const EARCUT_MAX_RINGS = 500;
 
 const UNIMPLEMENTED_SYNTHETICS: string[] = [
-    "0x01/multipoint_morton",
-    "0x01/poly_morton_hole_morton",
     "0x01/poly_multi_morton_hole_morton",
     "0x01/poly_multi_morton_ring_morton",
     "0x01/poly_multi_morton_ring_no_morton",
@@ -28,10 +31,8 @@ describe("MLT Decoder - Synthetic tests", () => {
         });
     }
 
-    for (const skippedTest of testCases.skipped) {
-        it.skip(skippedTest, () => {
-            // Test is skipped since it is not supported yet
-        });
+    for (const { name, content, fileName } of testCases.skipped) {
+        it(`${name} (unsupported)`, () => expectUnsupported(() => decodeMLT(fileName), content));
     }
 });