Skip to content

Commit e9d5bcb

Browse files
trekawekclaude
andcommitted
Guard the mealybug scores in CI with per-test baselines
MealybugRomTest now asserts each test's diff-pixel count against its known baseline instead of requiring an exact match: the pixel-perfect tests (m2_win_en_toggle, m3_obp0_change, m3_scx_low_3_bits) are locked at 0 and the rest cannot regress silently. The profile joins the CI workflow; when accuracy work improves a score, the baseline should be tightened in the same commit. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J1wLWscyUGS7CFwc3CjJR8
1 parent 4cc8881 commit e9d5bcb

2 files changed

Lines changed: 63 additions & 17 deletions

File tree

.github/workflows/maven.yml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@ jobs:
3737
run: mvn -B -pl core test -Ptest-blargg-individual
3838
- name: dmg-acid2
3939
run: mvn -B -pl core test -Ptest-dmgacid2
40+
- name: Mealybug Tearoom (baseline-guarded)
41+
run: mvn -B -pl core test -Ptest-mealybug
4042

4143
# Optional: Uploads the full dependency graph to GitHub to improve the quality of Dependabot alerts this repository can receive
4244
- name: Update dependency graph
Lines changed: 61 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1,97 @@
11
package eu.rekawek.coffeegb.core.integration.mealybug;
22

3+
import eu.rekawek.coffeegb.core.Gameboy;
34
import eu.rekawek.coffeegb.core.GameboyType;
5+
import eu.rekawek.coffeegb.core.integration.support.ImageTestRunner;
46
import org.junit.Test;
57
import org.junit.runner.RunWith;
68
import org.junit.runners.Parameterized;
79

810
import java.io.File;
9-
import java.nio.file.Path;
1011
import java.nio.file.Paths;
1112
import java.util.ArrayList;
12-
import java.util.Arrays;
1313
import java.util.Collection;
1414
import java.util.List;
15+
import java.util.Map;
1516

16-
import static eu.rekawek.coffeegb.core.integration.support.RomTestUtils.testRomWithImage;
17+
import static org.junit.Assert.assertTrue;
1718

1819
/**
19-
* Mealybug Tearoom tests: mid-scanline writes to the PPU registers (SCX/SCY, LCDC bits, BGP)
20-
* during mode 3. The expected images were captured from real hardware (DMG-CPU B where the
21-
* revisions differ, DMG-blob otherwise); a test passes when the frame at the LD B,B breakpoint
22-
* is pixel-identical.
20+
* Mealybug Tearoom tests: mid-scanline writes to the PPU registers during mode 3, compared
21+
* against photos of a real DMG-CPU B (DMG-blob where no CPU B photo exists). The tests run
22+
* with the FAST_FORWARD boot because they reuse the boot ROM's VRAM leftovers (the (r) logo
23+
* tile) as sprite data.
24+
*
25+
* <p>Each ROM asserts that the number of differing pixels does not exceed its known
26+
* baseline, so the pixel-exact tests are locked at 0 and the rest cannot regress silently.
27+
* When an accuracy change improves a score, tighten the baseline here.
2328
*/
2429
@RunWith(Parameterized.class)
2530
public class MealybugRomTest {
2631

27-
private final Path rom;
32+
private static final Map<String, Integer> BASELINES = Map.ofEntries(
33+
Map.entry("m2_win_en_toggle.gb", 0),
34+
Map.entry("m3_bgp_change.gb", 21),
35+
Map.entry("m3_bgp_change_sprites.gb", 20),
36+
Map.entry("m3_lcdc_bg_en_change.gb", 11),
37+
Map.entry("m3_lcdc_bg_map_change.gb", 192),
38+
Map.entry("m3_lcdc_obj_en_change.gb", 2),
39+
Map.entry("m3_lcdc_obj_en_change_variant.gb", 54),
40+
Map.entry("m3_lcdc_obj_size_change.gb", 360),
41+
Map.entry("m3_lcdc_obj_size_change_scx.gb", 230),
42+
Map.entry("m3_lcdc_tile_sel_change.gb", 776),
43+
Map.entry("m3_lcdc_tile_sel_win_change.gb", 1062),
44+
Map.entry("m3_lcdc_win_en_change_multiple.gb", 8316),
45+
Map.entry("m3_lcdc_win_en_change_multiple_wx.gb", 15756),
46+
Map.entry("m3_lcdc_win_map_change.gb", 212),
47+
Map.entry("m3_obp0_change.gb", 0),
48+
Map.entry("m3_scx_high_5_bits.gb", 84),
49+
Map.entry("m3_scx_low_3_bits.gb", 0),
50+
Map.entry("m3_scy_change.gb", 4269),
51+
Map.entry("m3_window_timing.gb", 3),
52+
Map.entry("m3_window_timing_wx_0.gb", 130),
53+
Map.entry("m3_wx_4_change.gb", 229),
54+
Map.entry("m3_wx_4_change_sprites.gb", 10),
55+
Map.entry("m3_wx_5_change.gb", 638),
56+
Map.entry("m3_wx_6_change.gb", 13799));
2857

29-
public MealybugRomTest(String name, Path rom) {
58+
private final File rom;
59+
60+
private final int baseline;
61+
62+
public MealybugRomTest(String name, File rom, int baseline) {
3063
this.rom = rom;
64+
this.baseline = baseline;
3165
}
3266

3367
@Parameterized.Parameters(name = "{0}")
3468
public static Collection<Object[]> data() {
3569
File dir = Paths.get("src/test/resources/roms/mealybug").toFile();
36-
File[] roms = dir.listFiles((d, n) -> n.endsWith(".gb"));
37-
Arrays.sort(roms);
3870
List<Object[]> params = new ArrayList<>();
39-
for (File rom : roms) {
40-
// ROMs without an expected image are CGB-only variants
41-
if (new File(dir, rom.getName().replace(".gb", ".png")).isFile()) {
42-
params.add(new Object[]{rom.getName(), rom.toPath()});
71+
for (Map.Entry<String, Integer> e : BASELINES.entrySet()) {
72+
File rom = new File(dir, e.getKey());
73+
if (rom.isFile()) {
74+
params.add(new Object[]{e.getKey(), rom, e.getValue()});
4375
}
4476
}
77+
params.sort((a, b) -> ((String) a[0]).compareTo((String) b[0]));
4578
return params;
4679
}
4780

4881
@Test
4982
public void test() throws Exception {
50-
// FAST_FORWARD: the tests reuse the boot ROM's VRAM leftovers (the (r) logo tile)
51-
testRomWithImage(rom, GameboyType.DMG, eu.rekawek.coffeegb.core.Gameboy.BootstrapMode.FAST_FORWARD);
83+
ImageTestRunner runner =
84+
new ImageTestRunner(rom, GameboyType.DMG, Gameboy.BootstrapMode.FAST_FORWARD);
85+
ImageTestRunner.TestResult result = runner.runTest();
86+
int[] actual = result.getResultRGB();
87+
int[] expected = result.getExpectedRGB();
88+
int diff = 0;
89+
for (int i = 0; i < expected.length; i++) {
90+
if (actual[i] != expected[i]) {
91+
diff++;
92+
}
93+
}
94+
assertTrue("diff pixels " + diff + " exceeds the known baseline " + baseline
95+
+ " - an accuracy regression", diff <= baseline);
5296
}
5397
}

0 commit comments

Comments
 (0)