Add built-in webcam capture for the Game Boy Camera (#60)

The Pocket Camera sensor could only read from an image file kept fresh by
an external command (ffmpeg). This adds a real webcam feed with no
external tooling, via the sarxos webcam-capture library.

- core: a small CameraSource interface (getFrame -> BufferedImage) that
  the front end registers on PocketCamera; the camera prefers it over the
  image file, then the file, then the synthetic test pattern. The core
  stays free of any capture dependency. The frame scale/luminance step is
  factored into a shared toLuma() helper.
- swing: WebcamCameraSource opens the default webcam in async mode (a
  cheap non-blocking getImage() per capture), picking the smallest
  supported resolution that still covers the 128x112 sensor. A
  "Use webcam as camera" toggle in the Game menu wires it in and closes
  it again; if no webcam opens it reports the error and unchecks.

Verified end to end: with a synthetic CameraSource the Game Boy Camera's
SHOOT viewfinder renders the live frame dithered in camera style.

Refs #60

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J1wLWscyUGS7CFwc3CjJR8

Author: trekawek

core/src/main/java/eu/rekawek/coffeegb/core/memory/cart/type/CameraSource.java

@@ -0,0 +1,18 @@
+package eu.rekawek.coffeegb.core.memory.cart.type;
+
+import java.awt.image.BufferedImage;
+
+/**
+ * A live source of frames for the Pocket Camera sensor. A front end (e.g. the Swing UI
+ * with a real webcam) registers one through {@link PocketCamera#setCameraSource}; the
+ * camera scales each returned frame to the sensor's 128x112 and dithers it like the real
+ * ASIC. Keeping this out of the core means the core stays free of any capture dependency.
+ */
+public interface CameraSource {
+
+    /**
+     * @return the most recent camera frame, or {@code null} if none is available yet
+     * (the camera then falls back to the image file / test pattern)
+     */
+    BufferedImage getFrame();
+}

core/src/main/java/eu/rekawek/coffeegb/core/memory/cart/type/PocketCamera.java

@@ -13,11 +13,12 @@
 /**
  * The Pocket Camera cartridge (type 0xFC): MBC5-like ROM banking, 128 KB of banked RAM,
  * and the camera's register file mapped in place of RAM when RAM-bank bit 4 is set.
- * A capture completes instantly (the busy bit always reads 0). The sensor image comes
- * from the file named by the {@code coffeegb.camera.image} system property, re-read on
- * every capture, so pointing it at a file periodically refreshed from a webcam (e.g.
- * {@code ffmpeg -f v4l2 -i /dev/video0 -update 1 cam.jpg}) gives a live camera; without
- * the property a synthetic test pattern is photographed. The image is scaled to the
+ * A capture completes instantly (the busy bit always reads 0). The sensor image comes,
+ * in priority order, from a live {@link CameraSource} registered by the front end (the
+ * Swing UI wires a real webcam through it), then the file named by the
+ * {@code coffeegb.camera.image} system property (re-read on every capture, e.g. one kept
+ * fresh by {@code ffmpeg -f v4l2 -i /dev/video0 -update 1 cam.jpg}), and finally a
+ * synthetic test pattern. The image is scaled to the
  * sensor's 128x112, converted to luminance and dithered through the game-supplied
  * threshold matrix (registers 0x06-0x35), and the result is written as 2bpp tiles to the
  * capture buffer at RAM offset 0x100 like the real ASIC does.
@@ -54,6 +55,16 @@ public class PocketCamera implements MemoryController {
 
     private transient int[] sourceLuma;
 
+    // a live capture source (e.g. a webcam) registered by the front end; when present it
+    // takes priority over the image file. Static because the cartridge is created deep in
+    // the emulator and there is only ever one camera.
+    private static volatile CameraSource cameraSource;
+
+    /** Registers (or clears, with {@code null}) the live sensor source. */
+    public static void setCameraSource(CameraSource source) {
+        cameraSource = source;
+    }
+
     public PocketCamera(Rom rom, Battery battery) {
         this.rom = rom.getRom();
         this.ram = new int[0x20000];
@@ -148,6 +159,18 @@ private void capture() {
     }
 
     private int[] sensorImage() {
+        // a registered live source (webcam) wins over the file; it is re-read every capture
+        CameraSource source = cameraSource;
+        if (source != null) {
+            try {
+                BufferedImage frame = source.getFrame();
+                if (frame != null) {
+                    return toLuma(frame);
+                }
+            } catch (Exception e) {
+                // fall through to the file / test pattern
+            }
+        }
         String path = System.getProperty("coffeegb.camera.image");
         if (path != null) {
             File f = new File(path);
@@ -156,18 +179,7 @@ private int[] sensorImage() {
                     if (sourceLuma == null || f.lastModified() != sourceTimestamp) {
                         BufferedImage src = ImageIO.read(f);
                         if (src != null) {
-                            BufferedImage scaled = new BufferedImage(128, 112, BufferedImage.TYPE_INT_RGB);
-                            Graphics2D g = scaled.createGraphics();
-                            g.drawImage(src, 0, 0, 128, 112, null);
-                            g.dispose();
-                            int[] luma = new int[128 * 112];
-                            for (int y = 0; y < 112; y++) {
-                                for (int x = 0; x < 128; x++) {
-                                    int rgb = scaled.getRGB(x, y);
-                                    luma[y * 128 + x] = ((rgb >> 16 & 0xff) * 77 + (rgb >> 8 & 0xff) * 151 + (rgb & 0xff) * 28) >> 8;
-                                }
-                            }
-                            sourceLuma = luma;
+                            sourceLuma = toLuma(src);
                             sourceTimestamp = f.lastModified();
                         }
                     }
@@ -191,6 +203,22 @@ private int[] sensorImage() {
         return luma;
     }
 
+    /** Scales a frame to the sensor's 128x112 and converts it to per-pixel luminance. */
+    private static int[] toLuma(BufferedImage src) {
+        BufferedImage scaled = new BufferedImage(128, 112, BufferedImage.TYPE_INT_RGB);
+        Graphics2D g = scaled.createGraphics();
+        g.drawImage(src, 0, 0, 128, 112, null);
+        g.dispose();
+        int[] luma = new int[128 * 112];
+        for (int y = 0; y < 112; y++) {
+            for (int x = 0; x < 128; x++) {
+                int rgb = scaled.getRGB(x, y);
+                luma[y * 128 + x] = ((rgb >> 16 & 0xff) * 77 + (rgb >> 8 & 0xff) * 151 + (rgb & 0xff) * 28) >> 8;
+            }
+        }
+        return luma;
+    }
+
     @Override
     public void flushRam() {
         battery.saveRam(ram);

swing/pom.xml

@@ -58,5 +58,10 @@
             <artifactId>core</artifactId>
             <version>1.7.2-SNAPSHOT</version>
         </dependency>
+        <dependency>
+            <groupId>com.github.sarxos</groupId>
+            <artifactId>webcam-capture</artifactId>
+            <version>0.3.12</version>
+        </dependency>
     </dependencies>
 </project>

swing/src/main/java/eu/rekawek/coffeegb/swing/SwingMenu.kt

@@ -28,6 +28,8 @@ import eu.rekawek.coffeegb.controller.properties.EmulatorProperties.Key.DmgGames
 import eu.rekawek.coffeegb.core.GameboyType
 import eu.rekawek.coffeegb.core.events.EventBus
 import eu.rekawek.coffeegb.core.genie.AddPatches
+import eu.rekawek.coffeegb.core.memory.cart.type.PocketCamera
+import eu.rekawek.coffeegb.swing.io.WebcamCameraSource
 import eu.rekawek.coffeegb.core.genie.PatchFactory
 import eu.rekawek.coffeegb.core.sgb.SgbDisplay
 import eu.rekawek.coffeegb.core.sound.Sound
@@ -58,6 +60,8 @@ class SwingMenu(
 
   private var pauseSupport: Boolean = false
 
+  private var webcamSource: WebcamCameraSource? = null
+
   init {
     eventBus.register<SessionSnapshotSupportEvent> { snapshotSupport = it.snapshotSupport }
     eventBus.register<Controller.SessionPauseSupportEvent> { pauseSupport = it.enabled }
@@ -236,6 +240,30 @@ class SwingMenu(
     }
     enableWhenEmulationActive(scanBarcode)
 
+    val webcam = JCheckBoxMenuItem("Use webcam as camera", false)
+    gameMenu.add(webcam)
+    webcam.addActionListener {
+      if (webcam.state) {
+        val source = WebcamCameraSource.open()
+        if (source == null) {
+          webcam.state = false
+          JOptionPane.showMessageDialog(
+              window,
+              "No webcam could be opened.",
+              "Game Boy Camera",
+              JOptionPane.ERROR_MESSAGE,
+          )
+        } else {
+          webcamSource = source
+          PocketCamera.setCameraSource(source)
+        }
+      } else {
+        PocketCamera.setCameraSource(null)
+        webcamSource?.close()
+        webcamSource = null
+      }
+    }
+
     return gameMenu
   }
 

swing/src/main/java/eu/rekawek/coffeegb/swing/io/WebcamCameraSource.java

@@ -0,0 +1,75 @@
+package eu.rekawek.coffeegb.swing.io;
+
+import com.github.sarxos.webcam.Webcam;
+import eu.rekawek.coffeegb.core.memory.cart.type.CameraSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.awt.Dimension;
+import java.awt.image.BufferedImage;
+
+/**
+ * A live {@link CameraSource} for the Game Boy (Pocket) Camera, backed by a real webcam
+ * through the sarxos webcam-capture library. The webcam runs in async mode: a background
+ * thread keeps the latest frame buffered, so {@link #getFrame()} is a cheap non-blocking
+ * read the emulator can call on every in-game capture.
+ */
+public class WebcamCameraSource implements CameraSource, AutoCloseable {
+
+    private static final Logger LOG = LoggerFactory.getLogger(WebcamCameraSource.class);
+
+    private final Webcam webcam;
+
+    private WebcamCameraSource(Webcam webcam) {
+        this.webcam = webcam;
+    }
+
+    /**
+     * Opens the default webcam.
+     *
+     * @return the source, or {@code null} if no webcam is available or it cannot be opened
+     */
+    public static WebcamCameraSource open() {
+        try {
+            Webcam webcam = Webcam.getDefault();
+            if (webcam == null) {
+                LOG.warn("No webcam detected");
+                return null;
+            }
+            // the frame is downscaled to the 128x112 sensor anyway, so prefer the smallest
+            // supported resolution that still covers it (lower latency and CPU)
+            Dimension best = null;
+            for (Dimension d : webcam.getViewSizes()) {
+                if (d.width < 128 || d.height < 112) {
+                    continue;
+                }
+                if (best == null || (long) d.width * d.height < (long) best.width * best.height) {
+                    best = d;
+                }
+            }
+            if (best != null) {
+                webcam.setViewSize(best);
+            }
+            webcam.open(true);
+            LOG.info("Opened webcam {} at {}", webcam.getName(), webcam.getViewSize());
+            return new WebcamCameraSource(webcam);
+        } catch (Throwable t) {
+            LOG.warn("Failed to open the webcam", t);
+            return null;
+        }
+    }
+
+    @Override
+    public BufferedImage getFrame() {
+        return webcam.isOpen() ? webcam.getImage() : null;
+    }
+
+    @Override
+    public void close() {
+        try {
+            webcam.close();
+        } catch (Throwable t) {
+            LOG.warn("Failed to close the webcam", t);
+        }
+    }
+}