Auto Camera Registration Pipeline

client/dive-common/apispec.ts

@@ -9,7 +9,7 @@ import { CustomStyle } from 'vue-media-annotator/StyleManager';
 import { AttributeTrackFilter } from 'vue-media-annotator/AttributeTrackFilterControls';
 import { ImageEnhancements } from 'vue-media-annotator/use/useImageEnhancements';
 import type {
-  CameraHomographies, CameraCorrespondences, CameraTransformTypes, RegistrationSource,
+  CameraHomographies, CameraObservations, CameraTransformTypes, RegistrationSource,
 } from 'vue-media-annotator/alignedView/CameraRegistrationStore';
 import type { PercentileStretch } from 'vue-media-annotator/use/useImageEnhancements';
 
@@ -84,6 +84,15 @@ interface PipeMetadata {
 
 interface PipelineRuntimeParams {
   frameRange?: [number, number] | null;
+  /**
+   * Multicam registration subset: camera name -> ordered image identifiers
+   * for exactly the frames the job should process. Row i of one camera's
+   * list pairs with row i of every other's. Identifiers are the camera's
+   * own image names (the platform backend resolves them to real paths) or
+   * `frame://N` pseudo-names for video cameras (the backend extracts those
+   * frames to temp images before the job).
+   */
+  imagePairs?: Record<string, string[]>;
 }
 
 interface PipelineParams {
@@ -268,7 +277,12 @@ interface DatasetConfigMutable {
   attributeTrackFilters?: Readonly<Record<string, AttributeTrackFilter>>;
   datasetInfo?: DatasetInfoFields;
   cameraHomographies?: CameraHomographies;
-  cameraCorrespondences?: CameraCorrespondences;
+  /**
+   * Per-image-pair correspondence observations, keyed by directional
+   * "left::right". Each entry lists the observations (image-pair identity,
+   * enabled flag, producer source, stats, and points) behind that pair's fit.
+   */
+  cameraCorrespondences?: CameraObservations;
   cameraTransformTypes?: CameraTransformTypes;
   /** Producer provenance of the camera registration (see RegistrationSource). */
   cameraRegistrationSource?: RegistrationSource | null;
@@ -366,9 +380,25 @@ interface DatasetCalibrationResult {
   conversionError?: string;
 }
 
+/** Terminal state of a pipeline job, as reported by {@link Api.watchPipelineJob}. */
+export interface PipelineJobResult {
+  /** True when the job exited successfully. */
+  ok: boolean;
+  /** Human-readable reason when `ok` is false. */
+  message?: string;
+}
+
 interface Api {
   getPipelineList(): Promise<Pipelines>;
   runPipeline(itemId: string, pipeline: Pipe, pipelineParams?: PipelineParams): Promise<unknown>;
+  /**
+   * Resolve once the pipeline job this dataset just launched reaches a terminal
+   * state, so a caller can key completion off the job instead of off whatever
+   * the job was expected to write. Optional: a platform without a job feed
+   * leaves it undefined and callers fall back to watching for the artifact,
+   * which cannot tell "finished, output identical" from "still running".
+   */
+  watchPipelineJob?(datasetId: string, pipeline: Pipe): Promise<PipelineJobResult>;
   deleteTrainedPipeline(pipeline: Pipe): Promise<void>;
   exportTrainedPipeline(path: string, pipeline: Pipe): Promise<unknown>;
   getDatasetCalibration(datasetId: string): Promise<DatasetCalibrationResult | null>;
@@ -700,4 +730,4 @@ export type {
   MediaImportResponse,
 };
 
-export type { PercentileStretch, CameraCorrespondences };
+export type { PercentileStretch, CameraObservations };

client/dive-common/autoRegisterSelection.spec.ts

@@ -0,0 +1,88 @@
+import { describe, expect, it } from 'vitest';
+
+import proposeRegistrationFrames from './autoRegisterSelection';
+
+describe('proposeRegistrationFrames', () => {
+  it('spreads candidates across every temporal bin (no single-scene bias)', () => {
+    const frames = proposeRegistrationFrames({
+      counts: [1200, 1200],
+      bins: 12,
+      perBin: 2,
+    });
+    expect(frames.length).toBe(24);
+    // Every 100-frame bin contributes exactly its share: a scene-rich
+    // stretch can never supply all the candidates.
+    for (let bin = 0; bin < 12; bin += 1) {
+      const inBin = frames.filter((f) => f >= bin * 100 && f < (bin + 1) * 100);
+      expect(inBin.length).toBe(2);
+    }
+    expect(frames).toEqual([...frames].sort((a, b) => a - b));
+  });
+
+  it('spans only the shortest camera', () => {
+    const frames = proposeRegistrationFrames({
+      counts: [1000, 300],
+      bins: 10,
+      perBin: 1,
+    });
+    expect(Math.max(...frames)).toBeLessThan(300);
+    expect(frames.length).toBe(10);
+  });
+
+  it('ranks within a bin by inter-camera timestamp skew', () => {
+    // Two cameras, 10 frames, one bin: frame 6 is perfectly synced, frame 3
+    // is close, everything else is badly skewed.
+    const base = 1_700_000_000;
+    const camA = Array.from({ length: 10 }, (_, i) => base + i);
+    const camB = camA.map((t, i) => {
+      if (i === 6) return t;
+      if (i === 3) return t + 0.1;
+      return t + 5;
+    });
+    const frames = proposeRegistrationFrames({
+      counts: [10, 10],
+      timestamps: [camA, camB],
+      bins: 1,
+      perBin: 2,
+      maxSkewSeconds: 0.5,
+    });
+    expect(frames).toEqual([3, 6]);
+  });
+
+  it('drops candidates whose skew exceeds the threshold entirely', () => {
+    const base = 1_700_000_000;
+    const camA = Array.from({ length: 4 }, (_, i) => base + i);
+    const camB = camA.map((t) => t + 10); // hopeless sync everywhere
+    const frames = proposeRegistrationFrames({
+      counts: [4, 4],
+      timestamps: [camA, camB],
+      bins: 1,
+      perBin: 2,
+      maxSkewSeconds: 0.5,
+    });
+    expect(frames).toEqual([]);
+  });
+
+  it('falls back to even spread for frames without timestamps', () => {
+    const frames = proposeRegistrationFrames({
+      counts: [100, 100],
+      timestamps: [
+        Array.from({ length: 100 }, () => undefined),
+        Array.from({ length: 100 }, () => undefined),
+      ],
+      bins: 2,
+      perBin: 2,
+    });
+    expect(frames.length).toBe(4);
+    expect(frames.filter((f) => f < 50).length).toBe(2);
+    expect(frames.filter((f) => f >= 50).length).toBe(2);
+  });
+
+  it('handles degenerate inputs', () => {
+    expect(proposeRegistrationFrames({ counts: [0, 10], bins: 5, perBin: 2 })).toEqual([]);
+    expect(proposeRegistrationFrames({ counts: [], bins: 5, perBin: 2 })).toEqual([]);
+    // More bins than frames: every frame proposed once.
+    const tiny = proposeRegistrationFrames({ counts: [3, 3], bins: 12, perBin: 2 });
+    expect(tiny).toEqual([0, 1, 2]);
+  });
+});

client/dive-common/autoRegisterSelection.ts

@@ -0,0 +1,108 @@
+/**
+ * Candidate-frame proposal for the auto-register pipeline (stage 0 of the
+ * selection contract): DIVE picks for diversity and synchronization, VIAME
+ * picks for image quality.
+ *
+ * Stratified, not "evenly spread" and not "most featureful": the flight is
+ * divided into `bins` equal time bins and `perBin` candidates are proposed
+ * within each, so temporal spread is guaranteed structurally -- a single
+ * scene-rich stretch can never supply every frame, which would reconstitute
+ * exactly the single-scene bias the multi-pair restructure exists to remove.
+ * The pipeline then keeps the best candidate per bin by image quality
+ * (`max_frames` is the bin count), so oversampling here is deliberate.
+ *
+ * Within a bin, candidates rank by inter-camera timestamp skew when every
+ * camera carries per-frame timestamps: on a survey aircraft at ~100 kt,
+ * 100 ms of desync is ~5 m of ground motion baked straight into the "ground
+ * truth" points, and RANSAC cannot reject it because it is consistent
+ * within the frame. Without timestamps (positional alignment), candidates
+ * spread evenly within the bin and skew is unknowable -- no threshold
+ * applies.
+ */
+
+export interface ProposalOptions {
+  /** Per-camera usable frame counts; the proposal spans [0, min(counts)). */
+  counts: number[];
+  /**
+   * Optional per-camera per-frame capture timestamps (epoch seconds;
+   * undefined entries = unknown). Skew ranking applies only when every
+   * camera has a timestamp for the frame under consideration.
+   */
+  timestamps?: (number | undefined)[][];
+  /** Number of temporal bins (the pipeline's max_frames budget). */
+  bins: number;
+  /** Candidates proposed per bin. */
+  perBin: number;
+  /** Candidates with a larger inter-camera skew (seconds) are dropped. */
+  maxSkewSeconds?: number;
+}
+
+/** Largest pairwise timestamp difference across cameras, or null if unknowable. */
+function frameSkew(
+  timestamps: (number | undefined)[][],
+  frame: number,
+): number | null {
+  const stamps: number[] = [];
+  for (let cam = 0; cam < timestamps.length; cam += 1) {
+    const t = timestamps[cam]?.[frame];
+    if (t === undefined) {
+      return null;
+    }
+    stamps.push(t);
+  }
+  return Math.max(...stamps) - Math.min(...stamps);
+}
+
+/**
+ * Propose candidate frame indices, sorted ascending. Returns at most
+ * `bins * perBin` frames; short datasets simply yield fewer.
+ */
+export default function proposeRegistrationFrames(options: ProposalOptions): number[] {
+  const usable = Math.min(...options.counts);
+  if (!Number.isFinite(usable) || usable <= 0 || options.bins <= 0 || options.perBin <= 0) {
+    return [];
+  }
+  const bins = Math.min(options.bins, usable);
+  const { timestamps } = options;
+  const maxSkew = options.maxSkewSeconds ?? 0.5;
+  const chosen = new Set<number>();
+  for (let bin = 0; bin < bins; bin += 1) {
+    const start = Math.floor((bin * usable) / bins);
+    const end = Math.floor(((bin + 1) * usable) / bins); // exclusive
+    // bins <= usable, so every bin spans at least one frame.
+    const size = end - start;
+    const perBin = Math.min(options.perBin, size);
+    if (timestamps && timestamps.length) {
+      // Rank the whole bin by skew; unknowable-skew frames rank after
+      // measured ones (evenly spread among themselves), over-threshold
+      // frames are dropped outright.
+      const measured: { frame: number; skew: number }[] = [];
+      const unknowable: number[] = [];
+      for (let frame = start; frame < end; frame += 1) {
+        const skew = frameSkew(timestamps, frame);
+        if (skew === null) {
+          unknowable.push(frame);
+        } else if (skew <= maxSkew) {
+          measured.push({ frame, skew });
+        }
+      }
+      measured.sort((a, b) => a.skew - b.skew || a.frame - b.frame);
+      measured.slice(0, perBin).forEach(({ frame }) => chosen.add(frame));
+      let still = perBin - Math.min(measured.length, perBin);
+      if (still > 0 && unknowable.length) {
+        const step = unknowable.length / still;
+        for (let i = 0; i < still; i += 1) {
+          chosen.add(unknowable[Math.floor(i * step)]);
+        }
+        still = 0;
+      }
+    } else {
+      // No timeline: spread evenly within the bin.
+      const step = size / perBin;
+      for (let i = 0; i < perBin; i += 1) {
+        chosen.add(start + Math.floor(i * step + step / 2));
+      }
+    }
+  }
+  return [...chosen].sort((a, b) => a - b);
+}