Add overlay-dx metric (#3270)

Author: DresdenGman

src/gluonts/evaluation/__init__.py

@@ -29,3 +29,4 @@
     "aggregate_all",
     "aggregate_valid",
 ]
+from gluonts.evaluation.metrics import overlay_dx  # noqa: F401

src/gluonts/evaluation/metrics.py

@@ -185,3 +185,58 @@ def num_masked_values(target) -> float:
         return np.ma.count_masked(target)
     else:
         return 0
+
+
+def overlay_dx(
+    target: np.ndarray,
+    forecast: np.ndarray,
+    max_percentage: float = 100.0,
+    min_percentage: float = 0.1,
+    step: float = 0.1,
+) -> float:
+    """Overlay-dx metric: tolerance-sweep visual alignment score.
+
+    Measures alignment between target and forecast by computing coverage
+    at varying tolerance levels and returning the normalized AUC.
+
+    Parameters
+    ----------
+    target : np.ndarray
+        Ground truth values.
+    forecast : np.ndarray
+        Predicted values.
+    max_percentage : float, default 100.0
+        Upper bound of tolerance sweep (percentage of value range).
+    min_percentage : float, default 0.1
+        Lower bound of tolerance sweep (percentage of value range).
+    step : float, default 0.1
+        Step size for tolerance sweep (percentage points).
+
+    Returns
+    -------
+    float
+        Normalized AUC score in [0, 1]. Higher is better.
+        Returns 0.0 if value_range is 0 (constant target).
+    """
+    value_range = np.max(target) - np.min(target)
+
+    if value_range == 0:
+        return 0.0
+
+    abs_errors = np.abs(target - forecast)
+    n = len(forecast)
+
+    percentages = np.arange(max_percentage, min_percentage - step, -step)
+    coverages = np.empty(len(percentages))
+
+    for i, pct in enumerate(percentages):
+        tolerance = pct / 100.0 * value_range / 2.0
+        coverages[i] = np.sum(abs_errors <= tolerance) / n
+
+    area = np.trapz(coverages, dx=step / 100.0 * value_range / 2.0)
+    max_area = value_range * (max_percentage - min_percentage) / 200.0
+
+    if max_area == 0:
+        return 0.0
+
+    return area / max_area

test/evaluation/test_metrics.py

@@ -26,6 +26,7 @@
     msis,
     quantile_loss,
     smape,
+    overlay_dx,
 )
 
 ZEROES = np.array([0.0] * 5)
@@ -202,3 +203,53 @@ def test_seasonal_error(past_data, seasonality, expected):
         ),
         expected,
     )
+
+
+class TestOverlayDx:
+    """Tests for overlay_dx metric."""
+
+    def test_perfect_forecast(self):
+        target = np.array([1.0, 2.0, 3.0, 4.0, 5.0])
+        forecast = np.array([1.0, 2.0, 3.0, 4.0, 5.0])
+        result = overlay_dx(target, forecast)
+        np.testing.assert_almost_equal(result, 1.0)
+
+    def test_constant_target_returns_zero(self):
+        target = np.array([5.0, 5.0, 5.0, 5.0])
+        forecast = np.array([4.0, 5.0, 6.0, 7.0])
+        result = overlay_dx(target, forecast)
+        np.testing.assert_almost_equal(result, 0.0)
+
+    def test_score_in_unit_interval(self):
+        rng = np.random.default_rng(42)
+        target = rng.standard_normal(100)
+        forecast = target + rng.standard_normal(100) * 0.5
+        result = overlay_dx(target, forecast)
+        assert 0.0 <= result <= 1.0
+
+    def test_better_forecast_scores_higher(self):
+        target = np.array([1.0, 2.0, 3.0, 4.0, 5.0])
+        good_forecast = target + 0.1
+        bad_forecast = target + 2.0
+        score_good = overlay_dx(target, good_forecast)
+        score_bad = overlay_dx(target, bad_forecast)
+        assert score_good > score_bad
+
+    def test_custom_parameters(self):
+        target = np.array([0.0, 1.0, 2.0, 3.0, 4.0])
+        forecast = np.array([0.1, 1.1, 2.1, 3.1, 4.1])
+        result = overlay_dx(
+            target, forecast,
+            max_percentage=50.0,
+            min_percentage=1.0,
+            step=0.5,
+        )
+        assert 0.0 <= result <= 1.0
+
+    def test_symmetric(self):
+        target = np.array([1.0, 2.0, 3.0, 4.0, 5.0])
+        forecast_above = target + 0.5
+        forecast_below = target - 0.5
+        score_above = overlay_dx(target, forecast_above)
+        score_below = overlay_dx(target, forecast_below)
+        np.testing.assert_almost_equal(score_above, score_below)