From 0eb3fd0d2f7568d8145b0e67a6423d789fb83716 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Fri, 4 Sep 2026 16:46:45 -0400 Subject: [PATCH 1/3] fix: build the fold split toggles from the strategy A forecasting run scores no train partition, so the chart's train default asked for fold metrics that were never written. --- .../evaluation/base_evaluation_strategy.py | 11 +++- .../components/models/FoldMetricsChart.jsx | 50 +++++++++++++++---- DashAI/front/src/hooks/useStrategyKind.js | 46 +++++++++-------- .../evaluation/test_forecasting_strategies.py | 25 ++++++++++ 4 files changed, 99 insertions(+), 33 deletions(-) diff --git a/DashAI/back/evaluation/base_evaluation_strategy.py b/DashAI/back/evaluation/base_evaluation_strategy.py index 0e23ff3ef..f100bad2d 100644 --- a/DashAI/back/evaluation/base_evaluation_strategy.py +++ b/DashAI/back/evaluation/base_evaluation_strategy.py @@ -36,9 +36,16 @@ def get_metadata(cls) -> dict: ------- dict Mapping with ``kind``, which says whether this strategy splits the - dataset once or into folds. + dataset once or into folds, and ``scored_splits``, the partitions + it writes metrics for. A screen that offers one control per + partition reads the latter instead of assuming all three exist: + a forecasting strategy scores no training partition, so asking it + for train metrics finds nothing. """ - return {"kind": cls.KIND} + return { + "kind": cls.KIND, + "scored_splits": [split.value for split in cls.SCORED_SPLITS], + } def __init__( self, diff --git a/DashAI/front/src/components/models/FoldMetricsChart.jsx b/DashAI/front/src/components/models/FoldMetricsChart.jsx index 695e634cb..972ae61be 100644 --- a/DashAI/front/src/components/models/FoldMetricsChart.jsx +++ b/DashAI/front/src/components/models/FoldMetricsChart.jsx @@ -26,6 +26,10 @@ import ResultsGraphsParameters from "../../pages/results/components/ResultsGraph import PillToggleButtonGroup from "../shared/PillToggleButtonGroup"; import PlotActions from "../shared/PlotActions"; import { getTraceColors } from "../../utils/chartColors"; +import { useStrategyMetadata } from "../../hooks/useStrategyKind"; +import { useModels } from "./ModelsContext"; + +const FOLD_SPLITS = ["TRAIN", "VALIDATION"]; // ─── Pure helpers (defined outside component — stable references) ───────────── @@ -175,17 +179,37 @@ export default function FoldMetricsChart({ run }) { const [error, setError] = useState(null); const [chartType, setChartType] = useState("boxplot"); const [foldScope, setFoldScope] = useState("default"); - const [split, setSplit] = useState("TRAIN"); + const [split, setSplit] = useState(null); const [selectedMetrics, setSelectedMetrics] = useState([]); const selectedMetricsRef = useRef([]); - const metricSplit = split.toLowerCase(); + const metricSplit = split?.toLowerCase() ?? null; const isNestedCV = !!run.nested; + const strategyName = + useModels()?.selectedSession?.evaluation_strategy ?? null; + const strategyMetadata = useStrategyMetadata(strategyName); + const awaitingStrategy = !!strategyName && strategyMetadata === null; + + const foldSplits = useMemo(() => { + const scored = strategyMetadata?.scored_splits; + if (!Array.isArray(scored)) return FOLD_SPLITS; + const available = FOLD_SPLITS.filter((name) => + scored.includes(name.toLowerCase()), + ); + return available.length > 0 ? available : FOLD_SPLITS; + }, [strategyMetadata]); + + useEffect(() => { + setSplit((current) => + foldSplits.includes(current) ? current : foldSplits[0], + ); + }, [foldSplits]); + // ── Fetch ────────────────────────────────────────────────────────────────── useEffect(() => { - if (!run) return; + if (!run || !split || awaitingStrategy) return; setLoading(true); setError(null); @@ -238,7 +262,7 @@ export default function FoldMetricsChart({ run }) { return () => controller.abort(); // selectedRepetition intentionally excluded — we only read it as "current value at fetch time" // eslint-disable-next-line react-hooks/exhaustive-deps - }, [run, metricSplit, foldScope, isNestedCV]); + }, [run, metricSplit, foldScope, isNestedCV, awaitingStrategy]); // ── Derived data ─────────────────────────────────────────────────────────── @@ -395,7 +419,7 @@ export default function FoldMetricsChart({ run }) { ); } - if (loading) { + if (loading || awaitingStrategy || !split) { return ( @@ -456,14 +480,18 @@ export default function FoldMetricsChart({ run }) { if (v) setSplit(v); }} > - - {t("models:label.train")} - + {foldSplits.includes("TRAIN") && ( + + {t("models:label.train")} + + )} {/* A fold is scored on its validation partition; the reserved rows produce a single value, with nothing to chart per fold. */} - - {t("models:label.validation")} - + {foldSplits.includes("VALIDATION") && ( + + {t("models:label.validation")} + + )} diff --git a/DashAI/front/src/hooks/useStrategyKind.js b/DashAI/front/src/hooks/useStrategyKind.js index 919514df9..f3236c401 100644 --- a/DashAI/front/src/hooks/useStrategyKind.js +++ b/DashAI/front/src/hooks/useStrategyKind.js @@ -2,59 +2,65 @@ import { useEffect, useState } from "react"; import { getComponents } from "../api/component"; /** - * How each evaluation strategy divides the dataset, once fetched. + * What each evaluation strategy declares about itself, once fetched. * * Strategy names never change for a given session, so the answer is cached * across components: several screens ask the same question about the same * session and none of them should trigger its own request. */ -const kindCache = new Map(); +const metadataCache = new Map(); /** - * Read the split shape of the strategy a session uses. + * Read the metadata an evaluation strategy declares. * - * Screens used to ask this by comparing the stored strategy name against a - * literal, which meant every new strategy silently read as "not cross - * validation" and rendered the wrong controls. The backend reports the shape, - * so it is read rather than guessed. + * Screens used to answer these questions by comparing the stored strategy + * name against a literal, which meant every new strategy silently read as + * "not cross validation" and rendered the wrong controls. The backend reports + * what it does, so it is read rather than guessed. * * @param {string|null} strategyName a session's evaluation_strategy - * @returns {string|null} "holdout", "cv", or null while unknown + * @returns {object|null} the strategy metadata, or null while it is unknown */ -export function useStrategyKind(strategyName) { - const [kind, setKind] = useState(() => kindCache.get(strategyName) ?? null); +export function useStrategyMetadata(strategyName) { + const [metadata, setMetadata] = useState( + () => metadataCache.get(strategyName) ?? null, + ); useEffect(() => { if (!strategyName) { - setKind(null); + setMetadata(null); return undefined; } - if (kindCache.has(strategyName)) { - setKind(kindCache.get(strategyName)); + if (metadataCache.has(strategyName)) { + setMetadata(metadataCache.get(strategyName)); return undefined; } let cancelled = false; - const fetchKind = async () => { + const fetchMetadata = async () => { try { const component = await getComponents({ model: strategyName }); - const value = component?.metadata?.kind ?? null; - kindCache.set(strategyName, value); - if (!cancelled) setKind(value); + const value = component?.metadata ?? {}; + metadataCache.set(strategyName, value); + if (!cancelled) setMetadata(value); } catch (error) { console.error(`Error fetching the ${strategyName} metadata`, error); - if (!cancelled) setKind(null); + if (!cancelled) setMetadata({}); } }; - fetchKind(); + fetchMetadata(); return () => { cancelled = true; }; }, [strategyName]); - return kind; + return metadata; +} + +export function useStrategyKind(strategyName) { + return useStrategyMetadata(strategyName)?.kind ?? null; } export default useStrategyKind; diff --git a/tests/back/evaluation/test_forecasting_strategies.py b/tests/back/evaluation/test_forecasting_strategies.py index 44a887a55..a2f8b3c17 100644 --- a/tests/back/evaluation/test_forecasting_strategies.py +++ b/tests/back/evaluation/test_forecasting_strategies.py @@ -82,6 +82,31 @@ def test_each_strategy_declares_the_shape_of_its_splits(): # --- which partitions get scored --------------------------------------------- +def test_each_strategy_declares_which_partitions_it_scores(): + """The fold charts build one toggle per scored partition. + + Reading it from the strategy is what stops them from asking for the train + fold metrics a forecasting run never wrote. + """ + assert ForecastingCrossValidationEvaluationStrategy.get_metadata()[ + "scored_splits" + ] == ["validation", "test"] + assert ForecastingHoldoutEvaluationStrategy.get_metadata()["scored_splits"] == [ + "validation", + "test", + ] + assert CrossValidationEvaluationStrategy.get_metadata()["scored_splits"] == [ + "train", + "validation", + "test", + ] + assert HoldoutEvaluationStrategy.get_metadata()["scored_splits"] == [ + "train", + "validation", + "test", + ] + + @pytest.mark.parametrize("strategy", FORECASTING_STRATEGIES) def test_forecasting_strategies_do_not_score_the_training_partition(strategy): assert SplitEnum.TRAIN not in strategy.SCORED_SPLITS From 93a1951c195c72e4e414859bcdb04f05d5aa959b Mon Sep 17 00:00:00 2001 From: Irozuku Date: Fri, 4 Sep 2026 16:52:03 -0400 Subject: [PATCH 2/3] fix: hide the live train toggle when nothing scores train A forecasting strategy writes no train metrics, so the toggle opened on an empty panel. --- .../components/models/LiveMetricsChart.jsx | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/DashAI/front/src/components/models/LiveMetricsChart.jsx b/DashAI/front/src/components/models/LiveMetricsChart.jsx index 51b9792de..b9a475642 100644 --- a/DashAI/front/src/components/models/LiveMetricsChart.jsx +++ b/DashAI/front/src/components/models/LiveMetricsChart.jsx @@ -14,6 +14,10 @@ import ResultsGraphsParameters from "../../pages/results/components/ResultsGraph import PillToggleButtonGroup from "../shared/PillToggleButtonGroup"; import PlotActions from "../shared/PlotActions"; import { getTraceColors } from "../../utils/chartColors"; +import { useStrategyMetadata } from "../../hooks/useStrategyKind"; +import { useModels } from "./ModelsContext"; + +const SPLITS = ["TRAIN", "VALIDATION", "TEST"]; function toFinalValue(value) { const resolved = Array.isArray(value) @@ -315,10 +319,27 @@ export function LiveMetricsChart({ run, modelSessionDetail = null }) { return (modelSessionDetail.test_metrics ?? []).length > 0; }, [modelSessionDetail]); + const strategyName = + useModels()?.selectedSession?.evaluation_strategy ?? null; + const strategyMetadata = useStrategyMetadata(strategyName); + + const availableSplits = useMemo(() => { + const scored = strategyMetadata?.scored_splits; + const offered = Array.isArray(scored) + ? SPLITS.filter((name) => scored.includes(name.toLowerCase())) + : SPLITS; + const withTest = hasTestSplit + ? offered + : offered.filter((name) => name !== "TEST"); + return withTest.length > 0 ? withTest : SPLITS; + }, [strategyMetadata, hasTestSplit]); + // Never leave the group pointing at a button that is no longer rendered. useEffect(() => { - if (!hasTestSplit && split === "TEST") setSplit("TRAIN"); - }, [hasTestSplit, split]); + setSplit((current) => + availableSplits.includes(current) ? current : availableSplits[0], + ); + }, [availableSplits]); return ( - - {t("models:label.train")} - - - {t("models:label.validation")} - - {hasTestSplit && ( + {availableSplits.includes("TRAIN") && ( + + {t("models:label.train")} + + )} + {availableSplits.includes("VALIDATION") && ( + + {t("models:label.validation")} + + )} + {availableSplits.includes("TEST") && ( {t("models:label.test")} From 17794eba53d4549d179f483965aab4e2c8db7955 Mon Sep 17 00:00:00 2001 From: Irozuku Date: Mon, 7 Sep 2026 11:36:02 -0300 Subject: [PATCH 3/3] fix: keep a nested run from crashing the comparison table The style set only defined nestedCv for a cross-validation session, but getRunType returns it for any run flagged nested. The strategy kind arrives asynchronously, so on the first render of a session with nested runs the key was missing and destructuring undefined took the whole table down. The style is now always defined and the legend does the filtering, with a fallback on the lookup so no row can crash the table over a style. --- .../models/ModelComparisonTable.jsx | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/DashAI/front/src/components/models/ModelComparisonTable.jsx b/DashAI/front/src/components/models/ModelComparisonTable.jsx index 1a173406b..150f9c8f6 100644 --- a/DashAI/front/src/components/models/ModelComparisonTable.jsx +++ b/DashAI/front/src/components/models/ModelComparisonTable.jsx @@ -115,22 +115,20 @@ function ModelComparisonTable({ color: theme.palette.accent.teal, label: "HPO", }, - ...(isCrossValidation - ? { - nestedCv: { - bg: "#585370", - border: "#585370", - color: "#585370", - label: "CV anidado", - }, - } - : {}), + nestedCv: { + bg: "#585370", + border: "#585370", + color: "#585370", + label: "CV anidado", + }, }; - const runTypeLegend = Object.entries(runTypeStyles).map(([key, value]) => ({ - key, - ...value, - })); + const runTypeLegend = Object.entries(runTypeStyles) + .filter(([key]) => key !== "nestedCv" || isCrossValidation) + .map(([key, value]) => ({ + key, + ...value, + })); const getMetricColumns = () => { const metricsSet = new Set(); @@ -442,7 +440,7 @@ function ModelComparisonTable({ state: { columnOrder }, muiTableBodyRowProps: ({ row }) => { const runType = getRunType(row.original); - const { bg, border } = runTypeStyles[runType]; + const { bg, border } = runTypeStyles[runType] ?? runTypeStyles.withoutHpo; return { onClick: () => { if (onRowClick) onRowClick(row.original.id);