From 164f3fe488f8d5467e79abcc1f3b3e7acc2e1fda Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Mon, 13 Jul 2026 14:00:32 +0530 Subject: [PATCH 01/48] experiment client API + /experiments/log endpoint Add vis.experiment/log_metrics/finish_experiment client methods that POST to a new /experiments/log Tornado handler. The handler records metadata through ExperimentStore over the server's DataStore and mirrors the blob into in-memory env state so a later full-env save preserves it. Covers create/update, metric append+autocreate, and finish (finished/failed). Once an experiment is terminal, further log/metrics writes are rejected: the store raises ExperimentFinishedError and the handler maps it to 409 Conflict, so a finished run's recorded data cannot change after the fact. Validation returns 400 (bad action/params/status), 404 (finish without experiment), 409 (write to terminal). Adds end-to-end + client-shape tests. --- openapi.yaml | 148 +++++++++++++++ py/tests/test_experiment_log_handler.py | 211 ++++++++++++++++++++++ py/tests/test_experiment_store.py | 37 ++++ py/visdom/__init__.py | 62 +++++++ py/visdom/__init__.pyi | 17 ++ py/visdom/experiments/__init__.py | 2 + py/visdom/experiments/models.py | 8 + py/visdom/experiments/store.py | 19 +- py/visdom/server/app.py | 6 + py/visdom/server/handlers/web_handlers.py | 101 +++++++++++ py/visdom/utils/server_utils.py | 8 +- 11 files changed, 614 insertions(+), 5 deletions(-) create mode 100644 py/tests/test_experiment_log_handler.py diff --git a/openapi.yaml b/openapi.yaml index 36870371c..a7cb1fcca 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -52,6 +52,8 @@ tags: description: Query, close, and retrieve window data - name: Environment description: Manage environments (create, delete, fork, list, save, compare) + - name: Experiments + description: Track experiment metadata (hyper-parameters, metrics, tags) - name: Authentication description: Login and session management - name: Socket Polling @@ -382,6 +384,92 @@ paths: "500": description: Server error. Occurs if the source environment does not exist (unhandled assertion error). + /experiments/log: + post: + operationId: logExperiment + tags: [Experiments] + summary: Record experiment metadata for an environment + description: > + Attaches experiment metadata (hyper-parameters, metric observations, + and tags) to an environment, stored under the environment's + `experiment` key and persisted through the server's data store. The + `action` field selects the operation: + + + - `log` (default): create or update the experiment. Repeated calls + merge new `params`/`tags` and overwrite `name`/`description`. + + + - `metrics`: append one or more `{name: value}` observations at an + optional `step`, creating the experiment if it does not exist yet. + + + - `finish`: mark the experiment terminal (`finished` or `failed`). + + + Once an experiment is terminal, `log` and `metrics` are rejected with + `409` so a finished run's recorded data cannot change after the fact. + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + eid: + type: string + description: Target environment ID. Defaults to `main`. + action: + type: string + enum: [log, metrics, finish] + default: log + description: Operation to perform. + name: + type: string + description: Display name (action `log`). Defaults to the eid. + description: + type: string + description: Free-form description (action `log`). + params: + type: object + additionalProperties: true + description: 'Hyper-parameters as `{name: value}` (action `log`).' + tags: + type: object + additionalProperties: true + description: 'Free-form tags as `{name: value}` (action `log`).' + metrics: + type: object + additionalProperties: + type: number + description: > + Metric observations as `{name: value}` (action `metrics`). + Must be a non-empty object. + step: + type: integer + description: Optional training step for the metrics (action `metrics`). + status: + type: string + enum: [finished, failed] + default: finished + description: Terminal status (action `finish`). + responses: + "200": + description: The stored experiment as JSON. + content: + application/json: + schema: + $ref: "#/components/schemas/Experiment" + "400": + description: > + Invalid request — unknown `action`, non-object `params`/`tags`/`metrics`, + empty `metrics`, or a non-terminal `finish` status. Also returned when + authentication is required but not provided. + "404": + description: A `finish` was requested for an env that has no experiment. + "409": + description: A `log`/`metrics` write was attempted on a terminal (finished/failed) experiment. + /upload_env: post: operationId: uploadEnvironment @@ -845,6 +933,66 @@ components: schemas: + Experiment: + type: object + description: Experiment metadata attached to an environment. + properties: + env_id: + type: string + description: Environment the experiment belongs to. + name: + type: string + description: Display name. Defaults to the env_id. + description: + type: string + status: + type: string + enum: [running, finished, failed] + description: Lifecycle state. New experiments start `running`. + created_at: + type: number + description: Unix timestamp when the experiment was created. + finished_at: + type: [number, "null"] + description: Unix timestamp when finished, or `null` while running. + params: + type: array + description: Hyper-parameters, keyed by name. + items: + type: object + properties: + key: + type: string + value: {} + dtype: + type: string + enum: [bool, int, float, str] + description: Inferred type, so a stored value can be cast back. + metrics: + type: array + description: Metric observations, appended over time. + items: + type: object + properties: + key: + type: string + value: + type: number + step: + type: [integer, "null"] + timestamp: + type: number + tags: + type: array + description: Free-form key/value labels. + items: + type: object + properties: + key: + type: string + value: + type: string + UploadErrorResponse: type: object description: Error response returned by the /upload_env endpoint. diff --git a/py/tests/test_experiment_log_handler.py b/py/tests/test_experiment_log_handler.py new file mode 100644 index 000000000..dd4cc1bd0 --- /dev/null +++ b/py/tests/test_experiment_log_handler.py @@ -0,0 +1,211 @@ +"""End-to-end tests for the ``/experiments/log`` endpoint (Layer 2, PR 2). + +Drives a real :class:`~visdom.server.app.Application` over a temp env dir with +Tornado's ``AsyncHTTPTestCase``, so the full route -> handler -> ``ExperimentStore`` +-> ``JSONStore`` path is exercised. Also unit-tests the client-side +``Visdom.experiment``/``log_metrics``/``finish_experiment`` message shapes with +``send=False`` (no server needed). +""" + +import json +import tempfile +import unittest + +import tornado.testing + +from visdom import Visdom +from visdom.data_model import JSONStore +from visdom.experiments import ExperimentStore +from visdom.server.app import Application + + +class TestExperimentLogEndpoint(tornado.testing.AsyncHTTPTestCase): + def setUp(self): + self._tmp_dir = tempfile.mkdtemp(prefix="visdom_exp_test_") + super().setUp() + + def get_app(self): + return Application(port=self.get_http_port(), env_path=self._tmp_dir) + + def post_json(self, path, body): + return self.fetch( + path, + method="POST", + body=json.dumps(body), + headers={"Content-Type": "application/json"}, + ) + + def read_experiment(self, eid): + """Read the persisted experiment straight from disk via a fresh store.""" + return ExperimentStore(JSONStore(self._tmp_dir)).get_experiment(eid) + + def test_log_creates_and_persists_experiment(self): + resp = self.post_json( + "/experiments/log", + { + "eid": "main", + "action": "log", + "name": "run-1", + "params": {"lr": 0.01, "epochs": 10}, + "tags": {"dataset": "mnist"}, + "description": "first run", + }, + ) + self.assertEqual(resp.code, 200) + body = json.loads(resp.body) + self.assertEqual(body["name"], "run-1") + self.assertEqual(body["params"][0]["key"], "lr") + + exp = self.read_experiment("main") + self.assertIsNotNone(exp) + self.assertEqual(exp.get_param("epochs").value, 10) + self.assertEqual(exp.get_param("epochs").dtype, "int") + self.assertEqual(exp.tags[0].value, "mnist") + + def test_action_defaults_to_log(self): + resp = self.post_json( + "/experiments/log", {"eid": "main", "params": {"lr": 0.5}} + ) + self.assertEqual(resp.code, 200) + self.assertEqual(self.read_experiment("main").get_param("lr").value, 0.5) + + def test_metrics_append_and_autocreate(self): + resp = self.post_json( + "/experiments/log", + { + "eid": "main", + "action": "metrics", + "metrics": {"acc": 0.9, "loss": 0.1}, + "step": 3, + }, + ) + self.assertEqual(resp.code, 200) + exp = self.read_experiment("main") + self.assertEqual(len(exp.metrics), 2) + self.assertEqual(exp.latest_metric("acc").value, 0.9) + self.assertEqual(exp.latest_metric("acc").step, 3) + + def test_finish_sets_terminal_status(self): + self.post_json("/experiments/log", {"eid": "main", "params": {"lr": 0.01}}) + resp = self.post_json( + "/experiments/log", + {"eid": "main", "action": "finish", "status": "failed"}, + ) + self.assertEqual(resp.code, 200) + self.assertEqual(self.read_experiment("main").status, "failed") + + def test_finish_without_experiment_is_404(self): + resp = self.post_json("/experiments/log", {"eid": "ghost", "action": "finish"}) + self.assertEqual(resp.code, 404) + + def test_finish_with_running_status_is_400(self): + self.post_json("/experiments/log", {"eid": "main", "params": {"lr": 0.01}}) + resp = self.post_json( + "/experiments/log", + {"eid": "main", "action": "finish", "status": "running"}, + ) + self.assertEqual(resp.code, 400) + + def test_log_to_finished_is_409(self): + self.post_json("/experiments/log", {"eid": "main", "params": {"lr": 0.01}}) + self.post_json("/experiments/log", {"eid": "main", "action": "finish"}) + resp = self.post_json( + "/experiments/log", {"eid": "main", "params": {"lr": 0.02}} + ) + self.assertEqual(resp.code, 409) + self.assertEqual(self.read_experiment("main").get_param("lr").value, 0.01) + + def test_metrics_to_finished_is_409(self): + self.post_json( + "/experiments/log", + {"eid": "main", "action": "metrics", "metrics": {"acc": 0.9}}, + ) + self.post_json("/experiments/log", {"eid": "main", "action": "finish"}) + resp = self.post_json( + "/experiments/log", + {"eid": "main", "action": "metrics", "metrics": {"acc": 0.95}}, + ) + self.assertEqual(resp.code, 409) + self.assertEqual(len(self.read_experiment("main").metrics), 1) + + def test_unknown_action_is_400(self): + resp = self.post_json("/experiments/log", {"eid": "main", "action": "bogus"}) + self.assertEqual(resp.code, 400) + + def test_empty_metrics_is_400(self): + resp = self.post_json( + "/experiments/log", {"eid": "main", "action": "metrics", "metrics": {}} + ) + self.assertEqual(resp.code, 400) + + def test_non_mapping_params_is_400(self): + resp = self.post_json("/experiments/log", {"eid": "main", "params": [1, 2, 3]}) + self.assertEqual(resp.code, 400) + + def test_experiment_survives_full_env_save(self): + """A window save must not clobber a previously logged experiment. + + This guards the in-memory/on-disk sync: logging writes the blob to disk + and mirrors it into server state, so persisting that env (which writes + the in-memory state) keeps the experiment instead of dropping it. + """ + self.post_json("/experiments/log", {"eid": "main", "params": {"lr": 0.01}}) + win_resp = self.post_json( + "/events", {"eid": "main", "data": [{"type": "text", "content": "hi"}]} + ) + self.assertEqual(win_resp.code, 200) + save_resp = self.post_json("/save", {"data": ["main"]}) + self.assertEqual(save_resp.code, 200) + + exp = self.read_experiment("main") + self.assertIsNotNone(exp, "experiment was clobbered by the env save") + self.assertEqual(exp.get_param("lr").value, 0.01) + + +class TestClientMessageShapes(unittest.TestCase): + """Client methods build the right request without needing a server. + + A ``send=False`` client short-circuits ``_send`` to return the + ``(msg, endpoint)`` it would have posted, so we can assert on it directly. + """ + + def _client(self): + return Visdom(send=False, env="expenv") + + def test_experiment_message(self): + msg, endpoint = self._client().experiment( + name="r1", params={"lr": 0.01}, tags={"ds": "mnist"}, description="d" + ) + self.assertEqual(endpoint, "experiments/log") + self.assertEqual(msg["action"], "log") + self.assertEqual(msg["eid"], "expenv") + self.assertEqual(msg["params"], {"lr": 0.01}) + self.assertEqual(msg["tags"], {"ds": "mnist"}) + + def test_experiment_env_override(self): + msg, _ = self._client().experiment(params={"lr": 0.01}, env="other") + self.assertEqual(msg["eid"], "other") + + def test_log_metrics_message(self): + msg, endpoint = self._client().log_metrics({"acc": 0.9}, step=5) + self.assertEqual(endpoint, "experiments/log") + self.assertEqual(msg["action"], "metrics") + self.assertEqual(msg["metrics"], {"acc": 0.9}) + self.assertEqual(msg["step"], 5) + + def test_finish_experiment_message(self): + msg, _ = self._client().finish_experiment(status="failed") + self.assertEqual(msg["action"], "finish") + self.assertEqual(msg["status"], "failed") + + def test_experiment_rejects_bad_params(self): + with self.assertRaises(TypeError): + self._client().experiment(params=[1, 2, 3]) + + def test_log_metrics_rejects_empty(self): + with self.assertRaises(TypeError): + self._client().log_metrics({}) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/tests/test_experiment_store.py b/py/tests/test_experiment_store.py index ef390690c..d330d9d67 100644 --- a/py/tests/test_experiment_store.py +++ b/py/tests/test_experiment_store.py @@ -11,8 +11,10 @@ import unittest from visdom.data_model import JSONStore +from visdom.utils.server_utils import LazyEnvData from visdom.experiments import ( Experiment, + ExperimentFinishedError, ExperimentStore, Metric, Param, @@ -163,6 +165,22 @@ def test_finish_missing_experiment_raises(self): with self.assertRaises(KeyError): self.store.finish_experiment("nope") + def test_log_experiment_on_finished_raises(self): + """Updating an experiment that is already terminal is rejected.""" + self.store.log_experiment("main", params={"lr": 0.1}) + self.store.finish_experiment("main") + with self.assertRaises(ExperimentFinishedError): + self.store.log_experiment("main", params={"lr": 0.2}) + self.assertEqual(self.store.get_experiment("main").get_param("lr").value, 0.1) + + def test_log_metric_on_finished_raises(self): + """Appending a metric to a terminal experiment is rejected.""" + self.store.log_metric("main", "acc", 0.9) + self.store.finish_experiment("main", STATUS_FAILED) + with self.assertRaises(ExperimentFinishedError): + self.store.log_metric("main", "acc", 0.95) + self.assertEqual(len(self.store.get_experiment("main").metrics), 1) + def test_list_experiments(self): """list_experiments returns only envs that actually have a blob.""" self.store.log_experiment("a") @@ -198,6 +216,25 @@ def test_experiment_coexists_with_window_data(self): self.assertEqual(env["reload"], {"foo": 1}) self.assertEqual(env["experiment"]["params"][0]["value"], 0.01) + def test_experiment_survives_lazy_env_reload_and_save(self): + """A prior-session experiment is not clobbered by a later full-env save. + + Mirrors the server's cross-restart path: an env logged in one session is + reloaded as a LazyEnvData, materialised by an unrelated window write, and + persisted again by the shutdown save_all. The experiment blob must ride + through rather than being stripped to jsons/reload. + """ + self.store.log_experiment("main", params={"lr": 0.01}) + + state = {"main": LazyEnvData(self.backend, "main")} + state["main"]["jsons"]["win_1"] = {"id": "win_1"} + self.backend.save_all(state) + + reopened = ExperimentStore(JSONStore(self.env_path)) + exp = reopened.get_experiment("main") + self.assertIsNotNone(exp, "experiment was clobbered by the full-env save") + self.assertEqual(exp.get_param("lr").value, 0.01) + class TestExperimentStoreNoPersistence(unittest.TestCase): """With persistence disabled the store degrades gracefully (no crashes).""" diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index 795b9d20c..1bc3ec3ff 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -1137,6 +1137,68 @@ def fork_env(self, prev_eid, eid): return self._send(msg={"prev_eid": prev_eid, "eid": eid}, endpoint="fork_env") + def _experiment_send(self, msg, env): + """POST an experiment action to the server and decode the JSON reply. + + Shared plumbing for :meth:`experiment`, :meth:`log_metrics` and + :meth:`finish_experiment`. Returns the stored experiment as a dict when + the server replies with JSON, otherwise the raw response (e.g. an error + string, or the `(msg, endpoint)` tuple when this client has `send=False`). + """ + msg["eid"] = env if env is not None else self.env + response = self._send(msg, endpoint="experiments/log", quiet=True) + if not isstr(response): + return response + try: + return json.loads(response) + except ValueError: + return response + + def experiment(self, name=None, params=None, tags=None, description=None, env=None): + """Create or update the experiment metadata for an environment. + + Records the hyper-parameters (`params`), free-form `tags` (both dicts of + `{name: value}`), a display `name`, and a `description` against `env` + (defaults to this client's env). Calling it again for the same env merges + in new params/tags and overwrites name/description, so it is safe to call + at the start of and again during a run. Returns the stored experiment as + a dict. + """ + if params is not None and not isinstance(params, dict): + raise TypeError("params must be a dict of {name: value}") + if tags is not None and not isinstance(tags, dict): + raise TypeError("tags must be a dict of {name: value}") + return self._experiment_send( + { + "action": "log", + "name": name, + "params": params, + "tags": tags, + "description": description, + }, + env, + ) + + def log_metrics(self, metrics, step=None, env=None): + """Append one or more metric observations to an env's experiment. + + `metrics` is a dict of `{name: value}` recorded at an optional training + `step`; the experiment is created automatically if it does not exist yet. + Returns the updated experiment as a dict. + """ + if not isinstance(metrics, dict) or not metrics: + raise TypeError("metrics must be a non-empty dict of {name: value}") + return self._experiment_send( + {"action": "metrics", "metrics": metrics, "step": step}, env + ) + + def finish_experiment(self, status="finished", env=None): + """Mark an env's experiment terminal (`"finished"` or `"failed"`). + + Returns the stored experiment as a dict. + """ + return self._experiment_send({"action": "finish", "status": status}, env) + def get_window_data(self, win=None, env=None): """ This function returns all the window data for a specified window in diff --git a/py/visdom/__init__.pyi b/py/visdom/__init__.pyi index 8a1d2a384..37635d44d 100644 --- a/py/visdom/__init__.pyi +++ b/py/visdom/__init__.pyi @@ -46,6 +46,23 @@ class Visdom: ) -> _SendReturn: ... def save(self, envs: List[Text]) -> _SendReturn: ... def close(self, win: _OptStr = ..., env: _OptStr = ...) -> _SendReturn: ... + def experiment( + self, + name: _OptStr = ..., + params: _OptOps = ..., + tags: _OptOps = ..., + description: _OptStr = ..., + env: _OptStr = ..., + ) -> Mapping[Text, Any]: ... + def log_metrics( + self, + metrics: Mapping[Text, Any], + step: Optional[int] = ..., + env: _OptStr = ..., + ) -> Mapping[Text, Any]: ... + def finish_experiment( + self, status: Text = ..., env: _OptStr = ... + ) -> Mapping[Text, Any]: ... def get_window_data( self, win: _OptStr = ..., env: _OptStr = ... ) -> _SendReturn: ... diff --git a/py/visdom/experiments/__init__.py b/py/visdom/experiments/__init__.py index f2593b7fb..3fd9e0d16 100644 --- a/py/visdom/experiments/__init__.py +++ b/py/visdom/experiments/__init__.py @@ -8,6 +8,7 @@ from visdom.experiments.models import ( Experiment, + ExperimentFinishedError, Metric, Param, Tag, @@ -20,6 +21,7 @@ __all__ = [ "Experiment", + "ExperimentFinishedError", "ExperimentStore", "Metric", "Param", diff --git a/py/visdom/experiments/models.py b/py/visdom/experiments/models.py index 11ed00ad0..f92f1e505 100644 --- a/py/visdom/experiments/models.py +++ b/py/visdom/experiments/models.py @@ -32,6 +32,10 @@ VALID_STATUSES = (STATUS_RUNNING, STATUS_FINISHED, STATUS_FAILED) +class ExperimentFinishedError(Exception): + """Raised when logging to an experiment already in a terminal state.""" + + def infer_dtype(value: Any) -> str: """Return a stable dtype tag (``bool``/``int``/``float``/``str``) for ``value``. @@ -184,6 +188,10 @@ def set_tag(self, key: str, value: str) -> Tag: self.tags.append(tag) return tag + def is_terminal(self) -> bool: + """True if the experiment is in a terminal state (finished/failed).""" + return self.status in (STATUS_FINISHED, STATUS_FAILED) + def finish(self, status: str = STATUS_FINISHED) -> None: """Mark the experiment terminal and stamp ``finished_at``. diff --git a/py/visdom/experiments/store.py b/py/visdom/experiments/store.py index 640351229..14ee29a2f 100644 --- a/py/visdom/experiments/store.py +++ b/py/visdom/experiments/store.py @@ -17,7 +17,11 @@ """ from visdom.data_model.base import DataStore -from visdom.experiments.models import Experiment, STATUS_FINISHED +from visdom.experiments.models import ( + Experiment, + ExperimentFinishedError, + STATUS_FINISHED, +) METADATA_KEY = "experiment" @@ -55,6 +59,16 @@ def _write(self, env_id, env, experiment): self.datastore.save_env(env_id, env) return experiment + @staticmethod + def _reject_if_terminal(env_id, experiment): + """Raise if ``experiment`` is finished/failed and so must not be logged to.""" + if experiment.is_terminal(): + raise ExperimentFinishedError( + "experiment {0!r} is {1}; cannot log to a terminal experiment".format( + env_id, experiment.status + ) + ) + def log_experiment( self, env_id, name=None, params=None, tags=None, description=None ): @@ -73,6 +87,7 @@ def log_experiment( description=description or "", ) else: + self._reject_if_terminal(env_id, experiment) if name is not None: experiment.name = name if description is not None: @@ -88,6 +103,8 @@ def log_metric(self, env_id, key, value, step=None): env, experiment = self._read(env_id) if experiment is None: experiment = Experiment(env_id=env_id, name=env_id) + else: + self._reject_if_terminal(env_id, experiment) experiment.add_metric(key, value, step) return self._write(env_id, env, experiment) diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index 2381ee643..26db0f56c 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -36,6 +36,7 @@ EnvStateHandler, ErrorHandler, ExistsHandler, + ExperimentLogHandler, ForkEnvHandler, HealthHandler, IndexHandler, @@ -120,6 +121,11 @@ def __init__( (r"%s/delete_env" % self.base_url, DeleteEnvHandler, {"app": self}), (r"%s/env_state" % self.base_url, EnvStateHandler, {"app": self}), (r"%s/fork_env" % self.base_url, ForkEnvHandler, {"app": self}), + ( + r"%s/experiments/log" % self.base_url, + ExperimentLogHandler, + {"app": self}, + ), (r"%s/user/(.*)" % self.base_url, UserSettingsHandler, {"app": self}), (r"%s/health" % self.base_url, HealthHandler), (r"%s(.*)" % self.base_url, IndexHandler, {"app": self}), diff --git a/py/visdom/server/handlers/web_handlers.py b/py/visdom/server/handlers/web_handlers.py index dcaa8f7c5..fd7d7d139 100644 --- a/py/visdom/server/handlers/web_handlers.py +++ b/py/visdom/server/handlers/web_handlers.py @@ -45,6 +45,11 @@ clear_deleted, ) from visdom.server.handlers.base_handlers import BaseHandler +from visdom.experiments import ( + ExperimentStore, + ExperimentFinishedError, + STATUS_FINISHED, +) logger = logging.getLogger(__name__) @@ -774,6 +779,102 @@ def post(self): ) +class ExperimentLogHandler(BaseHandler): + """POST ``/experiments/log`` — record experiment metadata for an environment. + + The JSON body carries an ``action`` selecting one of three operations: + + * ``"log"`` (default) — create or update the experiment (``name``/``params``/ + ``tags``/``description``); repeated calls merge rather than replace. + * ``"metrics"`` — append one or more ``{name: value}`` metric observations at + an optional ``step``, creating the experiment if it does not exist yet. + * ``"finish"`` — mark the experiment terminal (``status`` finished/failed). + + Once an experiment is terminal, ``"log"``/``"metrics"`` are rejected with + 409 Conflict so a finished run's recorded data cannot change after the fact. + + Metadata is persisted through the server's existing ``DataStore`` + (:class:`ExperimentStore` over ``handler.storage``) and mirrored into the + in-memory env state so a later full-env save writes it back rather than + dropping it. The stored experiment is written back to the client as JSON. + """ + + VALID_ACTIONS = ("log", "metrics", "finish") + + @staticmethod + def _require_mapping(args, field): + """Return ``args[field]`` if it is a mapping (or absent); else raise 400.""" + value = args.get(field) + if value is not None and not isinstance(value, Mapping): + raise tornado.web.HTTPError( + 400, reason="'{0}' must be an object".format(field) + ) + return value + + @staticmethod + def wrap_func(handler, args): + action = args.get("action", "log") + if action not in ExperimentLogHandler.VALID_ACTIONS: + raise tornado.web.HTTPError( + 400, reason="unknown action {0!r}".format(action) + ) + + eid = extract_eid(args) + store = ExperimentStore(handler.storage) + + if action == "metrics": + metrics = ExperimentLogHandler._require_mapping(args, "metrics") + if not metrics: + raise tornado.web.HTTPError( + 400, reason="'metrics' must be a non-empty object" + ) + elif action == "log": + params = ExperimentLogHandler._require_mapping(args, "params") + tags = ExperimentLogHandler._require_mapping(args, "tags") + + try: + if action == "log": + experiment = store.log_experiment( + eid, + name=args.get("name"), + params=params, + tags=tags, + description=args.get("description"), + ) + elif action == "metrics": + step = args.get("step") + for key, value in metrics.items(): + experiment = store.log_metric(eid, key, value, step) + else: + experiment = store.finish_experiment( + eid, args.get("status", STATUS_FINISHED) + ) + except ExperimentFinishedError as e: + raise tornado.web.HTTPError(409, reason=str(e)) + except KeyError: + raise tornado.web.HTTPError( + 404, reason="no experiment logged for env {0!r}".format(eid) + ) + except ValueError as e: + raise tornado.web.HTTPError(400, reason=str(e)) + + is_new_env = eid not in handler.state + if is_new_env: + handler.state[eid] = {"jsons": {}, "reload": {}} + handler.state[eid]["experiment"] = experiment.to_dict() + if is_new_env: + broadcast_envs(handler) + + handler.write(json.dumps(experiment.to_dict(), cls=NanSafeEncoder)) + + @check_auth + def post(self): + args = tornado.escape.json_decode( + tornado.escape.to_basestring(self.request.body) + ) + self.wrap_func(self, args) + + class HealthHandler(BaseHandler): def get(self): self.write({"status": "ok"}) diff --git a/py/visdom/utils/server_utils.py b/py/visdom/utils/server_utils.py index 8b3d9d074..97b34b014 100644 --- a/py/visdom/utils/server_utils.py +++ b/py/visdom/utils/server_utils.py @@ -96,10 +96,10 @@ def lazy_load_data(self): try: env_data = self._store.load_env(self._eid) - self._raw_dict = { - "jsons": env_data["jsons"], - "reload": env_data["reload"], - } + raw = dict(env_data) + raw["jsons"] = env_data["jsons"] + raw["reload"] = env_data["reload"] + self._raw_dict = raw except (KeyError, TypeError) as e: raise ValueError( "Failed loading environment json: {} - {}".format(self._eid, repr(e)) From fbd8c4519e199c06b48b428f84eb61afad66d9bd Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Tue, 14 Jul 2026 14:30:07 +0530 Subject: [PATCH 02/48] experiment query parser + heavy tests Add py/visdom/experiments/query.py: a small, injection-safe query language ('lr < 0.01 AND acc > 90') that tokenizes and parses into a predicate AST evaluated as a pure Python walk over a dict (no eval/exec, no SQL). Supports < <= > >= = != contains, AND/OR with correct precedence, parentheses, and dtype-aware casting (numeric/string/bool). build_record() flattens an Experiment into a queryable dict with bare and namespaced (param./metric./tag.) keys. Exported from the experiments package. Covered by py/tests/test_query.py (45 tests: tokenizing, grammar, precedence, type handling, malformed input, injection-style strings, build_record). --- py/tests/test_query.py | 314 +++++++++++++++++++++ py/visdom/experiments/__init__.py | 18 ++ py/visdom/experiments/query.py | 454 ++++++++++++++++++++++++++++++ 3 files changed, 786 insertions(+) create mode 100644 py/tests/test_query.py create mode 100644 py/visdom/experiments/query.py diff --git a/py/tests/test_query.py b/py/tests/test_query.py new file mode 100644 index 000000000..1f881ed1d --- /dev/null +++ b/py/tests/test_query.py @@ -0,0 +1,314 @@ +"""Unit tests for the experiments query parser (Layer 2, PR 3). + +The query language turns a human string such as ``lr < 0.01 AND acc > 90`` +into a predicate tree that is evaluated against a plain ``dict``. These tests +cover tokenising, the full grammar (comparisons, AND/OR precedence, nested +parentheses), type-aware casting (numeric vs string vs bool), the ``contains`` +operator, malformed input, injection-style strings (which must be inert), and +the :func:`build_record` bridge from a real :class:`Experiment`. +""" + +import unittest + +from visdom.experiments import ( + Experiment, + Query, + QueryParseError, + build_record, + parse_query, +) +from visdom.experiments.query import And, Comparison, Or, tokenize + + +def match(text, record): + """Parse ``text`` and evaluate it against ``record`` in one step.""" + return parse_query(text).matches(record) + + +class TestTokenizer(unittest.TestCase): + """The tokeniser classifies numbers, strings, ops and keywords.""" + + def test_numbers_int_vs_float(self): + self.assertEqual(tokenize("3")[0].value, 3) + self.assertIsInstance(tokenize("3")[0].value, int) + self.assertEqual(tokenize("3.5")[0].value, 3.5) + self.assertIsInstance(tokenize("3.5")[0].value, float) + self.assertEqual(tokenize("1e3")[0].value, 1000.0) + self.assertIsInstance(tokenize("1e3")[0].value, float) + + def test_negative_number(self): + toks = tokenize("acc > -0.5") + self.assertEqual([t.kind for t in toks], ["IDENT", "OP", "NUMBER"]) + self.assertEqual(toks[2].value, -0.5) + + def test_quoted_strings_and_escapes(self): + self.assertEqual(tokenize('"hello world"')[0].value, "hello world") + self.assertEqual(tokenize("'single'")[0].value, "single") + self.assertEqual(tokenize(r'"a\"b"')[0].value, 'a"b') + self.assertEqual(tokenize(r'"line\nbreak"')[0].value, "line\nbreak") + + def test_operator_normalisation(self): + self.assertEqual(tokenize("==")[0].value, "=") + self.assertEqual(tokenize("!=")[0].value, "!=") + self.assertEqual(tokenize("<=")[0].value, "<=") + + def test_keywords_are_case_insensitive(self): + kinds = [t.kind for t in tokenize("a = 1 and b = 2 OR c = 3")] + self.assertIn("AND", kinds) + self.assertIn("OR", kinds) + contains = tokenize("name Contains x")[1] + self.assertEqual((contains.kind, contains.value), ("OP", "contains")) + + def test_unexpected_character_raises(self): + with self.assertRaises(QueryParseError): + tokenize("a = 1 @ b") + + +class TestComparisons(unittest.TestCase): + """Individual comparison operators against numeric and string fields.""" + + def setUp(self): + self.record = {"lr": 0.01, "acc": 92.5, "name": "resnet50", "epochs": 10} + + def test_less_than(self): + self.assertTrue(match("lr < 0.05", self.record)) + self.assertFalse(match("lr < 0.001", self.record)) + + def test_all_ordering_operators(self): + self.assertTrue(match("acc > 90", self.record)) + self.assertTrue(match("acc >= 92.5", self.record)) + self.assertTrue(match("acc <= 92.5", self.record)) + self.assertFalse(match("acc < 92.5", self.record)) + self.assertTrue(match("epochs = 10", self.record)) + self.assertTrue(match("epochs != 5", self.record)) + + def test_equality_is_type_aware(self): + self.assertTrue(match("epochs = 10", {"epochs": "10"})) + self.assertTrue(match("lr = 0.010", {"lr": 0.01})) + self.assertFalse(match("lr = 0.02", {"lr": 0.01})) + + def test_string_equality_exact(self): + self.assertTrue(match("name = resnet50", self.record)) + self.assertFalse(match("name = resnet", self.record)) + + def test_string_ordering_is_lexicographic(self): + self.assertTrue(match("name > abc", {"name": "xyz"})) + self.assertFalse(match("name < abc", {"name": "xyz"})) + + def test_missing_field_never_matches(self): + self.assertFalse(match("missing = 1", self.record)) + self.assertFalse(match("missing != 1", self.record)) + self.assertFalse(match("missing contains x", self.record)) + + def test_non_numeric_field_vs_numeric_literal(self): + self.assertFalse(match("name > 5", {"name": "abc"})) + + +class TestBooleans(unittest.TestCase): + """Unquoted true/false parse as booleans and cast type-aware.""" + + def test_bool_literals(self): + self.assertTrue(match("amp = true", {"amp": True})) + self.assertTrue(match("amp = false", {"amp": False})) + self.assertFalse(match("amp = true", {"amp": False})) + + def test_bool_from_string_field(self): + self.assertTrue(match("amp = true", {"amp": "true"})) + self.assertTrue(match("amp = false", {"amp": "False"})) + + def test_number_literal_does_not_match_bool(self): + self.assertFalse(match("amp = 1", {"amp": True})) + + +class TestContains(unittest.TestCase): + """The contains operator does substring / membership matching.""" + + def test_substring(self): + self.assertTrue(match("name contains res", {"name": "resnet"})) + self.assertFalse(match("name contains xyz", {"name": "resnet"})) + + def test_quoted_substring_with_space(self): + self.assertTrue(match('desc contains "big model"', {"desc": "a big model"})) + + def test_list_membership(self): + self.assertTrue(match("tags contains vision", {"tags": ["vision", "nlp"]})) + self.assertFalse(match("tags contains audio", {"tags": ["vision", "nlp"]})) + + def test_numeric_substring(self): + self.assertTrue(match("name contains 50", {"name": "resnet50"})) + + +class TestLogicAndPrecedence(unittest.TestCase): + """AND binds tighter than OR; parentheses override precedence.""" + + def setUp(self): + self.record = {"lr": 0.01, "acc": 92.5, "status": "finished"} + + def test_and(self): + self.assertTrue(match("lr < 0.05 AND acc > 90", self.record)) + self.assertFalse(match("lr < 0.05 AND acc > 99", self.record)) + + def test_or(self): + self.assertTrue(match("lr > 1 OR acc > 90", self.record)) + self.assertFalse(match("lr > 1 OR acc > 99", self.record)) + + def test_and_binds_tighter_than_or(self): + record = {"a": 0, "b": 1, "c": 1} + node = parse_query("a = 1 AND b = 1 OR c = 1") + self.assertIsInstance(node, Or) + self.assertIsInstance(node.children[0], And) + self.assertTrue(node.matches(record)) + self.assertFalse(node.matches({"a": 0, "b": 1, "c": 0})) + + def test_parentheses_override(self): + record = {"a": 0, "b": 1, "c": 1} + self.assertFalse(match("a = 1 AND (b = 1 OR c = 1)", record)) + self.assertTrue(match("(a = 1 OR b = 1) AND c = 1", record)) + + def test_nested_parentheses(self): + record = {"a": 1, "b": 0, "c": 1, "d": 1} + self.assertTrue(match("a = 1 AND ((b = 1 OR c = 1) AND d = 1)", record)) + + def test_chained_and(self): + node = parse_query("a = 1 AND b = 1 AND c = 1") + self.assertIsInstance(node, And) + self.assertEqual(len(node.children), 3) + self.assertTrue(node.matches({"a": 1, "b": 1, "c": 1})) + self.assertFalse(node.matches({"a": 1, "b": 1, "c": 0})) + + +class TestSingleComparisonAst(unittest.TestCase): + """A lone comparison is not needlessly wrapped in And/Or.""" + + def test_lone_comparison_shape(self): + node = parse_query("lr < 0.01") + self.assertIsInstance(node, Comparison) + self.assertEqual((node.key, node.op, node.value), ("lr", "<", 0.01)) + + def test_dotted_key_preserved(self): + node = parse_query("tag.owner = alice") + self.assertEqual(node.key, "tag.owner") + + +class TestMalformedInput(unittest.TestCase): + """Malformed queries raise QueryParseError, never crash or silently pass.""" + + def test_empty_query(self): + for text in ("", " "): + with self.assertRaises(QueryParseError): + parse_query(text) + + def test_missing_operator(self): + with self.assertRaises(QueryParseError): + parse_query("lr 0.01") + + def test_missing_value(self): + with self.assertRaises(QueryParseError): + parse_query("lr <") + + def test_missing_key(self): + with self.assertRaises(QueryParseError): + parse_query("< 0.01") + + def test_trailing_tokens(self): + with self.assertRaises(QueryParseError): + parse_query("lr < 0.01 0.02") + + def test_dangling_boolean_operator(self): + with self.assertRaises(QueryParseError): + parse_query("lr < 0.01 AND") + + def test_unbalanced_parentheses(self): + with self.assertRaises(QueryParseError): + parse_query("(lr < 0.01") + with self.assertRaises(QueryParseError): + parse_query("lr < 0.01)") + + def test_empty_parentheses(self): + with self.assertRaises(QueryParseError): + parse_query("()") + + def test_reserved_word_as_bare_value(self): + with self.assertRaises(QueryParseError): + parse_query("owner = and") + + +class TestInjectionSafety(unittest.TestCase): + """Injection-style payloads are inert: they parse to string literals or fail.""" + + def test_sql_injection_is_a_string_literal(self): + record = {"name": "resnet"} + self.assertFalse(match("name = '1; DROP TABLE experiments'", record)) + self.assertTrue( + match( + "name = '1; DROP TABLE experiments'", + {"name": "1; DROP TABLE experiments"}, + ) + ) + + def test_python_expression_is_not_evaluated(self): + self.assertTrue( + match("cmd contains import", {"cmd": "__import__('os').system('x')"}) + ) + self.assertFalse(match("x = 2", {"x": "1+1"})) + + def test_unquoted_injection_fails_to_parse(self): + with self.assertRaises(QueryParseError): + parse_query("name = 1); DROP TABLE experiments; --") + + +class TestQueryWrapper(unittest.TestCase): + """The Query convenience wrapper compiles once and filters many records.""" + + def test_matches_and_filter(self): + query = Query("acc > 90") + records = [{"acc": 95}, {"acc": 80}, {"acc": 91}] + self.assertEqual(list(query.filter(records)), [{"acc": 95}, {"acc": 91}]) + self.assertTrue(query.matches({"acc": 95})) + + def test_wrapper_reports_parse_error_eagerly(self): + with self.assertRaises(QueryParseError): + Query("broken <") + + +class TestBuildRecord(unittest.TestCase): + """build_record flattens a real Experiment into a queryable dict.""" + + def _experiment(self): + exp = Experiment(env_id="run1", name="resnet-run", description="baseline") + exp.set_param("lr", 0.01) + exp.set_param("epochs", 30) + exp.add_metric("acc", 88.0, step=1) + exp.add_metric("acc", 92.5, step=2) + exp.set_tag("owner", "alice") + return exp + + def test_builtins_params_metrics_tags(self): + record = build_record(self._experiment()) + self.assertEqual(record["status"], "running") + self.assertEqual(record["name"], "resnet-run") + self.assertEqual(record["lr"], 0.01) + self.assertEqual(record["param.lr"], 0.01) + self.assertEqual(record["acc"], 92.5) + self.assertEqual(record["metric.acc"], 92.5) + self.assertEqual(record["tag.owner"], "alice") + self.assertEqual(record["owner"], "alice") + + def test_query_against_built_record(self): + record = build_record(self._experiment()) + self.assertTrue(Query("lr < 0.05 AND acc > 90").matches(record)) + self.assertTrue(Query("status = running").matches(record)) + self.assertTrue(Query("name contains resnet").matches(record)) + self.assertTrue(Query("tag.owner = alice").matches(record)) + self.assertFalse(Query("epochs > 100").matches(record)) + + def test_builtin_takes_precedence_over_bare_param(self): + exp = Experiment(env_id="run2") + exp.set_param("status", "custom") + record = build_record(exp) + self.assertEqual(record["status"], "running") + self.assertEqual(record["param.status"], "custom") + + +if __name__ == "__main__": + unittest.main() diff --git a/py/visdom/experiments/__init__.py b/py/visdom/experiments/__init__.py index 3fd9e0d16..c95cd3f84 100644 --- a/py/visdom/experiments/__init__.py +++ b/py/visdom/experiments/__init__.py @@ -17,15 +17,33 @@ STATUS_RUNNING, VALID_STATUSES, ) +from visdom.experiments.query import ( + And, + Comparison, + Node, + Or, + Query, + QueryParseError, + build_record, + parse_query, +) from visdom.experiments.store import ExperimentStore __all__ = [ + "And", + "Comparison", "Experiment", "ExperimentFinishedError", "ExperimentStore", "Metric", + "Node", + "Or", "Param", + "Query", + "QueryParseError", "Tag", + "build_record", + "parse_query", "STATUS_FAILED", "STATUS_FINISHED", "STATUS_RUNNING", diff --git a/py/visdom/experiments/query.py b/py/visdom/experiments/query.py new file mode 100644 index 000000000..e6f487b35 --- /dev/null +++ b/py/visdom/experiments/query.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, The Visdom Authors +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""A small, injection-safe query language for filtering experiments. + +The search layer (added in a later PR) lets a user filter experiments with a +human-readable expression such as:: + + lr < 0.01 AND acc > 90 + status = finished AND (name contains resnet OR tag.owner = alice) + +This module turns such a string into a **predicate tree** and evaluates it +against a plain ``dict`` describing one experiment. Because evaluation is a +pure Python walk over the tree — never ``eval``/``exec`` and never SQL — a +value like ``"1; DROP TABLE experiments"`` is just an ordinary string literal +that can never do anything but fail to match; there is no injection surface. + +The grammar (lowest-to-highest precedence):: + + expr := or_expr + or_expr := and_expr ( OR and_expr )* + and_expr := term ( AND term )* + term := "(" expr ")" | comparison + comparison := IDENT OP value + OP := "<" | "<=" | ">" | ">=" | "=" | "==" | "!=" | "contains" + value := NUMBER | STRING | BOOL | BAREWORD + +``AND``/``OR``/``contains`` are case-insensitive keywords; ``true``/``false`` +(unquoted) parse as booleans. Anything else on the value side is a string — +quote it (``"..."`` or ``'...'``) if it contains spaces or a reserved word. + +The public surface is deliberately tiny: :func:`parse_query` (text → AST), +the :class:`Query` convenience wrapper, :func:`build_record` (Experiment → +queryable dict), and :class:`QueryParseError`. +""" + +from __future__ import annotations + +import re +from typing import Any, Callable, Iterable, Iterator, List, NamedTuple, Optional + +__all__ = [ + "QueryParseError", + "parse_query", + "Query", + "build_record", + "Node", + "And", + "Or", + "Comparison", +] + +_COMPARISON_OPS = ("<", "<=", ">", ">=", "=", "!=", "contains") + +_ORDER_OPS: dict[str, Callable[[Any, Any], bool]] = { + "<": lambda a, b: a < b, + "<=": lambda a, b: a <= b, + ">": lambda a, b: a > b, + ">=": lambda a, b: a >= b, +} + +_MISSING = object() + + +class QueryParseError(ValueError): + """Raised when a query string is malformed and cannot be parsed.""" + + +class Token(NamedTuple): + """A single lexical token: its kind, parsed value, and source position.""" + + kind: str + value: Any + pos: int + + +_TOKEN_RE = re.compile( + r""" + (?P\s+) + | (?P-?(?:\d+\.\d*|\.\d+|\d+)(?:[eE][+-]?\d+)?) + | (?P"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*') + | (?P<=|>=|!=|==|<|>|=) + | (?P\() + | (?P\)) + | (?P[A-Za-z_][A-Za-z0-9_.]*) + """, + re.VERBOSE, +) + +_STRING_ESCAPES = {'\\"': '"', "\\'": "'", "\\\\": "\\", "\\n": "\n", "\\t": "\t"} + + +def _unescape(raw: str) -> str: + """Strip the surrounding quotes from a STRING token and unescape it.""" + body = raw[1:-1] + return re.sub(r"\\[\"'\\nt]", lambda m: _STRING_ESCAPES[m.group(0)], body) + + +def _number(raw: str) -> Any: + """Parse a NUMBER token as ``int`` when it has no fractional/exponent part.""" + if "." in raw or "e" in raw or "E" in raw: + return float(raw) + return int(raw) + + +def tokenize(text: str) -> List[Token]: + """Split ``text`` into tokens, raising :class:`QueryParseError` on garbage. + + Keywords are recognised here so the parser sees dedicated ``AND``/``OR`` + tokens and a normalised ``contains`` operator, and never has to special-case + identifiers by spelling. + """ + tokens: List[Token] = [] + pos = 0 + length = len(text) + while pos < length: + match = _TOKEN_RE.match(text, pos) + if match is None: + raise QueryParseError( + "unexpected character {0!r} at position {1}".format(text[pos], pos) + ) + pos = match.end() + kind = match.lastgroup + raw = match.group() + if kind == "WS": + continue + if kind == "NUMBER": + tokens.append(Token("NUMBER", _number(raw), match.start())) + elif kind == "STRING": + tokens.append(Token("STRING", _unescape(raw), match.start())) + elif kind == "OP": + tokens.append(Token("OP", "=" if raw == "==" else raw, match.start())) + elif kind == "LPAREN": + tokens.append(Token("LPAREN", raw, match.start())) + elif kind == "RPAREN": + tokens.append(Token("RPAREN", raw, match.start())) + else: + upper = raw.upper() + if upper == "AND": + tokens.append(Token("AND", raw, match.start())) + elif upper == "OR": + tokens.append(Token("OR", raw, match.start())) + elif raw.lower() == "contains": + tokens.append(Token("OP", "contains", match.start())) + else: + tokens.append(Token("IDENT", raw, match.start())) + return tokens + + +class Node: + """Base class for a node in the predicate tree.""" + + def matches(self, record: dict) -> bool: + raise NotImplementedError + + +class And(Node): + """Logical AND: matches when *every* child matches.""" + + def __init__(self, children: List[Node]): + self.children = children + + def matches(self, record: dict) -> bool: + return all(child.matches(record) for child in self.children) + + def __repr__(self) -> str: + return "And({0!r})".format(self.children) + + +class Or(Node): + """Logical OR: matches when *any* child matches.""" + + def __init__(self, children: List[Node]): + self.children = children + + def matches(self, record: dict) -> bool: + return any(child.matches(record) for child in self.children) + + def __repr__(self) -> str: + return "Or({0!r})".format(self.children) + + +class Comparison(Node): + """A single ``key OP value`` leaf, evaluated against a record dict.""" + + def __init__(self, key: str, op: str, value: Any): + if op not in _COMPARISON_OPS: + raise QueryParseError("unknown operator {0!r}".format(op)) + self.key = key + self.op = op + self.value = value + + def matches(self, record: dict) -> bool: + actual = _lookup(record, self.key) + if actual is _MISSING: + return False + return _compare(actual, self.op, self.value) + + def __repr__(self) -> str: + return "Comparison({0!r}, {1!r}, {2!r})".format(self.key, self.op, self.value) + + +class _Parser: + """Turns a token stream into a :class:`Node` tree by recursive descent.""" + + def __init__(self, tokens: List[Token]): + self._tokens = tokens + self._index = 0 + + def parse(self) -> Node: + if not self._tokens: + raise QueryParseError("empty query") + node = self._parse_or() + if not self._at_end(): + token = self._peek() + raise QueryParseError( + "unexpected {0!r} at position {1}".format(token.value, token.pos) + ) + return node + + def _parse_or(self) -> Node: + children = [self._parse_and()] + while self._match("OR"): + children.append(self._parse_and()) + return children[0] if len(children) == 1 else Or(children) + + def _parse_and(self) -> Node: + children = [self._parse_term()] + while self._match("AND"): + children.append(self._parse_term()) + return children[0] if len(children) == 1 else And(children) + + def _parse_term(self) -> Node: + if self._match("LPAREN"): + node = self._parse_or() + self._expect("RPAREN") + return node + return self._parse_comparison() + + def _parse_comparison(self) -> Node: + key_token = self._expect("IDENT", what="a field name") + op_token = self._expect("OP", what="a comparison operator") + value = self._parse_value() + return Comparison(key_token.value, op_token.value, value) + + def _parse_value(self) -> Any: + token = self._peek() + if token is None: + raise QueryParseError("expected a value but reached end of query") + if token.kind in ("NUMBER", "STRING"): + self._advance() + return token.value + if token.kind == "IDENT": + self._advance() + lowered = token.value.lower() + if lowered == "true": + return True + if lowered == "false": + return False + return token.value + raise QueryParseError( + "expected a value but found {0!r} at position {1}".format( + token.value, token.pos + ) + ) + + def _peek(self) -> Optional[Token]: + if self._index < len(self._tokens): + return self._tokens[self._index] + return None + + def _advance(self) -> Token: + token = self._tokens[self._index] + self._index += 1 + return token + + def _at_end(self) -> bool: + return self._index >= len(self._tokens) + + def _match(self, kind: str) -> bool: + token = self._peek() + if token is not None and token.kind == kind: + self._advance() + return True + return False + + def _expect(self, kind: str, what: Optional[str] = None) -> Token: + token = self._peek() + if token is None: + raise QueryParseError( + "expected {0} but reached end of query".format(what or kind) + ) + if token.kind != kind: + raise QueryParseError( + "expected {0} but found {1!r} at position {2}".format( + what or kind, token.value, token.pos + ) + ) + return self._advance() + + +def parse_query(text: str) -> Node: + """Parse ``text`` into a predicate :class:`Node`. + + Raises :class:`QueryParseError` for an empty or malformed query. + """ + return _Parser(tokenize(text)).parse() + + +def _lookup(record: dict, key: str) -> Any: + """Resolve ``key`` in ``record``: exact key first, then dotted traversal.""" + if not isinstance(record, dict): + return _MISSING + if key in record: + return record[key] + cursor: Any = record + for part in key.split("."): + if isinstance(cursor, dict) and part in cursor: + cursor = cursor[part] + else: + return _MISSING + return cursor + + +def _to_number(value: Any) -> Optional[float]: + """Coerce ``value`` to a float for numeric comparison, or ``None``. + + ``bool`` is intentionally not treated as a number so that ``amp = 1`` does + not silently match a boolean ``True``; booleans compare via :func:`_to_bool`. + """ + if isinstance(value, bool): + return None + if isinstance(value, (int, float)): + return float(value) + if isinstance(value, str): + try: + return float(value) + except ValueError: + return None + return None + + +def _to_bool(value: Any) -> Optional[bool]: + """Coerce ``value`` to a bool for boolean comparison, or ``None``.""" + if isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.lower() + if lowered == "true": + return True + if lowered == "false": + return False + return None + + +def _equals(actual: Any, expected: Any) -> bool: + """Type-aware equality between a stored value and a query literal.""" + if isinstance(expected, bool): + actual_bool = _to_bool(actual) + return actual_bool is not None and actual_bool == expected + if isinstance(expected, (int, float)): + actual_number = _to_number(actual) + return actual_number is not None and actual_number == float(expected) + return str(actual) == str(expected) + + +def _order(actual: Any, op: str, expected: Any) -> bool: + """Apply an ordering operator, numerically when the literal is numeric.""" + if isinstance(expected, (int, float)) and not isinstance(expected, bool): + left = _to_number(actual) + if left is None: + return False + right: Any = float(expected) + else: + left = str(actual) + right = str(expected) + return _ORDER_OPS[op](left, right) + + +def _contains(actual: Any, expected: Any) -> bool: + """Membership for collections, substring match for everything else.""" + if isinstance(actual, (list, tuple, set)): + if expected in actual: + return True + return any(str(item) == str(expected) for item in actual) + return str(expected) in str(actual) + + +def _compare(actual: Any, op: str, expected: Any) -> bool: + """Dispatch a single comparison to the right type-aware helper.""" + if op == "contains": + return _contains(actual, expected) + if op == "=": + return _equals(actual, expected) + if op == "!=": + return not _equals(actual, expected) + return _order(actual, op, expected) + + +class Query: + """A compiled query: parse once, then match/filter many records.""" + + def __init__(self, text: str): + self.text = text + self.root = parse_query(text) + + def matches(self, record: dict) -> bool: + """Return ``True`` if ``record`` satisfies the query.""" + return self.root.matches(record) + + def filter(self, records: Iterable[dict]) -> Iterator[dict]: + """Yield the records from ``records`` that satisfy the query.""" + return (record for record in records if self.root.matches(record)) + + def __repr__(self) -> str: + return "Query({0!r})".format(self.text) + + +def build_record(experiment: Any) -> dict: + """Flatten an :class:`~visdom.experiments.models.Experiment` into a queryable dict. + + Each param, latest-metric value and tag is exposed both under its bare name + (``lr``, ``acc``, ``owner``) and under a namespaced key (``param.lr``, + ``metric.acc``, ``tag.owner``) so a query can disambiguate when names + collide. Built-in fields (``name``, ``status`` …) take precedence over a + bare param/metric/tag of the same name; the namespaced form always reaches + the specific value. This is a duck-typed helper — it never imports the model + class — so :mod:`query` stays free of storage dependencies. + """ + record: dict = { + "env_id": experiment.env_id, + "name": experiment.name, + "description": experiment.description, + "status": experiment.status, + "created_at": experiment.created_at, + "finished_at": experiment.finished_at, + } + for param in experiment.params: + record.setdefault(param.key, param.value) + record["param." + param.key] = param.value + for key in {metric.key for metric in experiment.metrics}: + latest = experiment.latest_metric(key) + if latest is not None: + record.setdefault(key, latest.value) + record["metric." + key] = latest.value + for tag in experiment.tags: + record.setdefault(tag.key, tag.value) + record["tag." + tag.key] = tag.value + return record From dd23e317a7c493342c615be0919dfcb75ee22de7 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Tue, 14 Jul 2026 19:50:55 +0530 Subject: [PATCH 03/48] type build_record with a Protocol, single-pass metrics build_record() was annotated `experiment: Any`, justified in its docstring as keeping query.py free of storage dependencies. That rationale was wrong: models.py imports only stdlib and never touches store.py, so importing it costs nothing and cannot cycle. The real cost of `Any` was an invisible, uncheckable contract. Replace it with an ExperimentLike Protocol (structural, so Experiment satisfies it without importing or inheriting anything) plus a _KeyValueLike protocol for Param/Metric/Tag. params/metrics/tags are declared read-only via @property: as plain attributes they are invariant, which rejects list[Param] against Sequence[_KeyValueLike] and would have failed the concrete Experiment. Collect metrics in one reverse pass instead of a set comprehension plus an O(keys x metrics) latest_metric() re-scan per key. Semantics are unchanged, including latest meaning last logged rather than highest step. Document the two behaviours the code relied on but never stated: bare and namespaced keys are duplicated deliberately (in-memory, per query; the persisted shape is still to_dict()), and bare-name precedence runs built-ins > params > metrics > tags. Tests 45 -> 48: bare-name precedence across a param/metric/tag collision, latest-metric ordering, and a non-Experiment fake proving the decoupling. --- py/tests/test_query.py | 54 ++++++++++++++++++++ py/visdom/experiments/__init__.py | 2 + py/visdom/experiments/query.py | 82 +++++++++++++++++++++++++------ 3 files changed, 123 insertions(+), 15 deletions(-) diff --git a/py/tests/test_query.py b/py/tests/test_query.py index 1f881ed1d..71be3bbf1 100644 --- a/py/tests/test_query.py +++ b/py/tests/test_query.py @@ -309,6 +309,60 @@ def test_builtin_takes_precedence_over_bare_param(self): self.assertEqual(record["status"], "running") self.assertEqual(record["param.status"], "custom") + def test_bare_name_precedence_is_param_then_metric_then_tag(self): + exp = Experiment(env_id="run3") + exp.set_param("score", "from-param") + exp.add_metric("score", 1.0) + exp.set_tag("score", "from-tag") + exp.add_metric("loss", 0.5) + exp.set_tag("loss", "from-tag") + record = build_record(exp) + + self.assertEqual(record["score"], "from-param") + self.assertEqual(record["loss"], 0.5) + self.assertEqual(record["param.score"], "from-param") + self.assertEqual(record["metric.score"], 1.0) + self.assertEqual(record["tag.score"], "from-tag") + self.assertEqual(record["tag.loss"], "from-tag") + + def test_latest_metric_wins_regardless_of_step_order(self): + exp = Experiment(env_id="run4") + exp.add_metric("acc", 90.0, step=5) + exp.add_metric("acc", 70.0, step=1) + self.assertEqual(build_record(exp)["acc"], 70.0) + + +class _FakeNamedValue: + def __init__(self, key, value): + self.key = key + self.value = value + + +class _FakeExperiment: + """Experiment-shaped without being an Experiment; satisfies ExperimentLike.""" + + def __init__(self): + self.env_id = "fake1" + self.name = "fake-run" + self.description = "" + self.status = "finished" + self.created_at = 0.0 + self.finished_at = 1.0 + self.params = [_FakeNamedValue("lr", 0.5)] + self.metrics = [_FakeNamedValue("acc", 99.0)] + self.tags = [_FakeNamedValue("owner", "bob")] + + +class TestBuildRecordIsDecoupledFromModels(unittest.TestCase): + """build_record reads a structural shape, not the concrete Experiment class.""" + + def test_accepts_any_experiment_shaped_object(self): + record = build_record(_FakeExperiment()) + self.assertEqual(record["lr"], 0.5) + self.assertEqual(record["metric.acc"], 99.0) + self.assertEqual(record["tag.owner"], "bob") + self.assertTrue(Query("lr < 1 AND acc > 90").matches(record)) + if __name__ == "__main__": unittest.main() diff --git a/py/visdom/experiments/__init__.py b/py/visdom/experiments/__init__.py index c95cd3f84..ed3823310 100644 --- a/py/visdom/experiments/__init__.py +++ b/py/visdom/experiments/__init__.py @@ -20,6 +20,7 @@ from visdom.experiments.query import ( And, Comparison, + ExperimentLike, Node, Or, Query, @@ -34,6 +35,7 @@ "Comparison", "Experiment", "ExperimentFinishedError", + "ExperimentLike", "ExperimentStore", "Metric", "Node", diff --git a/py/visdom/experiments/query.py b/py/visdom/experiments/query.py index e6f487b35..005cc8019 100644 --- a/py/visdom/experiments/query.py +++ b/py/visdom/experiments/query.py @@ -36,19 +36,31 @@ The public surface is deliberately tiny: :func:`parse_query` (text → AST), the :class:`Query` convenience wrapper, :func:`build_record` (Experiment → -queryable dict), and :class:`QueryParseError`. +queryable dict) with the :class:`ExperimentLike` shape it accepts, and +:class:`QueryParseError`. """ from __future__ import annotations import re -from typing import Any, Callable, Iterable, Iterator, List, NamedTuple, Optional +from typing import ( + Any, + Callable, + Iterable, + Iterator, + List, + NamedTuple, + Optional, + Protocol, + Sequence, +) __all__ = [ "QueryParseError", "parse_query", "Query", "build_record", + "ExperimentLike", "Node", "And", "Or", @@ -421,16 +433,55 @@ def __repr__(self) -> str: return "Query({0!r})".format(self.text) -def build_record(experiment: Any) -> dict: +class _KeyValueLike(Protocol): + """A named value: :class:`~visdom.experiments.models.Param`, ``Metric``, ``Tag``.""" + + key: str + value: Any + + +class ExperimentLike(Protocol): + """The shape :func:`build_record` reads. + + Structural, so :class:`~visdom.experiments.models.Experiment` satisfies it + without inheriting from or knowing about it, and :mod:`query` needs no import + of :mod:`~visdom.experiments.models`. Anything experiment-shaped — including + a test fake — is accepted. + """ + + env_id: str + name: str + description: str + status: str + created_at: float + finished_at: Optional[float] + + @property + def params(self) -> Sequence[_KeyValueLike]: + ... + + @property + def metrics(self) -> Sequence[_KeyValueLike]: + ... + + @property + def tags(self) -> Sequence[_KeyValueLike]: + ... + + +def build_record(experiment: ExperimentLike) -> dict: """Flatten an :class:`~visdom.experiments.models.Experiment` into a queryable dict. - Each param, latest-metric value and tag is exposed both under its bare name - (``lr``, ``acc``, ``owner``) and under a namespaced key (``param.lr``, - ``metric.acc``, ``tag.owner``) so a query can disambiguate when names - collide. Built-in fields (``name``, ``status`` …) take precedence over a - bare param/metric/tag of the same name; the namespaced form always reaches - the specific value. This is a duck-typed helper — it never imports the model - class — so :mod:`query` stays free of storage dependencies. + Each param, latest-metric value and tag is exposed twice on purpose: once + under its bare name (``lr``, ``acc``, ``owner``) and once under a namespaced + key (``param.lr``, ``metric.acc``, ``tag.owner``). The bare form keeps the + common query readable, the namespaced form disambiguates when names collide. + The duplication costs only this dict, which lives for the length of one + query; the persisted shape is still :meth:`Experiment.to_dict`. + + Where a bare name is claimed more than once, first writer wins: + built-ins (``name``, ``status`` …) > params > metrics > tags. The namespaced + form always reaches the specific value regardless. """ record: dict = { "env_id": experiment.env_id, @@ -443,11 +494,12 @@ class — so :mod:`query` stays free of storage dependencies. for param in experiment.params: record.setdefault(param.key, param.value) record["param." + param.key] = param.value - for key in {metric.key for metric in experiment.metrics}: - latest = experiment.latest_metric(key) - if latest is not None: - record.setdefault(key, latest.value) - record["metric." + key] = latest.value + latest_metrics: dict[str, Any] = {} + for metric in reversed(experiment.metrics): + latest_metrics.setdefault(metric.key, metric.value) + for key, value in latest_metrics.items(): + record.setdefault(key, value) + record["metric." + key] = value for tag in experiment.tags: record.setdefault(tag.key, tag.value) record["tag." + tag.key] = tag.value From 31a68389232bd8e0092f81683fe19af0f109729e Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 15 Jul 2026 14:25:37 +0530 Subject: [PATCH 04/48] experiment search endpoint + vis.search_experiments Adds the search layer on top of the L2-3 query parser, so the experiments logged by /experiments/log can actually be found again. ExperimentStore.search(query, sort_by, descending) filters experiments through the parser's predicate and sorts them (newest first by default). Records are built once per experiment and reused for both the filter and the sort, and a run missing the sort field always lands last rather than jumping to the front when the sort is reversed. Mixed-type fields sort without raising, since params are user-supplied. POST /experiments/search wires that to HTTP: it validates query/sort_by/ limit/offset/descending, pages the result, and reports the unpaged total so a caller can walk the pages. A malformed query is a 400 with the parser's reason. Queries stay evaluated in Python -- never eval'd, never SQL -- so an injection payload is a parse error, not an execution. Client side, vis.search_experiments(query, limit, offset, sort_by, descending) returns the server's reply; the JSON-decoding plumbing is factored out of _experiment_send so both endpoints share one path. Endpoint documented in openapi.yaml. Tests: py/tests/test_experiment_search.py (42) covering the store filter/sort, the endpoint e2e over a real Application, validation and paging, and the client message shape. --- openapi.yaml | 99 ++++++ py/tests/test_experiment_search.py | 366 ++++++++++++++++++++++ py/visdom/__init__.py | 61 +++- py/visdom/experiments/__init__.py | 3 +- py/visdom/experiments/store.py | 79 +++++ py/visdom/server/app.py | 6 + py/visdom/server/handlers/web_handlers.py | 113 +++++++ 7 files changed, 718 insertions(+), 9 deletions(-) create mode 100644 py/tests/test_experiment_search.py diff --git a/openapi.yaml b/openapi.yaml index a7cb1fcca..c4044c53a 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -470,6 +470,105 @@ paths: "409": description: A `log`/`metrics` write was attempted on a terminal (finished/failed) experiment. + /experiments/search: + post: + operationId: searchExperiments + tags: [Experiments] + summary: Search experiments across all environments + description: > + Returns the experiments matching `query`, sorted and paged. The query is + a small readable syntax of comparisons (`<`, `<=`, `>`, `>=`, `=`, `!=`, + `contains`) combined with `AND`/`OR` and parentheses, for example + `lr < 0.01 AND (acc > 0.9 OR status = finished)`. Omitting `query` + matches every experiment. + + + Comparison names are the experiment's built-in fields (`name`, + `status`, `created_at`, ...), its params, its metrics and its tags. A + name may be given bare (`acc`) or namespaced when it is ambiguous + (`metric.acc`, `param.lr`, `tag.owner`); metrics compare on their latest + logged value. A name no experiment has simply matches nothing. + + + Queries are parsed into a predicate and evaluated in Python — never + eval'd and never turned into SQL — so a hostile query is rejected as a + parse error rather than executed. + + + Experiments are read back through the server's data store, so a server + running with no persistence path configured has nothing to search. + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + query: + type: string + nullable: true + description: > + Filter expression. Omitted, null or blank matches every + experiment. + sort_by: + type: string + nullable: true + default: created_at + description: > + Field to sort by — any name a query can compare on. + Experiments lacking the field sort last in both directions. + descending: + type: boolean + default: true + description: Sort direction; newest/highest first by default. + limit: + type: integer + nullable: true + minimum: 0 + default: 100 + description: > + Maximum number of experiments in the reply. `0` returns just + the count; `null` returns all matches. + offset: + type: integer + minimum: 0 + default: 0 + description: Number of matches to skip before the returned page. + responses: + "200": + description: The matching page of experiments. + content: + application/json: + schema: + type: object + required: [experiments, total, limit, offset, query] + properties: + experiments: + type: array + description: One page of matching experiments. + items: + $ref: "#/components/schemas/Experiment" + total: + type: integer + description: > + Total matches for the query, ignoring `limit`/`offset`. + limit: + type: integer + nullable: true + description: The limit applied to this reply. + offset: + type: integer + description: The offset applied to this reply. + query: + type: string + description: The query used, `""` if none was given. + "400": + description: > + Invalid request — malformed `query` syntax, a non-string + `query`/`sort_by`, or a `limit`/`offset` that is not a non-negative + integer. Also returned when authentication is required but not + provided. + /upload_env: post: operationId: uploadEnvironment diff --git a/py/tests/test_experiment_search.py b/py/tests/test_experiment_search.py new file mode 100644 index 000000000..9b1beee82 --- /dev/null +++ b/py/tests/test_experiment_search.py @@ -0,0 +1,366 @@ +"""Tests for experiment search (Layer 2, PR 4). + +Covers the three pieces the search layer is built from: ``ExperimentStore.search`` +(filtering via the query parser, plus sorting) against a real ``JSONStore`` over a +temporary directory; the ``/experiments/search`` endpoint end-to-end through a real +:class:`~visdom.server.app.Application` with Tornado's ``AsyncHTTPTestCase``; and +the ``Visdom.search_experiments`` message shape with ``send=False`` (no server). +""" + +import json +import tempfile +import unittest + +import tornado.testing + +from visdom import Visdom +from visdom.data_model import JSONStore +from visdom.experiments import ExperimentStore, QueryParseError +from visdom.server.app import Application + + +def seed_experiments(store): + """Log three experiments with known params/metrics/tags and created_at order. + + ``created_at`` is stamped explicitly rather than left to wall-clock time: + the runs are logged microseconds apart, so the default newest-first sort + would otherwise be testing the resolution of ``time.time()``. + """ + store.log_experiment( + "run-a", + name="alpha", + params={"lr": 0.1, "epochs": 10}, + tags={"dataset": "mnist"}, + ) + store.log_metric("run-a", "acc", 0.80) + + store.log_experiment( + "run-b", + name="beta", + params={"lr": 0.001, "epochs": 20}, + tags={"dataset": "cifar10"}, + ) + store.log_metric("run-b", "acc", 0.55) + store.log_metric("run-b", "acc", 0.95) # latest wins when querying "acc" + store.finish_experiment("run-b") + + store.log_experiment("run-c", name="gamma", params={"lr": 0.5}) + + for env_id, created_at in (("run-a", 100.0), ("run-b", 200.0), ("run-c", 300.0)): + env, experiment = store._read(env_id) + experiment.created_at = created_at + store._write(env_id, env, experiment) + + +def env_ids(experiments): + return [experiment.env_id for experiment in experiments] + + +class TestStoreSearch(unittest.TestCase): + """ExperimentStore.search filters by query and sorts the results.""" + + def setUp(self): + self._tmp_dir = tempfile.mkdtemp(prefix="visdom_exp_search_") + self.store = ExperimentStore(JSONStore(self._tmp_dir)) + seed_experiments(self.store) + + def test_no_query_returns_all(self): + """A None query matches every logged experiment.""" + self.assertEqual( + sorted(env_ids(self.store.search())), ["run-a", "run-b", "run-c"] + ) + + def test_blank_query_returns_all(self): + """A blank/whitespace query matches everything rather than failing to parse.""" + self.assertEqual(len(self.store.search(query="")), 3) + self.assertEqual(len(self.store.search(query=" ")), 3) + + def test_search_ignores_envs_without_experiments(self): + """Environments that carry no experiment blob are simply not results.""" + self.store.datastore.save_env("plain", {"jsons": {}, "reload": {}}) + self.assertEqual(len(self.store.search()), 3) + + def test_filters_by_param(self): + """A comparison on a param name selects on that param's value.""" + self.assertEqual(env_ids(self.store.search(query="lr < 0.01")), ["run-b"]) + + def test_filters_by_latest_metric(self): + """Metrics compare on their most recent value, not their first.""" + # run-b logged acc 0.55 and then 0.95; only the latter should match. + self.assertEqual(env_ids(self.store.search(query="acc > 0.9")), ["run-b"]) + + def test_filters_by_namespaced_name(self): + """The namespaced spelling reaches the same values as the bare one.""" + self.assertEqual(env_ids(self.store.search(query="param.lr = 0.5")), ["run-c"]) + self.assertEqual( + env_ids(self.store.search(query="tag.dataset contains mnist")), ["run-a"] + ) + + def test_filters_by_builtin_field(self): + """Built-in fields (status, name) are queryable alongside user data.""" + self.assertEqual( + env_ids(self.store.search(query="status = finished")), ["run-b"] + ) + self.assertEqual(env_ids(self.store.search(query="name = alpha")), ["run-a"]) + + def test_filters_with_boolean_operators(self): + """AND/OR/parentheses combine comparisons as the parser defines.""" + self.assertEqual( + env_ids(self.store.search(query="lr > 0.05 AND epochs = 10")), ["run-a"] + ) + self.assertEqual( + sorted(env_ids(self.store.search(query="lr < 0.01 OR lr > 0.4"))), + ["run-b", "run-c"], + ) + + def test_no_matches_returns_empty(self): + """A query nothing satisfies returns an empty list, not an error.""" + self.assertEqual(self.store.search(query="lr > 100"), []) + + def test_unknown_field_matches_nothing(self): + """A field no experiment has is absent, and absent never matches.""" + self.assertEqual(self.store.search(query="nonexistent > 1"), []) + + def test_invalid_query_raises_parse_error(self): + """Malformed query syntax surfaces as QueryParseError.""" + with self.assertRaises(QueryParseError): + self.store.search(query="lr <") + + def test_non_string_query_raises_type_error(self): + """A non-string query is a caller bug, not a parse error.""" + with self.assertRaises(TypeError): + self.store.search(query=42) + + def test_sorts_newest_first_by_default(self): + """The default sort is created_at, descending.""" + self.assertEqual(env_ids(self.store.search()), ["run-c", "run-b", "run-a"]) + + def test_sort_ascending(self): + """descending=False reverses the order.""" + self.assertEqual( + env_ids(self.store.search(descending=False)), ["run-a", "run-b", "run-c"] + ) + + def test_sort_by_param(self): + """Sorting works on any queryable name, not just built-ins.""" + self.assertEqual( + env_ids(self.store.search(sort_by="lr")), ["run-c", "run-a", "run-b"] + ) + + def test_sort_by_metric_puts_missing_last_in_both_directions(self): + """A run missing the sort field sorts last however the sort is directed.""" + # run-c logged no metrics at all, so it has no "acc" to be ranked by. + self.assertEqual( + env_ids(self.store.search(sort_by="acc")), ["run-b", "run-a", "run-c"] + ) + self.assertEqual( + env_ids(self.store.search(sort_by="acc", descending=False)), + ["run-a", "run-b", "run-c"], + ) + + def test_sort_by_none_keeps_backend_order(self): + """sort_by=None leaves the store's own ordering untouched.""" + unsorted = env_ids(self.store.search(sort_by=None)) + self.assertEqual(sorted(unsorted), ["run-a", "run-b", "run-c"]) + + def test_sort_by_mixed_types_does_not_raise(self): + """A field holding a number in one run and a string in another still sorts.""" + self.store.log_experiment("run-d", params={"lr": "auto"}) + results = env_ids(self.store.search(sort_by="lr", descending=False)) + # Numbers order among themselves and ahead of the string. + self.assertEqual(results, ["run-b", "run-a", "run-c", "run-d"]) + + def test_filter_and_sort_combine(self): + """Sorting applies to the filtered set.""" + self.assertEqual( + env_ids(self.store.search(query="lr > 0.01", sort_by="lr")), + ["run-c", "run-a"], + ) + + def test_search_reads_what_another_store_wrote(self): + """Results come off disk, so a fresh store sees the same experiments.""" + fresh = ExperimentStore(JSONStore(self._tmp_dir)) + self.assertEqual(env_ids(fresh.search(query="acc > 0.9")), ["run-b"]) + + +class TestSearchEndpoint(tornado.testing.AsyncHTTPTestCase): + """POST /experiments/search returns a paged, sorted, filtered result set.""" + + def setUp(self): + self._tmp_dir = tempfile.mkdtemp(prefix="visdom_exp_search_api_") + super().setUp() + seed_experiments(ExperimentStore(JSONStore(self._tmp_dir))) + + def get_app(self): + return Application(port=self.get_http_port(), env_path=self._tmp_dir) + + def search(self, body): + return self.fetch( + "/experiments/search", + method="POST", + body=json.dumps(body), + headers={"Content-Type": "application/json"}, + ) + + def search_ok(self, body): + resp = self.search(body) + self.assertEqual(resp.code, 200) + return json.loads(resp.body) + + def test_empty_body_returns_everything(self): + """A search with no query returns all experiments, newest first.""" + body = self.search_ok({}) + self.assertEqual( + [e["env_id"] for e in body["experiments"]], ["run-c", "run-b", "run-a"] + ) + self.assertEqual(body["total"], 3) + self.assertEqual(body["offset"], 0) + self.assertEqual(body["query"], "") + + def test_query_filters_results(self): + """The query reaches the parser and filters the reply.""" + body = self.search_ok({"query": "lr < 0.01 AND acc > 0.9"}) + self.assertEqual(body["total"], 1) + self.assertEqual(body["experiments"][0]["env_id"], "run-b") + self.assertEqual(body["query"], "lr < 0.01 AND acc > 0.9") + + def test_experiments_are_returned_in_full(self): + """Each result is the full experiment dict, params/metrics/tags included.""" + body = self.search_ok({"query": "name = alpha"}) + experiment = body["experiments"][0] + self.assertEqual(experiment["name"], "alpha") + self.assertEqual(experiment["params"][0]["key"], "lr") + self.assertEqual(experiment["metrics"][0]["key"], "acc") + self.assertEqual(experiment["tags"][0]["value"], "mnist") + + def test_limit_pages_results_and_total_ignores_it(self): + """limit caps the page while total still counts every match.""" + body = self.search_ok({"limit": 2}) + self.assertEqual([e["env_id"] for e in body["experiments"]], ["run-c", "run-b"]) + self.assertEqual(body["total"], 3) + self.assertEqual(body["limit"], 2) + + def test_offset_walks_the_pages(self): + """offset skips the results already seen.""" + body = self.search_ok({"limit": 2, "offset": 2}) + self.assertEqual([e["env_id"] for e in body["experiments"]], ["run-a"]) + self.assertEqual(body["total"], 3) + + def test_offset_past_the_end_is_empty(self): + """Paging past the last result is an empty page, not an error.""" + body = self.search_ok({"offset": 99}) + self.assertEqual(body["experiments"], []) + self.assertEqual(body["total"], 3) + + def test_limit_zero_returns_count_only(self): + """limit=0 is a legitimate way to ask only how many match.""" + body = self.search_ok({"limit": 0}) + self.assertEqual(body["experiments"], []) + self.assertEqual(body["total"], 3) + + def test_null_limit_returns_all(self): + """An explicit null limit lifts the cap.""" + body = self.search_ok({"limit": None}) + self.assertEqual(len(body["experiments"]), 3) + self.assertIsNone(body["limit"]) + + def test_default_limit_is_applied(self): + """A body that omits limit is capped at the handler's default.""" + body = self.search_ok({}) + self.assertEqual(body["limit"], 100) + + def test_integral_float_limit_is_accepted(self): + """JSON has no int type, so 2.0 is honoured as the index 2.""" + body = self.search_ok({"limit": 2.0}) + self.assertEqual(len(body["experiments"]), 2) + + def test_sort_by_and_direction(self): + """sort_by/descending order the reply.""" + body = self.search_ok({"sort_by": "lr", "descending": False}) + self.assertEqual( + [e["env_id"] for e in body["experiments"]], ["run-b", "run-a", "run-c"] + ) + + def test_invalid_query_is_400(self): + """A malformed query is the caller's error, and says why.""" + resp = self.search({"query": "lr <"}) + self.assertEqual(resp.code, 400) + self.assertIn("end of query", resp.reason) + + def test_query_wrong_type_is_400(self): + """A non-string query is rejected before it reaches the parser.""" + resp = self.search({"query": {"lr": 1}}) + self.assertEqual(resp.code, 400) + self.assertIn("query", resp.reason) + + def test_negative_offset_is_400(self): + """A negative index would silently wrap around the list, so reject it.""" + self.assertEqual(self.search({"offset": -1}).code, 400) + self.assertEqual(self.search({"limit": -5}).code, 400) + + def test_non_integer_limit_is_400(self): + """limit must be a whole number.""" + self.assertEqual(self.search({"limit": "10"}).code, 400) + self.assertEqual(self.search({"limit": 1.5}).code, 400) + + def test_non_string_sort_by_is_400(self): + """sort_by must name a field.""" + self.assertEqual(self.search({"sort_by": 7}).code, 400) + + def test_non_boolean_descending_is_400(self): + """The string "false" is rejected rather than coerced to true.""" + self.assertEqual(self.search({"descending": "false"}).code, 400) + + def test_sql_injection_payload_is_inert(self): + """A SQL-ish payload is either a parse error or a plain string compare.""" + resp = self.search({"query": "name = 'x'; DROP TABLE experiments'"}) + self.assertIn(resp.code, (200, 400)) + # Whatever happened, the data is untouched. + self.assertEqual(self.search_ok({})["total"], 3) + + def test_search_sees_an_experiment_logged_over_http(self): + """An experiment logged through /experiments/log is searchable at once.""" + self.fetch( + "/experiments/log", + method="POST", + body=json.dumps({"eid": "run-e", "action": "log", "params": {"lr": 0.02}}), + headers={"Content-Type": "application/json"}, + ) + body = self.search_ok({"query": "lr = 0.02"}) + self.assertEqual(body["total"], 1) + self.assertEqual(body["experiments"][0]["env_id"], "run-e") + + +class TestSearchClientMessage(unittest.TestCase): + """Visdom.search_experiments builds the message the endpoint expects.""" + + def setUp(self): + self.vis = Visdom(send=False, raise_exceptions=True) + + def test_search_message_shape(self): + """The client sends the query and paging to the search endpoint.""" + msg, endpoint = self.vis.search_experiments("acc > 0.9", limit=5, offset=10) + self.assertEqual(endpoint, "experiments/search") + self.assertEqual(msg["query"], "acc > 0.9") + self.assertEqual(msg["limit"], 5) + self.assertEqual(msg["offset"], 10) + self.assertTrue(msg["descending"]) + + def test_search_defaults(self): + """Called bare, it asks for the first page of everything.""" + msg, _ = self.vis.search_experiments() + self.assertIsNone(msg["query"]) + self.assertIsNone(msg["sort_by"]) + self.assertEqual(msg["limit"], 100) + self.assertEqual(msg["offset"], 0) + + def test_search_rejects_non_string_query(self): + """The client type-checks the query before any request is made.""" + with self.assertRaises(TypeError): + self.vis.search_experiments(query=42) + with self.assertRaises(TypeError): + self.vis.search_experiments(sort_by=42) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index 1bc3ec3ff..c88b9bcfe 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -1137,16 +1137,14 @@ def fork_env(self, prev_eid, eid): return self._send(msg={"prev_eid": prev_eid, "eid": eid}, endpoint="fork_env") - def _experiment_send(self, msg, env): - """POST an experiment action to the server and decode the JSON reply. + def _experiment_request(self, msg, endpoint): + """POST to an experiment `endpoint` and decode the JSON reply. - Shared plumbing for :meth:`experiment`, :meth:`log_metrics` and - :meth:`finish_experiment`. Returns the stored experiment as a dict when - the server replies with JSON, otherwise the raw response (e.g. an error - string, or the `(msg, endpoint)` tuple when this client has `send=False`). + Returns the decoded reply when the server replies with JSON, otherwise + the raw response (e.g. an error string, or the `(msg, endpoint)` tuple + when this client has `send=False`). """ - msg["eid"] = env if env is not None else self.env - response = self._send(msg, endpoint="experiments/log", quiet=True) + response = self._send(msg, endpoint=endpoint, quiet=True) if not isstr(response): return response try: @@ -1154,6 +1152,16 @@ def _experiment_send(self, msg, env): except ValueError: return response + def _experiment_send(self, msg, env): + """POST an experiment action for `env` and decode the JSON reply. + + Shared plumbing for :meth:`experiment`, :meth:`log_metrics` and + :meth:`finish_experiment`, each of which acts on a single environment. + Returns the stored experiment as a dict. + """ + msg["eid"] = env if env is not None else self.env + return self._experiment_request(msg, "experiments/log") + def experiment(self, name=None, params=None, tags=None, description=None, env=None): """Create or update the experiment metadata for an environment. @@ -1199,6 +1207,43 @@ def finish_experiment(self, status="finished", env=None): """ return self._experiment_send({"action": "finish", "status": status}, env) + def search_experiments( + self, query=None, limit=100, offset=0, sort_by=None, descending=True + ): + """Search the experiments logged on the server, across all environments. + + `query` filters the results using a small readable syntax — comparisons + (`<`, `<=`, `>`, `>=`, `=`, `!=`, `contains`) over param, metric and tag + names, combined with `AND`/`OR` and parentheses: + + vis.search_experiments("lr < 0.01 AND acc > 0.9") + vis.search_experiments("status = finished AND (dataset contains mnist)") + + A name is matched bare (`acc`) or namespaced (`metric.acc`, `param.lr`, + `tag.owner`) when it is ambiguous; metrics compare on their latest value. + `query=None` returns everything. Results are sorted by `sort_by` (any of + those same names, newest-created first by default) and paged with + `limit`/`offset`; pass `limit=None` for all of them. + + Returns the server's reply as a dict of `experiments` (a list of + experiment dicts, one page worth), the unpaged `total` matching the + query, and the `limit`/`offset`/`query` used. + """ + if query is not None and not isstr(query): + raise TypeError("query must be a string") + if sort_by is not None and not isstr(sort_by): + raise TypeError("sort_by must be a string") + return self._experiment_request( + { + "query": query, + "limit": limit, + "offset": offset, + "sort_by": sort_by, + "descending": descending, + }, + "experiments/search", + ) + def get_window_data(self, win=None, env=None): """ This function returns all the window data for a specified window in diff --git a/py/visdom/experiments/__init__.py b/py/visdom/experiments/__init__.py index ed3823310..78a1c519f 100644 --- a/py/visdom/experiments/__init__.py +++ b/py/visdom/experiments/__init__.py @@ -28,11 +28,12 @@ build_record, parse_query, ) -from visdom.experiments.store import ExperimentStore +from visdom.experiments.store import DEFAULT_SORT_FIELD, ExperimentStore __all__ = [ "And", "Comparison", + "DEFAULT_SORT_FIELD", "Experiment", "ExperimentFinishedError", "ExperimentLike", diff --git a/py/visdom/experiments/store.py b/py/visdom/experiments/store.py index 14ee29a2f..999c85657 100644 --- a/py/visdom/experiments/store.py +++ b/py/visdom/experiments/store.py @@ -22,9 +22,53 @@ ExperimentFinishedError, STATUS_FINISHED, ) +from visdom.experiments.query import Query, build_record METADATA_KEY = "experiment" +DEFAULT_SORT_FIELD = "created_at" + +# Distinguishes "the record has no such field" from "the field is present and +# None": both sort last, but only the former is absent from the record at all. +_MISSING = object() + + +def _order_key(value): + """Return a sort key that totally orders values of mixed types. + + Records come from user-supplied params/metrics/tags, so one field can hold a + number in one experiment and a string in another; sorting them directly + would raise ``TypeError``. Numbers sort before strings, and everything that + is neither is compared by its text form. Booleans are ordered as text rather + than as 0/1, matching :mod:`~visdom.experiments.query`, which likewise + refuses to treat a bool as a number. + """ + if isinstance(value, bool): + return (1, 0.0, str(value)) + if isinstance(value, (int, float)): + return (0, float(value), "") + return (1, 0.0, str(value)) + + +def _sort_pairs(pairs, field, descending): + """Sort ``(record, experiment)`` pairs by ``record[field]``. + + Experiments whose record lacks ``field`` (or holds ``None`` there) keep + their relative order and always land last, in both directions: a run that + never logged ``acc`` is not the best-scoring run just because the sort was + reversed. + """ + present = [] + missing = [] + for record, experiment in pairs: + value = record.get(field, _MISSING) + if value is _MISSING or value is None: + missing.append((record, experiment)) + else: + present.append((record, experiment)) + present.sort(key=lambda pair: _order_key(pair[0][field]), reverse=descending) + return present + missing + class ExperimentStore: """Read/write experiment metadata attached to environments via a DataStore.""" @@ -130,6 +174,41 @@ def list_experiments(self): experiments.append(experiment) return experiments + def search(self, query=None, sort_by=DEFAULT_SORT_FIELD, descending=True): + """Return the experiments matching ``query``, sorted by ``sort_by``. + + ``query`` is the human-readable syntax of + :mod:`~visdom.experiments.query` (``"lr < 0.01 AND acc > 90"``); ``None`` + or a blank string matches every experiment. Matching runs against the + flattened record of :func:`~visdom.experiments.query.build_record`, so a + param, latest metric or tag is reachable both bare (``acc``) and + namespaced (``metric.acc``) — and ``sort_by`` accepts either spelling of + the same names. + + Sorting defaults to newest-first; pass ``descending=False`` for oldest + first, or ``sort_by=None`` to keep the backend's own ordering. Results + are ordinary :class:`Experiment` objects, and paging through them is left + to the caller — the whole set is scanned regardless, since every + environment must be read to know whether it matches. + + Raises :class:`~visdom.experiments.query.QueryParseError` if ``query`` + is not valid query syntax. + """ + if query is not None and not isinstance(query, str): + raise TypeError( + "query must be a string or None, got {0}".format(type(query).__name__) + ) + pairs = [ + (build_record(experiment), experiment) + for experiment in self.list_experiments() + ] + if query is not None and query.strip(): + compiled = Query(query) + pairs = [pair for pair in pairs if compiled.matches(pair[0])] + if sort_by: + pairs = _sort_pairs(pairs, sort_by, descending) + return [experiment for _, experiment in pairs] + def delete_experiment(self, env_id): """Drop the experiment blob from ``env_id`` (keeping the env itself). diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index 26db0f56c..0e2e146f7 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -37,6 +37,7 @@ ErrorHandler, ExistsHandler, ExperimentLogHandler, + ExperimentSearchHandler, ForkEnvHandler, HealthHandler, IndexHandler, @@ -126,6 +127,11 @@ def __init__( ExperimentLogHandler, {"app": self}, ), + ( + r"%s/experiments/search" % self.base_url, + ExperimentSearchHandler, + {"app": self}, + ), (r"%s/user/(.*)" % self.base_url, UserSettingsHandler, {"app": self}), (r"%s/health" % self.base_url, HealthHandler), (r"%s(.*)" % self.base_url, IndexHandler, {"app": self}), diff --git a/py/visdom/server/handlers/web_handlers.py b/py/visdom/server/handlers/web_handlers.py index 4e4529b1f..4bdaac4c8 100644 --- a/py/visdom/server/handlers/web_handlers.py +++ b/py/visdom/server/handlers/web_handlers.py @@ -47,8 +47,10 @@ ) from visdom.server.handlers.base_handlers import BaseHandler from visdom.experiments import ( + DEFAULT_SORT_FIELD, ExperimentStore, ExperimentFinishedError, + QueryParseError, STATUS_FINISHED, ) @@ -913,6 +915,117 @@ def post(self): self.wrap_func(self, args) +class ExperimentSearchHandler(BaseHandler): + """POST ``/experiments/search`` — find experiments across all environments. + + The JSON body carries a ``query`` in the syntax of + :mod:`~visdom.experiments.query` (``"lr < 0.01 AND acc > 90"``); an absent or + blank query matches every experiment. Queries are parsed into a predicate and + evaluated in Python — never eval'd — so a hostile query is a parse error, not + an execution. + + Results are sorted (``sort_by``/``descending``, newest first by default) and + then paged with ``limit``/``offset``, and the reply reports the unpaged + ``total`` so a caller can page through it: + + {"experiments": [...], "total": 42, "limit": 100, "offset": 0, "query": ""} + + Experiments are read back through the server's ``DataStore``, which means a + server running with ``env_path=None`` — where nothing is persisted at all — + has nothing to search and returns no results. + """ + + DEFAULT_LIMIT = 100 + + @staticmethod + def _require_index(args, field, default): + """Return ``args[field]`` as a non-negative int (``None`` = unbounded).""" + value = args.get(field, default) + if value is None: + return None + # A JSON body has no int/float distinction, so a client that sends 10.0 + # means the index 10; anything with a fractional part is a mistake. + if isinstance(value, float) and value.is_integer(): + value = int(value) + if isinstance(value, bool) or not isinstance(value, int): + raise tornado.web.HTTPError( + 400, reason="'{0}' must be an integer".format(field) + ) + if value < 0: + raise tornado.web.HTTPError( + 400, reason="'{0}' must not be negative".format(field) + ) + return value + + @staticmethod + def _require_text(args, field): + """Return ``args[field]`` if it is a string (or absent); else raise 400.""" + value = args.get(field) + if value is not None and not isinstance(value, str): + raise tornado.web.HTTPError( + 400, reason="'{0}' must be a string".format(field) + ) + return value + + @staticmethod + def _require_flag(args, field, default): + """Return ``args[field]`` as a bool; else raise 400. + + Deliberately not ``bool(value)``: JSON has real booleans, so a client + sending the *string* ``"false"`` means false, and coercing it would + truthily flip the result to its opposite without a word. + """ + value = args.get(field, default) + if not isinstance(value, bool): + raise tornado.web.HTTPError( + 400, reason="'{0}' must be a boolean".format(field) + ) + return value + + @staticmethod + def wrap_func(handler, args): + query = ExperimentSearchHandler._require_text(args, "query") + sort_by = ExperimentSearchHandler._require_text(args, "sort_by") + limit = ExperimentSearchHandler._require_index( + args, "limit", ExperimentSearchHandler.DEFAULT_LIMIT + ) + offset = ExperimentSearchHandler._require_index(args, "offset", 0) + descending = ExperimentSearchHandler._require_flag(args, "descending", True) + + store = ExperimentStore(handler.storage) + try: + experiments = store.search( + query=query, + sort_by=sort_by or DEFAULT_SORT_FIELD, + descending=descending, + ) + except QueryParseError as e: + raise tornado.web.HTTPError(400, reason=str(e)) + + end = None if limit is None else offset + limit + page = experiments[offset:end] + + handler.write( + json.dumps( + { + "experiments": [e.to_dict() for e in page], + "total": len(experiments), + "limit": limit, + "offset": offset, + "query": query or "", + }, + cls=NanSafeEncoder, + ) + ) + + @check_auth + def post(self): + args = tornado.escape.json_decode( + tornado.escape.to_basestring(self.request.body) + ) + self.wrap_func(self, args) + + class HealthHandler(BaseHandler): def get(self): self.write({"status": "ok"}) From eb998dc72586bee005b11734ea49fe0ad41c5e01 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 15 Jul 2026 14:50:40 +0530 Subject: [PATCH 05/48] drop explanatory comments from search layer Remove the inline # comments PR-4 added and fold the useful context into docstrings instead, matching the experiments package style. No behaviour change. --- py/tests/test_experiment_search.py | 26 +++++++++++++++-------- py/visdom/experiments/store.py | 2 -- py/visdom/server/handlers/web_handlers.py | 8 ++++--- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/py/tests/test_experiment_search.py b/py/tests/test_experiment_search.py index 9b1beee82..97c46cd64 100644 --- a/py/tests/test_experiment_search.py +++ b/py/tests/test_experiment_search.py @@ -41,7 +41,7 @@ def seed_experiments(store): tags={"dataset": "cifar10"}, ) store.log_metric("run-b", "acc", 0.55) - store.log_metric("run-b", "acc", 0.95) # latest wins when querying "acc" + store.log_metric("run-b", "acc", 0.95) store.finish_experiment("run-b") store.log_experiment("run-c", name="gamma", params={"lr": 0.5}) @@ -85,8 +85,10 @@ def test_filters_by_param(self): self.assertEqual(env_ids(self.store.search(query="lr < 0.01")), ["run-b"]) def test_filters_by_latest_metric(self): - """Metrics compare on their most recent value, not their first.""" - # run-b logged acc 0.55 and then 0.95; only the latter should match. + """Metrics compare on their most recent value, not their first. + + run-b logged acc 0.55 and then 0.95; only the latter should match. + """ self.assertEqual(env_ids(self.store.search(query="acc > 0.9")), ["run-b"]) def test_filters_by_namespaced_name(self): @@ -148,8 +150,10 @@ def test_sort_by_param(self): ) def test_sort_by_metric_puts_missing_last_in_both_directions(self): - """A run missing the sort field sorts last however the sort is directed.""" - # run-c logged no metrics at all, so it has no "acc" to be ranked by. + """A run missing the sort field sorts last however the sort is directed. + + run-c logged no metrics at all, so it has no "acc" to be ranked by. + """ self.assertEqual( env_ids(self.store.search(sort_by="acc")), ["run-b", "run-a", "run-c"] ) @@ -164,10 +168,12 @@ def test_sort_by_none_keeps_backend_order(self): self.assertEqual(sorted(unsorted), ["run-a", "run-b", "run-c"]) def test_sort_by_mixed_types_does_not_raise(self): - """A field holding a number in one run and a string in another still sorts.""" + """A field holding a number in one run and a string in another still sorts. + + Numbers order among themselves and ahead of the string. + """ self.store.log_experiment("run-d", params={"lr": "auto"}) results = env_ids(self.store.search(sort_by="lr", descending=False)) - # Numbers order among themselves and ahead of the string. self.assertEqual(results, ["run-b", "run-a", "run-c", "run-d"]) def test_filter_and_sort_combine(self): @@ -312,10 +318,12 @@ def test_non_boolean_descending_is_400(self): self.assertEqual(self.search({"descending": "false"}).code, 400) def test_sql_injection_payload_is_inert(self): - """A SQL-ish payload is either a parse error or a plain string compare.""" + """A SQL-ish payload is either a parse error or a plain string compare. + + Whatever happened, the data must be untouched. + """ resp = self.search({"query": "name = 'x'; DROP TABLE experiments'"}) self.assertIn(resp.code, (200, 400)) - # Whatever happened, the data is untouched. self.assertEqual(self.search_ok({})["total"], 3) def test_search_sees_an_experiment_logged_over_http(self): diff --git a/py/visdom/experiments/store.py b/py/visdom/experiments/store.py index 999c85657..2246bbb51 100644 --- a/py/visdom/experiments/store.py +++ b/py/visdom/experiments/store.py @@ -28,8 +28,6 @@ DEFAULT_SORT_FIELD = "created_at" -# Distinguishes "the record has no such field" from "the field is present and -# None": both sort last, but only the former is absent from the record at all. _MISSING = object() diff --git a/py/visdom/server/handlers/web_handlers.py b/py/visdom/server/handlers/web_handlers.py index 4bdaac4c8..975107a1e 100644 --- a/py/visdom/server/handlers/web_handlers.py +++ b/py/visdom/server/handlers/web_handlers.py @@ -939,12 +939,14 @@ class ExperimentSearchHandler(BaseHandler): @staticmethod def _require_index(args, field, default): - """Return ``args[field]`` as a non-negative int (``None`` = unbounded).""" + """Return ``args[field]`` as a non-negative int (``None`` = unbounded). + + A JSON body has no int/float distinction, so a client that sends ``10.0`` + means the index 10; anything with a fractional part is a mistake. + """ value = args.get(field, default) if value is None: return None - # A JSON body has no int/float distinction, so a client that sends 10.0 - # means the index 10; anything with a fractional part is a mistake. if isinstance(value, float) and value.is_integer(): value = int(value) if isinstance(value, bool) or not isinstance(value, int): From 4b2cc8a4c1c038eca78b879e209d881ab7bd3a45 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 15 Jul 2026 17:09:25 +0530 Subject: [PATCH 06/48] experiment compare endpoint + vis.compare_experiments Layer 2 continues: after search, ask what actually differs between the runs it found. New visdom/experiments/compare.py holds the diff itself, pure and free of storage the way query.py's build_record is: build_comparison() lines the experiments up and reports, per section (params/metrics/tags), the union of fields, the shared ones every run agrees on, the differing rest, and the per-run values. Metrics diff on their latest observation, the same value a search compares on, so a run found by "acc > 0.9" shows that acc here. A field only some runs carry is a difference rather than a consensus among those that have it. Two comparisons are deliberately not ==. Bools are not numbers, so amp=True and amp=1 differ, matching query.py's refusal to treat a bool as a number. NaN agrees with itself, since a metric NaN in every run is not a difference and calling it one would bury the real ones. ExperimentStore.compare() selects the runs and delegates. Selection is by name or by query, mutually exclusive: both or neither raises rather than guessing which was meant. env_ids compares in the order given, dedupes, and raises KeyError naming any id without an experiment -- a comparison silently missing a run it was asked for reads as a comparison of the rest. A query selects via search() ordered by sort_by/descending and capped by limit; matching nothing is an empty comparison, not an error. ExperimentCompareHandler (POST /experiments/compare) mirrors the search handler's shape and maps those to 400/404. It rejects a bare-string env_ids, which would otherwise iterate into a comparison of runs "r", "u", "n". The three request validators move from ExperimentSearchHandler statics to module level so both handlers share one copy. Client: vis.compare_experiments(env_ids=None, query=None, limit=None, sort_by=None, descending=True). Documented in openapi.yaml (compareExperiments + ExperimentComparisonSection schema). Tests: py/tests/test_experiment_compare.py (50 -- pure diff, store selection, endpoint e2e, validation, client shape); 414 py/tests pass. Verified live against a real server and client: both selection modes, limit, 404/400 paths, injection payload inert with data intact. --- openapi.yaml | 138 +++++++ py/tests/test_experiment_compare.py | 463 ++++++++++++++++++++++ py/visdom/__init__.py | 48 +++ py/visdom/experiments/__init__.py | 8 + py/visdom/experiments/compare.py | 161 ++++++++ py/visdom/experiments/store.py | 82 ++++ py/visdom/server/app.py | 6 + py/visdom/server/handlers/web_handlers.py | 193 ++++++--- 8 files changed, 1045 insertions(+), 54 deletions(-) create mode 100644 py/tests/test_experiment_compare.py create mode 100644 py/visdom/experiments/compare.py diff --git a/openapi.yaml b/openapi.yaml index c4044c53a..48bf5652b 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -569,6 +569,110 @@ paths: integer. Also returned when authentication is required but not provided. + /experiments/compare: + post: + operationId: compareExperiments + tags: [Experiments] + summary: Compare experiments field by field + description: > + Lines several experiments up beside each other and reports, per section + (params, metrics and tags), which fields they agree on and which they do + not — the short list of knobs that actually changed between runs. + + + The runs to compare are selected either by name (`env_ids`) or by search + (`query`), and the two are mutually exclusive: sending both, or neither, + is a `400` rather than a silent guess at which was meant. With `env_ids` + the runs are compared in the order given and every id must have an + experiment, otherwise `404`. With `query` the syntax and the + `sort_by`/`descending`/`limit` handling are those of + `/experiments/search`, and a query matching nothing is an empty + comparison rather than an error. + + + Metrics are a time series, so a comparison uses each metric's latest + observation — the same value a search compares on. + + + Experiments are read back through the server's data store, so a server + running with no persistence path configured has nothing to compare. + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + env_ids: + type: array + nullable: true + minItems: 1 + description: > + Environments to compare, in the order given. Mutually + exclusive with `query`. Duplicate ids collapse to one. + items: + type: string + query: + type: string + nullable: true + description: > + Compare every experiment this filter matches, in the syntax + of `/experiments/search`. Mutually exclusive with `env_ids`. + sort_by: + type: string + nullable: true + default: created_at + description: > + Field ordering the compared runs (`query` selection only). + descending: + type: boolean + default: true + description: Sort direction; newest/highest first by default. + limit: + type: integer + nullable: true + minimum: 0 + default: null + description: > + Maximum number of runs to compare (`query` selection only; + `null` compares every match). Note that the diff describes + the runs actually compared, so a limit that truncates the + matches narrows what `shared`/`differing` are computed + over; the returned `env_ids` say which runs those were. + responses: + "200": + description: The comparison of the selected experiments. + content: + application/json: + schema: + type: object + required: [env_ids, experiments, params, metrics, tags] + properties: + env_ids: + type: array + description: The runs compared, in the order compared. + items: + type: string + experiments: + type: array + description: The compared experiments, in full. + items: + $ref: "#/components/schemas/Experiment" + params: + $ref: "#/components/schemas/ExperimentComparisonSection" + metrics: + $ref: "#/components/schemas/ExperimentComparisonSection" + tags: + $ref: "#/components/schemas/ExperimentComparisonSection" + "400": + description: > + Invalid request — both `env_ids` and `query` given, neither given, + an empty or non-string `env_ids`, malformed `query` syntax, or a + `limit` that is not a non-negative integer. Also returned when + authentication is required but not provided. + "404": + description: One or more of the given `env_ids` has no experiment. + /upload_env: post: operationId: uploadEnvironment @@ -1032,6 +1136,40 @@ components: schemas: + ExperimentComparisonSection: + type: object + description: > + One section (params, metrics or tags) of an experiment comparison, + diffed across the compared runs. + required: [fields, shared, differing, values] + properties: + fields: + type: array + description: Every field name any compared run has, sorted. + items: + type: string + shared: + type: object + additionalProperties: true + description: > + The fields every compared run carries with the same value, as + `{name: value}`. + differing: + type: array + description: > + The remaining fields — those whose value varies between runs, or + that some run is missing. + items: + type: string + values: + type: object + additionalProperties: + type: object + additionalProperties: true + description: > + Per-field, per-run values as `{field: {env_id: value}}`. A run that + never logged the field is omitted from that field's map. + Experiment: type: object description: Experiment metadata attached to an environment. diff --git a/py/tests/test_experiment_compare.py b/py/tests/test_experiment_compare.py new file mode 100644 index 000000000..aae54f66a --- /dev/null +++ b/py/tests/test_experiment_compare.py @@ -0,0 +1,463 @@ +"""Tests for experiment comparison (Layer 2, PR 5). + +Covers the four pieces the compare layer is built from: the pure +``build_comparison`` diff over experiment objects; ``ExperimentStore.compare`` +selecting runs by name or by query against a real ``JSONStore`` over a temporary +directory; the ``/experiments/compare`` endpoint end-to-end through a real +:class:`~visdom.server.app.Application` with Tornado's ``AsyncHTTPTestCase``; and +the ``Visdom.compare_experiments`` message shape with ``send=False`` (no server). +""" + +import json +import math +import tempfile +import unittest + +import tornado.testing + +from visdom import Visdom +from visdom.data_model import JSONStore +from visdom.experiments import ( + Experiment, + ExperimentStore, + QueryParseError, + build_comparison, +) +from visdom.server.app import Application + + +def seed_experiments(store): + """Log three experiments with known params/metrics/tags and created_at order. + + run-a and run-b share ``epochs`` and differ on ``lr``; run-c has neither + ``epochs`` nor any metric, so it exercises the missing-field paths. + ``created_at`` is stamped explicitly rather than left to wall-clock time, + since the runs are logged microseconds apart. + """ + store.log_experiment( + "run-a", + name="alpha", + params={"lr": 0.1, "epochs": 10}, + tags={"dataset": "mnist"}, + ) + store.log_metric("run-a", "acc", 0.80) + + store.log_experiment( + "run-b", + name="beta", + params={"lr": 0.001, "epochs": 10}, + tags={"dataset": "cifar10"}, + ) + store.log_metric("run-b", "acc", 0.55) + store.log_metric("run-b", "acc", 0.95) + store.finish_experiment("run-b") + + store.log_experiment("run-c", name="gamma", params={"lr": 0.5}) + + for env_id, created_at in (("run-a", 100.0), ("run-b", 200.0), ("run-c", 300.0)): + env, experiment = store._read(env_id) + experiment.created_at = created_at + store._write(env_id, env, experiment) + + +def make_experiment(env_id, params=None, metrics=None, tags=None): + """Build an in-memory Experiment, for the storage-free diff tests.""" + experiment = Experiment(env_id=env_id) + for key, value in (params or {}).items(): + experiment.set_param(key, value) + for key, value in metrics or []: + experiment.add_metric(key, value) + for key, value in (tags or {}).items(): + experiment.set_tag(key, value) + return experiment + + +class TestBuildComparison(unittest.TestCase): + """build_comparison diffs experiment objects without touching storage.""" + + def test_reports_shared_and_differing_params(self): + """A field all runs agree on is shared; one that varies is differing.""" + comparison = build_comparison( + [ + make_experiment("a", params={"lr": 0.1, "epochs": 10}), + make_experiment("b", params={"lr": 0.2, "epochs": 10}), + ] + ) + params = comparison["params"] + self.assertEqual(params["fields"], ["epochs", "lr"]) + self.assertEqual(params["shared"], {"epochs": 10}) + self.assertEqual(params["differing"], ["lr"]) + self.assertEqual(params["values"]["lr"], {"a": 0.1, "b": 0.2}) + + def test_echoes_compared_runs_in_order(self): + """env_ids and experiments come back in the order compared.""" + comparison = build_comparison( + [make_experiment("b"), make_experiment("a")], + ) + self.assertEqual(comparison["env_ids"], ["b", "a"]) + self.assertEqual([e["env_id"] for e in comparison["experiments"]], ["b", "a"]) + + def test_field_missing_from_one_run_is_differing(self): + """A field only some runs carry is a difference, not a consensus.""" + comparison = build_comparison( + [ + make_experiment("a", params={"lr": 0.1, "seed": 7}), + make_experiment("b", params={"lr": 0.1}), + ] + ) + params = comparison["params"] + self.assertEqual(params["shared"], {"lr": 0.1}) + self.assertEqual(params["differing"], ["seed"]) + self.assertEqual(params["values"]["seed"], {"a": 7}) + + def test_metrics_compare_on_latest_value(self): + """Metrics are a time series; the comparison uses the most recent one.""" + comparison = build_comparison( + [ + make_experiment("a", metrics=[("acc", 0.5), ("acc", 0.9)]), + make_experiment("b", metrics=[("acc", 0.9)]), + ] + ) + self.assertEqual(comparison["metrics"]["shared"], {"acc": 0.9}) + + def test_tags_are_compared(self): + """Tags get the same treatment as params and metrics.""" + comparison = build_comparison( + [ + make_experiment("a", tags={"owner": "mk", "stage": "dev"}), + make_experiment("b", tags={"owner": "mk", "stage": "prod"}), + ] + ) + self.assertEqual(comparison["tags"]["shared"], {"owner": "mk"}) + self.assertEqual(comparison["tags"]["differing"], ["stage"]) + + def test_bool_is_not_the_same_as_one(self): + """True == 1 in Python, but a run using amp=True is not one using amp=1.""" + comparison = build_comparison( + [ + make_experiment("a", params={"amp": True}), + make_experiment("b", params={"amp": 1}), + ] + ) + self.assertEqual(comparison["params"]["differing"], ["amp"]) + + def test_nan_agrees_with_itself(self): + """NaN != NaN, but a metric NaN in every run is not a difference.""" + comparison = build_comparison( + [ + make_experiment("a", metrics=[("loss", float("nan"))]), + make_experiment("b", metrics=[("loss", float("nan"))]), + ] + ) + self.assertEqual(comparison["metrics"]["differing"], []) + self.assertTrue(math.isnan(comparison["metrics"]["shared"]["loss"])) + + def test_single_experiment_shares_everything(self): + """Comparing one run is degenerate but legal: nothing can differ.""" + comparison = build_comparison([make_experiment("a", params={"lr": 0.1})]) + self.assertEqual(comparison["params"]["shared"], {"lr": 0.1}) + self.assertEqual(comparison["params"]["differing"], []) + + def test_no_experiments_yields_empty_sections(self): + """Comparing nothing is empty, not an error.""" + comparison = build_comparison([]) + self.assertEqual(comparison["env_ids"], []) + for section in ("params", "metrics", "tags"): + self.assertEqual(comparison[section]["fields"], []) + self.assertEqual(comparison[section]["shared"], {}) + + def test_sections_are_independent(self): + """A name used as both a param and a tag is not conflated across sections.""" + comparison = build_comparison( + [ + make_experiment("a", params={"mode": "fast"}, tags={"mode": "slow"}), + make_experiment("b", params={"mode": "fast"}, tags={"mode": "slow"}), + ] + ) + self.assertEqual(comparison["params"]["shared"], {"mode": "fast"}) + self.assertEqual(comparison["tags"]["shared"], {"mode": "slow"}) + + +class TestStoreCompare(unittest.TestCase): + """ExperimentStore.compare selects runs by name or by query.""" + + def setUp(self): + self._tmp_dir = tempfile.mkdtemp(prefix="visdom_exp_compare_") + self.store = ExperimentStore(JSONStore(self._tmp_dir)) + seed_experiments(self.store) + + def test_compare_by_env_ids(self): + """An explicit list compares exactly those runs, in order.""" + comparison = self.store.compare(env_ids=["run-b", "run-a"]) + self.assertEqual(comparison["env_ids"], ["run-b", "run-a"]) + self.assertEqual(comparison["params"]["shared"], {"epochs": 10}) + self.assertEqual(comparison["params"]["differing"], ["lr"]) + + def test_compare_reads_from_storage(self): + """Runs come off disk, so a fresh store compares the same experiments.""" + fresh = ExperimentStore(JSONStore(self._tmp_dir)) + comparison = fresh.compare(env_ids=["run-a", "run-b"]) + self.assertEqual( + comparison["metrics"]["values"]["acc"], + { + "run-a": 0.80, + "run-b": 0.95, + }, + ) + + def test_duplicate_env_ids_collapse(self): + """Naming a run twice cannot mean anything beyond naming it once.""" + comparison = self.store.compare(env_ids=["run-a", "run-a"]) + self.assertEqual(comparison["env_ids"], ["run-a"]) + + def test_unknown_env_id_raises_key_error(self): + """A comparison silently missing a requested run would mislead.""" + with self.assertRaises(KeyError) as ctx: + self.store.compare(env_ids=["run-a", "nope"]) + self.assertIn("nope", str(ctx.exception)) + + def test_env_without_experiment_raises_key_error(self): + """An env that exists but has no experiment is still nothing to compare.""" + self.store.datastore.save_env("plain", {"jsons": {}, "reload": {}}) + with self.assertRaises(KeyError): + self.store.compare(env_ids=["run-a", "plain"]) + + def test_empty_env_ids_raises_value_error(self): + """Comparing an empty list cannot mean anything.""" + with self.assertRaises(ValueError): + self.store.compare(env_ids=[]) + + def test_string_env_ids_raises_type_error(self): + """A bare string would iterate into a comparison of single characters.""" + with self.assertRaises(TypeError): + self.store.compare(env_ids="run-a") + + def test_non_string_env_id_raises_type_error(self): + with self.assertRaises(TypeError): + self.store.compare(env_ids=["run-a", 7]) + + def test_compare_by_query(self): + """A query selects the runs to compare.""" + comparison = self.store.compare(query="lr > 0.05") + self.assertEqual(sorted(comparison["env_ids"]), ["run-a", "run-c"]) + self.assertEqual(comparison["params"]["differing"], ["epochs", "lr"]) + + def test_query_selection_is_sorted(self): + """sort_by/descending order the compared runs as search does.""" + comparison = self.store.compare(query="lr > 0.05", descending=False) + self.assertEqual(comparison["env_ids"], ["run-a", "run-c"]) + comparison = self.store.compare(query="lr > 0.05", sort_by="lr") + self.assertEqual(comparison["env_ids"], ["run-c", "run-a"]) + + def test_query_limit_caps_the_compared_set(self): + """limit narrows which runs are compared, and env_ids says which.""" + comparison = self.store.compare(query="lr > 0.0001", limit=2) + self.assertEqual(comparison["env_ids"], ["run-c", "run-b"]) + self.assertEqual(len(comparison["experiments"]), 2) + + def test_query_matching_nothing_compares_nothing(self): + """An empty answer to a valid question is not an error.""" + comparison = self.store.compare(query="lr > 100") + self.assertEqual(comparison["env_ids"], []) + self.assertEqual(comparison["params"]["fields"], []) + + def test_invalid_query_raises_parse_error(self): + with self.assertRaises(QueryParseError): + self.store.compare(query="lr <") + + def test_both_selectors_raises_value_error(self): + """Passing both is ambiguous, so refuse rather than pick one.""" + with self.assertRaises(ValueError): + self.store.compare(env_ids=["run-a"], query="lr > 0") + + def test_neither_selector_raises_value_error(self): + with self.assertRaises(ValueError): + self.store.compare() + + def test_limit_is_ignored_for_env_ids(self): + """An explicit list is already the exact set; limit has nothing to cap.""" + comparison = self.store.compare(env_ids=["run-a", "run-b"], limit=1) + self.assertEqual(comparison["env_ids"], ["run-a", "run-b"]) + + +class TestCompareEndpoint(tornado.testing.AsyncHTTPTestCase): + """POST /experiments/compare diffs the selected experiments.""" + + def setUp(self): + self._tmp_dir = tempfile.mkdtemp(prefix="visdom_exp_compare_api_") + super().setUp() + seed_experiments(ExperimentStore(JSONStore(self._tmp_dir))) + + def get_app(self): + return Application(port=self.get_http_port(), env_path=self._tmp_dir) + + def compare(self, body): + return self.fetch( + "/experiments/compare", + method="POST", + body=json.dumps(body), + headers={"Content-Type": "application/json"}, + ) + + def compare_ok(self, body): + resp = self.compare(body) + self.assertEqual(resp.code, 200) + return json.loads(resp.body) + + def test_compare_by_env_ids(self): + """The named runs are compared, in the order given.""" + body = self.compare_ok({"env_ids": ["run-a", "run-b"]}) + self.assertEqual(body["env_ids"], ["run-a", "run-b"]) + self.assertEqual(body["params"]["shared"], {"epochs": 10}) + self.assertEqual(body["params"]["differing"], ["lr"]) + self.assertEqual(body["params"]["values"]["lr"], {"run-a": 0.1, "run-b": 0.001}) + + def test_all_sections_are_present(self): + """params, metrics and tags each come back diffed.""" + body = self.compare_ok({"env_ids": ["run-a", "run-b"]}) + self.assertEqual(body["metrics"]["values"]["acc"]["run-b"], 0.95) + self.assertEqual(body["tags"]["differing"], ["dataset"]) + self.assertEqual( + body["tags"]["values"]["dataset"], {"run-a": "mnist", "run-b": "cifar10"} + ) + + def test_experiments_are_returned_in_full(self): + """The compared runs are echoed as full experiment dicts.""" + body = self.compare_ok({"env_ids": ["run-a"]}) + experiment = body["experiments"][0] + self.assertEqual(experiment["name"], "alpha") + self.assertEqual(experiment["params"][0]["key"], "lr") + + def test_compare_by_query(self): + """A query selects the runs to compare.""" + body = self.compare_ok({"query": "lr > 0.05", "descending": False}) + self.assertEqual(body["env_ids"], ["run-a", "run-c"]) + + def test_query_with_limit(self): + """limit caps how many matches are compared.""" + body = self.compare_ok({"query": "lr > 0.0001", "limit": 2}) + self.assertEqual(len(body["env_ids"]), 2) + + def test_query_matching_nothing_is_200(self): + """An empty comparison is a valid answer.""" + body = self.compare_ok({"query": "lr > 100"}) + self.assertEqual(body["env_ids"], []) + + def test_unknown_env_id_is_404(self): + """A run that has no experiment is named in the error.""" + resp = self.compare({"env_ids": ["run-a", "nope"]}) + self.assertEqual(resp.code, 404) + self.assertIn("nope", resp.reason) + + def test_both_selectors_is_400(self): + """Both selectors together is refused, not silently resolved.""" + resp = self.compare({"env_ids": ["run-a"], "query": "lr > 0"}) + self.assertEqual(resp.code, 400) + self.assertIn("not both", resp.reason) + + def test_neither_selector_is_400(self): + resp = self.compare({}) + self.assertEqual(resp.code, 400) + self.assertIn("required", resp.reason) + + def test_explicit_nulls_are_treated_as_absent(self): + """The client always sends both keys, one of them null.""" + body = self.compare_ok({"env_ids": ["run-a"], "query": None}) + self.assertEqual(body["env_ids"], ["run-a"]) + body = self.compare_ok({"env_ids": None, "query": "lr = 0.5"}) + self.assertEqual(body["env_ids"], ["run-c"]) + + def test_empty_env_ids_is_400(self): + resp = self.compare({"env_ids": []}) + self.assertEqual(resp.code, 400) + + def test_string_env_ids_is_400(self): + """A bare string is rejected rather than iterated into characters.""" + resp = self.compare({"env_ids": "run-a"}) + self.assertEqual(resp.code, 400) + self.assertIn("env_ids", resp.reason) + + def test_non_string_env_id_is_400(self): + self.assertEqual(self.compare({"env_ids": ["run-a", 7]}).code, 400) + + def test_invalid_query_is_400(self): + """A malformed query is the caller's error, and says why.""" + resp = self.compare({"query": "lr <"}) + self.assertEqual(resp.code, 400) + self.assertIn("end of query", resp.reason) + + def test_non_boolean_descending_is_400(self): + """The string "false" is rejected rather than coerced to true.""" + self.assertEqual( + self.compare({"query": "lr > 0", "descending": "false"}).code, 400 + ) + + def test_negative_limit_is_400(self): + self.assertEqual(self.compare({"query": "lr > 0", "limit": -1}).code, 400) + + def test_non_string_sort_by_is_400(self): + self.assertEqual(self.compare({"query": "lr > 0", "sort_by": 7}).code, 400) + + def test_injection_payload_is_inert(self): + """A SQL-ish payload is a parse error or a plain string compare, not an act.""" + resp = self.compare({"query": "name = 'x'; DROP TABLE experiments'"}) + self.assertIn(resp.code, (200, 400)) + self.assertEqual(len(self.compare_ok({"query": "lr > 0.0001"})["env_ids"]), 3) + + def test_compare_sees_an_experiment_logged_over_http(self): + """An experiment logged through /experiments/log is comparable at once.""" + self.fetch( + "/experiments/log", + method="POST", + body=json.dumps({"eid": "run-e", "action": "log", "params": {"lr": 0.1}}), + headers={"Content-Type": "application/json"}, + ) + body = self.compare_ok({"env_ids": ["run-a", "run-e"]}) + self.assertEqual(body["params"]["shared"], {"lr": 0.1}) + + +class TestCompareClientMessage(unittest.TestCase): + """Visdom.compare_experiments builds the message the endpoint expects.""" + + def setUp(self): + self.vis = Visdom(send=False, raise_exceptions=True) + + def test_compare_by_env_ids_message_shape(self): + msg, endpoint = self.vis.compare_experiments(["run-a", "run-b"]) + self.assertEqual(endpoint, "experiments/compare") + self.assertEqual(msg["env_ids"], ["run-a", "run-b"]) + self.assertIsNone(msg["query"]) + self.assertIsNone(msg["limit"]) + + def test_compare_by_query_message_shape(self): + msg, _ = self.vis.compare_experiments(query="lr < 0.01", limit=10) + self.assertIsNone(msg["env_ids"]) + self.assertEqual(msg["query"], "lr < 0.01") + self.assertEqual(msg["limit"], 10) + self.assertTrue(msg["descending"]) + + def test_tuple_env_ids_is_sent_as_a_list(self): + """A tuple is a fine way to name runs, but JSON only has arrays.""" + msg, _ = self.vis.compare_experiments(("run-a", "run-b")) + self.assertEqual(msg["env_ids"], ["run-a", "run-b"]) + + def test_client_rejects_bad_types(self): + """The client type-checks before any request is made.""" + with self.assertRaises(TypeError): + self.vis.compare_experiments("run-a") + with self.assertRaises(TypeError): + self.vis.compare_experiments([1, 2]) + with self.assertRaises(TypeError): + self.vis.compare_experiments(query=42) + with self.assertRaises(TypeError): + self.vis.compare_experiments(query="lr > 0", sort_by=42) + + def test_client_rejects_both_or_neither_selector(self): + with self.assertRaises(ValueError): + self.vis.compare_experiments(["run-a"], query="lr > 0") + with self.assertRaises(ValueError): + self.vis.compare_experiments() + + +if __name__ == "__main__": + unittest.main() diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index c88b9bcfe..479f81f49 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -1244,6 +1244,54 @@ def search_experiments( "experiments/search", ) + def compare_experiments( + self, env_ids=None, query=None, limit=None, sort_by=None, descending=True + ): + """Compare experiments field by field, to see what differs between runs. + + Choose the runs either by name or by search — one or the other, not both: + + vis.compare_experiments(["run-a", "run-b"]) + vis.compare_experiments(query="lr < 0.01", limit=10) + + With `env_ids` the runs are compared in the order given and each must + exist. With `query` the syntax is :meth:`search_experiments`', ordered by + `sort_by`/`descending` and capped at `limit` (`None` for all of them); + the comparison then describes exactly the runs the query selected. + + Returns the server's reply as a dict: the compared runs (`env_ids` and + the full `experiments`), plus a `params`, `metrics` and `tags` section. + Each section holds the union of `fields`, the `shared` ones every run + agrees on, the `differing` rest, and the per-run `values`:: + + cmp = vis.compare_experiments(["run-a", "run-b"]) + cmp["params"]["differing"] # ['lr'] + cmp["params"]["values"]["lr"] # {'run-a': 0.1, 'run-b': 0.001} + """ + if env_ids is not None: + if isstr(env_ids) or not isinstance(env_ids, (list, tuple)): + raise TypeError("env_ids must be a list of environment ids") + if not all(isstr(env_id) for env_id in env_ids): + raise TypeError("env_ids must contain strings") + if query is not None and not isstr(query): + raise TypeError("query must be a string") + if sort_by is not None and not isstr(sort_by): + raise TypeError("sort_by must be a string") + if env_ids is not None and query is not None: + raise ValueError("pass either env_ids or query, not both") + if env_ids is None and query is None: + raise ValueError("one of env_ids or query is required") + return self._experiment_request( + { + "env_ids": list(env_ids) if env_ids is not None else None, + "query": query, + "limit": limit, + "sort_by": sort_by, + "descending": descending, + }, + "experiments/compare", + ) + def get_window_data(self, win=None, env=None): """ This function returns all the window data for a specified window in diff --git a/py/visdom/experiments/__init__.py b/py/visdom/experiments/__init__.py index 78a1c519f..88f5e6e48 100644 --- a/py/visdom/experiments/__init__.py +++ b/py/visdom/experiments/__init__.py @@ -6,6 +6,11 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. +from visdom.experiments.compare import ( + ComparableExperiment, + SECTIONS, + build_comparison, +) from visdom.experiments.models import ( Experiment, ExperimentFinishedError, @@ -32,6 +37,7 @@ __all__ = [ "And", + "ComparableExperiment", "Comparison", "DEFAULT_SORT_FIELD", "Experiment", @@ -44,7 +50,9 @@ "Param", "Query", "QueryParseError", + "SECTIONS", "Tag", + "build_comparison", "build_record", "parse_query", "STATUS_FAILED", diff --git a/py/visdom/experiments/compare.py b/py/visdom/experiments/compare.py new file mode 100644 index 000000000..a78dd5855 --- /dev/null +++ b/py/visdom/experiments/compare.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, The Visdom Authors +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Diffing a set of experiments field by field. + +Where :mod:`~visdom.experiments.query` flattens one experiment into a record to +match a predicate against, this module lines several experiments up beside each +other and reports, per section (params/metrics/tags), which fields they agree on +and which they do not. It is pure: it reads experiment objects and returns a +JSON-serialisable dict, touching no storage. :meth:`ExperimentStore.compare` +selects the experiments and delegates the arithmetic here. + +The interesting output is ``differing``: given a dozen runs of the same model, +it is the short list of knobs that actually changed between them. +""" + +from __future__ import annotations + +import math +from typing import Any, Iterable, Protocol, Sequence + +from visdom.experiments.query import ExperimentLike + + +class ComparableExperiment(ExperimentLike, Protocol): + """An :class:`ExperimentLike` that can also serialise itself. + + :func:`build_comparison` echoes the compared experiments back in full, which + the query layer's protocol alone does not cover. + """ + + def to_dict(self) -> dict: + ... + + +def _param_values(experiment: ComparableExperiment) -> dict: + """Return ``{name: value}`` for the experiment's params.""" + return {param.key: param.value for param in experiment.params} + + +def _metric_values(experiment: ComparableExperiment) -> dict: + """Return ``{name: latest value}`` for the experiment's metrics. + + Metrics are a time series, but a comparison wants one number per run, so the + most recent observation wins — the same value + :func:`~visdom.experiments.query.build_record` compares on, so a run found by + ``acc > 0.9`` shows that same ``acc`` here. + """ + latest: dict[str, Any] = {} + for metric in reversed(experiment.metrics): + latest.setdefault(metric.key, metric.value) + return latest + + +def _tag_values(experiment: ComparableExperiment) -> dict: + """Return ``{name: value}`` for the experiment's tags.""" + return {tag.key: tag.value for tag in experiment.tags} + + +_SECTION_READERS = { + "params": _param_values, + "metrics": _metric_values, + "tags": _tag_values, +} + +SECTIONS = tuple(_SECTION_READERS) +"""The section names :func:`build_comparison` diffs, in the order it emits them.""" + + +def _same_value(a: Any, b: Any) -> bool: + """Return ``True`` if two logged values should count as the same. + + Stricter than ``==`` about bools, because ``True == 1`` in Python and a run + launched with ``amp=True`` did not use the same setting as one launched with + ``amp=1``; :mod:`~visdom.experiments.query` draws the same line by refusing to + treat a bool as a number. Looser than ``==`` about NaN, which is never equal + to itself: a metric that is NaN in every run agrees across them, and calling + that a difference would bury the real ones. + """ + if isinstance(a, bool) != isinstance(b, bool): + return False + if isinstance(a, float) and isinstance(b, float): + if math.isnan(a) and math.isnan(b): + return True + return bool(a == b) + + +def _compare_section(per_env: dict) -> dict: + """Diff one section across the experiments in ``per_env``. + + ``per_env`` maps env_id to that experiment's ``{field: value}`` for the + section. A field counts as shared only when every experiment carries it *and* + they all agree: a value one run never logged is a difference between the runs, + not a consensus among those that happen to have it. + """ + fields = sorted({field for values in per_env.values() for field in values}) + values = { + field: { + env_id: env_values[field] + for env_id, env_values in per_env.items() + if field in env_values + } + for field in fields + } + shared = {} + for field in fields: + present = values[field] + if len(present) != len(per_env): + continue + found = list(present.values()) + if all(_same_value(found[0], other) for other in found[1:]): + shared[field] = found[0] + return { + "fields": fields, + "shared": shared, + "differing": [field for field in fields if field not in shared], + "values": values, + } + + +def build_comparison(experiments: Iterable[ComparableExperiment]) -> dict: + """Compare ``experiments`` field by field and return the result as a dict. + + The reply echoes the compared runs (``env_ids`` in the order given, and the + full ``experiments``) alongside one diff per section:: + + { + "env_ids": ["run-a", "run-b"], + "experiments": [{...}, {...}], + "params": { + "fields": ["epochs", "lr"], + "shared": {"epochs": 10}, + "differing": ["lr"], + "values": {"lr": {"run-a": 0.1, "run-b": 0.001}, + "epochs": {"run-a": 10, "run-b": 10}} + }, + "metrics": {...}, "tags": {...} + } + + ``fields`` is every name any run has, sorted; ``shared`` holds the fields all + runs carry with the same value; ``differing`` is the rest — the ones that vary + or that some run is missing; ``values`` gives the raw per-run value, omitting + the runs that never logged the field. Comparing a single experiment is legal + and puts everything it has in ``shared``; comparing none yields empty + sections. + """ + ordered: Sequence[ComparableExperiment] = list(experiments) + comparison = { + "env_ids": [experiment.env_id for experiment in ordered], + "experiments": [experiment.to_dict() for experiment in ordered], + } + for section, read in _SECTION_READERS.items(): + comparison[section] = _compare_section( + {experiment.env_id: read(experiment) for experiment in ordered} + ) + return comparison diff --git a/py/visdom/experiments/store.py b/py/visdom/experiments/store.py index 2246bbb51..beb6c42cf 100644 --- a/py/visdom/experiments/store.py +++ b/py/visdom/experiments/store.py @@ -17,6 +17,7 @@ """ from visdom.data_model.base import DataStore +from visdom.experiments.compare import build_comparison from visdom.experiments.models import ( Experiment, ExperimentFinishedError, @@ -207,6 +208,87 @@ def search(self, query=None, sort_by=DEFAULT_SORT_FIELD, descending=True): pairs = _sort_pairs(pairs, sort_by, descending) return [experiment for _, experiment in pairs] + def _load_named(self, env_ids): + """Return the experiments for ``env_ids``, in the order asked for. + + Duplicate ids collapse to their first occurrence: a comparison is keyed + by env_id, so a run named twice cannot mean anything beyond once. + """ + if isinstance(env_ids, str) or not isinstance(env_ids, (list, tuple)): + raise TypeError( + "env_ids must be a list of environment ids, got {0}".format( + type(env_ids).__name__ + ) + ) + unique = list(dict.fromkeys(env_ids)) + for env_id in unique: + if not isinstance(env_id, str): + raise TypeError( + "env_ids must contain strings, got {0}".format( + type(env_id).__name__ + ) + ) + if not unique: + raise ValueError("env_ids must name at least one environment") + experiments = [] + missing = [] + for env_id in unique: + experiment = self.get_experiment(env_id) + if experiment is None: + missing.append(env_id) + else: + experiments.append(experiment) + if missing: + raise KeyError( + "no experiment logged for env(s) {0}".format( + ", ".join(repr(env_id) for env_id in missing) + ) + ) + return experiments + + def compare( + self, + env_ids=None, + query=None, + sort_by=DEFAULT_SORT_FIELD, + descending=True, + limit=None, + ): + """Compare experiments field by field; see :func:`build_comparison`. + + The experiments are chosen either by name or by search, and the two are + mutually exclusive — passing both, or neither, is a ``ValueError`` rather + than a guess at which was meant: + + * ``env_ids`` — an explicit list, compared in the order given. Every id + must have an experiment; a :class:`KeyError` names those that do not, + since a comparison silently missing a run it was asked for would be + read as a comparison of the rest. + * ``query`` — compare everything the query matches, ordered by + ``sort_by``/``descending`` as in :meth:`search` and capped at ``limit`` + (``None`` = uncapped). A query matching nothing compares nothing and + yields empty sections; that is an empty answer, not an error. + + ``limit`` applies only to ``query`` selection — an explicit ``env_ids`` + list is already the exact set to compare. Note that the diff describes + the runs actually compared, so a ``limit`` that truncates the matches + narrows what ``shared``/``differing`` are computed over; the returned + ``env_ids`` always say which runs those were. + """ + if env_ids is not None and query is not None: + raise ValueError("pass either env_ids or query, not both") + if env_ids is None and query is None: + raise ValueError("one of env_ids or query is required") + if env_ids is not None: + experiments = self._load_named(env_ids) + else: + experiments = self.search( + query=query, sort_by=sort_by, descending=descending + ) + if limit is not None: + experiments = experiments[:limit] + return build_comparison(experiments) + def delete_experiment(self, env_id): """Drop the experiment blob from ``env_id`` (keeping the env itself). diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index 0e2e146f7..a7acbdb74 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -36,6 +36,7 @@ EnvStateHandler, ErrorHandler, ExistsHandler, + ExperimentCompareHandler, ExperimentLogHandler, ExperimentSearchHandler, ForkEnvHandler, @@ -132,6 +133,11 @@ def __init__( ExperimentSearchHandler, {"app": self}, ), + ( + r"%s/experiments/compare" % self.base_url, + ExperimentCompareHandler, + {"app": self}, + ), (r"%s/user/(.*)" % self.base_url, UserSettingsHandler, {"app": self}), (r"%s/health" % self.base_url, HealthHandler), (r"%s(.*)" % self.base_url, IndexHandler, {"app": self}), diff --git a/py/visdom/server/handlers/web_handlers.py b/py/visdom/server/handlers/web_handlers.py index 975107a1e..c3a95ffb3 100644 --- a/py/visdom/server/handlers/web_handlers.py +++ b/py/visdom/server/handlers/web_handlers.py @@ -819,6 +819,49 @@ def post(self): ) +def _require_index(args, field, default): + """Return ``args[field]`` as a non-negative int (``None`` = unbounded). + + A JSON body has no int/float distinction, so a client that sends ``10.0`` + means the index 10; anything with a fractional part is a mistake. + """ + value = args.get(field, default) + if value is None: + return None + if isinstance(value, float) and value.is_integer(): + value = int(value) + if isinstance(value, bool) or not isinstance(value, int): + raise tornado.web.HTTPError( + 400, reason="'{0}' must be an integer".format(field) + ) + if value < 0: + raise tornado.web.HTTPError( + 400, reason="'{0}' must not be negative".format(field) + ) + return value + + +def _require_text(args, field): + """Return ``args[field]`` if it is a string (or absent); else raise 400.""" + value = args.get(field) + if value is not None and not isinstance(value, str): + raise tornado.web.HTTPError(400, reason="'{0}' must be a string".format(field)) + return value + + +def _require_flag(args, field, default): + """Return ``args[field]`` as a bool; else raise 400. + + Deliberately not ``bool(value)``: JSON has real booleans, so a client + sending the *string* ``"false"`` means false, and coercing it would + truthily flip the result to its opposite without a word. + """ + value = args.get(field, default) + if not isinstance(value, bool): + raise tornado.web.HTTPError(400, reason="'{0}' must be a boolean".format(field)) + return value + + class ExperimentLogHandler(BaseHandler): """POST ``/experiments/log`` — record experiment metadata for an environment. @@ -937,62 +980,13 @@ class ExperimentSearchHandler(BaseHandler): DEFAULT_LIMIT = 100 - @staticmethod - def _require_index(args, field, default): - """Return ``args[field]`` as a non-negative int (``None`` = unbounded). - - A JSON body has no int/float distinction, so a client that sends ``10.0`` - means the index 10; anything with a fractional part is a mistake. - """ - value = args.get(field, default) - if value is None: - return None - if isinstance(value, float) and value.is_integer(): - value = int(value) - if isinstance(value, bool) or not isinstance(value, int): - raise tornado.web.HTTPError( - 400, reason="'{0}' must be an integer".format(field) - ) - if value < 0: - raise tornado.web.HTTPError( - 400, reason="'{0}' must not be negative".format(field) - ) - return value - - @staticmethod - def _require_text(args, field): - """Return ``args[field]`` if it is a string (or absent); else raise 400.""" - value = args.get(field) - if value is not None and not isinstance(value, str): - raise tornado.web.HTTPError( - 400, reason="'{0}' must be a string".format(field) - ) - return value - - @staticmethod - def _require_flag(args, field, default): - """Return ``args[field]`` as a bool; else raise 400. - - Deliberately not ``bool(value)``: JSON has real booleans, so a client - sending the *string* ``"false"`` means false, and coercing it would - truthily flip the result to its opposite without a word. - """ - value = args.get(field, default) - if not isinstance(value, bool): - raise tornado.web.HTTPError( - 400, reason="'{0}' must be a boolean".format(field) - ) - return value - @staticmethod def wrap_func(handler, args): - query = ExperimentSearchHandler._require_text(args, "query") - sort_by = ExperimentSearchHandler._require_text(args, "sort_by") - limit = ExperimentSearchHandler._require_index( - args, "limit", ExperimentSearchHandler.DEFAULT_LIMIT - ) - offset = ExperimentSearchHandler._require_index(args, "offset", 0) - descending = ExperimentSearchHandler._require_flag(args, "descending", True) + query = _require_text(args, "query") + sort_by = _require_text(args, "sort_by") + limit = _require_index(args, "limit", ExperimentSearchHandler.DEFAULT_LIMIT) + offset = _require_index(args, "offset", 0) + descending = _require_flag(args, "descending", True) store = ExperimentStore(handler.storage) try: @@ -1028,6 +1022,97 @@ def post(self): self.wrap_func(self, args) +class ExperimentCompareHandler(BaseHandler): + """POST ``/experiments/compare`` — diff experiments field by field. + + Selects the experiments to compare either by name or by search, and the two + are mutually exclusive:: + + {"env_ids": ["run-a", "run-b"]} + {"query": "lr < 0.01", "limit": 10} + + Sending both is a 400 rather than a silent precedence rule, as is sending + neither. With ``env_ids``, every id must have an experiment — a 404 names the + ones that do not, since quietly comparing the remainder would answer a + question the caller did not ask. With ``query``, the syntax and the + ``sort_by``/``descending``/``limit`` handling are exactly + :class:`ExperimentSearchHandler`'s, and a query matching nothing is an empty + comparison rather than an error. + + The reply carries the compared runs (``env_ids``, ``experiments``) and a diff + per section, each listing the union of ``fields``, the ``shared`` ones every + run agrees on, the ``differing`` rest, and the per-run ``values``:: + + {"env_ids": [...], "experiments": [...], + "params": {"fields": [...], "shared": {...}, + "differing": [...], "values": {...}}, + "metrics": {...}, "tags": {...}} + + Experiments are read through the server's ``DataStore``, so as with search a + server running with ``env_path=None`` has nothing to compare. + """ + + @staticmethod + def _require_env_ids(args): + """Return ``args["env_ids"]`` as a list of ids, or ``None`` if absent. + + A bare string is rejected rather than treated as a one-id list: it would + otherwise be iterated character by character into a comparison of runs + named ``"r"``, ``"u"``, ``"n"``. + """ + value = args.get("env_ids") + if value is None: + return None + if not isinstance(value, list): + raise tornado.web.HTTPError(400, reason="'env_ids' must be a list of ids") + if not value: + raise tornado.web.HTTPError( + 400, reason="'env_ids' must name at least one environment" + ) + if not all(isinstance(env_id, str) for env_id in value): + raise tornado.web.HTTPError(400, reason="'env_ids' must contain strings") + return value + + @staticmethod + def wrap_func(handler, args): + env_ids = ExperimentCompareHandler._require_env_ids(args) + query = _require_text(args, "query") + if env_ids is not None and query is not None: + raise tornado.web.HTTPError( + 400, reason="pass either 'env_ids' or 'query', not both" + ) + if env_ids is None and query is None: + raise tornado.web.HTTPError( + 400, reason="one of 'env_ids' or 'query' is required" + ) + sort_by = _require_text(args, "sort_by") + limit = _require_index(args, "limit", None) + descending = _require_flag(args, "descending", True) + + store = ExperimentStore(handler.storage) + try: + comparison = store.compare( + env_ids=env_ids, + query=query, + sort_by=sort_by or DEFAULT_SORT_FIELD, + descending=descending, + limit=limit, + ) + except QueryParseError as e: + raise tornado.web.HTTPError(400, reason=str(e)) + except KeyError as e: + raise tornado.web.HTTPError(404, reason=str(e.args[0])) + + handler.write(json.dumps(comparison, cls=NanSafeEncoder)) + + @check_auth + def post(self): + args = tornado.escape.json_decode( + tornado.escape.to_basestring(self.request.body) + ) + self.wrap_func(self, args) + + class HealthHandler(BaseHandler): def get(self): self.write({"status": "ok"}) From 5599953a90d9abfda623394abfc145b249a92c67 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 15 Jul 2026 17:17:05 +0530 Subject: [PATCH 07/48] drop the inline annotations from the compare docstring Fold the example's trailing # annotations into the prose instead, matching the experiments package style. No behaviour change. --- py/visdom/__init__.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index 479f81f49..7b15bf99e 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -1262,11 +1262,10 @@ def compare_experiments( Returns the server's reply as a dict: the compared runs (`env_ids` and the full `experiments`), plus a `params`, `metrics` and `tags` section. Each section holds the union of `fields`, the `shared` ones every run - agrees on, the `differing` rest, and the per-run `values`:: - - cmp = vis.compare_experiments(["run-a", "run-b"]) - cmp["params"]["differing"] # ['lr'] - cmp["params"]["values"]["lr"] # {'run-a': 0.1, 'run-b': 0.001} + agrees on, the `differing` rest, and the per-run `values`. Comparing two + runs that differ only in learning rate gives a `params` section whose + `differing` is `['lr']` and whose `values['lr']` is + `{'run-a': 0.1, 'run-b': 0.001}`. """ if env_ids is not None: if isstr(env_ids) or not isinstance(env_ids, (list, tuple)): From 6f28f6eea604eda988404ca7887d029ee8c40128 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 15 Jul 2026 17:20:48 +0530 Subject: [PATCH 08/48] type the search and compare client APIs in __init__.pyi The stubs cover experiment/log_metrics/finish_experiment but stop there, so search_experiments (L2-4) and compare_experiments (L2-5) were the two client methods a type checker could not see. env_ids is spelled Union[List[Text], Tuple[Text, ...]] rather than the obvious Sequence[Text]: a bare str is itself a Sequence[str], and compare_experiments rejects one at runtime, so Sequence would have a checker bless the very call the client raises TypeError on. A tuple is accepted, hence the union rather than a plain List. Both replies are the decoded JSON of their endpoint, named _ExperimentReply for the Mapping the existing experiment stubs return. Verified the stub parameter names and defaults against inspect.signature of the real methods; no type checker is configured in this repo, so nothing else guards the drift. --- py/visdom/__init__.pyi | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/py/visdom/__init__.pyi b/py/visdom/__init__.pyi index 37635d44d..c42a0826a 100644 --- a/py/visdom/__init__.pyi +++ b/py/visdom/__init__.pyi @@ -4,7 +4,7 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. -from typing import Optional, List, Any, Union, Mapping, overload, Text, Callable +from typing import Optional, List, Any, Union, Mapping, overload, Text, Tuple, Callable ### Type aliases for commonly-used types. # For optional 'options' parameters. @@ -13,6 +13,14 @@ from typing import Optional, List, Any, Union, Mapping, overload, Text, Callable _OptOps = Optional[Mapping[Text, Any]] _OptStr = Optional[Text] # For optional string parameters, like 'window' and 'env'. +# For the list of environments to compare. Spelled out rather than Sequence[Text] +# because a bare str is itself a Sequence[str]: 'compare_experiments' rejects one +# at runtime, so a checker must not accept it here. +_EnvIds = Optional[Union[List[Text], Tuple[Text, ...]]] + +# The decoded JSON reply of the experiment endpoints. +_ExperimentReply = Mapping[Text, Any] + # No widely-deployed stubs exist at the moment for torch or numpy. When they are available, the correct type of the tensor-like inputs # to the plotting commands should be # Tensor = Union[torch.Tensor, numpy.ndarray, List] @@ -63,6 +71,22 @@ class Visdom: def finish_experiment( self, status: Text = ..., env: _OptStr = ... ) -> Mapping[Text, Any]: ... + def search_experiments( + self, + query: _OptStr = ..., + limit: Optional[int] = ..., + offset: int = ..., + sort_by: _OptStr = ..., + descending: bool = ..., + ) -> _ExperimentReply: ... + def compare_experiments( + self, + env_ids: _EnvIds = ..., + query: _OptStr = ..., + limit: Optional[int] = ..., + sort_by: _OptStr = ..., + descending: bool = ..., + ) -> _ExperimentReply: ... def get_window_data( self, win: _OptStr = ..., env: _OptStr = ... ) -> _SendReturn: ... From eb933d39513a77824f0841809598328fa1442f30 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 15 Jul 2026 17:53:28 +0530 Subject: [PATCH 09/48] drop query selection from compare, leave it to search Compare took either env_ids or a query. The query mode was redundant: search already answers "which runs match?", so compare's copy of it was a second way to do the same thing, reachable only by duplicating search's syntax, sorting and paging into a second endpoint. It also carried a caveat that could mislead. limit truncated the compared set, so shared/differing were computed over only the runs that survived the cap -- correct, but easily read as a diff of everything matching. Compare is now purely "diff these runs": compare(env_ids). To compare a query's matches, search first and pass the ids on, which is one extra call and keeps the diff honest -- it always describes exactly the runs named. The two endpoints now have one job each. Removed with it: the mutual-exclusion checks (nothing to be exclusive with), sort_by/descending/limit on compare, and the QueryParseError path. env_ids is now required, so a missing one is a 400 rather than a fallback to query. The three request validators move back from module level to ExperimentSearchHandler statics, since compare no longer shares them -- web_handlers.py is byte-identical to L2-4 apart from the new handler. Client compare_experiments(env_ids) loses its optional selection knobs; __init__.pyi follows, and _EnvIds drops its Optional now that env_ids is required. Tests: the query-mode cases go; added that search-then-compare composes, that a stale caller still sending query/limit is ignored rather than a 500, and that a traversal env id degrades to 404 rather than a file read (JSONStore._primary_path already guards this) -- the latter replacing the injection test, whose parser surface compare no longer has. 398 py/tests pass. Verified live against a real server and client: compare, search-then-compare, the 400/404 paths, and the traversal id. --- openapi.yaml | 55 ++------ py/tests/test_experiment_compare.py | 158 ++++++---------------- py/visdom/__init__.py | 45 ++---- py/visdom/__init__.pyi | 11 +- py/visdom/experiments/store.py | 55 ++------ py/visdom/server/handlers/web_handlers.py | 147 +++++++++----------- 6 files changed, 150 insertions(+), 321 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 48bf5652b..27e7e2e97 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -580,14 +580,13 @@ paths: not — the short list of knobs that actually changed between runs. - The runs to compare are selected either by name (`env_ids`) or by search - (`query`), and the two are mutually exclusive: sending both, or neither, - is a `400` rather than a silent guess at which was meant. With `env_ids` - the runs are compared in the order given and every id must have an - experiment, otherwise `404`. With `query` the syntax and the - `sort_by`/`descending`/`limit` handling are those of - `/experiments/search`, and a query matching nothing is an empty - comparison rather than an error. + The runs are named by `env_ids` and compared in the order given; every id + must have an experiment, otherwise `404`. + + + Finding the runs is `/experiments/search`'s job: it answers "which runs + match?", this answers "how do these runs differ?". To compare a query's + matches, search first and pass the resulting ids on. Metrics are a time series, so a comparison uses each metric's latest @@ -602,43 +601,16 @@ paths: application/json: schema: type: object + required: [env_ids] properties: env_ids: type: array - nullable: true minItems: 1 description: > - Environments to compare, in the order given. Mutually - exclusive with `query`. Duplicate ids collapse to one. + Environments to compare, in the order given. Duplicate ids + collapse to one. items: type: string - query: - type: string - nullable: true - description: > - Compare every experiment this filter matches, in the syntax - of `/experiments/search`. Mutually exclusive with `env_ids`. - sort_by: - type: string - nullable: true - default: created_at - description: > - Field ordering the compared runs (`query` selection only). - descending: - type: boolean - default: true - description: Sort direction; newest/highest first by default. - limit: - type: integer - nullable: true - minimum: 0 - default: null - description: > - Maximum number of runs to compare (`query` selection only; - `null` compares every match). Note that the diff describes - the runs actually compared, so a limit that truncates the - matches narrows what `shared`/`differing` are computed - over; the returned `env_ids` say which runs those were. responses: "200": description: The comparison of the selected experiments. @@ -666,10 +638,9 @@ paths: $ref: "#/components/schemas/ExperimentComparisonSection" "400": description: > - Invalid request — both `env_ids` and `query` given, neither given, - an empty or non-string `env_ids`, malformed `query` syntax, or a - `limit` that is not a non-negative integer. Also returned when - authentication is required but not provided. + Invalid request — `env_ids` missing, empty, not a list, or holding + something other than strings. Also returned when authentication is + required but not provided. "404": description: One or more of the given `env_ids` has no experiment. diff --git a/py/tests/test_experiment_compare.py b/py/tests/test_experiment_compare.py index aae54f66a..745c507b5 100644 --- a/py/tests/test_experiment_compare.py +++ b/py/tests/test_experiment_compare.py @@ -2,7 +2,7 @@ Covers the four pieces the compare layer is built from: the pure ``build_comparison`` diff over experiment objects; ``ExperimentStore.compare`` -selecting runs by name or by query against a real ``JSONStore`` over a temporary +over the named runs against a real ``JSONStore`` over a temporary directory; the ``/experiments/compare`` endpoint end-to-end through a real :class:`~visdom.server.app.Application` with Tornado's ``AsyncHTTPTestCase``; and the ``Visdom.compare_experiments`` message shape with ``send=False`` (no server). @@ -17,12 +17,7 @@ from visdom import Visdom from visdom.data_model import JSONStore -from visdom.experiments import ( - Experiment, - ExperimentStore, - QueryParseError, - build_comparison, -) +from visdom.experiments import Experiment, ExperimentStore, build_comparison from visdom.server.app import Application @@ -179,7 +174,7 @@ def test_sections_are_independent(self): class TestStoreCompare(unittest.TestCase): - """ExperimentStore.compare selects runs by name or by query.""" + """ExperimentStore.compare diffs the runs it is given by name.""" def setUp(self): self._tmp_dir = tempfile.mkdtemp(prefix="visdom_exp_compare_") @@ -236,52 +231,20 @@ def test_non_string_env_id_raises_type_error(self): with self.assertRaises(TypeError): self.store.compare(env_ids=["run-a", 7]) - def test_compare_by_query(self): - """A query selects the runs to compare.""" - comparison = self.store.compare(query="lr > 0.05") - self.assertEqual(sorted(comparison["env_ids"]), ["run-a", "run-c"]) - self.assertEqual(comparison["params"]["differing"], ["epochs", "lr"]) + def test_searching_then_comparing_the_matches(self): + """Comparing a query's matches is search's job, then compare's. - def test_query_selection_is_sorted(self): - """sort_by/descending order the compared runs as search does.""" - comparison = self.store.compare(query="lr > 0.05", descending=False) + The two compose: search answers which runs match, compare answers how + they differ. This is the path that replaced compare's own query mode. + """ + found = self.store.search(query="lr > 0.05", descending=False) + comparison = self.store.compare([e.env_id for e in found]) self.assertEqual(comparison["env_ids"], ["run-a", "run-c"]) - comparison = self.store.compare(query="lr > 0.05", sort_by="lr") - self.assertEqual(comparison["env_ids"], ["run-c", "run-a"]) - - def test_query_limit_caps_the_compared_set(self): - """limit narrows which runs are compared, and env_ids says which.""" - comparison = self.store.compare(query="lr > 0.0001", limit=2) - self.assertEqual(comparison["env_ids"], ["run-c", "run-b"]) - self.assertEqual(len(comparison["experiments"]), 2) - - def test_query_matching_nothing_compares_nothing(self): - """An empty answer to a valid question is not an error.""" - comparison = self.store.compare(query="lr > 100") - self.assertEqual(comparison["env_ids"], []) - self.assertEqual(comparison["params"]["fields"], []) - - def test_invalid_query_raises_parse_error(self): - with self.assertRaises(QueryParseError): - self.store.compare(query="lr <") - - def test_both_selectors_raises_value_error(self): - """Passing both is ambiguous, so refuse rather than pick one.""" - with self.assertRaises(ValueError): - self.store.compare(env_ids=["run-a"], query="lr > 0") - - def test_neither_selector_raises_value_error(self): - with self.assertRaises(ValueError): - self.store.compare() - - def test_limit_is_ignored_for_env_ids(self): - """An explicit list is already the exact set; limit has nothing to cap.""" - comparison = self.store.compare(env_ids=["run-a", "run-b"], limit=1) - self.assertEqual(comparison["env_ids"], ["run-a", "run-b"]) + self.assertEqual(comparison["params"]["differing"], ["epochs", "lr"]) class TestCompareEndpoint(tornado.testing.AsyncHTTPTestCase): - """POST /experiments/compare diffs the selected experiments.""" + """POST /experiments/compare diffs the named experiments.""" def setUp(self): self._tmp_dir = tempfile.mkdtemp(prefix="visdom_exp_compare_api_") @@ -323,49 +286,23 @@ def test_all_sections_are_present(self): def test_experiments_are_returned_in_full(self): """The compared runs are echoed as full experiment dicts.""" - body = self.compare_ok({"env_ids": ["run-a"]}) + body = self.compare_ok({"env_ids": ["run-a", "run-b"]}) experiment = body["experiments"][0] self.assertEqual(experiment["name"], "alpha") self.assertEqual(experiment["params"][0]["key"], "lr") - def test_compare_by_query(self): - """A query selects the runs to compare.""" - body = self.compare_ok({"query": "lr > 0.05", "descending": False}) - self.assertEqual(body["env_ids"], ["run-a", "run-c"]) - - def test_query_with_limit(self): - """limit caps how many matches are compared.""" - body = self.compare_ok({"query": "lr > 0.0001", "limit": 2}) - self.assertEqual(len(body["env_ids"]), 2) - - def test_query_matching_nothing_is_200(self): - """An empty comparison is a valid answer.""" - body = self.compare_ok({"query": "lr > 100"}) - self.assertEqual(body["env_ids"], []) - def test_unknown_env_id_is_404(self): """A run that has no experiment is named in the error.""" resp = self.compare({"env_ids": ["run-a", "nope"]}) self.assertEqual(resp.code, 404) self.assertIn("nope", resp.reason) - def test_both_selectors_is_400(self): - """Both selectors together is refused, not silently resolved.""" - resp = self.compare({"env_ids": ["run-a"], "query": "lr > 0"}) - self.assertEqual(resp.code, 400) - self.assertIn("not both", resp.reason) - - def test_neither_selector_is_400(self): + def test_missing_env_ids_is_400(self): + """env_ids is the only way to select runs, so it is required.""" resp = self.compare({}) self.assertEqual(resp.code, 400) self.assertIn("required", resp.reason) - - def test_explicit_nulls_are_treated_as_absent(self): - """The client always sends both keys, one of them null.""" - body = self.compare_ok({"env_ids": ["run-a"], "query": None}) - self.assertEqual(body["env_ids"], ["run-a"]) - body = self.compare_ok({"env_ids": None, "query": "lr = 0.5"}) - self.assertEqual(body["env_ids"], ["run-c"]) + self.assertEqual(self.compare({"env_ids": None}).code, 400) def test_empty_env_ids_is_400(self): resp = self.compare({"env_ids": []}) @@ -380,29 +317,26 @@ def test_string_env_ids_is_400(self): def test_non_string_env_id_is_400(self): self.assertEqual(self.compare({"env_ids": ["run-a", 7]}).code, 400) - def test_invalid_query_is_400(self): - """A malformed query is the caller's error, and says why.""" - resp = self.compare({"query": "lr <"}) - self.assertEqual(resp.code, 400) - self.assertIn("end of query", resp.reason) + def test_unknown_keys_are_ignored(self): + """A stale caller still sending query/limit is not an error, just ignored. - def test_non_boolean_descending_is_400(self): - """The string "false" is rejected rather than coerced to true.""" - self.assertEqual( - self.compare({"query": "lr > 0", "descending": "false"}).code, 400 + The endpoint took a `query` until compare's query mode was dropped in + favour of search; an old client's extra keys must not 500. + """ + body = self.compare_ok( + {"env_ids": ["run-a", "run-b"], "query": "lr > 0", "limit": 1} ) + self.assertEqual(body["env_ids"], ["run-a", "run-b"]) - def test_negative_limit_is_400(self): - self.assertEqual(self.compare({"query": "lr > 0", "limit": -1}).code, 400) - - def test_non_string_sort_by_is_400(self): - self.assertEqual(self.compare({"query": "lr > 0", "sort_by": 7}).code, 400) + def test_traversal_env_id_is_404_not_a_file_read(self): + """A crafted id cannot escape env_path; it simply names no experiment. - def test_injection_payload_is_inert(self): - """A SQL-ish payload is a parse error or a plain string compare, not an act.""" - resp = self.compare({"query": "name = 'x'; DROP TABLE experiments'"}) - self.assertIn(resp.code, (200, 400)) - self.assertEqual(len(self.compare_ok({"query": "lr > 0.0001"})["env_ids"]), 3) + JSONStore._primary_path resolves the id under env_path and rejects + anything that would climb out, so this degrades to "no such experiment" + rather than reading the filesystem. + """ + resp = self.compare({"env_ids": ["run-a", "../../../../etc/passwd"]}) + self.assertEqual(resp.code, 404) def test_compare_sees_an_experiment_logged_over_http(self): """An experiment logged through /experiments/log is comparable at once.""" @@ -422,19 +356,16 @@ class TestCompareClientMessage(unittest.TestCase): def setUp(self): self.vis = Visdom(send=False, raise_exceptions=True) - def test_compare_by_env_ids_message_shape(self): + def test_compare_message_shape(self): + """The message names the runs and carries no selection knobs. + + ``_send`` stamps an ``eid`` on every message; the endpoint ignores it. + """ msg, endpoint = self.vis.compare_experiments(["run-a", "run-b"]) self.assertEqual(endpoint, "experiments/compare") self.assertEqual(msg["env_ids"], ["run-a", "run-b"]) - self.assertIsNone(msg["query"]) - self.assertIsNone(msg["limit"]) - - def test_compare_by_query_message_shape(self): - msg, _ = self.vis.compare_experiments(query="lr < 0.01", limit=10) - self.assertIsNone(msg["env_ids"]) - self.assertEqual(msg["query"], "lr < 0.01") - self.assertEqual(msg["limit"], 10) - self.assertTrue(msg["descending"]) + for dropped in ("query", "limit", "sort_by", "descending"): + self.assertNotIn(dropped, msg) def test_tuple_env_ids_is_sent_as_a_list(self): """A tuple is a fine way to name runs, but JSON only has arrays.""" @@ -447,15 +378,10 @@ def test_client_rejects_bad_types(self): self.vis.compare_experiments("run-a") with self.assertRaises(TypeError): self.vis.compare_experiments([1, 2]) - with self.assertRaises(TypeError): - self.vis.compare_experiments(query=42) - with self.assertRaises(TypeError): - self.vis.compare_experiments(query="lr > 0", sort_by=42) - def test_client_rejects_both_or_neither_selector(self): - with self.assertRaises(ValueError): - self.vis.compare_experiments(["run-a"], query="lr > 0") - with self.assertRaises(ValueError): + def test_env_ids_is_required(self): + """There is no other way to select runs, so it cannot be omitted.""" + with self.assertRaises(TypeError): self.vis.compare_experiments() diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index 7b15bf99e..51e27ec6c 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -1244,20 +1244,20 @@ def search_experiments( "experiments/search", ) - def compare_experiments( - self, env_ids=None, query=None, limit=None, sort_by=None, descending=True - ): - """Compare experiments field by field, to see what differs between runs. + def compare_experiments(self, env_ids): + """Compare the named experiments field by field, to see what differs. - Choose the runs either by name or by search — one or the other, not both: + `env_ids` names the runs to compare, in the order given, and each must + exist: vis.compare_experiments(["run-a", "run-b"]) - vis.compare_experiments(query="lr < 0.01", limit=10) - With `env_ids` the runs are compared in the order given and each must - exist. With `query` the syntax is :meth:`search_experiments`', ordered by - `sort_by`/`descending` and capped at `limit` (`None` for all of them); - the comparison then describes exactly the runs the query selected. + Finding the runs is :meth:`search_experiments`' job — it answers "which + runs match?", this answers "how do these runs differ?". To compare a + query's matches, search first and pass the ids on: + + found = vis.search_experiments("lr < 0.01") + vis.compare_experiments([e["env_id"] for e in found["experiments"]]) Returns the server's reply as a dict: the compared runs (`env_ids` and the full `experiments`), plus a `params`, `metrics` and `tags` section. @@ -1267,27 +1267,12 @@ def compare_experiments( `differing` is `['lr']` and whose `values['lr']` is `{'run-a': 0.1, 'run-b': 0.001}`. """ - if env_ids is not None: - if isstr(env_ids) or not isinstance(env_ids, (list, tuple)): - raise TypeError("env_ids must be a list of environment ids") - if not all(isstr(env_id) for env_id in env_ids): - raise TypeError("env_ids must contain strings") - if query is not None and not isstr(query): - raise TypeError("query must be a string") - if sort_by is not None and not isstr(sort_by): - raise TypeError("sort_by must be a string") - if env_ids is not None and query is not None: - raise ValueError("pass either env_ids or query, not both") - if env_ids is None and query is None: - raise ValueError("one of env_ids or query is required") + if isstr(env_ids) or not isinstance(env_ids, (list, tuple)): + raise TypeError("env_ids must be a list of environment ids") + if not all(isstr(env_id) for env_id in env_ids): + raise TypeError("env_ids must contain strings") return self._experiment_request( - { - "env_ids": list(env_ids) if env_ids is not None else None, - "query": query, - "limit": limit, - "sort_by": sort_by, - "descending": descending, - }, + {"env_ids": list(env_ids)}, "experiments/compare", ) diff --git a/py/visdom/__init__.pyi b/py/visdom/__init__.pyi index c42a0826a..64bb86c48 100644 --- a/py/visdom/__init__.pyi +++ b/py/visdom/__init__.pyi @@ -16,7 +16,7 @@ _OptStr = Optional[Text] # For optional string parameters, like 'window' and 'e # For the list of environments to compare. Spelled out rather than Sequence[Text] # because a bare str is itself a Sequence[str]: 'compare_experiments' rejects one # at runtime, so a checker must not accept it here. -_EnvIds = Optional[Union[List[Text], Tuple[Text, ...]]] +_EnvIds = Union[List[Text], Tuple[Text, ...]] # The decoded JSON reply of the experiment endpoints. _ExperimentReply = Mapping[Text, Any] @@ -79,14 +79,7 @@ class Visdom: sort_by: _OptStr = ..., descending: bool = ..., ) -> _ExperimentReply: ... - def compare_experiments( - self, - env_ids: _EnvIds = ..., - query: _OptStr = ..., - limit: Optional[int] = ..., - sort_by: _OptStr = ..., - descending: bool = ..., - ) -> _ExperimentReply: ... + def compare_experiments(self, env_ids: _EnvIds) -> _ExperimentReply: ... def get_window_data( self, win: _OptStr = ..., env: _OptStr = ... ) -> _SendReturn: ... diff --git a/py/visdom/experiments/store.py b/py/visdom/experiments/store.py index beb6c42cf..0b85cd41c 100644 --- a/py/visdom/experiments/store.py +++ b/py/visdom/experiments/store.py @@ -246,48 +246,21 @@ def _load_named(self, env_ids): ) return experiments - def compare( - self, - env_ids=None, - query=None, - sort_by=DEFAULT_SORT_FIELD, - descending=True, - limit=None, - ): - """Compare experiments field by field; see :func:`build_comparison`. - - The experiments are chosen either by name or by search, and the two are - mutually exclusive — passing both, or neither, is a ``ValueError`` rather - than a guess at which was meant: - - * ``env_ids`` — an explicit list, compared in the order given. Every id - must have an experiment; a :class:`KeyError` names those that do not, - since a comparison silently missing a run it was asked for would be - read as a comparison of the rest. - * ``query`` — compare everything the query matches, ordered by - ``sort_by``/``descending`` as in :meth:`search` and capped at ``limit`` - (``None`` = uncapped). A query matching nothing compares nothing and - yields empty sections; that is an empty answer, not an error. - - ``limit`` applies only to ``query`` selection — an explicit ``env_ids`` - list is already the exact set to compare. Note that the diff describes - the runs actually compared, so a ``limit`` that truncates the matches - narrows what ``shared``/``differing`` are computed over; the returned - ``env_ids`` always say which runs those were. + def compare(self, env_ids): + """Compare the named experiments field by field; see :func:`build_comparison`. + + ``env_ids`` is an explicit list, compared in the order given. Every id must + have an experiment; a :class:`KeyError` names those that do not, since a + comparison silently missing a run it was asked for would be read as a + comparison of the rest. + + Finding the runs to compare is :meth:`search`'s job, not this one: search + answers "which runs match?" and compare answers "how do these runs differ?". + Callers that want to compare a query's matches search first and pass the + ids on, which also keeps the diff honest — it always describes exactly the + runs that were named. """ - if env_ids is not None and query is not None: - raise ValueError("pass either env_ids or query, not both") - if env_ids is None and query is None: - raise ValueError("one of env_ids or query is required") - if env_ids is not None: - experiments = self._load_named(env_ids) - else: - experiments = self.search( - query=query, sort_by=sort_by, descending=descending - ) - if limit is not None: - experiments = experiments[:limit] - return build_comparison(experiments) + return build_comparison(self._load_named(env_ids)) def delete_experiment(self, env_id): """Drop the experiment blob from ``env_id`` (keeping the env itself). diff --git a/py/visdom/server/handlers/web_handlers.py b/py/visdom/server/handlers/web_handlers.py index c3a95ffb3..44943e694 100644 --- a/py/visdom/server/handlers/web_handlers.py +++ b/py/visdom/server/handlers/web_handlers.py @@ -819,49 +819,6 @@ def post(self): ) -def _require_index(args, field, default): - """Return ``args[field]`` as a non-negative int (``None`` = unbounded). - - A JSON body has no int/float distinction, so a client that sends ``10.0`` - means the index 10; anything with a fractional part is a mistake. - """ - value = args.get(field, default) - if value is None: - return None - if isinstance(value, float) and value.is_integer(): - value = int(value) - if isinstance(value, bool) or not isinstance(value, int): - raise tornado.web.HTTPError( - 400, reason="'{0}' must be an integer".format(field) - ) - if value < 0: - raise tornado.web.HTTPError( - 400, reason="'{0}' must not be negative".format(field) - ) - return value - - -def _require_text(args, field): - """Return ``args[field]`` if it is a string (or absent); else raise 400.""" - value = args.get(field) - if value is not None and not isinstance(value, str): - raise tornado.web.HTTPError(400, reason="'{0}' must be a string".format(field)) - return value - - -def _require_flag(args, field, default): - """Return ``args[field]`` as a bool; else raise 400. - - Deliberately not ``bool(value)``: JSON has real booleans, so a client - sending the *string* ``"false"`` means false, and coercing it would - truthily flip the result to its opposite without a word. - """ - value = args.get(field, default) - if not isinstance(value, bool): - raise tornado.web.HTTPError(400, reason="'{0}' must be a boolean".format(field)) - return value - - class ExperimentLogHandler(BaseHandler): """POST ``/experiments/log`` — record experiment metadata for an environment. @@ -980,13 +937,62 @@ class ExperimentSearchHandler(BaseHandler): DEFAULT_LIMIT = 100 + @staticmethod + def _require_index(args, field, default): + """Return ``args[field]`` as a non-negative int (``None`` = unbounded). + + A JSON body has no int/float distinction, so a client that sends ``10.0`` + means the index 10; anything with a fractional part is a mistake. + """ + value = args.get(field, default) + if value is None: + return None + if isinstance(value, float) and value.is_integer(): + value = int(value) + if isinstance(value, bool) or not isinstance(value, int): + raise tornado.web.HTTPError( + 400, reason="'{0}' must be an integer".format(field) + ) + if value < 0: + raise tornado.web.HTTPError( + 400, reason="'{0}' must not be negative".format(field) + ) + return value + + @staticmethod + def _require_text(args, field): + """Return ``args[field]`` if it is a string (or absent); else raise 400.""" + value = args.get(field) + if value is not None and not isinstance(value, str): + raise tornado.web.HTTPError( + 400, reason="'{0}' must be a string".format(field) + ) + return value + + @staticmethod + def _require_flag(args, field, default): + """Return ``args[field]`` as a bool; else raise 400. + + Deliberately not ``bool(value)``: JSON has real booleans, so a client + sending the *string* ``"false"`` means false, and coercing it would + truthily flip the result to its opposite without a word. + """ + value = args.get(field, default) + if not isinstance(value, bool): + raise tornado.web.HTTPError( + 400, reason="'{0}' must be a boolean".format(field) + ) + return value + @staticmethod def wrap_func(handler, args): - query = _require_text(args, "query") - sort_by = _require_text(args, "sort_by") - limit = _require_index(args, "limit", ExperimentSearchHandler.DEFAULT_LIMIT) - offset = _require_index(args, "offset", 0) - descending = _require_flag(args, "descending", True) + query = ExperimentSearchHandler._require_text(args, "query") + sort_by = ExperimentSearchHandler._require_text(args, "sort_by") + limit = ExperimentSearchHandler._require_index( + args, "limit", ExperimentSearchHandler.DEFAULT_LIMIT + ) + offset = ExperimentSearchHandler._require_index(args, "offset", 0) + descending = ExperimentSearchHandler._require_flag(args, "descending", True) store = ExperimentStore(handler.storage) try: @@ -1023,21 +1029,17 @@ def post(self): class ExperimentCompareHandler(BaseHandler): - """POST ``/experiments/compare`` — diff experiments field by field. + """POST ``/experiments/compare`` — diff the named experiments field by field. - Selects the experiments to compare either by name or by search, and the two - are mutually exclusive:: + The JSON body names the runs to compare:: {"env_ids": ["run-a", "run-b"]} - {"query": "lr < 0.01", "limit": 10} - Sending both is a 400 rather than a silent precedence rule, as is sending - neither. With ``env_ids``, every id must have an experiment — a 404 names the - ones that do not, since quietly comparing the remainder would answer a - question the caller did not ask. With ``query``, the syntax and the - ``sort_by``/``descending``/``limit`` handling are exactly - :class:`ExperimentSearchHandler`'s, and a query matching nothing is an empty - comparison rather than an error. + Every id must have an experiment — a 404 names the ones that do not, since + quietly comparing the remainder would answer a question the caller did not + ask. Finding the runs is ``/experiments/search``'s job: it answers "which runs + match?", this answers "how do these runs differ?". A caller comparing a + query's matches searches first and passes the ids on. The reply carries the compared runs (``env_ids``, ``experiments``) and a diff per section, each listing the union of ``fields``, the ``shared`` ones every @@ -1054,7 +1056,7 @@ class ExperimentCompareHandler(BaseHandler): @staticmethod def _require_env_ids(args): - """Return ``args["env_ids"]`` as a list of ids, or ``None`` if absent. + """Return ``args["env_ids"]`` as a list of ids; raise 400 if unusable. A bare string is rejected rather than treated as a one-id list: it would otherwise be iterated character by character into a comparison of runs @@ -1062,7 +1064,7 @@ def _require_env_ids(args): """ value = args.get("env_ids") if value is None: - return None + raise tornado.web.HTTPError(400, reason="'env_ids' is required") if not isinstance(value, list): raise tornado.web.HTTPError(400, reason="'env_ids' must be a list of ids") if not value: @@ -1076,30 +1078,9 @@ def _require_env_ids(args): @staticmethod def wrap_func(handler, args): env_ids = ExperimentCompareHandler._require_env_ids(args) - query = _require_text(args, "query") - if env_ids is not None and query is not None: - raise tornado.web.HTTPError( - 400, reason="pass either 'env_ids' or 'query', not both" - ) - if env_ids is None and query is None: - raise tornado.web.HTTPError( - 400, reason="one of 'env_ids' or 'query' is required" - ) - sort_by = _require_text(args, "sort_by") - limit = _require_index(args, "limit", None) - descending = _require_flag(args, "descending", True) - store = ExperimentStore(handler.storage) try: - comparison = store.compare( - env_ids=env_ids, - query=query, - sort_by=sort_by or DEFAULT_SORT_FIELD, - descending=descending, - limit=limit, - ) - except QueryParseError as e: - raise tornado.web.HTTPError(400, reason=str(e)) + comparison = store.compare(env_ids) except KeyError as e: raise tornado.web.HTTPError(404, reason=str(e.args[0])) From 29baa49947d16f697df99c35f623fbf35528c29b Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 15 Jul 2026 19:15:59 +0530 Subject: [PATCH 10/48] group the runs that agree, per field shared/differing was all-or-nothing: a field counted as shared only if every run agreed, so with run-a lr=0.1, run-b lr=0.001, run-c lr=0.1 the answer was just "lr differs". That run-a and run-c actually match was in the values map, but the reader had to spot it. Each section now also carries groups: per field, the runs clustered by the value they used. "lr": [{"value": 0.1, "env_ids": ["run-a", "run-c"]}, {"value": 0.001, "env_ids": ["run-b"]}] shared/differing stay as the at-a-glance "what changed?"; groups answers the finer "which runs agree?". They cannot disagree: a field is in shared exactly when its groups are a single cluster holding every compared run, and shared is now derived from the groups rather than computed twice. Grouping deliberately does not use a dict keyed by value. Values need not be hashable (a param may hold a list), hash(True) == hash(1) with True == 1 so a dict would silently merge them and undo the bool rule _same_value exists to enforce, and NaN never equals itself so it would never group. _group_values scans with _same_value instead, keeping one definition of sameness for the module; the run count per comparison is small. Groups are ordered by first appearance and env_ids within a group keep the compared order, so the output is deterministic. A run that never logged the field is in no group for it. Tests: 7 new in TestBuildComparison (clustering, shared<->groups agreement, missing field, bool-vs-1, NaN, unhashable lists, ordering) plus a JSON round-trip through the endpoint. 406 py/tests pass. Verified live: three runs on two learning rates cluster as intended, including list-valued params, with shared and groups agreeing on every field. --- openapi.yaml | 25 +++++- py/tests/test_experiment_compare.py | 121 ++++++++++++++++++++++++++++ py/visdom/__init__.py | 13 ++- py/visdom/experiments/compare.py | 85 ++++++++++++++----- 4 files changed, 218 insertions(+), 26 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 27e7e2e97..fbd663a72 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1112,7 +1112,7 @@ components: description: > One section (params, metrics or tags) of an experiment comparison, diffed across the compared runs. - required: [fields, shared, differing, values] + required: [fields, shared, differing, values, groups] properties: fields: type: array @@ -1140,6 +1140,29 @@ components: description: > Per-field, per-run values as `{field: {env_id: value}}`. A run that never logged the field is omitted from that field's map. + groups: + type: object + description: > + Per field, the runs clustered by the value they used — the finer + question "which runs agree?", where `shared`/`differing` answer only + "do they all?". With three runs on two learning rates, the two that + match share a group. A field appears in `shared` exactly when its + groups are a single cluster holding every compared run, so the two + never disagree. Groups are ordered by first appearance, and a run + that never logged the field is in no group. + additionalProperties: + type: array + items: + type: object + required: [value, env_ids] + properties: + value: + description: The value these runs share. + env_ids: + type: array + description: The runs that used it, in compared order. + items: + type: string Experiment: type: object diff --git a/py/tests/test_experiment_compare.py b/py/tests/test_experiment_compare.py index 745c507b5..361e9edef 100644 --- a/py/tests/test_experiment_compare.py +++ b/py/tests/test_experiment_compare.py @@ -161,6 +161,112 @@ def test_no_experiments_yields_empty_sections(self): self.assertEqual(comparison[section]["fields"], []) self.assertEqual(comparison[section]["shared"], {}) + def test_groups_cluster_the_runs_that_agree(self): + """groups answers which runs match, not just whether all of them do.""" + comparison = build_comparison( + [ + make_experiment("a", params={"lr": 0.1}), + make_experiment("b", params={"lr": 0.001}), + make_experiment("c", params={"lr": 0.1}), + ] + ) + self.assertEqual(comparison["params"]["differing"], ["lr"]) + self.assertEqual( + comparison["params"]["groups"]["lr"], + [ + {"value": 0.1, "env_ids": ["a", "c"]}, + {"value": 0.001, "env_ids": ["b"]}, + ], + ) + + def test_a_shared_field_is_one_group_of_everyone(self): + """shared and groups cannot disagree: shared == a single full cluster.""" + comparison = build_comparison( + [ + make_experiment("a", params={"epochs": 10}), + make_experiment("b", params={"epochs": 10}), + ] + ) + self.assertEqual(comparison["params"]["shared"], {"epochs": 10}) + self.assertEqual( + comparison["params"]["groups"]["epochs"], + [{"value": 10, "env_ids": ["a", "b"]}], + ) + + def test_a_run_missing_the_field_is_in_no_group(self): + """Groups cover only the runs that logged the field.""" + comparison = build_comparison( + [ + make_experiment("a", params={"lr": 0.1}), + make_experiment("b", params={"lr": 0.1}), + make_experiment("c", params={"seed": 7}), + ] + ) + self.assertEqual( + comparison["params"]["groups"]["lr"], + [{"value": 0.1, "env_ids": ["a", "b"]}], + ) + self.assertEqual(comparison["params"]["differing"], ["lr", "seed"]) + + def test_groups_do_not_merge_a_bool_with_one(self): + """A dict keyed by value would merge these: hash(True) == hash(1).""" + comparison = build_comparison( + [ + make_experiment("a", params={"amp": True}), + make_experiment("b", params={"amp": 1}), + ] + ) + self.assertEqual( + comparison["params"]["groups"]["amp"], + [ + {"value": True, "env_ids": ["a"]}, + {"value": 1, "env_ids": ["b"]}, + ], + ) + + def test_groups_cluster_nan_together(self): + """NaN never equals itself, so a dict would never group these.""" + comparison = build_comparison( + [ + make_experiment("a", metrics=[("loss", float("nan"))]), + make_experiment("b", metrics=[("loss", float("nan"))]), + ] + ) + groups = comparison["metrics"]["groups"]["loss"] + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0]["env_ids"], ["a", "b"]) + + def test_groups_handle_unhashable_values(self): + """A param may hold a list, which could not be a dict key.""" + comparison = build_comparison( + [ + make_experiment("a", params={"layers": [64, 32]}), + make_experiment("b", params={"layers": [64, 32]}), + make_experiment("c", params={"layers": [128]}), + ] + ) + self.assertEqual( + comparison["params"]["groups"]["layers"], + [ + {"value": [64, 32], "env_ids": ["a", "b"]}, + {"value": [128], "env_ids": ["c"]}, + ], + ) + + def test_group_order_follows_the_compared_order(self): + """Groups appear in the order their value was first seen.""" + comparison = build_comparison( + [ + make_experiment("b", params={"lr": 0.001}), + make_experiment("a", params={"lr": 0.1}), + make_experiment("c", params={"lr": 0.1}), + ] + ) + self.assertEqual( + [g["value"] for g in comparison["params"]["groups"]["lr"]], [0.001, 0.1] + ) + self.assertEqual(comparison["params"]["groups"]["lr"][1]["env_ids"], ["a", "c"]) + def test_sections_are_independent(self): """A name used as both a param and a tag is not conflated across sections.""" comparison = build_comparison( @@ -284,6 +390,21 @@ def test_all_sections_are_present(self): body["tags"]["values"]["dataset"], {"run-a": "mnist", "run-b": "cifar10"} ) + def test_groups_survive_the_json_round_trip(self): + """The clusters reach the client intact, epochs shared and lr split.""" + body = self.compare_ok({"env_ids": ["run-a", "run-b"]}) + self.assertEqual( + body["params"]["groups"]["epochs"], + [{"value": 10, "env_ids": ["run-a", "run-b"]}], + ) + self.assertEqual( + body["params"]["groups"]["lr"], + [ + {"value": 0.1, "env_ids": ["run-a"]}, + {"value": 0.001, "env_ids": ["run-b"]}, + ], + ) + def test_experiments_are_returned_in_full(self): """The compared runs are echoed as full experiment dicts.""" body = self.compare_ok({"env_ids": ["run-a", "run-b"]}) diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index 51e27ec6c..2bcdda91c 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -1262,10 +1262,15 @@ def compare_experiments(self, env_ids): Returns the server's reply as a dict: the compared runs (`env_ids` and the full `experiments`), plus a `params`, `metrics` and `tags` section. Each section holds the union of `fields`, the `shared` ones every run - agrees on, the `differing` rest, and the per-run `values`. Comparing two - runs that differ only in learning rate gives a `params` section whose - `differing` is `['lr']` and whose `values['lr']` is - `{'run-a': 0.1, 'run-b': 0.001}`. + agrees on, the `differing` rest, the per-run `values`, and `groups`. + + `shared`/`differing` answer "what changed?"; `groups` answers "which runs + agree?", clustering the runs by value per field. Comparing three runs on + two learning rates gives a `params` section whose `differing` is `['lr']`, + whose `values['lr']` is `{'run-a': 0.1, 'run-b': 0.001, 'run-c': 0.1}`, + and whose `groups['lr']` is + `[{'value': 0.1, 'env_ids': ['run-a', 'run-c']}, + {'value': 0.001, 'env_ids': ['run-b']}]`. """ if isstr(env_ids) or not isinstance(env_ids, (list, tuple)): raise TypeError("env_ids must be a list of environment ids") diff --git a/py/visdom/experiments/compare.py b/py/visdom/experiments/compare.py index a78dd5855..ce474bd35 100644 --- a/py/visdom/experiments/compare.py +++ b/py/visdom/experiments/compare.py @@ -90,13 +90,45 @@ def _same_value(a: Any, b: Any) -> bool: return bool(a == b) +def _group_values(present: dict) -> list: + """Cluster ``{env_id: value}`` into ``[{"value": v, "env_ids": [...]}, ...]``. + + Answers "which runs used the same value?" rather than only "did they all?" — + with three runs on two learning rates, the two that match end up in one group + and the odd one out in another. + + The obvious ``defaultdict`` keyed by value cannot be used here. Values need + not be hashable (a param may hold a list), ``hash(True) == hash(1)`` with + ``True == 1`` so a dict would silently merge them and undo the bool rule + :func:`_same_value` exists to enforce, and NaN never equals itself so it would + never group. Scanning the groups with :func:`_same_value` keeps one definition + of sameness for the whole module; the run count per comparison is small. + + Groups appear in the order their value was first seen, and env_ids within a + group keep the order the runs were compared in. + """ + groups: list = [] + for env_id, value in present.items(): + for group in groups: + if _same_value(group["value"], value): + group["env_ids"].append(env_id) + break + else: + groups.append({"value": value, "env_ids": [env_id]}) + return groups + + def _compare_section(per_env: dict) -> dict: """Diff one section across the experiments in ``per_env``. ``per_env`` maps env_id to that experiment's ``{field: value}`` for the - section. A field counts as shared only when every experiment carries it *and* - they all agree: a value one run never logged is a difference between the runs, - not a consensus among those that happen to have it. + section. Each field gets its per-run ``values``, its ``groups`` of runs that + agree, and a place in ``shared`` or ``differing``. + + A field counts as shared only when every experiment carries it *and* they all + agree — one group holding every run. A value one run never logged is a + difference between the runs, not a consensus among those that happen to have + it, and that run appears in no group for the field. """ fields = sorted({field for values in per_env.values() for field in values}) values = { @@ -107,19 +139,18 @@ def _compare_section(per_env: dict) -> dict: } for field in fields } - shared = {} - for field in fields: - present = values[field] - if len(present) != len(per_env): - continue - found = list(present.values()) - if all(_same_value(found[0], other) for other in found[1:]): - shared[field] = found[0] + groups = {field: _group_values(values[field]) for field in fields} + shared = { + field: groups[field][0]["value"] + for field in fields + if len(groups[field]) == 1 and len(values[field]) == len(per_env) + } return { "fields": fields, "shared": shared, "differing": [field for field in fields if field not in shared], "values": values, + "groups": groups, } @@ -130,24 +161,36 @@ def build_comparison(experiments: Iterable[ComparableExperiment]) -> dict: full ``experiments``) alongside one diff per section:: { - "env_ids": ["run-a", "run-b"], - "experiments": [{...}, {...}], + "env_ids": ["run-a", "run-b", "run-c"], + "experiments": [{...}, {...}, {...}], "params": { "fields": ["epochs", "lr"], "shared": {"epochs": 10}, "differing": ["lr"], - "values": {"lr": {"run-a": 0.1, "run-b": 0.001}, - "epochs": {"run-a": 10, "run-b": 10}} + "values": {"lr": {"run-a": 0.1, "run-b": 0.001, "run-c": 0.1}, + "epochs": {"run-a": 10, "run-b": 10, "run-c": 10}}, + "groups": { + "epochs": [{"value": 10, "env_ids": ["run-a", "run-b", "run-c"]}], + "lr": [{"value": 0.1, "env_ids": ["run-a", "run-c"]}, + {"value": 0.001, "env_ids": ["run-b"]}] + } }, "metrics": {...}, "tags": {...} } - ``fields`` is every name any run has, sorted; ``shared`` holds the fields all - runs carry with the same value; ``differing`` is the rest — the ones that vary - or that some run is missing; ``values`` gives the raw per-run value, omitting - the runs that never logged the field. Comparing a single experiment is legal - and puts everything it has in ``shared``; comparing none yields empty - sections. + ``fields`` is every name any run has, sorted. ``shared`` holds the fields all + runs carry with the same value, and ``differing`` is the rest — the ones that + vary or that some run is missing; together they answer "what changed?" at a + glance. ``values`` gives the raw per-run value, omitting the runs that never + logged the field. + + ``groups`` answers the finer question "*which* runs agree?", clustering the + runs by value per field: above, ``lr`` differs overall, but run-a and run-c + still used the same one. A field is in ``shared`` exactly when its ``groups`` + is a single cluster holding every compared run, so the two never disagree. + + Comparing a single experiment is legal and puts everything it has in + ``shared``; comparing none yields empty sections. """ ordered: Sequence[ComparableExperiment] = list(experiments) comparison = { From 7a6e022e3dbdfb06b40d4e37154646bb3dc16472 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Thu, 16 Jul 2026 16:44:47 +0530 Subject: [PATCH 11/48] experiment suggest endpoint stub + vis.suggest_experiment Reserve /experiments/suggest for the next-run hyper-parameter suggestion strategy (Optuna-backed), which belongs to a later layer. The endpoint is a stub: it parses the request like its siblings and replies 501 Not Implemented with a JSON body ({"status": "not_implemented", "suggestion": null, ...}) so a caller gets a stable, decodable answer it can tell apart from a real result. - ExperimentSuggestHandler (web_handlers.py) + route (app.py), mirroring the log/search/compare handler shape (static wrap_func, @check_auth post). - Visdom.suggest_experiment(params=None, env=None) client method + .pyi stub; type-checks params and posts the search space through for the eventual strategy. - openapi.yaml: /experiments/suggest documented (operationId suggestExperiment, 501 stub response schema). - test_experiment_suggest.py: endpoint 501/JSON-body contract and client message shape. --- openapi.yaml | 50 +++++++++++++ py/tests/test_experiment_suggest.py | 91 +++++++++++++++++++++++ py/visdom/__init__.py | 19 +++++ py/visdom/__init__.pyi | 3 + py/visdom/server/app.py | 6 ++ py/visdom/server/handlers/web_handlers.py | 42 +++++++++++ 6 files changed, 211 insertions(+) create mode 100644 py/tests/test_experiment_suggest.py diff --git a/openapi.yaml b/openapi.yaml index fbd663a72..148d539af 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -644,6 +644,56 @@ paths: "404": description: One or more of the given `env_ids` has no experiment. + /experiments/suggest: + post: + operationId: suggestExperiment + tags: [Experiments] + summary: Suggest parameters for the next run (reserved) + description: > + Reserved endpoint for hyper-parameter suggestion. Choosing the next set + of parameters to try is a search-strategy problem (Optuna-backed) that + lands in a later layer, so this is currently a stub: it accepts the + request and replies `501 Not Implemented` with a JSON body carrying a + `suggestion: null` placeholder, rather than a made-up suggestion. The + route, the `suggest_experiment` client method and this documentation are + in place so the strategy can be wired in later without changing the + surface. + requestBody: + required: false + content: + application/json: + schema: + type: object + properties: + eid: + type: string + description: Target environment ID. Defaults to `main`. + params: + type: object + additionalProperties: true + description: > + The search space to suggest over, as `{name: spec}`. Ignored + by the stub; reserved for the eventual strategy. + responses: + "501": + description: > + Not implemented — the suggestion strategy is reserved for a later + layer. The body is a JSON stub. + content: + application/json: + schema: + type: object + required: [status, detail, suggestion] + properties: + status: + type: string + example: not_implemented + detail: + type: string + suggestion: + nullable: true + description: Always `null` while the endpoint is a stub. + /upload_env: post: operationId: uploadEnvironment diff --git a/py/tests/test_experiment_suggest.py b/py/tests/test_experiment_suggest.py new file mode 100644 index 000000000..6d9e1bc52 --- /dev/null +++ b/py/tests/test_experiment_suggest.py @@ -0,0 +1,91 @@ +"""Tests for the experiment suggest stub (Layer 2, PR 6). + +``/experiments/suggest`` is a reserved endpoint: the suggestion strategy +(Optuna-backed) lands in a later layer, so for now the server replies +``501 Not Implemented`` with a JSON stub. These tests pin that contract from +both ends — the endpoint through a real +:class:`~visdom.server.app.Application` with Tornado's ``AsyncHTTPTestCase``, +and the ``Visdom.suggest_experiment`` message shape with ``send=False`` (no +server) — so the surface stays stable until the strategy is wired in. +""" + +import json +import tempfile +import unittest + +import tornado.testing + +from visdom import Visdom +from visdom.server.app import Application + + +class TestSuggestEndpoint(tornado.testing.AsyncHTTPTestCase): + """POST /experiments/suggest is a 501 stub with a JSON body.""" + + def setUp(self): + self._tmp_dir = tempfile.mkdtemp(prefix="visdom_exp_suggest_api_") + super().setUp() + + def get_app(self): + return Application(port=self.get_http_port(), env_path=self._tmp_dir) + + def suggest(self, body): + return self.fetch( + "/experiments/suggest", + method="POST", + body=json.dumps(body), + headers={"Content-Type": "application/json"}, + ) + + def test_stub_returns_501(self): + """The reserved endpoint reports that it is not implemented.""" + self.assertEqual(self.suggest({}).code, 501) + + def test_stub_body_is_json_not_implemented(self): + """The body is a decodable stub a caller can recognise as such.""" + body = json.loads(self.suggest({}).body) + self.assertEqual(body["status"], "not_implemented") + self.assertIsNone(body["suggestion"]) + self.assertIn("detail", body) + + def test_params_body_is_accepted_and_ignored(self): + """A search space in the body is parsed without error, then ignored.""" + resp = self.suggest({"eid": "run-a", "params": {"lr": [0.1, 0.01]}}) + self.assertEqual(resp.code, 501) + self.assertIsNone(json.loads(resp.body)["suggestion"]) + + +class TestSuggestClientMessage(unittest.TestCase): + """Visdom.suggest_experiment builds the message the endpoint expects.""" + + def setUp(self): + self.vis = Visdom(send=False, raise_exceptions=True) + + def test_suggest_message_shape(self): + """The message posts to the suggest endpoint and carries params. + + ``_send`` stamps an ``eid`` on every message; the stub ignores it. + """ + msg, endpoint = self.vis.suggest_experiment() + self.assertEqual(endpoint, "experiments/suggest") + self.assertIsNone(msg["params"]) + self.assertIn("eid", msg) + + def test_params_are_passed_through(self): + """The search space rides along untouched for the eventual strategy.""" + msg, _ = self.vis.suggest_experiment(params={"lr": [0.1, 0.01]}) + self.assertEqual(msg["params"], {"lr": [0.1, 0.01]}) + + def test_env_overrides_the_target_eid(self): + """An explicit env names the study to suggest against.""" + msg, _ = self.vis.suggest_experiment(env="run-x") + self.assertEqual(msg["eid"], "run-x") + + def test_client_rejects_bad_params(self): + """The client type-checks before any request is made.""" + with self.assertRaises(TypeError): + self.vis.suggest_experiment(params="lr") + + +if __name__ == "__main__": + unittest.main() diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index 2bcdda91c..11ee31b4c 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -1281,6 +1281,25 @@ def compare_experiments(self, env_ids): "experiments/compare", ) + def suggest_experiment(self, params=None, env=None): + """Ask the server to suggest parameters for the next run. + + Reserved: the suggestion strategy (Optuna-backed) is not implemented + yet, so this returns the server's stub reply — a dict of + `{"status": "not_implemented", "suggestion": None, ...}` — rather than a + real suggestion. The method, its `params` search space (a dict of + `{name: spec}`) and the endpoint are in place so callers and the docs + are ready for when the strategy is wired in. + + Returns the server's reply as a dict. + """ + if params is not None and not isinstance(params, dict): + raise TypeError("params must be a dict of {name: spec}") + msg = {"params": params} + if env is not None: + msg["eid"] = env + return self._experiment_request(msg, "experiments/suggest") + def get_window_data(self, win=None, env=None): """ This function returns all the window data for a specified window in diff --git a/py/visdom/__init__.pyi b/py/visdom/__init__.pyi index 64bb86c48..95a2ea100 100644 --- a/py/visdom/__init__.pyi +++ b/py/visdom/__init__.pyi @@ -80,6 +80,9 @@ class Visdom: descending: bool = ..., ) -> _ExperimentReply: ... def compare_experiments(self, env_ids: _EnvIds) -> _ExperimentReply: ... + def suggest_experiment( + self, params: _OptOps = ..., env: _OptStr = ... + ) -> _ExperimentReply: ... def get_window_data( self, win: _OptStr = ..., env: _OptStr = ... ) -> _SendReturn: ... diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index a7acbdb74..2f9bdf7d2 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -39,6 +39,7 @@ ExperimentCompareHandler, ExperimentLogHandler, ExperimentSearchHandler, + ExperimentSuggestHandler, ForkEnvHandler, HealthHandler, IndexHandler, @@ -138,6 +139,11 @@ def __init__( ExperimentCompareHandler, {"app": self}, ), + ( + r"%s/experiments/suggest" % self.base_url, + ExperimentSuggestHandler, + {"app": self}, + ), (r"%s/user/(.*)" % self.base_url, UserSettingsHandler, {"app": self}), (r"%s/health" % self.base_url, HealthHandler), (r"%s(.*)" % self.base_url, IndexHandler, {"app": self}), diff --git a/py/visdom/server/handlers/web_handlers.py b/py/visdom/server/handlers/web_handlers.py index 44943e694..63d5d511c 100644 --- a/py/visdom/server/handlers/web_handlers.py +++ b/py/visdom/server/handlers/web_handlers.py @@ -1094,6 +1094,48 @@ def post(self): self.wrap_func(self, args) +class ExperimentSuggestHandler(BaseHandler): + """POST ``/experiments/suggest`` — suggest parameters for the next run. + + Reserved endpoint. Choosing the next set of hyper-parameters to try is a + search-strategy problem (Optuna-backed) that belongs to a later layer, so + this is a stub: it accepts the request and replies ``501 Not Implemented`` + with a JSON body rather than a made-up suggestion. Wiring the route, the + :meth:`Visdom.suggest_experiment` client method and the API docs now means + the strategy can be dropped in later without changing the surface, and a + caller gets a stable, decodable answer it can tell apart from a real one:: + + {"status": "not_implemented", "detail": "...", "suggestion": null} + + The request body is parsed and passed through like the sibling handlers so + that shape is already in place, but it is otherwise ignored until the + strategy lands. + """ + + #: The stub reply, carrying ``suggestion: null`` so the eventual field is + #: already named and a caller can distinguish the stub from a real result. + NOT_IMPLEMENTED = { + "status": "not_implemented", + "detail": ( + "experiment suggestion is not implemented yet; the endpoint is " + "reserved for a later layer" + ), + "suggestion": None, + } + + @staticmethod + def wrap_func(handler, args): + handler.set_status(501) + handler.write(json.dumps(ExperimentSuggestHandler.NOT_IMPLEMENTED)) + + @check_auth + def post(self): + args = tornado.escape.json_decode( + tornado.escape.to_basestring(self.request.body) + ) + self.wrap_func(self, args) + + class HealthHandler(BaseHandler): def get(self): self.write({"status": "ok"}) From 7c9b1f19bc70b73f572ba6b60abc8e55f368dc3e Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Thu, 16 Jul 2026 16:49:44 +0530 Subject: [PATCH 12/48] document the experiments API in the README MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Layer-2 experiments client API had no README coverage (L2-1..L2-5 were local-only with docs deferred here). Add an Experiments section — both the API list entry and the per-method Details — covering the full workflow: experiment / log_metrics / finish_experiment / search_experiments / compare_experiments, plus suggest_experiment documented honestly as a reserved 501 stub. Argument names and defaults match the client signatures. --- README.md | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/README.md b/README.md index 8dbc90c1a..35e6067c5 100644 --- a/README.md +++ b/README.md @@ -331,6 +331,15 @@ vis._send({'data': [trace], 'layout': layout, 'win': 'mywin'}) - [`vis.check_connection`](#vischeck_connection): check if the server is connected - [`vis.replay_log`](#visreplay_log): replay the actions from the provided log file +### Experiments +Track experiment metadata (hyper-parameters, metrics, tags) alongside your plots, then search and compare runs across your server: +- [`vis.experiment`](#visexperiment) : create or update experiment metadata for an env +- [`vis.log_metrics`](#vislog_metrics) : append metric observations to an env's experiment +- [`vis.finish_experiment`](#visfinish_experiment) : mark an experiment terminal (finished/failed) +- [`vis.search_experiments`](#vissearch_experiments) : search experiments across envs with a query +- [`vis.compare_experiments`](#viscompare_experiments) : diff experiments field by field +- [`vis.suggest_experiment`](#vissuggest_experiment) : suggest parameters for the next run (reserved) + ## Loggers @@ -1024,6 +1033,75 @@ This function takes the contents of a visdom log and replays them to the current Arguments: - `log_filename`: log file to replay the contents of. +### Experiments + +Attach experiment metadata — hyper-parameters, metric observations and tags — to an environment, then search and compare runs across your server. Metadata is stored under the environment's `experiment` key and persisted through the server's data store, so a server started without a persistence path (`env_path=None`) has nothing to store, search or compare. Each function returns the server's reply decoded as a dict. + +#### vis.experiment + +This function creates or updates the experiment metadata for an environment. Calling it again for the same env merges in new params/tags and overwrites the name/description, so it is safe to call at the start of and again during a run. + +Arguments: +- `name`: Display name for the experiment. Defaults to the env id. +- `params`: Hyper-parameters as a dict of `{name: value}`. +- `tags`: Free-form tags as a dict of `{name: value}`. +- `description`: Free-form description. +- `env`: Environment to attach the experiment to. Defaults to the client's env. + +#### vis.log_metrics + +This function appends one or more metric observations to an env's experiment, creating the experiment automatically if it does not exist yet. + +Arguments: +- `metrics`: A non-empty dict of `{name: value}` observations. +- `step`: Optional training step the observations were recorded at. +- `env`: Environment whose experiment to log to. Defaults to the client's env. + +> **Note**: once an experiment is finished (see `finish_experiment`), further `experiment`/`log_metrics` writes are rejected so a completed run's recorded data cannot change after the fact. + +#### vis.finish_experiment + +This function marks an env's experiment terminal so its recorded data is frozen. + +Arguments: +- `status`: Terminal status, either `finished` (default) or `failed`. +- `env`: Environment whose experiment to finish. Defaults to the client's env. + +#### vis.search_experiments + +This function searches the experiments logged on the server, across all environments. The `query` uses a small readable syntax — comparisons (`<`, `<=`, `>`, `>=`, `=`, `!=`, `contains`) over param, metric and tag names, combined with `AND`/`OR` and parentheses. A name is matched bare (`acc`) or namespaced (`metric.acc`, `param.lr`, `tag.owner`) when ambiguous; metrics compare on their latest value. Queries are evaluated in Python, never `eval`'d, so a hostile query is a parse error rather than code execution. + +```python +vis.search_experiments("lr < 0.01 AND acc > 0.9") +vis.search_experiments("status = finished AND (dataset contains mnist)") +``` + +Arguments: +- `query`: The filter string. `None` (default) returns everything. +- `limit`: Maximum results to return (default `100`). `None` returns all matches. +- `offset`: Number of results to skip, for paging (default `0`). +- `sort_by`: Field to sort by (any of the same names). Defaults to newest-created first. +- `descending`: Sort direction (default `True`). + +Returns a dict of `experiments` (one page's worth), the unpaged `total` matching the query, and the `limit`/`offset`/`query` used. + +#### vis.compare_experiments + +This function compares the named experiments field by field to see what differs. Finding the runs is `search_experiments`' job — it answers "which runs match?"; this answers "how do these runs differ?". To compare a query's matches, search first and pass the resulting ids on. + +Arguments: +- `env_ids`: A list (or tuple) of environment ids to compare, in the order given. Each must have an experiment. + +Returns a dict with the compared runs (`env_ids`, full `experiments`) and a `params`, `metrics` and `tags` section. Each section lists the union of `fields`, the `shared` ones every run agrees on, the `differing` rest, the per-run `values`, and `groups` clustering the runs that agree per field. + +#### vis.suggest_experiment + +> **Reserved**: this endpoint is a stub. Choosing the next set of hyper-parameters to try (an Optuna-backed search strategy) is planned for a later layer, so the server currently replies `501 Not Implemented` and this function returns a stub dict `{"status": "not_implemented", "suggestion": None, ...}` rather than a real suggestion. The method, its arguments and the endpoint are in place so callers and docs are ready for when the strategy is wired in. + +Arguments: +- `params`: The search space to suggest over, as a dict of `{name: spec}`. Currently ignored by the stub. +- `env`: Environment (study) to suggest against. Defaults to the client's env. + ## Customizing Visdom The user config directory for visdom is - `~/.config/visdom` for Linux From 5b5c8fbeb2b8857e5a8b03f04796f635661a5b1b Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Thu, 16 Jul 2026 16:51:56 +0530 Subject: [PATCH 13/48] docs: document the data_model storage abstraction layer Add py/visdom/data_model/README.md describing the storage abstraction: the DataStore interface (env/layout/undo operations), the JSONStore backend (on-disk layout, in-memory mode, id sanitisation/path-traversal guard, long-id hash fallback, byte-stable JSON, atomic undo writes), how it is wired through Application.storage, and how to add a new backend. Point the env-persistence skill at the layer, which previously referenced only the old serialization path. --- .agents/skills/env-persistence/SKILL.md | 2 + py/visdom/data_model/README.md | 110 ++++++++++++++++++++++++ 2 files changed, 112 insertions(+) create mode 100644 py/visdom/data_model/README.md diff --git a/.agents/skills/env-persistence/SKILL.md b/.agents/skills/env-persistence/SKILL.md index 98c4b8a07..bbe05a680 100644 --- a/.agents/skills/env-persistence/SKILL.md +++ b/.agents/skills/env-persistence/SKILL.md @@ -28,6 +28,8 @@ Use this skill for persistence format changes and env reload/view layout behavio ## Documentation - [Skill reference](references/REFERENCE.md) +- `py/visdom/data_model/README.md` — the storage abstraction layer (`DataStore`/`JSONStore`); persistence now funnels through it +- `py/visdom/data_model/base.py`, `py/visdom/data_model/json_store.py` - `py/visdom/utils/server_utils.py` - `py/visdom/server/handlers/web_handlers.py` - `py/visdom/server/app.py` diff --git a/py/visdom/data_model/README.md b/py/visdom/data_model/README.md new file mode 100644 index 000000000..b144f2fee --- /dev/null +++ b/py/visdom/data_model/README.md @@ -0,0 +1,110 @@ +# `visdom.data_model` — the storage abstraction layer + +This package is the single place Visdom's server goes to persist and read back +its state. It defines **what** Visdom needs from storage (`DataStore`) apart from +**how** that storage works (`JSONStore`, the JSON-file backend), so a different +backend — a database, an object store — can be dropped in later without touching +any caller. + +## Why an abstraction + +Historically the server read and wrote environment JSON files directly from the +handlers and app code, with the same path-building, hashing and sanitisation +logic copied across several call sites. This layer replaces that with one +interface: + +- Every save/load/delete/list, every layout and undo read/write, funnels through + a `DataStore` instance held on the application as `Application.storage`. +- Nothing outside this package touches env/layout/undo files on disk directly. +- Swapping the backend is a one-line change (construct a different `DataStore`); + the rest of the server is unaware of the storage medium. + +## The `DataStore` interface + +`base.py` defines the abstract contract. An *environment* is the in-memory dict +the server holds, of the form `{"jsons": {...}, "reload": {...}}`, keyed by its +id (`eid`). + +| Group | Method | Purpose | +|-------|--------|---------| +| Environments | `save_env(eid, env_data)` | Persist one environment. | +| | `save_envs(state, eids)` | Persist a named subset of `state`; returns the ids written. | +| | `save_all(state)` | Persist every environment in `state`. | +| | `load_env(eid)` | Read one environment's data (`{}` if absent). | +| | `list_envs()` | Ids of all stored environments. | +| | `delete_env(eid)` | Remove one environment. | +| | `env_exists(eid)` | Whether an environment is stored. | +| Layouts | `save_layouts(layouts)` | Persist the saved-views layout blob. | +| | `load_layouts()` | Read the layout blob (`""` if none). | +| Undo | `load_undo(eid)` | An env's closed-pane undo stack (`[]` if none). | +| | `save_undo(eid, stack)` | Persist an env's undo stack. | +| | `clear_undo(eid)` | Remove an env's undo history. | + +Three separate save operations exist on purpose: callers persist one env, a +named subset, or everything, and each maps to a distinct server code path +(fork/upload/main-write, the save handler, and the atexit/save-all flow). + +## The `JSONStore` backend + +`json_store.py` is the default backend, backward compatible with the classic +`~/.visdom/*.json` layout. It is constructed with an `env_path`: + +```python +from visdom.data_model import JSONStore +store = JSONStore("/path/to/env_dir") # env_path=None ⇒ in-memory, no-op +``` + +On-disk layout under `env_path`: + +``` +/ + .json # one file per environment + hash_.json # fallback for ids too long to be a filename + view/layouts.json # saved-views layout blob + .undo/.json # per-env closed-pane undo stack (atomic writes) + .undo/hash_.json # undo fallback for over-long ids +``` + +Key behaviours: + +- **In-memory mode.** When `env_path is None` persistence is disabled and every + operation is a no-op (`load_*` return empty, `save_*`/`delete` do nothing). + This matches Visdom's in-memory-only server mode — there is nothing on disk to + store, search or compare. +- **Id sanitisation & path-traversal guard.** Every id is run through + `_safe_eid` (strips whitespace, neutralises `/`, `\`, newlines) and resolved + under `env_path`; an id such as `../evil` that would escape `env_path` is + rejected, so a crafted id can never read or write outside the env directory. + Saves, loads, deletes and existence checks all funnel through the same path + helpers so they agree on the file a given id maps to. +- **Long-id fallback.** An id whose plain filename would exceed the filesystem + limit is stored as `hash_.json` with its real id kept in a `name` + field inside the file; `list_envs` resolves those back to the real id. +- **Byte-stable JSON.** Env files are written with `NanSafeEncoder` so `NaN`/inf + survive the round trip; `load_env` returns only the canonical `jsons`/`reload` + fields (plus an `experiment` metadata blob when present), dropping internal + bookkeeping like the hashed-file `name`. +- **Atomic undo writes.** `save_undo` writes to a temporary file and `os.replace`s + it into place so a crash cannot leave a half-written undo stack. + +## How it is wired + +```python +# server/app.py +self.storage = JSONStore(env_path) # set before load_state() +self.state = self.load_state() # iterates storage.list_envs()/load_env() +``` + +Handlers receive it via `BaseHandler.initialize` (`self.storage = app.storage`) +and use it for every persistence operation — saving, deleting, forking, +uploading, layouts and undo. `LazyEnvData` (in `utils/server_utils.py`) also +reads through the store, so an env is only loaded from disk on first access. + +## Adding a new backend + +Subclass `DataStore`, implement every abstract method, and construct it in place +of `JSONStore` in `app.py`. Because callers depend only on the interface, no +handler or app logic changes. A backend is free to ignore the on-disk layout +above entirely (e.g. a database backend stores rows, not files) as long as it +honours the method contracts — notably returning `{}`/`[]`/`""` for absent data +and treating a null/disabled destination as a no-op. From 75f3601030775b22bcfb84aa0564d15d42b9d875 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Fri, 17 Jul 2026 14:57:40 +0530 Subject: [PATCH 14/48] vis.hparams() opens a hyper-parameter pane Add the client-side entry point for the Layer 3 hyper-parameter view. `vis.hparams()` gathers experiments through the existing `experiments/search` endpoint (same query syntax, optional `env_ids` filter), flattens them into a compact records payload via the new module-level `_flatten_experiments` helper, and creates a dedicated `hparams` window that renders from its content like the properties/embeddings panes. `_flatten_experiments` collapses each run's params/metrics lists into per-run maps, keeping only each metric's latest value, and returns the sorted param/metric key unions so the frontend renders a table/parallel-coordinates view without re-deriving columns. Includes the `.pyi` stub for the new method and pytest covering the flatten helper (unions, latest-metric, heterogeneous/empty/malformed input) and the client message shape (window type, env/win pass-through, env_ids validation and ordering). --- py/tests/test_hparams.py | 157 +++++++++++++++++++++++++++++++++++++++ py/visdom/__init__.py | 100 +++++++++++++++++++++++++ py/visdom/__init__.pyi | 8 ++ 3 files changed, 265 insertions(+) create mode 100644 py/tests/test_hparams.py diff --git a/py/tests/test_hparams.py b/py/tests/test_hparams.py new file mode 100644 index 000000000..5579de92f --- /dev/null +++ b/py/tests/test_hparams.py @@ -0,0 +1,157 @@ +"""Tests for the hyper-parameter pane (Layer 3, PR A1). + +Covers the two pieces ``Visdom.hparams`` is built from: the module-level +``_flatten_experiments`` helper, which collapses experiment dicts (as returned +by ``experiments/search``) into the compact records payload the pane renders; +and the ``Visdom.hparams`` message shape with ``send=False`` (no server), which +pins the window type and the search-then-flatten wiring. Realistic input is +produced through a real ``ExperimentStore`` over a temporary ``JSONStore``, the +same way the other experiment tests seed their fixtures. +""" + +import tempfile +import unittest + +from visdom import Visdom, _flatten_experiments +from visdom.data_model import JSONStore +from visdom.experiments import ExperimentStore + + +def seed_experiments(store): + """Log three runs with heterogeneous params and a metric time series.""" + store.log_experiment("run-a", name="alpha", params={"lr": 0.1, "epochs": 10}) + store.log_metric("run-a", "acc", 0.80) + + store.log_experiment("run-b", name="beta", params={"lr": 0.001, "epochs": 20}) + store.log_metric("run-b", "acc", 0.55) + store.log_metric("run-b", "acc", 0.95) # latest wins + store.log_metric("run-b", "loss", 0.1) + store.finish_experiment("run-b") + + # run-c only carries a param that the other two do not. + store.log_experiment("run-c", name="gamma", params={"momentum": 0.9}) + + +def search_dicts(store): + """Return experiment dicts as the search endpoint would hand them back.""" + return [experiment.to_dict() for experiment in store.search()] + + +class TestFlattenExperiments(unittest.TestCase): + """_flatten_experiments collapses lists of params/metrics into per-run maps.""" + + def setUp(self): + self._tmp_dir = tempfile.mkdtemp(prefix="visdom_hparams_flatten_") + self.store = ExperimentStore(JSONStore(self._tmp_dir)) + seed_experiments(self.store) + self.payload = _flatten_experiments(search_dicts(self.store)) + + def _record(self, env_id): + for record in self.payload["records"]: + if record["env_id"] == env_id: + return record + self.fail("no record for {0!r}".format(env_id)) + + def test_one_record_per_run(self): + """Every experiment becomes exactly one flattened row.""" + self.assertEqual(len(self.payload["records"]), 3) + + def test_param_keys_are_sorted_union(self): + """param_keys is the sorted union of every run's param names.""" + self.assertEqual(self.payload["param_keys"], ["epochs", "lr", "momentum"]) + + def test_metric_keys_are_sorted_union(self): + """metric_keys is the sorted union of every run's metric names.""" + self.assertEqual(self.payload["metric_keys"], ["acc", "loss"]) + + def test_params_collapse_to_a_map(self): + """A run's params flatten to a {name: value} map.""" + self.assertEqual(self._record("run-a")["params"], {"lr": 0.1, "epochs": 10}) + + def test_latest_metric_value_is_kept(self): + """Metrics are a time series; only the last value per key survives.""" + self.assertEqual(self._record("run-b")["metrics"]["acc"], 0.95) + + def test_record_carries_identity_fields(self): + """Name and status ride along for the table header.""" + record = self._record("run-b") + self.assertEqual(record["name"], "beta") + self.assertEqual(record["status"], "finished") + + def test_missing_param_is_absent_not_null(self): + """A run without a param simply omits it (columns are unioned client-side).""" + self.assertNotIn("momentum", self._record("run-a")["params"]) + self.assertNotIn("lr", self._record("run-c")["params"]) + + def test_empty_input_is_empty_payload(self): + """No experiments yields empty records and empty key unions.""" + payload = _flatten_experiments([]) + self.assertEqual(payload["records"], []) + self.assertEqual(payload["param_keys"], []) + self.assertEqual(payload["metric_keys"], []) + + def test_non_dict_entries_are_skipped(self): + """Defensive: a malformed entry does not abort the whole flatten.""" + payload = _flatten_experiments([None, "oops", {"env_id": "x"}]) + self.assertEqual(len(payload["records"]), 1) + self.assertEqual(payload["records"][0]["env_id"], "x") + + +class TestHparamsClientMessage(unittest.TestCase): + """Visdom.hparams builds the window the pane expects.""" + + def setUp(self): + self.vis = Visdom(send=False, raise_exceptions=True) + + def test_creates_an_hparams_window(self): + """The message posts to events and carries a single hparams pane.""" + msg, endpoint = self.vis.hparams() + self.assertEqual(endpoint, "events") + self.assertEqual(len(msg["data"]), 1) + self.assertEqual(msg["data"][0]["type"], "hparams") + + def test_content_has_records_shape(self): + """The pane content always exposes the three keys the frontend reads.""" + msg, _ = self.vis.hparams() + content = msg["data"][0]["content"] + self.assertIn("records", content) + self.assertIn("param_keys", content) + self.assertIn("metric_keys", content) + + def test_env_and_win_pass_through(self): + """win/env target a specific pane like the other plotting methods.""" + msg, _ = self.vis.hparams(win="hp1", env="run-x") + self.assertEqual(msg["win"], "hp1") + self.assertEqual(msg["eid"], "run-x") + + def test_rejects_non_list_env_ids(self): + """env_ids must be a list/tuple of ids, not a bare string.""" + with self.assertRaises(TypeError): + self.vis.hparams(env_ids="run-a") + + def test_rejects_non_string_env_ids(self): + """env_ids must contain strings.""" + with self.assertRaises(TypeError): + self.vis.hparams(env_ids=["run-a", 3]) + + def test_env_ids_filter_and_order(self): + """env_ids selects and orders runs out of the fetched set. + + The search call is stubbed so the filter/flatten path runs without a + server; hparams should keep only the named runs, in the order given. + """ + canned = { + "experiments": [ + {"env_id": "run-a", "name": "a", "params": [], "metrics": []}, + {"env_id": "run-b", "name": "b", "params": [], "metrics": []}, + {"env_id": "run-c", "name": "c", "params": [], "metrics": []}, + ] + } + self.vis.search_experiments = lambda query=None, limit=None: canned + msg, _ = self.vis.hparams(env_ids=["run-c", "run-a"]) + ordered = [r["env_id"] for r in msg["data"][0]["content"]["records"]] + self.assertEqual(ordered, ["run-c", "run-a"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index cef627bbd..f5d2da85d 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -656,6 +656,58 @@ def _decode_binary_arrays(obj): return obj +def _flatten_experiments(experiments): + """Flatten experiment dicts into a compact records payload for the hparams pane. + + ``experiments`` is a list of experiment dicts as returned by the + ``experiments/search`` endpoint (see :class:`visdom.experiments.Experiment`). + Each experiment carries ``params``/``metrics``/``tags`` as lists of + ``{"key": ..., "value": ...}`` dicts; this collapses them into per-run maps so + the frontend renders a table/parallel-coordinates view without re-deriving the + column set. Metrics form a time series, so only each metric's *latest* logged + value is kept (the last observation for that key). + + Returns a dict of ``records`` (one flattened row per run), the sorted + ``param_keys`` union and the sorted ``metric_keys`` union across all runs. + """ + records = [] + param_keys = set() + metric_keys = set() + for exp in experiments: + if not isinstance(exp, dict): + continue + params = {} + for param in exp.get("params", []) or []: + key = param.get("key") + if key is None: + continue + params[key] = param.get("value") + param_keys.add(key) + # Metrics are appended in order; keep the last value seen per key. + metrics = {} + for metric in exp.get("metrics", []) or []: + key = metric.get("key") + if key is None: + continue + metrics[key] = metric.get("value") + metric_keys.add(key) + records.append( + { + "env_id": exp.get("env_id"), + "name": exp.get("name", exp.get("env_id")), + "status": exp.get("status"), + "created_at": exp.get("created_at"), + "params": params, + "metrics": metrics, + } + ) + return { + "records": records, + "param_keys": sorted(param_keys), + "metric_keys": sorted(metric_keys), + } + + class Visdom(object): def __init__( self, @@ -1300,6 +1352,54 @@ def suggest_experiment(self, params=None, env=None): msg["eid"] = env return self._experiment_request(msg, "experiments/suggest") + def hparams(self, query=None, env_ids=None, win=None, env=None, opts=None): + """Open a hyper-parameter pane over the experiments logged on the server. + + Gathers experiments via :meth:`search_experiments` (so the same `query` + syntax applies, and `query=None` pulls every logged run), optionally + restricting to an explicit list of `env_ids`, then flattens them into a + table of hyper-parameters against their latest metric values and renders + it in a dedicated ``hparams`` window. The window travels through the + normal window machinery, so it persists and reloads like any other pane. + + vis.hparams() # every logged experiment + vis.hparams("lr < 0.01 AND acc > 0.9") + vis.hparams(env_ids=["run-a", "run-b"]) + + `win`/`env`/`opts` behave as they do for the other plotting methods. + Returns the created window id (or the raw send result when this client is + constructed with `send=False`). + """ + if env_ids is not None: + if isstr(env_ids) or not isinstance(env_ids, (list, tuple)): + raise TypeError("env_ids must be a list of environment ids") + if not all(isstr(env_id) for env_id in env_ids): + raise TypeError("env_ids must contain strings") + + opts = {} if opts is None else opts + _title2str(opts) + _assert_opts(opts) + + reply = self.search_experiments(query=query, limit=None) + experiments = reply.get("experiments", []) if isinstance(reply, dict) else [] + if env_ids is not None: + wanted = list(dict.fromkeys(env_ids)) + by_id = {exp.get("env_id"): exp for exp in experiments} + experiments = [by_id[eid] for eid in wanted if eid in by_id] + + content = _flatten_experiments(experiments) + data = [{"content": content, "type": "hparams"}] + + return self._send( + { + "data": data, + "win": win, + "eid": env, + "opts": opts, + }, + endpoint="events", + ) + def get_window_data(self, win=None, env=None): """ This function returns all the window data for a specified window in diff --git a/py/visdom/__init__.pyi b/py/visdom/__init__.pyi index ab6aaac18..dcd8b8f1d 100644 --- a/py/visdom/__init__.pyi +++ b/py/visdom/__init__.pyi @@ -83,6 +83,14 @@ class Visdom: def suggest_experiment( self, params: _OptOps = ..., env: _OptStr = ... ) -> _ExperimentReply: ... + def hparams( + self, + query: _OptStr = ..., + env_ids: Optional[_EnvIds] = ..., + win: _OptStr = ..., + env: _OptStr = ..., + opts: _OptOps = ..., + ) -> _SendReturn: ... def get_window_data( self, win: _OptStr = ..., env: _OptStr = ... ) -> _SendReturn: ... From a0e821924b4b1e4721c3c8c1eae97695ee171e60 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Fri, 17 Jul 2026 16:37:39 +0530 Subject: [PATCH 15/48] hparams mode select, tags in flatten, test cleanup Add a `mode` argument to `vis.hparams` ("query" | "env_ids" | "both", default "both") so the caller explicitly chooses how the shown runs are selected: by query alone, by an explicit env_ids list alone, or the intersection of the two. mode="env_ids" requires env_ids and skips the query; mode="query" ignores env_ids. Include tags in `_flatten_experiments`: each run now carries a tags map alongside params/metrics, and the payload exposes a sorted tag_keys union, matching the server-side build_record which also flattens tags. Switch the flatten test fixture to TemporaryDirectory with a tearDown so the seeded JSONStore is removed after each test instead of leaking temp directories, and extend coverage for the mode paths and tag flattening. --- py/tests/test_hparams.py | 103 ++++++++++++++++++++++++++++++++------- py/visdom/__init__.py | 73 +++++++++++++++++++-------- py/visdom/__init__.pyi | 1 + 3 files changed, 141 insertions(+), 36 deletions(-) diff --git a/py/tests/test_hparams.py b/py/tests/test_hparams.py index 5579de92f..ea2a406b9 100644 --- a/py/tests/test_hparams.py +++ b/py/tests/test_hparams.py @@ -18,17 +18,26 @@ def seed_experiments(store): - """Log three runs with heterogeneous params and a metric time series.""" - store.log_experiment("run-a", name="alpha", params={"lr": 0.1, "epochs": 10}) + """Log three runs with heterogeneous params/tags and a metric time series.""" + store.log_experiment( + "run-a", + name="alpha", + params={"lr": 0.1, "epochs": 10}, + tags={"dataset": "mnist"}, + ) store.log_metric("run-a", "acc", 0.80) - store.log_experiment("run-b", name="beta", params={"lr": 0.001, "epochs": 20}) + store.log_experiment( + "run-b", + name="beta", + params={"lr": 0.001, "epochs": 20}, + tags={"dataset": "cifar10", "owner": "mira"}, + ) store.log_metric("run-b", "acc", 0.55) - store.log_metric("run-b", "acc", 0.95) # latest wins + store.log_metric("run-b", "acc", 0.95) store.log_metric("run-b", "loss", 0.1) store.finish_experiment("run-b") - # run-c only carries a param that the other two do not. store.log_experiment("run-c", name="gamma", params={"momentum": 0.9}) @@ -41,11 +50,14 @@ class TestFlattenExperiments(unittest.TestCase): """_flatten_experiments collapses lists of params/metrics into per-run maps.""" def setUp(self): - self._tmp_dir = tempfile.mkdtemp(prefix="visdom_hparams_flatten_") - self.store = ExperimentStore(JSONStore(self._tmp_dir)) + self._tmp = tempfile.TemporaryDirectory() + self.store = ExperimentStore(JSONStore(self._tmp.name)) seed_experiments(self.store) self.payload = _flatten_experiments(search_dicts(self.store)) + def tearDown(self): + self._tmp.cleanup() + def _record(self, env_id): for record in self.payload["records"]: if record["env_id"] == env_id: @@ -64,6 +76,20 @@ def test_metric_keys_are_sorted_union(self): """metric_keys is the sorted union of every run's metric names.""" self.assertEqual(self.payload["metric_keys"], ["acc", "loss"]) + def test_tag_keys_are_sorted_union(self): + """tag_keys is the sorted union of every run's tag names.""" + self.assertEqual(self.payload["tag_keys"], ["dataset", "owner"]) + + def test_tags_collapse_to_a_map(self): + """A run's tags flatten to a {name: value} map on the record.""" + self.assertEqual( + self._record("run-b")["tags"], {"dataset": "cifar10", "owner": "mira"} + ) + + def test_run_without_tags_has_empty_tag_map(self): + """A run with no tags still exposes a tags key (an empty map).""" + self.assertEqual(self._record("run-c")["tags"], {}) + def test_params_collapse_to_a_map(self): """A run's params flatten to a {name: value} map.""" self.assertEqual(self._record("run-a")["params"], {"lr": 0.1, "epochs": 10}) @@ -89,6 +115,7 @@ def test_empty_input_is_empty_payload(self): self.assertEqual(payload["records"], []) self.assertEqual(payload["param_keys"], []) self.assertEqual(payload["metric_keys"], []) + self.assertEqual(payload["tag_keys"], []) def test_non_dict_entries_are_skipped(self): """Defensive: a malformed entry does not abort the whole flatten.""" @@ -111,12 +138,13 @@ def test_creates_an_hparams_window(self): self.assertEqual(msg["data"][0]["type"], "hparams") def test_content_has_records_shape(self): - """The pane content always exposes the three keys the frontend reads.""" + """The pane content always exposes the keys the frontend reads.""" msg, _ = self.vis.hparams() content = msg["data"][0]["content"] self.assertIn("records", content) self.assertIn("param_keys", content) self.assertIn("metric_keys", content) + self.assertIn("tag_keys", content) def test_env_and_win_pass_through(self): """win/env target a specific pane like the other plotting methods.""" @@ -134,12 +162,9 @@ def test_rejects_non_string_env_ids(self): with self.assertRaises(TypeError): self.vis.hparams(env_ids=["run-a", 3]) - def test_env_ids_filter_and_order(self): - """env_ids selects and orders runs out of the fetched set. - - The search call is stubbed so the filter/flatten path runs without a - server; hparams should keep only the named runs, in the order given. - """ + def _stub_search(self): + """Stub search_experiments to record its query and return three runs.""" + self._seen = {} canned = { "experiments": [ {"env_id": "run-a", "name": "a", "params": [], "metrics": []}, @@ -147,10 +172,54 @@ def test_env_ids_filter_and_order(self): {"env_id": "run-c", "name": "c", "params": [], "metrics": []}, ] } - self.vis.search_experiments = lambda query=None, limit=None: canned + + def fake_search(query=None, limit=None): + self._seen["query"] = query + return canned + + self.vis.search_experiments = fake_search + + def _ordered(self, msg): + return [r["env_id"] for r in msg["data"][0]["content"]["records"]] + + def test_env_ids_filter_and_order(self): + """env_ids selects and orders runs out of the fetched set (default mode).""" + self._stub_search() msg, _ = self.vis.hparams(env_ids=["run-c", "run-a"]) - ordered = [r["env_id"] for r in msg["data"][0]["content"]["records"]] - self.assertEqual(ordered, ["run-c", "run-a"]) + self.assertEqual(self._ordered(msg), ["run-c", "run-a"]) + + def test_mode_query_ignores_env_ids(self): + """mode='query' forwards the query and does not narrow by env_ids.""" + self._stub_search() + msg, _ = self.vis.hparams(query="acc > 0.9", env_ids=["run-a"], mode="query") + self.assertEqual(self._seen["query"], "acc > 0.9") + self.assertEqual(self._ordered(msg), ["run-a", "run-b", "run-c"]) + + def test_mode_env_ids_ignores_query(self): + """mode='env_ids' sends no query and narrows to the named runs.""" + self._stub_search() + msg, _ = self.vis.hparams(query="acc > 0.9", env_ids=["run-b"], mode="env_ids") + self.assertIsNone(self._seen["query"]) + self.assertEqual(self._ordered(msg), ["run-b"]) + + def test_mode_both_intersects(self): + """mode='both' forwards the query and then narrows by env_ids.""" + self._stub_search() + msg, _ = self.vis.hparams( + query="acc > 0.9", env_ids=["run-c", "run-a"], mode="both" + ) + self.assertEqual(self._seen["query"], "acc > 0.9") + self.assertEqual(self._ordered(msg), ["run-c", "run-a"]) + + def test_mode_env_ids_requires_env_ids(self): + """mode='env_ids' without env_ids is a usage error.""" + with self.assertRaises(ValueError): + self.vis.hparams(mode="env_ids") + + def test_rejects_unknown_mode(self): + """An unrecognised mode is rejected before any request.""" + with self.assertRaises(ValueError): + self.vis.hparams(mode="sideways") if __name__ == "__main__": diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index f5d2da85d..155562073 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -662,17 +662,19 @@ def _flatten_experiments(experiments): ``experiments`` is a list of experiment dicts as returned by the ``experiments/search`` endpoint (see :class:`visdom.experiments.Experiment`). Each experiment carries ``params``/``metrics``/``tags`` as lists of - ``{"key": ..., "value": ...}`` dicts; this collapses them into per-run maps so - the frontend renders a table/parallel-coordinates view without re-deriving the - column set. Metrics form a time series, so only each metric's *latest* logged - value is kept (the last observation for that key). - - Returns a dict of ``records`` (one flattened row per run), the sorted - ``param_keys`` union and the sorted ``metric_keys`` union across all runs. + ``{"key": ..., "value": ...}`` dicts; this collapses each of the three into a + per-run ``{name: value}`` map so the frontend renders a table/parallel- + coordinates view without re-deriving the column set. Metrics form a time + series, so only each metric's *latest* logged value is kept (the last + observation for that key). + + Returns a dict of ``records`` (one flattened row per run) plus the sorted + ``param_keys``, ``metric_keys`` and ``tag_keys`` unions across all runs. """ records = [] param_keys = set() metric_keys = set() + tag_keys = set() for exp in experiments: if not isinstance(exp, dict): continue @@ -683,7 +685,6 @@ def _flatten_experiments(experiments): continue params[key] = param.get("value") param_keys.add(key) - # Metrics are appended in order; keep the last value seen per key. metrics = {} for metric in exp.get("metrics", []) or []: key = metric.get("key") @@ -691,6 +692,13 @@ def _flatten_experiments(experiments): continue metrics[key] = metric.get("value") metric_keys.add(key) + tags = {} + for tag in exp.get("tags", []) or []: + key = tag.get("key") + if key is None: + continue + tags[key] = tag.get("value") + tag_keys.add(key) records.append( { "env_id": exp.get("env_id"), @@ -699,12 +707,14 @@ def _flatten_experiments(experiments): "created_at": exp.get("created_at"), "params": params, "metrics": metrics, + "tags": tags, } ) return { "records": records, "param_keys": sorted(param_keys), "metric_keys": sorted(metric_keys), + "tag_keys": sorted(tag_keys), } @@ -1352,25 +1362,50 @@ def suggest_experiment(self, params=None, env=None): msg["eid"] = env return self._experiment_request(msg, "experiments/suggest") - def hparams(self, query=None, env_ids=None, win=None, env=None, opts=None): + def hparams( + self, query=None, env_ids=None, mode="both", win=None, env=None, opts=None + ): """Open a hyper-parameter pane over the experiments logged on the server. - Gathers experiments via :meth:`search_experiments` (so the same `query` - syntax applies, and `query=None` pulls every logged run), optionally - restricting to an explicit list of `env_ids`, then flattens them into a - table of hyper-parameters against their latest metric values and renders - it in a dedicated ``hparams`` window. The window travels through the - normal window machinery, so it persists and reloads like any other pane. + Gathers experiments, flattens them into a table of hyper-parameters + against their latest metric values (and tags), and renders it in a + dedicated ``hparams`` window that persists/reloads like any other pane. + + `mode` chooses how the runs to show are selected: - vis.hparams() # every logged experiment + * ``"query"`` — only the runs matching `query` (the readable syntax of + :meth:`search_experiments`; `query=None` means every run). `env_ids` + is ignored. + * ``"env_ids"`` — only the runs named in `env_ids`, in that order. + `query` is ignored, and `env_ids` is required. + * ``"both"`` (default) — the intersection: runs that match `query` *and* + are named in `env_ids`, ordered by `env_ids`. With only one of the two + supplied it degrades to that one (so the single-argument calls below + work under the default). + + :: + + vis.hparams() vis.hparams("lr < 0.01 AND acc > 0.9") vis.hparams(env_ids=["run-a", "run-b"]) + vis.hparams("acc > 0.9", ["run-a", "run-b"]) + vis.hparams("acc > 0.9", ["run-a"], mode="query") `win`/`env`/`opts` behave as they do for the other plotting methods. Returns the created window id (or the raw send result when this client is constructed with `send=False`). """ - if env_ids is not None: + valid_modes = ("query", "env_ids", "both") + if mode not in valid_modes: + raise ValueError( + "mode must be one of {0}, got {1!r}".format(valid_modes, mode) + ) + use_query = mode in ("query", "both") + use_env_ids = mode in ("env_ids", "both") + + if mode == "env_ids" and env_ids is None: + raise ValueError("mode='env_ids' requires env_ids") + if use_env_ids and env_ids is not None: if isstr(env_ids) or not isinstance(env_ids, (list, tuple)): raise TypeError("env_ids must be a list of environment ids") if not all(isstr(env_id) for env_id in env_ids): @@ -1380,9 +1415,9 @@ def hparams(self, query=None, env_ids=None, win=None, env=None, opts=None): _title2str(opts) _assert_opts(opts) - reply = self.search_experiments(query=query, limit=None) + reply = self.search_experiments(query=query if use_query else None, limit=None) experiments = reply.get("experiments", []) if isinstance(reply, dict) else [] - if env_ids is not None: + if use_env_ids and env_ids is not None: wanted = list(dict.fromkeys(env_ids)) by_id = {exp.get("env_id"): exp for exp in experiments} experiments = [by_id[eid] for eid in wanted if eid in by_id] diff --git a/py/visdom/__init__.pyi b/py/visdom/__init__.pyi index dcd8b8f1d..6ab739e33 100644 --- a/py/visdom/__init__.pyi +++ b/py/visdom/__init__.pyi @@ -87,6 +87,7 @@ class Visdom: self, query: _OptStr = ..., env_ids: Optional[_EnvIds] = ..., + mode: Text = ..., win: _OptStr = ..., env: _OptStr = ..., opts: _OptOps = ..., From 05aefbb47a94caffe32dfe901dd9d857564e959c Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Fri, 17 Jul 2026 16:55:44 +0530 Subject: [PATCH 16/48] fetch hparams runs by id when no query is used Previously the env_ids selection always went through search_experiments, which reads every environment on disk and returns every experiment over the wire, only for the client to keep the few it named. When no query is in play, route the selection through compare_experiments instead, which reads only the named environments and returns just those runs; a query, when present, still uses search. Also type-check query up front and treat a blank/whitespace query as no query so it takes the by-id path too. Tests stub both read endpoints and assert which one each mode reaches. --- py/tests/test_hparams.py | 87 +++++++++++++++++++++++++++------------- py/visdom/__init__.py | 24 +++++++++-- 2 files changed, 80 insertions(+), 31 deletions(-) diff --git a/py/tests/test_hparams.py b/py/tests/test_hparams.py index ea2a406b9..977cfa996 100644 --- a/py/tests/test_hparams.py +++ b/py/tests/test_hparams.py @@ -162,55 +162,86 @@ def test_rejects_non_string_env_ids(self): with self.assertRaises(TypeError): self.vis.hparams(env_ids=["run-a", 3]) - def _stub_search(self): - """Stub search_experiments to record its query and return three runs.""" - self._seen = {} - canned = { - "experiments": [ - {"env_id": "run-a", "name": "a", "params": [], "metrics": []}, - {"env_id": "run-b", "name": "b", "params": [], "metrics": []}, - {"env_id": "run-c", "name": "c", "params": [], "metrics": []}, - ] - } + def _stub_endpoints(self): + """Stub both read endpoints, recording which one each call reaches. + + ``search`` records the query it was handed; ``compare`` records the ids + and, like the real endpoint, returns only those runs. This lets the + tests assert that a query-less env_ids selection fetches by id rather + than pulling every experiment. + """ + self._seen = {"search": None, "compare": None} + runs = [ + {"env_id": "run-a", "name": "a", "params": [], "metrics": [], "tags": []}, + {"env_id": "run-b", "name": "b", "params": [], "metrics": [], "tags": []}, + {"env_id": "run-c", "name": "c", "params": [], "metrics": [], "tags": []}, + ] + by_id = {run["env_id"]: run for run in runs} def fake_search(query=None, limit=None): - self._seen["query"] = query - return canned + self._seen["search"] = query + return {"experiments": runs} + + def fake_compare(env_ids): + self._seen["compare"] = list(env_ids) + return {"experiments": [by_id[e] for e in env_ids if e in by_id]} self.vis.search_experiments = fake_search + self.vis.compare_experiments = fake_compare def _ordered(self, msg): return [r["env_id"] for r in msg["data"][0]["content"]["records"]] - def test_env_ids_filter_and_order(self): - """env_ids selects and orders runs out of the fetched set (default mode).""" - self._stub_search() + def test_no_selection_searches_everything(self): + """No query and no env_ids searches (query=None) for the full set.""" + self._stub_endpoints() + msg, _ = self.vis.hparams() + self.assertIsNone(self._seen["search"]) + self.assertIsNone(self._seen["compare"]) + self.assertEqual(self._ordered(msg), ["run-a", "run-b", "run-c"]) + + def test_env_ids_without_query_fetches_by_id(self): + """env_ids and no query fetches only the named runs via compare.""" + self._stub_endpoints() msg, _ = self.vis.hparams(env_ids=["run-c", "run-a"]) + self.assertEqual(self._seen["compare"], ["run-c", "run-a"]) + self.assertIsNone(self._seen["search"]) self.assertEqual(self._ordered(msg), ["run-c", "run-a"]) + def test_mode_env_ids_fetches_by_id_ignoring_query(self): + """mode='env_ids' fetches by id and never runs the query.""" + self._stub_endpoints() + msg, _ = self.vis.hparams(query="acc > 0.9", env_ids=["run-b"], mode="env_ids") + self.assertEqual(self._seen["compare"], ["run-b"]) + self.assertIsNone(self._seen["search"]) + self.assertEqual(self._ordered(msg), ["run-b"]) + def test_mode_query_ignores_env_ids(self): """mode='query' forwards the query and does not narrow by env_ids.""" - self._stub_search() + self._stub_endpoints() msg, _ = self.vis.hparams(query="acc > 0.9", env_ids=["run-a"], mode="query") - self.assertEqual(self._seen["query"], "acc > 0.9") + self.assertEqual(self._seen["search"], "acc > 0.9") + self.assertIsNone(self._seen["compare"]) self.assertEqual(self._ordered(msg), ["run-a", "run-b", "run-c"]) - def test_mode_env_ids_ignores_query(self): - """mode='env_ids' sends no query and narrows to the named runs.""" - self._stub_search() - msg, _ = self.vis.hparams(query="acc > 0.9", env_ids=["run-b"], mode="env_ids") - self.assertIsNone(self._seen["query"]) - self.assertEqual(self._ordered(msg), ["run-b"]) - - def test_mode_both_intersects(self): - """mode='both' forwards the query and then narrows by env_ids.""" - self._stub_search() + def test_mode_both_with_query_searches_then_narrows(self): + """mode='both' with a query searches, then narrows by env_ids.""" + self._stub_endpoints() msg, _ = self.vis.hparams( query="acc > 0.9", env_ids=["run-c", "run-a"], mode="both" ) - self.assertEqual(self._seen["query"], "acc > 0.9") + self.assertEqual(self._seen["search"], "acc > 0.9") + self.assertIsNone(self._seen["compare"]) self.assertEqual(self._ordered(msg), ["run-c", "run-a"]) + def test_blank_query_with_env_ids_fetches_by_id(self): + """A blank query is treated as no query, so it fetches by id.""" + self._stub_endpoints() + msg, _ = self.vis.hparams(query=" ", env_ids=["run-b"]) + self.assertEqual(self._seen["compare"], ["run-b"]) + self.assertIsNone(self._seen["search"]) + self.assertEqual(self._ordered(msg), ["run-b"]) + def test_mode_env_ids_requires_env_ids(self): """mode='env_ids' without env_ids is a usage error.""" with self.assertRaises(ValueError): diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index 155562073..030e255cc 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -1383,6 +1383,12 @@ def hparams( supplied it degrades to that one (so the single-argument calls below work under the default). + When no query is in play but `env_ids` is given, the named runs are + fetched directly (through :meth:`compare_experiments`, which reads only + those environments) rather than pulling every experiment and discarding + the rest; a query, when present, still goes through + :meth:`search_experiments`. + :: vis.hparams() @@ -1403,6 +1409,8 @@ def hparams( use_query = mode in ("query", "both") use_env_ids = mode in ("env_ids", "both") + if query is not None and not isstr(query): + raise TypeError("query must be a string") if mode == "env_ids" and env_ids is None: raise ValueError("mode='env_ids' requires env_ids") if use_env_ids and env_ids is not None: @@ -1415,10 +1423,20 @@ def hparams( _title2str(opts) _assert_opts(opts) - reply = self.search_experiments(query=query if use_query else None, limit=None) + active_query = query if (use_query and query and query.strip()) else None + wanted = ( + list(dict.fromkeys(env_ids)) + if (use_env_ids and env_ids is not None) + else None + ) + + if wanted and active_query is None: + reply = self.compare_experiments(wanted) + else: + reply = self.search_experiments(query=active_query, limit=None) experiments = reply.get("experiments", []) if isinstance(reply, dict) else [] - if use_env_ids and env_ids is not None: - wanted = list(dict.fromkeys(env_ids)) + + if wanted is not None: by_id = {exp.get("env_id"): exp for exp in experiments} experiments = [by_id[eid] for eid in wanted if eid in by_id] From 0e995d3ff180dd8ceeb2097384c92accc00a38a3 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Fri, 17 Jul 2026 17:42:20 +0530 Subject: [PATCH 17/48] strict per-mode selection for hparams Make the three selection modes strict and mutually exclusive in their arguments instead of leniently combining them: query -> non-empty query, no env_ids; fetched via search env_ids -> non-empty env_ids, no query; fetched via compare (reads only the named environments) both -> both required and non-empty; search then narrow by env_ids mode defaults to None and is inferred from which of query/env_ids are given; an explicit mode enforces its rule and rejects the wrong argument. A blank or whitespace-only query counts as no query. There is no "show everything" call: with neither query nor env_ids a ValueError is raised. --- py/tests/test_hparams.py | 149 +++++++++++++++++++-------------------- py/visdom/__init__.py | 88 +++++++++++++---------- py/visdom/__init__.pyi | 2 +- 3 files changed, 127 insertions(+), 112 deletions(-) diff --git a/py/tests/test_hparams.py b/py/tests/test_hparams.py index 977cfa996..5e0f47e3d 100644 --- a/py/tests/test_hparams.py +++ b/py/tests/test_hparams.py @@ -125,50 +125,19 @@ def test_non_dict_entries_are_skipped(self): class TestHparamsClientMessage(unittest.TestCase): - """Visdom.hparams builds the window the pane expects.""" + """Visdom.hparams selects runs per mode and builds the pane window.""" def setUp(self): self.vis = Visdom(send=False, raise_exceptions=True) - - def test_creates_an_hparams_window(self): - """The message posts to events and carries a single hparams pane.""" - msg, endpoint = self.vis.hparams() - self.assertEqual(endpoint, "events") - self.assertEqual(len(msg["data"]), 1) - self.assertEqual(msg["data"][0]["type"], "hparams") - - def test_content_has_records_shape(self): - """The pane content always exposes the keys the frontend reads.""" - msg, _ = self.vis.hparams() - content = msg["data"][0]["content"] - self.assertIn("records", content) - self.assertIn("param_keys", content) - self.assertIn("metric_keys", content) - self.assertIn("tag_keys", content) - - def test_env_and_win_pass_through(self): - """win/env target a specific pane like the other plotting methods.""" - msg, _ = self.vis.hparams(win="hp1", env="run-x") - self.assertEqual(msg["win"], "hp1") - self.assertEqual(msg["eid"], "run-x") - - def test_rejects_non_list_env_ids(self): - """env_ids must be a list/tuple of ids, not a bare string.""" - with self.assertRaises(TypeError): - self.vis.hparams(env_ids="run-a") - - def test_rejects_non_string_env_ids(self): - """env_ids must contain strings.""" - with self.assertRaises(TypeError): - self.vis.hparams(env_ids=["run-a", 3]) + self._stub_endpoints() def _stub_endpoints(self): """Stub both read endpoints, recording which one each call reaches. ``search`` records the query it was handed; ``compare`` records the ids and, like the real endpoint, returns only those runs. This lets the - tests assert that a query-less env_ids selection fetches by id rather - than pulling every experiment. + tests assert that an env_ids selection fetches by id rather than pulling + every experiment through search. """ self._seen = {"search": None, "compare": None} runs = [ @@ -192,65 +161,95 @@ def fake_compare(env_ids): def _ordered(self, msg): return [r["env_id"] for r in msg["data"][0]["content"]["records"]] - def test_no_selection_searches_everything(self): - """No query and no env_ids searches (query=None) for the full set.""" - self._stub_endpoints() - msg, _ = self.vis.hparams() - self.assertIsNone(self._seen["search"]) + def test_query_only_creates_hparams_window_via_search(self): + """A query builds one hparams pane and is fetched through search.""" + msg, endpoint = self.vis.hparams("acc > 0.9") + self.assertEqual(endpoint, "events") + self.assertEqual(len(msg["data"]), 1) + self.assertEqual(msg["data"][0]["type"], "hparams") + self.assertEqual(self._seen["search"], "acc > 0.9") self.assertIsNone(self._seen["compare"]) - self.assertEqual(self._ordered(msg), ["run-a", "run-b", "run-c"]) - def test_env_ids_without_query_fetches_by_id(self): - """env_ids and no query fetches only the named runs via compare.""" - self._stub_endpoints() + def test_content_has_records_shape(self): + """The pane content exposes the keys the frontend reads.""" + msg, _ = self.vis.hparams("acc > 0.9") + content = msg["data"][0]["content"] + self.assertIn("records", content) + self.assertIn("param_keys", content) + self.assertIn("metric_keys", content) + self.assertIn("tag_keys", content) + + def test_env_and_win_pass_through(self): + """win/env target a specific pane like the other plotting methods.""" + msg, _ = self.vis.hparams("acc > 0.9", win="hp1", env="run-x") + self.assertEqual(msg["win"], "hp1") + self.assertEqual(msg["eid"], "run-x") + + def test_env_ids_only_fetches_by_id_via_compare(self): + """env_ids alone infers env_ids mode and fetches only the named runs.""" msg, _ = self.vis.hparams(env_ids=["run-c", "run-a"]) self.assertEqual(self._seen["compare"], ["run-c", "run-a"]) self.assertIsNone(self._seen["search"]) self.assertEqual(self._ordered(msg), ["run-c", "run-a"]) - def test_mode_env_ids_fetches_by_id_ignoring_query(self): - """mode='env_ids' fetches by id and never runs the query.""" - self._stub_endpoints() - msg, _ = self.vis.hparams(query="acc > 0.9", env_ids=["run-b"], mode="env_ids") - self.assertEqual(self._seen["compare"], ["run-b"]) - self.assertIsNone(self._seen["search"]) - self.assertEqual(self._ordered(msg), ["run-b"]) - - def test_mode_query_ignores_env_ids(self): - """mode='query' forwards the query and does not narrow by env_ids.""" - self._stub_endpoints() - msg, _ = self.vis.hparams(query="acc > 0.9", env_ids=["run-a"], mode="query") - self.assertEqual(self._seen["search"], "acc > 0.9") - self.assertIsNone(self._seen["compare"]) - self.assertEqual(self._ordered(msg), ["run-a", "run-b", "run-c"]) - - def test_mode_both_with_query_searches_then_narrows(self): - """mode='both' with a query searches, then narrows by env_ids.""" - self._stub_endpoints() - msg, _ = self.vis.hparams( - query="acc > 0.9", env_ids=["run-c", "run-a"], mode="both" - ) + def test_both_infers_and_searches_then_narrows(self): + """query + env_ids infers both mode: search, then narrow by env_ids.""" + msg, _ = self.vis.hparams("acc > 0.9", ["run-c", "run-a"]) self.assertEqual(self._seen["search"], "acc > 0.9") self.assertIsNone(self._seen["compare"]) self.assertEqual(self._ordered(msg), ["run-c", "run-a"]) - def test_blank_query_with_env_ids_fetches_by_id(self): - """A blank query is treated as no query, so it fetches by id.""" - self._stub_endpoints() - msg, _ = self.vis.hparams(query=" ", env_ids=["run-b"]) - self.assertEqual(self._seen["compare"], ["run-b"]) - self.assertIsNone(self._seen["search"]) - self.assertEqual(self._ordered(msg), ["run-b"]) + def test_no_arguments_is_an_error(self): + """With neither query nor env_ids there is nothing to select.""" + with self.assertRaises(ValueError): + self.vis.hparams() + + def test_blank_query_alone_is_an_error(self): + """A blank query counts as no query, so it selects nothing.""" + with self.assertRaises(ValueError): + self.vis.hparams(" ") + + def test_mode_query_rejects_env_ids(self): + """mode='query' does not accept env_ids.""" + with self.assertRaises(ValueError): + self.vis.hparams("acc > 0.9", ["run-a"], mode="query") + + def test_mode_query_requires_nonempty_query(self): + """mode='query' needs an actual query.""" + with self.assertRaises(ValueError): + self.vis.hparams(mode="query") + + def test_mode_env_ids_rejects_query(self): + """mode='env_ids' does not accept a query.""" + with self.assertRaises(ValueError): + self.vis.hparams("acc > 0.9", ["run-b"], mode="env_ids") def test_mode_env_ids_requires_env_ids(self): - """mode='env_ids' without env_ids is a usage error.""" + """mode='env_ids' needs a non-empty env_ids.""" with self.assertRaises(ValueError): self.vis.hparams(mode="env_ids") + def test_mode_both_requires_both(self): + """mode='both' needs both a query and env_ids.""" + with self.assertRaises(ValueError): + self.vis.hparams("acc > 0.9", mode="both") + with self.assertRaises(ValueError): + self.vis.hparams(env_ids=["run-a"], mode="both") + + def test_rejects_non_list_env_ids(self): + """env_ids must be a list/tuple of ids, not a bare string.""" + with self.assertRaises(TypeError): + self.vis.hparams(env_ids="run-a") + + def test_rejects_non_string_env_ids(self): + """env_ids must contain strings.""" + with self.assertRaises(TypeError): + self.vis.hparams(env_ids=["run-a", 3]) + def test_rejects_unknown_mode(self): """An unrecognised mode is rejected before any request.""" with self.assertRaises(ValueError): - self.vis.hparams(mode="sideways") + self.vis.hparams("acc > 0.9", mode="sideways") if __name__ == "__main__": diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index 030e255cc..51eaa0e82 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -1363,7 +1363,7 @@ def suggest_experiment(self, params=None, env=None): return self._experiment_request(msg, "experiments/suggest") def hparams( - self, query=None, env_ids=None, mode="both", win=None, env=None, opts=None + self, query=None, env_ids=None, mode=None, win=None, env=None, opts=None ): """Open a hyper-parameter pane over the experiments logged on the server. @@ -1371,69 +1371,85 @@ def hparams( against their latest metric values (and tags), and renders it in a dedicated ``hparams`` window that persists/reloads like any other pane. - `mode` chooses how the runs to show are selected: + `mode` chooses how the runs to show are selected; when it is left as + `None` it is inferred from which of `query`/`env_ids` were supplied: - * ``"query"`` — only the runs matching `query` (the readable syntax of - :meth:`search_experiments`; `query=None` means every run). `env_ids` - is ignored. - * ``"env_ids"`` — only the runs named in `env_ids`, in that order. - `query` is ignored, and `env_ids` is required. - * ``"both"`` (default) — the intersection: runs that match `query` *and* - are named in `env_ids`, ordered by `env_ids`. With only one of the two - supplied it degrades to that one (so the single-argument calls below - work under the default). + * ``"query"`` — the runs matching `query` (the readable syntax of + :meth:`search_experiments`). The query must be non-empty and `env_ids` + must not be given. Fetched with :meth:`search_experiments`. + * ``"env_ids"`` — the runs named in `env_ids`, in that order. `env_ids` + must be non-empty and `query` must not be given. Fetched with + :meth:`compare_experiments`, which reads only those environments + instead of every experiment. + * ``"both"`` — the intersection: runs that match `query` *and* are named + in `env_ids`, ordered by `env_ids`. Both must be given and non-empty. - When no query is in play but `env_ids` is given, the named runs are - fetched directly (through :meth:`compare_experiments`, which reads only - those environments) rather than pulling every experiment and discarding - the rest; a query, when present, still goes through - :meth:`search_experiments`. + There is no "show everything" call: with neither `query` nor `env_ids` + there is nothing to select, so a :class:`ValueError` is raised. A blank + or whitespace-only `query` counts as no query. :: - vis.hparams() - vis.hparams("lr < 0.01 AND acc > 0.9") - vis.hparams(env_ids=["run-a", "run-b"]) - vis.hparams("acc > 0.9", ["run-a", "run-b"]) - vis.hparams("acc > 0.9", ["run-a"], mode="query") + vis.hparams("lr < 0.01 AND acc > 0.9") # query + vis.hparams(env_ids=["run-a", "run-b"]) # env_ids + vis.hparams("acc > 0.9", ["run-a", "run-b"]) # both + vis.hparams("acc > 0.9", ["run-a"], mode="query") # forced, errors `win`/`env`/`opts` behave as they do for the other plotting methods. Returns the created window id (or the raw send result when this client is constructed with `send=False`). """ valid_modes = ("query", "env_ids", "both") - if mode not in valid_modes: + if mode is not None and mode not in valid_modes: raise ValueError( "mode must be one of {0}, got {1!r}".format(valid_modes, mode) ) - use_query = mode in ("query", "both") - use_env_ids = mode in ("env_ids", "both") - if query is not None and not isstr(query): raise TypeError("query must be a string") - if mode == "env_ids" and env_ids is None: - raise ValueError("mode='env_ids' requires env_ids") - if use_env_ids and env_ids is not None: + if env_ids is not None: if isstr(env_ids) or not isinstance(env_ids, (list, tuple)): raise TypeError("env_ids must be a list of environment ids") if not all(isstr(env_id) for env_id in env_ids): raise TypeError("env_ids must contain strings") + has_query = isstr(query) and query.strip() != "" + has_env_ids = env_ids is not None and len(env_ids) > 0 + + if mode is None: + if has_query and has_env_ids: + mode = "both" + elif has_query: + mode = "query" + elif has_env_ids: + mode = "env_ids" + else: + raise ValueError("hparams needs a query, env_ids, or both") + elif mode == "query": + if not has_query: + raise ValueError("mode='query' requires a non-empty query") + if env_ids is not None: + raise ValueError("mode='query' does not accept env_ids") + elif mode == "env_ids": + if query is not None: + raise ValueError("mode='env_ids' does not accept a query") + if not has_env_ids: + raise ValueError("mode='env_ids' requires a non-empty env_ids") + else: + if not has_query: + raise ValueError("mode='both' requires a non-empty query") + if not has_env_ids: + raise ValueError("mode='both' requires a non-empty env_ids") + opts = {} if opts is None else opts _title2str(opts) _assert_opts(opts) - active_query = query if (use_query and query and query.strip()) else None - wanted = ( - list(dict.fromkeys(env_ids)) - if (use_env_ids and env_ids is not None) - else None - ) + wanted = list(dict.fromkeys(env_ids)) if mode in ("env_ids", "both") else None - if wanted and active_query is None: + if mode == "env_ids": reply = self.compare_experiments(wanted) else: - reply = self.search_experiments(query=active_query, limit=None) + reply = self.search_experiments(query=query, limit=None) experiments = reply.get("experiments", []) if isinstance(reply, dict) else [] if wanted is not None: diff --git a/py/visdom/__init__.pyi b/py/visdom/__init__.pyi index 6ab739e33..062c62f9f 100644 --- a/py/visdom/__init__.pyi +++ b/py/visdom/__init__.pyi @@ -87,7 +87,7 @@ class Visdom: self, query: _OptStr = ..., env_ids: Optional[_EnvIds] = ..., - mode: Text = ..., + mode: _OptStr = ..., win: _OptStr = ..., env: _OptStr = ..., opts: _OptOps = ..., From a2f3358b3b830592b0002c5eac0b38b84efdfe1d Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Fri, 17 Jul 2026 18:46:11 +0530 Subject: [PATCH 18/48] Add /experiments/hparams endpoint that builds the pane; hparams calls it Move the hyper-parameter selection, flattening and window creation off the Visdom.hparams client and into a POST /experiments/hparams handler. The handler resolves the strict query/env_ids/both modes the client used to resolve itself (invalid combinations are 400s), flattens the selected runs via visdom.experiments.flatten_experiments, and writes an hparams window with that content into the env state. For now the handler only writes to state; it does not broadcast the window to connected clients, since the frontend has no dedicated hparams pane to render it yet. The pane is served on the next env load, and broadcasting lands with that pane. vis.hparams is now a thin call: it validates opts and posts query/env_ids/mode/win/opts to the endpoint, returning the created window id, instead of gathering runs and posting to the events endpoint itself. window() learns the hparams type so the content is stored as-is rather than as a plot. --- py/tests/test_experiment_hparams.py | 199 +++++++++++++ py/tests/test_hparams.py | 263 ++---------------- py/visdom/__init__.py | 147 +--------- py/visdom/experiments/__init__.py | 2 + py/visdom/experiments/records.py | 79 ++++++ py/visdom/server/app.py | 6 + .../server/handlers/experiments_handler.py | 190 +++++++++++++ py/visdom/utils/server_utils.py | 2 +- 8 files changed, 520 insertions(+), 368 deletions(-) create mode 100644 py/tests/test_experiment_hparams.py create mode 100644 py/visdom/experiments/records.py create mode 100644 py/visdom/server/handlers/experiments_handler.py diff --git a/py/tests/test_experiment_hparams.py b/py/tests/test_experiment_hparams.py new file mode 100644 index 000000000..21bf85b9f --- /dev/null +++ b/py/tests/test_experiment_hparams.py @@ -0,0 +1,199 @@ +"""Tests for the hyper-parameter endpoint (Layer 3). + +Covers the two pieces the endpoint is built from: the ``flatten_experiments`` +transform, and the ``/experiments/hparams`` endpoint end-to-end through a real +:class:`~visdom.server.app.Application` with Tornado's ``AsyncHTTPTestCase``. +The endpoint selects experiments (the strict query/env_ids/both modes the +client used to resolve itself), flattens them, and registers an ``hparams`` +window; the tests inspect the created window in the app state. Experiments are +seeded through a real ``ExperimentStore`` over a temporary directory. +""" + +import json +import shutil +import tempfile +import unittest + +import tornado.testing + +from visdom.data_model import JSONStore +from visdom.experiments import ExperimentStore, flatten_experiments +from visdom.server.app import Application + + +def seed_experiments(store): + """Log three runs with heterogeneous params/tags and a metric time series.""" + store.log_experiment( + "run-a", + name="alpha", + params={"lr": 0.1, "epochs": 10}, + tags={"dataset": "mnist"}, + ) + store.log_metric("run-a", "acc", 0.80) + + store.log_experiment( + "run-b", + name="beta", + params={"lr": 0.001, "epochs": 20}, + tags={"dataset": "cifar10", "owner": "mira"}, + ) + store.log_metric("run-b", "acc", 0.55) + store.log_metric("run-b", "acc", 0.95) + store.log_metric("run-b", "loss", 0.1) + + store.log_experiment("run-c", name="gamma", params={"momentum": 0.9}) + + +class TestFlattenTransform(unittest.TestCase): + """flatten_experiments collapses experiment dicts into the records payload.""" + + def setUp(self): + self._tmp = tempfile.TemporaryDirectory() + self.store = ExperimentStore(JSONStore(self._tmp.name)) + seed_experiments(self.store) + dicts = [experiment.to_dict() for experiment in self.store.search()] + self.payload = flatten_experiments(dicts) + + def tearDown(self): + self._tmp.cleanup() + + def _record(self, env_id): + for record in self.payload["records"]: + if record["env_id"] == env_id: + return record + self.fail("no record for {0!r}".format(env_id)) + + def test_one_record_per_run(self): + self.assertEqual(len(self.payload["records"]), 3) + + def test_key_unions_are_sorted(self): + self.assertEqual(self.payload["param_keys"], ["epochs", "lr", "momentum"]) + self.assertEqual(self.payload["metric_keys"], ["acc", "loss"]) + self.assertEqual(self.payload["tag_keys"], ["dataset", "owner"]) + + def test_latest_metric_value_is_kept(self): + self.assertEqual(self._record("run-b")["metrics"]["acc"], 0.95) + + def test_params_and_tags_are_maps(self): + record = self._record("run-a") + self.assertEqual(record["params"], {"lr": 0.1, "epochs": 10}) + self.assertEqual(record["tags"], {"dataset": "mnist"}) + + def test_missing_key_is_absent(self): + self.assertNotIn("momentum", self._record("run-a")["params"]) + self.assertEqual(self._record("run-c")["tags"], {}) + + def test_empty_input_is_empty_payload(self): + payload = flatten_experiments([]) + self.assertEqual(payload["records"], []) + self.assertEqual(payload["param_keys"], []) + self.assertEqual(payload["metric_keys"], []) + self.assertEqual(payload["tag_keys"], []) + + def test_non_dict_entries_are_skipped(self): + payload = flatten_experiments([None, "oops", {"env_id": "x"}]) + self.assertEqual(len(payload["records"]), 1) + self.assertEqual(payload["records"][0]["env_id"], "x") + + +class TestHparamsEndpoint(tornado.testing.AsyncHTTPTestCase): + """POST /experiments/hparams selects runs and registers the pane window.""" + + def setUp(self): + self._tmp_dir = tempfile.mkdtemp(prefix="visdom_exp_hparams_api_") + super().setUp() + seed_experiments(ExperimentStore(self._app.storage)) + + def tearDown(self): + super().tearDown() + shutil.rmtree(self._tmp_dir, ignore_errors=True) + + def get_app(self): + self._app = Application(port=self.get_http_port(), env_path=self._tmp_dir) + return self._app + + def hparams(self, body): + return self.fetch( + "/experiments/hparams", + method="POST", + body=json.dumps(body), + headers={"Content-Type": "application/json"}, + ) + + def _window(self, resp): + win_id = resp.body.decode() + return self._app.state["main"]["jsons"][win_id] + + def _env_ids(self, resp): + return [record["env_id"] for record in self._window(resp)["content"]["records"]] + + def test_query_creates_hparams_window(self): + """A query registers an hparams window holding the matching runs.""" + resp = self.hparams({"query": "lr < 0.01"}) + self.assertEqual(resp.code, 200) + window = self._window(resp) + self.assertEqual(window["type"], "hparams") + self.assertEqual(self._env_ids(resp), ["run-b"]) + + def test_content_carries_column_unions(self): + """The window content is the flattened matrix with column-name unions.""" + resp = self.hparams({"query": "epochs > 0"}) + content = self._window(resp)["content"] + self.assertEqual(content["param_keys"], ["epochs", "lr"]) + self.assertEqual(content["metric_keys"], ["acc", "loss"]) + + def test_env_ids_selects_and_orders(self): + """env_ids alone selects only the named runs, in the order given.""" + resp = self.hparams({"env_ids": ["run-c", "run-a"]}) + self.assertEqual(self._env_ids(resp), ["run-c", "run-a"]) + + def test_env_ids_skips_missing(self): + """An env_id without an experiment is skipped, not an error.""" + resp = self.hparams({"env_ids": ["run-a", "ghost"]}) + self.assertEqual(resp.code, 200) + self.assertEqual(self._env_ids(resp), ["run-a"]) + + def test_both_intersects_ordered(self): + """query + env_ids returns the intersection, ordered by env_ids.""" + resp = self.hparams({"query": "acc > 0.9", "env_ids": ["run-b", "run-a"]}) + self.assertEqual(self._env_ids(resp), ["run-b"]) + + def test_win_id_is_honoured(self): + """A supplied win id is the window the pane is registered under.""" + resp = self.hparams({"query": "epochs > 0", "win": "hp1"}) + self.assertEqual(resp.body.decode(), "hp1") + self.assertIn("hp1", self._app.state["main"]["jsons"]) + + def test_no_selection_is_400(self): + """With neither query nor env_ids there is nothing to select.""" + self.assertEqual(self.hparams({}).code, 400) + + def test_mode_query_rejects_env_ids(self): + resp = self.hparams( + {"query": "acc > 0.9", "env_ids": ["run-a"], "mode": "query"} + ) + self.assertEqual(resp.code, 400) + + def test_mode_env_ids_rejects_query(self): + resp = self.hparams( + {"query": "acc > 0.9", "env_ids": ["run-b"], "mode": "env_ids"} + ) + self.assertEqual(resp.code, 400) + + def test_mode_both_requires_both(self): + self.assertEqual(self.hparams({"query": "acc > 0.9", "mode": "both"}).code, 400) + + def test_bad_query_is_400(self): + self.assertEqual(self.hparams({"query": "lr <<< 3"}).code, 400) + + def test_env_ids_must_be_a_list(self): + self.assertEqual(self.hparams({"env_ids": "run-a"}).code, 400) + + def test_unknown_mode_is_400(self): + self.assertEqual( + self.hparams({"query": "acc > 0.9", "mode": "sideways"}).code, 400 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/py/tests/test_hparams.py b/py/tests/test_hparams.py index 5e0f47e3d..2707a5505 100644 --- a/py/tests/test_hparams.py +++ b/py/tests/test_hparams.py @@ -1,255 +1,48 @@ -"""Tests for the hyper-parameter pane (Layer 3, PR A1). - -Covers the two pieces ``Visdom.hparams`` is built from: the module-level -``_flatten_experiments`` helper, which collapses experiment dicts (as returned -by ``experiments/search``) into the compact records payload the pane renders; -and the ``Visdom.hparams`` message shape with ``send=False`` (no server), which -pins the window type and the search-then-flatten wiring. Realistic input is -produced through a real ``ExperimentStore`` over a temporary ``JSONStore``, the -same way the other experiment tests seed their fixtures. +"""Tests for the hyper-parameter pane client method (Layer 3, PR A1). + +``Visdom.hparams`` is a thin wrapper: it validates ``opts`` like the other +plotting methods and posts the selection (``query``/``env_ids``/``mode``) to the +``experiments/hparams`` endpoint, which does the selecting, flattening and +window creation. These tests pin the request it builds with ``send=False`` (no +server); the selection rules and flattening are tested against the endpoint in +``test_experiment_hparams``. """ -import tempfile import unittest -from visdom import Visdom, _flatten_experiments -from visdom.data_model import JSONStore -from visdom.experiments import ExperimentStore - - -def seed_experiments(store): - """Log three runs with heterogeneous params/tags and a metric time series.""" - store.log_experiment( - "run-a", - name="alpha", - params={"lr": 0.1, "epochs": 10}, - tags={"dataset": "mnist"}, - ) - store.log_metric("run-a", "acc", 0.80) - - store.log_experiment( - "run-b", - name="beta", - params={"lr": 0.001, "epochs": 20}, - tags={"dataset": "cifar10", "owner": "mira"}, - ) - store.log_metric("run-b", "acc", 0.55) - store.log_metric("run-b", "acc", 0.95) - store.log_metric("run-b", "loss", 0.1) - store.finish_experiment("run-b") - - store.log_experiment("run-c", name="gamma", params={"momentum": 0.9}) - - -def search_dicts(store): - """Return experiment dicts as the search endpoint would hand them back.""" - return [experiment.to_dict() for experiment in store.search()] - - -class TestFlattenExperiments(unittest.TestCase): - """_flatten_experiments collapses lists of params/metrics into per-run maps.""" - - def setUp(self): - self._tmp = tempfile.TemporaryDirectory() - self.store = ExperimentStore(JSONStore(self._tmp.name)) - seed_experiments(self.store) - self.payload = _flatten_experiments(search_dicts(self.store)) - - def tearDown(self): - self._tmp.cleanup() - - def _record(self, env_id): - for record in self.payload["records"]: - if record["env_id"] == env_id: - return record - self.fail("no record for {0!r}".format(env_id)) - - def test_one_record_per_run(self): - """Every experiment becomes exactly one flattened row.""" - self.assertEqual(len(self.payload["records"]), 3) - - def test_param_keys_are_sorted_union(self): - """param_keys is the sorted union of every run's param names.""" - self.assertEqual(self.payload["param_keys"], ["epochs", "lr", "momentum"]) - - def test_metric_keys_are_sorted_union(self): - """metric_keys is the sorted union of every run's metric names.""" - self.assertEqual(self.payload["metric_keys"], ["acc", "loss"]) - - def test_tag_keys_are_sorted_union(self): - """tag_keys is the sorted union of every run's tag names.""" - self.assertEqual(self.payload["tag_keys"], ["dataset", "owner"]) - - def test_tags_collapse_to_a_map(self): - """A run's tags flatten to a {name: value} map on the record.""" - self.assertEqual( - self._record("run-b")["tags"], {"dataset": "cifar10", "owner": "mira"} - ) - - def test_run_without_tags_has_empty_tag_map(self): - """A run with no tags still exposes a tags key (an empty map).""" - self.assertEqual(self._record("run-c")["tags"], {}) - - def test_params_collapse_to_a_map(self): - """A run's params flatten to a {name: value} map.""" - self.assertEqual(self._record("run-a")["params"], {"lr": 0.1, "epochs": 10}) - - def test_latest_metric_value_is_kept(self): - """Metrics are a time series; only the last value per key survives.""" - self.assertEqual(self._record("run-b")["metrics"]["acc"], 0.95) - - def test_record_carries_identity_fields(self): - """Name and status ride along for the table header.""" - record = self._record("run-b") - self.assertEqual(record["name"], "beta") - self.assertEqual(record["status"], "finished") - - def test_missing_param_is_absent_not_null(self): - """A run without a param simply omits it (columns are unioned client-side).""" - self.assertNotIn("momentum", self._record("run-a")["params"]) - self.assertNotIn("lr", self._record("run-c")["params"]) - - def test_empty_input_is_empty_payload(self): - """No experiments yields empty records and empty key unions.""" - payload = _flatten_experiments([]) - self.assertEqual(payload["records"], []) - self.assertEqual(payload["param_keys"], []) - self.assertEqual(payload["metric_keys"], []) - self.assertEqual(payload["tag_keys"], []) - - def test_non_dict_entries_are_skipped(self): - """Defensive: a malformed entry does not abort the whole flatten.""" - payload = _flatten_experiments([None, "oops", {"env_id": "x"}]) - self.assertEqual(len(payload["records"]), 1) - self.assertEqual(payload["records"][0]["env_id"], "x") +from visdom import Visdom class TestHparamsClientMessage(unittest.TestCase): - """Visdom.hparams selects runs per mode and builds the pane window.""" + """Visdom.hparams posts the selection to the experiments/hparams endpoint.""" def setUp(self): self.vis = Visdom(send=False, raise_exceptions=True) - self._stub_endpoints() - - def _stub_endpoints(self): - """Stub both read endpoints, recording which one each call reaches. - - ``search`` records the query it was handed; ``compare`` records the ids - and, like the real endpoint, returns only those runs. This lets the - tests assert that an env_ids selection fetches by id rather than pulling - every experiment through search. - """ - self._seen = {"search": None, "compare": None} - runs = [ - {"env_id": "run-a", "name": "a", "params": [], "metrics": [], "tags": []}, - {"env_id": "run-b", "name": "b", "params": [], "metrics": [], "tags": []}, - {"env_id": "run-c", "name": "c", "params": [], "metrics": [], "tags": []}, - ] - by_id = {run["env_id"]: run for run in runs} - - def fake_search(query=None, limit=None): - self._seen["search"] = query - return {"experiments": runs} - def fake_compare(env_ids): - self._seen["compare"] = list(env_ids) - return {"experiments": [by_id[e] for e in env_ids if e in by_id]} - - self.vis.search_experiments = fake_search - self.vis.compare_experiments = fake_compare - - def _ordered(self, msg): - return [r["env_id"] for r in msg["data"][0]["content"]["records"]] - - def test_query_only_creates_hparams_window_via_search(self): - """A query builds one hparams pane and is fetched through search.""" + def test_posts_to_hparams_endpoint(self): + """The selection goes to the experiments/hparams endpoint.""" msg, endpoint = self.vis.hparams("acc > 0.9") - self.assertEqual(endpoint, "events") - self.assertEqual(len(msg["data"]), 1) - self.assertEqual(msg["data"][0]["type"], "hparams") - self.assertEqual(self._seen["search"], "acc > 0.9") - self.assertIsNone(self._seen["compare"]) - - def test_content_has_records_shape(self): - """The pane content exposes the keys the frontend reads.""" - msg, _ = self.vis.hparams("acc > 0.9") - content = msg["data"][0]["content"] - self.assertIn("records", content) - self.assertIn("param_keys", content) - self.assertIn("metric_keys", content) - self.assertIn("tag_keys", content) - - def test_env_and_win_pass_through(self): + self.assertEqual(endpoint, "experiments/hparams") + self.assertEqual(msg["query"], "acc > 0.9") + self.assertIsNone(msg["mode"]) + self.assertIsNone(msg["env_ids"]) + + def test_env_ids_and_mode_pass_through(self): + """env_ids and an explicit mode ride along untouched for the server.""" + msg, _ = self.vis.hparams(env_ids=["run-a", "run-b"], mode="env_ids") + self.assertEqual(msg["env_ids"], ["run-a", "run-b"]) + self.assertEqual(msg["mode"], "env_ids") + + def test_win_and_env_pass_through(self): """win/env target a specific pane like the other plotting methods.""" msg, _ = self.vis.hparams("acc > 0.9", win="hp1", env="run-x") self.assertEqual(msg["win"], "hp1") self.assertEqual(msg["eid"], "run-x") - def test_env_ids_only_fetches_by_id_via_compare(self): - """env_ids alone infers env_ids mode and fetches only the named runs.""" - msg, _ = self.vis.hparams(env_ids=["run-c", "run-a"]) - self.assertEqual(self._seen["compare"], ["run-c", "run-a"]) - self.assertIsNone(self._seen["search"]) - self.assertEqual(self._ordered(msg), ["run-c", "run-a"]) - - def test_both_infers_and_searches_then_narrows(self): - """query + env_ids infers both mode: search, then narrow by env_ids.""" - msg, _ = self.vis.hparams("acc > 0.9", ["run-c", "run-a"]) - self.assertEqual(self._seen["search"], "acc > 0.9") - self.assertIsNone(self._seen["compare"]) - self.assertEqual(self._ordered(msg), ["run-c", "run-a"]) - - def test_no_arguments_is_an_error(self): - """With neither query nor env_ids there is nothing to select.""" - with self.assertRaises(ValueError): - self.vis.hparams() - - def test_blank_query_alone_is_an_error(self): - """A blank query counts as no query, so it selects nothing.""" - with self.assertRaises(ValueError): - self.vis.hparams(" ") - - def test_mode_query_rejects_env_ids(self): - """mode='query' does not accept env_ids.""" - with self.assertRaises(ValueError): - self.vis.hparams("acc > 0.9", ["run-a"], mode="query") - - def test_mode_query_requires_nonempty_query(self): - """mode='query' needs an actual query.""" - with self.assertRaises(ValueError): - self.vis.hparams(mode="query") - - def test_mode_env_ids_rejects_query(self): - """mode='env_ids' does not accept a query.""" - with self.assertRaises(ValueError): - self.vis.hparams("acc > 0.9", ["run-b"], mode="env_ids") - - def test_mode_env_ids_requires_env_ids(self): - """mode='env_ids' needs a non-empty env_ids.""" - with self.assertRaises(ValueError): - self.vis.hparams(mode="env_ids") - - def test_mode_both_requires_both(self): - """mode='both' needs both a query and env_ids.""" - with self.assertRaises(ValueError): - self.vis.hparams("acc > 0.9", mode="both") - with self.assertRaises(ValueError): - self.vis.hparams(env_ids=["run-a"], mode="both") - - def test_rejects_non_list_env_ids(self): - """env_ids must be a list/tuple of ids, not a bare string.""" - with self.assertRaises(TypeError): - self.vis.hparams(env_ids="run-a") - - def test_rejects_non_string_env_ids(self): - """env_ids must contain strings.""" - with self.assertRaises(TypeError): - self.vis.hparams(env_ids=["run-a", 3]) - - def test_rejects_unknown_mode(self): - """An unrecognised mode is rejected before any request.""" - with self.assertRaises(ValueError): - self.vis.hparams("acc > 0.9", mode="sideways") + def test_opts_are_validated_client_side(self): + """opts are asserted before the request, like the other methods.""" + with self.assertRaises(AssertionError): + self.vis.hparams("acc > 0.9", opts={"opacity": 5}) if __name__ == "__main__": diff --git a/py/visdom/__init__.py b/py/visdom/__init__.py index 51eaa0e82..feceead11 100644 --- a/py/visdom/__init__.py +++ b/py/visdom/__init__.py @@ -656,68 +656,6 @@ def _decode_binary_arrays(obj): return obj -def _flatten_experiments(experiments): - """Flatten experiment dicts into a compact records payload for the hparams pane. - - ``experiments`` is a list of experiment dicts as returned by the - ``experiments/search`` endpoint (see :class:`visdom.experiments.Experiment`). - Each experiment carries ``params``/``metrics``/``tags`` as lists of - ``{"key": ..., "value": ...}`` dicts; this collapses each of the three into a - per-run ``{name: value}`` map so the frontend renders a table/parallel- - coordinates view without re-deriving the column set. Metrics form a time - series, so only each metric's *latest* logged value is kept (the last - observation for that key). - - Returns a dict of ``records`` (one flattened row per run) plus the sorted - ``param_keys``, ``metric_keys`` and ``tag_keys`` unions across all runs. - """ - records = [] - param_keys = set() - metric_keys = set() - tag_keys = set() - for exp in experiments: - if not isinstance(exp, dict): - continue - params = {} - for param in exp.get("params", []) or []: - key = param.get("key") - if key is None: - continue - params[key] = param.get("value") - param_keys.add(key) - metrics = {} - for metric in exp.get("metrics", []) or []: - key = metric.get("key") - if key is None: - continue - metrics[key] = metric.get("value") - metric_keys.add(key) - tags = {} - for tag in exp.get("tags", []) or []: - key = tag.get("key") - if key is None: - continue - tags[key] = tag.get("value") - tag_keys.add(key) - records.append( - { - "env_id": exp.get("env_id"), - "name": exp.get("name", exp.get("env_id")), - "status": exp.get("status"), - "created_at": exp.get("created_at"), - "params": params, - "metrics": metrics, - "tags": tags, - } - ) - return { - "records": records, - "param_keys": sorted(param_keys), - "metric_keys": sorted(metric_keys), - "tag_keys": sorted(tag_keys), - } - - class Visdom(object): def __init__( self, @@ -1367,26 +1305,26 @@ def hparams( ): """Open a hyper-parameter pane over the experiments logged on the server. - Gathers experiments, flattens them into a table of hyper-parameters - against their latest metric values (and tags), and renders it in a - dedicated ``hparams`` window that persists/reloads like any other pane. + Posts the selection to the ``experiments/hparams`` endpoint, which picks + the runs, flattens them into a table of hyper-parameters against their + latest metric values (and tags), and registers a dedicated ``hparams`` + window with that content. The window persists/reloads like any pane. `mode` chooses how the runs to show are selected; when it is left as - `None` it is inferred from which of `query`/`env_ids` were supplied: + `None` the server infers it from which of `query`/`env_ids` were given: * ``"query"`` — the runs matching `query` (the readable syntax of :meth:`search_experiments`). The query must be non-empty and `env_ids` - must not be given. Fetched with :meth:`search_experiments`. + must not be given. * ``"env_ids"`` — the runs named in `env_ids`, in that order. `env_ids` - must be non-empty and `query` must not be given. Fetched with - :meth:`compare_experiments`, which reads only those environments - instead of every experiment. + must be non-empty and `query` must not be given; only those + environments are read rather than every experiment. * ``"both"`` — the intersection: runs that match `query` *and* are named in `env_ids`, ordered by `env_ids`. Both must be given and non-empty. There is no "show everything" call: with neither `query` nor `env_ids` - there is nothing to select, so a :class:`ValueError` is raised. A blank - or whitespace-only `query` counts as no query. + the server has nothing to select and rejects the request. A blank or + whitespace-only `query` counts as no query. :: @@ -1396,77 +1334,22 @@ def hparams( vis.hparams("acc > 0.9", ["run-a"], mode="query") # forced, errors `win`/`env`/`opts` behave as they do for the other plotting methods. - Returns the created window id (or the raw send result when this client is - constructed with `send=False`). + Returns the created window id. """ - valid_modes = ("query", "env_ids", "both") - if mode is not None and mode not in valid_modes: - raise ValueError( - "mode must be one of {0}, got {1!r}".format(valid_modes, mode) - ) - if query is not None and not isstr(query): - raise TypeError("query must be a string") - if env_ids is not None: - if isstr(env_ids) or not isinstance(env_ids, (list, tuple)): - raise TypeError("env_ids must be a list of environment ids") - if not all(isstr(env_id) for env_id in env_ids): - raise TypeError("env_ids must contain strings") - - has_query = isstr(query) and query.strip() != "" - has_env_ids = env_ids is not None and len(env_ids) > 0 - - if mode is None: - if has_query and has_env_ids: - mode = "both" - elif has_query: - mode = "query" - elif has_env_ids: - mode = "env_ids" - else: - raise ValueError("hparams needs a query, env_ids, or both") - elif mode == "query": - if not has_query: - raise ValueError("mode='query' requires a non-empty query") - if env_ids is not None: - raise ValueError("mode='query' does not accept env_ids") - elif mode == "env_ids": - if query is not None: - raise ValueError("mode='env_ids' does not accept a query") - if not has_env_ids: - raise ValueError("mode='env_ids' requires a non-empty env_ids") - else: - if not has_query: - raise ValueError("mode='both' requires a non-empty query") - if not has_env_ids: - raise ValueError("mode='both' requires a non-empty env_ids") - opts = {} if opts is None else opts _title2str(opts) _assert_opts(opts) - wanted = list(dict.fromkeys(env_ids)) if mode in ("env_ids", "both") else None - - if mode == "env_ids": - reply = self.compare_experiments(wanted) - else: - reply = self.search_experiments(query=query, limit=None) - experiments = reply.get("experiments", []) if isinstance(reply, dict) else [] - - if wanted is not None: - by_id = {exp.get("env_id"): exp for exp in experiments} - experiments = [by_id[eid] for eid in wanted if eid in by_id] - - content = _flatten_experiments(experiments) - data = [{"content": content, "type": "hparams"}] - return self._send( { - "data": data, + "query": query, + "env_ids": env_ids, + "mode": mode, "win": win, "eid": env, "opts": opts, }, - endpoint="events", + endpoint="experiments/hparams", ) def get_window_data(self, win=None, env=None): diff --git a/py/visdom/experiments/__init__.py b/py/visdom/experiments/__init__.py index 88f5e6e48..f47b1b0fb 100644 --- a/py/visdom/experiments/__init__.py +++ b/py/visdom/experiments/__init__.py @@ -33,6 +33,7 @@ build_record, parse_query, ) +from visdom.experiments.records import flatten_experiments from visdom.experiments.store import DEFAULT_SORT_FIELD, ExperimentStore __all__ = [ @@ -54,6 +55,7 @@ "Tag", "build_comparison", "build_record", + "flatten_experiments", "parse_query", "STATUS_FAILED", "STATUS_FINISHED", diff --git a/py/visdom/experiments/records.py b/py/visdom/experiments/records.py new file mode 100644 index 000000000..6d035cf00 --- /dev/null +++ b/py/visdom/experiments/records.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, The Visdom Authors +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Flatten experiments into the compact records payload the hparams pane reads. + +An :class:`~visdom.experiments.models.Experiment` stores its params, metrics +and tags as lists of ``{"key": ..., "value": ...}`` dicts. The hyper-parameter +pane instead wants one row per run with those collapsed into ``{name: value}`` +maps, plus the union of names to use as columns. This module is the single +definition of that transform, used by the ``experiments/hparams`` endpoint over +:meth:`~visdom.experiments.models.Experiment.to_dict`. +""" + + +def flatten_experiments(experiments): + """Flatten experiment dicts into a compact records payload. + + ``experiments`` is a list of experiment dicts in the shape of + :meth:`~visdom.experiments.models.Experiment.to_dict`. Each run's + ``params``/``metrics``/``tags`` lists are collapsed into per-run + ``{name: value}`` maps so a table/parallel-coordinates view renders without + re-deriving the column set. Metrics form a time series, so only each + metric's *latest* logged value is kept (the last observation for that key). + + Entries that are not dicts are skipped rather than raising, so a partial or + malformed reply still yields a usable payload. Returns a dict of ``records`` + (one flattened row per run) plus the sorted ``param_keys``, ``metric_keys`` + and ``tag_keys`` unions across all runs. + """ + records = [] + param_keys = set() + metric_keys = set() + tag_keys = set() + for exp in experiments: + if not isinstance(exp, dict): + continue + params = {} + for param in exp.get("params", []) or []: + key = param.get("key") + if key is None: + continue + params[key] = param.get("value") + param_keys.add(key) + metrics = {} + for metric in exp.get("metrics", []) or []: + key = metric.get("key") + if key is None: + continue + metrics[key] = metric.get("value") + metric_keys.add(key) + tags = {} + for tag in exp.get("tags", []) or []: + key = tag.get("key") + if key is None: + continue + tags[key] = tag.get("value") + tag_keys.add(key) + records.append( + { + "env_id": exp.get("env_id"), + "name": exp.get("name", exp.get("env_id")), + "status": exp.get("status"), + "created_at": exp.get("created_at"), + "params": params, + "metrics": metrics, + "tags": tags, + } + ) + return { + "records": records, + "param_keys": sorted(param_keys), + "metric_keys": sorted(metric_keys), + "tag_keys": sorted(tag_keys), + } diff --git a/py/visdom/server/app.py b/py/visdom/server/app.py index 2f9bdf7d2..4cc7fc259 100644 --- a/py/visdom/server/app.py +++ b/py/visdom/server/app.py @@ -27,6 +27,7 @@ VisSocketHandler, VisSocketWrap, ) +from visdom.server.handlers.experiments_handler import ExperimentHparamsHandler from visdom.server.handlers.web_handlers import ( CloseHandler, CompareHandler, @@ -144,6 +145,11 @@ def __init__( ExperimentSuggestHandler, {"app": self}, ), + ( + r"%s/experiments/hparams" % self.base_url, + ExperimentHparamsHandler, + {"app": self}, + ), (r"%s/user/(.*)" % self.base_url, UserSettingsHandler, {"app": self}), (r"%s/health" % self.base_url, HealthHandler), (r"%s(.*)" % self.base_url, IndexHandler, {"app": self}), diff --git a/py/visdom/server/handlers/experiments_handler.py b/py/visdom/server/handlers/experiments_handler.py new file mode 100644 index 000000000..4e2382902 --- /dev/null +++ b/py/visdom/server/handlers/experiments_handler.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 + +# Copyright 2017-present, The Visdom Authors +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. + +"""Handler that builds the hyper-parameter pane. + +``/experiments/hparams`` selects experiments, flattens them into the records +matrix the pane renders from, and stores an ``hparams`` window with that content +in the env state — so the ``Visdom.hparams`` client is a thin call to this +endpoint rather than gathering, flattening and creating the window itself. It +reads through the server's ``DataStore`` (:class:`ExperimentStore` over +``handler.storage``), so it stays backend-agnostic. + +For now the window is only written into the state; it is served on the next env +load but not broadcast to connected clients, because the frontend does not yet +have a dedicated ``hparams`` pane to render it. Broadcasting is added together +with that pane. +""" + +import tornado.escape +import tornado.web + +from visdom.experiments import ( + ExperimentStore, + QueryParseError, + flatten_experiments, +) +from visdom.server.handlers.base_handlers import BaseHandler +from visdom.utils.server_utils import ( + check_auth, + extract_eid, + window, +) + +VALID_MODES = ("query", "env_ids", "both") + + +class ExperimentHparamsHandler(BaseHandler): + """POST ``/experiments/hparams`` — select experiments and open the pane. + + The JSON body selects which runs to show and how: + + * ``query`` — filter with the syntax of :mod:`~visdom.experiments.query` + (``"lr < 0.01 AND acc > 90"``). + * ``env_ids`` — an explicit list of environments, kept in the order given. + * ``mode`` — ``"query"``, ``"env_ids"`` or ``"both"``; when omitted it is + inferred from which of ``query``/``env_ids`` were supplied. Each mode + rejects the argument it does not accept, and with neither supplied there is + nothing to select (400). A blank query counts as no query. + + ``win``/``eid``/``opts`` behave as for any other window. The selected runs + are flattened (:func:`~visdom.experiments.flatten_experiments`) into the + window content and written into the env state; the reply is the created + window id. + """ + + @staticmethod + def _select(store, query, env_ids, mode): + """Resolve the mode, fetch the runs, and return them as experiments. + + Mirrors the selection the ``Visdom.hparams`` client used to do: a query + goes through search, an ``env_ids`` selection reads only those + environments, and ``both`` searches then narrows by ``env_ids``. Invalid + argument combinations raise ``HTTPError(400)``. + """ + if mode is not None and mode not in VALID_MODES: + raise tornado.web.HTTPError( + 400, reason="mode must be one of {0}".format(VALID_MODES) + ) + if query is not None and not isinstance(query, str): + raise tornado.web.HTTPError(400, reason="'query' must be a string") + if env_ids is not None: + if not isinstance(env_ids, list): + raise tornado.web.HTTPError( + 400, reason="'env_ids' must be a list of ids" + ) + if not all(isinstance(env_id, str) for env_id in env_ids): + raise tornado.web.HTTPError( + 400, reason="'env_ids' must contain strings" + ) + + has_query = isinstance(query, str) and query.strip() != "" + has_env_ids = env_ids is not None and len(env_ids) > 0 + + if mode is None: + if has_query and has_env_ids: + mode = "both" + elif has_query: + mode = "query" + elif has_env_ids: + mode = "env_ids" + else: + raise tornado.web.HTTPError( + 400, reason="a query, env_ids, or both is required" + ) + elif mode == "query": + if not has_query: + raise tornado.web.HTTPError( + 400, reason="mode='query' requires a non-empty query" + ) + if env_ids is not None: + raise tornado.web.HTTPError( + 400, reason="mode='query' does not accept env_ids" + ) + elif mode == "env_ids": + if query is not None: + raise tornado.web.HTTPError( + 400, reason="mode='env_ids' does not accept a query" + ) + if not has_env_ids: + raise tornado.web.HTTPError( + 400, reason="mode='env_ids' requires a non-empty env_ids" + ) + else: + if not has_query: + raise tornado.web.HTTPError( + 400, reason="mode='both' requires a non-empty query" + ) + if not has_env_ids: + raise tornado.web.HTTPError( + 400, reason="mode='both' requires a non-empty env_ids" + ) + + wanted = list(dict.fromkeys(env_ids)) if mode in ("env_ids", "both") else None + + if mode == "env_ids": + experiments = [] + for env_id in wanted: + experiment = store.get_experiment(env_id) + if experiment is not None: + experiments.append(experiment) + return experiments + + try: + experiments = store.search(query=query) + except QueryParseError as e: + raise tornado.web.HTTPError(400, reason=str(e)) + if wanted is not None: + by_id = {experiment.env_id: experiment for experiment in experiments} + experiments = [by_id[eid] for eid in wanted if eid in by_id] + return experiments + + @staticmethod + def wrap_func(handler, args): + store = ExperimentStore(handler.storage) + experiments = ExperimentHparamsHandler._select( + store, args.get("query"), args.get("env_ids"), args.get("mode") + ) + content = flatten_experiments( + [experiment.to_dict() for experiment in experiments] + ) + + eid = extract_eid(args) + p = window( + { + "data": [{"content": content, "type": "hparams"}], + "win": args.get("win"), + "opts": args.get("opts", {}), + } + ) + ExperimentHparamsHandler._store_window(handler, p, eid) + handler.write(p["id"]) + + @staticmethod + def _store_window(handler, p, eid): + """Write ``p`` into ``eid``'s state so it is served on the next env load. + + This is the state half of :func:`register_window` without the socket + broadcast: the pane is persisted but not pushed to connected clients, + since the frontend has no ``hparams`` pane to render it yet. + """ + if eid not in handler.state: + handler.state[eid] = {"jsons": {}, "reload": {}} + env = handler.state[eid]["jsons"] + if p["id"] in env: + p["i"] = env[p["id"]]["i"] + else: + p["i"] = len(env) + env[p["id"]] = p + + @check_auth + def post(self): + args = tornado.escape.json_decode( + tornado.escape.to_basestring(self.request.body) + ) + self.wrap_func(self, args) diff --git a/py/visdom/utils/server_utils.py b/py/visdom/utils/server_utils.py index 2c2be5dd1..6fd436875 100644 --- a/py/visdom/utils/server_utils.py +++ b/py/visdom/utils/server_utils.py @@ -197,7 +197,7 @@ def window(args): "show_slider": opts.get("show_slider", True), } ) - elif ptype in ["image", "text", "properties"] and is_visdom_type: + elif ptype in ["image", "text", "properties", "hparams"] and is_visdom_type: p.update({"content": args["data"][0]["content"], "type": ptype}) elif ptype == "network" and is_visdom_type: p.update( From c770fc340e117e76cab28ebe0a1762e0ac169bb8 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Sat, 18 Jul 2026 17:06:59 +0530 Subject: [PATCH 19/48] feat(hparams): render the hyper-parameter pane on the frontend Register an `hparams` pane so `vis.hparams()` windows are drawn in the browser instead of falling through to the plot fallback: - js/panes/HParamsPane.js: functional pane (PlotPane-style React.memo) reading the flattened records payload from window content; renders empty / error / summary states and leaves an .hparams-views seam for the table / parallel-coordinates / SPLOM / filter views of later PRs. - js/settings.js: register the pane in PANES and PANE_SIZE. - py/visdom/static/css/hparams.css + index.html link: pane styling. Switch the /experiments/hparams handler from the persist-only _store_window to the standard register_window, so the pane is broadcast to connected clients and appears live (broadcast was deferred until this pane existed). --- js/panes/HParamsPane.js | 112 ++++++++++++++++++ js/settings.js | 3 + .../server/handlers/experiments_handler.py | 40 ++----- py/visdom/static/css/hparams.css | 100 ++++++++++++++++ py/visdom/static/index.html | 2 + 5 files changed, 228 insertions(+), 29 deletions(-) create mode 100644 js/panes/HParamsPane.js create mode 100644 py/visdom/static/css/hparams.css diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js new file mode 100644 index 000000000..e266e13f2 --- /dev/null +++ b/js/panes/HParamsPane.js @@ -0,0 +1,112 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import React from 'react'; + +import Pane from './Pane'; + +function readContent(content) { + if (!content || typeof content !== 'object' || Array.isArray(content)) { + return null; + } + if (!Array.isArray(content.records)) { + return null; + } + const asKeys = (value) => (Array.isArray(value) ? value : []); + return { + records: content.records, + paramKeys: asKeys(content.param_keys), + metricKeys: asKeys(content.metric_keys), + tagKeys: asKeys(content.tag_keys), + }; +} + +var HParamsPane = (props) => { + const { content } = props; + const data = readContent(content); + + const handleDownload = () => { + let blob = new Blob([JSON.stringify(content)], { + type: 'application/json', + }); + let url = window.URL.createObjectURL(blob); + let link = document.createElement('a'); + link.download = 'visdom_hparams.json'; + link.href = url; + link.click(); + }; + + let body; + if (data === null) { + body = ( +
+ Could not read hyper-parameter data for this window. +
+ ); + } else if (data.records.length === 0) { + body = ( +
+ No experiments match this selection. +
+ ); + } else { + body = ( +
+
+ + {data.records.length} runs + + + {data.paramKeys.length} params + + + {data.metricKeys.length} metrics + + + {data.tagKeys.length} tags + +
+
    + {data.records.map((record, index) => ( +
  • + + {record.name || record.env_id || 'run ' + index} + + {record.status ? ( + + {record.status} + + ) : null} +
    • + ))} +
+
+
+ ); + } + + return ( + +
{body}
+ + ); +}; + +HParamsPane = React.memo(HParamsPane, (props, nextProps) => { + if (props.contentID !== nextProps.contentID) return false; + else if (props.h !== nextProps.h || props.w !== nextProps.w) return false; + else if (props.isFocused !== nextProps.isFocused) return false; + return true; +}); + +export default HParamsPane; diff --git a/js/settings.js b/js/settings.js index 352d35d8f..5537f84ba 100644 --- a/js/settings.js +++ b/js/settings.js @@ -1,4 +1,5 @@ import EmbeddingsPane from './panes/EmbeddingsPane'; +import HParamsPane from './panes/HParamsPane'; import ImageComparePane from './panes/ImageComparePane'; import ImagePane from './panes/ImagePane'; import NetworkPane from './panes/NetworkPane'; @@ -19,6 +20,7 @@ const PANES = { properties: PropertiesPane, embeddings: EmbeddingsPane, network: NetworkPane, + hparams: HParamsPane, }; const PANE_SIZE = { image: [20, 20], @@ -30,6 +32,7 @@ const PANE_SIZE = { embeddings: [20, 20], properties: [20, 20], network: [20, 20], + hparams: [40, 24], }; const MODAL_STYLE = { content: { diff --git a/py/visdom/server/handlers/experiments_handler.py b/py/visdom/server/handlers/experiments_handler.py index 4e2382902..a889578cd 100644 --- a/py/visdom/server/handlers/experiments_handler.py +++ b/py/visdom/server/handlers/experiments_handler.py @@ -9,16 +9,15 @@ """Handler that builds the hyper-parameter pane. ``/experiments/hparams`` selects experiments, flattens them into the records -matrix the pane renders from, and stores an ``hparams`` window with that content -in the env state — so the ``Visdom.hparams`` client is a thin call to this -endpoint rather than gathering, flattening and creating the window itself. It -reads through the server's ``DataStore`` (:class:`ExperimentStore` over +matrix the pane renders from, and registers an ``hparams`` window with that +content — so the ``Visdom.hparams`` client is a thin call to this endpoint +rather than gathering, flattening and creating the window itself. It reads +through the server's ``DataStore`` (:class:`ExperimentStore` over ``handler.storage``), so it stays backend-agnostic. -For now the window is only written into the state; it is served on the next env -load but not broadcast to connected clients, because the frontend does not yet -have a dedicated ``hparams`` pane to render it. Broadcasting is added together -with that pane. +The window is registered like any other pane (:func:`register_window`): written +into the env state and broadcast to connected clients, so it appears live and is +also served on the next env load. """ import tornado.escape @@ -33,6 +32,7 @@ from visdom.utils.server_utils import ( check_auth, extract_eid, + register_window, window, ) @@ -54,8 +54,8 @@ class ExperimentHparamsHandler(BaseHandler): ``win``/``eid``/``opts`` behave as for any other window. The selected runs are flattened (:func:`~visdom.experiments.flatten_experiments`) into the - window content and written into the env state; the reply is the created - window id. + window content and registered as a window (env state + broadcast); the reply + is the created window id. """ @staticmethod @@ -162,25 +162,7 @@ def wrap_func(handler, args): "opts": args.get("opts", {}), } ) - ExperimentHparamsHandler._store_window(handler, p, eid) - handler.write(p["id"]) - - @staticmethod - def _store_window(handler, p, eid): - """Write ``p`` into ``eid``'s state so it is served on the next env load. - - This is the state half of :func:`register_window` without the socket - broadcast: the pane is persisted but not pushed to connected clients, - since the frontend has no ``hparams`` pane to render it yet. - """ - if eid not in handler.state: - handler.state[eid] = {"jsons": {}, "reload": {}} - env = handler.state[eid]["jsons"] - if p["id"] in env: - p["i"] = env[p["id"]]["i"] - else: - p["i"] = len(env) - env[p["id"]] = p + register_window(handler, p, eid) @check_auth def post(self): diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css new file mode 100644 index 000000000..0a9eca674 --- /dev/null +++ b/py/visdom/static/css/hparams.css @@ -0,0 +1,100 @@ +/* + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + */ + +.content-hparams { + height: 100%; + overflow: auto; + background-color: #fff; + font-family: "Open Sans", sans-serif; + font-size: 12px; + color: #333; +} + +.hparams-body { + display: flex; + flex-direction: column; + height: 100%; +} + +.hparams-summary { + position: sticky; + top: 0; + z-index: 1; + display: flex; + flex-wrap: wrap; + gap: 12px; + padding: 6px 10px; + background-color: #f0f0f0; + border-bottom: 1px solid #dedede; +} + +.hparams-stat { + color: #3b5998; + white-space: nowrap; +} + +.hparams-stat b { + font-weight: 600; +} + +.hparams-runs { + flex: 1 1 auto; + margin: 0; + padding: 4px 0; + list-style: none; +} + +.hparams-run { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + padding: 4px 10px; + border-bottom: 1px solid #f0f0f0; +} + +.hparams-run-name { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.hparams-run-status { + flex: 0 0 auto; + padding: 1px 8px; + border-radius: 10px; + font-size: 11px; + text-transform: capitalize; + background-color: #dedede; + color: #333; +} + +.hparams-status-running { + background-color: #6389d8; + color: #fff; +} + +.hparams-status-finished { + background-color: #3b5998; + color: #fff; +} + +.hparams-status-failed { + background-color: #d85f5f; + color: #fff; +} + +.hparams-message { + padding: 16px; + text-align: center; + color: grey; +} + +.hparams-error { + color: #b94a48; +} diff --git a/py/visdom/static/index.html b/py/visdom/static/index.html index edbc0b3d9..499df039f 100644 --- a/py/visdom/static/index.html +++ b/py/visdom/static/index.html @@ -79,6 +79,8 @@ + + visdom From 7bde22356698ec4e01b7402bef8d9591e30d9c45 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Sun, 19 Jul 2026 16:53:01 +0530 Subject: [PATCH 20/48] feat(hparams): add sortable, filterable run table with color spine Replace the placeholder run list in the hparams pane with HParamsTable, mounted in the .hparams-views seam. Adds: - hparamsUtils: pure sort/format/filter/color helpers; the comparator mirrors the backend order rule (numbers before strings, missing/NaN last in both directions) so the table and server-side search agree. - Column-header sorting plus a tree 'sort by' dropdown with a 3-state direction toggle (asc/desc/off); active sort column is highlighted. - Client-side text filter and per-run selection checkboxes. - 'color by' shades a numeric param or metric column on a ramp off the Visdom blue (#3b5998). - Both dropdowns reuse rc-tree-select to match the environment selector. --- js/panes/HParamsPane.js | 28 +-- js/panes/hparams/HParamsTable.js | 414 +++++++++++++++++++++++++++++++ js/panes/hparams/hparamsUtils.js | 152 ++++++++++++ py/visdom/static/css/hparams.css | 275 ++++++++++++++++++++ 4 files changed, 850 insertions(+), 19 deletions(-) create mode 100644 js/panes/hparams/HParamsTable.js create mode 100644 js/panes/hparams/hparamsUtils.js diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index e266e13f2..8b0bce8e7 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -9,6 +9,7 @@ import React from 'react'; +import HParamsTable from './hparams/HParamsTable'; import Pane from './Pane'; function readContent(content) { @@ -72,25 +73,14 @@ var HParamsPane = (props) => { {data.tagKeys.length} tags
-
    - {data.records.map((record, index) => ( -
  • - - {record.name || record.env_id || 'run ' + index} - - {record.status ? ( - - {record.status} - - ) : null} -
    • - ))} -
-
+
+ +
); } diff --git a/js/panes/hparams/HParamsTable.js b/js/panes/hparams/HParamsTable.js new file mode 100644 index 000000000..f68e55728 --- /dev/null +++ b/js/panes/hparams/HParamsTable.js @@ -0,0 +1,414 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import TreeSelect from 'rc-tree-select'; +import React, { useCallback, useMemo, useState } from 'react'; + +import { + buildColumns, + filterRecords, + formatValue, + isNumeric, + makeComparator, + numericExtent, + spineStyle, +} from './hparamsUtils'; + +const RUN_COLUMN_ID = 'run:name'; +const CONTROL_STYLE = { width: 150 }; + +function nextSort(current, columnId) { + if (current.by !== columnId) return { by: columnId, dir: 'asc' }; + if (current.dir === 'asc') return { by: columnId, dir: 'desc' }; + return { by: null, dir: null }; +} + +function ariaSort(sort, columnId) { + if (sort.by !== columnId) return 'none'; + return sort.dir === 'asc' ? 'ascending' : 'descending'; +} + +function accessorFor(sortBy, columns) { + if (sortBy === RUN_COLUMN_ID) { + return (record) => record.name || record.env_id; + } + const col = columns.find((c) => c.id === sortBy); + return col ? col.accessor : null; +} + +const SortCaret = ({ sort, columnId }) => { + const active = sort.by === columnId; + const glyph = !active ? '⇅' : sort.dir === 'asc' ? '▲' : '▼'; + return ( + + ); +}; + +const SortHeader = ({ sort, columnId, label, scopeClass, onSort }) => { + const active = sort.by === columnId; + return ( + + + + ); +}; + +const HParamsRow = React.memo(function HParamsRow({ + record, + columns, + colorBy, + extent, + isSelected, + onToggle, +}) { + const runLabel = record.name || record.env_id || 'run'; + return ( + + + onToggle(record.env_id)} + aria-label={'Select ' + runLabel} + /> + + +
+ {runLabel} + {record.status ? ( + + {record.status} + + ) : null} +
+ + {columns.map((col) => { + const value = col.accessor(record); + const style = + colorBy && col.id === colorBy ? spineStyle(value, extent) : null; + const cls = + 'hparams-cell' + + (isNumeric(value) ? ' hparams-cell-num' : '') + + (style ? ' hparams-cell-spine' : ''); + return ( + + {formatValue(value)} + + ); + })} + + ); +}); + +const HParamsTable = ({ records, paramKeys, metricKeys, tagKeys }) => { + const [sort, setSort] = useState({ by: null, dir: null }); + const [filter, setFilter] = useState(''); + const [colorBy, setColorBy] = useState(null); + const [selected, setSelected] = useState(() => new Set()); + + const columns = useMemo( + () => buildColumns(paramKeys, metricKeys, tagKeys), + [paramKeys, metricKeys, tagKeys] + ); + const colorCols = useMemo( + () => + columns.filter( + (c) => + (c.group === 'param' || c.group === 'metric') && + numericExtent(records, c.accessor) + ), + [columns, records] + ); + const colorParamCols = colorCols.filter((c) => c.group === 'param'); + const colorMetricCols = colorCols.filter((c) => c.group === 'metric'); + + const filtered = useMemo( + () => filterRecords(records, filter, columns), + [records, filter, columns] + ); + + const rows = useMemo(() => { + if (!sort.by) return filtered; + const accessor = accessorFor(sort.by, columns); + if (!accessor) return filtered; + return filtered.slice().sort(makeComparator(accessor, sort.dir)); + }, [filtered, sort, columns]); + + const extent = useMemo(() => { + if (!colorBy) return null; + const col = columns.find((c) => c.id === colorBy); + if (!col) return null; + return numericExtent(records, col.accessor); + }, [records, colorBy, columns]); + + const handleSort = useCallback((columnId) => { + setSort((prev) => nextSort(prev, columnId)); + }, []); + + const handleSortSelect = useCallback((value) => { + setSort((prev) => + value ? { by: value, dir: prev.dir || 'asc' } : { by: null, dir: null } + ); + }, []); + + const cycleDir = useCallback(() => { + setSort((prev) => { + if (!prev.by) return prev; + if (prev.dir === 'asc') return { by: prev.by, dir: 'desc' }; + return { by: null, dir: null }; + }); + }, []); + + const toggle = useCallback((envId) => { + setSelected((prev) => { + const next = new Set(prev); + if (next.has(envId)) next.delete(envId); + else next.add(envId); + return next; + }); + }, []); + + const allSelected = + rows.length > 0 && rows.every((r) => selected.has(r.env_id)); + + const handleSelectAll = useCallback(() => { + setSelected((prev) => { + const next = new Set(prev); + const everyOn = rows.length > 0 && rows.every((r) => next.has(r.env_id)); + rows.forEach((r) => + everyOn ? next.delete(r.env_id) : next.add(r.env_id) + ); + return next; + }); + }, [rows]); + + const bands = [ + { key: 'param', label: 'params' }, + { key: 'metric', label: 'metrics' }, + { key: 'tag', label: 'tags' }, + ] + .map((b) => ({ + ...b, + span: columns.filter((c) => c.group === b.key).length, + })) + .filter((b) => b.span > 0); + + const sortTreeData = [ + { key: RUN_COLUMN_ID, value: RUN_COLUMN_ID, title: 'run' }, + ...bands.map((b) => ({ + key: '__g_' + b.key, + value: '__g_' + b.key, + title: b.label, + selectable: false, + children: columns + .filter((c) => c.group === b.key) + .map((c) => ({ key: c.id, value: c.id, title: c.label })), + })), + ]; + + const colorTreeData = []; + if (colorParamCols.length) { + colorTreeData.push({ + key: '__cg_param', + value: '__cg_param', + title: 'params', + selectable: false, + children: colorParamCols.map((c) => ({ + key: c.id, + value: c.id, + title: c.label, + })), + }); + } + if (colorMetricCols.length) { + colorTreeData.push({ + key: '__cg_metric', + value: '__cg_metric', + title: 'metrics', + selectable: false, + children: colorMetricCols.map((c) => ({ + key: c.id, + value: c.id, + title: c.label, + })), + }); + } + + return ( +
+
+ setFilter(e.target.value)} + aria-label="Filter runs" + /> + + sort by: + handleSortSelect(value || '')} + /> + + + {colorCols.length ? ( + + color by: + setColorBy(value || null)} + /> + + ) : null} + {selected.size ? ( + + {selected.size} selected + + ) : null} +
+ +
+ + + + + ))} + + + + + {columns.map((col) => ( + + ))} + + + + {rows.length === 0 ? ( + + + + ) : ( + rows.map((record, index) => ( + + )) + )} + +
+ {b.label} +
+ +
+ No runs match “{filter}”. +
+
+
+ ); +}; + +export default HParamsTable; diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js new file mode 100644 index 000000000..880d665b3 --- /dev/null +++ b/js/panes/hparams/hparamsUtils.js @@ -0,0 +1,152 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +/* + * Pure helpers for the hyper-parameter table. Kept free of React so the + * ordering and formatting rules live in one place and can mirror the Python + * backend exactly. The comparator below intentionally matches + * visdom.experiments.store._order_key / _sort_pairs so the table and the + * server-side search agree on which run ranks first. + */ + +const SPINE_LIGHT = [235, 240, 249]; +const SPINE_DARK = [59, 89, 152]; + +export function isMissing(value) { + return ( + value === undefined || + value === null || + (typeof value === 'number' && Number.isNaN(value)) + ); +} + +export function isNumeric(value) { + return typeof value === 'number' && Number.isFinite(value); +} + +export function orderKey(value) { + if (typeof value === 'boolean') { + return [1, 0, String(value)]; + } + if (isNumeric(value)) { + return [0, value, '']; + } + return [1, 0, String(value)]; +} + +export function compareOrderKeys(a, b) { + if (a[0] !== b[0]) return a[0] - b[0]; + if (a[1] !== b[1]) return a[1] - b[1]; + if (a[2] < b[2]) return -1; + if (a[2] > b[2]) return 1; + return 0; +} + +export function makeComparator(accessor, direction) { + const descending = direction === 'desc'; + return (rowA, rowB) => { + const va = accessor(rowA); + const vb = accessor(rowB); + const ma = isMissing(va); + const mb = isMissing(vb); + if (ma && mb) return 0; + if (ma) return 1; + if (mb) return -1; + const cmp = compareOrderKeys(orderKey(va), orderKey(vb)); + return descending ? -cmp : cmp; + }; +} + +export function buildColumns(paramKeys, metricKeys, tagKeys) { + const columns = []; + (paramKeys || []).forEach((key) => { + columns.push({ + id: 'param:' + key, + label: key, + group: 'param', + accessor: (record) => (record.params ? record.params[key] : undefined), + }); + }); + (metricKeys || []).forEach((key) => { + columns.push({ + id: 'metric:' + key, + label: key, + group: 'metric', + metricKey: key, + accessor: (record) => (record.metrics ? record.metrics[key] : undefined), + }); + }); + (tagKeys || []).forEach((key) => { + columns.push({ + id: 'tag:' + key, + label: key, + group: 'tag', + accessor: (record) => (record.tags ? record.tags[key] : undefined), + }); + }); + return columns; +} + +export function formatValue(value) { + if (isMissing(value)) return '—'; + if (typeof value === 'number') { + if (!Number.isFinite(value)) return '—'; + if (Number.isInteger(value)) return String(value); + return String(parseFloat(value.toPrecision(4))); + } + return String(value); +} + +export function filterRecords(records, query, columns) { + const q = (query || '').trim().toLowerCase(); + if (!q) return records; + return records.filter((record) => { + const label = String(record.name || record.env_id || '').toLowerCase(); + if (label.indexOf(q) !== -1) return true; + for (let i = 0; i < columns.length; i++) { + const v = columns[i].accessor(record); + if (!isMissing(v) && String(v).toLowerCase().indexOf(q) !== -1) { + return true; + } + } + return false; + }); +} + +export function numericExtent(records, accessor) { + let min = Infinity; + let max = -Infinity; + records.forEach((record) => { + const v = accessor(record); + if (isNumeric(v)) { + if (v < min) min = v; + if (v > max) max = v; + } + }); + if (min === Infinity) return null; + return { min, max }; +} + +export function spineStyle(value, extent) { + if (!extent || !isNumeric(value)) return null; + const { min, max } = extent; + let t = max > min ? (value - min) / (max - min) : 1; + if (t < 0) t = 0; + else if (t > 1) t = 1; + const mix = (a, b) => Math.round(a + (b - a) * t); + const bg = + 'rgb(' + + mix(SPINE_LIGHT[0], SPINE_DARK[0]) + + ', ' + + mix(SPINE_LIGHT[1], SPINE_DARK[1]) + + ', ' + + mix(SPINE_LIGHT[2], SPINE_DARK[2]) + + ')'; + return { backgroundColor: bg, color: t > 0.62 ? '#fff' : '#333' }; +} diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index 0a9eca674..cf2fbe112 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -98,3 +98,278 @@ .hparams-error { color: #b94a48; } + +/* ---- HParamsTable (B2) ---- */ + +.hparams-views { + flex: 1 1 auto; + min-height: 0; + display: flex; + flex-direction: column; +} + +.hparams-table-wrap { + flex: 1 1 auto; + min-height: 0; + display: flex; + flex-direction: column; +} + +.hparams-toolbar { + display: flex; + align-items: center; + gap: 10px; + padding: 6px 10px; + background-color: #fff; + border-bottom: 1px solid #f0f0f0; +} + +.hparams-filter { + flex: 0 1 200px; + padding: 3px 8px; + font-family: "Open Sans", sans-serif; + font-size: 12px; + color: #333; + border: 1px solid #dedede; + border-radius: 3px; +} + +.hparams-filter:focus { + outline: none; + border-color: #3b5998; +} + +.hparams-colorby, +.hparams-sortby { + display: flex; + align-items: center; + gap: 4px; + color: #666; +} + +.hparams-treeselect { + font-family: "Open Sans", sans-serif; + font-size: 13px; + color: #333; +} + +.hparams-treeselect .rc-tree-select-selector { + border-radius: 4px !important; + border-color: #dedede !important; +} + +.hparams-dir-btn { + padding: 1px 6px; + font-size: 10px; + line-height: 1.4; + color: #3b5998; + background-color: #fff; + border: 1px solid #dedede; + border-radius: 3px; + cursor: pointer; +} + +.hparams-dir-btn:hover:not(:disabled) { + border-color: #3b5998; +} + +.hparams-dir-btn:disabled { + color: #b8b8b8; + cursor: default; +} + +.hparams-dir-btn:focus-visible { + outline: 2px solid #3b5998; + outline-offset: -2px; +} + +.hparams-selected-count { + margin-left: auto; + color: #3b5998; + font-weight: 600; + white-space: nowrap; +} + +.hparams-table-scroll { + flex: 1 1 auto; + min-height: 0; + overflow: auto; +} + +.hparams-table { + border-collapse: separate; + border-spacing: 0; + width: 100%; + font-size: 12px; + color: #333; +} + +.hparams-table th, +.hparams-table td { + border-bottom: 1px solid #f0f0f0; + border-right: 1px solid #f0f0f0; + padding: 3px 8px; + text-align: left; + white-space: nowrap; + font-weight: normal; +} + +/* Column group bands (params | metrics | tags) */ +.hparams-col-group { + font-size: 10px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + color: #444; + text-align: center; + background-color: #f7f7f7; + border-bottom: 1px solid #dedede !important; +} + +.hparams-group-metric, +.hparams-group-tag { + border-left: 1px solid #dedede; +} + +.hparams-col-blank { + background-color: #fff; + border-right-color: transparent !important; +} + +/* Sticky column headers */ +.hparams-head-row th { + position: sticky; + top: 0; + z-index: 2; + background-color: #f0f0f0; + border-bottom: 1px solid #dedede; +} + +.hparams-sort-btn { + display: inline-flex; + align-items: center; + gap: 3px; + width: 100%; + max-width: 140px; + padding: 0; + border: none; + background: none; + font: inherit; + font-weight: 600; + color: #222; + cursor: pointer; + text-align: left; +} + +.hparams-sort-btn:focus-visible { + outline: 2px solid #3b5998; + outline-offset: -2px; +} + +.hparams-th-label { + overflow: hidden; + text-overflow: ellipsis; +} + +.hparams-caret { + flex: 0 0 auto; + width: 11px; + font-size: 9px; + line-height: 1; + text-align: center; +} + +.hparams-caret-idle { + color: #6b7280; +} + +.hparams-caret-active { + color: #3b5998; +} + +.hparams-head-row th:hover .hparams-sort-btn, +.hparams-head-row th:hover .hparams-caret-idle { + color: #3b5998; +} + +.hparams-head-row th.hparams-th-active { + background-color: #e6ecfa; +} + +.hparams-head-row th.hparams-th-active .hparams-sort-btn { + color: #3b5998; + font-weight: 700; +} + +/* Sticky first columns: selection checkbox + run name */ +.hparams-cell-select, +.hparams-th-select { + position: sticky; + left: 0; + z-index: 1; + width: 28px; + min-width: 28px; + text-align: center; + background-color: #fff; +} + +.hparams-cell-run, +.hparams-th-run { + position: sticky; + left: 28px; + z-index: 1; + background-color: #fff; + max-width: 180px; +} + +.hparams-head-row .hparams-th-select, +.hparams-head-row .hparams-th-run { + z-index: 3; + background-color: #f0f0f0; +} + +.hparams-run-cell { + display: flex; + align-items: center; + justify-content: space-between; + gap: 6px; +} + +.hparams-run-cell .hparams-run-name { + max-width: 120px; +} + +/* Numeric cells: right aligned, aligned digits */ +.hparams-cell-num { + text-align: right; + font-variant-numeric: tabular-nums; +} + +/* The color spine transition (disabled when the user prefers reduced motion) */ +.hparams-cell-spine { + transition: background-color 0.2s ease; +} + +@media (prefers-reduced-motion: reduce) { + .hparams-cell-spine, + .hparams-caret-idle { + transition: none; + } +} + +/* Row states */ +.hparams-row:hover td, +.hparams-row:hover th { + background-color: #f7f9fd; +} + +.hparams-row-selected td, +.hparams-row-selected th { + background-color: #eaf0fb; +} + +.hparams-nomatch { + padding: 16px; + text-align: center; + color: grey; +} From 0c6e425b0b8866be27a24d8c6acda3ba4da743cf Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Sun, 19 Jul 2026 17:17:20 +0530 Subject: [PATCH 21/48] feat(hparams): centered title, group dividers, and layout fixes - Add a centered, larger pane title: the experiments handler defaults the hparams window title, and Pane tags its bar with a per-type class so the hparams title can be styled without touching other panes. - Draw solid vertical dividers at the param/metric/tag group boundaries, aligned across the header band, header row, and body. - Fix the double horizontal scrollbar by scrolling only the inner table and adding min-width:0 to the flex chain. --- js/panes/Pane.js | 10 ++++-- js/panes/hparams/HParamsTable.js | 21 +++++++++-- .../server/handlers/experiments_handler.py | 4 ++- py/visdom/static/css/hparams.css | 36 +++++++++++++++++-- 4 files changed, 64 insertions(+), 7 deletions(-) diff --git a/js/panes/Pane.js b/js/panes/Pane.js index 79d0234f1..22d8fa33a 100644 --- a/js/panes/Pane.js +++ b/js/panes/Pane.js @@ -34,8 +34,14 @@ var Pane = forwardRef((props, ref) => { // rendering // --------- - let windowClassNames = classNames({ window: true, focus: props.isFocused }); - let barClassNames = classNames({ bar: true, focus: props.isFocused }); + let windowClassNames = classNames( + { window: true, focus: props.isFocused }, + props.type && 'window-' + props.type + ); + let barClassNames = classNames( + { bar: true, focus: props.isFocused }, + props.type && 'bar-' + props.type + ); // add property list button to barwidgets if ( diff --git a/js/panes/hparams/HParamsTable.js b/js/panes/hparams/HParamsTable.js index f68e55728..d6472be0d 100644 --- a/js/panes/hparams/HParamsTable.js +++ b/js/panes/hparams/HParamsTable.js @@ -88,6 +88,7 @@ const HParamsRow = React.memo(function HParamsRow({ extent, isSelected, onToggle, + groupStartIds, }) { const runLabel = record.name || record.env_id || 'run'; return ( @@ -123,7 +124,8 @@ const HParamsRow = React.memo(function HParamsRow({ const cls = 'hparams-cell' + (isNumeric(value) ? ' hparams-cell-num' : '') + - (style ? ' hparams-cell-spine' : ''); + (style ? ' hparams-cell-spine' : '') + + (groupStartIds.has(col.id) ? ' hparams-col-sep' : ''); return ( {formatValue(value)} @@ -156,6 +158,16 @@ const HParamsTable = ({ records, paramKeys, metricKeys, tagKeys }) => { const colorParamCols = colorCols.filter((c) => c.group === 'param'); const colorMetricCols = colorCols.filter((c) => c.group === 'metric'); + const groupStartIds = useMemo(() => { + const ids = new Set(); + let prev = null; + columns.forEach((c) => { + if (c.group !== prev) ids.add(c.id); + prev = c.group; + }); + return ids; + }, [columns]); + const filtered = useMemo( () => filterRecords(records, filter, columns), [records, filter, columns] @@ -378,7 +390,11 @@ const HParamsTable = ({ records, paramKeys, metricKeys, tagKeys }) => { sort={sort} columnId={col.id} label={col.label} - scopeClass={'hparams-th-' + col.group} + scopeClass={ + 'hparams-th-' + + col.group + + (groupStartIds.has(col.id) ? ' hparams-col-sep' : '') + } onSort={handleSort} /> ))} @@ -401,6 +417,7 @@ const HParamsTable = ({ records, paramKeys, metricKeys, tagKeys }) => { extent={extent} isSelected={selected.has(record.env_id)} onToggle={toggle} + groupStartIds={groupStartIds} /> )) )} diff --git a/py/visdom/server/handlers/experiments_handler.py b/py/visdom/server/handlers/experiments_handler.py index a889578cd..e1102434b 100644 --- a/py/visdom/server/handlers/experiments_handler.py +++ b/py/visdom/server/handlers/experiments_handler.py @@ -155,11 +155,13 @@ def wrap_func(handler, args): ) eid = extract_eid(args) + opts = dict(args.get("opts") or {}) + opts.setdefault("title", "Hyperparameters") p = window( { "data": [{"content": content, "type": "hparams"}], "win": args.get("win"), - "opts": args.get("opts", {}), + "opts": opts, } ) register_window(handler, p, eid) diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index cf2fbe112..cf4e825f8 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -8,7 +8,7 @@ .content-hparams { height: 100%; - overflow: auto; + overflow: hidden; background-color: #fff; font-family: "Open Sans", sans-serif; font-size: 12px; @@ -99,11 +99,37 @@ color: #b94a48; } +/* ---- Title bar (centered, like a proper pane heading) ---- */ + +.bar-hparams { + height: 22px; +} + +.bar-hparams button { + line-height: 22px; +} + +.bar-hparams .pull-right { + float: none !important; + position: absolute; + left: 0; + right: 0; + top: 0; + line-height: 22px; + text-align: center; + font-size: 14px; + font-weight: 600; + letter-spacing: 0.02em; + color: #3b5998; + pointer-events: none; +} + /* ---- HParamsTable (B2) ---- */ .hparams-views { flex: 1 1 auto; min-height: 0; + min-width: 0; display: flex; flex-direction: column; } @@ -111,6 +137,7 @@ .hparams-table-wrap { flex: 1 1 auto; min-height: 0; + min-width: 0; display: flex; flex-direction: column; } @@ -226,9 +253,14 @@ border-bottom: 1px solid #dedede !important; } +.hparams-group-param, .hparams-group-metric, .hparams-group-tag { - border-left: 1px solid #dedede; + border-left: 1px solid #ccc; +} + +.hparams-col-sep { + border-left: 1px solid #ccc; } .hparams-col-blank { From f3a731276209969450a0a2deba057532a88fcbf6 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Mon, 20 Jul 2026 15:44:51 +0530 Subject: [PATCH 22/48] feat(hparams): add scatter matrix (SPLOM) view with dimension picker and color-by Adds a Table | Scatter matrix switcher to the hyper-parameter pane and a new HParamsSplom view that renders a Plotly splom trace from the window records. Users pick up to six numeric param/metric axes and an optional color-by metric (same light-to-dark ramp as the table color spine). NaN/missing values are dropped so empty axes never appear. Reuses hparamsUtils for column building and numeric detection; global Plotly and the PlotPane resize pattern; no new deps and no backend/API changes. --- js/panes/HParamsPane.js | 47 +++- js/panes/hparams/HParamsSplom.js | 357 +++++++++++++++++++++++++++++++ js/panes/hparams/HParamsTable.js | 7 +- js/panes/hparams/hparamsUtils.js | 34 +++ js/settings.js | 2 +- py/visdom/static/css/hparams.css | 83 +++++++ 6 files changed, 517 insertions(+), 13 deletions(-) create mode 100644 js/panes/hparams/HParamsSplom.js diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index 8b0bce8e7..090970f86 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -7,11 +7,17 @@ * */ -import React from 'react'; +import React, { useState } from 'react'; +import HParamsSplom from './hparams/HParamsSplom'; import HParamsTable from './hparams/HParamsTable'; import Pane from './Pane'; +const VIEWS = [ + { key: 'table', label: 'Table' }, + { key: 'splom', label: 'Scatter matrix' }, +]; + function readContent(content) { if (!content || typeof content !== 'object' || Array.isArray(content)) { return null; @@ -31,6 +37,7 @@ function readContent(content) { var HParamsPane = (props) => { const { content } = props; const data = readContent(content); + const [view, setView] = useState('table'); const handleDownload = () => { let blob = new Blob([JSON.stringify(content)], { @@ -74,12 +81,38 @@ var HParamsPane = (props) => {
- +
+ {VIEWS.map((v) => ( + + ))} +
+ {view === 'splom' ? ( + + ) : ( + + )}
); diff --git a/js/panes/hparams/HParamsSplom.js b/js/panes/hparams/HParamsSplom.js new file mode 100644 index 000000000..5872016ae --- /dev/null +++ b/js/panes/hparams/HParamsSplom.js @@ -0,0 +1,357 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import TreeSelect from 'rc-tree-select'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; + +import { + buildColumns, + isNumeric, + numericExtent, + selectNumericColumns, +} from './hparamsUtils'; + +const MAX_DIMS = 6; + +const SPLOM_COLORSCALE = 'Viridis'; + +const AXIS_STYLE = { + showline: true, + linecolor: '#aab8d8', + linewidth: 1, + mirror: 'all', + gridcolor: '#f0f2f8', + zeroline: false, + ticklen: 3, + tickfont: { size: 9, color: '#666' }, + automargin: true, +}; + +const SNAPSHOT_NOTICE_DELAY = 700; + +function notify(message, kind) { + const lib = window.Plotly && window.Plotly.Lib; + if (lib && typeof lib.notifier === 'function') lib.notifier(message, kind); +} + +function downloadSnapshot(gd) { + if (!window.Plotly || typeof window.Plotly.toImage !== 'function') return; + let done = false; + const timer = setTimeout(() => { + if (!done) notify('Taking snapshot - this may take a few seconds', 'long'); + }, SNAPSHOT_NOTICE_DELAY); + + window.Plotly.toImage(gd, { + format: 'png', + width: gd.offsetWidth || 900, + height: gd.offsetHeight || 600, + }) + .then((url) => { + done = true; + clearTimeout(timer); + const link = document.createElement('a'); + link.href = url; + link.download = 'hparams_scatter.png'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }) + .catch(() => { + done = true; + clearTimeout(timer); + notify('Snapshot failed', 'long'); + }); +} + +function groupedTreeData(cols) { + const params = cols.filter((c) => c.group === 'param'); + const metrics = cols.filter((c) => c.group === 'metric'); + const branch = (key, title, children) => + children.length + ? [ + { + key: '__g_' + key, + value: '__g_' + key, + title, + selectable: false, + checkable: false, + children: children.map((c) => ({ + key: c.id, + value: c.id, + title: c.label, + })), + }, + ] + : []; + return [ + ...branch('param', 'params', params), + ...branch('metric', 'metrics', metrics), + ]; +} + +const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { + const plotRef = useRef(null); + const prevDimCount = useRef(0); + + const columns = useMemo( + () => buildColumns(paramKeys, metricKeys, tagKeys), + [paramKeys, metricKeys, tagKeys] + ); + const numericCols = useMemo( + () => selectNumericColumns(records, columns), + [records, columns] + ); + + const [selectedDims, setSelectedDims] = useState(null); + const [colorBy, setColorBy] = useState(null); + + const effectiveDims = useMemo(() => { + const validIds = new Set(numericCols.map((c) => c.id)); + let ids = (selectedDims || []).filter((id) => validIds.has(id)); + if (ids.length === 0) ids = numericCols.slice(0, MAX_DIMS).map((c) => c.id); + return ids.slice(0, MAX_DIMS); + }, [selectedDims, numericCols]); + + const effectiveColorBy = useMemo(() => { + if (!colorBy) return null; + return numericCols.some((c) => c.id === colorBy) ? colorBy : null; + }, [colorBy, numericCols]); + + const truncated = + (selectedDims || []).filter((id) => numericCols.some((c) => c.id === id)) + .length > MAX_DIMS; + + const dimTreeData = useMemo( + () => groupedTreeData(numericCols), + [numericCols] + ); + const colorTreeData = useMemo( + () => groupedTreeData(numericCols), + [numericCols] + ); + + const hasPlot = numericCols.length >= 2; + + useEffect(() => { + const el = plotRef.current; + if (!el) return; + const isDisplayed = (node) => + !!(node && node.offsetWidth > 0 && node.offsetHeight > 0); + const resizeObserver = new ResizeObserver(() => { + if (window.Plotly && el._fullLayout && isDisplayed(el)) { + window.Plotly.Plots.resize(el); + } + }); + resizeObserver.observe(el); + return () => { + resizeObserver.disconnect(); + if (window.Plotly && el._fullLayout) window.Plotly.purge(el); + }; + }, []); + + useEffect(() => { + const el = plotRef.current; + if (!el || !window.Plotly) return; + + const activeCols = effectiveDims + .map((id) => columns.find((c) => c.id === id)) + .filter((col) => col && records.some((r) => isNumeric(col.accessor(r)))); + if (activeCols.length < 2) { + window.Plotly.purge(el); + prevDimCount.current = 0; + return; + } + + const dimensions = activeCols.map((col) => ({ + label: col.label, + values: records.map((r) => { + const v = col.accessor(r); + return isNumeric(v) ? v : null; + }), + })); + + const label = (r) => r.name || r.env_id || 'run'; + const names = records.map(label); + + const colorCol = effectiveColorBy + ? columns.find((c) => c.id === effectiveColorBy) + : null; + let colorValues; + let colorLabel; + let cmin; + let cmax; + if (colorCol) { + colorValues = records.map((r) => { + const v = colorCol.accessor(r); + return isNumeric(v) ? v : null; + }); + colorLabel = colorCol.label; + const ext = numericExtent(records, colorCol.accessor); + if (ext) { + cmin = ext.min; + cmax = ext.max; + } + } else { + colorValues = records.map((_, i) => i + 1); + colorLabel = 'run order'; + cmin = 1; + cmax = Math.max(records.length, 1); + } + + const data = [ + { + type: 'splom', + dimensions, + text: names, + hovertemplate: '%{text}
x: %{x}
y: %{y}', + marker: { + size: 7, + line: { color: '#ffffff', width: 0.6 }, + color: colorValues, + colorscale: SPLOM_COLORSCALE, + showscale: true, + cmin, + cmax, + colorbar: { + title: { text: colorLabel, side: 'right', font: { size: 11 } }, + thickness: 12, + len: 0.6, + outlinewidth: 0, + }, + }, + diagonal: { visible: true }, + showupperhalf: true, + showlowerhalf: true, + opacity: 1, + }, + ]; + + const layout = { + margin: { l: 60, r: 20, t: 34, b: 44 }, + dragmode: 'select', + hovermode: 'closest', + showlegend: false, + font: { family: '"Open Sans", sans-serif', size: 11, color: '#333' }, + paper_bgcolor: '#ffffff', + plot_bgcolor: '#ffffff', + datarevision: + effectiveDims.join('|') + + '::' + + (effectiveColorBy || 'order') + + '::' + + records.length, + }; + for (let i = 1; i <= activeCols.length; i++) { + const suffix = i === 1 ? '' : String(i); + layout['xaxis' + suffix] = { ...AXIS_STYLE }; + layout['yaxis' + suffix] = { ...AXIS_STYLE }; + } + + if (el._fullLayout && prevDimCount.current !== activeCols.length) { + window.Plotly.purge(el); + } + prevDimCount.current = activeCols.length; + + const config = { + showLink: false, + displaylogo: false, + responsive: true, + doubleClick: 'reset', + }; + const cameraIcon = window.Plotly.Icons && window.Plotly.Icons.camera; + if (cameraIcon) { + config.modeBarButtonsToRemove = ['toImage']; + config.modeBarButtonsToAdd = [ + { + name: 'downloadPng', + title: 'Download plot as PNG', + icon: cameraIcon, + click: downloadSnapshot, + }, + ]; + } + + try { + window.Plotly.react(el, data, layout, config) + .then(() => { + if (el._fullLayout && el.offsetWidth > 0) { + window.Plotly.Plots.resize(el); + } + }) + .catch(() => window.Plotly.purge(el)); + } catch (e) { + window.Plotly.purge(el); + } + }, [records, columns, effectiveDims, effectiveColorBy]); + + const handleDims = (value) => { + setSelectedDims(Array.isArray(value) ? value.slice(0, MAX_DIMS) : []); + }; + + if (!hasPlot) { + return ( +
+
+ A scatter matrix needs at least two numeric params or metrics. +
+
+ ); + } + + return ( +
+
+ + dimensions: + + + + color by: + setColorBy(value || null)} + aria-label="Color scatter matrix by" + /> + + {truncated ? ( + showing first {MAX_DIMS} + ) : null} +
+
+ {effectiveDims.length < 2 ? ( +
+ Select at least two dimensions to plot. +
+ ) : null} +
+ ); +}; + +export default HParamsSplom; diff --git a/js/panes/hparams/HParamsTable.js b/js/panes/hparams/HParamsTable.js index d6472be0d..cd09ee594 100644 --- a/js/panes/hparams/HParamsTable.js +++ b/js/panes/hparams/HParamsTable.js @@ -21,7 +21,6 @@ import { } from './hparamsUtils'; const RUN_COLUMN_ID = 'run:name'; -const CONTROL_STYLE = { width: 150 }; function nextSort(current, columnId) { if (current.by !== columnId) return { by: columnId, dir: 'asc' }; @@ -294,8 +293,7 @@ const HParamsTable = ({ records, paramKeys, metricKeys, tagKeys }) => { sort by: { color by: 0.62 ? '#fff' : '#333' }; } + +/* + * Numeric param/metric columns only — the axes a scatter matrix (SPLOM) or a + * "color by" ramp can actually plot. Tags are excluded (categorical) and any + * column whose values are all missing/non-numeric is dropped. + */ +export function selectNumericColumns(records, columns) { + return (columns || []).filter( + (col) => + (col.group === 'param' || col.group === 'metric') && + numericExtent(records, col.accessor) !== null + ); +} + +/* + * Build Plotly `splom` dimensions from the chosen column ids. Order follows + * selectedIds; unknown ids are skipped; missing/non-numeric cells become null + * so Plotly leaves a gap instead of plotting a bogus 0. + */ +export function buildSplomDimensions(records, columns, selectedIds) { + const byId = new Map((columns || []).map((col) => [col.id, col])); + const dimensions = []; + (selectedIds || []).forEach((id) => { + const col = byId.get(id); + if (!col) return; + const values = (records || []).map((record) => { + const value = col.accessor(record); + return isNumeric(value) ? value : null; + }); + if (values.every((v) => v === null)) return; + dimensions.push({ label: col.label, values }); + }); + return dimensions; +} diff --git a/js/settings.js b/js/settings.js index 5537f84ba..358e128a0 100644 --- a/js/settings.js +++ b/js/settings.js @@ -32,7 +32,7 @@ const PANE_SIZE = { embeddings: [20, 20], properties: [20, 20], network: [20, 20], - hparams: [40, 24], + hparams: [98, 46], }; const MODAL_STYLE = { content: { diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index cf4e825f8..2062f2ffe 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -180,6 +180,14 @@ color: #333; } +.hparams-select-narrow { + width: 150px; +} + +.hparams-select-wide { + width: 220px; +} + .hparams-treeselect .rc-tree-select-selector { border-radius: 4px !important; border-color: #dedede !important; @@ -405,3 +413,78 @@ text-align: center; color: grey; } + +/* ---- View switcher (Table | Scatter matrix) ---- */ + +.hparams-viewtabs { + display: flex; + gap: 2px; + padding: 4px 10px 0; + background-color: #fff; + border-bottom: 1px solid #dedede; +} + +.hparams-viewtab { + padding: 3px 12px; + font-family: "Open Sans", sans-serif; + font-size: 12px; + color: #666; + background: none; + border: 1px solid transparent; + border-bottom: none; + border-radius: 4px 4px 0 0; + cursor: pointer; +} + +.hparams-viewtab:hover { + color: #3b5998; +} + +.hparams-viewtab-active { + color: #3b5998; + font-weight: 600; + background-color: #f0f0f0; + border-color: #dedede; +} + +.hparams-viewtab:focus-visible { + outline: 2px solid #3b5998; + outline-offset: -2px; +} + +/* ---- HParamsSplom (B4) ---- */ + +.hparams-splom-wrap { + position: relative; + flex: 1 1 auto; + min-height: 0; + min-width: 0; + display: flex; + flex-direction: column; +} + +.hparams-splom-plot { + flex: 1 1 auto; + min-height: 0; + min-width: 0; +} + +.hparams-splom-overlay { + position: absolute; + top: 40px; + right: 0; + bottom: 0; + left: 0; + display: flex; + align-items: center; + justify-content: center; + color: grey; + pointer-events: none; +} + +.hparams-splom-note { + margin-left: auto; + color: #888; + font-style: italic; + white-space: nowrap; +} From 7134e02901385816b90f5bbf980c6f005c982b1a Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Tue, 21 Jul 2026 16:53:57 +0530 Subject: [PATCH 23/48] feat(hparams): add parallel coordinates view --- js/panes/HParamsPane.js | 29 +- js/panes/hparams/HParamsParallelCoords.js | 317 ++++++++++++++++++++++ js/panes/hparams/hparamsUtils.js | 23 ++ py/visdom/static/css/hparams.css | 19 +- 4 files changed, 372 insertions(+), 16 deletions(-) create mode 100644 js/panes/hparams/HParamsParallelCoords.js diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index 090970f86..2ef0eb649 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -9,12 +9,14 @@ import React, { useState } from 'react'; +import HParamsParallelCoords from './hparams/HParamsParallelCoords'; import HParamsSplom from './hparams/HParamsSplom'; import HParamsTable from './hparams/HParamsTable'; import Pane from './Pane'; const VIEWS = [ { key: 'table', label: 'Table' }, + { key: 'parcoords', label: 'Parallel coordinates' }, { key: 'splom', label: 'Scatter matrix' }, ]; @@ -98,21 +100,18 @@ var HParamsPane = (props) => { ))}
- {view === 'splom' ? ( - - ) : ( - - )} + {(() => { + const viewProps = { + records: data.records, + paramKeys: data.paramKeys, + metricKeys: data.metricKeys, + tagKeys: data.tagKeys, + }; + if (view === 'splom') return ; + if (view === 'parcoords') + return ; + return ; + })()} ); diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js new file mode 100644 index 000000000..0ed6e801a --- /dev/null +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -0,0 +1,317 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import TreeSelect from 'rc-tree-select'; +import React, { useEffect, useMemo, useRef, useState } from 'react'; + +import { + buildColumns, + buildParcoordsDimensions, + numericExtent, + selectNumericColumns, +} from './hparamsUtils'; + +const MAX_DIMS = 10; + +const PARCOORDS_COLORSCALE = 'Viridis'; + +const SNAPSHOT_NOTICE_DELAY = 700; + +function notify(message, kind) { + const lib = window.Plotly && window.Plotly.Lib; + if (lib && typeof lib.notifier === 'function') lib.notifier(message, kind); +} + +function downloadSnapshot(gd) { + if (!window.Plotly || typeof window.Plotly.toImage !== 'function') return; + let done = false; + const timer = setTimeout(() => { + if (!done) notify('Taking snapshot - this may take a few seconds', 'long'); + }, SNAPSHOT_NOTICE_DELAY); + + window.Plotly.toImage(gd, { + format: 'png', + width: gd.offsetWidth || 900, + height: gd.offsetHeight || 600, + }) + .then((url) => { + done = true; + clearTimeout(timer); + const link = document.createElement('a'); + link.href = url; + link.download = 'hparams_parcoords.png'; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }) + .catch(() => { + done = true; + clearTimeout(timer); + notify('Snapshot failed', 'long'); + }); +} + +function groupedTreeData(cols) { + const params = cols.filter((c) => c.group === 'param'); + const metrics = cols.filter((c) => c.group === 'metric'); + const branch = (key, title, children) => + children.length + ? [ + { + key: '__g_' + key, + value: '__g_' + key, + title, + selectable: false, + checkable: false, + children: children.map((c) => ({ + key: c.id, + value: c.id, + title: c.label, + })), + }, + ] + : []; + return [ + ...branch('param', 'params', params), + ...branch('metric', 'metrics', metrics), + ]; +} + +const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { + const plotRef = useRef(null); + + const columns = useMemo( + () => buildColumns(paramKeys, metricKeys, tagKeys), + [paramKeys, metricKeys, tagKeys] + ); + const numericCols = useMemo( + () => selectNumericColumns(records, columns), + [records, columns] + ); + + const [selectedDims, setSelectedDims] = useState(null); + const [colorBy, setColorBy] = useState(null); + + const effectiveDims = useMemo(() => { + const validIds = new Set(numericCols.map((c) => c.id)); + let ids = (selectedDims || []).filter((id) => validIds.has(id)); + if (ids.length === 0) ids = numericCols.slice(0, MAX_DIMS).map((c) => c.id); + return ids.slice(0, MAX_DIMS); + }, [selectedDims, numericCols]); + + const effectiveColorBy = useMemo(() => { + if (!colorBy) return null; + return numericCols.some((c) => c.id === colorBy) ? colorBy : null; + }, [colorBy, numericCols]); + + const truncated = + (selectedDims || []).filter((id) => numericCols.some((c) => c.id === id)) + .length > MAX_DIMS; + + const dimTreeData = useMemo( + () => groupedTreeData(numericCols), + [numericCols] + ); + const colorTreeData = useMemo( + () => groupedTreeData(numericCols), + [numericCols] + ); + + const hasPlot = numericCols.length >= 2; + + useEffect(() => { + const el = plotRef.current; + if (!el) return; + const isDisplayed = (node) => + !!(node && node.offsetWidth > 0 && node.offsetHeight > 0); + const resizeObserver = new ResizeObserver(() => { + if (window.Plotly && el._fullLayout && isDisplayed(el)) { + window.Plotly.Plots.resize(el); + } + }); + resizeObserver.observe(el); + return () => { + resizeObserver.disconnect(); + if (window.Plotly && el._fullLayout) window.Plotly.purge(el); + }; + }, []); + + useEffect(() => { + const el = plotRef.current; + if (!el || !window.Plotly) return; + + const dimensions = buildParcoordsDimensions( + records, + columns, + effectiveDims + ); + if (dimensions.length < 2) { + window.Plotly.purge(el); + return; + } + + const colorCol = effectiveColorBy + ? columns.find((c) => c.id === effectiveColorBy) + : null; + let line; + if (colorCol) { + const ext = numericExtent(records, colorCol.accessor); + line = { + color: records.map((r) => { + const v = colorCol.accessor(r); + return typeof v === 'number' && Number.isFinite(v) ? v : null; + }), + colorscale: PARCOORDS_COLORSCALE, + showscale: true, + cmin: ext ? ext.min : 0, + cmax: ext ? ext.max : 1, + colorbar: { + title: { text: colorCol.label, side: 'right', font: { size: 11 } }, + thickness: 12, + len: 0.6, + outlinewidth: 0, + }, + }; + } else { + line = { + color: records.map((_, i) => i + 1), + colorscale: PARCOORDS_COLORSCALE, + showscale: true, + cmin: 1, + cmax: Math.max(records.length, 1), + colorbar: { + title: { text: 'run order', side: 'right', font: { size: 11 } }, + thickness: 12, + len: 0.6, + outlinewidth: 0, + }, + }; + } + + const data = [ + { + type: 'parcoords', + dimensions, + line, + labelfont: { size: 12 }, + tickfont: { size: 10 }, + rangefont: { size: 10 }, + }, + ]; + + const layout = { + margin: { l: 60, r: 40, t: 30, b: 24 }, + font: { family: '"Open Sans", sans-serif', size: 11, color: '#333' }, + paper_bgcolor: '#ffffff', + plot_bgcolor: '#ffffff', + datarevision: + effectiveDims.join('|') + + '::' + + (effectiveColorBy || 'order') + + '::' + + records.length, + }; + + const config = { + showLink: false, + displaylogo: false, + responsive: true, + doubleClick: 'reset', + }; + const cameraIcon = window.Plotly.Icons && window.Plotly.Icons.camera; + if (cameraIcon) { + config.modeBarButtonsToRemove = ['toImage']; + config.modeBarButtonsToAdd = [ + { + name: 'downloadPng', + title: 'Download plot as PNG', + icon: cameraIcon, + click: downloadSnapshot, + }, + ]; + } + + try { + window.Plotly.react(el, data, layout, config) + .then(() => { + if (el._fullLayout && el.offsetWidth > 0) { + window.Plotly.Plots.resize(el); + } + }) + .catch(() => window.Plotly.purge(el)); + } catch (e) { + window.Plotly.purge(el); + } + }, [records, columns, effectiveDims, effectiveColorBy]); + + const handleDims = (value) => { + setSelectedDims(Array.isArray(value) ? value.slice(0, MAX_DIMS) : []); + }; + + if (!hasPlot) { + return ( +
+
+ Parallel coordinates need at least two numeric params or metrics. +
+
+ ); + } + + return ( +
+
+ + axes: + + + + color by: + setColorBy(value || null)} + aria-label="Color parallel coordinates by" + /> + + {truncated ? ( + showing first {MAX_DIMS} + ) : null} +
+
+ {effectiveDims.length < 2 ? ( +
+ Select at least two dimensions to plot. +
+ ) : null} +
+ ); +}; + +export default HParamsParallelCoords; diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index 541e2e9f2..ba45c78de 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -184,3 +184,26 @@ export function buildSplomDimensions(records, columns, selectedIds) { }); return dimensions; } + +export function buildParcoordsDimensions(records, columns, selectedIds) { + const byId = new Map((columns || []).map((col) => [col.id, col])); + const dimensions = []; + (selectedIds || []).forEach((id) => { + const col = byId.get(id); + if (!col) return; + const values = (records || []).map((record) => { + const value = col.accessor(record); + return isNumeric(value) ? value : null; + }); + const extent = numericExtent(records, col.accessor); + if (extent === null) return; + const span = extent.max - extent.min; + const pad = span > 0 ? span * 0.05 : Math.abs(extent.max) * 0.05 || 1; + dimensions.push({ + label: col.label, + values, + range: [extent.min - pad, extent.max + pad], + }); + }); + return dimensions; +} diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index 2062f2ffe..529766942 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -414,7 +414,7 @@ color: grey; } -/* ---- View switcher (Table | Scatter matrix) ---- */ +/* ---- View switcher (Table | Parallel coordinates | Scatter matrix) ---- */ .hparams-viewtabs { display: flex; @@ -488,3 +488,20 @@ font-style: italic; white-space: nowrap; } + +/* ---- HParamsParallelCoords (B3) ---- */ + +.hparams-parcoords-wrap { + position: relative; + flex: 1 1 auto; + min-height: 0; + min-width: 0; + display: flex; + flex-direction: column; +} + +.hparams-parcoords-plot { + flex: 1 1 auto; + min-height: 0; + min-width: 0; +} From eb3f6a170dd16d4941d5c448f1c3882478907862 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 22 Jul 2026 15:56:30 +0530 Subject: [PATCH 24/48] refactor(hparams): share column, tree, and plot helpers across the three views Centralize the numeric-column selection, grouped TreeSelect data, run labelling, and numeric-or-null mapping in hparamsUtils, and move the Plotly-facing snapshot download, notifier, snapshot mode-bar button, and resize observer into a new hparamsPlot module. The table, scatter matrix, and parallel coordinates views now consume these instead of each carrying their own copy. --- js/panes/hparams/HParamsParallelCoords.js | 123 +++--------------- js/panes/hparams/HParamsSplom.js | 149 ++++------------------ js/panes/hparams/HParamsTable.js | 76 +++-------- js/panes/hparams/hparamsPlot.js | 81 ++++++++++++ js/panes/hparams/hparamsUtils.js | 51 ++++++-- 5 files changed, 187 insertions(+), 293 deletions(-) create mode 100644 js/panes/hparams/hparamsPlot.js diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index 0ed6e801a..359d14d53 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -10,79 +10,21 @@ import TreeSelect from 'rc-tree-select'; import React, { useEffect, useMemo, useRef, useState } from 'react'; +import { applySnapshotButton, observePlotResize } from './hparamsPlot'; import { buildColumns, buildParcoordsDimensions, + groupColumnTree, + NUMERIC_GROUPS, numericExtent, selectNumericColumns, + toNumericColumn, } from './hparamsUtils'; const MAX_DIMS = 10; const PARCOORDS_COLORSCALE = 'Viridis'; -const SNAPSHOT_NOTICE_DELAY = 700; - -function notify(message, kind) { - const lib = window.Plotly && window.Plotly.Lib; - if (lib && typeof lib.notifier === 'function') lib.notifier(message, kind); -} - -function downloadSnapshot(gd) { - if (!window.Plotly || typeof window.Plotly.toImage !== 'function') return; - let done = false; - const timer = setTimeout(() => { - if (!done) notify('Taking snapshot - this may take a few seconds', 'long'); - }, SNAPSHOT_NOTICE_DELAY); - - window.Plotly.toImage(gd, { - format: 'png', - width: gd.offsetWidth || 900, - height: gd.offsetHeight || 600, - }) - .then((url) => { - done = true; - clearTimeout(timer); - const link = document.createElement('a'); - link.href = url; - link.download = 'hparams_parcoords.png'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - }) - .catch(() => { - done = true; - clearTimeout(timer); - notify('Snapshot failed', 'long'); - }); -} - -function groupedTreeData(cols) { - const params = cols.filter((c) => c.group === 'param'); - const metrics = cols.filter((c) => c.group === 'metric'); - const branch = (key, title, children) => - children.length - ? [ - { - key: '__g_' + key, - value: '__g_' + key, - title, - selectable: false, - checkable: false, - children: children.map((c) => ({ - key: c.id, - value: c.id, - title: c.label, - })), - }, - ] - : []; - return [ - ...branch('param', 'params', params), - ...branch('metric', 'metrics', metrics), - ]; -} - const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { const plotRef = useRef(null); @@ -114,12 +56,8 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { (selectedDims || []).filter((id) => numericCols.some((c) => c.id === id)) .length > MAX_DIMS; - const dimTreeData = useMemo( - () => groupedTreeData(numericCols), - [numericCols] - ); - const colorTreeData = useMemo( - () => groupedTreeData(numericCols), + const treeData = useMemo( + () => groupColumnTree(numericCols, NUMERIC_GROUPS), [numericCols] ); @@ -128,18 +66,7 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { useEffect(() => { const el = plotRef.current; if (!el) return; - const isDisplayed = (node) => - !!(node && node.offsetWidth > 0 && node.offsetHeight > 0); - const resizeObserver = new ResizeObserver(() => { - if (window.Plotly && el._fullLayout && isDisplayed(el)) { - window.Plotly.Plots.resize(el); - } - }); - resizeObserver.observe(el); - return () => { - resizeObserver.disconnect(); - if (window.Plotly && el._fullLayout) window.Plotly.purge(el); - }; + return observePlotResize(el); }, []); useEffect(() => { @@ -163,10 +90,7 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { if (colorCol) { const ext = numericExtent(records, colorCol.accessor); line = { - color: records.map((r) => { - const v = colorCol.accessor(r); - return typeof v === 'number' && Number.isFinite(v) ? v : null; - }), + color: toNumericColumn(records, colorCol.accessor), colorscale: PARCOORDS_COLORSCALE, showscale: true, cmin: ext ? ext.min : 0, @@ -218,24 +142,15 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { records.length, }; - const config = { - showLink: false, - displaylogo: false, - responsive: true, - doubleClick: 'reset', - }; - const cameraIcon = window.Plotly.Icons && window.Plotly.Icons.camera; - if (cameraIcon) { - config.modeBarButtonsToRemove = ['toImage']; - config.modeBarButtonsToAdd = [ - { - name: 'downloadPng', - title: 'Download plot as PNG', - icon: cameraIcon, - click: downloadSnapshot, - }, - ]; - } + const config = applySnapshotButton( + { + showLink: false, + displaylogo: false, + responsive: true, + doubleClick: 'reset', + }, + 'hparams_parcoords.png' + ); try { window.Plotly.react(el, data, layout, config) @@ -280,7 +195,7 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { treeDefaultExpandAll maxTagCount={3} dropdownMatchSelectWidth={false} - treeData={dimTreeData} + treeData={treeData} onChange={handleDims} aria-label="Parallel coordinates axes" /> @@ -295,7 +210,7 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { treeLine treeDefaultExpandAll dropdownMatchSelectWidth={false} - treeData={colorTreeData} + treeData={treeData} onChange={(value) => setColorBy(value || null)} aria-label="Color parallel coordinates by" /> diff --git a/js/panes/hparams/HParamsSplom.js b/js/panes/hparams/HParamsSplom.js index 5872016ae..2bbc04344 100644 --- a/js/panes/hparams/HParamsSplom.js +++ b/js/panes/hparams/HParamsSplom.js @@ -10,11 +10,16 @@ import TreeSelect from 'rc-tree-select'; import React, { useEffect, useMemo, useRef, useState } from 'react'; +import { applySnapshotButton, observePlotResize } from './hparamsPlot'; import { buildColumns, - isNumeric, + buildSplomDimensions, + groupColumnTree, + NUMERIC_GROUPS, numericExtent, + runLabel, selectNumericColumns, + toNumericColumn, } from './hparamsUtils'; const MAX_DIMS = 6; @@ -33,68 +38,6 @@ const AXIS_STYLE = { automargin: true, }; -const SNAPSHOT_NOTICE_DELAY = 700; - -function notify(message, kind) { - const lib = window.Plotly && window.Plotly.Lib; - if (lib && typeof lib.notifier === 'function') lib.notifier(message, kind); -} - -function downloadSnapshot(gd) { - if (!window.Plotly || typeof window.Plotly.toImage !== 'function') return; - let done = false; - const timer = setTimeout(() => { - if (!done) notify('Taking snapshot - this may take a few seconds', 'long'); - }, SNAPSHOT_NOTICE_DELAY); - - window.Plotly.toImage(gd, { - format: 'png', - width: gd.offsetWidth || 900, - height: gd.offsetHeight || 600, - }) - .then((url) => { - done = true; - clearTimeout(timer); - const link = document.createElement('a'); - link.href = url; - link.download = 'hparams_scatter.png'; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - }) - .catch(() => { - done = true; - clearTimeout(timer); - notify('Snapshot failed', 'long'); - }); -} - -function groupedTreeData(cols) { - const params = cols.filter((c) => c.group === 'param'); - const metrics = cols.filter((c) => c.group === 'metric'); - const branch = (key, title, children) => - children.length - ? [ - { - key: '__g_' + key, - value: '__g_' + key, - title, - selectable: false, - checkable: false, - children: children.map((c) => ({ - key: c.id, - value: c.id, - title: c.label, - })), - }, - ] - : []; - return [ - ...branch('param', 'params', params), - ...branch('metric', 'metrics', metrics), - ]; -} - const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { const plotRef = useRef(null); const prevDimCount = useRef(0); @@ -127,12 +70,8 @@ const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { (selectedDims || []).filter((id) => numericCols.some((c) => c.id === id)) .length > MAX_DIMS; - const dimTreeData = useMemo( - () => groupedTreeData(numericCols), - [numericCols] - ); - const colorTreeData = useMemo( - () => groupedTreeData(numericCols), + const treeData = useMemo( + () => groupColumnTree(numericCols, NUMERIC_GROUPS), [numericCols] ); @@ -141,43 +80,21 @@ const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { useEffect(() => { const el = plotRef.current; if (!el) return; - const isDisplayed = (node) => - !!(node && node.offsetWidth > 0 && node.offsetHeight > 0); - const resizeObserver = new ResizeObserver(() => { - if (window.Plotly && el._fullLayout && isDisplayed(el)) { - window.Plotly.Plots.resize(el); - } - }); - resizeObserver.observe(el); - return () => { - resizeObserver.disconnect(); - if (window.Plotly && el._fullLayout) window.Plotly.purge(el); - }; + return observePlotResize(el); }, []); useEffect(() => { const el = plotRef.current; if (!el || !window.Plotly) return; - const activeCols = effectiveDims - .map((id) => columns.find((c) => c.id === id)) - .filter((col) => col && records.some((r) => isNumeric(col.accessor(r)))); - if (activeCols.length < 2) { + const dimensions = buildSplomDimensions(records, columns, effectiveDims); + if (dimensions.length < 2) { window.Plotly.purge(el); prevDimCount.current = 0; return; } - const dimensions = activeCols.map((col) => ({ - label: col.label, - values: records.map((r) => { - const v = col.accessor(r); - return isNumeric(v) ? v : null; - }), - })); - - const label = (r) => r.name || r.env_id || 'run'; - const names = records.map(label); + const names = records.map(runLabel); const colorCol = effectiveColorBy ? columns.find((c) => c.id === effectiveColorBy) @@ -187,10 +104,7 @@ const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { let cmin; let cmax; if (colorCol) { - colorValues = records.map((r) => { - const v = colorCol.accessor(r); - return isNumeric(v) ? v : null; - }); + colorValues = toNumericColumn(records, colorCol.accessor); colorLabel = colorCol.label; const ext = numericExtent(records, colorCol.accessor); if (ext) { @@ -247,35 +161,26 @@ const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { '::' + records.length, }; - for (let i = 1; i <= activeCols.length; i++) { + for (let i = 1; i <= dimensions.length; i++) { const suffix = i === 1 ? '' : String(i); layout['xaxis' + suffix] = { ...AXIS_STYLE }; layout['yaxis' + suffix] = { ...AXIS_STYLE }; } - if (el._fullLayout && prevDimCount.current !== activeCols.length) { + if (el._fullLayout && prevDimCount.current !== dimensions.length) { window.Plotly.purge(el); } - prevDimCount.current = activeCols.length; + prevDimCount.current = dimensions.length; - const config = { - showLink: false, - displaylogo: false, - responsive: true, - doubleClick: 'reset', - }; - const cameraIcon = window.Plotly.Icons && window.Plotly.Icons.camera; - if (cameraIcon) { - config.modeBarButtonsToRemove = ['toImage']; - config.modeBarButtonsToAdd = [ - { - name: 'downloadPng', - title: 'Download plot as PNG', - icon: cameraIcon, - click: downloadSnapshot, - }, - ]; - } + const config = applySnapshotButton( + { + showLink: false, + displaylogo: false, + responsive: true, + doubleClick: 'reset', + }, + 'hparams_scatter.png' + ); try { window.Plotly.react(el, data, layout, config) @@ -320,7 +225,7 @@ const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { treeDefaultExpandAll maxTagCount={3} dropdownMatchSelectWidth={false} - treeData={dimTreeData} + treeData={treeData} onChange={handleDims} aria-label="Scatter matrix dimensions" /> @@ -335,7 +240,7 @@ const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { treeLine treeDefaultExpandAll dropdownMatchSelectWidth={false} - treeData={colorTreeData} + treeData={treeData} onChange={(value) => setColorBy(value || null)} aria-label="Color scatter matrix by" /> diff --git a/js/panes/hparams/HParamsTable.js b/js/panes/hparams/HParamsTable.js index cd09ee594..4cd2fd47f 100644 --- a/js/panes/hparams/HParamsTable.js +++ b/js/panes/hparams/HParamsTable.js @@ -12,11 +12,16 @@ import React, { useCallback, useMemo, useState } from 'react'; import { buildColumns, + COLUMN_GROUPS, filterRecords, formatValue, + groupColumnTree, isNumeric, makeComparator, + NUMERIC_GROUPS, numericExtent, + runLabel, + selectNumericColumns, spineStyle, } from './hparamsUtils'; @@ -89,7 +94,7 @@ const HParamsRow = React.memo(function HParamsRow({ onToggle, groupStartIds, }) { - const runLabel = record.name || record.env_id || 'run'; + const label = runLabel(record); return ( onToggle(record.env_id)} - aria-label={'Select ' + runLabel} + aria-label={'Select ' + label} /> -
- {runLabel} +
+ {label} {record.status ? ( { [paramKeys, metricKeys, tagKeys] ); const colorCols = useMemo( - () => - columns.filter( - (c) => - (c.group === 'param' || c.group === 'metric') && - numericExtent(records, c.accessor) - ), - [columns, records] + () => selectNumericColumns(records, columns), + [records, columns] ); - const colorParamCols = colorCols.filter((c) => c.group === 'param'); - const colorMetricCols = colorCols.filter((c) => c.group === 'metric'); const groupStartIds = useMemo(() => { const ids = new Set(); @@ -227,57 +225,17 @@ const HParamsTable = ({ records, paramKeys, metricKeys, tagKeys }) => { }); }, [rows]); - const bands = [ - { key: 'param', label: 'params' }, - { key: 'metric', label: 'metrics' }, - { key: 'tag', label: 'tags' }, - ] - .map((b) => ({ - ...b, - span: columns.filter((c) => c.group === b.key).length, - })) - .filter((b) => b.span > 0); + const bands = COLUMN_GROUPS.map((b) => ({ + ...b, + span: columns.filter((c) => c.group === b.key).length, + })).filter((b) => b.span > 0); const sortTreeData = [ { key: RUN_COLUMN_ID, value: RUN_COLUMN_ID, title: 'run' }, - ...bands.map((b) => ({ - key: '__g_' + b.key, - value: '__g_' + b.key, - title: b.label, - selectable: false, - children: columns - .filter((c) => c.group === b.key) - .map((c) => ({ key: c.id, value: c.id, title: c.label })), - })), + ...groupColumnTree(columns, COLUMN_GROUPS), ]; - const colorTreeData = []; - if (colorParamCols.length) { - colorTreeData.push({ - key: '__cg_param', - value: '__cg_param', - title: 'params', - selectable: false, - children: colorParamCols.map((c) => ({ - key: c.id, - value: c.id, - title: c.label, - })), - }); - } - if (colorMetricCols.length) { - colorTreeData.push({ - key: '__cg_metric', - value: '__cg_metric', - title: 'metrics', - selectable: false, - children: colorMetricCols.map((c) => ({ - key: c.id, - value: c.id, - title: c.label, - })), - }); - } + const colorTreeData = groupColumnTree(colorCols, NUMERIC_GROUPS); return (
diff --git a/js/panes/hparams/hparamsPlot.js b/js/panes/hparams/hparamsPlot.js new file mode 100644 index 000000000..005f0c90f --- /dev/null +++ b/js/panes/hparams/hparamsPlot.js @@ -0,0 +1,81 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +/* + * Plotly-facing helpers shared by the hyper-parameter plot views (scatter + * matrix and parallel coordinates). Unlike hparamsUtils these touch the global + * Plotly instance and the DOM, so they live apart from the pure helpers. + */ + +const SNAPSHOT_NOTICE_DELAY = 700; + +export function notify(message, kind) { + const lib = window.Plotly && window.Plotly.Lib; + if (lib && typeof lib.notifier === 'function') lib.notifier(message, kind); +} + +export function downloadPlotPng(gd, filename) { + if (!window.Plotly || typeof window.Plotly.toImage !== 'function') return; + let done = false; + const timer = setTimeout(() => { + if (!done) notify('Taking snapshot - this may take a few seconds', 'long'); + }, SNAPSHOT_NOTICE_DELAY); + + window.Plotly.toImage(gd, { + format: 'png', + width: gd.offsetWidth || 900, + height: gd.offsetHeight || 600, + }) + .then((url) => { + done = true; + clearTimeout(timer); + const link = document.createElement('a'); + link.href = url; + link.download = filename; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + }) + .catch(() => { + done = true; + clearTimeout(timer); + notify('Snapshot failed', 'long'); + }); +} + +export function applySnapshotButton(config, filename) { + const icons = window.Plotly && window.Plotly.Icons; + const icon = icons && icons.camera; + if (!icon) return config; + config.modeBarButtonsToRemove = ['toImage']; + config.modeBarButtonsToAdd = [ + { + name: 'downloadPng', + title: 'Download plot as PNG', + icon, + click: (gd) => downloadPlotPng(gd, filename), + }, + ]; + return config; +} + +export function observePlotResize(el) { + const isDisplayed = (node) => + !!(node && node.offsetWidth > 0 && node.offsetHeight > 0); + const resizeObserver = new ResizeObserver(() => { + if (window.Plotly && el._fullLayout && isDisplayed(el)) { + window.Plotly.Plots.resize(el); + } + }); + resizeObserver.observe(el); + return () => { + resizeObserver.disconnect(); + if (window.Plotly && el._fullLayout) window.Plotly.purge(el); + }; +} diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index ba45c78de..631c19a86 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -18,6 +18,18 @@ const SPINE_LIGHT = [235, 240, 249]; const SPINE_DARK = [59, 89, 152]; +export const COLUMN_GROUPS = [ + { key: 'param', label: 'params' }, + { key: 'metric', label: 'metrics' }, + { key: 'tag', label: 'tags' }, +]; + +export const NUMERIC_GROUPS = COLUMN_GROUPS.slice(0, 2); + +export function runLabel(record) { + return (record && (record.name || record.env_id)) || 'run'; +} + export function isMissing(value) { return ( value === undefined || @@ -93,6 +105,28 @@ export function buildColumns(paramKeys, metricKeys, tagKeys) { return columns; } +export function groupColumnTree(columns, groups) { + const cols = columns || []; + return (groups || []) + .map(({ key, label }) => { + const children = cols.filter((col) => col.group === key); + if (children.length === 0) return null; + return { + key: '__g_' + key, + value: '__g_' + key, + title: label, + selectable: false, + checkable: false, + children: children.map((col) => ({ + key: col.id, + value: col.id, + title: col.label, + })), + }; + }) + .filter(Boolean); +} + export function formatValue(value) { if (isMissing(value)) return '—'; if (typeof value === 'number') { @@ -119,6 +153,13 @@ export function filterRecords(records, query, columns) { }); } +export function toNumericColumn(records, accessor) { + return (records || []).map((record) => { + const value = accessor(record); + return isNumeric(value) ? value : null; + }); +} + export function numericExtent(records, accessor) { let min = Infinity; let max = -Infinity; @@ -175,10 +216,7 @@ export function buildSplomDimensions(records, columns, selectedIds) { (selectedIds || []).forEach((id) => { const col = byId.get(id); if (!col) return; - const values = (records || []).map((record) => { - const value = col.accessor(record); - return isNumeric(value) ? value : null; - }); + const values = toNumericColumn(records, col.accessor); if (values.every((v) => v === null)) return; dimensions.push({ label: col.label, values }); }); @@ -191,10 +229,7 @@ export function buildParcoordsDimensions(records, columns, selectedIds) { (selectedIds || []).forEach((id) => { const col = byId.get(id); if (!col) return; - const values = (records || []).map((record) => { - const value = col.accessor(record); - return isNumeric(value) ? value : null; - }); + const values = toNumericColumn(records, col.accessor); const extent = numericExtent(records, col.accessor); if (extent === null) return; const span = extent.max - extent.min; From 696c4a28a98642033cfd7cb77e39464a6c27b269 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 22 Jul 2026 15:56:44 +0530 Subject: [PATCH 25/48] fix(hparams): stop clipping parallel-coordinates axis labels The top margin was 30px, so Plotly drew the axis titles above the plotting area and the container clipped them, leaving the names half-cut. Widen the margins so titles, the top/bottom range values, and the colorbar all have room, and pin the label, tick, and range font colors for consistent contrast. --- js/panes/hparams/HParamsParallelCoords.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index 359d14d53..a7bbf958a 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -123,14 +123,16 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { type: 'parcoords', dimensions, line, - labelfont: { size: 12 }, - tickfont: { size: 10 }, - rangefont: { size: 10 }, + labelangle: 0, + labelside: 'top', + labelfont: { size: 12, color: '#333' }, + tickfont: { size: 10, color: '#666' }, + rangefont: { size: 10, color: '#888' }, }, ]; const layout = { - margin: { l: 60, r: 40, t: 30, b: 24 }, + margin: { l: 70, r: 80, t: 64, b: 48 }, font: { family: '"Open Sans", sans-serif', size: 11, color: '#333' }, paper_bgcolor: '#ffffff', plot_bgcolor: '#ffffff', From 51e0f8a181a26de235fd36f72983662cb675fe56 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 22 Jul 2026 16:00:38 +0530 Subject: [PATCH 26/48] fix(hparams): persist each view's selection across view switches HParamsPane rendered one view at a time, so switching unmounted the active view and reset its picks. Lift the table's sort/filter/color/selection and each plot view's axes and color-by into HParamsPane, giving every view its own persistent state, and make the three views controlled components driven by it. --- js/panes/HParamsPane.js | 43 +++++++++++++++++++++-- js/panes/hparams/HParamsParallelCoords.js | 20 +++++++---- js/panes/hparams/HParamsSplom.js | 20 +++++++---- js/panes/hparams/HParamsTable.js | 22 ++++++++---- 4 files changed, 81 insertions(+), 24 deletions(-) diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index 2ef0eb649..6bf6f9c9b 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -40,6 +40,14 @@ var HParamsPane = (props) => { const { content } = props; const data = readContent(content); const [view, setView] = useState('table'); + const [tableSort, setTableSort] = useState({ by: null, dir: null }); + const [tableFilter, setTableFilter] = useState(''); + const [tableColorBy, setTableColorBy] = useState(null); + const [tableSelected, setTableSelected] = useState(() => new Set()); + const [splomDims, setSplomDims] = useState(null); + const [splomColorBy, setSplomColorBy] = useState(null); + const [parcoordsDims, setParcoordsDims] = useState(null); + const [parcoordsColorBy, setParcoordsColorBy] = useState(null); const handleDownload = () => { let blob = new Blob([JSON.stringify(content)], { @@ -107,10 +115,39 @@ var HParamsPane = (props) => { metricKeys: data.metricKeys, tagKeys: data.tagKeys, }; - if (view === 'splom') return ; + if (view === 'splom') + return ( + + ); if (view === 'parcoords') - return ; - return ; + return ( + + ); + return ( + + ); })()}
diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index a7bbf958a..1d156422a 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -8,7 +8,7 @@ */ import TreeSelect from 'rc-tree-select'; -import React, { useEffect, useMemo, useRef, useState } from 'react'; +import React, { useEffect, useMemo, useRef } from 'react'; import { applySnapshotButton, observePlotResize } from './hparamsPlot'; import { @@ -25,7 +25,16 @@ const MAX_DIMS = 10; const PARCOORDS_COLORSCALE = 'Viridis'; -const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { +const HParamsParallelCoords = ({ + records, + paramKeys, + metricKeys, + tagKeys, + selectedDims, + onSelectedDims, + colorBy, + onColorBy, +}) => { const plotRef = useRef(null); const columns = useMemo( @@ -37,9 +46,6 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { [records, columns] ); - const [selectedDims, setSelectedDims] = useState(null); - const [colorBy, setColorBy] = useState(null); - const effectiveDims = useMemo(() => { const validIds = new Set(numericCols.map((c) => c.id)); let ids = (selectedDims || []).filter((id) => validIds.has(id)); @@ -168,7 +174,7 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { }, [records, columns, effectiveDims, effectiveColorBy]); const handleDims = (value) => { - setSelectedDims(Array.isArray(value) ? value.slice(0, MAX_DIMS) : []); + onSelectedDims(Array.isArray(value) ? value.slice(0, MAX_DIMS) : []); }; if (!hasPlot) { @@ -213,7 +219,7 @@ const HParamsParallelCoords = ({ records, paramKeys, metricKeys, tagKeys }) => { treeDefaultExpandAll dropdownMatchSelectWidth={false} treeData={treeData} - onChange={(value) => setColorBy(value || null)} + onChange={(value) => onColorBy(value || null)} aria-label="Color parallel coordinates by" /> diff --git a/js/panes/hparams/HParamsSplom.js b/js/panes/hparams/HParamsSplom.js index 2bbc04344..d6f946edb 100644 --- a/js/panes/hparams/HParamsSplom.js +++ b/js/panes/hparams/HParamsSplom.js @@ -8,7 +8,7 @@ */ import TreeSelect from 'rc-tree-select'; -import React, { useEffect, useMemo, useRef, useState } from 'react'; +import React, { useEffect, useMemo, useRef } from 'react'; import { applySnapshotButton, observePlotResize } from './hparamsPlot'; import { @@ -38,7 +38,16 @@ const AXIS_STYLE = { automargin: true, }; -const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { +const HParamsSplom = ({ + records, + paramKeys, + metricKeys, + tagKeys, + selectedDims, + onSelectedDims, + colorBy, + onColorBy, +}) => { const plotRef = useRef(null); const prevDimCount = useRef(0); @@ -51,9 +60,6 @@ const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { [records, columns] ); - const [selectedDims, setSelectedDims] = useState(null); - const [colorBy, setColorBy] = useState(null); - const effectiveDims = useMemo(() => { const validIds = new Set(numericCols.map((c) => c.id)); let ids = (selectedDims || []).filter((id) => validIds.has(id)); @@ -196,7 +202,7 @@ const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { }, [records, columns, effectiveDims, effectiveColorBy]); const handleDims = (value) => { - setSelectedDims(Array.isArray(value) ? value.slice(0, MAX_DIMS) : []); + onSelectedDims(Array.isArray(value) ? value.slice(0, MAX_DIMS) : []); }; if (!hasPlot) { @@ -241,7 +247,7 @@ const HParamsSplom = ({ records, paramKeys, metricKeys, tagKeys }) => { treeDefaultExpandAll dropdownMatchSelectWidth={false} treeData={treeData} - onChange={(value) => setColorBy(value || null)} + onChange={(value) => onColorBy(value || null)} aria-label="Color scatter matrix by" /> diff --git a/js/panes/hparams/HParamsTable.js b/js/panes/hparams/HParamsTable.js index 4cd2fd47f..ce6866316 100644 --- a/js/panes/hparams/HParamsTable.js +++ b/js/panes/hparams/HParamsTable.js @@ -8,7 +8,7 @@ */ import TreeSelect from 'rc-tree-select'; -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useMemo } from 'react'; import { buildColumns, @@ -140,12 +140,20 @@ const HParamsRow = React.memo(function HParamsRow({ ); }); -const HParamsTable = ({ records, paramKeys, metricKeys, tagKeys }) => { - const [sort, setSort] = useState({ by: null, dir: null }); - const [filter, setFilter] = useState(''); - const [colorBy, setColorBy] = useState(null); - const [selected, setSelected] = useState(() => new Set()); - +const HParamsTable = ({ + records, + paramKeys, + metricKeys, + tagKeys, + sort, + setSort, + filter, + setFilter, + colorBy, + setColorBy, + selected, + setSelected, +}) => { const columns = useMemo( () => buildColumns(paramKeys, metricKeys, tagKeys), [paramKeys, metricKeys, tagKeys] From ea1ec85d6ba7d997044bcb74652d05e499222913 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 22 Jul 2026 16:26:01 +0530 Subject: [PATCH 27/48] fix(hparams): give parallel-coordinates more bottom margin The 48px bottom margin left the axis min-value range labels and the lowest line segments jammed against the pane's bottom edge. Widen it to 76px so the plotting area lifts up and the bottom of every axis is fully visible. --- js/panes/hparams/HParamsParallelCoords.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index 1d156422a..9c00cabc7 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -138,7 +138,7 @@ const HParamsParallelCoords = ({ ]; const layout = { - margin: { l: 70, r: 80, t: 64, b: 48 }, + margin: { l: 70, r: 80, t: 64, b: 76 }, font: { family: '"Open Sans", sans-serif', size: 11, color: '#333' }, paper_bgcolor: '#ffffff', plot_bgcolor: '#ffffff', From 9a14a07a79ee3bd859ae3f98ae29af0d7e6f8637 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 22 Jul 2026 16:29:51 +0530 Subject: [PATCH 28/48] fix(hparams): span parallel-coordinates axes to the exact data range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The axes were padded 5% below their minimum, which produced meaningless negative lower bounds (batch -8.8, lr -0.00895, weight_decay -35µ) and left empty space beneath the lowest line so runs looked like they trailed off the bottom. Range each axis to its true [min, max] instead, keeping a small symmetric range only when every value on an axis is identical. --- js/panes/hparams/hparamsUtils.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index 631c19a86..774cbccab 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -232,13 +232,14 @@ export function buildParcoordsDimensions(records, columns, selectedIds) { const values = toNumericColumn(records, col.accessor); const extent = numericExtent(records, col.accessor); if (extent === null) return; - const span = extent.max - extent.min; - const pad = span > 0 ? span * 0.05 : Math.abs(extent.max) * 0.05 || 1; - dimensions.push({ - label: col.label, - values, - range: [extent.min - pad, extent.max + pad], - }); + const dimension = { label: col.label, values }; + if (extent.min === extent.max) { + const delta = Math.abs(extent.max) * 0.05 || 1; + dimension.range = [extent.min - delta, extent.max + delta]; + } else { + dimension.range = [extent.min, extent.max]; + } + dimensions.push(dimension); }); return dimensions; } From aee589afddd99cf8461d2ea52e5ebb945b110321 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 22 Jul 2026 16:57:06 +0530 Subject: [PATCH 29/48] fix(hparams): drop null-holed runs from parallel coordinates Plotly parcoords cannot render null/NaN cells: a single sparse axis (e.g. a metric only some runs logged) corrupted every line and dropped their colour. Draw a line only for runs that have a numeric value on every selected axis and the colour field (completeRecords), align the colour array to those runs, and default the axes to the fully-populated columns so the initial view shows all runs. A toolbar note reports when runs are excluded by the current axes. --- js/panes/hparams/HParamsParallelCoords.js | 56 +++++++++++++++-------- js/panes/hparams/hparamsUtils.js | 15 +++++- 2 files changed, 52 insertions(+), 19 deletions(-) diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index 9c00cabc7..d15dd8876 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -14,7 +14,9 @@ import { applySnapshotButton, observePlotResize } from './hparamsPlot'; import { buildColumns, buildParcoordsDimensions, + completeRecords, groupColumnTree, + isNumeric, NUMERIC_GROUPS, numericExtent, selectNumericColumns, @@ -49,9 +51,15 @@ const HParamsParallelCoords = ({ const effectiveDims = useMemo(() => { const validIds = new Set(numericCols.map((c) => c.id)); let ids = (selectedDims || []).filter((id) => validIds.has(id)); - if (ids.length === 0) ids = numericCols.slice(0, MAX_DIMS).map((c) => c.id); + if (ids.length === 0) { + const dense = numericCols.filter((c) => + records.every((r) => isNumeric(c.accessor(r))) + ); + const pick = dense.length >= 2 ? dense : numericCols; + ids = pick.slice(0, MAX_DIMS).map((c) => c.id); + } return ids.slice(0, MAX_DIMS); - }, [selectedDims, numericCols]); + }, [selectedDims, numericCols, records]); const effectiveColorBy = useMemo(() => { if (!colorBy) return null; @@ -67,6 +75,17 @@ const HParamsParallelCoords = ({ [numericCols] ); + const rows = useMemo(() => { + const colorCol = effectiveColorBy + ? columns.find((c) => c.id === effectiveColorBy) + : null; + const axisCols = effectiveDims + .map((id) => columns.find((c) => c.id === id)) + .filter(Boolean); + const requiredCols = colorCol ? axisCols.concat(colorCol) : axisCols; + return completeRecords(records, requiredCols); + }, [records, columns, effectiveDims, effectiveColorBy]); + const hasPlot = numericCols.length >= 2; useEffect(() => { @@ -79,24 +98,21 @@ const HParamsParallelCoords = ({ const el = plotRef.current; if (!el || !window.Plotly) return; - const dimensions = buildParcoordsDimensions( - records, - columns, - effectiveDims - ); - if (dimensions.length < 2) { + const colorCol = effectiveColorBy + ? columns.find((c) => c.id === effectiveColorBy) + : null; + + const dimensions = buildParcoordsDimensions(rows, columns, effectiveDims); + if (dimensions.length < 2 || rows.length === 0) { window.Plotly.purge(el); return; } - const colorCol = effectiveColorBy - ? columns.find((c) => c.id === effectiveColorBy) - : null; let line; if (colorCol) { - const ext = numericExtent(records, colorCol.accessor); + const ext = numericExtent(rows, colorCol.accessor); line = { - color: toNumericColumn(records, colorCol.accessor), + color: toNumericColumn(rows, colorCol.accessor), colorscale: PARCOORDS_COLORSCALE, showscale: true, cmin: ext ? ext.min : 0, @@ -110,11 +126,11 @@ const HParamsParallelCoords = ({ }; } else { line = { - color: records.map((_, i) => i + 1), + color: rows.map((_, i) => i + 1), colorscale: PARCOORDS_COLORSCALE, showscale: true, cmin: 1, - cmax: Math.max(records.length, 1), + cmax: Math.max(rows.length, 1), colorbar: { title: { text: 'run order', side: 'right', font: { size: 11 } }, thickness: 12, @@ -147,7 +163,7 @@ const HParamsParallelCoords = ({ '::' + (effectiveColorBy || 'order') + '::' + - records.length, + rows.length, }; const config = applySnapshotButton( @@ -171,7 +187,7 @@ const HParamsParallelCoords = ({ } catch (e) { window.Plotly.purge(el); } - }, [records, columns, effectiveDims, effectiveColorBy]); + }, [rows, columns, effectiveDims, effectiveColorBy]); const handleDims = (value) => { onSelectedDims(Array.isArray(value) ? value.slice(0, MAX_DIMS) : []); @@ -223,7 +239,11 @@ const HParamsParallelCoords = ({ aria-label="Color parallel coordinates by" /> - {truncated ? ( + {rows.length < records.length ? ( + + {rows.length} of {records.length} runs have all selected axes + + ) : truncated ? ( showing first {MAX_DIMS} ) : null}
diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index 774cbccab..50b27758e 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -223,15 +223,28 @@ export function buildSplomDimensions(records, columns, selectedIds) { return dimensions; } +export function completeRecords(records, cols) { + return (records || []).filter((record) => + (cols || []).every((col) => isNumeric(col.accessor(record))) + ); +} + +/* + * Build Plotly `parcoords` dimensions. Plotly cannot render null/NaN cells — + * one sparse axis corrupts every line — so callers pass records that already + * hold a numeric value on every axis (see completeRecords). Each axis spans its + * exact data range; an axis whose values are all equal gets a small symmetric + * range so it does not collapse to zero height. + */ export function buildParcoordsDimensions(records, columns, selectedIds) { const byId = new Map((columns || []).map((col) => [col.id, col])); const dimensions = []; (selectedIds || []).forEach((id) => { const col = byId.get(id); if (!col) return; - const values = toNumericColumn(records, col.accessor); const extent = numericExtent(records, col.accessor); if (extent === null) return; + const values = (records || []).map((record) => col.accessor(record)); const dimension = { label: col.label, values }; if (extent.min === extent.max) { const delta = Math.abs(extent.max) * 0.05 || 1; From 65ae35d9a5193840cb10e39eb06ae898b0349345 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 22 Jul 2026 17:11:08 +0530 Subject: [PATCH 30/48] fix(hparams): explain empty parallel coordinates instead of blanking Adding a sparse axis can shrink the complete-case run set to zero, which purged the plot and looked like nothing rendered. Show a message telling the user to remove a sparse axis when no run has a value on every selected axis. --- js/panes/hparams/HParamsParallelCoords.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index d15dd8876..1713954dc 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -252,6 +252,11 @@ const HParamsParallelCoords = ({
Select at least two dimensions to plot.
+ ) : rows.length === 0 ? ( +
+ No run has a value on every selected axis. Remove a sparse axis to see + lines. +
) : null}
); From b6d19cfa38e8966255f23c32e27ed6f622f974d6 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Wed, 22 Jul 2026 17:24:47 +0530 Subject: [PATCH 31/48] feat(hparams): add a run identity axis to parallel coordinates Plotly parcoords has no per-line hover, so lines could not be traced back to a run. Prepend a categorical 'run' axis whose ticks are the run names (aligned to the drawn rows and colour), long names elided, with a wider left margin to fit them, so every line can be read back to its run. --- js/panes/hparams/HParamsParallelCoords.js | 28 ++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index 1713954dc..8350b9598 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -19,10 +19,13 @@ import { isNumeric, NUMERIC_GROUPS, numericExtent, + runLabel, selectNumericColumns, toNumericColumn, } from './hparamsUtils'; +const RUN_LABEL_MAX = 18; + const MAX_DIMS = 10; const PARCOORDS_COLORSCALE = 'Viridis'; @@ -102,12 +105,31 @@ const HParamsParallelCoords = ({ ? columns.find((c) => c.id === effectiveColorBy) : null; - const dimensions = buildParcoordsDimensions(rows, columns, effectiveDims); - if (dimensions.length < 2 || rows.length === 0) { + const numericDimensions = buildParcoordsDimensions( + rows, + columns, + effectiveDims + ); + if (numericDimensions.length < 2 || rows.length === 0) { window.Plotly.purge(el); return; } + const runName = (record) => { + const name = runLabel(record); + return name.length > RUN_LABEL_MAX + ? name.slice(0, RUN_LABEL_MAX - 1) + '…' + : name; + }; + const runDimension = { + label: 'run', + values: rows.map((_, i) => i), + tickvals: rows.map((_, i) => i), + ticktext: rows.map(runName), + range: [-0.5, Math.max(rows.length - 1, 0.5)], + }; + const dimensions = [runDimension, ...numericDimensions]; + let line; if (colorCol) { const ext = numericExtent(rows, colorCol.accessor); @@ -154,7 +176,7 @@ const HParamsParallelCoords = ({ ]; const layout = { - margin: { l: 70, r: 80, t: 64, b: 76 }, + margin: { l: 120, r: 80, t: 64, b: 76 }, font: { family: '"Open Sans", sans-serif', size: 11, color: '#333' }, paper_bgcolor: '#ffffff', plot_bgcolor: '#ffffff', From 28ed13c4250d5480dabd348e3c1ec3d1f12087bd Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Thu, 23 Jul 2026 14:35:22 +0530 Subject: [PATCH 32/48] feat(hparams): filter runs across every view from a shared sidebar Until now the table filtered its own rows through a local text query while parallel coordinates and the scatter matrix always plotted the full record set, so switching tabs silently changed which runs were on screen. Lift the filter state into HParamsPane, apply it once, and hand the same reduced records to all three views. Adds a collapsible sidebar offering a range slider per numeric param or metric, a checkbox list for low-cardinality and categorical columns, and a run status group derived from the data rather than hardcoded. Filters combine as a facet: every active column must pass, but any checked value within one list is enough. A missing value survives only while its filter keeps missing values, which is what makes a range safe to apply to a sparse metric column. The axis pickers in the two plot views now derive their column lists from the unfiltered records, so narrowing a filter cannot make a column disappear from the picker and reset the user's selection mid-interaction. Range changes are committed on handle release rather than per drag frame, since every commit rebuilds the Plotly views. Pulls in rc-slider for the range control, wired the same way rc-tree-select already is: library stylesheet imported in main.js, accent colours applied through a static overrides file. --- js/main.js | 1 + js/panes/HParamsPane.js | 152 ++++++++--- js/panes/hparams/HParamsFilters.js | 249 +++++++++++++++++++ js/panes/hparams/HParamsParallelCoords.js | 12 +- js/panes/hparams/HParamsSplom.js | 12 +- js/panes/hparams/hparamsUtils.js | 164 ++++++++++++ package.json | 3 +- py/visdom/static/css/hparams.css | 179 +++++++++++++ py/visdom/static/css/rc-slider-overrides.css | 44 ++++ py/visdom/static/index.html | 1 + yarn.lock | 11 +- 11 files changed, 784 insertions(+), 44 deletions(-) create mode 100644 js/panes/hparams/HParamsFilters.js create mode 100644 py/visdom/static/css/rc-slider-overrides.css diff --git a/js/main.js b/js/main.js index 79446e6c5..21dc0fbf5 100644 --- a/js/main.js +++ b/js/main.js @@ -12,6 +12,7 @@ 'use strict'; import 'fetch'; +import 'rc-slider/assets/index.css'; import 'rc-tree-select/assets/index.less'; import React, { useContext, useEffect, useRef, useState } from 'react'; diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index 6bf6f9c9b..ec4451f3d 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -7,11 +7,20 @@ * */ -import React, { useState } from 'react'; +import React, { useMemo, useState } from 'react'; +import HParamsFilters from './hparams/HParamsFilters'; import HParamsParallelCoords from './hparams/HParamsParallelCoords'; import HParamsSplom from './hparams/HParamsSplom'; import HParamsTable from './hparams/HParamsTable'; +import { + applyFilters, + buildColumns, + buildFilterSpecs, + collectStatuses, + countActiveFilters, + filterRecords, +} from './hparams/hparamsUtils'; import Pane from './Pane'; const VIEWS = [ @@ -20,6 +29,8 @@ const VIEWS = [ { key: 'splom', label: 'Scatter matrix' }, ]; +const NO_RECORDS = []; + function readContent(content) { if (!content || typeof content !== 'object' || Array.isArray(content)) { return null; @@ -48,6 +59,31 @@ var HParamsPane = (props) => { const [splomColorBy, setSplomColorBy] = useState(null); const [parcoordsDims, setParcoordsDims] = useState(null); const [parcoordsColorBy, setParcoordsColorBy] = useState(null); + const [filtersOpen, setFiltersOpen] = useState(false); + const [filters, setFilters] = useState({ statuses: [], columns: {} }); + + const records = data ? data.records : NO_RECORDS; + const columns = useMemo( + () => + data ? buildColumns(data.paramKeys, data.metricKeys, data.tagKeys) : [], + [data] + ); + const specs = useMemo( + () => buildFilterSpecs(records, columns), + [records, columns] + ); + const statuses = useMemo(() => collectStatuses(records), [records]); + const visibleRecords = useMemo( + () => + applyFilters( + filterRecords(records, tableFilter, columns), + specs, + filters + ), + [records, tableFilter, columns, specs, filters] + ); + const activeFilters = + countActiveFilters(filters, specs) + (tableFilter.trim() ? 1 : 0); const handleDownload = () => { let blob = new Blob([JSON.stringify(content)], { @@ -89,6 +125,11 @@ var HParamsPane = (props) => { {data.tagKeys.length} tags + {visibleRecords.length !== records.length ? ( + + showing {visibleRecords.length} of {records.length} + + ) : null}
@@ -107,48 +148,83 @@ var HParamsPane = (props) => { {v.label} ))} +
- {(() => { - const viewProps = { - records: data.records, - paramKeys: data.paramKeys, - metricKeys: data.metricKeys, - tagKeys: data.tagKeys, - }; - if (view === 'splom') - return ( - - ); - if (view === 'parcoords') +
+ {filtersOpen ? ( + setFiltersOpen(false)} + visibleCount={visibleRecords.length} + totalCount={records.length} + /> + ) : null} + {(() => { + if (visibleRecords.length === 0) { + return ( +
+ No runs match your filters. +
+ ); + } + const viewProps = { + records: visibleRecords, + columnRecords: records, + paramKeys: data.paramKeys, + metricKeys: data.metricKeys, + tagKeys: data.tagKeys, + }; + if (view === 'splom') + return ( + + ); + if (view === 'parcoords') + return ( + + ); return ( - ); - return ( - - ); - })()} + })()} +
); diff --git a/js/panes/hparams/HParamsFilters.js b/js/panes/hparams/HParamsFilters.js new file mode 100644 index 000000000..12b37e79c --- /dev/null +++ b/js/panes/hparams/HParamsFilters.js @@ -0,0 +1,249 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import Slider from 'rc-slider'; +import React, { useCallback, useEffect, useState } from 'react'; + +import { COLUMN_GROUPS, formatValue } from './hparamsUtils'; + +/* + * A range control that tracks the drag locally and lifts the value only once + * the handle is released. Every commit re-filters the records feeding the + * Plotly views, so committing per drag frame would rebuild those plots + * continuously. + */ +const RangeFilter = ({ spec, entry, onChange }) => { + const bounds = [entry ? entry.lo : spec.min, entry ? entry.hi : spec.max]; + const [dragging, setDragging] = useState(null); + const value = dragging || bounds; + + useEffect(() => { + setDragging(null); + }, [entry, spec.min, spec.max]); + + const commit = (next) => { + setDragging(null); + onChange(spec.id, { + lo: next[0], + hi: next[1], + includeMissing: entry ? entry.includeMissing !== false : true, + }); + }; + + return ( +
+
+ {formatValue(value[0])} + {formatValue(value[1])} +
+ setDragging(next)} + onChangeComplete={commit} + ariaLabelForHandle={[spec.label + ' minimum', spec.label + ' maximum']} + /> +
+ ); +}; + +const CategoryFilter = ({ spec, entry, onChange }) => { + const selected = (entry && entry.values) || []; + const toggle = (value) => { + const next = selected.slice(); + const at = next.indexOf(value); + if (at === -1) next.push(value); + else next.splice(at, 1); + onChange(spec.id, { + values: next, + includeMissing: entry ? entry.includeMissing !== false : true, + }); + }; + + return ( +
+ {spec.values.map((value, index) => ( + + ))} +
+ ); +}; + +const FilterSection = ({ spec, entry, onChange }) => { + const toggleMissing = () => { + const base = + entry || + (spec.kind === 'range' ? { lo: spec.min, hi: spec.max } : { values: [] }); + onChange(spec.id, { + ...base, + includeMissing: entry ? entry.includeMissing === false : false, + }); + }; + + return ( +
+
+ {spec.label} +
+ {spec.kind === 'range' ? ( + + ) : ( + + )} + {spec.missing > 0 ? ( + + ) : null} +
+ ); +}; + +const HParamsFilters = ({ + specs, + statuses, + filters, + setFilters, + search, + setSearch, + onClose, + visibleCount, + totalCount, +}) => { + const setColumn = useCallback( + (id, entry) => { + setFilters((prev) => ({ + ...prev, + columns: { ...prev.columns, [id]: entry }, + })); + }, + [setFilters] + ); + + const toggleStatus = useCallback( + (status) => { + setFilters((prev) => { + const next = (prev.statuses || []).slice(); + const at = next.indexOf(status); + if (at === -1) next.push(status); + else next.splice(at, 1); + return { ...prev, statuses: next }; + }); + }, + [setFilters] + ); + + const clearAll = useCallback(() => { + setFilters({ statuses: [], columns: {} }); + setSearch(''); + }, [setFilters, setSearch]); + + const groups = COLUMN_GROUPS.map((group) => ({ + ...group, + specs: specs.filter((spec) => spec.group === group.key), + })).filter((group) => group.specs.length > 0); + + return ( +
+
+ Filters + +
+ +
+ setSearch(e.target.value)} + aria-label="Search runs" + /> + + {statuses.length > 0 ? ( +
+
status
+
+ {statuses.map((status) => ( + + ))} +
+
+ ) : null} + + {groups.map((group) => ( +
+
{group.label}
+ {group.specs.map((spec) => ( + + ))} +
+ ))} + + {groups.length === 0 ? ( +
+ No params or metrics can be filtered on. +
+ ) : null} +
+ +
+ + showing {visibleCount} of {totalCount} runs + + +
+
+ ); +}; + +export default HParamsFilters; diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index 8350b9598..45af8d2fe 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -32,6 +32,7 @@ const PARCOORDS_COLORSCALE = 'Viridis'; const HParamsParallelCoords = ({ records, + columnRecords, paramKeys, metricKeys, tagKeys, @@ -42,13 +43,20 @@ const HParamsParallelCoords = ({ }) => { const plotRef = useRef(null); + /* + * Which columns can be plotted is derived from the unfiltered records, so a + * filter that empties a column does not make it vanish from the axis picker + * and silently reset the user's selection. + */ + const pickerRecords = columnRecords || records; + const columns = useMemo( () => buildColumns(paramKeys, metricKeys, tagKeys), [paramKeys, metricKeys, tagKeys] ); const numericCols = useMemo( - () => selectNumericColumns(records, columns), - [records, columns] + () => selectNumericColumns(pickerRecords, columns), + [pickerRecords, columns] ); const effectiveDims = useMemo(() => { diff --git a/js/panes/hparams/HParamsSplom.js b/js/panes/hparams/HParamsSplom.js index d6f946edb..b1a3c097e 100644 --- a/js/panes/hparams/HParamsSplom.js +++ b/js/panes/hparams/HParamsSplom.js @@ -40,6 +40,7 @@ const AXIS_STYLE = { const HParamsSplom = ({ records, + columnRecords, paramKeys, metricKeys, tagKeys, @@ -51,13 +52,20 @@ const HParamsSplom = ({ const plotRef = useRef(null); const prevDimCount = useRef(0); + /* + * Which columns can be plotted is derived from the unfiltered records, so a + * filter that empties a column does not make it vanish from the axis picker + * and silently reset the user's selection. + */ + const pickerRecords = columnRecords || records; + const columns = useMemo( () => buildColumns(paramKeys, metricKeys, tagKeys), [paramKeys, metricKeys, tagKeys] ); const numericCols = useMemo( - () => selectNumericColumns(records, columns), - [records, columns] + () => selectNumericColumns(pickerRecords, columns), + [pickerRecords, columns] ); const effectiveDims = useMemo(() => { diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index 50b27758e..397e351f8 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -256,3 +256,167 @@ export function buildParcoordsDimensions(records, columns, selectedIds) { }); return dimensions; } + +/* + * A column with more distinct values than this is not offered as a checkbox + * list — the free-text search is the better tool for high-cardinality strings. + */ +const MAX_CATEGORIES = 12; + +/* + * Statuses come from the Python model (visdom.experiments.models), but the + * order here is lifecycle order rather than alphabetical so the sidebar reads + * the way a run progresses. + */ +export const STATUS_ORDER = ['running', 'finished', 'failed']; + +function distinctValues(records, accessor) { + const seen = new Map(); + let missing = 0; + (records || []).forEach((record) => { + const value = accessor(record); + if (isMissing(value)) { + missing += 1; + return; + } + const key = typeof value + ':' + String(value); + if (!seen.has(key)) seen.set(key, value); + }); + return { values: Array.from(seen.values()), missing }; +} + +/* + * Derive one filter control per column. A numeric param/metric with more than + * two distinct values becomes a range slider; anything with few enough distinct + * values becomes a checkbox list (this deliberately catches low-cardinality + * numerics such as batch size, where discrete choices beat a slider). Columns + * that are neither are skipped rather than rendered as an unusable control. + * + * Category values are ordered with the same comparator the table uses, which + * in turn mirrors the backend's _sort_pairs, so numbers precede strings. + */ +export function buildFilterSpecs(records, columns) { + const specs = []; + (columns || []).forEach((col) => { + const { values, missing } = distinctValues(records, col.accessor); + if (values.length === 0) return; + const numericColumn = + (col.group === 'param' || col.group === 'metric') && + values.every((value) => isNumeric(value)); + if (numericColumn && values.length > 2) { + const extent = numericExtent(records, col.accessor); + const integral = values.every((value) => Number.isInteger(value)); + const span = extent.max - extent.min; + specs.push({ + id: col.id, + label: col.label, + group: col.group, + accessor: col.accessor, + kind: 'range', + min: extent.min, + max: extent.max, + step: integral ? 1 : span / 100 || 1, + missing, + }); + return; + } + if (values.length <= MAX_CATEGORIES) { + specs.push({ + id: col.id, + label: col.label, + group: col.group, + accessor: col.accessor, + kind: 'category', + values: values + .slice() + .sort((a, b) => compareOrderKeys(orderKey(a), orderKey(b))), + missing, + }); + } + }); + return specs; +} + +/* + * The statuses actually present in the data, in lifecycle order. Derived rather + * than hardcoded so a run in a status this build does not know about still gets + * a checkbox instead of silently becoming unfilterable. + */ +export function collectStatuses(records) { + const present = new Set(); + (records || []).forEach((record) => { + if (record.status) present.add(record.status); + }); + const known = STATUS_ORDER.filter((status) => present.has(status)); + const extra = Array.from(present) + .filter((status) => STATUS_ORDER.indexOf(status) === -1) + .sort(); + return known.concat(extra); +} + +function passesSpec(record, spec, state, accessor) { + if (!state) return true; + const value = accessor(record); + if (isMissing(value)) return state.includeMissing !== false; + if (spec.kind === 'range') { + if (!isNumeric(value)) return state.includeMissing !== false; + return value >= state.lo && value <= state.hi; + } + if (!state.values || state.values.length === 0) return true; + return state.values.indexOf(value) !== -1; +} + +/* + * Faceted semantics: every active column filter must pass (AND), but within one + * category list any checked value is enough (OR). A run whose value is missing + * survives only while that filter keeps missing values, which is what makes a + * range filter safe to apply to sparse metric columns. + */ +export function applyFilters(records, specs, filters) { + const state = filters || {}; + const statuses = state.statuses; + const columns = state.columns || {}; + const active = (specs || []).filter((spec) => columns[spec.id]); + if ((!statuses || statuses.length === 0) && active.length === 0) { + return records; + } + return (records || []).filter((record) => { + if (statuses && statuses.length > 0) { + if (statuses.indexOf(record.status) === -1) return false; + } + for (let i = 0; i < active.length; i++) { + const spec = active[i]; + if (!passesSpec(record, spec, columns[spec.id], spec.accessor)) { + return false; + } + } + return true; + }); +} + +/* + * How many filters the badge should report. A range left at its full extent is + * not counted: it excludes nothing, so claiming it as active would make the + * badge disagree with what the user sees. + */ +export function countActiveFilters(filters, specs) { + const state = filters || {}; + const columns = state.columns || {}; + let count = state.statuses && state.statuses.length > 0 ? 1 : 0; + (specs || []).forEach((spec) => { + const entry = columns[spec.id]; + if (!entry) return; + if (spec.kind === 'range') { + const bounded = entry.lo > spec.min || entry.hi < spec.max; + if (bounded || entry.includeMissing === false) count += 1; + return; + } + if ( + (entry.values && entry.values.length > 0) || + entry.includeMissing === false + ) { + count += 1; + } + }); + return count; +} diff --git a/package.json b/package.json index 65dad409d..15b064a2a 100644 --- a/package.json +++ b/package.json @@ -57,13 +57,14 @@ "https-browserify": "^1.0.0", "jquery": "^4.0.0", "ml-savitzky-golay-generalized": "^5.0.0", + "rc-slider": "^11.1.9", "rc-tree-select": "^5.27.0", "react": "^17.0.2", "react-dom": "^17.0.2", "react-grid-layout": "^2.2.3", "react-modal": "^3.16.3", - "react-scroll-to-bottom": "^4.2.0", "react-resize-detector": "^9.1.1", + "react-scroll-to-bottom": "^4.2.0", "stream-browserify": "^3.0.0", "stream-http": "^3.2.0", "style-loader": "^4.0.0", diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index 529766942..b52a1ea2f 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -505,3 +505,182 @@ min-height: 0; min-width: 0; } + +/* ---- HParamsFilters (B5) ---- */ + +.hparams-layout { + flex: 1 1 auto; + min-height: 0; + min-width: 0; + display: flex; + flex-direction: row; +} + +.hparams-filters-toggle { + margin-left: auto; + padding: 2px 10px; + font-family: "Open Sans", sans-serif; + font-size: 12px; + color: #3b5998; + background-color: #fff; + border: 1px solid #dedede; + border-radius: 3px; + cursor: pointer; + white-space: nowrap; +} + +.hparams-filters-toggle:hover { + border-color: #3b5998; +} + +.hparams-filters-toggle-active { + color: #fff; + background-color: #3b5998; + border-color: #3b5998; +} + +.hparams-filters-toggle:focus-visible { + outline: 2px solid #3b5998; + outline-offset: -2px; +} + +.hparams-filters { + flex: 0 0 190px; + display: flex; + flex-direction: column; + min-height: 0; + background-color: #fbfbfc; + border-right: 1px solid #e6e6e6; +} + +.hparams-filters-head { + display: flex; + align-items: center; + padding: 6px 8px 6px 10px; + border-bottom: 1px solid #ececec; +} + +.hparams-filters-title { + font-size: 11px; + font-weight: 600; + letter-spacing: 0.06em; + text-transform: uppercase; + color: #666; +} + +.hparams-filters-close { + margin-left: auto; + padding: 0 4px; + font-size: 11px; + line-height: 1; + color: #888; + background: none; + border: none; + cursor: pointer; +} + +.hparams-filters-close:hover { + color: #3b5998; +} + +.hparams-filters-close:focus-visible { + outline: 2px solid #3b5998; + outline-offset: 1px; +} + +.hparams-filters-body { + flex: 1 1 auto; + min-height: 0; + overflow-y: auto; + padding: 8px 10px; +} + +.hparams-filters-body .hparams-filter { + flex: none; + width: 100%; + margin-bottom: 10px; +} + +.hparams-filter-group { + margin-bottom: 12px; +} + +.hparams-filter-eyebrow { + margin-bottom: 4px; + font-size: 10px; + font-weight: 600; + letter-spacing: 0.08em; + text-transform: uppercase; + color: #9aa2b1; + border-bottom: 1px solid #ececec; +} + +.hparams-filter-row { + margin-bottom: 10px; +} + +.hparams-filter-label { + overflow: hidden; + font-size: 12px; + color: #333; + text-overflow: ellipsis; + white-space: nowrap; +} + +.hparams-filter-bounds { + display: flex; + justify-content: space-between; + font-size: 11px; + color: #666; + font-variant-numeric: tabular-nums; +} + +.hparams-filter-categories { + display: flex; + flex-wrap: wrap; + gap: 2px 8px; +} + +.hparams-filter-check { + display: flex; + align-items: center; + gap: 4px; + font-size: 12px; + color: #444; + cursor: pointer; +} + +.hparams-filter-check input { + margin: 0; +} + +.hparams-filter-missing { + margin-top: 2px; + font-size: 11px; + color: #888; +} + +.hparams-filter-none { + font-size: 12px; + font-style: italic; + color: #888; +} + +.hparams-filters-foot { + display: flex; + align-items: center; + gap: 8px; + padding: 6px 10px; + font-size: 11px; + color: #666; + background-color: #fff; + border-top: 1px solid #ececec; +} + +.hparams-filters-count { + flex: 1 1 auto; +} + +.hparams-stat-filtered { + color: #3b5998; +} diff --git a/py/visdom/static/css/rc-slider-overrides.css b/py/visdom/static/css/rc-slider-overrides.css new file mode 100644 index 000000000..83b35989a --- /dev/null +++ b/py/visdom/static/css/rc-slider-overrides.css @@ -0,0 +1,44 @@ +/* + * Recolours rc-slider's default blue to the Visdom accent so the range filters + * in the hyper-parameter pane match the rest of the chrome. Declarations are + * marked important for the same reason as the rc-tree-select overrides: the + * library ships its stylesheet through the bundle, which loads after this file. + */ + +.hparams-filter-range .rc-slider { + height: 14px; + padding: 5px 0; + margin: 0 6px; +} + +.hparams-filter-range .rc-slider-rail { + height: 3px; + background-color: #e4e4e4 !important; +} + +.hparams-filter-range .rc-slider-track { + height: 3px; + background-color: #3b5998 !important; +} + +.hparams-filter-range .rc-slider-handle { + width: 12px; + height: 12px; + margin-top: -5px; + opacity: 1; + border: 2px solid #3b5998 !important; + background-color: #fff !important; + box-shadow: none !important; +} + +.hparams-filter-range .rc-slider-handle:hover, +.hparams-filter-range .rc-slider-handle-dragging { + border-color: #2c4479 !important; + box-shadow: 0 0 0 3px rgba(59, 89, 152, 0.18) !important; +} + +.hparams-filter-range .rc-slider-handle:focus-visible { + outline: 2px solid #3b5998; + outline-offset: 2px; + box-shadow: none !important; +} diff --git a/py/visdom/static/index.html b/py/visdom/static/index.html index 499df039f..43f83a100 100644 --- a/py/visdom/static/index.html +++ b/py/visdom/static/index.html @@ -81,6 +81,7 @@ + visdom diff --git a/yarn.lock b/yarn.lock index 3d473f354..0ed86c429 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2050,7 +2050,7 @@ classnames@2.x, classnames@^2.2.1, classnames@^2.2.6: resolved "https://registry.npmjs.org/classnames/-/classnames-2.3.2.tgz" integrity sha512-CSbhY4cFEJRe6/GQzIk5qXZ4Jeg5pcsP7b5peFSDpffpe1cqjASH/n9UTjBwOp6XpMSTwQ8Za2K5V02ueA7Tmw== -classnames@^2.3.2: +classnames@^2.2.5, classnames@^2.3.2: version "2.5.1" resolved "https://registry.yarnpkg.com/classnames/-/classnames-2.5.1.tgz#ba774c614be0f016da105c858e7159eae8e7687b" integrity sha512-saHYOzhIQs6wy2sVxTM6bUDsQO4F50V9RQ22qBpEdCW+I+/Wmke2HOl6lS6dTpdxVhb88/I6+Hs+438c3lfUow== @@ -5028,6 +5028,15 @@ rc-select@~14.16.2: rc-util "^5.16.1" rc-virtual-list "^3.5.2" +rc-slider@^11.1.9: + version "11.1.9" + resolved "https://registry.yarnpkg.com/rc-slider/-/rc-slider-11.1.9.tgz#d872130fbf4ec51f28543d62e90451091d6f5208" + integrity sha512-h8IknhzSh3FEM9u8ivkskh+Ef4Yo4JRIY2nj7MrH6GQmrwV6mcpJf5/4KgH5JaVI1H3E52yCdpOlVyGZIeph5A== + dependencies: + "@babel/runtime" "^7.10.1" + classnames "^2.2.5" + rc-util "^5.36.0" + rc-tree-select@^5.27.0: version "5.27.0" resolved "https://registry.yarnpkg.com/rc-tree-select/-/rc-tree-select-5.27.0.tgz#3daa62972ae80846dac96bf4776d1a9dc9c7c4c6" From 30406e0b509f90a72199a73b9c23b38cbfe129b2 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Thu, 23 Jul 2026 15:28:43 +0530 Subject: [PATCH 33/48] fix(hparams): show a single run search shared by every view The search box was rendered twice: once in the table toolbar and again at the top of the filters sidebar, both bound to the same state, so opening the sidebar put two identical inputs on screen side by side. Search now filters all three views rather than just the table, so it is a pane level control and belongs next to the Filters toggle instead of inside either of them. Move the single input there and drop both copies. The table no longer filters its own rows either, since the pane has already applied the same query before handing the records over. --- js/panes/HParamsPane.js | 37 ++++++++++++++++++------------ js/panes/hparams/HParamsFilters.js | 10 -------- js/panes/hparams/HParamsTable.js | 26 ++++----------------- py/visdom/static/css/hparams.css | 20 ++++++++++------ 4 files changed, 40 insertions(+), 53 deletions(-) diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index ec4451f3d..9f29d6d8f 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -148,18 +148,28 @@ var HParamsPane = (props) => { {v.label} ))} - + + setTableFilter(e.target.value)} + aria-label="Search runs" + /> + +
{filtersOpen ? ( @@ -168,7 +178,6 @@ var HParamsPane = (props) => { statuses={statuses} filters={filters} setFilters={setFilters} - search={tableFilter} setSearch={setTableFilter} onClose={() => setFiltersOpen(false)} visibleCount={visibleRecords.length} @@ -215,8 +224,6 @@ var HParamsPane = (props) => { {...viewProps} sort={tableSort} setSort={setTableSort} - filter={tableFilter} - setFilter={setTableFilter} colorBy={tableColorBy} setColorBy={setTableColorBy} selected={tableSelected} diff --git a/js/panes/hparams/HParamsFilters.js b/js/panes/hparams/HParamsFilters.js index 12b37e79c..972e868b7 100644 --- a/js/panes/hparams/HParamsFilters.js +++ b/js/panes/hparams/HParamsFilters.js @@ -126,7 +126,6 @@ const HParamsFilters = ({ statuses, filters, setFilters, - search, setSearch, onClose, visibleCount, @@ -181,15 +180,6 @@ const HParamsFilters = ({
- setSearch(e.target.value)} - aria-label="Search runs" - /> - {statuses.length > 0 ? (
status
diff --git a/js/panes/hparams/HParamsTable.js b/js/panes/hparams/HParamsTable.js index ce6866316..3b0f07e70 100644 --- a/js/panes/hparams/HParamsTable.js +++ b/js/panes/hparams/HParamsTable.js @@ -13,7 +13,6 @@ import React, { useCallback, useMemo } from 'react'; import { buildColumns, COLUMN_GROUPS, - filterRecords, formatValue, groupColumnTree, isNumeric, @@ -147,8 +146,6 @@ const HParamsTable = ({ tagKeys, sort, setSort, - filter, - setFilter, colorBy, setColorBy, selected, @@ -173,17 +170,12 @@ const HParamsTable = ({ return ids; }, [columns]); - const filtered = useMemo( - () => filterRecords(records, filter, columns), - [records, filter, columns] - ); - const rows = useMemo(() => { - if (!sort.by) return filtered; + if (!sort.by) return records; const accessor = accessorFor(sort.by, columns); - if (!accessor) return filtered; - return filtered.slice().sort(makeComparator(accessor, sort.dir)); - }, [filtered, sort, columns]); + if (!accessor) return records; + return records.slice().sort(makeComparator(accessor, sort.dir)); + }, [records, sort, columns]); const extent = useMemo(() => { if (!colorBy) return null; @@ -248,14 +240,6 @@ const HParamsTable = ({ return (
- setFilter(e.target.value)} - aria-label="Filter runs" - /> sort by: - No runs match “{filter}”. + No runs to show. ) : ( diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index b52a1ea2f..65c268560 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -516,8 +516,20 @@ flex-direction: row; } -.hparams-filters-toggle { +.hparams-viewtools { + display: flex; + align-items: center; + gap: 6px; margin-left: auto; + padding-bottom: 4px; +} + +.hparams-viewtools .hparams-filter { + flex: 0 1 160px; + padding: 2px 8px; +} + +.hparams-filters-toggle { padding: 2px 10px; font-family: "Open Sans", sans-serif; font-size: 12px; @@ -595,12 +607,6 @@ padding: 8px 10px; } -.hparams-filters-body .hparams-filter { - flex: none; - width: 100%; - margin-bottom: 10px; -} - .hparams-filter-group { margin-bottom: 12px; } From 02da57aa0b84f27551981e2ff3998a3bcd0365d8 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Thu, 23 Jul 2026 15:32:47 +0530 Subject: [PATCH 34/48] fix(hparams): show the include-missing toggle on every filter The toggle only appeared on columns that happened to have a gap, so the sidebar showed it under some filters and not others with nothing to explain the difference. Render it for every filter and let the count carry the information instead. Where a column has no missing values the box is disabled rather than merely present, since toggling it could not change the result, and a tooltip on each one says how many runs are affected. --- js/panes/hparams/HParamsFilters.js | 29 +++++++++++++++++++---------- py/visdom/static/css/hparams.css | 9 +++++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/js/panes/hparams/HParamsFilters.js b/js/panes/hparams/HParamsFilters.js index 972e868b7..72a20e69b 100644 --- a/js/panes/hparams/HParamsFilters.js +++ b/js/panes/hparams/HParamsFilters.js @@ -107,16 +107,25 @@ const FilterSection = ({ spec, entry, onChange }) => { ) : ( )} - {spec.missing > 0 ? ( - - ) : null} +
); }; diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index 65c268560..dc7c38b2f 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -666,6 +666,15 @@ color: #888; } +.hparams-filter-missing-none { + color: #b8b8b8; + cursor: default; +} + +.hparams-filter-missing-none input { + cursor: default; +} + .hparams-filter-none { font-size: 12px; font-style: italic; From ca76ae542d7ccf369c90722208216deae56d12f9 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Thu, 23 Jul 2026 15:36:38 +0530 Subject: [PATCH 35/48] feat(hparams): open the plots on one param against one metric Both plots opened on every numeric column they could find, up to six axes for the scatter matrix and ten for parallel coordinates. That gives a first impression of a dense mesh rather than a readable chart, and the axes it picks are whichever happen to sort first rather than a pairing that means anything. Default instead to a single param against a single metric, which is the comparison these views exist to show. Parallel coordinates keeps preferring columns with a value on every run, since a sparse axis costs it whole lines, but no longer at the expense of pairing a param with a metric. Explicit axis choices are untouched. --- js/panes/hparams/HParamsParallelCoords.js | 7 +++---- js/panes/hparams/HParamsSplom.js | 3 ++- js/panes/hparams/hparamsUtils.js | 19 +++++++++++++++++++ 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index 45af8d2fe..da473605a 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -15,6 +15,7 @@ import { buildColumns, buildParcoordsDimensions, completeRecords, + defaultDimIds, groupColumnTree, isNumeric, NUMERIC_GROUPS, @@ -63,11 +64,9 @@ const HParamsParallelCoords = ({ const validIds = new Set(numericCols.map((c) => c.id)); let ids = (selectedDims || []).filter((id) => validIds.has(id)); if (ids.length === 0) { - const dense = numericCols.filter((c) => - records.every((r) => isNumeric(c.accessor(r))) + ids = defaultDimIds(numericCols, (col) => + records.every((record) => isNumeric(col.accessor(record))) ); - const pick = dense.length >= 2 ? dense : numericCols; - ids = pick.slice(0, MAX_DIMS).map((c) => c.id); } return ids.slice(0, MAX_DIMS); }, [selectedDims, numericCols, records]); diff --git a/js/panes/hparams/HParamsSplom.js b/js/panes/hparams/HParamsSplom.js index b1a3c097e..5af5992e3 100644 --- a/js/panes/hparams/HParamsSplom.js +++ b/js/panes/hparams/HParamsSplom.js @@ -14,6 +14,7 @@ import { applySnapshotButton, observePlotResize } from './hparamsPlot'; import { buildColumns, buildSplomDimensions, + defaultDimIds, groupColumnTree, NUMERIC_GROUPS, numericExtent, @@ -71,7 +72,7 @@ const HParamsSplom = ({ const effectiveDims = useMemo(() => { const validIds = new Set(numericCols.map((c) => c.id)); let ids = (selectedDims || []).filter((id) => validIds.has(id)); - if (ids.length === 0) ids = numericCols.slice(0, MAX_DIMS).map((c) => c.id); + if (ids.length === 0) ids = defaultDimIds(numericCols); return ids.slice(0, MAX_DIMS); }, [selectedDims, numericCols]); diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index 397e351f8..6253ad4f9 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -205,6 +205,25 @@ export function selectNumericColumns(records, columns) { ); } +/* + * The axes a plot opens on: one param against one metric, so a freshly opened + * view reads as a single relationship rather than every numeric column at once. + * Callers may pass `isDense` to prefer columns that hold a value on every run, + * since a sparse axis costs a parallel-coordinates plot whole lines. Falls back + * to the first two columns when the data has no param/metric pair to offer. + */ +export function defaultDimIds(numericCols, isDense) { + const cols = numericCols || []; + const dense = isDense ? cols.filter(isDense) : cols; + const pick = (group) => + dense.find((col) => col.group === group) || + cols.find((col) => col.group === group); + const param = pick('param'); + const metric = pick('metric'); + if (param && metric) return [param.id, metric.id]; + return cols.slice(0, 2).map((col) => col.id); +} + /* * Build Plotly `splom` dimensions from the chosen column ids. Order follows * selectedIds; unknown ids are skipped; missing/non-numeric cells become null From 2fa9ac82fff0bfc22f1cf849e969793819a4e147 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Thu, 23 Jul 2026 15:43:25 +0530 Subject: [PATCH 36/48] fix(hparams): leave the include-missing toggle usable on every column Disabling the box where a column had no gaps made it look broken next to the ones that were live, and it decided on the user's behalf that the control was not worth offering. Leave every toggle interactive; the count and the tooltip already say whether it will change anything. --- js/panes/hparams/HParamsFilters.js | 6 +----- py/visdom/static/css/hparams.css | 9 --------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/js/panes/hparams/HParamsFilters.js b/js/panes/hparams/HParamsFilters.js index 72a20e69b..d8431bd95 100644 --- a/js/panes/hparams/HParamsFilters.js +++ b/js/panes/hparams/HParamsFilters.js @@ -108,10 +108,7 @@ const FilterSection = ({ spec, entry, onChange }) => { )} diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index dc7c38b2f..65c268560 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -666,15 +666,6 @@ color: #888; } -.hparams-filter-missing-none { - color: #b8b8b8; - cursor: default; -} - -.hparams-filter-missing-none input { - cursor: default; -} - .hparams-filter-none { font-size: 12px; font-style: italic; From 3f12ec999a8bef58f1cb9f14b09206651ecb1d48 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Thu, 23 Jul 2026 16:06:05 +0530 Subject: [PATCH 37/48] refactor(hparams): share the axis logic the two plot views had each copied The scatter matrix and parallel coordinates resolved their axes with the same forty lines and drew the same thirty-line toolbar, differing only in a label, two aria strings and a placeholder. Parallel coordinates was also styling itself with splom-prefixed classes, a leftover from having been cloned from it. Move the shared derivation into useHParamsAxes and the shared markup into HParamsAxisToolbar, with the wording each view differs on passed in, and rename the two classes both now use to say plot rather than splom. Collapse the include-missing idiom in the same pass. Four places asked whether a filter still admits runs with no value and three built the same next entry by hand; both now go through one helper, so the default lives in a single place instead of being restated at every call site. Open the sidebar on first render. It is the control that explains what the numbers on screen are being narrowed by, and hiding it made the pane look like it had fewer runs than it does. --- js/panes/HParamsPane.js | 2 +- js/panes/hparams/HParamsAxisToolbar.js | 71 +++++++++++ js/panes/hparams/HParamsFilters.js | 33 +++--- js/panes/hparams/HParamsParallelCoords.js | 138 +++++++--------------- js/panes/hparams/HParamsSplom.js | 121 ++++++------------- js/panes/hparams/hparamsUtils.js | 39 +++--- js/panes/hparams/useHParamsAxes.js | 91 ++++++++++++++ py/visdom/static/css/hparams.css | 4 +- 8 files changed, 277 insertions(+), 222 deletions(-) create mode 100644 js/panes/hparams/HParamsAxisToolbar.js create mode 100644 js/panes/hparams/useHParamsAxes.js diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index 9f29d6d8f..403c66de5 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -59,7 +59,7 @@ var HParamsPane = (props) => { const [splomColorBy, setSplomColorBy] = useState(null); const [parcoordsDims, setParcoordsDims] = useState(null); const [parcoordsColorBy, setParcoordsColorBy] = useState(null); - const [filtersOpen, setFiltersOpen] = useState(false); + const [filtersOpen, setFiltersOpen] = useState(true); const [filters, setFilters] = useState({ statuses: [], columns: {} }); const records = data ? data.records : NO_RECORDS; diff --git a/js/panes/hparams/HParamsAxisToolbar.js b/js/panes/hparams/HParamsAxisToolbar.js new file mode 100644 index 000000000..7f4938332 --- /dev/null +++ b/js/panes/hparams/HParamsAxisToolbar.js @@ -0,0 +1,71 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import TreeSelect from 'rc-tree-select'; +import React from 'react'; + +/* + * The axis picker and colour picker both plot views carry. Only the wording + * differs between them: a scatter matrix talks about dimensions and falls back + * to no colour, parallel coordinates talks about axes and falls back to run + * order. `note` is the trailing status each view fills for itself. + */ +const HParamsAxisToolbar = ({ + axesLabel, + axesName, + colorFallback, + treeData, + dims, + onDims, + colorBy, + onColorBy, + maxDims, + note, +}) => ( +
+ + {axesLabel}: + + onDims(Array.isArray(value) ? value.slice(0, maxDims) : []) + } + aria-label={axesName + ' ' + axesLabel} + /> + + + color by: + onColorBy(value || null)} + aria-label={'Color ' + axesName + ' by'} + /> + + {note ? {note} : null} +
+); + +export default HParamsAxisToolbar; diff --git a/js/panes/hparams/HParamsFilters.js b/js/panes/hparams/HParamsFilters.js index d8431bd95..fc3f7d34a 100644 --- a/js/panes/hparams/HParamsFilters.js +++ b/js/panes/hparams/HParamsFilters.js @@ -10,7 +10,18 @@ import Slider from 'rc-slider'; import React, { useCallback, useEffect, useState } from 'react'; -import { COLUMN_GROUPS, formatValue } from './hparamsUtils'; +import { COLUMN_GROUPS, formatValue, keepsMissing } from './hparamsUtils'; + +/* + * Every control edits one field of its filter and leaves the rest alone, so + * they all build the next entry the same way: the current one where it exists, + * the spec's full range or an empty tick list where it does not. + */ +const entryFor = (spec, entry) => + entry || + (spec.kind === 'range' + ? { lo: spec.min, hi: spec.max, includeMissing: true } + : { values: [], includeMissing: true }); /* * A range control that tracks the drag locally and lifts the value only once @@ -29,11 +40,7 @@ const RangeFilter = ({ spec, entry, onChange }) => { const commit = (next) => { setDragging(null); - onChange(spec.id, { - lo: next[0], - hi: next[1], - includeMissing: entry ? entry.includeMissing !== false : true, - }); + onChange(spec.id, { ...entryFor(spec, entry), lo: next[0], hi: next[1] }); }; return ( @@ -64,10 +71,7 @@ const CategoryFilter = ({ spec, entry, onChange }) => { const at = next.indexOf(value); if (at === -1) next.push(value); else next.splice(at, 1); - onChange(spec.id, { - values: next, - includeMissing: entry ? entry.includeMissing !== false : true, - }); + onChange(spec.id, { ...entryFor(spec, entry), values: next }); }; return ( @@ -88,12 +92,9 @@ const CategoryFilter = ({ spec, entry, onChange }) => { const FilterSection = ({ spec, entry, onChange }) => { const toggleMissing = () => { - const base = - entry || - (spec.kind === 'range' ? { lo: spec.min, hi: spec.max } : { values: [] }); onChange(spec.id, { - ...base, - includeMissing: entry ? entry.includeMissing === false : false, + ...entryFor(spec, entry), + includeMissing: !keepsMissing(entry), }); }; @@ -117,7 +118,7 @@ const FilterSection = ({ spec, entry, onChange }) => { > include missing ({spec.missing}) diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index da473605a..a3c61b370 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -7,23 +7,18 @@ * */ -import TreeSelect from 'rc-tree-select'; import React, { useEffect, useMemo, useRef } from 'react'; +import HParamsAxisToolbar from './HParamsAxisToolbar'; import { applySnapshotButton, observePlotResize } from './hparamsPlot'; import { - buildColumns, buildParcoordsDimensions, completeRecords, - defaultDimIds, - groupColumnTree, - isNumeric, - NUMERIC_GROUPS, numericExtent, runLabel, - selectNumericColumns, toNumericColumn, } from './hparamsUtils'; +import useHParamsAxes from './useHParamsAxes'; const RUN_LABEL_MAX = 18; @@ -44,46 +39,24 @@ const HParamsParallelCoords = ({ }) => { const plotRef = useRef(null); - /* - * Which columns can be plotted is derived from the unfiltered records, so a - * filter that empties a column does not make it vanish from the axis picker - * and silently reset the user's selection. - */ - const pickerRecords = columnRecords || records; - - const columns = useMemo( - () => buildColumns(paramKeys, metricKeys, tagKeys), - [paramKeys, metricKeys, tagKeys] - ); - const numericCols = useMemo( - () => selectNumericColumns(pickerRecords, columns), - [pickerRecords, columns] - ); - - const effectiveDims = useMemo(() => { - const validIds = new Set(numericCols.map((c) => c.id)); - let ids = (selectedDims || []).filter((id) => validIds.has(id)); - if (ids.length === 0) { - ids = defaultDimIds(numericCols, (col) => - records.every((record) => isNumeric(col.accessor(record))) - ); - } - return ids.slice(0, MAX_DIMS); - }, [selectedDims, numericCols, records]); - - const effectiveColorBy = useMemo(() => { - if (!colorBy) return null; - return numericCols.some((c) => c.id === colorBy) ? colorBy : null; - }, [colorBy, numericCols]); - - const truncated = - (selectedDims || []).filter((id) => numericCols.some((c) => c.id === id)) - .length > MAX_DIMS; - - const treeData = useMemo( - () => groupColumnTree(numericCols, NUMERIC_GROUPS), - [numericCols] - ); + const { + columns, + dims: effectiveDims, + colorBy: effectiveColorBy, + treeData, + truncated, + hasPlot, + } = useHParamsAxes({ + records, + columnRecords, + paramKeys, + metricKeys, + tagKeys, + selectedDims, + colorBy, + maxDims: MAX_DIMS, + preferDense: true, + }); const rows = useMemo(() => { const colorCol = effectiveColorBy @@ -96,8 +69,6 @@ const HParamsParallelCoords = ({ return completeRecords(records, requiredCols); }, [records, columns, effectiveDims, effectiveColorBy]); - const hasPlot = numericCols.length >= 2; - useEffect(() => { const el = plotRef.current; if (!el) return; @@ -218,10 +189,6 @@ const HParamsParallelCoords = ({ } }, [rows, columns, effectiveDims, effectiveColorBy]); - const handleDims = (value) => { - onSelectedDims(Array.isArray(value) ? value.slice(0, MAX_DIMS) : []); - }; - if (!hasPlot) { return (
@@ -232,57 +199,34 @@ const HParamsParallelCoords = ({ ); } + const note = + rows.length < records.length + ? rows.length + ' of ' + records.length + ' runs have all selected axes' + : truncated + ? 'showing first ' + MAX_DIMS + : null; + return (
-
- - axes: - - - - color by: - onColorBy(value || null)} - aria-label="Color parallel coordinates by" - /> - - {rows.length < records.length ? ( - - {rows.length} of {records.length} runs have all selected axes - - ) : truncated ? ( - showing first {MAX_DIMS} - ) : null} -
+
{effectiveDims.length < 2 ? ( -
+
Select at least two dimensions to plot.
) : rows.length === 0 ? ( -
+
No run has a value on every selected axis. Remove a sparse axis to see lines.
diff --git a/js/panes/hparams/HParamsSplom.js b/js/panes/hparams/HParamsSplom.js index 5af5992e3..7961fc968 100644 --- a/js/panes/hparams/HParamsSplom.js +++ b/js/panes/hparams/HParamsSplom.js @@ -7,21 +7,17 @@ * */ -import TreeSelect from 'rc-tree-select'; -import React, { useEffect, useMemo, useRef } from 'react'; +import React, { useEffect, useRef } from 'react'; +import HParamsAxisToolbar from './HParamsAxisToolbar'; import { applySnapshotButton, observePlotResize } from './hparamsPlot'; import { - buildColumns, buildSplomDimensions, - defaultDimIds, - groupColumnTree, - NUMERIC_GROUPS, numericExtent, runLabel, - selectNumericColumns, toNumericColumn, } from './hparamsUtils'; +import useHParamsAxes from './useHParamsAxes'; const MAX_DIMS = 6; @@ -53,44 +49,23 @@ const HParamsSplom = ({ const plotRef = useRef(null); const prevDimCount = useRef(0); - /* - * Which columns can be plotted is derived from the unfiltered records, so a - * filter that empties a column does not make it vanish from the axis picker - * and silently reset the user's selection. - */ - const pickerRecords = columnRecords || records; - - const columns = useMemo( - () => buildColumns(paramKeys, metricKeys, tagKeys), - [paramKeys, metricKeys, tagKeys] - ); - const numericCols = useMemo( - () => selectNumericColumns(pickerRecords, columns), - [pickerRecords, columns] - ); - - const effectiveDims = useMemo(() => { - const validIds = new Set(numericCols.map((c) => c.id)); - let ids = (selectedDims || []).filter((id) => validIds.has(id)); - if (ids.length === 0) ids = defaultDimIds(numericCols); - return ids.slice(0, MAX_DIMS); - }, [selectedDims, numericCols]); - - const effectiveColorBy = useMemo(() => { - if (!colorBy) return null; - return numericCols.some((c) => c.id === colorBy) ? colorBy : null; - }, [colorBy, numericCols]); - - const truncated = - (selectedDims || []).filter((id) => numericCols.some((c) => c.id === id)) - .length > MAX_DIMS; - - const treeData = useMemo( - () => groupColumnTree(numericCols, NUMERIC_GROUPS), - [numericCols] - ); - - const hasPlot = numericCols.length >= 2; + const { + columns, + dims: effectiveDims, + colorBy: effectiveColorBy, + treeData, + truncated, + hasPlot, + } = useHParamsAxes({ + records, + columnRecords, + paramKeys, + metricKeys, + tagKeys, + selectedDims, + colorBy, + maxDims: MAX_DIMS, + }); useEffect(() => { const el = plotRef.current; @@ -210,10 +185,6 @@ const HParamsSplom = ({ } }, [records, columns, effectiveDims, effectiveColorBy]); - const handleDims = (value) => { - onSelectedDims(Array.isArray(value) ? value.slice(0, MAX_DIMS) : []); - }; - if (!hasPlot) { return (
@@ -226,47 +197,21 @@ const HParamsSplom = ({ return (
-
- - dimensions: - - - - color by: - onColorBy(value || null)} - aria-label="Color scatter matrix by" - /> - - {truncated ? ( - showing first {MAX_DIMS} - ) : null} -
+
{effectiveDims.length < 2 ? ( -
+
Select at least two dimensions to plot.
) : null} diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index 6253ad4f9..8d9e7105c 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -373,16 +373,25 @@ export function collectStatuses(records) { return known.concat(extra); } -function passesSpec(record, spec, state, accessor) { - if (!state) return true; +/* + * Whether a filter still admits runs that have no value for its column. Absent + * means yes: a filter should only ever remove runs it actually judged, so + * excluding the unmeasured ones has to be asked for. + */ +export function keepsMissing(entry) { + return !entry || entry.includeMissing !== false; +} + +function passesSpec(record, spec, entry, accessor) { + if (!entry) return true; const value = accessor(record); - if (isMissing(value)) return state.includeMissing !== false; + if (isMissing(value)) return keepsMissing(entry); if (spec.kind === 'range') { - if (!isNumeric(value)) return state.includeMissing !== false; - return value >= state.lo && value <= state.hi; + if (!isNumeric(value)) return keepsMissing(entry); + return value >= entry.lo && value <= entry.hi; } - if (!state.values || state.values.length === 0) return true; - return state.values.indexOf(value) !== -1; + if (!entry.values || entry.values.length === 0) return true; + return entry.values.indexOf(value) !== -1; } /* @@ -425,17 +434,11 @@ export function countActiveFilters(filters, specs) { (specs || []).forEach((spec) => { const entry = columns[spec.id]; if (!entry) return; - if (spec.kind === 'range') { - const bounded = entry.lo > spec.min || entry.hi < spec.max; - if (bounded || entry.includeMissing === false) count += 1; - return; - } - if ( - (entry.values && entry.values.length > 0) || - entry.includeMissing === false - ) { - count += 1; - } + const narrowed = + spec.kind === 'range' + ? entry.lo > spec.min || entry.hi < spec.max + : entry.values && entry.values.length > 0; + if (narrowed || !keepsMissing(entry)) count += 1; }); return count; } diff --git a/js/panes/hparams/useHParamsAxes.js b/js/panes/hparams/useHParamsAxes.js new file mode 100644 index 000000000..05e8d1d38 --- /dev/null +++ b/js/panes/hparams/useHParamsAxes.js @@ -0,0 +1,91 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import { useMemo } from 'react'; + +import { + buildColumns, + defaultDimIds, + groupColumnTree, + isNumeric, + NUMERIC_GROUPS, + selectNumericColumns, +} from './hparamsUtils'; + +/* + * Resolves which axes a plot should draw, shared by the scatter matrix and + * parallel coordinates because both answer the question identically. + * + * `columnRecords` is the unfiltered set: which columns can be plotted at all is + * decided from it, so a filter that empties a column cannot make it vanish from + * the picker and silently reset a selection. `records` is what actually gets + * drawn, and only informs which columns are dense. + * + * `preferDense` suits parallel coordinates, where a column with gaps costs the + * plot whole lines and is a poor opening axis; a scatter matrix just leaves a + * marker out, so it does not care. + */ +export default function useHParamsAxes({ + records, + columnRecords, + paramKeys, + metricKeys, + tagKeys, + selectedDims, + colorBy, + maxDims, + preferDense = false, +}) { + const pickerRecords = columnRecords || records; + + const columns = useMemo( + () => buildColumns(paramKeys, metricKeys, tagKeys), + [paramKeys, metricKeys, tagKeys] + ); + + const numericCols = useMemo( + () => selectNumericColumns(pickerRecords, columns), + [pickerRecords, columns] + ); + + const dims = useMemo(() => { + const valid = new Set(numericCols.map((col) => col.id)); + const chosen = (selectedDims || []).filter((id) => valid.has(id)); + if (chosen.length > 0) return chosen.slice(0, maxDims); + const isDense = preferDense + ? (col) => records.every((record) => isNumeric(col.accessor(record))) + : null; + return defaultDimIds(numericCols, isDense).slice(0, maxDims); + }, [selectedDims, numericCols, records, maxDims, preferDense]); + + const activeColorBy = useMemo(() => { + if (!colorBy) return null; + return numericCols.some((col) => col.id === colorBy) ? colorBy : null; + }, [colorBy, numericCols]); + + const treeData = useMemo( + () => groupColumnTree(numericCols, NUMERIC_GROUPS), + [numericCols] + ); + + const truncated = + (selectedDims || []).filter((id) => + numericCols.some((col) => col.id === id) + ).length > maxDims; + + return { + columns, + numericCols, + dims, + colorBy: activeColorBy, + treeData, + truncated, + hasPlot: numericCols.length >= 2, + }; +} diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index 65c268560..60f0969c5 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -469,7 +469,7 @@ min-width: 0; } -.hparams-splom-overlay { +.hparams-plot-overlay { position: absolute; top: 40px; right: 0; @@ -482,7 +482,7 @@ pointer-events: none; } -.hparams-splom-note { +.hparams-plot-note { margin-left: auto; color: #888; font-style: italic; From d46a6d17d344ca26f8d0472d495fc2215e39fcf5 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Fri, 24 Jul 2026 15:42:08 +0530 Subject: [PATCH 38/48] feat(hparams): compare view, linked selection, and per-cell scatter hover - add a Compare tab that diffs the selected runs: a field-by-run table with differing params and tags first, a color spine across each metric row, and identical fields folded into an expandable strip (open by default) - narrow the parallel coordinates and scatter matrix to the runs ticked in the table, surfaced by a removable selection chip in the summary bar - name every run stacked on a scatter-matrix marker through a live per-cell hover, so runs that coincide on one cell are no longer hidden - share the Plotly plumbing (resize hook, colorbar, base layout, render and snapshot, colour ramp) and the run-status badge across the views --- js/panes/HParamsPane.js | 93 ++++++-- js/panes/hparams/HParamsAxisToolbar.js | 6 - js/panes/hparams/HParamsCompare.js | 164 ++++++++++++++ js/panes/hparams/HParamsFilters.js | 11 - js/panes/hparams/HParamsParallelCoords.js | 89 ++------ js/panes/hparams/HParamsSplom.js | 198 ++++++++++------- js/panes/hparams/HParamsTable.js | 9 +- js/panes/hparams/StatusBadge.js | 19 ++ js/panes/hparams/hparamsPlot.js | 58 ++++- js/panes/hparams/hparamsUtils.js | 152 ++++++------- js/panes/hparams/useHParamsAxes.js | 13 -- py/visdom/static/css/hparams.css | 253 ++++++++++++++++++++++ 12 files changed, 786 insertions(+), 279 deletions(-) create mode 100644 js/panes/hparams/HParamsCompare.js create mode 100644 js/panes/hparams/StatusBadge.js diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index 403c66de5..ac4f269a3 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -9,6 +9,7 @@ import React, { useMemo, useState } from 'react'; +import HParamsCompare from './hparams/HParamsCompare'; import HParamsFilters from './hparams/HParamsFilters'; import HParamsParallelCoords from './hparams/HParamsParallelCoords'; import HParamsSplom from './hparams/HParamsSplom'; @@ -27,6 +28,7 @@ const VIEWS = [ { key: 'table', label: 'Table' }, { key: 'parcoords', label: 'Parallel coordinates' }, { key: 'splom', label: 'Scatter matrix' }, + { key: 'compare', label: 'Compare' }, ]; const NO_RECORDS = []; @@ -85,6 +87,18 @@ var HParamsPane = (props) => { const activeFilters = countActiveFilters(filters, specs) + (tableFilter.trim() ? 1 : 0); + const selectionActive = tableSelected.size > 0; + const selectedVisible = useMemo( + () => + selectionActive + ? visibleRecords.filter((r) => tableSelected.has(r.env_id)) + : visibleRecords, + [selectionActive, visibleRecords, tableSelected] + ); + const clearSelection = () => setTableSelected(new Set()); + + const comparisonRecords = selectionActive ? selectedVisible : NO_RECORDS; + const handleDownload = () => { let blob = new Blob([JSON.stringify(content)], { type: 'application/json', @@ -130,6 +144,20 @@ var HParamsPane = (props) => { showing {visibleRecords.length} of {records.length} ) : null} + {selectionActive ? ( + + {tableSelected.size} selected for plots + + + ) : null}
@@ -192,43 +220,82 @@ var HParamsPane = (props) => {
); } + const isPlot = view === 'splom' || view === 'parcoords'; + const plotRecords = + isPlot && selectionActive ? selectedVisible : visibleRecords; const viewProps = { - records: visibleRecords, columnRecords: records, paramKeys: data.paramKeys, metricKeys: data.metricKeys, tagKeys: data.tagKeys, }; - if (view === 'splom') + + if (view === 'compare') return ( + + ); + + let viewEl; + if (view === 'splom') + viewEl = ( ); - if (view === 'parcoords') - return ( + else if (view === 'parcoords') + viewEl = ( ); + else + viewEl = ( + + ); + + if (!isPlot || !selectionActive) return viewEl; return ( - +
+
+ + Plotting {plotRecords.length} selected{' '} + {plotRecords.length === 1 ? 'run' : 'runs'} + + +
+ {plotRecords.length === 0 ? ( +
+ Every selected run is hidden by the current filters. +
+ ) : ( + viewEl + )} +
); })()}
diff --git a/js/panes/hparams/HParamsAxisToolbar.js b/js/panes/hparams/HParamsAxisToolbar.js index 7f4938332..39283c494 100644 --- a/js/panes/hparams/HParamsAxisToolbar.js +++ b/js/panes/hparams/HParamsAxisToolbar.js @@ -10,12 +10,6 @@ import TreeSelect from 'rc-tree-select'; import React from 'react'; -/* - * The axis picker and colour picker both plot views carry. Only the wording - * differs between them: a scatter matrix talks about dimensions and falls back - * to no colour, parallel coordinates talks about axes and falls back to run - * order. `note` is the trailing status each view fills for itself. - */ const HParamsAxisToolbar = ({ axesLabel, axesName, diff --git a/js/panes/hparams/HParamsCompare.js b/js/panes/hparams/HParamsCompare.js new file mode 100644 index 000000000..1db533e12 --- /dev/null +++ b/js/panes/hparams/HParamsCompare.js @@ -0,0 +1,164 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import React, { useMemo, useState } from 'react'; + +import { + buildColumns, + buildComparison, + formatValue, + isNumeric, + numericExtent, + runLabel, + spineStyle, +} from './hparamsUtils'; +import StatusBadge from './StatusBadge'; + +const HParamsCompare = ({ records, paramKeys, metricKeys, tagKeys }) => { + const [showIdentical, setShowIdentical] = useState(true); + + const columns = useMemo( + () => buildColumns(paramKeys, metricKeys, tagKeys), + [paramKeys, metricKeys, tagKeys] + ); + const comparison = useMemo( + () => buildComparison(records, columns), + [records, columns] + ); + + if (records.length < 2) { + return ( +
+
+ Pick two or more runs in the table to compare them. +
+
+ ); + } + + const inputs = comparison.param.concat(comparison.tag); + const differing = inputs.filter((field) => !field.shared); + const identical = inputs.filter((field) => field.shared); + const metrics = comparison.metric; + const span = records.length + 1; + + const fieldRow = (field, spine) => { + const extent = spine ? numericExtent(records, field.accessor) : null; + return ( + + + {field.label} + + {field.cells.map((value, i) => { + const style = extent ? spineStyle(value, extent) : null; + const cls = + 'hparams-cell' + + (isNumeric(value) ? ' hparams-cell-num' : '') + + (style ? ' hparams-cell-spine' : ''); + return ( + + {formatValue(value)} + + ); + })} + + ); + }; + + const sectionRow = (key, label) => ( + + + {label} + + + ); + + const rows = []; + rows.push(sectionRow('s-differs', 'what differs')); + if (differing.length === 0) { + rows.push( + + + These runs share every parameter and tag. + + + ); + } else { + differing.forEach((field) => rows.push(fieldRow(field, false))); + } + + if (metrics.length) { + rows.push(sectionRow('s-metrics', 'metrics')); + metrics.forEach((field) => rows.push(fieldRow(field, true))); + } + + if (identical.length) { + rows.push( + + + + + + ); + if (showIdentical) { + identical.forEach((field) => rows.push(fieldRow(field, false))); + } + } + + return ( +
+
+ Comparing {records.length} runs — {differing.length} differ,{' '} + {identical.length} identical +
+
+ + + + + ); + })} + + + {rows} +
+ {label} + +
+
+
+ ); +}; + +export default HParamsCompare; diff --git a/js/panes/hparams/HParamsFilters.js b/js/panes/hparams/HParamsFilters.js index fc3f7d34a..ea959321c 100644 --- a/js/panes/hparams/HParamsFilters.js +++ b/js/panes/hparams/HParamsFilters.js @@ -12,23 +12,12 @@ import React, { useCallback, useEffect, useState } from 'react'; import { COLUMN_GROUPS, formatValue, keepsMissing } from './hparamsUtils'; -/* - * Every control edits one field of its filter and leaves the rest alone, so - * they all build the next entry the same way: the current one where it exists, - * the spec's full range or an empty tick list where it does not. - */ const entryFor = (spec, entry) => entry || (spec.kind === 'range' ? { lo: spec.min, hi: spec.max, includeMissing: true } : { values: [], includeMissing: true }); -/* - * A range control that tracks the drag locally and lifts the value only once - * the handle is released. Every commit re-filters the records feeding the - * Plotly views, so committing per drag frame would rebuild those plots - * continuously. - */ const RangeFilter = ({ spec, entry, onChange }) => { const bounds = [entry ? entry.lo : spec.min, entry ? entry.hi : spec.max]; const [dragging, setDragging] = useState(null); diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index a3c61b370..44bd25eb1 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -10,13 +10,18 @@ import React, { useEffect, useMemo, useRef } from 'react'; import HParamsAxisToolbar from './HParamsAxisToolbar'; -import { applySnapshotButton, observePlotResize } from './hparamsPlot'; +import { + PLOT_COLORSCALE, + plotBaseLayout, + plotColorbar, + renderPlot, + usePlotResize, +} from './hparamsPlot'; import { buildParcoordsDimensions, completeRecords, - numericExtent, + resolveColor, runLabel, - toNumericColumn, } from './hparamsUtils'; import useHParamsAxes from './useHParamsAxes'; @@ -24,8 +29,6 @@ const RUN_LABEL_MAX = 18; const MAX_DIMS = 10; -const PARCOORDS_COLORSCALE = 'Viridis'; - const HParamsParallelCoords = ({ records, columnRecords, @@ -69,20 +72,12 @@ const HParamsParallelCoords = ({ return completeRecords(records, requiredCols); }, [records, columns, effectiveDims, effectiveColorBy]); - useEffect(() => { - const el = plotRef.current; - if (!el) return; - return observePlotResize(el); - }, []); + usePlotResize(plotRef); useEffect(() => { const el = plotRef.current; if (!el || !window.Plotly) return; - const colorCol = effectiveColorBy - ? columns.find((c) => c.id === effectiveColorBy) - : null; - const numericDimensions = buildParcoordsDimensions( rows, columns, @@ -108,37 +103,15 @@ const HParamsParallelCoords = ({ }; const dimensions = [runDimension, ...numericDimensions]; - let line; - if (colorCol) { - const ext = numericExtent(rows, colorCol.accessor); - line = { - color: toNumericColumn(rows, colorCol.accessor), - colorscale: PARCOORDS_COLORSCALE, - showscale: true, - cmin: ext ? ext.min : 0, - cmax: ext ? ext.max : 1, - colorbar: { - title: { text: colorCol.label, side: 'right', font: { size: 11 } }, - thickness: 12, - len: 0.6, - outlinewidth: 0, - }, - }; - } else { - line = { - color: rows.map((_, i) => i + 1), - colorscale: PARCOORDS_COLORSCALE, - showscale: true, - cmin: 1, - cmax: Math.max(rows.length, 1), - colorbar: { - title: { text: 'run order', side: 'right', font: { size: 11 } }, - thickness: 12, - len: 0.6, - outlinewidth: 0, - }, - }; - } + const color = resolveColor(rows, columns, effectiveColorBy); + const line = { + color: color.values, + colorscale: PLOT_COLORSCALE, + showscale: true, + cmin: color.cmin, + cmax: color.cmax, + colorbar: plotColorbar(color.label), + }; const data = [ { @@ -154,10 +127,8 @@ const HParamsParallelCoords = ({ ]; const layout = { + ...plotBaseLayout(), margin: { l: 120, r: 80, t: 64, b: 76 }, - font: { family: '"Open Sans", sans-serif', size: 11, color: '#333' }, - paper_bgcolor: '#ffffff', - plot_bgcolor: '#ffffff', datarevision: effectiveDims.join('|') + '::' + @@ -166,27 +137,7 @@ const HParamsParallelCoords = ({ rows.length, }; - const config = applySnapshotButton( - { - showLink: false, - displaylogo: false, - responsive: true, - doubleClick: 'reset', - }, - 'hparams_parcoords.png' - ); - - try { - window.Plotly.react(el, data, layout, config) - .then(() => { - if (el._fullLayout && el.offsetWidth > 0) { - window.Plotly.Plots.resize(el); - } - }) - .catch(() => window.Plotly.purge(el)); - } catch (e) { - window.Plotly.purge(el); - } + renderPlot(el, data, layout, 'hparams_parcoords.png'); }, [rows, columns, effectiveDims, effectiveColorBy]); if (!hasPlot) { diff --git a/js/panes/hparams/HParamsSplom.js b/js/panes/hparams/HParamsSplom.js index 7961fc968..89db56c2e 100644 --- a/js/panes/hparams/HParamsSplom.js +++ b/js/panes/hparams/HParamsSplom.js @@ -10,19 +10,23 @@ import React, { useEffect, useRef } from 'react'; import HParamsAxisToolbar from './HParamsAxisToolbar'; -import { applySnapshotButton, observePlotResize } from './hparamsPlot'; import { - buildSplomDimensions, - numericExtent, - runLabel, + PLOT_COLORSCALE, + plotBaseLayout, + plotColorbar, + renderPlot, + usePlotResize, +} from './hparamsPlot'; +import { + coincidentRuns, + formatValue, + resolveColor, toNumericColumn, } from './hparamsUtils'; import useHParamsAxes from './useHParamsAxes'; const MAX_DIMS = 6; -const SPLOM_COLORSCALE = 'Viridis'; - const AXIS_STYLE = { showline: true, linecolor: '#aab8d8', @@ -35,6 +39,49 @@ const AXIS_STYLE = { automargin: true, }; +function axisDimIndex(axis) { + const id = (axis && (axis._id || axis.id)) || ''; + const n = parseInt(String(id).replace(/[^0-9]/g, ''), 10); + return Number.isNaN(n) ? 0 : n - 1; +} + +function escapeHtml(text) { + return String(text) + .replace(/&/g, '&') + .replace(//g, '>'); +} + +function tipHtml(names, colX, colY, x, y) { + const head = + names.length > 1 ? names.length + ' runs here' : escapeHtml(names[0]); + const list = + names.length > 1 + ? '
    ' + + names.map((n) => '
  • ' + escapeHtml(n) + '
    • ').join('') + + '
' + : ''; + const coord = + colX.id === colY.id + ? escapeHtml(colX.label) + ': ' + formatValue(x) + : escapeHtml(colX.label) + + ': ' + + formatValue(x) + + '
' + + escapeHtml(colY.label) + + ': ' + + formatValue(y); + return ( + '
' + + head + + '
' + + list + + '
' + + coord + + '
' + ); +} + const HParamsSplom = ({ records, columnRecords, @@ -46,7 +93,9 @@ const HParamsSplom = ({ colorBy, onColorBy, }) => { + const wrapRef = useRef(null); const plotRef = useRef(null); + const tipRef = useRef(null); const prevDimCount = useRef(0); const { @@ -67,67 +116,44 @@ const HParamsSplom = ({ maxDims: MAX_DIMS, }); - useEffect(() => { - const el = plotRef.current; - if (!el) return; - return observePlotResize(el); - }, []); + usePlotResize(plotRef); useEffect(() => { const el = plotRef.current; - if (!el || !window.Plotly) return; + if (!el || !window.Plotly) return undefined; - const dimensions = buildSplomDimensions(records, columns, effectiveDims); + const plotted = []; + const dimensions = []; + effectiveDims.forEach((id) => { + const col = columns.find((c) => c.id === id); + if (!col) return; + const values = toNumericColumn(records, col.accessor); + if (values.every((v) => v === null)) return; + plotted.push(col); + dimensions.push({ label: col.label, values }); + }); if (dimensions.length < 2) { window.Plotly.purge(el); prevDimCount.current = 0; - return; + return undefined; } - const names = records.map(runLabel); - - const colorCol = effectiveColorBy - ? columns.find((c) => c.id === effectiveColorBy) - : null; - let colorValues; - let colorLabel; - let cmin; - let cmax; - if (colorCol) { - colorValues = toNumericColumn(records, colorCol.accessor); - colorLabel = colorCol.label; - const ext = numericExtent(records, colorCol.accessor); - if (ext) { - cmin = ext.min; - cmax = ext.max; - } - } else { - colorValues = records.map((_, i) => i + 1); - colorLabel = 'run order'; - cmin = 1; - cmax = Math.max(records.length, 1); - } + const color = resolveColor(records, columns, effectiveColorBy); const data = [ { type: 'splom', dimensions, - text: names, - hovertemplate: '%{text}
x: %{x}
y: %{y}', + hoverinfo: 'none', marker: { size: 7, line: { color: '#ffffff', width: 0.6 }, - color: colorValues, - colorscale: SPLOM_COLORSCALE, + color: color.values, + colorscale: PLOT_COLORSCALE, showscale: true, - cmin, - cmax, - colorbar: { - title: { text: colorLabel, side: 'right', font: { size: 11 } }, - thickness: 12, - len: 0.6, - outlinewidth: 0, - }, + cmin: color.cmin, + cmax: color.cmax, + colorbar: plotColorbar(color.label), }, diagonal: { visible: true }, showupperhalf: true, @@ -137,13 +163,11 @@ const HParamsSplom = ({ ]; const layout = { + ...plotBaseLayout(), margin: { l: 60, r: 20, t: 34, b: 44 }, dragmode: 'select', hovermode: 'closest', showlegend: false, - font: { family: '"Open Sans", sans-serif', size: 11, color: '#333' }, - paper_bgcolor: '#ffffff', - plot_bgcolor: '#ffffff', datarevision: effectiveDims.join('|') + '::' + @@ -162,27 +186,54 @@ const HParamsSplom = ({ } prevDimCount.current = dimensions.length; - const config = applySnapshotButton( - { - showLink: false, - displaylogo: false, - responsive: true, - doubleClick: 'reset', - }, - 'hparams_scatter.png' - ); + const hideTip = () => { + if (tipRef.current) tipRef.current.style.display = 'none'; + }; + const showTip = (ev) => { + const tip = tipRef.current; + const wrap = wrapRef.current; + if (!tip || !wrap || !ev || !ev.points || !ev.points.length) return; + const p = ev.points[0]; + const colX = plotted[axisDimIndex(p.xaxis)]; + const colY = plotted[axisDimIndex(p.yaxis)]; + if (!colX || !colY) return; + const names = coincidentRuns(records, colX, colY, p.x, p.y); + if (names.length === 0) return; - try { - window.Plotly.react(el, data, layout, config) - .then(() => { - if (el._fullLayout && el.offsetWidth > 0) { - window.Plotly.Plots.resize(el); - } - }) - .catch(() => window.Plotly.purge(el)); - } catch (e) { - window.Plotly.purge(el); - } + tip.innerHTML = tipHtml(names, colX, colY, p.x, p.y); + tip.style.display = 'block'; + const rect = wrap.getBoundingClientRect(); + const me = ev.event; + let left = (me ? me.clientX - rect.left : 0) + 14; + let top = (me ? me.clientY - rect.top : 0) + 12; + if (left + tip.offsetWidth > wrap.clientWidth) { + left = wrap.clientWidth - tip.offsetWidth - 6; + } + if (top + tip.offsetHeight > wrap.clientHeight) { + top = wrap.clientHeight - tip.offsetHeight - 6; + } + tip.style.left = Math.max(0, left) + 'px'; + tip.style.top = Math.max(0, top) + 'px'; + }; + + renderPlot(el, data, layout, 'hparams_scatter.png', (gd) => { + if (!gd || typeof gd.on !== 'function') return; + if (gd.removeAllListeners) { + gd.removeAllListeners('plotly_hover'); + gd.removeAllListeners('plotly_unhover'); + } + gd.on('plotly_hover', showTip); + gd.on('plotly_unhover', hideTip); + }); + + return () => { + const gd = plotRef.current; + if (gd && gd.removeAllListeners) { + gd.removeAllListeners('plotly_hover'); + gd.removeAllListeners('plotly_unhover'); + } + hideTip(); + }; }, [records, columns, effectiveDims, effectiveColorBy]); if (!hasPlot) { @@ -196,7 +247,7 @@ const HParamsSplom = ({ } return ( -
+
+
{effectiveDims.length < 2 ? (
Select at least two dimensions to plot. diff --git a/js/panes/hparams/HParamsTable.js b/js/panes/hparams/HParamsTable.js index 3b0f07e70..e3c11aa72 100644 --- a/js/panes/hparams/HParamsTable.js +++ b/js/panes/hparams/HParamsTable.js @@ -23,6 +23,7 @@ import { selectNumericColumns, spineStyle, } from './hparamsUtils'; +import StatusBadge from './StatusBadge'; const RUN_COLUMN_ID = 'run:name'; @@ -111,13 +112,7 @@ const HParamsRow = React.memo(function HParamsRow({
{label} - {record.status ? ( - - {record.status} - - ) : null} +
{columns.map((col) => { diff --git a/js/panes/hparams/StatusBadge.js b/js/panes/hparams/StatusBadge.js new file mode 100644 index 000000000..ecb3a5a49 --- /dev/null +++ b/js/panes/hparams/StatusBadge.js @@ -0,0 +1,19 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import React from 'react'; + +const StatusBadge = ({ status }) => + status ? ( + + {status} + + ) : null; + +export default StatusBadge; diff --git a/js/panes/hparams/hparamsPlot.js b/js/panes/hparams/hparamsPlot.js index 005f0c90f..88a2f1886 100644 --- a/js/panes/hparams/hparamsPlot.js +++ b/js/panes/hparams/hparamsPlot.js @@ -7,14 +7,12 @@ * */ -/* - * Plotly-facing helpers shared by the hyper-parameter plot views (scatter - * matrix and parallel coordinates). Unlike hparamsUtils these touch the global - * Plotly instance and the DOM, so they live apart from the pure helpers. - */ +import { useEffect } from 'react'; const SNAPSHOT_NOTICE_DELAY = 700; +export const PLOT_COLORSCALE = 'Viridis'; + export function notify(message, kind) { const lib = window.Plotly && window.Plotly.Lib; if (lib && typeof lib.notifier === 'function') lib.notifier(message, kind); @@ -79,3 +77,53 @@ export function observePlotResize(el) { if (window.Plotly && el._fullLayout) window.Plotly.purge(el); }; } + +export function usePlotResize(ref) { + useEffect(() => { + const el = ref.current; + if (!el) return undefined; + return observePlotResize(el); + }, [ref]); +} + +export function plotBaseLayout() { + return { + font: { family: '"Open Sans", sans-serif', size: 11, color: '#333' }, + paper_bgcolor: '#ffffff', + plot_bgcolor: '#ffffff', + }; +} + +export function plotColorbar(label) { + return { + title: { text: label, side: 'right', font: { size: 11 } }, + thickness: 12, + len: 0.6, + outlinewidth: 0, + }; +} + +export function renderPlot(el, data, layout, filename, onReady) { + if (!el || !window.Plotly) return; + const config = applySnapshotButton( + { + showLink: false, + displaylogo: false, + responsive: true, + doubleClick: 'reset', + }, + filename + ); + try { + window.Plotly.react(el, data, layout, config) + .then(() => { + if (el._fullLayout && el.offsetWidth > 0) { + window.Plotly.Plots.resize(el); + } + if (onReady) onReady(el); + }) + .catch(() => window.Plotly.purge(el)); + } catch (e) { + window.Plotly.purge(el); + } +} diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index 8d9e7105c..bd107d700 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -7,14 +7,6 @@ * */ -/* - * Pure helpers for the hyper-parameter table. Kept free of React so the - * ordering and formatting rules live in one place and can mirror the Python - * backend exactly. The comparator below intentionally matches - * visdom.experiments.store._order_key / _sort_pairs so the table and the - * server-side search agree on which run ranks first. - */ - const SPINE_LIGHT = [235, 240, 249]; const SPINE_DARK = [59, 89, 152]; @@ -192,11 +184,6 @@ export function spineStyle(value, extent) { return { backgroundColor: bg, color: t > 0.62 ? '#fff' : '#333' }; } -/* - * Numeric param/metric columns only — the axes a scatter matrix (SPLOM) or a - * "color by" ramp can actually plot. Tags are excluded (categorical) and any - * column whose values are all missing/non-numeric is dropped. - */ export function selectNumericColumns(records, columns) { return (columns || []).filter( (col) => @@ -205,13 +192,6 @@ export function selectNumericColumns(records, columns) { ); } -/* - * The axes a plot opens on: one param against one metric, so a freshly opened - * view reads as a single relationship rather than every numeric column at once. - * Callers may pass `isDense` to prefer columns that hold a value on every run, - * since a sparse axis costs a parallel-coordinates plot whole lines. Falls back - * to the first two columns when the data has no param/metric pair to offer. - */ export function defaultDimIds(numericCols, isDense) { const cols = numericCols || []; const dense = isDense ? cols.filter(isDense) : cols; @@ -224,22 +204,38 @@ export function defaultDimIds(numericCols, isDense) { return cols.slice(0, 2).map((col) => col.id); } -/* - * Build Plotly `splom` dimensions from the chosen column ids. Order follows - * selectedIds; unknown ids are skipped; missing/non-numeric cells become null - * so Plotly leaves a gap instead of plotting a bogus 0. - */ -export function buildSplomDimensions(records, columns, selectedIds) { - const byId = new Map((columns || []).map((col) => [col.id, col])); - const dimensions = []; - (selectedIds || []).forEach((id) => { - const col = byId.get(id); - if (!col) return; - const values = toNumericColumn(records, col.accessor); - if (values.every((v) => v === null)) return; - dimensions.push({ label: col.label, values }); +export function coincidentRuns(records, colX, colY, x, y) { + if (!colX || !colY) return []; + const names = []; + (records || []).forEach((record) => { + const vx = colX.accessor(record); + const vy = colY.accessor(record); + if (isNumeric(vx) && isNumeric(vy) && vx === x && vy === y) { + names.push(runLabel(record)); + } }); - return dimensions; + return names; +} + +export function resolveColor(records, columns, colorById) { + const col = colorById + ? (columns || []).find((c) => c.id === colorById) + : null; + if (col) { + const ext = numericExtent(records, col.accessor); + return { + values: toNumericColumn(records, col.accessor), + label: col.label, + cmin: ext ? ext.min : 0, + cmax: ext ? ext.max : 1, + }; + } + return { + values: (records || []).map((_, i) => i + 1), + label: 'run order', + cmin: 1, + cmax: Math.max((records || []).length, 1), + }; } export function completeRecords(records, cols) { @@ -248,13 +244,6 @@ export function completeRecords(records, cols) { ); } -/* - * Build Plotly `parcoords` dimensions. Plotly cannot render null/NaN cells — - * one sparse axis corrupts every line — so callers pass records that already - * hold a numeric value on every axis (see completeRecords). Each axis spans its - * exact data range; an axis whose values are all equal gets a small symmetric - * range so it does not collapse to zero height. - */ export function buildParcoordsDimensions(records, columns, selectedIds) { const byId = new Map((columns || []).map((col) => [col.id, col])); const dimensions = []; @@ -276,17 +265,8 @@ export function buildParcoordsDimensions(records, columns, selectedIds) { return dimensions; } -/* - * A column with more distinct values than this is not offered as a checkbox - * list — the free-text search is the better tool for high-cardinality strings. - */ const MAX_CATEGORIES = 12; -/* - * Statuses come from the Python model (visdom.experiments.models), but the - * order here is lifecycle order rather than alphabetical so the sidebar reads - * the way a run progresses. - */ export const STATUS_ORDER = ['running', 'finished', 'failed']; function distinctValues(records, accessor) { @@ -304,16 +284,6 @@ function distinctValues(records, accessor) { return { values: Array.from(seen.values()), missing }; } -/* - * Derive one filter control per column. A numeric param/metric with more than - * two distinct values becomes a range slider; anything with few enough distinct - * values becomes a checkbox list (this deliberately catches low-cardinality - * numerics such as batch size, where discrete choices beat a slider). Columns - * that are neither are skipped rather than rendered as an unusable control. - * - * Category values are ordered with the same comparator the table uses, which - * in turn mirrors the backend's _sort_pairs, so numbers precede strings. - */ export function buildFilterSpecs(records, columns) { const specs = []; (columns || []).forEach((col) => { @@ -356,11 +326,6 @@ export function buildFilterSpecs(records, columns) { return specs; } -/* - * The statuses actually present in the data, in lifecycle order. Derived rather - * than hardcoded so a run in a status this build does not know about still gets - * a checkbox instead of silently becoming unfilterable. - */ export function collectStatuses(records) { const present = new Set(); (records || []).forEach((record) => { @@ -373,11 +338,6 @@ export function collectStatuses(records) { return known.concat(extra); } -/* - * Whether a filter still admits runs that have no value for its column. Absent - * means yes: a filter should only ever remove runs it actually judged, so - * excluding the unmeasured ones has to be asked for. - */ export function keepsMissing(entry) { return !entry || entry.includeMissing !== false; } @@ -394,12 +354,6 @@ function passesSpec(record, spec, entry, accessor) { return entry.values.indexOf(value) !== -1; } -/* - * Faceted semantics: every active column filter must pass (AND), but within one - * category list any checked value is enough (OR). A run whose value is missing - * survives only while that filter keeps missing values, which is what makes a - * range filter safe to apply to sparse metric columns. - */ export function applyFilters(records, specs, filters) { const state = filters || {}; const statuses = state.statuses; @@ -422,11 +376,6 @@ export function applyFilters(records, specs, filters) { }); } -/* - * How many filters the badge should report. A range left at its full extent is - * not counted: it excludes nothing, so claiming it as active would make the - * badge disagree with what the user sees. - */ export function countActiveFilters(filters, specs) { const state = filters || {}; const columns = state.columns || {}; @@ -442,3 +391,42 @@ export function countActiveFilters(filters, specs) { }); return count; } + +export function sameValue(a, b) { + if (typeof a === 'boolean' || typeof b === 'boolean') return a === b; + if (typeof a === 'number' && typeof b === 'number') { + if (Number.isNaN(a) && Number.isNaN(b)) return true; + return a === b; + } + return a === b; +} + +export function buildComparison(records, columns) { + const runs = records || []; + const sections = { param: [], metric: [], tag: [] }; + (columns || []).forEach((col) => { + const cells = runs.map((record) => col.accessor(record)); + let present = 0; + const groups = []; + cells.forEach((value) => { + if (isMissing(value)) return; + present += 1; + const group = groups.find((g) => sameValue(g.value, value)); + if (group) group.count += 1; + else groups.push({ value, count: 1 }); + }); + if (present === 0) return; + const shared = present === runs.length && groups.length === 1; + if (!sections[col.group]) return; + sections[col.group].push({ + id: col.id, + label: col.label, + group: col.group, + accessor: col.accessor, + cells, + groups, + shared, + }); + }); + return sections; +} diff --git a/js/panes/hparams/useHParamsAxes.js b/js/panes/hparams/useHParamsAxes.js index 05e8d1d38..659c36d46 100644 --- a/js/panes/hparams/useHParamsAxes.js +++ b/js/panes/hparams/useHParamsAxes.js @@ -18,19 +18,6 @@ import { selectNumericColumns, } from './hparamsUtils'; -/* - * Resolves which axes a plot should draw, shared by the scatter matrix and - * parallel coordinates because both answer the question identically. - * - * `columnRecords` is the unfiltered set: which columns can be plotted at all is - * decided from it, so a filter that empties a column cannot make it vanish from - * the picker and silently reset a selection. `records` is what actually gets - * drawn, and only informs which columns are dense. - * - * `preferDense` suits parallel coordinates, where a column with gaps costs the - * plot whole lines and is a poor opening axis; a scatter matrix just leaves a - * marker out, so it does not care. - */ export default function useHParamsAxes({ records, columnRecords, diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index 60f0969c5..722276aa5 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -469,6 +469,40 @@ min-width: 0; } +/* Live hover for the scatter matrix: names every run stacked on the marker + under the cursor, which a shared marker would otherwise hide. */ +.hparams-splom-tip { + position: absolute; + z-index: 4; + display: none; + max-width: 220px; + padding: 6px 9px; + font-size: 11px; + line-height: 1.4; + color: #f5f7fb; + background-color: rgba(30, 40, 62, 0.95); + border-radius: 4px; + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.25); + pointer-events: none; +} + +.hparams-splom-tip-head { + font-weight: 600; +} + +.hparams-splom-tip-list { + margin: 3px 0 0; + padding-left: 16px; + max-height: 160px; + overflow-y: auto; +} + +.hparams-splom-tip-coord { + margin-top: 4px; + color: #b9c4de; + font-variant-numeric: tabular-nums; +} + .hparams-plot-overlay { position: absolute; top: 40px; @@ -506,6 +540,139 @@ min-width: 0; } +/* ---- HParamsCompare (C1) ---- */ + +.hparams-compare-wrap { + flex: 1 1 auto; + min-height: 0; + min-width: 0; + display: flex; + flex-direction: column; +} + +.hparams-compare-lead { + padding: 8px 12px; + color: #666; + border-bottom: 1px solid #f0f0f0; +} + +.hparams-compare-lead b { + color: #3b5998; + font-weight: 600; +} + +.hparams-compare-scroll { + flex: 1 1 auto; + min-height: 0; + overflow: auto; +} + +.hparams-compare-table { + border-collapse: separate; + border-spacing: 0; + font-size: 12px; + color: #333; +} + +.hparams-compare-table th, +.hparams-compare-table td { + border-bottom: 1px solid #f0f0f0; + border-right: 1px solid #f0f0f0; + padding: 4px 12px; + text-align: left; + white-space: nowrap; + font-weight: normal; +} + +/* Run columns are the comparison; their headers stay put on both scroll axes. */ +.hparams-compare-run { + position: sticky; + top: 0; + z-index: 2; + min-width: 96px; + background-color: #f0f0f0; + border-bottom: 1px solid #dedede !important; + vertical-align: bottom; +} + +.hparams-compare-run-name { + display: block; + max-width: 160px; + overflow: hidden; + font-weight: 600; + color: #222; + text-overflow: ellipsis; +} + +.hparams-compare-run .hparams-run-status { + margin-top: 3px; +} + +.hparams-compare-corner { + position: sticky; + top: 0; + left: 0; + z-index: 3; + background-color: #f0f0f0; + border-bottom: 1px solid #dedede !important; +} + +/* The field name is the row's identity — pin it left so a wide run set scrolls + under a label that stays readable. */ +.hparams-compare-field { + position: sticky; + left: 0; + z-index: 1; + max-width: 200px; + overflow: hidden; + color: #444; + background-color: #fff; + text-overflow: ellipsis; +} + +.hparams-compare-row:hover td, +.hparams-compare-row:hover .hparams-compare-field { + background-color: #f7f9fd; +} + +/* Section eyebrow: what-differs / metrics / identical — the structure that tells + the reader where the signal is. */ +.hparams-compare-section th { + position: sticky; + left: 0; + padding: 8px 12px 3px; + font-size: 10px; + font-weight: 600; + letter-spacing: 0.08em; + text-transform: uppercase; + color: #9aa2b1; + background-color: #fff; + border-right-color: transparent !important; +} + +.hparams-compare-toggle { + padding: 0; + font: inherit; + font-size: 10px; + font-weight: 600; + letter-spacing: 0.08em; + text-transform: uppercase; + color: #3b5998; + background: none; + border: none; + cursor: pointer; +} + +.hparams-compare-toggle:focus-visible { + outline: 2px solid #3b5998; + outline-offset: 2px; +} + +.hparams-compare-note { + color: #888; + font-style: italic; +} + /* ---- HParamsFilters (B5) ---- */ .hparams-layout { @@ -690,3 +857,89 @@ .hparams-stat-filtered { color: #3b5998; } + +/* A removable chip: the count reads as a filter that is currently on, and the + × is the way to turn it off. */ +.hparams-stat-selected { + display: inline-flex; + align-items: center; + gap: 7px; + padding: 1px 3px 1px 9px; + color: #3b5998; + background-color: #e6ecfa; + border: 1px solid #cddaf3; + border-radius: 11px; +} + +.hparams-chip-clear { + display: inline-flex; + align-items: center; + justify-content: center; + width: 16px; + height: 16px; + padding: 0; + font-size: 15px; + line-height: 1; + color: #5c73a8; + background: none; + border: none; + border-radius: 50%; + cursor: pointer; +} + +.hparams-chip-clear:hover { + color: #fff; + background-color: #3b5998; +} + +.hparams-chip-clear:focus-visible { + outline: 2px solid #3b5998; + outline-offset: 1px; +} + +/* A quiet inline action — reads as a link, not a button, so it sits inside the + plot banner's sentence without stealing weight from the text next to it. */ +.hparams-link-btn { + padding: 0; + font: inherit; + font-size: 11px; + color: #3b5998; + background: none; + border: none; + text-decoration: underline; + cursor: pointer; +} + +.hparams-link-btn:hover { + color: #24386b; +} + +.hparams-link-btn:focus-visible { + outline: 2px solid #3b5998; + outline-offset: 1px; +} + +/* Wraps a plot when a selection is active so the banner and the plot share the + column the plot alone usually fills. */ +.hparams-plot-area { + flex: 1 1 auto; + min-height: 0; + min-width: 0; + display: flex; + flex-direction: column; +} + +.hparams-selection-banner { + display: flex; + align-items: center; + gap: 10px; + padding: 5px 12px; + font-size: 12px; + color: #3b5998; + background-color: #eaf0fb; + border-bottom: 1px solid #d4e0f6; +} + +.hparams-selection-banner b { + font-weight: 600; +} From dd930d22acafbea8f0d8bb5cb97270d2f880fe3f Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Fri, 24 Jul 2026 18:30:11 +0530 Subject: [PATCH 39/48] refactor(hparams): share the data-cell class builder across table and compare extract cellClass into hparamsUtils and consume it from HParamsTable and HParamsCompare, dropping the duplicated inline className concatenation --- js/panes/hparams/HParamsCompare.js | 7 ++----- js/panes/hparams/HParamsTable.js | 11 +++++------ js/panes/hparams/hparamsUtils.js | 10 ++++++++++ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/js/panes/hparams/HParamsCompare.js b/js/panes/hparams/HParamsCompare.js index 1db533e12..55216a673 100644 --- a/js/panes/hparams/HParamsCompare.js +++ b/js/panes/hparams/HParamsCompare.js @@ -12,8 +12,8 @@ import React, { useMemo, useState } from 'react'; import { buildColumns, buildComparison, + cellClass, formatValue, - isNumeric, numericExtent, runLabel, spineStyle, @@ -57,10 +57,7 @@ const HParamsCompare = ({ records, paramKeys, metricKeys, tagKeys }) => { {field.cells.map((value, i) => { const style = extent ? spineStyle(value, extent) : null; - const cls = - 'hparams-cell' + - (isNumeric(value) ? ' hparams-cell-num' : '') + - (style ? ' hparams-cell-spine' : ''); + const cls = cellClass(value, { spine: !!style }); return ( {formatValue(value)} diff --git a/js/panes/hparams/HParamsTable.js b/js/panes/hparams/HParamsTable.js index e3c11aa72..501daf41e 100644 --- a/js/panes/hparams/HParamsTable.js +++ b/js/panes/hparams/HParamsTable.js @@ -12,10 +12,10 @@ import React, { useCallback, useMemo } from 'react'; import { buildColumns, + cellClass, COLUMN_GROUPS, formatValue, groupColumnTree, - isNumeric, makeComparator, NUMERIC_GROUPS, numericExtent, @@ -119,11 +119,10 @@ const HParamsRow = React.memo(function HParamsRow({ const value = col.accessor(record); const style = colorBy && col.id === colorBy ? spineStyle(value, extent) : null; - const cls = - 'hparams-cell' + - (isNumeric(value) ? ' hparams-cell-num' : '') + - (style ? ' hparams-cell-spine' : '') + - (groupStartIds.has(col.id) ? ' hparams-col-sep' : ''); + const cls = cellClass(value, { + spine: !!style, + separator: groupStartIds.has(col.id), + }); return ( {formatValue(value)} diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index bd107d700..70c982434 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -184,6 +184,16 @@ export function spineStyle(value, extent) { return { backgroundColor: bg, color: t > 0.62 ? '#fff' : '#333' }; } +export function cellClass(value, options) { + const opts = options || {}; + return ( + 'hparams-cell' + + (isNumeric(value) ? ' hparams-cell-num' : '') + + (opts.spine ? ' hparams-cell-spine' : '') + + (opts.separator ? ' hparams-col-sep' : '') + ); +} + export function selectNumericColumns(records, columns) { return (columns || []).filter( (col) => From b7e1ad7765ce177be1d785d7987382839da838ba Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Fri, 24 Jul 2026 18:52:10 +0530 Subject: [PATCH 40/48] perf(hparams): stop the pane and its views re-rendering on unrelated updates memoize the parsed pane content so the column, spec, and visible-record chain no longer rebuilds every render; memo the scatter matrix, parallel coordinates, and filter panel; render only the filter row whose entry changed --- js/panes/HParamsPane.js | 13 ++++++------ js/panes/hparams/HParamsFilters.js | 24 +++++++++++++++-------- js/panes/hparams/HParamsParallelCoords.js | 2 +- js/panes/hparams/HParamsSplom.js | 2 +- 4 files changed, 25 insertions(+), 16 deletions(-) diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index ac4f269a3..19354d428 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -7,7 +7,7 @@ * */ -import React, { useMemo, useState } from 'react'; +import React, { useCallback, useMemo, useState } from 'react'; import HParamsCompare from './hparams/HParamsCompare'; import HParamsFilters from './hparams/HParamsFilters'; @@ -51,7 +51,7 @@ function readContent(content) { var HParamsPane = (props) => { const { content } = props; - const data = readContent(content); + const data = useMemo(() => readContent(content), [content]); const [view, setView] = useState('table'); const [tableSort, setTableSort] = useState({ by: null, dir: null }); const [tableFilter, setTableFilter] = useState(''); @@ -95,11 +95,12 @@ var HParamsPane = (props) => { : visibleRecords, [selectionActive, visibleRecords, tableSelected] ); - const clearSelection = () => setTableSelected(new Set()); + const clearSelection = useCallback(() => setTableSelected(new Set()), []); + const closeFilters = useCallback(() => setFiltersOpen(false), []); const comparisonRecords = selectionActive ? selectedVisible : NO_RECORDS; - const handleDownload = () => { + const handleDownload = useCallback(() => { let blob = new Blob([JSON.stringify(content)], { type: 'application/json', }); @@ -108,7 +109,7 @@ var HParamsPane = (props) => { link.download = 'visdom_hparams.json'; link.href = url; link.click(); - }; + }, [content]); let body; if (data === null) { @@ -207,7 +208,7 @@ var HParamsPane = (props) => { filters={filters} setFilters={setFilters} setSearch={setTableFilter} - onClose={() => setFiltersOpen(false)} + onClose={closeFilters} visibleCount={visibleRecords.length} totalCount={records.length} /> diff --git a/js/panes/hparams/HParamsFilters.js b/js/panes/hparams/HParamsFilters.js index ea959321c..850b4608c 100644 --- a/js/panes/hparams/HParamsFilters.js +++ b/js/panes/hparams/HParamsFilters.js @@ -8,7 +8,7 @@ */ import Slider from 'rc-slider'; -import React, { useCallback, useEffect, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { COLUMN_GROUPS, formatValue, keepsMissing } from './hparamsUtils'; @@ -79,7 +79,11 @@ const CategoryFilter = ({ spec, entry, onChange }) => { ); }; -const FilterSection = ({ spec, entry, onChange }) => { +const FilterSection = React.memo(function FilterSection({ + spec, + entry, + onChange, +}) { const toggleMissing = () => { onChange(spec.id, { ...entryFor(spec, entry), @@ -114,7 +118,7 @@ const FilterSection = ({ spec, entry, onChange }) => {
); -}; +}); const HParamsFilters = ({ specs, @@ -154,10 +158,14 @@ const HParamsFilters = ({ setSearch(''); }, [setFilters, setSearch]); - const groups = COLUMN_GROUPS.map((group) => ({ - ...group, - specs: specs.filter((spec) => spec.group === group.key), - })).filter((group) => group.specs.length > 0); + const groups = useMemo( + () => + COLUMN_GROUPS.map((group) => ({ + ...group, + specs: specs.filter((spec) => spec.group === group.key), + })).filter((group) => group.specs.length > 0), + [specs] + ); return (
@@ -231,4 +239,4 @@ const HParamsFilters = ({ ); }; -export default HParamsFilters; +export default React.memo(HParamsFilters); diff --git a/js/panes/hparams/HParamsParallelCoords.js b/js/panes/hparams/HParamsParallelCoords.js index 44bd25eb1..dcae41827 100644 --- a/js/panes/hparams/HParamsParallelCoords.js +++ b/js/panes/hparams/HParamsParallelCoords.js @@ -186,4 +186,4 @@ const HParamsParallelCoords = ({ ); }; -export default HParamsParallelCoords; +export default React.memo(HParamsParallelCoords); diff --git a/js/panes/hparams/HParamsSplom.js b/js/panes/hparams/HParamsSplom.js index 89db56c2e..944861f1d 100644 --- a/js/panes/hparams/HParamsSplom.js +++ b/js/panes/hparams/HParamsSplom.js @@ -271,4 +271,4 @@ const HParamsSplom = ({ ); }; -export default HParamsSplom; +export default React.memo(HParamsSplom); From 640620daa37ce3c8d9ea819cbb4ec8a0792a8878 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Sat, 25 Jul 2026 17:12:24 +0530 Subject: [PATCH 41/48] feat(hparams): plot the metric history of the selected runs The pane could compare final numbers but not show how a run got there: every flatten site keeps only each metric's latest value, so the window content carries no history at all. A fifth tab reads it from the server. - new Metrics tab, gated on the table selection like Compare, since falling back to all visible runs would download every run's full history on tab open and draw an unreadable chart - one line per selected run for a metric picked from the union of the keys the selection logged, coloured by position in the unfiltered records so a run keeps its colour as filters and selection change - reads experiments/compare through window.fetch, never jQuery: the document-level ajaxError handler navigates the whole page to error/500, which would destroy the dashboard on a 404 for a deleted run - caches per run, so ticking one more checkbox fetches one run rather than re-reading the whole selection; Refresh re-reads a running run - a series plots against real steps only when every observation has one, otherwise against its own ordinal; logging without a step is the SDK default and mixing the two would place points the data never claimed - repeated steps collapse to one point with the later write winning, so a line ends on the number the table shows; NaN stays a gap in the line - lifts correctPathname out of ApiProvider so the request keeps working under -base_url The view is fetch-on-demand, not live: the pane memoises on window id, so Refresh is the only way to pick up new observations. Plotly's legend toggles a run, and double-click isolates one, but that visibility resets when the metric changes. --- js/api/ApiProvider.js | 16 +- js/api/experimentsApi.js | 44 +++++ js/api/serverPath.js | 19 +++ js/panes/HParamsPane.js | 22 ++- js/panes/hparams/HParamsMetrics.js | 194 +++++++++++++++++++++++ js/panes/hparams/hparamsPlot.js | 35 ++++ js/panes/hparams/hparamsUtils.js | 83 ++++++++++ js/panes/hparams/useExperimentMetrics.js | 102 ++++++++++++ py/visdom/static/css/hparams.css | 34 ++++ 9 files changed, 532 insertions(+), 17 deletions(-) create mode 100644 js/api/experimentsApi.js create mode 100644 js/api/serverPath.js create mode 100644 js/panes/hparams/HParamsMetrics.js create mode 100644 js/panes/hparams/useExperimentMetrics.js diff --git a/js/api/ApiProvider.js b/js/api/ApiProvider.js index 5bb8df0f4..6f5b24115 100644 --- a/js/api/ApiProvider.js +++ b/js/api/ApiProvider.js @@ -4,6 +4,7 @@ import React, { useEffect, useRef, useState } from 'react'; import { showToast } from '../toasts/toastEvents'; import ApiContext from './ApiContext'; import Poller from './Legacy'; +import serverPath from './serverPath'; const ApiProvider = ({ children }) => { const [connected, setConnected] = useState(false); @@ -15,20 +16,7 @@ const ApiProvider = ({ children }) => { // helper functions // // ---------------- // - // Normalize window.location by removing specific path segments - // and ensuring the pathname ends with a '/' - const correctPathname = () => { - var pathname = window.location.pathname; - if (pathname.indexOf('/env/') > -1) { - pathname = pathname.split('/env/')[0]; - } else if (pathname.indexOf('/compare/') > -1) { - pathname = pathname.split('/compare/')[0]; - } - if (pathname.slice(-1) != '/') { - pathname = pathname + '/'; - } - return pathname; - }; + const correctPathname = serverPath; // ------------------- // // basic communication // diff --git a/js/api/experimentsApi.js b/js/api/experimentsApi.js new file mode 100644 index 000000000..1fa9363c4 --- /dev/null +++ b/js/api/experimentsApi.js @@ -0,0 +1,44 @@ +import serverPath from './serverPath'; + +/** + * Read the comparison payload for a set of runs. + * + * Deliberately window.fetch and not jQuery: ApiProvider installs a + * document-level ajaxError handler that navigates the whole page to + * error/500, so a 404 for a deleted run would destroy the dashboard. + * fetch keeps the failure local to the caller. + * + * The response carries params/metrics/tags diff sections alongside the + * raw experiments; only experiments[].metrics holds per-step history. + */ +export function fetchExperimentComparison(envIds, signal) { + const ids = (envIds || []).filter((id) => typeof id === 'string'); + if (ids.length === 0) { + return Promise.reject(new Error('No runs to load.')); + } + return window + .fetch(serverPath() + 'experiments/compare', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + credentials: 'same-origin', + body: JSON.stringify({ env_ids: ids }), + signal, + }) + .catch((err) => { + /* A dead server rejects with a bare "Failed to fetch", which reads + like a bug rather than a server that is not answering. */ + if (err && err.name === 'AbortError') throw err; + throw new Error('Could not reach the server.'); + }) + .then((res) => { + /* An empty 401 body from check_auth would make res.json() throw a + SyntaxError that reads like a parsing bug, so branch on ok first. */ + if (!res.ok) { + const reason = res.statusText || 'request failed'; + throw new Error('Could not load metric history (' + reason + ').'); + } + return res.json(); + }); +} + +export default fetchExperimentComparison; diff --git a/js/api/serverPath.js b/js/api/serverPath.js new file mode 100644 index 000000000..8a136e27c --- /dev/null +++ b/js/api/serverPath.js @@ -0,0 +1,19 @@ +/** + * Normalize window.location by removing specific path segments + * and ensuring the pathname ends with a '/'. + * + * The pathname already carries the server's base_url, so deriving the + * prefix from it keeps requests correct under -base_url deployments. + */ +export default function serverPath() { + var pathname = window.location.pathname; + if (pathname.indexOf('/env/') > -1) { + pathname = pathname.split('/env/')[0]; + } else if (pathname.indexOf('/compare/') > -1) { + pathname = pathname.split('/compare/')[0]; + } + if (pathname.slice(-1) != '/') { + pathname = pathname + '/'; + } + return pathname; +} diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index 19354d428..218bab171 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -7,10 +7,11 @@ * */ -import React, { useCallback, useMemo, useState } from 'react'; +import React, { useCallback, useMemo, useRef, useState } from 'react'; import HParamsCompare from './hparams/HParamsCompare'; import HParamsFilters from './hparams/HParamsFilters'; +import HParamsMetrics from './hparams/HParamsMetrics'; import HParamsParallelCoords from './hparams/HParamsParallelCoords'; import HParamsSplom from './hparams/HParamsSplom'; import HParamsTable from './hparams/HParamsTable'; @@ -29,6 +30,7 @@ const VIEWS = [ { key: 'parcoords', label: 'Parallel coordinates' }, { key: 'splom', label: 'Scatter matrix' }, { key: 'compare', label: 'Compare' }, + { key: 'metrics', label: 'Metrics' }, ]; const NO_RECORDS = []; @@ -61,6 +63,9 @@ var HParamsPane = (props) => { const [splomColorBy, setSplomColorBy] = useState(null); const [parcoordsDims, setParcoordsDims] = useState(null); const [parcoordsColorBy, setParcoordsColorBy] = useState(null); + const [metricsKey, setMetricsKey] = useState(null); + const metricsCache = useRef(null); + if (metricsCache.current === null) metricsCache.current = new Map(); const [filtersOpen, setFiltersOpen] = useState(true); const [filters, setFilters] = useState({ statuses: [], columns: {} }); @@ -98,7 +103,7 @@ var HParamsPane = (props) => { const clearSelection = useCallback(() => setTableSelected(new Set()), []); const closeFilters = useCallback(() => setFiltersOpen(false), []); - const comparisonRecords = selectionActive ? selectedVisible : NO_RECORDS; + const selectedRecords = selectionActive ? selectedVisible : NO_RECORDS; const handleDownload = useCallback(() => { let blob = new Blob([JSON.stringify(content)], { @@ -233,7 +238,18 @@ var HParamsPane = (props) => { if (view === 'compare') return ( - + + ); + + if (view === 'metrics') + return ( + ); let viewEl; diff --git a/js/panes/hparams/HParamsMetrics.js b/js/panes/hparams/HParamsMetrics.js new file mode 100644 index 000000000..247fe9f5b --- /dev/null +++ b/js/panes/hparams/HParamsMetrics.js @@ -0,0 +1,194 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import React, { useEffect, useMemo, useRef } from 'react'; + +import { + plotAxisStyle, + plotBaseLayout, + renderPlot, + runColor, + usePlotResize, +} from './hparamsPlot'; +import { selectMetricSeries } from './hparamsUtils'; +import useExperimentMetrics from './useExperimentMetrics'; + +const MAX_MISSING_NAMED = 3; + +function missingNote(missing, metric) { + if (missing.length === 0) return null; + const named = missing.slice(0, MAX_MISSING_NAMED).join(', '); + const rest = missing.length - MAX_MISSING_NAMED; + const who = rest > 0 ? named + ' +' + rest + ' more' : named; + return 'no ' + metric + ' logged by ' + who; +} + +var HParamsMetrics = (props) => { + const { records, columnRecords, metric, onMetric, cacheRef } = props; + const plotRef = useRef(null); + const { status, error, runs, metricKeys, refresh } = useExperimentMetrics( + records, + cacheRef + ); + + /* Colour by position in the unfiltered records so a run keeps its line + colour when the selection or the filters change. */ + const colorIndex = useMemo(() => { + const index = new Map(); + (columnRecords || []).forEach((record, i) => { + if (record && record.env_id) index.set(record.env_id, i); + }); + return index; + }, [columnRecords]); + + const activeMetric = useMemo(() => { + if (metric && metricKeys.indexOf(metric) > -1) return metric; + return metricKeys.length > 0 ? metricKeys[0] : null; + }, [metric, metricKeys]); + + const { plotted, missing } = useMemo( + () => selectMetricSeries(runs, activeMetric, colorIndex), + [runs, activeMetric, colorIndex] + ); + + const xLabel = useMemo(() => { + if (plotted.length === 0) return 'step'; + const indexed = plotted.filter((run) => run.usesIndex).length; + if (indexed === 0) return 'step'; + return indexed === plotted.length ? 'observation' : 'step / observation'; + }, [plotted]); + + usePlotResize(plotRef); + + useEffect(() => { + const el = plotRef.current; + if (!el || !window.Plotly) return; + if (plotted.length === 0 || !activeMetric) { + window.Plotly.purge(el); + return; + } + + const data = plotted.map((run) => ({ + type: 'scatter', + mode: 'lines', + name: run.label, + x: run.x, + y: run.y, + connectgaps: false, + line: { color: runColor(run.colorIndex), width: 1.6 }, + hovertemplate: + '%{fullData.name}
' + + xLabel + + ' %{x}
' + + activeMetric + + ' %{y}', + })); + + const points = plotted.reduce((total, run) => total + run.x.length, 0); + const layout = { + ...plotBaseLayout(), + margin: { l: 56, r: 16, t: 12, b: 44 }, + xaxis: { + ...plotAxisStyle(), + title: { text: xLabel, font: { size: 11 } }, + }, + yaxis: { + ...plotAxisStyle(), + title: { text: activeMetric, font: { size: 11 } }, + }, + showlegend: true, + legend: { font: { size: 10 } }, + hovermode: 'closest', + datarevision: + activeMetric + + '::' + + plotted.map((run) => run.env_id).join('|') + + '::' + + points, + }; + + renderPlot(el, data, layout, 'hparams_metrics.png'); + }, [plotted, activeMetric, xLabel]); + + if (records.length === 0) { + return ( +
+
+ Pick one or more runs in the table to plot their metric history. +
+
+ ); + } + + if (status === 'error') { + return ( +
+
+ {error}{' '} + +
+
+ ); + } + + if (status === 'ready' && metricKeys.length === 0) { + return ( +
+
+ None of these runs logged any metrics. +
+
+ ); + } + + const note = missingNote(missing, activeMetric); + + return ( +
+
+ + + {plotted.length} of {records.length}{' '} + {records.length === 1 ? 'run' : 'runs'} + {note ? ' · ' + note : ''} + + +
+
+ {status === 'loading' ? ( +
Loading metric history…
+ ) : plotted.length === 0 && activeMetric ? ( +
+ No run logged {activeMetric} yet. +
+ ) : null} +
+ ); +}; + +export default React.memo(HParamsMetrics); diff --git a/js/panes/hparams/hparamsPlot.js b/js/panes/hparams/hparamsPlot.js index 88a2f1886..c1aca64db 100644 --- a/js/panes/hparams/hparamsPlot.js +++ b/js/panes/hparams/hparamsPlot.js @@ -13,6 +13,26 @@ const SNAPSHOT_NOTICE_DELAY = 700; export const PLOT_COLORSCALE = 'Viridis'; +/* Plotly's own default colorway. PLOT_COLORSCALE is a continuous ramp + and cannot tell a handful of discrete runs apart. */ +export const RUN_PALETTE = [ + '#1f77b4', + '#ff7f0e', + '#2ca02c', + '#d62728', + '#9467bd', + '#8c564b', + '#e377c2', + '#7f7f7f', + '#bcbd22', + '#17becf', +]; + +export function runColor(index) { + const i = Number.isFinite(index) ? Math.abs(Math.trunc(index)) : 0; + return RUN_PALETTE[i % RUN_PALETTE.length]; +} + export function notify(message, kind) { const lib = window.Plotly && window.Plotly.Lib; if (lib && typeof lib.notifier === 'function') lib.notifier(message, kind); @@ -94,6 +114,21 @@ export function plotBaseLayout() { }; } +/* A fresh object per call: axis styles are spread per axis and a shared + constant would be a mutation hazard. */ +export function plotAxisStyle() { + return { + showline: true, + linecolor: '#aab8d8', + linewidth: 1, + gridcolor: '#f0f2f8', + zeroline: false, + ticklen: 3, + tickfont: { size: 10, color: '#666' }, + automargin: true, + }; +} + export function plotColorbar(label) { return { title: { text: label, side: 'right', font: { size: 11 } }, diff --git a/js/panes/hparams/hparamsUtils.js b/js/panes/hparams/hparamsUtils.js index 70c982434..5d4d26494 100644 --- a/js/panes/hparams/hparamsUtils.js +++ b/js/panes/hparams/hparamsUtils.js @@ -440,3 +440,86 @@ export function buildComparison(records, columns) { }); return sections; } + +/** + * Turn the raw observation list a comparison carries into per-run, + * per-metric series. + * + * A series plots against real steps only when every one of its + * observations has one; logging without a step is the SDK default, and + * mixing real steps with fallback ordinals on a single axis would place + * points at positions the data never claimed. + * + * Values arrive as null where the server encoded NaN or an infinity. + * They stay in the series as null so the line breaks at the right place + * rather than closing over the gap or shifting later ordinals. + */ +export function buildMetricSeries(experiments) { + const runs = []; + const metricKeys = new Set(); + (experiments || []).forEach((exp) => { + if (!exp || typeof exp !== 'object') return; + if (typeof exp.env_id !== 'string') return; + const raw = new Map(); + (exp.metrics || []).forEach((metric) => { + if (!metric || typeof metric.key !== 'string') return; + if (!raw.has(metric.key)) raw.set(metric.key, []); + raw.get(metric.key).push(metric); + metricKeys.add(metric.key); + }); + const series = {}; + raw.forEach((observations, key) => { + const usesIndex = !observations.every((obs) => isNumeric(obs.step)); + const x = []; + const y = []; + if (usesIndex) { + observations.forEach((obs, index) => { + x.push(index); + y.push(isNumeric(obs.value) ? obs.value : null); + }); + } else { + /* Later arrivals win a repeated step, matching how the backend + resolves a run's latest value for the table. */ + const byStep = new Map(); + observations.forEach((obs) => { + byStep.set(obs.step, isNumeric(obs.value) ? obs.value : null); + }); + Array.from(byStep.keys()) + .sort((a, b) => a - b) + .forEach((step) => { + x.push(step); + y.push(byStep.get(step)); + }); + } + series[key] = { x, y, usesIndex }; + }); + runs.push({ + env_id: exp.env_id, + label: runLabel(exp), + series, + }); + }); + return { runs, metricKeys: Array.from(metricKeys).sort() }; +} + +export function selectMetricSeries(runs, metricKey, colorIndex) { + const plotted = []; + const missing = []; + (runs || []).forEach((run) => { + const series = metricKey && run.series ? run.series[metricKey] : null; + if (!series || !series.y.some((value) => isNumeric(value))) { + missing.push(run.label); + return; + } + const index = colorIndex ? colorIndex.get(run.env_id) : undefined; + plotted.push({ + env_id: run.env_id, + label: run.label, + x: series.x, + y: series.y, + usesIndex: series.usesIndex, + colorIndex: index === undefined ? plotted.length : index, + }); + }); + return { plotted, missing }; +} diff --git a/js/panes/hparams/useExperimentMetrics.js b/js/panes/hparams/useExperimentMetrics.js new file mode 100644 index 000000000..45d43feef --- /dev/null +++ b/js/panes/hparams/useExperimentMetrics.js @@ -0,0 +1,102 @@ +/** + * Copyright 2017-present, The Visdom Authors + * All rights reserved. + * + * This source code is licensed under the license found in the + * LICENSE file in the root directory of this source tree. + * + */ + +import { useCallback, useEffect, useMemo, useState } from 'react'; + +import { fetchExperimentComparison } from '../../api/experimentsApi'; +import { buildMetricSeries } from './hparamsUtils'; + +const NO_EXPERIMENTS = []; + +/** + * Load per-step metric history for a set of runs. + * + * History is not part of the window content, which keeps only each + * metric's latest value, so it has to be read from the server. The cache + * is keyed per run rather than per selection so that ticking one more + * checkbox fetches one run instead of re-downloading the whole set. + */ +export default function useExperimentMetrics(records, cacheRef) { + const [nonce, setNonce] = useState(0); + const [state, setState] = useState({ + status: 'idle', + error: null, + experiments: NO_EXPERIMENTS, + }); + + const envIds = useMemo( + () => (records || []).map((r) => r.env_id).filter((id) => !!id), + [records] + ); + + const refresh = useCallback(() => { + const cache = cacheRef.current; + if (cache) envIds.forEach((id) => cache.delete(id)); + setNonce((n) => n + 1); + }, [cacheRef, envIds]); + + useEffect(() => { + const cache = cacheRef.current; + if (envIds.length === 0) { + setState({ status: 'idle', error: null, experiments: NO_EXPERIMENTS }); + return undefined; + } + + /* Read back in selection order so the traces and the legend follow + the table rather than whatever order the server replied in. */ + const readCache = () => envIds.map((id) => cache.get(id)).filter(Boolean); + const wanted = envIds.filter((id) => !cache.has(id)); + if (wanted.length === 0) { + setState({ status: 'ready', error: null, experiments: readCache() }); + return undefined; + } + + /* abort() alone still leaves an already-resolved json() microtask + able to set state on an unmounted view, so guard with a flag too. */ + let cancelled = false; + const controller = new AbortController(); + setState((prev) => ({ ...prev, status: 'loading', error: null })); + + fetchExperimentComparison(wanted, controller.signal) + .then((reply) => { + if (cancelled) return; + const loaded = (reply && reply.experiments) || []; + loaded.forEach((exp) => { + if (exp && typeof exp.env_id === 'string') cache.set(exp.env_id, exp); + }); + setState({ status: 'ready', error: null, experiments: readCache() }); + }) + .catch((err) => { + if (cancelled || (err && err.name === 'AbortError')) return; + setState({ + status: 'error', + error: (err && err.message) || 'Could not load metric history.', + experiments: NO_EXPERIMENTS, + }); + }); + + return () => { + cancelled = true; + controller.abort(); + }; + }, [envIds, nonce, cacheRef]); + + const parsed = useMemo( + () => buildMetricSeries(state.experiments), + [state.experiments] + ); + + return { + status: state.status, + error: state.error, + runs: parsed.runs, + metricKeys: parsed.metricKeys, + refresh, + }; +} diff --git a/py/visdom/static/css/hparams.css b/py/visdom/static/css/hparams.css index 722276aa5..1e2129803 100644 --- a/py/visdom/static/css/hparams.css +++ b/py/visdom/static/css/hparams.css @@ -943,3 +943,37 @@ .hparams-selection-banner b { font-weight: 600; } + +/* ---- HParamsMetrics (C2) ---- */ + +/* min-height/min-width 0 lets the plot shrink with the pane; without them the + resize observer only ever grows it. */ +.hparams-metrics-wrap { + position: relative; + flex: 1 1 auto; + min-height: 0; + min-width: 0; + display: flex; + flex-direction: column; +} + +.hparams-metrics-plot { + flex: 1 1 auto; + min-height: 0; + min-width: 0; +} + +.hparams-metric-select { + padding: 3px 8px; + font-family: "Open Sans", sans-serif; + font-size: 12px; + color: #333; + border: 1px solid #dedede; + border-radius: 3px; + background-color: #fff; +} + +.hparams-metric-select:focus { + outline: none; + border-color: #3b5998; +} From dda79eff39cbad1ebdfa792858e7cff33351da80 Mon Sep 17 00:00:00 2001 From: Manik-Khajuria-5 Date: Sat, 25 Jul 2026 17:46:35 +0530 Subject: [PATCH 42/48] feat(hparams): export the runs on screen and finish the keyboard pass Closes out the dashboard: the numbers were only readable inside the pane, and the view tabs claimed the tab role without honouring its keyboard contract. - CSV and JSON export of whatever the pane is currently showing: the selected runs when a selection is active, otherwise the filtered ones, so what lands on disk is what is on screen - CSV headers carry the group (param.lr, metric.acc) because a param and a metric may share a name; missing values and NaN both become empty cells rather than the strings null or NaN, and commas, quotes, newlines and list params are escaped - JSON keeps the window-content shape, so an export can be read back by anything that already reads a hparams window - the tabs now implement the roving tabindex the role implies: arrows move and wrap, Home and End jump to the ends, focus follows selection, and only the active tab is in the tab order - the tab panel is linked to its tab both ways, so a screen reader can say which view it is announcing - Escape closes the filter sidebar and returns focus to the button that opened it, instead of stranding focus in a hidden subtree - a polite live region announces how many runs survive the filters and how many are selected; the table gets an off-screen caption Comments dropped from the metrics files added in the previous commit. --- js/api/experimentsApi.js | 15 ---- js/api/serverPath.js | 7 -- js/panes/HParamsPane.js | 104 ++++++++++++++++++++++- js/panes/hparams/HParamsFilters.js | 24 +++++- js/panes/hparams/HParamsMetrics.js | 2 - js/panes/hparams/HParamsTable.js | 4 + js/panes/hparams/hparamsExport.js | 78 +++++++++++++++++ js/panes/hparams/hparamsPlot.js | 4 - js/panes/hparams/hparamsUtils.js | 15 ---- js/panes/hparams/useExperimentMetrics.js | 12 --- py/visdom/static/css/hparams.css | 53 +++++++++++- 11 files changed, 256 insertions(+), 62 deletions(-) create mode 100644 js/panes/hparams/hparamsExport.js diff --git a/js/api/experimentsApi.js b/js/api/experimentsApi.js index 1fa9363c4..0f4a043ff 100644 --- a/js/api/experimentsApi.js +++ b/js/api/experimentsApi.js @@ -1,16 +1,5 @@ import serverPath from './serverPath'; -/** - * Read the comparison payload for a set of runs. - * - * Deliberately window.fetch and not jQuery: ApiProvider installs a - * document-level ajaxError handler that navigates the whole page to - * error/500, so a 404 for a deleted run would destroy the dashboard. - * fetch keeps the failure local to the caller. - * - * The response carries params/metrics/tags diff sections alongside the - * raw experiments; only experiments[].metrics holds per-step history. - */ export function fetchExperimentComparison(envIds, signal) { const ids = (envIds || []).filter((id) => typeof id === 'string'); if (ids.length === 0) { @@ -25,14 +14,10 @@ export function fetchExperimentComparison(envIds, signal) { signal, }) .catch((err) => { - /* A dead server rejects with a bare "Failed to fetch", which reads - like a bug rather than a server that is not answering. */ if (err && err.name === 'AbortError') throw err; throw new Error('Could not reach the server.'); }) .then((res) => { - /* An empty 401 body from check_auth would make res.json() throw a - SyntaxError that reads like a parsing bug, so branch on ok first. */ if (!res.ok) { const reason = res.statusText || 'request failed'; throw new Error('Could not load metric history (' + reason + ').'); diff --git a/js/api/serverPath.js b/js/api/serverPath.js index 8a136e27c..b4af9865a 100644 --- a/js/api/serverPath.js +++ b/js/api/serverPath.js @@ -1,10 +1,3 @@ -/** - * Normalize window.location by removing specific path segments - * and ensuring the pathname ends with a '/'. - * - * The pathname already carries the server's base_url, so deriving the - * prefix from it keeps requests correct under -base_url deployments. - */ export default function serverPath() { var pathname = window.location.pathname; if (pathname.indexOf('/env/') > -1) { diff --git a/js/panes/HParamsPane.js b/js/panes/HParamsPane.js index 218bab171..cd413775a 100644 --- a/js/panes/HParamsPane.js +++ b/js/panes/HParamsPane.js @@ -10,6 +10,7 @@ import React, { useCallback, useMemo, useRef, useState } from 'react'; import HParamsCompare from './hparams/HParamsCompare'; +import { exportCsv, exportJson } from './hparams/hparamsExport'; import HParamsFilters from './hparams/HParamsFilters'; import HParamsMetrics from './hparams/HParamsMetrics'; import HParamsParallelCoords from './hparams/HParamsParallelCoords'; @@ -35,6 +36,21 @@ const VIEWS = [ const NO_RECORDS = []; +const TAB_KEYS = { + ArrowRight: 1, + ArrowLeft: -1, + ArrowDown: 1, + ArrowUp: -1, +}; + +function tabId(contentID, key) { + return 'hparams-tab-' + contentID + '-' + key; +} + +function panelId(contentID) { + return 'hparams-panel-' + contentID; +} + function readContent(content) { if (!content || typeof content !== 'object' || Array.isArray(content)) { return null; @@ -101,10 +117,52 @@ var HParamsPane = (props) => { [selectionActive, visibleRecords, tableSelected] ); const clearSelection = useCallback(() => setTableSelected(new Set()), []); - const closeFilters = useCallback(() => setFiltersOpen(false), []); + const filtersToggleRef = useRef(null); + const tablistRef = useRef(null); + const closeFilters = useCallback(() => { + setFiltersOpen(false); + if (filtersToggleRef.current) filtersToggleRef.current.focus(); + }, []); + + const handleTabKeyDown = useCallback( + (e) => { + let next = null; + if (e.key === 'Home') next = 0; + else if (e.key === 'End') next = VIEWS.length - 1; + else if (TAB_KEYS[e.key]) { + const at = VIEWS.findIndex((v) => v.key === view); + next = (at + TAB_KEYS[e.key] + VIEWS.length) % VIEWS.length; + } + if (next === null) return; + e.preventDefault(); + setView(VIEWS[next].key); + const buttons = tablistRef.current + ? tablistRef.current.querySelectorAll('.hparams-viewtab') + : null; + if (buttons && buttons[next]) buttons[next].focus(); + }, + [view] + ); const selectedRecords = selectionActive ? selectedVisible : NO_RECORDS; + const exportRecords = selectionActive ? selectedVisible : visibleRecords; + const exportScope = selectionActive ? 'selected' : 'shown'; + const handleExportCsv = useCallback(() => { + exportCsv(exportRecords, columns, 'visdom_hparams.csv'); + }, [exportRecords, columns]); + const handleExportJson = useCallback(() => { + exportJson( + exportRecords, + { + paramKeys: data ? data.paramKeys : [], + metricKeys: data ? data.metricKeys : [], + tagKeys: data ? data.tagKeys : [], + }, + 'visdom_hparams.json' + ); + }, [exportRecords, data]); + const handleDownload = useCallback(() => { let blob = new Blob([JSON.stringify(content)], { type: 'application/json', @@ -166,18 +224,27 @@ var HParamsPane = (props) => { ) : null}
-
+
{VIEWS.map((v) => ( @@ -193,6 +260,7 @@ var HParamsPane = (props) => { /> + + export + + +
-
+

+ {visibleRecords.length} of {records.length} runs shown + {selectionActive ? ', ' + tableSelected.size + ' selected' : ''} +

+
{filtersOpen ? ( { + const el = rootRef.current; + if (!el) return undefined; + const onKeyDown = (e) => { + if (e.key !== 'Escape') return; + e.stopPropagation(); + onClose(); + }; + el.addEventListener('keydown', onKeyDown); + return () => el.removeEventListener('keydown', onKeyDown); + }, [onClose]); + return ( -
+
Filters