From 3168fbd720083cd6a08eca0e643ad2b97d8c1ee9 Mon Sep 17 00:00:00 2001 From: Dmitrii Branitskii Date: Sun, 30 Aug 2026 18:27:37 +0100 Subject: [PATCH] docs: add hyperparameter sweep recipes Grid, random, and Optuna recipes built on init(group=, config=), plus dashboard viewing, SQL querying of results, and delegating sweeps to coding agents. Includes a runnable example. --- docs/source/_toctree.yml | 2 + docs/source/sweeps.md | 126 ++++++++++++++++++++++++++++++++++++++ examples/sweep-recipes.py | 69 +++++++++++++++++++++ 3 files changed, 197 insertions(+) create mode 100644 docs/source/sweeps.md create mode 100644 examples/sweep-recipes.py diff --git a/docs/source/_toctree.yml b/docs/source/_toctree.yml index fde4227b5..2602eba80 100644 --- a/docs/source/_toctree.yml +++ b/docs/source/_toctree.yml @@ -11,6 +11,8 @@ - sections: - local: track title: Track + - local: sweeps + title: Hyperparameter Sweeps - local: traces title: Traces - local: artifacts diff --git a/docs/source/sweeps.md b/docs/source/sweeps.md new file mode 100644 index 000000000..c5d998acb --- /dev/null +++ b/docs/source/sweeps.md @@ -0,0 +1,126 @@ +# Hyperparameter Sweeps + +Trackio intentionally does not ship a first-class sweep API. A sweep is a plain Python loop: one run per configuration, grouped so that the dashboard and queries can treat the sweep as a unit. This keeps sweeps fully under your control — any search strategy you can write (or ask a coding agent to write) works, with nothing new to learn. + +This page collects recipes for the common cases. + +## The Pattern + +Every recipe below is a variation of the same three ingredients: + +- **One run per configuration** — call [`init`] once per trial. +- **`group`** — set it to a sweep identifier so all trials of one sweep stay together (see [Grouping runs](./track#grouping-runs)). +- **`config`** — record the hyperparameters, so results are comparable later. + +```python +import trackio + +def train(config: dict, group: str, name: str) -> float: + trackio.init(project="my_project", name=name, group=group, config=config) + + final_loss = None + for epoch in range(10): + final_loss = ... # your actual training step + trackio.log({"loss": final_loss, "epoch": epoch}) + + trackio.finish() + return final_loss +``` + +## Grid Search + +Enumerate the full cartesian product of the search space with `itertools.product`: + +```python +import itertools + +search_space = { + "lr": [1e-2, 1e-3, 1e-4], + "batch_size": [32, 128], +} + +best = None +for values in itertools.product(*search_space.values()): + config = dict(zip(search_space.keys(), values)) + name = "grid-" + "-".join(f"{k}={v}" for k, v in config.items()) + loss = train(config, group="grid-search", name=name) + if best is None or loss < best[1]: + best = (config, loss) + +print(f"best config: {best[0]} (loss {best[1]:.3f})") +``` + +## Random Search + +Sample configurations instead of enumerating them — usually a better use of the same budget when some hyperparameters matter much more than others. Seed the generator so the sweep is reproducible: + +```python +import random + +N_TRIALS = 20 +rng = random.Random(42) + +best = None +for trial in range(N_TRIALS): + config = { + "lr": 10 ** rng.uniform(-5, -1), + "batch_size": rng.choice([16, 32, 64, 128, 256]), + } + loss = train(config, group="random-search", name=f"random-{trial}") + if best is None or loss < best[1]: + best = (config, loss) +``` + +## Bayesian Optimization with Optuna + +For smarter search strategies, use a dedicated library and let Trackio do the tracking. With [Optuna](https://optuna.org) (`pip install optuna`), each trial becomes one Trackio run: + +```python +import optuna + +def objective(trial): + config = { + "lr": trial.suggest_float("lr", 1e-5, 1e-1, log=True), + "batch_size": trial.suggest_categorical("batch_size", [16, 32, 64, 128]), + } + return train(config, group="optuna-sweep", name=f"optuna-{trial.number}") + +study = optuna.create_study(direction="minimize") +study.optimize(objective, n_trials=20) + +print("best:", study.best_params, study.best_value) +``` + +## Viewing Sweep Results + +In the dashboard (`trackio show`), runs that share a `group` are grouped together in the sidebar, so a sweep can be toggled on and off as a unit and its runs compared on the same charts. Since every trial records its hyperparameters in `config`, sweeps are also a natural fit for comparing run configurations side by side. + +## Querying the Best Configuration + +Because everything is in SQLite, "which configuration won?" is a query. The run's group is stored inside the config JSON under the `_Group` key: + +```bash +trackio query project --project "my_project" --sql " +SELECT m.run_name, + json_extract(m.metrics, '$.loss') AS final_loss +FROM metrics m +JOIN configs c ON c.run_name = m.run_name +WHERE json_extract(c.config, '$._Group') = 'grid-search' + AND m.id = (SELECT MAX(id) FROM metrics WHERE run_name = m.run_name) +ORDER BY final_loss ASC +LIMIT 5" +``` + +Add `--json` for machine-readable output. See [Storage Schema and Direct Queries](./storage_schema) for the full schema. + +## Running Sweeps with a Coding Agent + +Sweeps are a natural task to delegate: the loop is mechanical, and Trackio gives agents programmatic access to results (CLI with `--json`, the Python API, and direct SQL). A prompt as simple as: + +> Run a random search over lr (log-uniform, 1e-5 to 1e-1) and batch_size (16–256) for my training script. Create one Trackio run per trial in the group "sweep-aug30", 20 trials, then query the results and report the best three configurations. + +is enough for an agent to write the loop, run it, and read back the results. See [Running ML Experiments with Agents](./ml_agents) for how to structure the full feedback loop, including alerts and monitoring. + +## Runnable Example + +A complete, self-contained script with the grid and random recipes lives at [`examples/sweep-recipes.py`](https://github.com/gradio-app/trackio/blob/main/examples/sweep-recipes.py). diff --git a/examples/sweep-recipes.py b/examples/sweep-recipes.py new file mode 100644 index 000000000..55e441273 --- /dev/null +++ b/examples/sweep-recipes.py @@ -0,0 +1,69 @@ +"""Hyperparameter sweep recipes using plain Python loops. + +Trackio intentionally has no first-class sweep API: a sweep is just a loop +that creates one run per configuration. This example shows a grid search and +a random search, grouped so they are easy to compare in the dashboard. + +Run with: python examples/sweep-recipes.py +""" + +import itertools +import math +import random + +import trackio + +PROJECT_ID = random.randint(100000, 999999) +PROJECT = f"fake-sweep-{PROJECT_ID}" +EPOCHS = 10 + + +def train(config: dict, group: str, name: str) -> float: + """Fake training loop: logs a loss curve shaped by the config.""" + trackio.init(project=PROJECT, name=name, group=group, config=config) + + # Pretend lower lr + larger batch converge better, with noise. + quality = 1.0 / (1 + abs(math.log10(config["lr"]) + 3)) + config["batch_size"] / 512 + final_loss = None + for epoch in range(EPOCHS): + progress = (epoch + 1) / EPOCHS + loss = 2.5 * math.exp(-3 * progress * quality) + random.gauss(0, 0.05) + final_loss = max(0.05, loss) + trackio.log({"loss": final_loss, "epoch": epoch}) + + trackio.finish() + return final_loss + + +# --- Recipe 1: grid search ------------------------------------------------- +search_space = { + "lr": [1e-2, 1e-3, 1e-4], + "batch_size": [32, 128], +} + +best = None +for values in itertools.product(*search_space.values()): + config = dict(zip(search_space.keys(), values)) + name = "grid-" + "-".join(f"{k}={v}" for k, v in config.items()) + loss = train(config, group="grid-search", name=name) + if best is None or loss < best[1]: + best = (config, loss) + +print(f"[grid] best config: {best[0]} (loss {best[1]:.3f})") + +# --- Recipe 2: random search ----------------------------------------------- +N_TRIALS = 6 +rng = random.Random(42) # seed so the sweep is reproducible + +best = None +for trial in range(N_TRIALS): + config = { + "lr": 10 ** rng.uniform(-5, -1), + "batch_size": rng.choice([16, 32, 64, 128, 256]), + } + loss = train(config, group="random-search", name=f"random-{trial}") + if best is None or loss < best[1]: + best = (config, loss) + +print(f"[random] best config: {best[0]} (loss {best[1]:.3f})") +print(f"\nView results: trackio show --project {PROJECT}")