Add training e2e eval flows

Author: ShubyM

Makefile

@@ -13,13 +13,31 @@ HOST           ?= 127.0.0.1
 PORT           ?= 9003
 # The fully qualified base URL used by local CLI tools and clients
 BASE_URL       ?= http://$(HOST):$(PORT)
-TEST_PYTHONPATH ?= examples/sft/pig-latin
+UNIT_TESTS ?= tests.test_gateway_paths tests.test_snapshot_agent tests.test_trainer_optimizer_correctness tests.test_worker_launch_processor
+# Only forward BASE_URL to e2e when the user supplied it. The Makefile default
+# is for local CLI usage; e2e should start its own backend by default.
+TRAINING_TEST_BASE_URL ?= $(if $(filter environment command line,$(origin BASE_URL)),$(BASE_URL),)
+TRAINING_TEST_EXTRA ?= gpu
+TRAINING_TEST_ARGS ?=
+PIGLATIN_TEST_PYTHONPATH ?= examples/sft/pig-latin
+
+# CUDA_VISIBLE_DEVICES can be provided either as an environment variable or as a
+# Make variable, and is inherited by the backend/eval subprocesses.
+ifneq ($(origin CUDA_VISIBLE_DEVICES),undefined)
+  export CUDA_VISIBLE_DEVICES
+endif
 
 help:
 	@echo "make server                              # $(BASE_MODEL), SAMPLING_BACKEND=$(SAMPLING_BACKEND), port $(PORT)"
 	@echo "make server BASE_MODEL=google/gemma-4-e2b SAMPLING_BACKEND=vllm"
 	@echo "VLLM_ARCHITECTURE_OVERRIDE=Gemma4ForCausalLM make vllm BASE_MODEL=google/gemma-4-e2b"
-	@echo "make test | lint | fmt"
+	@echo "make test                               # fast unit tests"
+	@echo "make test e2e lora-sft|lora-rl|fft-sft  # starts a local backend, then runs existing training/eval examples"
+	@echo "make test e2e lora-sft BASE_URL=http://host:9003"
+	@echo "CUDA_VISIBLE_DEVICES=0 make test e2e fft-sft"
+	@echo "make test e2e fft-sft TRAINING_TEST_ARGS='gsm8k.steps=10'"
+	@echo "make test piglatin                      # pig-latin example end-to-end tests"
+	@echo "make lint | fmt"
 
 # ---------------------------------------------------------------------------
 # Server
@@ -42,14 +60,40 @@ ifeq (cli,$(firstword $(MAKECMDGOALS)))
   $(eval $(CLI_ARGS):;@:)
 endif
 
+ifeq (test,$(firstword $(MAKECMDGOALS)))
+  TEST_MODE := $(word 2,$(MAKECMDGOALS))
+  TEST_SCENARIO := $(word 3,$(MAKECMDGOALS))
+  TEST_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+  ifneq ($(TEST_ARGS),)
+    $(eval $(TEST_ARGS):;@:)
+  endif
+endif
+
 cli:
 	@cd dev/tools && BASE_URL="$(BASE_URL)" uv run python cli.py $(CLI_ARGS)
 
 # ---------------------------------------------------------------------------
 # Dev
 # ---------------------------------------------------------------------------
 test:
-	PYTHONPATH="$(TEST_PYTHONPATH)" uv --project examples run python -m unittest discover -s tests
+	@mode="$(TEST_MODE)"; \
+	scenario="$(TEST_SCENARIO)"; \
+	if [ -z "$$mode" ] || [ "$$mode" = "unit" ]; then \
+	  uv run --frozen --extra cpu python -m unittest $(UNIT_TESTS); \
+	elif [ "$$mode" = "e2e" ]; then \
+	  if [ -z "$$scenario" ]; then \
+	    echo "Missing e2e scenario. Expected lora-sft, lora-rl, or fft-sft."; \
+	    exit 2; \
+	  fi; \
+	  set -- "scenario=$$scenario" "backend.uv_extra=$(TRAINING_TEST_EXTRA)"; \
+	  if [ -n "$(TRAINING_TEST_BASE_URL)" ]; then set -- "$$@" "backend.base_url=$(TRAINING_TEST_BASE_URL)"; fi; \
+	  uv run --extra "$(TRAINING_TEST_EXTRA)" python scripts/run_training_e2e.py "$$@" $(TRAINING_TEST_ARGS); \
+	elif [ "$$mode" = "piglatin" ]; then \
+	  PYTHONPATH="$(PIGLATIN_TEST_PYTHONPATH)" uv --project examples run python -m unittest discover -s tests; \
+	else \
+	  echo "Unknown test mode '$$mode'. Expected unit, e2e, or piglatin."; \
+	  exit 2; \
+	fi
 
 lint:
 	uvx ruff check .

examples/sft/gsm8k/vllm_eval.py

@@ -22,12 +22,19 @@ def main() -> None:
   parser = argparse.ArgumentParser()
   parser.add_argument("--path", required=True)
   parser.add_argument("--data", default="gsm8k_test.json")
+  parser.add_argument("--gpu-memory-utilization", type=float, default=0.85)
   args = parser.parse_args()
 
   with open(args.data) as f:
     data = json.load(f)
 
-  llm = LLM(model=args.path, dtype="bfloat16", gpu_memory_utilization=0.85, max_model_len=1024, enforce_eager=True)
+  llm = LLM(
+    model=args.path,
+    dtype="bfloat16",
+    gpu_memory_utilization=args.gpu_memory_utilization,
+    max_model_len=1024,
+    enforce_eager=True,
+  )
   sampling_params = SamplingParams(temperature=0.0, max_tokens=256, stop=["\nQuestion:"])
   start = time.time()
   outputs = llm.generate([datum["prompt"] for datum in data], sampling_params)

pyproject.toml

@@ -9,6 +9,7 @@ description = "Open-RL server and training runtime."
 readme = "README.md"
 requires-python = ">=3.12, <3.13"
 dependencies = [
+    "chz>=0.4.0",
     "fastapi",
     "opentelemetry-api",
     "opentelemetry-sdk",