chore: sync from NMP MR !5254 - e2e tests (#55)

# Summary
Sync CI tests from NMP

> Files changed in MR but NOT synced (outside src/ and tests/, may need
manual review):
Makefile
README.md
images/ci_pipeline.png

## Pre-Review Checklist

<!-- These checks should be completed before a PR is reviewed, -->
<!-- but you can submit a draft early to indicate that the issue is
being worked on. -->

Ensure that the following pass:

- [x] `make format && make lint` or via prek validation.
- [ ] `make test` passes locally
- [ ] `make e2e` passes locally
- [ ] `make test-ci-container` passes locally (recommended)

## Pre-Merge Checklist

<!-- These checks need to be completed before a PR is merged, -->
<!-- but as PRs often change significantly during review, -->
<!-- it's OK for them to be incomplete when review is first requested.
-->

- [x] New or updated tests for any fix or new behavior
- [ ] Updated documentation for new features and behaviors, including
docstrings for API docs.

## Other Notes

<!-- Please add the issue number that should be closed when this PR is
merged. -->
- Closes #<issue>

---------

Signed-off-by: aagonzales <aagonzales@nvidia.com>
Co-authored-by: Nina Xu <ninaxu@cw-dfw-cs-001-vscode-01.cm.cluster>
Co-authored-by: aagonzales <aagonzales@nvidia.com>

Author: 3 people

.agent/skills/diagnose-failures/SKILL.md

@@ -59,6 +59,7 @@ Map GitHub Actions job names to local commands:
 | `Lint` | `make lint` (runs ruff + ty + copyright) |
 | `Typecheck` | `bash tools/lint/run-ty-check.sh` |
 | `Unit Tests` | `make test-ci` or `make test` |
+| `Config-Dataset e2e` | `make test-nss-<config>-<dataset>-ci` |
 
 Fetch CI logs:
 ```bash

.agent/skills/python-testing-patterns/SKILL.md

@@ -14,7 +14,6 @@ compatibility: "Python 3.11+, pytest with xdist and asyncio_mode=auto."
 # Make targets (preferred)
 make test                 # Unit tests, excluding slow
 make test-slow            # All tests including slow, excluding e2e
-make test-sdk-related     # Config/SDK/CLI/API tests only
 make test-ci-container    # CI tests in a Linux container
 
 # Direct pytest (via uv)

.claude/commands/gpu-test.md

@@ -9,4 +9,5 @@ Run GPU-dependent tests. Requires CUDA.
 * All e2e tests: `make test-e2e`
 * Default e2e only: `make test-e2e-default`
 * DP e2e only: `make test-e2e-dp`
+* Config-dataset combo: `make test-nss-tinyllama_unsloth-clinc_oos-ci` (12 combos total, see `tests/TESTING.md`)
 * Note: e2e tests run with `-n 0` (single process)

.claude/commands/test-sdk-related.md

@@ -10,6 +10,7 @@ README.md
 SECURITY.md
 THIRD_PARTY.md
 design.md
+tests/stub_datasets/LICENSES.md
 
 # AI / IDE / CI configuration directories
 .agent/

.copyrightignore

@@ -106,13 +106,21 @@ make test
 # All tests including slow (excludes e2e)
 make test-slow
 
-# SDK-related tests only
-make test-sdk-related
+# GPU integration (requires CUDA)
+make test-gpu-integration
+
+# End-to-end tests (requires CUDA)
+make test-e2e
+
+# Specific config-dataset e2e combo (12 total)
+make test-nss-tinyllama_unsloth-clinc_oos-ci
 
 # CI tests in a Linux container
 make test-ci-container
 ```
 
+See [tests/TESTING.md](tests/TESTING.md) for the full test matrix and details.
+
 Test runner: `uv run --frozen pytest -n auto --dist loadscope -vv`
 
 Markers (defined in `pytest.ini`):

AGENTS.md

@@ -265,15 +265,15 @@ make test
 # Run all tests including slow tests (excludes e2e)
 make test-slow
 
-# Run SDK-related tests (config, sdk, cli, api)
-make test-sdk-related
-
 # Run GPU integration tests (requires CUDA)
 make test-gpu-integration
 
 # Run end-to-end tests (requires CUDA)
 make test-e2e
 
+# Run a specific config-dataset e2e combo (12 total, see tests/TESTING.md)
+make test-nss-tinyllama_unsloth-clinc_oos-ci
+
 # Run CI tests locally in a Linux container (Docker/Podman)
 make test-ci-container
 

CONTRIBUTING.md

@@ -28,6 +28,7 @@ PYTORCH_DEPS ?= cpu
 PYTEST_ADDOPTS := -n auto --dist loadscope -vv
 PYTEST_CI_OPTS := --cov --cov-report json:coverage.json
 PYTEST_CMD := uv run --frozen pytest $(PYTEST_ADDOPTS)
+PYTEST_NO_XDIST_CMD := $(PYTEST_CMD) -n 0
 
 # Display platform info
 $(info local system architecture: $(PLATFORM)/$(ARCH))
@@ -153,14 +154,6 @@ test-slow: ## Run all tests including slow tests (excludes e2e)
 	pushd $(NSS_ROOT_PATH) && \
 	$(PYTEST_CMD) $(NSS_ROOT_PATH)/tests -m "not e2e" --run-slow
 
-.PHONY: test-sdk-related
-test-sdk-related: ## Run SDK-related tests (config, sdk, cli, api)
-	$(PYTEST_CMD) \
-		$(NSS_ROOT_PATH)/tests/config \
-		$(NSS_ROOT_PATH)/tests/sdk \
-		$(NSS_ROOT_PATH)/tests/cli \
-		$(NSS_ROOT_PATH)/tests/api
-
 .PHONY: test-ci
 test-ci: ## Run CI unit tests excluding slow and GPU tests
 	pushd $(NSS_ROOT_PATH) && \
@@ -368,3 +361,28 @@ synchronize-from-nmp: synchronize-py-files-from-nmp synchronize-metafiles-from-n
 		echo "NMP_REPO_PATH '$(NMP_REPO_PATH)' is not a valid directory."; \
 		exit 1; \
 	fi
+
+
+# ============================================================
+# Config-Dataset Combination Tests (12 total)
+# ============================================================
+# Generated targets: test-nss-{CONFIG}-{DATASET}-ci
+#   CONFIGS : tinyllama_unsloth tinyllama_dp smollm3_unsloth smollm3_dp mistral_nodp mistral_dp
+#   DATASETS: clinc_oos dow_jones_index
+# Example usage:
+#   make test-nss-tinyllama_unsloth-clinc_oos-ci
+#   make test-nss-tinyllama_dp-dow_jones_index-ci
+
+NSS_CONFIGS  := tinyllama_unsloth tinyllama_dp smollm3_unsloth smollm3_dp mistral_nodp mistral_dp
+NSS_DATASETS := clinc_oos dow_jones_index
+
+define nss_combo_test
+test-nss-$(1)-$(2)-ci: ## Run pytest test for $(shell echo $(1) | tr '_' '-') config with $(shell echo $(2) | tr '_' '-') dataset
+	$(MAKE) bootstrap-nss cu128
+	$(PYTEST_NO_XDIST_CMD) -vv $(PYTEST_CI_OPTS) $(NSS_ROOT_PATH)/tests/e2e/test_dataset_config.py -k "test_$(2)_dataset[$(subst _,-,$(1))]"
+endef
+
+$(foreach config,$(NSS_CONFIGS),\
+  $(foreach dataset,$(NSS_DATASETS),\
+    $(eval $(call nss_combo_test,$(config),$(dataset)))))
+

Makefile

@@ -380,19 +380,21 @@ make test
 # Run all tests including slow tests (excludes e2e)
 make test-slow
 
-# Run SDK-related tests (config, sdk, cli, api)
-make test-sdk-related
-
 # Run GPU integration tests (requires CUDA)
 make test-gpu-integration
 
 # Run end-to-end tests (requires CUDA)
 make test-e2e
 
+# Run a specific config-dataset e2e combo (12 total, see tests/TESTING.md)
+make test-nss-tinyllama_unsloth-clinc_oos-ci
+
 # Run specific test files directly
 uv run pytest tests/cli/test_run.py
 ```
 
+See [tests/TESTING.md](tests/TESTING.md) for the full test matrix and usage.
+
 ### Container-Based Testing
 
 You can run the CI test suite locally in a Linux container using Docker or Podman:

README.md

@@ -363,6 +363,10 @@ def generate(self) -> SafeSynthesizer:
         if self._total_start is None:
             self._total_start = time.monotonic()
 
+        # Clean up trainer model if it exists (only present when train->generate in same session)
+        if hasattr(self, "trainer") and self.trainer is not None:
+            self.trainer.delete_trainable_model()
+
         # Select backend based on time_series configuration
         if self._nss_config.time_series and self._nss_config.time_series.is_timeseries:
             self.generator = TimeseriesBackend(