update build scripts to automate oldest env local validation, bump tensorboard version

Author: speediedan

.github/copilot-instructions.md

@@ -97,6 +97,9 @@ Use the provided build script for automated setup:
 
 # Build from Lightning source
 ./scripts/build_fts_env.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --from-source="lightning:${HOME}/repos/lightning"
+
+# Build with oldest compatible dependencies (Python 3.10, mirrors CI oldest matrix)
+./scripts/build_fts_env.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_oldest --oldest
 ```
 
 **Venv Location Options:**
@@ -266,17 +269,65 @@ python -m pytest src/finetuning_scheduler tests -v
 
 ### Coverage Collection
 
+Use the `manage_standalone_processes.sh` harness with `--use-nohup` to run coverage collection in an isolated process. Output is written to `/tmp/gen_fts_coverage_<env>_<timestamp>.log`.
+
+**Monitoring progress:**
+
 ```bash
-# Generate coverage with rebuild
-./scripts/gen_fts_coverage.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest
+# Tail the most recent coverage log
+tail -f `ls -rt /tmp/gen_fts_coverage_fts_* | tail -1`
+```
+
+**Common coverage commands:**
+
+```bash
+# Generate coverage with rebuild (fts_latest with stable PyTorch)
+~/repos/finetuning-scheduler/scripts/manage_standalone_processes.sh --use-nohup \
+  ~/repos/finetuning-scheduler/scripts/gen_fts_coverage.sh \
+  --repo_home=${HOME}/repos/finetuning-scheduler \
+  --target_env_name=fts_latest \
+  --venv-dir=/mnt/cache/${USER}/.venvs
 
 # Generate coverage without rebuild
-./scripts/gen_fts_coverage.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --no_rebuild_base
+~/repos/finetuning-scheduler/scripts/manage_standalone_processes.sh --use-nohup \
+  ~/repos/finetuning-scheduler/scripts/gen_fts_coverage.sh \
+  --repo_home=${HOME}/repos/finetuning-scheduler \
+  --target_env_name=fts_latest \
+  --venv-dir=/mnt/cache/${USER}/.venvs \
+  --no_rebuild_base
 
 # Include experimental patch tests
-./scripts/gen_fts_coverage.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --include_experimental
+~/repos/finetuning-scheduler/scripts/manage_standalone_processes.sh --use-nohup \
+  ~/repos/finetuning-scheduler/scripts/gen_fts_coverage.sh \
+  --repo_home=${HOME}/repos/finetuning-scheduler \
+  --target_env_name=fts_latest \
+  --venv-dir=/mnt/cache/${USER}/.venvs \
+  --include_experimental
+
+# Generate coverage with oldest dependencies (Python 3.10, mirrors CI oldest matrix)
+~/repos/finetuning-scheduler/scripts/manage_standalone_processes.sh --use-nohup \
+  ~/repos/finetuning-scheduler/scripts/gen_fts_coverage.sh \
+  --repo_home=${HOME}/repos/finetuning-scheduler \
+  --target_env_name=fts_oldest \
+  --venv-dir=/mnt/cache/${USER}/.venvs \
+  --oldest
+
+# Generate coverage with oldest deps, skip special tests (faster CI-like run)
+~/repos/finetuning-scheduler/scripts/manage_standalone_processes.sh --use-nohup \
+  ~/repos/finetuning-scheduler/scripts/gen_fts_coverage.sh \
+  --repo_home=${HOME}/repos/finetuning-scheduler \
+  --target_env_name=fts_oldest \
+  --venv-dir=/mnt/cache/${USER}/.venvs \
+  --oldest \
+  --no-special
 ```
 
+**Flags:**
+
+- `--oldest`: Uses Python 3.10 and `requirements/ci/requirements-oldest.txt` (mirrors CI oldest matrix)
+- `--no-special`: Skips `special_tests.sh` standalone and experimental patch tests (faster iteration)
+- `--venv-dir`: Base directory for venvs (recommended: `/mnt/cache/${USER}/.venvs` for hardlink performance)
+
 ## Special Dependencies and Known Issues
 
 ### Lightning Dependency

pyproject.toml

@@ -60,7 +60,7 @@ examples = [
     "transformers>=4.26.0",
     "scikit-learn>=1.2.0",
     "sentencepiece>=0.2.0",
-    "tensorboardX>=2.5",
+    "tensorboardX>=2.6.1",  # 2.6.1+ regenerated protos for protobuf 4.x
     "tabulate>=0.8.9",
     "psutil>=5.9.0",
     # Include CLI deps
@@ -98,7 +98,7 @@ all = [
     "transformers>=4.26.0",
     "scikit-learn>=1.2.0",
     "sentencepiece>=0.2.0",
-    "tensorboardX>=2.5",
+    "tensorboardX>=2.6.1",  # 2.6.1+ regenerated protos for protobuf 4.x
     "tabulate>=0.8.9",
     "psutil>=5.9.0",
     # ipynb

requirements/ci/requirements-oldest.txt

@@ -391,6 +391,7 @@ packaging==23.2
     #   mlflow
     #   pytest
     #   pytorch-lightning
+    #   tensorboardx
     #   torchmetrics
     #   transformers
 pandas==2.3.3
@@ -563,7 +564,6 @@ six==1.17.0
     #   nbval
     #   python-dateutil
     #   querystring-parser
-    #   tensorboardx
 smmap==5.0.2
     # via gitdb
 sqlalchemy==2.0.44
@@ -578,7 +578,7 @@ tabulate==0.8.9
     # via
     #   finetuning-scheduler (pyproject.toml)
     #   databricks-cli
-tensorboardx==2.5
+tensorboardx==2.6.1
     # via finetuning-scheduler (pyproject.toml)
 terminado==0.18.1
     # via notebook

requirements/ci/requirements.txt

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile /home/speediedan/repos/finetuning-scheduler/pyproject.toml --extra all --group dev --group test --output-file /tmp/tmp.wCfF6eMKZC --no-strip-extras --resolution highest --universal --python-version 3.10
+#    uv pip compile /home/speediedan/repos/finetuning-scheduler/pyproject.toml --extra all --group dev --group test --output-file /tmp/tmp.flfkowf3vK --no-strip-extras --resolution highest --universal --python-version 3.10
 aiohappyeyeballs==2.6.1
     # via aiohttp
 aiohttp==3.13.2

scripts/build_fts_env.sh

@@ -6,6 +6,8 @@
 #   ./build_fts_env.sh --repo_home=~/repos/finetuning-scheduler --target_env_name=fts_latest
 # build latest with explicit venv directory (recommended for hardlink performance):
 #   ./build_fts_env.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --venv-dir=/mnt/cache/${USER}/.venvs
+# build oldest (CI oldest build simulation with Python 3.10 and oldest deps):
+#   ./build_fts_env.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_oldest --oldest
 # build release:
 #   ./build_fts_env.sh --repo_home=${HOME}/repos/fts-release --target_env_name=fts_release
 # build latest with torch test channel:
@@ -22,6 +24,7 @@ unset torch_test_channel
 unset uv_install_flags
 unset no_commit_pin
 unset venv_dir
+unset oldest
 declare -a from_source_specs=()
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -32,6 +35,7 @@ usage(){
 Usage: $0
    [ --repo_home input]
    [ --target_env_name input ]
+   [ --oldest ]                # Use oldest CI requirements (Python 3.10, requirements-oldest.txt)
    [ --torch_test_channel ]    # Use PyTorch test/RC channel
    [ --uv_install_flags "flags" ]
    [ --no_commit_pin ]
@@ -43,6 +47,8 @@ Usage: $0
     #   ./build_fts_env.sh --repo_home=\${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest
     # build latest with explicit venv directory (recommended for hardlink performance):
     #   ./build_fts_env.sh --repo_home=\${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --venv-dir=/mnt/cache/\${USER}/.venvs
+    # build oldest (CI oldest build simulation):
+    #   ./build_fts_env.sh --repo_home=\${HOME}/repos/finetuning-scheduler --target_env_name=fts_oldest --oldest --venv-dir=/mnt/cache/\${USER}/.venvs
     # build release:
     #   ./build_fts_env.sh --repo_home=\${HOME}/repos/fts-release --target_env_name=fts_release
     # build latest with torch test channel:
@@ -61,7 +67,7 @@ EOF
 exit 1
 }
 
-args=$(getopt -o '' --long repo_home:,target_env_name:,torch_test_channel,uv_install_flags:,no_commit_pin,venv-dir:,from-source:,help -- "$@")
+args=$(getopt -o '' --long repo_home:,target_env_name:,oldest,torch_test_channel,uv_install_flags:,no_commit_pin,venv-dir:,from-source:,help -- "$@")
 if [[ $? -gt 0 ]]; then
   usage
 fi
@@ -72,6 +78,7 @@ do
   case $1 in
     --repo_home)  repo_home=$2    ; shift 2  ;;
     --target_env_name)  target_env_name=$2  ; shift 2 ;;
+    --oldest)   oldest=1 ; shift  ;;
     --torch_test_channel)   torch_test_channel=1 ; shift  ;;
     --uv_install_flags)   uv_install_flags=$2 ; shift 2 ;;
     --no_commit_pin)   no_commit_pin=1 ; shift  ;;
@@ -143,16 +150,26 @@ log_torch_version(){
 }
 
 base_env_build(){
+    # Use Python 3.10 for oldest builds, 3.12 for latest
     local python_version="python3.12"
+    if [[ -n ${oldest} ]]; then
+        python_version="python3.10"
+        echo "Using Python 3.10 for oldest build"
+    fi
 
     clear_activate_env ${python_version}
 
-    # Check for torch nightly configuration
-    read_torch_nightly_config
+    # Check for torch nightly configuration (skip for oldest builds)
+    if [[ -z ${oldest} ]]; then
+        read_torch_nightly_config
+    fi
 
     # Handle PyTorch version selection (pre-install before FTS dependencies)
-    # Priority: torch nightly from config > torch test channel > stable (via --torch-backend in fts_install)
-    if [[ -n "${TORCH_NIGHTLY_VERSION}" ]]; then
+    # Priority: oldest (stable from lock) > torch nightly from config > torch test channel > stable (via --torch-backend in fts_install)
+    if [[ -n ${oldest} ]]; then
+        # For oldest builds, torch is installed from requirements-oldest.txt (stable version)
+        echo "Using torch stable from requirements-oldest.txt for oldest build"
+    elif [[ -n "${TORCH_NIGHTLY_VERSION}" ]]; then
         # Nightly version from torch-nightly.txt with specified CUDA backend
         local cuda_target="${TORCH_NIGHTLY_CUDA:-cu128}"  # Default to cu128 if not specified
         local torch_pkg="torch==${TORCH_NIGHTLY_VERSION}"
@@ -195,7 +212,13 @@ fts_install(){
     local req_file="${repo_home}/requirements/ci/requirements.txt"
     local torch_backend_flag=""
 
-    if [[ -n "${TORCH_NIGHTLY_VERSION}" || -n ${torch_test_channel} ]]; then
+    # For oldest builds, use requirements-oldest.txt
+    if [[ -n ${oldest} ]]; then
+        req_file="${repo_home}/requirements/ci/requirements-oldest.txt"
+        echo "Using oldest requirements file: ${req_file}"
+        # Oldest builds use torch stable from lock file, need --torch-backend=auto
+        torch_backend_flag="--torch-backend=auto"
+    elif [[ -n "${TORCH_NIGHTLY_VERSION}" || -n ${torch_test_channel} ]]; then
         # Torch already pre-installed (nightly or test channel)
         # When nightly: requirements.txt already has torch filtered during lock generation
         # When test channel: filter at runtime
@@ -227,12 +250,11 @@ fts_install(){
     uv pip install ${uv_install_flags} -r requirements/docs.txt ${torch_backend_flag}
     log_torch_version "after docs requirements install"
 
-    # Install pip for mypy and pre-commit (they use pip internally)
+    # Install pip for pre-commit (it uses pip internally)
     uv pip install pip
 
     # Development setup
-    rm -rf .mypy_cache
-    mypy --install-types --non-interactive
+    pyright -p pyproject.toml || echo "⚠ pyright check had issues, continuing..."
     pre-commit install
     git lfs install
 

scripts/gen_fts_coverage.sh

@@ -13,6 +13,8 @@ unset uv_install_flags
 unset no_commit_pin
 unset venv_dir
 unset dry_run
+unset oldest
+unset no_special
 declare -a from_source_specs=()
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -23,9 +25,11 @@ usage(){
 Usage: $0
    [ --repo_home input]
    [ --target_env_name input ]
+   [ --oldest ]                  # Use oldest CI requirements (Python 3.10, requirements-oldest.txt)
    [ --torch_dev_ver input ]
    [ --torch_test_channel ]
    [ --no_rebuild_base ]
+   [ --no-special ]              # Skip special tests (standalone/experimental), run only main test suite
    [ --include_experimental ]
    [ --uv_install_flags "flags" ]
    [ --no_commit_pin ]
@@ -35,26 +39,28 @@ Usage: $0
    [ --help ]
    Examples:
 	# generate fts_latest coverage without rebuilding the fts_latest base environment:
-	#   ./gen_fts_coverage.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --no_rebuild_base
+	#   ./gen_fts_coverage.sh --repo_home=\${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --no_rebuild_base
+	# generate oldest CI build coverage (matches CI oldest matrix):
+	#   ./gen_fts_coverage.sh --repo_home=\${HOME}/repos/finetuning-scheduler --target_env_name=fts_oldest --oldest --no-special --venv-dir=/mnt/cache/\${USER}/.venvs
 	# generate fts_latest coverage with a given torch_dev_version:
-	#   ./gen_fts_coverage.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --torch_dev_ver=dev20240201
+	#   ./gen_fts_coverage.sh --repo_home=\${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --torch_dev_ver=dev20240201
     # generate fts_latest coverage, rebuilding base fts_latest with PyTorch test channel and run tests that require experimental patches:
-    #   ./gen_fts_coverage.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --torch_test_channel --include_experimental
+    #   ./gen_fts_coverage.sh --repo_home=\${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --torch_test_channel --include_experimental
 	# generate fts_release coverage, rebuilding the base fts_release environment with PyTorch stable channel:
-	#   ./gen_fts_coverage.sh --repo_home=${HOME}/repos/fts-release --target_env_name=fts_release
+	#   ./gen_fts_coverage.sh --repo_home=\${HOME}/repos/fts-release --target_env_name=fts_release
 	# generate fts_release coverage, rebuilding the base fts_release environment with PyTorch test channel:
-	#   ./gen_fts_coverage.sh --repo_home=${HOME}/repos/fts-release --target_env_name=fts_release --torch_test_channel
+	#   ./gen_fts_coverage.sh --repo_home=\${HOME}/repos/fts-release --target_env_name=fts_release --torch_test_channel
 	# generate fts_latest coverage with explicit venv directory (recommended for hardlink performance):
-	#   ./gen_fts_coverage.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --venv-dir=/mnt/cache/\${USER}/.venvs
+	#   ./gen_fts_coverage.sh --repo_home=\${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --venv-dir=/mnt/cache/\${USER}/.venvs
 	# generate fts_release coverage without using CI commit pinning:
-	#   ./gen_fts_coverage.sh --repo_home=${HOME}/repos/fts-release --target_env_name=fts_release --no_commit_pin
+	#   ./gen_fts_coverage.sh --repo_home=\${HOME}/repos/fts-release --target_env_name=fts_release --no_commit_pin
 	# dry-run mode: setup environment and show what tests would run without executing them:
-	#   ./gen_fts_coverage.sh --repo_home=${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --torch_dev_ver=dev20240201 --dry-run
+	#   ./gen_fts_coverage.sh --repo_home=\${HOME}/repos/finetuning-scheduler --target_env_name=fts_latest --torch_dev_ver=dev20240201 --dry-run
 EOF
 exit 1
 }
 
-args=$(getopt -o '' --long repo_home:,target_env_name:,torch_dev_ver:,torch_test_channel,no_rebuild_base,include_experimental,uv_install_flags:,no_commit_pin,venv-dir:,from-source:,dry-run,help -- "$@")
+args=$(getopt -o '' --long repo_home:,target_env_name:,oldest,torch_dev_ver:,torch_test_channel,no_rebuild_base,no-special,include_experimental,uv_install_flags:,no_commit_pin,venv-dir:,from-source:,dry-run,help -- "$@")
 if [[ $? -gt 0 ]]; then
   usage
 fi
@@ -65,9 +71,11 @@ do
   case $1 in
     --repo_home)  repo_home=$2    ; shift 2  ;;
     --target_env_name)  target_env_name=$2  ; shift 2 ;;
+    --oldest)   oldest=1 ; shift  ;;
     --torch_dev_ver)   torch_dev_ver=$2   ; shift 2 ;;
     --torch_test_channel)   torch_test_channel=1 ; shift  ;;
     --no_rebuild_base)   no_rebuild_base=1 ; shift  ;;
+    --no-special)   no_special=1 ; shift  ;;
     --include_experimental)   include_experimental=1 ; shift  ;;
     --uv_install_flags)   uv_install_flags=$2 ; shift 2 ;;
     --no_commit_pin)   no_commit_pin=1 ; shift  ;;
@@ -121,6 +129,11 @@ env_rebuild(){
     # Build command arguments array
     local -a cmd_args=("${repo_home}/scripts/build_fts_env.sh" "--repo_home=${repo_home}" "--target_env_name=$1")
 
+    # Add oldest flag if specified
+    if [[ $oldest -eq 1 ]]; then
+        cmd_args+=("--oldest")
+    fi
+
     # Add uv_install_flags if specified
     if [[ -n "${uv_install_flags}" ]]; then
         cmd_args+=("--uv_install_flags=${uv_install_flags}")
@@ -145,7 +158,7 @@ env_rebuild(){
     log_msg "Executing build command: ${cmd_args[*]}"
 
     case $1 in
-        fts_latest)
+        fts_latest|fts_oldest)
             if [[ -n ${torch_dev_ver} ]]; then
                 cmd_args+=("--torch_dev_ver=${torch_dev_ver}")
 			elif [[ $torch_test_channel -eq 1 ]]; then
@@ -212,18 +225,23 @@ collect_env_coverage(){
     fi
 
     case $1 in
-	    fts_latest|fts_release|$all_supported_pattern)
+	    fts_latest|fts_oldest|fts_release|$all_supported_pattern)
             log_msg "Erasing previous coverage data"
 			python -m coverage erase
             log_msg "Running main test suite with coverage"
 			python -m coverage run --append --source src/finetuning_scheduler -m pytest src/finetuning_scheduler tests -v 2>&1 >> $coverage_session_log
-            log_msg "Running standalone tests (pattern: test_f)"
-            (./tests/special_tests.sh --mark_type=standalone --filter_pattern='test_f' --log_file=${coverage_session_log} 2>&1 >> ${temp_special_log}) > /dev/null
-            if [[ $include_experimental -eq 1 ]]; then
-                log_msg "Running tests that require experimental patches using $1"
-                (./tests/special_tests.sh --mark_type=exp_patch --filter_pattern='test_f' --log_file=${coverage_session_log} --experiment_patch_mask="1 0 0 1" 2>&1 >> ${temp_special_log}) > /dev/null
+            # Skip special tests if --no-special flag is set
+            if [[ $no_special -eq 1 ]]; then
+                log_msg "Skipping special tests (--no-special flag set)"
             else
-                log_msg "Skipping tests that require experimental patches."
+                log_msg "Running standalone tests (pattern: test_f)"
+                (./tests/special_tests.sh --mark_type=standalone --filter_pattern='test_f' --log_file=${coverage_session_log} 2>&1 >> ${temp_special_log}) > /dev/null
+                if [[ $include_experimental -eq 1 ]]; then
+                    log_msg "Running tests that require experimental patches using $1"
+                    (./tests/special_tests.sh --mark_type=exp_patch --filter_pattern='test_f' --log_file=${coverage_session_log} --experiment_patch_mask="1 0 0 1" 2>&1 >> ${temp_special_log}) > /dev/null
+                else
+                    log_msg "Skipping tests that require experimental patches."
+                fi
             fi
 	        ;;
 	    *)
@@ -255,7 +273,7 @@ fi
 log_msg "Generating base coverage for the FTS env ${target_env_name}"
 env_rebuild_collect "${target_env_name}"
 case ${target_env_name} in
-    fts_latest|$supported_fts_latest_pattern)
+    fts_latest|fts_oldest|$supported_fts_latest_pattern)
         log_msg "No env-specific additional coverage currently required for ${target_env_name}"
         ;;
     fts_release|$supported_fts_release_pattern)