Complete Pyright Migration and Remove Mypy Parity Configuration

.github/workflows/copilot-setup-steps.yml

@@ -0,0 +1,45 @@
+name: "Copilot Setup Steps"
+
+on:
+  workflow_dispatch:
+  push:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+  pull_request:
+    paths:
+      - .github/workflows/copilot-setup-steps.yml
+
+jobs:
+  copilot-setup-steps:
+    runs-on: ubuntu-latest
+    environment: copilot
+    permissions:
+      contents: read
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install CI dependencies
+        uses: ./.github/actions/install-ci-dependencies
+        with:
+          python_version: '3.12'
+          show_pip_list: 'true'
+
+      - name: Install pre-commit
+        shell: bash
+        run: |
+          pre-commit install
+
+      - name: Run type checking
+        shell: bash
+        run: |
+          pyright -p pyproject.toml
+
+      - name: Run pre-commit hooks
+        shell: bash
+        run: |
+          pre-commit run --all-files
+
+      - name: Run Copilot setup steps
+        run: echo "Copilot setup steps completed successfully."

pyproject.toml

@@ -179,33 +179,9 @@ known-first-party = ["docs","fts_examples","finetuning_scheduler","tests"]
 force-sort-within-sections = false
 order-by-type = false
 
-[tool.mypy]
-files = ["src/finetuning_scheduler"]
-disallow_untyped_defs = "True"
-ignore_missing_imports = "True"
-show_error_codes = "True"
-warn_redundant_casts = "True"
-warn_unused_configs = "True"
-warn_unused_ignores = "False"
-allow_redefinition = "True"
-# disable this rule as the PL Trainer attributes are defined in the connectors, not in its __init__
-disable_error_code = "attr-defined"
-# style choices
-warn_no_return = "False"
-exclude = ['tests/.*']
-
-# Ignore mypy errors for these files
-# TODO: the goal is for this to be empty
-#[[tool.mypy.overrides]]
-# the list can be generated with:
-# mypy | tr ':' ' ' |  awk '{print $1}' | sort | uniq | sed 's/\.py//g' | sed 's|\/|\.|g' | xargs -I {} echo '"{}",'
-# module = []
-# ignore_errors = "True"
-
 [tool.pyright]
-# Using "basic" mode for initial migration from mypy
-# TODO: Progressively tighten to "standard" as type issues are resolved
-typeCheckingMode = "basic"
+# Using "standard" mode for comprehensive type checking
+typeCheckingMode = "standard"
 include = ["src/finetuning_scheduler"]
 exclude = [
     "tests",
@@ -214,22 +190,21 @@ exclude = [
     "dist",
     ".git",
 ]
-# Match mypy's ignore_missing_imports behavior
+# Ignore missing imports from third-party libraries without type stubs
 reportMissingImports = "none"
-# Match mypy's disable_error_code = "attr-defined" for PL Trainer attributes
+# Disable attribute access checks - PL Trainer attributes are defined in connectors, not __init__
 reportAttributeAccessIssue = "none"
-# Match mypy's allow_redefinition behavior
+# Allow constant redefinition for dynamic configuration patterns
 reportConstantRedefinition = "none"
 # Disable private import usage warnings - required for Lightning internals
 reportPrivateImportUsage = "none"
-# Disable general type issues for generator context managers until properly annotated
-reportGeneralTypeIssues = "warning"
+# Enable comprehensive type checking for standard mode
+reportGeneralTypeIssues = "error"
+reportAssignmentType = "error"
+reportCallIssue = "error"
+reportIndexIssue = "error"
 # Disable invalid type form warnings for dynamic type expressions
 reportInvalidTypeForm = "none"
-# Temporarily disable these until code is updated with proper type annotations
-reportAssignmentType = "none"
-reportCallIssue = "none"
-reportIndexIssue = "none"
 
 [tool.coverage.report]
 exclude_lines = [

requirements/ci/requirements-oldest.txt

@@ -125,7 +125,7 @@ evaluate==0.3.0
     # via finetuning-scheduler (pyproject.toml)
 exceptiongroup==1.3.1 ; python_full_version < '3.11'
     # via anyio
-fastapi==0.123.10
+fastapi==0.124.0
     # via mlflow-skinny
 filelock==3.20.0
     # via
@@ -474,7 +474,7 @@ propcache==0.4.1
     # via
     #   aiohttp
     #   yarl
-protobuf==6.33.1
+protobuf==6.33.2
     # via
     #   databricks-sdk
     #   mlflow-skinny

requirements/ci/requirements.txt

@@ -1,5 +1,5 @@
 # This file was autogenerated by uv via the following command:
-#    uv pip compile /home/speediedan/repos/finetuning-scheduler/pyproject.toml --extra all --group dev --group test --output-file /home/speediedan/repos/finetuning-scheduler/requirements/ci/requirements.txt --no-strip-extras --resolution highest --universal --python-version 3.10 --prerelease=if-necessary-or-explicit --override /tmp/tmp.bOqxnBiEpd --index-strategy unsafe-best-match --no-emit-package torch
+#    uv pip compile /home/speediedan/repos/finetuning-scheduler/pyproject.toml --extra all --group dev --group test --output-file /home/speediedan/repos/finetuning-scheduler/requirements/ci/requirements.txt --no-strip-extras --resolution highest --universal --python-version 3.10 --prerelease=if-necessary-or-explicit --override /tmp/tmp.CNVCWjH0nH --index-strategy unsafe-best-match --no-emit-package torch
 aiohappyeyeballs==2.6.1
     # via aiohttp
 aiohttp==3.13.2
@@ -137,7 +137,7 @@ exceptiongroup==1.3.1 ; python_full_version < '3.11'
     #   pytest
 executing==2.2.1
     # via stack-data
-fastapi==0.123.10
+fastapi==0.124.0
     # via mlflow-skinny
 fastjsonschema==2.21.2
     # via nbformat
@@ -381,8 +381,6 @@ more-itertools==10.8.0 ; platform_machine != 'ppc64le' and platform_machine != '
     # via
     #   jaraco-classes
     #   jaraco-functools
-mpmath==1.3.0
-    # via sympy
 multidict==6.7.0
     # via
     #   aiohttp
@@ -411,10 +409,6 @@ nbval==0.11.0
     # via finetuning-scheduler (pyproject.toml)
 nest-asyncio==1.6.0
     # via ipykernel
-networkx==3.4.2 ; python_full_version < '3.11'
-    # via torch
-networkx==3.6 ; python_full_version >= '3.11'
-    # via torch
 nh3==0.3.2
     # via readme-renderer
 nodeenv==1.9.1
@@ -534,7 +528,7 @@ propcache==0.4.1
     # via
     #   aiohttp
     #   yarl
-protobuf==6.33.1
+protobuf==6.33.2
     # via
     #   databricks-sdk
     #   mlflow-skinny
@@ -725,8 +719,6 @@ stack-data==0.6.3
     # via ipython
 starlette==0.50.0
     # via fastapi
-sympy==1.14.0
-    # via torch
 tabulate==0.9.0
     # via finetuning-scheduler (pyproject.toml)
 tensorboardx==2.6.4

requirements/utils/lock_ci_requirements.sh

@@ -14,6 +14,7 @@
 #   - Lock file is generated with torch pinned to the nightly version
 #   - Uses PyTorch nightly index for resolution
 #   - Docker image and CI both use the same nightly version
+#   - Post-processing prunes torch-only dependencies (see prune_torch_only_deps)
 # - Without torch-nightly.txt:
 #   - Uses stable torch from PyPI
 #   - CI uses --torch-backend=cpu for CPU variant
@@ -67,6 +68,15 @@ get_torch_nightly_version() {
 # Check if torch nightly is configured
 TORCH_NIGHTLY_VERSION=$(get_torch_nightly_version)
 
+# Prune packages that are ONLY dependencies of torch from the lockfile.
+# This reduces the dependency confusion attack surface when using unsafe-best-match
+# by removing any packages that could potentially be resolved from the nightly index only.
+# See prune_torch_deps.py for detailed documentation and implementation.
+prune_torch_only_deps() {
+    local lockfile=$1
+    python "${SCRIPT_DIR}/prune_torch_deps.py" "${lockfile}"
+}
+
 # Generate/update torch_override.txt if nightly is configured, remove if not
 generate_torch_override() {
     if [[ -n "${TORCH_NIGHTLY_VERSION}" ]]; then
@@ -125,24 +135,33 @@ generate_lockfile() {
     # 1. Create a temporary override file to pin torch to the nightly version for dependency resolution
     # 2. Use --prerelease=if-necessary-or-explicit to only allow prereleases for explicitly specified packages (torch)
     #    or where all versions of the package are pre-release
-    # 3. Use --extra-index-url with nightly CPU index for torch resolution
+    # 3. Use --index with nightly CPU index for torch resolution
     # 4. Use --index-strategy=unsafe-best-match for lockfile GENERATION only
     #    This is required because with first-index (default), uv would either:
     #      - Find torch on PyPI first (no nightly version), or
     #      - Find scipy/etc on nightly index first (missing versions)
-    #    Security impact is minimal because:
-    #      - Lockfile generation runs on maintainer machines, not user machines
-    #      - Generated lockfile pins exact package versions from PyPI
-    #      - User INSTALLATION uses secure two-step approach (no unsafe-best-match)
+    #
+    #    Security rationale for using unsafe-best-match during lockfile generation:
+    #    a) User INSTALLATION uses a secure two-step approach, only ever installing torch nightly
+    #       from the explicitly specified nightly index (no unsafe-best-match at install time)
+    #    b) The marginal dependency confusion attack surface is limited to the closely monitored
+    #       PyTorch nightly index, which is maintained by PyTorch team. Post-processing prunes any packages that are
+    #       ONLY dependencies of torch, eliminating potential attack vectors from torch-exclusive dependencies that
+    #       might only exist on the nightly index. If a package is shared with other dependencies, it's already
+    #       being resolved from PyPI and subject to normal security scanning.
+    #    c) Lockfile generation runs on maintainer machines, not user machines
+    #    d) Generated lockfile pins exact package versions from PyPI
+    #
     # 5. Use --no-emit-package=torch to exclude torch from output (installed separately with backend)
+    # 6. Post-process to prune torch-only dependencies (see prune_torch_only_deps)
     if [[ "${use_nightly}" == "true" && -n "${TORCH_NIGHTLY_VERSION}" ]]; then
         local torch_override_file=$(mktemp)
         echo "torch==${TORCH_NIGHTLY_VERSION}" > "${torch_override_file}"
 
         compile_cmd+=(
             --prerelease=if-necessary-or-explicit
             --override "${torch_override_file}"
-            --extra-index-url "https://download.pytorch.org/whl/nightly/cpu"
+            --index "https://download.pytorch.org/whl/nightly/cpu"
             --index-strategy unsafe-best-match  # for lockfile generation only, see comment above
             --no-emit-package torch
         )
@@ -151,7 +170,11 @@ generate_lockfile() {
         "${compile_cmd[@]}"
 
         rm -f "${torch_override_file}"
-        echo "✓ Generated ${output_file} (torch ${TORCH_NIGHTLY_VERSION} excluded, install separately)"
+
+        # Prune torch-only dependencies to minimize dependency confusion attack surface
+        prune_torch_only_deps "${output_file}"
+
+        echo "✓ Generated ${output_file} (torch ${TORCH_NIGHTLY_VERSION} excluded, torch-only deps pruned)"
     else
         "${compile_cmd[@]}"
         echo "✓ Generated ${output_file}"