[WIP] Canonical v2 migration

.gitattributes

@@ -1,3 +1,5 @@
+* text=auto eol=lf
+
 s2and/data/production_model_v*.pickle filter=lfs diff=lfs merge=lfs -text
 s2and/data/production_model_v*.pkl filter=lfs diff=lfs merge=lfs -text
 s2and/data/production_model_v*/** filter=lfs diff=lfs merge=lfs -text

.githooks/pre-commit

@@ -9,18 +9,14 @@ if ! git diff --cached --name-only --diff-filter=ACM | grep -qx "VERSION"; then
   exit 0
 fi
 
-VERSION_FILES=(
-  "VERSION"
-  "pyproject.toml"
-  "s2and_rust/pyproject.toml"
-  "s2and_rust/Cargo.toml"
-  "s2and/runtime.py"
-  "README.md"
-  "docs/development.md"
-  "s2and_rust/README.md"
-  "uv.lock"
-  "s2and_rust/Cargo.lock"
-)
+VERSION_TARGETS="$(uv run python scripts/sync_version.py --print-targets)"
+VERSION_FILES=()
+while IFS= read -r version_file; do
+  version_file="${version_file%$'\r'}"
+  if [[ -n "$version_file" ]]; then
+    VERSION_FILES+=("$version_file")
+  fi
+done <<< "$VERSION_TARGETS"
 
 if uv run python scripts/sync_version.py --check; then
   exit 0

.github/workflows/main.yaml

@@ -26,39 +26,14 @@ jobs:
             ~/.cache/uv
           key: uv-cache-${{ runner.os }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
 
-      # Sync environment from lock if present so tool versions come from pyproject.toml/uv.lock
-      - name: Sync deps (locked if available)
-        shell: bash
-        run: |
-          if [[ -f uv.lock ]]; then
-            uv sync --extra dev --frozen
-          else
-            uv sync --extra dev
-          fi
-
-      - name: version sync check
-        run: uv run python scripts/sync_version.py --check
-      - name: ruff check
-        run: uv run ruff check s2and scripts tests
-      - name: ruff format (s2and/)
-        run: uv run ruff format --check s2and
-      - name: ruff format (scripts/*.py)
-        shell: bash
-        run: |
-          shopt -s nullglob
-          files=(scripts/*.py)
-          if (( ${#files[@]} )); then
-            uv run ruff format --check "${files[@]}"
-          fi
+      - name: Run shared lint policy
+        run: uv run --no-project python scripts/run_ci_locally.py lint
 
   typecheck-and-test:
-    name: typecheck-and-test (${{ matrix.lane }})
     runs-on: ubuntu-latest
     needs: [lint]
-    strategy:
-      fail-fast: false
-      matrix:
-        lane: [py-only, rust-enabled]
+    env:
+      S2AND_TEST_REQUIRE_RUST: "1"
     steps:
       - uses: actions/checkout@v5
         with:
@@ -73,73 +48,12 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Hydrate CI LFS artifacts
-        run: git lfs pull --include="s2and/data/production_model_v*.pickle,s2and/data/production_model_v*/**,tests/fixtures/arrow/pubmed_specter2/**"
-
-      - name: Verify hydrated CI LFS artifacts
-        run: |
-          uv run --no-project python - <<'PY'
-          from pathlib import Path
-
-          pointer_prefix = b"version https://git-lfs.github.com/spec/v1"
-          required_paths = [
-              Path("s2and/data/production_model_v1.0.pickle"),
-              Path("s2and/data/production_model_v1.1.pickle"),
-              Path("s2and/data/production_model_v1.2.pickle"),
-          ]
-          required_paths.extend(
-              sorted(path for path in Path("s2and/data/production_model_v1.21").rglob("*") if path.is_file())
-          )
-          required_paths.extend(
-              [
-                  Path("tests/fixtures/arrow/pubmed_specter2/pubmed/signatures.arrow"),
-                  Path("tests/fixtures/arrow/pubmed_specter2/pubmed/papers.arrow"),
-                  Path("tests/fixtures/arrow/pubmed_specter2/pubmed/paper_authors.arrow"),
-                  Path("tests/fixtures/arrow/pubmed_specter2/pubmed/specter2.arrow"),
-                  Path("tests/fixtures/arrow/pubmed_specter2/pubmed/signatures.signatures_batch_index.bin"),
-                  Path("tests/fixtures/arrow/pubmed_specter2/pubmed/papers.papers_batch_index.bin"),
-                  Path("tests/fixtures/arrow/pubmed_specter2/pubmed/paper_authors.paper_authors_batch_index.bin"),
-                  Path("tests/fixtures/arrow/pubmed_specter2/pubmed/specter2.specter_batch_index.bin"),
-                  Path("tests/fixtures/arrow/pubmed_specter2/pubmed/name_counts_index/manifest.json"),
-                  Path(
-                      "tests/fixtures/arrow/pubmed_specter2/pubmed/name_counts_index/"
-                      "generations/pubmed-specter2/first.bin"
-                  ),
-                  Path(
-                      "tests/fixtures/arrow/pubmed_specter2/pubmed/name_counts_index/"
-                      "generations/pubmed-specter2/last.bin"
-                  ),
-                  Path(
-                      "tests/fixtures/arrow/pubmed_specter2/pubmed/name_counts_index/"
-                      "generations/pubmed-specter2/first_last.bin"
-                  ),
-                  Path(
-                      "tests/fixtures/arrow/pubmed_specter2/pubmed/name_counts_index/"
-                      "generations/pubmed-specter2/last_first_initial.bin"
-                  ),
-              ]
-          )
-
-          failures = []
-          for path in required_paths:
-              if not path.exists():
-                  failures.append(f"missing: {path}")
-                  continue
-              head = path.read_bytes()[: len(pointer_prefix)]
-              if head == pointer_prefix:
-                  failures.append(f"LFS pointer was not hydrated: {path}")
-
-          if failures:
-              raise SystemExit("\n".join(failures))
-          print(f"Verified {len(required_paths)} hydrated CI LFS artifact files.")
-          PY
-
       - name: Setup Rust
-        if: matrix.lane == 'rust-enabled'
         uses: dtolnay/rust-toolchain@stable
+        with:
+          components: rustfmt, clippy
 
       - name: Cache cargo build
-        if: matrix.lane == 'rust-enabled'
         uses: actions/cache@v5
         with:
           path: |
@@ -157,86 +71,10 @@ jobs:
           path: |
             ~/.cache/uv
             .venv
-          key: uv-venv-${{ runner.os }}-py311-${{ matrix.lane }}-${{ hashFiles('pyproject.toml', 'uv.lock') }}
+          key: uv-venv-${{ runner.os }}-py311-${{ hashFiles('pyproject.toml', 'uv.lock') }}
           restore-keys: |
-            uv-venv-${{ runner.os }}-py311-${{ matrix.lane }}-
             uv-venv-${{ runner.os }}-py311-
             uv-venv-
 
-      # Sync environment from lock if present (fast; no network if cached)
-      - name: Sync deps (locked if available)
-        shell: bash
-        run: |
-          if [[ -f uv.lock ]]; then
-            if [[ "${{ matrix.lane }}" == "rust-enabled" ]]; then
-              uv sync --extra dev --extra rust --frozen --no-install-package s2and-rust
-            else
-              uv sync --extra dev --frozen
-            fi
-          else
-            # No lock present; resolve once, then install
-            if [[ "${{ matrix.lane }}" == "rust-enabled" ]]; then
-              uv sync --extra dev --extra rust --no-install-package s2and-rust
-            else
-              uv sync --extra dev
-            fi
-          fi
-
-      # Build/install Rust extension for parity tests
-      - name: Build Rust extension
-        if: matrix.lane == 'rust-enabled'
-        run: uv run --with maturin maturin develop -m s2and_rust/Cargo.toml
-
-      - name: "Rust artifact guardrail: default incremental linker"
-        if: matrix.lane == 'rust-enabled'
-        run: uv run pytest -q tests/test_incremental_linking_default_artifact.py
-
-      - name: "Rust parity guardrail: feature parity"
-        if: matrix.lane == 'rust-enabled'
-        run: uv run pytest -q tests/test_feature_port_parity.py
-
-      - name: "Rust parity guardrail: signature preprocess parity"
-        if: matrix.lane == 'rust-enabled'
-        run: uv run pytest -q tests/test_rust_signature_preprocess.py
-
-      - name: "Rust parity guardrail: batch chunking/fallback parity"
-        if: matrix.lane == 'rust-enabled'
-        run: uv run pytest -q tests/test_rust_batch_chunking.py
-
-      # Type checking (ty with tuned migration rules)
-      - name: ty (s2and)
-        run: |
-          uv run ty check s2and \
-            --ignore unresolved-import \
-            --ignore unused-type-ignore-comment \
-            --ignore possibly-missing-attribute \
-            --ignore unresolved-global
-      - name: ty (scripts)
-        shell: bash
-        run: |
-          shopt -s nullglob
-          files=(scripts/*.py)
-          if (( ${#files[@]} )); then
-            uv run ty check "${files[@]}" \
-              --ignore unresolved-import \
-              --ignore unused-type-ignore-comment \
-              --ignore possibly-missing-attribute \
-              --ignore unresolved-global \
-              --ignore unresolved-reference \
-              --ignore unresolved-attribute
-          fi
-
-      - name: pytest (coverage, py-only)
-        if: matrix.lane == 'py-only'
-        env:
-          # keep startup lean; avoid user-level plugins on hosted runners
-          S2AND_BACKEND: python
-        run: |
-          uv run pytest tests/ \
-            --cov=s2and --cov-report=term-missing --cov-fail-under=40
-
-      - name: pytest (coverage, rust-enabled)
-        if: matrix.lane == 'rust-enabled'
-        run: |
-          uv run pytest tests/ \
-            --cov=s2and --cov-report=term-missing --cov-fail-under=40
+      - name: Run shared typecheck-and-test policy
+        run: uv run --no-project python scripts/run_ci_locally.py typecheck-and-test