[CRCR] Add SHA validator for nightly/periodic callbacks (Phase 1.2) (#8304)

## Summary

Phase 1.2 of the CRCR nightly/periodic self-report implementation ([RFC
98](pytorch/rfcs#98)).

Adds `utils/sha_validator.py` — validates that a self-reported commit
SHA exists on `pytorch/pytorch` via the GitHub API before accepting
nightly/periodic callback results.

**Key features:**
- `validate_sha(upstream_repo, sha)` → `True` if SHA exists, `False` on
404, raises on other errors
- Module-level TTL cache (1 hour) avoids redundant API calls when
multiple downstream repos report against the same nightly SHA
- Expired entries are evicted lazily on each call

**Depends on:** Phase 1 (#8302) — will be called from
`_handle_nightly_callback()`

**Changes:**
| File | Change |
|------|--------|
| `utils/sha_validator.py` | New module: SHA validation with TTL cache |
| `tests/test_sha_validator.py` | 7 tests: valid/invalid SHA, cache
hit/miss, TTL expiry, error propagation |

## Test plan
- [x] 7 unit tests covering all paths
- [ ] CI passes

Author: subinz1

aws/lambda/cross_repo_ci_relay/tests/test_sha_validator.py

@@ -0,0 +1,159 @@
+import time
+import unittest
+from unittest.mock import MagicMock, patch
+
+import github
+from utils.sha_validator import _CacheEntry, _REPO_CACHE, _SHA_CACHE, validate_sha
+
+
+class TestShaValidator(unittest.TestCase):
+    def setUp(self):
+        _SHA_CACHE.clear()
+        _REPO_CACHE.clear()
+
+    def tearDown(self):
+        _SHA_CACHE.clear()
+        _REPO_CACHE.clear()
+
+    def test_valid_sha_returns_true(self):
+        mock_gh = MagicMock()
+        mock_gh.get_repo.return_value.get_commit.return_value = MagicMock()
+
+        result = validate_sha("pytorch/pytorch", "abc123", gh_client=mock_gh)
+
+        self.assertTrue(result)
+        mock_gh.get_repo.assert_called_once_with("pytorch/pytorch")
+
+    def test_invalid_sha_returns_false(self):
+        mock_gh = MagicMock()
+        mock_gh.get_repo.return_value.get_commit.side_effect = github.GithubException(
+            404, {"message": "Not Found"}, None
+        )
+
+        result = validate_sha("pytorch/pytorch", "bad_sha", gh_client=mock_gh)
+
+        self.assertFalse(result)
+
+    def test_cache_hit_skips_api_call(self):
+        mock_gh = MagicMock()
+        mock_gh.get_repo.return_value.get_commit.return_value = MagicMock()
+
+        validate_sha("pytorch/pytorch", "abc123", gh_client=mock_gh)
+        self.assertEqual(mock_gh.get_repo.call_count, 1)
+
+        validate_sha("pytorch/pytorch", "abc123", gh_client=mock_gh)
+        # repo handle is cached, so get_repo is only called once
+        self.assertEqual(mock_gh.get_repo.call_count, 1)
+
+    def test_different_sha_not_cached(self):
+        mock_gh = MagicMock()
+        mock_repo = MagicMock()
+        mock_gh.get_repo.return_value = mock_repo
+
+        validate_sha("pytorch/pytorch", "sha1", gh_client=mock_gh)
+        validate_sha("pytorch/pytorch", "sha2", gh_client=mock_gh)
+
+        # repo handle cached, but get_commit called twice (different SHAs)
+        self.assertEqual(mock_repo.get_commit.call_count, 2)
+
+    @patch("utils.sha_validator.time")
+    def test_expired_cache_entry_evicted(self, mock_time):
+        mock_time.monotonic.return_value = 1000.0
+        _SHA_CACHE["pytorch/pytorch:old_sha"] = _CacheEntry(
+            exists=True, timestamp=1000.0
+        )
+
+        mock_time.monotonic.return_value = 1000.0 + 3601
+        mock_gh = MagicMock()
+        mock_gh.get_repo.return_value.get_commit.return_value = MagicMock()
+
+        validate_sha("pytorch/pytorch", "new_sha", gh_client=mock_gh)
+
+        self.assertNotIn("pytorch/pytorch:old_sha", _SHA_CACHE)
+
+    def test_transient_error_fails_open(self):
+        mock_gh = MagicMock()
+        mock_gh.get_repo.return_value.get_commit.side_effect = github.GithubException(
+            500, {"message": "Server Error"}, None
+        )
+
+        result = validate_sha("pytorch/pytorch", "abc123", gh_client=mock_gh)
+
+        self.assertTrue(result)
+
+    def test_rate_limit_403_fails_open(self):
+        mock_gh = MagicMock()
+        mock_gh.get_repo.return_value.get_commit.side_effect = github.GithubException(
+            403, {"message": "rate limit exceeded"}, None
+        )
+
+        result = validate_sha("pytorch/pytorch", "abc123", gh_client=mock_gh)
+
+        self.assertTrue(result)
+
+    def test_negative_cache_hit_returns_false(self):
+        mock_gh = MagicMock()
+        mock_gh.get_repo.return_value.get_commit.side_effect = github.GithubException(
+            404, {"message": "Not Found"}, None
+        )
+
+        validate_sha("pytorch/pytorch", "bad_sha", gh_client=mock_gh)
+        self.assertIn("pytorch/pytorch:bad_sha", _SHA_CACHE)
+        self.assertFalse(_SHA_CACHE["pytorch/pytorch:bad_sha"].exists)
+
+        # Second call hits the negative cache — no additional API call
+        mock_gh.get_repo.return_value.get_commit.reset_mock()
+        result = validate_sha("pytorch/pytorch", "bad_sha", gh_client=mock_gh)
+        self.assertFalse(result)
+        mock_gh.get_repo.return_value.get_commit.assert_not_called()
+
+    @patch("utils.sha_validator.time")
+    def test_negative_cache_expires_sooner(self, mock_time):
+        mock_time.monotonic.return_value = 1000.0
+        _SHA_CACHE["pytorch/pytorch:bad_sha"] = _CacheEntry(
+            exists=False, timestamp=1000.0
+        )
+
+        # After 301 seconds (> 300s negative TTL), entry should be evicted
+        mock_time.monotonic.return_value = 1000.0 + 301
+        mock_gh = MagicMock()
+        mock_gh.get_repo.return_value.get_commit.side_effect = github.GithubException(
+            404, {"message": "Not Found"}, None
+        )
+
+        result = validate_sha("pytorch/pytorch", "bad_sha", gh_client=mock_gh)
+        self.assertFalse(result)
+        # API was called again because cache was evicted
+        mock_gh.get_repo.return_value.get_commit.assert_called_once()
+
+    def test_cached_valid_sha_short_circuits_even_if_api_would_404(self):
+        """A previously cached valid SHA returns True without hitting the API,
+        even if the API would now return 404."""
+        _SHA_CACHE["pytorch/pytorch:abc123"] = _CacheEntry(
+            exists=True, timestamp=time.monotonic()
+        )
+
+        mock_gh = MagicMock()
+        mock_gh.get_repo.return_value.get_commit.side_effect = github.GithubException(
+            404, {"message": "Not Found"}, None
+        )
+
+        result = validate_sha("pytorch/pytorch", "abc123", gh_client=mock_gh)
+
+        self.assertTrue(result)
+        mock_gh.get_repo.return_value.get_commit.assert_not_called()
+
+    def test_repo_handle_cached(self):
+        mock_gh = MagicMock()
+        mock_repo = MagicMock()
+        mock_gh.get_repo.return_value = mock_repo
+
+        validate_sha("pytorch/pytorch", "sha1", gh_client=mock_gh)
+        validate_sha("pytorch/pytorch", "sha2", gh_client=mock_gh)
+
+        # get_repo only called once due to repo handle caching
+        mock_gh.get_repo.assert_called_once_with("pytorch/pytorch")
+
+
+if __name__ == "__main__":
+    unittest.main()

aws/lambda/cross_repo_ci_relay/utils/sha_validator.py

@@ -0,0 +1,91 @@
+"""Validate that a commit SHA exists on pytorch/pytorch via GitHub API.
+
+Used by the nightly/periodic callback path to verify that the self-reported
+dispatch_id (commit SHA) is real before accepting the result.  A TTL cache
+avoids redundant API calls when multiple downstream repos report against the
+same SHA.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from dataclasses import dataclass
+
+import github
+
+
+logger = logging.getLogger(__name__)
+
+_CACHE_TTL_SECONDS = 3600  # 1 hour
+_NEGATIVE_CACHE_TTL_SECONDS = 300  # 5 minutes for 404s
+
+
+@dataclass
+class _CacheEntry:
+    exists: bool
+    timestamp: float
+
+
+_SHA_CACHE: dict[str, _CacheEntry] = {}
+_REPO_CACHE: dict[str, object] = {}
+
+
+def _evict_expired() -> None:
+    now = time.monotonic()
+    expired = []
+    for k, entry in _SHA_CACHE.items():
+        ttl = _CACHE_TTL_SECONDS if entry.exists else _NEGATIVE_CACHE_TTL_SECONDS
+        if now - entry.timestamp > ttl:
+            expired.append(k)
+    for k in expired:
+        del _SHA_CACHE[k]
+
+
+def _get_repo(gh_client: github.Github, upstream_repo: str):
+    """Return a cached repo handle to avoid redundant API calls."""
+    if upstream_repo not in _REPO_CACHE:
+        _REPO_CACHE[upstream_repo] = gh_client.get_repo(upstream_repo)
+    return _REPO_CACHE[upstream_repo]
+
+
+def validate_sha(
+    upstream_repo: str,
+    sha: str,
+    gh_client: github.Github,
+) -> bool:
+    """Return True if ``sha`` exists on ``upstream_repo``, False otherwise.
+
+    Results are cached: valid SHAs for 1 hour, invalid (404) SHAs for 5 minutes.
+    The repo handle is also cached to halve API calls per cache miss.
+
+    Transient API errors (500, 403 rate-limit) fail open — since nightly results
+    are informational, a GitHub outage should not reject valid callbacks.
+    """
+    _evict_expired()
+
+    cache_key = f"{upstream_repo}:{sha}"
+    if cache_key in _SHA_CACHE:
+        return _SHA_CACHE[cache_key].exists
+
+    try:
+        repo = _get_repo(gh_client, upstream_repo)
+        repo.get_commit(sha)
+        _SHA_CACHE[cache_key] = _CacheEntry(exists=True, timestamp=time.monotonic())
+        return True
+    except github.GithubException as exc:
+        if exc.status == 404:
+            logger.warning("SHA %s does not exist on %s", sha, upstream_repo)
+            _SHA_CACHE[cache_key] = _CacheEntry(
+                exists=False, timestamp=time.monotonic()
+            )
+            return False
+        # Transient errors (500, 403 rate-limit, etc.) — fail open since
+        # nightly is informational and should not be blocked by GitHub outages.
+        logger.warning(
+            "GitHub API error (%d) validating SHA %s on %s, failing open",
+            exc.status,
+            sha,
+            upstream_repo,
+        )
+        return True