Address review: hash installed files not versions; fix loop/import ordering

Build fingerprint (blocking finding): now hashes the on-disk content
of every file in the installed moss and inferedge-moss-core
distributions (Python sources, data, and native .so/.pyd bindings)
instead of just their version strings. A native binding rebuilt
without a version bump — e.g. a locally compiled wheel, or a binding
change shipped under an unchanged pin — previously kept the old
fingerprint and could reuse a stale index; it now changes the file
bytes and forces a rebuild.

MossClient construction moved inside the moss_client fixture's
_setup() coroutine, after _run() has created and installed the
shared event loop, so any async initialization the client performs
binds to the loop it will actually run on.

Made python-dotenv optional in test_bench_ci_moss.py (try/except
ModuleNotFoundError) so a repo-wide pytest run that collects this
module doesn't hard-fail on an unrelated missing dependency before
the credential-based skips even run.

Extended ALLOW_BENCHMARK_SKIP to also cover same-repo Dependabot
PRs, which GitHub denies normal secrets to just like fork PRs, but
which the previous fork-only check would have made fail as if
secrets were misconfigured.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: Sravan1011

.github/workflows/benchmark.yml

@@ -54,11 +54,13 @@ jobs:
         env:
           MOSS_PROJECT_ID: ${{ secrets.MOSS_PROJECT_ID }}
           MOSS_PROJECT_KEY: ${{ secrets.MOSS_PROJECT_KEY }}
-          # Fork PRs cannot read repository secrets, so missing credentials
-          # are expected there and the suite may skip. On trusted runs
-          # (push to main, same-repo PRs, manual dispatch) missing secrets
-          # make the suite FAIL instead of passing as a green no-op.
-          ALLOW_BENCHMARK_SKIP: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository) && '1' || '0' }}
+          # Fork PRs, and same-repo Dependabot PRs (which GitHub also denies
+          # normal secrets), cannot read repository secrets, so missing
+          # credentials are expected there and the suite may skip. On other
+          # trusted runs (push to main, same-repo human PRs, manual
+          # dispatch) missing secrets make the suite FAIL instead of
+          # passing as a green no-op.
+          ALLOW_BENCHMARK_SKIP: ${{ (github.event_name == 'pull_request' && (github.event.pull_request.head.repo.full_name != github.repository || github.actor == 'dependabot[bot]')) && '1' || '0' }}
         run: |
           # Baseline-update runs skip the regression comparison: comparing
           # against the baseline being replaced would fail the run (and skip

benchmarks/ci/bench_queries.py

@@ -42,28 +42,36 @@ def corpus_signature(docs: list[dict[str, Any]]) -> str:
 
 
 def build_fingerprint() -> str:
-    """Fingerprint of the code path that builds the index.
-
-    Covers the installed SDK/bindings versions and, when running from the
-    repository, a content hash of the Python SDK source tree. Any change to
-    the indexing/build path yields a new fingerprint — and therefore a new
-    index name via ``index_name_for`` — so the benchmark rebuilds the index
-    and exercises ``create_index``/document serialization instead of loading
-    an index built by older code (which could pass on stale embeddings).
+    """Fingerprint of every file belonging to the packages that build the index.
+
+    Hashes the on-disk content of every file in the installed ``moss`` and
+    ``inferedge-moss-core`` distributions — Python sources, data files, and
+    native bindings (``.so``/``.pyd``) alike — rather than just their
+    version strings. A rebuilt native binding (e.g. a locally compiled wheel
+    during development, or a binding change that ships under an unchanged
+    version pin) still changes the file bytes even when the version string
+    doesn't, so it isn't missed. Any change to the indexing/build path
+    yields a new fingerprint — and therefore a new index name via
+    ``index_name_for`` — so the benchmark rebuilds the index and exercises
+    ``create_index``/document serialization instead of loading an index
+    built by older code (which could pass on stale embeddings).
     """
-    from importlib.metadata import PackageNotFoundError, version
+    from importlib.metadata import PackageNotFoundError, distribution
 
     h = hashlib.sha256()
-    for pkg in ("moss", "inferedge-moss", "inferedge-moss-core"):
+    for pkg in ("moss", "inferedge-moss-core"):
         try:
-            h.update(f"{pkg}={version(pkg)}".encode())
+            dist = distribution(pkg)
         except PackageNotFoundError:
-            pass
-    sdk_src = Path(__file__).resolve().parents[2] / "sdks" / "python" / "sdk" / "src"
-    if sdk_src.is_dir():
-        for p in sorted(sdk_src.rglob("*.py")):
-            h.update(str(p.relative_to(sdk_src)).encode())
-            h.update(p.read_bytes())
+            continue
+        h.update(f"{pkg}={dist.version}".encode())
+        for rel_path in sorted(dist.files or (), key=str):
+            try:
+                data = rel_path.read_binary()
+            except (FileNotFoundError, IsADirectoryError, OSError):
+                continue
+            h.update(str(rel_path).encode())
+            h.update(data)
     return h.hexdigest()[:12]
 
 

benchmarks/ci/test_bench_ci_moss.py

@@ -38,7 +38,13 @@
     load_corpus_slice,
     query_set_hash,
 )
-from dotenv import load_dotenv
+try:
+    from dotenv import load_dotenv
+except ModuleNotFoundError:  # pragma: no cover - optional outside benchmarks/ci
+
+    def load_dotenv() -> None:
+        return None
+
 
 load_dotenv()
 
@@ -204,10 +210,15 @@ def moss_client(corpus_slice, corpus_sig, build_fp):
     # native bindings may not be installed in every env.
     from moss import DocumentInfo, MossClient
 
-    client = MossClient(project_id, project_key)
     index_name = os.getenv("MOSS_INDEX_NAME") or index_name_for(corpus_sig, build_fp)
 
     async def _setup():
+        # Construct the client inside the coroutine, after _run() has
+        # created and installed the shared event loop — MossClient may bind
+        # an async session or call get_event_loop() internally, and must do
+        # so against the loop it will actually run on.
+        client = MossClient(project_id, project_key)
+
         # Determine existence explicitly (rather than treating any get_index
         # failure as "missing") so auth/network errors surface instead of
         # silently triggering index creation.