Multiple fixes to improve detection (#1974)

* Allow vulnerability scans on arbitrary N|Solid refs

Known branches still use curated dependency lists, but unknown refs now fall
back to probing every tracked dependency definition and keeping only the ones
whose version parsers succeed against the checked-out tree.

This lets the scanner run against release branches and ad hoc refs without
having to pre-register each branch name in dependencies_per_branch.

Also document the workflow input as an arbitrary N|Solid branch/ref and mark
curated scans as incomplete when a curated dependency fails to resolve.

* Harden dependency resolution, npm audit edge cases, and reconciliation

Make arbitrary-ref dependency probing more tolerant of parser shape errors and
mark curated scans as incomplete when curated dependencies fail to resolve.

Tighten npm audit handling by:
- recording discovery failures and empty installed bundle trees as incomplete
- generating prod-only lockfiles with --package-lock-only when no lockfile is present
- retrying ENOLOCK recovery even when node_modules exists without a lockfile
- handling boolean bundleDependencies and avoiding duplicate installed-tree walks
- skipping invalid GitHub advisory specifiers or versions per advisory while
  marking the scan incomplete
- making per-package GitHub advisory query failures non-fatal
- normalizing npm advisory IDs across modern and legacy audit payloads
- preferring CVE identifiers when GitHub exposes them

Preserve reconciliation when a vendored npm advisory changes from GHSA to CVE
by carrying alternate advisory identifiers in the scan payload and matching
existing issues through those aliases before create/close decisions.

Author: santigimeno

.github/workflows/check-vulns.yml

@@ -6,6 +6,7 @@ on:
       nsolidStream:
         type: string
         default: 'main'
+        description: 'N|Solid branch or ref to scan'
     secrets:
       NVD_API_KEY:
         required: true
@@ -14,6 +15,7 @@ on:
       nsolidStream:
         type: string
         default: 'main'
+        description: 'N|Solid branch or ref to scan'
 
 
 permissions:

dep_checker/main.py

@@ -35,7 +35,7 @@ class Vulnerability:
     def __init__(self, id: str, url: str, dependency: str, version: str, source: str = "binary", 
                  severity: Optional[str] = None, via: Optional[list] = None, 
                  fix_available: Optional[bool] = None, main_dep_name: Optional[str] = None,
-                 main_dep_path: Optional[str] = None):
+                 main_dep_path: Optional[str] = None, advisory_aliases: Optional[list[str]] = None):
         self.id = id
         self.url = url
         self.dependency = dependency
@@ -46,6 +46,7 @@ def __init__(self, id: str, url: str, dependency: str, version: str, source: str
         self.fix_available = fix_available  # whether fix is available
         self.main_dep_name = main_dep_name  # main dependency name for npm vulnerabilities
         self.main_dep_path = main_dep_path  # path to the main dependency
+        self.advisory_aliases = advisory_aliases or []  # alternate IDs for reconciliation migration
 
 
 class VulnerabilityEncoder(json.JSONEncoder):
@@ -69,6 +70,8 @@ def default(self, obj):
                 result["main_dep_path"] = obj.main_dep_path
             if obj.fix_available is not None:
                 result["fix_available"] = obj.fix_available
+            if obj.advisory_aliases:
+                result["advisory_aliases"] = obj.advisory_aliases
             return result
         # Let the base class default method raise the TypeError
         return json.JSONEncoder.default(self, obj)
@@ -103,6 +106,64 @@ def default(self, obj):
 )
 
 
+def resolve_dependencies(
+    repo_path: Path, repo_branch: str
+) -> tuple[dict[str, Dependency], list[str]]:
+    """Return the dependencies that can be parsed from the checked-out repo.
+
+    Known branches still use their curated dependency list. Unknown branches fall
+    back to probing every tracked dependency definition and keeping only the ones
+    whose version parser succeeds against the checkout.
+    """
+
+    configured_names = dependencies_per_branch.get(repo_branch)
+    if configured_names is None:
+        candidate_dependencies = dependencies_info
+        print(
+            f"Info: '{repo_branch}' is not explicitly configured; scanning all known dependency definitions"
+        )
+    else:
+        candidate_dependencies = {
+            name: dep
+            for name, dep in dependencies_info.items()
+            if name in configured_names
+        }
+
+    available_dependencies: dict[str, Dependency] = {}
+    skipped_dependencies: list[str] = []
+
+    for name, dep in candidate_dependencies.items():
+        try:
+            dep.version_parser(repo_path)
+        except (
+            FileNotFoundError,
+            RuntimeError,
+            ValueError,
+            KeyError,
+            IndexError,
+            AttributeError,
+            TypeError,
+            OSError,
+        ) as exc:
+            skipped_dependencies.append(f"{name}: {exc}")
+            continue
+        available_dependencies[name] = dep
+
+    if not available_dependencies:
+        raise RuntimeError(
+            f"No supported dependencies could be resolved from '{repo_branch}'"
+        )
+
+    if skipped_dependencies:
+        print(
+            f"Info: Skipping {len(skipped_dependencies)} dependencies that are not present or not parseable in '{repo_branch}'"
+        )
+        for skipped in skipped_dependencies:
+            print(f"  - {skipped}")
+
+    return available_dependencies, skipped_dependencies
+
+
 def query_ghad(
     dependencies: dict[str, Dependency], gh_token: str, repo_path: Path
 ) -> list[Vulnerability]:
@@ -237,7 +298,10 @@ def main() -> int:
     parser.add_argument(
         "node_repo_branch",
         metavar="NODE_REPO_BRANCH",
-        help=f"the current branch of the Node repository (supports {supported_branches})",
+        help=(
+            "the current branch of the Node/N|Solid repository; known branches use "
+            f"curated dependency lists, other branches are scanned by probing available dependencies ({supported_branches})"
+        ),
     )
     parser.add_argument(
         "--gh-token",
@@ -281,10 +345,6 @@ def main() -> int:
         raise RuntimeError(
             "Invalid argument: '{repo_path}' is not a valid Node git repository"
         )
-    if repo_branch not in dependencies_per_branch:
-        raise RuntimeError(
-            f"Invalid argument: '{repo_branch}' is not a supported branch. Please use one of: {supported_branches}"
-        )
     if gh_token is None:
         print(
             "Warning: GitHub authentication token not provided, skipping GitHub Advisory Database queries",
@@ -296,15 +356,17 @@ def main() -> int:
             file=sys.stderr,
         )
 
-    dependencies = {
-        name: dep
-        for name, dep in dependencies_info.items()
-        if name in dependencies_per_branch[repo_branch]
-    }
+    dependencies, skipped_dependencies = resolve_dependencies(repo_path, repo_branch)
 
     # Track whether every vulnerability source completed successfully. A partial scan must not
     # cause the reconciler to close issues for vulns that simply weren't queried this run.
     scan_complete = True
+    if repo_branch in dependencies_per_branch and skipped_dependencies:
+        scan_complete = False
+        print(
+            f"Warning: {len(skipped_dependencies)} curated dependencies could not be resolved for '{repo_branch}'",
+            file=sys.stderr,
+        )
 
     ghad_vulnerabilities: list[Vulnerability] = []
     if gh_token is not None:
@@ -336,7 +398,7 @@ def main() -> int:
 
             from npm_audit import NPMAuditChecker
             print("Running npm package vulnerability audit...", file=sys.stderr)
-            npm_checker = NPMAuditChecker(repo_path, npm_timeout)
+            npm_checker = NPMAuditChecker(repo_path, npm_timeout, gh_token=gh_token)
             npm_vulnerabilities = npm_checker.check_npm_vulnerabilities(Vulnerability)
             if npm_checker.failed_packages:
                 scan_complete = False