Raise an error when a requested core version can't be found (and accept v-prefixed versions) (#121)

* update-logic-to-fetch-core-meta

* update-docs

* add-tests

* handle-npm-style-versions

* fix-ci

* debug-ci

* fix-github-tags-from-wildcard-versions

Author: Darshan808

jupyter_builder/core_path.py

@@ -18,6 +18,12 @@
 
 _MAX_CORE_META_BYTES = 5 * 1024 * 1024  # 5 MB — generous upper bound for core.package.json
 
+#: GitHub API endpoint used to resolve wildcard versions to concrete release tags
+_GITHUB_TAGS_API_URL = "https://api.github.com/repos/jupyterlab/jupyterlab/tags"
+
+#: Upper bound on tag-list pages fetched when resolving a wildcard (100 tags per page)
+_MAX_GITHUB_TAG_PAGES = 10
+
 
 def _home_dir() -> Path:
     home = os.environ.get("HOME")
@@ -56,26 +62,73 @@ def get_core_meta(
                     "@jupyterlab/builder.\n \033[0m",
                 )
         requested_version = "main"
+    else:
+        # Accept both "vX.Y.Z" and "X.Y.Z" for an explicitly requested version.
+        requested_version = _normalize_version(requested_version)
 
     cache_root = _home_dir() / ".cache" / "jupyterlab_builder" / "core"
     cached_file = _get_cached_core_meta_file(cache_root, requested_version)
     if cached_file is not None:
         return str(cached_file)
 
-    # Try to retrieve core meta from npm first
+    # Try to retrieve core meta from npm first, then fall back to GitHub. If the
+    # requested version cannot be found in either source, raise an error.
     try:
         npm_version = _resolve_npm_version(requested_version)
         npm_cache_file = cache_root / npm_version / "core.package.json"
         if npm_cache_file.exists():
             return str(npm_cache_file)
         _download_npm_core_meta(npm_version, npm_cache_file)
         return str(npm_cache_file)
-    except urllib.error.URLError:
-        pass  # Fallback to GitHub below
-
-    github_cache_file = cache_root / requested_version / "core.package.json"
-    _download_github_core_meta(requested_version, github_cache_file)
-    return str(github_cache_file)
+    except urllib.error.URLError as npm_error:
+        try:
+            # Wildcards (e.g. "4.5.x") have no single git ref, so resolve them
+            # to a concrete tag from the GitHub tag list before downloading.
+            github_version = (
+                _resolve_wildcard_github_version(requested_version)
+                if _is_wildcard_version(requested_version)
+                else requested_version
+            )
+            github_cache_file = cache_root / github_version / "core.package.json"
+            if github_cache_file.exists():
+                return str(github_cache_file)
+            _download_github_core_meta(_github_ref(github_version), github_cache_file)
+        except urllib.error.URLError as github_error:
+            msg = (
+                f"Could not resolve @jupyterlab/core-meta for requested version "
+                f"{requested_version!r}: not found on the npm registry "
+                f"({npm_error}) or in the jupyterlab/jupyterlab GitHub repository "
+                f"({github_error}). Verify that the version exists "
+                f"(both '4.5.7' and 'v4.5.7' are accepted)."
+            )
+            raise RuntimeError(msg) from github_error
+        return str(github_cache_file)
+
+
+def _normalize_version(version: str) -> str:
+    """Strip a leading 'v' from a numeric version so 'vX.Y.Z' and 'X.Y.Z' are equivalent."""
+    return version[1:] if re.match(r"v\d", version) else version
+
+
+def _github_ref(version: str) -> str:
+    """Map a resolved version to its jupyterlab/jupyterlab git ref.
+
+    Numeric releases are published as git tags. Stable releases are tagged like
+    'v4.5.7', while npm-style prereleases such as '4.6.0-alpha.4' correspond to
+    the PEP 440 tag form JupyterLab uses, 'v4.6.0a4'. Branch names such as
+    'main' (and other non-numeric refs) are used verbatim.
+    """
+    if not re.match(r"\d", version):
+        return version
+    release, separator, prerelease = version.partition("-")
+    if separator:
+        # Translate npm prerelease identifiers (alpha.4 / beta.1 / rc.2) to the
+        # PEP 440 form JupyterLab tags use (a4 / b1 / rc2).
+        prerelease = re.sub(r"alpha\.?", "a", prerelease)
+        prerelease = re.sub(r"beta\.?", "b", prerelease)
+        prerelease = re.sub(r"rc\.", "rc", prerelease)
+        version = release + prerelease
+    return f"v{version}"
 
 
 def _is_wildcard_version(version: str) -> bool:
@@ -126,14 +179,53 @@ def _resolve_wildcard_npm_version(version: str) -> str:
         msg = f"No published @jupyterlab/core-meta versions match range '{version}'"
         raise urllib.error.URLError(msg)
 
-    def semver_key(v: str) -> tuple[tuple[int, ...], int, tuple[int, ...]]:
-        release, _, prerelease = v.partition("-")
-        numeric = tuple(int(p) for p in release.split(".") if p.isdigit())
-        # Stable releases sort higher than pre-releases; within pre-releases,
-        pre_numeric = tuple(int(p) for p in prerelease.split(".") if p.isdigit())
-        return (numeric, 0 if prerelease else 1, pre_numeric)
+    return max(matching, key=_semver_key)
+
+
+def _semver_key(v: str) -> tuple[tuple[int, ...], int, tuple[int, ...]]:
+    release, _, prerelease = v.partition("-")
+    numeric = tuple(int(p) for p in release.split(".") if p.isdigit())
+    # Stable releases sort higher than pre-releases; within pre-releases,
+    # order by the numeric identifiers (e.g. alpha.3 < alpha.4).
+    pre_numeric = tuple(int(p) for p in prerelease.split(".") if p.isdigit())
+    return (numeric, 0 if prerelease else 1, pre_numeric)
+
+
+def _resolve_wildcard_github_version(version: str) -> str:
+    """Resolve a wildcard range like '4.5.x' to the highest matching git tag.
+
+    JupyterLab publishes stable releases as git tags (e.g. 'v4.5.9') that may
+    predate @jupyterlab/core-meta on npm, so wildcards that npm cannot satisfy
+    are resolved here against the jupyterlab/jupyterlab tag list.
+
+    Raises urllib.error.URLError if no matching tag is found.
+    """
+    escaped = re.escape(version)
+    wildcard_pattern = re.sub(r"x", r"\\d+", escaped, flags=re.IGNORECASE)
+    pattern = re.compile("^" + wildcard_pattern + r"$")
+
+    matching: list[str] = []
+    for page in range(1, _MAX_GITHUB_TAG_PAGES + 1):
+        data = _http_get(f"{_GITHUB_TAGS_API_URL}?per_page=100&page={page}")
+        tags = json.loads(data)
+        if not tags:
+            break
+        page_matches = [
+            normalized
+            for tag in tags
+            if (normalized := _normalize_version(tag.get("name", ""))) and pattern.match(normalized)
+        ]
+        # Tags are returned newest-first, so once matches stop appearing
+        # (after they have started) the rest are older and can be skipped.
+        if matching and not page_matches:
+            break
+        matching.extend(page_matches)
+
+    if not matching:
+        msg = f"No jupyterlab/jupyterlab git tags match range '{version}'"
+        raise urllib.error.URLError(msg)
 
-    return max(matching, key=semver_key)
+    return max(matching, key=_semver_key)
 
 
 def _get_cached_core_meta_file(cache_root: Path, version: str) -> Path | None:

jupyter_builder/extension_commands/build.py

@@ -42,7 +42,9 @@ class BuildLabExtensionApp(BaseExtensionApp):
         "",
         config=True,
         help=(
-            "Version of JupyterLab core to use when building (ignored if core-package-file is set)"
+            "Version of JupyterLab core to use when building, e.g. 'X.Y.Z' or 'vX.Y.Z' "
+            "(ignored if core-package-file is set). Building fails if the version cannot "
+            "be resolved."
         ),
     )
 

jupyter_builder/extension_commands/watch.py

@@ -34,7 +34,9 @@ class WatchLabExtensionApp(BaseExtensionApp):
         "",
         config=True,
         help=(
-            "Version of JupyterLab core to use when watching (ignored if core-package-file is set)"
+            "Version of JupyterLab core to use when watching, e.g. 'X.Y.Z' or 'vX.Y.Z' "
+            "(ignored if core-package-file is set). Watching fails if the version cannot "
+            "be resolved."
         ),
     )
 

tests/test_core_path.py

@@ -126,9 +126,106 @@ def test_get_core_meta_falls_back_to_github_when_npm_fails(tmp_path, monkeypatch
     ext_path.mkdir()
     monkeypatch.setenv("HOME", str(tmp_path))
 
+    # JupyterLab releases are published as git tags like "v4.5.7", so a numeric
+    # version requested from GitHub must be looked up with the "v" prefix.
     github_url = (
         "https://raw.githubusercontent.com/"
-        "jupyterlab/jupyterlab/4.6.0-alpha.4/"
+        "jupyterlab/jupyterlab/v4.5.7/"
+        "jupyterlab/staging/package.json"
+    )
+    calls = []
+
+    def fake_urlopen(req_or_url, **_kwargs):
+        url = getattr(req_or_url, "full_url", req_or_url)
+        calls.append(url)
+        if url == f"{core_path.JPBLD_NPM_URL}/@jupyterlab/core-meta/4.5.7":
+            msg = "Not Found"
+            raise urllib.error.URLError(msg)
+        if url == github_url:
+            return io.BytesIO(b'{"dependencies": {}}')
+        msg = f"Unexpected URL {url}"
+        raise AssertionError(msg)
+
+    monkeypatch.setattr(core_path.urllib.request, "urlopen", fake_urlopen)
+
+    location = core_path.get_core_meta(version="4.5.7", ext_path=ext_path)
+
+    assert calls == [
+        f"{core_path.JPBLD_NPM_URL}/@jupyterlab/core-meta/4.5.7",
+        github_url,
+    ]
+    assert location == str(
+        tmp_path / ".cache" / "jupyterlab_builder" / "core" / "4.5.7" / "core.package.json",
+    )
+
+
+def test_get_core_meta_accepts_v_prefixed_version(tmp_path, monkeypatch):
+    """A 'v'-prefixed version is normalized so it resolves identically to the bare form."""
+    ext_path = tmp_path / "ext"
+    ext_path.mkdir()
+    monkeypatch.setenv("HOME", str(tmp_path))
+
+    github_url = (
+        "https://raw.githubusercontent.com/"
+        "jupyterlab/jupyterlab/v4.5.7/"
+        "jupyterlab/staging/package.json"
+    )
+    calls = []
+
+    def fake_urlopen(req_or_url, **_kwargs):
+        url = getattr(req_or_url, "full_url", req_or_url)
+        calls.append(url)
+        if url == f"{core_path.JPBLD_NPM_URL}/@jupyterlab/core-meta/4.5.7":
+            msg = "Not Found"
+            raise urllib.error.URLError(msg)
+        if url == github_url:
+            return io.BytesIO(b'{"dependencies": {}}')
+        msg = f"Unexpected URL {url}"
+        raise AssertionError(msg)
+
+    monkeypatch.setattr(core_path.urllib.request, "urlopen", fake_urlopen)
+
+    location = core_path.get_core_meta(version="v4.5.7", ext_path=ext_path)
+
+    # Both the npm lookup and the cache directory use the normalized "4.5.7".
+    assert calls == [
+        f"{core_path.JPBLD_NPM_URL}/@jupyterlab/core-meta/4.5.7",
+        github_url,
+    ]
+    assert location == str(
+        tmp_path / ".cache" / "jupyterlab_builder" / "core" / "4.5.7" / "core.package.json",
+    )
+
+
+@pytest.mark.parametrize(
+    ("version", "expected_ref"),
+    [
+        ("4.5.7", "v4.5.7"),
+        ("4.6.0-alpha.4", "v4.6.0a4"),
+        ("4.6.0-beta.1", "v4.6.0b1"),
+        ("4.6.0-rc.2", "v4.6.0rc2"),
+        ("main", "main"),
+        ("some-branch", "some-branch"),
+    ],
+)
+def test_github_ref_maps_versions_to_git_tags(version, expected_ref):
+    assert core_path._github_ref(version) == expected_ref
+
+
+@pytest.mark.parametrize("version", ["4.5.7", "v4.5.7"])
+def test_normalize_version_accepts_both_forms(version):
+    assert core_path._normalize_version(version) == "4.5.7"
+
+
+def test_get_core_meta_falls_back_to_github_for_prerelease(tmp_path, monkeypatch):
+    """An npm-style prerelease that npm lacks is fetched from the matching git tag."""
+    ext_path = tmp_path / "ext"
+    ext_path.mkdir()
+    monkeypatch.setenv("HOME", str(tmp_path))
+
+    github_url = (
+        "https://raw.githubusercontent.com/"
+        "jupyterlab/jupyterlab/v4.6.0a4/"
         "jupyterlab/staging/package.json"
     )
     calls = []
@@ -157,6 +254,78 @@ def fake_urlopen(req_or_url, **_kwargs):
     )
 
 
+def test_get_core_meta_wildcard_resolves_from_github_when_npm_has_no_match(tmp_path, monkeypatch):
+    """A wildcard that npm cannot satisfy is resolved against GitHub release tags."""
+    ext_path = tmp_path / "ext"
+    ext_path.mkdir()
+    monkeypatch.setenv("HOME", str(tmp_path))
+
+    tags_page1 = f"{core_path._GITHUB_TAGS_API_URL}?per_page=100&page=1"
+    tags_page2 = f"{core_path._GITHUB_TAGS_API_URL}?per_page=100&page=2"
+    # 4.5.10 must win over 4.5.2 — confirms numeric (not lexical) ordering.
+    github_url = (
+        "https://raw.githubusercontent.com/"
+        "jupyterlab/jupyterlab/v4.5.10/"
+        "jupyterlab/staging/package.json"
+    )
+    calls = []
+
+    def fake_urlopen(req_or_url, **_kwargs):
+        url = getattr(req_or_url, "full_url", req_or_url)
+        calls.append(url)
+        if url == f"{core_path.JPBLD_NPM_URL}/@jupyterlab/core-meta":
+            # npm publishes no 4.5.x, so the npm wildcard lookup finds no match.
+            return io.BytesIO(json.dumps({"versions": {"4.6.0": {}}}).encode())
+        if url == tags_page1:
+            return io.BytesIO(
+                json.dumps(
+                    [
+                        {"name": "v4.6.0"},
+                        {"name": "v4.5.2"},
+                        {"name": "v4.5.10"},
+                        {"name": "v4.5.1"},
+                    ],
+                ).encode(),
+            )
+        if url == tags_page2:
+            # Older tags with no 4.5.x match — resolution stops here.
+            return io.BytesIO(json.dumps([{"name": "v4.4.9"}]).encode())
+        if url == github_url:
+            return io.BytesIO(b'{"dependencies": {}}')
+        msg = f"Unexpected URL {url}"
+        raise AssertionError(msg)
+
+    monkeypatch.setattr(core_path.urllib.request, "urlopen", fake_urlopen)
+
+    location = core_path.get_core_meta(version="4.5.x", ext_path=ext_path)
+
+    assert calls == [
+        f"{core_path.JPBLD_NPM_URL}/@jupyterlab/core-meta",
+        tags_page1,
+        tags_page2,
+        github_url,
+    ]
+    assert location == str(
+        tmp_path / ".cache" / "jupyterlab_builder" / "core" / "4.5.10" / "core.package.json",
+    )
+
+
+def test_get_core_meta_raises_when_requested_version_is_unresolvable(tmp_path, monkeypatch):
+    """An explicitly requested version that exists nowhere must fail loudly."""
+    ext_path = tmp_path / "ext"
+    ext_path.mkdir()
+    monkeypatch.setenv("HOME", str(tmp_path))
+
+    def fake_urlopen(*_args, **_kwargs):
+        msg = "Not Found"
+        raise urllib.error.URLError(msg)
+
+    monkeypatch.setattr(core_path.urllib.request, "urlopen", fake_urlopen)
+
+    with pytest.raises(RuntimeError, match="Could not resolve @jupyterlab/core-meta"):
+        core_path.get_core_meta(version="9.9.9", ext_path=ext_path)
+
+
 def test_get_core_meta_wildcard_version_resolves_from_npm_and_downloads_core_meta(
     tmp_path,
     monkeypatch,

tests/test_tpl.py

@@ -292,10 +292,11 @@ def test_builder_version_mismatch(tmp_path):
             text=True,
         )
     # Check if the expected error message is in the output
+    output = excinfo.value.stderr
     assert re.search(
         (
-            r"ValueError: Extensions require a devDependency on @jupyterlab/builder@\^.+?, "
+            r"ValueError: Extensions require a devDependency on @jupyterlab/builder@\^[^,]+, "
             r"you have a dependency on 4\.0\.0"
         ),
-        excinfo.value.stderr,
+        output,
     ), "Expected version mismatch error message not found in output!"