Add a collection visualisation page

Author: satyaog

config/autofocuses.yaml

@@ -8,15 +8,21 @@ paperoni:
     paperoni_v2: _
   fetch:
     $class: RulesFetcher
+    simultaneous:
+      "*": 3
     rules:
+      "^https://api.ror.org/": standard
       "^https://export.arxiv.org/": standard
       "^https://api.crossref.org/": cached
       "^https://api.semanticscholar.org/": cached
+      "^https://api.openalex.org/": cached
+      "^https://api.datacite.org/": cached
+      "^https://api.unpaywall.org/": cached
       "^https://www.biorxiv.org/": sequence
       ".*": sequence
     fetchers:
       standard:
-        $class: RequestsFetcher
+        $class: HTTPXFetcher
         user_agent: chrome
       cached:
         $class: CachedFetcher
@@ -28,20 +34,20 @@ paperoni:
       sequence:
         $class: SequenceFetcher
         fetchers:
+        - $class: HTTPXFetcher
+          user_agent: chrome
         - $class: CloudFlareFetcher
           user_agent: chrome
         - $class: ScraperAPIFetcher
           cache_path: ${paperoni.cache_path}/scraper-cache.sql
           expire_after: 100d
           api_key: ${paperoni.api_keys.scraperapi}
-  focuses:
-    $include:
-    - focuses.yaml
-    - autofocuses.yaml
+  focuses: focuses.yaml
   autofocus:
     author:
       score: 1
       institution_score_threshold: 1
+      threshold: 1
   autovalidate:
     score_threshold: 10.0
   discovery:
@@ -68,6 +74,9 @@ paperoni:
     $class: paperoni.collection.filecoll:FileCollection
     file: ${paperoni.data_path}/collection.json
   server:
+    host: localhost
+    port: 8000
+    protocol: http
     max_results: 10000
     auth:
       server_metadata_url: https://accounts.google.com/.well-known/openid-configuration
@@ -89,4 +98,4 @@ paperoni:
           user: []
           search: [user]
           validate: [search]
-          dev: []
+          dev: [validate]

config/basic.yaml

@@ -63,6 +63,9 @@ dev = [
     "pytest-asyncio>=1.3.0",
     "ruff>=0.11.3",
 ]
+lint = [
+    "ruff>=0.11.3",
+]
 
 [project.entry-points."paperoni.discovery"]
 semantic_scholar = "paperoni.discovery.semantic_scholar:SemanticScholar"
@@ -150,6 +153,19 @@ mongo = [
     "hatch run mongo:stop",
 ]
 
+[tool.hatch.envs.lint]
+installer = "uv"
+detached = true
+dependency-groups = ["lint"]
+
+[tool.hatch.envs.lint.scripts]
+uv = 'UV_PROJECT_ENVIRONMENT="{env:VIRTUAL_ENV}" "$(which uv)" {args}'
+lint = [
+    "uv sync --no-dev --no-install-workspace --group lint",
+    "ruff check --fix",
+    "ruff format",
+]
+
 [tool.hatch.envs.pixi]
 detached = true
 installer = "uv"

pyproject.toml

@@ -553,7 +553,7 @@ async def run(self, work: "Work"):
     # [alias: -f]
     focus_file: Path = None
 
-    # Collection dir
+    # Collection file
     # [alias: -c]
     collection_file: Path = None
 
@@ -848,17 +848,17 @@ async def run(self, coll: "Coll"):
     # Command to execute
     command: TaggedUnion[Search, Import, Export, Drop, Validate, Diff, Operate]
 
-    # Collection dir
+    # Collection string. Can be a remote collection URL or a path.
     # [alias: -c]
-    collection_path: str = None
+    collection_path: Path = None
 
     @cached_property
     def collection(self) -> PaperCollection:
         if self.collection_path:
-            if self.collection_path.startswith("http"):
-                return RemoteCollection(endpoint=self.collection_path)
+            if str(self.collection_path).startswith("http"):
+                return RemoteCollection(endpoint=str(self.collection_path))
             else:
-                return FileCollection(file=Path(self.collection_path))
+                return FileCollection(file=self.collection_path)
         else:
             return config.collection
 
@@ -914,7 +914,7 @@ async def run(self, focus: "Focus"):
     # [option: -f]
     focuses: Focuses @ FileProxy() = None
 
-    # Collection dir
+    # Collection file
     # [alias: -c]
     collection_file: Path = None
 
@@ -947,6 +947,10 @@ class Serve:
     # Whether to enable auth
     auth: bool = True
 
+    # Collection file
+    # [alias: -c]
+    collection_file: Path = None
+
     async def run(self):
         from .web import create_app
 
@@ -959,6 +963,11 @@ async def run(self):
                     "__admin__": ["admin"]
                 },
             }
+        if self.collection_file:
+            overrides["paperoni.collection"] = {
+                "$class": f"{FileCollection.__module__}:{FileCollection.__qualname__}",
+                "file": str(self.collection_file.resolve()),
+            }
         with gifnoc.overlay(overrides):
             app = create_app()
             ssl_config = config.server.ssl

src/paperoni/__main__.py

@@ -1,3 +1,5 @@
+import stat
+import warnings
 from dataclasses import dataclass, field
 from pathlib import Path
 
@@ -10,10 +12,22 @@
 @dataclass(kw_only=True)
 class FileCollection(MemCollection):
     file: Path = field(compare=False)
+    read_only: bool = False
 
     def __post_init__(self):
         ann = FileProxy(default_factory=PaperIndex, refresh=True)
         self._index = deserialize(PaperIndex @ ann, str(self.file))
 
+        # Check if file is read-only
+        if self.read_only or (
+            self.file.exists() and self.file.stat().st_mode & stat.S_IWRITE == 0
+        ):
+            self.read_only = True
+
     def _commit(self) -> None:
+        if self.read_only:
+            warnings.warn(
+                f"Collection {self.file} is open in read-only mode, skipping commit."
+            )
+            return
         self._index.save()

src/paperoni/collection/filecoll.py

@@ -1,8 +1,8 @@
 import { html } from './common.js';
 import {
     createAuthorsSection,
-    createReleasesSection,
     createDetailsSection,
+    createReleasesSection,
     getScoreClass
 } from './paper.js';
 
@@ -30,6 +30,22 @@ async function fetchWorksets(offset = 0, size = 100) {
     return await response.json();
 }
 
+async function fetchCollection(offset = 0, limit = 100) {
+    const queryParams = new URLSearchParams({
+        offset: offset.toString(),
+        limit: limit.toString(),
+    });
+
+    const url = `/api/v1/search?${queryParams.toString()}`;
+    const response = await fetch(url);
+
+    if (!response.ok) {
+        throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    return await response.json();
+}
+
 function createInfoValue(value) {
     if (value === null || value === undefined) {
         return html`<span class="info-null">null</span>`;
@@ -105,11 +121,11 @@ function createWorksetPaperElement(paper) {
     `;
 }
 
-function createWorksetElement(scoredWorkset, index) {
-    // scoredWorkset is Scored[PaperWorkingSet]
+function createWorksetElement(scoredWorksetOrPaper, index) {
+    // scoredWorkset is Scored[PaperWorkingSet] or Paper
     // It has: { score: float, value: PaperWorkingSet }
-    const score = scoredWorkset.score;
-    const workset = scoredWorkset.value;
+    const score = scoredWorksetOrPaper.score;
+    const workset = scoredWorksetOrPaper.value ?? { current: scoredWorksetOrPaper, collected: [scoredWorksetOrPaper] };
     const current = workset.current;
     const collected = workset.collected || [];
 
@@ -340,3 +356,15 @@ export async function displayWorksets() {
         displayError(error);
     }
 }
+
+export async function displayCollection() {
+    displayLoading();
+
+    try {
+        const data = await fetchCollection();
+        renderWorksets(data);
+    } catch (error) {
+        console.error('Failed to load collection:', error);
+        displayError(error);
+    }
+}

src/paperoni/web/assets/workset.js

@@ -65,4 +65,19 @@ async def exclusions_page(request: Request):
             validation_buttons=False,
         )
 
+    @app.get("/visualize")
+    async def visualize_page(
+        request: Request,
+        user: str = Depends(hascap("dev", redirect=True)),
+    ):
+        """Render the collection visualization page."""
+        validate = deserialize(app.auth.capabilities.captype, "validate")
+        is_validator = app.auth.capabilities.check(user, validate)
+        return render_template(
+            "visualize.html",
+            request,
+            is_validator=is_validator,
+            validation_buttons=False,
+        )
+
     return app

src/paperoni/web/search.py

@@ -0,0 +1,18 @@
+{% extends "main.html" %}
+
+{% block title %}Visualize{% endblock %}
+
+{% block page_title %}Visualize{% endblock %}
+
+{% block content %}
+<div class="workset-container">
+    <div id="worksetContainer"></div>
+</div>
+{% endblock %}
+
+{% block scripts %}
+<script type="module">
+    import { displayCollection } from '/assets/workset.js';
+    displayCollection();
+</script>
+{% endblock %}