Discovery of Paperoni v2, auto-validation, collection diff (#131)

* Add coll validate to insert validated v2 papers back to the db

* Use paperoni.mila.quebec v2 endpoint and [in]valid flags

* Add coll diff cli

Author: satyaog

config/basic.yaml

@@ -5,6 +5,7 @@ paperoni:
   api_keys:
     gemini: _
     scraperapi: _
+    paperoni_v2: _
   fetch:
     $class: RulesFetcher
     rules:
@@ -41,10 +42,17 @@ paperoni:
     author:
       score: 1
       institution_score_threshold: 1
+  autovalidate:
+    score_threshold: 10.0
   discovery:
     scrape:
       urls:
         - https://dadelani.github.io/publications
+    v2:
+      $class: paperoni.discovery.paperoni_v2:PaperoniV2
+      endpoint: https://paperoni.mila.quebec
+      token: ${paperoni.api_keys.paperoni_v2}
+      cache: ${paperoni.data_path}/paperoniv2.json
   refine:
     prompt:
       $class: paperoni.prompt:GenAIPrompt

pyproject.toml

@@ -71,6 +71,7 @@ pmlr = "paperoni.discovery.pmlr:PMLR"
 jmlr = "paperoni.discovery.jmlr:JMLR"
 synth = "paperoni.discovery.synth:Synth"
 scrape = "paperoni.discovery.scrape:Scrape"
+v2 = "paperoni.discovery.paperoni_v2:PaperoniV2"
 
 [project.entry-points."outsight.fixtures"]
 dash = "paperoni.dash:Dash"

src/paperoni/__main__.py

@@ -43,6 +43,7 @@
 from .collection.remotecoll import RemoteCollection
 from .config import config
 from .dash import History
+from .discovery.paperoni_v2 import PaperoniV2
 from .display import display, print_field, terminal_width
 from .fulltext.locate import URL, locate_all
 from .fulltext.pdf import PDF, CachePolicies, get_pdf
@@ -247,6 +248,7 @@ class Configure:
         """Configure the workset."""
 
         n: int
+        drop_zero: bool = True
         clear: bool = False
 
         async def run(self, work: "Work"):
@@ -255,14 +257,15 @@ async def run(self, work: "Work"):
                 top = deserialize(
                     Top[Scored[CommentRec[PaperWorkingSet, float]]], work_file
                 )
+                top.drop_zero = self.drop_zero
                 if self.clear:
                     top.entries = []
                 elif top.n > self.n:
                     top.entries = list(top)[: self.n]
                     top.resort()
                 top.n = self.n
             else:
-                top = Top(self.n)
+                top = Top(self.n, drop_zero=self.drop_zero)
             work.save(top)
             print(f"Configured {work_file.resolve()} for n={self.n}")
 
@@ -701,8 +704,122 @@ async def run(self, coll: "Coll"):
             elif not len(coll.collection) and not len(await coll.collection.exclusions()):
                 logging.warning("Collection is not empty. Use --force to drop it.")
 
+    @dataclass
+    class Validate:
+        """Validate the papers in the collection using the paperoni v2 database."""
+
+        # The paperoni v2 database
+        # [optional]
+        # [metavar v2]
+        paperoni_v2: Auto[PaperoniV2.query] = None
+
+        # Validate papers having a score greater than the threshold
+        # [metavar FLOAT]
+        threshold: float = None
+
+        async def iterate(
+            self, coll: "Coll" = None, **kwargs
+        ) -> AsyncGenerator[Paper, None]:
+            if self.paperoni_v2 is not None:
+                validated = 0
+                total = 0
+                async for paper_v2 in self.paperoni_v2(**kwargs):
+                    paper_v2: Paper
+                    total += 1
+
+                    if "valid" not in paper_v2.flags:
+                        continue
+
+                    validated += 1
+
+                    yield paper_v2
+
+                    send(progress=("Validated v2 papers", validated, total))
+
+                send(progress=("Validated v2 papers", None, total))
+
+            else:
+                score_threshold = self.threshold or config.autovalidate.score_threshold
+
+                async for paper in coll.collection.search():
+                    paper: Paper
+
+                    if (
+                        "valid" in paper.flags
+                        or (paper.score or config.focuses.score(paper)) < score_threshold
+                    ):
+                        continue
+
+                    yield paper
+
+        async def run(self, coll: "Coll"):
+            ignored = 0
+            validated = 0
+            count = 0
+
+            async for paper in self.iterate(coll=coll):
+                count += 1
+
+                if coll_paper := await coll.collection.find_paper(paper):
+                    if "invalid" in coll_paper.flags:
+                        ignored += 1
+                        continue
+
+                    validated += 1
+                    coll_paper.flags.add("valid")
+                    await coll.collection.edit_paper(coll_paper)
+
+                if ignored and ignored != count:
+                    send(progress=("Ignored papers", ignored, count))
+
+                send(progress=("Validated papers", validated, count))
+
+    @dataclass
+    class Diff:
+        """Diff the paper collection and another collection.
+
+        The output directory will contain two files:
+        - missing.json: Papers in the other collection that are not in the current collection
+        - extra.json: Papers in the current collection that are not in the other collection
+        """
+
+        # The other collection
+        # [positional]
+        other_collection_path: str
+
+        # Output directory
+        out: Path
+
+        # Format of the output files
+        # [alias: --fmt]
+        format: Literal["json", "yaml"] = "json"
+
+        async def run(self, coll: "Coll"):
+            other_collection = FileCollection(file=Path(self.other_collection_path))
+            missings = []
+            extras = []
+
+            async for paper in other_collection.search():
+                if not await coll.collection.find_paper(paper):
+                    missings.append(paper)
+
+            self.out.mkdir(exist_ok=True, parents=True)
+            (self.out / f"missing.{self.format}").unlink(missing_ok=True)
+            await FileCollection(file=self.out / f"missing.{self.format}").add_papers(
+                missings
+            )
+
+            async for paper in coll.collection.search():
+                if not await other_collection.find_paper(paper):
+                    extras.append(paper)
+
+            (self.out / f"extra.{self.format}").unlink(missing_ok=True)
+            await FileCollection(file=self.out / f"extra.{self.format}").add_papers(
+                extras
+            )
+
     # Command to execute
-    command: TaggedUnion[Search, Import, Export, Drop]
+    command: TaggedUnion[Search, Import, Export, Drop, Validate, Diff]
 
     # Collection dir
     # [alias: -c]

src/paperoni/collection/abc.py

@@ -1,5 +1,5 @@
 from datetime import date
-from typing import AsyncIterable, Iterable
+from typing import AsyncGenerator, Iterable
 
 from ..model.classes import Paper
 
@@ -62,7 +62,7 @@ async def search(
         include_flags: list[str] = None,
         # Flags that must be False
         exclude_flags: list[str] = None,
-    ) -> AsyncIterable[Paper]:
+    ) -> AsyncGenerator[Paper, None]:
         raise NotImplementedError()
 
     def __len__(self) -> int:

src/paperoni/collection/remotecoll.py

@@ -1,7 +1,7 @@
 import os
 from dataclasses import dataclass, field
 from datetime import date
-from typing import Iterable
+from typing import AsyncGenerator, Iterable
 
 from fastapi import HTTPException
 from serieux import deserialize
@@ -81,7 +81,7 @@ async def search(
         include_flags: list[str] = None,
         # Flags that must be False
         exclude_flags: list[str] = None,
-    ):
+    ) -> AsyncGenerator[Paper, None]:
         params = {}
         if paper_id:
             params["paper_id"] = paper_id
@@ -106,7 +106,7 @@ async def search(
         while True:
             query_params = params.copy()
             query_params["offset"] = offset
-            resp = await self.fetch.read(
+            resp: dict = await self.fetch.read(
                 url,
                 format="json",
                 cache_into=None,

src/paperoni/config.py

@@ -7,7 +7,7 @@
 import gifnoc
 from easy_oauth import OAuthManager
 from rapporteur.report import Reporter
-from serieux import JSON, TaggedSubclass
+from serieux import TaggedSubclass
 from serieux.features.encrypt import Secret
 
 from .collection.abc import PaperCollection
@@ -52,16 +52,21 @@ def __post_init__(self):
             self.process_pool = ProcessPoolExecutor(**self.process_pool_executor)
 
 
+@dataclass
+class AutoValidate:
+    score_threshold: float = 10.0
+
+
 @dataclass
 class PaperoniConfig:
     cache_path: Path = None
     data_path: Path = None
     mailto: str = ""
     api_keys: Keys[str, Secret[str]] = field(default_factory=Keys)
-    discovery: JSON = None
     fetch: TaggedSubclass[Fetcher] = field(default_factory=RequestsFetcher)
     focuses: Focuses = field(default_factory=Focuses)
     autofocus: AutoFocus[str, AutoFocus.Author] = field(default_factory=AutoFocus)
+    autovalidate: AutoValidate = field(default_factory=AutoValidate)
     refine: Refine = None
     work_file: Path = None
     collection: TaggedSubclass[PaperCollection] = None