Paper merging system (#68)

* Add AugmentedProxy, quality, merge algo
* Update dependencies and lock file

Author: breuleux

pyproject.toml

@@ -13,14 +13,18 @@ readme = "README.md"
 license = "MIT"
 requires-python = ">=3.12"
 dependencies = [
+    "ovld>=0.5.7",
+    "serieux>=0.1.1",
+    "gifnoc>=0.5.3",
     "backoff>=2.2.1",
     "beautifulsoup4>=4.13.4",
     "blessed>=1.21.0",
-    "gifnoc>=0.5.3",
     "requests>=2.32.3",
     "openreview-py>=1.50.0",
     "requests-cache>=1.2.1",
     "lxml>=6.0.0",
+    "wrapt>=1.17.2",
+    "unidecode>=1.4.0",
 ]
 
 [project.urls]

src/paperoni/model/merge.py

@@ -0,0 +1,159 @@
+from dataclasses import dataclass
+from difflib import SequenceMatcher
+from numbers import Number
+from typing import Any
+
+from ovld import Dataclass, Medley, call_next, ovld, recurse
+from ovld.dependent import HasKey
+from serieux import Context, Serieux
+from wrapt import ObjectProxy
+
+from ..model.classes import Institution, PaperAuthor
+from ..utils import associate, plainify
+
+
+@dataclass
+class Annotations:
+    quality: float
+
+
+class AugmentedProxy(ObjectProxy):
+    def __init__(self, wrapped, aug):
+        super().__init__(wrapped)
+        self._self_ann = aug
+
+    @property
+    def _(self):
+        return self._self_ann
+
+    def __repr__(self):  # pragma: no cover
+        return f"<{self.__wrapped__!r}>"
+
+
+@Serieux.extend
+class HandleProxy(Medley):
+    @ovld(priority=1)
+    def serialize(self, t: Any, obj: AugmentedProxy, ctx: Context):
+        return {
+            "$ann": recurse(Annotations, obj._self_ann, ctx),
+            "$value": recurse(t, obj.__wrapped__, ctx),
+        }
+
+    @ovld(priority=1)
+    def deserialize(self, t: Any, obj: HasKey["$ann"], ctx: Context):
+        return AugmentedProxy(
+            recurse(t, obj["$value"], ctx),
+            recurse(Annotations, obj["$ann"], ctx),
+        )
+
+
+def qual(x, q):
+    if isinstance(x, AugmentedProxy):
+        x = x.__wrapped__
+    return AugmentedProxy(x, Annotations(quality=q))
+
+
+@ovld
+def merge(x: object, y: object):
+    return recurse(x, y, 0.0, 0.0)
+
+
+@ovld(priority=10)
+def merge(x: object, y: object, qx: Number, qy: Number):
+    if qx <= -10:
+        return y
+    elif qy <= -10:
+        return x
+    else:
+        return call_next(x, y, qx, qy)
+
+
+@ovld
+def merge(x: object, y: object, qx: Number, qy: Number):
+    return x if qx > qy else y
+
+
+@ovld(priority=1)
+def merge(x: AugmentedProxy, y: object, qx: Number, qy: Number):
+    qx = x._.quality
+    return qual(recurse(x.__wrapped__, y, qx, qy), max(qx, qy))
+
+
+@ovld
+def merge(x: object, y: AugmentedProxy, qx: Number, qy: Number):
+    qy = y._.quality
+    return qual(recurse(x, y.__wrapped__, qx, qy), max(qx, qy))
+
+
+@ovld
+def merge(x: dict, y: dict, qx: Number, qy: Number):
+    main, other, qx, qy = (x, y, qx, qy) if qx >= qy else (y, x, qy, qx)
+    results = dict(other)
+    for k, v in main.items():
+        if k in other:
+            results[k] = recurse(v, other[k], qx, qy)
+        else:
+            results[k] = v
+    return results
+
+
+@ovld
+def merge(x: Dataclass, y: Dataclass, qx: Number, qy: Number):
+    return type(x)(**recurse(vars(x), vars(y), qx, qy))
+
+
+@ovld
+def merge(x: list, y: list, qx: Number, qy: Number):
+    main, other, qx, qy = (x, y, qx, qy) if qx >= qy else (y, x, qy, qx)
+    if not main:
+        return other
+    elif not other:
+        return main
+    first = x[0]
+    if isinstance(first, AugmentedProxy):
+        first = first.__wrapped__
+    return recurse(x, y, qx, qy, type(first))
+
+
+@ovld
+def merge(
+    x: list, y: list, qx: Number, qy: Number, et: type[PaperAuthor] | type[Institution]
+):
+    results = []
+    ass = associate(x, y, key=similarity, threshold=0.5)
+    for x1, x2 in ass:
+        merged = recurse(x1, x2, qx, qy) if x2 is not None else x1
+        results.append(merged)
+    return results
+
+
+@ovld
+def merge(x: list, y: list, qx: Number, qy: Number, et: type[object]):
+    return x + [a for a in y if a not in x]
+
+
+@ovld(priority=1)
+def similarity(a: AugmentedProxy, b: object):
+    return recurse(a.__wrapped__, b)
+
+
+@ovld
+def similarity(a: object, b: AugmentedProxy):
+    return recurse(a, b.__wrapped__)
+
+
+@ovld
+def similarity(a: PaperAuthor, b: PaperAuthor):
+    return similarity(a.display_name, b.display_name)
+
+
+@ovld
+def similarity(a: Institution, b: Institution):
+    return similarity(a.name, b.name)
+
+
+@ovld
+def similarity(a: str, b: str):
+    a = plainify(a)
+    b = plainify(b)
+    return SequenceMatcher(a=a, b=b).ratio()

src/paperoni/utils.py

@@ -1,5 +1,9 @@
+import itertools
+import re
 import unicodedata
 
+from unidecode import unidecode
+
 link_generators = {
     "arxiv": {
         "abstract": "https://arxiv.org/abs/{}",
@@ -111,3 +115,29 @@ def mostly_latin(s: str, threshold: float = 0.9) -> bool:
             if "LATIN" in unicodedata.name(base, ""):
                 good += 1
     return good / total >= threshold
+
+
+def plainify(name):
+    name = unidecode(name).lower()
+    name = re.sub(string=name, pattern="[()-]", repl=" ")
+    name = re.sub(string=name, pattern="['.]", repl="")
+    return name
+
+
+def associate(l1, l2, key, threshold=0):
+    el1 = list(enumerate(l1))
+    el2 = list(enumerate(l2))
+    sims = [
+        (value, i1, i2)
+        for (i1, x1), (i2, x2) in itertools.product(el1, el2)
+        if (value := key(x1, x2)) > threshold
+    ]
+    sims.sort(key=lambda tup: -tup[0])
+    mapping = {}
+    n = len(l1)
+    for _, i1, i2 in sims:
+        if i1 not in mapping:
+            mapping[i1] = l2[i2]
+        if len(mapping) == n:
+            break
+    return [(x1, mapping.get(i1, None)) for i1, x1 in el1]

tests/model/test_merge.py

@@ -0,0 +1,157 @@
+from dataclasses import dataclass
+
+from serieux import deserialize, serialize
+
+from paperoni.model.classes import (
+    Author,
+    Institution,
+    InstitutionCategory,
+    Link,
+    PaperAuthor,
+)
+from paperoni.model.merge import merge, qual, similarity
+
+
+@dataclass
+class Point:
+    x: int
+    y: int
+
+
+@dataclass
+class Person:
+    name: str
+    job: str
+
+
+def test_similarity():
+    assert similarity("bonjour", "bonjour") == 1
+    assert similarity("bon jour", "bon-jour.") == 1
+
+
+def test_similarity_2():
+    assert similarity("Hugo Larochelle", "Marc Bellemare") < 0.5
+
+
+def test_augment():
+    pt = Point(qual(3, 2.5), 4)
+    ser = serialize(Point, pt)
+    assert ser == {"x": {"$ann": {"quality": 2.5}, "$value": 3}, "y": 4}
+    deser = deserialize(Point, ser)
+    assert deser.x._.quality == 2.5
+
+
+def test_merge_dicts():
+    d1 = {"a": 1, "b": 2}
+    d2 = {"c": 3}
+
+    assert merge(d1, d2) == {"a": 1, "b": 2, "c": 3}
+    assert merge(d1, qual(d2, -10)) == {"a": 1, "b": 2}
+    assert merge(qual(d1, -10), d2) == {"c": 3}
+
+
+def test_merge():
+    p1 = Person(name=qual("John", 2), job="Carpenter")
+    p2 = Person(name=qual("Johnny", 1), job=qual("Lawyer", 3))
+    p12 = merge(p1, p2)
+    assert p12 == Person(name="John", job="Lawyer")
+
+    p3 = Person(name=qual("NO!", 1.5), job=qual("Philosopher", 4))
+    p123 = merge(p12, p3)
+    assert p123 == Person(name="John", job="Philosopher")
+
+    p4 = qual(Person(name="Gunther", job="Unemployed"), 3)
+    p34 = merge(p3, p4)
+    assert p34 == Person(name="Gunther", job="Philosopher")
+
+
+def test_merge_lists():
+    p1 = Person(name="John", job="Carpenter")
+    p2 = Person(name="John", job=qual("Lawyer", 3))
+    p3 = qual(Person(name="Gunther", job="Unemployed"), 3)
+
+    l1 = [p3, p1]
+    l2 = [p2, p1]
+
+    assert merge(l1, l2) == [p3, p1, p2]
+
+
+def test_merge_lists_empty():
+    l1 = [1, 2]
+    l2 = []
+    assert merge(l1, l2) == l1
+    assert merge(l2, l1) == l1
+
+
+def test_merge_author_lists():
+    p1 = PaperAuthor(
+        display_name="John",
+        author=Author(name="John", links=[Link(type="job", link="baker")]),
+    )
+    p2 = PaperAuthor(
+        display_name="John",
+        author=Author(name="John", links=[Link(type="hair", link="yes")]),
+    )
+    p3 = qual(
+        PaperAuthor(display_name="Gunther", author=Author(name="Gunther", links=[])), 3
+    )
+
+    l1 = [p3, p1]
+    l2 = [p2]
+
+    assert merge(l1, l2) == [
+        PaperAuthor(display_name="Gunther", author=Author(name="Gunther", links=[])),
+        PaperAuthor(
+            display_name="John",
+            author=Author(
+                name="John",
+                links=[Link(type="job", link="baker"), Link(type="hair", link="yes")],
+            ),
+        ),
+    ]
+
+
+def test_merge_author_lists_similarity():
+    p1 = PaperAuthor(
+        display_name="J. Smith",
+        author=Author(name="J.", links=[Link(type="job", link="baker")]),
+    )
+    p2 = qual(
+        PaperAuthor(
+            display_name="John Smith",
+            author=Author(name="John Smith", links=[Link(type="hair", link="yes")]),
+        ),
+        2,
+    )
+    p3 = qual(
+        PaperAuthor(display_name="Gunther", author=Author(name="Gunther", links=[])), 3
+    )
+
+    l1 = [p3, p1]
+    l2 = [p2]
+
+    assert merge(l1, l2) == [
+        PaperAuthor(display_name="Gunther", author=Author(name="Gunther", links=[])),
+        PaperAuthor(
+            display_name="John Smith",
+            author=Author(
+                name="John Smith",
+                links=[Link(type="hair", link="yes"), Link(type="job", link="baker")],
+            ),
+        ),
+    ]
+
+
+def test_merge_institution_lists():
+    i1 = Institution(name="MIT")
+    i2 = qual(Institution(name="MIT", category=InstitutionCategory.academia), 2)
+    i3 = qual(Institution(name="Stanford University"), 3)
+
+    l1 = [i3, i1]
+    l2 = [i2]
+
+    merged = merge(l1, l2)
+    assert merged == [
+        Institution(name="Stanford University"),
+        Institution(name="MIT", category=InstitutionCategory.academia),
+    ]