WIP Moving Assessments and Comparisons to UnnamedDataFrame

Author: lenhoanglnh

solidago/src/solidago/primitives/datastructure/unnamed_dataframe.py

@@ -9,6 +9,8 @@
 
 
 class UnnamedDataFrame(DataFrame):
+    row_cls: Optional[type]=None
+    
     def __init__(self, 
         key_names: Optional[Union[str, list[str]]]=None, 
         value_names: Optional[Union[str, list[str]]]=None,
@@ -19,14 +21,15 @@ def __init__(self,
         **kwargs
     ):
         """ Defines a DataFrame wrapper """
-        super().__init__(*args, **kwargs)
         to_list = lambda l: [l] if isinstance(l, str) else l
+        key_names, value_names = to_list(key_names), to_list(value_names)
+        columns = sum([ n if n else list() for n in (key_names, value_names) ], list())
+        super().__init__(*args, **kwargs, columns=columns)
         self.meta = SimpleNamespace()
         self.meta.name = name
-        self.meta.key_names, self.meta.value_names = to_list(key_names), to_list(value_names)
+        self.meta.key_names, self.meta.value_names = key_names, value_names
         assert isinstance(self.key_names, list) or not self.key_names
         assert isinstance(self.value_names, list) or not self.value_names
-        columns = sum([ n if n else list() for n in (self.key_names, self.value_names) ], list())
         for column in columns:
             if column not in self.columns:
                 self[column] = float("nan")
@@ -46,17 +49,6 @@ def value_names(self):
     def default_value(self) -> Any:
         return self.meta._default_value
 
-    def value2row(self, value: Optional[Any]=None, **kwargs) -> Series:
-        if value is None:
-            value = dict()
-        elif isinstance(value, (dict, Series)):
-            value = { key: v for key, v in value.items() }
-        elif isinstance(value, Iterable):
-            value = { self.value_names[index]: v for index, v in enumerate(value) }
-        else:
-            value = { self.value_names[0]: value }
-        return Series(kwargs | value)
-    
     def row2key(self, row: Series) -> Any:
         if not self.key_names:
             return row
@@ -66,7 +58,7 @@ def row2key(self, row: Series) -> Any:
 
     def row2value(self, row: Series) -> Any:
         if not self.value_names:
-            return row
+            return row if row_cls is None else row_cls(row)
         if len(self.value_names) == 1:
             return row[self.value_names[0]]
         return tuple( row[name] for name in self.value_names )
@@ -75,44 +67,51 @@ def df2value(self, df: DataFrame, last_only: Optional[bool]=None) -> Any:
         last_only = self.meta._last_only if last_only is None else last_only
         if last_only:
             return self.row2value(df.iloc[-1])
-        return df
+        return type(self)(df)
 
     """ The following methods are are more standard """
-    def add_row(self, value: Optional[Any]=None, **kwargs) -> None:
+    def input2dict(self, *args, keys_only: bool=False, **kwargs) -> dict:
+        """ args is assumed to list keys and then values, 
+        though some may be specified through kwargs """
+        key_value_columns = self.key_names if keys_only else (self.key_names + self.value_names)
+        assert len(args) <= len(key_value_columns)
+        assert all({ key not in key_value_columns[:len(args)] for key in kwargs })
+        assert (not keys_only) or all({ key in self.key_names for key in kwargs })
+        to_value = lambda v, k: str(v) if k in self.key_names else v
+        kwargs = { k: to_value(v, k) for k, v in kwargs.items() }
+        return kwargs | { k: to_value(v, k) for k, v in zip(key_value_columns[:len(args)], args) }
+    
+    def add_row(self, *args, **kwargs) -> None:
         self.index = list(range(len(self)))
-        kwargs = { k: (str(v) if k in self.key_names else v) for k, v in kwargs.items() }
-        self.loc[len(self)] = Series(kwargs) if value is None else self.value2row(value, **kwargs)
+        self.loc[len(self)] = Series(self.input2dict(*args, **kwargs))
 
     def get(self, 
         *args, 
         process: bool=True, 
         last_only: Optional[bool]=None, 
         **kwargs
     ) -> Union["UnnamedDataFrame", tuple]:
-        assert len(args) <= len(self.key_names)
-        assert all({ key not in self.key_names[:len(args)] for key in kwargs })
-        kwargs = { k: str(v) for k, v in kwargs.items() }
-        kwargs |= { key: str(value) for key, value in zip(self.key_names[:len(args)], args) }
+        kwargs = self.input2dict(*args, keys_only=True, **kwargs)
         df = self[reduce(lambda a, x: a & x, [ self[k] == v for k, v in kwargs.items() ], True)]
-        key_names = [ n for n in self.key_names if n not in kwargs ]
+        key_names = [ key_name for key_name in self.key_names if key_name not in kwargs ]
         if key_names or not process:
             return type(self)(df, key_names=key_names)
         return self.default_value if df.empty else self.df2value(df, last_only)
 
     def __contains__(self, *args, **kwargs) -> bool:
-        return not self.get(*args, **kwargs).empty
+        return not self.get(*args, process=False, **kwargs).empty
 
-    def set(self, value: Optional[Any]=None, *args, **kwargs) -> None:
-        assert len(args) <= len(self.key_names)
-        assert all({ key not in self.key_names[:len(args)] for key in kwargs })
-        kwargs = { k: str(v) for k, v in kwargs.items() }
-        kwargs |= { k: str(v) for k, v in zip(self.key_names[:len(args)], args) }
-        df = self.get(process=False, **kwargs)
+    def set(self, *args, **kwargs) -> None:
+        """ args is assumed to list keys and then values, 
+        though some may be specified through kwargs """
+        kwargs_keys_only = self.input2dict(*args[:len(self.key_names)], **kwargs)
+        kwargs = self.input2dict(*args, **kwargs)
+        df = self.get(process=False, **kwargs_keys_only)
         if df.empty:
-            self.add_row(value, **kwargs)
+            self.add_row(**kwargs)
         else: # Updates the last row of df
             name = df.iloc[-1].name
-            self.loc[name] = self.value2row(value, **kwargs)
+            self.loc[name] = Series(kwargs)
 
     def __or__(self, other: "UnnamedDataFrame") -> "UnnamedDataFrame":
         return type(self)(pd.concat([self, other]))
@@ -122,11 +121,26 @@ def load(cls, filename: str) -> "UnnamedDataFrame":
         try: return cls(pd.read_csv(filename, keep_default_na=False))
         except pd.errors.EmptyDataError: return cls()
 
+    def last_only(self) -> "UnnamedDataFrame":
+        return type(self)(
+            data=[ row for _, row in self.iter(process=False, last_only=True) ],
+            key_names=self.key_names,
+            value_names=self.value_names,
+            name=self.meta.name, 
+            default_value=self.meta._default_value,
+            last_only=self.meta._last_only,
+        )
+    
     def groupby(self, columns: Optional[list[str]]=None, process: bool=True) -> dict:
         return { key: value for key, value in self.iter(columns, process) }
 
-    def iter(self, columns: Optional[list[str]]=None, process: bool=True) -> Iterable:
+    def iter(self, 
+        columns: Optional[list[str]]=None, 
+        process: bool=True, 
+        last_only: Optional[bool]=None
+    ) -> Iterable:
         columns = columns if columns else self.key_names
+        last_only = self.meta._last_only if last_only is None else last_only
         if columns is None:
             for _, row in self.iterrows():
                 if process:
@@ -135,14 +149,15 @@ def iter(self, columns: Optional[list[str]]=None, process: bool=True) -> Iterabl
                     yield row
             return None            
         if not columns:
-            yield list(), self.df2value(self) if process else self
+            yield list(), self.df2value(self, last_only) if process else self
             return None
         groups = DataFrame(self).groupby(columns)
         kn = [ n for n in self.key_names if n not in columns ]
         for key in list(groups.groups.keys()):
             key_tuple = key if isinstance(key, tuple) else (key,)
             df = groups.get_group(key_tuple)
-            yield key, type(self)(df, key_names=kn) if kn or not process else self.df2value(df)
+            v = type(self)(df, key_names=kn) if kn or not process else self.df2value(df, last_only)
+            yield key, v
 
     def __iter__(self, process: bool=True) -> Iterable:
         return self.iter(process=process)

solidago/src/solidago/state/assessments/base.py

@@ -1,24 +1,26 @@
-from typing import Optional, Union
+from typing import Optional, Union, Any
 from pandas import DataFrame, Series
 
-from solidago.primitives.datastructure import NestedDictOfRowLists
+from solidago.primitives.datastructure import UnnamedDataFrame
 
 
 class Assessment(Series):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
 
-class Assessments(NestedDictOfRowLists):
+class Assessments(UnnamedDataFrame):
     row_cls: type=Assessment
 
     def __init__(self, 
-        d: Optional[Union[NestedDictOfRowLists, dict, DataFrame]]=None, 
+        data: Optional[Any]=None, 
         key_names=["username", "criterion", "entity_name"],
-        save_filename="assessments.csv"
+        name="assessments",
+        last_only=True,
+        **kwargs
     ):
-        super().__init__(d, key_names, save_filename)
+        super().__init__(key_names, None, name, None, last_only, data, **kwargs)
 
     def get_evaluators(self, entity: Union[str, "Entity"]) -> set[str]:
-        return self[{ "entity_name": entity }].get_set("username")
+        return set(self.get(entity_name=entity)["username"])
 

solidago/src/solidago/state/comparisons/base.py

@@ -1,106 +1,46 @@
 import numpy as np
 
-from typing import Optional, Union, Mapping, Literal
+from typing import Optional, Union, Mapping, Literal, Any
 from pandas import DataFrame, Series
 
-from solidago.primitives.datastructure import NestedDictOfRowLists
+from solidago.primitives.datastructure import UnnamedDataFrame
 
 
 class Comparison(Series):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
 
-class Comparisons(NestedDictOfRowLists):
+class Comparisons(UnnamedDataFrame):
     row_cls: type=Comparison
 
     def __init__(self, 
-        d: Optional[Union[NestedDictOfRowLists, dict, DataFrame]]=None, 
+        data: Optional[Any]=None, 
         key_names=["username", "criterion", "left_name", "right_name"],
-        save_filename="comparisons.csv"
+        name="comparisons",
+        last_only=True,
+        **kwargs
     ):
-        super().__init__(d, key_names, save_filename)
+        super().__init__(key_names, None, name, None, last_only, data, **kwargs)
 
     def get_evaluators(self, entity: Union[str, "Entity"]) -> set[str]:
-        evaluators = self[{ "left_name": entity }].get_set("username") 
-        return evaluators | self[{ "right_name": entity }].get_set("username")
-
-    def order_by_entities(self, other_keys_first: bool=False) -> "Comparisons":
-        """ Returns an object Comparison, with the same set of comparisons,
-        but now ordered by entities. Key names in self are replugged into the result,
-        except for "left_name" and "right_name". Instead, an "other_name" is added
-        to account for the other entity that the comparison is against.
-        Moreover, we add an entry to each dict, which says whether "entity_name" 
-        was the left or the right video.
-        
-        Returns
-        -------
-        ordered_comparisons: Comparisons
-            With key_names == ["entity_name", "other_name", *] or [*, "entity_name", "other_name"]
-            depending on parameter other_keys_first
-        """
-        other_key_names = [ 
-            kn for kn in self.key_names 
-            if kn not in ("entity_name", "other_name", "left_name", "right_name")
-        ]
-        if other_keys_first:
-            key_names = other_key_names + ["entity_name", "other_name"]
-        else:
-            key_names = ["entity_name", "other_name"] + other_key_names
-        
-        if "entity_name" in self.key_names:
-            return self.reorder_keys(key_names)
-        assert "left_name" in self.key_names and "right_name" in self.key_names, "" \
-            "Comparisons must have columns `left_name` and `right_name`"
-        
-        def to_keys(non_entity_keys: list[str], entity_name: str, other_name: str) -> list[str]:
-            if other_keys_first:
-                return non_entity_keys + [entity_name, other_name]
-            return [entity_name, other_name] + non_entity_keys
-                
-        def invert(comparison):
-            if "comparison" in comparison:
-                comparison["comparison"] = - comparison["comparison"]
-            return comparison
-        
-        result = Comparisons(key_names=key_names)
-        left_key_index = self.key_names.index("left_name")
-        right_key_index = self.key_names.index("right_name")
-        for keys, comparison in self:
-            left_name, right_name = keys[left_key_index], keys[right_key_index]
-            non_entity_keys = [ 
-                key for index, key in enumerate(keys) 
-                if index not in (left_key_index, right_key_index) 
-            ]
-            new_comparison = dict(zip(self.key_names, keys)) | dict(comparison)
-            result.add_row(
-                to_keys(non_entity_keys, left_name, right_name),
-                new_comparison | dict(location="left")
-            )
-            result.add_row(
-                to_keys(non_entity_keys, right_name, left_name),
-                invert(new_comparison) | dict(location="right")
-            )
-        return result
+        evaluators = set(self.get(left_name=entity)["username"])
+        return evaluators | set(self.get(right_name=entity)["username"])
 
     def compared_entity_indices(self, 
         entity_name2index: dict[str, int], 
-        last_comparison_only: bool=True,
+        last_only: bool=True,
     ) -> dict[str, list[int]]:
         key_indices = { loc: self.key_names.index(f"{loc}_name") for loc in ("left", "right") }
-        returns = "last_row" if last_comparison_only else "rows"
         return {
             location: [ 
                 entity_name2index[keys[key_indices[location]]] 
-                for keys, _ in self.iter(returns)
+                for keys, _ in self.iter(last_only=last_only)
             ] for location in ("left", "right")
         }
 
-    def normalized_comparisons(self, last_comparison_only: bool) -> Series:
-        df = self.to_df(last_row_only=last_comparison_only)
-        if df.empty:
-            return Series()
-        return df["comparison"] / df["comparison_max"]
+    def normalized_comparisons(self) -> Series:
+        return Series() if self.empty else self["comparison"] / self["comparison_max"]
 
     def to_comparison_dict(self, 
         entities: "Entities", 

solidago/tests/modules/test_trust_propagation.py

@@ -15,7 +15,7 @@ def test_lipschitrust_simple():
         ["0", "2", "Personhood", 1, 0],
         ["2", "3", "Personhood", 1, 0],
         ["3", "4", "Personhood", 1, 0]
-    ], columns=["by", "to", "kind", "weight", "priority"])
+    ])
     users = LipschiTrust(pretrust_value=0.8, decay=0.8, sink_vouch=5.0, error=1e-8)(users, vouches)
     assert users.get("0")["trust_score"] == 0.8
     assert users.get("4")["trust_score"] > 0
@@ -50,7 +50,7 @@ def test_lipschitrust_ten_users():
         ["8", "3", "Personhood", 1, 0],
         ["9", "4", "Personhood", 1, 0],
         ["9", "5", "Personhood", 1, 0],
-    ], columns=["by", "to", "kind", "weight", "priority"])
+    ])
 
     trust_propagator = LipschiTrust(pretrust_value=0.8, decay=0.8, sink_vouch=5.0, error=1e-8)
     users = trust_propagator(users, vouches)

solidago/tests/modules/test_voting_right_assignment.py

@@ -178,11 +178,11 @@ def test_affine_overtrust():
     users = Users(dict(username=list(range(5)), trust_score=[0.5, 0.6, 0.0, 0.4, 1]))
     entities = Entities(list(range(6)))
     made_public = MadePublic()
-    made_public["0", "0"] = True
-    made_public["0", "3"] = True
-    made_public["1", "5"] = True
-    made_public["2", "1"] = True
-    made_public["4", "3"] = True
+    made_public.set(True, "0", "0")
+    made_public.set(True, "0", "3")
+    made_public.set(True, "1", "5")
+    made_public.set(True, "2", "1")
+    made_public.set(True, "4", "3")
 
     assessments = Assessments()
     comparisons = Comparisons()

solidago/tests/state/test_made_public.py

@@ -4,7 +4,7 @@
 
 def test_made_public():
     made_public = MadePublic()
-    made_public.set(False, "aidjango", "entity_4")
-    made_public.set(True, "le_science4all", "entity_4")
+    made_public.set("aidjango", "entity_4", False)
+    made_public.set("le_science4all", "entity_4", True)
     assert not made_public.get("aidjango", "entity_4")
     assert made_public.get("le_science4all", "entity_4")

solidago/tests/state/test_voting_rights.py

@@ -4,8 +4,8 @@
 
 def test_voting_rights():
     voting_rights = VotingRights()
-    voting_rights.set(0.5, "aidjango", "entity_4", "default")
-    voting_rights.set(1., "le_science4all", "entity_4", "largely_recommended")
+    voting_rights.set("aidjango", "entity_4", "default", 0.5)
+    voting_rights.set("le_science4all", "entity_4", "largely_recommended", 1)
     assert voting_rights.get("aidjango", "entity_4", "default") == 0.5
     assert voting_rights.get("le_science4all", "entity_4", "largely_recommended") == 1
     assert voting_rights.get("le_science4all", "entity_4", "default") == 0

solidago/tests/state/test_vouches.py

@@ -4,6 +4,6 @@
 
 def test_vouches():
     vouches = Vouches()
-    vouches.set((0.5, 0), "aidjango", "le_science4all", "Personhood")
+    vouches.set("aidjango", "le_science4all", "Personhood", 0.5, 0)
     assert vouches.get("aidjango", "le_science4all", "Personhood") == (0.5, 0)
     assert vouches.get("le_science4all", "aidjango", "Personhood") == (0, - float("inf"))