✨ Update llm ranking scripts to only update changed entries

Author: Zeta611

scripts/rank_translations_llm.py

@@ -24,6 +24,7 @@
 import time
 import argparse
 import re
+import shutil
 import pandas as pd
 from tqdm import tqdm
 from typing import List, Tuple, Dict

scripts/update_llm_rank.py

@@ -3,15 +3,19 @@
 Batch update Supabase translation.llm_rank using CSV produced by rank_translations_llm.py
 
 Inputs:
-  - llm_ranks.csv with columns: translation_id, llm_rank
+  - llm_ranks.csv with columns: translation_id, llm_rank (new rankings)
+  - llm_ranks.csv.bak (previous rankings for diffing)
 
 Environment:
   - For direct Postgres: use same .env as dump.py (user, password, host, port, dbname)
     or set DATABASE_URL explicitly (postgresql+psycopg2://...)
 
 Behavior:
-  - Performs efficient batched updates using SQLAlchemy executemany.
-  - On missing rows, skips silently.
+  - Diffs llm_ranks.csv against llm_ranks.csv.bak to find changed/new rows
+  - Only updates rows where llm_rank has changed (typically ~10 rows)
+  - Does not update updated_at timestamp
+  - If no backup file exists, updates all rows
+  - On missing rows in DB, skips silently
 
 Note:
   - New translations added after a ranking run should keep llm_rank NULL until next batch.
@@ -46,26 +50,59 @@ def get_engine():
 def main():
     parser = argparse.ArgumentParser(description="Update translation.llm_rank from CSV")
     parser.add_argument("--ranks_csv", default="llm_ranks.csv")
+    parser.add_argument("--backup_csv", default="llm_ranks.csv.bak")
     parser.add_argument("--batch_size", type=int, default=1000)
     args = parser.parse_args()
 
-    df = pd.read_csv(args.ranks_csv)
-    if "translation_id" not in df.columns or "llm_rank" not in df.columns:
+    # Read new and backup CSVs
+    df_new = pd.read_csv(args.ranks_csv)
+    if "translation_id" not in df_new.columns or "llm_rank" not in df_new.columns:
         raise ValueError("llm_ranks.csv must have columns: translation_id, llm_rank")
 
+    # Read backup CSV (if it doesn't exist, update all rows)
+    if not os.path.exists(args.backup_csv):
+        print(f"Backup file {args.backup_csv} not found; updating all rows")
+        df_changed = df_new
+    else:
+        df_old = pd.read_csv(args.backup_csv)
+        if "translation_id" not in df_old.columns or "llm_rank" not in df_old.columns:
+            raise ValueError("Backup CSV must have columns: translation_id, llm_rank")
+
+        # Merge to find changes
+        merged = df_new.merge(
+            df_old,
+            on="translation_id",
+            how="left",
+            suffixes=("_new", "_old")
+        )
+
+        # Filter to only rows where llm_rank changed or is new
+        df_changed = merged[
+            (merged["llm_rank_old"].isna()) |
+            (merged["llm_rank_new"] != merged["llm_rank_old"])
+        ][["translation_id", "llm_rank_new"]].rename(columns={"llm_rank_new": "llm_rank"})
+
+        print(f"Found {len(df_changed)} changed/new rows out of {len(df_new)} total")
+
+    if len(df_changed) == 0:
+        print("No changes to update")
+        return
+
+    if len(df_changed) <= 200:
+      print("df_changed:", df_changed, sep="\n")
+
     engine = get_engine()
 
     sql = text("""
         update public.translation as t
-           set llm_rank = v.llm_rank,
-               updated_at = now()
+           set llm_rank = v.llm_rank
           from (values (:translation_id, :llm_rank)) as v(translation_id, llm_rank)
          where t.id = v.translation_id::uuid
     """)
 
     total = 0
     with engine.begin() as conn:
-        rows = df.to_dict("records")
+        rows = df_changed.to_dict("records")
         for i in range(0, len(rows), args.batch_size):
             batch = rows[i : i + args.batch_size]
             # executemany: param style is dict per row