Some fixes related to evaluation

Author: mseeger

keys_values/finetune/longcontext_eval_ext.py

@@ -700,7 +700,7 @@ def eval_for_setup_internal(
                     result_path = eval_metrics_path
                 else:
                     eval_fname = eval_metrics_path.stem
-                    suffix = "_".split(eval_fname)[-1]
+                    suffix = eval_fname.split("_")[-1]
                     result_path = (
                         eval_metrics_path.parent
                         / GENERATED_SAMPLES_FILENAME.format(suffix)

keys_values/scripts/cleanup_gen_samples.py

@@ -34,10 +34,12 @@ def main(control_file: Path):
 
 
 if __name__ == "__main__":
-    # dataset_size = "64k"
-    dataset_size = "128k"
+    dataset_size = "64k"
+    # dataset_size = "128k"
+    # control_file = (
+    #    Path.home() / "sync" / "keys_values" / f"eval_inst1_{dataset_size}_h2o.yaml"
+    # )
     control_file = (
-        Path.home() / "sync" / "keys_values" / f"eval_inst1_{dataset_size}.yaml"
+        Path.home() / "git" / "keys_values" / f"eval_inst2_3_{dataset_size}_h2o.yaml"
     )
-    # control_file = Path.home() / "git" / "keys_values" / f"eval_inst2_3_{dataset_size}.yaml"
     main(control_file)

keys_values/scripts/collect_eval_results.py

@@ -59,6 +59,8 @@ def main(
     print(f"Total number of records: {len(all_data)}")
     if all_data:
         combined_path = out_dir / EVAL_METRICS_ALL_FILENAME
+        if combined_path.exists():
+            combined_path.unlink()
         with open(combined_path, "w") as fp:
             writer = csv.writer(fp, delimiter=",")
             writer.writerow(column_names)
@@ -88,6 +90,8 @@ def main(
         "qh2onorm_4gpu_cs2048_lr5",
         "lr_4gpu_cs1024_lr5",
         "h2o_4gpu_cs1024_lr5",
+        "slr_4gpu_cs1024_lr5",
+        "h2onorm_4gpu_cs1024_lr5",
     ]
     model_type = "lora"
     if mode == "collect":

keys_values/scripts/collect_gen_samples.py

@@ -56,6 +56,8 @@ def main(
     print(f"Total number of records: {num_total}")
     if all_data:
         combined_path = out_dir / GENERATED_SAMPLES_ALL_FILENAME
+        if combined_path.exists():
+            combined_path.unlink()
         with open(combined_path, "w") as fp:
             yaml.safe_dump(all_data, fp)
 
@@ -82,6 +84,8 @@ def main(
         "qh2onorm_4gpu_cs2048_lr5",
         "lr_4gpu_cs1024_lr5",
         "h2o_4gpu_cs1024_lr5",
+        "slr_4gpu_cs1024_lr5",
+        "h2onorm_4gpu_cs1024_lr5",
     ]
     model_type = "lora"
     if mode == "collect":

keys_values/scripts/create_result_table.py

@@ -29,7 +29,28 @@ def _sort_entries(entries):
     return non_fin + [(st, v) for st, v in entries if st == "fin"]
 
 
-def main(datasets, cases, result_path):
+# We ran evaluations for more than the task for which evaluation loss was
+# lowest. With this predicate, we filter for the winning tasks only.
+def _filter_dataset_case(
+    dataset: str,
+    case: str,
+    task: str,
+) -> bool:
+    if dataset.endswith("_128k"):
+        # Not yet implemented!!
+        return True
+    # Filter out error in results:
+    if task == "380" and case.startswith("lr_") and dataset.startswith("helmet_trivia"):
+        return False
+    if task == "fin":
+        # Only those for which "fin" is the only result
+        return dataset.startswith("helmet_pop") and (
+            case.startswith("slr_") or case.startswith("h2onorm_")
+        )
+    return task != "010"
+
+
+def main(datasets, cases, result_path, final_table: bool):
     base_path = result_path.parent
     col_labels = [
         d.removeprefix("helmet_").rsplit("_", 1)[0].replace("_", r"\_")
@@ -48,23 +69,49 @@ def main(datasets, cases, result_path):
             else:
                 df = pd.read_csv(csv_path)
                 avg = df.groupby("task")["sub_exact_match"].mean()
-                row.append(_sort_entries([(_short_task(t), v) for t, v in avg.items()]))
+                row.append(
+                    _sort_entries(
+                        [
+                            (_short_task(t), v)
+                            for t, v in avg.items()
+                            if not final_table
+                            or _filter_dataset_case(dataset, case_key, _short_task(t))
+                        ]
+                    )
+                )
         table.append(row)
 
-    # Each dataset gets 2 sub-columns (l for task, r for value) for cross-cell alignment.
+    # - final_table == False:
+    #   Each dataset gets 2 sub-columns (l for task, r for value) for cross-cell alignment.
+    # - final_table == True:
+    #   Each dataset column features a single entry (r for value)
     N = len(datasets)
-    col_spec = "l" + "lr" * N
-    tex_lines = [
-        r"\begin{tabular}{" + col_spec + "}",
-        r"\noalign{\smallskip}\hline\noalign{\smallskip}",
-        " & ".join([""] + [r"\multicolumn{2}{c}{" + lbl + "}" for lbl in col_labels])
-        + r" \\",
-        r"\noalign{\smallskip}\hline\hline\noalign{\smallskip}",
-    ]
-    for i, case_label in enumerate(case_labels):
-        row_entries = table[i]
+    if final_table:
+        col_spec = "l" + "r" * N
+        tex_lines = [
+            r"\begin{tabular}{" + col_spec + "}",
+            r"\noalign{\smallskip}\hline\noalign{\smallskip}",
+            " & ".join([""] + col_labels) + r" \\",
+            r"\noalign{\smallskip}\hline\hline\noalign{\smallskip}",
+        ]
+    else:
+        col_spec = "l" + "lr" * N
+        tex_lines = [
+            r"\begin{tabular}{" + col_spec + "}",
+            r"\noalign{\smallskip}\hline\noalign{\smallskip}",
+            " & ".join(
+                [""] + [r"\multicolumn{2}{c}{" + lbl + "}" for lbl in col_labels]
+            )
+            + r" \\",
+            r"\noalign{\smallskip}\hline\hline\noalign{\smallskip}",
+        ]
+    for case_label, row_entries in zip(case_labels, table):
         max_rows = max((len(e) for e in row_entries), default=0)
         max_rows = max(max_rows, 1)
+        if final_table and max_rows > 1:
+            print(
+                f"{case_label}: max_rows = {max_rows} > 1, must not happen for final_table=True"
+            )
         for k in range(max_rows):
             if k == 0 and max_rows > 1:
                 label_cell = r"\multirow{" + str(max_rows) + r"}{*}{" + case_label + "}"
@@ -76,23 +123,28 @@ def main(datasets, cases, result_path):
             for entries in row_entries:
                 if k < len(entries):
                     st, v = entries[k]
-                    cells.append(r"{\small " + st + r":}")
+                    if not final_table:
+                        cells.append(r"{\small " + st + r":}")
                     cells.append(r"{\small\!" + f"{v * 100:.2f}" + "}")
                 else:
-                    cells.append("")
+                    if not final_table:
+                        cells.append("")
                     cells.append("")
             tex_lines.append(" & ".join(cells) + r" \\")
         tex_lines.append(r"\noalign{\smallskip}\hline\noalign{\smallskip}")
     tex_lines.append(r"\end{tabular}")
 
+    if result_path.exists():
+        result_path.unlink()
     result_path.write_text("\n".join(tex_lines) + "\n")
 
 
+# TODO: If `final_table = True`, do not print the task ID, just the metric value
 if __name__ == "__main__":
     base_path = Path.home() / "out/finetune/neurips_exp/lora/qwen3_4b"
 
-    # dataset_size = "64k"
-    dataset_size = "128k"
+    dataset_size = "64k"
+    # dataset_size = "128k"
     datasets = [
         f"helmet_nq_{dataset_size}",
         f"helmet_trivia_qa_{dataset_size}",
@@ -101,14 +153,16 @@ def main(datasets, cases, result_path):
     ]
     cases = [
         ("lr_4gpu_cs2048_lr5", "lr_2048"),
-        ("h2o_4gpu_cs2048_lr5", "h2o_2048"),
         ("slr_4gpu_cs2048_lr5", "slr_2048"),
-        #     ("qh2o_4gpu_cs2048_lr5", "qh2o_2048"),
-        #     ("h2onorm_4gpu_cs2048_lr5", "h2onorm_2048"),
-        #     ("qh2onorm_4gpu_cs2048_lr5", "qh2onorm_2048"),
-        #     ("lr_4gpu_cs1024_lr5", "lr_1024"),
-        #     ("h2o_4gpu_cs1024_lr5", "h2o_1024"),
+        ("h2o_4gpu_cs2048_lr5", "h2o_2048"),
+        ("qh2o_4gpu_cs2048_lr5", "qh2o_2048"),
+        ("h2onorm_4gpu_cs2048_lr5", "h2onorm_2048"),
+        ("qh2onorm_4gpu_cs2048_lr5", "qh2onorm_2048"),
+        ("lr_4gpu_cs1024_lr5", "lr_1024"),
+        ("h2o_4gpu_cs1024_lr5", "h2o_1024"),
     ]
     result_path = base_path / f"results_{dataset_size}.tex"
+    # final_table = False
+    final_table = True
 
-    main(datasets, cases, result_path)
+    main(datasets, cases, result_path, final_table)