Run second pretraining iteration

Author: eric-czech

experiments/plantcad/README.md

@@ -121,6 +121,11 @@ rm -rf local_store/evaluation/dna-conservation*; python -m experiments.plantcad.
 
 # Checkpoint upload
 find local_store | grep -E 'hf/step-[0-9]+$' | xargs -I {} echo "hf upload plantcad/_dev_marin_plantcad1_v2_train {} {} --repo-type model" | bash /dev/stdin
+
+find local_store | grep -E 'checkpoints/step-[0-9]+$' | \
+grep -E 'step-26780$|step-24102|step-21424|step-18746$' | \
+xargs -I {} echo "hf upload plantcad/_dev_marin_plantcad1_v2_train {} {} --repo-type model" | \
+bash /dev/stdin
 ```
 
 ```bash
@@ -151,6 +156,12 @@ Second iteration:
 0.604521  8034  hf://plantcad/_dev_marin_plantcad1_v2_train/local_store/checkpoints/plantcad-train-600m-r12-7ea0fc/hf/step-8034
 0.626729 10712 hf://plantcad/_dev_marin_plantcad1_v2_train/local_store/checkpoints/plantcad-train-600m-r12-7ea0fc/hf/step-10712
 0.631095 13390 hf://plantcad/_dev_marin_plantcad1_v2_train/local_store/checkpoints/plantcad-train-600m-r12-7ea0fc/hf/step-13390
+0.607316 16068 hf://plantcad/_dev_marin_plantcad1_v2_train/local_store/checkpoints/plantcad-train-600m-r12-7ea0fc/hf/step-16068
+0.622988 18746 hf://plantcad/_dev_marin_plantcad1_v2_train/local_store/checkpoints/plantcad-train-600m-r12-7ea0fc/hf/step-18746
+0.639093 21424 hf://plantcad/_dev_marin_plantcad1_v2_train/local_store/checkpoints/plantcad-train-600m-r12-7ea0fc/hf/step-21424
+0.650973 24102 hf://plantcad/_dev_marin_plantcad1_v2_train/local_store/checkpoints/plantcad-train-600m-r12-7ea0fc/hf/step-24102
+0.657882 26780 hf://plantcad/_dev_marin_plantcad1_v2_train/local_store/checkpoints/plantcad-train-600m-r12-7ea0fc/hf/step-26780
+0.657452 26782 hf://plantcad/_dev_marin_plantcad1_v2_train/local_store/checkpoints/plantcad-train-600m-r12-7ea0fc/hf/step-26782
 ```
 
 ## EDA

experiments/plantcad/evaluation.py

@@ -660,7 +660,7 @@ def evaluate_conservation_scores(scores: ConservationResult) -> dict[str, float]
     logger.info("EVALUATION RESULTS")
     logger.info("=" * 50)
     logger.info(f"Total examples: {results['n_total']}")
-    logger.info(f"ROC AUC: {results['roc_auc']:.4f}")
+    logger.info(f"ROC AUC: {results['roc_auc']:.6f}")
     logger.info(f"Balance: {results['balance']:.3f} ({results['n_positive']}/{results['n_total']})")
     logger.info("=" * 50)
 

experiments/plantcad/misc/agg_eval_results.py

@@ -21,9 +21,9 @@
 
 def main():
     data = []
-    base_dir = "/home/ubuntu/sky_workdir/local_store/evaluation"
+    base_dir = "~/sky_workdir/local_store/evaluation"
 
-    for eval_dir in Path(base_dir).glob("dna-conservation-*"):
+    for eval_dir in Path(base_dir).expanduser().glob("dna-conservation-*"):
         with open(eval_dir / "results.json") as f:
             result = json.load(f)
         config = result.get("config", {})