Include task scores with mmlu results + adjust default api retries

Signed-off-by: Dan McPherson <[email protected]>

src/instructlab/eval/mmlu.py

@@ -194,12 +194,15 @@ def run(self) -> tuple:
         )
         results = mmlu_output["results"]
 
-        for task in self.tasks:
-            mmlu_res = results[task]
-            agg_score += float(mmlu_res["acc,none"])
-            individual_scores[task] = {}
-            individual_scores[task]["score"] = float(mmlu_res["acc,none"])
-            individual_scores[task]["stderr"] = float(mmlu_res["acc_stderr,none"])
+        for task, result in results.items():
+            if task in self.tasks:
+                agg_score += float(result["acc,none"])
+            else:
+                individual_scores[task] = {
+                    "score": float(result["acc,none"]),
+                    "stderr": float(result["acc_stderr,none"]),
+                }
 
         overall_score = float(agg_score / len(self.tasks))
+
         return overall_score, individual_scores

src/instructlab/eval/mt_bench_common.py

@@ -17,8 +17,8 @@
 import openai
 
 # API setting constants
-API_MAX_RETRY = 16
-API_RETRY_SLEEP = 10
+API_MAX_RETRY = 4
+API_RETRY_SLEEP = 4
 API_ERROR_OUTPUT = "$ERROR$"
 
 # Categories that need reference answers