Vidhisha/ba benchmarks (#88)

Updates to ba calendar:
1. usage reporting
2. constrainedness computing + grouping results by constrainedness
bucket

---------

Co-authored-by: Vidhisha Balachandran <[email protected]>

Author: vidhishanair

eureka_ml_insights/data_utils/__init__.py

@@ -79,4 +79,5 @@
     TokenCounterTransform,
     MajorityVoteTransform,
     NumpyEncoder,
+    ExtractUsageTransform,
 ]

eureka_ml_insights/data_utils/ba_calendar_utils.py

@@ -28,11 +28,12 @@ def parse_output_answer(response):
 
         if response is None:
             return ""
+        
+        response = response.replace("**", "").replace("\n", "")
 
-        # Try to find an answer in the "Final Answer: X" format
-        match = re.search(r"(?i)(?<=Final Answer: ).*", response)
+        match = re.findall(r"(?i)(?<=Final Answer: )(\w+)(?=\s|\W|$)", response)
 
         if match:
-            answer = match.group(0)
+            answer = match[len(match) - 1]
 
         return answer

eureka_ml_insights/metrics/ba_calendar_metrics.py

@@ -5,6 +5,7 @@
 import ast
 import json
 import re
+import math
 import numpy as np
 from datetime import datetime, timedelta
 
@@ -48,11 +49,13 @@ def filter_slots_by_constraints(time_slots, constraints, day):
     for slot in time_slots:
         start_time, end_time = slot
         if constraints['no_meetings_before']:
-            no_meetings_before = datetime.strptime(f"{constraints['no_meetings_before']}:00", "%H:%M")
+            nb = int(constraints['no_meetings_before'])
+            no_meetings_before = datetime.strptime(f"{nb}:00", "%H:%M")
             if start_time < no_meetings_before:
                 continue
         if constraints['no_meetings_after']:
-            no_meetings_after = datetime.strptime(f"{constraints['no_meetings_after']}:00", "%H:%M")
+            na = int(constraints['no_meetings_after'])
+            no_meetings_after = datetime.strptime(f"{na}:00", "%H:%M")
             if end_time >= no_meetings_after:
                 continue
         if constraints['no_meetings_on_weekends'] and day in ['Saturday', 'Sunday']:
@@ -86,6 +89,8 @@ def run_programmatic_tests(self, instance):
         solution = solution.strip('"').strip('`').strip('\n')
         if check_time_slot_format(solution):
             result['format_programmatic'] = 1
+        else:
+            result['format_programmatic'] = 0
         result.update(self.check_availability_programmatic(instance, solution))
         result.update(self.check_meeting_duration_programmatic(instance, solution))
         result.update(self.check_buffer_time_programmatic(instance, solution))
@@ -102,6 +107,7 @@ def run_programmatic_tests(self, instance):
                 passed_constraints.append(value)
         result['all_correct'] = all_correct
         result['fraction_passed'] = np.mean(passed_constraints)
+        result.update(self.compute_constrainedness_programmatic(instance))
         return result
 
     def is_formatted(self, solution):
@@ -224,12 +230,14 @@ def check_time_restrictions_programmatic(self, instance, solution):
         no_meetings_after = instance['constraints'].get('no_meetings_after')
 
         if no_meetings_before:
-            no_meetings_before = datetime.strptime(f"{no_meetings_before}:00", "%H:%M")
+            nb = int(no_meetings_before)
+            no_meetings_before = datetime.strptime(f"{nb}:00", "%H:%M")
             if start_time < no_meetings_before:
                 return {'time_restrictions_programmatic_check': 0}
 
         if no_meetings_after:
-            no_meetings_after = datetime.strptime(f"{no_meetings_after}:00", '%H:%M')
+            na = int(no_meetings_after)
+            no_meetings_after = datetime.strptime(f"{na}:00", '%H:%M')
             if end_time > no_meetings_after:
                 return {'time_restrictions_programmatic_check': 0}
         return {'time_restrictions_programmatic_check': 1}
@@ -290,4 +298,61 @@ def check_specific_times_programmatic(self, instance, solution):
             result = 0
         else:
             result = 1
-        return {'specific_times_programmatic_check': result}
+        return {'specific_times_programmatic_check': result}
+
+    def compute_constrainedness_programmatic(self, instance):
+        """
+        Compute the constrainedness of the problem based on the constraints and availability.
+        The constrainedness is defined as (1 - the ratio of feasible slots to total slots).
+        The higher the constrainedness, the more constrained the problem is.
+        """
+        params = instance['params']
+        constraints = instance['constraints']
+        metadata = instance['metadata']
+        if not instance['constraints']['buffer_time_before_and_after_meeting']:
+            buffer_time_before_and_after_meeting = 0
+        else:
+            buffer_time_before_and_after_meeting = instance['constraints']['buffer_time_before_and_after_meeting']
+        total_slots = 0
+        feasible_slots = 0       
+        for day in params['days_of_week']:
+            common_time_slots = None
+            union_time_slots = None
+            availability = json.loads(metadata['availability'].replace("'", '"'))
+            for participant, schedule in availability.items():
+                if day in schedule:
+                    participant_time_slots = []
+                    participant_time_slots_unconstrained = []
+                    for time_slot in schedule[day]:
+                        start_time, end_time = parse_time_block(time_slot)
+                        time_slots = generate_time_slots(start_time, end_time, params['granularity'])
+                        time_slots = filter_slots_by_duration(time_slots, constraints['meeting_duration'])
+                        participant_time_slots_unconstrained.extend(time_slots)
+                        time_slots = generate_time_slots(start_time, end_time, params['granularity'])
+                        time_slots = filter_slots_by_duration(time_slots, constraints['meeting_duration'] + buffer_time_before_and_after_meeting*2)
+                        time_slots = filter_slots_by_constraints(time_slots, constraints, day=day)
+                        participant_time_slots.extend(time_slots)
+                    if common_time_slots is None:
+                        common_time_slots = set(participant_time_slots)
+                    else:
+                        common_time_slots = common_time_slots.intersection(participant_time_slots)
+                    if union_time_slots is None:
+                        union_time_slots = set(participant_time_slots_unconstrained)
+                    else:
+                        union_time_slots = union_time_slots.union(participant_time_slots_unconstrained)
+            if common_time_slots:
+                feasible_slots +=len(common_time_slots)
+            if union_time_slots:
+                total_slots+=len(union_time_slots)
+
+        # Calculate constrainedness ratio
+        if total_slots > 0:
+            constrainedness_ratio = 1 - (feasible_slots / total_slots)
+        else:
+            constrainedness_ratio = 1
+
+        # Bucket the constrainedness ratio into intervals of 0.2
+        constrainedness_bucket = round(math.floor(constrainedness_ratio / 0.1) * 0.1, 4)
+
+        # Add test result
+        return {'constrainedness': constrainedness_ratio, 'constrainedness_bucket': constrainedness_bucket}

eureka_ml_insights/metrics/reports.py

@@ -131,7 +131,7 @@ def _aggregate(self, data):
         if len(data) == 0:
             averages = {col: 0 for col in self.column_names}
         else:
-            averages = {col: data[col].mean().round(3) for col in self.column_names}
+            averages = {col: round(data[col].mean(), 3) for col in self.column_names}
         self.aggregated_result = averages
 
     def _aggregate_grouped(self, data):

eureka_ml_insights/user_configs/__init__.py

@@ -8,7 +8,11 @@
     AIME_PIPELINE512Run,
     AIME_PIPELINE1024Run,
 )
-from .ba_calendar import BA_Calendar_Parallel_PIPELINE, BA_Calendar_PIPELINE
+from .ba_calendar import (
+        BA_Calendar_PIPELINE,
+        BA_Calendar_Parallel_PIPELINE,
+        BA_Calendar_RunEvals_PIPELINE,
+)
 from .dna import DNA_PIPELINE
 from .drop import Drop_Experiment_Pipeline
 from .flenqa import FlenQA_Experiment_Pipeline
@@ -120,6 +124,7 @@
     DNA_PIPELINE,
     BA_Calendar_PIPELINE,
     BA_Calendar_Parallel_PIPELINE,
+    BA_Calendar_RunEvals_PIPELINE,
     ToxiGen_Discriminative_PIPELINE,
     ToxiGen_Generative_PIPELINE,
     Geo_Nondeterminism,