Restore scoring file

Author: dbobrenko

prompting/rewards/scoring.py

@@ -73,16 +73,16 @@ async def run_step(self) -> RewardLoggingEvent:
         await asyncio.sleep(0.1)
         # Only score responses for which the model is loaded
         await self.model_scheduler.llm_model_manager.lock.acquire()
-        # with self.mp_lock:
-        scorable = [
-            scoring_config
-            for scoring_config in self.scoring_queue
-            if (scoring_config.task.llm_model in self.model_scheduler.llm_model_manager.active_models.keys())
-            or (scoring_config.task.llm_model is None)
-        ]
-        if len(scorable) == 0:
-            return
-        self.scoring_queue.remove(scorable[0])
+        with self.mp_lock:
+            scorable = [
+                scoring_config
+                for scoring_config in self.scoring_queue
+                if (scoring_config.task.llm_model in self.model_scheduler.llm_model_manager.active_models.keys())
+                or (scoring_config.task.llm_model is None)
+            ]
+            if len(scorable) == 0:
+                return
+            self.scoring_queue.remove(scorable[0])
         scoring_config: ScoringConfig = scorable.pop(0)
 
         # here we generate the actual reference
@@ -106,15 +106,13 @@ async def run_step(self) -> RewardLoggingEvent:
                 model_manager=self.model_scheduler.llm_model_manager,
                 task_queue=self.task_queue,
             )
-
-        # with self.mp_lock:
         self.reward_events.append(reward_events)
 
         # TODO: Remove this once we have a better way to handle organic tasks
-        # if scoring_config.task.organic:
-        #     self.reward_events.append(
-        #         reward_events
-        #     )  # Add the organic a second time, doubling the weight of the organic
+        if scoring_config.task.organic:
+            self.reward_events.append(
+                reward_events
+            )  # Add the organic a second time, doubling the weight of the organic
         logger.debug(
             f"Scored {scoring_config.task.__class__.__name__} {scoring_config.task.task_id} with model "
             f"{scoring_config.task.llm_model_id}"