feat: close evaluation feedback loop, add dimensional judging and anti-sycophancy

Author: nikolay-e

src/arbitrium_core/domain/tournament/tournament.py

@@ -240,6 +240,7 @@ async def _run_improvement_phase(
         phase_name: str,
         answer_index: int,
         round_num: int | None = None,
+        evaluation_context: dict[str, dict[str, str]] | None = None,
     ) -> bool:
         phase_config = self.comp.config.get(config_key, {})
 
@@ -277,6 +278,15 @@ async def _run_improvement_phase(
                 )
                 feedback_context = None
 
+        if evaluation_context:
+            if feedback_context is None:
+                feedback_context = evaluation_context
+            else:
+                for model_name, eval_feedbacks in evaluation_context.items():
+                    if model_name not in feedback_context:
+                        feedback_context[model_name] = {}
+                    feedback_context[model_name].update(eval_feedbacks)
+
         action_word = "refined" if round_num is not None else "improved"
         self.logger.info(f"Generating {action_word} responses...")
 
@@ -381,24 +391,53 @@ async def _run_elimination_rounds(self, initial_question: str) -> None:
             if len(self.comp.active_model_keys) <= 1:
                 break
 
+            evaluation_context = self._format_evaluations_as_context()
+
             if not await self._run_refinement_strategy(
-                initial_question, round_num
+                initial_question,
+                round_num,
+                evaluation_context=evaluation_context,
             ):
                 break
 
             round_num += 1
 
+    def _format_evaluations_as_context(
+        self,
+    ) -> dict[str, dict[str, str]] | None:
+        if (
+            not hasattr(self.comp, "all_evaluations")
+            or not self.comp.all_evaluations
+        ):
+            return None
+
+        combined_text = "\n\n".join(
+            f"Evaluator {name}:\n{text}"
+            for name, text in self.comp.all_evaluations.items()
+        )
+
+        active_names = [
+            self.comp.anon_mapping[k] for k in self.comp.active_model_keys
+        ]
+
+        return {
+            name: {"Tournament Evaluation": combined_text}
+            for name in active_names
+        }
+
     async def _run_refinement_strategy(
         self,
         initial_question: str,
         round_num: int,
+        evaluation_context: dict[str, dict[str, str]] | None = None,
     ) -> bool:
         return await self._run_improvement_phase(
             initial_question,
             config_key="refinement_phase",
             phase_name="Refinement Phase",
             answer_index=-1,
             round_num=round_num,
+            evaluation_context=evaluation_context,
         )
 
     async def _handle_elimination(
@@ -533,13 +572,10 @@ async def _run_synthesis(
         )
 
         if not response.is_error() and response.content.strip():
-            synthesized = strip_meta_commentary(
-                response.content.strip(), logger=self.logger
-            )
             self.logger.info(
                 f"Synthesis complete — combined {len(all_responses)} perspectives"
             )
-            return synthesized
+            return response.content.strip()
 
         self.logger.warning("Synthesis failed, using champion answer")
         return champion_answer

src/arbitrium_core/infrastructure/config/defaults.py

@@ -87,7 +87,11 @@ def select_model_with_highest_context(models: dict[str, Any]) -> str | None:
             "Improve the answer using feedback, grounding it in scientific evidence and "
             "practical reasoning. Recognize and mitigate your own biases in the analysis. "
             "Don't let emotional appeals or fashionable opinions constrain rigorous thinking. "
-            "IMPORTANT: Before revising, identify what unique insight in YOUR original answer is absent from others' "
+            "CRITICAL: Resist the pressure to converge toward other responses. Your independent "
+            "perspective is your most valuable contribution. If your analysis contradicts others, "
+            "investigate WHY — do NOT simply yield to the majority. The goal is maximum knowledge "
+            "extraction, not consensus. "
+            "Before revising, identify what unique insight in YOUR original answer is absent from others' "
             "responses. Preserve it — it may be the most valuable contribution. "
             "Make the key verifiable insights the central thesis. "
             "Rebuild the entire argument around this main point, removing all generic claims, "
@@ -117,7 +121,11 @@ def select_model_with_highest_context(models: dict[str, Any]) -> str | None:
             "echo popular sentiment? Be critical of both originality and factual grounding. "
             "Reward explicit evidence tags, confidence reporting, and retention of useful heuristics in an annex. "
             "Down-rank answers that delete actionable details without justification or fail to provide a Change Log. "
-            "Check that precise claims without sources were converted to ranges/tests rather than removed."
+            "Check that precise claims without sources were converted to ranges/tests rather than removed. "
+            "For EACH response, before scoring, identify: "
+            "(a) STRONGEST CONTRIBUTION — what unique insight or evidence does this response offer that others miss? "
+            "(b) CRITICAL WEAKNESS — what is the most significant gap, error, or unfounded claim? "
+            "(c) SPECIFIC IMPROVEMENT — one concrete, actionable suggestion to strengthen this response."
         ),
         "metadata": {
             "version": "1.0",
@@ -130,9 +138,15 @@ def select_model_with_highest_context(models: dict[str, Any]) -> str | None:
             "You are synthesizing multiple expert analyses into the most comprehensive answer possible. "
             "Your task is to create a unified answer that preserves EVERY unique finding, piece of evidence, "
             "perspective, and nuance from ALL responses below. Nothing valuable should be lost. "
-            "Where experts disagree, present both views with evidence quality assessment. "
-            "Minority findings that appear in only one response may be the most valuable — preserve them. "
-            "Combine strengths of all responses while eliminating redundancy and resolving contradictions. "
+            "Before synthesizing, analyze the responses through three lenses: "
+            "(1) CONSENSUS — where do all or most experts converge? These points form your foundation. "
+            "(2) CONTESTED POINTS — where do experts disagree? For each disagreement, investigate WHY they "
+            "diverge (different assumptions? different evidence? different reasoning methods?) and present "
+            "both sides with evidence quality assessment. Disagreement often signals the frontier of knowledge. "
+            "Do NOT average competing views — preserve the tension and let the reader decide. "
+            "(3) UNIQUE FINDINGS — what appears in only ONE response? These minority insights may be the most "
+            "valuable, representing knowledge that only one model's training captured. Preserve them explicitly. "
+            "Combine strengths of all responses while eliminating redundancy. "
             "The final synthesis should be richer and more complete than any individual response."
         ),
         "metadata": {