trying again

Signed-off-by: dafnapension <[email protected]>

Author: dafnapension

prepare/metrics/llm_as_judge/pairwise_rating/llama_3_arena_hard_template.py

@@ -6,7 +6,11 @@
 )
 from unitxt.llm_as_judge import LLMAsJudge
 
-model_list = ["meta-llama/llama-3-8b-instruct", "meta-llama/llama-3-70b-instruct"]
+model_list = [
+    "meta-llama/llama-3-8b-instruct",
+    "meta-llama/llama-3-70b-instruct",
+    "meta-llama/llama-3-3-70b-instruct",
+]
 format = "formats.llama3_instruct"
 templates = [
     "templates.response_assessment.pairwise_comparative_rating.arena_hard",

src/unitxt/catalog/metrics/llm_as_judge/pairwise_comparative_rating/llama_3_3_70b_instruct_generic_engine_template_arena_hard.json

@@ -0,0 +1,11 @@
+{
+    "__type__": "llm_as_judge",
+    "inference_model": {
+        "__type__": "generic_inference_engine",
+        "default": "engines.ibm_gen_ai.llama_3_70b_instruct"
+    },
+    "template": "templates.response_assessment.pairwise_comparative_rating.arena_hard",
+    "task": "pairwise_comparative_rating.single_turn",
+    "format": "formats.llama3_instruct",
+    "main_score": "llama_3_3_70b_instruct_generic_engine_template_arena_hard"
+}

src/unitxt/catalog/metrics/llm_as_judge/pairwise_comparative_rating/llama_3_3_70b_instruct_generic_engine_template_arena_hard_with_shuffling.json

@@ -0,0 +1,11 @@
+{
+    "__type__": "llm_as_judge",
+    "inference_model": {
+        "__type__": "generic_inference_engine",
+        "default": "engines.ibm_gen_ai.llama_3_70b_instruct"
+    },
+    "template": "templates.response_assessment.pairwise_comparative_rating.arena_hard_with_shuffling",
+    "task": "pairwise_comparative_rating.single_turn",
+    "format": "formats.llama3_instruct",
+    "main_score": "llama_3_3_70b_instruct_generic_engine_template_arena_hard_with_shuffling"
+}

src/unitxt/catalog/metrics/llm_as_judge/pairwise_comparative_rating/llama_3_3_70b_instruct_ibm_wml_template_arena_hard.json

@@ -0,0 +1,13 @@
+{
+    "__type__": "llm_as_judge",
+    "inference_model": {
+        "__type__": "wml_inference_engine",
+        "model_name": "meta-llama/llama-3-3-70b-instruct",
+        "max_new_tokens": 2048,
+        "random_seed": 42
+    },
+    "template": "templates.response_assessment.pairwise_comparative_rating.arena_hard",
+    "task": "pairwise_comparative_rating.single_turn",
+    "format": "formats.llama3_instruct",
+    "main_score": "llama_3_3_70b_instruct_ibm_wml_template_arena_hard"
+}

src/unitxt/catalog/metrics/llm_as_judge/pairwise_comparative_rating/llama_3_3_70b_instruct_ibm_wml_template_arena_hard_with_shuffling.json

@@ -0,0 +1,13 @@
+{
+    "__type__": "llm_as_judge",
+    "inference_model": {
+        "__type__": "wml_inference_engine",
+        "model_name": "meta-llama/llama-3-3-70b-instruct",
+        "max_new_tokens": 2048,
+        "random_seed": 42
+    },
+    "template": "templates.response_assessment.pairwise_comparative_rating.arena_hard_with_shuffling",
+    "task": "pairwise_comparative_rating.single_turn",
+    "format": "formats.llama3_instruct",
+    "main_score": "llama_3_3_70b_instruct_ibm_wml_template_arena_hard_with_shuffling"
+}