Catalog changes because of code refactor

The order of the items in a dict changes Signed-off-by: Martín Santillán Cooper <[email protected]>
IBM · Feb 18, 2025 · 0afe6c6 · 0afe6c6
1 parent 5f89294
commit 0afe6c6
Show file tree

Hide file tree

Showing 34 changed files with 68 additions and 68 deletions.
diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/openai/gpt_4o.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/openai/gpt_4o.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "openai/gpt-4o-2024-08-06",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "openai/gpt-4o-2024-08-06"
     },
     "evaluator_name": "GPT4",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/openai/o1_mini.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/openai/o1_mini.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "openai/o1-mini-2024-09-12",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "openai/o1-mini-2024-09-12"
     },
     "evaluator_name": "O1_MINI",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/openai/o1_preview.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/openai/o1_preview.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "openai/o1-preview-2024-09-12",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "openai/o1-preview-2024-09-12"
     },
     "evaluator_name": "O1_PREVIEW",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/granite3_0_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/granite3_0_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "ibm-granite/granite-3.0-8b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "ibm-granite/granite-3.0-8b-instruct"
     },
     "evaluator_name": "GRANITE3_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/granite3_1_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/granite3_1_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "ibm-granite/granite-3.1-8b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "ibm-granite/granite-3.1-8b-instruct"
     },
     "evaluator_name": "GRANITE3_1_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/llama3_1_405b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/llama3_1_405b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "meta-llama/llama-3-1-405b-instruct-fp8",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "meta-llama/llama-3-1-405b-instruct-fp8"
     },
     "evaluator_name": "LLAMA3_1_405B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/llama3_1_70b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/llama3_1_70b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "meta-llama/llama-3-1-70b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "meta-llama/llama-3-1-70b-instruct"
     },
     "evaluator_name": "LLAMA3_1_70B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/llama3_1_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/llama3_1_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "meta-llama/Llama-3.1-8B-Instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "meta-llama/Llama-3.1-8B-Instruct"
     },
     "evaluator_name": "LLAMA3_1_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/mixtral8_22b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/mixtral8_22b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "mistralai/mixtral-8x22B-instruct-v0.1",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "mistralai/mixtral-8x22B-instruct-v0.1"
     },
     "evaluator_name": "MIXTRAL8_22b",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/mixtral8_7b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/mixtral8_7b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "mistralai/mixtral-8x7B-instruct-v0.1",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "mistralai/mixtral-8x7B-instruct-v0.1"
     },
     "evaluator_name": "MIXTRAL8_7b",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/mixtral_large.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/rits/mixtral_large.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "mistralai/mistral-large-instruct-2407",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "mistralai/mistral-large-instruct-2407"
     },
     "evaluator_name": "MIXTRAL_LARGE",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/granite3_0_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/granite3_0_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/ibm/granite-3-8b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/ibm/granite-3-8b-instruct"
     },
     "evaluator_name": "GRANITE3_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/llama3_1_405b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/llama3_1_405b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/meta-llama/llama-3-405b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/meta-llama/llama-3-405b-instruct"
     },
     "evaluator_name": "LLAMA3_1_405B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/llama3_1_70b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/llama3_1_70b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/meta-llama/llama-3-1-70b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/meta-llama/llama-3-1-70b-instruct"
     },
     "evaluator_name": "LLAMA3_1_70B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/llama3_1_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/llama3_1_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/meta-llama/llama-3-1-8b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/meta-llama/llama-3-1-8b-instruct"
     },
     "evaluator_name": "LLAMA3_1_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/mixtral8_7b.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/mixtral8_7b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/mistralai/mixtral-8x7b-instruct-v01",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/mistralai/mixtral-8x7b-instruct-v01"
     },
     "evaluator_name": "MIXTRAL8_7b",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/mixtral_large.json b/src/unitxt/catalog/metrics/llm_as_judge/direct/watsonx/mixtral_large.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_direct",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/mistralai/mistral-large",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/mistralai/mistral-large"
     },
     "evaluator_name": "MIXTRAL_LARGE",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/openai/gpt_4o.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/openai/gpt_4o.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "openai/gpt-4o-2024-08-06",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "openai/gpt-4o-2024-08-06"
     },
     "evaluator_name": "GPT4",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/openai/o1_mini.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/openai/o1_mini.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "openai/o1-mini-2024-09-12",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "openai/o1-mini-2024-09-12"
     },
     "evaluator_name": "O1_MINI",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/openai/o1_preview.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/openai/o1_preview.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "openai/o1-preview-2024-09-12",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "openai/o1-preview-2024-09-12"
     },
     "evaluator_name": "O1_PREVIEW",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/granite3_0_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/granite3_0_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "ibm-granite/granite-3.0-8b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "ibm-granite/granite-3.0-8b-instruct"
     },
     "evaluator_name": "GRANITE3_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/granite3_1_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/granite3_1_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "ibm-granite/granite-3.1-8b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "ibm-granite/granite-3.1-8b-instruct"
     },
     "evaluator_name": "GRANITE3_1_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/llama3_1_405b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/llama3_1_405b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "meta-llama/llama-3-1-405b-instruct-fp8",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "meta-llama/llama-3-1-405b-instruct-fp8"
     },
     "evaluator_name": "LLAMA3_1_405B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/llama3_1_70b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/llama3_1_70b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "meta-llama/llama-3-1-70b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "meta-llama/llama-3-1-70b-instruct"
     },
     "evaluator_name": "LLAMA3_1_70B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/llama3_1_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/llama3_1_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "meta-llama/Llama-3.1-8B-Instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "meta-llama/Llama-3.1-8B-Instruct"
     },
     "evaluator_name": "LLAMA3_1_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/mixtral8_22b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/mixtral8_22b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "mistralai/mixtral-8x22B-instruct-v0.1",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "mistralai/mixtral-8x22B-instruct-v0.1"
     },
     "evaluator_name": "MIXTRAL8_22b",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/mixtral8_7b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/mixtral8_7b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "mistralai/mixtral-8x7B-instruct-v0.1",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "mistralai/mixtral-8x7B-instruct-v0.1"
     },
     "evaluator_name": "MIXTRAL8_7b",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/mixtral_large.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/rits/mixtral_large.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "rits_inference_engine",
-        "model_name": "mistralai/mistral-large-instruct-2407",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model_name": "mistralai/mistral-large-instruct-2407"
     },
     "evaluator_name": "MIXTRAL_LARGE",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/granite3_0_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/granite3_0_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/ibm/granite-3-8b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/ibm/granite-3-8b-instruct"
     },
     "evaluator_name": "GRANITE3_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/llama3_1_405b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/llama3_1_405b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/meta-llama/llama-3-405b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/meta-llama/llama-3-405b-instruct"
     },
     "evaluator_name": "LLAMA3_1_405B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/llama3_1_70b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/llama3_1_70b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/meta-llama/llama-3-1-70b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/meta-llama/llama-3-1-70b-instruct"
     },
     "evaluator_name": "LLAMA3_1_70B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/llama3_1_8b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/llama3_1_8b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/meta-llama/llama-3-1-8b-instruct",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/meta-llama/llama-3-1-8b-instruct"
     },
     "evaluator_name": "LLAMA3_1_8B",
     "generate_summaries": false

diff --git a/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/mixtral8_7b.json b/src/unitxt/catalog/metrics/llm_as_judge/pairwise/watsonx/mixtral8_7b.json
@@ -2,9 +2,9 @@
     "__type__": "llm_judge_pairwise",
     "inference_engine": {
         "__type__": "lite_llm_inference_engine",
-        "model": "watsonx/mistralai/mixtral-8x7b-instruct-v01",
         "max_tokens": 1024,
-        "seed": 42
+        "seed": 42,
+        "model": "watsonx/mistralai/mixtral-8x7b-instruct-v01"
     },
     "evaluator_name": "MIXTRAL8_7b",
     "generate_summaries": false