Update BlueBench to match the original implementation (#1855)

* Updates to BlueBench to better match the internal version.

Signed-off-by: Jonathan Bnayahu <[email protected]>

* Resolve warning on undelared fields.

Signed-off-by: Jonathan Bnayahu <[email protected]>

* Switch to the detailed entity extraction template

Signed-off-by: Jonathan Bnayahu <[email protected]>

---------

Signed-off-by: Jonathan Bnayahu <[email protected]>
Co-authored-by: Elron Bandel <[email protected]>

Author: bnayahu

examples/evaluate_bluebench.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+
+######################################################################################################
+# BlueBench is an open-source benchmark developed by domain experts to represent required needs of   #
+# Enterprise users. It is constructed using state-of-the-art benchmarking methodologies to ensure    #
+# validity, robustness, and efficiency by utilizing unitxt’s abilities for dynamic and flexible text #
+# processing. As a dynamic and evolving benchmark, BlueBench currently encompasses diverse domains   #
+# such as legal, finance, customer support, and news. It also evaluates a range of capabilities,     #
+# including RAG, pro-social behavior, summarization, and chatbot performance, with additional tasks  #
+# and domains to be integrated over time.                                                            #
+#                                                                                                    #
+# Further details: https://ibm.biz/bluebench                                                         #
+######################################################################################################
+
+if [ $# -eq 0 ]
+  then
+    echo "Usage: evaluate_bluebench.sh model-to-evaluate-in-litellm-format"
+    exit 1
+fi
+
+unitxt-evaluate \
+    --tasks "benchmarks.bluebench" \
+    --model cross_provider \
+    --model_args "model_name=$1,max_tokens=1024" \
+    --output_path ./results/bluebench \
+    --log_samples \
+    --trust_remote_code \
+    --batch_size 8 \
+    --verbosity ERROR
+
+unitxt-summarize ./results/bluebench

prepare/cards/attaq_500.py

@@ -519,8 +519,9 @@
         Shuffle(page_size=2800),
     ],
     task=Task(
-        input_fields=["input"],
-        reference_fields=["label"],
+        input_fields={"input": str},
+        reference_fields={"label": str},
+        prediction_type=str,
         metrics=["metrics.safety_metric"],
     ),
     templates=[

prepare/recipes/bluebench.py

@@ -66,7 +66,7 @@
     "template_card_index": 1,
     "max_train_instances": 1000,
     "max_validation_instances": 1000,
-    "max_test_instances": 1000,
+    "max_test_instances": 100,
 }
 
 
@@ -82,6 +82,8 @@ def prepare_recipe(default_args, specific_args):
 
     if "template" in recipe and "template_card_index" in recipe:
         del recipe["template_card_index"]
+
+    # Note: BlueBench only uses the chat_api format.
     return DatasetRecipe(**recipe, format="formats.chat_api")
 
 
@@ -145,6 +147,7 @@ def prepare_recipe(default_args, specific_args):
 ingridients = {
     "card": "cards.20_newsgroups_short",
     "template": "templates.classification.multi_class.bluebench",
+    "num_demos": 1,
 }
 recipe = prepare_recipe(default_args, ingridients)
 add_to_catalog(
@@ -159,7 +162,7 @@ def prepare_recipe(default_args, specific_args):
         "card": f"cards.safety.bbq.{subset}",
         "demos_pool_size": 20,
         "num_demos": 5,
-        "template": "templates.qa.multiple_choice.with_context.match",
+        "template": "templates.qa.multiple_choice.with_context.bluebench",
         "demos_taken_from": "test",
     }
     recipe = prepare_recipe(default_args, ingridients)
@@ -178,6 +181,7 @@ def prepare_recipe(default_args, specific_args):
         "demos_pool_size": 10,
         "template": "templates.classification.multi_class.bluebench",
         "demos_taken_from": "test",
+        "num_demos": 1,
     }
     recipe = prepare_recipe(default_args, ingridients)
     add_to_catalog(
@@ -235,10 +239,11 @@ def prepare_recipe(default_args, specific_args):
         "card": f"cards.universal_ner.{subset}",
         "demos_pool_size": 10000,
         "num_demos": 5,
-        "template": "templates.span_labeling.extraction.title",
+        "template": "templates.span_labeling.extraction.detailed",
         "metrics": ["metrics.ner[zero_division=1.0]"],
         "train_refiner": "operators.balancers.ner.zero_vs_many_entities[segments_boundaries=[0,1,2]]",
         "demos_taken_from": "test" if "pud" in subset else "train",
+        "max_train_instances": 10000,
     }
     recipe = prepare_recipe(default_args, ingridients)
     add_to_catalog(
@@ -293,6 +298,7 @@ def prepare_recipe(default_args, specific_args):
 ingridients = {
     "card": "cards.rag.response_generation.clapnq",
     "template": "templates.rag.response_generation.bluebench",
+    "num_demos": 1,
 }
 recipe = prepare_recipe(default_args, ingridients)
 add_to_catalog(
@@ -306,7 +312,7 @@ def prepare_recipe(default_args, specific_args):
 
 ingridients = {
     "card": "cards.fin_qa",
-    "num_demos": 1,
+    "num_demos": 2,
     "template_card_index": 0,
 }
 recipe = prepare_recipe(default_args, ingridients)

src/unitxt/catalog/cards/attaq_500.json

@@ -526,12 +526,13 @@
     ],
     "task": {
         "__type__": "task",
-        "input_fields": [
-            "input"
-        ],
-        "reference_fields": [
-            "label"
-        ],
+        "input_fields": {
+            "input": "str"
+        },
+        "reference_fields": {
+            "label": "str"
+        },
+        "prediction_type": "str",
         "metrics": [
             "metrics.safety_metric"
         ]

src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_age.json

@@ -5,8 +5,8 @@
     "demos_taken_from": "test",
     "max_train_instances": 1000,
     "max_validation_instances": 1000,
-    "max_test_instances": 90,
+    "max_test_instances": 9,
     "card": "cards.safety.bbq.Age",
-    "template": "templates.qa.multiple_choice.with_context.match",
+    "template": "templates.qa.multiple_choice.with_context.bluebench",
     "format": "formats.chat_api"
 }

src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_disability_status.json

@@ -5,8 +5,8 @@
     "demos_taken_from": "test",
     "max_train_instances": 1000,
     "max_validation_instances": 1000,
-    "max_test_instances": 90,
+    "max_test_instances": 9,
     "card": "cards.safety.bbq.Disability_status",
-    "template": "templates.qa.multiple_choice.with_context.match",
+    "template": "templates.qa.multiple_choice.with_context.bluebench",
     "format": "formats.chat_api"
 }

src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_gender_identity.json

@@ -5,8 +5,8 @@
     "demos_taken_from": "test",
     "max_train_instances": 1000,
     "max_validation_instances": 1000,
-    "max_test_instances": 90,
+    "max_test_instances": 9,
     "card": "cards.safety.bbq.Gender_identity",
-    "template": "templates.qa.multiple_choice.with_context.match",
+    "template": "templates.qa.multiple_choice.with_context.bluebench",
     "format": "formats.chat_api"
 }

src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_nationality.json

@@ -5,8 +5,8 @@
     "demos_taken_from": "test",
     "max_train_instances": 1000,
     "max_validation_instances": 1000,
-    "max_test_instances": 90,
+    "max_test_instances": 9,
     "card": "cards.safety.bbq.Nationality",
-    "template": "templates.qa.multiple_choice.with_context.match",
+    "template": "templates.qa.multiple_choice.with_context.bluebench",
     "format": "formats.chat_api"
 }

src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_physical_appearance.json

@@ -5,8 +5,8 @@
     "demos_taken_from": "test",
     "max_train_instances": 1000,
     "max_validation_instances": 1000,
-    "max_test_instances": 90,
+    "max_test_instances": 9,
     "card": "cards.safety.bbq.Physical_appearance",
-    "template": "templates.qa.multiple_choice.with_context.match",
+    "template": "templates.qa.multiple_choice.with_context.bluebench",
     "format": "formats.chat_api"
 }

src/unitxt/catalog/recipes/bluebench/bias/safety_bbq_race_ethnicity.json

@@ -5,8 +5,8 @@
     "demos_taken_from": "test",
     "max_train_instances": 1000,
     "max_validation_instances": 1000,
-    "max_test_instances": 90,
+    "max_test_instances": 9,
     "card": "cards.safety.bbq.Race_ethnicity",
-    "template": "templates.qa.multiple_choice.with_context.match",
+    "template": "templates.qa.multiple_choice.with_context.bluebench",
     "format": "formats.chat_api"
 }