lighteval/tests/unit/metrics/test_cases/maj_at_n.json at 150bbd3e9e6455a0ad4b16549632024f54dbbd35 · huggingface/lighteval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
{
  "name": "Maj At N Test Suite",
  "description": "Test cases for maj_at_n metric",
  "test_cases": [
    {
      "name": "Maj at N - Majority Correct",
      "metric_class": "maj_at_n",
      "metric_params": {"n": 3},
      "doc": {
        "query": "What is the capital of France?",
        "choices": ["London", "Paris", "Berlin"],
        "gold_index": [1],
        "task_name": "geography"
      },
      "model_response": {
        "text": ["Paris", "Paris", "London"]
      },
      "expected_output": {
        "maj@n:n=3": 1
      },
      "tolerance": 0.01,
      "description": "Test maj at n with majority correct"
    },
    {
      "name": "Maj at N - No Majority",
      "metric_class": "maj_at_n",
      "metric_params": {"n": 3},
      "doc": {
        "query": "What is the capital of France?",
        "choices": ["London", "Paris", "Berlin"],
        "gold_index": [1],
        "task_name": "geography"
      },
      "model_response": {
        "text": ["Paris", "London", "Berlin"]
      },
      "expected_output": {
        "maj@n:n=3": 1
      },
      "tolerance": 0.01,
      "description": "Test maj at n with no majority"
    },
    {
      "name": "Maj at N - All Correct",
      "metric_class": "maj_at_n",
      "metric_params": {"n": 3},
      "doc": {
        "query": "What is the capital of France?",
        "choices": ["London", "Paris", "Berlin"],
        "gold_index": [1],
        "task_name": "geography"
      },
      "model_response": {
        "text": ["Paris", "Paris", "Paris"]
      },
      "expected_output": {
        "maj@n:n=3": 1
      },
      "tolerance": 0.01,
      "description": "Test maj at n with all correct"
    },
    {
      "name": "Maj at N - Wrong Answer",
      "metric_class": "maj_at_n",
      "metric_params": {"n": 3},
      "doc": {
        "query": "What is the capital of France?",
        "choices": ["London", "Paris", "Berlin"],
        "gold_index": [1],
        "task_name": "geography"
      },
      "model_response": {
        "text": ["London", "London", "London"]
      },
      "expected_output": {
        "maj@n:n=3": 0
      },
      "tolerance": 0.01,
      "description": "Test maj at n with wrong answer"
    }
  ]
}