fix(eval): handle variable retrieval depths in MKQA recall

FlagEmbedding/evaluation/mkqa/utils/compute_metrics.py

@@ -77,7 +77,8 @@ def evaluate_qa_recall(ctxs, answers, k_values: Union[int, List[int]]=100):
 
     scores = map(get_score_partial, data)
 
-    n_docs = len(data[0]['ctxs'])
+    # Retrieval depth can vary by query, so size the buckets to the deepest result list.
+    n_docs = max(len(example['ctxs']) for example in data)
     top_k_hits = [0] * n_docs
     for question_hits in scores:
         best_hit = next((i for i, x in enumerate(question_hits) if x), None)

tests/test_mkqa_metrics.py

@@ -0,0 +1,13 @@
+from FlagEmbedding.evaluation.mkqa.utils.compute_metrics import evaluate_qa_recall
+
+
+def test_evaluate_qa_recall_supports_variable_retrieval_depths():
+    contexts = [
+        ["no answer here"],
+        ["still no answer", "the target answer is here"],
+    ]
+    answers = [["target answer"], ["target answer"]]
+
+    recall = evaluate_qa_recall(contexts, answers, k_values=[1, 2])
+
+    assert recall == [0.0, 0.5]