msmarco-genqa/configs/baseline.yaml at main · GioiaZheng/msmarco-genqa · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
# Baseline configuration for the MS MARCO Passage Ranking pipeline.
# Paths are interpreted relative to the project root.

seed: 42

data:
  cache_dir: data/raw            # ir_datasets cache (will be ~3 GB after first run)
  expected_corpus_size: "~8.8M"  # for reporting only
  expected_dev_queries: 6980     # for reporting only
  # Set ``corpus_limit`` to a small int (e.g. 200000) for a development smoke test.
  # Leave null for the official baseline.
  corpus_limit: null

retrieval:
  backend: bm25s
  k1: 1.5
  b: 0.75
  stopwords: en
  top_k: 1000
  index_dir: data/processed/bm25_index_msmarco

  # Checkpoint cadence: ``run.tsv`` is flushed after every ``chunk_size``
  # queries. If the process is killed, ``--resume`` will pick up at the next
  # chunk boundary instead of starting from query 0.
  chunk_size: 200

  # bm25s parallel-retrieve. ``0`` = sequential (the safe default that
  # produced the official MRR@10 = 0.1703 baseline). ``-1`` uses all CPUs;
  # positive integers spawn that many worker processes. On macOS,
  # multiprocessing-based parallelism is currently opt-in: it can be a 4-8x
  # win or strictly worse depending on whether bm25s ends up forking or
  # spawning. Try ``-1`` for new runs and compare wall-clock; if it doesn't
  # help, set it back to ``0``.
  n_threads: 0
  bm25s_chunksize: 50

query_transform:
  # Default stays off so historical retrieval baselines remain reproducible.
  # Set to normalize or lexical_expansion for controlled ablations.
  method: none
  output_dir: outputs/query_transform/none
  lowercase: true
  strip_terminal_punctuation: true
  max_query_terms_for_expansion: 6
  max_expansion_terms: 4
  expansion_terms:
    covid: [coronavirus]
    fda: [food and drug administration]
    flu: [influenza]
    heart attack: [myocardial infarction]
    irs: [internal revenue service]
    nyc: [new york city]
    u.s.: [united states]
    uk: [united kingdom]
    usa: [united states]

eval_retrieval:
  ks_mrr: [10]
  ks_ndcg: [10]
  ks_recall: [100, 1000]
  output_dir: outputs/bm25_baseline
  n_examples: 20                # qualitative examples dumped to examples.jsonl

generation:
  model_name: t5-small
  # HF revision pin. The full 40-hex commit SHA on the HuggingFace Hub repo.
  # Captured so the manifest records *which* checkpoint produced the run --
  # HF repos are not immutable, the maintainer can force-push the main
  # branch. To bump: look up the latest SHA via huggingface_hub.HfApi
  # (see commit infra(deps): pin HF model revisions).
  revision: df1b051c49625cf57a3d0d8d3863ed4d13564fe4
  max_input_length: 512
  max_new_tokens: 64
  top_k_passages: 3
  num_eval_queries: 200         # subset for the generation baseline (CPU-friendly)
  output_dir: outputs/generation
  # Disabled for canonical historical baselines. Enable in a separate
  # output directory when comparing prompt cost versus answer quality.
  context_packing:
    enabled: false
    max_context_chars: 1200
    max_passage_chars: 500
    sentence_selection: query_overlap
    deduplicate: true
    ordering: rank

eval_generation:
  metrics: [rouge-l, bleu, exact-match, token-f1]

# Dense retrieval baseline on a *sampled* sub-corpus.
# We do NOT embed the full 8.8M corpus on a CPU laptop. The sample is
# qrels-anchored: every dev/small relevant doc is included, then random
# distractors fill to ``sample_size``. This makes BM25-on-sample vs
# dense-on-sample an apples-to-apples comparison; both numbers are
# **not** comparable to the full-corpus BM25 baseline.
dense:
  model_name: sentence-transformers/all-MiniLM-L6-v2
  # HF revision pin (40-hex). See note under generation.revision.
  revision: c9745ed1d9f207416be6d2e6f8de32d1f16199bf
  sample_size: 50000
  encode_batch_size: 32
  device: null            # null -> auto (cuda if available, else cpu)
  index_dir: data/processed/dense_index_minilm_50k
  bm25_sample_index_dir: data/processed/bm25_sample_index_50k
  output_dir: outputs/dense_retrieval
  top_k: 1000
  compare_bm25_on_sample: true
  n_examples: 20          # qualitative examples dumped to examples.jsonl

# Cross-encoder reranking on top of the dense run.
# We score (query, passage) pairs with a small MS MARCO cross-encoder.
# Cost is O(rerank_top_k) forward passes per query -- do NOT default to
# 1000. Top-100 is the standard reranking depth.
reranker:
  model_name: cross-encoder/ms-marco-MiniLM-L-6-v2
  # HF revision pin (40-hex). See note under generation.revision.
  revision: c5ee24cb16019beea0893ab7796b1df96625c6b8
  rerank_top_k: 100
  batch_size: 64
  max_length: 512
  device: null            # null -> auto (cuda if available, else cpu)
  output_dir: outputs/cross_encoder_rerank
  n_examples: 20

  # Resume / chunking. ``chunk_size`` queries are scored and appended to
  # ``run.tsv`` per flush. With top_k=100 on CPU (~30 pairs/s) a chunk
  # of 200 queries is ~11 min of work, so a SIGKILL costs at most ~11
  # min and ``--resume`` picks up from the next chunk. Smaller =
  # more durable but more file-flush overhead.
  chunk_size: 200

# Research evaluation workflow facade. `rag-eval run --config configs/baseline.yaml`
# reads this section to build an auditable BM25 -> dense -> rerank -> generation
# -> context packing -> bootstrap -> grounding -> triad plan without duplicating
# stage logic.
rag_eval:
  stages:
    - query_transformation
    - bm25_retrieval
    - dense_retrieval
    - cross_encoder_rerank
    - retrieval_quality_report
    - retrieval_lift_analysis
    - generation_bm25
    - generation_reranked
    - generation_reranked_packed
    - context_packing_report
    - paired_bootstrap_ci
    - grounding_audit
    - rag_triad
  num_eval_queries: 9999
  bm25_generation_dir: outputs/generation_bm25_full
  reranker_output_dir: outputs/cross_encoder_rerank_full
  reranker_resume: true
  reranked_generation_dir: outputs/generation_reranked_full
  packed_generation_dir: outputs/generation_reranked_packed
  context_packing_report_dir: outputs/context_packing
  context_max_chars: 900
  context_max_passage_chars: 320
  context_sentence_selection: query_overlap
  context_ordering: rank
  context_deduplicate: true
  bootstrap_output_dir: outputs/generation_bootstrap_full
  retrieval_report_output_dir: outputs/retrieval_reports/dense_vs_reranked
  retrieval_lift_output_dir: outputs/retrieval_lift_analysis
  grounding_output_dir: outputs/grounding
  triad_output_dir: outputs/rag_triad
  bootstrap_resamples: 10000
  grounding_nli_pairs: 0
  tracking:
    backend: jsonl
    output_dir: outputs/rag_eval_tracking