regression.msmarco-v1-doc.yaml Maven / Gradle / Ivy
---
corpus: msmarco-doc
corpus_path: collections/msmarco/msmarco-doc/
index_path: indexes/lucene-inverted.msmarco-v1-doc/
collection_class: JsonCollection
generator_class: DefaultLuceneDocumentGenerator
index_threads: 7
index_options: -storePositions -storeDocvectors -storeRaw
index_stats:
documents: 3213835
documents (non-empty): 3213835
total terms: 2742219865
metrics:
- metric: AP@1000
command: bin/trec_eval
params: -c -m map
separator: "\t"
parse_index: 2
metric_precision: 4
can_combine: false
- metric: RR@100
command: bin/trec_eval
params: -c -M 100 -m recip_rank
separator: "\t"
parse_index: 2
metric_precision: 4
can_combine: false
- metric: R@100
command: bin/trec_eval
params: -c -m recall.100
separator: "\t"
parse_index: 2
metric_precision: 4
can_combine: false
- metric: R@1000
command: bin/trec_eval
params: -c -m recall.1000
separator: "\t"
parse_index: 2
metric_precision: 4
can_combine: false
topic_reader: TsvInt
topics:
- name: "[MS MARCO Doc: Dev](https://github.com/microsoft/MSMARCO-Document-Ranking)"
id: dev
path: topics.msmarco-doc.dev.txt
qrel: qrels.msmarco-doc.dev.txt
models:
- name: bm25-default
display: BM25 (default)
params: -bm25
results:
AP@1000:
- 0.2305
RR@100:
- 0.2299
R@100:
- 0.7281
R@1000:
- 0.8856
# PRF regressions are no longer maintained for sparse judgments to reduce running times.
# (commenting out instead of removing; in case these numbers are needed, just uncomment and rerun.)
#
# - name: bm25-default+rm3
# display: +RM3
# params: -bm25 -rm3
# results:
# AP@1000:
# - 0.1627
# RR@100:
# - 0.1618
# R@100:
# - 0.6771
# R@1000:
# - 0.8783
# - name: bm25-default+rocchio
# display: +Rocchio
# params: -bm25 -rocchio
# results:
# AP@1000:
# - 0.1632
# RR@100:
# - 0.1624
# R@100:
# - 0.6765
# R@1000:
# - 0.8789
# - name: bm25-default+rocchio-neg
# display: +Rocchio*
# params: -bm25 -rocchio -rocchio.useNegative -rerankCutoff 1000
# results:
# AP@1000:
# - 0.1630
# RR@100:
# - 0.1622
# R@100:
# - 0.6794
# R@1000:
# - 0.8808
# - name: bm25-default+ax
# display: +Ax
# params: -bm25 -axiom -axiom.deterministic -rerankCutoff 20
# results:
# AP@1000:
# - 0.1146
# RR@100:
# - 0.1135
# R@100:
# - 0.5754
# R@1000:
# - 0.8373
# - name: bm25-default+prf
# display: +PRF
# params: -bm25 -bm25prf
# results:
# AP@1000:
# - 0.1357
# RR@100:
# - 0.1347
# R@100:
# - 0.6374
# R@1000:
# - 0.8471
- name: bm25-tuned
display: BM25 (tuned)
params: -bm25 -bm25.k1 3.44 -bm25.b 0.87
results:
AP@1000:
- 0.2784
RR@100:
- 0.2778
R@100:
- 0.8069
R@1000:
- 0.9324
# PRF regressions are no longer maintained for sparse judgments to reduce running times.
# (commenting out instead of removing; in case these numbers are needed, just uncomment and rerun.)
#
# - name: bm25-tuned+rm3
# display: +RM3
# params: -bm25 -bm25.k1 3.44 -bm25.b 0.87 -rm3
# results:
# AP@1000:
# - 0.2271
# RR@100:
# - 0.2264
# R@100:
# - 0.7870
# R@1000:
# - 0.9322
# - name: bm25-tuned+rocchio
# display: +Rocchio
# params: -bm25 -bm25.k1 3.44 -bm25.b 0.87 -rocchio
# results:
# AP@1000:
# - 0.2280
# RR@100:
# - 0.2274
# R@100:
# - 0.7901
# R@1000:
# - 0.9334
# - name: bm25-tuned+rocchio-neg
# display: +Rocchio*
# params: -bm25 -bm25.k1 3.44 -bm25.b 0.87 -rocchio -rocchio.useNegative -rerankCutoff 1000
# results:
# AP@1000:
# - 0.2271
# RR@100:
# - 0.2265
# R@100:
# - 0.7924
# R@1000:
# - 0.9324
# - name: bm25-tuned+ax
# display: +Ax
# params: -bm25 -bm25.k1 3.44 -bm25.b 0.87 -axiom -axiom.deterministic -rerankCutoff 20
# results:
# AP@1000:
# - 0.1888
# RR@100:
# - 0.1880
# R@100:
# - 0.7560
# R@1000:
# - 0.9268
# - name: bm25-tuned+prf
# display: +PRF
# params: -bm25 -bm25.k1 3.44 -bm25.b 0.87 -bm25prf
# results:
# AP@1000:
# - 0.1559
# RR@100:
# - 0.1550
# R@100:
# - 0.6852
# R@1000:
# - 0.8760
- name: bm25-tuned2
display: BM25 (tuned2)
params: -bm25 -bm25.k1 4.46 -bm25.b 0.82
results:
AP@1000:
- 0.2773
RR@100:
- 0.2767
R@100:
- 0.8070
R@1000:
- 0.9357
# PRF regressions are no longer maintained for sparse judgments to reduce running times.
# (commenting out instead of removing; in case these numbers are needed, just uncomment and rerun.)
#
# - name: bm25-tuned2+rm3
# display: +RM3
# params: -bm25 -bm25.k1 4.46 -bm25.b 0.82 -rm3
# results:
# AP@1000:
# - 0.2234
# RR@100:
# - 0.2227
# R@100:
# - 0.7785
# R@1000:
# - 0.9303
# - name: bm25-tuned2+rocchio
# display: +Rocchio
# params: -bm25 -bm25.k1 4.46 -bm25.b 0.82 -rocchio
# results:
# AP@1000:
# - 0.2248
# RR@100:
# - 0.2242
# R@100:
# - 0.7878
# R@1000:
# - 0.9314
# - name: bm25-tuned2+rocchio-neg
# display: +Rocchio*
# params: -bm25 -bm25.k1 4.46 -bm25.b 0.82 -rocchio -rocchio.useNegative -rerankCutoff 1000
# results:
# AP@1000:
# - 0.2231
# RR@100:
# - 0.2224
# R@100:
# - 0.7863
# R@1000:
# - 0.9316
# - name: bm25-tuned2+ax
# display: +Ax
# params: -bm25 -bm25.k1 4.46 -bm25.b 0.82 -axiom -axiom.deterministic -rerankCutoff 20
# results:
# AP@1000:
# - 0.1886
# RR@100:
# - 0.1877
# R@100:
# - 0.7526
# R@1000:
# - 0.9249
# - name: bm25-tuned2+prf
# display: +PRF
# params: -bm25 -bm25.k1 4.46 -bm25.b 0.82 -bm25prf
# results:
# AP@1000:
# - 0.1530
# RR@100:
# - 0.1521
# R@100:
# - 0.6825
# R@1000:
# - 0.8766
© 2015 - 2025 Weber Informatics LLC | Privacy Policy