regression.cw09b.yaml Maven / Gradle / Ivy

Go to download
---
corpus: cw09b
corpus_path: collections/web/ClueWeb09b/

index_path: indexes/lucene-index.cw09b/
collection_class: ClueWeb09Collection
generator_class: DefaultLuceneDocumentGenerator
index_threads: 44
index_options: -storeRaw
index_stats:
  documents: 50220186
  documents (non-empty): 50220156
  total terms: 31303110383

metrics:
  - metric: MAP
    command: tools/eval/trec_eval.9.0.4/trec_eval
    params: -m map
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: true
  - metric: P30
    command: tools/eval/trec_eval.9.0.4/trec_eval
    params: -m P.30
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: true
  - metric: nDCG@20
    command: tools/eval/gdeval.pl
    separator: ","
    parse_index: -2
    metric_precision: 4
  - metric: ERR@20
    command: tools/eval/gdeval.pl
    separator: ","
    parse_index: -1
    metric_precision: 4

topic_reader: Webxml
topic_root: src/main/resources/topics-and-qrels/
qrels_root: src/main/resources/topics-and-qrels/
topics:
  - name: "[TREC 2010 Web Track (Topics 51-100)](../src/main/resources/topics-and-qrels/topics.web.51-100.txt)"
    id: trec2010
    path: topics.web.51-100.txt
    qrel: qrels.web.51-100.txt
  - name: "[TREC 2011 Web Track (Topics 101-150)](../src/main/resources/topics-and-qrels/topics.web.101-150.txt)"
    id: trec2011
    path: topics.web.101-150.txt
    qrel: qrels.web.101-150.txt
  - name: "[TREC 2012 Web Track (Topics 151-200)](../src/main/resources/topics-and-qrels/topics.web.151-200.txt)"
    id: trec2012
    path: topics.web.151-200.txt
    qrel: qrels.web.151-200.txt

models:
  - name: bm25
    display: BM25
    params: -parallelism 16 -bm25
    results:
      MAP:
        - 0.1126
        - 0.1094
        - 0.1105
      P30:
        - 0.2688
        - 0.2513
        - 0.2167
      nDCG@20:
        - 0.1351
        - 0.1894
        - 0.1014
      ERR@20:
        - 0.0733
        - 0.0959
        - 0.1304
  - name: bm25+rm3
    display: +RM3
    params: -parallelism 16 -bm25 -rm3 -collection ClueWeb09Collection
    results:
      MAP:
        - 0.0929
        - 0.1080
        - 0.1108
      P30:
        - 0.2375
        - 0.2433
        - 0.1920
      nDCG@20:
        - 0.1371
        - 0.1922
        - 0.0918
      ERR@20:
        - 0.0756
        - 0.0970
        - 0.1491
  - name: bm25+ax
    display: +Ax
    params: -parallelism 16 -bm25 -axiom -axiom.deterministic -axiom.beta 0.1 -rerankCutoff 20 -collection ClueWeb09Collection
    results:
      MAP:
        - 0.0961
        - 0.0986
        - 0.1355
      P30:
        - 0.2528
        - 0.2367
        - 0.2547
      nDCG@20:
        - 0.1767
        - 0.1854
        - 0.1388
      ERR@20:
        - 0.1019
        - 0.0950
        - 0.2399
  - name: ql
    display: QL
    params: -parallelism 16 -qld
    results:
      MAP:
        - 0.1060
        - 0.0959
        - 0.1070
      P30:
        - 0.2438
        - 0.2147
        - 0.2080
      nDCG@20:
        - 0.1143
        - 0.1631
        - 0.0876
      ERR@20:
        - 0.0599
        - 0.0850
        - 0.1306
  - name: ql+rm3
    display: +RM3
    params: -parallelism 16 -qld -rm3 -collection ClueWeb09Collection
    results:
      MAP:
        - 0.1018
        - 0.0835
        - 0.1057
      P30:
        - 0.2306
        - 0.2047
        - 0.1973
      nDCG@20:
        - 0.1181
        - 0.1444
        - 0.0896
      ERR@20:
        - 0.0591
        - 0.0781
        - 0.1333
  - name: ql+ax
    display: +Ax
    params: -parallelism 16 -qld -axiom -axiom.deterministic -axiom.beta 0.1 -rerankCutoff 20 -collection ClueWeb09Collection
    results:
      MAP:
        - 0.1088
        - 0.0860
        - 0.1224
      P30:
        - 0.2625
        - 0.2120
        - 0.2220
      nDCG@20:
        - 0.1495
        - 0.1537
        - 0.1091
      ERR@20:
        - 0.0751
        - 0.0861
        - 0.1564