regression.neuclir22-zh-qt.yaml Maven / Gradle / Ivy

Go to download
---
corpus: neuclir22-zh
corpus_path: collections/multilingual/neuclir22-zh/

index_path: indexes/lucene-index.neuclir22-zh
collection_class: NeuClirCollection
generator_class: DefaultLuceneDocumentGenerator
index_threads: 8
index_options: -storePositions -storeDocvectors -storeRaw -language zh
index_stats:
  documents: 3179206
  documents (non-empty): 3179206

metrics:
  - metric: MAP
    command: bin/trec_eval
    params: -c -m map
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: true
  - metric: nDCG@20
    command: bin/trec_eval
    params: -c -m ndcg_cut.20
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: false
  - metric: J@20
    command: python -m pyserini.eval.trec_eval
    params: -c -m judged.20
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: false
  - metric: Recall@1000
    command: bin/trec_eval
    params: -c -m recall.1000
    separator: "\t"
    parse_index: 2
    metric_precision: 4
    can_combine: false

topic_reader: TsvInt
topics:
  - name: "[NeuCLIR 2022 (Chinese): title (human-translated queries)](https://neuclir.github.io/)"
    id: ht-title
    path: topics.neuclir22-zh.ht-title.txt
    qrel: qrels.neuclir22-zh.txt
  - name: "[NeuCLIR 2022 (Chinese): desc (human-translated queries)](https://neuclir.github.io/)"
    id: ht-desc
    path: topics.neuclir22-zh.ht-desc.txt
    qrel: qrels.neuclir22-zh.txt
  - name: "[NeuCLIR 2022 (Chinese): desc+title (human-translated queries)](https://neuclir.github.io/)"
    id: ht-desc_title
    path: topics.neuclir22-zh.ht-desc_title.txt
    qrel: qrels.neuclir22-zh.txt
  - name: "[NeuCLIR 2022 (Chinese): title (machine-translated queries)](https://neuclir.github.io/)"
    id: mt-title
    path: topics.neuclir22-zh.mt-title.txt
    qrel: qrels.neuclir22-zh.txt
  - name: "[NeuCLIR 2022 (Chinese): desc (machine-translated queries)](https://neuclir.github.io/)"
    id: mt-desc
    path: topics.neuclir22-zh.mt-desc.txt
    qrel: qrels.neuclir22-zh.txt
  - name: "[NeuCLIR 2022 (Chinese): desc+title (machine-translated queries)](https://neuclir.github.io/)"
    id: mt-desc_title
    path: topics.neuclir22-zh.mt-desc_title.txt
    qrel: qrels.neuclir22-zh.txt

models:
  - name: bm25-default
    display: BM25 (default)
    params: -bm25 -language zh
    results:
      MAP:
        - 0.1505
        - 0.1306
        - 0.1734
        - 0.1009
        - 0.0791
        - 0.1053
      nDCG@20:
        - 0.2478
        - 0.2068
        - 0.2572
        - 0.1830
        - 0.1498
        - 0.1754
      J@20:
        - 0.3895
        - 0.3588
        - 0.4298
        - 0.3750
        - 0.2925
        - 0.3961
      Recall@1000:
        - 0.4759
        - 0.4577
        - 0.4940
        - 0.3829
        - 0.2989
        - 0.4028
  - name: bm25-default+rm3
    display: +RM3
    params: -bm25 -rm3 -language zh
    results:
      MAP:
        - 0.1501
        - 0.0925
        - 0.1367
        - 0.0995
        - 0.0454
        - 0.0731
      nDCG@20:
        - 0.2127
        - 0.1519
        - 0.1891
        - 0.1571
        - 0.0868
        - 0.1152
      J@20:
        - 0.3132
        - 0.2461
        - 0.2925
        - 0.3303
        - 0.2031
        - 0.2522
      Recall@1000:
        - 0.4651
        - 0.3703
        - 0.4373
        - 0.3621
        - 0.2462
        - 0.2746
  - name: bm25-default+rocchio
    display: +Rocchio
    params: -bm25 -rocchio -language zh
    results:
      MAP:
        - 0.1692
        - 0.1251
        - 0.1888
        - 0.1134
        - 0.0800
        - 0.1095
      nDCG@20:
        - 0.2544
        - 0.1985
        - 0.2734
        - 0.1861
        - 0.1464
        - 0.1785
      J@20:
        - 0.4004
        - 0.3689
        - 0.4298
        - 0.3842
        - 0.3276
        - 0.3934
      Recall@1000:
        - 0.5230
        - 0.5113
        - 0.5327
        - 0.4361
        - 0.3748
        - 0.4341