examples/trec_29_deep_documents_overlap_depth_20.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTerrier 0.8.0 has loaded Terrier 5.6 (built by craigmacdonald on 2021-09-17 13:27)\n",
      "\n",
      "No etc/terrier.properties, using terrier.default.properties for bootstrap configuration.\n"
     ]
    }
   ],
   "source": [
    "# Start/initialize PyTerrier.\n",
    "from pyterrier import started, init\n",
    "\n",
    "if not started():\n",
    "    init(tqdm=\"auto\")"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "edition = 29\n",
    "track = \"deep.documents\"\n",
    "dataset_name = \"msmarco-document/trec-dl-2020/judged\"\n",
    "depth = 20"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "result_dir = Path(\n",
    "    \"/mnt/ceph/storage/data-in-progress/data-research/\"\n",
    "    \"web-search/web-search-trec/trec-system-runs\"\n",
    ") / f\"trec{edition}\" / track\n",
    "result_files = list(result_dir.iterdir())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "outputs": [
    {
     "data": {
      "text/plain": "Load results:   0%|          | 0/64 [00:00<?, ?it/s]",
      "application/vnd.jupyter.widget-view+json": {
       "version_major": 2,
       "version_minor": 0,
       "model_id": "dad3c68c8565414baa57968cee88d2e2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from pandas import concat\n",
    "from pyterrier.io import read_results\n",
    "from tqdm.auto import tqdm\n",
    "\n",
    "results = concat([\n",
    "    read_results(result_file).groupby(by=\"qid\").head(20)\n",
    "    for result_file in tqdm(result_files, desc=\"Load results\")\n",
    "])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "outputs": [
    {
     "data": {
      "text/plain": "           qid     docno  rank     score           name\n0         3505  D2097653     2  0.996275     uob_runid3\n1         3505    D21135     1  0.996275     uob_runid3\n2         3505   D569493     3  0.995899     uob_runid3\n3         3505   D970726     4  0.995527     uob_runid3\n4         3505  D2136980     5  0.995293     uob_runid3\n...        ...       ...   ...       ...            ...\n19915  1136962   D949811    16 -0.102991  bigIR-DH-T5-R\n19916  1136962  D3074938    17 -0.115630  bigIR-DH-T5-R\n19917  1136962  D1631353    18 -0.129743  bigIR-DH-T5-R\n19918  1136962  D2354862    19 -0.132265  bigIR-DH-T5-R\n19919  1136962   D650851    20 -0.143323  bigIR-DH-T5-R\n\n[255651 rows x 5 columns]",
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>qid</th>\n      <th>docno</th>\n      <th>rank</th>\n      <th>score</th>\n      <th>name</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>3505</td>\n      <td>D2097653</td>\n      <td>2</td>\n      <td>0.996275</td>\n      <td>uob_runid3</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>3505</td>\n      <td>D21135</td>\n      <td>1</td>\n      <td>0.996275</td>\n      <td>uob_runid3</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>3505</td>\n      <td>D569493</td>\n      <td>3</td>\n      <td>0.995899</td>\n      <td>uob_runid3</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>3505</td>\n      <td>D970726</td>\n      <td>4</td>\n      <td>0.995527</td>\n      <td>uob_runid3</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>3505</td>\n      <td>D2136980</td>\n      <td>5</td>\n      <td>0.995293</td>\n      <td>uob_runid3</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>19915</th>\n      <td>1136962</td>\n      <td>D949811</td>\n      <td>16</td>\n      <td>-0.102991</td>\n      <td>bigIR-DH-T5-R</td>\n    </tr>\n    <tr>\n      <th>19916</th>\n      <td>1136962</td>\n      <td>D3074938</td>\n      <td>17</td>\n      <td>-0.115630</td>\n      <td>bigIR-DH-T5-R</td>\n    </tr>\n    <tr>\n      <th>19917</th>\n      <td>1136962</td>\n      <td>D1631353</td>\n      <td>18</td>\n      <td>-0.129743</td>\n      <td>bigIR-DH-T5-R</td>\n    </tr>\n    <tr>\n      <th>19918</th>\n      <td>1136962</td>\n      <td>D2354862</td>\n      <td>19</td>\n      <td>-0.132265</td>\n      <td>bigIR-DH-T5-R</td>\n    </tr>\n    <tr>\n      <th>19919</th>\n      <td>1136962</td>\n      <td>D650851</td>\n      <td>20</td>\n      <td>-0.143323</td>\n      <td>bigIR-DH-T5-R</td>\n    </tr>\n  </tbody>\n</table>\n<p>255651 rows × 5 columns</p>\n</div>"
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "outputs": [],
   "source": [
    "docs = results[\"qid\"] + results[\"docno\"]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "outputs": [
    {
     "data": {
      "text/plain": "0           3505D2097653\n1             3505D21135\n2            3505D569493\n3            3505D970726\n4           3505D2136980\n              ...       \n19915     1136962D949811\n19916    1136962D3074938\n19917    1136962D1631353\n19918    1136962D2354862\n19919     1136962D650851\nLength: 255651, dtype: object"
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "outputs": [],
   "source": [
    "num_docs = len(docs)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "outputs": [
    {
     "data": {
      "text/plain": "array(['3505D2097653', '3505D21135', '3505D569493', ..., '1110678D483502',\n       '1113256D653486', '1134939D210328'], dtype=object)"
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "docs.unique()"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "outputs": [],
   "source": [
    "num_uniq_docs = len(docs.unique())"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [],
   "source": [
    "uniq = num_uniq_docs / num_docs"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "outputs": [
    {
     "data": {
      "text/plain": "0.8774031785520103"
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1 - uniq"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}