summarize.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "source": [
    "# Sumarize Runs\n",
    "\n",
    "This is really just used for aggregating the results of different runs (stored in `./summaries` dir).\n",
    "At this point you should have done the runs you want (hint hint: maybe that means running `./run.sh` for you)."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "outputs": [],
   "source": [
    "import json\n",
    "from pathlib import Path\n",
    "from pprint import pprint\n",
    "\n",
    "import numpy as np\n",
    "from tabulate import tabulate"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-07T22:35:57.017114Z",
     "start_time": "2023-03-07T22:35:57.010532Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'bbc-news', 'sst2', 'SentEval-CR', 'imdb', '.nomedia', 'enron'}\n"
     ]
    }
   ],
   "source": [
    "# Lets see what folders exist\n",
    "dumpdir = Path('summaries')\n",
    "datasets = set(str(foldername.name.split('_')[0]) for foldername in dumpdir.glob('*'))\n",
    "pprint(datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-07T22:35:57.398825Z",
     "start_time": "2023-03-07T22:35:57.395842Z"
    }
   },
   "outputs": [],
   "source": [
    "# We now have to select all instances (per case) across all variations\n",
    "# So we have one list where all the case-n runs are"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-07T23:15:02.788676Z",
     "start_time": "2023-03-07T23:15:02.778232Z"
    }
   },
   "outputs": [],
   "source": [
    "def pull_case_runs(casenum):\n",
    "    runs = {}\n",
    "    for filename in dumpdir.glob(f'*/case_{casenum}.json'):\n",
    "        with filename.open(\"r\") as f:\n",
    "            runs.setdefault('_'.join(str(filename.parent.name).split('_')[:-1]),[]).append(json.load(f))\n",
    "    return runs\n",
    "    \n",
    "runs_0 = pull_case_runs(0)\n",
    "runs_1 = pull_case_runs(1)\n",
    "runs_2 = pull_case_runs(2)\n",
    "runs_3 = pull_case_runs(3)\n",
    "runs_4 = pull_case_runs(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-07T23:16:09.507801Z",
     "start_time": "2023-03-07T23:16:09.473034Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "def filter_runs(runs, num_sents:int):\n",
    "    filtered = {}\n",
    "    for datasetname, dataset_runs in runs.items():\n",
    "        for run in dataset_runs:\n",
    "            if not \"config\" in run or run['config']['num_sents'] == num_sents:\n",
    "                filtered.setdefault(datasetname, []).append(run)\n",
    "                \n",
    "    assert max(len(v) for v in filtered.values()) == 1\n",
    "    return {k:v[0] for k, v in filtered.items()}\n",
    "\n",
    "def summarize(runs):\n",
    "    # assume data is filtered\n",
    "    # output format = {dataset: {metricname: (mean, std)}}\n",
    "    output = {}\n",
    "    for datasetname, datasetrun in runs.items():\n",
    "        output[datasetname] = {}\n",
    "        for metricname, metricvalues in datasetrun.items():\n",
    "            if metricname == 'config':\n",
    "                continue\n",
    "            mean, std = np.mean(metricvalues), np.std(metricvalues)\n",
    "            output[datasetname][metricname] = (mean, std)\n",
    "            \n",
    "    return output\n",
    "\n",
    "def gettable(runs, rownames = None):\n",
    "    # give summarized filtered stuff\n",
    "    # get all datasets\n",
    "    \n",
    "    if not rownames:\n",
    "        rownames = list(range(len(runs)))\n",
    "        \n",
    "    datasets = set()\n",
    "    for datasetruns in runs:\n",
    "        print(datasetruns.keys())\n",
    "        datasets = datasets.union(set(datasetruns.keys()))\n",
    "    datasets = list(datasets)\n",
    "    print(datasets)\n",
    "    \n",
    "    # Each dataset is a column (header is dataset)\n",
    "    header = datasets[:]\n",
    "    \n",
    "    # Each cell in the colum should be 0.34 +- 0.11 or whatever\n",
    "    # Each row is a element of the runs list\n",
    "    table = []\n",
    "    for rowname, run in zip(rownames, runs):\n",
    "        row = [rowname]\n",
    "        for dataset in datasets:\n",
    "            \n",
    "            try:\n",
    "                data = run[dataset]['accuracy']\n",
    "                row.append(f\"{data[0]:.3f} ± {data[1]:.3f}\")\n",
    "            except KeyError:\n",
    "                row.append('N/A')\n",
    "        table.append(row)\n",
    "                \n",
    "    return table, header\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-07T23:16:14.364247Z",
     "start_time": "2023-03-07T23:16:14.354979Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['imdb', 'SentEval-CR'])\n",
      "dict_keys(['imdb', 'SentEval-CR'])\n",
      "dict_keys(['imdb', 'SentEval-CR'])\n",
      "dict_keys(['enron_spam', 'sst2', 'imdb', 'SentEval-CR', 'bbc-news'])\n",
      "dict_keys(['imdb', 'bbc-news', 'sst2'])\n",
      "['bbc-news', 'sst2', 'SentEval-CR', 'imdb', 'enron_spam']\n"
     ]
    }
   ],
   "source": [
    "s0 = summarize(filter_runs(runs_0, 100))\n",
    "s1 = summarize(filter_runs(runs_1, 100))\n",
    "s2 = summarize(filter_runs(runs_2, 100))\n",
    "s3 = summarize(filter_runs(runs_3, 10))\n",
    "s4 = summarize(filter_runs(runs_4, 100))\n",
    "\n",
    "table, header = gettable((s0, s2, s1, s3, s4),\n",
    "                         rownames=['SetFit FT', 'No Contrastive SetFit FT', 'Regular FT', 'LLM Prompting','Constrastive AL' ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-03-07T23:16:15.434618Z",
     "start_time": "2023-03-07T23:16:15.421964Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|                          | bbc-news      | sst2          | SentEval-CR   | imdb          | enron_spam    |\n",
      "|--------------------------|---------------|---------------|---------------|---------------|---------------|\n",
      "| SetFit FT                | N/A           | N/A           | 0.882 ± 0.029 | 0.924 ± 0.026 | N/A           |\n",
      "| No Contrastive SetFit FT | N/A           | N/A           | 0.886 ± 0.005 | 0.902 ± 0.019 | N/A           |\n",
      "| Regular FT               | N/A           | N/A           | 0.582 ± 0.054 | 0.836 ± 0.166 | N/A           |\n",
      "| LLM Prompting            | 0.950 ± 0.000 | 0.930 ± 0.000 | 0.900 ± 0.000 | 0.930 ± 0.000 | 0.820 ± 0.000 |\n",
      "| Constrastive AL          | 0.974 ± 0.000 | 0.925 ± 0.000 | N/A           | 0.926 ± 0.000 | N/A           |\n"
     ]
    }
   ],
   "source": [
    "print(tabulate(table, header, tablefmt='github'))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}