calculating_stats_from_results.ipynb

{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"id":"toPxGtnPI6J-"},"outputs":[],"source":["from glob import glob\n","import os\n","import re\n","import pandas as pd\n","pd.set_option(\"display.max_colwidth\", None)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"oerS3BPOJVHR"},"outputs":[],"source":["data_path = \"/content/drive/MyDrive/CS 159/\"\n","assert os.path.isdir(data_path)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":739},"executionInfo":{"elapsed":355,"status":"ok","timestamp":1717489858504,"user":{"displayName":"Yulun Wu","userId":"15728323746379503625"},"user_tz":420},"id":"M-OayExtJPUV","outputId":"f3c375ab-0299-4922-afae-a8659e18f8e2"},"outputs":[{"data":{"application/vnd.google.colaboratory.intrinsic+json":{"summary":"{\n  \"name\": \"internal_rep_df\",\n  \"rows\": 22,\n  \"fields\": [\n    {\n      \"column\": \"file\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 22,\n        \"samples\": [\n          \"/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_internalrep-baseline.csv\",\n          \"/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_grid.csv\",\n          \"/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_grid.csv\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"created_at\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": \"2024-05-24 01:03:38\",\n        \"max\": \"2024-05-25 06:47:20\",\n        \"num_unique_values\": 22,\n        \"samples\": [\n          \"2024-05-24 01:03:38\",\n          \"2024-05-24 07:33:59\",\n          \"2024-05-24 04:49:55\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task_type\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"square\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task_size\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"3\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task_steps\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"4\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"num_questions\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 100,\n        \"max\": 100,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          100\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"model\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"Llama70b\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"prompt_type\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"internalrep-baseline\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.1,\n        \"max\": 0.98,\n        \"num_unique_values\": 20,\n        \"samples\": [\n          0.19\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}","type":"dataframe","variable_name":"internal_rep_df"},"text/html":["\n","  <div id=\"df-22ed2973-7ae9-40cc-924d-df816f6d69b4\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>file</th>\n","      <th>created_at</th>\n","      <th>task_type</th>\n","      <th>task_size</th>\n","      <th>task_steps</th>\n","      <th>num_questions</th>\n","      <th>model</th>\n","      <th>prompt_type</th>\n","      <th>accuracy</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_internalrep-baseline.csv</td>\n","      <td>2024-05-24 01:03:38</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama8b</td>\n","      <td>internalrep-baseline</td>\n","      <td>0.19</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_internalrep-baseline.csv</td>\n","      <td>2024-05-24 01:11:04</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama70b</td>\n","      <td>internalrep-baseline</td>\n","      <td>0.63</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_base.csv</td>\n","      <td>2024-05-24 04:14:28</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama70b</td>\n","      <td>base</td>\n","      <td>0.8</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_grid.csv</td>\n","      <td>2024-05-24 04:21:47</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama70b</td>\n","      <td>grid</td>\n","      <td>0.53</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_csv.csv</td>\n","      <td>2024-05-24 04:26:28</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama70b</td>\n","      <td>csv</td>\n","      <td>0.56</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_coord.csv</td>\n","      <td>2024-05-24 04:35:40</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama70b</td>\n","      <td>coord</td>\n","      <td>0.58</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_colbycol.csv</td>\n","      <td>2024-05-24 04:40:38</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama70b</td>\n","      <td>colbycol</td>\n","      <td>0.49</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_base.csv</td>\n","      <td>2024-05-24 04:44:00</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama8b</td>\n","      <td>base</td>\n","      <td>0.27</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_grid.csv</td>\n","      <td>2024-05-24 04:49:55</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama8b</td>\n","      <td>grid</td>\n","      <td>0.17</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_csv.csv</td>\n","      <td>2024-05-24 04:54:05</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama8b</td>\n","      <td>csv</td>\n","      <td>0.21</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_coord.csv</td>\n","      <td>2024-05-24 05:01:28</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama8b</td>\n","      <td>coord</td>\n","      <td>0.3</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_colbycol.csv</td>\n","      <td>2024-05-24 05:05:37</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>Llama8b</td>\n","      <td>colbycol</td>\n","      <td>0.18</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_base.csv</td>\n","      <td>2024-05-24 06:59:32</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT4</td>\n","      <td>base</td>\n","      <td>0.98</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_grid.csv</td>\n","      <td>2024-05-24 07:33:59</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT4</td>\n","      <td>grid</td>\n","      <td>0.95</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_csv.csv</td>\n","      <td>2024-05-24 07:55:00</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT4</td>\n","      <td>csv</td>\n","      <td>0.9</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_coord.csv</td>\n","      <td>2024-05-24 08:31:34</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT4</td>\n","      <td>coord</td>\n","      <td>0.96</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_colbycol.csv</td>\n","      <td>2024-05-25 06:32:09</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT4</td>\n","      <td>colbycol</td>\n","      <td>0.9</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_base.csv</td>\n","      <td>2024-05-25 06:34:18</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT3</td>\n","      <td>base</td>\n","      <td>0.11</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_grid.csv</td>\n","      <td>2024-05-25 06:38:27</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT3</td>\n","      <td>grid</td>\n","      <td>0.1</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_csv.csv</td>\n","      <td>2024-05-25 06:40:40</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT3</td>\n","      <td>csv</td>\n","      <td>0.13</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_coord.csv</td>\n","      <td>2024-05-25 06:44:12</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT3</td>\n","      <td>coord</td>\n","      <td>0.21</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>/content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_colbycol.csv</td>\n","      <td>2024-05-25 06:47:20</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>GPT3</td>\n","      <td>colbycol</td>\n","      <td>0.14</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-22ed2973-7ae9-40cc-924d-df816f6d69b4')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-22ed2973-7ae9-40cc-924d-df816f6d69b4 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-22ed2973-7ae9-40cc-924d-df816f6d69b4');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-ba326660-914f-42f3-9d9c-33df73244409\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-ba326660-914f-42f3-9d9c-33df73244409')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-ba326660-914f-42f3-9d9c-33df73244409 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"text/plain":["                                                                                                                                file  \\\n","0    /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_internalrep-baseline.csv   \n","1   /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_internalrep-baseline.csv   \n","2                   /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_base.csv   \n","3                   /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_grid.csv   \n","4                    /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_csv.csv   \n","5                  /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_coord.csv   \n","6               /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama70b_colbycol.csv   \n","7                    /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_base.csv   \n","8                    /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_grid.csv   \n","9                     /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_csv.csv   \n","10                  /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_coord.csv   \n","11               /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_Llama8b_colbycol.csv   \n","12                      /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_base.csv   \n","13                      /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_grid.csv   \n","14                       /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_csv.csv   \n","15                     /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_coord.csv   \n","16                  /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT4_colbycol.csv   \n","17                      /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_base.csv   \n","18                      /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_grid.csv   \n","19                       /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_csv.csv   \n","20                     /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_coord.csv   \n","21                  /content/drive/MyDrive/CS 159/internal_representations/type-square_size-3_steps-4_seed-3_n-100_GPT3_colbycol.csv   \n","\n","            created_at task_type task_size task_steps num_questions     model  \\\n","0  2024-05-24 01:03:38    square         3          4           100   Llama8b   \n","1  2024-05-24 01:11:04    square         3          4           100  Llama70b   \n","2  2024-05-24 04:14:28    square         3          4           100  Llama70b   \n","3  2024-05-24 04:21:47    square         3          4           100  Llama70b   \n","4  2024-05-24 04:26:28    square         3          4           100  Llama70b   \n","5  2024-05-24 04:35:40    square         3          4           100  Llama70b   \n","6  2024-05-24 04:40:38    square         3          4           100  Llama70b   \n","7  2024-05-24 04:44:00    square         3          4           100   Llama8b   \n","8  2024-05-24 04:49:55    square         3          4           100   Llama8b   \n","9  2024-05-24 04:54:05    square         3          4           100   Llama8b   \n","10 2024-05-24 05:01:28    square         3          4           100   Llama8b   \n","11 2024-05-24 05:05:37    square         3          4           100   Llama8b   \n","12 2024-05-24 06:59:32    square         3          4           100      GPT4   \n","13 2024-05-24 07:33:59    square         3          4           100      GPT4   \n","14 2024-05-24 07:55:00    square         3          4           100      GPT4   \n","15 2024-05-24 08:31:34    square         3          4           100      GPT4   \n","16 2024-05-25 06:32:09    square         3          4           100      GPT4   \n","17 2024-05-25 06:34:18    square         3          4           100      GPT3   \n","18 2024-05-25 06:38:27    square         3          4           100      GPT3   \n","19 2024-05-25 06:40:40    square         3          4           100      GPT3   \n","20 2024-05-25 06:44:12    square         3          4           100      GPT3   \n","21 2024-05-25 06:47:20    square         3          4           100      GPT3   \n","\n","             prompt_type accuracy  \n","0   internalrep-baseline     0.19  \n","1   internalrep-baseline     0.63  \n","2                   base      0.8  \n","3                   grid     0.53  \n","4                    csv     0.56  \n","5                  coord     0.58  \n","6               colbycol     0.49  \n","7                   base     0.27  \n","8                   grid     0.17  \n","9                    csv     0.21  \n","10                 coord      0.3  \n","11              colbycol     0.18  \n","12                  base     0.98  \n","13                  grid     0.95  \n","14                   csv      0.9  \n","15                 coord     0.96  \n","16              colbycol      0.9  \n","17                  base     0.11  \n","18                  grid      0.1  \n","19                   csv     0.13  \n","20                 coord     0.21  \n","21              colbycol     0.14  "]},"execution_count":7,"metadata":{},"output_type":"execute_result"}],"source":["# internal representations\n","internal_rep_files = glob(data_path + \"internal_representations/*.csv\")\n","internal_rep_df = pd.DataFrame(\n","    columns=[\n","        \"file\", \"created_at\", \"task_type\", \"task_size\", \"task_steps\",\n","        \"num_questions\", \"model\", \"prompt_type\", \"accuracy\"\n","    ]\n",")\n","\n","for file in internal_rep_files:\n","  created_at = os.path.getmtime(file) * (10**9)\n","  task_type = re.search(\"(?<=type-)(.*)(?=_size)\", file).group(0)\n","  task_size = re.search(\"(?<=_size-)(.*)(?=_steps)\", file).group(0)\n","  task_steps = re.search(\"(?<=steps-)(.*)(?=_seed)\", file).group(0)\n","  splits = file.split(\"_\")\n","  model_name = splits[-2]\n","  prompt_type = splits[-1].replace(\".csv\", \"\")\n","  this_df = pd.read_csv(file)\n","  num_questions = this_df.shape[0]\n","  accuracy = this_df[\"Is Correct\"].astype(int).mean()\n","  internal_rep_df.loc[len(internal_rep_df), :] = [\n","      file, created_at, task_type, task_size, task_steps, num_questions,\n","      model_name, prompt_type, accuracy\n","  ]\n","internal_rep_df[\"created_at\"] = pd.to_datetime(internal_rep_df[\"created_at\"])\n","\n","internal_rep_df"]},{"cell_type":"code","execution_count":9,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":293},"executionInfo":{"elapsed":94696,"status":"ok","timestamp":1717490922534,"user":{"displayName":"Yulun Wu","userId":"15728323746379503625"},"user_tz":420},"id":"W9646sS0KJRR","outputId":"ea173c6f-6ac1-40ab-c780-b8b26a44ca3c"},"outputs":[{"data":{"application/vnd.google.colaboratory.intrinsic+json":{"summary":"{\n  \"name\": \"other_results_df\",\n  \"rows\": 285,\n  \"fields\": [\n    {\n      \"column\": \"file\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 285,\n        \"samples\": [\n          \"/content/drive/MyDrive/CS 159/AnthropicClient/type-tree_size-9_steps-4_seed-9_n-100_evaluation_no_code_baseline.csv\",\n          \"/content/drive/MyDrive/CS 159/Llama8b/type-square_size-3_steps-4_seed-3_n-100_evaluation_no_code_zero_shot_vot.csv\",\n          \"/content/drive/MyDrive/CS 159/Llama70b/type-ring_size-9_steps-4_seed-9_n-100_evaluation_no_code_baseline.csv\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"filetype\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 15,\n        \"samples\": [\n          \"type-hexagon_size-2_seed-2_n-10000_label-imagenetsimple_nsampled-100_nstep-8\",\n          \"type-ring_size-12_seed-12_n-10000_label-imagenetsimple_nsampled-100_nstep-8\",\n          \"type-ring_size-12_steps-8_seed-12_n-100\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task_type\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"ring\",\n          \"square\",\n          \"triangle\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task_size\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"9\",\n          \"2\",\n          \"12\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"task_steps\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"4\",\n          \"8\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"num_questions\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 100,\n        \"max\": 100,\n        \"num_unique_values\": 1,\n        \"samples\": [\n          100\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"created_at\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": \"2024-05-04 06:25:12\",\n        \"max\": \"2024-05-26 10:16:17\",\n        \"num_unique_values\": 284,\n        \"samples\": [\n          \"2024-05-23 16:23:07\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"model\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 6,\n        \"samples\": [\n          \"Claude\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"use_code\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          true\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"prompt_type\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 13,\n        \"samples\": [\n          \"colbycol\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.0,\n        \"max\": 0.98,\n        \"num_unique_values\": 84,\n        \"samples\": [\n          0.48\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}","type":"dataframe","variable_name":"other_results_df"},"text/html":["\n","  <div id=\"df-5063143c-6d53-4f28-b956-771d543db2b5\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>file</th>\n","      <th>filetype</th>\n","      <th>task_type</th>\n","      <th>task_size</th>\n","      <th>task_steps</th>\n","      <th>num_questions</th>\n","      <th>created_at</th>\n","      <th>model</th>\n","      <th>use_code</th>\n","      <th>prompt_type</th>\n","      <th>accuracy</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>/content/drive/MyDrive/CS 159/AnthropicClient/type-ring_size-12_steps-8_seed-12_n-100_evaluation_no_code_baseline.csv</td>\n","      <td>type-ring_size-12_steps-8_seed-12_n-100</td>\n","      <td>ring</td>\n","      <td>12</td>\n","      <td>8</td>\n","      <td>100</td>\n","      <td>2024-05-23 16:01:30</td>\n","      <td>Claude</td>\n","      <td>False</td>\n","      <td>baseline</td>\n","      <td>0.08</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>/content/drive/MyDrive/CS 159/AnthropicClient/type-ring_size-9_steps-4_seed-9_n-100_evaluation_no_code_baseline.csv</td>\n","      <td>type-ring_size-9_steps-4_seed-9_n-100</td>\n","      <td>ring</td>\n","      <td>9</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>2024-05-23 16:04:21</td>\n","      <td>Claude</td>\n","      <td>False</td>\n","      <td>baseline</td>\n","      <td>0.13</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>/content/drive/MyDrive/CS 159/AnthropicClient/type-square_size-3_steps-4_seed-3_n-100_evaluation_no_code_baseline.csv</td>\n","      <td>type-square_size-3_steps-4_seed-3_n-100</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>4</td>\n","      <td>100</td>\n","      <td>2024-05-23 16:07:05</td>\n","      <td>Claude</td>\n","      <td>False</td>\n","      <td>baseline</td>\n","      <td>0.5</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>/content/drive/MyDrive/CS 159/AnthropicClient/type-square_size-3_steps-8_seed-3_n-100_evaluation_no_code_baseline.csv</td>\n","      <td>type-square_size-3_steps-8_seed-3_n-100</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>8</td>\n","      <td>100</td>\n","      <td>2024-05-23 16:09:51</td>\n","      <td>Claude</td>\n","      <td>False</td>\n","      <td>baseline</td>\n","      <td>0.31</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>/content/drive/MyDrive/CS 159/AnthropicClient/type-square_size-3_steps-8_seed-3_n-100_special_order-random_order_evaluation_no_code_baseline.csv</td>\n","      <td>type-square_size-3_steps-8_seed-3_n-100_special_order-random_order</td>\n","      <td>square</td>\n","      <td>3</td>\n","      <td>8</td>\n","      <td>100</td>\n","      <td>2024-05-23 16:12:55</td>\n","      <td>Claude</td>\n","      <td>False</td>\n","      <td>baseline</td>\n","      <td>0.09</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5063143c-6d53-4f28-b956-771d543db2b5')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-5063143c-6d53-4f28-b956-771d543db2b5 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-5063143c-6d53-4f28-b956-771d543db2b5');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-a4374943-03a1-4e24-bab4-66394057cbf8\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-a4374943-03a1-4e24-bab4-66394057cbf8')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-a4374943-03a1-4e24-bab4-66394057cbf8 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"],"text/plain":["                                                                                                                                               file  \\\n","0                             /content/drive/MyDrive/CS 159/AnthropicClient/type-ring_size-12_steps-8_seed-12_n-100_evaluation_no_code_baseline.csv   \n","1                               /content/drive/MyDrive/CS 159/AnthropicClient/type-ring_size-9_steps-4_seed-9_n-100_evaluation_no_code_baseline.csv   \n","2                             /content/drive/MyDrive/CS 159/AnthropicClient/type-square_size-3_steps-4_seed-3_n-100_evaluation_no_code_baseline.csv   \n","3                             /content/drive/MyDrive/CS 159/AnthropicClient/type-square_size-3_steps-8_seed-3_n-100_evaluation_no_code_baseline.csv   \n","4  /content/drive/MyDrive/CS 159/AnthropicClient/type-square_size-3_steps-8_seed-3_n-100_special_order-random_order_evaluation_no_code_baseline.csv   \n","\n","                                                             filetype  \\\n","0                             type-ring_size-12_steps-8_seed-12_n-100   \n","1                               type-ring_size-9_steps-4_seed-9_n-100   \n","2                             type-square_size-3_steps-4_seed-3_n-100   \n","3                             type-square_size-3_steps-8_seed-3_n-100   \n","4  type-square_size-3_steps-8_seed-3_n-100_special_order-random_order   \n","\n","  task_type task_size task_steps num_questions          created_at   model  \\\n","0      ring        12          8           100 2024-05-23 16:01:30  Claude   \n","1      ring         9          4           100 2024-05-23 16:04:21  Claude   \n","2    square         3          4           100 2024-05-23 16:07:05  Claude   \n","3    square         3          8           100 2024-05-23 16:09:51  Claude   \n","4    square         3          8           100 2024-05-23 16:12:55  Claude   \n","\n","  use_code prompt_type accuracy  \n","0    False    baseline     0.08  \n","1    False    baseline     0.13  \n","2    False    baseline      0.5  \n","3    False    baseline     0.31  \n","4    False    baseline     0.09  "]},"execution_count":9,"metadata":{},"output_type":"execute_result"}],"source":["# other baselines\n","model_mappings = {\n","    \"AnthropicClient\": \"Claude\",\n","    \"GPT3Client\": \"GPT3\",\n","    \"GPT4Client\": \"GPT4\",\n","    \"Llama70b\": \"Llama70b\",\n","    \"Llama8b\": \"Llama8b\",\n","    \"OpenAIClient\": \"`OpenAI`\"\n","}\n","folders = list(model_mappings.keys())\n","other_results_df = pd.DataFrame(\n","    columns=[\n","        \"file\", \"filetype\", \"task_type\", \"task_size\", \"task_steps\",\n","        \"num_questions\", \"created_at\", \"model\", \"use_code\", \"prompt_type\",\n","        \"accuracy\"\n","    ]\n",")\n","\n","for folder in folders:\n","\n","  files = glob(data_path+folder+\"/*.csv\")\n","  for file in files:\n","\n","    # task type stuff\n","    task_type = re.search(\"(?<=type-)(.*)(?=_size)\", file).group(0)\n","    task_size = re.search(\"(?<=_size-)(.*)(?=_seed)\", file).group(0)\n","    task_steps = None\n","    if \"steps\" in task_size:\n","      task_steps = task_size.split(\"-\")[-1]\n","      task_size = task_size.split(\"_\")[0]\n","\n","    # results analysis\n","    this_df = pd.read_csv(file)\n","    num_questions = this_df.shape[0]\n","    accuracy = this_df[\"Is Correct\"].astype(int).mean()\n","\n","    # code, prompting flags\n","    uses_code = \"no_code\" not in file\n","    prompting = re.findall(\"(?<=code_)(.*$)\", file)\n","    filetype = re.findall(\".+?(?=_evaluation)\", file.split(\"/\")[-1])[0]\n","    if len(prompting) != 0:\n","      prompting = prompting[-1].replace(\".csv\", \"\")\n","    else:\n","      prompting = None\n","\n","    # metadata\n","    created_at = os.path.getmtime(file) * (10**9)\n","\n","    # add to master df\n","    other_results_df.loc[len(other_results_df), :] = [\n","        file, filetype, task_type, task_size, task_steps, num_questions,\n","        created_at, folder, uses_code, prompting, accuracy\n","    ]\n","\n","# post-processing\n","other_results_df[\"model\"] = other_results_df[\"model\"].map(model_mappings)\n","other_results_df[\"created_at\"] = pd.to_datetime(other_results_df[\"created_at\"])\n","\n","other_results_df.head(5)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"elapsed":126,"status":"ok","timestamp":1716756792844,"user":{"displayName":"Ethan Wilk","userId":"04266046801556009728"},"user_tz":420},"id":"-q176fyMndP0","outputId":"f666f950-a94f-4293-8928-b6f5b8ca5d5c"},"outputs":[{"data":{"application/vnd.google.colaboratory.intrinsic+json":{"summary":"{\n  \"name\": \"compare_df\",\n  \"rows\": 40,\n  \"fields\": [\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.09500000000000001,\n        \"max\": 0.98,\n        \"num_unique_values\": 38,\n        \"samples\": [\n          0.16,\n          0.3707142857142857,\n          0.09500000000000001\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}","type":"dataframe","variable_name":"compare_df"},"text/html":["\n","  <div id=\"df-0d3a246f-19dd-4b02-bf83-907e950de766\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th></th>\n","      <th>accuracy</th>\n","    </tr>\n","    <tr>\n","      <th>prompt_type</th>\n","      <th>model</th>\n","      <th></th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th rowspan=\"3\" valign=\"top\">base</th>\n","      <th>GPT4</th>\n","      <td>0.98</td>\n","    </tr>\n","    <tr>\n","      <th>Llama70b</th>\n","      <td>0.8</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.27</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"5\" valign=\"top\">baseline</th>\n","      <th>Claude</th>\n","      <td>0.362143</td>\n","    </tr>\n","    <tr>\n","      <th>GPT3</th>\n","      <td>0.095</td>\n","    </tr>\n","    <tr>\n","      <th>GPT4</th>\n","      <td>0.357143</td>\n","    </tr>\n","    <tr>\n","      <th>Llama70b</th>\n","      <td>0.257143</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.146429</td>\n","    </tr>\n","    <tr>\n","      <th>chain_of_thought</th>\n","      <th>`OpenAI`</th>\n","      <td>0.92</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"2\" valign=\"top\">code_use</th>\n","      <th>GPT4</th>\n","      <td>0.87</td>\n","    </tr>\n","    <tr>\n","      <th>Llama70b</th>\n","      <td>0.77</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"2\" valign=\"top\">colbycol</th>\n","      <th>Llama70b</th>\n","      <td>0.49</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.18</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"3\" valign=\"top\">coord</th>\n","      <th>GPT4</th>\n","      <td>0.96</td>\n","    </tr>\n","    <tr>\n","      <th>Llama70b</th>\n","      <td>0.58</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.3</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"3\" valign=\"top\">csv</th>\n","      <th>GPT4</th>\n","      <td>0.9</td>\n","    </tr>\n","    <tr>\n","      <th>Llama70b</th>\n","      <td>0.56</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.21</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"3\" valign=\"top\">grid</th>\n","      <th>GPT4</th>\n","      <td>0.95</td>\n","    </tr>\n","    <tr>\n","      <th>Llama70b</th>\n","      <td>0.53</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.17</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"5\" valign=\"top\">indexing</th>\n","      <th>Claude</th>\n","      <td>0.717</td>\n","    </tr>\n","    <tr>\n","      <th>GPT3</th>\n","      <td>0.106429</td>\n","    </tr>\n","    <tr>\n","      <th>GPT4</th>\n","      <td>0.715</td>\n","    </tr>\n","    <tr>\n","      <th>Llama70b</th>\n","      <td>0.435714</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.137143</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"2\" valign=\"top\">internalrep-baseline</th>\n","      <th>Llama70b</th>\n","      <td>0.63</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.19</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"2\" valign=\"top\">visualize</th>\n","      <th>Llama70b</th>\n","      <td>0.57</td>\n","    </tr>\n","    <tr>\n","      <th>`OpenAI`</th>\n","      <td>0.77</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"4\" valign=\"top\">zero_shot_cot</th>\n","      <th>GPT3</th>\n","      <td>0.096429</td>\n","    </tr>\n","    <tr>\n","      <th>GPT4</th>\n","      <td>0.677143</td>\n","    </tr>\n","    <tr>\n","      <th>Llama70b</th>\n","      <td>0.433571</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.16</td>\n","    </tr>\n","    <tr>\n","      <th rowspan=\"5\" valign=\"top\">zero_shot_vot</th>\n","      <th>Claude</th>\n","      <td>0.434444</td>\n","    </tr>\n","    <tr>\n","      <th>GPT3</th>\n","      <td>0.095</td>\n","    </tr>\n","    <tr>\n","      <th>GPT4</th>\n","      <td>0.623571</td>\n","    </tr>\n","    <tr>\n","      <th>Llama70b</th>\n","      <td>0.370714</td>\n","    </tr>\n","    <tr>\n","      <th>Llama8b</th>\n","      <td>0.132857</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0d3a246f-19dd-4b02-bf83-907e950de766')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-0d3a246f-19dd-4b02-bf83-907e950de766 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-0d3a246f-19dd-4b02-bf83-907e950de766');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-b00b6413-4010-440c-b8ea-4e9ffe4387af\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-b00b6413-4010-440c-b8ea-4e9ffe4387af')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-b00b6413-4010-440c-b8ea-4e9ffe4387af button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","\n","  <div id=\"id_e388e11a-e16d-4570-976a-6701e4f918ec\">\n","    <style>\n","      .colab-df-generate {\n","        background-color: #E8F0FE;\n","        border: none;\n","        border-radius: 50%;\n","        cursor: pointer;\n","        display: none;\n","        fill: #1967D2;\n","        height: 32px;\n","        padding: 0 0 0 0;\n","        width: 32px;\n","      }\n","\n","      .colab-df-generate:hover {\n","        background-color: #E2EBFA;\n","        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","        fill: #174EA6;\n","      }\n","\n","      [theme=dark] .colab-df-generate {\n","        background-color: #3B4455;\n","        fill: #D2E3FC;\n","      }\n","\n","      [theme=dark] .colab-df-generate:hover {\n","        background-color: #434B5C;\n","        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","        fill: #FFFFFF;\n","      }\n","    </style>\n","    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('compare_df')\"\n","            title=\"Generate code using this dataframe.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n","  </svg>\n","    </button>\n","    <script>\n","      (() => {\n","      const buttonEl =\n","        document.querySelector('#id_e388e11a-e16d-4570-976a-6701e4f918ec button.colab-df-generate');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      buttonEl.onclick = () => {\n","        google.colab.notebook.generateWithVariable('compare_df');\n","      }\n","      })();\n","    </script>\n","  </div>\n","\n","    </div>\n","  </div>\n"],"text/plain":["                               accuracy\n","prompt_type          model             \n","base                 GPT4          0.98\n","                     Llama70b       0.8\n","                     Llama8b       0.27\n","baseline             Claude    0.362143\n","                     GPT3         0.095\n","                     GPT4      0.357143\n","                     Llama70b  0.257143\n","                     Llama8b   0.146429\n","chain_of_thought     `OpenAI`      0.92\n","code_use             GPT4          0.87\n","                     Llama70b      0.77\n","colbycol             Llama70b      0.49\n","                     Llama8b       0.18\n","coord                GPT4          0.96\n","                     Llama70b      0.58\n","                     Llama8b        0.3\n","csv                  GPT4           0.9\n","                     Llama70b      0.56\n","                     Llama8b       0.21\n","grid                 GPT4          0.95\n","                     Llama70b      0.53\n","                     Llama8b       0.17\n","indexing             Claude       0.717\n","                     GPT3      0.106429\n","                     GPT4         0.715\n","                     Llama70b  0.435714\n","                     Llama8b   0.137143\n","internalrep-baseline Llama70b      0.63\n","                     Llama8b       0.19\n","visualize            Llama70b      0.57\n","                     `OpenAI`      0.77\n","zero_shot_cot        GPT3      0.096429\n","                     GPT4      0.677143\n","                     Llama70b  0.433571\n","                     Llama8b       0.16\n","zero_shot_vot        Claude    0.434444\n","                     GPT3         0.095\n","                     GPT4      0.623571\n","                     Llama70b  0.370714\n","                     Llama8b   0.132857"]},"execution_count":39,"metadata":{},"output_type":"execute_result"}],"source":["# comparing results\n","compare_df = pd.DataFrame(\n","    other_results_df\n","    .groupby([\"prompt_type\", \"model\"])\n","    [\"accuracy\"]\n","    .mean()\n",")\n","compare_df"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":488},"executionInfo":{"elapsed":129,"status":"ok","timestamp":1716756854927,"user":{"displayName":"Ethan Wilk","userId":"04266046801556009728"},"user_tz":420},"id":"-T6RRh3Kpjjn","outputId":"e86bd3a9-0ad5-4640-cfc1-c8c0a4175c98"},"outputs":[{"data":{"application/vnd.google.colaboratory.intrinsic+json":{"summary":"{\n  \"name\": \"compare_df\",\n  \"rows\": 13,\n  \"fields\": [\n    {\n      \"column\": \"prompt_type\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 13,\n        \"samples\": [\n          \"zero_shot_cot\",\n          \"internalrep-baseline\",\n          \"base\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"accuracy\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.24357142857142855,\n        \"max\": 0.92,\n        \"num_unique_values\": 13,\n        \"samples\": [\n          0.3417857142857143,\n          0.41000000000000003,\n          0.6833333333333332\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}","type":"dataframe"},"text/html":["\n","  <div id=\"df-0dce331b-9ff3-46ee-a3b8-8dd739f72650\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>accuracy</th>\n","    </tr>\n","    <tr>\n","      <th>prompt_type</th>\n","      <th></th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>base</th>\n","      <td>0.683333</td>\n","    </tr>\n","    <tr>\n","      <th>baseline</th>\n","      <td>0.243571</td>\n","    </tr>\n","    <tr>\n","      <th>chain_of_thought</th>\n","      <td>0.92</td>\n","    </tr>\n","    <tr>\n","      <th>code_use</th>\n","      <td>0.82</td>\n","    </tr>\n","    <tr>\n","      <th>colbycol</th>\n","      <td>0.335</td>\n","    </tr>\n","    <tr>\n","      <th>coord</th>\n","      <td>0.613333</td>\n","    </tr>\n","    <tr>\n","      <th>csv</th>\n","      <td>0.556667</td>\n","    </tr>\n","    <tr>\n","      <th>grid</th>\n","      <td>0.55</td>\n","    </tr>\n","    <tr>\n","      <th>indexing</th>\n","      <td>0.422257</td>\n","    </tr>\n","    <tr>\n","      <th>internalrep-baseline</th>\n","      <td>0.41</td>\n","    </tr>\n","    <tr>\n","      <th>visualize</th>\n","      <td>0.67</td>\n","    </tr>\n","    <tr>\n","      <th>zero_shot_cot</th>\n","      <td>0.341786</td>\n","    </tr>\n","    <tr>\n","      <th>zero_shot_vot</th>\n","      <td>0.331317</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-0dce331b-9ff3-46ee-a3b8-8dd739f72650')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-0dce331b-9ff3-46ee-a3b8-8dd739f72650 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-0dce331b-9ff3-46ee-a3b8-8dd739f72650');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-2c02be17-58e2-4532-b58d-825cc3613ebd\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-2c02be17-58e2-4532-b58d-825cc3613ebd')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-2c02be17-58e2-4532-b58d-825cc3613ebd button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","\n","    </div>\n","  </div>\n"],"text/plain":["                      accuracy\n","prompt_type                   \n","base                  0.683333\n","baseline              0.243571\n","chain_of_thought          0.92\n","code_use                  0.82\n","colbycol                 0.335\n","coord                 0.613333\n","csv                   0.556667\n","grid                      0.55\n","indexing              0.422257\n","internalrep-baseline      0.41\n","visualize                 0.67\n","zero_shot_cot         0.341786\n","zero_shot_vot         0.331317"]},"execution_count":41,"metadata":{},"output_type":"execute_result"}],"source":["compare_df.groupby([\"prompt_type\"]).mean()"]}],"metadata":{"colab":{"provenance":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}