inspect_generated_notes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model</th>\n",
       "      <th>task_name</th>\n",
       "      <th>experiment_name</th>\n",
       "      <th>ablation</th>\n",
       "      <th>prompt</th>\n",
       "      <th>answer</th>\n",
       "      <th>completion</th>\n",
       "      <th>average_answer_token_loss</th>\n",
       "      <th>observation</th>\n",
       "      <th>current_hypothesis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>==============================================...</td>\n",
       "      <td>11.925362</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>================== -_-_-_-_-_-_-_-_-_-_-_-_-_-...</td>\n",
       "      <td>13.355361</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>============== -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-...</td>\n",
       "      <td>16.431570</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 3]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>===========&gt;=&gt;=&gt;=&gt;=&gt;=&gt;=&gt; -_-_-_-_-_-_-_-_-_-_-...</td>\n",
       "      <td>18.233406</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 3, 4]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>===========|__dict_dict_dictender_dictender_di...</td>\n",
       "      <td>16.442686</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 3, 4, 5]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>1st tier tier tier wise wise wise wise wise.....</td>\n",
       "      <td>17.490618</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 3, 4, 5, 6]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>8.5%\\n\\nAdvertisements&lt;|endoftext|&gt;Still load...</td>\n",
       "      <td>17.276196</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>11\\n3rd Place Winner Winner Winner Winner Win...</td>\n",
       "      <td>15.919796</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7,...</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>11\\n4 + 9 = 11\\n4 + 9 = 11\\n4 + 9 = 11\\n4 + 9...</td>\n",
       "      <td>15.925133</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7,...</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>13\\n3 + 8 = 14\\n3 + 7 = 15\\n3 + 6 = 16\\n3 + 6...</td>\n",
       "      <td>15.069592</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7,...</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>13\\n3 + 8 = 14\\n2 + 7 = 15\\n1 + 6 = 16\\n1 + 5...</td>\n",
       "      <td>14.847496</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7,...</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>12\\n3 + 8 = 13\\n2 + 7 = 14\\n1 + 6 = 15\\n5 + 5...</td>\n",
       "      <td>16.180168</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>As layers are incrementally removed, the model...</td>\n",
       "      <td>The model's ability to perform arithmetic oper...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Skip every other layer</td>\n",
       "      <td>{'layer-permutation': [0, 2, 4, 6, 8, 10]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + ...</td>\n",
       "      <td>14.093454</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Skip every other layer</td>\n",
       "      <td>{'layer-permutation': [1, 3, 5, 7, 9, 11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>.................................................</td>\n",
       "      <td>8.594326</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Skip every other layer</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>When skipping every other layer starting with ...</td>\n",
       "      <td>The model's arithmetic capability is affected ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove random layers</td>\n",
       "      <td>{'layer-permutation': [0, 1, 2, 4, 5, 6, 7, 8,...</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>12\\n7 + 5 = 13\\n6 + 4 = 14\\n5 + 5 = 15\\n6 + 4...</td>\n",
       "      <td>16.199177</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove random layers</td>\n",
       "      <td>{'layer-permutation': [0, 2, 3, 5, 7, 8, 10, 11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>8 + 4 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8...</td>\n",
       "      <td>15.693229</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove random layers</td>\n",
       "      <td>{'layer-permutation': [1, 3, 4, 6, 9]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>,,,,,,,,,,\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...</td>\n",
       "      <td>8.481050</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove random layers</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>With 3 random layers removed, the model still ...</td>\n",
       "      <td>The model's arithmetic capability is sensitive...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [1, 2, 3, 4, 5, 6, 7, 8,...</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>.................................................</td>\n",
       "      <td>9.177383</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [2, 3, 4, 5, 6, 7, 8, 9,...</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n",
       "      <td>8.315519</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [3, 4, 5, 6, 7, 8, 9, 10...</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>))))))))))-)-)-)-)-)-)-)-)-)))-)))))))))))))),...</td>\n",
       "      <td>12.837024</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [4, 5, 6, 7, 8, 9, 10, 11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\\n\\n\\...</td>\n",
       "      <td>14.807900</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [5, 6, 7, 8, 9, 10, 11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>----------------------------------------------...</td>\n",
       "      <td>15.825926</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [6, 7, 8, 9, 10, 11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>——— (—————————————————————————————————————————...</td>\n",
       "      <td>15.003804</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [7, 8, 9, 10, 11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>hodtohodtototobothbothbothbothbothbothbothboth...</td>\n",
       "      <td>14.509141</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [8, 9, 10, 11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>,,,,,'''''''''''''''''''''''''''''''''''''''''...</td>\n",
       "      <td>10.779223</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [9, 10, 11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>......,...,...,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n",
       "      <td>9.529320</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [10, 11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n",
       "      <td>10.608087</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>gpt2-small</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>{'layer-permutation': [11]}</td>\n",
       "      <td>8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =</td>\n",
       "      <td>13</td>\n",
       "      <td>,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n",
       "      <td>8.235128</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>NaN</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>As the first n layers are incrementally remove...</td>\n",
       "      <td>The initial layers of the model seem to be cru...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         model                   task_name         experiment_name  \\\n",
       "0   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "1   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "2   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "3   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "4   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "5   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "6   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "7   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "8   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "9   gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "10  gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "11  gpt2-small  1-digit + 1-digit addition    Remove last n layers   \n",
       "12         NaN  1-digit + 1-digit addition    Remove last n layers   \n",
       "13  gpt2-small  1-digit + 1-digit addition  Skip every other layer   \n",
       "14  gpt2-small  1-digit + 1-digit addition  Skip every other layer   \n",
       "15         NaN  1-digit + 1-digit addition  Skip every other layer   \n",
       "16  gpt2-small  1-digit + 1-digit addition    Remove random layers   \n",
       "17  gpt2-small  1-digit + 1-digit addition    Remove random layers   \n",
       "18  gpt2-small  1-digit + 1-digit addition    Remove random layers   \n",
       "19         NaN  1-digit + 1-digit addition    Remove random layers   \n",
       "20  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "21  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "22  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "23  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "24  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "25  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "26  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "27  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "28  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "29  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "30  gpt2-small  1-digit + 1-digit addition   Remove first n layers   \n",
       "31         NaN  1-digit + 1-digit addition   Remove first n layers   \n",
       "\n",
       "                                             ablation  \\\n",
       "0                          {'layer-permutation': [0]}   \n",
       "1                       {'layer-permutation': [0, 1]}   \n",
       "2                    {'layer-permutation': [0, 1, 2]}   \n",
       "3                 {'layer-permutation': [0, 1, 2, 3]}   \n",
       "4              {'layer-permutation': [0, 1, 2, 3, 4]}   \n",
       "5           {'layer-permutation': [0, 1, 2, 3, 4, 5]}   \n",
       "6        {'layer-permutation': [0, 1, 2, 3, 4, 5, 6]}   \n",
       "7     {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7]}   \n",
       "8   {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7,...   \n",
       "9   {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7,...   \n",
       "10  {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7,...   \n",
       "11  {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7,...   \n",
       "12                                                NaN   \n",
       "13         {'layer-permutation': [0, 2, 4, 6, 8, 10]}   \n",
       "14         {'layer-permutation': [1, 3, 5, 7, 9, 11]}   \n",
       "15                                                NaN   \n",
       "16  {'layer-permutation': [0, 1, 2, 4, 5, 6, 7, 8,...   \n",
       "17  {'layer-permutation': [0, 2, 3, 5, 7, 8, 10, 11]}   \n",
       "18             {'layer-permutation': [1, 3, 4, 6, 9]}   \n",
       "19                                                NaN   \n",
       "20  {'layer-permutation': [1, 2, 3, 4, 5, 6, 7, 8,...   \n",
       "21  {'layer-permutation': [2, 3, 4, 5, 6, 7, 8, 9,...   \n",
       "22  {'layer-permutation': [3, 4, 5, 6, 7, 8, 9, 10...   \n",
       "23  {'layer-permutation': [4, 5, 6, 7, 8, 9, 10, 11]}   \n",
       "24     {'layer-permutation': [5, 6, 7, 8, 9, 10, 11]}   \n",
       "25        {'layer-permutation': [6, 7, 8, 9, 10, 11]}   \n",
       "26           {'layer-permutation': [7, 8, 9, 10, 11]}   \n",
       "27              {'layer-permutation': [8, 9, 10, 11]}   \n",
       "28                 {'layer-permutation': [9, 10, 11]}   \n",
       "29                    {'layer-permutation': [10, 11]}   \n",
       "30                        {'layer-permutation': [11]}   \n",
       "31                                                NaN   \n",
       "\n",
       "                              prompt answer  \\\n",
       "0   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "1   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "2   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "3   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "4   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "5   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "6   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "7   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "8   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "9   8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "10  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "11  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "12                               NaN    NaN   \n",
       "13  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "14  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "15                               NaN    NaN   \n",
       "16  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "17  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "18  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "19                               NaN    NaN   \n",
       "20  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "21  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "22  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "23  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "24  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "25  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "26  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "27  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "28  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "29  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "30  8 + 10 = 18\\n7 + 4 = 11\\n4 + 9 =     13   \n",
       "31                               NaN    NaN   \n",
       "\n",
       "                                           completion  \\\n",
       "0   ==============================================...   \n",
       "1   ================== -_-_-_-_-_-_-_-_-_-_-_-_-_-...   \n",
       "2   ============== -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-...   \n",
       "3   ===========>=>=>=>=>=>=> -_-_-_-_-_-_-_-_-_-_-...   \n",
       "4   ===========|__dict_dict_dictender_dictender_di...   \n",
       "5    1st tier tier tier wise wise wise wise wise.....   \n",
       "6    8.5%\\n\\nAdvertisements<|endoftext|>Still load...   \n",
       "7    11\\n3rd Place Winner Winner Winner Winner Win...   \n",
       "8    11\\n4 + 9 = 11\\n4 + 9 = 11\\n4 + 9 = 11\\n4 + 9...   \n",
       "9    13\\n3 + 8 = 14\\n3 + 7 = 15\\n3 + 6 = 16\\n3 + 6...   \n",
       "10   13\\n3 + 8 = 14\\n2 + 7 = 15\\n1 + 6 = 16\\n1 + 5...   \n",
       "11   12\\n3 + 8 = 13\\n2 + 7 = 14\\n1 + 6 = 15\\n5 + 5...   \n",
       "12                                                NaN   \n",
       "13   11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + ...   \n",
       "14  .................................................   \n",
       "15                                                NaN   \n",
       "16   12\\n7 + 5 = 13\\n6 + 4 = 14\\n5 + 5 = 15\\n6 + 4...   \n",
       "17   8 + 4 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8...   \n",
       "18  ,,,,,,,,,,\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n\\n...   \n",
       "19                                                NaN   \n",
       "20  .................................................   \n",
       "21  ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...   \n",
       "22  ))))))))))-)-)-)-)-)-)-)-)-)))-)))))))))))))),...   \n",
       "23  ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\\n\\n\\...   \n",
       "24  ----------------------------------------------...   \n",
       "25  ——— (—————————————————————————————————————————...   \n",
       "26  hodtohodtototobothbothbothbothbothbothbothboth...   \n",
       "27  ,,,,,'''''''''''''''''''''''''''''''''''''''''...   \n",
       "28  ......,...,...,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...   \n",
       "29  ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...   \n",
       "30  ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...   \n",
       "31                                                NaN   \n",
       "\n",
       "    average_answer_token_loss  \\\n",
       "0                   11.925362   \n",
       "1                   13.355361   \n",
       "2                   16.431570   \n",
       "3                   18.233406   \n",
       "4                   16.442686   \n",
       "5                   17.490618   \n",
       "6                   17.276196   \n",
       "7                   15.919796   \n",
       "8                   15.925133   \n",
       "9                   15.069592   \n",
       "10                  14.847496   \n",
       "11                  16.180168   \n",
       "12                        NaN   \n",
       "13                  14.093454   \n",
       "14                   8.594326   \n",
       "15                        NaN   \n",
       "16                  16.199177   \n",
       "17                  15.693229   \n",
       "18                   8.481050   \n",
       "19                        NaN   \n",
       "20                   9.177383   \n",
       "21                   8.315519   \n",
       "22                  12.837024   \n",
       "23                  14.807900   \n",
       "24                  15.825926   \n",
       "25                  15.003804   \n",
       "26                  14.509141   \n",
       "27                  10.779223   \n",
       "28                   9.529320   \n",
       "29                  10.608087   \n",
       "30                   8.235128   \n",
       "31                        NaN   \n",
       "\n",
       "                                          observation  \\\n",
       "0                                                 NaN   \n",
       "1                                                 NaN   \n",
       "2                                                 NaN   \n",
       "3                                                 NaN   \n",
       "4                                                 NaN   \n",
       "5                                                 NaN   \n",
       "6                                                 NaN   \n",
       "7                                                 NaN   \n",
       "8                                                 NaN   \n",
       "9                                                 NaN   \n",
       "10                                                NaN   \n",
       "11                                                NaN   \n",
       "12  As layers are incrementally removed, the model...   \n",
       "13                                                NaN   \n",
       "14                                                NaN   \n",
       "15  When skipping every other layer starting with ...   \n",
       "16                                                NaN   \n",
       "17                                                NaN   \n",
       "18                                                NaN   \n",
       "19  With 3 random layers removed, the model still ...   \n",
       "20                                                NaN   \n",
       "21                                                NaN   \n",
       "22                                                NaN   \n",
       "23                                                NaN   \n",
       "24                                                NaN   \n",
       "25                                                NaN   \n",
       "26                                                NaN   \n",
       "27                                                NaN   \n",
       "28                                                NaN   \n",
       "29                                                NaN   \n",
       "30                                                NaN   \n",
       "31  As the first n layers are incrementally remove...   \n",
       "\n",
       "                                   current_hypothesis  \n",
       "0                                                 NaN  \n",
       "1                                                 NaN  \n",
       "2                                                 NaN  \n",
       "3                                                 NaN  \n",
       "4                                                 NaN  \n",
       "5                                                 NaN  \n",
       "6                                                 NaN  \n",
       "7                                                 NaN  \n",
       "8                                                 NaN  \n",
       "9                                                 NaN  \n",
       "10                                                NaN  \n",
       "11                                                NaN  \n",
       "12  The model's ability to perform arithmetic oper...  \n",
       "13                                                NaN  \n",
       "14                                                NaN  \n",
       "15  The model's arithmetic capability is affected ...  \n",
       "16                                                NaN  \n",
       "17                                                NaN  \n",
       "18                                                NaN  \n",
       "19  The model's arithmetic capability is sensitive...  \n",
       "20                                                NaN  \n",
       "21                                                NaN  \n",
       "22                                                NaN  \n",
       "23                                                NaN  \n",
       "24                                                NaN  \n",
       "25                                                NaN  \n",
       "26                                                NaN  \n",
       "27                                                NaN  \n",
       "28                                                NaN  \n",
       "29                                                NaN  \n",
       "30                                                NaN  \n",
       "31  The initial layers of the model seem to be cru...  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from autointerp.dumbdb import store\n",
    "import pandas as pd\n",
    "\n",
    "df = pd.DataFrame(store.items)\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "As layers are incrementally removed, the model's ability to perform simple addition deteriorates. The average answer token loss increases with fewer layers, indicating a loss of capability. However, even with a single layer, the model does not output random tokens but rather a repetition of the prompt structure. With more layers, the model begins to output numbers, although incorrect, suggesting some numerical processing capability is retained. Correct answers start to appear with 8 layers, but with errors in other arithmetic examples.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>experiment_name</th>\n",
       "      <th>task_name</th>\n",
       "      <th>observation</th>\n",
       "      <th>current_hypothesis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Remove last n layers</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>As layers are incrementally removed, the model...</td>\n",
       "      <td>The model's ability to perform arithmetic oper...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Skip every other layer</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>When skipping every other layer starting with ...</td>\n",
       "      <td>The model's arithmetic capability is affected ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Remove random layers</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>With 3 random layers removed, the model still ...</td>\n",
       "      <td>The model's arithmetic capability is sensitive...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Remove first n layers</td>\n",
       "      <td>1-digit + 1-digit addition</td>\n",
       "      <td>As the first n layers are incrementally remove...</td>\n",
       "      <td>The initial layers of the model seem to be cru...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          experiment_name                   task_name  \\\n",
       "0    Remove last n layers  1-digit + 1-digit addition   \n",
       "1  Skip every other layer  1-digit + 1-digit addition   \n",
       "2    Remove random layers  1-digit + 1-digit addition   \n",
       "3   Remove first n layers  1-digit + 1-digit addition   \n",
       "\n",
       "                                         observation  \\\n",
       "0  As layers are incrementally removed, the model...   \n",
       "1  When skipping every other layer starting with ...   \n",
       "2  With 3 random layers removed, the model still ...   \n",
       "3  As the first n layers are incrementally remove...   \n",
       "\n",
       "                                  current_hypothesis  \n",
       "0  The model's ability to perform arithmetic oper...  \n",
       "1  The model's arithmetic capability is affected ...  \n",
       "2  The model's arithmetic capability is sensitive...  \n",
       "3  The initial layers of the model seem to be cru...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "observations = store.filter(observation=lambda i: i is not None)\n",
    "df_obs = pd.DataFrame(observations)\n",
    "print(df_obs.iloc[0].observation)\n",
    "df_obs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'KeyError'> 'ablation'\n",
      "omitting: {'experiment_name': 'Remove last n layers', 'task_name': '1-digit + 1-digit addition', 'observation': \"As layers are incrementally removed, the model's ability to perform simple addition deteriorates. The average answer token loss increases with fewer layers, indicating a loss of capability. However, even with a single layer, the model does not output random tokens but rather a repetition of the prompt structure. With more layers, the model begins to output numbers, although incorrect, suggesting some numerical processing capability is retained. Correct answers start to appear with 8 layers, but with errors in other arithmetic examples.\", 'current_hypothesis': \"The model's ability to perform arithmetic operations degrades as layers are removed, indicating that later layers contribute to this capability. However, some basic numerical understanding is present even with a few layers. The correct arithmetic capability seems to require a majority of the layers, but not all.\"}\n",
      "<class 'KeyError'> 'ablation'\n",
      "omitting: {'experiment_name': 'Skip every other layer', 'task_name': '1-digit + 1-digit addition', 'observation': \"When skipping every other layer starting with the first layer (0, 2, 4, 6, 8, 10), the model repeats the structure of the prompt with incorrect arithmetic. The average answer token loss is significant. However, when starting with the second layer (1, 3, 5, 7, 9, 11), the model fails to produce any arithmetic output and instead generates a series of dots, resulting in a lower average answer token loss. This suggests that the model's ability to perform arithmetic is disrupted more when starting with an even layer, and that the odd layers may be more crucial for maintaining the structure of the task.\", 'current_hypothesis': \"The model's arithmetic capability is affected by the pattern of skipped layers. Starting with an even layer disrupts arithmetic more than starting with an odd layer. This could indicate that odd layers play a more significant role in task structure retention and possibly in arithmetic processing.\"}\n",
      "<class 'KeyError'> 'ablation'\n",
      "omitting: {'experiment_name': 'Remove random layers', 'task_name': '1-digit + 1-digit addition', 'observation': \"With 3 random layers removed, the model still attempts to perform arithmetic but with incorrect results and repetition of incorrect calculations. The average answer token loss is high. Removing 6 random layers leads to a pattern of repetition with numbers and the '+' sign, but no valid arithmetic. The average answer token loss is slightly lower. With 9 layers removed, the model outputs commas and the word 'the', with no arithmetic structure, resulting in the lowest average answer token loss among the configurations. The model's performance degrades with more layers removed, but the type of degradation varies depending on which layers are removed.\", 'current_hypothesis': \"The model's arithmetic capability is sensitive to the removal of specific layers. The presence of certain layers seems to be critical for maintaining arithmetic structure, while others contribute to the actual computation. The degradation pattern suggests that the model's ability to perform arithmetic is not uniformly distributed across layers.\"}\n",
      "<class 'KeyError'> 'ablation'\n",
      "omitting: {'experiment_name': 'Remove first n layers', 'task_name': '1-digit + 1-digit addition', 'observation': \"As the first n layers are incrementally removed, the model's output becomes less structured and more chaotic, with a mix of punctuation and repeated characters. The average answer token loss generally increases as more initial layers are removed, peaking when the first five layers are omitted. However, when only the last layer is used, the average answer token loss decreases again, suggesting that the last layer alone is not sufficient to produce structured output but is less detrimental than removing the first five layers. The model fails to perform arithmetic in all cases.\", 'current_hypothesis': \"The initial layers of the model seem to be crucial for establishing the structure of the task and possibly for early processing of the input. As these layers are removed, the model's ability to maintain task structure and perform arithmetic deteriorates. The last layer alone does not have the capability to produce coherent output, indicating that the processing capabilities are distributed across multiple layers and that the initial layers play a significant role.\"}\n",
      "# Remove last n layers \n",
      "## 1-digit + 1-digit addition \n",
      "### Raw results\n",
      "Ablation: {'layer-permutation': [0]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================================\n",
      "Avg loss of: '13': 11.925361633300781\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ================== -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-\n",
      "Avg loss of: '13': 13.355360984802246\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ============== -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-\n",
      "Avg loss of: '13': 16.431570053100586\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 3]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ===========>=>=>=>=>=>=> -_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-_-\n",
      "Avg loss of: '13': 18.23340606689453\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 3, 4]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ===========|__dict_dict_dictender_dictender_dictender_dictender_type_type_type_type_type_type_type_type_type_type_type_type_type_type_type_type_type_type.\n",
      "                                          (*)$112080802111234567890808080211.html\n",
      "\n",
      "(*) ERROR REPORT REPORTINGLYLYSTEMSTEMSTEMOOLOOLOOLOOLOOLOOLING\n",
      "Avg loss of: '13': 16.442686080932617\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 3, 4, 5]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  1st tier tier tier wise wise wise wise wise..-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.-.\n",
      "Avg loss of: '13': 17.490617752075195\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 3, 4, 5, 6]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  8.5%\n",
      "\n",
      "Advertisements<|endoftext|>Still loading loading loading loading loading loading loading loading loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading Loading\n",
      "Avg loss of: '13': 17.276195526123047\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  11\n",
      "3rd Place Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner Winner\n",
      "Avg loss of: '13': 15.919795989990234\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7, 8]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  11\n",
      "4 + 9 = 11\n",
      "4 + 9 = 11\n",
      "4 + 9 = 11\n",
      "4 + 9 = 11\n",
      "#define USE_CURRENT_TARGET_type_name__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__attribute__\n",
      "Avg loss of: '13': 15.925132751464844\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  13\n",
      "3 + 8 = 14\n",
      "3 + 7 = 15\n",
      "3 + 6 = 16\n",
      "3 + 6 = 17\n",
      "3 + 6 = 18\n",
      "3 + 6 = 19\n",
      "3 + 6 = 20\n",
      "3 + 6 = 21\n",
      "3 + 6 = 22\n",
      "3 + 6 = 23\n",
      "3 + 6 = 24\n",
      "3 + 6 = 25\n",
      "3 + 6 = 26\n",
      "3 + 6 = 27\n",
      "3 + 6 = 28\n",
      "3 + 6 = 29\n",
      "3 +\n",
      "Avg loss of: '13': 15.069592475891113\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  13\n",
      "3 + 8 = 14\n",
      "2 + 7 = 15\n",
      "1 + 6 = 16\n",
      "1 + 5 = 17\n",
      "1 + 4 = 18\n",
      "1 + 3 = 19\n",
      "1 + 2 = 20\n",
      "1 + 1 = 21\n",
      "1 + 0 = 22\n",
      "9 + 8 = 23\n",
      "9 + 7 = 24\n",
      "9 + 6 = 25\n",
      "9 + 5 = 26\n",
      "9 + 4 = 27\n",
      "9 + 3 = 28\n",
      "9 + 2 = 29\n",
      "9 +\n",
      "Avg loss of: '13': 14.847496032714844\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  12\n",
      "3 + 8 = 13\n",
      "2 + 7 = 14\n",
      "1 + 6 = 15\n",
      "5 + 5 = 16\n",
      "4 + 4 = 17\n",
      "3 + 3 = 18\n",
      "2 + 2 = 19\n",
      "1 + 1 = 20\n",
      "\n",
      "10 + 9 = 21\n",
      "\n",
      "9 + 8 = 22\n",
      "\n",
      "8 + 7 = 23\n",
      "\n",
      "7 + 6 = 24\n",
      "\n",
      "6 + 5 = 25\n",
      "\n",
      "5 + 4 = 26\n",
      "\n",
      "4 + 3 = 27\n",
      "\n",
      "\n",
      "Avg loss of: '13': 16.18016815185547\n",
      "---\n",
      "### Notes\n",
      "As layers are incrementally removed, the model's ability to perform simple addition deteriorates. The average answer token loss increases with fewer layers, indicating a loss of capability. However, even with a single layer, the model does not output random tokens but rather a repetition of the prompt structure. With more layers, the model begins to output numbers, although incorrect, suggesting some numerical processing capability is retained. Correct answers start to appear with 8 layers, but with errors in other arithmetic examples.\n",
      "Current hypothesis:\n",
      "The model's ability to perform arithmetic operations degrades as layers are removed, indicating that later layers contribute to this capability. However, some basic numerical understanding is present even with a few layers. The correct arithmetic capability seems to require a majority of the layers, but not all.\n",
      "# Skip every other layer \n",
      "## 1-digit + 1-digit addition \n",
      "### Raw results\n",
      "Ablation: {'layer-permutation': [0, 2, 4, 6, 8, 10]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 + 10 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 = 11 =\n",
      "Avg loss of: '13': 14.093454360961914\n",
      "---\n",
      "Ablation: {'layer-permutation': [1, 3, 5, 7, 9, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ....................................................................................................\n",
      "Avg loss of: '13': 8.59432601928711\n",
      "---\n",
      "### Notes\n",
      "When skipping every other layer starting with the first layer (0, 2, 4, 6, 8, 10), the model repeats the structure of the prompt with incorrect arithmetic. The average answer token loss is significant. However, when starting with the second layer (1, 3, 5, 7, 9, 11), the model fails to produce any arithmetic output and instead generates a series of dots, resulting in a lower average answer token loss. This suggests that the model's ability to perform arithmetic is disrupted more when starting with an even layer, and that the odd layers may be more crucial for maintaining the structure of the task.\n",
      "Current hypothesis:\n",
      "The model's arithmetic capability is affected by the pattern of skipped layers. Starting with an even layer disrupts arithmetic more than starting with an odd layer. This could indicate that odd layers play a more significant role in task structure retention and possibly in arithmetic processing.\n",
      "# Remove random layers \n",
      "## 1-digit + 1-digit addition \n",
      "### Raw results\n",
      "Ablation: {'layer-permutation': [0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  12\n",
      "7 + 5 = 13\n",
      "6 + 4 = 14\n",
      "5 + 5 = 15\n",
      "6 + 4 = 16\n",
      "6 + 4 = 17\n",
      "6 + 4 = 18\n",
      "6 + 4 = 19\n",
      "6 + 4 = 20\n",
      "6 + 4 = 21\n",
      "6 + 4 = 22\n",
      "6 + 4 = 23\n",
      "6 + 4 = 24\n",
      "6 + 4 = 25\n",
      "6 + 4 = 26\n",
      "6 + 4 = 27\n",
      "6 + 4 = 28\n",
      "6 +\n",
      "Avg loss of: '13': 16.199176788330078\n",
      "---\n",
      "Ablation: {'layer-permutation': [0, 2, 3, 5, 7, 8, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion:  8 + 4 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 4 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 + 8 = 8 +\n",
      "Avg loss of: '13': 15.693228721618652\n",
      "---\n",
      "Ablation: {'layer-permutation': [1, 3, 4, 6, 9]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ,,,,,,,,,,\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      ",,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, the the the,,,,,\n",
      "Avg loss of: '13': 8.481050491333008\n",
      "---\n",
      "### Notes\n",
      "With 3 random layers removed, the model still attempts to perform arithmetic but with incorrect results and repetition of incorrect calculations. The average answer token loss is high. Removing 6 random layers leads to a pattern of repetition with numbers and the '+' sign, but no valid arithmetic. The average answer token loss is slightly lower. With 9 layers removed, the model outputs commas and the word 'the', with no arithmetic structure, resulting in the lowest average answer token loss among the configurations. The model's performance degrades with more layers removed, but the type of degradation varies depending on which layers are removed.\n",
      "Current hypothesis:\n",
      "The model's arithmetic capability is sensitive to the removal of specific layers. The presence of certain layers seems to be critical for maintaining arithmetic structure, while others contribute to the actual computation. The degradation pattern suggests that the model's ability to perform arithmetic is not uniformly distributed across layers.\n",
      "# Remove first n layers \n",
      "## 1-digit + 1-digit addition \n",
      "### Raw results\n",
      "Ablation: {'layer-permutation': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ....................................................................................................\n",
      "Avg loss of: '13': 9.177383422851562\n",
      "---\n",
      "Ablation: {'layer-permutation': [2, 3, 4, 5, 6, 7, 8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,, the,,,, the,,, the,,, the, the, the, the, the the, the the, the the the the the the the the the the the the the the the the the the\n",
      "Avg loss of: '13': 8.315519332885742\n",
      "---\n",
      "Ablation: {'layer-permutation': [3, 4, 5, 6, 7, 8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ))))))))))-)-)-)-)-)-)-)-)-)))-)))))))))))))),),) \"),) \") \") \") \") \") \"- \"- \", \",, \", \", \", \", \", \", \", \", \", \", \" ( ( ( ( ( ( (\n",
      "Avg loss of: '13': 12.837023735046387\n",
      "---\n",
      "Ablation: {'layer-permutation': [4, 5, 6, 7, 8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "- \",-------------\n",
      "Avg loss of: '13': 14.807900428771973\n",
      "---\n",
      "Ablation: {'layer-permutation': [5, 6, 7, 8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ----------------------------------------------------------------------------------------------------\n",
      "Avg loss of: '13': 15.825925827026367\n",
      "---\n",
      "Ablation: {'layer-permutation': [6, 7, 8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ——— (———————————————————————————————————————————————----------------------------- ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (\n",
      "Avg loss of: '13': 15.003804206848145\n",
      "---\n",
      "Ablation: {'layer-permutation': [7, 8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: hodtohodtototobothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothbothtotototototototototototototototototototototototototototototototototototototototototototototototototototo\n",
      "Avg loss of: '13': 14.509140968322754\n",
      "---\n",
      "Ablation: {'layer-permutation': [8, 9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ,,,,,'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' '\n",
      "Avg loss of: '13': 10.779223442077637\n",
      "---\n",
      "Ablation: {'layer-permutation': [9, 10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ......,...,...,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\n",
      "Avg loss of: '13': 9.529319763183594\n",
      "---\n",
      "Ablation: {'layer-permutation': [10, 11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\n",
      "Avg loss of: '13': 10.608086585998535\n",
      "---\n",
      "Ablation: {'layer-permutation': [11]}\n",
      "Prompt: 8 + 10 = 18\n",
      "7 + 4 = 11\n",
      "4 + 9 =\n",
      "Completion: ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,\n",
      "Avg loss of: '13': 8.235128402709961\n",
      "---\n",
      "### Notes\n",
      "As the first n layers are incrementally removed, the model's output becomes less structured and more chaotic, with a mix of punctuation and repeated characters. The average answer token loss generally increases as more initial layers are removed, peaking when the first five layers are omitted. However, when only the last layer is used, the average answer token loss decreases again, suggesting that the last layer alone is not sufficient to produce structured output but is less detrimental than removing the first five layers. The model fails to perform arithmetic in all cases.\n",
      "Current hypothesis:\n",
      "The initial layers of the model seem to be crucial for establishing the structure of the task and possibly for early processing of the input. As these layers are removed, the model's ability to maintain task structure and perform arithmetic deteriorates. The last layer alone does not have the capability to produce coherent output, indicating that the processing capabilities are distributed across multiple layers and that the initial layers play a significant role.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def get_notes():\n",
    "    items = store.filter(experiment_name=lambda i: i is not None)\n",
    "    df_items = pd.DataFrame(items)\n",
    "    experiment_names = df_items.experiment_name.unique()\n",
    "    res = \"\"\n",
    "    for experiment_name in experiment_names:\n",
    "        res += get_experiment(experiment_name)\n",
    "    return res\n",
    "\n",
    "def get_experiment(experiment_name):\n",
    "    res = f\"# {experiment_name} \\n\"\n",
    "    items = store.filter(experiment_name=experiment_name)\n",
    "    df_items = pd.DataFrame(items)\n",
    "    task_names = df_items.task_name.unique()\n",
    "    for task_name in task_names:\n",
    "        res += get_experiment_task(experiment_name, task_name)\n",
    "    return res\n",
    "    \n",
    "def get_experiment_task(experiment_name, task_name):\n",
    "    items = store.filter(experiment_name=experiment_name, task_name=task_name)\n",
    "    observations = store.filter(observation=lambda i: i is not None, experiment_name=experiment_name, task_name=task_name)\n",
    "    res = f\"## {task_name} \\n### Raw results\\n\"\n",
    "    for i in items:\n",
    "        res += format_item(i)\n",
    "    if len(observations) > 0:\n",
    "        res += f\"### Notes\\n\"\n",
    "        for observation in observations:\n",
    "            res += observation['observation']\n",
    "            res += f\"\\nCurrent hypothesis:\\n{observation['current_hypothesis']}\\n\"\n",
    "    return res\n",
    "\n",
    "\n",
    "def format_item(i):\n",
    "    try:\n",
    "        outputs_str = f\"Ablation: {i['ablation']}\\nPrompt: {i['prompt']}\\nCompletion: {i['completion']}\\nAvg loss of: '{i['answer']}': {i['average_answer_token_loss']}\\n---\\n\"\n",
    "        return outputs_str\n",
    "    except Exception as e:\n",
    "        # print(type(e), e)\n",
    "        # print(\"omitting:\", i)\n",
    "        return \"\"\n",
    "\n",
    "\n",
    "print(get_notes())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lens",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}