scripts/fafb_missing_some_synapses.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "946dc5c7",
   "metadata": {},
   "source": [
    "# It looks like the FAFB graph doesn't have all the connections that the connections table does"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4e1fc82e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attribute class_1 not found in the graph. Adding it.\n"
     ]
    }
   ],
   "source": [
    "import polars as pl\n",
    "\n",
    "import vnc_networks\n",
    "\n",
    "connections = vnc_networks.connections.Connections(\n",
    "    CR=vnc_networks.connectome_reader.FAFB_v783()\n",
    ")\n",
    "connections_table = pl.from_pandas(connections.connections)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "15eb1f75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "connections table has 2709829 connections\n",
      "connections graph has 2484163 connections\n"
     ]
    }
   ],
   "source": [
    "print(f\"connections table has {len(connections_table)} connections\")\n",
    "print(f\"connections graph has {len(connections.graph.edges)} connections\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a484a36f",
   "metadata": {},
   "source": [
    "It turns out that the FAFB connections table sometimes has multiple entries between the same pair of neurons.\n",
    "\n",
    "If we check the documentation [here](https://codex.flywire.ai/api/download) it says \"More than one row can be present for the same pair of cells if they synapse in multiple neuropils (regions).\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f2166d15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (169_972, 3)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>start_bid</th><th>end_bid</th><th>number_of_connections</th></tr><tr><td>i64</td><td>i64</td><td>u32</td></tr></thead><tbody><tr><td>720575940624402173</td><td>720575940622160705</td><td>12</td></tr><tr><td>720575940620540507</td><td>720575940618932763</td><td>11</td></tr><tr><td>720575940612718563</td><td>720575940623122125</td><td>11</td></tr><tr><td>720575940639242303</td><td>720575940622160705</td><td>11</td></tr><tr><td>720575940622915060</td><td>720575940616551029</td><td>10</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>720575940613455986</td><td>720575940637085503</td><td>2</td></tr><tr><td>720575940620795784</td><td>720575940607689394</td><td>2</td></tr><tr><td>720575940609658377</td><td>720575940630758418</td><td>2</td></tr><tr><td>720575940630403269</td><td>720575940636250292</td><td>2</td></tr><tr><td>720575940633593548</td><td>720575940633923479</td><td>2</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (169_972, 3)\n",
       "┌────────────────────┬────────────────────┬───────────────────────┐\n",
       "│ start_bid          ┆ end_bid            ┆ number_of_connections │\n",
       "│ ---                ┆ ---                ┆ ---                   │\n",
       "│ i64                ┆ i64                ┆ u32                   │\n",
       "╞════════════════════╪════════════════════╪═══════════════════════╡\n",
       "│ 720575940624402173 ┆ 720575940622160705 ┆ 12                    │\n",
       "│ 720575940620540507 ┆ 720575940618932763 ┆ 11                    │\n",
       "│ 720575940612718563 ┆ 720575940623122125 ┆ 11                    │\n",
       "│ 720575940639242303 ┆ 720575940622160705 ┆ 11                    │\n",
       "│ 720575940622915060 ┆ 720575940616551029 ┆ 10                    │\n",
       "│ …                  ┆ …                  ┆ …                     │\n",
       "│ 720575940613455986 ┆ 720575940637085503 ┆ 2                     │\n",
       "│ 720575940620795784 ┆ 720575940607689394 ┆ 2                     │\n",
       "│ 720575940609658377 ┆ 720575940630758418 ┆ 2                     │\n",
       "│ 720575940630403269 ┆ 720575940636250292 ┆ 2                     │\n",
       "│ 720575940633593548 ┆ 720575940633923479 ┆ 2                     │\n",
       "└────────────────────┴────────────────────┴───────────────────────┘"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "connections_table.group_by([\"start_bid\", \"end_bid\"]).agg(\n",
    "    pl.col(\"syn_count\").len().alias(\"number_of_connections\")\n",
    ").filter(pl.col(\"number_of_connections\") > 1).sort(\n",
    "    \"number_of_connections\", descending=True\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "57b25dcd",
   "metadata": {},
   "source": [
    "Looking at all the connections between the first pair of neurons:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e574e2d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (12, 11)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>start_bid</th><th>end_bid</th><th>syn_count</th><th>nt_type</th><th>eff_weight</th><th>subdivision_start</th><th>subdivision_end</th><th>syn_count_norm</th><th>eff_weight_norm</th><th>start_uid</th><th>end_uid</th></tr><tr><td>i64</td><td>i64</td><td>i64</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>f64</td><td>f64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>720575940624402173</td><td>720575940622160705</td><td>45</td><td>&quot;ACH&quot;</td><td>45</td><td>0</td><td>0</td><td>0.034695</td><td>0.034695</td><td>131982</td><td>72220</td></tr><tr><td>720575940624402173</td><td>720575940622160705</td><td>12</td><td>&quot;ACH&quot;</td><td>12</td><td>0</td><td>0</td><td>0.009252</td><td>0.009252</td><td>131982</td><td>72220</td></tr><tr><td>720575940624402173</td><td>720575940622160705</td><td>11</td><td>&quot;ACH&quot;</td><td>11</td><td>0</td><td>0</td><td>0.008481</td><td>0.008481</td><td>131982</td><td>72220</td></tr><tr><td>720575940624402173</td><td>720575940622160705</td><td>6</td><td>&quot;ACH&quot;</td><td>6</td><td>0</td><td>0</td><td>0.004626</td><td>0.004626</td><td>131982</td><td>72220</td></tr><tr><td>720575940624402173</td><td>720575940622160705</td><td>13</td><td>&quot;ACH&quot;</td><td>13</td><td>0</td><td>0</td><td>0.010023</td><td>0.010023</td><td>131982</td><td>72220</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>720575940624402173</td><td>720575940622160705</td><td>6</td><td>&quot;ACH&quot;</td><td>6</td><td>0</td><td>0</td><td>0.004626</td><td>0.004626</td><td>131982</td><td>72220</td></tr><tr><td>720575940624402173</td><td>720575940622160705</td><td>25</td><td>&quot;ACH&quot;</td><td>25</td><td>0</td><td>0</td><td>0.019275</td><td>0.019275</td><td>131982</td><td>72220</td></tr><tr><td>720575940624402173</td><td>720575940622160705</td><td>35</td><td>&quot;ACH&quot;</td><td>35</td><td>0</td><td>0</td><td>0.026985</td><td>0.026985</td><td>131982</td><td>72220</td></tr><tr><td>720575940624402173</td><td>720575940622160705</td><td>61</td><td>&quot;ACH&quot;</td><td>61</td><td>0</td><td>0</td><td>0.047032</td><td>0.047032</td><td>131982</td><td>72220</td></tr><tr><td>720575940624402173</td><td>720575940622160705</td><td>33</td><td>&quot;ACH&quot;</td><td>33</td><td>0</td><td>0</td><td>0.025443</td><td>0.025443</td><td>131982</td><td>72220</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (12, 11)\n",
       "┌────────────┬────────────┬───────────┬─────────┬───┬────────────┬───────────┬───────────┬─────────┐\n",
       "│ start_bid  ┆ end_bid    ┆ syn_count ┆ nt_type ┆ … ┆ syn_count_ ┆ eff_weigh ┆ start_uid ┆ end_uid │\n",
       "│ ---        ┆ ---        ┆ ---       ┆ ---     ┆   ┆ norm       ┆ t_norm    ┆ ---       ┆ ---     │\n",
       "│ i64        ┆ i64        ┆ i64       ┆ str     ┆   ┆ ---        ┆ ---       ┆ i64       ┆ i64     │\n",
       "│            ┆            ┆           ┆         ┆   ┆ f64        ┆ f64       ┆           ┆         │\n",
       "╞════════════╪════════════╪═══════════╪═════════╪═══╪════════════╪═══════════╪═══════════╪═════════╡\n",
       "│ 7205759406 ┆ 7205759406 ┆ 45        ┆ ACH     ┆ … ┆ 0.034695   ┆ 0.034695  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "│ 7205759406 ┆ 7205759406 ┆ 12        ┆ ACH     ┆ … ┆ 0.009252   ┆ 0.009252  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "│ 7205759406 ┆ 7205759406 ┆ 11        ┆ ACH     ┆ … ┆ 0.008481   ┆ 0.008481  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "│ 7205759406 ┆ 7205759406 ┆ 6         ┆ ACH     ┆ … ┆ 0.004626   ┆ 0.004626  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "│ 7205759406 ┆ 7205759406 ┆ 13        ┆ ACH     ┆ … ┆ 0.010023   ┆ 0.010023  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "│ …          ┆ …          ┆ …         ┆ …       ┆ … ┆ …          ┆ …         ┆ …         ┆ …       │\n",
       "│ 7205759406 ┆ 7205759406 ┆ 6         ┆ ACH     ┆ … ┆ 0.004626   ┆ 0.004626  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "│ 7205759406 ┆ 7205759406 ┆ 25        ┆ ACH     ┆ … ┆ 0.019275   ┆ 0.019275  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "│ 7205759406 ┆ 7205759406 ┆ 35        ┆ ACH     ┆ … ┆ 0.026985   ┆ 0.026985  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "│ 7205759406 ┆ 7205759406 ┆ 61        ┆ ACH     ┆ … ┆ 0.047032   ┆ 0.047032  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "│ 7205759406 ┆ 7205759406 ┆ 33        ┆ ACH     ┆ … ┆ 0.025443   ┆ 0.025443  ┆ 131982    ┆ 72220   │\n",
       "│ 24402173   ┆ 22160705   ┆           ┆         ┆   ┆            ┆           ┆           ┆         │\n",
       "└────────────┴────────────┴───────────┴─────────┴───┴────────────┴───────────┴───────────┴─────────┘"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "connections_table.filter(\n",
    "    (pl.col(\"start_bid\") == 720575940624402173)\n",
    "    & (pl.col(\"end_bid\") == 720575940622160705)\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fd5bb318",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'syn_count': 33,\n",
       " 'nt_type': 'ACH',\n",
       " 'eff_weight': 33,\n",
       " 'syn_count_norm': 0.025443330763299923,\n",
       " 'eff_weight_norm': 0.025443330763299923,\n",
       " 'weight': 33}"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "connections.graph.edges[131982, 72220]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e5de8e7c",
   "metadata": {},
   "source": [
    "The graph just remembered that last connection in the table - the rest were dropped by networkx"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d7fa48e6",
   "metadata": {},
   "source": [
    "## Quick check for MANC"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d91b9b1f",
   "metadata": {},
   "source": [
    "MANC v1.2 must have already aggregated synapses over pairs of neurons (actually I think I did this when downloading the data from neuprint), so here there's no problem.\n",
    "\n",
    "This is because the neurotransmitter predictions are only applied at the level of neurons, so we first aggregate and then label the synapse type according to the neuron's neurotransmitter type."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "676a805a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attribute class_1 not found in the graph. Adding it.\n",
      "connections table has 1372588 connections with 24151003 synapses\n",
      "connections graph has 1372588 connections with 24151003 synapses\n"
     ]
    }
   ],
   "source": [
    "def check_connections_table_and_graph_count(\n",
    "    connectome_reader: vnc_networks.connectome_reader.ConnectomeReader,\n",
    "):\n",
    "    c = vnc_networks.connections.Connections(connectome_reader)\n",
    "    print(\n",
    "        f\"connections table has {len(c.connections)} connections with {c.connections[\"syn_count\"].sum()} synapses\"\n",
    "    )\n",
    "    print(\n",
    "        f\"connections graph has {len(c.graph.edges)} connections with {sum(\n",
    "        c.graph.edges[e][\"syn_count\"] for e in c.graph.edges\n",
    "    )} synapses\"\n",
    "    )\n",
    "\n",
    "check_connections_table_and_graph_count(vnc_networks.connectome_reader.MANC_v_1_2())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b87f95d5",
   "metadata": {},
   "source": [
    "MANC v1.0 is all good too"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "819775ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attribute class_1 not found in the graph. Adding it.\n",
      "connections table has 1548657 connections with 27387970 synapses\n",
      "connections graph has 1548657 connections with 27387970 synapses\n"
     ]
    }
   ],
   "source": [
    "check_connections_table_and_graph_count(vnc_networks.connectome_reader.MANC_v_1_0())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5823402",
   "metadata": {},
   "source": [
    "## How many synapses are affected?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d064083e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We lost 3549543 out of 31574890 synapses, which is 11.2%\n"
     ]
    }
   ],
   "source": [
    "connections_table_synapses = connections_table[\"syn_count\"].sum()\n",
    "graph_synapses = sum(\n",
    "    connections.graph.edges[e][\"syn_count\"] for e in connections.graph.edges\n",
    ")\n",
    "\n",
    "print(\n",
    "    f\"We lost {(diff := connections_table_synapses - graph_synapses)} out of {connections_table_synapses} synapses, which is {diff/connections_table_synapses*100:.1f}%\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "20f974c7",
   "metadata": {},
   "source": [
    "Weirdly, there are 12285 neuron pairs that have multiple different neurotransmitters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "9647a987",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (12_285, 5)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>start_bid</th><th>end_bid</th><th>different_neurotransmitters</th><th>num_different_neurotransmitters</th><th>num_synapses</th></tr><tr><td>i64</td><td>i64</td><td>list[struct[2]]</td><td>u32</td><td>i64</td></tr></thead><tbody><tr><td>720575940623317321</td><td>720575940639278399</td><td>[{&quot;GLUT&quot;,2}, {&quot;GABA&quot;,3}, … {&quot;ACH&quot;,1}]</td><td>4</td><td>156</td></tr><tr><td>720575940641501648</td><td>720575940621268651</td><td>[{&quot;SER&quot;,1}, {&quot;GABA&quot;,1}, … {&quot;ACH&quot;,1}]</td><td>4</td><td>57</td></tr><tr><td>720575940631763909</td><td>720575940613096089</td><td>[{&quot;GABA&quot;,1}, {&quot;GLUT&quot;,1}, … {&quot;ACH&quot;,1}]</td><td>4</td><td>49</td></tr><tr><td>720575940641501648</td><td>720575940629399370</td><td>[{&quot;GABA&quot;,2}, {&quot;ACH&quot;,1}, {&quot;GLUT&quot;,1}]</td><td>3</td><td>76</td></tr><tr><td>720575940622263066</td><td>720575940607384242</td><td>[{&quot;GLUT&quot;,1}, {&quot;ACH&quot;,1}, {&quot;GABA&quot;,1}]</td><td>3</td><td>25</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>720575940610384082</td><td>720575940623682195</td><td>[{&quot;ACH&quot;,4}, {&quot;SER&quot;,1}]</td><td>2</td><td>39</td></tr><tr><td>720575940627209990</td><td>720575940624202424</td><td>[{&quot;GABA&quot;,1}, {&quot;GLUT&quot;,1}]</td><td>2</td><td>10</td></tr><tr><td>720575940617505629</td><td>720575940618392912</td><td>[{&quot;GLUT&quot;,1}, {&quot;GABA&quot;,2}]</td><td>2</td><td>48</td></tr><tr><td>720575940621801866</td><td>720575940609819512</td><td>[{&quot;GLUT&quot;,2}, {&quot;GABA&quot;,1}]</td><td>2</td><td>27</td></tr><tr><td>720575940632032460</td><td>720575940624705514</td><td>[{&quot;GLUT&quot;,1}, {&quot;GABA&quot;,1}]</td><td>2</td><td>51</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (12_285, 5)\n",
       "┌────────────────────┬────────────────────┬────────────────────┬────────────────────┬──────────────┐\n",
       "│ start_bid          ┆ end_bid            ┆ different_neurotra ┆ num_different_neur ┆ num_synapses │\n",
       "│ ---                ┆ ---                ┆ nsmitters          ┆ otransmitter…      ┆ ---          │\n",
       "│ i64                ┆ i64                ┆ ---                ┆ ---                ┆ i64          │\n",
       "│                    ┆                    ┆ list[struct[2]]    ┆ u32                ┆              │\n",
       "╞════════════════════╪════════════════════╪════════════════════╪════════════════════╪══════════════╡\n",
       "│ 720575940623317321 ┆ 720575940639278399 ┆ [{\"GLUT\",2},       ┆ 4                  ┆ 156          │\n",
       "│                    ┆                    ┆ {\"GABA\",3}, … {\"A… ┆                    ┆              │\n",
       "│ 720575940641501648 ┆ 720575940621268651 ┆ [{\"SER\",1},        ┆ 4                  ┆ 57           │\n",
       "│                    ┆                    ┆ {\"GABA\",1}, …      ┆                    ┆              │\n",
       "│                    ┆                    ┆ {\"AC…              ┆                    ┆              │\n",
       "│ 720575940631763909 ┆ 720575940613096089 ┆ [{\"GABA\",1},       ┆ 4                  ┆ 49           │\n",
       "│                    ┆                    ┆ {\"GLUT\",1}, … {\"A… ┆                    ┆              │\n",
       "│ 720575940641501648 ┆ 720575940629399370 ┆ [{\"GABA\",2},       ┆ 3                  ┆ 76           │\n",
       "│                    ┆                    ┆ {\"ACH\",1}, {\"GLUT… ┆                    ┆              │\n",
       "│ 720575940622263066 ┆ 720575940607384242 ┆ [{\"GLUT\",1},       ┆ 3                  ┆ 25           │\n",
       "│                    ┆                    ┆ {\"ACH\",1}, {\"GABA… ┆                    ┆              │\n",
       "│ …                  ┆ …                  ┆ …                  ┆ …                  ┆ …            │\n",
       "│ 720575940610384082 ┆ 720575940623682195 ┆ [{\"ACH\",4},        ┆ 2                  ┆ 39           │\n",
       "│                    ┆                    ┆ {\"SER\",1}]         ┆                    ┆              │\n",
       "│ 720575940627209990 ┆ 720575940624202424 ┆ [{\"GABA\",1},       ┆ 2                  ┆ 10           │\n",
       "│                    ┆                    ┆ {\"GLUT\",1}]        ┆                    ┆              │\n",
       "│ 720575940617505629 ┆ 720575940618392912 ┆ [{\"GLUT\",1},       ┆ 2                  ┆ 48           │\n",
       "│                    ┆                    ┆ {\"GABA\",2}]        ┆                    ┆              │\n",
       "│ 720575940621801866 ┆ 720575940609819512 ┆ [{\"GLUT\",2},       ┆ 2                  ┆ 27           │\n",
       "│                    ┆                    ┆ {\"GABA\",1}]        ┆                    ┆              │\n",
       "│ 720575940632032460 ┆ 720575940624705514 ┆ [{\"GLUT\",1},       ┆ 2                  ┆ 51           │\n",
       "│                    ┆                    ┆ {\"GABA\",1}]        ┆                    ┆              │\n",
       "└────────────────────┴────────────────────┴────────────────────┴────────────────────┴──────────────┘"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "connections_table.group_by([\"start_bid\", \"end_bid\"]).agg(\n",
    "    pl.col(\"nt_type\").value_counts().alias(\"different_neurotransmitters\"),\n",
    "    pl.col(\"nt_type\").n_unique().alias(\"num_different_neurotransmitters\"),\n",
    "    pl.col(\"syn_count\").sum().alias(\"num_synapses\"),\n",
    ").filter(pl.col(\"num_different_neurotransmitters\") > 1).sort(\n",
    "    pl.col(\"num_different_neurotransmitters\"), descending=True\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0ff833e3",
   "metadata": {},
   "source": [
    "If we count the total number of synapses per neurotransmitter type, things look kind of messy. We even have a lot of neurons that have multiple of acetylcholine, glutamate and GABA, which shouldn't really be possible? Maybe it's just mislabelling? Would need to check the [neurotransmitter paper](https://doi.org/10.1016/j.cell.2024.03.016)...\n",
    "\n",
    "I think it could be important to know that a neuron which primarily uses a fast acting neurotransmitter can also have neuromodulatory connections, and it would be nice to keep these."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "1824ca1f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (12_285, 9)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>start_bid</th><th>end_bid</th><th>num_connections_per_nt</th><th>ACH</th><th>GLUT</th><th>GABA</th><th>SER</th><th>DA</th><th>OCT</th></tr><tr><td>i64</td><td>i64</td><td>list[struct[2]]</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td></tr></thead><tbody><tr><td>720575940631763909</td><td>720575940613096089</td><td>[{&quot;GABA&quot;,1}, {&quot;GLUT&quot;,1}, … {&quot;SER&quot;,1}]</td><td>33</td><td>5</td><td>6</td><td>5</td><td>null</td><td>null</td></tr><tr><td>720575940641501648</td><td>720575940621268651</td><td>[{&quot;ACH&quot;,1}, {&quot;SER&quot;,1}, … {&quot;GLUT&quot;,1}]</td><td>5</td><td>8</td><td>39</td><td>5</td><td>null</td><td>null</td></tr><tr><td>720575940623317321</td><td>720575940639278399</td><td>[{&quot;GLUT&quot;,1}, {&quot;GABA&quot;,1}, … {&quot;ACH&quot;,1}]</td><td>11</td><td>53</td><td>86</td><td>null</td><td>6</td><td>null</td></tr><tr><td>720575940630911991</td><td>720575940625310014</td><td>[{&quot;GLUT&quot;,1}, {&quot;GABA&quot;,1}, {&quot;ACH&quot;,1}]</td><td>7</td><td>11</td><td>7</td><td>null</td><td>null</td><td>null</td></tr><tr><td>720575940622263066</td><td>720575940620543025</td><td>[{&quot;GABA&quot;,1}, {&quot;GLUT&quot;,1}, {&quot;ACH&quot;,1}]</td><td>8</td><td>13</td><td>31</td><td>null</td><td>null</td><td>null</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>720575940633356883</td><td>720575940636316023</td><td>[{&quot;GLUT&quot;,1}, {&quot;GABA&quot;,1}]</td><td>null</td><td>65</td><td>23</td><td>null</td><td>null</td><td>null</td></tr><tr><td>720575940639071977</td><td>720575940620818017</td><td>[{&quot;GABA&quot;,1}, {&quot;ACH&quot;,1}]</td><td>7</td><td>null</td><td>6</td><td>null</td><td>null</td><td>null</td></tr><tr><td>720575940630026745</td><td>720575940617190361</td><td>[{&quot;GLUT&quot;,1}, {&quot;GABA&quot;,1}]</td><td>null</td><td>26</td><td>16</td><td>null</td><td>null</td><td>null</td></tr><tr><td>720575940626995880</td><td>720575940629010356</td><td>[{&quot;ACH&quot;,1}, {&quot;GABA&quot;,1}]</td><td>23</td><td>null</td><td>25</td><td>null</td><td>null</td><td>null</td></tr><tr><td>720575940623433725</td><td>720575940604088288</td><td>[{&quot;ACH&quot;,1}, {&quot;SER&quot;,1}]</td><td>36</td><td>null</td><td>null</td><td>5</td><td>null</td><td>null</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (12_285, 9)\n",
       "┌───────────────────┬───────────────────┬───────────────────┬──────┬───┬──────┬──────┬──────┬──────┐\n",
       "│ start_bid         ┆ end_bid           ┆ num_connections_p ┆ ACH  ┆ … ┆ GABA ┆ SER  ┆ DA   ┆ OCT  │\n",
       "│ ---               ┆ ---               ┆ er_nt             ┆ ---  ┆   ┆ ---  ┆ ---  ┆ ---  ┆ ---  │\n",
       "│ i64               ┆ i64               ┆ ---               ┆ i64  ┆   ┆ i64  ┆ i64  ┆ i64  ┆ i64  │\n",
       "│                   ┆                   ┆ list[struct[2]]   ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "╞═══════════════════╪═══════════════════╪═══════════════════╪══════╪═══╪══════╪══════╪══════╪══════╡\n",
       "│ 72057594063176390 ┆ 72057594061309608 ┆ [{\"GABA\",1},      ┆ 33   ┆ … ┆ 6    ┆ 5    ┆ null ┆ null │\n",
       "│ 9                 ┆ 9                 ┆ {\"GLUT\",1}, …     ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│                   ┆                   ┆ {\"S…              ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│ 72057594064150164 ┆ 72057594062126865 ┆ [{\"ACH\",1},       ┆ 5    ┆ … ┆ 39   ┆ 5    ┆ null ┆ null │\n",
       "│ 8                 ┆ 1                 ┆ {\"SER\",1}, …      ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│                   ┆                   ┆ {\"GLU…            ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│ 72057594062331732 ┆ 72057594063927839 ┆ [{\"GLUT\",1},      ┆ 11   ┆ … ┆ 86   ┆ null ┆ 6    ┆ null │\n",
       "│ 1                 ┆ 9                 ┆ {\"GABA\",1}, …     ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│                   ┆                   ┆ {\"A…              ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│ 72057594063091199 ┆ 72057594062531001 ┆ [{\"GLUT\",1},      ┆ 7    ┆ … ┆ 7    ┆ null ┆ null ┆ null │\n",
       "│ 1                 ┆ 4                 ┆ {\"GABA\",1},       ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│                   ┆                   ┆ {\"ACH…            ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│ 72057594062226306 ┆ 72057594062054302 ┆ [{\"GABA\",1},      ┆ 8    ┆ … ┆ 31   ┆ null ┆ null ┆ null │\n",
       "│ 6                 ┆ 5                 ┆ {\"GLUT\",1},       ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│                   ┆                   ┆ {\"ACH…            ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│ …                 ┆ …                 ┆ …                 ┆ …    ┆ … ┆ …    ┆ …    ┆ …    ┆ …    │\n",
       "│ 72057594063335688 ┆ 72057594063631602 ┆ [{\"GLUT\",1},      ┆ null ┆ … ┆ 23   ┆ null ┆ null ┆ null │\n",
       "│ 3                 ┆ 3                 ┆ {\"GABA\",1}]       ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│ 72057594063907197 ┆ 72057594062081801 ┆ [{\"GABA\",1},      ┆ 7    ┆ … ┆ 6    ┆ null ┆ null ┆ null │\n",
       "│ 7                 ┆ 7                 ┆ {\"ACH\",1}]        ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│ 72057594063002674 ┆ 72057594061719036 ┆ [{\"GLUT\",1},      ┆ null ┆ … ┆ 16   ┆ null ┆ null ┆ null │\n",
       "│ 5                 ┆ 1                 ┆ {\"GABA\",1}]       ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│ 72057594062699588 ┆ 72057594062901035 ┆ [{\"ACH\",1},       ┆ 23   ┆ … ┆ 25   ┆ null ┆ null ┆ null │\n",
       "│ 0                 ┆ 6                 ┆ {\"GABA\",1}]       ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "│ 72057594062343372 ┆ 72057594060408828 ┆ [{\"ACH\",1},       ┆ 36   ┆ … ┆ null ┆ 5    ┆ null ┆ null │\n",
       "│ 5                 ┆ 8                 ┆ {\"SER\",1}]        ┆      ┆   ┆      ┆      ┆      ┆      │\n",
       "└───────────────────┴───────────────────┴───────────────────┴──────┴───┴──────┴──────┴──────┴──────┘"
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "connections_table.group_by([\"start_bid\", \"end_bid\", \"nt_type\"]).agg(\n",
    "    pl.col(\"syn_count\").sum().alias(\"num_nt_synapses\"),\n",
    ").group_by([\"start_bid\", \"end_bid\"]).agg(\n",
    "    pl.struct(\"nt_type\", \"num_nt_synapses\").alias(\"num_synapses_per_nt\"),\n",
    "    pl.col(\"nt_type\").value_counts().alias(\"num_connections_per_nt\"),\n",
    ").filter(pl.col(\"num_synapses_per_nt\").list.len() > 1).explode(\n",
    "    \"num_synapses_per_nt\"\n",
    ").unnest(\"num_synapses_per_nt\").pivot(\n",
    "    on=\"nt_type\",\n",
    "    index=[\"start_bid\", \"end_bid\", \"num_connections_per_nt\"],\n",
    "    values=\"num_nt_synapses\",\n",
    ").sort(pl.col(\"num_connections_per_nt\").list.len(), descending=True)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c95ff1ba",
   "metadata": {},
   "source": [
    "## What can we do?\n",
    "\n",
    "If all the neurotransmitter types for the same neuron pair were the same we could just aggregate the synapse counts, but this isn't the case...\n",
    "\n",
    "One easy thing we can do that maybe works is to use the networkx `MultiDiGraph` - a directed graph that can have multiple edges between pairs of nodes. Maybe this works with some things and not others though?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d3d60a29",
   "metadata": {},
   "outputs": [],
   "source": [
    "import networkx as nx\n",
    "\n",
    "fafb_multigraph = nx.from_pandas_edgelist(\n",
    "    connections.connections,\n",
    "    source=\"start_uid\",\n",
    "    target=\"end_uid\",\n",
    "    edge_attr=[\n",
    "        \"syn_count\",  # absolute synapse count\n",
    "        \"nt_type\",\n",
    "        \"eff_weight\",  # signed synapse count (nt weighted)\n",
    "        \"syn_count_norm\",  # input normalized synapse count\n",
    "        \"eff_weight_norm\",  # input normalized signed synapse count\n",
    "    ],\n",
    "    create_using=nx.MultiDiGraph,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4ee1b8eb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "connections table has 2709829 connections with 31574890 synapses\n",
      "connections graph has 2709829 connections with 31574890 synapses\n"
     ]
    }
   ],
   "source": [
    "print(\n",
    "        f\"connections table has {len(connections.connections)} connections with {connections.connections[\"syn_count\"].sum()} synapses\"\n",
    "    )\n",
    "print(\n",
    "    f\"connections graph has {len(fafb_multigraph.edges)} connections with {sum(\n",
    "    fafb_multigraph.edges[e][\"syn_count\"] for e in fafb_multigraph.edges\n",
    ")} synapses\"\n",
    ")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7c2c16fe",
   "metadata": {},
   "source": [
    "Whether this works for all the methods that operate on the graph, I'm not sure"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25bccb83",
   "metadata": {},
   "source": [
    "Alternatively, we can reuse the existing functionality of splitting neurons to split each neuron with multiple different neurotransmitter outputs into \"virtual neurons\", each with only one type of neurotransmitter.\n",
    "\n",
    "Then everything still fits in a normal digraph, and we can still represent it as an adjacency matrix"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}