{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Co-plotting CATMAID skeleton with flywire dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### This section contains example of fetching skeleton from a public catmaid instance and co-plotting with a flywire instance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import neccesary library modules now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import navis\n",
    "import fafbseg\n",
    "import pymaid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from copy import deepcopy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyroglancer.localserver import startdataserver, closedataserver\n",
    "from pyroglancer.flywire import flywireurl2dict, add_flywirelayer, set_flywireviewerstate\n",
    "from pyroglancer.ngviewer import openviewer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set configurations to fetch from data from CATMAID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "publicurl = 'https://fafb.catmaid.virtualflybrain.org/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO  : Global CATMAID instance set. Caching is ON. (pymaid)\n",
      "INFO - 2021-05-19 23:30:41,498 - client - Global CATMAID instance set. Caching is ON.\n"
     ]
    }
   ],
   "source": [
    "working_rm = pymaid.CatmaidInstance(publicurl, api_token=None, project_id = 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get sample skids and neuropil meshes from CATMAID"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "acf601e9752a4f2b9311b665e58be255",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Fetch neurons'), FloatProgress(value=0.0, max=15.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING - 2021-05-19 23:30:42,839 - connectionpool - Connection pool is full, discarding connection: fafb.catmaid.virtualflybrain.org\n",
      "WARNING - 2021-05-19 23:30:42,887 - connectionpool - Connection pool is full, discarding connection: fafb.catmaid.virtualflybrain.org\n",
      "WARNING - 2021-05-19 23:30:43,086 - connectionpool - Connection pool is full, discarding connection: fafb.catmaid.virtualflybrain.org\n",
      "WARNING - 2021-05-19 23:30:43,132 - connectionpool - Connection pool is full, discarding connection: fafb.catmaid.virtualflybrain.org\n",
      "WARNING - 2021-05-19 23:30:43,178 - connectionpool - Connection pool is full, discarding connection: fafb.catmaid.virtualflybrain.org\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc61310a263f48a481c701b13cee50db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Make nrn'), FloatProgress(value=0.0, max=15.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO - 2021-05-19 23:30:43,574 - utils - NumExpr defaulting to 8 threads.\n"
     ]
    }
   ],
   "source": [
    "catmiad_neuronlist=pymaid.get_neuron('/DA1 lPN',remote_instance = working_rm) #get some DA1 PNs.."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06171cb374924846b5e9dadca04bfbea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Volumes'), FloatProgress(value=0.0, max=2.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "vols = pymaid.get_volume(['v14.neuropil', 'AL_R'], color=(255, 0, 0, .2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Transform them to flywire space"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transform path: FAFB14 -> FAFB14raw -> FLYWIREraw -> FLYWIRE\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e75fdb83574b49bc8210cded99f43703",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value='Xforming'), FloatProgress(value=0.0, max=15.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "flywire_neuron=navis.xform_brain(catmiad_neuronlist,source='FAFB14', target='FLYWIRE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Transform path: FAFB14 -> FAFB14raw -> FLYWIREraw -> FLYWIRE\n",
      "Transform path: FAFB14 -> FAFB14raw -> FLYWIREraw -> FLYWIRE\n"
     ]
    }
   ],
   "source": [
    "flywirevol = {}\n",
    "flywirevol['v14.neuropil']=navis.xform_brain(vols['v14.neuropil'],source='FAFB14', target='FLYWIRE')\n",
    "flywirevol['AL_R']=navis.xform_brain(vols['AL_R'],source='FAFB14', target='FLYWIRE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'v14.neuropil': <navis.Volume(name=None, id=200, color=(0.85, 0.85, 0.85, 0.2), vertices.shape=(8997, 3), faces.shape=(18000, 3))>,\n",
       " 'AL_R': <navis.Volume(name=None, id=300, color=(0.85, 0.85, 0.85, 0.2), vertices.shape=(622, 3), faces.shape=(1240, 3))>}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "flywirevol['v14.neuropil'].id = 200\n",
    "flywirevol['AL_R'].id = 300\n",
    "flywirevol"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Start the dataserver to host precomputed data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Serving data from:  /var/folders/_l/lrfvj_8j3ps0c37ncbr3c8dh0000gn/T/tmp_5o756gr\n"
     ]
    }
   ],
   "source": [
    "startdataserver()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Get the url from flywire to append data onto"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Serving directory at http://127.0.0.1:8000\n"
     ]
    }
   ],
   "source": [
    "shorturl = 'https://ngl.flywire.ai/?json_url=https://globalv1.flywire-daf.com/nglstate/5644175227748352'\n",
    "#This flywire segments has some stuff similar to the DA1 PNs, you will notice their closeness in the final plot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add skids to flywire layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/private/var/folders/_l/lrfvj_8j3ps0c37ncbr3c8dh0000gn/T/tmp_5o756gr/precomputed/catmaid_skels/skeletons/57323\n",
      "/private/var/folders/_l/lrfvj_8j3ps0c37ncbr3c8dh0000gn/T/tmp_5o756gr/precomputed/catmaid_skels/skeletons/57353\n",
      "creating: /private/var/folders/_l/lrfvj_8j3ps0c37ncbr3c8dh0000gn/T/tmp_5o756gr/precomputed/catmaid_skels/skeletons/seg_props\n",
      "{'57323': ['#ff0000'], '57353': ['#ff0000']}\n",
      "flywire url at: https://ngl.flywire.ai/?json_url=https://globalv1.flywire-daf.com/nglstate/5174672299655168\n"
     ]
    }
   ],
   "source": [
    "tmpviewer = add_flywirelayer(flywireurl2dict(shorturl), layer_kws = {'type': 'skeletons',\n",
    "                                                                     'source': flywire_neuron[7:9],\n",
    "                                                                     'name': 'catmaid_skels',\n",
    "                                                                     'color': 'red'}) \n",
    "#'alpha': 0.3, doesn't work yet in skeleton layers.."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add neuropil meshes to flywire layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "shorturl = tmpviewer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mesh/200\n",
      "Seg id is: 200\n",
      "Full filepath: /private/var/folders/_l/lrfvj_8j3ps0c37ncbr3c8dh0000gn/T/tmp_5o756gr/precomputed/catmaid_neuropils/mesh/200\n",
      "mesh/300\n",
      "Seg id is: 300\n",
      "Full filepath: /private/var/folders/_l/lrfvj_8j3ps0c37ncbr3c8dh0000gn/T/tmp_5o756gr/precomputed/catmaid_neuropils/mesh/300\n",
      "creating: /private/var/folders/_l/lrfvj_8j3ps0c37ncbr3c8dh0000gn/T/tmp_5o756gr/precomputed/catmaid_neuropils/mesh/segment_properties\n",
      "creating: /private/var/folders/_l/lrfvj_8j3ps0c37ncbr3c8dh0000gn/T/tmp_5o756gr/precomputed/catmaid_neuropils/mesh/segment_names\n",
      "flywire url at: https://ngl.flywire.ai/?json_url=https://globalv1.flywire-daf.com/nglstate/6708937697001472\n"
     ]
    }
   ],
   "source": [
    "tmpviewer = add_flywirelayer(flywireurl2dict(shorturl), layer_kws = {'type': 'volumes',\n",
    "                                                                     'source': [flywirevol['v14.neuropil'],\n",
    "                                                                                flywirevol['AL_R']],\n",
    "                                                                     'name': 'catmaid_neuropils',\n",
    "                                                                     'color': ['grey', 'magenta'], \n",
    "                                                                     'alpha': 0.3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "shorturl = tmpviewer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "layout = {'type':'xy-3d','orthographicProjection':'True'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "flywire url at: https://ngl.flywire.ai/?json_url=https://globalv1.flywire-daf.com/nglstate/4984286184734720\n"
     ]
    }
   ],
   "source": [
    "tmpviewer,shorturl = set_flywireviewerstate(shorturl, axis_lines=False, bounding_box=False, layout = layout)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Screenshot of the flywire instance"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### The grey mesh is the v14.1 neuropil, magenta is the Antenna lobe (Right), you can see the skeletons from CATMAID (red) plotted in the skeleton layer, and some similar neurons (meshes) of the flywire dataset plotted in green and yellow"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "![alt text](pics/catmaid2flywire.png \"catmaid2flywire\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add synapses and other annotations to flywire layers (not yet working)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ngdict = flywireurl2dict(shorturl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmpviewer = add_flywirelayer(ngdict, layer_kws = {'type': 'synapses','source': flywire_neuron,\n",
    "                                                 'ngspace': 'FAFBv14.1'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "temp_pts = pd.DataFrame([[123072, 47001, 3375]],columns=['x','y','z'])\n",
    "temp_pts = pd.DataFrame([[123072, 47001, 3375], [120000, 17001, 3000]], columns=['x', 'y', 'z'])\n",
    "temp_pts['description'] = ['center_pt','above_pt']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plot landmarks..\n",
    "tmpviewer = add_flywirelayer(ngdict, layer_kws = {'type': 'points','name': 'landmarks',\n",
    "                                                  'ngspace': 'FAFBv14.1',\n",
    "                                                  'source': temp_pts,'color': 'red'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmpviewer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Close the dataserver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Closing server at http://127.0.0.1:8000\n",
      "Cleaning directory at /private/var/folders/_l/lrfvj_8j3ps0c37ncbr3c8dh0000gn/T/tmp_5o756gr\n"
     ]
    }
   ],
   "source": [
    "closedataserver()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}