00_core.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# default_exp core"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# ONi csv processor\n",
    "\n",
    "> Read localizations .csv from oxford nanoimager ONi and work with it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#hide\n",
    "from nbdev.showdoc import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import vaex as v\n",
    "\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]full_truncated.csv\n",
      "[1]long_truncated.csv\n",
      "[2]short_truncated.csv\n"
     ]
    }
   ],
   "source": [
    "PATH=\"test_csvs/\"\n",
    "\n",
    "afiles = os.listdir(PATH)\n",
    "\n",
    "files=[]\n",
    "for i in afiles:\n",
    "    if '.csv' in i:\n",
    "        files.append(i)\n",
    "        \n",
    "for i in range(len(files)):\n",
    "    print('['+str(i)+']'+ files[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# csvs were saved with unnecessary 'Unnamed: 0' column, read without it.\n",
    "full = pd.read_csv(PATH + files[0]).drop('Unnamed: 0', axis=1)\n",
    "long = pd.read_csv(PATH + files[1]).drop('Unnamed: 0', axis=1)\n",
    "short = pd.read_csv(PATH + files[2]).drop('Unnamed: 0', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Channel', 'Frame', 'X (nm)', 'Y (nm)', 'Z (nm)', 'X precision (nm)',\n",
      "       'Y precision (nm)', 'X (pix)', 'Y (pix)', 'Z (pix)',\n",
      "       'X precision (pix)', 'Y precision (pix)', 'Photons', 'Background',\n",
      "       'PSF Sigma X (pix)', 'PSF Sigma Y (pix)', 'Sigma X var', 'Sigma Y var'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Channel</th>\n",
       "      <th>Frame</th>\n",
       "      <th>X (nm)</th>\n",
       "      <th>Y (nm)</th>\n",
       "      <th>Z (nm)</th>\n",
       "      <th>X precision (nm)</th>\n",
       "      <th>Y precision (nm)</th>\n",
       "      <th>X (pix)</th>\n",
       "      <th>Y (pix)</th>\n",
       "      <th>Z (pix)</th>\n",
       "      <th>X precision (pix)</th>\n",
       "      <th>Y precision (pix)</th>\n",
       "      <th>Photons</th>\n",
       "      <th>Background</th>\n",
       "      <th>PSF Sigma X (pix)</th>\n",
       "      <th>PSF Sigma Y (pix)</th>\n",
       "      <th>Sigma X var</th>\n",
       "      <th>Sigma Y var</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>21573.433594</td>\n",
       "      <td>2105.305420</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.211508</td>\n",
       "      <td>29.349554</td>\n",
       "      <td>184.388321</td>\n",
       "      <td>17.994064</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.206936</td>\n",
       "      <td>0.250851</td>\n",
       "      <td>259.793915</td>\n",
       "      <td>195.626770</td>\n",
       "      <td>0.755919</td>\n",
       "      <td>0.923537</td>\n",
       "      <td>0.093245</td>\n",
       "      <td>0.104666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>41139.273438</td>\n",
       "      <td>2168.957764</td>\n",
       "      <td>0.0</td>\n",
       "      <td>33.111080</td>\n",
       "      <td>25.369190</td>\n",
       "      <td>351.617737</td>\n",
       "      <td>18.538101</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.283001</td>\n",
       "      <td>0.216831</td>\n",
       "      <td>5430.423340</td>\n",
       "      <td>168.029678</td>\n",
       "      <td>2.791028</td>\n",
       "      <td>2.491404</td>\n",
       "      <td>3.739474</td>\n",
       "      <td>2.983588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>42598.929688</td>\n",
       "      <td>3193.350586</td>\n",
       "      <td>0.0</td>\n",
       "      <td>20.898561</td>\n",
       "      <td>15.947609</td>\n",
       "      <td>364.093414</td>\n",
       "      <td>27.293594</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.178620</td>\n",
       "      <td>0.136304</td>\n",
       "      <td>1709.374146</td>\n",
       "      <td>231.349258</td>\n",
       "      <td>1.795385</td>\n",
       "      <td>1.443872</td>\n",
       "      <td>0.347870</td>\n",
       "      <td>0.268366</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>17339.099609</td>\n",
       "      <td>4047.465088</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.807699</td>\n",
       "      <td>16.724777</td>\n",
       "      <td>148.197433</td>\n",
       "      <td>34.593719</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.160750</td>\n",
       "      <td>0.142947</td>\n",
       "      <td>747.525696</td>\n",
       "      <td>213.677063</td>\n",
       "      <td>1.134157</td>\n",
       "      <td>1.078705</td>\n",
       "      <td>0.062254</td>\n",
       "      <td>0.045783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38394.988281</td>\n",
       "      <td>4650.355469</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.195688</td>\n",
       "      <td>21.187086</td>\n",
       "      <td>328.162292</td>\n",
       "      <td>39.746628</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.266630</td>\n",
       "      <td>0.181086</td>\n",
       "      <td>383.547546</td>\n",
       "      <td>254.557709</td>\n",
       "      <td>1.162104</td>\n",
       "      <td>0.764033</td>\n",
       "      <td>0.116336</td>\n",
       "      <td>0.062114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>0</td>\n",
       "      <td>273</td>\n",
       "      <td>13216.037109</td>\n",
       "      <td>17281.587891</td>\n",
       "      <td>0.0</td>\n",
       "      <td>21.525955</td>\n",
       "      <td>26.180573</td>\n",
       "      <td>112.957581</td>\n",
       "      <td>147.705872</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.183983</td>\n",
       "      <td>0.223766</td>\n",
       "      <td>993.084961</td>\n",
       "      <td>346.975739</td>\n",
       "      <td>1.257754</td>\n",
       "      <td>1.448853</td>\n",
       "      <td>0.250722</td>\n",
       "      <td>0.306024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>0</td>\n",
       "      <td>273</td>\n",
       "      <td>18898.595703</td>\n",
       "      <td>17573.136719</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.859333</td>\n",
       "      <td>13.248240</td>\n",
       "      <td>161.526459</td>\n",
       "      <td>150.197754</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.152644</td>\n",
       "      <td>0.113233</td>\n",
       "      <td>810.720825</td>\n",
       "      <td>354.611847</td>\n",
       "      <td>1.121001</td>\n",
       "      <td>0.833053</td>\n",
       "      <td>0.044136</td>\n",
       "      <td>0.028074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>0</td>\n",
       "      <td>273</td>\n",
       "      <td>23631.041016</td>\n",
       "      <td>17587.605469</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.528518</td>\n",
       "      <td>12.263875</td>\n",
       "      <td>201.974716</td>\n",
       "      <td>150.321411</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.089987</td>\n",
       "      <td>0.104819</td>\n",
       "      <td>802.625549</td>\n",
       "      <td>346.061493</td>\n",
       "      <td>0.748456</td>\n",
       "      <td>0.885574</td>\n",
       "      <td>0.011437</td>\n",
       "      <td>0.016879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>0</td>\n",
       "      <td>273</td>\n",
       "      <td>12578.090820</td>\n",
       "      <td>17767.861328</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.902623</td>\n",
       "      <td>15.916443</td>\n",
       "      <td>107.505051</td>\n",
       "      <td>151.862061</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.076091</td>\n",
       "      <td>0.136038</td>\n",
       "      <td>702.948486</td>\n",
       "      <td>347.657440</td>\n",
       "      <td>0.580553</td>\n",
       "      <td>1.051115</td>\n",
       "      <td>0.032322</td>\n",
       "      <td>0.027976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>0</td>\n",
       "      <td>273</td>\n",
       "      <td>19592.984375</td>\n",
       "      <td>17779.855469</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.485478</td>\n",
       "      <td>7.871193</td>\n",
       "      <td>167.461411</td>\n",
       "      <td>151.964569</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.072525</td>\n",
       "      <td>0.067275</td>\n",
       "      <td>1612.821289</td>\n",
       "      <td>364.054413</td>\n",
       "      <td>1.000615</td>\n",
       "      <td>0.934429</td>\n",
       "      <td>0.010802</td>\n",
       "      <td>0.009923</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 18 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Channel  Frame        X (nm)        Y (nm)  Z (nm)  X precision (nm)  \\\n",
       "0            0      0  21573.433594   2105.305420     0.0         24.211508   \n",
       "1            0      0  41139.273438   2168.957764     0.0         33.111080   \n",
       "2            0      0  42598.929688   3193.350586     0.0         20.898561   \n",
       "3            0      0  17339.099609   4047.465088     0.0         18.807699   \n",
       "4            0      0  38394.988281   4650.355469     0.0         31.195688   \n",
       "...        ...    ...           ...           ...     ...               ...   \n",
       "99995        0    273  13216.037109  17281.587891     0.0         21.525955   \n",
       "99996        0    273  18898.595703  17573.136719     0.0         17.859333   \n",
       "99997        0    273  23631.041016  17587.605469     0.0         10.528518   \n",
       "99998        0    273  12578.090820  17767.861328     0.0          8.902623   \n",
       "99999        0    273  19592.984375  17779.855469     0.0          8.485478   \n",
       "\n",
       "       Y precision (nm)     X (pix)     Y (pix)  Z (pix)  X precision (pix)  \\\n",
       "0             29.349554  184.388321   17.994064      0.0           0.206936   \n",
       "1             25.369190  351.617737   18.538101      0.0           0.283001   \n",
       "2             15.947609  364.093414   27.293594      0.0           0.178620   \n",
       "3             16.724777  148.197433   34.593719      0.0           0.160750   \n",
       "4             21.187086  328.162292   39.746628      0.0           0.266630   \n",
       "...                 ...         ...         ...      ...                ...   \n",
       "99995         26.180573  112.957581  147.705872      0.0           0.183983   \n",
       "99996         13.248240  161.526459  150.197754      0.0           0.152644   \n",
       "99997         12.263875  201.974716  150.321411      0.0           0.089987   \n",
       "99998         15.916443  107.505051  151.862061      0.0           0.076091   \n",
       "99999          7.871193  167.461411  151.964569      0.0           0.072525   \n",
       "\n",
       "       Y precision (pix)      Photons  Background  PSF Sigma X (pix)  \\\n",
       "0               0.250851   259.793915  195.626770           0.755919   \n",
       "1               0.216831  5430.423340  168.029678           2.791028   \n",
       "2               0.136304  1709.374146  231.349258           1.795385   \n",
       "3               0.142947   747.525696  213.677063           1.134157   \n",
       "4               0.181086   383.547546  254.557709           1.162104   \n",
       "...                  ...          ...         ...                ...   \n",
       "99995           0.223766   993.084961  346.975739           1.257754   \n",
       "99996           0.113233   810.720825  354.611847           1.121001   \n",
       "99997           0.104819   802.625549  346.061493           0.748456   \n",
       "99998           0.136038   702.948486  347.657440           0.580553   \n",
       "99999           0.067275  1612.821289  364.054413           1.000615   \n",
       "\n",
       "       PSF Sigma Y (pix)  Sigma X var  Sigma Y var  \n",
       "0               0.923537     0.093245     0.104666  \n",
       "1               2.491404     3.739474     2.983588  \n",
       "2               1.443872     0.347870     0.268366  \n",
       "3               1.078705     0.062254     0.045783  \n",
       "4               0.764033     0.116336     0.062114  \n",
       "...                  ...          ...          ...  \n",
       "99995           1.448853     0.250722     0.306024  \n",
       "99996           0.833053     0.044136     0.028074  \n",
       "99997           0.885574     0.011437     0.016879  \n",
       "99998           1.051115     0.032322     0.027976  \n",
       "99999           0.934429     0.010802     0.009923  \n",
       "\n",
       "[100000 rows x 18 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# full column set from ONi localizations export, includes all possible parameters for each localization\n",
    "print(full.columns)\n",
    "full"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Channel', 'Frame', 'X (nm)', 'Y (nm)', 'Z (nm)', 'X precision (nm)',\n",
      "       'Y precision (nm)', 'X (pix)', 'Y (pix)', 'Z (pix)',\n",
      "       'X precision (pix)', 'Y precision (pix)', 'Photons', 'Background',\n",
      "       'PSF Sigma X (pix)', 'PSF Sigma Y (pix)'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Channel</th>\n",
       "      <th>Frame</th>\n",
       "      <th>X (nm)</th>\n",
       "      <th>Y (nm)</th>\n",
       "      <th>Z (nm)</th>\n",
       "      <th>X precision (nm)</th>\n",
       "      <th>Y precision (nm)</th>\n",
       "      <th>X (pix)</th>\n",
       "      <th>Y (pix)</th>\n",
       "      <th>Z (pix)</th>\n",
       "      <th>X precision (pix)</th>\n",
       "      <th>Y precision (pix)</th>\n",
       "      <th>Photons</th>\n",
       "      <th>Background</th>\n",
       "      <th>PSF Sigma X (pix)</th>\n",
       "      <th>PSF Sigma Y (pix)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>20523.187500</td>\n",
       "      <td>311.265076</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.973742</td>\n",
       "      <td>12.443872</td>\n",
       "      <td>175.411865</td>\n",
       "      <td>2.660385</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.110887</td>\n",
       "      <td>0.106358</td>\n",
       "      <td>195.261139</td>\n",
       "      <td>67.741257</td>\n",
       "      <td>0.615442</td>\n",
       "      <td>0.483970</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>25530.263672</td>\n",
       "      <td>286.637146</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.987727</td>\n",
       "      <td>21.506376</td>\n",
       "      <td>218.207382</td>\n",
       "      <td>2.449890</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.136647</td>\n",
       "      <td>0.183815</td>\n",
       "      <td>248.447495</td>\n",
       "      <td>53.025429</td>\n",
       "      <td>0.746685</td>\n",
       "      <td>1.031305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>49439.746094</td>\n",
       "      <td>1198.334106</td>\n",
       "      <td>0.0</td>\n",
       "      <td>18.197208</td>\n",
       "      <td>16.926477</td>\n",
       "      <td>422.561920</td>\n",
       "      <td>10.242172</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.155532</td>\n",
       "      <td>0.144671</td>\n",
       "      <td>177.823624</td>\n",
       "      <td>29.500187</td>\n",
       "      <td>0.836590</td>\n",
       "      <td>0.760550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>27679.429688</td>\n",
       "      <td>1668.451538</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10.311979</td>\n",
       "      <td>12.141504</td>\n",
       "      <td>236.576324</td>\n",
       "      <td>14.260269</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.088137</td>\n",
       "      <td>0.103774</td>\n",
       "      <td>249.217850</td>\n",
       "      <td>68.126373</td>\n",
       "      <td>0.562145</td>\n",
       "      <td>0.663490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>28931.167969</td>\n",
       "      <td>1842.923218</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.982113</td>\n",
       "      <td>55.536037</td>\n",
       "      <td>247.274948</td>\n",
       "      <td>15.751480</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.222069</td>\n",
       "      <td>0.474667</td>\n",
       "      <td>272.386047</td>\n",
       "      <td>58.780933</td>\n",
       "      <td>1.009058</td>\n",
       "      <td>1.375971</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>0</td>\n",
       "      <td>248</td>\n",
       "      <td>33962.218750</td>\n",
       "      <td>8102.107422</td>\n",
       "      <td>0.0</td>\n",
       "      <td>24.376728</td>\n",
       "      <td>18.027479</td>\n",
       "      <td>290.275360</td>\n",
       "      <td>69.248779</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.208348</td>\n",
       "      <td>0.154081</td>\n",
       "      <td>361.813446</td>\n",
       "      <td>67.260818</td>\n",
       "      <td>1.265213</td>\n",
       "      <td>0.955205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>0</td>\n",
       "      <td>248</td>\n",
       "      <td>6915.722656</td>\n",
       "      <td>8277.325195</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14.814459</td>\n",
       "      <td>37.465008</td>\n",
       "      <td>59.108742</td>\n",
       "      <td>70.746368</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.126619</td>\n",
       "      <td>0.320214</td>\n",
       "      <td>781.411133</td>\n",
       "      <td>73.695068</td>\n",
       "      <td>1.156711</td>\n",
       "      <td>1.909840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>0</td>\n",
       "      <td>248</td>\n",
       "      <td>27743.181641</td>\n",
       "      <td>8566.104492</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.744524</td>\n",
       "      <td>12.352351</td>\n",
       "      <td>237.121216</td>\n",
       "      <td>73.214569</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.108928</td>\n",
       "      <td>0.105576</td>\n",
       "      <td>754.584412</td>\n",
       "      <td>60.801632</td>\n",
       "      <td>1.239387</td>\n",
       "      <td>1.200127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>0</td>\n",
       "      <td>248</td>\n",
       "      <td>27464.937500</td>\n",
       "      <td>9160.863281</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.259060</td>\n",
       "      <td>12.362824</td>\n",
       "      <td>234.743057</td>\n",
       "      <td>78.297974</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.113325</td>\n",
       "      <td>0.105665</td>\n",
       "      <td>492.055298</td>\n",
       "      <td>65.740189</td>\n",
       "      <td>1.023156</td>\n",
       "      <td>0.951723</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>0</td>\n",
       "      <td>248</td>\n",
       "      <td>29643.779297</td>\n",
       "      <td>9491.146484</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.973066</td>\n",
       "      <td>24.521162</td>\n",
       "      <td>253.365631</td>\n",
       "      <td>81.120911</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.273274</td>\n",
       "      <td>0.209583</td>\n",
       "      <td>373.610321</td>\n",
       "      <td>73.098152</td>\n",
       "      <td>1.344178</td>\n",
       "      <td>1.163003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 16 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Channel  Frame        X (nm)       Y (nm)  Z (nm)  X precision (nm)  \\\n",
       "0            0      0  20523.187500   311.265076     0.0         12.973742   \n",
       "1            0      0  25530.263672   286.637146     0.0         15.987727   \n",
       "2            0      0  49439.746094  1198.334106     0.0         18.197208   \n",
       "3            0      0  27679.429688  1668.451538     0.0         10.311979   \n",
       "4            0      0  28931.167969  1842.923218     0.0         25.982113   \n",
       "...        ...    ...           ...          ...     ...               ...   \n",
       "99995        0    248  33962.218750  8102.107422     0.0         24.376728   \n",
       "99996        0    248   6915.722656  8277.325195     0.0         14.814459   \n",
       "99997        0    248  27743.181641  8566.104492     0.0         12.744524   \n",
       "99998        0    248  27464.937500  9160.863281     0.0         13.259060   \n",
       "99999        0    248  29643.779297  9491.146484     0.0         31.973066   \n",
       "\n",
       "       Y precision (nm)     X (pix)    Y (pix)  Z (pix)  X precision (pix)  \\\n",
       "0             12.443872  175.411865   2.660385      0.0           0.110887   \n",
       "1             21.506376  218.207382   2.449890      0.0           0.136647   \n",
       "2             16.926477  422.561920  10.242172      0.0           0.155532   \n",
       "3             12.141504  236.576324  14.260269      0.0           0.088137   \n",
       "4             55.536037  247.274948  15.751480      0.0           0.222069   \n",
       "...                 ...         ...        ...      ...                ...   \n",
       "99995         18.027479  290.275360  69.248779      0.0           0.208348   \n",
       "99996         37.465008   59.108742  70.746368      0.0           0.126619   \n",
       "99997         12.352351  237.121216  73.214569      0.0           0.108928   \n",
       "99998         12.362824  234.743057  78.297974      0.0           0.113325   \n",
       "99999         24.521162  253.365631  81.120911      0.0           0.273274   \n",
       "\n",
       "       Y precision (pix)     Photons  Background  PSF Sigma X (pix)  \\\n",
       "0               0.106358  195.261139   67.741257           0.615442   \n",
       "1               0.183815  248.447495   53.025429           0.746685   \n",
       "2               0.144671  177.823624   29.500187           0.836590   \n",
       "3               0.103774  249.217850   68.126373           0.562145   \n",
       "4               0.474667  272.386047   58.780933           1.009058   \n",
       "...                  ...         ...         ...                ...   \n",
       "99995           0.154081  361.813446   67.260818           1.265213   \n",
       "99996           0.320214  781.411133   73.695068           1.156711   \n",
       "99997           0.105576  754.584412   60.801632           1.239387   \n",
       "99998           0.105665  492.055298   65.740189           1.023156   \n",
       "99999           0.209583  373.610321   73.098152           1.344178   \n",
       "\n",
       "       PSF Sigma Y (pix)  \n",
       "0               0.483970  \n",
       "1               1.031305  \n",
       "2               0.760550  \n",
       "3               0.663490  \n",
       "4               1.375971  \n",
       "...                  ...  \n",
       "99995           0.955205  \n",
       "99996           1.909840  \n",
       "99997           1.200127  \n",
       "99998           0.951723  \n",
       "99999           1.163003  \n",
       "\n",
       "[100000 rows x 16 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# long column set, includes all parameters for localizations except 'Sigma X var' and 'Sigma Y var'\n",
    "print(long.columns)\n",
    "long"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['Channel', 'Frame', 'X (nm)', 'Y (nm)', 'Z (nm)', 'Photons',\n",
      "       'Background'],\n",
      "      dtype='object')\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Channel</th>\n",
       "      <th>Frame</th>\n",
       "      <th>X (nm)</th>\n",
       "      <th>Y (nm)</th>\n",
       "      <th>Z (nm)</th>\n",
       "      <th>Photons</th>\n",
       "      <th>Background</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>934.454285</td>\n",
       "      <td>4771.859863</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3851.816406</td>\n",
       "      <td>1.189330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11303.127930</td>\n",
       "      <td>24269.955078</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14689.367188</td>\n",
       "      <td>10.747117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>11340.823242</td>\n",
       "      <td>25342.900391</td>\n",
       "      <td>0.0</td>\n",
       "      <td>799.065002</td>\n",
       "      <td>158.214005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>38650.664063</td>\n",
       "      <td>38161.367188</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4679.799316</td>\n",
       "      <td>289.868988</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>40417.269531</td>\n",
       "      <td>38402.472656</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6161.399414</td>\n",
       "      <td>343.919800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99995</th>\n",
       "      <td>0</td>\n",
       "      <td>4394</td>\n",
       "      <td>20608.373047</td>\n",
       "      <td>67361.250000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>844.858398</td>\n",
       "      <td>420.960754</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99996</th>\n",
       "      <td>0</td>\n",
       "      <td>4394</td>\n",
       "      <td>19592.402344</td>\n",
       "      <td>68234.210938</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4496.629883</td>\n",
       "      <td>286.038605</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99997</th>\n",
       "      <td>0</td>\n",
       "      <td>4395</td>\n",
       "      <td>44750.890625</td>\n",
       "      <td>38707.031250</td>\n",
       "      <td>0.0</td>\n",
       "      <td>964.338623</td>\n",
       "      <td>161.554016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99998</th>\n",
       "      <td>0</td>\n",
       "      <td>4395</td>\n",
       "      <td>37233.082031</td>\n",
       "      <td>39393.871094</td>\n",
       "      <td>0.0</td>\n",
       "      <td>460.955048</td>\n",
       "      <td>445.787323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99999</th>\n",
       "      <td>0</td>\n",
       "      <td>4395</td>\n",
       "      <td>42169.113281</td>\n",
       "      <td>39793.804688</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1696.422363</td>\n",
       "      <td>348.532654</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>100000 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Channel  Frame        X (nm)        Y (nm)  Z (nm)       Photons  \\\n",
       "0            0      0    934.454285   4771.859863     0.0   3851.816406   \n",
       "1            0      0  11303.127930  24269.955078     0.0  14689.367188   \n",
       "2            0      0  11340.823242  25342.900391     0.0    799.065002   \n",
       "3            0      0  38650.664063  38161.367188     0.0   4679.799316   \n",
       "4            0      0  40417.269531  38402.472656     0.0   6161.399414   \n",
       "...        ...    ...           ...           ...     ...           ...   \n",
       "99995        0   4394  20608.373047  67361.250000     0.0    844.858398   \n",
       "99996        0   4394  19592.402344  68234.210938     0.0   4496.629883   \n",
       "99997        0   4395  44750.890625  38707.031250     0.0    964.338623   \n",
       "99998        0   4395  37233.082031  39393.871094     0.0    460.955048   \n",
       "99999        0   4395  42169.113281  39793.804688     0.0   1696.422363   \n",
       "\n",
       "       Background  \n",
       "0        1.189330  \n",
       "1       10.747117  \n",
       "2      158.214005  \n",
       "3      289.868988  \n",
       "4      343.919800  \n",
       "...           ...  \n",
       "99995  420.960754  \n",
       "99996  286.038605  \n",
       "99997  161.554016  \n",
       "99998  445.787323  \n",
       "99999  348.532654  \n",
       "\n",
       "[100000 rows x 7 columns]"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# short column set, includes only Channel, Frame, X/Y/Z in nm, Photons and Background\n",
    "print(short.columns)\n",
    "short"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# need to rename columns with space in names. Vaex can't correctly read data from pandas columns including space in names.\n",
    "# So we remove all spaces in names (and brackets just in case).\n",
    "vat = long.rename(columns={'X (nm)':'Xnm', 'Y (nm)':'Ynm', 'Z (nm)':'Znm', 'X precision (nm)':'Xprecisionnm',\n",
    "       'Y precision (nm)':'Yprecisionnm', 'X (pix)':'Xpix', 'Y (pix)':'Ypix', 'Z (pix)':'Zpix',\n",
    "       'X precision (pix)':'Xprecisionpix', 'Y precision (pix)':'Yprecisionpix',\n",
    "       'PSF Sigma X (pix)':'PSFSigmaXpix', 'PSF Sigma Y (pix)':'PSFSigmaYpix'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read data from pandas dataframe, make vaex fast dataframe\n",
    "faster = v.from_pandas(vat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead>\n",
       "<tr><th>#                                 </th><th>Channel  </th><th>Frame  </th><th>Xnm               </th><th>Ynm        </th><th>Znm  </th><th>Xprecisionnm  </th><th>Yprecisionnm      </th><th>Xpix              </th><th>Ypix              </th><th>Zpix  </th><th>Xprecisionpix     </th><th>Yprecisionpix  </th><th>Photons           </th><th>Background       </th><th>PSFSigmaXpix  </th><th>PSFSigmaYpix      </th></tr>\n",
       "</thead>\n",
       "<tbody>\n",
       "<tr><td><i style='opacity: 0.6'>0</i>     </td><td>0        </td><td>0      </td><td>20523.1875        </td><td>311.265076 </td><td>0.0  </td><td>12.973742     </td><td>12.443872         </td><td>175.41186499999998</td><td>2.660385          </td><td>0.0   </td><td>0.1108869999999999</td><td>0.106358       </td><td>195.261139        </td><td>67.74125699999999</td><td>0.615442      </td><td>0.4839699999999999</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>1</i>     </td><td>0        </td><td>0      </td><td>25530.263672      </td><td>286.637146 </td><td>0.0  </td><td>15.987727     </td><td>21.506376         </td><td>218.207382        </td><td>2.4498900000000003</td><td>0.0   </td><td>0.136647          </td><td>0.183815       </td><td>248.44749500000003</td><td>53.025429        </td><td>0.746685      </td><td>1.031305          </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>2</i>     </td><td>0        </td><td>0      </td><td>49439.746094      </td><td>1198.334106</td><td>0.0  </td><td>18.197208     </td><td>16.926477         </td><td>422.56192         </td><td>10.242172         </td><td>0.0   </td><td>0.155532          </td><td>0.144671       </td><td>177.823624        </td><td>29.500187        </td><td>0.83659       </td><td>0.76055           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>3</i>     </td><td>0        </td><td>0      </td><td>27679.429688      </td><td>1668.451538</td><td>0.0  </td><td>10.311979     </td><td>12.141504         </td><td>236.57632400000003</td><td>14.260269         </td><td>0.0   </td><td>0.088137          </td><td>0.103774       </td><td>249.21785         </td><td>68.126373        </td><td>0.562145      </td><td>0.6634899999999999</td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>4</i>     </td><td>0        </td><td>0      </td><td>28931.167969      </td><td>1842.923218</td><td>0.0  </td><td>25.982113     </td><td>55.536037         </td><td>247.274948        </td><td>15.75148          </td><td>0.0   </td><td>0.222069          </td><td>0.474667       </td><td>272.386047        </td><td>58.780933        </td><td>1.009058      </td><td>1.3759709999999998</td></tr>\n",
       "<tr><td>...                               </td><td>...      </td><td>...    </td><td>...               </td><td>...        </td><td>...  </td><td>...           </td><td>...               </td><td>...               </td><td>...               </td><td>...   </td><td>...               </td><td>...            </td><td>...               </td><td>...              </td><td>...           </td><td>...               </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,995</i></td><td>0        </td><td>248    </td><td>33962.21875       </td><td>8102.107422</td><td>0.0  </td><td>24.376728     </td><td>18.027479         </td><td>290.27536000000003</td><td>69.248779         </td><td>0.0   </td><td>0.208348          </td><td>0.154081       </td><td>361.813446        </td><td>67.260818        </td><td>1.265213      </td><td>0.955205          </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,996</i></td><td>0        </td><td>248    </td><td>6915.722656       </td><td>8277.325195</td><td>0.0  </td><td>14.814459     </td><td>37.465008000000005</td><td>59.10874200000001 </td><td>70.746368         </td><td>0.0   </td><td>0.126619          </td><td>0.320214       </td><td>781.4111330000001 </td><td>73.695068        </td><td>1.156711      </td><td>1.90984           </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,997</i></td><td>0        </td><td>248    </td><td>27743.181641000003</td><td>8566.104492</td><td>0.0  </td><td>12.744524     </td><td>12.352351         </td><td>237.121216        </td><td>73.214569         </td><td>0.0   </td><td>0.108928          </td><td>0.105576       </td><td>754.584412        </td><td>60.801632        </td><td>1.239387      </td><td>1.200127          </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,998</i></td><td>0        </td><td>248    </td><td>27464.9375        </td><td>9160.863281</td><td>0.0  </td><td>13.25906      </td><td>12.362824         </td><td>234.743057        </td><td>78.297974         </td><td>0.0   </td><td>0.113325          </td><td>0.105665       </td><td>492.0552980000001 </td><td>65.740189        </td><td>1.023156      </td><td>0.951723          </td></tr>\n",
       "<tr><td><i style='opacity: 0.6'>99,999</i></td><td>0        </td><td>248    </td><td>29643.779297000005</td><td>9491.146484</td><td>0.0  </td><td>31.973066     </td><td>24.521162         </td><td>253.365631        </td><td>81.12091099999999 </td><td>0.0   </td><td>0.2732739999999999</td><td>0.209583       </td><td>373.610321        </td><td>73.098152        </td><td>1.344178      </td><td>1.163003          </td></tr>\n",
       "</tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "#       Channel    Frame    Xnm                 Ynm          Znm    Xprecisionnm    Yprecisionnm        Xpix                Ypix                Zpix    Xprecisionpix       Yprecisionpix    Photons             Background         PSFSigmaXpix    PSFSigmaYpix\n",
       "0       0          0        20523.1875          311.265076   0.0    12.973742       12.443872           175.41186499999998  2.660385            0.0     0.1108869999999999  0.106358         195.261139          67.74125699999999  0.615442        0.4839699999999999\n",
       "1       0          0        25530.263672        286.637146   0.0    15.987727       21.506376           218.207382          2.4498900000000003  0.0     0.136647            0.183815         248.44749500000003  53.025429          0.746685        1.031305\n",
       "2       0          0        49439.746094        1198.334106  0.0    18.197208       16.926477           422.56192           10.242172           0.0     0.155532            0.144671         177.823624          29.500187          0.83659         0.76055\n",
       "3       0          0        27679.429688        1668.451538  0.0    10.311979       12.141504           236.57632400000003  14.260269           0.0     0.088137            0.103774         249.21785           68.126373          0.562145        0.6634899999999999\n",
       "4       0          0        28931.167969        1842.923218  0.0    25.982113       55.536037           247.274948          15.75148            0.0     0.222069            0.474667         272.386047          58.780933          1.009058        1.3759709999999998\n",
       "...     ...        ...      ...                 ...          ...    ...             ...                 ...                 ...                 ...     ...                 ...              ...                 ...                ...             ...\n",
       "99,995  0          248      33962.21875         8102.107422  0.0    24.376728       18.027479           290.27536000000003  69.248779           0.0     0.208348            0.154081         361.813446          67.260818          1.265213        0.955205\n",
       "99,996  0          248      6915.722656         8277.325195  0.0    14.814459       37.465008000000005  59.10874200000001   70.746368           0.0     0.126619            0.320214         781.4111330000001   73.695068          1.156711        1.90984\n",
       "99,997  0          248      27743.181641000003  8566.104492  0.0    12.744524       12.352351           237.121216          73.214569           0.0     0.108928            0.105576         754.584412          60.801632          1.239387        1.200127\n",
       "99,998  0          248      27464.9375          9160.863281  0.0    13.25906        12.362824           234.743057          78.297974           0.0     0.113325            0.105665         492.0552980000001   65.740189          1.023156        0.951723\n",
       "99,999  0          248      29643.779297000005  9491.146484  0.0    31.973066       24.521162           253.365631          81.12091099999999   0.0     0.2732739999999999  0.209583         373.610321          73.098152          1.344178        1.163003"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "faster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b429f310dd049a9bb29e17c7863d444",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Heatmap(children=[ToolsToolbar(interact_value=None, supports_normalize=False, template='<template>\\n  <v-toolb…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# widget heatmap that plots our localizations. log1p transform need for correct intensity scaling. \n",
    "faster.widget.heatmap(faster['Ypix'], faster['Xpix'], transform='log1p', shape=400)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}