Feature Calculation Example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "np.set_printoptions(threshold=sys.maxsize)\n",
    "import pandas as pd\n",
    "sys.path.append('python')\n",
    "import keplerml as fc # keplerml is a feature calculator, fc for short."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Features can be generated from the terminal using the following command:\n",
    "```\n",
    "python keplerml.py data/filelists/ex_filelist.txt data/lightcurves/ data/output/Example_output.p \n",
    "```\n",
    "This requires a Python 3+ to be the default version, replace `python` with an appropriate version if this is not the case, for example, use `python3.7`\n",
    "\n",
    "The above terminal command is equivalent to the following:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading ./data/filelists/ex_filelist.txt...\n",
      "Using 47 cpus to calculate features...\n",
      "Importing 247 lightcurves...\n",
      "Lightcurve import took 0:00:01.129264\n",
      "Processing 247 files...\n",
      "247/247 completed. Time for chunk: 0:01:06.296283\n",
      "Features have been calculated, total time to calculate features: 0:01:06.308606\n",
      "Saving output to data/output/Example_output.p\n",
      "Cleaning up...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "path_to_filelist = './data/filelists/ex_filelist.txt'\n",
    "path_to_fits = './data/lightcurves/'\n",
    "output_file = 'data/output/Example_output.p'\n",
    "features = fc.features_from_filelist(path_to_filelist,path_to_fits,output_file,fl_as_array=False,verbose=True,prime_feats=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Features are returned in a Pandas DataFrame, and saved to the specified output file as a pickled dataframe, which can be read in using the pickle module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longtermtrend</th>\n",
       "      <th>meanmedrat</th>\n",
       "      <th>skews</th>\n",
       "      <th>varss</th>\n",
       "      <th>coeffvar</th>\n",
       "      <th>stds</th>\n",
       "      <th>numoutliers</th>\n",
       "      <th>numnegoutliers</th>\n",
       "      <th>numposoutliers</th>\n",
       "      <th>numout1s</th>\n",
       "      <th>...</th>\n",
       "      <th>percentamp</th>\n",
       "      <th>magratio</th>\n",
       "      <th>sautocorrcoef</th>\n",
       "      <th>autocorrcoef</th>\n",
       "      <th>flatmean</th>\n",
       "      <th>tflatmean</th>\n",
       "      <th>roundmean</th>\n",
       "      <th>troundmean</th>\n",
       "      <th>roundrat</th>\n",
       "      <th>flatrat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kplr001026032-2011271113734_llc.fits</th>\n",
       "      <td>-6.415193e-07</td>\n",
       "      <td>0.998781</td>\n",
       "      <td>-7.109529</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>0.008196</td>\n",
       "      <td>0.008186</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>163</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086309</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.742404</td>\n",
       "      <td>0.937333</td>\n",
       "      <td>0.034865</td>\n",
       "      <td>0.108202</td>\n",
       "      <td>-0.723083</td>\n",
       "      <td>0.031835</td>\n",
       "      <td>-22.713754</td>\n",
       "      <td>0.322218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kplr001026957-2011271113734_llc.fits</th>\n",
       "      <td>8.002484e-07</td>\n",
       "      <td>1.000915</td>\n",
       "      <td>0.396550</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>0.005688</td>\n",
       "      <td>0.005694</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1474</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013276</td>\n",
       "      <td>0.616403</td>\n",
       "      <td>-0.294359</td>\n",
       "      <td>0.999479</td>\n",
       "      <td>0.002733</td>\n",
       "      <td>0.003749</td>\n",
       "      <td>0.047433</td>\n",
       "      <td>-0.012429</td>\n",
       "      <td>-3.816354</td>\n",
       "      <td>0.728774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kplr001433962-2011271113734_llc.fits</th>\n",
       "      <td>2.901455e-07</td>\n",
       "      <td>0.998547</td>\n",
       "      <td>-0.267555</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>0.005488</td>\n",
       "      <td>0.005480</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1789</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013524</td>\n",
       "      <td>0.502052</td>\n",
       "      <td>-0.486633</td>\n",
       "      <td>0.978182</td>\n",
       "      <td>0.019154</td>\n",
       "      <td>0.018673</td>\n",
       "      <td>-0.003119</td>\n",
       "      <td>0.165303</td>\n",
       "      <td>-0.018868</td>\n",
       "      <td>1.025801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kplr001571511-2011271113734_llc.fits</th>\n",
       "      <td>-3.231966e-08</td>\n",
       "      <td>0.999617</td>\n",
       "      <td>-6.939159</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.002572</td>\n",
       "      <td>0.002571</td>\n",
       "      <td>81</td>\n",
       "      <td>81</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>...</td>\n",
       "      <td>0.021482</td>\n",
       "      <td>0.069174</td>\n",
       "      <td>0.510130</td>\n",
       "      <td>0.968878</td>\n",
       "      <td>0.007759</td>\n",
       "      <td>0.006047</td>\n",
       "      <td>-0.089364</td>\n",
       "      <td>0.196277</td>\n",
       "      <td>-0.455296</td>\n",
       "      <td>1.283076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kplr001725193-2011271113734_llc.fits</th>\n",
       "      <td>-1.793919e-06</td>\n",
       "      <td>0.998418</td>\n",
       "      <td>-3.711617</td>\n",
       "      <td>0.000052</td>\n",
       "      <td>0.007213</td>\n",
       "      <td>0.007202</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>0</td>\n",
       "      <td>259</td>\n",
       "      <td>...</td>\n",
       "      <td>0.042875</td>\n",
       "      <td>0.136161</td>\n",
       "      <td>0.789440</td>\n",
       "      <td>0.957196</td>\n",
       "      <td>0.055880</td>\n",
       "      <td>0.210136</td>\n",
       "      <td>-1.015293</td>\n",
       "      <td>0.573217</td>\n",
       "      <td>-1.771219</td>\n",
       "      <td>0.265923</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      longtermtrend  meanmedrat     skews  \\\n",
       "kplr001026032-2011271113734_llc.fits  -6.415193e-07    0.998781 -7.109529   \n",
       "kplr001026957-2011271113734_llc.fits   8.002484e-07    1.000915  0.396550   \n",
       "kplr001433962-2011271113734_llc.fits   2.901455e-07    0.998547 -0.267555   \n",
       "kplr001571511-2011271113734_llc.fits  -3.231966e-08    0.999617 -6.939159   \n",
       "kplr001725193-2011271113734_llc.fits  -1.793919e-06    0.998418 -3.711617   \n",
       "\n",
       "                                         varss  coeffvar      stds  \\\n",
       "kplr001026032-2011271113734_llc.fits  0.000067  0.008196  0.008186   \n",
       "kplr001026957-2011271113734_llc.fits  0.000032  0.005688  0.005694   \n",
       "kplr001433962-2011271113734_llc.fits  0.000030  0.005488  0.005480   \n",
       "kplr001571511-2011271113734_llc.fits  0.000007  0.002572  0.002571   \n",
       "kplr001725193-2011271113734_llc.fits  0.000052  0.007213  0.007202   \n",
       "\n",
       "                                     numoutliers numnegoutliers  \\\n",
       "kplr001026032-2011271113734_llc.fits          56             56   \n",
       "kplr001026957-2011271113734_llc.fits           0              0   \n",
       "kplr001433962-2011271113734_llc.fits           0              0   \n",
       "kplr001571511-2011271113734_llc.fits          81             81   \n",
       "kplr001725193-2011271113734_llc.fits         117            117   \n",
       "\n",
       "                                     numposoutliers numout1s  ...  percentamp  \\\n",
       "kplr001026032-2011271113734_llc.fits              0      163  ...    0.086309   \n",
       "kplr001026957-2011271113734_llc.fits              0     1474  ...    0.013276   \n",
       "kplr001433962-2011271113734_llc.fits              0     1789  ...    0.013524   \n",
       "kplr001571511-2011271113734_llc.fits              0       94  ...    0.021482   \n",
       "kplr001725193-2011271113734_llc.fits              0      259  ...    0.042875   \n",
       "\n",
       "                                      magratio  sautocorrcoef  autocorrcoef  \\\n",
       "kplr001026032-2011271113734_llc.fits  0.076923       0.742404      0.937333   \n",
       "kplr001026957-2011271113734_llc.fits  0.616403      -0.294359      0.999479   \n",
       "kplr001433962-2011271113734_llc.fits  0.502052      -0.486633      0.978182   \n",
       "kplr001571511-2011271113734_llc.fits  0.069174       0.510130      0.968878   \n",
       "kplr001725193-2011271113734_llc.fits  0.136161       0.789440      0.957196   \n",
       "\n",
       "                                      flatmean  tflatmean  roundmean  \\\n",
       "kplr001026032-2011271113734_llc.fits  0.034865   0.108202  -0.723083   \n",
       "kplr001026957-2011271113734_llc.fits  0.002733   0.003749   0.047433   \n",
       "kplr001433962-2011271113734_llc.fits  0.019154   0.018673  -0.003119   \n",
       "kplr001571511-2011271113734_llc.fits  0.007759   0.006047  -0.089364   \n",
       "kplr001725193-2011271113734_llc.fits  0.055880   0.210136  -1.015293   \n",
       "\n",
       "                                      troundmean   roundrat   flatrat  \n",
       "kplr001026032-2011271113734_llc.fits    0.031835 -22.713754  0.322218  \n",
       "kplr001026957-2011271113734_llc.fits   -0.012429  -3.816354  0.728774  \n",
       "kplr001433962-2011271113734_llc.fits    0.165303  -0.018868  1.025801  \n",
       "kplr001571511-2011271113734_llc.fits    0.196277  -0.455296  1.283076  \n",
       "kplr001725193-2011271113734_llc.fits    0.573217  -1.771219  0.265923  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>longtermtrend</th>\n",
       "      <th>meanmedrat</th>\n",
       "      <th>skews</th>\n",
       "      <th>varss</th>\n",
       "      <th>coeffvar</th>\n",
       "      <th>stds</th>\n",
       "      <th>numoutliers</th>\n",
       "      <th>numnegoutliers</th>\n",
       "      <th>numposoutliers</th>\n",
       "      <th>numout1s</th>\n",
       "      <th>...</th>\n",
       "      <th>percentamp</th>\n",
       "      <th>magratio</th>\n",
       "      <th>sautocorrcoef</th>\n",
       "      <th>autocorrcoef</th>\n",
       "      <th>flatmean</th>\n",
       "      <th>tflatmean</th>\n",
       "      <th>roundmean</th>\n",
       "      <th>troundmean</th>\n",
       "      <th>roundrat</th>\n",
       "      <th>flatrat</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>kplr001026032-2011271113734_llc.fits</th>\n",
       "      <td>-6.415193e-07</td>\n",
       "      <td>0.998781</td>\n",
       "      <td>-7.109529</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>0.008196</td>\n",
       "      <td>0.008186</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>0</td>\n",
       "      <td>163</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086309</td>\n",
       "      <td>0.076923</td>\n",
       "      <td>0.742404</td>\n",
       "      <td>0.937333</td>\n",
       "      <td>0.034865</td>\n",
       "      <td>0.108202</td>\n",
       "      <td>-0.723083</td>\n",
       "      <td>0.031835</td>\n",
       "      <td>-22.713754</td>\n",
       "      <td>0.322218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kplr001026957-2011271113734_llc.fits</th>\n",
       "      <td>8.002484e-07</td>\n",
       "      <td>1.000915</td>\n",
       "      <td>0.396550</td>\n",
       "      <td>0.000032</td>\n",
       "      <td>0.005688</td>\n",
       "      <td>0.005694</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1474</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013276</td>\n",
       "      <td>0.616403</td>\n",
       "      <td>-0.294359</td>\n",
       "      <td>0.999479</td>\n",
       "      <td>0.002733</td>\n",
       "      <td>0.003749</td>\n",
       "      <td>0.047433</td>\n",
       "      <td>-0.012429</td>\n",
       "      <td>-3.816354</td>\n",
       "      <td>0.728774</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kplr001433962-2011271113734_llc.fits</th>\n",
       "      <td>2.901455e-07</td>\n",
       "      <td>0.998547</td>\n",
       "      <td>-0.267555</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>0.005488</td>\n",
       "      <td>0.005480</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1789</td>\n",
       "      <td>...</td>\n",
       "      <td>0.013524</td>\n",
       "      <td>0.502052</td>\n",
       "      <td>-0.486633</td>\n",
       "      <td>0.978182</td>\n",
       "      <td>0.019154</td>\n",
       "      <td>0.018673</td>\n",
       "      <td>-0.003119</td>\n",
       "      <td>0.165303</td>\n",
       "      <td>-0.018868</td>\n",
       "      <td>1.025801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kplr001571511-2011271113734_llc.fits</th>\n",
       "      <td>-3.231966e-08</td>\n",
       "      <td>0.999617</td>\n",
       "      <td>-6.939159</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.002572</td>\n",
       "      <td>0.002571</td>\n",
       "      <td>81</td>\n",
       "      <td>81</td>\n",
       "      <td>0</td>\n",
       "      <td>94</td>\n",
       "      <td>...</td>\n",
       "      <td>0.021482</td>\n",
       "      <td>0.069174</td>\n",
       "      <td>0.510130</td>\n",
       "      <td>0.968878</td>\n",
       "      <td>0.007759</td>\n",
       "      <td>0.006047</td>\n",
       "      <td>-0.089364</td>\n",
       "      <td>0.196277</td>\n",
       "      <td>-0.455296</td>\n",
       "      <td>1.283076</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>kplr001725193-2011271113734_llc.fits</th>\n",
       "      <td>-1.793919e-06</td>\n",
       "      <td>0.998418</td>\n",
       "      <td>-3.711617</td>\n",
       "      <td>0.000052</td>\n",
       "      <td>0.007213</td>\n",
       "      <td>0.007202</td>\n",
       "      <td>117</td>\n",
       "      <td>117</td>\n",
       "      <td>0</td>\n",
       "      <td>259</td>\n",
       "      <td>...</td>\n",
       "      <td>0.042875</td>\n",
       "      <td>0.136161</td>\n",
       "      <td>0.789440</td>\n",
       "      <td>0.957196</td>\n",
       "      <td>0.055880</td>\n",
       "      <td>0.210136</td>\n",
       "      <td>-1.015293</td>\n",
       "      <td>0.573217</td>\n",
       "      <td>-1.771219</td>\n",
       "      <td>0.265923</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 60 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                      longtermtrend  meanmedrat     skews  \\\n",
       "kplr001026032-2011271113734_llc.fits  -6.415193e-07    0.998781 -7.109529   \n",
       "kplr001026957-2011271113734_llc.fits   8.002484e-07    1.000915  0.396550   \n",
       "kplr001433962-2011271113734_llc.fits   2.901455e-07    0.998547 -0.267555   \n",
       "kplr001571511-2011271113734_llc.fits  -3.231966e-08    0.999617 -6.939159   \n",
       "kplr001725193-2011271113734_llc.fits  -1.793919e-06    0.998418 -3.711617   \n",
       "\n",
       "                                         varss  coeffvar      stds  \\\n",
       "kplr001026032-2011271113734_llc.fits  0.000067  0.008196  0.008186   \n",
       "kplr001026957-2011271113734_llc.fits  0.000032  0.005688  0.005694   \n",
       "kplr001433962-2011271113734_llc.fits  0.000030  0.005488  0.005480   \n",
       "kplr001571511-2011271113734_llc.fits  0.000007  0.002572  0.002571   \n",
       "kplr001725193-2011271113734_llc.fits  0.000052  0.007213  0.007202   \n",
       "\n",
       "                                     numoutliers numnegoutliers  \\\n",
       "kplr001026032-2011271113734_llc.fits          56             56   \n",
       "kplr001026957-2011271113734_llc.fits           0              0   \n",
       "kplr001433962-2011271113734_llc.fits           0              0   \n",
       "kplr001571511-2011271113734_llc.fits          81             81   \n",
       "kplr001725193-2011271113734_llc.fits         117            117   \n",
       "\n",
       "                                     numposoutliers numout1s  ...  percentamp  \\\n",
       "kplr001026032-2011271113734_llc.fits              0      163  ...    0.086309   \n",
       "kplr001026957-2011271113734_llc.fits              0     1474  ...    0.013276   \n",
       "kplr001433962-2011271113734_llc.fits              0     1789  ...    0.013524   \n",
       "kplr001571511-2011271113734_llc.fits              0       94  ...    0.021482   \n",
       "kplr001725193-2011271113734_llc.fits              0      259  ...    0.042875   \n",
       "\n",
       "                                      magratio  sautocorrcoef  autocorrcoef  \\\n",
       "kplr001026032-2011271113734_llc.fits  0.076923       0.742404      0.937333   \n",
       "kplr001026957-2011271113734_llc.fits  0.616403      -0.294359      0.999479   \n",
       "kplr001433962-2011271113734_llc.fits  0.502052      -0.486633      0.978182   \n",
       "kplr001571511-2011271113734_llc.fits  0.069174       0.510130      0.968878   \n",
       "kplr001725193-2011271113734_llc.fits  0.136161       0.789440      0.957196   \n",
       "\n",
       "                                      flatmean  tflatmean  roundmean  \\\n",
       "kplr001026032-2011271113734_llc.fits  0.034865   0.108202  -0.723083   \n",
       "kplr001026957-2011271113734_llc.fits  0.002733   0.003749   0.047433   \n",
       "kplr001433962-2011271113734_llc.fits  0.019154   0.018673  -0.003119   \n",
       "kplr001571511-2011271113734_llc.fits  0.007759   0.006047  -0.089364   \n",
       "kplr001725193-2011271113734_llc.fits  0.055880   0.210136  -1.015293   \n",
       "\n",
       "                                      troundmean   roundrat   flatrat  \n",
       "kplr001026032-2011271113734_llc.fits    0.031835 -22.713754  0.322218  \n",
       "kplr001026957-2011271113734_llc.fits   -0.012429  -3.816354  0.728774  \n",
       "kplr001433962-2011271113734_llc.fits    0.165303  -0.018868  1.025801  \n",
       "kplr001571511-2011271113734_llc.fits    0.196277  -0.455296  1.283076  \n",
       "kplr001725193-2011271113734_llc.fits    0.573217  -1.771219  0.265923  \n",
       "\n",
       "[5 rows x 60 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output_file = 'data/output/Example_output.p'\n",
    "with open(output_file,'rb') as f:\n",
    "    feats = pickle.load(f)\n",
    "feats.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The features are optimized using the `@njit` decorator from the `numba` package. To make full use of this, the code to be optimized by numba needs to run once. This can be done manually as follows:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "lc_path = './data/lightcurves/kplr001026032-2011271113734_llc.fits'\n",
    "lc = fc.import_lcs(lc_path)\n",
    "t = lc[1]\n",
    "nf = lc[2]\n",
    "err = lc[3]\n",
    "lc_feats = fc.feats(t,nf,err)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After priming the feature calculation, the features for a filelist can be run in the same way as before.\n",
    "As a note, the `features_from_filelist` method will run a primer by default using the first lightcurve. Whether primed manually as above, with the default, or even run specifically without priming, runs following the first will be optimized by `numba` and be quicker.\n",
    "\n",
    "Note the drastically improved runtime following the manual priming above:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading ./data/filelists/ex_filelist.txt...\n",
      "Using 47 cpus to calculate features...\n",
      "Importing 247 lightcurves...\n",
      "Lightcurve import took 0:00:01.126179\n",
      "Processing 247 files...\n",
      "247/247 completed. Time for chunk: 0:00:12.281421\n",
      "Features have been calculated, total time to calculate features: 0:00:12.294860\n",
      "Saving output to data/output/Example_output.p\n",
      "Cleaning up...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "path_to_filelist = './data/filelists/ex_filelist.txt'\n",
    "path_to_fits = './data/lightcurves/'\n",
    "output_file = 'data/output/Example_output.p'\n",
    "features = fc.features_from_filelist(path_to_filelist,path_to_fits,output_file,fl_as_array=False,verbose=True,prime_feats=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The file list can also be fed into the feature calculator as a list of filenames, produced however. Two examples below."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using 47 cpus to calculate features...\n",
      "Importing 247 lightcurves...\n",
      "Lightcurve import took 0:00:01.606876\n",
      "Processing 247 files...\n",
      "247/247 completed. Time for chunk: 0:00:12.205176\n",
      "Features have been calculated, total time to calculate features: 0:00:12.274446\n",
      "Saving output to data/output/Example_output.p\n",
      "Cleaning up...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "path_to_filelist = './data/filelists/ex_filelist.txt'\n",
    "with open(path_to_filelist,'r') as f:\n",
    "    files = f.read().splitlines()\n",
    "path_to_fits = './data/lightcurves/'\n",
    "output_file = 'data/output/Example_output.p'\n",
    "feats = fc.features_from_filelist(files,path_to_fits,output_file,fl_as_array=True,verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using 47 cpus to calculate features...\n",
      "Importing 247 lightcurves...\n",
      "Lightcurve import took 0:00:01.874902\n",
      "Processing 247 files...\n",
      "247/247 completed. Time for chunk: 0:00:14.005037\n",
      "Features have been calculated, total time to calculate features: 0:00:14.058610\n",
      "Saving output to data/output/Example_output.p\n",
      "Cleaning up...\n",
      "Done.\n"
     ]
    }
   ],
   "source": [
    "path_to_fits = './data/lightcurves/'\n",
    "files = os.listdir('data/lightcurves')\n",
    "output_file = 'data/output/Example_output.p'\n",
    "feats = fc.features_from_filelist(files,path_to_fits,output_file,fl_as_array=True,verbose=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving as a Cluster Outlier Object"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import clusterOutliers as coo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "!!! DOES NOT WORK FOR THIS EXAMPLE DATA\n",
    "!!! CLUSTER OUTLIER OBJECT IS DESIGNED TO WORK ON LARGE DATASETS, EXAMPLE IS TOO SMALL\n",
    "\"\"\"\n",
    "example_coo = coo.clusterOutliers(feats=feats,fitsDir=path_to_fits,output_file='example.coo')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Speed tests "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time to prime: 0:00:08.198919\n"
     ]
    }
   ],
   "source": [
    "start = datetime.now()\n",
    "lc_path = './data/lightcurves/kplr001026032-2011271113734_llc.fits'\n",
    "lc = fc.import_lcs(lc_path)\n",
    "t = lc[1]\n",
    "nf = lc[2]\n",
    "err = lc[3]\n",
    "lc_feats = fc.feats(t,nf,err)\n",
    "print(\"Time to prime: {}\".format(datetime.now()-start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29.7 ms ± 31.2 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "lc_path = './data/lightcurves/kplr001026032-2011271113734_llc.fits'\n",
    "lc = fc.import_lcs(lc_path)\n",
    "t = lc[1]\n",
    "nf = lc[2]\n",
    "err = lc[3]\n",
    "lc_feats = fc.feats(t,nf,err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "8.12 ms ± 9.96 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "lc_feats = fc.feats(t,nf,err)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19.1 ms ± 1.24 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "lc = fc.import_lcs(lc_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}