CleverSys_Analyzer_2023_01_10.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "# Define all Functions\n",
    "\n",
    "# save it\n",
    "def saveit(dark, MasterFile0, summary):\n",
    "    date = str(dark['Start Time'].values[0])[:10]\n",
    "    savename = str(fc_saver.selected_path) + \"\\\\\" + \" \" + date + \\\n",
    "        \" Processed \" + str(dt.today().strftime('%Y-%m-%d')) + \".xlsx\"\n",
    "\n",
    "# modify for excel writing\n",
    "    dark[['Behavior', 'Bin No', 'Start Time', 'Sex', 'Genotype', 'Start']] = dark[[\n",
    "        'Behavior', 'Bin No', 'Start Time', 'Sex', 'Genotype', 'Start']].astype(str)\n",
    "    MasterFile0[['Behavior', 'Bin No', 'Start Time', 'Sex', 'Genotype', 'Start']] = MasterFile0[[\n",
    "        'Behavior', 'Bin No', 'Start Time', 'Sex', 'Genotype', 'Start']].astype(str)\n",
    "\n",
    "    with pd.ExcelWriter(savename) as writer:\n",
    "        dark.to_excel(writer, sheet_name='Dark')\n",
    "        MasterFile0.to_excel(writer, sheet_name='All Data')\n",
    "        summary.to_excel(writer, sheet_name='Sum in Dark')\n",
    "        \n",
    "        button\n",
    "        \n",
    "    return(savename)\n",
    "\n",
    "\n",
    "# remaining functions\n",
    "\n",
    "def analyze_cleversys(xls, mouse1, mouse2, mouse3, mouse4, \n",
    "                      m1geno, m2geno, m3geno,m4geno,\n",
    "                      m1sex, m2sex, m3sex, m4sex):\n",
    "    # import info\n",
    "    names_and_dates = pd.read_excel(xls, 'HCSInfo')\n",
    "    data = pd.read_excel(xls, 'HCSGrpBinTimesDuration')\n",
    "\n",
    "    # make the numbers numeric\n",
    "    x = range(2, len(data.columns), 1)\n",
    "    for n in x:\n",
    "        data.iloc[:, n] = pd.to_numeric(data.iloc[:, n], errors='coerce')\n",
    "\n",
    "    data2 = data.drop([0, 1, 2, 3, 4, 5], axis=0).fillna(method='ffill')\n",
    "\n",
    "    mice = [mouse1, mouse2, mouse3, mouse4]\n",
    "    mice = [i for i in mice if i != 0]\n",
    "    num_mice = len(mice)\n",
    "\n",
    "\n",
    "# if there are 4 mice\n",
    "    if num_mice == 4:\n",
    "        data2.columns = [\n",
    "            'Behavior', 'Bin No', 'Mouse 1 Times', 'Mouse 1 Duration',\n",
    "            'Mouse 2 Times', 'Mouse 2 Duration', 'Mouse 3 Times',\n",
    "            'Mouse 3 Duration', 'Mouse 4 Times', 'Mouse 4 Duration'\n",
    "        ]\n",
    "\n",
    "        data2.to_clipboard()\n",
    "\n",
    "# if there are 3 mice\n",
    "    if num_mice == 3:\n",
    "        data2 = data2.iloc[:, :8]\n",
    "        print(len(data2.columns))\n",
    "        data2.columns = [\n",
    "            'Behavior', 'Bin No', 'Mouse 1 Times', 'Mouse 1 Duration',\n",
    "            'Mouse 2 Times', 'Mouse 2 Duration', 'Mouse 3 Times',\n",
    "            'Mouse 3 Duration'\n",
    "        ]\n",
    "\n",
    "\n",
    "# if there are 2 mice\n",
    "    if num_mice == 2:\n",
    "        data2 = data2.iloc[:, :6]\n",
    "        data2.columns = [\n",
    "            'Behavior', 'Bin No', 'Mouse 1 Times', 'Mouse 1 Duration',\n",
    "            'Mouse 2 Times', 'Mouse 2 Duration'\n",
    "        ]\n",
    "\n",
    "\n",
    "# if there is 1 mouse\n",
    "    if num_mice == 1:\n",
    "        data2 = data2.iloc[:, :4]\n",
    "        data2.columns = [\n",
    "            'Behavior', 'Bin No', 'Mouse 1 Times', 'Mouse 1 Duration'\n",
    "        ]\n",
    "\n",
    "\n",
    "# filter behaviors\n",
    "    data3 = data2[data2['Behavior'].isin([\n",
    "        'Groom', 'HVfromHC', 'HVfromRU', 'HangCudl', 'Jump', 'Pause', 'RearUp',\n",
    "        'RemainHC', 'RemainHV', 'RemainLw', 'RemainPR', 'RemainRU', 'ReptJump',\n",
    "        'Sleep', 'Stationa', 'Stretch', 'TravDist.', 'Turn'\n",
    "    ])]\n",
    "\n",
    "\n",
    "# get start times sorted out\n",
    "    start0 = names_and_dates.iloc[18:19, 1:2]\n",
    "    val = start0['Unnamed: 1'].values[0]\n",
    "\n",
    "    dstart = datetime.datetime.strptime(val, '%Y-%m-%d %H:%M:%S')\n",
    "    starts, starts1 = [], []\n",
    "    starts.append(dstart)\n",
    "    for i in range(0, 47):\n",
    "        starts.append(starts[i] + timedelta(minutes=30))\n",
    "\n",
    "    for time in range(0, (len(starts))):\n",
    "        starts[time] = starts[time].strftime('%Y-%m-%d %H:%M:%S')\n",
    "\n",
    "    all_start = starts * len(set(data3['Behavior']))\n",
    "\n",
    "    data3['Start Time'] = all_start\n",
    "\n",
    "# split into one dataframe per mouse\n",
    "\n",
    "    Mdf = data3[[\n",
    "        'Behavior', 'Bin No', 'Mouse 1 Times', 'Mouse 1 Duration', 'Start Time'\n",
    "    ]].copy()\n",
    "\n",
    "    Mdf['Mouse'] = mouse1\n",
    "    Mdf['Sex'] = m1sex\n",
    "    Mdf['Genotype'] = m1geno\n",
    "    Mdf.rename(columns={'Mouse 1 Times': 'Times'}, inplace=True)\n",
    "    Mdf.rename(columns={'Mouse 1 Duration': 'Duration'}, inplace=True)\n",
    "\n",
    "    if num_mice >= 2:\n",
    "        M2df = data3[[\n",
    "            'Behavior', 'Bin No', 'Mouse 2 Times', 'Mouse 2 Duration', 'Start Time'\n",
    "        ]].copy()\n",
    "\n",
    "        M2df['Mouse'] = mouse2\n",
    "        M2df['Sex'] = m2sex\n",
    "        M2df['Genotype'] = m2geno\n",
    "        M2df.rename(columns={'Mouse 2 Times': 'Times'}, inplace=True)\n",
    "        M2df.rename(columns={'Mouse 2 Duration': 'Duration'}, inplace=True)\n",
    "\n",
    "    if num_mice >= 3:\n",
    "        M3df = data3[[\n",
    "            'Behavior', 'Bin No', 'Mouse 3 Times', 'Mouse 3 Duration', 'Start Time'\n",
    "        ]].copy()\n",
    "\n",
    "        M3df['Mouse'] = mouse3\n",
    "        M3df['Sex'] = m3sex\n",
    "        M3df['Genotype'] = m3geno\n",
    "        M3df.rename(columns={'Mouse 3 Times': 'Times'}, inplace=True)\n",
    "        M3df.rename(columns={'Mouse 3 Duration': 'Duration'}, inplace=True)\n",
    "\n",
    "    if num_mice == 4:\n",
    "        M4df = data3[[\n",
    "            'Behavior', 'Bin No', 'Mouse 4 Times', 'Mouse 4 Duration', 'Start Time'\n",
    "        ]].copy()\n",
    "\n",
    "        M4df['Mouse'] = mouse4\n",
    "        M4df['Sex'] = m4sex\n",
    "        M4df['Genotype'] = m4geno\n",
    "        M4df.rename(columns={'Mouse 4 Times': 'Times'}, inplace=True)\n",
    "        M4df.rename(columns={'Mouse 4 Duration': 'Duration'}, inplace=True)\n",
    "\n",
    "# put it back together\n",
    "    if num_mice == 2:\n",
    "        Mdf = Mdf.append(M2df, ignore_index=True)\n",
    "\n",
    "    if num_mice == 3:\n",
    "        Mdf = Mdf.append(M2df, ignore_index=True).append(M3df,\n",
    "                                                         ignore_index=True)\n",
    "    if num_mice == 4:\n",
    "        Mdf = Mdf.append(M2df, ignore_index=True).append(\n",
    "            M3df, ignore_index=True).append(M4df, ignore_index=True)\n",
    "\n",
    "    MasterFile0 = Mdf.copy()\n",
    "# Set the dates up to be usable\n",
    "    MasterFile0['Start Time'] = pd.to_datetime(MasterFile0['Start Time'],\n",
    "                                               utc=True)\n",
    "    MasterFile0['Start'] = [\n",
    "        datetime.datetime.time(d) for d in MasterFile0['Start Time']\n",
    "    ]\n",
    "\n",
    "# put a fake day value in so I can compare across days\n",
    "    fakedate = str(datetime.datetime.strptime('2000-01-01', '%Y-%m-%d').date())\n",
    "    MasterFile0['Start'] = pd.to_datetime(fakedate + \" \" +\n",
    "                                          MasterFile0['Start'].astype(str))\n",
    "\n",
    "# Make the values the right types so we can use them\n",
    "   # MasterFile0[[\"Times\",\n",
    "    #            \"Duration\"]] = MasterFile0[[\"Times\",\n",
    "    #                   \"Duration\"]].apply(pd.to_numeric)\n",
    "    MasterFile0['Mouse'] = MasterFile0['Mouse'].astype(str)\n",
    "    MasterFile0['Start'] = pd.to_datetime(MasterFile0['Start'], utc=True)\n",
    "\n",
    "# extract times in the dark\n",
    "    dark = MasterFile0[~((MasterFile0['Start'] > '2000-01-01 06:00:00') &\n",
    "                         (MasterFile0['Start'] <= '2000-01-01 18:00:00'))]\n",
    "\n",
    "# get summary data\n",
    "    summary = dark.drop(columns=['Bin No', 'Start'])\n",
    "    summary = dark.drop(columns=['Bin No', 'Start']).groupby([\n",
    "        'Genotype',\n",
    "        'Behavior',\n",
    "        'Sex',\n",
    "        'Mouse',\n",
    "    ]).agg(['sum', 'mean', 'sem']).reset_index()\n",
    "    summary.columns = [\n",
    "        'Genotype', 'Behavior', 'Sex', 'Mouse', 'Sum_Times', 'Mean_Times',\n",
    "        'SEM_Times', 'Sum_Duration', 'Mean_Duration', 'SEM_Duration'\n",
    "    ]\n",
    "    print(\"File Processed\")\n",
    "    return (dark, MasterFile0, summary)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def analyze(): \n",
    "    # Process files and save them\n",
    "\n",
    "    # read each file and process it\n",
    "    import warnings\n",
    "    import datetime as datetime\n",
    "    globber = fc_processed.selected_path + \"\\*.xlsx\"\n",
    "    globbed_files = glob.glob(globber)\n",
    "\n",
    "    warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "\n",
    "    for xl in globbed_files:\n",
    "        # get each file and parse out the mouse / geno / hemisphere\n",
    "        names_and_dates = pd.read_excel(xl, 'HCSInfo', index_col=0)\n",
    "        data = pd.read_excel(xl, 'HCSGrpBinTimesDuration')\n",
    "\n",
    "        mouse_ids = list(names_and_dates.loc['Mouse ID'])\n",
    "        pad(mouse_ids, 0, 4)\n",
    "        mouse_ids = mouse_ids[:4]\n",
    "\n",
    "        if \"Sex\" in names_and_dates.index:\n",
    "            sexes = list(names_and_dates.loc['Sex'])\n",
    "            pad(sexes, 0, 4)\n",
    "            sexes = sexes[:4]\n",
    "\n",
    "        if \"Genotype\" in names_and_dates.index:\n",
    "            genos = list(names_and_dates.loc['Genotype'])\n",
    "            pad(genos, 0, 4)\n",
    "            genos = genos[:4]\n",
    "\n",
    "        m1geno = genos[0]\n",
    "        m2geno = genos[1]\n",
    "        m3geno = genos[2]\n",
    "        m4geno = genos[3]\n",
    "\n",
    "        mouse1 = mouse_ids[0]\n",
    "        mouse2 = mouse_ids[1]\n",
    "        mouse3 = mouse_ids[2]\n",
    "        mouse4 = mouse_ids[3]\n",
    "\n",
    "        m1sex = sexes[0]\n",
    "        m2sex = sexes[1]\n",
    "        m3sex = sexes[2]\n",
    "        m4sex = sexes[3]\n",
    "\n",
    "        dark, MasterFile0, summary = analyze_cleversys(xl, \n",
    "                                                       mouse1, mouse2, mouse3, mouse4, \n",
    "                                                       m1geno, m2geno, m3geno, m4geno,\n",
    "                                                       m1sex, m2sex, m3sex, m4sex)\n",
    "        savename = saveit(dark, MasterFile0, summary)\n",
    "        \n",
    "        \n",
    "def pad(l, content, width):\n",
    "    l.extend([content] * (width - len(l)))\n",
    "    return l\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# set up the App: Fill in info here, then run next cells\n",
    "\n",
    "import pandas as pd\n",
    "from datetime import timedelta\n",
    "import datetime as datetime\n",
    "from datetime import datetime as dt\n",
    "from ipyfilechooser import FileChooser\n",
    "import ipywidgets as widgets\n",
    "from ipywidgets import AppLayout\n",
    "import glob\n",
    "pd.options.mode.chained_assignment = None\n",
    "\n",
    "\n",
    "# set up GUI\n",
    "# Create and display a FileChooser widgets\n",
    "fc_processed = FileChooser()\n",
    "fc_processed.show_only_dirs = True\n",
    "fc_processed.title = '<b>Locate the folder containing your raw .csv files</b>'\n",
    "\n",
    "fc_saver = FileChooser()\n",
    "fc_saver.show_only_dirs = True\n",
    "fc_saver.title = '<b> Where do you want your outputs saved?: NOTE they must be alone in a folder for the next step of processing, so create a new folder for them to be in first </b>'\n",
    "\n",
    "\n",
    "final_save = widgets.Text(\n",
    "    value='Enter Final Save Name',\n",
    "    placeholder='Type something',\n",
    "    description='',\n",
    "    disabled=False\n",
    ")\n",
    "\n",
    "\n",
    "header0 = widgets.Label(value= \"Requirements:\")\n",
    "header00 = widgets.Label(value= \"Cleversys output files consisting of 48 30-minute blocks of behavior with all behaviors selected for output\")\n",
    "header000 = widgets.Label(value= \"Make sure to check the box for interleaved data so the columns are Mouse 1 time, then Mouse 1 duration, etc, not all time then all durations.\")\n",
    "header0000 = widgets.Label(value= \"All mice used must have a value in their mouse number info spot at Row 9 cells B,C,D,E in your excel output\")\n",
    "header00000 = widgets.Label(value = \"Date and Time must be in cell 20B in the format YYYY-MM-DD HH:MM:SS\")\n",
    "\n",
    "\n",
    "\n",
    "header = widgets.Label(\n",
    "    value=\"Program Instructions:\")\n",
    "header1 = widgets.Label(\"Locate your raw data files (ALL .csv files in the selected folder will be processed\")\n",
    "header2 = widgets.Label(value=\"Then locate an EMPTY folder for your outputs.\")\n",
    "header3 = widgets.Label(value=\"Finally, name your output file.\")\n",
    "spacer = widgets.Label(value=\"    \")\n",
    "saveinstructions = widgets.Label(\n",
    "    value=\"Enter the name you want to save the final file as.\")\n",
    "\n",
    "button = widgets.Button(description=\"Click to Analyze\",button_style='success')\n",
    "\n",
    "def on_button_clicked(b):\n",
    "    analyze()\n",
    "\n",
    "button.on_click(on_button_clicked)\n",
    "\n",
    "finder = widgets.VBox([header0,header00,header000, header0000, header00000,spacer,header, header1, header2, header3, spacer, fc_processed,\n",
    "                      spacer, fc_saver, spacer, saveinstructions, final_save, spacer, button])\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "finder\n",
    "# deploy GUI"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "# pick up processed files and produce final file\n",
    "#  This needs to run only when ready\n",
    "headera = widgets.Label(value= \"Click to Aggregate Data ONLY AFTER processed files are created:\")\n",
    "\n",
    "from ipywidgets import HBox, Label, VBox\n",
    "\n",
    "def final_analysis():\n",
    "    \n",
    "    globber2 = fc_saver.selected_path + \"\\*.xlsx\"\n",
    "    globbed_files2 = glob.glob(globber2)\n",
    "\n",
    "    dataframe = pd.DataFrame(\n",
    "        columns=['Genotype', 'Behavior', 'Sex', 'Mouse', 'Sum_Duration', 'Sum_Times'])\n",
    "\n",
    "    for xl in globbed_files2:\n",
    "        # get each file\n",
    "        data = pd.read_excel(xl, 'Sum in Dark', index_col=0)\n",
    "        data2 = data[['Genotype', 'Behavior', 'Sex',\n",
    "                      'Mouse', 'Sum_Duration', 'Sum_Times']]\n",
    "        dataframe = pd.concat([dataframe, data2])\n",
    "\n",
    "\n",
    "    # format the final save file\n",
    "    # empty frame to add to\n",
    "    allmice = pd.DataFrame(columns=['Mouse', 'Groom', 'HVfromHC', 'HVfromRU', 'HangCudl', 'Jump', 'Pause',\n",
    "                                    'RearUp', 'RemainHC', 'RemainHV', 'RemainLw', 'RemainPR', 'RemainRU',\n",
    "                                    'ReptJump', 'Sleep', 'Stationa', 'Stretch', 'TravDist.', 'Turn'])\n",
    "\n",
    "\n",
    "    for m in set(dataframe['Mouse']):\n",
    "        mouseframe = dataframe[dataframe['Mouse'] == m]\n",
    "\n",
    "       # remove travel for now\n",
    "        travelframe = mouseframe[mouseframe['Behavior'] == \"TravDist.\"\n",
    "                                 ]\n",
    "        travelframe['Sum_Duration'] = travelframe['Sum_Times']\n",
    "\n",
    "        notravframe = mouseframe[mouseframe['Behavior'] != \"TravDist\"]\n",
    "        allframe = pd.concat([travelframe, notravframe], sort=False)\n",
    "\n",
    "        allmice0 = allframe.pivot_table(index=['Mouse', 'Genotype', 'Sex'], columns='Behavior',\n",
    "                                        values='Sum_Duration', aggfunc='sum').reset_index()\n",
    "\n",
    "        allmice = pd.concat([allmice0, allmice], sort=False)\n",
    "\n",
    "\n",
    "    # save the file\n",
    "    saveitt = fc_saver.selected_path + \"\\\\\" + str(final_save.value) + \".xlsx\"\n",
    "    allmice.to_excel(saveitt)\n",
    "    print(\"File saved!\")\n",
    "    \n",
    "\n",
    "button2 = widgets.Button(description=\"Final Analysis\",\n",
    "                         button_style='danger'\n",
    "                        )\n",
    "\n",
    "def on_button_clicked(b):\n",
    "    final_analysis()\n",
    "\n",
    "button2.on_click(on_button_clicked)\n",
    "\n",
    "VBox([spacer, spacer, spacer, Label('Click to Aggregate Data ONLY AFTER processed files are created'), button2])"
   ]
  }
 ],
 "metadata": {
  "celltoolbar": "Hide code",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "164.993px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}