process_AOD_data_leading_jets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import uproot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import TensorDataset\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_data = '../data/aods/'\n",
    "\n",
    "# Load a ROOT file\n",
    "folder = 'data18_13TeV.00364292.calibration_DataScouting_05_Jets.deriv.DAOD_TRIG6.r10657_p3592_p3754/'\n",
    "fname = 'DAOD_TRIG6.16825104._000035.pool.root.1'\n",
    "filePath = path_to_data + folder + fname\n",
    "#ttree = uproot.open(filePath)['outTree']['nominal']\n",
    "tree = uproot.open(filePath)['CollectionTree']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[b'ByteStreamEventInfo',\n",
       " b'TrigConfKeys',\n",
       " b'EventInfoAux.',\n",
       " b'xTrigDecisionAux.',\n",
       " b'EventInfo',\n",
       " b'xTrigDecision',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAux.',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollection',\n",
       " b'EventInfoAuxDyn.streamTagRobs',\n",
       " b'EventInfoAuxDyn.streamTagDets',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.pt',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.eta',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.phi',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.m',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.constituentLinks',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.constituentWeights',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ConstituentScale',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetEMScaleMomentum_pt',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetEMScaleMomentum_eta',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetEMScaleMomentum_phi',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetEMScaleMomentum_m',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetConstitScaleMomentum_pt',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetConstitScaleMomentum_eta',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetConstitScaleMomentum_phi',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetConstitScaleMomentum_m',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.InputType',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.AlgorithmType',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.SizeParameter',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.GhostTrack',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.FracSamplingMaxIndex',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.OriginCorrected',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.PileupCorrected',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea4vec_eta',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea4vec_m',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea4vec_phi',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea4vec_pt',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.AverageLArQF',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.BchCorrCell',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.CentroidR',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.DetectorEta',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.EMFrac',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.FracSamplingMax',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.HECFrac',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.HECQuality',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetEtaJESScaleMomentum_eta',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetEtaJESScaleMomentum_m',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetEtaJESScaleMomentum_phi',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetEtaJESScaleMomentum_pt',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetGSCScaleMomentum_eta',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetGSCScaleMomentum_m',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetGSCScaleMomentum_phi',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetGSCScaleMomentum_pt',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetGhostArea',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetInsituScaleMomentum_eta',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetInsituScaleMomentum_m',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetInsituScaleMomentum_phi',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetInsituScaleMomentum_pt',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetPileupScaleMomentum_eta',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetPileupScaleMomentum_m',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetPileupScaleMomentum_phi',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetPileupScaleMomentum_pt',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LArQuality',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LeadingClusterCenterLambda',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LeadingClusterPt',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LeadingClusterSecondLambda',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LeadingClusterSecondR',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.N90Constituents',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.NegativeE',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.OotFracClusters10',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.OotFracClusters5',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.Timing',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.Width',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.WidthPhi',\n",
       " b'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.EnergyPerSampling']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2260895,)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tree.array('HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.NegativeE').shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "branchnames = [\n",
    "    # 4-momentum\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.pt',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.eta',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.phi',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.m',\n",
    "    # Energy deposition in each calorimeter layer\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.EnergyPerSampling',\n",
    "    # Area of jet,used for pile-up suppression (4-vector)\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea4vec_eta',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea4vec_m',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea4vec_phi',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.ActiveArea4vec_pt',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.JetGhostArea',\n",
    "    # Variables related to quality of jet\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.AverageLArQF',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.BchCorrCell',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.NegativeE',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.HECQuality',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LArQuality',\n",
    "    # Shape and position, most energetic cluster\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.Width',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.WidthPhi',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.CentroidR',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.DetectorEta',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LeadingClusterCenterLambda',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LeadingClusterPt',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LeadingClusterSecondLambda',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.LeadingClusterSecondR',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.N90Constituents',\n",
    "    # Energy released in each calorimeter\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.EMFrac',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.HECFrac',\n",
    "    # Variables related to the time of arrival of a jet\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.Timing',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.OotFracClusters10',\n",
    "    'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.OotFracClusters5',\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(branchnames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.EnergyPerSampling'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "branchnames[4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2260895"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "EnergyPerSampling = tree.array(branchnames[4])\n",
    "n_events = len(EnergyPerSampling)\n",
    "n_events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0%\n",
      "13%\n",
      "26%\n",
      "39%\n",
      "53%\n",
      "66%\n",
      "79%\n",
      "92%\n",
      "100%\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(2260895, 28)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arr = np.zeros((len(EnergyPerSampling), 28))\n",
    "for ii, samp in enumerate(EnergyPerSampling):\n",
    "    layers = np.array(samp[0])\n",
    "    arr[ii, :] = layers\n",
    "    if ii % 300000 == 0:\n",
    "        print(str((100 * ii) // n_events) + '%')\n",
    "print('100%')\n",
    "arr.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>EnergyPerSampling0</th>\n",
       "      <th>EnergyPerSampling1</th>\n",
       "      <th>EnergyPerSampling2</th>\n",
       "      <th>EnergyPerSampling3</th>\n",
       "      <th>EnergyPerSampling4</th>\n",
       "      <th>EnergyPerSampling5</th>\n",
       "      <th>EnergyPerSampling6</th>\n",
       "      <th>EnergyPerSampling7</th>\n",
       "      <th>EnergyPerSampling8</th>\n",
       "      <th>EnergyPerSampling9</th>\n",
       "      <th>...</th>\n",
       "      <th>EnergyPerSampling18</th>\n",
       "      <th>EnergyPerSampling19</th>\n",
       "      <th>EnergyPerSampling20</th>\n",
       "      <th>EnergyPerSampling21</th>\n",
       "      <th>EnergyPerSampling22</th>\n",
       "      <th>EnergyPerSampling23</th>\n",
       "      <th>EnergyPerSampling24</th>\n",
       "      <th>EnergyPerSampling25</th>\n",
       "      <th>EnergyPerSampling26</th>\n",
       "      <th>EnergyPerSampling27</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4463.817383</td>\n",
       "      <td>23248.812500</td>\n",
       "      <td>32758.755859</td>\n",
       "      <td>175.359970</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>700.465637</td>\n",
       "      <td>2787.192871</td>\n",
       "      <td>39659.492188</td>\n",
       "      <td>3555.119873</td>\n",
       "      <td>1273.725708</td>\n",
       "      <td>1467.259521</td>\n",
       "      <td>1702.037476</td>\n",
       "      <td>25.156708</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>57268.914062</td>\n",
       "      <td>39543.589844</td>\n",
       "      <td>1214.614746</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>4481.879883</td>\n",
       "      <td>17487.234375</td>\n",
       "      <td>31605.128906</td>\n",
       "      <td>819.519958</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1268.775146</td>\n",
       "      <td>25783.349609</td>\n",
       "      <td>123290.929688</td>\n",
       "      <td>11596.035156</td>\n",
       "      <td>52947.421875</td>\n",
       "      <td>38970.355469</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>26491.923828</td>\n",
       "      <td>142611.765625</td>\n",
       "      <td>23353.757812</td>\n",
       "      <td>59680.585938</td>\n",
       "      <td>26988.935547</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260890</td>\n",
       "      <td>298.998657</td>\n",
       "      <td>322.053650</td>\n",
       "      <td>174.562683</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>7434.597656</td>\n",
       "      <td>36035.804688</td>\n",
       "      <td>94699.312500</td>\n",
       "      <td>6494.367676</td>\n",
       "      <td>7786.124023</td>\n",
       "      <td>17215.060547</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260891</td>\n",
       "      <td>14950.682617</td>\n",
       "      <td>16882.427734</td>\n",
       "      <td>55113.578125</td>\n",
       "      <td>15053.662109</td>\n",
       "      <td>204.860580</td>\n",
       "      <td>518.438904</td>\n",
       "      <td>14.297218</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>8076.918457</td>\n",
       "      <td>1731.564697</td>\n",
       "      <td>306.276306</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260892</td>\n",
       "      <td>5381.773926</td>\n",
       "      <td>13260.756836</td>\n",
       "      <td>48842.007812</td>\n",
       "      <td>2307.945801</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-113.000000</td>\n",
       "      <td>224.189087</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260893</td>\n",
       "      <td>12455.039062</td>\n",
       "      <td>20173.287109</td>\n",
       "      <td>84696.546875</td>\n",
       "      <td>2507.416504</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>3838.447754</td>\n",
       "      <td>3840.161621</td>\n",
       "      <td>913.936890</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260894</td>\n",
       "      <td>2126.402100</td>\n",
       "      <td>341.992096</td>\n",
       "      <td>-36.726974</td>\n",
       "      <td>52.962128</td>\n",
       "      <td>18672.910156</td>\n",
       "      <td>32004.253906</td>\n",
       "      <td>98466.593750</td>\n",
       "      <td>9607.082031</td>\n",
       "      <td>35036.175781</td>\n",
       "      <td>28562.531250</td>\n",
       "      <td>...</td>\n",
       "      <td>10539.982422</td>\n",
       "      <td>1920.685303</td>\n",
       "      <td>-91.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2260895 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         EnergyPerSampling0  EnergyPerSampling1  EnergyPerSampling2  \\\n",
       "0               4463.817383        23248.812500        32758.755859   \n",
       "1                700.465637         2787.192871        39659.492188   \n",
       "2               4481.879883        17487.234375        31605.128906   \n",
       "3                  0.000000            0.000000            0.000000   \n",
       "4                  0.000000            0.000000            0.000000   \n",
       "...                     ...                 ...                 ...   \n",
       "2260890          298.998657          322.053650          174.562683   \n",
       "2260891        14950.682617        16882.427734        55113.578125   \n",
       "2260892         5381.773926        13260.756836        48842.007812   \n",
       "2260893        12455.039062        20173.287109        84696.546875   \n",
       "2260894         2126.402100          341.992096          -36.726974   \n",
       "\n",
       "         EnergyPerSampling3  EnergyPerSampling4  EnergyPerSampling5  \\\n",
       "0                175.359970            0.000000            0.000000   \n",
       "1               3555.119873         1273.725708         1467.259521   \n",
       "2                819.519958            0.000000            0.000000   \n",
       "3                  0.000000         1268.775146        25783.349609   \n",
       "4                  0.000000            0.000000        26491.923828   \n",
       "...                     ...                 ...                 ...   \n",
       "2260890            0.000000         7434.597656        36035.804688   \n",
       "2260891        15053.662109          204.860580          518.438904   \n",
       "2260892         2307.945801            0.000000            0.000000   \n",
       "2260893         2507.416504            0.000000            0.000000   \n",
       "2260894           52.962128        18672.910156        32004.253906   \n",
       "\n",
       "         EnergyPerSampling6  EnergyPerSampling7  EnergyPerSampling8  \\\n",
       "0                  0.000000            0.000000            0.000000   \n",
       "1               1702.037476           25.156708            0.000000   \n",
       "2                  0.000000            0.000000            0.000000   \n",
       "3             123290.929688        11596.035156        52947.421875   \n",
       "4             142611.765625        23353.757812        59680.585938   \n",
       "...                     ...                 ...                 ...   \n",
       "2260890        94699.312500         6494.367676         7786.124023   \n",
       "2260891           14.297218            0.000000            0.000000   \n",
       "2260892            0.000000            0.000000            0.000000   \n",
       "2260893            0.000000            0.000000            0.000000   \n",
       "2260894        98466.593750         9607.082031        35036.175781   \n",
       "\n",
       "         EnergyPerSampling9  ...  EnergyPerSampling18  EnergyPerSampling19  \\\n",
       "0                  0.000000  ...             0.000000             0.000000   \n",
       "1                  0.000000  ...         57268.914062         39543.589844   \n",
       "2                  0.000000  ...             0.000000             0.000000   \n",
       "3              38970.355469  ...             0.000000             0.000000   \n",
       "4              26988.935547  ...             0.000000             0.000000   \n",
       "...                     ...  ...                  ...                  ...   \n",
       "2260890        17215.060547  ...             0.000000             0.000000   \n",
       "2260891            0.000000  ...          8076.918457          1731.564697   \n",
       "2260892            0.000000  ...             0.000000          -113.000000   \n",
       "2260893            0.000000  ...          3838.447754          3840.161621   \n",
       "2260894        28562.531250  ...         10539.982422          1920.685303   \n",
       "\n",
       "         EnergyPerSampling20  EnergyPerSampling21  EnergyPerSampling22  \\\n",
       "0                   0.000000                  0.0                  0.0   \n",
       "1                1214.614746                  0.0                  0.0   \n",
       "2                   0.000000                  0.0                  0.0   \n",
       "3                   0.000000                  0.0                  0.0   \n",
       "4                   0.000000                  0.0                  0.0   \n",
       "...                      ...                  ...                  ...   \n",
       "2260890             0.000000                  0.0                  0.0   \n",
       "2260891           306.276306                  0.0                  0.0   \n",
       "2260892           224.189087                  0.0                  0.0   \n",
       "2260893           913.936890                  0.0                  0.0   \n",
       "2260894           -91.000000                  0.0                  0.0   \n",
       "\n",
       "         EnergyPerSampling23  EnergyPerSampling24  EnergyPerSampling25  \\\n",
       "0                        0.0                  0.0                  0.0   \n",
       "1                        0.0                  0.0                  0.0   \n",
       "2                        0.0                  0.0                  0.0   \n",
       "3                        0.0                  0.0                  0.0   \n",
       "4                        0.0                  0.0                  0.0   \n",
       "...                      ...                  ...                  ...   \n",
       "2260890                  0.0                  0.0                  0.0   \n",
       "2260891                  0.0                  0.0                  0.0   \n",
       "2260892                  0.0                  0.0                  0.0   \n",
       "2260893                  0.0                  0.0                  0.0   \n",
       "2260894                  0.0                  0.0                  0.0   \n",
       "\n",
       "         EnergyPerSampling26  EnergyPerSampling27  \n",
       "0                        0.0                  0.0  \n",
       "1                        0.0                  0.0  \n",
       "2                        0.0                  0.0  \n",
       "3                        0.0                  0.0  \n",
       "4                        0.0                  0.0  \n",
       "...                      ...                  ...  \n",
       "2260890                  0.0                  0.0  \n",
       "2260891                  0.0                  0.0  \n",
       "2260892                  0.0                  0.0  \n",
       "2260893                  0.0                  0.0  \n",
       "2260894                  0.0                  0.0  \n",
       "\n",
       "[2260895 rows x 28 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "e_samp_df = pd.DataFrame(data=arr, columns=['EnergyPerSampling%d' % kk for kk in np.arange(28)])\n",
    "e_samp_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_leading(jaggedX):\n",
    "    return jaggedX[jaggedX.counts > 0, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0%\n",
      "23%\n",
      "46%\n",
      "70%\n",
      "93%\n",
      "100%\n"
     ]
    }
   ],
   "source": [
    "df_dict = {}\n",
    "for ii, branchname in enumerate(branchnames):\n",
    "    variable = branchname.split('.')[1]\n",
    "    if branchname == 'HLT_xAOD__JetContainer_TrigHLTJetDSSelectorCollectionAuxDyn.EnergyPerSampling':\n",
    "        pass\n",
    "    else:\n",
    "        jaggedX = tree.array(branchname)\n",
    "        leadingX = get_leading(jaggedX)\n",
    "        df_dict.update({variable: leadingX})\n",
    "    if ii % 7 == 0:\n",
    "        print(str((ii * 100) // len(branchnames)) + '%')\n",
    "print('100%')\n",
    "partial_df = pd.DataFrame(data=df_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pt</th>\n",
       "      <th>eta</th>\n",
       "      <th>phi</th>\n",
       "      <th>m</th>\n",
       "      <th>ActiveArea</th>\n",
       "      <th>ActiveArea4vec_eta</th>\n",
       "      <th>ActiveArea4vec_m</th>\n",
       "      <th>ActiveArea4vec_phi</th>\n",
       "      <th>ActiveArea4vec_pt</th>\n",
       "      <th>JetGhostArea</th>\n",
       "      <th>...</th>\n",
       "      <th>LeadingClusterCenterLambda</th>\n",
       "      <th>LeadingClusterPt</th>\n",
       "      <th>LeadingClusterSecondLambda</th>\n",
       "      <th>LeadingClusterSecondR</th>\n",
       "      <th>N90Constituents</th>\n",
       "      <th>EMFrac</th>\n",
       "      <th>HECFrac</th>\n",
       "      <th>Timing</th>\n",
       "      <th>OotFracClusters10</th>\n",
       "      <th>OotFracClusters5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>90306.070312</td>\n",
       "      <td>0.308789</td>\n",
       "      <td>-1.921283</td>\n",
       "      <td>6840.930176</td>\n",
       "      <td>0.498666</td>\n",
       "      <td>0.318246</td>\n",
       "      <td>0.140055</td>\n",
       "      <td>-1.924426</td>\n",
       "      <td>0.489299</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>178.461990</td>\n",
       "      <td>60102.636719</td>\n",
       "      <td>15984.612305</td>\n",
       "      <td>13437.013672</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.888970</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.078428</td>\n",
       "      <td>0.004533</td>\n",
       "      <td>0.117972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>123364.226562</td>\n",
       "      <td>-1.344482</td>\n",
       "      <td>1.154525</td>\n",
       "      <td>10920.023438</td>\n",
       "      <td>0.488692</td>\n",
       "      <td>-1.343978</td>\n",
       "      <td>0.136743</td>\n",
       "      <td>1.147211</td>\n",
       "      <td>0.479669</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>5170.814453</td>\n",
       "      <td>44042.378906</td>\n",
       "      <td>165553.234375</td>\n",
       "      <td>76716.382812</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.324276</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.707971</td>\n",
       "      <td>0.019895</td>\n",
       "      <td>0.026191</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>78104.867188</td>\n",
       "      <td>0.472603</td>\n",
       "      <td>-2.727408</td>\n",
       "      <td>7270.276855</td>\n",
       "      <td>0.478719</td>\n",
       "      <td>0.468219</td>\n",
       "      <td>0.133986</td>\n",
       "      <td>-2.730499</td>\n",
       "      <td>0.470848</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>269.738586</td>\n",
       "      <td>28077.353516</td>\n",
       "      <td>108412.906250</td>\n",
       "      <td>10236.585938</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.905376</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.363695</td>\n",
       "      <td>0.006068</td>\n",
       "      <td>0.006068</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>110898.640625</td>\n",
       "      <td>1.884189</td>\n",
       "      <td>-2.434126</td>\n",
       "      <td>13122.060547</td>\n",
       "      <td>0.528585</td>\n",
       "      <td>1.914493</td>\n",
       "      <td>0.151015</td>\n",
       "      <td>-2.437690</td>\n",
       "      <td>0.516863</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>627.649658</td>\n",
       "      <td>16306.850586</td>\n",
       "      <td>139001.937500</td>\n",
       "      <td>2557.068359</td>\n",
       "      <td>11.0</td>\n",
       "      <td>0.587340</td>\n",
       "      <td>0.412595</td>\n",
       "      <td>0.525843</td>\n",
       "      <td>0.006622</td>\n",
       "      <td>0.023643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>101825.992188</td>\n",
       "      <td>-2.036684</td>\n",
       "      <td>-3.113684</td>\n",
       "      <td>8041.627441</td>\n",
       "      <td>0.478719</td>\n",
       "      <td>-2.056497</td>\n",
       "      <td>0.132225</td>\n",
       "      <td>-3.115452</td>\n",
       "      <td>0.469632</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>869.004639</td>\n",
       "      <td>36063.585938</td>\n",
       "      <td>166999.343750</td>\n",
       "      <td>17401.626953</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.663741</td>\n",
       "      <td>0.335038</td>\n",
       "      <td>-0.329519</td>\n",
       "      <td>0.011073</td>\n",
       "      <td>0.011073</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260890</td>\n",
       "      <td>91249.914062</td>\n",
       "      <td>-1.745303</td>\n",
       "      <td>-2.623895</td>\n",
       "      <td>6592.313965</td>\n",
       "      <td>0.468746</td>\n",
       "      <td>-1.774570</td>\n",
       "      <td>0.125520</td>\n",
       "      <td>-2.642066</td>\n",
       "      <td>0.459769</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>339.393738</td>\n",
       "      <td>33378.828125</td>\n",
       "      <td>166918.890625</td>\n",
       "      <td>3192.991211</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.771015</td>\n",
       "      <td>0.228985</td>\n",
       "      <td>-1.044461</td>\n",
       "      <td>0.004217</td>\n",
       "      <td>0.004217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260891</td>\n",
       "      <td>101260.921875</td>\n",
       "      <td>-1.253928</td>\n",
       "      <td>-2.815772</td>\n",
       "      <td>9903.192383</td>\n",
       "      <td>0.528585</td>\n",
       "      <td>-1.253075</td>\n",
       "      <td>0.151641</td>\n",
       "      <td>-2.804190</td>\n",
       "      <td>0.518223</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>263.608734</td>\n",
       "      <td>30561.958984</td>\n",
       "      <td>29329.003906</td>\n",
       "      <td>2992.535400</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.865037</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.036002</td>\n",
       "      <td>0.006210</td>\n",
       "      <td>0.006210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260892</td>\n",
       "      <td>81322.601562</td>\n",
       "      <td>0.825275</td>\n",
       "      <td>-2.666970</td>\n",
       "      <td>9852.224609</td>\n",
       "      <td>0.478719</td>\n",
       "      <td>0.857738</td>\n",
       "      <td>0.134604</td>\n",
       "      <td>-2.664757</td>\n",
       "      <td>0.469947</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>243.027466</td>\n",
       "      <td>45031.343750</td>\n",
       "      <td>14059.222656</td>\n",
       "      <td>5731.004883</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.974370</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.120335</td>\n",
       "      <td>0.016170</td>\n",
       "      <td>0.023511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260893</td>\n",
       "      <td>112891.523438</td>\n",
       "      <td>1.165162</td>\n",
       "      <td>1.432389</td>\n",
       "      <td>8749.023438</td>\n",
       "      <td>0.498666</td>\n",
       "      <td>1.187786</td>\n",
       "      <td>0.141278</td>\n",
       "      <td>1.419347</td>\n",
       "      <td>0.489429</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>257.263977</td>\n",
       "      <td>64777.597656</td>\n",
       "      <td>25375.058594</td>\n",
       "      <td>3136.577393</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.850187</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.226453</td>\n",
       "      <td>0.030247</td>\n",
       "      <td>0.030247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260894</td>\n",
       "      <td>136593.500000</td>\n",
       "      <td>-1.626654</td>\n",
       "      <td>-0.872492</td>\n",
       "      <td>13577.865234</td>\n",
       "      <td>0.528585</td>\n",
       "      <td>-1.646330</td>\n",
       "      <td>0.150062</td>\n",
       "      <td>-0.876254</td>\n",
       "      <td>0.519145</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>166.846436</td>\n",
       "      <td>57454.226562</td>\n",
       "      <td>36886.175781</td>\n",
       "      <td>9861.752930</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.659191</td>\n",
       "      <td>0.259862</td>\n",
       "      <td>0.205212</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2260895 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    pt       eta       phi             m  ActiveArea  \\\n",
       "0         90306.070312  0.308789 -1.921283   6840.930176    0.498666   \n",
       "1        123364.226562 -1.344482  1.154525  10920.023438    0.488692   \n",
       "2         78104.867188  0.472603 -2.727408   7270.276855    0.478719   \n",
       "3        110898.640625  1.884189 -2.434126  13122.060547    0.528585   \n",
       "4        101825.992188 -2.036684 -3.113684   8041.627441    0.478719   \n",
       "...                ...       ...       ...           ...         ...   \n",
       "2260890   91249.914062 -1.745303 -2.623895   6592.313965    0.468746   \n",
       "2260891  101260.921875 -1.253928 -2.815772   9903.192383    0.528585   \n",
       "2260892   81322.601562  0.825275 -2.666970   9852.224609    0.478719   \n",
       "2260893  112891.523438  1.165162  1.432389   8749.023438    0.498666   \n",
       "2260894  136593.500000 -1.626654 -0.872492  13577.865234    0.528585   \n",
       "\n",
       "         ActiveArea4vec_eta  ActiveArea4vec_m  ActiveArea4vec_phi  \\\n",
       "0                  0.318246          0.140055           -1.924426   \n",
       "1                 -1.343978          0.136743            1.147211   \n",
       "2                  0.468219          0.133986           -2.730499   \n",
       "3                  1.914493          0.151015           -2.437690   \n",
       "4                 -2.056497          0.132225           -3.115452   \n",
       "...                     ...               ...                 ...   \n",
       "2260890           -1.774570          0.125520           -2.642066   \n",
       "2260891           -1.253075          0.151641           -2.804190   \n",
       "2260892            0.857738          0.134604           -2.664757   \n",
       "2260893            1.187786          0.141278            1.419347   \n",
       "2260894           -1.646330          0.150062           -0.876254   \n",
       "\n",
       "         ActiveArea4vec_pt  JetGhostArea  ...  LeadingClusterCenterLambda  \\\n",
       "0                 0.489299          0.01  ...                  178.461990   \n",
       "1                 0.479669          0.01  ...                 5170.814453   \n",
       "2                 0.470848          0.01  ...                  269.738586   \n",
       "3                 0.516863          0.01  ...                  627.649658   \n",
       "4                 0.469632          0.01  ...                  869.004639   \n",
       "...                    ...           ...  ...                         ...   \n",
       "2260890           0.459769          0.01  ...                  339.393738   \n",
       "2260891           0.518223          0.01  ...                  263.608734   \n",
       "2260892           0.469947          0.01  ...                  243.027466   \n",
       "2260893           0.489429          0.01  ...                  257.263977   \n",
       "2260894           0.519145          0.01  ...                  166.846436   \n",
       "\n",
       "         LeadingClusterPt  LeadingClusterSecondLambda  LeadingClusterSecondR  \\\n",
       "0            60102.636719                15984.612305           13437.013672   \n",
       "1            44042.378906               165553.234375           76716.382812   \n",
       "2            28077.353516               108412.906250           10236.585938   \n",
       "3            16306.850586               139001.937500            2557.068359   \n",
       "4            36063.585938               166999.343750           17401.626953   \n",
       "...                   ...                         ...                    ...   \n",
       "2260890      33378.828125               166918.890625            3192.991211   \n",
       "2260891      30561.958984                29329.003906            2992.535400   \n",
       "2260892      45031.343750                14059.222656            5731.004883   \n",
       "2260893      64777.597656                25375.058594            3136.577393   \n",
       "2260894      57454.226562                36886.175781            9861.752930   \n",
       "\n",
       "         N90Constituents    EMFrac   HECFrac    Timing  OotFracClusters10  \\\n",
       "0                    3.0  0.888970  0.000000  1.078428           0.004533   \n",
       "1                    3.0  0.324276  0.000000  0.707971           0.019895   \n",
       "2                    4.0  0.905376  0.000000  0.363695           0.006068   \n",
       "3                   11.0  0.587340  0.412595  0.525843           0.006622   \n",
       "4                    5.0  0.663741  0.335038 -0.329519           0.011073   \n",
       "...                  ...       ...       ...       ...                ...   \n",
       "2260890              3.0  0.771015  0.228985 -1.044461           0.004217   \n",
       "2260891              5.0  0.865037  0.000000  0.036002           0.006210   \n",
       "2260892              3.0  0.974370  0.000000 -0.120335           0.016170   \n",
       "2260893              3.0  0.850187  0.000000 -0.226453           0.030247   \n",
       "2260894              5.0  0.659191  0.259862  0.205212           0.000000   \n",
       "\n",
       "         OotFracClusters5  \n",
       "0                0.117972  \n",
       "1                0.026191  \n",
       "2                0.006068  \n",
       "3                0.023643  \n",
       "4                0.011073  \n",
       "...                   ...  \n",
       "2260890          0.004217  \n",
       "2260891          0.006210  \n",
       "2260892          0.023511  \n",
       "2260893          0.030247  \n",
       "2260894          0.000000  \n",
       "\n",
       "[2260895 rows x 29 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "partial_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "full_df = partial_df.join(e_samp_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pt</th>\n",
       "      <th>eta</th>\n",
       "      <th>phi</th>\n",
       "      <th>m</th>\n",
       "      <th>ActiveArea</th>\n",
       "      <th>ActiveArea4vec_eta</th>\n",
       "      <th>ActiveArea4vec_m</th>\n",
       "      <th>ActiveArea4vec_phi</th>\n",
       "      <th>ActiveArea4vec_pt</th>\n",
       "      <th>JetGhostArea</th>\n",
       "      <th>...</th>\n",
       "      <th>EnergyPerSampling18</th>\n",
       "      <th>EnergyPerSampling19</th>\n",
       "      <th>EnergyPerSampling20</th>\n",
       "      <th>EnergyPerSampling21</th>\n",
       "      <th>EnergyPerSampling22</th>\n",
       "      <th>EnergyPerSampling23</th>\n",
       "      <th>EnergyPerSampling24</th>\n",
       "      <th>EnergyPerSampling25</th>\n",
       "      <th>EnergyPerSampling26</th>\n",
       "      <th>EnergyPerSampling27</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>90306.070312</td>\n",
       "      <td>0.308789</td>\n",
       "      <td>-1.921283</td>\n",
       "      <td>6840.930176</td>\n",
       "      <td>0.498666</td>\n",
       "      <td>0.318246</td>\n",
       "      <td>0.140055</td>\n",
       "      <td>-1.924426</td>\n",
       "      <td>0.489299</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>123364.226562</td>\n",
       "      <td>-1.344482</td>\n",
       "      <td>1.154525</td>\n",
       "      <td>10920.023438</td>\n",
       "      <td>0.488692</td>\n",
       "      <td>-1.343978</td>\n",
       "      <td>0.136743</td>\n",
       "      <td>1.147211</td>\n",
       "      <td>0.479669</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>57268.914062</td>\n",
       "      <td>39543.589844</td>\n",
       "      <td>1214.614746</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>78104.867188</td>\n",
       "      <td>0.472603</td>\n",
       "      <td>-2.727408</td>\n",
       "      <td>7270.276855</td>\n",
       "      <td>0.478719</td>\n",
       "      <td>0.468219</td>\n",
       "      <td>0.133986</td>\n",
       "      <td>-2.730499</td>\n",
       "      <td>0.470848</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>110898.640625</td>\n",
       "      <td>1.884189</td>\n",
       "      <td>-2.434126</td>\n",
       "      <td>13122.060547</td>\n",
       "      <td>0.528585</td>\n",
       "      <td>1.914493</td>\n",
       "      <td>0.151015</td>\n",
       "      <td>-2.437690</td>\n",
       "      <td>0.516863</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>101825.992188</td>\n",
       "      <td>-2.036684</td>\n",
       "      <td>-3.113684</td>\n",
       "      <td>8041.627441</td>\n",
       "      <td>0.478719</td>\n",
       "      <td>-2.056497</td>\n",
       "      <td>0.132225</td>\n",
       "      <td>-3.115452</td>\n",
       "      <td>0.469632</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260890</td>\n",
       "      <td>91249.914062</td>\n",
       "      <td>-1.745303</td>\n",
       "      <td>-2.623895</td>\n",
       "      <td>6592.313965</td>\n",
       "      <td>0.468746</td>\n",
       "      <td>-1.774570</td>\n",
       "      <td>0.125520</td>\n",
       "      <td>-2.642066</td>\n",
       "      <td>0.459769</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260891</td>\n",
       "      <td>101260.921875</td>\n",
       "      <td>-1.253928</td>\n",
       "      <td>-2.815772</td>\n",
       "      <td>9903.192383</td>\n",
       "      <td>0.528585</td>\n",
       "      <td>-1.253075</td>\n",
       "      <td>0.151641</td>\n",
       "      <td>-2.804190</td>\n",
       "      <td>0.518223</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>8076.918457</td>\n",
       "      <td>1731.564697</td>\n",
       "      <td>306.276306</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260892</td>\n",
       "      <td>81322.601562</td>\n",
       "      <td>0.825275</td>\n",
       "      <td>-2.666970</td>\n",
       "      <td>9852.224609</td>\n",
       "      <td>0.478719</td>\n",
       "      <td>0.857738</td>\n",
       "      <td>0.134604</td>\n",
       "      <td>-2.664757</td>\n",
       "      <td>0.469947</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-113.000000</td>\n",
       "      <td>224.189087</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260893</td>\n",
       "      <td>112891.523438</td>\n",
       "      <td>1.165162</td>\n",
       "      <td>1.432389</td>\n",
       "      <td>8749.023438</td>\n",
       "      <td>0.498666</td>\n",
       "      <td>1.187786</td>\n",
       "      <td>0.141278</td>\n",
       "      <td>1.419347</td>\n",
       "      <td>0.489429</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>3838.447754</td>\n",
       "      <td>3840.161621</td>\n",
       "      <td>913.936890</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2260894</td>\n",
       "      <td>136593.500000</td>\n",
       "      <td>-1.626654</td>\n",
       "      <td>-0.872492</td>\n",
       "      <td>13577.865234</td>\n",
       "      <td>0.528585</td>\n",
       "      <td>-1.646330</td>\n",
       "      <td>0.150062</td>\n",
       "      <td>-0.876254</td>\n",
       "      <td>0.519145</td>\n",
       "      <td>0.01</td>\n",
       "      <td>...</td>\n",
       "      <td>10539.982422</td>\n",
       "      <td>1920.685303</td>\n",
       "      <td>-91.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2260895 rows × 57 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                    pt       eta       phi             m  ActiveArea  \\\n",
       "0         90306.070312  0.308789 -1.921283   6840.930176    0.498666   \n",
       "1        123364.226562 -1.344482  1.154525  10920.023438    0.488692   \n",
       "2         78104.867188  0.472603 -2.727408   7270.276855    0.478719   \n",
       "3        110898.640625  1.884189 -2.434126  13122.060547    0.528585   \n",
       "4        101825.992188 -2.036684 -3.113684   8041.627441    0.478719   \n",
       "...                ...       ...       ...           ...         ...   \n",
       "2260890   91249.914062 -1.745303 -2.623895   6592.313965    0.468746   \n",
       "2260891  101260.921875 -1.253928 -2.815772   9903.192383    0.528585   \n",
       "2260892   81322.601562  0.825275 -2.666970   9852.224609    0.478719   \n",
       "2260893  112891.523438  1.165162  1.432389   8749.023438    0.498666   \n",
       "2260894  136593.500000 -1.626654 -0.872492  13577.865234    0.528585   \n",
       "\n",
       "         ActiveArea4vec_eta  ActiveArea4vec_m  ActiveArea4vec_phi  \\\n",
       "0                  0.318246          0.140055           -1.924426   \n",
       "1                 -1.343978          0.136743            1.147211   \n",
       "2                  0.468219          0.133986           -2.730499   \n",
       "3                  1.914493          0.151015           -2.437690   \n",
       "4                 -2.056497          0.132225           -3.115452   \n",
       "...                     ...               ...                 ...   \n",
       "2260890           -1.774570          0.125520           -2.642066   \n",
       "2260891           -1.253075          0.151641           -2.804190   \n",
       "2260892            0.857738          0.134604           -2.664757   \n",
       "2260893            1.187786          0.141278            1.419347   \n",
       "2260894           -1.646330          0.150062           -0.876254   \n",
       "\n",
       "         ActiveArea4vec_pt  JetGhostArea  ...  EnergyPerSampling18  \\\n",
       "0                 0.489299          0.01  ...             0.000000   \n",
       "1                 0.479669          0.01  ...         57268.914062   \n",
       "2                 0.470848          0.01  ...             0.000000   \n",
       "3                 0.516863          0.01  ...             0.000000   \n",
       "4                 0.469632          0.01  ...             0.000000   \n",
       "...                    ...           ...  ...                  ...   \n",
       "2260890           0.459769          0.01  ...             0.000000   \n",
       "2260891           0.518223          0.01  ...          8076.918457   \n",
       "2260892           0.469947          0.01  ...             0.000000   \n",
       "2260893           0.489429          0.01  ...          3838.447754   \n",
       "2260894           0.519145          0.01  ...         10539.982422   \n",
       "\n",
       "         EnergyPerSampling19  EnergyPerSampling20  EnergyPerSampling21  \\\n",
       "0                   0.000000             0.000000                  0.0   \n",
       "1               39543.589844          1214.614746                  0.0   \n",
       "2                   0.000000             0.000000                  0.0   \n",
       "3                   0.000000             0.000000                  0.0   \n",
       "4                   0.000000             0.000000                  0.0   \n",
       "...                      ...                  ...                  ...   \n",
       "2260890             0.000000             0.000000                  0.0   \n",
       "2260891          1731.564697           306.276306                  0.0   \n",
       "2260892          -113.000000           224.189087                  0.0   \n",
       "2260893          3840.161621           913.936890                  0.0   \n",
       "2260894          1920.685303           -91.000000                  0.0   \n",
       "\n",
       "         EnergyPerSampling22  EnergyPerSampling23  EnergyPerSampling24  \\\n",
       "0                        0.0                  0.0                  0.0   \n",
       "1                        0.0                  0.0                  0.0   \n",
       "2                        0.0                  0.0                  0.0   \n",
       "3                        0.0                  0.0                  0.0   \n",
       "4                        0.0                  0.0                  0.0   \n",
       "...                      ...                  ...                  ...   \n",
       "2260890                  0.0                  0.0                  0.0   \n",
       "2260891                  0.0                  0.0                  0.0   \n",
       "2260892                  0.0                  0.0                  0.0   \n",
       "2260893                  0.0                  0.0                  0.0   \n",
       "2260894                  0.0                  0.0                  0.0   \n",
       "\n",
       "         EnergyPerSampling25  EnergyPerSampling26  EnergyPerSampling27  \n",
       "0                        0.0                  0.0                  0.0  \n",
       "1                        0.0                  0.0                  0.0  \n",
       "2                        0.0                  0.0                  0.0  \n",
       "3                        0.0                  0.0                  0.0  \n",
       "4                        0.0                  0.0                  0.0  \n",
       "...                      ...                  ...                  ...  \n",
       "2260890                  0.0                  0.0                  0.0  \n",
       "2260891                  0.0                  0.0                  0.0  \n",
       "2260892                  0.0                  0.0                  0.0  \n",
       "2260893                  0.0                  0.0                  0.0  \n",
       "2260894                  0.0                  0.0                  0.0  \n",
       "\n",
       "[2260895 rows x 57 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "full_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "partial_train, partial_test = train_test_split(partial_df, test_size=0.1, random_state=41)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "full_train, full_test = train_test_split(full_df, test_size=0.1, random_state=41)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2034805, 29) (226090, 29)\n"
     ]
    }
   ],
   "source": [
    "print(partial_train.shape, partial_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2034805, 57) (226090, 57)\n"
     ]
    }
   ],
   "source": [
    "print(full_train.shape, full_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save train and test sets\n",
    "#partial_train.to_pickle('processed_data/aod/partial_train.pkl')\n",
    "#partial_test.to_pickle('processed_data/aod/partial_test.pkl')\n",
    "#full_train.to_pickle('processed_data/aod/full_train.pkl')\n",
    "#full_test.to_pickle('processed_data/aod/full_test.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3.7 (fastairoot)",
   "language": "python",
   "name": "fastairoot"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}