params-grounped_gbr-new-groupbycomorbidities (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %pip install multigroupGP\n",
    "import numpy as np \n",
    "import matplotlib.pyplot as plt\n",
    "#from multigroupGP import GP, MultiGroupRBF\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "import pickle\n",
    "import configs_ as cf\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"AllFrameCleaned.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'timestamp', 'hadm_id', 'anchor_age', 'patientweight',\n",
       "       'ethnicity', 'los', 'gender', 'dod', 'cad', 'afib', 'chf', 'ckd',\n",
       "       'esrd', 'paralysis', 'parathyroid', 'rhabdo', 'sarcoid', 'sepsis',\n",
       "       'dialysis', 'hypokalemia', 'Hyperkalemia', 'Obesity', 'cancer',\n",
       "       'asthma', 'pneumonia', 'diabetics', 'HIV', 'bpdia', 'bpsys', 'hr', 'rr',\n",
       "       'spo2', 'temp', 'alt', 'aniongap', 'bun', 'cpk', 'ca', 'chloride',\n",
       "       'creatinine', 'glucose', 'hgb', 'k', 'ldh', 'mg', 'na', 'p', 'wbc',\n",
       "       'betablockers', 'ca-iv', 'ca-noniv', 'cablockers', 'dextrose',\n",
       "       'hours-dextrose', 'fluids', 'insulin', 'k-iv', 'k-noniv',\n",
       "       'loopdiuretics', 'hours-loopdiuretics', 'mg-iv', 'mg-noniv',\n",
       "       'hours-mg-noniv', 'p-iv', 'p-noniv', 'pnutrition', 'ponutrition',\n",
       "       'tpnutrition', 'vasopressors', 'hours-betablockers', 'hours-cablockers',\n",
       "       'hours-insulin', 'hours-k-iv', 'hours-ca-noniv', 'hours-vasopressors',\n",
       "       'hours-pnutrition', 'hours-tpnutrition', 'hours-p-iv',\n",
       "       'hours-ponutrition', 'hours-fluids', 'hours-ca-iv', 'hours-mg-iv'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_feats = ['anchor_age', 'patientweight',\n",
    "       'ethnicity', 'los', 'gender', 'dod', 'cad', 'afib', 'chf', 'ckd',\n",
    "       'esrd', 'paralysis', 'parathyroid', 'rhabdo', 'sarcoid', 'sepsis',\n",
    "       'dialysis', 'hypokalemia', 'Hyperkalemia', 'Obesity', 'cancer',\n",
    "       'asthma', 'pneumonia', 'diabetics', 'HIV', 'bpdia', 'bpsys', 'hr', 'rr',\n",
    "       'spo2', 'temp', 'alt', 'aniongap', 'bun', 'cpk', 'ca', 'chloride',\n",
    "       'creatinine', 'glucose', 'hgb', 'k', 'ldh', 'mg', 'na', 'p', 'wbc',\n",
    "       'betablockers', 'ca-iv', 'ca-noniv', 'cablockers', 'dextrose',\n",
    "       'hours-dextrose', 'fluids', 'insulin', 'k-iv', 'k-noniv',\n",
    "       'loopdiuretics', 'hours-loopdiuretics', 'mg-iv', 'mg-noniv',\n",
    "       'hours-mg-noniv', 'p-iv', 'p-noniv', 'pnutrition', 'ponutrition',\n",
    "       'tpnutrition', 'vasopressors', 'hours-betablockers', 'hours-cablockers',\n",
    "       'hours-insulin', 'hours-k-iv', 'hours-ca-noniv', 'hours-vasopressors',\n",
    "       'hours-pnutrition', 'hours-tpnutrition', 'hours-p-iv',\n",
    "       'hours-ponutrition', 'hours-fluids', 'hours-ca-iv', 'hours-mg-iv']\n",
    "\n",
    "\n",
    "comorbidities = ['esrd', 'hypokalemia', 'Hyperkalemia', 'Obesity', 'cancer',\n",
    "       'asthma', 'diabetics', 'HIV']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "state_feats_names = ['anchor_age', 'patientweight',\n",
    "       'ethnicity', 'los', 'gender', 'dod', 'cad', 'afib', 'chf', 'ckd',\n",
    "       'esrd', 'paralysis', 'parathyroid', 'rhabdo', 'sarcoid', 'sepsis',\n",
    "       'dialysis', 'hypokalemia', 'Hyperkalemia', 'Obesity', 'cancer',\n",
    "       'asthma', 'pneumonia', 'diabetics', 'HIV', 'bpdia', 'bpsys', 'hr', 'rr',\n",
    "       'spo2', 'temp', 'alt', 'aniongap', 'bun', 'cpk', 'ca', 'chloride',\n",
    "       'creatinine', 'glucose', 'hgb', 'k', 'ldh', 'mg', 'na', 'p', 'wbc',\n",
    "       'betablockers', 'ca-iv', 'ca-noniv', 'cablockers', 'dextrose',\n",
    "       'hours-dextrose', 'fluids', 'insulin', 'k-iv', 'k-noniv',\n",
    "       'loopdiuretics', 'hours-loopdiuretics', 'mg-iv', 'mg-noniv',\n",
    "       'hours-mg-noniv', 'p-iv', 'p-noniv', 'pnutrition', 'ponutrition',\n",
    "       'tpnutrition', 'vasopressors', 'hours-betablockers', 'hours-cablockers',\n",
    "       'hours-insulin', 'hours-k-iv', 'hours-ca-noniv', 'hours-vasopressors',\n",
    "       'hours-pnutrition', 'hours-tpnutrition', 'hours-p-iv',\n",
    "       'hours-ponutrition', 'hours-fluids', 'hours-ca-iv', 'hours-mg-iv']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a group column based on comorbidities conditions\n",
    "conditions = [\n",
    "    (df['esrd'] + df['Hyperkalemia'] + df['Obesity'] +df['asthma'] + df['diabetics'] + df['HIV'] == 0),  # when all comorbidities are 0\n",
    "    (((df['esrd'] == 1 ) & (df['Hyperkalemia'] + df['Obesity'] +df['asthma'] + df['diabetics'] + df['HIV'] == 0))), # when esrd is 1 and all other comorbidities are 0\n",
    "    (((df['Hyperkalemia'] == 1 ) & (df['esrd'] + df['Obesity'] +df['asthma'] + df['diabetics'] + df['HIV'] == 0))), # when Hyperkalemia is 1 and all other comorbidities are 0\n",
    "    (((df['Obesity'] == 1 ) & (df['Hyperkalemia'] + df['esrd'] +df['asthma'] + df['diabetics'] + df['HIV'] == 0))), # when Obesity is 1 and all other comorbidities are 0\n",
    "    (((df['asthma'] == 1 ) & (df['Hyperkalemia'] + df['Obesity'] +df['esrd'] + df['diabetics'] + df['HIV'] == 0))), # when asthma is 1 and all other comorbidities are 0\n",
    "    (((df['diabetics'] == 1 ) & (df['Hyperkalemia'] + df['Obesity'] +df['esrd'] + df['asthma'] + df['HIV'] == 0))), # when diabetics is 1 and all other comorbidities are 0\n",
    "    #(((df['HIV'] == 1 ) & (df['Hyperkalemia'] + df['Obesity'] +df['asthma'] + df['diabetics'] + df['esrd'] == 0))), # when HIV is 1 and all other comorbidities are 0\n",
    "    (df['esrd'] + df['Hyperkalemia'] + df['Obesity'] +df['asthma'] + df['diabetics'] + df['HIV']  >= 2) # when more than 1 comorbidities are 1\n",
    "    ]\n",
    "\n",
    "values = ['0', '1', '2', '3', '4', '5', '6']\n",
    "\n",
    "df[\"group\"] = np.select(conditions, values)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    76481\n",
       "5    12441\n",
       "6     8410\n",
       "3     5935\n",
       "2     2819\n",
       "4     2641\n",
       "1     2284\n",
       "Name: group, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.group.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "group_0 = df[df['group']=='0'] #no comorbidities\n",
    "group_1 = df[df['group']=='1'] #esrd\n",
    "group_2 = df[df['group']=='2'] #hyperkalemia\n",
    "group_3 = df[df['group']=='3'] #obesity\n",
    "group_4 = df[df['group']=='4'] #asthma\n",
    "group_5 = df[df['group']=='5'] #diabetics\n",
    "group_6 = df[df['group']=='6'] #multiple comorbdities\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(76481, 84)\n",
      "(2284, 84)\n",
      "(2819, 84)\n",
      "(5935, 84)\n",
      "(2641, 84)\n",
      "(12441, 84)\n",
      "(8410, 84)\n"
     ]
    }
   ],
   "source": [
    "print(group_0.shape)\n",
    "print(group_1.shape)\n",
    "print(group_2.shape)\n",
    "print(group_3.shape)\n",
    "print(group_4.shape)\n",
    "print(group_5.shape)\n",
    "print(group_6.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def state_transformer(frames, states=None, feats=state_feats, transformer=cf.outputdir+'transformer.pkl'):\n",
    "   \n",
    "    print('#:', len(feats))\n",
    "    if states is None:\n",
    "        states = np.vstack([np.array(frames.loc[i, feats]).astype(float) for i in range(len(frames))])\n",
    "    if False: #os.path.isfile(transformer):\n",
    "        scaler = pickle.load(open(transformer, 'rb'))\n",
    "    else:\n",
    "        scaler = preprocessing.StandardScaler().fit(states)\n",
    "        pickle.dump(scaler, open(cf.outputdir+'transformer.pkl', 'wb'))\n",
    "    transformed_states = scaler.transform(states)\n",
    "    return transformed_states\n",
    "\n",
    "def state_invtransformer(tstates, transformer=cf.outputdir+'transformer.pkl'):\n",
    "    \n",
    "    scaler = pickle.load(open(transformer, 'rb'))\n",
    "    states = scaler.inverse_transform(tstates)\n",
    "    \n",
    "    return states\n",
    "\n",
    "def transform(state, transformer=cf.outputdir+'transformer.pkl'):\n",
    "    if os.path.isfile(transformer):\n",
    "        scaler = pickle.load(open(transformer, 'rb'))\n",
    "        return scaler.transform([state])[0]\n",
    "    else:\n",
    "        return state\n",
    "    return \n",
    "\n",
    "def discretize(a, el='K'):\n",
    "    \n",
    "    if el=='K':\n",
    "        adict = {'none': 0, 'low2-iv': 0, 'low4-iv': 0, 'low6-iv': 0,'high1-iv': 0, \n",
    "                 'high2-iv': 0, 'high3-iv': 0, 'low-po': 0, 'med-po': 0, 'high-po': 0}\n",
    "        # What is ivd, ivh?\n",
    "        ivd = float(a[0])\n",
    "        ivh = float(a[2])\n",
    "        orald = float(a[1])\n",
    "        #print(ivd, ivh, orald)\n",
    "\n",
    "        if ivd > 0:\n",
    "            if ivh == 0.0: ivh = 1.0    \n",
    "            rate = ivd/ivh\n",
    "\n",
    "            if rate <= 10:\n",
    "                if ivh <= 2: adict['low2-iv'] = 1\n",
    "                elif ivh <= 4: adict['low4-iv'] = 1\n",
    "                else: adict['low6-iv'] = 1\n",
    "\n",
    "            if rate > 10:\n",
    "                if ivh <= 1: adict['high1-iv'] = 1\n",
    "                elif ivh <= 2: adict['high2-iv'] = 1\n",
    "                else: adict['high3-iv'] = 1\n",
    "\n",
    "        if orald > 0:\n",
    "            if orald <= 20: adict['low-po'] = 1\n",
    "            elif orald <= 40: adict['med-po'] = 1\n",
    "            else: adict['high-po'] = 1  \n",
    "\n",
    "                \n",
    "    elif el=='Mg':\n",
    "        \n",
    "        adict = {'none': 0, 'low4-iv': 0,'high1-iv': 0, 'high2-iv': 0, 'high3-iv': 0, 'low-po': 0, 'med-po': 0, 'high-po': 0}\n",
    "        \n",
    "        ivd = float(a[0])\n",
    "        ivh = float(a[2])\n",
    "        orald = float(a[1])\n",
    "        #print(ivd, ivh, orald)\n",
    "\n",
    "        if ivd > 0:\n",
    "            if ivd > 4: ivd = 4\n",
    "            if ivh == 0.0: ivh = 1.0    \n",
    "            rate = ivd/ivh    \n",
    "            if rate < 1: adict['low4-iv'] = 1\n",
    "            if rate >= 1:\n",
    "                if ivh <= 1: adict['high1-iv'] = 1\n",
    "                elif ivh <= 2: adict['high2-iv'] = 1\n",
    "                else: adict['high3-iv'] = 1\n",
    "\n",
    "        if orald > 0:\n",
    "            if orald < 400: adict['low-po'] = 1\n",
    "            elif orald < 800: adict['med-po'] = 1\n",
    "            else: adict['high-po'] = 1  \n",
    "                \n",
    "                \n",
    "    elif el=='P':\n",
    "        adict = {'none': 0, 'low2-iv': 0, 'high1-iv': 0, 'high3-iv': 0, 'low-po': 0, 'med-po': 0, 'high-po': 0}\n",
    "        \n",
    "        ivd = float(a[0])\n",
    "        ivh = float(a[2])\n",
    "        orald = float(a[1])\n",
    "        #print(ivd, ivh, orald)\n",
    "\n",
    "        if ivd > 0:\n",
    "            if ivh == 0.0: ivh = 1.0    \n",
    "            rate = ivd/ivh\n",
    "\n",
    "            if rate <= 1: adict['low2-iv'] = 1\n",
    "            if rate > 1:\n",
    "                if ivh < 6: adict['high1-iv'] = 1\n",
    "                else: adict['high3-iv'] = 1\n",
    "\n",
    "        if orald > 0:\n",
    "            if orald < 250: adict['low-po'] = 1\n",
    "            elif orald < 500: adict['med-po'] = 1\n",
    "            else: adict['high-po'] = 1          \n",
    "    \n",
    "        \n",
    "    da = list(adict.values())\n",
    "    if sum(da) == 0: da[0] = 1\n",
    "\n",
    "    return da\n",
    "\n",
    "def reward(s, a, ns, w=np.array([1, 1, 1, 1, 1])/5., el='K'):\n",
    "    \n",
    "    rdict = {'cost-iv': 0, 'cost-po': 0, 'high': 0, 'low': 0, 'other': 0}\n",
    "    \n",
    "    if a[0] > 0 : rdict['cost-iv'] -= 1 \n",
    "    if a[1] > 0 : rdict['cost-po'] -= 1\n",
    "    \n",
    "    if el=='K': rdict['high'], rdict['low'] = sigmoid(ns[0], el=el)\n",
    "    if el=='Mg': rdict['high'], rdict['low'] = sigmoid(ns[1], el=el)\n",
    "    if el=='P': rdict['high'], rdict['low'] = sigmoid(ns[2], el=el)\n",
    "    # What does this mean? And-ing a set of floats\n",
    "#     if el == 'K':\n",
    "#         print(str(s[30]))\n",
    "#         print(str(s[31]))\n",
    "#         print(str(s[32]))\n",
    "#         rdict['other'] = -1 * (s[30] & s[31] & s[32])\n",
    "    \n",
    "    phi = np.array(list(rdict.values()))\n",
    "    r = np.dot(phi, w)\n",
    "    \n",
    "    return phi, r\n",
    "\n",
    "def sigmoid(x, el='K'):\n",
    "    \n",
    "    minmax = {'K': [3.5, 4.5], 'Mg': [1.5, 2.5], 'P': [2.5, 4.5]}\n",
    "    lmin, lmax = minmax[el]\n",
    "    \n",
    "    if x < lmin:\n",
    "        z = 1/(1 + np.exp(-3.5*(x-(lmin-1)))) - 1\n",
    "        return (0, z)\n",
    "    elif x > lmax: \n",
    "        z = - 1/(1 + np.exp(-3.5*(x-(lmax+1))))\n",
    "        return (z, 0)\n",
    "    else:\n",
    "        z = 0\n",
    "        return (z, z)\n",
    "    \n",
    "def generate_samples(vnum, trainFrames, el='K'):\n",
    "\n",
    "    frame = trainFrames[trainFrames.hadm_id==vnum]\n",
    "    all_st = []\n",
    "    all_nst = []\n",
    "    all_a = []\n",
    "    all_phi = []\n",
    "    all_r = []\n",
    "    all_G = []\n",
    "\n",
    "    for i in frame.index[:-1]:\n",
    "        \n",
    "        s = list(frame.loc[i, state_feats])\n",
    "        st = transform(s)\n",
    "        all_st.append(st)\n",
    "        if el=='K':\n",
    "            a = list(frame.loc[i+1, ['k-iv', 'k-noniv', 'hours-k-iv']])\n",
    "            #a = list(frame.loc[i + 1, ['k-iv', 'k-noniv']])\n",
    "        elif el=='Mg':\n",
    "            a = list(frame.loc[i+1, ['mg-iv', 'mg-noniv', 'hours-mg-iv']])\n",
    "            #a = list(frame.loc[i + 1, ['mg-iv', 'mg-noniv']])\n",
    "        elif el=='P':\n",
    "            a = list(frame.loc[i+1, ['p-iv', 'p-noniv', 'hours-p-iv']])\n",
    "            #a = list(frame.loc[i + 1, ['p-iv', 'p-noniv']])\n",
    "        da = discretize(a, el=el)\n",
    "        all_a.append(da)\n",
    "        ns = list(frame.loc[i+1, state_feats])\n",
    "        nst = transform(ns)\n",
    "        all_nst.append(nst)\n",
    "        phi, r = reward(s, a, ns, el=el) \n",
    "        all_phi.append(phi)\n",
    "        all_r.append(r)\n",
    "        all_G.append(frame.loc[i, \"group\"])\n",
    "        #print('s:', s, '\\n\\na:', a, '\\n\\nns', ns, '\\n\\nr', phi, r)\n",
    "        \n",
    "    return (all_st, all_a, all_nst, all_phi, all_r, all_G)\n",
    "\n",
    "def combine(ent):\n",
    "    return np.concatenate(np.array(ent))\n",
    "\n",
    "def get_tuples(frames, filename='tuples.pkl', el='K'):\n",
    "    \n",
    "    transition_tuples = {'s': [], 'a': [], 'ns': [], 'phi': [], 'r': [], 'vnum': [], 'G': []}\n",
    "    \n",
    "    if el=='K':\n",
    "        visits = frames[(frames['k-iv']!=0) | (frames['k-noniv']!=0)].hadm_id.unique()\n",
    "    elif el=='Mg':\n",
    "        visits = frames[(frames['mg-iv']!=0) | (frames['mg-noniv']!=0)].hadm_id.unique()\n",
    "    elif el=='P':\n",
    "        visits = frames[(frames['p-iv']!=0) | (frames['p-noniv']!=0)].hadm_id.unique()    \n",
    "    else: \n",
    "        visits = frames.visit_num.unique()\n",
    "        \n",
    "    for vnum in visits:\n",
    "        if len(frames[frames.hadm_id==vnum]) > 1:\n",
    "            s, a, ns, phi, r, G = generate_samples(vnum, frames, el)\n",
    "            transition_tuples['s'].append(np.array(s))\n",
    "            transition_tuples['a'].append(np.array(a))\n",
    "            transition_tuples['ns'].append(ns)\n",
    "            transition_tuples['phi'].append(phi)\n",
    "            transition_tuples['r'].append(r)\n",
    "            transition_tuples['vnum'].append(np.repeat(vnum, len(r)))\n",
    "            transition_tuples['G'].append(G)\n",
    "\n",
    "    for k in transition_tuples.keys():\n",
    "        transition_tuples[k] = combine(transition_tuples[k])\n",
    "        \n",
    "    pickle.dump(transition_tuples, open(cf.outputdir + filename, 'wb'))\n",
    "    \n",
    "    return transition_tuples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'../mimic_iv/'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cf.outputdir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datadir = '../data-processed/vnum_lists/'\n",
    "# dfs = [df, group_1, group_2, group_3, group_4, group_0]\n",
    "# title = ['All', 'group_1', 'group_2', 'group_3', 'group_4', 'group_0']\n",
    "dfs = [df, group_0, group_1, group_2, group_3, group_4, group_5, group_6]\n",
    "title = ['All', 'group_0', 'group_1', 'group_2', 'group_3', 'group_4', 'group_5', 'group_6']\n",
    "i = 0\n",
    "for df in dfs:\n",
    "    allFrames = df #pd.read_csv(datadir + 'AllFrameCleaned.csv')\n",
    "\n",
    "    print('Total number of processed adms =', len(allFrames.hadm_id.unique()),'; number of transitions =', len(allFrames))    \n",
    "    allFrames = allFrames.sort_values(by=['hadm_id', 'timestamp'])\n",
    "    d = 1000\n",
    "    if len(df) < 10000:\n",
    "        d = 150\n",
    "    bp = np.sort(allFrames.hadm_id.unique())[d]\n",
    "    print(bp)\n",
    "    trainFrames = allFrames[allFrames['hadm_id'] < bp].reset_index()\n",
    "    testFrames = allFrames[allFrames['hadm_id'] >= bp].reset_index()\n",
    "    # trainFrames.to_csv(cf.outputdir+'trainFrames.csv', index=False)\n",
    "    # testFrames.to_csv(cf.outputdir+'testFrames.csv', index=False)\n",
    "    print('Transformer')\n",
    "    ts = state_transformer(trainFrames)\n",
    "    visits = allFrames.hadm_id.unique()\n",
    "    print('Get tuples')\n",
    "    train_tuple = get_tuples(trainFrames, filename=f'{title[i]}_GroupedtrainKtuples.pkl', el='K')\n",
    "    test_tuple = get_tuples(testFrames, filename=f'{title[i]}_GroupedtestKtuples.pkl', el='K')\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_group = pickle.load(open(cf.outputdir + 'All_GroupedtrainKtuples.pkl', 'rb'))       \n",
    "all_group_test = pickle.load(open(cf.outputdir + 'All_GroupedtestKtuples.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(all_group['G']))\n",
    "print(len(all_group_test['G']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(np.unique(all_group['G'], return_counts=True))\n",
    "print(np.unique(all_group_test['G'], return_counts=True))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "vscode": {
   "interpreter": {
    "hash": "e8656658b91e58fc427d957c511da0a238718ef3a47a940e73158e087ae07829"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}